diff --git a/.env.example b/.env.example index 089bb975..e146c9a0 100644 --- a/.env.example +++ b/.env.example @@ -10,14 +10,15 @@ OPENAI_API_KEY=sk-... # For local development: localhost:5000 EVAL_REGISTRY=ghcr.io/eval-containers -# Default agent and model (can be overridden per run) +# Default agent and model (can be overridden per run). EVAL_MODEL is the +# / handle the shared gateway routes to at runtime. EVAL_AGENT=claude-code -EVAL_MODEL=claude-sonnet-4 +EVAL_MODEL=anthropic/claude-sonnet-4-5 # Default timeout in seconds (can be overridden per run or per benchmark) EVAL_TIMEOUT=300 -# Hard cap on model spend in USD per run. The litellm proxy enforces +# Hard cap on model spend in USD per run. The gateway enforces # it and crashes the run (BudgetExceededError) once crossed. Default $1 # is strict on purpose — bump after observing real runs. Overridable # via `eval-containers run --max-budget `. diff --git a/.github/.trivyignore b/.github/.trivyignore index 521d3e6e..85b465a5 100644 --- a/.github/.trivyignore +++ b/.github/.trivyignore @@ -106,7 +106,7 @@ CVE-2026-45447 CVE-2026-44432 # --- undici in node tooling: npm bundle (no reachable fix) ------------------ -# core/agent-base-node and core/litellm both carry npm's bundled undici 6.26.0 / +# core/agent-base-node carries npm's bundled undici 6.26.0 / # 6.25.0. Fixed in undici 6.27.0, but `npm@latest` IS 11.17.0 (still bundles # 6.26.0 — verified), so no npm bump reaches it. This is build-time npm tooling: # the agents' runtime HTTP uses node's OWN bundled undici, and the eval data path diff --git a/.github/ISSUE_TEMPLATE/05-new-agent-request.md b/.github/ISSUE_TEMPLATE/05-new-agent-request.md index e7de9bf2..32ceeb67 100644 --- a/.github/ISSUE_TEMPLATE/05-new-agent-request.md +++ b/.github/ISSUE_TEMPLATE/05-new-agent-request.md @@ -43,7 +43,7 @@ specifically. --> ## Endpoint compatibility ## Known obstacles diff --git a/.github/ISSUE_TEMPLATE/06-new-model-request.md b/.github/ISSUE_TEMPLATE/06-new-model-request.md index 7e4f8f08..53be9e24 100644 --- a/.github/ISSUE_TEMPLATE/06-new-model-request.md +++ b/.github/ISSUE_TEMPLATE/06-new-model-request.md @@ -7,12 +7,11 @@ labels: ["new-model"] ## Model: `/` @@ -30,9 +29,7 @@ bifrost / litellm / portkey. ## What's missing -- [ ] LiteLLM doesn't support this provider in the pinned `core/litellm` version — link a version bump -- [ ] Needs a new gateway backend (the existing bifrost / litellm / portkey can't reach it) — describe why -- [ ] Want a **pinned per-model image** (a shared, custom-configured artifact teams run against via `EVAL_GATEWAY_IMAGE=`) — not just runtime `EVAL_MODEL` +- [ ] Needs a new gateway flavor (the existing bifrost / litellm / portkey can't reach it) — describe why - [ ] It works already; requesting it be added to the docs / examples ## Who implements diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index c78b580b..1021a0ea 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -37,7 +37,7 @@ For PRs that modify existing code, fill in the sections below. - [ ] `cargo test` passes (sanity gates: check, compose, dockerfile_inspection, task_inspection, upstream unit tests) - [ ] If the change touches a Dockerfile or compose file: one affected benchmark/agent builds locally (`docker build` or `eval-containers build bench `) - [ ] If the change touches mechanical rule catalogs: every new rule has a unit test for the positive and negative case -- [ ] If the change touches shared infrastructure (core/entrypoint, core/litellm, core/combination.Dockerfile): one smoke run against `aime` or `mmlu` via the live driver passes +- [ ] If the change touches shared infrastructure (core/entrypoint, gateways/, core/combination.Dockerfile): one smoke run against `aime` or `mmlu` via the live driver passes ## RULES.md impact diff --git a/.github/PULL_REQUEST_TEMPLATE/agent.md b/.github/PULL_REQUEST_TEMPLATE/agent.md index af88826c..32a4b119 100644 --- a/.github/PULL_REQUEST_TEMPLATE/agent.md +++ b/.github/PULL_REQUEST_TEMPLATE/agent.md @@ -71,7 +71,7 @@ eval-containers run aime --agent --model openai/gpt-5.4 --task-id 0 --loc - [ ] `output/aime/0/model/trajectory.jsonl` non-empty, has real LLM calls - [ ] `output/aime/0/task/result.json` has a valid reward (0, 1, or fractional) -- [ ] `output/aime/0/model/result.json` has `cost_usd > 0` — the proxy logged the call (if 0, investigate: your agent's SDK path may not trigger the logging callback, see `core/litellm/eval_logger.py`) +- [ ] `output/aime/0/model/result.json` has `cost_usd > 0` — the proxy logged the call (if 0, investigate: your agent's SDK path may not trigger the logging callback, see `gateways/litellm/eval_logger.py`) Benchmark 2 (recommend `humaneval` for code generation or `gsm8k` for tool-less reasoning): @@ -97,7 +97,7 @@ Our proxy exposes `/v1/messages` (Anthropic) and `/v1/chat/completions` (OpenAI) - [ ] My agent uses `/v1/messages` (Anthropic SDK) - [ ] My agent uses `/v1/chat/completions` (OpenAI Chat Completions) -- [ ] My agent uses `/v1/responses` (OpenAI Responses API) — requires `core/litellm` pinned to v1.63.8+ +- [ ] My agent uses `/v1/responses` (OpenAI Responses API) — requires the `gateways/litellm` pin at v1.63.8+ - [ ] My agent uses a different endpoint (specify): — document which LiteLLM version is required ### Known limitations diff --git a/.github/PULL_REQUEST_TEMPLATE/model.md b/.github/PULL_REQUEST_TEMPLATE/model.md index d6bf3a73..6de665fc 100644 --- a/.github/PULL_REQUEST_TEMPLATE/model.md +++ b/.github/PULL_REQUEST_TEMPLATE/model.md @@ -1,46 +1,41 @@ -## Model image: `` - -- [ ] **Kind**: pinned per-model (bakes `/`) **/** generic backend (routes `EVAL_MODEL`) +## Gateway flavor: `` - + -### Contract ([.agents/models/RULES.md](../../.agents/models/RULES.md)) +### Contract ([.agents/gateways/RULES.md](../../.agents/gateways/RULES.md), [.agents/models/RULES.md](../../.agents/models/RULES.md)) -- [ ] `FROM ghcr.io/exgentic/gateways/:latest` or `core/litellm:latest` (inherits the eval-logger + budget wrapper) -- [ ] Routing matches the kind — a generic backend routes `EVAL_MODEL` via a wildcard (rules 1–2); a pinned image bakes one `/` -- [ ] Labels `eval.type="model"` (+ `gateway.kind` / `eval.model.*` per rule 15) -- [ ] Provider keys via `os.environ/_API_KEY` only — none in labels / compose / the agent (rules 4–5) -- [ ] Logs every request + response; the agent cannot reach `/output/model/` (rules 6–7) -- [ ] `EVAL_MODEL_MAX_BUDGET` hard cap enforced (rule 16) -- [ ] LiteLLM/proxy version pinned as a reproducible default; `EVAL_LITELLM_VERSION` honored (rules 12–13) +- [ ] `containers/models//Dockerfile` is `FROM ghcr.io/exgentic/gateways/:latest` and adds nothing but the config template (gateways rule 16) +- [ ] Routes `EVAL_MODEL` at runtime via a wildcard — no baked provider, model, or URL (gateways rules 1–2a; models rules 1–2) +- [ ] Labels `eval.type="model"` + `gateway.kind=""` (models rule 15) +- [ ] Provider keys via provider-native env vars only — none in labels / compose / the agent (models rules 4–5) +- [ ] Logs every request + response; the agent cannot reach `/output/model/` (models rules 6–7) +- [ ] `EVAL_MODEL_MAX_BUDGET` hard cap enforced (models rule 16) +- [ ] Proxy version pinned at build time; `gateway._version` label set (models rule 12; gateways rule 14) ### Evidence: a real run ```bash -# generic backend: EVAL_GATEWAY_IMAGE= + --model / -# pinned per-model: EVAL_GATEWAY_IMAGE= (model is baked; no --model) EVAL_GATEWAY_IMAGE= \ - eval-containers run aime --agent claude-code --task-id 0 --local --max-budget 1 + eval-containers run aime --agent claude-code --task-id 0 --local \ + --model / --max-budget 1 ``` - [ ] `output/aime/0/model/trajectory.jsonl` non-empty; `result.json` `cost_usd > 0` -- [ ] (generic) swapping `--model` to another `/` routes to the new provider with **no rebuild** +- [ ] Swapping `--model` to another `/` routes to the new provider with **no rebuild** ### Docs + changelog - [ ] [`docs/guides/add-a-model.md`](../../docs/guides/add-a-model.md) and any affected page updated ([.agents/docs/RULES.md](../../.agents/docs/RULES.md) rule 15) -- [ ] `.agents/models/RULES.md` changelog entry if rules changed; otherwise "no RULES.md changes needed" +- [ ] `.agents/models/RULES.md` / `.agents/gateways/RULES.md` changelog entry if rules changed; otherwise "no RULES.md changes needed" diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index bbb213ca..4766ac94 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -144,7 +144,7 @@ jobs: if: ${{ !cancelled() }} run: tests/static/helm.sweep.sh - - name: model paths — generic gateway + pinned per-model image + - name: model paths — one shared gateway, model picked at runtime if: ${{ !cancelled() }} run: tests/static/model-paths.sweep.sh diff --git a/cli/README.md b/cli/README.md index 8e8b66b4..5d3368fe 100644 --- a/cli/README.md +++ b/cli/README.md @@ -16,7 +16,7 @@ evaluation by hand. cargo install eval-containers # Run one evaluation (prints the plain `docker compose` command it stands for) -eval-containers run aime --task-id 0 --agent codex --model gpt-5.4 +eval-containers run aime --task-id 0 --agent codex --model openai/gpt-5.4 ``` See the [repository README](https://github.com/Exgentic/eval-containers) and the diff --git a/cli/src/gen_bake.rs b/cli/src/gen_bake.rs index 5f914a3f..ed9e10dc 100644 --- a/cli/src/gen_bake.rs +++ b/cli/src/gen_bake.rs @@ -176,7 +176,7 @@ mod tests { #[test] fn gateway_target_is_category_prefixed() { // Gateways follow - like agents/benchmarks, NOT a bare - // name — a bare `litellm` would collide with core/litellm's target when + // name — a bare `litellm` would collide with models/litellm's target when // both bake files load in one invocation (RULES.md principle 15.a). let out = render("gateways", "litellm", &[], false); assert!(out.contains("target \"gateway-litellm\"")); diff --git a/cli/src/naming.rs b/cli/src/naming.rs index a13d4b65..63ddad42 100644 --- a/cli/src/naming.rs +++ b/cli/src/naming.rs @@ -30,7 +30,8 @@ pub fn agent_image(registry: &str, agent: &str, tag: &str) -> String { format!("{registry}/agents/{agent}:{tag}") } -/// `{registry}/models/:` — the per-model gateway image. +/// `{registry}/models/:` — the shared model-gateway image +/// (a gateway flavor: bifrost/litellm/portkey, or replay). pub fn model_image(registry: &str, model: &str, tag: &str) -> String { format!("{registry}/models/{model}:{tag}") } diff --git a/cli/src/run.rs b/cli/src/run.rs index 67a0be8e..2390dfb8 100644 --- a/cli/src/run.rs +++ b/cli/src/run.rs @@ -100,7 +100,7 @@ pub struct RunArgs { model_tag: Option, // NOTE: upstream versions (benchmark dataset revision, agent CLI version, - // litellm version) are a BUILD-time axis (RULES.md principle 9): pinned via + // gateway version) are a BUILD-time axis (RULES.md principle 9): pinned via // `ARG *_VERSION` in each image and overridden at `build` time, not here. // There is no runtime override — the running version is whatever the image // was built with, recorded in its label. @@ -109,7 +109,7 @@ pub struct RunArgs { timeout: Option, /// Hard cap on model spend in USD for this run (maps to - /// $EVAL_MODEL_MAX_BUDGET). The litellm proxy enforces it and + /// $EVAL_MODEL_MAX_BUDGET). The gateway enforces it and /// returns an error once spend crosses the cap, which crashes /// the agent's next request. Default: $1. #[arg(long)] diff --git a/containers/benchmarks/_chart/templates/job.yaml b/containers/benchmarks/_chart/templates/job.yaml index 46cbdc25..bcfd5df5 100644 --- a/containers/benchmarks/_chart/templates/job.yaml +++ b/containers/benchmarks/_chart/templates/job.yaml @@ -107,9 +107,6 @@ spec: {{- with $v.maxBudget }} - { name: EVAL_MODEL_MAX_BUDGET, value: {{ . | quote }} } {{- end }} - {{- with $v.litellmVersion }} - - { name: EVAL_LITELLM_VERSION, value: {{ . | quote }} } - {{- end }} {{- with $v.gatewayExtraEnv }} {{- toYaml . | nindent 12 }} {{- end }} diff --git a/containers/benchmarks/_chart/values.yaml b/containers/benchmarks/_chart/values.yaml index 9f46f21e..cf220e5a 100644 --- a/containers/benchmarks/_chart/values.yaml +++ b/containers/benchmarks/_chart/values.yaml @@ -14,10 +14,10 @@ task: "0" # rejected when perTask is true. (benchmarks/RULES.md — eval-image naming.) perTask: false registry: ghcr.io/exgentic -# `gatewayImage` picks the proxy: a generic gateway (bifrost/litellm/portkey) -# routes `model`=/ at runtime; a pinned image bakes its own. -# REQUIRED, no default — the gateway errors on an empty handle. The runner's clean -# label is derived from it (last path segment). +# `gatewayImage` picks the proxy flavor (bifrost/litellm/portkey — or replay); +# the gateway routes `model`=/ at runtime. `model` is REQUIRED, +# no default — the gateway errors on an empty handle. The runner's clean label +# is derived from it (last path segment). gatewayImage: bifrost model: "" # Agent-side reasoning effort (e.g. high); empty → the agent's default. @@ -50,7 +50,6 @@ nameSuffix: "" # Internal upstream version overrides (CLI --*-version, --max-budget); empty = unset benchmarkVersion: "" agentVersion: "" -litellmVersion: "" maxBudget: "" # ── Knobs a benchmark MAY override in its values.yaml ──────────────────────── diff --git a/containers/compose/services.yaml b/containers/compose/services.yaml index 4d504b54..98e0de53 100644 --- a/containers/compose/services.yaml +++ b/containers/compose/services.yaml @@ -27,8 +27,8 @@ services: environment: OPENAI_API_KEY: ${OPENAI_API_KEY:?} OPENAI_API_BASE: ${OPENAI_API_BASE:?} - # The model authority: a generic gateway pins to this handle. Unset -> - # passthrough (client model); a pinned per-model image bakes its own. + # The model authority: the gateway pins every request to this handle. + # Unset -> passthrough (client model; recording only, not evals). EVAL_MODEL: ${EVAL_MODEL} HOST: 0.0.0.0 # Both gateway flavors emit OTel: bifrost via its native `otel` diff --git a/containers/core/litellm/Dockerfile b/containers/core/litellm/Dockerfile deleted file mode 100644 index 483bf6bc..00000000 --- a/containers/core/litellm/Dockerfile +++ /dev/null @@ -1,37 +0,0 @@ -FROM ghcr.io/berriai/litellm:v1.89.1 - -LABEL eval.type="core" -LABEL eval.core.component="litellm" -LABEL eval.core.description="LiteLLM proxy base with Eval Containers file logger" -LABEL eval.core.litellm_version="v1.89.1" - -ENV EVAL_LITELLM_VERSION_DEFAULT="v1.89.1" - -# CVE remediation: the litellm v1.89.1 base venv ships four transitive deps -# below their fixed versions (starlette 1.1.0, python-multipart 0.0.27, -# PyJWT 2.12.0, cryptography 46.0.7). Force-patch them in the base's pip-less uv -# venv via a build-time uv bind-mount (leaves no layer). cryptography 48.0.1 is -# two majors past litellm's tested 46.x ceiling — the GHSA-537c fix lands only -# there; the RS256 smoke below proves litellm's JWT path still works. -# (orjson/tornado/urllib3 already ship fixed in the base; pillow isn't installed.) -# starlette CVE-2026-48818/54283 · python-multipart CVE-2026-24486/42561/53539 -# PyJWT CVE-2026-48526 · cryptography GHSA-537c-gmf6-5ccf -RUN --mount=from=ghcr.io/astral-sh/uv:0.11.21,source=/uv,target=/usr/local/bin/uv \ - uv pip install --python /app/.venv/bin/python3 --no-cache \ - "starlette==1.3.1" \ - "python-multipart==0.0.30" \ - "PyJWT==2.13.0" \ - "cryptography==48.0.1" - -# Functional guard: cryptography 48 is past litellm's cap, so prove PyJWT's -# RS256 sign+verify (the proxy's JWT-auth path) still works before shipping. -RUN /app/.venv/bin/python3 -c "import jwt; from cryptography.hazmat.primitives.asymmetric import rsa; k=rsa.generate_private_key(public_exponent=65537, key_size=2048); assert jwt.decode(jwt.encode({'s':'e'}, k, algorithm='RS256'), k.public_key(), algorithms=['RS256'])['s']=='e'" - -COPY eval_logger.py /app/eval_logger.py -COPY eval-litellm-entrypoint.sh /eval-litellm-entrypoint.sh -RUN chmod +x /eval-litellm-entrypoint.sh - -# Wrap the upstream `litellm` entrypoint so we can rewrite -# config.yaml's max_budget from $EVAL_MODEL_MAX_BUDGET before starting -# the proxy. Child model images inherit this automatically via FROM. -ENTRYPOINT ["/eval-litellm-entrypoint.sh"] diff --git a/containers/core/litellm/docker-bake.hcl b/containers/core/litellm/docker-bake.hcl deleted file mode 100644 index ebb3922c..00000000 --- a/containers/core/litellm/docker-bake.hcl +++ /dev/null @@ -1,4 +0,0 @@ -target "litellm" { - context = "containers/core/litellm" - tags = ["${REGISTRY}/core/litellm:${TAG}"] -} diff --git a/containers/core/litellm/eval-litellm-entrypoint.sh b/containers/core/litellm/eval-litellm-entrypoint.sh deleted file mode 100644 index f6880e4f..00000000 --- a/containers/core/litellm/eval-litellm-entrypoint.sh +++ /dev/null @@ -1,58 +0,0 @@ -#!/bin/bash -# Eval Containers LiteLLM entrypoint wrapper. -# -# The upstream litellm image's ENTRYPOINT is `litellm` and CMD is -# `--port 4000`. This wrapper sits in front of that and does ONE thing: -# inject the per-run `EVAL_MODEL_MAX_BUDGET` value into `/app/config.yaml` -# so the proxy enforces a hard cap on cost before starting. -# -# Why a wrapper and not `os.environ/VAR` in config.yaml directly: litellm -# supports env var substitution for string fields in `litellm_params` -# (like `api_key`), but `litellm_settings.max_budget` is a numeric field -# and its env-var handling is unreliable across minor versions. Doing -# the substitution at container-start with sed is the portable answer. -# -# See compose/RULES.md rule 10 (.env is the single config), parent -# /RULES.md principle 9 (runtime version override), and -# compose/services.yaml for the passthrough. -set -euo pipefail - -BUDGET="${EVAL_MODEL_MAX_BUDGET:-1}" - -# Sanity check: must be a positive number. Refuse to start on garbage -# (fail-loud, per tests/RULES.md rule 8). -if ! [[ "$BUDGET" =~ ^[0-9]+(\.[0-9]+)?$ ]]; then - echo "eval-litellm: EVAL_MODEL_MAX_BUDGET=$BUDGET is not a valid number" >&2 - exit 64 -fi - -# Inject max_budget under litellm_settings if the key is absent; replace -# it if present. Idempotent across restarts. -python3 - "$BUDGET" <<'PY' -import sys, re -budget = float(sys.argv[1]) -with open('/app/config.yaml') as f: - text = f.read() - -# Replace an existing max_budget line if present -new, n = re.subn(r'(?m)^(\s*)max_budget:.*$', rf'\g<1>max_budget: {budget}', text) -if n == 0: - # Append under litellm_settings block, creating it if needed - if re.search(r'(?m)^litellm_settings:', new): - new = re.sub( - r'(?m)^(litellm_settings:\n)', - rf'\g<1> max_budget: {budget}\n', - new, - count=1, - ) - else: - new = new.rstrip() + f'\n\nlitellm_settings:\n max_budget: {budget}\n' - -with open('/app/config.yaml', 'w') as f: - f.write(new) -print(f'eval-litellm: enforced max_budget={budget} USD', file=sys.stderr) -PY - -# Hand off to the real litellm entrypoint. All args we receive come -# from the model image's CMD (e.g. `--port 4000 --config /app/config.yaml`). -exec litellm "$@" diff --git a/containers/core/litellm/eval_logger.py b/containers/core/litellm/eval_logger.py deleted file mode 100644 index 0434ff73..00000000 --- a/containers/core/litellm/eval_logger.py +++ /dev/null @@ -1,87 +0,0 @@ -""" -Eval Containers LiteLLM callback: writes LiteLLM's StandardLoggingPayload to /output/trajectory.jsonl -and maintains /output/result.json with aggregated model metadata. - -The trajectory format is LiteLLM's standard — Eval Containers only controls where it's written. -""" - -import json -import os -from litellm.integrations.custom_logger import CustomLogger - - -class EvalContainersLogger(CustomLogger): - # Fallback per-token rates when LiteLLM returns response_cost=0 for - # custom model paths (e.g. openai/azure/gpt-5.4). These match the - # model_info values in the litellm gateway config template so cost accounting - # stays consistent. - _FALLBACK_INPUT_COST = float( - os.environ.get("EVAL_FALLBACK_INPUT_COST_PER_TOKEN", 0.0000025) - ) # $2.50/1M - _FALLBACK_OUTPUT_COST = float( - os.environ.get("EVAL_FALLBACK_OUTPUT_COST_PER_TOKEN", 0.000010) - ) # $10.00/1M - - def __init__(self): - self.output_dir = os.environ.get("EVAL_OUTPUT_DIR", "/output") - self.log_file = os.path.join(self.output_dir, "trajectory.jsonl") - self.result_file = os.path.join(self.output_dir, "result.json") - os.makedirs(self.output_dir, exist_ok=True) - self.total_tokens = 0 - self.total_cost = 0.0 - self.model = "" - self.provider = "" - - def log_success_event(self, kwargs, response_obj, start_time, end_time): - self._write(kwargs) - - def log_failure_event(self, kwargs, response_obj, start_time, end_time): - self._write(kwargs) - - async def async_log_success_event(self, kwargs, response_obj, start_time, end_time): - self._write(kwargs) - - async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time): - self._write(kwargs) - - def _write(self, kwargs): - try: - payload = kwargs.get("standard_logging_object") - if payload is None: - return - - # Append to trajectory - with open(self.log_file, "a") as f: - f.write(json.dumps(payload, default=str) + "\n") - - # Update aggregated result - self.model = payload.get("model", self.model) - self.provider = payload.get("custom_llm_provider", self.provider) - self.total_tokens += payload.get("total_tokens", 0) or 0 - - cost = payload.get("response_cost", 0) or 0 - if cost == 0: - # LiteLLM lacks pricing for custom model paths like - # openai/azure/gpt-5.4 — compute from token counts. - prompt_tokens = payload.get("prompt_tokens", 0) or 0 - completion_tokens = payload.get("completion_tokens", 0) or 0 - cost = ( - prompt_tokens * self._FALLBACK_INPUT_COST - + completion_tokens * self._FALLBACK_OUTPUT_COST - ) - self.total_cost += cost - - result = { - "model": self.model, - "provider": self.provider, - "total_tokens": self.total_tokens, - "cost_usd": round(self.total_cost, 6), - } - with open(self.result_file, "w") as f: - json.dump(result, f) - - except Exception as e: - print(f"[eval_logger] error: {e}") - - -eval_logger_instance = EvalContainersLogger() diff --git a/containers/gateways/litellm/Dockerfile b/containers/gateways/litellm/Dockerfile index 7d23c3e1..5856d0cc 100644 --- a/containers/gateways/litellm/Dockerfile +++ b/containers/gateways/litellm/Dockerfile @@ -19,7 +19,7 @@ COPY --from=ghcr.io/astral-sh/uv:0.11.21 /uv /usr/local/bin/uv # self-contained under /opt/gateway/ — combination.Dockerfile only has # to COPY one path and gets the binary, the venv, and the config. # -# litellm 1.89.1 (same as core/litellm) + the same transitive-dep CVE +# litellm 1.89.1 + the transitive-dep CVE # force-patches, applied in a second pass that overrides litellm's caps # (cryptography 48.0.1 is past its <47 ceiling): # starlette CVE-2026-48818/54283 · python-multipart CVE-2026-24486/42561/53539 diff --git a/containers/models/claude-opus-4/Dockerfile b/containers/models/claude-opus-4/Dockerfile deleted file mode 100644 index 5cedf072..00000000 --- a/containers/models/claude-opus-4/Dockerfile +++ /dev/null @@ -1,13 +0,0 @@ -ARG REGISTRY=ghcr.io/exgentic -ARG REGISTRY_SUFFIX=/ -FROM ${REGISTRY}/core${REGISTRY_SUFFIX}litellm:latest - -LABEL eval.type="model" -LABEL eval.model.name="claude-opus-4" -LABEL eval.model.provider="anthropic" -LABEL eval.model.litellm_version="main-v1.83.3-stable" - -ENV EVAL_LITELLM_VERSION_DEFAULT="main-v1.83.3-stable" - -COPY config.yaml /app/config.yaml -CMD ["--port", "4000", "--config", "/app/config.yaml"] diff --git a/containers/models/claude-opus-4/config.yaml b/containers/models/claude-opus-4/config.yaml deleted file mode 100644 index c1f66335..00000000 --- a/containers/models/claude-opus-4/config.yaml +++ /dev/null @@ -1,8 +0,0 @@ -model_list: - - model_name: "*" - litellm_params: - model: "anthropic/claude-opus-4-20250514" - api_key: "os.environ/ANTHROPIC_API_KEY" - -litellm_settings: - callbacks: ["eval_logger.eval_logger_instance"] diff --git a/containers/models/claude-opus-4/docker-bake.hcl b/containers/models/claude-opus-4/docker-bake.hcl deleted file mode 100644 index c8a06399..00000000 --- a/containers/models/claude-opus-4/docker-bake.hcl +++ /dev/null @@ -1,7 +0,0 @@ -target "model-claude-opus-4" { - context = "containers/models/claude-opus-4" - contexts = { - "${REGISTRY}/core/litellm" = "target:litellm" - } - tags = ["${REGISTRY}/models/claude-opus-4:${TAG}"] -} diff --git a/containers/models/claude-sonnet-4/Dockerfile b/containers/models/claude-sonnet-4/Dockerfile deleted file mode 100644 index 90b54003..00000000 --- a/containers/models/claude-sonnet-4/Dockerfile +++ /dev/null @@ -1,13 +0,0 @@ -ARG REGISTRY=ghcr.io/exgentic -ARG REGISTRY_SUFFIX=/ -FROM ${REGISTRY}/core${REGISTRY_SUFFIX}litellm:latest - -LABEL eval.type="model" -LABEL eval.model.name="claude-sonnet-4" -LABEL eval.model.provider="anthropic" -LABEL eval.model.litellm_version="main-v1.83.3-stable" - -ENV EVAL_LITELLM_VERSION_DEFAULT="main-v1.83.3-stable" - -COPY config.yaml /app/config.yaml -CMD ["--port", "4000", "--config", "/app/config.yaml"] diff --git a/containers/models/claude-sonnet-4/config.yaml b/containers/models/claude-sonnet-4/config.yaml deleted file mode 100644 index 0b30c88e..00000000 --- a/containers/models/claude-sonnet-4/config.yaml +++ /dev/null @@ -1,8 +0,0 @@ -model_list: - - model_name: "*" - litellm_params: - model: "anthropic/claude-sonnet-4-20250514" - api_key: "os.environ/ANTHROPIC_API_KEY" - -litellm_settings: - callbacks: ["eval_logger.eval_logger_instance"] diff --git a/containers/models/claude-sonnet-4/docker-bake.hcl b/containers/models/claude-sonnet-4/docker-bake.hcl deleted file mode 100644 index c50934ff..00000000 --- a/containers/models/claude-sonnet-4/docker-bake.hcl +++ /dev/null @@ -1,7 +0,0 @@ -target "model-claude-sonnet-4" { - context = "containers/models/claude-sonnet-4" - contexts = { - "${REGISTRY}/core/litellm" = "target:litellm" - } - tags = ["${REGISTRY}/models/claude-sonnet-4:${TAG}"] -} diff --git a/containers/models/gpt-4.1-mini/Dockerfile b/containers/models/gpt-4.1-mini/Dockerfile deleted file mode 100644 index 33af11d8..00000000 --- a/containers/models/gpt-4.1-mini/Dockerfile +++ /dev/null @@ -1,13 +0,0 @@ -ARG REGISTRY=ghcr.io/exgentic -ARG REGISTRY_SUFFIX=/ -FROM ${REGISTRY}/core${REGISTRY_SUFFIX}litellm:latest - -LABEL eval.type="model" -LABEL eval.model.name="gpt-4.1-mini" -LABEL eval.model.provider="azure" -LABEL eval.model.litellm_version="main-v1.83.3-stable" - -ENV EVAL_LITELLM_VERSION_DEFAULT="main-v1.83.3-stable" - -COPY config.yaml /app/config.yaml -CMD ["--port", "4000", "--config", "/app/config.yaml"] diff --git a/containers/models/gpt-4.1-mini/config.yaml b/containers/models/gpt-4.1-mini/config.yaml deleted file mode 100644 index c8c2eb7d..00000000 --- a/containers/models/gpt-4.1-mini/config.yaml +++ /dev/null @@ -1,9 +0,0 @@ -model_list: - - model_name: "*" - litellm_params: - model: "openai/Azure/gpt-4.1-mini" - api_key: "os.environ/OPENAI_API_KEY" - api_base: "os.environ/OPENAI_API_BASE" - -litellm_settings: - callbacks: ["eval_logger.eval_logger_instance"] diff --git a/containers/models/gpt-4.1-mini/docker-bake.hcl b/containers/models/gpt-4.1-mini/docker-bake.hcl deleted file mode 100644 index 559eef3c..00000000 --- a/containers/models/gpt-4.1-mini/docker-bake.hcl +++ /dev/null @@ -1,7 +0,0 @@ -target "model-gpt-4_1-mini" { - context = "containers/models/gpt-4.1-mini" - contexts = { - "${REGISTRY}/core/litellm" = "target:litellm" - } - tags = ["${REGISTRY}/models/gpt-4.1-mini:${TAG}"] -} diff --git a/containers/models/gpt-5.4/Dockerfile b/containers/models/gpt-5.4/Dockerfile deleted file mode 100644 index 6c4f1106..00000000 --- a/containers/models/gpt-5.4/Dockerfile +++ /dev/null @@ -1,13 +0,0 @@ -ARG REGISTRY=ghcr.io/exgentic -ARG REGISTRY_SUFFIX=/ -FROM ${REGISTRY}/core${REGISTRY_SUFFIX}litellm:latest - -LABEL eval.type="model" -LABEL eval.model.name="gpt-5.4" -LABEL eval.model.provider="azure" -LABEL eval.model.litellm_version="main-v1.83.3-stable" - -ENV EVAL_LITELLM_VERSION_DEFAULT="main-v1.83.3-stable" - -COPY config.yaml /app/config.yaml -CMD ["--port", "4000", "--config", "/app/config.yaml"] diff --git a/containers/models/gpt-5.4/config.yaml b/containers/models/gpt-5.4/config.yaml deleted file mode 100644 index 4bd6b721..00000000 --- a/containers/models/gpt-5.4/config.yaml +++ /dev/null @@ -1,97 +0,0 @@ -# Every model name an agent might request aliases to azure/gpt-5.4 on -# the IBM gateway. The wildcard `model_name: "*"` pattern works for the -# Anthropic /v1/messages path but NOT for OpenAI /v1/chat/completions -# (the `openai/` provider prefix passes the request body through -# unchanged, so the upstream receives the inbound model name). The -# explicit enumeration below is the reliable fix. Add a new alias -# when an agent shows up with a model name that isn't on the list. - -model_list: - # ── Wildcard catch-all (Anthropic path) ───────────────────────── - - model_name: "*" - litellm_params: - model: "openai/azure/gpt-5.4" - api_key: "os.environ/OPENAI_API_KEY" - api_base: "os.environ/OPENAI_API_BASE" - service_tier: "default" - model_info: - input_cost_per_token: 0.0000025 # $2.50 / 1M tokens - output_cost_per_token: 0.000010 # $10.00 / 1M tokens - - # ── OpenAI-family explicit aliases (chat completions path) ────── - - model_name: "gpt-4o" - litellm_params: &gpt54 - model: "openai/azure/gpt-5.4" - api_key: "os.environ/OPENAI_API_KEY" - api_base: "os.environ/OPENAI_API_BASE" - service_tier: "default" - model_info: &gpt54_info - input_cost_per_token: 0.0000025 # $2.50 / 1M tokens - output_cost_per_token: 0.000010 # $10.00 / 1M tokens - - model_name: "gpt-4o-mini" - litellm_params: *gpt54 - model_info: *gpt54_info - - model_name: "gpt-4-turbo" - litellm_params: *gpt54 - model_info: *gpt54_info - - model_name: "gpt-4" - litellm_params: *gpt54 - model_info: *gpt54_info - - model_name: "gpt-4.1" - litellm_params: *gpt54 - model_info: *gpt54_info - - model_name: "gpt-4.1-mini" - litellm_params: *gpt54 - model_info: *gpt54_info - - model_name: "gpt-5" - litellm_params: *gpt54 - model_info: *gpt54_info - - model_name: "gpt-5-mini" - litellm_params: *gpt54 - model_info: *gpt54_info - - model_name: "gpt-5.4" - litellm_params: *gpt54 - model_info: *gpt54_info - - model_name: "o1" - litellm_params: *gpt54 - model_info: *gpt54_info - - model_name: "o1-mini" - litellm_params: *gpt54 - model_info: *gpt54_info - - # ── Anthropic-family explicit aliases ────────────────────────── - - model_name: "claude-3-5-sonnet-20241022" - litellm_params: *gpt54 - model_info: *gpt54_info - - model_name: "claude-3-5-haiku-20241022" - litellm_params: *gpt54 - model_info: *gpt54_info - - model_name: "claude-3-opus-20240229" - litellm_params: *gpt54 - model_info: *gpt54_info - - model_name: "claude-sonnet-4-5" - litellm_params: *gpt54 - model_info: *gpt54_info - - model_name: "claude-sonnet-4-5-20250929" - litellm_params: *gpt54 - model_info: *gpt54_info - - model_name: "claude-opus-4-5" - litellm_params: *gpt54 - model_info: *gpt54_info - - model_name: "claude-opus-4-5-20250929" - litellm_params: *gpt54 - model_info: *gpt54_info - - # ── Gemini-family explicit aliases ───────────────────────────── - - model_name: "gemini-2.5-pro" - litellm_params: *gpt54 - model_info: *gpt54_info - - model_name: "gemini-1.5-pro" - litellm_params: *gpt54 - model_info: *gpt54_info - - model_name: "gemini-pro" - litellm_params: *gpt54 - model_info: *gpt54_info - -litellm_settings: - callbacks: ["eval_logger.eval_logger_instance"] diff --git a/containers/models/gpt-5.4/docker-bake.hcl b/containers/models/gpt-5.4/docker-bake.hcl deleted file mode 100644 index c34deef7..00000000 --- a/containers/models/gpt-5.4/docker-bake.hcl +++ /dev/null @@ -1,7 +0,0 @@ -target "model-gpt-5_4" { - context = "containers/models/gpt-5.4" - contexts = { - "${REGISTRY}/core/litellm" = "target:litellm" - } - tags = ["${REGISTRY}/models/gpt-5.4:${TAG}"] -} diff --git a/containers/models/gpt-5/Dockerfile b/containers/models/gpt-5/Dockerfile deleted file mode 100644 index e6d79cb0..00000000 --- a/containers/models/gpt-5/Dockerfile +++ /dev/null @@ -1,13 +0,0 @@ -ARG REGISTRY=ghcr.io/exgentic -ARG REGISTRY_SUFFIX=/ -FROM ${REGISTRY}/core${REGISTRY_SUFFIX}litellm:latest - -LABEL eval.type="model" -LABEL eval.model.name="gpt-5" -LABEL eval.model.provider="openai" -LABEL eval.model.litellm_version="main-v1.83.3-stable" - -ENV EVAL_LITELLM_VERSION_DEFAULT="main-v1.83.3-stable" - -COPY config.yaml /app/config.yaml -CMD ["--port", "4000", "--config", "/app/config.yaml"] diff --git a/containers/models/gpt-5/config.yaml b/containers/models/gpt-5/config.yaml deleted file mode 100644 index 7425091e..00000000 --- a/containers/models/gpt-5/config.yaml +++ /dev/null @@ -1,8 +0,0 @@ -model_list: - - model_name: "*" - litellm_params: - model: "openai/gpt-5" - api_key: "os.environ/OPENAI_API_KEY" - -litellm_settings: - callbacks: ["eval_logger.eval_logger_instance"] diff --git a/containers/models/gpt-5/docker-bake.hcl b/containers/models/gpt-5/docker-bake.hcl deleted file mode 100644 index 756cd186..00000000 --- a/containers/models/gpt-5/docker-bake.hcl +++ /dev/null @@ -1,7 +0,0 @@ -target "model-gpt-5" { - context = "containers/models/gpt-5" - contexts = { - "${REGISTRY}/core/litellm" = "target:litellm" - } - tags = ["${REGISTRY}/models/gpt-5:${TAG}"] -} diff --git a/deploy/oc/README.md b/deploy/oc/README.md index 32359c2c..b12a2d06 100644 --- a/deploy/oc/README.md +++ b/deploy/oc/README.md @@ -31,20 +31,24 @@ every `--builder oc` consumer, not just this script. ## Quickstart +`--model` is the `/` routing handle (`EVAL_MODEL`) — the shared +gateway routes any model at runtime, no per-model image exists. `--gateway` +picks the proxy flavor (`bifrost`, default; `litellm`, `portkey`). + ```bash # one dataset eval (50 examples, 8 at a time), watch run progress, then results -./oc/run.sh --benchmark aime --agent codex --model bifrost --dataset-size 50 --parallelism 8 --watch +./oc/run.sh --benchmark aime --agent codex --model openai/gpt-5.4 --dataset-size 50 --parallelism 8 --watch ./oc/status.sh --benchmark aime # run progress (Jobs) -./oc/fetch.sh --benchmark aime --agent codex --model bifrost +./oc/fetch.sh --benchmark aime --agent codex --model openai/gpt-5.4 eval-containers report output/ # PASS/FAIL, reward, tokens, cost, traces health # a grid -./oc/sweep.sh --dataset-size 50 --model bifrost +./oc/sweep.sh --dataset-size 50 --model openai/gpt-5.4 ./oc/status.sh --sweep-id ./oc/fetch.sh --sweep-id && eval-containers report output/ # single example, for debugging -./oc/run.sh --benchmark aime --agent codex --model bifrost --task 0 --watch +./oc/run.sh --benchmark aime --agent codex --model openai/gpt-5.4 --task 0 --watch ``` ## Concurrency: with vs without Kueue @@ -54,14 +58,14 @@ The per-example cap inside one run is always the Job's `parallelism`. The questi **Without Kueue** (default) — `parallelism` is a *per-sweep* cap. Simple, zero infra, but ten concurrent sweeps run up to `10 × parallelism` pods: no global ceiling, so a busy cluster oversubscribes and the scheduler thrashes. Fine for one sweep at a time or a small team. ```bash -./oc/sweep.sh --dataset-size 50 --model bifrost --parallelism 8 +./oc/sweep.sh --dataset-size 50 --model openai/gpt-5.4 --parallelism 8 ``` **With Kueue** (`--queue eval-queue`) — every Job starts `suspend: true` and joins a queue; the **ClusterQueue's quota is the single global budget**. Kueue admits pods up to quota and *queues the rest* — many sweeps share one budget instead of fighting. You get fair-sharing, borrowing, and no oversubscription, at the cost of installing the operator and an admin defining quotas once. ```bash oc apply -f deploy/kueue.yaml # one-time, admin: defines the global budget -./oc/sweep.sh --dataset-size 50 --model bifrost --queue eval-queue +./oc/sweep.sh --dataset-size 50 --model openai/gpt-5.4 --queue eval-queue ``` | | without Kueue | with Kueue | diff --git a/deploy/oc/fetch.sh b/deploy/oc/fetch.sh index 5ada6677..c1d7120b 100755 --- a/deploy/oc/fetch.sh +++ b/deploy/oc/fetch.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash # fetch.sh — `oc cp` eval output off the PVC (paths read from Job labels). # -# ./oc/fetch.sh --benchmark aime --agent codex --model bifrost # whole dataset +# ./oc/fetch.sh --benchmark aime --agent codex --model openai/gpt-5.4 # whole dataset # ./oc/fetch.sh --sweep-id # every Job in a sweep set -euo pipefail source "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/_lib.sh" @@ -35,6 +35,7 @@ if [[ -n "$SWEEP_ID" ]]; then else [[ -z "$BENCHMARK" || -z "$AGENT" || -z "$MODEL" ]] && { echo "error: --sweep-id, or --benchmark/--agent/--model, required" >&2; exit 1; } - copy "$BENCHMARK" "$AGENT" "$MODEL" + # Results are keyed by the clean model label (last handle segment). + copy "$BENCHMARK" "$AGENT" "${MODEL##*/}" fi log "done → $DEST_ROOT" diff --git a/deploy/oc/run.sh b/deploy/oc/run.sh index bd04a72f..5fc80580 100755 --- a/deploy/oc/run.sh +++ b/deploy/oc/run.sh @@ -2,21 +2,22 @@ # run.sh — build + run one eval on OpenShift: a single --task, or --dataset # (whole dataset → an Indexed Job). Model + flags: oc/README.md and the case below. # -# ./oc/run.sh --benchmark aime --agent codex --model bifrost --dataset -# ./oc/run.sh --benchmark aime --agent codex --model bifrost --task 0 # single, debug +# ./oc/run.sh --benchmark aime --agent codex --model openai/gpt-5.4 --dataset +# ./oc/run.sh --benchmark aime --agent codex --model openai/gpt-5.4 --task 0 # single, debug set -euo pipefail source "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/_lib.sh" -BENCHMARK="" AGENT="" MODEL="" TASK="0" DATASET="" PARALLELISM="" RETRY="" QUEUE="" -EVAL_MODEL="" NAMESPACE="$NS_DEFAULT" REGISTRY="" PVC="eval-output-pvc" SWEEP_ID="" SUFFIX="" +BENCHMARK="" AGENT="" MODEL="" GATEWAY="bifrost" TASK="0" DATASET="" PARALLELISM="" RETRY="" QUEUE="" +NAMESPACE="$NS_DEFAULT" REGISTRY="" PVC="eval-output-pvc" SWEEP_ID="" SUFFIX="" DATASET_MODE=false NO_BUILD=false NO_RUN=false REBUILD=false TEST=false RERUN=false WATCH=false DRY_RUN=false while [[ $# -gt 0 ]]; do case "$1" in --benchmark) BENCHMARK="$2"; shift 2;; --agent) AGENT="$2"; shift 2;; --model) MODEL="$2"; shift 2;; --task) TASK="$2"; shift 2;; + --gateway) GATEWAY="$2"; shift 2;; --dataset) DATASET_MODE=true; shift;; --dataset-size) DATASET="$2"; DATASET_MODE=true; shift 2;; --parallelism) PARALLELISM="$2"; shift 2;; --retry) RETRY="$2"; shift 2;; --queue) QUEUE="$2"; shift 2;; - --eval-model) EVAL_MODEL="$2"; shift 2;; --namespace) NAMESPACE="$2"; shift 2;; + --namespace) NAMESPACE="$2"; shift 2;; --registry) REGISTRY="$2"; shift 2;; --pvc) PVC="$2"; shift 2;; --repo-dir) REPO_DIR="$2"; shift 2;; --sweep-id) SWEEP_ID="$2"; shift 2;; --rebuild) REBUILD=true; shift;; --no-build) NO_BUILD=true; shift;; @@ -26,7 +27,9 @@ while [[ $# -gt 0 ]]; do case "$1" in *) echo "Unknown argument: $1" >&2; exit 1;; esac; done [[ -z "$BENCHMARK" || -z "$AGENT" || -z "$MODEL" ]] && { - echo "error: --benchmark, --agent and --model are required" >&2; exit 1; } + echo "error: --benchmark, --agent and --model (/) are required" >&2; exit 1; } +# The runner's clean model label (chart's eval.modelLabel): last handle segment. +MLABEL="${MODEL##*/}" log() { echo "[run] $*"; } [[ -z "$REGISTRY" ]] && REGISTRY="$(oc_registry "$NAMESPACE")" @@ -41,7 +44,7 @@ RESULT_PREFIX="runs${SUFFIX}" # successfulBuildsHistoryLimit on it so the controller GCs old build pods (and # their ConfigMaps) natively; no shell housekeeping needed here. if ! $NO_BUILD; then - log "=== build ($BENCHMARK / $AGENT / $MODEL) ===" + log "=== build ($BENCHMARK / $AGENT / $GATEWAY) ===" ISFLAG=(); [[ -n "$SUFFIX" ]] && ISFLAG=(--imagestream-suffix="$SUFFIX") build() { local label="$1" is="$2"; shift 2 $DRY_RUN && { echo "[dry-run] eval-containers build $* --builder oc ${ISFLAG[*]:-}"; return; } @@ -50,8 +53,8 @@ if ! $NO_BUILD; then ( cd "$REPO_DIR" build "bench" "$(flat "$BENCHMARK")$SUFFIX" bench "$BENCHMARK" build "agent" "$(flat "$AGENT")$SUFFIX" agent "$AGENT" - build "model" "$(flat "$MODEL")$SUFFIX" model "$MODEL" - build "eval" "$(flat "$BENCHMARK-$AGENT")$SUFFIX" eval "$BENCHMARK" --agent "$AGENT" --model "$MODEL" ) + build "model" "$(flat "$GATEWAY")$SUFFIX" model "$GATEWAY" + build "eval" "$(flat "$BENCHMARK-$AGENT")$SUFFIX" eval "$BENCHMARK" --agent "$AGENT" --model "$GATEWAY" ) fi $NO_RUN && { log "--no-run: built only, not submitting."; exit 0; } @@ -67,13 +70,12 @@ if $DATASET_MODE && [[ -z "$DATASET" ]] && ! $DRY_RUN; then log "dataset size for $BENCHMARK (from image label): $DATASET" fi -if [[ -n "$DATASET" ]]; then JOB="${BENCHMARK}-${AGENT}${SUFFIX}"; SUB="${RESULT_PREFIX}/${BENCHMARK}/${AGENT}/${MODEL}"; -else JOB="${BENCHMARK}-${AGENT}-task-${TASK}${SUFFIX}"; SUB="${RESULT_PREFIX}/${BENCHMARK}/${AGENT}/${MODEL}/${TASK}/${JOB}"; fi +if [[ -n "$DATASET" ]]; then JOB="${BENCHMARK}-${AGENT}${SUFFIX}"; SUB="${RESULT_PREFIX}/${BENCHMARK}/${AGENT}/${MLABEL}"; +else JOB="${BENCHMARK}-${AGENT}-task-${TASK}${SUFFIX}"; SUB="${RESULT_PREFIX}/${BENCHMARK}/${AGENT}/${MLABEL}/${TASK}/${JOB}"; fi -[[ -z "$EVAL_MODEL" ]] && EVAL_MODEL="openai/azure/$(echo "$MODEL" | sed 's/--bifrost//;s/--litellm//;s/--portkey//')" # flatImages=true → the chart composes flat ImageStream refs for the OC registry. SET=(--set "benchmark=$BENCHMARK" --set "agent=$AGENT" --set "task=$TASK" - --set "model=$MODEL" --set "gatewayImage=$MODEL" --set "evalModel=$EVAL_MODEL" + --set "model=$MODEL" --set "gatewayImage=$GATEWAY" --set "registry=$REGISTRY" --set "flatImages=true" --set "outputVolume.persistentVolumeClaim.claimName=$PVC" --set "outputSubPath=$SUB") [[ -n "$SUFFIX" ]] && SET+=(--set "imageSuffix=$SUFFIX" --set "nameSuffix=$SUFFIX") diff --git a/deploy/oc/sweep.sh b/deploy/oc/sweep.sh index 882e547b..961c446c 100755 --- a/deploy/oc/sweep.sh +++ b/deploy/oc/sweep.sh @@ -2,24 +2,25 @@ # sweep.sh — loop run.sh over a benchmark×agent grid, each cell a dataset Indexed # Job tagged sweep-id=. Flags: see the case block. Default grid: the *.txt. # -# ./oc/sweep.sh --model bifrost --benchmarks "aime gsm8k" # each auto-sized -# ./oc/sweep.sh --model bifrost --dataset-size 50 # uniform cap +# ./oc/sweep.sh --model openai/gpt-5.4 --benchmarks "aime gsm8k" # each auto-sized +# ./oc/sweep.sh --model openai/gpt-5.4 --dataset-size 50 # uniform cap set -euo pipefail source "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/_lib.sh" RUN="$(dirname "${BASH_SOURCE[0]}")/run.sh" -MODEL="" DATASET="" PARALLELISM="" RETRY="" QUEUE="" BSET="" ASET="" -EVAL_MODEL="" NAMESPACE="$NS_DEFAULT" PVC="eval-output-pvc" NO_BUILD=false DRY_RUN=false +MODEL="" GATEWAY="" DATASET="" PARALLELISM="" RETRY="" QUEUE="" BSET="" ASET="" +NAMESPACE="$NS_DEFAULT" PVC="eval-output-pvc" NO_BUILD=false DRY_RUN=false while [[ $# -gt 0 ]]; do case "$1" in --model) MODEL="$2"; shift 2;; --dataset-size) DATASET="$2"; shift 2;; + --gateway) GATEWAY="$2"; shift 2;; --parallelism) PARALLELISM="$2"; shift 2;; --retry) RETRY="$2"; shift 2;; --queue) QUEUE="$2"; shift 2;; --benchmarks) BSET="$2"; shift 2;; --agents) ASET="$2"; shift 2;; - --eval-model) EVAL_MODEL="$2"; shift 2;; --namespace) NAMESPACE="$2"; shift 2;; + --namespace) NAMESPACE="$2"; shift 2;; --pvc) PVC="$2"; shift 2;; --repo-dir) REPO_DIR="$2"; shift 2;; --no-build) NO_BUILD=true; shift;; --dry-run) DRY_RUN=true; shift;; *) echo "Unknown argument: $1" >&2; exit 1;; esac; done -[[ -z "$MODEL" ]] && { echo "error: --model is required" >&2; exit 1; } +[[ -z "$MODEL" ]] && { echo "error: --model (/) is required" >&2; exit 1; } log() { echo "[sweep] $*"; } read_list() { grep -v '^[[:space:]]*#' "$1" | grep -v '^[[:space:]]*$'; } @@ -28,7 +29,7 @@ else BENCHMARKS=(); while IFS= read -r l; do BENCHMARKS+=("$l"); done < <(read_l if [[ -n "$ASET" ]]; then read -ra AGENTS <<<"$ASET" else AGENTS=(); while IFS= read -r l; do AGENTS+=("$l"); done < <(read_list "$REPO_DIR/deploy/oc/agents.txt"); fi -SWEEP_ID="$(date -u +%Y%m%dT%H%M%S)--$(flat "$MODEL")" +SWEEP_ID="$(date -u +%Y%m%dT%H%M%S)--$(flat "${MODEL##*/}")" log "sweep-id: $SWEEP_ID grid: ${#BENCHMARKS[@]} benchmarks × ${#AGENTS[@]} agents${QUEUE:+ queue: $QUEUE}" PASS=(--model "$MODEL" --namespace "$NAMESPACE" --pvc "$PVC" --repo-dir "$REPO_DIR" --sweep-id "$SWEEP_ID") @@ -38,7 +39,7 @@ if [[ -n "$DATASET" ]]; then PASS+=(--dataset-size "$DATASET"); else PASS+=(--da [[ -n "$PARALLELISM" ]] && PASS+=(--parallelism "$PARALLELISM") [[ -n "$RETRY" ]] && PASS+=(--retry "$RETRY") [[ -n "$QUEUE" ]] && PASS+=(--queue "$QUEUE") -[[ -n "$EVAL_MODEL" ]] && PASS+=(--eval-model "$EVAL_MODEL") +[[ -n "$GATEWAY" ]] && PASS+=(--gateway "$GATEWAY") $NO_BUILD && PASS+=(--no-build) $DRY_RUN && PASS+=(--dry-run) diff --git a/deploy/oc/test.sh b/deploy/oc/test.sh index 5b702400..17b7806d 100755 --- a/deploy/oc/test.sh +++ b/deploy/oc/test.sh @@ -3,7 +3,7 @@ # then assert on the PVC output (result written, agent exit 0, gen_ai traces). # Exits non-zero on the first failed check. # -# ./oc/test.sh --benchmark aime --agent codex --model bifrost +# ./oc/test.sh --benchmark aime --agent codex --model openai/gpt-5.4 set -euo pipefail source "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/_lib.sh" RUN="$(dirname "${BASH_SOURCE[0]}")/run.sh" @@ -13,7 +13,7 @@ while [[ $# -gt 0 ]]; do case "$1" in --benchmark) BENCHMARK="$2"; shift 2;; --agent) AGENT="$2"; shift 2;; --model) MODEL="$2"; shift 2;; --task) TASK="$2"; shift 2;; --namespace) NAMESPACE="$2"; PASS_ARGS+=(--namespace "$2"); shift 2;; - --eval-model) PASS_ARGS+=(--eval-model "$2"); shift 2;; + --gateway) PASS_ARGS+=(--gateway "$2"); shift 2;; --pvc) PASS_ARGS+=(--pvc "$2"); shift 2;; --repo-dir) PASS_ARGS+=(--repo-dir "$2"); shift 2;; --rebuild) PASS_ARGS+=(--rebuild); shift;; @@ -22,13 +22,14 @@ while [[ $# -gt 0 ]]; do case "$1" in *) echo "Unknown argument: $1" >&2; exit 1;; esac; done [[ -z "$BENCHMARK" || -z "$AGENT" || -z "$MODEL" ]] && { - echo "error: --benchmark, --agent and --model are required" >&2; exit 1; } + echo "error: --benchmark, --agent and --model (/) are required" >&2; exit 1; } pass() { echo "[test] PASS: $*"; } fail() { echo "[test] FAIL: $*" >&2; exit 1; } # Isolated run: job --task-, results under runs/. +# Result paths use the clean model label (last handle segment), same as run.sh. JOB="${BENCHMARK}-${AGENT}-task-${TASK}${SUFFIX}" -RESULT="/data/runs${SUFFIX}/${BENCHMARK}/${AGENT}/${MODEL}/${TASK}/${JOB}" +RESULT="/data/runs${SUFFIX}/${BENCHMARK}/${AGENT}/${MODEL##*/}/${TASK}/${JOB}" read_file() { oc exec eval-reader -n "$NAMESPACE" -- cat "$1" 2>/dev/null || true; } echo "[test] running $BENCHMARK/$AGENT/$MODEL task=$TASK (isolated $SUFFIX) …" diff --git a/docs/concepts/overview.md b/docs/concepts/overview.md index 46f26396..4c982613 100644 --- a/docs/concepts/overview.md +++ b/docs/concepts/overview.md @@ -50,7 +50,8 @@ Every image has a reproducible default and two independent version knobs: - **Container version** — *which image to pull* — set by the image **tag** (`EVAL_BENCHMARK_TAG`, `EVAL_AGENT_TAG`, `EVAL_MODEL_TAG`). - **Internal version** — *what runs inside* — set at runtime - (`EVAL_BENCHMARK_VERSION`, `EVAL_AGENT_VERSION`, `EVAL_LITELLM_VERSION`). + (`EVAL_BENCHMARK_VERSION`, `EVAL_AGENT_VERSION`; the model gateway's proxy + version is pinned at build time instead). Casual users never touch these; power users pin. Full list in [Environment variables](../reference/env-vars.md). diff --git a/docs/guides/add-a-model.md b/docs/guides/add-a-model.md index 808730cf..89592a52 100644 --- a/docs/guides/add-a-model.md +++ b/docs/guides/add-a-model.md @@ -1,16 +1,15 @@ -# Use (or build) a model +# Use (or add) a model *Guide · for everyone · the canonical rules are [`.agents/models/RULES.md`](../../.agents/models/RULES.md).* -The model is a **runtime** axis, and **using one never requires a build**. Two -paths — the generic gateway is the default; pinned per-model images are an -opt-in. You only build when you *author* one. +The model is a **runtime** axis: one shared gateway image routes any model, and +**using a model never requires a build**. There are no per-model images — the +only build case is authoring a new gateway *flavor*. -## Generic gateway (default) — any model, zero build +## Any model, zero build -The default gateway routes whatever `EVAL_MODEL=/` you set to -that provider, so any [LiteLLM-supported model](https://docs.litellm.ai/docs/providers) -works with no per-model image: +The shared gateway routes whatever `EVAL_MODEL=/` you set to +that provider, so any model works with no new image: ```bash echo "OPENAI_API_KEY=sk-..." > .env @@ -18,35 +17,51 @@ eval-containers run aime --task-id 0 --agent codex --model openai/gpt-5.4 # or --model anthropic/claude-sonnet-4-5, gemini/gemini-2.5-pro, openai/azure/, … ``` -`EVAL_MODEL` must be `/` form — the generic gateway errors on a -bare name or an empty value (no silent default). Pick the proxy backend with -`EVAL_GATEWAY_IMAGE` (default `bifrost`; `litellm` and `portkey` also ship). +`EVAL_MODEL` must be `/` form — the gateway errors on a +bare name or an empty value (no silent default). Pick the proxy flavor with +`EVAL_GATEWAY_IMAGE` (default `bifrost`; `litellm` and `portkey` also ship) — +the flavor changes *how* requests are proxied, never *which* model runs. -## Pinned per-model image — a shared, custom artifact (still zero build) +The gateway holds the real provider key; the runner only ever sees the proxy — +see [Isolation & gateways](../concepts/isolation-and-gateways.md). -A per-model image bakes one model plus its config (cost rates, endpoint, -params). It's a **named, versioned artifact** teams share — "run it against -`models/gpt-5.4`" gives everyone the exact same pinned setup, which is the point -for cross-team reproducibility and per-model customization. Use a published one -with no build and no `EVAL_MODEL` (the model is baked): +## Custom gateway config (advanced, non-default) + +If your team needs a shared, custom-configured gateway — a fixed corporate +endpoint, custom cost rates, an alias table — that is a **downstream artifact +you own**, not a framework image (models rule 1a). Two equivalent styles, both +selected through the standard seams: ```bash -EVAL_GATEWAY_IMAGE=gpt-5.4 eval-containers run aime --task-id 0 --agent codex +# Style A — mount your template on the published gateway image +docker run -e EVAL_MODEL=/ -e \ + -v ./config.yaml.template:/opt/gateway/config.yaml.template \ + ghcr.io/exgentic/gateways/litellm:latest ``` -The gateway holds the real provider key; the runner only ever sees the proxy — -see [Isolation & gateways](../concepts/isolation-and-gateways.md). +```dockerfile +# Style B — bake it in YOUR registry: template-only over a published flavor +FROM ghcr.io/exgentic/gateways/litellm:latest +COPY config.yaml.template /opt/gateway/config.yaml.template +``` + +Publish style B under your own registry (e.g. `ghcr.io//models/`) +and select it with `EVAL_REGISTRY=ghcr.io/ EVAL_GATEWAY_IMAGE=` +(compose) or `--set gatewayImageRef=` (k8s). The image must stay +template-only — same routing as the bare gateway with the template mounted. +Custom images never live in this repo or its registry: the framework's default +stays one shared gateway per flavor. -## Build a model image (the only build case) +## Add a gateway flavor (the only build case) -You build + publish a `containers/models/` image only to **author** one — -either a new pinned per-model artifact, or a new generic backend beside -`bifrost`/`litellm`/`portkey`. Then: +You build + publish an image only to **author a new gateway flavor** beside +`bifrost`/`litellm`/`portkey` — never for a model. Then: -1. **Read the rules** — [`.agents/models/RULES.md`](../../.agents/models/RULES.md) - (rule 1: the default is a generic runtime gateway; a pinned image is a - deliberate option; rules 4–7: key isolation + tamper-proof logging). -2. **Honor the version axes** — pin a reproducible LiteLLM version, expose - `EVAL_LITELLM_VERSION` (see [Environment variables](../reference/env-vars.md)). +1. **Read the rules** — [`.agents/gateways/RULES.md`](../../.agents/gateways/RULES.md) + (provider-agnostic, `EVAL_MODEL` selected at runtime) and + [`.agents/models/RULES.md`](../../.agents/models/RULES.md) + (rule 1: no per-model images; rules 4–7: key isolation + tamper-proof logging). +2. **Ship the pair** — the implementation at `containers/gateways//` and + its thin combo at `containers/models//` (gateways rules 16–17). 3. **Open the PR** with the [model PR template](../../.github/PULL_REQUEST_TEMPLATE/model.md). diff --git a/docs/reference/chart-values.md b/docs/reference/chart-values.md index cb72506f..f0846a5a 100644 --- a/docs/reference/chart-values.md +++ b/docs/reference/chart-values.md @@ -27,7 +27,6 @@ in the chart. Per-run axes arrive via `--set` (or the CLI). See | `runnerTag` | `latest` | `--agent-tag` / `--benchmark-tag` | | `benchmarkVersion` | `""` | `--benchmark-version` | | `agentVersion` | `""` | `--agent-version` | -| `litellmVersion` | `""` | `--litellm-version` | | `maxBudget` | `""` | `--max-budget` | | `reasoningEffort` | `""` | `--agent-reasoning-effort` | diff --git a/docs/reference/cli.md b/docs/reference/cli.md index f4355afa..5501f16f 100644 --- a/docs/reference/cli.md +++ b/docs/reference/cli.md @@ -51,7 +51,6 @@ overrides the env var. | `--model-tag ` | `EVAL_MODEL_TAG` | image tag | | `--benchmark-version ` | `EVAL_BENCHMARK_VERSION` | dataset revision inside the image | | `--agent-version ` | `EVAL_AGENT_VERSION` | upstream CLI version inside the image | -| `--litellm-version ` | `EVAL_LITELLM_VERSION` | LiteLLM version inside the image | | `--timeout ` | `EVAL_TIMEOUT` | default `300` | | `--max-budget ` | `EVAL_MODEL_MAX_BUDGET` | hard spend cap; default `$1` | | `--local` | — | use in-repo `containers/benchmarks//` instead of the registry | diff --git a/docs/reference/env-vars.md b/docs/reference/env-vars.md index 51a75209..4493b526 100644 --- a/docs/reference/env-vars.md +++ b/docs/reference/env-vars.md @@ -15,13 +15,12 @@ env var. | `EVAL_AGENT` | Which agent to run | — | | `EVAL_MODEL` | LiteLLM handle `/` the gateway routes to (e.g. `openai/gpt-5.4`) — **required**, must be `/` form | — | | `EVAL_TASK_ID` | Which task within the benchmark | `0` | -| `EVAL_GATEWAY_IMAGE` | Which proxy backend serves the model | `bifrost` | +| `EVAL_GATEWAY_IMAGE` | Which gateway flavor proxies the model | `bifrost` | -`EVAL_MODEL` is a *runtime handle, not an image*: any LiteLLM-supported -provider/model works with no per-model build — the generic gateway -(`EVAL_GATEWAY_IMAGE`, default `bifrost`; also `litellm`, `portkey`) routes it. -Or set `EVAL_GATEWAY_IMAGE` to a **pinned per-model image** (e.g. `gpt-5.4`) — a -baked, shared artifact that ignores `EVAL_MODEL`. Both are pull-not-build; see +`EVAL_MODEL` is a *runtime handle, not an image*: any provider/model works with +no build — the shared gateway (`EVAL_GATEWAY_IMAGE`, default `bifrost`; also +`litellm`, `portkey`) routes it. The flavor changes how requests are proxied, +never which model runs; there are no per-model images. See [Use a model](../guides/add-a-model.md). ## Container versions — *which image tag to pull* @@ -38,7 +37,10 @@ baked, shared artifact that ignores `EVAL_MODEL`. Both are pull-not-build; see |---|---|---| | `EVAL_BENCHMARK_VERSION` | Dataset revision inside the benchmark | built-in pin | | `EVAL_AGENT_VERSION` | Upstream CLI version inside the agent | built-in pin | -| `EVAL_LITELLM_VERSION` | LiteLLM version inside the model | built-in pin | + +Model images have no internal-version variable: the gateway proxy version is +pinned at build time and recorded in the image's `gateway._version` +label. ## Runtime diff --git a/tests/build/RULES.md b/tests/build/RULES.md index bf20648b..0908d412 100644 --- a/tests/build/RULES.md +++ b/tests/build/RULES.md @@ -20,7 +20,7 @@ Parent: [../RULES.md](../RULES.md) fails with `no space left on device`. 3. **Core images bootstrap first.** Benchmarks `COPY --from=` the core - images (`core/entrypoint`, `core/test-exact-match`, `core/litellm`, + images (`core/entrypoint`, `core/test-exact-match`, `core/llm-bridge`). The sweep MUST build these first, otherwise every benchmark build fails at its first `COPY --from=` step. diff --git a/tests/build/test.rs b/tests/build/test.rs index cf7780f9..c53afa26 100644 --- a/tests/build/test.rs +++ b/tests/build/test.rs @@ -859,7 +859,7 @@ fn dockerfile_bake_alignment() { let art = dir.file_name().and_then(|s| s.to_str()).unwrap_or(""); let expected_target = match cat { // Leaf core images: bare name. Other categories: - - // (gateways included — a bare `litellm` collides with core/litellm). + // (gateways included — a bare `litellm` collides with models/litellm). "core" => art.to_string(), "agents" => format!("agent-{art}"), "benchmarks" => format!("benchmark-{art}"), diff --git a/tests/run/replay/RULES.md b/tests/run/replay/RULES.md index e4e07b94..d7840c1f 100644 --- a/tests/run/replay/RULES.md +++ b/tests/run/replay/RULES.md @@ -82,7 +82,7 @@ Parent: [../RULES.md](../RULES.md) ## Core image dependency 10. **Core images MUST be available.** Replay's `ensure_images()` MUST - rebuild `core/entrypoint`, `core/test-exact-match`, `core/litellm`, - and `models/replay` before any replay test runs. The build sweep's + rebuild `core/entrypoint`, `core/test-exact-match`, and + `models/replay` before any replay test runs. The build sweep's `ImageGuard::Drop` deletes them after a prior sweep, so replay cannot assume they exist. diff --git a/tests/static/check.rs b/tests/static/check.rs index 8352a746..72978d95 100644 --- a/tests/static/check.rs +++ b/tests/static/check.rs @@ -464,14 +464,14 @@ fn otelcol_health_gate_is_consistent_across_modes() { eprintln!("✓ otelcol health gate consistent across all three modes (#45)"); } -/// The model axis supports BOTH paths, with the generic gateway as the default -/// (models/RULES.md rule 1, #187): -/// - **Generic** (default): `EVAL_GATEWAY_IMAGE=bifrost` routes whatever -/// `EVAL_MODEL=/` you set — any LiteLLM model, zero build. -/// The generic image errors on an empty handle, so there is no compose-level -/// default model (no silent fallback). -/// - **Pinned** (opt-in): a per-model image (`EVAL_GATEWAY_IMAGE=`) bakes -/// its model + custom config — a shared, versioned artifact. Allowed, not forbidden. +/// The model axis has exactly ONE image kind: the shared generic gateway +/// (models/RULES.md rule 1). `EVAL_MODEL=/` selects the model +/// at runtime — any model, zero build. `EVAL_GATEWAY_IMAGE` selects only the +/// gateway *flavor* (bifrost default; litellm/portkey/replay). Per-model images +/// were removed: every `containers/models/` must be a thin combo over its +/// `gateways/` image, with `replay` (a recorded-trajectory server, rule +/// 17) the single sanctioned exception. The generic image errors on an empty +/// handle, so there is no compose-level default model (no silent fallback). #[test] fn model_axis_generic_default_no_silent_model() { let svc = fs::read_to_string(repo_root().join("containers/compose/services.yaml")) @@ -484,7 +484,7 @@ fn model_axis_generic_default_no_silent_model() { ); // No silent fallback model: the compose never bakes a default EVAL_MODEL — an // unset handle surfaces as the generic gateway's own startup error, never a - // stale default route. (Pinned per-model images bake their model and ignore it.) + // stale default route. assert!( !svc.contains("${EVAL_MODEL:-"), "services.yaml must not default EVAL_MODEL to a baked handle — no silent fallback model (#187)" @@ -496,13 +496,37 @@ fn model_axis_generic_default_no_silent_model() { "services.yaml gateway must pass EVAL_MODEL through so it can pin the model (#269)" ); - // Both paths exist. The generic gateways are present… + // The generic gateways are present… for g in ["bifrost", "litellm", "portkey"] { assert!( repo_root().join("containers/models").join(g).is_dir(), "generic gateway containers/models/{g} must exist (#187)" ); } + // …and they are the ONLY kind: every models/ image is a thin combo + // over its gateways/ image. A Dockerfile FROMing anything else is a + // per-model image sneaking back in (models/RULES.md rule 1; `replay` is the + // single sanctioned pinned exception, rule 17). + for entry in + fs::read_dir(repo_root().join("containers/models")).expect("missing containers/models") + { + let dir = entry.expect("readable containers/models entry").path(); + if !dir.is_dir() { + continue; + } + let name = dir.file_name().unwrap().to_string_lossy().into_owned(); + if name == "replay" { + continue; + } + let dockerfile = fs::read_to_string(dir.join("Dockerfile")) + .unwrap_or_else(|_| panic!("missing containers/models/{name}/Dockerfile")); + assert!( + dockerfile.contains("/gateways${REGISTRY_SUFFIX}"), + "containers/models/{name} must be FROM a gateways/ image — \ + per-model images were removed; models are selected at runtime via \ + EVAL_MODEL (models/RULES.md rule 1)" + ); + } // …and the k8s chart likewise carries no hardcoded default handle. let vals = fs::read_to_string(repo_root().join("containers/benchmarks/_chart/values.yaml")) .expect("missing _chart/values.yaml"); @@ -511,9 +535,7 @@ fn model_axis_generic_default_no_silent_model() { "_chart/values.yaml must ship `model: \"\"` — the routing handle, no hardcoded default (#187)" ); - eprintln!( - "✓ generic gateway is the default; no silent fallback model; per-model images allowed (#187)" - ); + eprintln!("✓ one shared gateway per flavor; no silent fallback model; no per-model images"); } /// The stitched eval image must launch the pipeline (rule 12): the combination diff --git a/tests/static/model-paths.sweep.sh b/tests/static/model-paths.sweep.sh index a7132f1a..db7b1cfc 100755 --- a/tests/static/model-paths.sweep.sh +++ b/tests/static/model-paths.sweep.sh @@ -1,16 +1,17 @@ #!/usr/bin/env bash -# tests/static/model-paths.sweep.sh — assert BOTH model paths wire correctly via -# `docker compose config` (#187). The model axis has two supported paths; this gate -# proves each renders the right gateway with the right behaviour: +# tests/static/model-paths.sweep.sh — assert the single model path wires correctly +# via `docker compose config`. The model axis has ONE image kind: the shared +# generic gateway. `EVAL_MODEL=/` selects the model at runtime; +# `EVAL_GATEWAY_IMAGE` selects only the gateway *flavor* (bifrost default; +# litellm/portkey/replay). Per-model images were removed — no image bakes a model. # -# 1. GENERIC (default): EVAL_MODEL=/ through the default `bifrost` -# gateway → the gateway image is models/bifrost and the handle reaches it. -# 2. PINNED per-model image: EVAL_GATEWAY_IMAGE= with NO EVAL_MODEL → the -# gateway image is models/ and the stack STILL LOADS — the pinned image -# bakes its model, so it needs no handle and no compose-level require blocks it. +# 1. DEFAULT: EVAL_MODEL through the default gateway → the image is +# models/bifrost and the handle reaches it. +# 2. FLAVOR: EVAL_GATEWAY_IMAGE=litellm swaps the flavor only — the image is +# models/litellm and the SAME handle still reaches it. # # `config` is a client-side parse: no daemon, images, or creds. The gateway wiring is -# shared (compose/services.yaml), so one benchmark exercises both paths. OPENAI_API_* +# shared (compose/services.yaml), so one benchmark exercises both cases. OPENAI_API_* # are dummies (services.yaml marks them required); --env-file /dev/null ignores .env. set -uo pipefail ROOT=$(CDPATH='' cd -- "$(dirname -- "$0")/../.." && pwd) || exit 2 @@ -21,26 +22,28 @@ docker compose version >/dev/null 2>&1 || { echo "docker compose plugin not foun C="$ROOT/containers/benchmarks/aime/compose.yaml" fail=0 -# 1. GENERIC (default gateway) + a / handle. +# 1. DEFAULT gateway + a / handle. if out=$(OPENAI_API_KEY=x OPENAI_API_BASE=x EVAL_MODEL=openai/gpt-5.4 \ docker compose --env-file /dev/null -f "$C" config 2>&1); then grep -qE 'image:.*/models/bifrost:' <<<"$out" \ - || { echo "FAIL generic: default gateway is not models/bifrost"; fail=$((fail + 1)); } + || { echo "FAIL default: default gateway is not models/bifrost"; fail=$((fail + 1)); } grep -qE 'EVAL_MODEL:[[:space:]]*openai/gpt-5\.4' <<<"$out" \ - || { echo "FAIL generic: the EVAL_MODEL handle did not reach the gateway"; fail=$((fail + 1)); } + || { echo "FAIL default: the EVAL_MODEL handle did not reach the gateway"; fail=$((fail + 1)); } else - echo "FAIL generic: docker compose config failed:"; printf '%s\n' "$out" | sed 's/^/ /'; fail=$((fail + 1)) + echo "FAIL default: docker compose config failed:"; printf '%s\n' "$out" | sed 's/^/ /'; fail=$((fail + 1)) fi -# 2. PINNED per-model image, NO EVAL_MODEL — must still load (the model is baked). -if out=$(OPENAI_API_KEY=x OPENAI_API_BASE=x EVAL_GATEWAY_IMAGE=gpt-5.4 \ +# 2. FLAVOR swap: EVAL_GATEWAY_IMAGE picks a gateway flavor, never a model — +# the handle still comes from EVAL_MODEL. +if out=$(OPENAI_API_KEY=x OPENAI_API_BASE=x EVAL_MODEL=openai/gpt-5.4 EVAL_GATEWAY_IMAGE=litellm \ docker compose --env-file /dev/null -f "$C" config 2>&1); then - grep -qE 'image:.*/models/gpt-5\.4:' <<<"$out" \ - || { echo "FAIL pinned: gateway is not the pinned models/gpt-5.4"; fail=$((fail + 1)); } + grep -qE 'image:.*/models/litellm:' <<<"$out" \ + || { echo "FAIL flavor: gateway is not models/litellm"; fail=$((fail + 1)); } + grep -qE 'EVAL_MODEL:[[:space:]]*openai/gpt-5\.4' <<<"$out" \ + || { echo "FAIL flavor: the EVAL_MODEL handle did not reach the gateway"; fail=$((fail + 1)); } else - echo "FAIL pinned: a pinned per-model image must load with no EVAL_MODEL, but config failed:" - printf '%s\n' "$out" | sed 's/^/ /'; fail=$((fail + 1)) + echo "FAIL flavor: docker compose config failed:"; printf '%s\n' "$out" | sed 's/^/ /'; fail=$((fail + 1)) fi -echo "model paths: generic (default, handle) + pinned (per-model, no handle) — $fail failed" +echo "model paths: one shared gateway — default flavor + flavor swap, handle always runtime — $fail failed" [ "$fail" -eq 0 ] diff --git a/tests/static/policy/dockerfile/hygiene.rego b/tests/static/policy/dockerfile/hygiene.rego index 617a37f1..f9898036 100644 --- a/tests/static/policy/dockerfile/hygiene.rego +++ b/tests/static/policy/dockerfile/hygiene.rego @@ -18,7 +18,7 @@ # here): missing_dock_type, untagged_from, unpinned_pip, unpinned_npm, # from_arg_not_global, hardcoded_secret, label_dir_mismatch, todo_string_literal, # install_order_pip_before_apt, phantom_pip_uninstall, upstream_base_unpinned, -# agent_missing_version_arg, model_missing_litellm_version_*. +# agent_missing_version_arg. # # PARITY NOTE (heredocs): buildkit drops heredoc *bodies* from the parse, so the # RUN-scanning rules below see only what Rust sees after its strip_heredocs() — diff --git a/tests/static/policy/dockerfile/inspection.rego b/tests/static/policy/dockerfile/inspection.rego index 65b63c15..d97e6449 100644 --- a/tests/static/policy/dockerfile/inspection.rego +++ b/tests/static/policy/dockerfile/inspection.rego @@ -14,8 +14,6 @@ # - todo_string_literal a non-comment instruction with a quoted "TODO" / 'TODO' # - from_arg_not_global a FROM interpolates an ARG not declared in the global scope # - agent_missing_version_arg agent image with no ARG AGENT_VERSION -# - model_missing_litellm_version_label model image with no LABEL eval.model.litellm_version -# - model_missing_litellm_version_default model image with no ENV EVAL_LITELLM_VERSION_DEFAULT # warn (yellow): # - upstream_base_unpinned eval.benchmark.upstream_base ends :latest, or has no tag/digest # - phantom_pip_uninstall a RUN does `pip uninstall` with no `pip install` on it @@ -84,8 +82,6 @@ deny contains msg if { # ── type classification (eval.type label, reusing main.label_value) ── is_agent if main.label_value("eval.type") == `"agent"` -is_model if main.label_value("eval.type") == `"model"` - # ── missing_dock_type (red) ───────────────────────────────────────── # A benchmarks/agents/models Dockerfile MUST declare `LABEL eval.type=`. (Rust: # `!t.contains("LABEL eval.type=")`, scoped to those three roots — gateways carry @@ -216,42 +212,6 @@ deny contains msg if { msg := "agent Dockerfile is missing `ARG AGENT_VERSION` (dockerfile_inspection agent_missing_version_arg, RULES.md 9)" } -# ── model_missing_litellm_version_{label,default} (red, type-gated) ─ -# A model image that actually wraps the LiteLLM proxy MUST record the version on -# both axes: LABEL eval.model.litellm_version= and ENV EVAL_LITELLM_VERSION_DEFAULT=. -# Exempt: models/replay (its own minimal server, no litellm) and gateway-flavor -# models (LABEL gateway.kind=, thin wrappers over the gateways). Type-gated by -# is_model. -is_replay_model if data.params.dir == "replay" - -is_gateway_flavor_model if main.label_keys["gateway.kind"] - -litellm_model if { - is_model - not is_replay_model - not is_gateway_flavor_model -} - -# ENV keys present anywhere (ENV packs as flat (key, value, "=") triples like LABEL). -env_keys contains key if { - some instr in input - instr.Cmd == "env" - some idx, key in instr.Value - idx % 3 == 0 -} - -deny contains msg if { - litellm_model - not main.label_keys["eval.model.litellm_version"] - msg := "model Dockerfile is missing LABEL eval.model.litellm_version (dockerfile_inspection model_missing_litellm_version_label)" -} - -deny contains msg if { - litellm_model - not env_keys["EVAL_LITELLM_VERSION_DEFAULT"] - msg := "model Dockerfile is missing ENV EVAL_LITELLM_VERSION_DEFAULT (dockerfile_inspection model_missing_litellm_version_default)" -} - # ── upstream_base_unpinned (yellow) ───────────────────────────────── # `LABEL eval.benchmark.upstream_base` ending in `:latest`, or with no tag/digest at # all, is supply-chain debt (benchmarks/RULES.md 21b). Only benchmarks carry the diff --git a/tests/static/policy/dockerfile/inspection_test.rego b/tests/static/policy/dockerfile/inspection_test.rego index 0c066731..98b6f014 100644 --- a/tests/static/policy/dockerfile/inspection_test.rego +++ b/tests/static/policy/dockerfile/inspection_test.rego @@ -248,56 +248,6 @@ test_agent_version_arg_not_required_for_benchmark if { with data.params.category as "benchmarks" } -# ════════════════════════════════════════════════════════════════════ -# model_missing_litellm_version_{label,default} (red, type-gated) -# ════════════════════════════════════════════════════════════════════ - -complete_litellm_model := [ - from("alpine:3"), - label("eval.type", `"model"`), - label("eval.model.litellm_version", `"main-v1.83.3-stable"`), - env("EVAL_LITELLM_VERSION_DEFAULT", "main-v1.83.3-stable"), -] - -test_model_complete_ok if { - count(deny) == 0 with input as complete_litellm_model with data.params.dir as "gpt-5" -} - -test_model_missing_litellm_label_fires if { - bad := [ - from("alpine:3"), - label("eval.type", `"model"`), - env("EVAL_LITELLM_VERSION_DEFAULT", "main-v1.83.3-stable"), - ] - count(deny) == 1 with input as bad with data.params.dir as "gpt-5" -} - -test_model_missing_litellm_default_fires if { - bad := [ - from("alpine:3"), - label("eval.type", `"model"`), - label("eval.model.litellm_version", `"main-v1.83.3-stable"`), - ] - count(deny) == 1 with input as bad with data.params.dir as "gpt-5" -} - -test_model_missing_both_fires_twice if { - bad := [from("alpine:3"), label("eval.type", `"model"`)] - count(deny) == 2 with input as bad with data.params.dir as "gpt-5" -} - -# replay is the in-repo stub — exempt from both litellm requirements. -test_model_replay_exempt if { - replay := [from("alpine:3"), label("eval.type", `"model"`)] - count(deny) == 0 with input as replay with data.params.dir as "replay" -} - -# Gateway-flavor models (LABEL gateway.kind=) are thin wrappers — exempt. -test_model_gateway_flavor_exempt if { - gw := [from("alpine:3"), label("eval.type", `"model"`), label("gateway.kind", `"litellm"`)] - count(deny) == 0 with input as gw with data.params.dir as "litellm" -} - # ════════════════════════════════════════════════════════════════════ # upstream_base_unpinned (yellow) # ════════════════════════════════════════════════════════════════════