feat(models)!: remove per-model gateway images — one shared gateway, models picked at runtime (#294)

.env.example

@@ -10,14 +10,15 @@ OPENAI_API_KEY=sk-...
 # For local development: localhost:5000
 EVAL_REGISTRY=ghcr.io/eval-containers
 
-# Default agent and model (can be overridden per run)
+# Default agent and model (can be overridden per run). EVAL_MODEL is the
+# <provider>/<model> handle the shared gateway routes to at runtime.
 EVAL_AGENT=claude-code
-EVAL_MODEL=claude-sonnet-4
+EVAL_MODEL=anthropic/claude-sonnet-4-5
 
 # Default timeout in seconds (can be overridden per run or per benchmark)
 EVAL_TIMEOUT=300
 
-# Hard cap on model spend in USD per run. The litellm proxy enforces
+# Hard cap on model spend in USD per run. The gateway enforces
 # it and crashes the run (BudgetExceededError) once crossed. Default $1
 # is strict on purpose — bump after observing real runs. Overridable
 # via `eval-containers run --max-budget <N>`.

.github/.trivyignore

@@ -106,7 +106,7 @@ CVE-2026-45447
 CVE-2026-44432
 
 # --- undici in node tooling: npm bundle (no reachable fix) ------------------
-# core/agent-base-node and core/litellm both carry npm's bundled undici 6.26.0 /
+# core/agent-base-node carries npm's bundled undici 6.26.0 /
 # 6.25.0. Fixed in undici 6.27.0, but `npm@latest` IS 11.17.0 (still bundles
 # 6.26.0 — verified), so no npm bump reaches it. This is build-time npm tooling:
 # the agents' runtime HTTP uses node's OWN bundled undici, and the eval data path

.github/ISSUE_TEMPLATE/05-new-agent-request.md

@@ -43,7 +43,7 @@ specifically. -->
 ## Endpoint compatibility
 
 <!-- If this agent uses OpenAI's Responses API (`/v1/responses`),
-`core/litellm` must be pinned to v1.63.8 or newer. Check the
+`gateways/litellm` must be pinned to v1.63.8 or newer. Check the
 current pin before opening the PR. -->
 
 ## Known obstacles

.github/ISSUE_TEMPLATE/06-new-model-request.md

@@ -7,12 +7,11 @@ labels: ["new-model"]
 
 <!--
 The model is a RUNTIME axis: set `EVAL_MODEL=<provider>/<model>` and put the
-provider key in `.env` — any LiteLLM-supported model works with no image, no
-build, no request. See docs/guides/add-a-model.md.
+provider key in `.env` — any model works with no image, no build, no request.
+There are no per-model images. See docs/guides/add-a-model.md.
 
-Open this issue only if that DOESN'T work — e.g. the pinned LiteLLM version
-doesn't support the provider yet, or it needs a new gateway backend beside
-bifrost / litellm / portkey.
+Open this issue only if that DOESN'T work — e.g. it needs a new gateway
+flavor beside bifrost / litellm / portkey.
 -->
 
 ## Model: `<provider>/<model>`
@@ -30,9 +29,7 @@ bifrost / litellm / portkey.
 
 ## What's missing
 
-- [ ] LiteLLM doesn't support this provider in the pinned `core/litellm` version — link a version bump
-- [ ] Needs a new gateway backend (the existing bifrost / litellm / portkey can't reach it) — describe why
-- [ ] Want a **pinned per-model image** (a shared, custom-configured artifact teams run against via `EVAL_GATEWAY_IMAGE=<name>`) — not just runtime `EVAL_MODEL`
+- [ ] Needs a new gateway flavor (the existing bifrost / litellm / portkey can't reach it) — describe why
 - [ ] It works already; requesting it be added to the docs / examples
 
 ## Who implements

.github/PULL_REQUEST_TEMPLATE.md

@@ -37,7 +37,7 @@ For PRs that modify existing code, fill in the sections below.
 - [ ] `cargo test` passes (sanity gates: check, compose, dockerfile_inspection, task_inspection, upstream unit tests)
 - [ ] If the change touches a Dockerfile or compose file: one affected benchmark/agent builds locally (`docker build` or `eval-containers build bench <name>`)
 - [ ] If the change touches mechanical rule catalogs: every new rule has a unit test for the positive and negative case
-- [ ] If the change touches shared infrastructure (core/entrypoint, core/litellm, core/combination.Dockerfile): one smoke run against `aime` or `mmlu` via the live driver passes
+- [ ] If the change touches shared infrastructure (core/entrypoint, gateways/, core/combination.Dockerfile): one smoke run against `aime` or `mmlu` via the live driver passes
 
 ## RULES.md impact
 

.github/PULL_REQUEST_TEMPLATE/agent.md

@@ -71,7 +71,7 @@ eval-containers run aime --agent <name> --model openai/gpt-5.4 --task-id 0 --loc
 
 - [ ] `output/aime/0/model/trajectory.jsonl` non-empty, has real LLM calls
 - [ ] `output/aime/0/task/result.json` has a valid reward (0, 1, or fractional)
-- [ ] `output/aime/0/model/result.json` has `cost_usd > 0` — the proxy logged the call (if 0, investigate: your agent's SDK path may not trigger the logging callback, see `core/litellm/eval_logger.py`)
+- [ ] `output/aime/0/model/result.json` has `cost_usd > 0` — the proxy logged the call (if 0, investigate: your agent's SDK path may not trigger the logging callback, see `gateways/litellm/eval_logger.py`)
 
 Benchmark 2 (recommend `humaneval` for code generation or `gsm8k` for tool-less reasoning):
 
@@ -97,7 +97,7 @@ Our proxy exposes `/v1/messages` (Anthropic) and `/v1/chat/completions` (OpenAI)
 
 - [ ] My agent uses `/v1/messages` (Anthropic SDK)
 - [ ] My agent uses `/v1/chat/completions` (OpenAI Chat Completions)
-- [ ] My agent uses `/v1/responses` (OpenAI Responses API) — requires `core/litellm` pinned to v1.63.8+
+- [ ] My agent uses `/v1/responses` (OpenAI Responses API) — requires the `gateways/litellm` pin at v1.63.8+
 - [ ] My agent uses a different endpoint (specify): <!-- ... --> — document which LiteLLM version is required
 
 ### Known limitations

.github/PULL_REQUEST_TEMPLATE/model.md

@@ -1,46 +1,41 @@
 <!--
-Adding a MODEL? You almost certainly don't need a PR. The model is a RUNTIME
-axis: set EVAL_MODEL=<provider>/<model> (any LiteLLM-supported model) + the
-provider key in .env — no image, no build. See docs/guides/add-a-model.md.
-
-Open THIS PR to author a MODEL IMAGE under containers/models/<name>/ — either:
-  - a PINNED per-model artifact (bakes one model + its config; a shared,
-    versioned reference teams run against via EVAL_GATEWAY_IMAGE=<name>), or
-  - a new GENERIC backend (routes EVAL_MODEL at runtime, beside bifrost/litellm/portkey).
+Adding a MODEL? You don't need a PR. The model is a RUNTIME axis: set
+EVAL_MODEL=<provider>/<model> + the provider key in .env — no image, no build.
+There are no per-model images. See docs/guides/add-a-model.md.
+
+Open THIS PR only to author a new GATEWAY FLAVOR (routes EVAL_MODEL at
+runtime, beside bifrost/litellm/portkey): the implementation under
+containers/gateways/<flavor>/ + its thin combo under containers/models/<flavor>/.
 Fill every checkbox; reviewers reject on an empty evidence section.
 -->
 
-## Model image: `<name>`
-
-- [ ] **Kind**: pinned per-model (bakes `<provider>/<model>`) **/** generic backend (routes `EVAL_MODEL`)
+## Gateway flavor: `<name>`
 
-<!-- One paragraph: the model or proxy, and why it earns a shared image — team
-reproducibility, custom cost/endpoint/params, protocol coverage, or a provider
-the generic backends can't reach. -->
+<!-- One paragraph: the proxy, and why it earns a shared image — protocol
+coverage, observability, or a provider the existing flavors can't reach. -->
 
-### Contract ([.agents/models/RULES.md](../../.agents/models/RULES.md))
+### Contract ([.agents/gateways/RULES.md](../../.agents/gateways/RULES.md), [.agents/models/RULES.md](../../.agents/models/RULES.md))
 
-- [ ] `FROM ghcr.io/exgentic/gateways/<proxy>:latest` or `core/litellm:latest` (inherits the eval-logger + budget wrapper)
-- [ ] Routing matches the kind — a generic backend routes `EVAL_MODEL` via a wildcard (rules 1–2); a pinned image bakes one `<provider>/<model>`
-- [ ] Labels `eval.type="model"` (+ `gateway.kind` / `eval.model.*` per rule 15)
-- [ ] Provider keys via `os.environ/<PROVIDER>_API_KEY` only — none in labels / compose / the agent (rules 4–5)
-- [ ] Logs every request + response; the agent cannot reach `/output/model/` (rules 6–7)
-- [ ] `EVAL_MODEL_MAX_BUDGET` hard cap enforced (rule 16)
-- [ ] LiteLLM/proxy version pinned as a reproducible default; `EVAL_LITELLM_VERSION` honored (rules 12–13)
+- [ ] `containers/models/<name>/Dockerfile` is `FROM ghcr.io/exgentic/gateways/<name>:latest` and adds nothing but the config template (gateways rule 16)
+- [ ] Routes `EVAL_MODEL` at runtime via a wildcard — no baked provider, model, or URL (gateways rules 1–2a; models rules 1–2)
+- [ ] Labels `eval.type="model"` + `gateway.kind="<name>"` (models rule 15)
+- [ ] Provider keys via provider-native env vars only — none in labels / compose / the agent (models rules 4–5)
+- [ ] Logs every request + response; the agent cannot reach `/output/model/` (models rules 6–7)
+- [ ] `EVAL_MODEL_MAX_BUDGET` hard cap enforced (models rule 16)
+- [ ] Proxy version pinned at build time; `gateway.<name>_version` label set (models rule 12; gateways rule 14)
 
 ### Evidence: a real run
 
 ```bash
-# generic backend: EVAL_GATEWAY_IMAGE=<name> + --model <provider>/<model>
-# pinned per-model: EVAL_GATEWAY_IMAGE=<name>   (model is baked; no --model)
 EVAL_GATEWAY_IMAGE=<name> \
-  eval-containers run aime --agent claude-code --task-id 0 --local --max-budget 1
+  eval-containers run aime --agent claude-code --task-id 0 --local \
+  --model <provider>/<model> --max-budget 1
 ```
 
 - [ ] `output/aime/0/model/trajectory.jsonl` non-empty; `result.json` `cost_usd > 0`
-- [ ] (generic) swapping `--model` to another `<provider>/<model>` routes to the new provider with **no rebuild**
+- [ ] Swapping `--model` to another `<provider>/<model>` routes to the new provider with **no rebuild**
 
 ### Docs + changelog
 
 - [ ] [`docs/guides/add-a-model.md`](../../docs/guides/add-a-model.md) and any affected page updated ([.agents/docs/RULES.md](../../.agents/docs/RULES.md) rule 15)
-- [ ] `.agents/models/RULES.md` changelog entry if rules changed; otherwise "no RULES.md changes needed"
+- [ ] `.agents/models/RULES.md` / `.agents/gateways/RULES.md` changelog entry if rules changed; otherwise "no RULES.md changes needed"

.github/workflows/test.yml

@@ -144,7 +144,7 @@ jobs:
         if: ${{ !cancelled() }}
         run: tests/static/helm.sweep.sh
 
-      - name: model paths — generic gateway + pinned per-model image
+      - name: model paths — one shared gateway, model picked at runtime
         if: ${{ !cancelled() }}
         run: tests/static/model-paths.sweep.sh
 

cli/README.md

@@ -16,7 +16,7 @@ evaluation by hand.
 cargo install eval-containers
 
 # Run one evaluation (prints the plain `docker compose` command it stands for)
-eval-containers run aime --task-id 0 --agent codex --model gpt-5.4
+eval-containers run aime --task-id 0 --agent codex --model openai/gpt-5.4
 ```
 
 See the [repository README](https://github.com/Exgentic/eval-containers) and the

cli/src/gen_bake.rs

@@ -176,7 +176,7 @@ mod tests {
     #[test]
     fn gateway_target_is_category_prefixed() {
         // Gateways follow <category>-<name> like agents/benchmarks, NOT a bare
-        // name — a bare `litellm` would collide with core/litellm's target when
+        // name — a bare `litellm` would collide with models/litellm's target when
         // both bake files load in one invocation (RULES.md principle 15.a).
         let out = render("gateways", "litellm", &[], false);
         assert!(out.contains("target \"gateway-litellm\""));

cli/src/naming.rs

@@ -30,7 +30,8 @@ pub fn agent_image(registry: &str, agent: &str, tag: &str) -> String {
     format!("{registry}/agents/{agent}:{tag}")
 }
 
-/// `{registry}/models/<model>:<tag>` — the per-model gateway image.
+/// `{registry}/models/<name>:<tag>` — the shared model-gateway image
+/// (a gateway flavor: bifrost/litellm/portkey, or replay).
 pub fn model_image(registry: &str, model: &str, tag: &str) -> String {
     format!("{registry}/models/{model}:{tag}")
 }

cli/src/run.rs

@@ -100,7 +100,7 @@ pub struct RunArgs {
     model_tag: Option<String>,
 
     // NOTE: upstream versions (benchmark dataset revision, agent CLI version,
-    // litellm version) are a BUILD-time axis (RULES.md principle 9): pinned via
+    // gateway version) are a BUILD-time axis (RULES.md principle 9): pinned via
     // `ARG *_VERSION` in each image and overridden at `build` time, not here.
     // There is no runtime override — the running version is whatever the image
     // was built with, recorded in its label.
@@ -109,7 +109,7 @@ pub struct RunArgs {
     timeout: Option<u32>,
 
     /// Hard cap on model spend in USD for this run (maps to
-    /// $EVAL_MODEL_MAX_BUDGET). The litellm proxy enforces it and
+    /// $EVAL_MODEL_MAX_BUDGET). The gateway enforces it and
     /// returns an error once spend crosses the cap, which crashes
     /// the agent's next request. Default: $1.
     #[arg(long)]

containers/benchmarks/_chart/templates/job.yaml

@@ -107,9 +107,6 @@ spec:
             {{- with $v.maxBudget }}
             - { name: EVAL_MODEL_MAX_BUDGET, value: {{ . | quote }} }
             {{- end }}
-            {{- with $v.litellmVersion }}
-            - { name: EVAL_LITELLM_VERSION, value: {{ . | quote }} }
-            {{- end }}
             {{- with $v.gatewayExtraEnv }}
             {{- toYaml . | nindent 12 }}
             {{- end }}

containers/benchmarks/_chart/values.yaml

@@ -14,10 +14,10 @@ task: "0"
 # rejected when perTask is true. (benchmarks/RULES.md — eval-image naming.)
 perTask: false
 registry: ghcr.io/exgentic
-# `gatewayImage` picks the proxy: a generic gateway (bifrost/litellm/portkey)
-# routes `model`=<provider>/<model> at runtime; a pinned image bakes its own.
-# REQUIRED, no default — the gateway errors on an empty handle. The runner's clean
-# label is derived from it (last path segment).
+# `gatewayImage` picks the proxy flavor (bifrost/litellm/portkey — or replay);
+# the gateway routes `model`=<provider>/<model> at runtime. `model` is REQUIRED,
+# no default — the gateway errors on an empty handle. The runner's clean label
+# is derived from it (last path segment).
 gatewayImage: bifrost
 model: ""
 # Agent-side reasoning effort (e.g. high); empty → the agent's default.
@@ -50,7 +50,6 @@ nameSuffix: ""
 # Internal upstream version overrides (CLI --*-version, --max-budget); empty = unset
 benchmarkVersion: ""
 agentVersion: ""
-litellmVersion: ""
 maxBudget: ""
 
 # ── Knobs a benchmark MAY override in its values.yaml ────────────────────────

containers/compose/services.yaml

@@ -27,8 +27,8 @@ services:
     environment:
       OPENAI_API_KEY: ${OPENAI_API_KEY:?}
       OPENAI_API_BASE: ${OPENAI_API_BASE:?}
-      # The model authority: a generic gateway pins to this handle. Unset ->
-      # passthrough (client model); a pinned per-model image bakes its own.
+      # The model authority: the gateway pins every request to this handle.
+      # Unset -> passthrough (client model; recording only, not evals).
       EVAL_MODEL: ${EVAL_MODEL}
       HOST: 0.0.0.0
       # Both gateway flavors emit OTel: bifrost via its native `otel`