diff --git a/CHANGELOG.md b/CHANGELOG.md index 1a72d646..e72445f7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -42,6 +42,14 @@ addition; the patch on a bug fix that doesn't change the rule surface. ### Changed +- **The k8s `job` mode is now a self-contained Helm chart.** A benchmark + is selected with `--set benchmark=` instead of + `-f benchmarks//values.yaml`; the 4 benchmarks with bespoke topology + (`osworld`, `tau-bench`, `visualwebarena`, `webarena`) moved into the + chart as `benchmarks/_chart/presets/.yaml` (loaded via `.Files.Get`), + and the 98 one-line `values.yaml` files were deleted. The chart now + renders with no external file, so it can be packaged and published to an + OCI registry. Renders byte-identical to the prior `-f values.yaml` form. - **Agent Dockerfiles: 1957 → 585 lines (70% reduction)** across all 20 agents via the Rule 11 refactor onto shared bases. - **91 of 100 benchmarks** refactored to extend `core/benchmark-base-*`. diff --git a/README.md b/README.md index f6139920..3bbc422b 100644 --- a/README.md +++ b/README.md @@ -77,14 +77,14 @@ eval-containers run aime --task-id 0 --agent codex --mode job |---|---|---| | `compose` *(default)* | `docker compose -f benchmarks//compose.yaml up` | Local laptop, full stack with gateway + OTel sidecars, fastest iteration. | | `container` | `docker run -e EVAL_MODEL=... ` | CI smoke tests, one-shot runs against an existing model proxy, minimal footprint. | -| `job` | `helm template benchmarks/_chart -f benchmarks//values.yaml \| kubectl apply -f -` | Kubernetes clusters. Production-scale regressions (1000s of tasks in parallel). | +| `job` | `helm template benchmarks/_chart --set benchmark= \| kubectl apply -f -` | Kubernetes clusters. Production-scale regressions (1000s of tasks in parallel). | ### Kubernetes (`--mode job`) -Every benchmark is a small [Helm](https://helm.sh/) `values.yaml` over one shared chart (`benchmarks/_chart`) — render it and apply, no CLI needed: +Every benchmark renders from one shared [Helm](https://helm.sh/) chart (`benchmarks/_chart`) — select it with `--set benchmark=` and apply, no CLI needed. A benchmark with bespoke topology (extra Deployments/sidecars) adds a `presets/.yaml` inside the chart; standard ones need nothing: ```bash -helm template aime benchmarks/_chart -f benchmarks/aime/values.yaml \ +helm template aime benchmarks/_chart --set benchmark=aime \ --set agent=claude-code,task=0 | kubectl apply -f - ``` @@ -92,7 +92,7 @@ The CLI does exactly that, mapping every axis to a `--set`: ```bash eval-containers run aime --agent codex --task-id 42 --mode job -# → helm template aime-codex-task-42 benchmarks/_chart -f benchmarks/aime/values.yaml \ +# → helm template aime-codex-task-42 benchmarks/_chart --set benchmark=aime \ # --set registry=…,agent=codex,task=42 | kubectl apply -f - ``` @@ -102,7 +102,7 @@ Platform specifics (corp registry, NodeAffinity, NetworkPolicies, a different se eval-containers run aime --agent codex --mode job \ --overlay deploy/values-openshift.yaml \ --registry image-registry.openshift-image-registry.svc:5000/ -# → helm template … -f benchmarks/aime/values.yaml -f deploy/values-openshift.yaml … | kubectl apply -f - +# → helm template … --set benchmark=aime -f deploy/values-openshift.yaml … | kubectl apply -f - ``` On OpenShift, create the service account once and use `oc` in place of `kubectl`: diff --git a/benchmarks/osworld/values.yaml b/benchmarks/_chart/presets/osworld.yaml similarity index 98% rename from benchmarks/osworld/values.yaml rename to benchmarks/_chart/presets/osworld.yaml index 6d31ed69..dd574f4a 100644 --- a/benchmarks/osworld/values.yaml +++ b/benchmarks/_chart/presets/osworld.yaml @@ -1,4 +1,3 @@ -benchmark: osworld activeDeadlineSeconds: 2100 timeout: '1800' resources: diff --git a/benchmarks/tau-bench/values.yaml b/benchmarks/_chart/presets/tau-bench.yaml similarity index 99% rename from benchmarks/tau-bench/values.yaml rename to benchmarks/_chart/presets/tau-bench.yaml index c22f848f..d10ab04e 100644 --- a/benchmarks/tau-bench/values.yaml +++ b/benchmarks/_chart/presets/tau-bench.yaml @@ -1,4 +1,3 @@ -benchmark: tau-bench activeDeadlineSeconds: 2100 timeout: '1800' runnerArgs: python3 /app/agent.py; rc=$?; /usr/local/bin/reap-sidecars; exit $rc diff --git a/benchmarks/visualwebarena/values.yaml b/benchmarks/_chart/presets/visualwebarena.yaml similarity index 99% rename from benchmarks/visualwebarena/values.yaml rename to benchmarks/_chart/presets/visualwebarena.yaml index f49afd4a..cd78eb3f 100644 --- a/benchmarks/visualwebarena/values.yaml +++ b/benchmarks/_chart/presets/visualwebarena.yaml @@ -1,4 +1,3 @@ -benchmark: visualwebarena activeDeadlineSeconds: 2100 timeout: '1800' resources: diff --git a/benchmarks/webarena/values.yaml b/benchmarks/_chart/presets/webarena.yaml similarity index 99% rename from benchmarks/webarena/values.yaml rename to benchmarks/_chart/presets/webarena.yaml index a2d9bc64..a1330443 100644 --- a/benchmarks/webarena/values.yaml +++ b/benchmarks/_chart/presets/webarena.yaml @@ -1,4 +1,3 @@ -benchmark: webarena activeDeadlineSeconds: 2100 timeout: '1800' resources: diff --git a/benchmarks/_chart/templates/_helpers.tpl b/benchmarks/_chart/templates/_helpers.tpl index 00754e18..3a6d526e 100644 --- a/benchmarks/_chart/templates/_helpers.tpl +++ b/benchmarks/_chart/templates/_helpers.tpl @@ -1,6 +1,22 @@ +{{/* +Effective values = chart defaults / --set overrides (.Values) with the selected +benchmark's preset overlaid. The benchmark is named via `--set benchmark=`; +its bespoke topology (sidecars, resources, extra manifests) lives in +`presets/.yaml` inside the chart, loaded here so a `helm template` of the +packaged chart needs no external file. Standard benchmarks have no preset — +`.Files.Get` returns "" → empty overlay → the chart defaults apply unchanged. +Presets only set structural keys; the per-run axes (agent/task/model/…) come +from --set and are never in a preset, so preset-wins is safe. +*/}} +{{- define "eval.values" -}} +{{- $name := required "benchmark is required (--set benchmark=)" .Values.benchmark -}} +{{- $preset := .Files.Get (printf "presets/%s.yaml" $name) | fromYaml | default dict -}} +{{- mergeOverwrite (deepCopy .Values) $preset | toYaml -}} +{{- end -}} + {{/* Shared labels: benchmark/agent/task, plus sweep-id only when set. */}} {{- define "eval.labels" -}} -benchmark: {{ required "benchmark is required (set it in benchmarks//values.yaml)" .Values.benchmark }} +benchmark: {{ required "benchmark is required (--set benchmark=)" .Values.benchmark }} agent: {{ .Values.agent }} task: {{ .Values.task | quote }} {{- with .Values.sweepId }} diff --git a/benchmarks/_chart/templates/job.yaml b/benchmarks/_chart/templates/job.yaml index 98bbcef0..748458a6 100644 --- a/benchmarks/_chart/templates/job.yaml +++ b/benchmarks/_chart/templates/job.yaml @@ -1,13 +1,14 @@ +{{- $v := include "eval.values" . | fromYaml -}} apiVersion: batch/v1 kind: Job metadata: - name: {{ .Values.benchmark }}-{{ .Values.agent }}-task-{{ .Values.task }} + name: {{ $v.benchmark }}-{{ $v.agent }}-task-{{ $v.task }} labels: {{- include "eval.labels" . | nindent 4 }} spec: backoffLimit: 0 ttlSecondsAfterFinished: 3600 - activeDeadlineSeconds: {{ .Values.activeDeadlineSeconds }} + activeDeadlineSeconds: {{ $v.activeDeadlineSeconds }} template: metadata: labels: @@ -15,16 +16,16 @@ spec: spec: restartPolicy: Never shareProcessNamespace: true - {{- with .Values.serviceAccountName }} + {{- with $v.serviceAccountName }} serviceAccountName: {{ . }} {{- end }} - {{- with .Values.initContainers }} + {{- with $v.initContainers }} initContainers: {{- toYaml . | nindent 8 }} {{- end }} containers: - name: otelcol - image: {{ .Values.registry }}/core/otel:latest + image: {{ $v.registry }}/core/otel:latest imagePullPolicy: IfNotPresent ports: - { name: otlp-grpc, containerPort: 4317 } @@ -34,7 +35,7 @@ spec: - { name: output, mountPath: /output } - name: gateway - image: {{ .Values.registry }}/models/{{ .Values.gatewayImage }}:{{ .Values.gatewayTag }} + image: {{ $v.registry }}/models/{{ $v.gatewayImage }}:{{ $v.gatewayTag }} imagePullPolicy: IfNotPresent ports: - { name: gateway, containerPort: 4000 } @@ -45,44 +46,44 @@ spec: - name: OPENAI_API_BASE valueFrom: secretKeyRef: { name: eval-secrets, key: OPENAI_API_BASE } - - { name: EVAL_MODEL, value: {{ .Values.evalModel | quote }} } + - { name: EVAL_MODEL, value: {{ $v.evalModel | quote }} } - { name: HOST, value: "127.0.0.1" } # shareProcessNamespace + sidecar pod model puts otelcol on localhost. - { name: OTEL_EXPORTER_OTLP_ENDPOINT, value: "http://localhost:4318" } - {{- with .Values.maxBudget }} + {{- with $v.maxBudget }} - { name: EVAL_MODEL_MAX_BUDGET, value: {{ . | quote }} } {{- end }} - {{- with .Values.litellmVersion }} + {{- with $v.litellmVersion }} - { name: EVAL_LITELLM_VERSION, value: {{ . | quote }} } {{- end }} - {{- with .Values.gatewayExtraEnv }} + {{- with $v.gatewayExtraEnv }} {{- toYaml . | nindent 12 }} {{- end }} - name: runner - image: {{ .Values.registry }}/evals/{{ .Values.benchmark }}--{{ .Values.agent }}:{{ .Values.runnerTag }} + image: {{ $v.registry }}/evals/{{ $v.benchmark }}--{{ $v.agent }}:{{ $v.runnerTag }} imagePullPolicy: IfNotPresent command: ["/bin/bash", "-c"] # $? / $rc are the container's shell — Helm leaves them untouched # (they're not {{`{{ }}`}} actions). A few benchmarks override the # command (e.g. tau-bench runs python3 /app/agent.py). - args: [{{ .Values.runnerArgs | quote }}] + args: [{{ $v.runnerArgs | quote }}] env: # Framework axes in BOTH namespaces, mirroring compose/services.yaml: # EVAL_* is canonical (read by /eval-entrypoint.sh + per-benchmark # /entrypoint.sh — incl. the task materialization that sets $TASK); # the bare names are the process-compose legacy path. Set both so the # run works regardless of which entrypoint the eval image uses. - - { name: EVAL_BENCHMARK, value: {{ .Values.benchmark | quote }} } - - { name: EVAL_AGENT, value: {{ .Values.agent | quote }} } - - { name: EVAL_MODEL, value: {{ .Values.model | quote }} } - - { name: EVAL_TASK_ID, value: {{ .Values.task | quote }} } - - { name: EVAL_TIMEOUT, value: {{ .Values.timeout | quote }} } - - { name: BENCHMARK, value: {{ .Values.benchmark | quote }} } - - { name: AGENT, value: {{ .Values.agent | quote }} } - - { name: MODEL, value: {{ .Values.model | quote }} } - - { name: TASK_ID, value: {{ .Values.task | quote }} } - - { name: TIMEOUT, value: {{ .Values.timeout | quote }} } + - { name: EVAL_BENCHMARK, value: {{ $v.benchmark | quote }} } + - { name: EVAL_AGENT, value: {{ $v.agent | quote }} } + - { name: EVAL_MODEL, value: {{ $v.model | quote }} } + - { name: EVAL_TASK_ID, value: {{ $v.task | quote }} } + - { name: EVAL_TIMEOUT, value: {{ $v.timeout | quote }} } + - { name: BENCHMARK, value: {{ $v.benchmark | quote }} } + - { name: AGENT, value: {{ $v.agent | quote }} } + - { name: MODEL, value: {{ $v.model | quote }} } + - { name: TASK_ID, value: {{ $v.task | quote }} } + - { name: TIMEOUT, value: {{ $v.timeout | quote }} } - { name: ANTHROPIC_BASE_URL, value: "http://localhost:4000/anthropic" } - { name: OPENAI_BASE_URL, value: "http://localhost:4000/openai/v1" } - { name: GOOGLE_GEMINI_BASE_URL, value: "http://localhost:4000/genai" } @@ -90,22 +91,22 @@ spec: - { name: OPENAI_API_KEY, value: "sk-proxy" } - { name: GEMINI_API_KEY, value: "sk-proxy" } - { name: OTEL_EXPORTER_OTLP_ENDPOINT, value: "http://localhost:4318" } - {{- with .Values.benchmarkVersion }} + {{- with $v.benchmarkVersion }} - { name: EVAL_BENCHMARK_VERSION, value: {{ . | quote }} } {{- end }} - {{- with .Values.agentVersion }} + {{- with $v.agentVersion }} - { name: EVAL_AGENT_VERSION, value: {{ . | quote }} } {{- end }} - {{- with .Values.runnerExtraEnv }} + {{- with $v.runnerExtraEnv }} {{- toYaml . | nindent 12 }} {{- end }} resources: - {{- toYaml .Values.resources | nindent 12 }} + {{- toYaml $v.resources | nindent 12 }} volumeMounts: - { name: output, mountPath: /output } - { name: tmp, mountPath: /tmp } - { name: logs, mountPath: /logs } - {{- with .Values.runnerExtraVolumeMounts }} + {{- with $v.runnerExtraVolumeMounts }} {{- toYaml . | nindent 12 }} {{- end }} @@ -113,10 +114,10 @@ spec: - { name: output, emptyDir: {} } - { name: tmp, emptyDir: { medium: Memory, sizeLimit: 1Gi } } - { name: logs, emptyDir: { medium: Memory, sizeLimit: 100Mi } } - {{- with .Values.extraVolumes }} + {{- with $v.extraVolumes }} {{- toYaml . | nindent 8 }} {{- end }} -{{- range .Values.extraManifests }} +{{- range $v.extraManifests }} --- {{ toYaml . }} {{- end }} diff --git a/benchmarks/_chart/values.yaml b/benchmarks/_chart/values.yaml index 306b29ad..ede5db4e 100644 --- a/benchmarks/_chart/values.yaml +++ b/benchmarks/_chart/values.yaml @@ -1,7 +1,8 @@ -# Defaults for the shared eval Job. A benchmark's values.yaml sets `benchmark` -# (required) and overrides only what differs; the per-run axes (agent, task, -# model, …) come from `--set` (or the eval-containers CLI). envsubst is gone: -# Helm fills these and gets the types right (task stays a quoted string). +# Defaults for the shared eval Job. The benchmark is selected with +# `--set benchmark=` (required); a benchmark with bespoke topology overrides +# what differs via an optional `presets/.yaml` in this chart. The per-run axes +# (agent, task, model, …) come from `--set` (or the eval-containers CLI). envsubst +# is gone: Helm fills these and gets the types right (task stays a quoted string). # ── Per-run axes (overridden at deploy via --set / the eval-containers CLI) ── agent: claude-code diff --git a/benchmarks/acpbench/values.yaml b/benchmarks/acpbench/values.yaml deleted file mode 100644 index 8c340565..00000000 --- a/benchmarks/acpbench/values.yaml +++ /dev/null @@ -1 +0,0 @@ -benchmark: acpbench diff --git a/benchmarks/advbench/values.yaml b/benchmarks/advbench/values.yaml deleted file mode 100644 index 7a32fcd0..00000000 --- a/benchmarks/advbench/values.yaml +++ /dev/null @@ -1 +0,0 @@ -benchmark: advbench diff --git a/benchmarks/agentbench/values.yaml b/benchmarks/agentbench/values.yaml deleted file mode 100644 index ca5e0b94..00000000 --- a/benchmarks/agentbench/values.yaml +++ /dev/null @@ -1 +0,0 @@ -benchmark: agentbench diff --git a/benchmarks/agentcompany/values.yaml b/benchmarks/agentcompany/values.yaml deleted file mode 100644 index dc1520b5..00000000 --- a/benchmarks/agentcompany/values.yaml +++ /dev/null @@ -1 +0,0 @@ -benchmark: agentcompany diff --git a/benchmarks/agentdojo/values.yaml b/benchmarks/agentdojo/values.yaml deleted file mode 100644 index 2ed32b72..00000000 --- a/benchmarks/agentdojo/values.yaml +++ /dev/null @@ -1 +0,0 @@ -benchmark: agentdojo diff --git a/benchmarks/agentharm/values.yaml b/benchmarks/agentharm/values.yaml deleted file mode 100644 index ae400fe7..00000000 --- a/benchmarks/agentharm/values.yaml +++ /dev/null @@ -1 +0,0 @@ -benchmark: agentharm diff --git a/benchmarks/agents-smoke/values.yaml b/benchmarks/agents-smoke/values.yaml deleted file mode 100644 index 36bc849f..00000000 --- a/benchmarks/agents-smoke/values.yaml +++ /dev/null @@ -1 +0,0 @@ -benchmark: agents-smoke diff --git a/benchmarks/ai2d/values.yaml b/benchmarks/ai2d/values.yaml deleted file mode 100644 index 938b5453..00000000 --- a/benchmarks/ai2d/values.yaml +++ /dev/null @@ -1 +0,0 @@ -benchmark: ai2d diff --git a/benchmarks/aider-polyglot/values.yaml b/benchmarks/aider-polyglot/values.yaml deleted file mode 100644 index 11b62b6a..00000000 --- a/benchmarks/aider-polyglot/values.yaml +++ /dev/null @@ -1 +0,0 @@ -benchmark: aider-polyglot diff --git a/benchmarks/aime/README.md b/benchmarks/aime/README.md index 6ed532fa..cf1fb2e4 100644 --- a/benchmarks/aime/README.md +++ b/benchmarks/aime/README.md @@ -33,7 +33,7 @@ Uses the shared `core/test-exact-match` scorer: the agent's stdout is compared a - `Dockerfile` — builds the benchmark base image (tasks data + verifier). - `container.Dockerfile` — single-mode deployment artifact (1-line registry pin). - `compose.yaml` — compose-mode deployment artifact (`include:` shared base + aime overrides). -- `values.yaml` — k8s-mode deployment artifact (Helm values over the shared `benchmarks/_chart`). +- k8s — the shared chart `benchmarks/_chart`, selected with `--set benchmark=aime` (no per-benchmark file; aime has no bespoke topology). - `README.md` — this file. ## Running — three deployment surfaces @@ -42,7 +42,7 @@ Uses the shared `core/test-exact-match` scorer: the agent's stdout is compared a |------|------|------------| | **single** | `container.Dockerfile` | `docker run -e OPENAI_API_KEY=… -e OPENAI_API_BASE=… ` | | **compose** | `compose.yaml` | `docker compose -f benchmarks/aime/compose.yaml up` | -| **k8s** | `values.yaml` | `helm template aime benchmarks/_chart -f benchmarks/aime/values.yaml \| kubectl apply -f -` (needs `eval-secrets`) | +| **k8s** | shared chart | `helm template aime benchmarks/_chart --set benchmark=aime \| kubectl apply -f -` (needs `eval-secrets`) | ```bash # Single mode — just docker run @@ -60,7 +60,7 @@ OPENAI_API_KEY=… OPENAI_API_BASE=… \ kubectl create secret generic eval-secrets \ --from-literal=OPENAI_API_KEY="$OPENAI_API_KEY" \ --from-literal=OPENAI_API_BASE="$OPENAI_API_BASE" -helm template aime benchmarks/_chart -f benchmarks/aime/values.yaml | kubectl apply -f - +helm template aime benchmarks/_chart --set benchmark=aime | kubectl apply -f - ``` ## Different task @@ -74,7 +74,7 @@ TASK_ID=42 docker compose -f benchmarks/aime/compose.yaml up For k8s, the task is a Helm value: ```bash -helm template aime benchmarks/_chart -f benchmarks/aime/values.yaml --set task=42 | kubectl apply -f - +helm template aime benchmarks/_chart --set benchmark=aime --set task=42 | kubectl apply -f - ``` ## Build args diff --git a/benchmarks/aime/values.yaml b/benchmarks/aime/values.yaml deleted file mode 100644 index 9aee9fc3..00000000 --- a/benchmarks/aime/values.yaml +++ /dev/null @@ -1 +0,0 @@ -benchmark: aime diff --git a/benchmarks/alpaca-eval/values.yaml b/benchmarks/alpaca-eval/values.yaml deleted file mode 100644 index 940b1145..00000000 --- a/benchmarks/alpaca-eval/values.yaml +++ /dev/null @@ -1 +0,0 @@ -benchmark: alpaca-eval diff --git a/benchmarks/apps/values.yaml b/benchmarks/apps/values.yaml deleted file mode 100644 index 40fb96ac..00000000 --- a/benchmarks/apps/values.yaml +++ /dev/null @@ -1 +0,0 @@ -benchmark: apps diff --git a/benchmarks/appworld/values.yaml b/benchmarks/appworld/values.yaml deleted file mode 100644 index 2f469437..00000000 --- a/benchmarks/appworld/values.yaml +++ /dev/null @@ -1 +0,0 @@ -benchmark: appworld diff --git a/benchmarks/arc-agi/values.yaml b/benchmarks/arc-agi/values.yaml deleted file mode 100644 index d9f88b54..00000000 --- a/benchmarks/arc-agi/values.yaml +++ /dev/null @@ -1 +0,0 @@ -benchmark: arc-agi diff --git a/benchmarks/arc/values.yaml b/benchmarks/arc/values.yaml deleted file mode 100644 index 7cdbb978..00000000 --- a/benchmarks/arc/values.yaml +++ /dev/null @@ -1 +0,0 @@ -benchmark: arc diff --git a/benchmarks/arena-hard/values.yaml b/benchmarks/arena-hard/values.yaml deleted file mode 100644 index 83cb31c2..00000000 --- a/benchmarks/arena-hard/values.yaml +++ /dev/null @@ -1 +0,0 @@ -benchmark: arena-hard diff --git a/benchmarks/assetopsbench/values.yaml b/benchmarks/assetopsbench/values.yaml deleted file mode 100644 index a56c0efb..00000000 --- a/benchmarks/assetopsbench/values.yaml +++ /dev/null @@ -1 +0,0 @@ -benchmark: assetopsbench diff --git a/benchmarks/assistantbench/values.yaml b/benchmarks/assistantbench/values.yaml deleted file mode 100644 index 5f578d0b..00000000 --- a/benchmarks/assistantbench/values.yaml +++ /dev/null @@ -1 +0,0 @@ -benchmark: assistantbench diff --git a/benchmarks/bbh/values.yaml b/benchmarks/bbh/values.yaml deleted file mode 100644 index 617b0522..00000000 --- a/benchmarks/bbh/values.yaml +++ /dev/null @@ -1 +0,0 @@ -benchmark: bbh diff --git a/benchmarks/bfcl/values.yaml b/benchmarks/bfcl/values.yaml deleted file mode 100644 index 35f38fb6..00000000 --- a/benchmarks/bfcl/values.yaml +++ /dev/null @@ -1 +0,0 @@ -benchmark: bfcl diff --git a/benchmarks/bigcodebench/values.yaml b/benchmarks/bigcodebench/values.yaml deleted file mode 100644 index 349ad4de..00000000 --- a/benchmarks/bigcodebench/values.yaml +++ /dev/null @@ -1 +0,0 @@ -benchmark: bigcodebench diff --git a/benchmarks/browsecomp/values.yaml b/benchmarks/browsecomp/values.yaml deleted file mode 100644 index 4df14281..00000000 --- a/benchmarks/browsecomp/values.yaml +++ /dev/null @@ -1 +0,0 @@ -benchmark: browsecomp diff --git a/benchmarks/chartqa/values.yaml b/benchmarks/chartqa/values.yaml deleted file mode 100644 index 8fcae6b4..00000000 --- a/benchmarks/chartqa/values.yaml +++ /dev/null @@ -1 +0,0 @@ -benchmark: chartqa diff --git a/benchmarks/code-contests/values.yaml b/benchmarks/code-contests/values.yaml deleted file mode 100644 index daa89d3a..00000000 --- a/benchmarks/code-contests/values.yaml +++ /dev/null @@ -1 +0,0 @@ -benchmark: code-contests diff --git a/benchmarks/coderefine/values.yaml b/benchmarks/coderefine/values.yaml deleted file mode 100644 index 40099af0..00000000 --- a/benchmarks/coderefine/values.yaml +++ /dev/null @@ -1 +0,0 @@ -benchmark: coderefine diff --git a/benchmarks/commonsenseqa/values.yaml b/benchmarks/commonsenseqa/values.yaml deleted file mode 100644 index 037aff25..00000000 --- a/benchmarks/commonsenseqa/values.yaml +++ /dev/null @@ -1 +0,0 @@ -benchmark: commonsenseqa diff --git a/benchmarks/compilebench/values.yaml b/benchmarks/compilebench/values.yaml deleted file mode 100644 index 1d3aa91b..00000000 --- a/benchmarks/compilebench/values.yaml +++ /dev/null @@ -1 +0,0 @@ -benchmark: compilebench diff --git a/benchmarks/core-bench/values.yaml b/benchmarks/core-bench/values.yaml deleted file mode 100644 index fe7b1473..00000000 --- a/benchmarks/core-bench/values.yaml +++ /dev/null @@ -1 +0,0 @@ -benchmark: core-bench diff --git a/benchmarks/cybench/values.yaml b/benchmarks/cybench/values.yaml deleted file mode 100644 index a4e3556b..00000000 --- a/benchmarks/cybench/values.yaml +++ /dev/null @@ -1 +0,0 @@ -benchmark: cybench diff --git a/benchmarks/docvqa/values.yaml b/benchmarks/docvqa/values.yaml deleted file mode 100644 index 4612c90f..00000000 --- a/benchmarks/docvqa/values.yaml +++ /dev/null @@ -1 +0,0 @@ -benchmark: docvqa diff --git a/benchmarks/drop/values.yaml b/benchmarks/drop/values.yaml deleted file mode 100644 index 0e1016f0..00000000 --- a/benchmarks/drop/values.yaml +++ /dev/null @@ -1 +0,0 @@ -benchmark: drop diff --git a/benchmarks/flores200/values.yaml b/benchmarks/flores200/values.yaml deleted file mode 100644 index 6f5245d6..00000000 --- a/benchmarks/flores200/values.yaml +++ /dev/null @@ -1 +0,0 @@ -benchmark: flores200 diff --git a/benchmarks/frontiermath/values.yaml b/benchmarks/frontiermath/values.yaml deleted file mode 100644 index 47eb3932..00000000 --- a/benchmarks/frontiermath/values.yaml +++ /dev/null @@ -1 +0,0 @@ -benchmark: frontiermath diff --git a/benchmarks/gaia/values.yaml b/benchmarks/gaia/values.yaml deleted file mode 100644 index d73f4327..00000000 --- a/benchmarks/gaia/values.yaml +++ /dev/null @@ -1 +0,0 @@ -benchmark: gaia diff --git a/benchmarks/gdpval/values.yaml b/benchmarks/gdpval/values.yaml deleted file mode 100644 index b0024137..00000000 --- a/benchmarks/gdpval/values.yaml +++ /dev/null @@ -1 +0,0 @@ -benchmark: gdpval diff --git a/benchmarks/global-mmlu/values.yaml b/benchmarks/global-mmlu/values.yaml deleted file mode 100644 index f475b609..00000000 --- a/benchmarks/global-mmlu/values.yaml +++ /dev/null @@ -1 +0,0 @@ -benchmark: global-mmlu diff --git a/benchmarks/gpqa-diamond/values.yaml b/benchmarks/gpqa-diamond/values.yaml deleted file mode 100644 index 25e8ea29..00000000 --- a/benchmarks/gpqa-diamond/values.yaml +++ /dev/null @@ -1 +0,0 @@ -benchmark: gpqa-diamond diff --git a/benchmarks/gsm8k/values.yaml b/benchmarks/gsm8k/values.yaml deleted file mode 100644 index d135ac79..00000000 --- a/benchmarks/gsm8k/values.yaml +++ /dev/null @@ -1 +0,0 @@ -benchmark: gsm8k diff --git a/benchmarks/harmbench/values.yaml b/benchmarks/harmbench/values.yaml deleted file mode 100644 index 7f70ee51..00000000 --- a/benchmarks/harmbench/values.yaml +++ /dev/null @@ -1 +0,0 @@ -benchmark: harmbench diff --git a/benchmarks/healthbench/values.yaml b/benchmarks/healthbench/values.yaml deleted file mode 100644 index 06b7c927..00000000 --- a/benchmarks/healthbench/values.yaml +++ /dev/null @@ -1 +0,0 @@ -benchmark: healthbench diff --git a/benchmarks/hellaswag/values.yaml b/benchmarks/hellaswag/values.yaml deleted file mode 100644 index 7e9cacdb..00000000 --- a/benchmarks/hellaswag/values.yaml +++ /dev/null @@ -1 +0,0 @@ -benchmark: hellaswag diff --git a/benchmarks/hle/values.yaml b/benchmarks/hle/values.yaml deleted file mode 100644 index b1c20fc1..00000000 --- a/benchmarks/hle/values.yaml +++ /dev/null @@ -1 +0,0 @@ -benchmark: hle diff --git a/benchmarks/humaneval/values.yaml b/benchmarks/humaneval/values.yaml deleted file mode 100644 index 6ae3507a..00000000 --- a/benchmarks/humaneval/values.yaml +++ /dev/null @@ -1 +0,0 @@ -benchmark: humaneval diff --git a/benchmarks/humanevalplus/values.yaml b/benchmarks/humanevalplus/values.yaml deleted file mode 100644 index 0c1e587e..00000000 --- a/benchmarks/humanevalplus/values.yaml +++ /dev/null @@ -1 +0,0 @@ -benchmark: humanevalplus diff --git a/benchmarks/ifeval/values.yaml b/benchmarks/ifeval/values.yaml deleted file mode 100644 index 412c7ae0..00000000 --- a/benchmarks/ifeval/values.yaml +++ /dev/null @@ -1 +0,0 @@ -benchmark: ifeval diff --git a/benchmarks/itbench/values.yaml b/benchmarks/itbench/values.yaml deleted file mode 100644 index a2b2f9ed..00000000 --- a/benchmarks/itbench/values.yaml +++ /dev/null @@ -1 +0,0 @@ -benchmark: itbench diff --git a/benchmarks/kumo/values.yaml b/benchmarks/kumo/values.yaml deleted file mode 100644 index 6e063a9c..00000000 --- a/benchmarks/kumo/values.yaml +++ /dev/null @@ -1 +0,0 @@ -benchmark: kumo diff --git a/benchmarks/legalbench/values.yaml b/benchmarks/legalbench/values.yaml deleted file mode 100644 index 28acf130..00000000 --- a/benchmarks/legalbench/values.yaml +++ /dev/null @@ -1 +0,0 @@ -benchmark: legalbench diff --git a/benchmarks/livecodebench/values.yaml b/benchmarks/livecodebench/values.yaml deleted file mode 100644 index 51aa0339..00000000 --- a/benchmarks/livecodebench/values.yaml +++ /dev/null @@ -1 +0,0 @@ -benchmark: livecodebench diff --git a/benchmarks/longbench/values.yaml b/benchmarks/longbench/values.yaml deleted file mode 100644 index 2c04af31..00000000 --- a/benchmarks/longbench/values.yaml +++ /dev/null @@ -1 +0,0 @@ -benchmark: longbench diff --git a/benchmarks/math-500/values.yaml b/benchmarks/math-500/values.yaml deleted file mode 100644 index 87b9dc3a..00000000 --- a/benchmarks/math-500/values.yaml +++ /dev/null @@ -1 +0,0 @@ -benchmark: math-500 diff --git a/benchmarks/math/values.yaml b/benchmarks/math/values.yaml deleted file mode 100644 index ef2da116..00000000 --- a/benchmarks/math/values.yaml +++ /dev/null @@ -1 +0,0 @@ -benchmark: math diff --git a/benchmarks/mathvista/values.yaml b/benchmarks/mathvista/values.yaml deleted file mode 100644 index a0c98f5d..00000000 --- a/benchmarks/mathvista/values.yaml +++ /dev/null @@ -1 +0,0 @@ -benchmark: mathvista diff --git a/benchmarks/mbpp/values.yaml b/benchmarks/mbpp/values.yaml deleted file mode 100644 index b6944d45..00000000 --- a/benchmarks/mbpp/values.yaml +++ /dev/null @@ -1 +0,0 @@ -benchmark: mbpp diff --git a/benchmarks/mbppplus/values.yaml b/benchmarks/mbppplus/values.yaml deleted file mode 100644 index 2aafc174..00000000 --- a/benchmarks/mbppplus/values.yaml +++ /dev/null @@ -1 +0,0 @@ -benchmark: mbppplus diff --git a/benchmarks/medmcqa/values.yaml b/benchmarks/medmcqa/values.yaml deleted file mode 100644 index 7231d473..00000000 --- a/benchmarks/medmcqa/values.yaml +++ /dev/null @@ -1 +0,0 @@ -benchmark: medmcqa diff --git a/benchmarks/medqa/values.yaml b/benchmarks/medqa/values.yaml deleted file mode 100644 index c083d530..00000000 --- a/benchmarks/medqa/values.yaml +++ /dev/null @@ -1 +0,0 @@ -benchmark: medqa diff --git a/benchmarks/mgsm/values.yaml b/benchmarks/mgsm/values.yaml deleted file mode 100644 index fa64b584..00000000 --- a/benchmarks/mgsm/values.yaml +++ /dev/null @@ -1 +0,0 @@ -benchmark: mgsm diff --git a/benchmarks/mind2web/values.yaml b/benchmarks/mind2web/values.yaml deleted file mode 100644 index c5e8772d..00000000 --- a/benchmarks/mind2web/values.yaml +++ /dev/null @@ -1 +0,0 @@ -benchmark: mind2web diff --git a/benchmarks/minif2f/values.yaml b/benchmarks/minif2f/values.yaml deleted file mode 100644 index 94f31ac9..00000000 --- a/benchmarks/minif2f/values.yaml +++ /dev/null @@ -1 +0,0 @@ -benchmark: minif2f diff --git a/benchmarks/mle-bench/values.yaml b/benchmarks/mle-bench/values.yaml deleted file mode 100644 index 849aabb6..00000000 --- a/benchmarks/mle-bench/values.yaml +++ /dev/null @@ -1 +0,0 @@ -benchmark: mle-bench diff --git a/benchmarks/mmlu-pro/values.yaml b/benchmarks/mmlu-pro/values.yaml deleted file mode 100644 index ba426874..00000000 --- a/benchmarks/mmlu-pro/values.yaml +++ /dev/null @@ -1 +0,0 @@ -benchmark: mmlu-pro diff --git a/benchmarks/mmlu/values.yaml b/benchmarks/mmlu/values.yaml deleted file mode 100644 index fdc73100..00000000 --- a/benchmarks/mmlu/values.yaml +++ /dev/null @@ -1 +0,0 @@ -benchmark: mmlu diff --git a/benchmarks/mmmu/values.yaml b/benchmarks/mmmu/values.yaml deleted file mode 100644 index 8fba0750..00000000 --- a/benchmarks/mmmu/values.yaml +++ /dev/null @@ -1 +0,0 @@ -benchmark: mmmu diff --git a/benchmarks/mrcr/values.yaml b/benchmarks/mrcr/values.yaml deleted file mode 100644 index 5df3f759..00000000 --- a/benchmarks/mrcr/values.yaml +++ /dev/null @@ -1 +0,0 @@ -benchmark: mrcr diff --git a/benchmarks/mt-bench/values.yaml b/benchmarks/mt-bench/values.yaml deleted file mode 100644 index 7a4ab842..00000000 --- a/benchmarks/mt-bench/values.yaml +++ /dev/null @@ -1 +0,0 @@ -benchmark: mt-bench diff --git a/benchmarks/naturalquestions/values.yaml b/benchmarks/naturalquestions/values.yaml deleted file mode 100644 index 52728b30..00000000 --- a/benchmarks/naturalquestions/values.yaml +++ /dev/null @@ -1 +0,0 @@ -benchmark: naturalquestions diff --git a/benchmarks/niah/values.yaml b/benchmarks/niah/values.yaml deleted file mode 100644 index d6823da8..00000000 --- a/benchmarks/niah/values.yaml +++ /dev/null @@ -1 +0,0 @@ -benchmark: niah diff --git a/benchmarks/ocrbench/values.yaml b/benchmarks/ocrbench/values.yaml deleted file mode 100644 index 91bfe418..00000000 --- a/benchmarks/ocrbench/values.yaml +++ /dev/null @@ -1 +0,0 @@ -benchmark: ocrbench diff --git a/benchmarks/olympiad-bench/values.yaml b/benchmarks/olympiad-bench/values.yaml deleted file mode 100644 index 109f72b7..00000000 --- a/benchmarks/olympiad-bench/values.yaml +++ /dev/null @@ -1 +0,0 @@ -benchmark: olympiad-bench diff --git a/benchmarks/openbookqa/values.yaml b/benchmarks/openbookqa/values.yaml deleted file mode 100644 index 62b24877..00000000 --- a/benchmarks/openbookqa/values.yaml +++ /dev/null @@ -1 +0,0 @@ -benchmark: openbookqa diff --git a/benchmarks/piqa/values.yaml b/benchmarks/piqa/values.yaml deleted file mode 100644 index e5e59a1e..00000000 --- a/benchmarks/piqa/values.yaml +++ /dev/null @@ -1 +0,0 @@ -benchmark: piqa diff --git a/benchmarks/pubmedqa/values.yaml b/benchmarks/pubmedqa/values.yaml deleted file mode 100644 index f9093b7d..00000000 --- a/benchmarks/pubmedqa/values.yaml +++ /dev/null @@ -1 +0,0 @@ -benchmark: pubmedqa diff --git a/benchmarks/realworldqa/values.yaml b/benchmarks/realworldqa/values.yaml deleted file mode 100644 index f5dc9ae4..00000000 --- a/benchmarks/realworldqa/values.yaml +++ /dev/null @@ -1 +0,0 @@ -benchmark: realworldqa diff --git a/benchmarks/ruler/values.yaml b/benchmarks/ruler/values.yaml deleted file mode 100644 index af6aa3fd..00000000 --- a/benchmarks/ruler/values.yaml +++ /dev/null @@ -1 +0,0 @@ -benchmark: ruler diff --git a/benchmarks/scibench/values.yaml b/benchmarks/scibench/values.yaml deleted file mode 100644 index fb7aedb0..00000000 --- a/benchmarks/scibench/values.yaml +++ /dev/null @@ -1 +0,0 @@ -benchmark: scibench diff --git a/benchmarks/scicode/values.yaml b/benchmarks/scicode/values.yaml deleted file mode 100644 index 02cc0bf3..00000000 --- a/benchmarks/scicode/values.yaml +++ /dev/null @@ -1 +0,0 @@ -benchmark: scicode diff --git a/benchmarks/simpleqa/values.yaml b/benchmarks/simpleqa/values.yaml deleted file mode 100644 index 29c34ff2..00000000 --- a/benchmarks/simpleqa/values.yaml +++ /dev/null @@ -1 +0,0 @@ -benchmark: simpleqa diff --git a/benchmarks/socialiqa/values.yaml b/benchmarks/socialiqa/values.yaml deleted file mode 100644 index 711d37f4..00000000 --- a/benchmarks/socialiqa/values.yaml +++ /dev/null @@ -1 +0,0 @@ -benchmark: socialiqa diff --git a/benchmarks/swe-bench-pro/values.yaml b/benchmarks/swe-bench-pro/values.yaml deleted file mode 100644 index 0b579ded..00000000 --- a/benchmarks/swe-bench-pro/values.yaml +++ /dev/null @@ -1 +0,0 @@ -benchmark: swe-bench-pro diff --git a/benchmarks/swe-bench/values.yaml b/benchmarks/swe-bench/values.yaml deleted file mode 100644 index 7852e75b..00000000 --- a/benchmarks/swe-bench/values.yaml +++ /dev/null @@ -1 +0,0 @@ -benchmark: swe-bench diff --git a/benchmarks/swe-gym/values.yaml b/benchmarks/swe-gym/values.yaml deleted file mode 100644 index 9c707b9a..00000000 --- a/benchmarks/swe-gym/values.yaml +++ /dev/null @@ -1 +0,0 @@ -benchmark: swe-gym diff --git a/benchmarks/swe-lancer/values.yaml b/benchmarks/swe-lancer/values.yaml deleted file mode 100644 index e355d22f..00000000 --- a/benchmarks/swe-lancer/values.yaml +++ /dev/null @@ -1 +0,0 @@ -benchmark: swe-lancer diff --git a/benchmarks/terminal-bench/values.yaml b/benchmarks/terminal-bench/values.yaml deleted file mode 100644 index e8ffdcfe..00000000 --- a/benchmarks/terminal-bench/values.yaml +++ /dev/null @@ -1 +0,0 @@ -benchmark: terminal-bench diff --git a/benchmarks/theoremqa/values.yaml b/benchmarks/theoremqa/values.yaml deleted file mode 100644 index d9f4d714..00000000 --- a/benchmarks/theoremqa/values.yaml +++ /dev/null @@ -1 +0,0 @@ -benchmark: theoremqa diff --git a/benchmarks/triviaqa/values.yaml b/benchmarks/triviaqa/values.yaml deleted file mode 100644 index f6f82604..00000000 --- a/benchmarks/triviaqa/values.yaml +++ /dev/null @@ -1 +0,0 @@ -benchmark: triviaqa diff --git a/benchmarks/truthfulqa/values.yaml b/benchmarks/truthfulqa/values.yaml deleted file mode 100644 index 0ee2bafc..00000000 --- a/benchmarks/truthfulqa/values.yaml +++ /dev/null @@ -1 +0,0 @@ -benchmark: truthfulqa diff --git a/benchmarks/usaco/values.yaml b/benchmarks/usaco/values.yaml deleted file mode 100644 index 4be012f0..00000000 --- a/benchmarks/usaco/values.yaml +++ /dev/null @@ -1 +0,0 @@ -benchmark: usaco diff --git a/benchmarks/vakra/values.yaml b/benchmarks/vakra/values.yaml deleted file mode 100644 index 960b2c8b..00000000 --- a/benchmarks/vakra/values.yaml +++ /dev/null @@ -1 +0,0 @@ -benchmark: vakra diff --git a/benchmarks/winogrande/values.yaml b/benchmarks/winogrande/values.yaml deleted file mode 100644 index 7437196f..00000000 --- a/benchmarks/winogrande/values.yaml +++ /dev/null @@ -1 +0,0 @@ -benchmark: winogrande diff --git a/benchmarks/wmdp/values.yaml b/benchmarks/wmdp/values.yaml deleted file mode 100644 index 32ba2571..00000000 --- a/benchmarks/wmdp/values.yaml +++ /dev/null @@ -1 +0,0 @@ -benchmark: wmdp diff --git a/benchmarks/wmt/values.yaml b/benchmarks/wmt/values.yaml deleted file mode 100644 index 345be415..00000000 --- a/benchmarks/wmt/values.yaml +++ /dev/null @@ -1 +0,0 @@ -benchmark: wmt diff --git a/benchmarks/workarena/values.yaml b/benchmarks/workarena/values.yaml deleted file mode 100644 index a3662c1a..00000000 --- a/benchmarks/workarena/values.yaml +++ /dev/null @@ -1 +0,0 @@ -benchmark: workarena diff --git a/benchmarks/writingbench/values.yaml b/benchmarks/writingbench/values.yaml deleted file mode 100644 index abf2b2a2..00000000 --- a/benchmarks/writingbench/values.yaml +++ /dev/null @@ -1 +0,0 @@ -benchmark: writingbench diff --git a/benchmarks/xcopa/values.yaml b/benchmarks/xcopa/values.yaml deleted file mode 100644 index 534b1c26..00000000 --- a/benchmarks/xcopa/values.yaml +++ /dev/null @@ -1 +0,0 @@ -benchmark: xcopa diff --git a/benchmarks/xnli/values.yaml b/benchmarks/xnli/values.yaml deleted file mode 100644 index 80dd28c3..00000000 --- a/benchmarks/xnli/values.yaml +++ /dev/null @@ -1 +0,0 @@ -benchmark: xnli diff --git a/benchmarks/xstory-cloze/values.yaml b/benchmarks/xstory-cloze/values.yaml deleted file mode 100644 index 25c13f16..00000000 --- a/benchmarks/xstory-cloze/values.yaml +++ /dev/null @@ -1 +0,0 @@ -benchmark: xstory-cloze diff --git a/deploy/openshift-service-account.yaml b/deploy/openshift-service-account.yaml index 2e4dc51d..2e76fa96 100644 --- a/deploy/openshift-service-account.yaml +++ b/deploy/openshift-service-account.yaml @@ -8,7 +8,7 @@ # # Then deploy any benchmark with the OpenShift values layered on: # -# helm template benchmarks/_chart -f benchmarks//values.yaml \ +# helm template benchmarks/_chart --set benchmark= \ # -f deploy/values-openshift.yaml --set agent=,task= | oc apply -f - apiVersion: v1 kind: ServiceAccount diff --git a/docs/README.md b/docs/README.md index aff80770..c92755ac 100644 --- a/docs/README.md +++ b/docs/README.md @@ -22,7 +22,7 @@ way on a laptop, in CI, or on a Kubernetes cluster. - [Overview](concepts/overview.md) — the model: images as the product, three axes - [Triple-mode](concepts/triple-mode.md) — the same eval as container / compose / k8s job - [Isolation & gateways](concepts/isolation-and-gateways.md) — how trajectories stay honest -- [The Helm chart](concepts/the-helm-chart.md) — one chart, one `values.yaml` per benchmark +- [The Helm chart](concepts/the-helm-chart.md) — one chart, `--set benchmark=` to select; optional per-benchmark preset ## Guides — *how to do a task* diff --git a/docs/concepts/the-helm-chart.md b/docs/concepts/the-helm-chart.md index c6559ef5..f823a52b 100644 --- a/docs/concepts/the-helm-chart.md +++ b/docs/concepts/the-helm-chart.md @@ -3,17 +3,21 @@ *Concept · for operators · derives from [`doctrine/benchmarks/RULES.md`](../../doctrine/benchmarks/RULES.md) rules 24/29, [`doctrine/src/RULES.md`](../../doctrine/src/RULES.md).* In `job` mode, every benchmark deploys through **one shared Helm chart**, -`benchmarks/_chart`. A benchmark contributes only a small `values.yaml`; the -chart renders the otelcol + gateway + runner Job. +`benchmarks/_chart`. A benchmark is selected by name (`--set benchmark=`); +the chart renders the otelcol + gateway + runner Job. A benchmark with bespoke +topology adds an optional preset file inside the chart — standard benchmarks +contribute nothing, so the published chart is self-contained. ``` benchmarks/ _chart/ # the one chart — pod shape, defined once Chart.yaml values.yaml # defaults for every benchmark + presets/ # optional per-benchmark topology, bundled in the chart + osworld.yaml # adds sidecars/Deployments via composition hooks + tau-bench.yaml templates/job.yaml - aime/values.yaml # → just: benchmark: aime - osworld/values.yaml # a bespoke one: adds sidecars via composition hooks + aime/ # standard benchmark — no preset; just --set benchmark=aime ``` ## Why one chart @@ -26,25 +30,21 @@ Kustomize overlays did reuse + composition but couldn't interpolate variables.) ## What a benchmark overrides -Most benchmarks need a single line: - -```yaml -# benchmarks/aime/values.yaml -benchmark: aime -``` - -The chart's `values.yaml` supplies everything else (image tags, resources, -timeout). The per-run axes — agent, task, model — arrive at deploy time via -`--set` (or the CLI). The bespoke benchmarks (osworld, tau-bench, -visualwebarena, webarena) add sidecars/Deployments through composition hooks -(`initContainers`, `runnerExtraEnv`, `extraManifests`, …). +Most benchmarks override nothing — naming one with `--set benchmark=aime` is +enough, and the chart's `values.yaml` supplies everything else (image tags, +resources, timeout). The per-run axes — agent, task, model — arrive at deploy +time via `--set` (or the CLI). The bespoke benchmarks (osworld, tau-bench, +visualwebarena, webarena) ship a `presets/.yaml` in the chart that adds +sidecars/Deployments through composition hooks (`initContainers`, +`runnerExtraEnv`, `extraManifests`, …); the chart overlays it automatically when +that benchmark is selected. Full field list: [Chart values reference](../reference/chart-values.md). ## Rendering it ```bash -helm template aime benchmarks/_chart -f benchmarks/aime/values.yaml \ +helm template aime benchmarks/_chart --set benchmark=aime \ --set agent=claude-code,task=0 | kubectl apply -f - ``` @@ -56,5 +56,5 @@ in as a second values file via `--overlay` — see ## No drift One chart can't drift from itself — there is no `_base` to keep in sync. CI -renders every benchmark's `values.yaml` through `helm template | kubeconform` -(`tests/helm.rs`) so a broken values file fails the build. +renders every benchmark through `helm template --set benchmark= | kubeconform` +(`tests/helm.rs`) so a broken chart or preset fails the build. diff --git a/docs/concepts/triple-mode.md b/docs/concepts/triple-mode.md index 2812d592..d7076127 100644 --- a/docs/concepts/triple-mode.md +++ b/docs/concepts/triple-mode.md @@ -9,7 +9,7 @@ environment — the benchmark, agent, and model are identical across all three. |---|---|---| | `compose` *(default)* | `docker compose -f benchmarks//compose.yaml up` | Laptop, full stack (gateway + OTel), fastest iteration | | `container` | `docker run -e EVAL_MODEL=… ` | CI smoke tests, one-shot runs, minimal footprint | -| `job` | `helm template benchmarks/_chart -f benchmarks//values.yaml \| kubectl apply -f -` | Kubernetes, production-scale parallel runs | +| `job` | `helm template benchmarks/_chart --set benchmark= \| kubectl apply -f -` | Kubernetes, production-scale parallel runs | Select the mode with `--mode`: @@ -19,18 +19,18 @@ eval-containers run aime --task-id 0 --agent codex --mode container eval-containers run aime --task-id 0 --agent codex --mode job ``` -## Three artifacts per benchmark +## Artifacts per benchmark -Every benchmark carries exactly three deploy artifacts — this is the -"triple-mode" invariant ([rule 24](../../doctrine/benchmarks/RULES.md), enforced -by `tests/sanity/check.rs`): +The container and compose modes each need one file in the benchmark's dir +([rule 24](../../doctrine/benchmarks/RULES.md), enforced by `tests/sanity/check.rs`): - `container.Dockerfile` — the single-container image (`container` mode) - `compose.yaml` — the compose stack (`compose` mode) -- `values.yaml` — Helm values over the shared chart (`job` mode) -The container and compose modes wrap unmodified Docker. The job mode renders one -shared Helm chart — see [The Helm chart](the-helm-chart.md). +The `job` mode renders one shared Helm chart, selected with `--set benchmark=` +— no per-benchmark file required. A benchmark with bespoke topology adds an +optional `benchmarks/_chart/presets/.yaml`. See +[The Helm chart](the-helm-chart.md). ## The mental model diff --git a/docs/guides/add-a-benchmark.md b/docs/guides/add-a-benchmark.md index 18b22e15..e7962403 100644 --- a/docs/guides/add-a-benchmark.md +++ b/docs/guides/add-a-benchmark.md @@ -9,10 +9,12 @@ replacement — follow the skill and the rules it links. (what a finished benchmark must be). 2. **Follow the skill** — [`doctrine/benchmarks/add-benchmark/SKILL.md`](../../doctrine/benchmarks/add-benchmark/SKILL.md) (the step-by-step procedure, with a template). -3. **Ship the three deploy artifacts** — every benchmark needs all of - `container.Dockerfile`, `compose.yaml`, and `values.yaml` - (the triple-mode invariant; see [Triple-mode](../concepts/triple-mode.md)). - `tests/sanity/check.rs` enforces it. +3. **Ship the deploy artifacts** — every benchmark needs `container.Dockerfile` + and `compose.yaml` (the single-container and compose surfaces; + `tests/sanity/check.rs` enforces it). The k8s surface is the shared chart + selected with `--set benchmark=` — add a `benchmarks/_chart/presets/.yaml` + only if the benchmark needs bespoke topology. See + [Triple-mode](../concepts/triple-mode.md). 4. **Open the PR** using the [benchmark PR template](../../.github/PULL_REQUEST_TEMPLATE/benchmark.md), which lists every required label, env var, and evidence step. diff --git a/docs/guides/deploy-on-kubernetes.md b/docs/guides/deploy-on-kubernetes.md index 939665c6..774b82e0 100644 --- a/docs/guides/deploy-on-kubernetes.md +++ b/docs/guides/deploy-on-kubernetes.md @@ -21,7 +21,7 @@ kubectl create secret generic eval-secrets \ Plain Helm — no CLI required: ```bash -helm template aime benchmarks/_chart -f benchmarks/aime/values.yaml \ +helm template aime benchmarks/_chart --set benchmark=aime \ --set agent=claude-code,task=0 | kubectl apply -f - ``` @@ -52,7 +52,7 @@ service account — go in a Helm **values file you own**, layered on as an extra eval-containers run aime --agent codex --mode job \ --overlay my-cluster-values.yaml \ --registry my-registry.example.com/evals -# → helm template … -f benchmarks/aime/values.yaml -f my-cluster-values.yaml … | kubectl apply -f - +# → helm template … --set benchmark=aime -f my-cluster-values.yaml … | kubectl apply -f - ``` The eval axes and your platform settings merge; you never fork the chart. The diff --git a/docs/guides/deploy-on-openshift.md b/docs/guides/deploy-on-openshift.md index 45ac2d8d..bd3c8c52 100644 --- a/docs/guides/deploy-on-openshift.md +++ b/docs/guides/deploy-on-openshift.md @@ -35,14 +35,14 @@ sets the `anyuid-sa` service account. Layer it with `--overlay` and point eval-containers run aime --agent codex --mode job \ --overlay deploy/values-openshift.yaml \ --registry image-registry.openshift-image-registry.svc:5000/ -# → helm template … -f benchmarks/aime/values.yaml -f deploy/values-openshift.yaml … | oc apply -f - +# → helm template … --set benchmark=aime -f deploy/values-openshift.yaml … | oc apply -f - ``` Plain Helm equivalent (no CLI): ```bash helm template aime benchmarks/_chart \ - -f benchmarks/aime/values.yaml \ + --set benchmark=aime \ -f deploy/values-openshift.yaml \ --set agent=codex,task=0,registry=image-registry.openshift-image-registry.svc:5000/ \ | oc apply -f - diff --git a/docs/reference/chart-values.md b/docs/reference/chart-values.md index 4ea7bd70..345414c0 100644 --- a/docs/reference/chart-values.md +++ b/docs/reference/chart-values.md @@ -3,15 +3,16 @@ *Reference · for operators · derives from [`benchmarks/_chart/values.yaml`](../../benchmarks/_chart/values.yaml). That file is authoritative — these are its fields with defaults at the time of writing.* The shared chart `benchmarks/_chart` renders the otelcol + gateway + runner Job. -A benchmark's `values.yaml` sets `benchmark` (required) and overrides only what -differs; per-run axes arrive via `--set` (or the CLI). See +The benchmark is named via `--set benchmark=` (required); a benchmark with +bespoke topology overrides what differs through an optional `presets/.yaml` +in the chart. Per-run axes arrive via `--set` (or the CLI). See [The Helm chart](../concepts/the-helm-chart.md). ## Required | Field | Meaning | |---|---| -| `benchmark` | Benchmark name. The only field most `values.yaml` files set. | +| `benchmark` | Benchmark name, set via `--set benchmark=`. Selects the optional `presets/.yaml` overlay. | ## Per-run axes — *set at deploy via `--set` / the CLI* diff --git a/doctrine/benchmarks/RULES.md b/doctrine/benchmarks/RULES.md index feab8ac2..fe3dfa10 100644 --- a/doctrine/benchmarks/RULES.md +++ b/doctrine/benchmarks/RULES.md @@ -77,26 +77,26 @@ The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", "SHOULD", "S ### Three Deployment Surfaces -24. **Triple-mode contract — every benchmark ships exactly three deployment artifacts**, one per surface. The artifacts MUST share the same env contract (`EVAL_MODEL`, `EVAL_TASK_ID`, upstream credentials), MUST produce byte-equivalent `task/result.json` outputs for the same inputs (modulo timestamps and provider-side request IDs), and MUST exercise the same five-unit pipeline (otelcol → gateway → agent → verifier → result). +24. **Triple-mode contract — every benchmark supports three deployment surfaces.** The single and compose surfaces each ship a per-benchmark file; the k8s surface is the shared chart `benchmarks/_chart`, selected by name (`--set benchmark=`), with an optional per-benchmark preset for bespoke topology. All three MUST share the same env contract (`EVAL_MODEL`, `EVAL_TASK_ID`, upstream credentials), MUST produce byte-equivalent `task/result.json` outputs for the same inputs (modulo timestamps and provider-side request IDs), and MUST exercise the same five-unit pipeline (otelcol → gateway → agent → verifier → result). - | File | Mode | Topology | Invocation | + | Artifact | Mode | Topology | Invocation | |------|------|----------|------------| | `container.Dockerfile` | **single** | 1 container, 5 processes inside (process-compose orchestrates) | `docker build -f benchmarks//container.Dockerfile -t . && docker run ` | | `compose.yaml` | **compose** | 3 services on a compose network (otelcol + gateway + runner) | `docker compose -f benchmarks//compose.yaml up` | - | `values.yaml` | **k8s** | A Helm values file over the shared chart `benchmarks/_chart` — renders 1 `Job` + 1 Pod + 3 containers (`shareProcessNamespace` for sidecar reaping; isolation via credentials — see 24d), plus any bespoke `Deployment`s/`Service`s the benchmark composes | `helm template benchmarks/_chart -f benchmarks//values.yaml \| kubectl apply -f -` | + | shared chart + `presets/.yaml` *(optional)* | **k8s** | The shared chart `benchmarks/_chart` renders 1 `Job` + 1 Pod + 3 containers (`shareProcessNamespace` for sidecar reaping; isolation via credentials — see 24d), plus any bespoke `Deployment`s/`Service`s the benchmark composes via its preset | `helm template benchmarks/_chart --set benchmark= \| kubectl apply -f -` | - Single mode is the simplest surface (one `docker run`, no orchestrator); compose and k8s split the pipeline across containers so the agent process cannot see upstream credentials (rule 8). Benchmarks that ship only one or two surfaces are incomplete. + Single mode is the simplest surface (one `docker run`, no orchestrator); compose and k8s split the pipeline across containers so the agent process cannot see upstream credentials (rule 8). Benchmarks missing the single or compose file are incomplete; the k8s surface works for every benchmark with no per-benchmark file (a preset is added only for bespoke topology). 24a. **Universal eval-image recipe.** `core/combination.Dockerfile` is the single source of truth for the eval-image build. Per-benchmark `container.Dockerfile` files MUST be a single-line registry pin of the form `FROM /evals/--:` — nothing more. The canonical build args (`BENCHMARK_IMAGE`, `AGENT_IMAGE`, `AGENT_VERSION`, `MODEL_IMAGE`) MUST be recorded in the benchmark's `README.md` so CI can rebuild the eval image by invoking `core/combination.Dockerfile` with those args. Declaring inert `ARG` lines that the `FROM` does not consume is forbidden — they look load-bearing but aren't, and they drift. Duplicating the combination Dockerfile body across benchmarks is forbidden — there is exactly one eval-image recipe in the repo. 24b. **Both surfaces share one base; benchmarks override only what differs.** - `compose/services.yaml` — shared compose services (`otelcol`, `gateway`, `runner`); per-benchmark `compose.yaml` pulls it in via `include:` and overrides the benchmark-specific bits. - - `benchmarks/_chart/` — the shared Helm chart: the otelcol + gateway + runner Job, defined **once**. Per-benchmark `values.yaml` pins the benchmark and overrides only what differs. The ~97 standard benchmarks are a single line (`benchmark: `); the few with bespoke topology add their sidecars/`Deployment`s/`Service`s through the chart's composition hooks (`initContainers`, `runnerArgs`, `runnerExtraEnv`, `extraManifests`, …). + - `benchmarks/_chart/` — the shared Helm chart: the otelcol + gateway + runner Job, defined **once**. The benchmark is named via `--set benchmark=`; standard benchmarks override nothing. A benchmark with bespoke topology adds `benchmarks/_chart/presets/.yaml` — bundled in the chart and overlaid automatically when selected — to compose its sidecars/`Deployment`s/`Service`s through the chart's hooks (`initContainers`, `runnerArgs`, `runnerExtraEnv`, `extraManifests`, …). Presets MUST set only structural keys, never the per-run axes (agent/task/model), which arrive via `--set`. Both surfaces keep `service_healthy` ↔ `shareProcessNamespace` reaper, service-name DNS ↔ Pod loopback, and an identical env contract in lockstep — changes to the compose base or the chart MUST be reflected in the other in the same commit. 24c. **Task parameterization in deployment artifacts.** Rule 1 makes `EVAL_TASK_ID` the only required runtime input for shared-env benchmarks; the deployment artifacts MUST honor this: - - **Shared-env**: `compose.yaml` MUST read the task id from the shell environment as `TASK_ID: ${TASK_ID:-0}` (default 0, override via `TASK_ID=42 docker compose up`). Hardcoding a literal task id in `compose.yaml` is forbidden. The k8s surface parameterizes the task through Helm — `helm template … --set task=42` (default 0); a benchmark's `values.yaml` MUST NOT hardcode a task id. + - **Shared-env**: `compose.yaml` MUST read the task id from the shell environment as `TASK_ID: ${TASK_ID:-0}` (default 0, override via `TASK_ID=42 docker compose up`). Hardcoding a literal task id in `compose.yaml` is forbidden. The k8s surface parameterizes the task through Helm — `helm template … --set task=42` (default 0); a benchmark's preset MUST NOT hardcode a task id. - **Per-task**: `EVAL_TASK_ID` is a build-time `ARG` in `container.Dockerfile`; each image bakes exactly one task. The compose and k8s artifacts inherit the baked-in task — they do not parameterize. 24d. **Network isolation across surfaces.** Rule 9 (no agent internet by default) MUST be enforced in every shipped surface, by the mechanism native to that surface: @@ -106,11 +106,11 @@ The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", "SHOULD", "S Benchmarks that explicitly require agent internet MUST declare `eval.benchmark.internet=true` AND remove the relevant isolation primitive in every surface that ships. Asymmetry (e.g., compose blocks but k8s allows) is forbidden. -24e. **Resource limit parity.** Rule 10 (resource limits) MUST be expressed in both `compose.yaml` (`deploy.resources.limits` on the runner) and the k8s surface (`resources.limits` on the runner — the chart default, which a benchmark overrides via `values.yaml`'s `resources:`). The values MUST match modulo k8s unit syntax (`"2"` ↔ `2`, `"2Gi"` ↔ `2147483648`). GPU benchmarks declare via `deploy.resources.reservations.devices[].driver: nvidia` in compose and `resources.limits["nvidia.com/gpu"]` in k8s. +24e. **Resource limit parity.** Rule 10 (resource limits) MUST be expressed in both `compose.yaml` (`deploy.resources.limits` on the runner) and the k8s surface (`resources.limits` on the runner — the chart default, which a benchmark overrides via its preset's `resources:`). The values MUST match modulo k8s unit syntax (`"2"` ↔ `2`, `"2Gi"` ↔ `2147483648`). GPU benchmarks declare via `deploy.resources.reservations.devices[].driver: nvidia` in compose and `resources.limits["nvidia.com/gpu"]` in k8s. 25. **Use the surface's natural sharing approach.** - **compose** has native sharing (`include:`/`extends:`). Per-benchmark `compose.yaml` MUST pull `compose/services.yaml` in via `include:` and only declare overrides. Inlining a service/healthcheck/network/volume that already exists in `compose/services.yaml` is forbidden. - - **k8s** uses Helm. The shared Pod lives once in the chart's template (`benchmarks/_chart/templates/`); per-benchmark `values.yaml` MUST only pin the benchmark and override what differs (via the chart's composition hooks). Re-declaring the otelcol/gateway/runner Pod inside a benchmark is forbidden — there is exactly one k8s Pod definition in the repo. + - **k8s** uses Helm. The shared Pod lives once in the chart's template (`benchmarks/_chart/templates/`); a benchmark is selected with `--set benchmark=` and overrides what differs only via an optional `presets/.yaml` (the chart's composition hooks). Re-declaring the otelcol/gateway/runner Pod in a preset is forbidden — there is exactly one k8s Pod definition in the repo. Effect: when the canonical Pod shape evolves, compose changes 1 file and k8s changes 1 file (the chart) — every benchmark re-renders from it, so there is nothing to drift. @@ -123,12 +123,11 @@ The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", "SHOULD", "S 28. **Replay test.** Every benchmark MUST have at least one end-to-end test using the replay model with a recorded fixture. This test MUST verify that `result.json` is produced with the correct schema. 29. **Triple-mode existence + render test.** A CI test MUST walk every directory in `benchmarks/` and assert: - (a) `container.Dockerfile`, `compose.yaml`, and `values.yaml` all exist; + (a) `container.Dockerfile` and `compose.yaml` exist; (b) `container.Dockerfile` is a single-line `FROM` (rule 24a); (c) `docker compose -f compose.yaml config` succeeds (rule 27); - (d) `helm template benchmarks/_chart -f benchmarks//values.yaml` renders and its output validates against the k8s schema (kubeconform or equivalent); - (e) `values.yaml` pins the benchmark (`benchmark: `), so the chart renders `evals/--` and labels the Job from it; - (f) the env contract (`EVAL_MODEL`, `EVAL_TASK_ID`, upstream creds) is identical across all three surfaces. + (d) `helm template benchmarks/_chart --set benchmark=` renders (overlaying `presets/.yaml` if present) and its output validates against the k8s schema (kubeconform or equivalent), labelling the Job `evals/--`; + (e) the env contract (`EVAL_MODEL`, `EVAL_TASK_ID`, upstream creds) is identical across all three surfaces. Benchmarks failing any sub-test cannot be merged. There is no per-benchmark drift check — one chart cannot drift from itself. ## References @@ -147,3 +146,4 @@ The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", "SHOULD", "S | 2026-05-18 | Rule 24 rewritten as the triple-mode contract: every benchmark ships `container.Dockerfile` (single) + `compose.yaml` (compose) + `job.yaml` (k8s). Rule 24a forbids duplicating the universal `core/combination.Dockerfile` body per benchmark — per-benchmark Dockerfiles only pin build args. Rule 24b requires `compose.yaml` and `job.yaml` to stay in lockstep. Pre-rename `single.yaml` is gone (it was a one-container k8s adapter for single mode — but single mode's contract is the Dockerfile, not a YAML); `k8s.yaml` renamed to `job.yaml`. | | 2026-05-18 | Tightening pass before the 90-benchmark sweep. Rule 24a corrected: `container.Dockerfile` MUST be a single-line registry pin; inert `ARG` lines that the `FROM` doesn't consume are forbidden (they looked load-bearing but drifted). New rule 24c codifies task parameterization — shared-env `compose.yaml` MUST use `${TASK_ID:-0}`, `job.yaml` ships as a task-0 template; per-task benchmarks bake `EVAL_TASK_ID` via build ARG. New rule 24d makes network-isolation enforcement explicit per surface and honest about k8s achieving rule 9 via credential isolation (rule 8) rather than network policy. New rule 24e requires resource limits to be declared identically in both `compose.yaml` and `job.yaml`. Rule 25 strengthened to forbid inlining definitions that already exist in `compose/services.yaml`. New rule 29 mandates a triple-mode CI gate that walks `benchmarks/` and asserts artifact existence + parse + env-contract symmetry. | | 2026-06-01 | k8s surface moved from per-benchmark Kustomize to one shared **Helm chart** (`benchmarks/_chart`) + a per-benchmark `values.yaml`. Rule 24's k8s artifact is `values.yaml` (was `job.yaml`); 24b/25 replace the `benchmarks/_base/job.yaml` inline-and-drift model with "the Pod is defined once in the chart; `values.yaml` pins the benchmark and overrides only what differs"; 24c parameterizes the task via `helm --set task=`; 24e/24d retargeted at the chart. Rule 29 drops the canonical-drift sub-test (one chart can't drift) — it now renders each `values.yaml` via `helm template` and kubeconform-validates. `eval-containers run --mode job` and `--overlay` drive Helm; the OpenShift overlay is `deploy/values-openshift.yaml`. Deleted `benchmarks/_base` + 114 per-benchmark kustomize files (net −5.4k lines). | +| 2026-06-03 | Per-benchmark `values.yaml` removed; the benchmark is now selected with `--set benchmark=` and its bespoke topology (if any) lives in `benchmarks/_chart/presets/.yaml`, bundled inside the chart and overlaid via `.Files.Get`. The chart is now self-contained — `helm template … --set benchmark=` needs no external file, so it can be packaged/published to an OCI registry. Rule 24 drops `values.yaml` as a required artifact (k8s works for every benchmark with no per-benchmark file); 24b/24c/24e/25 retarget at the preset; rule 29(a)/(d) drop the values.yaml existence + pin checks and render via `--set benchmark`. The 4 bespoke benchmarks (osworld, tau-bench, visualwebarena, webarena) became presets; the 98 trivial one-line files were deleted. Renders byte-identical to the prior `-f values.yaml` form. | diff --git a/doctrine/benchmarks/add-benchmark/SKILL.md b/doctrine/benchmarks/add-benchmark/SKILL.md index f4ce2b02..4dea1413 100644 --- a/doctrine/benchmarks/add-benchmark/SKILL.md +++ b/doctrine/benchmarks/add-benchmark/SKILL.md @@ -37,11 +37,13 @@ answer, and any attached files from `EVAL_TASK_ID` alone 1. **Create the benchmark directory.** Make `benchmarks//` and copy [`assets/TEMPLATE.md`](assets/TEMPLATE.md) as your scaffold. Every benchmark - ships exactly five authored files: `Dockerfile` (builds the base image with - tasks + verifier), the triple-mode trio `container.Dockerfile` + - `compose.yaml` + `values.yaml`, and `README.md`. *Why:* the directory is the - unit a CI test walks; missing any of the trio makes the benchmark incomplete - (`doctrine/benchmarks/RULES.md:24`, `doctrine/benchmarks/RULES.md:29`). + ships four authored files: `Dockerfile` (builds the base image with + tasks + verifier), the per-benchmark deploy files `container.Dockerfile` + + `compose.yaml`, and `README.md`. The k8s surface is the shared chart selected + with `--set benchmark=` — author a `benchmarks/_chart/presets/.yaml` + only if the benchmark needs bespoke topology. *Why:* the directory is the + unit a CI test walks; missing `container.Dockerfile` or `compose.yaml` makes + the benchmark incomplete (`doctrine/benchmarks/RULES.md:24`, `doctrine/benchmarks/RULES.md:29`). 2. **Write the `Dockerfile` to materialize tasks as flat files.** Fetch the dataset and write each task as `/tasks//problem.txt` + `answer.txt` @@ -117,11 +119,10 @@ answer, and any attached files from `EVAL_TASK_ID` alone `eval.benchmark.released="true"` yet — that label is earned at the release gate (step 11). -9. **Author the three deployment surfaces with one shared env contract.** Every - benchmark ships exactly three artifacts, and they MUST share the same env - contract (`EVAL_MODEL`, `EVAL_TASK_ID`, upstream credentials) and produce - byte-equivalent `task/result.json` for the same inputs - (`doctrine/benchmarks/RULES.md:24`): +9. **Author the three deployment surfaces with one shared env contract.** The + surfaces MUST share the same env contract (`EVAL_MODEL`, `EVAL_TASK_ID`, + upstream credentials) and produce byte-equivalent `task/result.json` for the + same inputs (`doctrine/benchmarks/RULES.md:24`): - `container.Dockerfile` (**single**) — a *single-line* registry pin `FROM /evals/--:`, nothing more. Record the canonical build args (`BENCHMARK_IMAGE`, `AGENT_IMAGE`, `AGENT_VERSION`, @@ -132,27 +133,28 @@ answer, and any attached files from `EVAL_TASK_ID` alone `include:` and only declare overrides; do NOT inline a service, network, or volume that already exists there (`doctrine/benchmarks/RULES.md:24b`, `doctrine/benchmarks/RULES.md:25`). - - `values.yaml` (**k8s**) — a Helm values file over the shared chart - `benchmarks/_chart`. For a standard benchmark this is a single line - (`benchmark: `); benchmarks with bespoke topology add their - sidecars/`Deployment`s/`Service`s through the chart's composition hooks + - **k8s** — the shared chart `benchmarks/_chart`, selected with + `--set benchmark=`. A standard benchmark needs nothing here. One with + bespoke topology adds `benchmarks/_chart/presets/.yaml` to compose its + sidecars/`Deployment`s/`Service`s through the chart's hooks (`initContainers`, `runnerArgs`, `runnerExtraEnv`, `extraManifests`, …) — do NOT redeclare the otelcol/gateway/runner Pod (`doctrine/benchmarks/RULES.md:24b`, `doctrine/benchmarks/RULES.md:25`). For a simple shared-env benchmark, copy `benchmarks/aime/` and substitute the - name. Changes to the compose base (`compose/services.yaml`) or the chart - (`benchmarks/_chart`) MUST be reflected in the other in the same commit. + name (no preset needed). Changes to the compose base (`compose/services.yaml`) + or the chart (`benchmarks/_chart`) MUST be reflected in the other in the same + commit. 10. **Parameterize the task and enforce limits/isolation in every surface.** - Task: shared-env `compose.yaml` MUST read `TASK_ID: ${TASK_ID:-0}` (never hardcode a literal); the k8s task comes from `helm --set task=` (default - 0), so `values.yaml` MUST NOT hardcode a task. Per-task benchmarks bake + 0), so a preset MUST NOT hardcode a task. Per-task benchmarks bake `EVAL_TASK_ID` via build `ARG` and the artifacts inherit it (`doctrine/benchmarks/RULES.md:24c`). - Resource limits: declare CPU and memory in BOTH `compose.yaml` (`deploy.resources.limits` on the runner) and the k8s runner — the chart - default, overridden per benchmark via `values.yaml`'s `resources:` — + default, overridden per benchmark via its preset's `resources:` — matching modulo k8s unit syntax (`doctrine/benchmarks/RULES.md:10`, `doctrine/benchmarks/RULES.md:24e`). - Network: enforce no-agent-internet per surface — `internal: true` in @@ -167,8 +169,9 @@ answer, and any attached files from `EVAL_TASK_ID` alone (`doctrine/benchmarks/RULES.md:27`), and at least one end-to-end replay test with a recorded fixture that verifies `result.json` schema (`doctrine/benchmarks/RULES.md:28`). The triple-mode CI gate - (`doctrine/benchmarks/RULES.md:29`) checks all three artifacts exist, parse, - and share one env contract. Once the benchmark is proven end-to-end against + (`doctrine/benchmarks/RULES.md:29`) checks the per-benchmark files exist and + parse, the chart renders for the benchmark, and all surfaces share one env + contract. Once the benchmark is proven end-to-end against at least one agent with a replay fixture at `tests/fixtures/--.trajectory.jsonl`, add `LABEL eval.benchmark.released="true"` (`doctrine/benchmarks/RULES.md:21a`). @@ -183,7 +186,7 @@ answer, and any attached files from `EVAL_TASK_ID` alone ## References - [`assets/TEMPLATE.md`](assets/TEMPLATE.md) — copyable scaffold with the - Dockerfile, the triple-mode trio, the blanks-to-fill table, and gotchas. + Dockerfile, the per-benchmark deploy files, the blanks-to-fill table, and gotchas. - `doctrine/benchmarks/RULES.md` — the outcomes every benchmark MUST satisfy. - `benchmarks/aime/` — canonical simple shared-env reference. - `benchmarks/_chart/` — the shared k8s Helm chart (the canonical Pod, once). diff --git a/doctrine/benchmarks/add-benchmark/assets/TEMPLATE.md b/doctrine/benchmarks/add-benchmark/assets/TEMPLATE.md index cf9a70e5..dc8d2965 100644 --- a/doctrine/benchmarks/add-benchmark/assets/TEMPLATE.md +++ b/doctrine/benchmarks/add-benchmark/assets/TEMPLATE.md @@ -1,16 +1,16 @@ # Adding a Benchmark -Read `RULES.md` first. Every benchmark ships exactly four authored files plus the Dockerfile: +Read `RULES.md` first. Every benchmark ships three authored files plus the Dockerfile (and, only for bespoke k8s topology, a chart preset): | File | Purpose | Shape | |------|---------|-------| | `Dockerfile` | Build the benchmark base image (tasks + verifier) | Per-benchmark | | `container.Dockerfile` | Single-mode deployment artifact | 1 line — `FROM /evals/--:` | | `compose.yaml` | Compose-mode deployment artifact | ~7 lines — `include:` shared base + benchmark overrides | -| `values.yaml` | k8s-mode deployment artifact | 1 line for simple (`benchmark: `) — a Helm values file over the shared chart `benchmarks/_chart`. Complex benchmarks add sidecars/Deployments/Services via the chart's composition hooks | | `README.md` | Docs | At-a-glance table + agent contract + grading + run examples | +| `benchmarks/_chart/presets/.yaml` *(optional)* | k8s bespoke topology | Only for complex benchmarks — adds sidecars/Deployments/Services via the chart's composition hooks | -The triple-mode trio (`container.Dockerfile`, `compose.yaml`, `values.yaml`) is uniform across simple benchmarks — copy `benchmarks/aime/` and substitute the name. Complex benchmarks (with bespoke services) only diverge in `values.yaml` (where they add Deployments/Services via the chart's `extraManifests` and other hooks) and `compose.yaml` (where they add services after the `include:`). See `benchmarks/aime/` for the canonical reference and `benchmarks/_chart/` for the shared k8s chart. +The k8s surface is the shared chart `benchmarks/_chart`, selected with `--set benchmark=` — a simple benchmark needs no per-benchmark k8s file. So the per-benchmark files (`container.Dockerfile`, `compose.yaml`) are uniform across simple benchmarks — copy `benchmarks/aime/` and substitute the name. Complex benchmarks (with bespoke services) diverge in `compose.yaml` (extra services after the `include:`) and add a `benchmarks/_chart/presets/.yaml` (Deployments/Services via the chart's `extraManifests` and other hooks). See `benchmarks/aime/` for the canonical reference and `benchmarks/_chart/` for the shared k8s chart. ## Shared-env Benchmark (one image, many tasks) @@ -94,20 +94,18 @@ services: BENCHMARK: {name} ``` -### values.yaml +### k8s surface -For a simple shared-env benchmark this is a single line — the shared chart -(`benchmarks/_chart`) renders the otelcol+gateway+runner Job from it: - -```yaml -benchmark: {name} -``` +A simple shared-env benchmark needs no k8s file — the shared chart +(`benchmarks/_chart`) renders the otelcol+gateway+runner Job when selected with +`--set benchmark={name}`. For complex benchmarks (bespoke services like a VM, browser, or database -sidecar), set the chart's composition hooks in `values.yaml` — `initContainers`, -`runnerExtraEnv`, `runnerArgs`, and `extraManifests` (full `Deployment`/`Service` -docs). See `benchmarks/osworld/values.yaml` (a desktop `Deployment`/`Service`) or -`benchmarks/webarena/values.yaml` (proxy + 6 site `Deployment`s) for examples. +sidecar), add `benchmarks/_chart/presets/{name}.yaml` and set the chart's +composition hooks there — `initContainers`, `runnerExtraEnv`, `runnerArgs`, and +`extraManifests` (full `Deployment`/`Service` docs). See +`benchmarks/_chart/presets/osworld.yaml` (a desktop `Deployment`/`Service`) or +`benchmarks/_chart/presets/webarena.yaml` (proxy + 6 site `Deployment`s) for examples. ## Blanks to fill @@ -141,13 +139,14 @@ services: BENCHMARK: {name} ``` -```yaml -# values.yaml — set the non-default axes as chart values (no manifest editing): -benchmark: {name} -agent: # → runner image evals/{name}-- -gatewayImage: # → gateway image models/ -evalModel: -model: +```bash +# k8s — pass the non-default axes as --set values (no manifest editing): +helm template {name} benchmarks/_chart \ + --set benchmark={name} \ + --set agent= \ # → runner image evals/{name}-- + --set gatewayImage= \ # → gateway image models/ + --set evalModel= \ + --set model= ``` ## Gotchas diff --git a/doctrine/delivery/build/SKILL.md b/doctrine/delivery/build/SKILL.md index 92d1af9b..68e656b8 100644 --- a/doctrine/delivery/build/SKILL.md +++ b/doctrine/delivery/build/SKILL.md @@ -302,7 +302,7 @@ Dockerfile's `FROM` lines — lives in the build-test catalog - **Build orchestration** (which artifacts to build for a given combo): `src/build.rs` and the CLI. - **Compose runtime topology**: each benchmark's `compose.yaml`. -- **K8s runtime topology**: the shared Helm chart `benchmarks/_chart` + each benchmark's `values.yaml`. +- **K8s runtime topology**: the shared Helm chart `benchmarks/_chart`, selected with `--set benchmark=` + an optional `presets/.yaml` for bespoke topology. - **Per-task variant builds** (swe-bench's 1000+ tasks): the CLI's `--task-id` flow; not enumerated in bake. - **In-cluster OC builds**: a translator that reads bake files and emits diff --git a/doctrine/src/RULES.md b/doctrine/src/RULES.md index 9d314ca7..0942c060 100644 --- a/doctrine/src/RULES.md +++ b/doctrine/src/RULES.md @@ -86,3 +86,4 @@ Each tool MUST be a standard release the user can install and invoke themselves | 2026-05-31 | Principle 11 (Build): exposed in-cluster builds as `build --builder ` — a passthrough of buildx's `--builder` that implies `--push`; a missing builder fails with the `docker buildx create --driver kubernetes` command to run. | | 2026-05-31 | Principle 2 refined: scoped "maps to a tool command" to container/cluster-driving commands and carved out `report`/`gen-bake` as local file utilities; scoped the `--dry-run` requirement to state-changing/outward commands (`build`, `run`, `push`, `prune`). | | 2026-06-01 | `run --mode job` moved from synthesizing a Kustomize overlay to `helm template benchmarks/_chart -f benchmarks//values.yaml --set … \| kubectl apply -f -`. `--overlay` now takes a Helm values file (e.g. `deploy/values-openshift.yaml`), not a Kustomize component directory. | +| 2026-06-03 | `run --mode job` drops the per-benchmark `-f values.yaml`; the benchmark is named via `--set benchmark=` and its bespoke topology (if any) lives in the chart at `presets/.yaml`. The chart is now self-contained (renders with no external file), so it can be packaged/published. | diff --git a/src/run.rs b/src/run.rs index f0b25e7e..b1874147 100644 --- a/src/run.rs +++ b/src/run.rs @@ -5,7 +5,7 @@ //! //! --mode compose (default) → docker compose -f benchmarks//compose.yaml up //! --mode container → docker run -e EVAL_MODEL=... -//! --mode job → helm template benchmarks/_chart -f benchmarks//values.yaml | kubectl apply -f - +//! --mode job → helm template benchmarks/_chart --set benchmark= | kubectl apply -f - //! //! Mapping flags → manifest, by mode: //! @@ -13,9 +13,10 @@ //! `EVAL_*` environment variable on the spawned subprocess. Compose //! interpolates `${EVAL_FOO:-default}` in compose.yaml; container //! mode hands them in via `docker run -e`. -//! - **job** renders the shared Helm chart (`benchmarks/_chart`) with the -//! benchmark's `values.yaml` plus a `--set` for each axis (agent/task/ -//! model/tags/versions), then `helm template … | kubectl apply -f -`. +//! - **job** renders the shared Helm chart (`benchmarks/_chart`) with a +//! `--set` for each axis (benchmark/agent/task/model/tags/versions), +//! then `helm template … | kubectl apply -f -`. A benchmark's bespoke +//! topology, if any, lives in the chart at `presets/.yaml`. //! Helm interpolates the values (kubectl can't), keeps numeric fields //! like `task` quoted, and the Job name carries the agent + task so //! concurrent applies don't collide. @@ -31,7 +32,7 @@ //! `--dry-run=server` to `kubectl apply` (exercises admission, no state). //! //! With `--local`, uses the in-repo `benchmarks//{compose.yaml, -//! container.Dockerfile, values.yaml}` instead of the registry artifact. +//! container.Dockerfile}` and the local chart instead of the registry artifact. //! //! Two orthogonal versioning axes (see RULES.md principle 9): //! @@ -40,7 +41,7 @@ //! flags --*-version) //! //! With `--local`, uses the in-repo `benchmarks//{compose.yaml, -//! container.Dockerfile, values.yaml}` instead of the registry artifact. +//! container.Dockerfile}` and the local chart instead of the registry artifact. use clap::{Args, ValueEnum}; use std::process::Command; @@ -150,7 +151,7 @@ pub struct RunArgs { namespace: Option, /// (`--mode job`) Layer a platform Helm values file on top of the - /// benchmark's values — e.g. `deploy/values-openshift.yaml`, which sets + /// chart values — e.g. `deploy/values-openshift.yaml`, which sets /// the anyuid SCC service account. Passed to helm as an extra `-f`. #[arg(long)] overlay: Option, @@ -344,11 +345,12 @@ fn run_container( Ok(()) } -/// `--mode job` → `helm template benchmarks/_chart -f benchmarks//values.yaml … | kubectl apply -f -` +/// `--mode job` → `helm template benchmarks/_chart --set benchmark= … | kubectl apply -f -` /// /// The shared chart (`benchmarks/_chart`) renders the otelcol+gateway+runner -/// Job from the benchmark's `values.yaml`; the per-run axes (agent/task/model/ -/// tags/versions) come in via `--set`. Platform composition (e.g. the OpenShift +/// Job; the axes (benchmark/agent/task/model/tags/versions) come in via `--set`, +/// and a benchmark's bespoke topology (if any) from the chart's +/// `presets/.yaml`. Platform composition (e.g. the OpenShift /// anyuid SCC) layers in as an extra `-f ` via `--overlay`. Helm fills /// the values, keeps numeric fields (task) quoted, and leaves the runner /// command's `$?`/`$rc` untouched — no kustomize overlay to synthesize. @@ -361,12 +363,6 @@ fn run_job( args: &RunArgs, _envs: &[(&str, String)], ) -> Result<(), String> { - let values = format!("./benchmarks/{benchmark}/values.yaml"); - if !std::path::Path::new(&values).exists() { - return Err(format!( - "missing benchmarks/{benchmark}/values.yaml; run from repo root" - )); - } let chart = "./benchmarks/_chart"; if !std::path::Path::new(chart).exists() { return Err("missing benchmarks/_chart; run from repo root".into()); @@ -375,18 +371,14 @@ fn run_job( let agent = args.agent.as_deref().unwrap_or("claude-code"); let task = args.task_id.as_deref().unwrap_or("0"); - // helm template -f [-f ] --set … + // helm template [-f ] --set benchmark=… --set … + // The benchmark is named via --set; its bespoke topology (if any) lives in + // the chart at presets/.yaml, so no per-benchmark file is passed. let release = format!("{benchmark}-{agent}-task-{task}"); - let mut helm: Vec = vec![ - "template".into(), - release, - chart.into(), - "-f".into(), - values.clone(), - ]; + let mut helm: Vec = vec!["template".into(), release, chart.into()]; - // Platform composition: --overlay now points at a Helm values file (e.g. - // deploy/values-openshift.yaml), layered on top of the benchmark's values. + // Platform composition: --overlay points at a Helm values file (e.g. + // deploy/values-openshift.yaml), layered on top of the chart values. if let Some(ov) = &args.overlay { if !std::path::Path::new(ov).exists() { return Err(format!( @@ -402,6 +394,7 @@ fn run_job( // --model maps to EVAL_MODEL (the upstream the fixed gateway proxies to) // plus the runner's MODEL logging tag — matching the prior behavior. let mut sets: Vec = vec![ + format!("benchmark={benchmark}"), format!("registry={registry}"), format!("agent={agent}"), format!("task={task}"), diff --git a/tests/helm.rs b/tests/helm.rs index 0043ae19..7f1b1089 100644 --- a/tests/helm.rs +++ b/tests/helm.rs @@ -1,6 +1,7 @@ -//! Rule 29(d): every benchmark's `values.yaml` MUST render through the shared -//! Helm chart (`benchmarks/_chart`) and the output MUST validate against the -//! k8s schema. `helm` is the deploy tool, so it's a required CI dependency; +//! Rule 29(d): every benchmark MUST render through the shared Helm chart +//! (`benchmarks/_chart`, selected with `--set benchmark=`) and the output +//! MUST validate against the k8s schema. `helm` is the deploy tool, so it's a +//! required CI dependency; //! `kubeconform` is used as the schema validator when present (the render //! itself is the floor when it isn't). //! @@ -35,7 +36,7 @@ fn benchmark_dirs() -> Vec<(String, PathBuf)> { } #[test] -fn every_values_renders_and_validates() { +fn every_benchmark_renders_and_validates() { if Command::new("helm").arg("version").output().is_err() { panic!("helm not found — required by doctrine/benchmarks/RULES.md rule 29(d)"); } @@ -43,14 +44,17 @@ fn every_values_renders_and_validates() { let dirs = benchmark_dirs(); let mut issues: Vec = Vec::new(); - for (name, dir) in &dirs { - let values = dir.join("values.yaml"); - if !values.is_file() { - continue; // structural_validation (check.rs) already flags this - } + for (name, _dir) in &dirs { + // The benchmark is named via --set; its bespoke topology (if any) lives + // in the chart at presets/.yaml — no per-benchmark file is passed. let out = match Command::new("helm") - .args(["template", name, "benchmarks/_chart", "-f"]) - .arg(&values) + .args([ + "template", + name, + "benchmarks/_chart", + "--set", + &format!("benchmark={name}"), + ]) .output() { Ok(o) => o, diff --git a/tests/sanity/check.rs b/tests/sanity/check.rs index ba6fef4e..3b762c17 100644 --- a/tests/sanity/check.rs +++ b/tests/sanity/check.rs @@ -83,19 +83,18 @@ const REQUIRED_COMPOSE_MARKERS: &[&str] = &[ "BENCHMARK:", ]; -// Rule 24 (triple-mode contract): every benchmark ships container.Dockerfile, -// compose.yaml, and values.yaml — one per deployment surface. The k8s surface -// renders from the shared Helm chart (benchmarks/_chart) + the benchmark's -// values.yaml; there is no per-benchmark job.yaml to drift (rule 29 retired — -// one chart can't drift from itself). -const REQUIRED_TRIPLE_MODE_FILES: &[&str] = - &["container.Dockerfile", "compose.yaml", "values.yaml"]; +// Rule 24 (triple-mode contract): every benchmark ships container.Dockerfile +// and compose.yaml — the single-container and compose surfaces. The k8s surface +// is the shared Helm chart (benchmarks/_chart), selected with `--set +// benchmark=`; a benchmark with bespoke topology adds an optional +// `benchmarks/_chart/presets/.yaml` (no per-benchmark file required, so it +// is not part of the triple-mode contract). +const REQUIRED_TRIPLE_MODE_FILES: &[&str] = &["container.Dockerfile", "compose.yaml"]; fn check_benchmark_structure(name: &str, dir: &Path) -> Vec { let mut issues = Vec::new(); let dockerfile = dir.join("Dockerfile"); let compose = dir.join("compose.yaml"); - let values = dir.join("values.yaml"); if !dockerfile.is_file() { issues.push(format!("{name}: no Dockerfile")); @@ -122,15 +121,6 @@ fn check_benchmark_structure(name: &str, dir: &Path) -> Vec { } } - if values.is_file() { - // The k8s values must pin this benchmark — the chart renders - // evals/-- and labels the Job from it. - if !contains_line(&values, &format!("benchmark: {name}")) { - issues.push(format!( - "{name}: values.yaml must set `benchmark: {name}` (Helm chart pin)" - )); - } - } issues }