diff --git a/.agents/scripts/nel-next.sh b/.agents/scripts/nel-next.sh index 92f3f18f35d..8b8f1130b9a 100755 --- a/.agents/scripts/nel-next.sh +++ b/.agents/scripts/nel-next.sh @@ -32,21 +32,27 @@ # using the config's export_config.mlflow (resolves ${MLFLOW_TRACKING_URI}, forces # emit_traces=false to avoid the per-sample hang). Run after `source .env`. # -# Install source (env overrides): NEL_NEXT_SPEC (PyPI default, "nemo-evaluator[harbor,export]==0.3.*" -# — [export] pulls mlflow for mlflow-push; pin an exact 0.3.x here for reproducibility), or -# NEL_NEXT_ORIGIN [+ NEL_NEXT_REF] for the internal git build. uv caches the resolved env and -# refreshes it when the spec changes. +# Install source: NEL_NEXT_ORIGIN + NEL_NEXT_REF git build (default), or NEL_NEXT_SPEC +# to force a PyPI release. Upstream `main` is 0.4.0 and ships the vendored TB 2.1 registry +# override; PyPI stops at 0.3.0, so a 0.4.x pin there resolves to nothing. No v0.4.0 tag +# exists, so NEL_NEXT_REF defaults to a commit SHA — an unpinned branch HEAD would install +# a different harness for the baseline and the candidate run, folding a harness change into +# the pass@1 delta. NEL_NEXT_REF=main tracks HEAD (dev/canary only); bump the default below +# after re-validating a canary. Override NEL_NEXT_ORIGIN in `.env` to build from a mirror. +# `--version` prints the resolved spec — record it alongside scored results. set -euo pipefail # [harbor] = agentic/sandbox deps; [export] pulls mlflow for `mlflow-push`. -NEL_NEXT_SPEC="${NEL_NEXT_SPEC:-nemo-evaluator[harbor,export]==0.3.*}" -NEL_NEXT_ORIGIN="${NEL_NEXT_ORIGIN:-}" -NEL_NEXT_REF="${NEL_NEXT_REF:-}" +NEL_NEXT_SPEC="${NEL_NEXT_SPEC:-}" +NEL_NEXT_ORIGIN="${NEL_NEXT_ORIGIN:-git+https://github.com/NVIDIA-NeMo/Evaluator.git}" +# Reproducibility pin — NVIDIA-NeMo/Evaluator main @ 2026-08-04 (nemo-evaluator 0.4.0). +NEL_NEXT_REF="${NEL_NEXT_REF:-4d081325170aababd0c8f27c58bed31a81ce82ac}" -if [[ -n "$NEL_NEXT_ORIGIN" ]]; then - INSTALL_SPEC="nemo-evaluator[harbor,export] @ ${NEL_NEXT_ORIGIN}${NEL_NEXT_REF:+@${NEL_NEXT_REF}}" -else +# NEL_NEXT_SPEC wins when explicitly set (PyPI escape hatch); otherwise use the git origin. +if [[ -n "$NEL_NEXT_SPEC" ]]; then INSTALL_SPEC="$NEL_NEXT_SPEC" +else + INSTALL_SPEC="nemo-evaluator[harbor,export] @ ${NEL_NEXT_ORIGIN}${NEL_NEXT_REF:+@${NEL_NEXT_REF}}" fi _log() { printf '\033[2m %s\033[0m\n' "$*" >&2; } @@ -128,7 +134,7 @@ command -v uvx >/dev/null 2>&1 || { echo "ERROR: 'uvx' not found (curl -LsSf htt case "${1:-}" in --setup-only) _uvx nel --version >/dev/null 2>&1 && _log "nel-next ready — ${INSTALL_SPEC}"; exit 0 ;; --which) echo "uvx --python 3.12 --from \"${INSTALL_SPEC}\" nel"; exit 0 ;; - --version) _uvx python -c 'import nemo_evaluator; print(nemo_evaluator.__version__)'; exit 0 ;; + --version) _uvx python -c 'import nemo_evaluator; print(nemo_evaluator.__version__)'; _log "from ${INSTALL_SPEC}"; exit 0 ;; mlflow-push) _mlflow_push "${@:2}"; exit $? ;; "") echo "ERROR: no args. Try: nel-next.sh eval run [--dry-run] (or --help)" >&2; exit 2 ;; esac diff --git a/.agents/skills/evaluation/recipes/examples/example_eval_next.yaml b/.agents/skills/evaluation/recipes/examples/example_eval_next.yaml index c61908907d4..4aeb4390859 100644 --- a/.agents/skills/evaluation/recipes/examples/example_eval_next.yaml +++ b/.agents/skills/evaluation/recipes/examples/example_eval_next.yaml @@ -43,12 +43,18 @@ services: - ???:/cache/huggingface generation: {temperature: 1.0, top_p: 0.95} # from model card (reasoning mode); adjust per card — mandatory lookup (references/model-card-research.md), same as 0.2.6 proxy: - request_timeout: 1800 + request_timeout: 3600 # canonical; MUST be >= benchmarks[].solver.agent_kwargs.llm_kwargs.timeout extra_body: {skip_special_tokens: false} # add model-card sampling extras here if the card specifies them; mirror them in the export tags below interceptors: - name: drop_params # agents send max_tokens; many servers reject it - config: {params: [max_tokens, max_completion_tokens]} + # last two are sent by the 0.5.x harbor eval image; vLLM 400s on them unless stripped + config: {params: [max_tokens, max_completion_tokens, max_input_tokens_per_task, no_rebuild]} # SWE-bench (OpenHands, multi-turn) adds turn_counter + consolidate_system + a system_message — see swebench_verified.md + # FEP-1104/1120 diagnostics — uncomment for a CANARY/debug run, drop it for the scored run: + # first_n caps only 200s, so every error pair (full req+res bodies) is retained in memory for + # the whole run and re-serialized on each write — unbounded growth exactly when the server errors. + # - name: http_pairs_dump # canonical LAST in the chain (SWE-bench: first) + # config: {dump_path: "$${NEL_OUTPUT_DIR}/http_pairs_metrics.json", first_n: 50} # $$ defers expansion to run time node_pool: gpu benchmarks: @@ -72,7 +78,7 @@ cluster: account: ??? walltime: "04:00:00" # auto_resume chains across windows shards: 1 # N = N nodes (each redeploys vLLM) - eval_image: ${NEL_NEXT_EVAL_IMAGE} # from eval-config (TB2.1 needs ≥0.3.1.1-harbor, multi-arch; enroot creds per SKILL Step 7.5) + eval_image: ${NEL_NEXT_EVAL_IMAGE} # from eval-config (0.5.0.1-harbor, multi-arch; enroot creds per SKILL Step 7.5) sbatch_comment: '{"OccupiedIdleGPUsJobReaper":{"exemptIdleTimeMins":"480","reason":"benchmarking","description":"nel-next agentic eval"}}' sbatch_extra_flags: {switches: 1, exclusive: true} container_env: # AWS creds reach the eval container ONLY via here @@ -100,7 +106,7 @@ output: experiment_name: ??? # /- (hardcode; ${USER}=root in-container) log_config_params: true copy_logs: true - exclude_patterns: ["shard*"] + exclude_patterns: ["shard*", "model_traffic.jsonl"] # captured request bodies (FEA-224) stay in the run dir description: ??? # ' | T=1.0 top_p=0.95 | (timeout_strategy=…) | r8' # model/checkpoint_path/benchmark drive dashboard attribution (engine logs only a generic metric key); temperature/top_p mirror generation above. tags: {framework: vllm, model: "???", checkpoint_path: "???", benchmark: "???", temperature: '1.0', top_p: '0.95'} diff --git a/.agents/skills/evaluation/recipes/tasks/aa_next/swebench_verified.md b/.agents/skills/evaluation/recipes/tasks/aa_next/swebench_verified.md index 00287bc97e1..399ebceef5e 100644 --- a/.agents/skills/evaluation/recipes/tasks/aa_next/swebench_verified.md +++ b/.agents/skills/evaluation/recipes/tasks/aa_next/swebench_verified.md @@ -5,8 +5,9 @@ run flow). Same harbor/ECS-Fargate flow as Terminal-Bench; the deltas are the **OpenHands agent**, a larger problem set, longer timeouts, and a different ECR/region. Start from `recipes/examples/example_eval_next.yaml`. -> **Source of truth:** `configs/benchmarks/nel_next/swebench_verified/bench.yaml` -> in nvidia-eval-factory-benchmarking — match its values for a reference run. +> **Source of truth:** `configs/benchmarks/swe-bench-verified/bench.yaml` in +> nvidia-eval-factory-benchmarking (`dl/JoC/competitive_evaluation/…`), with the eval-image +> pin in `configs/shared/nel_next_containers.yaml` — match its values for a reference run. ## Task-specific values (canonical `bench.yaml`) @@ -15,20 +16,24 @@ ECR/region. Start from `recipes/examples/example_eval_next.yaml`. | `playbook` | `swebench_verified` (`harbor://swebench-verified@1.0`) | | agent | `openhands-sdk` (playbook; `agent_kwargs: {max_iterations: 200, version: "1.17.0"}`) | | scope | 500 Python tasks × `repeats: 5` | -| `max_concurrent` / `sandbox.concurrency` | `15` | +| `max_concurrent` / `sandbox.concurrency` | `15` in `bench.yaml`; per-model configs override it (MiniMax-M2.7 uses `20`) | | `solver` | `timeout_strategy: max`, `run_timeout: 10800` (3h), `agent_kwargs.llm_kwargs.timeout: 3600` | | `sandbox.region` | `us-east-2` | | `sandbox.ecr_repository` | `${HARBOR_SWEBENCH_ECR_REPOSITORY}` (dedicated `harbor-swebench` repo, **us-west-2**, regardless of sandbox region) | -| `cluster.eval_image` | `${NEL_NEXT_EVAL_IMAGE}` (needs **≥ `0.3.1.1-harbor`** — FEP-1085 reasoning fix) | +| `cluster.eval_image` | `${NEL_NEXT_EVAL_IMAGE}` → **`0.5.0.1-harbor`** (same pin as TB2.1: `configs/shared/nel_next_containers.yaml`) | | `cluster.container_env.AWS_DEFAULT_REGION` | `us-east-2` (match `sandbox.region`) | -| `instruction_template` | **must be MOUNTED** — the harbor image doesn't bundle the built-in (gotcha below) | +| `instruction_template` | `/configs/prompts/swebench_instruction.md`, **must be MOUNTED**; content is scoring-relevant (gotcha below) | +| `proxy.request_timeout` | `3600` (FEP-1104 paired HTTP timeout; leaves mirror it on the service proxy) | +| `drop_params` | `max_tokens`, `max_completion_tokens`, `max_input_tokens_per_task`, `no_rebuild` | +| `output.export_config.mlflow.exclude_patterns` | `["shard*", "model_traffic.jsonl"]` | +| `system_message` | `strategy: replace` + the OpenHands prompt from `bench.yaml` (verbatim) — scoring-relevant | ```yaml benchmarks: - playbook: swebench_verified repeats: 5 max_concurrent: 15 - instruction_template: /configs/swebench-instruction.md # mounted (see gotcha) + instruction_template: /configs/prompts/swebench_instruction.md # mounted (see gotcha) solver: service: timeout_strategy: max # canonical; "task" = leaderboard-comparable @@ -45,41 +50,65 @@ benchmarks: The playbook defaults `instruction_template: swebench-instruction.md`, but the harbor image doesn't ship that built-in → run dies at finalize with -`FileNotFoundError: instruction_template not found`. Mount it (the canonical -config uses the compeval OpenHands prompt; the public built-in from the host venv -works too): +`FileNotFoundError: instruction_template not found`. So it must be mounted. + +**Which file you mount changes the score.** Mount the canonical `swebench_instruction.md` +(underscore) at `/configs/prompts/swebench_instruction.md`, taken from the reference config or +run dir. The built-in in the `nemo_evaluator/templates/` venv directory is a **different +prompt** (`swebench-instruction.md`, hyphen) — it runs, but results are not comparable. Keep +whichever you use fixed across both sides of a comparison. ```bash -VENV="${NEL_NEXT_VENV:-$HOME/.local/share/nel/venvs/nel-next}" # same default as nel-next.sh (NEL_NEXT_VENV may be unset) -cp "$VENV/lib/python3.12/site-packages/nemo_evaluator/templates/swebench-instruction.md" /tmp/ -ssh 'mkdir -p //prompts' && scp /tmp/swebench-instruction.md ://prompts/ +ssh 'mkdir -p //prompts' +scp swebench_instruction.md ://prompts/ # canonical file; verify sha256 against the source ``` ```yaml -benchmarks: [{playbook: swebench_verified, instruction_template: /configs/swebench-instruction.md}] +benchmarks: [{playbook: swebench_verified, instruction_template: /configs/prompts/swebench_instruction.md}] cluster: - container_mounts: ["//prompts/swebench-instruction.md:/configs/swebench-instruction.md:ro"] + container_mounts: ["//prompts/swebench_instruction.md:/configs/prompts/swebench_instruction.md:ro"] ``` ### Deployment proxy (multi-turn agentic) OpenHands runs ~200 turns/task. The canonical config adds a `system_message` interceptor (a large OpenHands system prompt — copy it verbatim from `bench.yaml`) -plus `turn_counter`. Full stack on `services..proxy.interceptors`: +plus `turn_counter`. + +**Order differs from TB2.1**: `http_pairs_dump` is **first** (not last) and `drop_params` +comes **before** `consolidate_system`. `http_pairs_dump` is canary/diagnostic-only — it +retains every error pair in memory for the whole run (`references/nel-next.md`); drop it +from the scored config. ```yaml proxy: request_timeout: 3600 extra_body: {skip_special_tokens: false} # add model-card sampling extras if the card sets them + model_traffic: {capture_request_body: true} # FEA-224; pair with the exclude_patterns entry interceptors: + # - {name: http_pairs_dump, config: {dump_path: "$${NEL_OUTPUT_DIR}/http_pairs_metrics.json", first_n: 50}} # canary only - {name: system_message, config: {strategy: replace, system_message: ""}} - - {name: turn_counter, config: {max_turns: 200}} + - {name: turn_counter, config: {max_turns: 200, position: system_message}} + - {name: drop_params, config: {params: [max_tokens, max_completion_tokens, max_input_tokens_per_task, no_rebuild]}} - {name: consolidate_system} - - {name: drop_params, config: {params: [max_tokens, max_completion_tokens]}} - {name: reasoning} # reasoning models: normalize reasoning field … - - {name: reasoning_replay} # … and replay it across turns (drop both for instruct) + - {name: reasoning_replay} # … and replay across turns. Drop both for instruct models. ``` +**`reasoning_replay.mode` is per model, not per benchmark.** `think_tags` (Qwen-style), +`native` (GLM), omitted (MiniMax). Copying another model's mode is a silent output-parsing bug. + +**Omitting `system_message` is a scoring change**: without it the agent runs the +openhands-sdk default prompt instead of the canonical one. + +### Sharding + +`max_concurrent`/`sandbox.concurrency` are **per shard**, and each shard redeploys the model +on its own node — `shards: N` multiplies serving capacity *and* live Fargate sandboxes +(`N × concurrency`). 500 tasks × `repeats: 5` = 2500 trials; `shards: 10` at `concurrency: 15` +suits it. Score is unaffected — purely a wall-clock lever. Check `N × concurrency` against the +Fargate quota and `N × gpus_per_node` against your allocation. + ## Score Extraction Report **`pass@1`** only — benchmark `swebench-verified@1.0`, scorer `pass@1` (0–1): diff --git a/.agents/skills/evaluation/recipes/tasks/aa_next/terminal_bench_2_1.md b/.agents/skills/evaluation/recipes/tasks/aa_next/terminal_bench_2_1.md index d5b976513c7..114203cf5ed 100644 --- a/.agents/skills/evaluation/recipes/tasks/aa_next/terminal_bench_2_1.md +++ b/.agents/skills/evaluation/recipes/tasks/aa_next/terminal_bench_2_1.md @@ -24,12 +24,18 @@ pinned via a vendored registry override in nemo-evaluator-next. | `cluster.container_env.AWS_DEFAULT_REGION` | match `sandbox.region` | | `max_concurrent` / `sandbox.concurrency` | `50` (canonical bench.yaml) | | timeout_strategy | `max` (canonical bench.yaml) + `agent_kwargs.llm_kwargs.timeout: 3600`; use `task` for leaderboard-comparable | -| `cluster.eval_image` requirement | **≥ `0.3.1.1-harbor`** — TB 2.1's task set is pinned via a vendored registry override in that image (`${NEL_NEXT_EVAL_IMAGE}`, multi-arch) | +| `cluster.eval_image` | **`0.5.0.1-harbor`** (`${NEL_NEXT_EVAL_IMAGE}`, multi-arch) | +| `proxy.request_timeout` | `3600` — must be **≥** `agent_kwargs.llm_kwargs.timeout` | +| `drop_params` | `max_tokens`, `max_completion_tokens`, `max_input_tokens_per_task`, `no_rebuild` | +| `output.export_config.mlflow.exclude_patterns` | `["shard*", "model_traffic.jsonl"]` | +| `http_pairs_dump` | **last** in the interceptor chain — canary/diagnostic only, drop it for a scored run (unbounded error-pair retention) | +| scope | 89 tasks × `repeats: 8` | These values mirror the canonical TB2.1 config — re-check it before a scored run: -`configs/benchmarks/nel_next/terminal_bench_21/bench.yaml` in -nvidia-eval-factory-benchmarking (see `references/nel-next.md` + the eval-config -"source of truth" note). The `benchmarks:` block (drop into the example template): +`configs/benchmarks/terminal-bench-2.1/bench.yaml` (+ `manifest.yaml`) in +nvidia-eval-factory-benchmarking (`dl/JoC/competitive_evaluation/…`), with the image pin in +`configs/shared/nel_next_containers.yaml`. See `references/nel-next.md` + the eval-config +"source of truth" note. The `benchmarks:` block (drop into the example template): ```yaml benchmarks: @@ -50,9 +56,15 @@ benchmarks: log_stream_prefix: terminalbench21-- ``` -`cluster.eval_image: ${NEL_NEXT_EVAL_IMAGE}` (≥ `0.3.1.1-harbor`) and the AWS creds +`cluster.eval_image: ${NEL_NEXT_EVAL_IMAGE}` (`0.5.0.1-harbor`) and the AWS creds come from `modelopttools:eval-config` (run it first) + the workspace `.env`. +**Sharding.** `max_concurrent`/`sandbox.concurrency` are **per shard**, and each shard runs +its own vLLM on its own node — `shards: N` multiplies both serving capacity and live Fargate +sandboxes (`N × concurrency`). Trials are partitioned and merged, so the score is unaffected; +it is purely a wall-clock lever. `shards: 4` suits 89 × r8 = 712 trials. Check +`N × concurrency` against the Fargate quota and `N × gpus_per_node` against your allocation. + ## Score Extraction Report **`pass@1`** only — benchmark `terminal-bench@2.1`, scorer `pass@1` (0–1): diff --git a/.agents/skills/evaluation/references/nel-next.md b/.agents/skills/evaluation/references/nel-next.md index 5d1975f4e2f..59efcfa7c68 100644 --- a/.agents/skills/evaluation/references/nel-next.md +++ b/.agents/skills/evaluation/references/nel-next.md @@ -27,8 +27,14 @@ Installing 0.3.x into the 0.2.6 env clobbers `nel`, so it lives in its own venv: .agents/scripts/nel-next.sh eval run --dry-run | --submit | … ``` -Default install is public PyPI `nemo-evaluator[harbor]==0.3.*`; set -`NEL_NEXT_ORIGIN`/`NEL_NEXT_REF` for the internal git build (see script header). +Default install is a git build from `github.com/NVIDIA-NeMo/Evaluator` via `NEL_NEXT_ORIGIN` +(`main` → **0.4.0**). PyPI `nemo-evaluator` stops at **0.3.0**, so a version pin can't reach +0.4.x. No `v0.4.0` tag exists, so `NEL_NEXT_REF` **defaults to a pinned commit SHA** in +`nel-next.sh` — a floating branch HEAD installs a different harness for the baseline and the +candidate, so the pass@1 delta no longer isolates the model. `NEL_NEXT_REF=main` tracks HEAD +for dev/canary work only; to move the pin, canary against the new SHA and bump the default in +`nel-next.sh`. Record `nel-next.sh --version` (prints version + resolved spec) with scored +results. Set `NEL_NEXT_ORIGIN` in `.env` to build from a mirror. ## Credentials + internal infra (`.env`) @@ -75,7 +81,7 @@ services: extra_env: {...} # VLLM_* backend env (e.g. NVFP4 MoE flags) container_mounts: [/.cache/vllm:/cache/vllm, ...] generation: {temperature: 1.0, top_p: 0.95} - proxy: {request_timeout: 1800, extra_body: {...}, interceptors: [...]} + proxy: {request_timeout: 3600, extra_body: {...}, interceptors: [...]} # >= llm_kwargs.timeout node_pool: gpu benchmarks: # EXACTLY ONE entry — one benchmark per config (see "One benchmark per config") - playbook: # per recipe @@ -131,9 +137,25 @@ with its own `run_id`, copying the shared `services:` block. ## Rules & gotchas -- **`eval_image`** = `${NEL_NEXT_EVAL_IMAGE}`. `0.3.1.1-harbor` is multi-arch and is - the minimum for **TB 2.1**; older `0.17.x/0.18.x-harbor-` are arch-suffixed. +- **`eval_image`** = `${NEL_NEXT_EVAL_IMAGE}` → `0.5.0.1-harbor` (multi-arch). Re-check + against `configs/shared/nel_next_containers.yaml` in the eval-factory repo, which is the + pin and does move. Arch-suffixed `0.17.x/0.18.x-harbor-` are too old for TB 2.1. Private gitlab-master image → cluster needs enroot creds (SKILL Step 7.5). +- **`proxy.request_timeout` must be >= `agent_kwargs.llm_kwargs.timeout`** (both 3600). A + smaller proxy timeout silently truncates long agent turns. +- **`drop_params`** for harbor agentic benchmarks: `max_tokens`, `max_completion_tokens`, + `max_input_tokens_per_task`, `no_rebuild`. The last two are sent by the 0.5.x eval image; + vLLM 400s on them if they aren't stripped. +- **`exclude_patterns`** = `["shard*", "model_traffic.jsonl"]` — captured request bodies + stay in the run dir, never MLflow. +- **`http_pairs_dump` — canary/diagnostic runs only, leave it OUT of scored runs.** + `config: {dump_path: "$${NEL_OUTPUT_DIR}/http_pairs_metrics.json", first_n: 50}` (the `$$` + defers expansion to run time). `first_n` caps only the *successful* pairs: the keep rule is + `(total_seen <= first_n) or (status != 200)`, so **every** non-200 pair is held in memory + for the life of the run, full request + response bodies, and the whole list is re-serialized + on each write. A long agentic run that is 400ing or rate-limiting (the failure this dumps + diagnose) grows the proxy without bound — exactly the run you can least afford to lose. + Chain position is per benchmark (last for TB2.1, first for SWE-bench). - **Mount sources must pre-exist** — pyxis won't create the host side of a bind mount (invisible to `--dry-run`, fails at canary). `ssh 'mkdir -p //.cache/{vllm,huggingface}'`. diff --git a/.gitignore b/.gitignore index 89a4a8540dd..72089b06e7e 100644 --- a/.gitignore +++ b/.gitignore @@ -29,8 +29,7 @@ docs/source/reference/generated **/.ipynb_checkpoints # Environments -.env -.env-* +.env* .venv env/ venv/