From fe2cc5292d27a95029392c55a66fb6fccefde90f Mon Sep 17 00:00:00 2001 From: Zhiyu Cheng Date: Thu, 6 Aug 2026 16:42:50 -0700 Subject: [PATCH] Fix AA-next OpenHands timeout guidance Signed-off-by: Zhiyu Cheng --- .../recipes/examples/example_eval_next.yaml | 2 +- .../recipes/tasks/aa_next/swebench_verified.md | 16 ++++++++++++++-- .../recipes/tasks/aa_next/terminal_bench_2_1.md | 8 +++++--- .agents/skills/evaluation/references/nel-next.md | 16 +++++++++++++--- 4 files changed, 33 insertions(+), 9 deletions(-) diff --git a/.agents/skills/evaluation/recipes/examples/example_eval_next.yaml b/.agents/skills/evaluation/recipes/examples/example_eval_next.yaml index c61908907d4..6d4d1a33ba0 100644 --- a/.agents/skills/evaluation/recipes/examples/example_eval_next.yaml +++ b/.agents/skills/evaluation/recipes/examples/example_eval_next.yaml @@ -72,7 +72,7 @@ cluster: account: ??? walltime: "04:00:00" # auto_resume chains across windows shards: 1 # N = N nodes (each redeploys vLLM) - eval_image: ${NEL_NEXT_EVAL_IMAGE} # from eval-config (TB2.1 needs ≥0.3.1.1-harbor, multi-arch; enroot creds per SKILL Step 7.5) + eval_image: ${NEL_NEXT_EVAL_IMAGE} # from eval-config (current AA-next source of truth: 0.5.0.1-harbor, multi-arch; enroot creds per SKILL Step 7.5) sbatch_comment: '{"OccupiedIdleGPUsJobReaper":{"exemptIdleTimeMins":"480","reason":"benchmarking","description":"nel-next agentic eval"}}' sbatch_extra_flags: {switches: 1, exclusive: true} container_env: # AWS creds reach the eval container ONLY via here diff --git a/.agents/skills/evaluation/recipes/tasks/aa_next/swebench_verified.md b/.agents/skills/evaluation/recipes/tasks/aa_next/swebench_verified.md index 00287bc97e1..5bf6b881b53 100644 --- a/.agents/skills/evaluation/recipes/tasks/aa_next/swebench_verified.md +++ b/.agents/skills/evaluation/recipes/tasks/aa_next/swebench_verified.md @@ -5,7 +5,8 @@ run flow). Same harbor/ECS-Fargate flow as Terminal-Bench; the deltas are the **OpenHands agent**, a larger problem set, longer timeouts, and a different ECR/region. Start from `recipes/examples/example_eval_next.yaml`. -> **Source of truth:** `configs/benchmarks/nel_next/swebench_verified/bench.yaml` +> **Source of truth:** `configs/benchmarks/swe-bench-verified/bench.yaml` and +> `configs/shared/nel_next_containers.yaml` > in nvidia-eval-factory-benchmarking — match its values for a reference run. ## Task-specific values (canonical `bench.yaml`) @@ -19,7 +20,7 @@ ECR/region. Start from `recipes/examples/example_eval_next.yaml`. | `solver` | `timeout_strategy: max`, `run_timeout: 10800` (3h), `agent_kwargs.llm_kwargs.timeout: 3600` | | `sandbox.region` | `us-east-2` | | `sandbox.ecr_repository` | `${HARBOR_SWEBENCH_ECR_REPOSITORY}` (dedicated `harbor-swebench` repo, **us-west-2**, regardless of sandbox region) | -| `cluster.eval_image` | `${NEL_NEXT_EVAL_IMAGE}` (needs **≥ `0.3.1.1-harbor`** — FEP-1085 reasoning fix) | +| `cluster.eval_image` | `${NEL_NEXT_EVAL_IMAGE}` (current internal source of truth: **`0.5.0.1-harbor`**) | | `cluster.container_env.AWS_DEFAULT_REGION` | `us-east-2` (match `sandbox.region`) | | `instruction_template` | **must be MOUNTED** — the harbor image doesn't bundle the built-in (gotcha below) | @@ -41,6 +42,17 @@ benchmarks: log_stream_prefix: swebench-verified-- ``` +### Canary gate — verify the OpenHands request timeout + +`0.3.1.1-harbor` accepts the `llm_kwargs.timeout` field above but does not pass +it to the OpenHands `LLM` client, which silently retains its 300-second default. +Use an eval image containing +[NVIDIA-NeMo/Evaluator#1083](https://github.com/NVIDIA-NeMo/Evaluator/pull/1083) +(`0.5.0.1-harbor` in the current internal source of truth). After the first +canary task starts, inspect the OpenHands startup/agent log and require an +active timeout of `3600`; treat `timeout: 300` as an infrastructure failure and +do not launch the scored run. + ### Gotcha — mount the instruction template The playbook defaults `instruction_template: swebench-instruction.md`, but the diff --git a/.agents/skills/evaluation/recipes/tasks/aa_next/terminal_bench_2_1.md b/.agents/skills/evaluation/recipes/tasks/aa_next/terminal_bench_2_1.md index d5b976513c7..35551874081 100644 --- a/.agents/skills/evaluation/recipes/tasks/aa_next/terminal_bench_2_1.md +++ b/.agents/skills/evaluation/recipes/tasks/aa_next/terminal_bench_2_1.md @@ -24,10 +24,11 @@ pinned via a vendored registry override in nemo-evaluator-next. | `cluster.container_env.AWS_DEFAULT_REGION` | match `sandbox.region` | | `max_concurrent` / `sandbox.concurrency` | `50` (canonical bench.yaml) | | timeout_strategy | `max` (canonical bench.yaml) + `agent_kwargs.llm_kwargs.timeout: 3600`; use `task` for leaderboard-comparable | -| `cluster.eval_image` requirement | **≥ `0.3.1.1-harbor`** — TB 2.1's task set is pinned via a vendored registry override in that image (`${NEL_NEXT_EVAL_IMAGE}`, multi-arch) | +| `cluster.eval_image` requirement | current internal source of truth: **`0.5.0.1-harbor`** (`${NEL_NEXT_EVAL_IMAGE}`, multi-arch) | These values mirror the canonical TB2.1 config — re-check it before a scored run: -`configs/benchmarks/nel_next/terminal_bench_21/bench.yaml` in +`configs/benchmarks/terminal-bench-2.1/bench.yaml` and +`configs/shared/nel_next_containers.yaml` in nvidia-eval-factory-benchmarking (see `references/nel-next.md` + the eval-config "source of truth" note). The `benchmarks:` block (drop into the example template): @@ -50,7 +51,8 @@ benchmarks: log_stream_prefix: terminalbench21-- ``` -`cluster.eval_image: ${NEL_NEXT_EVAL_IMAGE}` (≥ `0.3.1.1-harbor`) and the AWS creds +`cluster.eval_image: ${NEL_NEXT_EVAL_IMAGE}` (`0.5.0.1-harbor` in the current +internal source of truth) and the AWS creds come from `modelopttools:eval-config` (run it first) + the workspace `.env`. ## Score Extraction diff --git a/.agents/skills/evaluation/references/nel-next.md b/.agents/skills/evaluation/references/nel-next.md index 5d1975f4e2f..81d682fee68 100644 --- a/.agents/skills/evaluation/references/nel-next.md +++ b/.agents/skills/evaluation/references/nel-next.md @@ -131,9 +131,14 @@ with its own `run_id`, copying the shared `services:` block. ## Rules & gotchas -- **`eval_image`** = `${NEL_NEXT_EVAL_IMAGE}`. `0.3.1.1-harbor` is multi-arch and is - the minimum for **TB 2.1**; older `0.17.x/0.18.x-harbor-` are arch-suffixed. - Private gitlab-master image → cluster needs enroot creds (SKILL Step 7.5). +- **`eval_image`** = `${NEL_NEXT_EVAL_IMAGE}`. The current internal source of + truth pins `0.5.0.1-harbor` (multi-arch) for TB 2.1 and SWE-bench. Do not use + `0.3.1.1-harbor` for OpenHands: although it accepts + `agent_kwargs.llm_kwargs.timeout`, the client still uses its 300-second + default. The runtime must include + [NVIDIA-NeMo/Evaluator#1083](https://github.com/NVIDIA-NeMo/Evaluator/pull/1083), + which propagates that setting into OpenHands. Private gitlab-master images + need enroot creds (SKILL Step 7.5). - **Mount sources must pre-exist** — pyxis won't create the host side of a bind mount (invisible to `--dry-run`, fails at canary). `ssh 'mkdir -p //.cache/{vllm,huggingface}'`. @@ -160,6 +165,11 @@ $NEL eval {status|logs -f|report -f markdown|merge} -r # lifecycle $NEL mlflow-push -r -c .yaml # post-run: push merged bundle(s) to MLflow ``` +For a SWE-bench canary, inspect the OpenHands startup/agent log before scaling +out. The configured `agent_kwargs.llm_kwargs.timeout` must appear as the active +LLM timeout (for the recipe below, `3600`); `timeout: 300` means the eval image +does not contain the propagation fix and the canary is invalid. + `eval run` on a slurm cluster scp's the sbatch + redacted `.secrets.env` and submits via SSH; a built-in afternotok chain auto-resumes across walltime windows; sharded runs auto-merge. **SLURM does not auto-export** — `mlflow-push` is the final