From c7b332b3cb1b3e1ee308aac4a7b401d2d17b9c19 Mon Sep 17 00:00:00 2001 From: Nick Goncharenko Date: Mon, 10 Aug 2026 14:37:38 -0700 Subject: [PATCH 01/10] feat(evaluator): expose per-task attempt values Add `AgentEvalSummary.task_metric_values`: the ordered per-attempt values for each task, keyed `.`, persisted into `summary.json`. Answering "which tasks were flaky, and on which attempt?" previously meant regrouping the flat task x trial x metric score list by hand. Rebuild pass@k on top of that mapping instead of rescanning the scores, so the per-attempt view and the published pass@k figures cannot disagree. pass@k means are unchanged; a task that produced no trial at all now surfaces in `nan_count` rather than silently shrinking the denominator. Retention follows the declared output schema: continuous, discrete and boolean values are kept, while labels and free models (token measurements) stay out even when their emitted value happens to be numeric. Rework the gym `inspect_results.py` example to read the summary directly rather than re-deriving per-task outcomes from `scores.jsonl`. Signed-off-by: Nick Goncharenko --- .../nemo_evaluator_sdk/examples/gym/README.md | 5 +- .../examples/gym/inspect_results.py | 83 ++---- .../nemo_evaluator_sdk/agent_eval/results.py | 180 +++++++++--- .../tests/agent_eval/test_pass_at_k.py | 16 +- .../tests/agent_eval/test_persistence.py | 19 ++ .../agent_eval/test_task_metric_values.py | 278 ++++++++++++++++++ .../beta/evaluator/agent_eval/results.py | 180 +++++++++--- 7 files changed, 625 insertions(+), 136 deletions(-) create mode 100644 packages/nemo_evaluator_sdk/tests/agent_eval/test_task_metric_values.py diff --git a/packages/nemo_evaluator_sdk/examples/gym/README.md b/packages/nemo_evaluator_sdk/examples/gym/README.md index ca04ba40b9..a3f7661460 100644 --- a/packages/nemo_evaluator_sdk/examples/gym/README.md +++ b/packages/nemo_evaluator_sdk/examples/gym/README.md @@ -57,7 +57,10 @@ Run bundle (run.json, trials.jsonl, scores.jsonl, report.html): /var/folders/... ## Read the results -`inspect_results.py` is the companion to the above: it reads a bundle and shows how to reach each kind of result — headline aggregates, `pass@k`, per-task outcomes, and the runner's own imported numbers. Its accessors (`aggregate`, `per_task_outcomes`) are written to be lifted into your own code, and everything it shows also works on the in-memory `AgentEvalResult` that `AgentEvaluator().run(...)` returns — reading a bundle just makes it runnable without a live run. +`inspect_results.py` reads `summary.json` and shows each result layer: run aggregates from +`summary.scores`, ordered per-task attempt values from `summary.task_metric_values`, and runner-owned +aggregates under `runner.gym.*`. Per-task keys use `.`; a `null` attempt is a trial +that failed before scoring, while an empty list means the metric produced no usable measurement. No bundle is checked in; the run above produces one. Give it a stable `--output-dir` and point the reader at the same path: diff --git a/packages/nemo_evaluator_sdk/examples/gym/inspect_results.py b/packages/nemo_evaluator_sdk/examples/gym/inspect_results.py index fae62e617b..c8a3405513 100644 --- a/packages/nemo_evaluator_sdk/examples/gym/inspect_results.py +++ b/packages/nemo_evaluator_sdk/examples/gym/inspect_results.py @@ -29,11 +29,9 @@ import argparse import json -from collections.abc import Sequence from pathlib import Path from nemo_evaluator_sdk.agent_eval.results import AgentEvalSummary -from nemo_evaluator_sdk.agent_eval.scores import AgentEvalScoreStatus, AgentEvalTaskScore, is_trial_failure from nemo_evaluator_sdk.values.results import AggregateScalarScore, AggregateScore #: Value at which an attempt counts as a pass, matching the SDK's pass@k definition (full credit). @@ -61,34 +59,22 @@ def aggregate(summary: AgentEvalSummary, name: str) -> AggregateScore: def per_task_outcomes( - scores: Sequence[AgentEvalTaskScore], + summary: AgentEvalSummary, *, metric_type: str, output_name: str, ) -> dict[str, list[float | None]]: - """Group per-trial score values by task: ``task_id -> [value per attempt]``, ``None`` if it died. + """Read ordered attempt values from the summary for one metric output. - A run with ``num_repeats=R`` produces R trials per task, and the scores are a flat - task x trial x metric list — so answering "which tasks failed?" means grouping them yourself. - - Failed trials are kept, as ``None``. Dropping them would show a task that passed once and crashed - once as solved, and disagrees with how the SDK computes pass@k (a dead rollout is an attempt that - did not pass). A failed *metric* is dropped instead: it leaves the attempt unmeasured rather than - unsuccessful, so counting it against the agent would turn a judge timeout into a failure. + ``None`` is a failed trial and therefore a failed attempt. An empty list means the task had no + usable measurement because its metric failed or omitted the output. """ - by_task: dict[str, list[float | None]] = {} - for score in scores: - if score.metric_type != metric_type: - continue - if is_trial_failure(score): - by_task.setdefault(score.task_id, []).append(None) - continue - if score.status == AgentEvalScoreStatus.FAILED: - continue - for output in score.outputs: - if output.name == output_name and isinstance(output.value, int | float): - by_task.setdefault(score.task_id, []).append(float(output.value)) - return by_task + key = f"{metric_type}.{output_name}" + return { + task_id: list(metric_values[key]) + for task_id, metric_values in summary.task_metric_values.items() + if key in metric_values + } # -------------------------------------------------------------------------------------------------- @@ -96,19 +82,9 @@ def per_task_outcomes( # -------------------------------------------------------------------------------------------------- -def load_bundle(bundle: Path) -> tuple[AgentEvalSummary, list[AgentEvalTaskScore]]: - """Hydrate the pieces of a persisted run bundle used below. - - A runner's own numbers need no separate file: they are imported into ``summary.scores`` under - ``runner..``, so one load covers both. - """ - summary = AgentEvalSummary.model_validate(json.loads((bundle / "summary.json").read_text(encoding="utf-8"))) - scores = [ - AgentEvalTaskScore.model_validate(json.loads(line)) - for line in (bundle / "scores.jsonl").read_text(encoding="utf-8").splitlines() - if line.strip() - ] - return summary, scores +def load_bundle(bundle: Path) -> AgentEvalSummary: + """Load the persisted summary, including native and runner aggregates and per-task attempts.""" + return AgentEvalSummary.model_validate(json.loads((bundle / "summary.json").read_text(encoding="utf-8"))) # -------------------------------------------------------------------------------------------------- @@ -146,21 +122,26 @@ def show_per_task(by_task: dict[str, list[float | None]]) -> None: once and crashed once reads as flaky rather than solved. """ print("\nPer-task outcomes (attempt values; an attempt passes at full credit)") - solved = flaky = failed = 0 + solved = flaky = failed = unmeasured = 0 for task_id, values in sorted(by_task.items()): - passes = sum(1 for value in values if value is not None and value >= PASS_VALUE) - if passes == len(values): - verdict, marker = "solved", "+" - solved += 1 - elif passes: - verdict, marker = f"flaky ({passes}/{len(values)})", "~" - flaky += 1 + if not values: + verdict, marker = "unmeasured", "?" + unmeasured += 1 + attempts = "" else: - verdict, marker = "failed", "-" - failed += 1 - attempts = ", ".join("died" if value is None else f"{value:g}" for value in values) + passes = sum(1 for value in values if value is not None and value >= PASS_VALUE) + if passes == len(values): + verdict, marker = "solved", "+" + solved += 1 + elif passes: + verdict, marker = f"flaky ({passes}/{len(values)})", "~" + flaky += 1 + else: + verdict, marker = "failed", "-" + failed += 1 + attempts = ", ".join("died" if value is None else f"{value:g}" for value in values) print(f" {marker} {task_id[:16]}… [{attempts}] {verdict}") - print(f"\n {solved} solved · {flaky} flaky · {failed} failed") + print(f"\n {solved} solved · {flaky} flaky · {failed} failed · {unmeasured} unmeasured") def show_runner_aggregations(summary: AgentEvalSummary) -> None: @@ -206,10 +187,10 @@ def main(argv: list[str] | None = None) -> int: if not (args.bundle / "summary.json").exists(): raise SystemExit(f"{args.bundle} is not a run bundle (no summary.json). Run run_gym_eval.py first.") - summary, scores = load_bundle(args.bundle) + summary = load_bundle(args.bundle) show_aggregates(summary) - by_task = per_task_outcomes(scores, metric_type=args.metric_type, output_name=args.output_name) + by_task = per_task_outcomes(summary, metric_type=args.metric_type, output_name=args.output_name) if by_task: show_per_task(by_task) show_runner_aggregations(summary) diff --git a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/results.py b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/results.py index bf658e90d5..3852e8bf8d 100644 --- a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/results.py +++ b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/results.py @@ -23,7 +23,7 @@ from nemo_evaluator_sdk.metrics.aggregation import compute_percentiles from nemo_evaluator_sdk.metrics.protocol import MetricOutput from nemo_evaluator_sdk.metrics.utils import metric_type_name -from nemo_evaluator_sdk.values.protocol import BooleanValue, ContinuousScore +from nemo_evaluator_sdk.values.protocol import BooleanValue, ContinuousScore, DiscreteScore from nemo_evaluator_sdk.values.results import ( AggregatedMetricResult, AggregateRangeScore, @@ -36,6 +36,9 @@ ) from pydantic import BaseModel, ConfigDict, Field +#: Metric-output value schemas retained in the ordered per-task attempt-value mapping. +_TASK_METRIC_VALUE_SCHEMAS = (ContinuousScore, DiscreteScore, BooleanValue) + #: Metric-output value schemas eligible for pass@k (a per-attempt "did it pass?" signal). Labels, #: discrete/count outputs, and free models (e.g. token measurements) are excluded. _PASS_AT_K_VALUE_SCHEMAS = (ContinuousScore, BooleanValue) @@ -59,7 +62,7 @@ class AgentEvalMetricOutputCoverage(BaseModel): class AgentEvalSummary(BaseModel): - """Aggregated metric and semantic-view scores, coverage, and run counts for an agent-eval run.""" + """Aggregated scores, coverage, per-task attempt values, and run counts for an agent-eval run.""" model_config = ConfigDict(extra="forbid") @@ -75,6 +78,16 @@ class AgentEvalSummary(BaseModel): default_factory=dict, description="Per-metric, per-output coverage counts (total/scored/failed/missing).", ) + task_metric_values: dict[str, dict[str, list[float | None]]] = Field( + default_factory=dict, + description=( + "Per task, the values each '.' measured, in trial order. A failed " + "trial is None: an attempt that did not pass. An unmeasured attempt (metric failed, output " + "absent) has no entry, so each key's list is independent and positions align within a key, " + "not across keys. An empty list means nothing was measured, including a task that produced " + "no trial." + ), + ) task_count: int = Field(default=0, description="Number of tasks represented in the run.") trial_count: int = Field(default=0, description="Number of distinct trials scored.") score_count: int = Field(default=0, description="Total number of metric scores.") @@ -99,15 +112,22 @@ def from_scores( tasks: Sequence[AgentEvalTask] | None = None, extra_scores: Sequence[AggregateScore] = (), ) -> AgentEvalSummary: - """Build aggregated scores and coverage for a set of metric scores. + """Build aggregated scores, task values, and coverage for a set of metric scores. ``extra_scores`` are already-aggregated scores contributed by the runner (namespaced ``runner..``), merged in so a backend's own figures are addressable the same way as ours. """ task_list = list(tasks) if tasks is not None else None + task_metric_values = _task_metric_values(scores, task_list) return AgentEvalSummary( - scores=_aggregate_scores(scores, task_list, extra_scores), + scores=_aggregate_scores( + scores, + task_list, + extra_scores, + task_metric_values=task_metric_values, + ), metric_coverage=_metric_coverage(scores, task_list), + task_metric_values=task_metric_values, task_count=len(task_list) if task_list is not None else len({score.task_id for score in scores}), trial_count=len({score.trial_id for score in scores}), score_count=len(scores), @@ -455,6 +475,8 @@ def _aggregate_scores( scores: Sequence[AgentEvalTaskScore], tasks: Sequence[AgentEvalTask] | None, extra_scores: Sequence[AggregateScore] = (), + *, + task_metric_values: dict[str, dict[str, list[float | None]]] | None = None, ) -> AggregatedMetricResult: """Aggregate per-metric-output, per-semantic-view, and task-level pass@k values into range scores. @@ -487,7 +509,8 @@ def _aggregate_scores( for view_name, (values, total) in sorted(_semantic_view_values(scores, tasks).items()): aggregated.append(_aggregate_range_score(f"view.{view_name}", values, total)) - aggregated.extend(_task_pass_at_k_scores(scores, tasks)) + attempt_values = task_metric_values if task_metric_values is not None else _task_metric_values(scores, tasks) + aggregated.extend(_task_pass_at_k_scores(attempt_values, tasks)) aggregated.extend(extra_scores) return AggregatedMetricResult(scores=aggregated) @@ -526,9 +549,95 @@ def _scorelike_outputs(tasks: Sequence[AgentEvalTask] | None) -> set[tuple[str, return scorelike -def _task_pass_at_k_scores( +def _task_metric_values( scores: Sequence[AgentEvalTaskScore], tasks: Sequence[AgentEvalTask] | None, +) -> dict[str, dict[str, list[float | None]]]: + """Ordered per-attempt values per task, keyed ``.``. + + ``task-a`` declares ``reward.score`` (continuous), ``steps.count`` (discrete) and + ``usage.prompt_tokens`` (a free model) and runs four trials:: + + in t0 reward 1.0 steps 5 usage 1200 + t1 reward steps 9 usage 1300 # the judge died, not the agent + t2 # every metric fails as a trial failure + t3 reward 0.0 steps 7 usage 1100 + + out {"task-a": {"reward.score": [1.0, None, 0.0], + "steps.count": [5.0, 9.0, None, 7.0]}} + + ``usage.prompt_tokens`` is absent because its declared schema is not in + :data:`_TASK_METRIC_VALUE_SCHEMAS`; t1 is missing from ``reward.score`` but present in + ``steps.count``; t2 is ``None`` in both. + + Which keys a task gets: + + - declared by its metric spec under :data:`_TASK_METRIC_VALUE_SCHEMAS` -> kept + - declared under any other schema -> dropped, even when the emitted value is numeric, so a + ``MetricOutputSpec.model("prompt_tokens", TokenCount)`` measurement never becomes a key + - undeclared, but some score emitted a numeric value for it -> kept + - ``tasks is None`` -> no specs to filter against, so every numeric output observed is kept + + What each score contributes to its key, in trial order: + + - failed trial (:func:`is_trial_failure`) -> ``None``, an attempt that did not pass + - failed metric, or the output absent -> no entry; the attempt is unmeasured, not unsuccessful + - otherwise -> the numeric value + + pass@k needs that asymmetry, and it is why a list is indexed by surviving measurement rather than + by attempt: above, index 1 is t2 under ``reward.score`` but t1 under ``steps.count``. Compare + positions within a key, never across keys. + """ + output_keys: dict[str, set[tuple[str, str]]] = {} + # Declared under a schema this mapping does not retain. Tracked so an emitted numeric value cannot + # add back what the spec filter just excluded. + excluded: set[tuple[str, str]] = set() + if tasks is not None: + for task in tasks: + task_keys = output_keys.setdefault(task.id, set()) + for metric in task.metrics: + metric_type = metric_type_name(metric) + for spec in metric.output_spec(): + if issubclass(spec.value_schema, _TASK_METRIC_VALUE_SCHEMAS): + task_keys.add((metric_type, spec.name)) + else: + excluded.add((metric_type, spec.name)) + + scores_by_task_metric: dict[tuple[str, str], list[AgentEvalTaskScore]] = {} + for score in scores: + scores_by_task_metric.setdefault((score.task_id, score.metric_type), []).append(score) + task_keys = output_keys.setdefault(score.task_id, set()) + if score.status not in (AgentEvalScoreStatus.COMPLETED, AgentEvalScoreStatus.PARTIAL): + continue + for output in score.outputs: + if (score.metric_type, output.name) in excluded: + continue + if _semantic_value(output) is not None: + task_keys.add((score.metric_type, output.name)) + + by_task: dict[str, dict[str, list[float | None]]] = {} + for task_id, keys in output_keys.items(): + task_values: dict[str, list[float | None]] = {} + for metric_type, output_name in sorted(keys): + values: list[float | None] = [] + for score in scores_by_task_metric.get((task_id, metric_type), []): + if is_trial_failure(score): + values.append(None) + continue + if score.status not in (AgentEvalScoreStatus.COMPLETED, AgentEvalScoreStatus.PARTIAL): + continue + output = _score_output(score, output_name) + value = _semantic_value(output) if output is not None else None + if value is not None: + values.append(value) + task_values[f"{metric_type}.{output_name}"] = values + by_task[task_id] = task_values + return by_task + + +def _task_pass_at_k_scores( + task_metric_values: dict[str, dict[str, list[float | None]]], + tasks: Sequence[AgentEvalTask] | None, ) -> list[AggregateScore]: """Task-level pass@k over the R trials per task, aggregated across tasks (uniform for any runner). @@ -544,50 +653,39 @@ def _task_pass_at_k_scores( all drop out of the estimate and are reported as ``nan_count``, uniform across ``k``, so a shrinking denominator is never silent. (Tasks excluded from a given ``k`` merely for having fewer than ``k`` attempts are *not* counted there — that is the estimator working as defined, not missing data.) + + "No usable attempt" includes a task that was never scored at all: a runner that returns no trial + for a requested task (Harbor logs a warning and carries on) leaves it declaring the metric with an + empty attempt list, and it lands in ``nan_count`` like any other unmeasured task. That is + deliberate — it is the same missing coverage whether the trial died or was never produced, and + excluding it would report pass@k over a denominator quietly smaller than the task set asked for. """ scorelike = _scorelike_outputs(tasks) if not scorelike: return [] aggregated: list[AggregateScore] = [] for metric_type, output_name in sorted(scorelike): - attempts_and_passes: dict[str, list[int]] = {} # task_id -> [n_attempts, n_passes] - tasks_seen: set[str] = set() - for score in scores: - if score.metric_type != metric_type: - continue - tasks_seen.add(score.task_id) - if is_trial_failure(score): - # The agent produced nothing to score. That is an attempt that did not pass, not an - # attempt that did not happen -- dropping it would report pass@1 = 1.0 for a run whose - # other rollout died, and make pass@2 vanish along with the attempt that justified it. - attempts_and_passes.setdefault(score.task_id, [0, 0])[0] += 1 - continue - if score.status not in (AgentEvalScoreStatus.COMPLETED, AgentEvalScoreStatus.PARTIAL): - # The metric raised, so whether this attempt passed is unknown. Charging it to the agent - # would let a judge timeout read as a task the agent failed; it lands in nan_count. - continue - output = _score_output(score, output_name) - value = _semantic_value(output) if output is not None else None - if value is None: - continue - counts = attempts_and_passes.setdefault(score.task_id, [0, 0]) - counts[0] += 1 - if value >= _PASS_VALUE: - counts[1] += 1 - if not attempts_and_passes: + key = f"{metric_type}.{output_name}" + values_by_task = [outputs[key] for outputs in task_metric_values.values() if key in outputs] + measured = [values for values in values_by_task if values] + if not measured: continue - # Tasks that were scored for this metric but yielded no usable attempt whatsoever. Constant - # across k, so pass@1 and pass@8 agree on how much of the task set went unmeasured. - unmeasured = len(tasks_seen - set(attempts_and_passes)) - max_n = max(n for n, _ in attempts_and_passes.values()) + # Empty attempt lists stay in nan_count (via total); for each k, mean the unbiased + # estimator over tasks with n >= k (None / < full credit do not count as passes). + unmeasured = sum(not values for values in values_by_task) + max_n = max(len(values) for values in measured) for k in range(1, max_n + 1): - per_task = [_pass_at_k(n, c, k) for n, c in attempts_and_passes.values() if n >= k] - if per_task: - aggregated.append( - _aggregate_range_score( - f"{metric_type}.{output_name}.pass@{k}", per_task, len(per_task) + unmeasured - ) + per_task = [ + _pass_at_k( + len(values), + sum(value is not None and value >= _PASS_VALUE for value in values), + k, ) + for values in measured + if len(values) >= k + ] + if per_task: + aggregated.append(_aggregate_range_score(f"{key}.pass@{k}", per_task, len(per_task) + unmeasured)) return aggregated diff --git a/packages/nemo_evaluator_sdk/tests/agent_eval/test_pass_at_k.py b/packages/nemo_evaluator_sdk/tests/agent_eval/test_pass_at_k.py index 00d3bf2625..eb4957a3b0 100644 --- a/packages/nemo_evaluator_sdk/tests/agent_eval/test_pass_at_k.py +++ b/packages/nemo_evaluator_sdk/tests/agent_eval/test_pass_at_k.py @@ -111,6 +111,14 @@ def test_task_pass_at_k_gated_and_uniform_across_metric_types() -> None: summary = AgentEvalSummary.from_scores(scores, tasks=tasks) by_name = {score.name: score for score in summary.scores.scores} + assert summary.task_metric_values["t1"] == { + "gym_reward.reward": [1.0, 0.0], + "harbor_reward.reward": [1.0, 0.0], + } + assert summary.task_metric_values["t2"] == { + "gym_reward.reward": [1.0, 1.0], + "harbor_reward.reward": [1.0, 1.0], + } for metric_type in ("gym_reward", "harbor_reward"): # uniform across runners assert by_name[f"{metric_type}.reward.pass@1"].mean == pytest.approx(0.75) assert by_name[f"{metric_type}.reward.pass@2"].mean == pytest.approx(1.0) @@ -167,8 +175,10 @@ def test_a_failed_trial_is_a_failed_attempt_not_an_absent_one() -> None: _failed_score("t1", "a1", "reward", details={TRIAL_STATUS_DETAIL: "failed"}), ] - by_name = {s.name: s for s in AgentEvalSummary.from_scores(scores, tasks=tasks).scores.scores} + summary = AgentEvalSummary.from_scores(scores, tasks=tasks) + by_name = {s.name: s for s in summary.scores.scores} + assert summary.task_metric_values["t1"]["reward.reward"] == [1.0, None] assert by_name["reward.reward.pass@1"].mean == pytest.approx(0.5) # 1 of 2 attempts, not 1 of 1 assert by_name["reward.reward.pass@2"].mean == pytest.approx(1.0) assert by_name["reward.reward.pass@1"].nan_count == 0 # the task was measured, so nothing is missing @@ -186,8 +196,10 @@ def test_a_metric_that_raised_leaves_the_attempt_unmeasured_rather_than_failed() _score("t2", "a1", "reward", "reward", 0.0), ] - by_name = {s.name: s for s in AgentEvalSummary.from_scores(scores, tasks=tasks).scores.scores} + summary = AgentEvalSummary.from_scores(scores, tasks=tasks) + by_name = {s.name: s for s in summary.scores.scores} + assert summary.task_metric_values["t1"]["reward.reward"] == [1.0] assert by_name["reward.reward.pass@1"].mean == pytest.approx(0.75) # mean(1.0, 0.5), not mean(0.5, 0.5) # t1 drops out of pass@2 for having fewer than k attempts. That is the estimator working as defined, # not missing data, so it is not counted as nan. diff --git a/packages/nemo_evaluator_sdk/tests/agent_eval/test_persistence.py b/packages/nemo_evaluator_sdk/tests/agent_eval/test_persistence.py index 521e3c6d68..4e4c0e9ab0 100644 --- a/packages/nemo_evaluator_sdk/tests/agent_eval/test_persistence.py +++ b/packages/nemo_evaluator_sdk/tests/agent_eval/test_persistence.py @@ -84,6 +84,25 @@ def test_persist_run_writes_bundle_relative_refs_that_survive_a_move(tmp_path: P assert resolved.is_dir() and resolved == moved / "evidence" / "fabric" / "r" / "000000-taskA" / "workspace" +def test_persist_run_writes_per_task_attempt_values_to_summary(tmp_path: Path) -> None: + summary = AgentEvalSummary( + task_metric_values={ + "task-a": {"harbor_reward.reward": [1.0, 0.0]}, + "task-b": {"harbor_reward.reward": [1.0, None]}, + } + ) + result = AgentEvalResult(run_id="run", tasks=[], trials=[], scores=[], summary=summary) + + persist_run(result, tmp_path) + + payload = json.loads((tmp_path / "summary.json").read_text(encoding="utf-8")) + assert payload["task_metric_values"] == { + "task-a": {"harbor_reward.reward": [1.0, 0.0]}, + "task-b": {"harbor_reward.reward": [1.0, None]}, + } + assert AgentEvalSummary.model_validate(payload).task_metric_values == summary.task_metric_values + + def test_read_trials_rejects_evidence_ref_that_escapes_the_bundle(tmp_path: Path) -> None: # A copied/untrusted bundle whose ref uses `..` to escape the bundle must NOT be resolved to the # outside path: the rebuilt ref is accepted only when it stays inside run_dir. diff --git a/packages/nemo_evaluator_sdk/tests/agent_eval/test_task_metric_values.py b/packages/nemo_evaluator_sdk/tests/agent_eval/test_task_metric_values.py new file mode 100644 index 0000000000..6909dab70d --- /dev/null +++ b/packages/nemo_evaluator_sdk/tests/agent_eval/test_task_metric_values.py @@ -0,0 +1,278 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +from collections.abc import Iterator +from pathlib import Path + +from nemo_evaluator_sdk.agent_eval.results import AgentEvalSummary, _task_metric_values +from nemo_evaluator_sdk.agent_eval.scores import ( + TRIAL_STATUS_DETAIL, + AgentEvalDiagnostic, + AgentEvalDiagnosticSeverity, + AgentEvalScoreStatus, + AgentEvalTaskScore, +) +from nemo_evaluator_sdk.agent_eval.tasks import AgentEvalTask +from nemo_evaluator_sdk.metrics.protocol import MetricInput, MetricOutput, MetricResult +from nemo_evaluator_sdk.values.protocol import MetricOutputSpec +from pydantic import RootModel + + +class _TokenCount(RootModel[int]): + """A free-model output: numeric, but a measurement rather than a per-attempt score.""" + + +class _Metric: + def __init__(self, metric_type: str, output: MetricOutputSpec) -> None: + self._type = metric_type + self._output = output + + @property + def type(self) -> str: + return self._type + + def output_spec(self) -> list[MetricOutputSpec]: + return [self._output] + + async def compute_scores(self, input: MetricInput) -> MetricResult: # pragma: no cover + raise NotImplementedError + + +class _SinglePassScores(list[AgentEvalTaskScore]): + def __init__(self, scores: list[AgentEvalTaskScore]) -> None: + super().__init__(scores) + self.iterations = 0 + + def __iter__(self) -> Iterator[AgentEvalTaskScore]: + self.iterations += 1 + assert self.iterations == 1, "scores were rescanned" + return super().__iter__() + + +def _task(task_id: str, *metrics: _Metric) -> AgentEvalTask: + return AgentEvalTask(id=task_id, intent="test", inputs={}, metrics=list(metrics)) + + +def _score( + task_id: str, + trial_id: str, + metric_type: str, + output_name: str, + value: object, + *, + status: AgentEvalScoreStatus = AgentEvalScoreStatus.COMPLETED, +) -> AgentEvalTaskScore: + return AgentEvalTaskScore( + id=f"run:{task_id}:{trial_id}:{metric_type}", + run_id="run", + task_id=task_id, + trial_id=trial_id, + metric_type=metric_type, + status=status, + outputs=[MetricOutput(name=output_name, value=value)], + ) + + +def _failed_score(task_id: str, trial_id: str, *, trial_failed: bool) -> AgentEvalTaskScore: + details = {TRIAL_STATUS_DETAIL: "failed"} if trial_failed else {"exception_type": "TimeoutError"} + return AgentEvalTaskScore( + id=f"run:{task_id}:{trial_id}:reward", + run_id="run", + task_id=task_id, + trial_id=trial_id, + metric_type="reward", + status=AgentEvalScoreStatus.FAILED, + diagnostics=[ + AgentEvalDiagnostic( + severity=AgentEvalDiagnosticSeverity.ERROR, + message="failed", + details=details, + ) + ], + ) + + +def test_summary_exposes_ordered_numeric_attempt_values_per_task() -> None: + reward = _Metric("reward", MetricOutputSpec.continuous_score("score")) + retries = _Metric("retries", MetricOutputSpec.discrete_score("count")) + verdict = _Metric("verdict", MetricOutputSpec.label("label")) + complete = _Metric("complete", MetricOutputSpec.boolean("passed")) + tasks = [_task("task-a", reward, retries, verdict, complete), _task("task-b", reward)] + scores = [ + _score("task-a", "attempt-0", "reward", "score", 1.0), + _score("task-a", "attempt-0", "retries", "count", 2), + _score("task-a", "attempt-0", "verdict", "label", "good"), + _score("task-a", "attempt-0", "complete", "passed", True), + _score( + "task-a", + "attempt-1", + "reward", + "score", + 0.25, + status=AgentEvalScoreStatus.PARTIAL, + ), + _score("task-a", "attempt-1", "retries", "count", 3), + _score("task-a", "attempt-1", "complete", "passed", False), + _score("task-b", "attempt-0", "reward", "score", 0.0), + ] + + summary = AgentEvalSummary.from_scores(scores, tasks=tasks) + + assert summary.task_metric_values == { + "task-a": { + "complete.passed": [1.0, 0.0], + "retries.count": [2.0, 3.0], + "reward.score": [1.0, 0.25], + }, + "task-b": {"reward.score": [0.0]}, + } + + +def test_task_metric_values_scans_scores_once() -> None: + tasks = [_task("task-a", _Metric("reward", MetricOutputSpec.continuous_score("score")))] + scores = _SinglePassScores( + [ + _score("task-a", "attempt-0", "reward", "score", 1.0), + _score("task-a", "attempt-1", "reward", "score", 0.0), + ] + ) + + assert _task_metric_values(scores, tasks) == {"task-a": {"reward.score": [1.0, 0.0]}} + + +def test_failed_trials_are_attempts_but_metric_failures_are_unmeasured() -> None: + tasks = [ + _task("flaky", _Metric("reward", MetricOutputSpec.continuous_score("score"))), + _task("unmeasured", _Metric("reward", MetricOutputSpec.continuous_score("score"))), + ] + scores = [ + _score("flaky", "attempt-0", "reward", "score", 1.0), + _failed_score("flaky", "attempt-1", trial_failed=True), + _failed_score("unmeasured", "attempt-0", trial_failed=False), + ] + + summary = AgentEvalSummary.from_scores(scores, tasks=tasks) + + assert summary.task_metric_values == { + "flaky": {"reward.score": [1.0, None]}, + "unmeasured": {"reward.score": []}, + } + + +def test_a_task_that_produced_no_trial_is_unmeasured_and_counted_in_pass_at_k_nan() -> None: + # A runner may return no trial at all for a requested task (Harbor warns and carries on). The task + # still declares the metric, so it holds an empty attempt list and counts as missing coverage -- + # excluding it would report pass@k over a denominator smaller than the task set that was asked for. + reward = _Metric("reward", MetricOutputSpec.continuous_score("score")) + tasks = [_task("scored", reward), _task("never-ran", reward)] + scores = [_score("scored", "attempt-0", "reward", "score", 1.0)] + + summary = AgentEvalSummary.from_scores(scores, tasks=tasks) + by_name = {score.name: score for score in summary.scores.scores} + + assert summary.task_metric_values == { + "scored": {"reward.score": [1.0]}, + "never-ran": {"reward.score": []}, + } + assert by_name["reward.score.pass@1"].mean == 1.0 # the one measured task passed + assert by_name["reward.score.pass@1"].count == 1 + assert by_name["reward.score.pass@1"].nan_count == 1 # ...and the unrun one is not hidden + + +def test_outputs_declared_under_an_unretained_schema_stay_out_even_when_numeric() -> None: + # Token measurements and other free models are excluded by their declared schema. Emitting a + # numeric value must not add them back: the value is a measurement, not a per-attempt score. + tasks = [ + _task( + "task-a", + _Metric("reward", MetricOutputSpec.continuous_score("score")), + _Metric("usage", MetricOutputSpec.model("prompt_tokens", _TokenCount)), + ) + ] + scores = [ + _score("task-a", "attempt-0", "reward", "score", 1.0), + _score("task-a", "attempt-0", "usage", "prompt_tokens", 1234), + ] + + assert AgentEvalSummary.from_scores(scores, tasks=tasks).task_metric_values == {"task-a": {"reward.score": [1.0]}} + + +def test_without_tasks_there_is_no_spec_to_filter_on() -> None: + # No tasks means no declared schemas to consult, so every numeric output observed is retained. + scores = [_score("task-a", "attempt-0", "usage", "prompt_tokens", 1234)] + + assert AgentEvalSummary.from_scores(scores).task_metric_values == {"task-a": {"usage.prompt_tokens": [1234.0]}} + + +def test_attempt_positions_are_comparable_only_within_one_key() -> None: + # A metric that raised drops its attempt entirely while a dead trial holds its slot as None, so a + # metric failure shortens its own key's list without shortening its neighbour's. Index i of two + # keys is then two different trials -- there is no trial id to join on, and callers must not try. + tasks = [ + _task( + "task-a", + _Metric("reward", MetricOutputSpec.continuous_score("score")), + _Metric("steps", MetricOutputSpec.discrete_score("count")), + ) + ] + scores = [ + _score("task-a", "attempt-0", "reward", "score", 1.0), + _score("task-a", "attempt-0", "steps", "count", 5), + _failed_score("task-a", "attempt-1", trial_failed=False), # the reward judge timed out + _score("task-a", "attempt-1", "steps", "count", 9), + _score("task-a", "attempt-2", "reward", "score", 0.0), + _score("task-a", "attempt-2", "steps", "count", 7), + ] + + values = AgentEvalSummary.from_scores(scores, tasks=tasks).task_metric_values["task-a"] + + # Index 1 of reward.score is attempt-2; index 1 of steps.count is attempt-1. + assert values == {"reward.score": [1.0, 0.0], "steps.count": [5.0, 9.0, 7.0]} + + +def test_summary_without_task_metric_values_loads_as_empty() -> None: + assert AgentEvalSummary.model_validate({}).task_metric_values == {} + + +def test_vendored_summary_accepts_task_metric_values() -> None: + from nemo_platform.beta.evaluator.agent_eval.results import AgentEvalSummary as VendoredAgentEvalSummary + + payload = {"task_metric_values": {"task-a": {"reward.score": [1.0, None]}}} + + assert VendoredAgentEvalSummary.model_validate(payload).task_metric_values == payload["task_metric_values"] + + +def test_vendored_results_module_is_a_verbatim_copy_of_this_one() -> None: + # `make vendor` mirrors this module into the SDK, rewriting only the package root. Validating the + # field shape (above) would still pass against a stale copy carrying older filtering or docs, so + # pin the whole file: any edit here that is not mirrored is drift between two live code paths. + import nemo_evaluator_sdk.agent_eval.results as source + import nemo_platform.beta.evaluator.agent_eval.results as vendored + + expected = ( + Path(source.__file__) + .read_text(encoding="utf-8") + .replace("from nemo_evaluator_sdk.", "from nemo_platform.beta.evaluator.") + ) + + assert Path(vendored.__file__).read_text(encoding="utf-8") == expected, ( + "sdk/python/.../beta/evaluator/agent_eval/results.py is out of sync; re-run `make vendor`" + ) + + +def test_gym_example_reads_task_outcomes_from_summary() -> None: + from packages.nemo_evaluator_sdk.examples.gym.inspect_results import per_task_outcomes + + summary = AgentEvalSummary( + task_metric_values={ + "task-a": {"gym_reward.reward": [1.0, 0.0]}, + "task-b": {"gym_reward.reward": []}, + } + ) + + assert per_task_outcomes(summary, metric_type="gym_reward", output_name="reward") == { + "task-a": [1.0, 0.0], + "task-b": [], + } diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/results.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/results.py index 2846465c06..ed13c26973 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/results.py +++ b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/results.py @@ -23,7 +23,7 @@ from nemo_platform.beta.evaluator.metrics.aggregation import compute_percentiles from nemo_platform.beta.evaluator.metrics.protocol import MetricOutput from nemo_platform.beta.evaluator.metrics.utils import metric_type_name -from nemo_platform.beta.evaluator.values.protocol import BooleanValue, ContinuousScore +from nemo_platform.beta.evaluator.values.protocol import BooleanValue, ContinuousScore, DiscreteScore from nemo_platform.beta.evaluator.values.results import ( AggregatedMetricResult, AggregateRangeScore, @@ -36,6 +36,9 @@ ) from pydantic import BaseModel, ConfigDict, Field +#: Metric-output value schemas retained in the ordered per-task attempt-value mapping. +_TASK_METRIC_VALUE_SCHEMAS = (ContinuousScore, DiscreteScore, BooleanValue) + #: Metric-output value schemas eligible for pass@k (a per-attempt "did it pass?" signal). Labels, #: discrete/count outputs, and free models (e.g. token measurements) are excluded. _PASS_AT_K_VALUE_SCHEMAS = (ContinuousScore, BooleanValue) @@ -59,7 +62,7 @@ class AgentEvalMetricOutputCoverage(BaseModel): class AgentEvalSummary(BaseModel): - """Aggregated metric and semantic-view scores, coverage, and run counts for an agent-eval run.""" + """Aggregated scores, coverage, per-task attempt values, and run counts for an agent-eval run.""" model_config = ConfigDict(extra="forbid") @@ -75,6 +78,16 @@ class AgentEvalSummary(BaseModel): default_factory=dict, description="Per-metric, per-output coverage counts (total/scored/failed/missing).", ) + task_metric_values: dict[str, dict[str, list[float | None]]] = Field( + default_factory=dict, + description=( + "Per task, the values each '.' measured, in trial order. A failed " + "trial is None: an attempt that did not pass. An unmeasured attempt (metric failed, output " + "absent) has no entry, so each key's list is independent and positions align within a key, " + "not across keys. An empty list means nothing was measured, including a task that produced " + "no trial." + ), + ) task_count: int = Field(default=0, description="Number of tasks represented in the run.") trial_count: int = Field(default=0, description="Number of distinct trials scored.") score_count: int = Field(default=0, description="Total number of metric scores.") @@ -99,15 +112,22 @@ def from_scores( tasks: Sequence[AgentEvalTask] | None = None, extra_scores: Sequence[AggregateScore] = (), ) -> AgentEvalSummary: - """Build aggregated scores and coverage for a set of metric scores. + """Build aggregated scores, task values, and coverage for a set of metric scores. ``extra_scores`` are already-aggregated scores contributed by the runner (namespaced ``runner..``), merged in so a backend's own figures are addressable the same way as ours. """ task_list = list(tasks) if tasks is not None else None + task_metric_values = _task_metric_values(scores, task_list) return AgentEvalSummary( - scores=_aggregate_scores(scores, task_list, extra_scores), + scores=_aggregate_scores( + scores, + task_list, + extra_scores, + task_metric_values=task_metric_values, + ), metric_coverage=_metric_coverage(scores, task_list), + task_metric_values=task_metric_values, task_count=len(task_list) if task_list is not None else len({score.task_id for score in scores}), trial_count=len({score.trial_id for score in scores}), score_count=len(scores), @@ -455,6 +475,8 @@ def _aggregate_scores( scores: Sequence[AgentEvalTaskScore], tasks: Sequence[AgentEvalTask] | None, extra_scores: Sequence[AggregateScore] = (), + *, + task_metric_values: dict[str, dict[str, list[float | None]]] | None = None, ) -> AggregatedMetricResult: """Aggregate per-metric-output, per-semantic-view, and task-level pass@k values into range scores. @@ -487,7 +509,8 @@ def _aggregate_scores( for view_name, (values, total) in sorted(_semantic_view_values(scores, tasks).items()): aggregated.append(_aggregate_range_score(f"view.{view_name}", values, total)) - aggregated.extend(_task_pass_at_k_scores(scores, tasks)) + attempt_values = task_metric_values if task_metric_values is not None else _task_metric_values(scores, tasks) + aggregated.extend(_task_pass_at_k_scores(attempt_values, tasks)) aggregated.extend(extra_scores) return AggregatedMetricResult(scores=aggregated) @@ -526,9 +549,95 @@ def _scorelike_outputs(tasks: Sequence[AgentEvalTask] | None) -> set[tuple[str, return scorelike -def _task_pass_at_k_scores( +def _task_metric_values( scores: Sequence[AgentEvalTaskScore], tasks: Sequence[AgentEvalTask] | None, +) -> dict[str, dict[str, list[float | None]]]: + """Ordered per-attempt values per task, keyed ``.``. + + ``task-a`` declares ``reward.score`` (continuous), ``steps.count`` (discrete) and + ``usage.prompt_tokens`` (a free model) and runs four trials:: + + in t0 reward 1.0 steps 5 usage 1200 + t1 reward steps 9 usage 1300 # the judge died, not the agent + t2 # every metric fails as a trial failure + t3 reward 0.0 steps 7 usage 1100 + + out {"task-a": {"reward.score": [1.0, None, 0.0], + "steps.count": [5.0, 9.0, None, 7.0]}} + + ``usage.prompt_tokens`` is absent because its declared schema is not in + :data:`_TASK_METRIC_VALUE_SCHEMAS`; t1 is missing from ``reward.score`` but present in + ``steps.count``; t2 is ``None`` in both. + + Which keys a task gets: + + - declared by its metric spec under :data:`_TASK_METRIC_VALUE_SCHEMAS` -> kept + - declared under any other schema -> dropped, even when the emitted value is numeric, so a + ``MetricOutputSpec.model("prompt_tokens", TokenCount)`` measurement never becomes a key + - undeclared, but some score emitted a numeric value for it -> kept + - ``tasks is None`` -> no specs to filter against, so every numeric output observed is kept + + What each score contributes to its key, in trial order: + + - failed trial (:func:`is_trial_failure`) -> ``None``, an attempt that did not pass + - failed metric, or the output absent -> no entry; the attempt is unmeasured, not unsuccessful + - otherwise -> the numeric value + + pass@k needs that asymmetry, and it is why a list is indexed by surviving measurement rather than + by attempt: above, index 1 is t2 under ``reward.score`` but t1 under ``steps.count``. Compare + positions within a key, never across keys. + """ + output_keys: dict[str, set[tuple[str, str]]] = {} + # Declared under a schema this mapping does not retain. Tracked so an emitted numeric value cannot + # add back what the spec filter just excluded. + excluded: set[tuple[str, str]] = set() + if tasks is not None: + for task in tasks: + task_keys = output_keys.setdefault(task.id, set()) + for metric in task.metrics: + metric_type = metric_type_name(metric) + for spec in metric.output_spec(): + if issubclass(spec.value_schema, _TASK_METRIC_VALUE_SCHEMAS): + task_keys.add((metric_type, spec.name)) + else: + excluded.add((metric_type, spec.name)) + + scores_by_task_metric: dict[tuple[str, str], list[AgentEvalTaskScore]] = {} + for score in scores: + scores_by_task_metric.setdefault((score.task_id, score.metric_type), []).append(score) + task_keys = output_keys.setdefault(score.task_id, set()) + if score.status not in (AgentEvalScoreStatus.COMPLETED, AgentEvalScoreStatus.PARTIAL): + continue + for output in score.outputs: + if (score.metric_type, output.name) in excluded: + continue + if _semantic_value(output) is not None: + task_keys.add((score.metric_type, output.name)) + + by_task: dict[str, dict[str, list[float | None]]] = {} + for task_id, keys in output_keys.items(): + task_values: dict[str, list[float | None]] = {} + for metric_type, output_name in sorted(keys): + values: list[float | None] = [] + for score in scores_by_task_metric.get((task_id, metric_type), []): + if is_trial_failure(score): + values.append(None) + continue + if score.status not in (AgentEvalScoreStatus.COMPLETED, AgentEvalScoreStatus.PARTIAL): + continue + output = _score_output(score, output_name) + value = _semantic_value(output) if output is not None else None + if value is not None: + values.append(value) + task_values[f"{metric_type}.{output_name}"] = values + by_task[task_id] = task_values + return by_task + + +def _task_pass_at_k_scores( + task_metric_values: dict[str, dict[str, list[float | None]]], + tasks: Sequence[AgentEvalTask] | None, ) -> list[AggregateScore]: """Task-level pass@k over the R trials per task, aggregated across tasks (uniform for any runner). @@ -544,50 +653,39 @@ def _task_pass_at_k_scores( all drop out of the estimate and are reported as ``nan_count``, uniform across ``k``, so a shrinking denominator is never silent. (Tasks excluded from a given ``k`` merely for having fewer than ``k`` attempts are *not* counted there — that is the estimator working as defined, not missing data.) + + "No usable attempt" includes a task that was never scored at all: a runner that returns no trial + for a requested task (Harbor logs a warning and carries on) leaves it declaring the metric with an + empty attempt list, and it lands in ``nan_count`` like any other unmeasured task. That is + deliberate — it is the same missing coverage whether the trial died or was never produced, and + excluding it would report pass@k over a denominator quietly smaller than the task set asked for. """ scorelike = _scorelike_outputs(tasks) if not scorelike: return [] aggregated: list[AggregateScore] = [] for metric_type, output_name in sorted(scorelike): - attempts_and_passes: dict[str, list[int]] = {} # task_id -> [n_attempts, n_passes] - tasks_seen: set[str] = set() - for score in scores: - if score.metric_type != metric_type: - continue - tasks_seen.add(score.task_id) - if is_trial_failure(score): - # The agent produced nothing to score. That is an attempt that did not pass, not an - # attempt that did not happen -- dropping it would report pass@1 = 1.0 for a run whose - # other rollout died, and make pass@2 vanish along with the attempt that justified it. - attempts_and_passes.setdefault(score.task_id, [0, 0])[0] += 1 - continue - if score.status not in (AgentEvalScoreStatus.COMPLETED, AgentEvalScoreStatus.PARTIAL): - # The metric raised, so whether this attempt passed is unknown. Charging it to the agent - # would let a judge timeout read as a task the agent failed; it lands in nan_count. - continue - output = _score_output(score, output_name) - value = _semantic_value(output) if output is not None else None - if value is None: - continue - counts = attempts_and_passes.setdefault(score.task_id, [0, 0]) - counts[0] += 1 - if value >= _PASS_VALUE: - counts[1] += 1 - if not attempts_and_passes: + key = f"{metric_type}.{output_name}" + values_by_task = [outputs[key] for outputs in task_metric_values.values() if key in outputs] + measured = [values for values in values_by_task if values] + if not measured: continue - # Tasks that were scored for this metric but yielded no usable attempt whatsoever. Constant - # across k, so pass@1 and pass@8 agree on how much of the task set went unmeasured. - unmeasured = len(tasks_seen - set(attempts_and_passes)) - max_n = max(n for n, _ in attempts_and_passes.values()) + # Empty attempt lists stay in nan_count (via total); for each k, mean the unbiased + # estimator over tasks with n >= k (None / < full credit do not count as passes). + unmeasured = sum(not values for values in values_by_task) + max_n = max(len(values) for values in measured) for k in range(1, max_n + 1): - per_task = [_pass_at_k(n, c, k) for n, c in attempts_and_passes.values() if n >= k] - if per_task: - aggregated.append( - _aggregate_range_score( - f"{metric_type}.{output_name}.pass@{k}", per_task, len(per_task) + unmeasured - ) + per_task = [ + _pass_at_k( + len(values), + sum(value is not None and value >= _PASS_VALUE for value in values), + k, ) + for values in measured + if len(values) >= k + ] + if per_task: + aggregated.append(_aggregate_range_score(f"{key}.pass@{k}", per_task, len(per_task) + unmeasured)) return aggregated From 04c799e20a0df092d58149ff56513514ccabca59 Mon Sep 17 00:00:00 2001 From: Nick Goncharenko Date: Tue, 11 Aug 2026 11:50:46 -0700 Subject: [PATCH 02/10] feat: add AgentEvalAttemptValue Signed-off-by: Nick Goncharenko --- docs/evaluator/agent-eval/reading-results.mdx | 15 ++ .../nemo_evaluator_sdk/examples/gym/README.md | 6 +- .../examples/gym/inspect_results.py | 28 ++- .../nemo_evaluator_sdk/agent_eval/results.py | 100 ++++++--- .../tests/agent_eval/test_harbor_runtime.py | 99 +++++++++ .../tests/agent_eval/test_pass_at_k.py | 10 +- .../tests/agent_eval/test_persistence.py | 63 +++++- ...values.py => test_task_metric_attempts.py} | 196 +++++++++++++++--- .../beta/evaluator/agent_eval/results.py | 98 ++++++--- 9 files changed, 510 insertions(+), 105 deletions(-) rename packages/nemo_evaluator_sdk/tests/agent_eval/{test_task_metric_values.py => test_task_metric_attempts.py} (50%) diff --git a/docs/evaluator/agent-eval/reading-results.mdx b/docs/evaluator/agent-eval/reading-results.mdx index 108fe55b9b..53bf03dceb 100644 --- a/docs/evaluator/agent-eval/reading-results.mdx +++ b/docs/evaluator/agent-eval/reading-results.mdx @@ -37,6 +37,21 @@ result = await AgentEvaluator().run(tasks=..., target=...) - **`summary.metric_coverage`** — per metric output, how many trials were `total` / `scored` / failed / missing, so you can tell a low mean from low coverage. +- **`summary.task_metric_attempts`** — per task, the individual attempts behind those means, keyed + `.`. Each attempt carries the `trial_id` that produced it and its `value`, so + you can answer "which tasks were flaky, and on which attempt?" without regrouping `result.scores` + yourself: + + ```python + for task_id, by_output in result.summary.task_metric_attempts.items(): + # .get: keys are per task, so a task scored by a different metric simply has none. + print(task_id, [(a.trial_id, a.value) for a in by_output.get("reward.score", [])]) + ``` + + A `value` of `None` is a trial that died before scoring — an attempt that did not pass. An attempt + whose *metric* failed is absent entirely, because that leaves it unmeasured rather than + unsuccessful. Join by `trial_id` rather than by position: the two rules above mean lists for + different outputs of one task need not be the same length. - **`summary.task_count`**, **`summary.trial_count`**, **`summary.score_count`**. ### Per-metric scores diff --git a/packages/nemo_evaluator_sdk/examples/gym/README.md b/packages/nemo_evaluator_sdk/examples/gym/README.md index a3f7661460..a7d39d6e24 100644 --- a/packages/nemo_evaluator_sdk/examples/gym/README.md +++ b/packages/nemo_evaluator_sdk/examples/gym/README.md @@ -58,10 +58,14 @@ Run bundle (run.json, trials.jsonl, scores.jsonl, report.html): /var/folders/... ## Read the results `inspect_results.py` reads `summary.json` and shows each result layer: run aggregates from -`summary.scores`, ordered per-task attempt values from `summary.task_metric_values`, and runner-owned +`summary.scores`, ordered per-task attempts from `summary.task_metric_attempts`, and runner-owned aggregates under `runner.gym.*`. Per-task keys use `.`; a `null` attempt is a trial that failed before scoring, while an empty list means the metric produced no usable measurement. +Each attempt names the trial that produced it, so `trial_id` — not list position — is what joins two +outputs of the same task, or joins out to `trials.jsonl`. An attempt whose metric failed is absent +rather than `null`, so two lists for one task need not be the same length. + No bundle is checked in; the run above produces one. Give it a stable `--output-dir` and point the reader at the same path: ```bash diff --git a/packages/nemo_evaluator_sdk/examples/gym/inspect_results.py b/packages/nemo_evaluator_sdk/examples/gym/inspect_results.py index c8a3405513..6a11b1b756 100644 --- a/packages/nemo_evaluator_sdk/examples/gym/inspect_results.py +++ b/packages/nemo_evaluator_sdk/examples/gym/inspect_results.py @@ -31,7 +31,7 @@ import json from pathlib import Path -from nemo_evaluator_sdk.agent_eval.results import AgentEvalSummary +from nemo_evaluator_sdk.agent_eval.results import AgentEvalAttemptValue, AgentEvalSummary, attempt_values from nemo_evaluator_sdk.values.results import AggregateScalarScore, AggregateScore #: Value at which an attempt counts as a pass, matching the SDK's pass@k definition (full credit). @@ -68,12 +68,32 @@ def per_task_outcomes( ``None`` is a failed trial and therefore a failed attempt. An empty list means the task had no usable measurement because its metric failed or omitted the output. + + Use :func:`per_task_attempts` when you need to know *which* trial produced a value. + """ + return { + task_id: attempt_values(attempts) + for task_id, attempts in per_task_attempts(summary, metric_type=metric_type, output_name=output_name).items() + } + + +def per_task_attempts( + summary: AgentEvalSummary, + *, + metric_type: str, + output_name: str, +) -> dict[str, list[AgentEvalAttemptValue]]: + """The same attempts, each still naming the trial that produced it. + + An attempt whose metric failed is absent rather than null, so lists for two different outputs of + one task need not be the same length — ``trial_id``, not position, is what lines them up. It is + also the join key out to ``trials.jsonl``, which is where a failed attempt's error lives. """ key = f"{metric_type}.{output_name}" return { - task_id: list(metric_values[key]) - for task_id, metric_values in summary.task_metric_values.items() - if key in metric_values + task_id: list(metric_attempts[key]) + for task_id, metric_attempts in summary.task_metric_attempts.items() + if key in metric_attempts } diff --git a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/results.py b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/results.py index 3852e8bf8d..d406d6ba2f 100644 --- a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/results.py +++ b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/results.py @@ -36,7 +36,7 @@ ) from pydantic import BaseModel, ConfigDict, Field -#: Metric-output value schemas retained in the ordered per-task attempt-value mapping. +#: Metric-output value schemas retained in the ordered per-task attempt mapping. _TASK_METRIC_VALUE_SCHEMAS = (ContinuousScore, DiscreteScore, BooleanValue) #: Metric-output value schemas eligible for pass@k (a per-attempt "did it pass?" signal). Labels, @@ -61,6 +61,32 @@ class AgentEvalMetricOutputCoverage(BaseModel): missing: int = Field(default=0, description="Scores where the output was expected but absent.") +class AgentEvalAttemptValue(BaseModel): + """One attempt at a task under one metric output: which trial made it, and what it measured. + + Frozen because these records are handed out by reference from the summary: a consumer rescaling + values in place (Gym reports reward on 0-100 where we use 0-1) would otherwise rewrite the run's + own results, and a later persist would save the rewrite. + """ + + model_config = ConfigDict(extra="forbid", frozen=True) + + trial_id: str = Field( + description=( + "Identifier of the trial that made this attempt. Joins to AgentEvalTrial.id " + "(trials.jsonl) and AgentEvalTaskScore.trial_id (scores.jsonl)." + ) + ) + value: float | None = Field( + description=( + "What the metric output measured, or None when the trial failed before it could be " + "measured -- an attempt that did not pass. Required rather than defaulted: None is a " + "load-bearing signal pass@k counts as a failed attempt, so an omitted value must not " + "quietly become one." + ), + ) + + class AgentEvalSummary(BaseModel): """Aggregated scores, coverage, per-task attempt values, and run counts for an agent-eval run.""" @@ -78,14 +104,15 @@ class AgentEvalSummary(BaseModel): default_factory=dict, description="Per-metric, per-output coverage counts (total/scored/failed/missing).", ) - task_metric_values: dict[str, dict[str, list[float | None]]] = Field( + task_metric_attempts: dict[str, dict[str, list[AgentEvalAttemptValue]]] = Field( default_factory=dict, description=( - "Per task, the values each '.' measured, in trial order. A failed " - "trial is None: an attempt that did not pass. An unmeasured attempt (metric failed, output " - "absent) has no entry, so each key's list is independent and positions align within a key, " - "not across keys. An empty list means nothing was measured, including a task that produced " - "no trial." + "Per task, the attempts each '.' measured, in trial order. Each " + "attempt names the trial that made it, so attempts join across keys -- and out to " + "trials.jsonl and scores.jsonl -- by trial_id. A failed trial has value None: an attempt " + "that did not pass. An unmeasured attempt (metric failed, output absent) has no entry at " + "all, so each key's list is independent: align by trial_id, never by position. An empty " + "list means nothing was measured, including a task that produced no trial." ), ) task_count: int = Field(default=0, description="Number of tasks represented in the run.") @@ -118,16 +145,16 @@ def from_scores( ``runner..``), merged in so a backend's own figures are addressable the same way as ours. """ task_list = list(tasks) if tasks is not None else None - task_metric_values = _task_metric_values(scores, task_list) + task_metric_attempts = _task_metric_attempts(scores, task_list) return AgentEvalSummary( scores=_aggregate_scores( scores, task_list, extra_scores, - task_metric_values=task_metric_values, + task_metric_attempts=task_metric_attempts, ), metric_coverage=_metric_coverage(scores, task_list), - task_metric_values=task_metric_values, + task_metric_attempts=task_metric_attempts, task_count=len(task_list) if task_list is not None else len({score.task_id for score in scores}), trial_count=len({score.trial_id for score in scores}), score_count=len(scores), @@ -476,7 +503,7 @@ def _aggregate_scores( tasks: Sequence[AgentEvalTask] | None, extra_scores: Sequence[AggregateScore] = (), *, - task_metric_values: dict[str, dict[str, list[float | None]]] | None = None, + task_metric_attempts: dict[str, dict[str, list[AgentEvalAttemptValue]]] | None = None, ) -> AggregatedMetricResult: """Aggregate per-metric-output, per-semantic-view, and task-level pass@k values into range scores. @@ -509,13 +536,25 @@ def _aggregate_scores( for view_name, (values, total) in sorted(_semantic_view_values(scores, tasks).items()): aggregated.append(_aggregate_range_score(f"view.{view_name}", values, total)) - attempt_values = task_metric_values if task_metric_values is not None else _task_metric_values(scores, tasks) - aggregated.extend(_task_pass_at_k_scores(attempt_values, tasks)) + # if the caller already passed attempts → use them (no second scan of all scores) + # if not (None) → compute them inside _aggregate_scores (no need to pass them in) + attempts = task_metric_attempts if task_metric_attempts is not None else _task_metric_attempts(scores, tasks) + aggregated.extend(_task_pass_at_k_scores(attempts, tasks)) aggregated.extend(extra_scores) return AggregatedMetricResult(scores=aggregated) +def attempt_values(attempts: Sequence[AgentEvalAttemptValue]) -> list[float | None]: + """The bare per-attempt values, for consumers scoring attempts without caring which trial made them. + + Preserves order, cardinality, and the None-versus-absent distinction exactly as recorded, so + anything counting attempts (pass@k above all) reads the same sequence it would have read before + attempts carried a trial id. + """ + return [attempt.value for attempt in attempts] + + def _pass_at_k(n: int, c: int, k: int) -> float: """Unbiased pass@k estimator (Chen et al., 2021): ``1 - C(n-c, k) / C(n, k)``. @@ -549,11 +588,11 @@ def _scorelike_outputs(tasks: Sequence[AgentEvalTask] | None) -> set[tuple[str, return scorelike -def _task_metric_values( +def _task_metric_attempts( scores: Sequence[AgentEvalTaskScore], tasks: Sequence[AgentEvalTask] | None, -) -> dict[str, dict[str, list[float | None]]]: - """Ordered per-attempt values per task, keyed ``.``. +) -> dict[str, dict[str, list[AgentEvalAttemptValue]]]: + """Ordered per-attempt records per task, keyed ``.``. ``task-a`` declares ``reward.score`` (continuous), ``steps.count`` (discrete) and ``usage.prompt_tokens`` (a free model) and runs four trials:: @@ -563,8 +602,10 @@ def _task_metric_values( t2 # every metric fails as a trial failure t3 reward 0.0 steps 7 usage 1100 - out {"task-a": {"reward.score": [1.0, None, 0.0], - "steps.count": [5.0, 9.0, None, 7.0]}} + out {"task-a": {"reward.score": [(t0, 1.0), (t2, None), (t3, 0.0)], + "steps.count": [(t0, 5.0), (t1, 9.0), (t2, None), (t3, 7.0)]}} + + (shown as ``(trial_id, value)`` pairs; each is an :class:`AgentEvalAttemptValue`) ``usage.prompt_tokens`` is absent because its declared schema is not in :data:`_TASK_METRIC_VALUE_SCHEMAS`; t1 is missing from ``reward.score`` but present in @@ -580,13 +621,16 @@ def _task_metric_values( What each score contributes to its key, in trial order: - - failed trial (:func:`is_trial_failure`) -> ``None``, an attempt that did not pass + - failed trial (:func:`is_trial_failure`) -> value ``None``, an attempt that did not pass - failed metric, or the output absent -> no entry; the attempt is unmeasured, not unsuccessful - otherwise -> the numeric value pass@k needs that asymmetry, and it is why a list is indexed by surviving measurement rather than - by attempt: above, index 1 is t2 under ``reward.score`` but t1 under ``steps.count``. Compare - positions within a key, never across keys. + by attempt: above, index 1 is t2 under ``reward.score`` but t1 under ``steps.count``. Every entry + therefore names its trial, and ``trial_id`` — not position — is what joins two keys of one task, + or joins out to ``trials.jsonl`` and ``scores.jsonl``. Ids are recorded as the runner reported + them and are never deduplicated: two attempts sharing an id stay two attempts, so a runner that + reuses one costs pass@k nothing. """ output_keys: dict[str, set[tuple[str, str]]] = {} # Declared under a schema this mapping does not retain. Tracked so an emitted numeric value cannot @@ -615,28 +659,28 @@ def _task_metric_values( if _semantic_value(output) is not None: task_keys.add((score.metric_type, output.name)) - by_task: dict[str, dict[str, list[float | None]]] = {} + by_task: dict[str, dict[str, list[AgentEvalAttemptValue]]] = {} for task_id, keys in output_keys.items(): - task_values: dict[str, list[float | None]] = {} + task_values: dict[str, list[AgentEvalAttemptValue]] = {} for metric_type, output_name in sorted(keys): - values: list[float | None] = [] + values: list[AgentEvalAttemptValue] = [] for score in scores_by_task_metric.get((task_id, metric_type), []): if is_trial_failure(score): - values.append(None) + values.append(AgentEvalAttemptValue(trial_id=score.trial_id, value=None)) continue if score.status not in (AgentEvalScoreStatus.COMPLETED, AgentEvalScoreStatus.PARTIAL): continue output = _score_output(score, output_name) value = _semantic_value(output) if output is not None else None if value is not None: - values.append(value) + values.append(AgentEvalAttemptValue(trial_id=score.trial_id, value=value)) task_values[f"{metric_type}.{output_name}"] = values by_task[task_id] = task_values return by_task def _task_pass_at_k_scores( - task_metric_values: dict[str, dict[str, list[float | None]]], + task_metric_attempts: dict[str, dict[str, list[AgentEvalAttemptValue]]], tasks: Sequence[AgentEvalTask] | None, ) -> list[AggregateScore]: """Task-level pass@k over the R trials per task, aggregated across tasks (uniform for any runner). @@ -666,7 +710,7 @@ def _task_pass_at_k_scores( aggregated: list[AggregateScore] = [] for metric_type, output_name in sorted(scorelike): key = f"{metric_type}.{output_name}" - values_by_task = [outputs[key] for outputs in task_metric_values.values() if key in outputs] + values_by_task = [attempt_values(outputs[key]) for outputs in task_metric_attempts.values() if key in outputs] measured = [values for values in values_by_task if values] if not measured: continue diff --git a/packages/nemo_evaluator_sdk/tests/agent_eval/test_harbor_runtime.py b/packages/nemo_evaluator_sdk/tests/agent_eval/test_harbor_runtime.py index bf19d2ac02..2a88607210 100644 --- a/packages/nemo_evaluator_sdk/tests/agent_eval/test_harbor_runtime.py +++ b/packages/nemo_evaluator_sdk/tests/agent_eval/test_harbor_runtime.py @@ -15,6 +15,7 @@ import pytest from nemo_evaluator_sdk.agent_eval.evaluator import AgentEvaluator +from nemo_evaluator_sdk.agent_eval.results import AgentEvalSummary from nemo_evaluator_sdk.agent_eval.runtimes.harbor_runtime import ( HarborAgentTaskRunner, HarborRewardMetric, @@ -105,6 +106,104 @@ async def _record(calls: list[str]) -> None: calls.append("ran") +def _reward_details_from_summary( + summary: AgentEvalSummary, + *, + metric_type: str = "harbor_reward", + output_name: str = "reward", +) -> dict[str, dict[str, list[str]]]: + """Rebuild ``reward_payload_from_result``'s ``reward_details`` from the summary alone. + + Inverts ``{task_id: [attempt]}`` back into the legacy ``{output: {value_str: [task_id, ...]}}``. + A ``None`` attempt is skipped: it is a trial that died before scoring, and the adapter drops + those too (it skips ``FAILED`` scores), so both sides agree on what is groupable. + """ + key = f"{metric_type}.{output_name}" + details: dict[str, dict[str, list[str]]] = {} + for task_id, attempts_by_key in summary.task_metric_attempts.items(): + for attempt in attempts_by_key.get(key, []): + if attempt.value is None: + continue + details.setdefault(output_name, {}).setdefault(str(float(attempt.value)), []).append(task_id) + return details + + +def _reward_stats_from_summary( + summary: AgentEvalSummary, + *, + metric_type: str = "harbor_reward", + output_name: str = "reward", +) -> dict[str, dict[float, list[str]]]: + """Rebuild Harbor's *own* ``reward_stats`` — ``{reward_key: {value: [trial_name, ...]}}``. + + Harbor builds it as ``reward_stats.setdefault(value, []).append(trial_result.trial_name)``: keyed + by the raw numeric value, listing trial names. ``_trial_from_harbor_result`` stamps Harbor's + ``trial_name`` straight onto ``AgentEvalTrial.id``, which is the ``trial_id`` each attempt now + carries, so this reproduces Harbor's shape rather than approximating it. + + A ``None`` attempt is a trial that died before the verifier ran. Harbor has no reward to file it + under either — it lands in ``exception_stats``/``n_errors`` instead — so it is skipped here. + """ + key = f"{metric_type}.{output_name}" + stats: dict[str, dict[float, list[str]]] = {} + for attempts_by_key in summary.task_metric_attempts.values(): + for attempt in attempts_by_key.get(key, []): + if attempt.value is None: + continue + stats.setdefault(output_name, {}).setdefault(attempt.value, []).append(attempt.trial_id) + return stats + + +@pytest.mark.asyncio +async def test_harbor_reward_stats_is_derivable_from_summary_task_metric_attempts(tmp_path: Path) -> None: + """The summary alone reproduces Harbor's ``reward_stats``, which is what AALGO-310 exists to enable. + + Harbor groups rewards by ``trial_name`` and keys them by the raw ``float | int``. Both were out of + reach while attempts were bare numbers indexed by position; now that each attempt names its trial, + AALGO-441 can rebuild the real shape without re-walking ``result.scores``. The legacy + ``reward_details`` (task-keyed, stringified) stays derivable too, so the rewiring loses nothing. + + Still out of scope here: ``exception_stats``, which needs the exception type per dead trial — + AALGO-428. The ``trial_id`` below is the join key that makes it a lookup against ``trials.jsonl``. + """ + job_dir = tmp_path / "job" + job_dir.mkdir() + # Two attempts each for alpha (flaky) and beta (solved); one for gamma, whose verifier emitted + # no reward at all -> PARTIAL trial that still scores 0.0. + _write_trial(job_dir, "alpha__a", "alpha", reward=1.0) + _write_trial(job_dir, "alpha__b", "alpha", reward=0.0) + _write_trial(job_dir, "beta__a", "beta", reward=1.0) + _write_trial(job_dir, "beta__b", "beta", reward=1.0) + _write_trial(job_dir, "gamma__a", "gamma", reward=None) + + tasks = [ + AgentEvalTask(id=task_id, intent="x", inputs={"instruction": "p"}, metrics=[HarborRewardMetric()]) + for task_id in ("alpha", "beta", "gamma") + ] + runner = HarborAgentTaskRunner(job_dir=job_dir, run_job=lambda: _record([])) + result = await AgentEvaluator().run(tasks=tasks, target=runner, config=AgentEvalRunConfig()) + + attempts = { + task_id: {key: [(a.trial_id, a.value) for a in records] for key, records in by_key.items()} + for task_id, by_key in result.summary.task_metric_attempts.items() + } + assert attempts == { + "alpha": {"harbor_reward.reward": [("alpha__a", 1.0), ("alpha__b", 0.0)]}, + "beta": {"harbor_reward.reward": [("beta__a", 1.0), ("beta__b", 1.0)]}, + "gamma": {"harbor_reward.reward": [("gamma__a", 0.0)]}, + } + + # Harbor's own shape: raw float keys, trial names in the lists. + assert _reward_stats_from_summary(result.summary) == { + "reward": {1.0: ["alpha__a", "beta__a", "beta__b"], 0.0: ["alpha__b", "gamma__a"]} + } + + # ...and the legacy task-keyed, stringified payload the current adapter emits stays derivable. + payload = reward_payload_from_result(result) + assert payload["reward_details"] == {"reward": {"1.0": ["alpha", "beta", "beta"], "0.0": ["alpha", "gamma"]}} + assert _reward_details_from_summary(result.summary) == payload["reward_details"] + + def test_reward_with_no_matching_reward_key_is_partial_and_warns(tmp_path: Path, caplog) -> None: # Verifier emitted a reward, but under a key we didn't ask for: no guessing — # the trial is treated as having no reward (None -> PARTIAL, scores 0.0) and warns. diff --git a/packages/nemo_evaluator_sdk/tests/agent_eval/test_pass_at_k.py b/packages/nemo_evaluator_sdk/tests/agent_eval/test_pass_at_k.py index eb4957a3b0..71ab361e25 100644 --- a/packages/nemo_evaluator_sdk/tests/agent_eval/test_pass_at_k.py +++ b/packages/nemo_evaluator_sdk/tests/agent_eval/test_pass_at_k.py @@ -8,7 +8,7 @@ import pytest from nemo_evaluator_sdk.agent_eval.evaluator import AgentEvaluator -from nemo_evaluator_sdk.agent_eval.results import AgentEvalSummary, _pass_at_k +from nemo_evaluator_sdk.agent_eval.results import AgentEvalSummary, _pass_at_k, attempt_values from nemo_evaluator_sdk.agent_eval.scores import ( TRIAL_STATUS_DETAIL, AgentEvalDiagnostic, @@ -111,11 +111,11 @@ def test_task_pass_at_k_gated_and_uniform_across_metric_types() -> None: summary = AgentEvalSummary.from_scores(scores, tasks=tasks) by_name = {score.name: score for score in summary.scores.scores} - assert summary.task_metric_values["t1"] == { + assert {key: attempt_values(a) for key, a in summary.task_metric_attempts["t1"].items()} == { "gym_reward.reward": [1.0, 0.0], "harbor_reward.reward": [1.0, 0.0], } - assert summary.task_metric_values["t2"] == { + assert {key: attempt_values(a) for key, a in summary.task_metric_attempts["t2"].items()} == { "gym_reward.reward": [1.0, 1.0], "harbor_reward.reward": [1.0, 1.0], } @@ -178,7 +178,7 @@ def test_a_failed_trial_is_a_failed_attempt_not_an_absent_one() -> None: summary = AgentEvalSummary.from_scores(scores, tasks=tasks) by_name = {s.name: s for s in summary.scores.scores} - assert summary.task_metric_values["t1"]["reward.reward"] == [1.0, None] + assert attempt_values(summary.task_metric_attempts["t1"]["reward.reward"]) == [1.0, None] assert by_name["reward.reward.pass@1"].mean == pytest.approx(0.5) # 1 of 2 attempts, not 1 of 1 assert by_name["reward.reward.pass@2"].mean == pytest.approx(1.0) assert by_name["reward.reward.pass@1"].nan_count == 0 # the task was measured, so nothing is missing @@ -199,7 +199,7 @@ def test_a_metric_that_raised_leaves_the_attempt_unmeasured_rather_than_failed() summary = AgentEvalSummary.from_scores(scores, tasks=tasks) by_name = {s.name: s for s in summary.scores.scores} - assert summary.task_metric_values["t1"]["reward.reward"] == [1.0] + assert attempt_values(summary.task_metric_attempts["t1"]["reward.reward"]) == [1.0] assert by_name["reward.reward.pass@1"].mean == pytest.approx(0.75) # mean(1.0, 0.5), not mean(0.5, 0.5) # t1 drops out of pass@2 for having fewer than k attempts. That is the estimator working as defined, # not missing data, so it is not counted as nan. diff --git a/packages/nemo_evaluator_sdk/tests/agent_eval/test_persistence.py b/packages/nemo_evaluator_sdk/tests/agent_eval/test_persistence.py index 4e4c0e9ab0..cb8dc071f1 100644 --- a/packages/nemo_evaluator_sdk/tests/agent_eval/test_persistence.py +++ b/packages/nemo_evaluator_sdk/tests/agent_eval/test_persistence.py @@ -11,7 +11,7 @@ import pytest from nemo_evaluator_sdk.agent_eval.persistence import persist_run, read_trials -from nemo_evaluator_sdk.agent_eval.results import AgentEvalResult, AgentEvalSummary +from nemo_evaluator_sdk.agent_eval.results import AgentEvalAttemptValue, AgentEvalResult, AgentEvalSummary from nemo_evaluator_sdk.agent_eval.trials import AgentEvalTrial, AgentEvalTrialStatus, AgentOutput from nemo_evaluator_sdk.values.evidence import CandidateEvidence, EvidenceDescriptor @@ -84,11 +84,21 @@ def test_persist_run_writes_bundle_relative_refs_that_survive_a_move(tmp_path: P assert resolved.is_dir() and resolved == moved / "evidence" / "fabric" / "r" / "000000-taskA" / "workspace" -def test_persist_run_writes_per_task_attempt_values_to_summary(tmp_path: Path) -> None: +def test_persist_run_writes_per_task_attempts_to_summary(tmp_path: Path) -> None: summary = AgentEvalSummary( - task_metric_values={ - "task-a": {"harbor_reward.reward": [1.0, 0.0]}, - "task-b": {"harbor_reward.reward": [1.0, None]}, + task_metric_attempts={ + "task-a": { + "harbor_reward.reward": [ + AgentEvalAttemptValue(trial_id="task-a__aaa", value=1.0), + AgentEvalAttemptValue(trial_id="task-a__bbb", value=0.0), + ] + }, + "task-b": { + "harbor_reward.reward": [ + AgentEvalAttemptValue(trial_id="task-b__ccc", value=1.0), + AgentEvalAttemptValue(trial_id="task-b__ddd", value=None), + ] + }, } ) result = AgentEvalResult(run_id="run", tasks=[], trials=[], scores=[], summary=summary) @@ -96,11 +106,46 @@ def test_persist_run_writes_per_task_attempt_values_to_summary(tmp_path: Path) - persist_run(result, tmp_path) payload = json.loads((tmp_path / "summary.json").read_text(encoding="utf-8")) - assert payload["task_metric_values"] == { - "task-a": {"harbor_reward.reward": [1.0, 0.0]}, - "task-b": {"harbor_reward.reward": [1.0, None]}, + assert payload["task_metric_attempts"] == { + "task-a": { + "harbor_reward.reward": [ + {"trial_id": "task-a__aaa", "value": 1.0}, + {"trial_id": "task-a__bbb", "value": 0.0}, + ] + }, + "task-b": { + "harbor_reward.reward": [ + {"trial_id": "task-b__ccc", "value": 1.0}, + {"trial_id": "task-b__ddd", "value": None}, + ] + }, } - assert AgentEvalSummary.model_validate(payload).task_metric_values == summary.task_metric_values + assert AgentEvalSummary.model_validate(payload).task_metric_attempts == summary.task_metric_attempts + + +def test_summary_json_round_trips_attempt_order_through_sort_keys(tmp_path: Path) -> None: + # persist_run writes with sort_keys=True. Attempts are a list precisely so that trial order + # survives that: keying them by trial id would come back sorted lexicographically instead. + summary = AgentEvalSummary( + task_metric_attempts={ + "task-a": { + "harbor_reward.reward": [ + AgentEvalAttemptValue(trial_id="z-trial", value=1.0), + AgentEvalAttemptValue(trial_id="a-trial", value=0.0), + ] + } + } + ) + result = AgentEvalResult(run_id="run", tasks=[], trials=[], scores=[], summary=summary) + + persist_run(result, tmp_path) + payload = json.loads((tmp_path / "summary.json").read_text(encoding="utf-8")) + reloaded = AgentEvalSummary.model_validate(payload) + + assert [a.trial_id for a in reloaded.task_metric_attempts["task-a"]["harbor_reward.reward"]] == [ + "z-trial", + "a-trial", + ] def test_read_trials_rejects_evidence_ref_that_escapes_the_bundle(tmp_path: Path) -> None: diff --git a/packages/nemo_evaluator_sdk/tests/agent_eval/test_task_metric_values.py b/packages/nemo_evaluator_sdk/tests/agent_eval/test_task_metric_attempts.py similarity index 50% rename from packages/nemo_evaluator_sdk/tests/agent_eval/test_task_metric_values.py rename to packages/nemo_evaluator_sdk/tests/agent_eval/test_task_metric_attempts.py index 6909dab70d..bb87decc6e 100644 --- a/packages/nemo_evaluator_sdk/tests/agent_eval/test_task_metric_values.py +++ b/packages/nemo_evaluator_sdk/tests/agent_eval/test_task_metric_attempts.py @@ -6,7 +6,13 @@ from collections.abc import Iterator from pathlib import Path -from nemo_evaluator_sdk.agent_eval.results import AgentEvalSummary, _task_metric_values +import pytest +from nemo_evaluator_sdk.agent_eval.results import ( + AgentEvalAttemptValue, + AgentEvalSummary, + _task_metric_attempts, + attempt_values, +) from nemo_evaluator_sdk.agent_eval.scores import ( TRIAL_STATUS_DETAIL, AgentEvalDiagnostic, @@ -55,6 +61,16 @@ def _task(task_id: str, *metrics: _Metric) -> AgentEvalTask: return AgentEvalTask(id=task_id, intent="test", inputs={}, metrics=list(metrics)) +def _pairs( + attempts: dict[str, dict[str, list[AgentEvalAttemptValue]]], +) -> dict[str, dict[str, list[tuple[str, float | None]]]]: + """Flatten attempt records to ``(trial_id, value)`` so assertions stay readable.""" + return { + task_id: {key: [(a.trial_id, a.value) for a in records] for key, records in by_key.items()} + for task_id, by_key in attempts.items() + } + + def _score( task_id: str, trial_id: str, @@ -75,14 +91,16 @@ def _score( ) -def _failed_score(task_id: str, trial_id: str, *, trial_failed: bool) -> AgentEvalTaskScore: +def _failed_score( + task_id: str, trial_id: str, *, trial_failed: bool, metric_type: str = "reward" +) -> AgentEvalTaskScore: details = {TRIAL_STATUS_DETAIL: "failed"} if trial_failed else {"exception_type": "TimeoutError"} return AgentEvalTaskScore( - id=f"run:{task_id}:{trial_id}:reward", + id=f"run:{task_id}:{trial_id}:{metric_type}", run_id="run", task_id=task_id, trial_id=trial_id, - metric_type="reward", + metric_type=metric_type, status=AgentEvalScoreStatus.FAILED, diagnostics=[ AgentEvalDiagnostic( @@ -120,17 +138,17 @@ def test_summary_exposes_ordered_numeric_attempt_values_per_task() -> None: summary = AgentEvalSummary.from_scores(scores, tasks=tasks) - assert summary.task_metric_values == { + assert _pairs(summary.task_metric_attempts) == { "task-a": { - "complete.passed": [1.0, 0.0], - "retries.count": [2.0, 3.0], - "reward.score": [1.0, 0.25], + "complete.passed": [("attempt-0", 1.0), ("attempt-1", 0.0)], + "retries.count": [("attempt-0", 2.0), ("attempt-1", 3.0)], + "reward.score": [("attempt-0", 1.0), ("attempt-1", 0.25)], }, - "task-b": {"reward.score": [0.0]}, + "task-b": {"reward.score": [("attempt-0", 0.0)]}, } -def test_task_metric_values_scans_scores_once() -> None: +def test_task_metric_attempts_scans_scores_once() -> None: tasks = [_task("task-a", _Metric("reward", MetricOutputSpec.continuous_score("score")))] scores = _SinglePassScores( [ @@ -139,7 +157,9 @@ def test_task_metric_values_scans_scores_once() -> None: ] ) - assert _task_metric_values(scores, tasks) == {"task-a": {"reward.score": [1.0, 0.0]}} + assert _pairs(_task_metric_attempts(scores, tasks)) == { + "task-a": {"reward.score": [("attempt-0", 1.0), ("attempt-1", 0.0)]} + } def test_failed_trials_are_attempts_but_metric_failures_are_unmeasured() -> None: @@ -155,8 +175,10 @@ def test_failed_trials_are_attempts_but_metric_failures_are_unmeasured() -> None summary = AgentEvalSummary.from_scores(scores, tasks=tasks) - assert summary.task_metric_values == { - "flaky": {"reward.score": [1.0, None]}, + assert _pairs(summary.task_metric_attempts) == { + # The dead trial keeps its identity, paired with its None; the unmeasured one has no entry at + # all, so it is nameable from neither -- that asymmetry is what pass@k depends on. + "flaky": {"reward.score": [("attempt-0", 1.0), ("attempt-1", None)]}, "unmeasured": {"reward.score": []}, } @@ -172,8 +194,8 @@ def test_a_task_that_produced_no_trial_is_unmeasured_and_counted_in_pass_at_k_na summary = AgentEvalSummary.from_scores(scores, tasks=tasks) by_name = {score.name: score for score in summary.scores.scores} - assert summary.task_metric_values == { - "scored": {"reward.score": [1.0]}, + assert _pairs(summary.task_metric_attempts) == { + "scored": {"reward.score": [("attempt-0", 1.0)]}, "never-ran": {"reward.score": []}, } assert by_name["reward.score.pass@1"].mean == 1.0 # the one measured task passed @@ -196,20 +218,24 @@ def test_outputs_declared_under_an_unretained_schema_stay_out_even_when_numeric( _score("task-a", "attempt-0", "usage", "prompt_tokens", 1234), ] - assert AgentEvalSummary.from_scores(scores, tasks=tasks).task_metric_values == {"task-a": {"reward.score": [1.0]}} + assert _pairs(AgentEvalSummary.from_scores(scores, tasks=tasks).task_metric_attempts) == { + "task-a": {"reward.score": [("attempt-0", 1.0)]} + } def test_without_tasks_there_is_no_spec_to_filter_on() -> None: # No tasks means no declared schemas to consult, so every numeric output observed is retained. scores = [_score("task-a", "attempt-0", "usage", "prompt_tokens", 1234)] - assert AgentEvalSummary.from_scores(scores).task_metric_values == {"task-a": {"usage.prompt_tokens": [1234.0]}} + assert _pairs(AgentEvalSummary.from_scores(scores).task_metric_attempts) == { + "task-a": {"usage.prompt_tokens": [("attempt-0", 1234.0)]} + } -def test_attempt_positions_are_comparable_only_within_one_key() -> None: +def test_attempts_align_across_keys_by_trial_id_not_position() -> None: # A metric that raised drops its attempt entirely while a dead trial holds its slot as None, so a # metric failure shortens its own key's list without shortening its neighbour's. Index i of two - # keys is then two different trials -- there is no trial id to join on, and callers must not try. + # keys is then two different trials -- which is exactly why every attempt names its trial. tasks = [ _task( "task-a", @@ -226,22 +252,123 @@ def test_attempt_positions_are_comparable_only_within_one_key() -> None: _score("task-a", "attempt-2", "steps", "count", 7), ] - values = AgentEvalSummary.from_scores(scores, tasks=tasks).task_metric_values["task-a"] + attempts = AgentEvalSummary.from_scores(scores, tasks=tasks).task_metric_attempts["task-a"] + + # Position lies: index 1 is attempt-2 under reward.score but attempt-1 under steps.count. + assert attempts["reward.score"][1].trial_id == "attempt-2" + assert attempts["steps.count"][1].trial_id == "attempt-1" + # trial_id tells the truth, so a join across the two keys is now possible and correct. + steps_by_trial = {a.trial_id: a.value for a in attempts["steps.count"]} + assert [(a.trial_id, a.value, steps_by_trial[a.trial_id]) for a in attempts["reward.score"]] == [ + ("attempt-0", 1.0, 5.0), + ("attempt-2", 0.0, 7.0), + ] + - # Index 1 of reward.score is attempt-2; index 1 of steps.count is attempt-1. - assert values == {"reward.score": [1.0, 0.0], "steps.count": [5.0, 9.0, 7.0]} +def test_dead_trials_are_nameable_from_the_summary_alone() -> None: + # AALGO-428 needs to say *which* trial died to roll up exception types. Before attempts carried a + # trial id the summary could count dead attempts but not name one; now it is a join key out to + # trials.jsonl, where the error lives. + tasks = [_task("task-a", _Metric("reward", MetricOutputSpec.continuous_score("score")))] + scores = [ + _score("task-a", "attempt-0", "reward", "score", 1.0), + _failed_score("task-a", "attempt-1", trial_failed=True), + _failed_score("task-a", "attempt-2", trial_failed=False), # metric raised: unmeasured, not dead + ] + attempts = AgentEvalSummary.from_scores(scores, tasks=tasks).task_metric_attempts["task-a"]["reward.score"] -def test_summary_without_task_metric_values_loads_as_empty() -> None: - assert AgentEvalSummary.model_validate({}).task_metric_values == {} + assert {a.trial_id for a in attempts if a.value is None} == {"attempt-1"} -def test_vendored_summary_accepts_task_metric_values() -> None: +def test_duplicate_trial_ids_are_two_attempts_not_one() -> None: + # Nothing enforces trial-id uniqueness, so the attempt list must never be re-keyed by trial id: + # collapsing two attempts into one would silently drop pass@k's n. A list cannot lose cardinality. + tasks = [_task("task-a", _Metric("reward", MetricOutputSpec.continuous_score("score")))] + scores = [ + _score("task-a", "dup", "reward", "score", 1.0), + _score("task-a", "dup", "reward", "score", 0.0), + ] + + summary = AgentEvalSummary.from_scores(scores, tasks=tasks) + by_name = {score.name: score for score in summary.scores.scores} + + assert _pairs(summary.task_metric_attempts) == {"task-a": {"reward.score": [("dup", 1.0), ("dup", 0.0)]}} + assert by_name["reward.score.pass@1"].mean == pytest.approx(0.5) # n=2, not n=1 + assert by_name["reward.score.pass@2"].mean == pytest.approx(1.0) + + +def test_attempt_values_projects_to_a_bare_value_list() -> None: + # The projection pass@k reads: order, cardinality and None-vs-absent preserved exactly. + attempts = [ + AgentEvalAttemptValue(trial_id="t0", value=1.0), + AgentEvalAttemptValue(trial_id="t1", value=None), + AgentEvalAttemptValue(trial_id="t2", value=0.0), + ] + + assert attempt_values(attempts) == [1.0, None, 0.0] + assert attempt_values([]) == [] + + +def test_pass_at_k_aggregates_are_unchanged_by_carrying_trial_ids() -> None: + """Golden table captured from the pre-change implementation, before attempts carried trial ids. + + Every branch pass@k distinguishes is present: a task that always passes, one whose attempts + include a dead trial (None counts toward ``n``), one whose metric raised on an attempt (dropped + from ``n``, so it falls out of ``k=2``), and two that yielded nothing at all (``nan_count``). + """ + reward = _Metric("reward", MetricOutputSpec.continuous_score("score")) + passed = _Metric("complete", MetricOutputSpec.boolean("passed")) + tasks = [_task(t, reward, passed) for t in ("solved", "flaky", "judged-out", "unmeasured", "never-ran")] + scores = [ + _score("solved", "s0", "reward", "score", 1.0), + _score("solved", "s0", "complete", "passed", True), + _score("solved", "s1", "reward", "score", 1.0), + _score("solved", "s1", "complete", "passed", True), + _score("solved", "s2", "reward", "score", 1.0), + _score("solved", "s2", "complete", "passed", True), + _score("flaky", "f0", "reward", "score", 1.0), + _score("flaky", "f0", "complete", "passed", True), + _failed_score("flaky", "f1", trial_failed=True), + _failed_score("flaky", "f1", trial_failed=True, metric_type="complete"), + _score("flaky", "f2", "reward", "score", 0.0), + _score("flaky", "f2", "complete", "passed", False), + _score("judged-out", "j0", "reward", "score", 1.0), + _score("judged-out", "j0", "complete", "passed", True), + _failed_score("judged-out", "j1", trial_failed=False), + _failed_score("judged-out", "j1", trial_failed=False, metric_type="complete"), + _failed_score("unmeasured", "u0", trial_failed=False), + _failed_score("unmeasured", "u0", trial_failed=False, metric_type="complete"), + ] + + summary = AgentEvalSummary.from_scores(scores, tasks=tasks) + actual = {s.name: (s.mean, s.count, s.nan_count) for s in summary.scores.scores if ".pass@" in s.name} + + assert actual == { + "complete.passed.pass@1": (pytest.approx(0.7777777777777777), 3, 2), + "complete.passed.pass@2": (pytest.approx(0.8333333333333333), 2, 2), + "complete.passed.pass@3": (pytest.approx(1.0), 2, 2), + "reward.score.pass@1": (pytest.approx(0.7777777777777777), 3, 2), + "reward.score.pass@2": (pytest.approx(0.8333333333333333), 2, 2), + "reward.score.pass@3": (pytest.approx(1.0), 2, 2), + } + + +def test_summary_without_task_metric_attempts_loads_as_empty() -> None: + assert AgentEvalSummary.model_validate({}).task_metric_attempts == {} + + +def test_vendored_summary_accepts_task_metric_attempts() -> None: from nemo_platform.beta.evaluator.agent_eval.results import AgentEvalSummary as VendoredAgentEvalSummary - payload = {"task_metric_values": {"task-a": {"reward.score": [1.0, None]}}} + payload = { + "task_metric_attempts": { + "task-a": {"reward.score": [{"trial_id": "t0", "value": 1.0}, {"trial_id": "t1", "value": None}]} + } + } - assert VendoredAgentEvalSummary.model_validate(payload).task_metric_values == payload["task_metric_values"] + attempts = VendoredAgentEvalSummary.model_validate(payload).task_metric_attempts + assert [(a.trial_id, a.value) for a in attempts["task-a"]["reward.score"]] == [("t0", 1.0), ("t1", None)] def test_vendored_results_module_is_a_verbatim_copy_of_this_one() -> None: @@ -263,16 +390,25 @@ def test_vendored_results_module_is_a_verbatim_copy_of_this_one() -> None: def test_gym_example_reads_task_outcomes_from_summary() -> None: - from packages.nemo_evaluator_sdk.examples.gym.inspect_results import per_task_outcomes + from packages.nemo_evaluator_sdk.examples.gym.inspect_results import per_task_attempts, per_task_outcomes summary = AgentEvalSummary( - task_metric_values={ - "task-a": {"gym_reward.reward": [1.0, 0.0]}, + task_metric_attempts={ + "task-a": { + "gym_reward.reward": [ + AgentEvalAttemptValue(trial_id="task-a__aaa", value=1.0), + AgentEvalAttemptValue(trial_id="task-a__bbb", value=0.0), + ] + }, "task-b": {"gym_reward.reward": []}, } ) + # The example's headline accessor keeps its bare-value shape... assert per_task_outcomes(summary, metric_type="gym_reward", output_name="reward") == { "task-a": [1.0, 0.0], "task-b": [], } + # ...and its sibling exposes the identity that makes an attempt traceable back to a rollout. + attempts = per_task_attempts(summary, metric_type="gym_reward", output_name="reward") + assert [(a.trial_id, a.value) for a in attempts["task-a"]] == [("task-a__aaa", 1.0), ("task-a__bbb", 0.0)] diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/results.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/results.py index ed13c26973..1cd2108976 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/results.py +++ b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/results.py @@ -36,7 +36,7 @@ ) from pydantic import BaseModel, ConfigDict, Field -#: Metric-output value schemas retained in the ordered per-task attempt-value mapping. +#: Metric-output value schemas retained in the ordered per-task attempt mapping. _TASK_METRIC_VALUE_SCHEMAS = (ContinuousScore, DiscreteScore, BooleanValue) #: Metric-output value schemas eligible for pass@k (a per-attempt "did it pass?" signal). Labels, @@ -61,6 +61,32 @@ class AgentEvalMetricOutputCoverage(BaseModel): missing: int = Field(default=0, description="Scores where the output was expected but absent.") +class AgentEvalAttemptValue(BaseModel): + """One attempt at a task under one metric output: which trial made it, and what it measured. + + Frozen because these records are handed out by reference from the summary: a consumer rescaling + values in place (Gym reports reward on 0-100 where we use 0-1) would otherwise rewrite the run's + own results, and a later persist would save the rewrite. + """ + + model_config = ConfigDict(extra="forbid", frozen=True) + + trial_id: str = Field( + description=( + "Identifier of the trial that made this attempt. Joins to AgentEvalTrial.id " + "(trials.jsonl) and AgentEvalTaskScore.trial_id (scores.jsonl)." + ) + ) + value: float | None = Field( + description=( + "What the metric output measured, or None when the trial failed before it could be " + "measured -- an attempt that did not pass. Required rather than defaulted: None is a " + "load-bearing signal pass@k counts as a failed attempt, so an omitted value must not " + "quietly become one." + ), + ) + + class AgentEvalSummary(BaseModel): """Aggregated scores, coverage, per-task attempt values, and run counts for an agent-eval run.""" @@ -78,14 +104,15 @@ class AgentEvalSummary(BaseModel): default_factory=dict, description="Per-metric, per-output coverage counts (total/scored/failed/missing).", ) - task_metric_values: dict[str, dict[str, list[float | None]]] = Field( + task_metric_attempts: dict[str, dict[str, list[AgentEvalAttemptValue]]] = Field( default_factory=dict, description=( - "Per task, the values each '.' measured, in trial order. A failed " - "trial is None: an attempt that did not pass. An unmeasured attempt (metric failed, output " - "absent) has no entry, so each key's list is independent and positions align within a key, " - "not across keys. An empty list means nothing was measured, including a task that produced " - "no trial." + "Per task, the attempts each '.' measured, in trial order. Each " + "attempt names the trial that made it, so attempts join across keys -- and out to " + "trials.jsonl and scores.jsonl -- by trial_id. A failed trial has value None: an attempt " + "that did not pass. An unmeasured attempt (metric failed, output absent) has no entry at " + "all, so each key's list is independent: align by trial_id, never by position. An empty " + "list means nothing was measured, including a task that produced no trial." ), ) task_count: int = Field(default=0, description="Number of tasks represented in the run.") @@ -118,16 +145,16 @@ def from_scores( ``runner..``), merged in so a backend's own figures are addressable the same way as ours. """ task_list = list(tasks) if tasks is not None else None - task_metric_values = _task_metric_values(scores, task_list) + task_metric_attempts = _task_metric_attempts(scores, task_list) return AgentEvalSummary( scores=_aggregate_scores( scores, task_list, extra_scores, - task_metric_values=task_metric_values, + task_metric_attempts=task_metric_attempts, ), metric_coverage=_metric_coverage(scores, task_list), - task_metric_values=task_metric_values, + task_metric_attempts=task_metric_attempts, task_count=len(task_list) if task_list is not None else len({score.task_id for score in scores}), trial_count=len({score.trial_id for score in scores}), score_count=len(scores), @@ -476,7 +503,7 @@ def _aggregate_scores( tasks: Sequence[AgentEvalTask] | None, extra_scores: Sequence[AggregateScore] = (), *, - task_metric_values: dict[str, dict[str, list[float | None]]] | None = None, + task_metric_attempts: dict[str, dict[str, list[AgentEvalAttemptValue]]] | None = None, ) -> AggregatedMetricResult: """Aggregate per-metric-output, per-semantic-view, and task-level pass@k values into range scores. @@ -509,13 +536,23 @@ def _aggregate_scores( for view_name, (values, total) in sorted(_semantic_view_values(scores, tasks).items()): aggregated.append(_aggregate_range_score(f"view.{view_name}", values, total)) - attempt_values = task_metric_values if task_metric_values is not None else _task_metric_values(scores, tasks) - aggregated.extend(_task_pass_at_k_scores(attempt_values, tasks)) + attempts = task_metric_attempts if task_metric_attempts is not None else _task_metric_attempts(scores, tasks) + aggregated.extend(_task_pass_at_k_scores(attempts, tasks)) aggregated.extend(extra_scores) return AggregatedMetricResult(scores=aggregated) +def attempt_values(attempts: Sequence[AgentEvalAttemptValue]) -> list[float | None]: + """The bare per-attempt values, for consumers scoring attempts without caring which trial made them. + + Preserves order, cardinality, and the None-versus-absent distinction exactly as recorded, so + anything counting attempts (pass@k above all) reads the same sequence it would have read before + attempts carried a trial id. + """ + return [attempt.value for attempt in attempts] + + def _pass_at_k(n: int, c: int, k: int) -> float: """Unbiased pass@k estimator (Chen et al., 2021): ``1 - C(n-c, k) / C(n, k)``. @@ -549,11 +586,11 @@ def _scorelike_outputs(tasks: Sequence[AgentEvalTask] | None) -> set[tuple[str, return scorelike -def _task_metric_values( +def _task_metric_attempts( scores: Sequence[AgentEvalTaskScore], tasks: Sequence[AgentEvalTask] | None, -) -> dict[str, dict[str, list[float | None]]]: - """Ordered per-attempt values per task, keyed ``.``. +) -> dict[str, dict[str, list[AgentEvalAttemptValue]]]: + """Ordered per-attempt records per task, keyed ``.``. ``task-a`` declares ``reward.score`` (continuous), ``steps.count`` (discrete) and ``usage.prompt_tokens`` (a free model) and runs four trials:: @@ -563,8 +600,10 @@ def _task_metric_values( t2 # every metric fails as a trial failure t3 reward 0.0 steps 7 usage 1100 - out {"task-a": {"reward.score": [1.0, None, 0.0], - "steps.count": [5.0, 9.0, None, 7.0]}} + out {"task-a": {"reward.score": [(t0, 1.0), (t2, None), (t3, 0.0)], + "steps.count": [(t0, 5.0), (t1, 9.0), (t2, None), (t3, 7.0)]}} + + (shown as ``(trial_id, value)`` pairs; each is an :class:`AgentEvalAttemptValue`) ``usage.prompt_tokens`` is absent because its declared schema is not in :data:`_TASK_METRIC_VALUE_SCHEMAS`; t1 is missing from ``reward.score`` but present in @@ -580,13 +619,16 @@ def _task_metric_values( What each score contributes to its key, in trial order: - - failed trial (:func:`is_trial_failure`) -> ``None``, an attempt that did not pass + - failed trial (:func:`is_trial_failure`) -> value ``None``, an attempt that did not pass - failed metric, or the output absent -> no entry; the attempt is unmeasured, not unsuccessful - otherwise -> the numeric value pass@k needs that asymmetry, and it is why a list is indexed by surviving measurement rather than - by attempt: above, index 1 is t2 under ``reward.score`` but t1 under ``steps.count``. Compare - positions within a key, never across keys. + by attempt: above, index 1 is t2 under ``reward.score`` but t1 under ``steps.count``. Every entry + therefore names its trial, and ``trial_id`` — not position — is what joins two keys of one task, + or joins out to ``trials.jsonl`` and ``scores.jsonl``. Ids are recorded as the runner reported + them and are never deduplicated: two attempts sharing an id stay two attempts, so a runner that + reuses one costs pass@k nothing. """ output_keys: dict[str, set[tuple[str, str]]] = {} # Declared under a schema this mapping does not retain. Tracked so an emitted numeric value cannot @@ -615,28 +657,28 @@ def _task_metric_values( if _semantic_value(output) is not None: task_keys.add((score.metric_type, output.name)) - by_task: dict[str, dict[str, list[float | None]]] = {} + by_task: dict[str, dict[str, list[AgentEvalAttemptValue]]] = {} for task_id, keys in output_keys.items(): - task_values: dict[str, list[float | None]] = {} + task_values: dict[str, list[AgentEvalAttemptValue]] = {} for metric_type, output_name in sorted(keys): - values: list[float | None] = [] + values: list[AgentEvalAttemptValue] = [] for score in scores_by_task_metric.get((task_id, metric_type), []): if is_trial_failure(score): - values.append(None) + values.append(AgentEvalAttemptValue(trial_id=score.trial_id, value=None)) continue if score.status not in (AgentEvalScoreStatus.COMPLETED, AgentEvalScoreStatus.PARTIAL): continue output = _score_output(score, output_name) value = _semantic_value(output) if output is not None else None if value is not None: - values.append(value) + values.append(AgentEvalAttemptValue(trial_id=score.trial_id, value=value)) task_values[f"{metric_type}.{output_name}"] = values by_task[task_id] = task_values return by_task def _task_pass_at_k_scores( - task_metric_values: dict[str, dict[str, list[float | None]]], + task_metric_attempts: dict[str, dict[str, list[AgentEvalAttemptValue]]], tasks: Sequence[AgentEvalTask] | None, ) -> list[AggregateScore]: """Task-level pass@k over the R trials per task, aggregated across tasks (uniform for any runner). @@ -666,7 +708,7 @@ def _task_pass_at_k_scores( aggregated: list[AggregateScore] = [] for metric_type, output_name in sorted(scorelike): key = f"{metric_type}.{output_name}" - values_by_task = [outputs[key] for outputs in task_metric_values.values() if key in outputs] + values_by_task = [attempt_values(outputs[key]) for outputs in task_metric_attempts.values() if key in outputs] measured = [values for values in values_by_task if values] if not measured: continue From 49406a3aeb8ca2c90a01a7311f7067dff3fc38fd Mon Sep 17 00:00:00 2001 From: Nick Goncharenko Date: Tue, 11 Aug 2026 12:04:31 -0700 Subject: [PATCH 03/10] chore: add examples Signed-off-by: Nick Goncharenko --- .../nemo_evaluator_sdk/agent_eval/results.py | 111 +++++++++++++++++ .../beta/evaluator/agent_eval/results.py | 113 ++++++++++++++++++ 2 files changed, 224 insertions(+) diff --git a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/results.py b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/results.py index d406d6ba2f..94cfb5ea24 100644 --- a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/results.py +++ b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/results.py @@ -99,10 +99,85 @@ class AgentEvalSummary(BaseModel): "'.', plus per-semantic-view rollups named 'view.'. " "Failed or missing scores are surfaced as nan_count." ), + examples=[ + # Emission order is real: metric outputs, then views, then pass@k. Note that pass@k + # counts *tasks* (4) where the metric output counts *trials* (10). + { + "scores": [ + { + "name": "harbor_reward.reward", + "score_type": "range", + "count": 10, + "nan_count": 2, + "mean": 0.6, + "min": 0.0, + "max": 1.0, + "std_dev": 0.4899, + }, + { + "name": "view.legal_quality", + "score_type": "range", + "count": 8, + "nan_count": 4, + "mean": 0.7375, + "min": 0.1, + "max": 1.0, + "std_dev": 0.3674, + }, + { + "name": "harbor_reward.reward.pass@1", + "score_type": "range", + "count": 4, + "nan_count": 1, + "mean": 0.5, + "min": 0.0, + "max": 1.0, + "std_dev": 0.3727, + }, + { + "name": "harbor_reward.reward.pass@2", + "score_type": "range", + "count": 4, + "nan_count": 1, + "mean": 0.6667, + "min": 0.0, + "max": 1.0, + "std_dev": 0.4082, + }, + ] + }, + # A separate run, because a runner's imported figures cannot co-occur with another + # runner's metrics. Scalars carry `value` and no distribution, and no `count` when the + # backend reports a figure without the sample size behind it. + { + "scores": [ + { + "name": "gym_reward.reward", + "score_type": "range", + "count": 20, + "nan_count": 0, + "mean": 0.65, + "min": 0.0, + "max": 1.0, + "std_dev": 0.477, + }, + {"name": "runner.gym.pass@1/accuracy", "score_type": "scalar", "nan_count": 0, "value": 0.68}, + ] + }, + ], ) metric_coverage: dict[str, dict[str, AgentEvalMetricOutputCoverage]] = Field( default_factory=dict, description="Per-metric, per-output coverage counts (total/scored/failed/missing).", + examples=[ + # Same 12 trials under two metrics, which is what distinguishes a low mean from low + # coverage. The two dead trials fail every metric; the judge failed once more on its own, + # and once completed without emitting its output at all (missing, not failed). + { + "harbor_reward": {"reward": {"total": 12, "scored": 10, "failed": 2, "missing": 0}}, + "rubric_judge": {"criteria_pass_rate": {"total": 12, "scored": 8, "failed": 3, "missing": 1}}, + } + ], ) task_metric_attempts: dict[str, dict[str, list[AgentEvalAttemptValue]]] = Field( default_factory=dict, @@ -114,6 +189,42 @@ class AgentEvalSummary(BaseModel): "all, so each key's list is independent: align by trial_id, never by position. An empty " "list means nothing was measured, including a task that produced no trial." ), + examples=[ + { + "contract-review-msa-indemnity": { + "harbor_reward.reward": [ + {"trial_id": "contract-review-msa-indemnity__k3f9wq2", "value": 1.0}, + {"trial_id": "contract-review-msa-indemnity__t7m2xb4", "value": 0.0}, + {"trial_id": "contract-review-msa-indemnity__9jr4vd1", "value": 1.0}, + ], + # t7m2xb4 is absent here rather than null: its judge timed out, so that attempt + # went unmeasured. Index 1 is therefore a different trial in each of these lists. + "rubric_judge.criteria_pass_rate": [ + {"trial_id": "contract-review-msa-indemnity__k3f9wq2", "value": 0.75}, + {"trial_id": "contract-review-msa-indemnity__9jr4vd1", "value": 1.0}, + ], + }, + "nda-scope-carveouts": { + # p2hn8sc died in the sandbox, so it is null in every key: an attempt that + # happened and did not pass, as opposed to one that was never measured. + "harbor_reward.reward": [ + {"trial_id": "nda-scope-carveouts__p2hn8sc", "value": None}, + {"trial_id": "nda-scope-carveouts__w5db3qy", "value": 1.0}, + {"trial_id": "nda-scope-carveouts__z8kt1nf", "value": 0.0}, + ], + "rubric_judge.criteria_pass_rate": [ + {"trial_id": "nda-scope-carveouts__p2hn8sc", "value": None}, + {"trial_id": "nda-scope-carveouts__w5db3qy", "value": 0.6}, + {"trial_id": "nda-scope-carveouts__z8kt1nf", "value": 0.2}, + ], + }, + # Requested, but the runner returned no trial for it: keys declared, nothing measured. + "merger-hsr-filing-threshold": { + "harbor_reward.reward": [], + "rubric_judge.criteria_pass_rate": [], + }, + } + ], ) task_count: int = Field(default=0, description="Number of tasks represented in the run.") trial_count: int = Field(default=0, description="Number of distinct trials scored.") diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/results.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/results.py index 1cd2108976..e833fd4ba4 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/results.py +++ b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/results.py @@ -99,10 +99,85 @@ class AgentEvalSummary(BaseModel): "'.', plus per-semantic-view rollups named 'view.'. " "Failed or missing scores are surfaced as nan_count." ), + examples=[ + # Emission order is real: metric outputs, then views, then pass@k. Note that pass@k + # counts *tasks* (4) where the metric output counts *trials* (10). + { + "scores": [ + { + "name": "harbor_reward.reward", + "score_type": "range", + "count": 10, + "nan_count": 2, + "mean": 0.6, + "min": 0.0, + "max": 1.0, + "std_dev": 0.4899, + }, + { + "name": "view.legal_quality", + "score_type": "range", + "count": 8, + "nan_count": 4, + "mean": 0.7375, + "min": 0.1, + "max": 1.0, + "std_dev": 0.3674, + }, + { + "name": "harbor_reward.reward.pass@1", + "score_type": "range", + "count": 4, + "nan_count": 1, + "mean": 0.5, + "min": 0.0, + "max": 1.0, + "std_dev": 0.3727, + }, + { + "name": "harbor_reward.reward.pass@2", + "score_type": "range", + "count": 4, + "nan_count": 1, + "mean": 0.6667, + "min": 0.0, + "max": 1.0, + "std_dev": 0.4082, + }, + ] + }, + # A separate run, because a runner's imported figures cannot co-occur with another + # runner's metrics. Scalars carry `value` and no distribution, and no `count` when the + # backend reports a figure without the sample size behind it. + { + "scores": [ + { + "name": "gym_reward.reward", + "score_type": "range", + "count": 20, + "nan_count": 0, + "mean": 0.65, + "min": 0.0, + "max": 1.0, + "std_dev": 0.477, + }, + {"name": "runner.gym.pass@1/accuracy", "score_type": "scalar", "nan_count": 0, "value": 0.68}, + ] + }, + ], ) metric_coverage: dict[str, dict[str, AgentEvalMetricOutputCoverage]] = Field( default_factory=dict, description="Per-metric, per-output coverage counts (total/scored/failed/missing).", + examples=[ + # Same 12 trials under two metrics, which is what distinguishes a low mean from low + # coverage. The two dead trials fail every metric; the judge failed once more on its own, + # and once completed without emitting its output at all (missing, not failed). + { + "harbor_reward": {"reward": {"total": 12, "scored": 10, "failed": 2, "missing": 0}}, + "rubric_judge": {"criteria_pass_rate": {"total": 12, "scored": 8, "failed": 3, "missing": 1}}, + } + ], ) task_metric_attempts: dict[str, dict[str, list[AgentEvalAttemptValue]]] = Field( default_factory=dict, @@ -114,6 +189,42 @@ class AgentEvalSummary(BaseModel): "all, so each key's list is independent: align by trial_id, never by position. An empty " "list means nothing was measured, including a task that produced no trial." ), + examples=[ + { + "contract-review-msa-indemnity": { + "harbor_reward.reward": [ + {"trial_id": "contract-review-msa-indemnity__k3f9wq2", "value": 1.0}, + {"trial_id": "contract-review-msa-indemnity__t7m2xb4", "value": 0.0}, + {"trial_id": "contract-review-msa-indemnity__9jr4vd1", "value": 1.0}, + ], + # t7m2xb4 is absent here rather than null: its judge timed out, so that attempt + # went unmeasured. Index 1 is therefore a different trial in each of these lists. + "rubric_judge.criteria_pass_rate": [ + {"trial_id": "contract-review-msa-indemnity__k3f9wq2", "value": 0.75}, + {"trial_id": "contract-review-msa-indemnity__9jr4vd1", "value": 1.0}, + ], + }, + "nda-scope-carveouts": { + # p2hn8sc died in the sandbox, so it is null in every key: an attempt that + # happened and did not pass, as opposed to one that was never measured. + "harbor_reward.reward": [ + {"trial_id": "nda-scope-carveouts__p2hn8sc", "value": None}, + {"trial_id": "nda-scope-carveouts__w5db3qy", "value": 1.0}, + {"trial_id": "nda-scope-carveouts__z8kt1nf", "value": 0.0}, + ], + "rubric_judge.criteria_pass_rate": [ + {"trial_id": "nda-scope-carveouts__p2hn8sc", "value": None}, + {"trial_id": "nda-scope-carveouts__w5db3qy", "value": 0.6}, + {"trial_id": "nda-scope-carveouts__z8kt1nf", "value": 0.2}, + ], + }, + # Requested, but the runner returned no trial for it: keys declared, nothing measured. + "merger-hsr-filing-threshold": { + "harbor_reward.reward": [], + "rubric_judge.criteria_pass_rate": [], + }, + } + ], ) task_count: int = Field(default=0, description="Number of tasks represented in the run.") trial_count: int = Field(default=0, description="Number of distinct trials scored.") @@ -536,6 +647,8 @@ def _aggregate_scores( for view_name, (values, total) in sorted(_semantic_view_values(scores, tasks).items()): aggregated.append(_aggregate_range_score(f"view.{view_name}", values, total)) + # if the caller already passed attempts → use them (no second scan of all scores) + # if not (None) → compute them inside _aggregate_scores (no need to pass them in) attempts = task_metric_attempts if task_metric_attempts is not None else _task_metric_attempts(scores, tasks) aggregated.extend(_task_pass_at_k_scores(attempts, tasks)) aggregated.extend(extra_scores) From 165a401043a63911a9ef734103e612536376d569 Mon Sep 17 00:00:00 2001 From: Nick Goncharenko Date: Tue, 11 Aug 2026 20:54:47 -0700 Subject: [PATCH 04/10] chore: cleanup Signed-off-by: Nick Goncharenko --- .../examples/gym/inspect_results.py | 16 ++++- .../nemo_evaluator_sdk/agent_eval/results.py | 42 ++++++++---- .../agent_eval/test_task_metric_attempts.py | 66 ++++++++++++++++++- .../beta/evaluator/agent_eval/results.py | 42 ++++++++---- 4 files changed, 139 insertions(+), 27 deletions(-) diff --git a/packages/nemo_evaluator_sdk/examples/gym/inspect_results.py b/packages/nemo_evaluator_sdk/examples/gym/inspect_results.py index 6a11b1b756..1b49272999 100644 --- a/packages/nemo_evaluator_sdk/examples/gym/inspect_results.py +++ b/packages/nemo_evaluator_sdk/examples/gym/inspect_results.py @@ -103,8 +103,20 @@ def per_task_attempts( def load_bundle(bundle: Path) -> AgentEvalSummary: - """Load the persisted summary, including native and runner aggregates and per-task attempts.""" - return AgentEvalSummary.model_validate(json.loads((bundle / "summary.json").read_text(encoding="utf-8"))) + """Load the persisted summary, including native and runner aggregates and per-task attempts. + + Rejects a bundle written before ``task_metric_attempts`` existed rather than reading one. The + field defaults to empty, so an older bundle would otherwise load cleanly and simply show no + per-task section — the reader would conclude the run had no per-task outcomes rather than that + this script cannot see them. + """ + payload = json.loads((bundle / "summary.json").read_text(encoding="utf-8")) + if "task_metric_attempts" not in payload: + raise SystemExit( + f"{bundle / 'summary.json'} predates summary.task_metric_attempts, which this script reads " + "per-task outcomes from. Re-run the eval to produce a current bundle." + ) + return AgentEvalSummary.model_validate(payload) # -------------------------------------------------------------------------------------------------- diff --git a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/results.py b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/results.py index 94cfb5ea24..4b7170a790 100644 --- a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/results.py +++ b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/results.py @@ -34,7 +34,7 @@ serialize_value, summary_aggregate_record, ) -from pydantic import BaseModel, ConfigDict, Field +from pydantic import BaseModel, ConfigDict, Field, field_serializer #: Metric-output value schemas retained in the ordered per-task attempt mapping. _TASK_METRIC_VALUE_SCHEMAS = (ContinuousScore, DiscreteScore, BooleanValue) @@ -86,6 +86,20 @@ class AgentEvalAttemptValue(BaseModel): ), ) + @field_serializer("value") + def serialize_nan(self, value: float | None) -> float | str | None: + """Emit NaN as the string ``"NaN"``, matching :class:`MetricOutput`. + + A metric may legitimately score an attempt NaN, and this is the first summary field to carry + a raw metric value rather than a filtered aggregate. ``json.dumps`` would write it as a bare + ``NaN`` token, which is valid Python but not valid JSON, so any strict reader of + ``summary.json`` would reject the whole bundle. Pydantic coerces the string back to a float + on load, so the round trip is lossless. + """ + if isinstance(value, float) and math.isnan(value): + return "NaN" + return value + class AgentEvalSummary(BaseModel): """Aggregated scores, coverage, per-task attempt values, and run counts for an agent-eval run.""" @@ -744,9 +758,10 @@ def _task_metric_attempts( reuses one costs pass@k nothing. """ output_keys: dict[str, set[tuple[str, str]]] = {} - # Declared under a schema this mapping does not retain. Tracked so an emitted numeric value cannot - # add back what the spec filter just excluded. - excluded: set[tuple[str, str]] = set() + # Per task, the outputs it declared under a schema this mapping does not retain. Tracked so an + # emitted numeric value cannot add back what that task's spec filter just excluded -- and keyed by + # task because tasks in one run need not declare the same output under the same schema. + excluded: dict[str, set[tuple[str, str]]] = {} if tasks is not None: for task in tasks: task_keys = output_keys.setdefault(task.id, set()) @@ -756,7 +771,7 @@ def _task_metric_attempts( if issubclass(spec.value_schema, _TASK_METRIC_VALUE_SCHEMAS): task_keys.add((metric_type, spec.name)) else: - excluded.add((metric_type, spec.name)) + excluded.setdefault(task.id, set()).add((metric_type, spec.name)) scores_by_task_metric: dict[tuple[str, str], list[AgentEvalTaskScore]] = {} for score in scores: @@ -764,8 +779,9 @@ def _task_metric_attempts( task_keys = output_keys.setdefault(score.task_id, set()) if score.status not in (AgentEvalScoreStatus.COMPLETED, AgentEvalScoreStatus.PARTIAL): continue + task_excluded = excluded.get(score.task_id, frozenset()) for output in score.outputs: - if (score.metric_type, output.name) in excluded: + if (score.metric_type, output.name) in task_excluded: continue if _semantic_value(output) is not None: task_keys.add((score.metric_type, output.name)) @@ -809,11 +825,15 @@ def _task_pass_at_k_scores( denominator is never silent. (Tasks excluded from a given ``k`` merely for having fewer than ``k`` attempts are *not* counted there — that is the estimator working as defined, not missing data.) - "No usable attempt" includes a task that was never scored at all: a runner that returns no trial - for a requested task (Harbor logs a warning and carries on) leaves it declaring the metric with an - empty attempt list, and it lands in ``nan_count`` like any other unmeasured task. That is - deliberate — it is the same missing coverage whether the trial died or was never produced, and - excluding it would report pass@k over a denominator quietly smaller than the task set asked for. + "No usable attempt" includes a task that was never scored at all: it declares the metric, holds an + empty attempt list, and lands in ``nan_count`` like any other unmeasured task. That is the same + missing coverage whether the trial died or was never produced, and excluding it would report + pass@k over a denominator quietly smaller than the task set asked for. + + Note this is reachable only through :meth:`AgentEvalSummary.from_scores` called directly with a + task list wider than the scores — a caller re-aggregating a subset, say. A full run cannot get + here: :meth:`AgentEvaluator._score_trials` refuses to score at all when a task produced no trial, + so a runner that drops one fails the run rather than reporting it as missing coverage. """ scorelike = _scorelike_outputs(tasks) if not scorelike: diff --git a/packages/nemo_evaluator_sdk/tests/agent_eval/test_task_metric_attempts.py b/packages/nemo_evaluator_sdk/tests/agent_eval/test_task_metric_attempts.py index bb87decc6e..03da1c559e 100644 --- a/packages/nemo_evaluator_sdk/tests/agent_eval/test_task_metric_attempts.py +++ b/packages/nemo_evaluator_sdk/tests/agent_eval/test_task_metric_attempts.py @@ -3,6 +3,8 @@ from __future__ import annotations +import json +import math from collections.abc import Iterator from pathlib import Path @@ -184,9 +186,11 @@ def test_failed_trials_are_attempts_but_metric_failures_are_unmeasured() -> None def test_a_task_that_produced_no_trial_is_unmeasured_and_counted_in_pass_at_k_nan() -> None: - # A runner may return no trial at all for a requested task (Harbor warns and carries on). The task - # still declares the metric, so it holds an empty attempt list and counts as missing coverage -- - # excluding it would report pass@k over a denominator smaller than the task set that was asked for. + # from_scores can be handed a task list wider than the scores -- a caller re-aggregating a subset. + # The task still declares the metric, so it holds an empty attempt list and counts as missing + # coverage: excluding it would report pass@k over a denominator smaller than the task set asked + # for. A full run cannot reach this state; AgentEvaluator._score_trials refuses to score when a + # task produced no trial, which test_evaluator.py::test_run_rejects_tasks_without_trials pins. reward = _Metric("reward", MetricOutputSpec.continuous_score("score")) tasks = [_task("scored", reward), _task("never-ran", reward)] scores = [_score("scored", "attempt-0", "reward", "score", 1.0)] @@ -223,6 +227,50 @@ def test_outputs_declared_under_an_unretained_schema_stay_out_even_when_numeric( } +def test_one_tasks_schema_exclusion_does_not_suppress_another_tasks_output() -> None: + # The spec filter is per task: tasks in one run need not declare the same output under the same + # schema. Task-a declaring usage.prompt_tokens as a free model must not strip it from task-b, + # which never declared it and whose only evidence is the numeric value it actually emitted. + tasks = [ + _task("task-a", _Metric("usage", MetricOutputSpec.model("prompt_tokens", _TokenCount))), + _task("task-b", _Metric("reward", MetricOutputSpec.continuous_score("score"))), + ] + scores = [ + _score("task-a", "attempt-0", "usage", "prompt_tokens", 100), + _score("task-b", "attempt-0", "reward", "score", 1.0), + _score("task-b", "attempt-0", "usage", "prompt_tokens", 250), # undeclared on task-b + ] + + attempts = AgentEvalSummary.from_scores(scores, tasks=tasks).task_metric_attempts + + # task-a declared it under an unretained schema, so it is not a key there at all -- not even an + # empty one -- and the numeric value it emitted cannot add it back. + assert attempts["task-a"] == {} + # task-b never declared it, so its emitted numeric value is the only evidence and it is kept. + assert sorted(attempts["task-b"]) == ["reward.score", "usage.prompt_tokens"] + assert _pairs(attempts)["task-b"]["usage.prompt_tokens"] == [("attempt-0", 250.0)] + + +def test_nan_attempt_values_survive_json_as_a_string() -> None: + # A metric may legitimately score NaN. json.dumps would write a bare NaN token, which is not + # valid JSON, so summary.json must carry the string form -- and read it back as a float. + tasks = [_task("task-a", _Metric("reward", MetricOutputSpec.continuous_score("score")))] + summary = AgentEvalSummary.from_scores( + [_score("task-a", "attempt-0", "reward", "score", float("nan"))], tasks=tasks + ) + + payload = summary.model_dump(mode="json") + assert payload["task_metric_attempts"]["task-a"]["reward.score"][0]["value"] == "NaN" + + # Strict JSON: no bare NaN/Infinity tokens anywhere in the serialized bundle. + def _reject(constant: str) -> float: + raise AssertionError(f"summary.json contains a bare {constant} token") + + reloaded = json.loads(json.dumps(payload), parse_constant=_reject) + value = AgentEvalSummary.model_validate(reloaded).task_metric_attempts["task-a"]["reward.score"][0].value + assert value is not None and math.isnan(value) + + def test_without_tasks_there_is_no_spec_to_filter_on() -> None: # No tasks means no declared schemas to consult, so every numeric output observed is retained. scores = [_score("task-a", "attempt-0", "usage", "prompt_tokens", 1234)] @@ -389,6 +437,18 @@ def test_vendored_results_module_is_a_verbatim_copy_of_this_one() -> None: ) +def test_gym_example_rejects_a_bundle_written_before_task_metric_attempts(tmp_path: Path) -> None: + # The field defaults to empty, so an older bundle would load cleanly and simply show no per-task + # section -- a reader would take that as "no per-task outcomes" rather than "this script cannot + # see them". Fail with a version message instead. + from packages.nemo_evaluator_sdk.examples.gym.inspect_results import load_bundle + + (tmp_path / "summary.json").write_text(json.dumps({"task_count": 2}), encoding="utf-8") + + with pytest.raises(SystemExit, match="predates summary.task_metric_attempts"): + load_bundle(tmp_path) + + def test_gym_example_reads_task_outcomes_from_summary() -> None: from packages.nemo_evaluator_sdk.examples.gym.inspect_results import per_task_attempts, per_task_outcomes diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/results.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/results.py index e833fd4ba4..39d67c527b 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/results.py +++ b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/results.py @@ -34,7 +34,7 @@ serialize_value, summary_aggregate_record, ) -from pydantic import BaseModel, ConfigDict, Field +from pydantic import BaseModel, ConfigDict, Field, field_serializer #: Metric-output value schemas retained in the ordered per-task attempt mapping. _TASK_METRIC_VALUE_SCHEMAS = (ContinuousScore, DiscreteScore, BooleanValue) @@ -86,6 +86,20 @@ class AgentEvalAttemptValue(BaseModel): ), ) + @field_serializer("value") + def serialize_nan(self, value: float | None) -> float | str | None: + """Emit NaN as the string ``"NaN"``, matching :class:`MetricOutput`. + + A metric may legitimately score an attempt NaN, and this is the first summary field to carry + a raw metric value rather than a filtered aggregate. ``json.dumps`` would write it as a bare + ``NaN`` token, which is valid Python but not valid JSON, so any strict reader of + ``summary.json`` would reject the whole bundle. Pydantic coerces the string back to a float + on load, so the round trip is lossless. + """ + if isinstance(value, float) and math.isnan(value): + return "NaN" + return value + class AgentEvalSummary(BaseModel): """Aggregated scores, coverage, per-task attempt values, and run counts for an agent-eval run.""" @@ -744,9 +758,10 @@ def _task_metric_attempts( reuses one costs pass@k nothing. """ output_keys: dict[str, set[tuple[str, str]]] = {} - # Declared under a schema this mapping does not retain. Tracked so an emitted numeric value cannot - # add back what the spec filter just excluded. - excluded: set[tuple[str, str]] = set() + # Per task, the outputs it declared under a schema this mapping does not retain. Tracked so an + # emitted numeric value cannot add back what that task's spec filter just excluded -- and keyed by + # task because tasks in one run need not declare the same output under the same schema. + excluded: dict[str, set[tuple[str, str]]] = {} if tasks is not None: for task in tasks: task_keys = output_keys.setdefault(task.id, set()) @@ -756,7 +771,7 @@ def _task_metric_attempts( if issubclass(spec.value_schema, _TASK_METRIC_VALUE_SCHEMAS): task_keys.add((metric_type, spec.name)) else: - excluded.add((metric_type, spec.name)) + excluded.setdefault(task.id, set()).add((metric_type, spec.name)) scores_by_task_metric: dict[tuple[str, str], list[AgentEvalTaskScore]] = {} for score in scores: @@ -764,8 +779,9 @@ def _task_metric_attempts( task_keys = output_keys.setdefault(score.task_id, set()) if score.status not in (AgentEvalScoreStatus.COMPLETED, AgentEvalScoreStatus.PARTIAL): continue + task_excluded = excluded.get(score.task_id, frozenset()) for output in score.outputs: - if (score.metric_type, output.name) in excluded: + if (score.metric_type, output.name) in task_excluded: continue if _semantic_value(output) is not None: task_keys.add((score.metric_type, output.name)) @@ -809,11 +825,15 @@ def _task_pass_at_k_scores( denominator is never silent. (Tasks excluded from a given ``k`` merely for having fewer than ``k`` attempts are *not* counted there — that is the estimator working as defined, not missing data.) - "No usable attempt" includes a task that was never scored at all: a runner that returns no trial - for a requested task (Harbor logs a warning and carries on) leaves it declaring the metric with an - empty attempt list, and it lands in ``nan_count`` like any other unmeasured task. That is - deliberate — it is the same missing coverage whether the trial died or was never produced, and - excluding it would report pass@k over a denominator quietly smaller than the task set asked for. + "No usable attempt" includes a task that was never scored at all: it declares the metric, holds an + empty attempt list, and lands in ``nan_count`` like any other unmeasured task. That is the same + missing coverage whether the trial died or was never produced, and excluding it would report + pass@k over a denominator quietly smaller than the task set asked for. + + Note this is reachable only through :meth:`AgentEvalSummary.from_scores` called directly with a + task list wider than the scores — a caller re-aggregating a subset, say. A full run cannot get + here: :meth:`AgentEvaluator._score_trials` refuses to score at all when a task produced no trial, + so a runner that drops one fails the run rather than reporting it as missing coverage. """ scorelike = _scorelike_outputs(tasks) if not scorelike: From aecc8d1c1ca04b27e3d778730396649cc5a71f70 Mon Sep 17 00:00:00 2001 From: Nick Goncharenko Date: Wed, 12 Aug 2026 21:00:15 -0700 Subject: [PATCH 05/10] feat: make values typed Signed-off-by: Nick Goncharenko --- docs/evaluator/agent-eval/reading-results.mdx | 28 +- .../nemo_evaluator_sdk/examples/gym/README.md | 12 +- .../examples/gym/inspect_results.py | 85 ++- .../examples/harbor/run_harbor_example.py | 26 +- .../nemo_evaluator_sdk/agent_eval/results.py | 527 ++++++++++---- .../nemo_evaluator_sdk/agent_eval/scores.py | 2 +- .../tests/agent_eval/test_harbor_runtime.py | 42 +- .../tests/agent_eval/test_pass_at_k.py | 40 +- .../tests/agent_eval/test_persistence.py | 38 +- .../tests/agent_eval/test_result_display.py | 2 +- .../agent_eval/test_task_metric_attempts.py | 474 ------------- .../agent_eval/test_task_metric_values.py | 656 ++++++++++++++++++ .../beta/evaluator/agent_eval/results.py | 527 ++++++++++---- .../beta/evaluator/agent_eval/scores.py | 2 +- 14 files changed, 1606 insertions(+), 855 deletions(-) delete mode 100644 packages/nemo_evaluator_sdk/tests/agent_eval/test_task_metric_attempts.py create mode 100644 packages/nemo_evaluator_sdk/tests/agent_eval/test_task_metric_values.py diff --git a/docs/evaluator/agent-eval/reading-results.mdx b/docs/evaluator/agent-eval/reading-results.mdx index 53bf03dceb..c5a014dc3a 100644 --- a/docs/evaluator/agent-eval/reading-results.mdx +++ b/docs/evaluator/agent-eval/reading-results.mdx @@ -37,21 +37,37 @@ result = await AgentEvaluator().run(tasks=..., target=...) - **`summary.metric_coverage`** — per metric output, how many trials were `total` / `scored` / failed / missing, so you can tell a low mean from low coverage. -- **`summary.task_metric_attempts`** — per task, the individual attempts behind those means, keyed - `.`. Each attempt carries the `trial_id` that produced it and its `value`, so - you can answer "which tasks were flaky, and on which attempt?" without regrouping `result.scores` +- **`summary.task_metric_values`** — per task, the individual trial values behind those means, keyed + `.`. Each record carries the `trial_id` that produced it and its metric `value`, so + you can answer "which tasks were flaky, and on which trial?" without regrouping `result.scores` yourself: ```python - for task_id, by_output in result.summary.task_metric_attempts.items(): + for task_id, by_output in result.summary.task_metric_values.items(): # .get: keys are per task, so a task scored by a different metric simply has none. print(task_id, [(a.trial_id, a.value) for a in by_output.get("reward.score", [])]) ``` - A `value` of `None` is a trial that died before scoring — an attempt that did not pass. An attempt + A `value` of `None` is a trial that died before scoring — a trial that did not pass. A trial whose *metric* failed is absent entirely, because that leaves it unmeasured rather than - unsuccessful. Join by `trial_id` rather than by position: the two rules above mean lists for + unsuccessful. Look up by `trial_id` rather than by position: the two rules above mean lists for different outputs of one task need not be the same length. + + Values keep the type the metric produced them in — a count stays an `int`, a flag stays a `bool`, + and a judge's verdict stays a `str`. Each record's `value_type` says which it is (`number`, + `label` or `missing`), which is what tells a real `NaN` apart from a label that reads `"NaN"`, since + strict JSON has no NaN literal and both travel as strings. Before doing arithmetic, project with + `numeric_metric_values`, which drops labels and keeps a dead trial's `None`: + + ```python + from nemo_evaluator_sdk.agent_eval.results import numeric_metric_values + + records = result.summary.task_metric_values["task-47"]["reward.score"] + scores = numeric_metric_values(records) # [1.0, None, 0.0] + ``` + + For a typed object rather than nested dicts, `result.summary.task_outcomes()` returns + `list[PerTaskOutcomes]`, each naming its `task_id` and its outcomes' `metric_name`. - **`summary.task_count`**, **`summary.trial_count`**, **`summary.score_count`**. ### Per-metric scores diff --git a/packages/nemo_evaluator_sdk/examples/gym/README.md b/packages/nemo_evaluator_sdk/examples/gym/README.md index a7d39d6e24..3c37df7da1 100644 --- a/packages/nemo_evaluator_sdk/examples/gym/README.md +++ b/packages/nemo_evaluator_sdk/examples/gym/README.md @@ -58,14 +58,18 @@ Run bundle (run.json, trials.jsonl, scores.jsonl, report.html): /var/folders/... ## Read the results `inspect_results.py` reads `summary.json` and shows each result layer: run aggregates from -`summary.scores`, ordered per-task attempts from `summary.task_metric_attempts`, and runner-owned -aggregates under `runner.gym.*`. Per-task keys use `.`; a `null` attempt is a trial +`summary.scores`, ordered per-task values from `summary.task_metric_values`, and runner-owned +aggregates under `runner.gym.*`. Per-task keys use `.`; a `null` value is a trial that failed before scoring, while an empty list means the metric produced no usable measurement. -Each attempt names the trial that produced it, so `trial_id` — not list position — is what joins two -outputs of the same task, or joins out to `trials.jsonl`. An attempt whose metric failed is absent +Each record names the trial that produced it, so `trial_id` — not list position — is what joins two +outputs of the same task, or looks up `trials.jsonl`. A trial whose metric failed is absent rather than `null`, so two lists for one task need not be the same length. +Values keep the metric's own type (a count stays an int, a judge's verdict stays a label), and each +carries a `value_type` of `number` / `label` / `missing`. Use `numeric_metric_values` before doing +arithmetic — it drops labels and keeps a dead trial's `null`. + No bundle is checked in; the run above produces one. Give it a stable `--output-dir` and point the reader at the same path: ```bash diff --git a/packages/nemo_evaluator_sdk/examples/gym/inspect_results.py b/packages/nemo_evaluator_sdk/examples/gym/inspect_results.py index 1b49272999..f1cad972b0 100644 --- a/packages/nemo_evaluator_sdk/examples/gym/inspect_results.py +++ b/packages/nemo_evaluator_sdk/examples/gym/inspect_results.py @@ -31,15 +31,24 @@ import json from pathlib import Path -from nemo_evaluator_sdk.agent_eval.results import AgentEvalAttemptValue, AgentEvalSummary, attempt_values +from nemo_evaluator_sdk.agent_eval.results import ( + AgentEvalSummary, + TrialMetricValue, + numeric_metric_values, +) from nemo_evaluator_sdk.values.results import AggregateScalarScore, AggregateScore -#: Value at which an attempt counts as a pass, matching the SDK's pass@k definition (full credit). +#: Value at which a trial counts as a pass, matching the SDK's pass@k definition (full credit). PASS_VALUE = 1.0 #: Namespace the Gym runner's own aggregations are imported under, so they never collide with ours. RUNNER_PREFIX = "runner.gym." + +class BundleFormatError(Exception): + """A run bundle this script cannot read (wrong directory, or written by an older SDK).""" + + # -------------------------------------------------------------------------------------------------- # Accessors — lift these into your own code. # -------------------------------------------------------------------------------------------------- @@ -64,56 +73,67 @@ def per_task_outcomes( metric_type: str, output_name: str, ) -> dict[str, list[float | None]]: - """Read ordered attempt values from the summary for one metric output. + """Read ordered trial values from the summary for one metric output. - ``None`` is a failed trial and therefore a failed attempt. An empty list means the task had no - usable measurement because its metric failed or omitted the output. + ``None`` is a failed trial and therefore did not pass. An empty list means the task had no + usable measurement — its metric failed, omitted the output, produced no trial at all, or scored + only non-numeric labels. - Use :func:`per_task_attempts` when you need to know *which* trial produced a value. + Use :func:`per_task_trial_values` when you need to know *which* trial produced a value. """ return { - task_id: attempt_values(attempts) - for task_id, attempts in per_task_attempts(summary, metric_type=metric_type, output_name=output_name).items() + task_id: numeric_metric_values(records) + for task_id, records in per_task_trial_values(summary, metric_type=metric_type, output_name=output_name).items() } -def per_task_attempts( +def per_task_trial_values( summary: AgentEvalSummary, *, metric_type: str, output_name: str, -) -> dict[str, list[AgentEvalAttemptValue]]: - """The same attempts, each still naming the trial that produced it. +) -> dict[str, list[TrialMetricValue]]: + """The same values, each still naming the trial that produced it. - An attempt whose metric failed is absent rather than null, so lists for two different outputs of + A trial whose metric failed is absent rather than null, so lists for two different outputs of one task need not be the same length — ``trial_id``, not position, is what lines them up. It is - also the join key out to ``trials.jsonl``, which is where a failed attempt's error lives. + also the lookup key into ``trials.jsonl``, which is where a failed trial's error lives. """ key = f"{metric_type}.{output_name}" return { - task_id: list(metric_attempts[key]) - for task_id, metric_attempts in summary.task_metric_attempts.items() - if key in metric_attempts + task_id: list(by_metric[key]) for task_id, by_metric in summary.task_metric_values.items() if key in by_metric } +# The typed view over the same data lives in the SDK: `summary.task_outcomes()` returns +# `list[PerTaskOutcomes]`, each naming its task and metric. Use it when you want an object to pass +# around rather than a dict keyed by strings. + # -------------------------------------------------------------------------------------------------- # Bundle loading (see the run.json manifest for the full artifact list). # -------------------------------------------------------------------------------------------------- def load_bundle(bundle: Path) -> AgentEvalSummary: - """Load the persisted summary, including native and runner aggregates and per-task attempts. + """Load the persisted summary, including native and runner aggregates and per-task values. - Rejects a bundle written before ``task_metric_attempts`` existed rather than reading one. The + Rejects a bundle written before ``task_metric_values`` existed rather than reading one. The field defaults to empty, so an older bundle would otherwise load cleanly and simply show no per-task section — the reader would conclude the run had no per-task outcomes rather than that this script cannot see them. """ - payload = json.loads((bundle / "summary.json").read_text(encoding="utf-8")) - if "task_metric_attempts" not in payload: - raise SystemExit( - f"{bundle / 'summary.json'} predates summary.task_metric_attempts, which this script reads " + summary_path = bundle / "summary.json" + if not summary_path.exists(): + raise BundleFormatError(f"{bundle} is not a run bundle (no summary.json). Run run_gym_eval.py first.") + try: + payload = json.loads(summary_path.read_text(encoding="utf-8")) + except json.JSONDecodeError as exc: + raise BundleFormatError(f"{summary_path} is not readable JSON): {exc}") from exc + # Checked explicitly because `model_validate` would *not* catch this: the field defaults to an + # empty dict, so an older bundle loads cleanly and simply shows no per-task section. + if "task_metric_values" not in payload: + raise BundleFormatError( + f"{summary_path} predates summary.task_metric_values, which this script reads " "per-task outcomes from. Re-run the eval to produce a current bundle." ) return AgentEvalSummary.model_validate(payload) @@ -149,17 +169,17 @@ def show_aggregates(summary: AgentEvalSummary) -> None: def show_per_task(by_task: dict[str, list[float | None]]) -> None: """Per-task outcomes: which tasks were solved, and how consistently. - An attempt passes on full credit (``>= PASS_VALUE``), matching how the SDK computes pass@k. A - ``None`` is a trial that died: it counts as an attempt and never as a pass, so a task that passed + A trial passes on full credit (``>= PASS_VALUE``), matching how the SDK computes pass@k. A + ``None`` is a trial that died: it counts toward ``n`` and never as a pass, so a task that passed once and crashed once reads as flaky rather than solved. """ - print("\nPer-task outcomes (attempt values; an attempt passes at full credit)") + print("\nPer-task outcomes (trial values; a trial passes at full credit)") solved = flaky = failed = unmeasured = 0 for task_id, values in sorted(by_task.items()): if not values: verdict, marker = "unmeasured", "?" unmeasured += 1 - attempts = "" + shown = "" else: passes = sum(1 for value in values if value is not None and value >= PASS_VALUE) if passes == len(values): @@ -171,8 +191,8 @@ def show_per_task(by_task: dict[str, list[float | None]]) -> None: else: verdict, marker = "failed", "-" failed += 1 - attempts = ", ".join("died" if value is None else f"{value:g}" for value in values) - print(f" {marker} {task_id[:16]}… [{attempts}] {verdict}") + shown = ", ".join("died" if value is None else f"{value:g}" for value in values) + print(f" {marker} {task_id[:16]}… [{shown}] {verdict}") print(f"\n {solved} solved · {flaky} flaky · {failed} failed · {unmeasured} unmeasured") @@ -216,10 +236,11 @@ def _parse_args(argv: list[str] | None = None) -> argparse.Namespace: def main(argv: list[str] | None = None) -> int: args = _parse_args(argv) - if not (args.bundle / "summary.json").exists(): - raise SystemExit(f"{args.bundle} is not a run bundle (no summary.json). Run run_gym_eval.py first.") - - summary = load_bundle(args.bundle) + # The CLI boundary is where a bad bundle becomes an exit code; the accessors above just raise. + try: + summary = load_bundle(args.bundle) + except BundleFormatError as exc: + raise SystemExit(str(exc)) from exc show_aggregates(summary) by_task = per_task_outcomes(summary, metric_type=args.metric_type, output_name=args.output_name) diff --git a/packages/nemo_evaluator_sdk/examples/harbor/run_harbor_example.py b/packages/nemo_evaluator_sdk/examples/harbor/run_harbor_example.py index a5c869a7f8..f0704cdbe9 100644 --- a/packages/nemo_evaluator_sdk/examples/harbor/run_harbor_example.py +++ b/packages/nemo_evaluator_sdk/examples/harbor/run_harbor_example.py @@ -21,6 +21,7 @@ Run it as a module from the repository root:: python -m packages.nemo_evaluator_sdk.examples.harbor.run_harbor_example --mode native + python -m packages.nemo_evaluator_sdk.examples.harbor.run_harbor_example --mode native --n-attempts 2 python -m packages.nemo_evaluator_sdk.examples.harbor.run_harbor_example --mode optimizer """ @@ -45,9 +46,17 @@ HELLO_WORLD_DATASET_DIR = Path(__file__).resolve().parent / "hello_world_dataset" -async def _main(mode: str, jobs_dir: Path) -> None: +async def _main(mode: str, jobs_dir: Path, *, n_attempts: int, job_name: str | None) -> None: # The entire caller-side plumbing: a config and one call. - config = HarborRuntimeConfig(jobs_dir=jobs_dir, agent_name="oracle") + # n_attempts>1 runs the same verifier criteria per attempt so summary can emit pass@k. + config = HarborRuntimeConfig( + jobs_dir=jobs_dir, + job_name=job_name, + agent_name="oracle", + n_attempts=n_attempts, + n_concurrent_trials=1, + quiet=False, + ) result = await run_harbor_eval(config, HELLO_WORLD_DATASET_DIR) if mode == "optimizer": @@ -78,5 +87,16 @@ async def _main(mode: str, jobs_dir: Path) -> None: default=Path(__file__).resolve().parent / "harbor-example-output", help="Directory Harbor writes its job results into.", ) + parser.add_argument( + "--n-attempts", + type=int, + default=1, + help="Harbor trials per task (same verifier criteria each attempt; enables pass@k when >1).", + ) + parser.add_argument( + "--job-name", + default=None, + help="Pin a stable Harbor job name to reuse the job-dir cache across debug runs.", + ) args = parser.parse_args() - asyncio.run(_main(args.mode, args.jobs_dir)) + asyncio.run(_main(args.mode, args.jobs_dir, n_attempts=args.n_attempts, job_name=args.job_name)) diff --git a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/results.py b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/results.py index 4b7170a790..26f23694b3 100644 --- a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/results.py +++ b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/results.py @@ -9,6 +9,7 @@ import math from collections.abc import Mapping, Sequence from datetime import datetime +from enum import Enum from pathlib import Path from typing import Any @@ -23,7 +24,7 @@ from nemo_evaluator_sdk.metrics.aggregation import compute_percentiles from nemo_evaluator_sdk.metrics.protocol import MetricOutput from nemo_evaluator_sdk.metrics.utils import metric_type_name -from nemo_evaluator_sdk.values.protocol import BooleanValue, ContinuousScore, DiscreteScore +from nemo_evaluator_sdk.values.protocol import BooleanValue, ContinuousScore, DiscreteScore, Label from nemo_evaluator_sdk.values.results import ( AggregatedMetricResult, AggregateRangeScore, @@ -34,16 +35,18 @@ serialize_value, summary_aggregate_record, ) -from pydantic import BaseModel, ConfigDict, Field, field_serializer +from pydantic import BaseModel, ConfigDict, Field, field_serializer, model_validator -#: Metric-output value schemas retained in the ordered per-task attempt mapping. -_TASK_METRIC_VALUE_SCHEMAS = (ContinuousScore, DiscreteScore, BooleanValue) +#: Metric-output value schemas retained in the ordered per-task value mapping. Broader than +#: :data:`_PASS_AT_K_VALUE_SCHEMAS` on purpose: a :class:`TrialMetricValue` is per-trial evidence, so a +#: count or a judge's label is worth keeping even though neither is a "did it pass?" signal. +_TASK_METRIC_VALUE_SCHEMAS = (ContinuousScore, DiscreteScore, BooleanValue, Label) -#: Metric-output value schemas eligible for pass@k (a per-attempt "did it pass?" signal). Labels, +#: Metric-output value schemas eligible for pass@k (a per-trial "did it pass?" signal). Labels, #: discrete/count outputs, and free models (e.g. token measurements) are excluded. _PASS_AT_K_VALUE_SCHEMAS = (ContinuousScore, BooleanValue) -#: Score value at or above which an attempt counts as a pass for pass@k. Full credit — pass@k answers +#: Score value at or above which a trial counts as a pass for pass@k. Full credit — pass@k answers #: "did the agent solve the task", so partial credit is not a pass. Deliberately not configurable: #: it's a reporting-time interpretation, and making it tunable would yield pass@k numbers that look #: comparable across runs but aren't. @@ -61,8 +64,53 @@ class AgentEvalMetricOutputCoverage(BaseModel): missing: int = Field(default=0, description="Scores where the output was expected but absent.") -class AgentEvalAttemptValue(BaseModel): - """One attempt at a task under one metric output: which trial made it, and what it measured. +#: Tokens :class:`TrialMetricValue` escapes non-finite floats as, and the floats they decode to. +#: Strict JSON has no literal for these, so they travel as strings -- which is the whole reason the +#: record carries ``value_type``: without it, a label that happens to read "NaN" is the same three +#: bytes as a real NaN. +_SPECIAL_FLOAT_TOKENS_MAP: dict[str, float] = { + "NaN": float("nan"), + "Infinity": float("inf"), + "-Infinity": float("-inf"), +} + + +def _escape_special_float(value: float) -> str: + """The token :data:`_SPECIAL_FLOAT_TOKENS_MAP` decodes back to ``value``. + + Looked up rather than spelled out a second time, so the encode and decode directions cannot + drift apart. NaN needs :func:`math.isnan` rather than equality: it is the one float that does + not equal itself, so a lookup keyed by value would miss it. + """ + for token, decoded in _SPECIAL_FLOAT_TOKENS_MAP.items(): + if decoded == value or (math.isnan(decoded) and math.isnan(value)): + return token + raise ValueError(f"{value!r} is a finite float and needs no escape") + + +class TrialMetricValueType(str, Enum): + """What kind of value one trial recorded under one metric output. + + Deliberately coarser than the declared value schemas: JSON already round-trips int, float and + bool distinctly, so a ``continuous``/``discrete``/``boolean`` split would restate what the payload + already says and give a reader two sources of truth for one fact. The only thing JSON cannot + carry is whether a string is a number's escape or a label, and that is exactly what this + discriminates. + """ + + NUMBER = "number" + LABEL = "label" + MISSING = "missing" + + +class TrialMetricValue(BaseModel): + """One trial's measured value under one metric output: which trial made it, and what it measured. + + Values keep the type the metric produced them in -- a count stays an int, a flag stays a bool, a + judge's verdict stays the string it was -- because this is one trial's measurement, not a mean or + other aggregate. Look up the matching trial by ``trial_id`` (in ``result.trials`` or + ``trials.jsonl``); do not assume list index lines up across metric outputs. Read it through + :func:`numeric_metric_values` when you intend to do arithmetic. Frozen because these records are handed out by reference from the summary: a consumer rescaling values in place (Gym reports reward on 0-100 where we use 0-1) would otherwise rewrite the run's @@ -73,36 +121,136 @@ class AgentEvalAttemptValue(BaseModel): trial_id: str = Field( description=( - "Identifier of the trial that made this attempt. Joins to AgentEvalTrial.id " + "Identifier of the trial that produced this value. Joins to AgentEvalTrial.id " "(trials.jsonl) and AgentEvalTaskScore.trial_id (scores.jsonl)." ) ) - value: float | None = Field( + # The default is never observed: `_derive_value_type` runs before validation and always supplies + # one. It exists so callers can write TrialMetricValue(trial_id=..., value=...) without + # restating what the value already says -- the type checker reads the signature, not the validator. + value_type: TrialMetricValueType = Field( + default=TrialMetricValueType.MISSING, + description=( + "Which kind of value this record holds: 'number' (float, int or bool), 'label' (a " + "categorical string), or 'missing' (the trial failed before it could be measured). " + "Always present in serialized output, because non-finite floats are escaped as strings " + "-- without it a genuine label reading 'NaN' and a real NaN are the same three bytes. " + "Derived from 'value' when omitted, so hand-built records and bundles written before " + "this field existed both load." + ), + ) + value: float | int | bool | str | None = Field( description=( - "What the metric output measured, or None when the trial failed before it could be " - "measured -- an attempt that did not pass. Required rather than defaulted: None is a " - "load-bearing signal pass@k counts as a failed attempt, so an omitted value must not " - "quietly become one." + "What the metric output measured, in the type the metric produced it in -- a number, a " + "label, or None when the trial failed before it could be measured: a trial that did " + "not pass. Required rather than defaulted: None is a load-bearing signal pass@k counts " + "as not passing, so an omitted value must not quietly become one. None never means " + "'no value of this kind'; that is what value_type is for." ), ) + @model_validator(mode="before") + @classmethod + def _derive_value_type(cls, data: Any) -> Any: + """Fill in ``value_type`` when absent, and decode the escaped-float form when present. + + Runs *before* the union so the escape is undone while the discriminator is still readable: + afterwards pydantic's smart mode has already committed ``"NaN"`` to ``str``, and the record + would be a label whatever the type said. + """ + if not isinstance(data, Mapping): + return data + value = data.get("value") + declared = data.get("value_type") + + if declared is None: + # No discriminator: a hand-built record, or a bundle written before this field existed. + # The old encoding gave a string exactly one meaning -- the escape -- so honour that + # rather than reading a pre-widening NaN as the label "NaN". A label that genuinely + # reads "NaN" must therefore name its value_type explicitly. + if isinstance(value, str): + if value in _SPECIAL_FLOAT_TOKENS_MAP: + return { + **data, + "value_type": TrialMetricValueType.NUMBER, + "value": _SPECIAL_FLOAT_TOKENS_MAP[value], + } + return {**data, "value_type": TrialMetricValueType.LABEL} + return { + **data, + "value_type": TrialMetricValueType.MISSING if value is None else TrialMetricValueType.NUMBER, + } + + if TrialMetricValueType(declared) is TrialMetricValueType.NUMBER and isinstance(value, str): + decoded = _SPECIAL_FLOAT_TOKENS_MAP.get(value) + if decoded is None: + raise ValueError( + f"value_type='number' but value {value!r} is not one of the escaped-float tokens " + f"{sorted(_SPECIAL_FLOAT_TOKENS_MAP)}; a categorical value must declare value_type='label'" + ) + return {**data, "value": decoded} + return data + + @model_validator(mode="after") + def _value_matches_its_type(self) -> TrialMetricValue: + """Re-narrow the union, so a record cannot claim one kind and carry another.""" + if self.value_type is TrialMetricValueType.MISSING: + if self.value is not None: + raise ValueError("value_type='missing' requires value None (a trial that died before measurement)") + elif self.value_type is TrialMetricValueType.LABEL: + if not isinstance(self.value, str): + raise ValueError(f"value_type='label' requires a string value, got {type(self.value).__name__}") + elif not isinstance(self.value, bool | int | float): + raise ValueError(f"value_type='number' requires a numeric value, got {type(self.value).__name__}") + return self + @field_serializer("value") - def serialize_nan(self, value: float | None) -> float | str | None: - """Emit NaN as the string ``"NaN"``, matching :class:`MetricOutput`. - - A metric may legitimately score an attempt NaN, and this is the first summary field to carry - a raw metric value rather than a filtered aggregate. ``json.dumps`` would write it as a bare - ``NaN`` token, which is valid Python but not valid JSON, so any strict reader of - ``summary.json`` would reject the whole bundle. Pydantic coerces the string back to a float - on load, so the round trip is lossless. + def serialize_nan(self, value: float | int | bool | str | None) -> float | int | bool | str | None: + """Escape non-finite floats as strings, so ``summary.json`` stays strict JSON. + + A metric may legitimately score a trial NaN, and this is the first summary field to carry + a raw metric value rather than a filtered aggregate. ``json.dumps`` would write a bare ``NaN`` + or ``Infinity`` token, which is valid Python but not valid JSON, so any strict reader of + ``summary.json`` would reject the whole bundle. ``value_type`` says which of these strings is + an escape and which is a label, so the round trip is lossless in both directions. + + This is deliberately *wider* than :meth:`MetricOutput.serialize_nan`, which escapes NaN only + and has no decoding validator -- an infinite value reaches ``scores.jsonl`` as ``null``. + Collapsing the SDK's several non-finite-float escapes into one pair belongs in ``values/``. """ - if isinstance(value, float) and math.isnan(value): - return "NaN" + if isinstance(value, float) and not math.isfinite(value): + return _escape_special_float(value) return value +#: One task's recorded values, keyed ``"."``. Named because the nesting is +#: otherwise spelled out at every producer, consumer and local that touches it, and because the key +#: format is the part a reader cannot infer from ``dict[str, ...]``. +TrialValuesByMetric = dict[str, list[TrialMetricValue]] + + +class PerTaskOutcome(BaseModel): + """Every trial's value at one task under one metric output.""" + + model_config = ConfigDict(extra="forbid") + + metric_name: str = Field(description="'.', e.g. 'gym_reward.reward'.") + trials: list[TrialMetricValue] = Field( + description="Values in trial order. A value of None is a trial that died before scoring." + ) + + +class PerTaskOutcomes(BaseModel): + """One task's values across every metric output that measured it.""" + + model_config = ConfigDict(extra="forbid") + + task_id: str = Field(description="The task these outcomes belong to.") + outcomes: list[PerTaskOutcome] = Field(description="One entry per metric output, sorted by metric_name.") + + class AgentEvalSummary(BaseModel): - """Aggregated scores, coverage, per-task attempt values, and run counts for an agent-eval run.""" + """Aggregated scores, coverage, per-task metric values, and run counts for an agent-eval run.""" model_config = ConfigDict(extra="forbid") @@ -193,49 +341,59 @@ class AgentEvalSummary(BaseModel): } ], ) - task_metric_attempts: dict[str, dict[str, list[AgentEvalAttemptValue]]] = Field( + task_metric_values: dict[str, TrialValuesByMetric] = Field( default_factory=dict, description=( - "Per task, the attempts each '.' measured, in trial order. Each " - "attempt names the trial that made it, so attempts join across keys -- and out to " - "trials.jsonl and scores.jsonl -- by trial_id. A failed trial has value None: an attempt " - "that did not pass. An unmeasured attempt (metric failed, output absent) has no entry at " - "all, so each key's list is independent: align by trial_id, never by position. An empty " - "list means nothing was measured, including a task that produced no trial." + "Per task, the values each '.' measured, in trial order. Each " + "record names the trial that produced it, so values join across keys -- and out to " + "trials.jsonl and scores.jsonl -- by trial_id. Values keep the type the metric produced " + "them in: a count stays an int, a flag stays a bool, a judge's verdict stays a label -- " + "read them through numeric_metric_values() before doing arithmetic. A failed trial has " + "value None: a trial that did not pass. An unmeasured trial (metric failed, output " + "absent) has no entry at all, so each key's list is independent: align by trial_id, " + "never by position. An empty list means nothing was measured, including a task that " + "produced no trial." ), examples=[ { "contract-review-msa-indemnity": { "harbor_reward.reward": [ - {"trial_id": "contract-review-msa-indemnity__k3f9wq2", "value": 1.0}, - {"trial_id": "contract-review-msa-indemnity__t7m2xb4", "value": 0.0}, - {"trial_id": "contract-review-msa-indemnity__9jr4vd1", "value": 1.0}, + {"trial_id": "contract-review-msa-indemnity__k3f9wq2", "value_type": "number", "value": 1.0}, + {"trial_id": "contract-review-msa-indemnity__t7m2xb4", "value_type": "number", "value": 0.0}, + {"trial_id": "contract-review-msa-indemnity__9jr4vd1", "value_type": "number", "value": 1.0}, + ], + # A count stays an int, and t7m2xb4's judge verdict is kept as a label -- neither + # is pass@k-eligible, but both are per-trial evidence worth recording. + "steps.count": [ + {"trial_id": "contract-review-msa-indemnity__k3f9wq2", "value_type": "number", "value": 14}, + {"trial_id": "contract-review-msa-indemnity__t7m2xb4", "value_type": "number", "value": 31}, + {"trial_id": "contract-review-msa-indemnity__9jr4vd1", "value_type": "number", "value": 12}, ], - # t7m2xb4 is absent here rather than null: its judge timed out, so that attempt + # t7m2xb4 is absent here rather than null: its judge timed out, so that trial # went unmeasured. Index 1 is therefore a different trial in each of these lists. "rubric_judge.criteria_pass_rate": [ - {"trial_id": "contract-review-msa-indemnity__k3f9wq2", "value": 0.75}, - {"trial_id": "contract-review-msa-indemnity__9jr4vd1", "value": 1.0}, + {"trial_id": "contract-review-msa-indemnity__k3f9wq2", "value_type": "number", "value": 0.75}, + {"trial_id": "contract-review-msa-indemnity__9jr4vd1", "value_type": "number", "value": 1.0}, ], }, "nda-scope-carveouts": { - # p2hn8sc died in the sandbox, so it is null in every key: an attempt that + # p2hn8sc died in the sandbox, so it is 'missing' in every key: a trial that # happened and did not pass, as opposed to one that was never measured. "harbor_reward.reward": [ - {"trial_id": "nda-scope-carveouts__p2hn8sc", "value": None}, - {"trial_id": "nda-scope-carveouts__w5db3qy", "value": 1.0}, - {"trial_id": "nda-scope-carveouts__z8kt1nf", "value": 0.0}, + {"trial_id": "nda-scope-carveouts__p2hn8sc", "value_type": "missing", "value": None}, + {"trial_id": "nda-scope-carveouts__w5db3qy", "value_type": "number", "value": 1.0}, + {"trial_id": "nda-scope-carveouts__z8kt1nf", "value_type": "number", "value": 0.0}, ], - "rubric_judge.criteria_pass_rate": [ - {"trial_id": "nda-scope-carveouts__p2hn8sc", "value": None}, - {"trial_id": "nda-scope-carveouts__w5db3qy", "value": 0.6}, - {"trial_id": "nda-scope-carveouts__z8kt1nf", "value": 0.2}, + "rubric_judge.verdict": [ + {"trial_id": "nda-scope-carveouts__p2hn8sc", "value_type": "missing", "value": None}, + {"trial_id": "nda-scope-carveouts__w5db3qy", "value_type": "label", "value": "compliant"}, + {"trial_id": "nda-scope-carveouts__z8kt1nf", "value_type": "label", "value": "overbroad"}, ], }, # Requested, but the runner returned no trial for it: keys declared, nothing measured. "merger-hsr-filing-threshold": { "harbor_reward.reward": [], - "rubric_judge.criteria_pass_rate": [], + "rubric_judge.verdict": [], }, } ], @@ -257,6 +415,24 @@ def score(self, name: str) -> AggregateScore: """ return self.scores.score(name) + def task_outcomes(self) -> list[PerTaskOutcomes]: + """:attr:`task_metric_values` as models that name their own keys, sorted by task then metric. + + A read-time *view*, not the wire format. The field itself stays a nested dict because it is + persisted per run: repeating "task_id"/"metric_name" on every row would grow ``summary.json`` + for no new information, and lookup by task and metric stays O(1). Reach for this when you + want a typed object to pass around or to hand to a template. + """ + return [ + PerTaskOutcomes( + task_id=task_id, + outcomes=[ + PerTaskOutcome(metric_name=key, trials=list(records)) for key, records in sorted(by_key.items()) + ], + ) + for task_id, by_key in sorted(self.task_metric_values.items()) + ] + @staticmethod def from_scores( scores: Sequence[AgentEvalTaskScore], @@ -270,16 +446,16 @@ def from_scores( ``runner..``), merged in so a backend's own figures are addressable the same way as ours. """ task_list = list(tasks) if tasks is not None else None - task_metric_attempts = _task_metric_attempts(scores, task_list) + task_metric_values = _task_metric_values(scores, task_list) return AgentEvalSummary( scores=_aggregate_scores( scores, task_list, extra_scores, - task_metric_attempts=task_metric_attempts, + task_metric_values=task_metric_values, ), metric_coverage=_metric_coverage(scores, task_list), - task_metric_attempts=task_metric_attempts, + task_metric_values=task_metric_values, task_count=len(task_list) if task_list is not None else len({score.task_id for score in scores}), trial_count=len({score.trial_id for score in scores}), score_count=len(scores), @@ -599,8 +775,8 @@ def _format_score_errors( ) -> list[str]: """Render the failed-score detail section, separating a failed trial from a failed metric. - Both arrive as ``FAILED``, but they mean different things to a reader: a failed trial is an - attempt the agent is answerable for, a failed metric is a measurement that never happened. The + Both arrive as ``FAILED``, but they mean different things to a reader: a failed trial is one + the agent is answerable for, a failed metric is a measurement that never happened. The dataset path has no equivalent distinction to make, so this section is agent-eval's own rather than a reuse of :func:`format_error_details`. """ @@ -628,7 +804,7 @@ def _aggregate_scores( tasks: Sequence[AgentEvalTask] | None, extra_scores: Sequence[AggregateScore] = (), *, - task_metric_attempts: dict[str, dict[str, list[AgentEvalAttemptValue]]] | None = None, + task_metric_values: dict[str, TrialValuesByMetric], ) -> AggregatedMetricResult: """Aggregate per-metric-output, per-semantic-view, and task-level pass@k values into range scores. @@ -661,29 +837,55 @@ def _aggregate_scores( for view_name, (values, total) in sorted(_semantic_view_values(scores, tasks).items()): aggregated.append(_aggregate_range_score(f"view.{view_name}", values, total)) - # if the caller already passed attempts → use them (no second scan of all scores) - # if not (None) → compute them inside _aggregate_scores (no need to pass them in) - attempts = task_metric_attempts if task_metric_attempts is not None else _task_metric_attempts(scores, tasks) - aggregated.extend(_task_pass_at_k_scores(attempts, tasks)) + # Required rather than recomputed here: the summary needs the same mapping, and deriving it + # twice is what this rewiring exists to stop. The one caller builds it once and shares it. + aggregated.extend(_task_pass_at_k_scores(task_metric_values, tasks)) aggregated.extend(extra_scores) return AggregatedMetricResult(scores=aggregated) -def attempt_values(attempts: Sequence[AgentEvalAttemptValue]) -> list[float | None]: - """The bare per-attempt values, for consumers scoring attempts without caring which trial made them. +def metric_values(records: Sequence[TrialMetricValue]) -> list[float | int | bool | str | None]: + """The bare per-trial values, for consumers reading records without caring which trial made them. + + Preserves order, cardinality, type, and the None-versus-absent distinction exactly as recorded. + Reach for :func:`numeric_metric_values` before doing arithmetic: this list may hold labels, and + ``value >= 1.0`` raises on one. + """ + return [record.value for record in records] + + +def numeric_metric_values(records: Sequence[TrialMetricValue]) -> list[float | None]: + """The values that can be compared and averaged, as floats, for consumers doing arithmetic. + + A number becomes a float (a bool becomes 1.0/0.0, matching how a pass/fail flag has always been + read). A dead trial stays ``None`` -- it is a trial that definitively did not pass, and + dropping it would let a crashed rollout flatter the agent. + + A label is **dropped**, not zeroed. A categorical verdict says nothing about whether the agent + solved the task, so it is an unmeasured trial rather than a failed one -- the same reading this + module gives a metric that raised (see :func:`is_trial_failure`). Charging it as a failure would + misattribute a measurement problem to the agent, and counting it as a pass is not defined. - Preserves order, cardinality, and the None-versus-absent distinction exactly as recorded, so - anything counting attempts (pass@k above all) reads the same sequence it would have read before - attempts carried a trial id. + A label can land under a score-like key: :func:`validate_metric_result` coerces and discards, so + a metric declaring a continuous score may still return the string ``"0.9"``, and an output one + task never declared may be score-like on another. This is where that stops being arithmetic. """ - return [attempt.value for attempt in attempts] + values: list[float | None] = [] + for record in records: + value = record.value + if value is None: + values.append(None) + elif isinstance(value, bool | int | float): + # bool first: it is a subclass of int, and False must become 0.0 rather than be dropped. + values.append(float(value)) + return values def _pass_at_k(n: int, c: int, k: int) -> float: """Unbiased pass@k estimator (Chen et al., 2021): ``1 - C(n-c, k) / C(n, k)``. - The probability that at least one of ``k`` samples drawn without replacement from ``n`` attempts + The probability that at least one of ``k`` samples drawn without replacement from ``n`` trials (``c`` of them passing) is a pass. Caller guarantees ``1 <= k <= n``. """ if n - c < k: @@ -697,7 +899,7 @@ def _pass_at_k(n: int, c: int, k: int) -> float: def _scorelike_outputs(tasks: Sequence[AgentEvalTask] | None) -> set[tuple[str, str]]: """``(metric_type, output_name)`` pairs whose declared value is a score (continuous or boolean). - pass@k is only meaningful for a per-attempt pass/fail signal, so labels, discrete/count outputs, + pass@k is only meaningful for a per-trial pass/fail signal, so labels, discrete/count outputs, and free models (e.g. token measurements) are excluded. Needs task metric specs; with no tasks the set is empty and pass@k is skipped. """ @@ -713,11 +915,11 @@ def _scorelike_outputs(tasks: Sequence[AgentEvalTask] | None) -> set[tuple[str, return scorelike -def _task_metric_attempts( +def _task_metric_values( scores: Sequence[AgentEvalTaskScore], tasks: Sequence[AgentEvalTask] | None, -) -> dict[str, dict[str, list[AgentEvalAttemptValue]]]: - """Ordered per-attempt records per task, keyed ``.``. +) -> dict[str, TrialValuesByMetric]: + """Ordered per-trial records per task, keyed ``.``. ``task-a`` declares ``reward.score`` (continuous), ``steps.count`` (discrete) and ``usage.prompt_tokens`` (a free model) and runs four trials:: @@ -728,40 +930,42 @@ def _task_metric_attempts( t3 reward 0.0 steps 7 usage 1100 out {"task-a": {"reward.score": [(t0, 1.0), (t2, None), (t3, 0.0)], - "steps.count": [(t0, 5.0), (t1, 9.0), (t2, None), (t3, 7.0)]}} + "steps.count": [(t0, 5), (t1, 9), (t2, None), (t3, 7)]}} - (shown as ``(trial_id, value)`` pairs; each is an :class:`AgentEvalAttemptValue`) + (shown as ``(trial_id, value)`` pairs; each is an :class:`TrialMetricValue`) ``usage.prompt_tokens`` is absent because its declared schema is not in :data:`_TASK_METRIC_VALUE_SCHEMAS`; t1 is missing from ``reward.score`` but present in - ``steps.count``; t2 is ``None`` in both. + ``steps.count``; t2 is ``None`` in both. ``steps.count`` keeps its ints -- values are recorded in + the type the metric produced them in, not flattened to float. Which keys a task gets: - declared by its metric spec under :data:`_TASK_METRIC_VALUE_SCHEMAS` -> kept - declared under any other schema -> dropped, even when the emitted value is numeric, so a ``MetricOutputSpec.model("prompt_tokens", TokenCount)`` measurement never becomes a key - - undeclared, but some score emitted a numeric value for it -> kept - - ``tasks is None`` -> no specs to filter against, so every numeric output observed is kept + - undeclared, but some score emitted a recordable value for it -> kept + - ``tasks is None`` -> no specs to filter against, so every recordable output observed is kept What each score contributes to its key, in trial order: - - failed trial (:func:`is_trial_failure`) -> value ``None``, an attempt that did not pass - - failed metric, or the output absent -> no entry; the attempt is unmeasured, not unsuccessful - - otherwise -> the numeric value + - failed trial (:func:`is_trial_failure`) -> value ``None``, a trial that did not pass + - failed metric, or the output absent -> no entry; the trial is unmeasured, not unsuccessful + - a value a metric can emit (number, bool or label) -> that value, in its own type + - anything else (a dict, a list, a literal null) -> no entry; see :func:`_native_value` pass@k needs that asymmetry, and it is why a list is indexed by surviving measurement rather than - by attempt: above, index 1 is t2 under ``reward.score`` but t1 under ``steps.count``. Every entry + by trial: above, index 1 is t2 under ``reward.score`` but t1 under ``steps.count``. Every entry therefore names its trial, and ``trial_id`` — not position — is what joins two keys of one task, or joins out to ``trials.jsonl`` and ``scores.jsonl``. Ids are recorded as the runner reported - them and are never deduplicated: two attempts sharing an id stay two attempts, so a runner that + them and are never deduplicated: two records sharing an id stay two records, so a runner that reuses one costs pass@k nothing. """ output_keys: dict[str, set[tuple[str, str]]] = {} - # Per task, the outputs it declared under a schema this mapping does not retain. Tracked so an - # emitted numeric value cannot add back what that task's spec filter just excluded -- and keyed by - # task because tasks in one run need not declare the same output under the same schema. - excluded: dict[str, set[tuple[str, str]]] = {} + # Outputs a task declared under a schema this mapping does not retain. Tracked so an emitted + # numeric value cannot add back what that task's spec filter just excluded, and carrying the task + # id because tasks in one run need not declare the same output under the same schema. + excluded: set[tuple[str, str, str]] = set() if tasks is not None: for task in tasks: task_keys = output_keys.setdefault(task.id, set()) @@ -771,62 +975,83 @@ def _task_metric_attempts( if issubclass(spec.value_schema, _TASK_METRIC_VALUE_SCHEMAS): task_keys.add((metric_type, spec.name)) else: - excluded.setdefault(task.id, set()).add((metric_type, spec.name)) - - scores_by_task_metric: dict[tuple[str, str], list[AgentEvalTaskScore]] = {} - for score in scores: - scores_by_task_metric.setdefault((score.task_id, score.metric_type), []).append(score) + excluded.add((task.id, metric_type, spec.name)) + + # Materialized once: the key set has to be settled before any record can be filed (a trial + # failure reaches every key of its metric, including keys only a later score reveals), and + # `scores` is walked exactly once so a one-shot sequence still works. + ordered = list(scores) + for score in ordered: + # setdefault, not add: a task whose every score failed still earns an entry, so it reads as + # measured-and-empty rather than absent. task_keys = output_keys.setdefault(score.task_id, set()) + if score.status in (AgentEvalScoreStatus.COMPLETED, AgentEvalScoreStatus.PARTIAL): + for output in score.outputs: + if (score.task_id, score.metric_type, output.name) in excluded: + continue + if _native_value(output) is not None: + task_keys.add((score.metric_type, output.name)) + + # Key set settled, so the records fill in score order -- which is what puts each key's list in + # trial order. + outputs_by_task_metric: dict[tuple[str, str], list[str]] = {} + by_task: dict[str, TrialValuesByMetric] = {} + for task_id, keys in output_keys.items(): + ordered_keys = sorted(keys) + by_task[task_id] = {f"{metric_type}.{name}": [] for metric_type, name in ordered_keys} + for metric_type, name in ordered_keys: + outputs_by_task_metric.setdefault((task_id, metric_type), []).append(name) + + for score in ordered: + output_names = outputs_by_task_metric.get((score.task_id, score.metric_type)) + if not output_names: + continue + task_values = by_task[score.task_id] + if is_trial_failure(score): + for name in output_names: + task_values[f"{score.metric_type}.{name}"].append( + TrialMetricValue(trial_id=score.trial_id, value_type=TrialMetricValueType.MISSING, value=None) + ) + continue if score.status not in (AgentEvalScoreStatus.COMPLETED, AgentEvalScoreStatus.PARTIAL): continue - task_excluded = excluded.get(score.task_id, frozenset()) + # Indexed once per score rather than rescanned per output, and first-wins on a duplicate name + # to match :func:`_score_output`. + outputs: dict[str, MetricOutput] = {} for output in score.outputs: - if (score.metric_type, output.name) in task_excluded: - continue - if _semantic_value(output) is not None: - task_keys.add((score.metric_type, output.name)) - - by_task: dict[str, dict[str, list[AgentEvalAttemptValue]]] = {} - for task_id, keys in output_keys.items(): - task_values: dict[str, list[AgentEvalAttemptValue]] = {} - for metric_type, output_name in sorted(keys): - values: list[AgentEvalAttemptValue] = [] - for score in scores_by_task_metric.get((task_id, metric_type), []): - if is_trial_failure(score): - values.append(AgentEvalAttemptValue(trial_id=score.trial_id, value=None)) - continue - if score.status not in (AgentEvalScoreStatus.COMPLETED, AgentEvalScoreStatus.PARTIAL): - continue - output = _score_output(score, output_name) - value = _semantic_value(output) if output is not None else None - if value is not None: - values.append(AgentEvalAttemptValue(trial_id=score.trial_id, value=value)) - task_values[f"{metric_type}.{output_name}"] = values - by_task[task_id] = task_values + outputs.setdefault(output.name, output) + for name in output_names: + output = outputs.get(name) + payload = _native_value(output) if output is not None else None + if payload is not None: + value_type, value = payload + task_values[f"{score.metric_type}.{name}"].append( + TrialMetricValue(trial_id=score.trial_id, value_type=value_type, value=value) + ) return by_task def _task_pass_at_k_scores( - task_metric_attempts: dict[str, dict[str, list[AgentEvalAttemptValue]]], + task_metric_values: dict[str, TrialValuesByMetric], tasks: Sequence[AgentEvalTask] | None, ) -> list[AggregateScore]: """Task-level pass@k over the R trials per task, aggregated across tasks (uniform for any runner). - For each score-like metric output, group trials by task, count attempts ``n`` and passes ``c`` + For each score-like metric output, group trials by task, count trials ``n`` and passes ``c`` (value ``>= _PASS_VALUE``), then emit ``..pass@k`` for ``k`` in ``1..max(n)`` as - the across-task mean of the unbiased per-task estimator (over tasks with at least ``k`` attempts). + the across-task mean of the unbiased per-task estimator (over tasks with at least ``k`` trials). ``pass@1`` equals the macro per-task pass rate, i.e. the task-level mean. - **A failed trial is a failed attempt.** It counts toward ``n`` and never toward ``c``: an agent that + **A failed trial did not pass.** It counts toward ``n`` and never toward ``c``: an agent that solved a task once and crashed once did not go one-for-one. A failed *metric* is different — it - leaves the attempt unmeasured rather than unsuccessful, so it stays out of ``n`` entirely rather - than being charged to the agent (see :func:`is_trial_failure`). Tasks left with no usable attempt at + leaves the trial unmeasured rather than unsuccessful, so it stays out of ``n`` entirely rather + than being charged to the agent (see :func:`is_trial_failure`). Tasks left with no usable value at all drop out of the estimate and are reported as ``nan_count``, uniform across ``k``, so a shrinking denominator is never silent. (Tasks excluded from a given ``k`` merely for having fewer than ``k`` - attempts are *not* counted there — that is the estimator working as defined, not missing data.) + trials are *not* counted there — that is the estimator working as defined, not missing data.) - "No usable attempt" includes a task that was never scored at all: it declares the metric, holds an - empty attempt list, and lands in ``nan_count`` like any other unmeasured task. That is the same + "No usable value" includes a task that was never scored at all: it declares the metric, holds an + empty value list, and lands in ``nan_count`` like any other unmeasured task. That is the same missing coverage whether the trial died or was never produced, and excluding it would report pass@k over a denominator quietly smaller than the task set asked for. @@ -841,24 +1066,23 @@ def _task_pass_at_k_scores( aggregated: list[AggregateScore] = [] for metric_type, output_name in sorted(scorelike): key = f"{metric_type}.{output_name}" - values_by_task = [attempt_values(outputs[key]) for outputs in task_metric_attempts.values() if key in outputs] + values_by_task = [ + numeric_metric_values(outputs[key]) for outputs in task_metric_values.values() if key in outputs + ] measured = [values for values in values_by_task if values] if not measured: continue - # Empty attempt lists stay in nan_count (via total); for each k, mean the unbiased + # Empty value lists stay in nan_count (via total); for each k, mean the unbiased # estimator over tasks with n >= k (None / < full credit do not count as passes). unmeasured = sum(not values for values in values_by_task) - max_n = max(len(values) for values in measured) + # (n, c) per task, counted once: neither depends on k, so counting inside the k loop would + # re-walk every task's values max_n times over. + counts = [ + (len(values), sum(value is not None and value >= _PASS_VALUE for value in values)) for values in measured + ] + max_n = max(n for n, _ in counts) for k in range(1, max_n + 1): - per_task = [ - _pass_at_k( - len(values), - sum(value is not None and value >= _PASS_VALUE for value in values), - k, - ) - for values in measured - if len(values) >= k - ] + per_task = [_pass_at_k(n, c, k) for n, c in counts if n >= k] if per_task: aggregated.append(_aggregate_range_score(f"{key}.pass@{k}", per_task, len(per_task) + unmeasured)) return aggregated @@ -1042,15 +1266,42 @@ def _numeric_value(output: MetricOutput) -> float | None: return None -def _semantic_value(output: MetricOutput) -> float | None: +def _native_value(output: MetricOutput) -> tuple[TrialMetricValueType, float | int | bool | str] | None: + """The payload for one metric output, in the type the metric produced it in. + + The *preserving* counterpart to :func:`_semantic_value`, which projects to a float because its + callers (aggregate stats, semantic views) do arithmetic. A :class:`TrialMetricValue` is not + arithmetic: it is the per-trial evidence a reader looks up by ``trial_id`` in ``result.trials`` + or ``trials.jsonl``, so a count stays an int, a flag stays a bool, and a judge's verdict stays + the string it was. + + Returns ``None`` -- "nothing a trial can record" -- rather than a value, so an output holding + a dict, a list, or a literal null stays *absent* from the value list. That is not the same as + the ``None`` a dead trial records, and conflating the two would charge pass@k a trial the + agent never made. + """ value = output.value - if isinstance(value, bool): - return 1.0 if value else 0.0 if isinstance(value, BaseModel): - root = getattr(value, "root", None) - if isinstance(root, bool): - return 1.0 if root else 0.0 - return _numeric_value(output) + value = getattr(value, "root", None) + # bool first: it is a subclass of int, and it is a pass/fail signal rather than a measurement. + if isinstance(value, bool | int | float): + return (TrialMetricValueType.NUMBER, value) + if isinstance(value, str): + return (TrialMetricValueType.LABEL, value) + return None + + +def _semantic_value(output: MetricOutput) -> float | None: + """:func:`_native_value` projected to a float, for the callers that do arithmetic. + + The two answer different questions -- preserve versus interpret -- but they must agree on what a + metric value *is*, so the RootModel unwrap and the "what counts as numeric" rule live in + :func:`_native_value` alone. A label projects to ``None``: a view or aggregate cannot average it. + """ + payload = _native_value(output) + if payload is None or payload[0] is not TrialMetricValueType.NUMBER: + return None + return float(payload[1]) def mean_numeric(values: list[float]) -> float | None: diff --git a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/scores.py b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/scores.py index 6231220e97..2750b85c94 100644 --- a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/scores.py +++ b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/scores.py @@ -23,7 +23,7 @@ class AgentEvalScoreStatus(str, Enum): #: Diagnostic detail key stamped when a score is ``FAILED`` because the *trial* failed — the agent #: produced nothing to score — rather than because the metric itself raised. Both are reported as -#: ``FAILED``, but they mean different things to a reader: a failed trial is a failed *attempt*, a +#: ``FAILED``, but they mean different things to a reader: a failed trial did not pass, a #: failed metric is a failed *measurement*. Consumers that must tell them apart read this key via #: :func:`is_trial_failure`. TRIAL_STATUS_DETAIL = "trial_status" diff --git a/packages/nemo_evaluator_sdk/tests/agent_eval/test_harbor_runtime.py b/packages/nemo_evaluator_sdk/tests/agent_eval/test_harbor_runtime.py index 2a88607210..39781e57a0 100644 --- a/packages/nemo_evaluator_sdk/tests/agent_eval/test_harbor_runtime.py +++ b/packages/nemo_evaluator_sdk/tests/agent_eval/test_harbor_runtime.py @@ -114,17 +114,18 @@ def _reward_details_from_summary( ) -> dict[str, dict[str, list[str]]]: """Rebuild ``reward_payload_from_result``'s ``reward_details`` from the summary alone. - Inverts ``{task_id: [attempt]}`` back into the legacy ``{output: {value_str: [task_id, ...]}}``. - A ``None`` attempt is skipped: it is a trial that died before scoring, and the adapter drops - those too (it skips ``FAILED`` scores), so both sides agree on what is groupable. + Inverts ``{task_id: [value]}`` back into the legacy ``{output: {value_str: [task_id, ...]}}``. + Only numeric values are groupable: a dead trial has no value, and a label is not a reward. + The adapter drops both too (it skips ``FAILED`` scores and non-numeric outputs), so the two sides + agree on what is groupable. """ key = f"{metric_type}.{output_name}" details: dict[str, dict[str, list[str]]] = {} - for task_id, attempts_by_key in summary.task_metric_attempts.items(): - for attempt in attempts_by_key.get(key, []): - if attempt.value is None: + for task_id, values_by_key in summary.task_metric_values.items(): + for record in values_by_key.get(key, []): + if not isinstance(record.value, bool | int | float): continue - details.setdefault(output_name, {}).setdefault(str(float(attempt.value)), []).append(task_id) + details.setdefault(output_name, {}).setdefault(str(float(record.value)), []).append(task_id) return details @@ -138,28 +139,29 @@ def _reward_stats_from_summary( Harbor builds it as ``reward_stats.setdefault(value, []).append(trial_result.trial_name)``: keyed by the raw numeric value, listing trial names. ``_trial_from_harbor_result`` stamps Harbor's - ``trial_name`` straight onto ``AgentEvalTrial.id``, which is the ``trial_id`` each attempt now + ``trial_name`` straight onto ``AgentEvalTrial.id``, which is the ``trial_id`` each record now carries, so this reproduces Harbor's shape rather than approximating it. - A ``None`` attempt is a trial that died before the verifier ran. Harbor has no reward to file it - under either — it lands in ``exception_stats``/``n_errors`` instead — so it is skipped here. + A ``None`` value is a trial that died before the verifier ran. Harbor has no reward to file it + under either — it lands in ``exception_stats``/``n_errors`` instead — so it is skipped here, as + is any non-numeric value: Harbor keys ``reward_stats`` by the reward value itself. """ key = f"{metric_type}.{output_name}" stats: dict[str, dict[float, list[str]]] = {} - for attempts_by_key in summary.task_metric_attempts.values(): - for attempt in attempts_by_key.get(key, []): - if attempt.value is None: + for values_by_key in summary.task_metric_values.values(): + for record in values_by_key.get(key, []): + if not isinstance(record.value, bool | int | float): continue - stats.setdefault(output_name, {}).setdefault(attempt.value, []).append(attempt.trial_id) + stats.setdefault(output_name, {}).setdefault(float(record.value), []).append(record.trial_id) return stats @pytest.mark.asyncio -async def test_harbor_reward_stats_is_derivable_from_summary_task_metric_attempts(tmp_path: Path) -> None: +async def test_harbor_reward_stats_is_derivable_from_summary_task_metric_values(tmp_path: Path) -> None: """The summary alone reproduces Harbor's ``reward_stats``, which is what AALGO-310 exists to enable. Harbor groups rewards by ``trial_name`` and keys them by the raw ``float | int``. Both were out of - reach while attempts were bare numbers indexed by position; now that each attempt names its trial, + reach while values were bare numbers indexed by position; now that each record names its trial, AALGO-441 can rebuild the real shape without re-walking ``result.scores``. The legacy ``reward_details`` (task-keyed, stringified) stays derivable too, so the rewiring loses nothing. @@ -168,7 +170,7 @@ async def test_harbor_reward_stats_is_derivable_from_summary_task_metric_attempt """ job_dir = tmp_path / "job" job_dir.mkdir() - # Two attempts each for alpha (flaky) and beta (solved); one for gamma, whose verifier emitted + # Two trials each for alpha (flaky) and beta (solved); one for gamma, whose verifier emitted # no reward at all -> PARTIAL trial that still scores 0.0. _write_trial(job_dir, "alpha__a", "alpha", reward=1.0) _write_trial(job_dir, "alpha__b", "alpha", reward=0.0) @@ -183,11 +185,11 @@ async def test_harbor_reward_stats_is_derivable_from_summary_task_metric_attempt runner = HarborAgentTaskRunner(job_dir=job_dir, run_job=lambda: _record([])) result = await AgentEvaluator().run(tasks=tasks, target=runner, config=AgentEvalRunConfig()) - attempts = { + recorded = { task_id: {key: [(a.trial_id, a.value) for a in records] for key, records in by_key.items()} - for task_id, by_key in result.summary.task_metric_attempts.items() + for task_id, by_key in result.summary.task_metric_values.items() } - assert attempts == { + assert recorded == { "alpha": {"harbor_reward.reward": [("alpha__a", 1.0), ("alpha__b", 0.0)]}, "beta": {"harbor_reward.reward": [("beta__a", 1.0), ("beta__b", 1.0)]}, "gamma": {"harbor_reward.reward": [("gamma__a", 0.0)]}, diff --git a/packages/nemo_evaluator_sdk/tests/agent_eval/test_pass_at_k.py b/packages/nemo_evaluator_sdk/tests/agent_eval/test_pass_at_k.py index 71ab361e25..fc507cac7d 100644 --- a/packages/nemo_evaluator_sdk/tests/agent_eval/test_pass_at_k.py +++ b/packages/nemo_evaluator_sdk/tests/agent_eval/test_pass_at_k.py @@ -8,7 +8,7 @@ import pytest from nemo_evaluator_sdk.agent_eval.evaluator import AgentEvaluator -from nemo_evaluator_sdk.agent_eval.results import AgentEvalSummary, _pass_at_k, attempt_values +from nemo_evaluator_sdk.agent_eval.results import AgentEvalSummary, _pass_at_k, metric_values from nemo_evaluator_sdk.agent_eval.scores import ( TRIAL_STATUS_DETAIL, AgentEvalDiagnostic, @@ -94,7 +94,7 @@ def test_pass_at_k_unbiased_estimator() -> None: def test_task_pass_at_k_gated_and_uniform_across_metric_types() -> None: - # Two tasks x 2 attempts, scored by two reward metric types (mimicking Gym + Harbor) plus a label + # Two tasks x 2 trials, scored by two reward metric types (mimicking Gym + Harbor) plus a label # metric. t1 rewards [1.0, 0.0] (1/2 pass); t2 [1.0, 1.0] (2/2). pass@1 = mean(0.5, 1.0) = 0.75. gym, harbor, label = _ScoreMetric("gym_reward"), _ScoreMetric("harbor_reward"), _LabelMetric() tasks = [_task("t1", gym, harbor, label), _task("t2", gym, harbor, label)] @@ -111,13 +111,17 @@ def test_task_pass_at_k_gated_and_uniform_across_metric_types() -> None: summary = AgentEvalSummary.from_scores(scores, tasks=tasks) by_name = {score.name: score for score in summary.scores.scores} - assert {key: attempt_values(a) for key, a in summary.task_metric_attempts["t1"].items()} == { + # The label is *recorded* -- it is per-trial evidence -- but stays out of pass@k. Those two facts + # together are the whole point of keeping _TASK_METRIC_VALUE_SCHEMAS wider than the pass@k set. + assert {key: metric_values(a) for key, a in summary.task_metric_values["t1"].items()} == { "gym_reward.reward": [1.0, 0.0], "harbor_reward.reward": [1.0, 0.0], + "verdict.category": ["good"], } - assert {key: attempt_values(a) for key, a in summary.task_metric_attempts["t2"].items()} == { + assert {key: metric_values(a) for key, a in summary.task_metric_values["t2"].items()} == { "gym_reward.reward": [1.0, 1.0], "harbor_reward.reward": [1.0, 1.0], + "verdict.category": [], # t2 declares the label metric but no verdict was scored for it } for metric_type in ("gym_reward", "harbor_reward"): # uniform across runners assert by_name[f"{metric_type}.reward.pass@1"].mean == pytest.approx(0.75) @@ -126,7 +130,7 @@ def test_task_pass_at_k_gated_and_uniform_across_metric_types() -> None: def test_partial_credit_is_not_a_pass() -> None: - # pass@k answers "did the agent solve the task", so only full credit counts: of attempts 0.5 and + # pass@k answers "did the agent solve the task", so only full credit counts: of values 0.5 and # 1.0, exactly one is a pass. tasks = [_task("t1", _ScoreMetric("reward"))] scores = [_score("t1", "a0", "reward", "reward", 0.5), _score("t1", "a1", "reward", "reward", 1.0)] @@ -134,7 +138,7 @@ def test_partial_credit_is_not_a_pass() -> None: by_name = {s.name: s for s in AgentEvalSummary.from_scores(scores, tasks=tasks).scores.scores} assert by_name["reward.reward.pass@1"].mean == pytest.approx(0.5) - assert by_name["reward.reward.pass@2"].mean == pytest.approx(1.0) # one of the two attempts passed + assert by_name["reward.reward.pass@2"].mean == pytest.approx(1.0) # one of the two trials passed def test_population_and_sample_stats_are_both_reported() -> None: @@ -166,9 +170,9 @@ def test_sample_stats_undefined_for_a_single_value() -> None: assert aggregate.sample_variance is None -def test_a_failed_trial_is_a_failed_attempt_not_an_absent_one() -> None: +def test_a_failed_trial_counts_as_not_passing_not_absent() -> None: # The agent solved the task once and its other rollout died. Dropping the dead one would report - # pass@1 = 1.0 ("solved it first try") and make pass@2 vanish along with the attempt that earned it. + # pass@1 = 1.0 ("solved it first try") and make pass@2 vanish along with the trial that earned it. tasks = [_task("t1", _ScoreMetric("reward"))] scores = [ _score("t1", "a0", "reward", "reward", 1.0), @@ -178,13 +182,13 @@ def test_a_failed_trial_is_a_failed_attempt_not_an_absent_one() -> None: summary = AgentEvalSummary.from_scores(scores, tasks=tasks) by_name = {s.name: s for s in summary.scores.scores} - assert attempt_values(summary.task_metric_attempts["t1"]["reward.reward"]) == [1.0, None] - assert by_name["reward.reward.pass@1"].mean == pytest.approx(0.5) # 1 of 2 attempts, not 1 of 1 + assert metric_values(summary.task_metric_values["t1"]["reward.reward"]) == [1.0, None] + assert by_name["reward.reward.pass@1"].mean == pytest.approx(0.5) # 1 of 2 trials, not 1 of 1 assert by_name["reward.reward.pass@2"].mean == pytest.approx(1.0) assert by_name["reward.reward.pass@1"].nan_count == 0 # the task was measured, so nothing is missing -def test_a_metric_that_raised_leaves_the_attempt_unmeasured_rather_than_failed() -> None: +def test_a_metric_that_raised_leaves_the_trial_unmeasured_rather_than_failed() -> None: # The distinction the trial_status detail exists for: a judge that timed out tells us nothing about # whether the agent passed, so it must not be charged to the agent the way a dead rollout is. # t1: [1.0, metric raised] -> n=1, pass@1 = 1.0. t2: [1.0, 0.0] -> n=2, pass@1 = 0.5, pass@2 = 1.0. @@ -199,16 +203,16 @@ def test_a_metric_that_raised_leaves_the_attempt_unmeasured_rather_than_failed() summary = AgentEvalSummary.from_scores(scores, tasks=tasks) by_name = {s.name: s for s in summary.scores.scores} - assert attempt_values(summary.task_metric_attempts["t1"]["reward.reward"]) == [1.0] + assert metric_values(summary.task_metric_values["t1"]["reward.reward"]) == [1.0] assert by_name["reward.reward.pass@1"].mean == pytest.approx(0.75) # mean(1.0, 0.5), not mean(0.5, 0.5) - # t1 drops out of pass@2 for having fewer than k attempts. That is the estimator working as defined, + # t1 drops out of pass@2 for having fewer than k trials. That is the estimator working as defined, # not missing data, so it is not counted as nan. assert by_name["reward.reward.pass@2"].count == 1 assert by_name["reward.reward.pass@2"].nan_count == 0 -def test_a_task_with_no_usable_attempt_is_reported_as_nan_count_uniformly_across_k() -> None: - # Every attempt on t2 was unmeasurable, so it contributes to no estimate at any k. Silently +def test_a_task_with_no_usable_value_is_reported_as_nan_count_uniformly_across_k() -> None: + # Every trial on t2 was unmeasurable, so it contributes to no estimate at any k. Silently # narrowing the denominator is what made a shrunken pass@k indistinguishable from a clean one. tasks = [_task("t1", _ScoreMetric("reward")), _task("t2", _ScoreMetric("reward"))] scores = [ @@ -245,7 +249,7 @@ def test_a_trial_status_detail_that_is_not_a_failure_does_not_make_one() -> None assert is_trial_failure(scores[1]) is False by_name = {s.name: s for s in AgentEvalSummary.from_scores(scores, tasks=tasks).scores.scores} - assert by_name["reward.reward.pass@1"].mean == pytest.approx(1.0) # n=1, not charged the bad attempt + assert by_name["reward.reward.pass@1"].mean == pytest.approx(1.0) # n=1, not charged the bad trial @pytest.mark.asyncio @@ -276,5 +280,5 @@ async def compute_scores(self, input: MetricInput) -> MetricResult: assert by_task["t1"].status is AgentEvalScoreStatus.FAILED assert by_task["t2"].status is AgentEvalScoreStatus.FAILED - assert is_trial_failure(by_task["t1"]) is True # the trial died -> a failed attempt - assert is_trial_failure(by_task["t2"]) is False # the metric died -> an unmeasured attempt + assert is_trial_failure(by_task["t1"]) is True # the trial died -> did not pass + assert is_trial_failure(by_task["t2"]) is False # the metric died -> unmeasured diff --git a/packages/nemo_evaluator_sdk/tests/agent_eval/test_persistence.py b/packages/nemo_evaluator_sdk/tests/agent_eval/test_persistence.py index cb8dc071f1..979fda7cc7 100644 --- a/packages/nemo_evaluator_sdk/tests/agent_eval/test_persistence.py +++ b/packages/nemo_evaluator_sdk/tests/agent_eval/test_persistence.py @@ -11,7 +11,7 @@ import pytest from nemo_evaluator_sdk.agent_eval.persistence import persist_run, read_trials -from nemo_evaluator_sdk.agent_eval.results import AgentEvalAttemptValue, AgentEvalResult, AgentEvalSummary +from nemo_evaluator_sdk.agent_eval.results import AgentEvalResult, AgentEvalSummary, TrialMetricValue from nemo_evaluator_sdk.agent_eval.trials import AgentEvalTrial, AgentEvalTrialStatus, AgentOutput from nemo_evaluator_sdk.values.evidence import CandidateEvidence, EvidenceDescriptor @@ -84,19 +84,19 @@ def test_persist_run_writes_bundle_relative_refs_that_survive_a_move(tmp_path: P assert resolved.is_dir() and resolved == moved / "evidence" / "fabric" / "r" / "000000-taskA" / "workspace" -def test_persist_run_writes_per_task_attempts_to_summary(tmp_path: Path) -> None: +def test_persist_run_writes_per_task_values_to_summary(tmp_path: Path) -> None: summary = AgentEvalSummary( - task_metric_attempts={ + task_metric_values={ "task-a": { "harbor_reward.reward": [ - AgentEvalAttemptValue(trial_id="task-a__aaa", value=1.0), - AgentEvalAttemptValue(trial_id="task-a__bbb", value=0.0), + TrialMetricValue(trial_id="task-a__aaa", value=1.0), + TrialMetricValue(trial_id="task-a__bbb", value=0.0), ] }, "task-b": { "harbor_reward.reward": [ - AgentEvalAttemptValue(trial_id="task-b__ccc", value=1.0), - AgentEvalAttemptValue(trial_id="task-b__ddd", value=None), + TrialMetricValue(trial_id="task-b__ccc", value=1.0), + TrialMetricValue(trial_id="task-b__ddd", value=None), ] }, } @@ -106,32 +106,32 @@ def test_persist_run_writes_per_task_attempts_to_summary(tmp_path: Path) -> None persist_run(result, tmp_path) payload = json.loads((tmp_path / "summary.json").read_text(encoding="utf-8")) - assert payload["task_metric_attempts"] == { + assert payload["task_metric_values"] == { "task-a": { "harbor_reward.reward": [ - {"trial_id": "task-a__aaa", "value": 1.0}, - {"trial_id": "task-a__bbb", "value": 0.0}, + {"trial_id": "task-a__aaa", "value_type": "number", "value": 1.0}, + {"trial_id": "task-a__bbb", "value_type": "number", "value": 0.0}, ] }, "task-b": { "harbor_reward.reward": [ - {"trial_id": "task-b__ccc", "value": 1.0}, - {"trial_id": "task-b__ddd", "value": None}, + {"trial_id": "task-b__ccc", "value_type": "number", "value": 1.0}, + {"trial_id": "task-b__ddd", "value_type": "missing", "value": None}, ] }, } - assert AgentEvalSummary.model_validate(payload).task_metric_attempts == summary.task_metric_attempts + assert AgentEvalSummary.model_validate(payload).task_metric_values == summary.task_metric_values -def test_summary_json_round_trips_attempt_order_through_sort_keys(tmp_path: Path) -> None: - # persist_run writes with sort_keys=True. Attempts are a list precisely so that trial order +def test_summary_json_round_trips_trial_order_through_sort_keys(tmp_path: Path) -> None: + # persist_run writes with sort_keys=True. Values are a list precisely so that trial order # survives that: keying them by trial id would come back sorted lexicographically instead. summary = AgentEvalSummary( - task_metric_attempts={ + task_metric_values={ "task-a": { "harbor_reward.reward": [ - AgentEvalAttemptValue(trial_id="z-trial", value=1.0), - AgentEvalAttemptValue(trial_id="a-trial", value=0.0), + TrialMetricValue(trial_id="z-trial", value=1.0), + TrialMetricValue(trial_id="a-trial", value=0.0), ] } } @@ -142,7 +142,7 @@ def test_summary_json_round_trips_attempt_order_through_sort_keys(tmp_path: Path payload = json.loads((tmp_path / "summary.json").read_text(encoding="utf-8")) reloaded = AgentEvalSummary.model_validate(payload) - assert [a.trial_id for a in reloaded.task_metric_attempts["task-a"]["harbor_reward.reward"]] == [ + assert [a.trial_id for a in reloaded.task_metric_values["task-a"]["harbor_reward.reward"]] == [ "z-trial", "a-trial", ] diff --git a/packages/nemo_evaluator_sdk/tests/agent_eval/test_result_display.py b/packages/nemo_evaluator_sdk/tests/agent_eval/test_result_display.py index f866620d55..4707326f40 100644 --- a/packages/nemo_evaluator_sdk/tests/agent_eval/test_result_display.py +++ b/packages/nemo_evaluator_sdk/tests/agent_eval/test_result_display.py @@ -194,7 +194,7 @@ def test_summary_header_lists_only_the_statuses_the_run_produced() -> None: def test_summary_separates_a_failed_trial_from_a_failed_metric() -> None: - # Both surface as FAILED but mean different things: a failed trial is an attempt the agent is + # Both surface as FAILED but mean different things: a failed trial is one the agent is # answerable for, a failed metric is a measurement that never happened. Each label is asserted # against the score it describes -- checking only that both strings appear would still pass if # the two were swapped. diff --git a/packages/nemo_evaluator_sdk/tests/agent_eval/test_task_metric_attempts.py b/packages/nemo_evaluator_sdk/tests/agent_eval/test_task_metric_attempts.py deleted file mode 100644 index 03da1c559e..0000000000 --- a/packages/nemo_evaluator_sdk/tests/agent_eval/test_task_metric_attempts.py +++ /dev/null @@ -1,474 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -from __future__ import annotations - -import json -import math -from collections.abc import Iterator -from pathlib import Path - -import pytest -from nemo_evaluator_sdk.agent_eval.results import ( - AgentEvalAttemptValue, - AgentEvalSummary, - _task_metric_attempts, - attempt_values, -) -from nemo_evaluator_sdk.agent_eval.scores import ( - TRIAL_STATUS_DETAIL, - AgentEvalDiagnostic, - AgentEvalDiagnosticSeverity, - AgentEvalScoreStatus, - AgentEvalTaskScore, -) -from nemo_evaluator_sdk.agent_eval.tasks import AgentEvalTask -from nemo_evaluator_sdk.metrics.protocol import MetricInput, MetricOutput, MetricResult -from nemo_evaluator_sdk.values.protocol import MetricOutputSpec -from pydantic import RootModel - - -class _TokenCount(RootModel[int]): - """A free-model output: numeric, but a measurement rather than a per-attempt score.""" - - -class _Metric: - def __init__(self, metric_type: str, output: MetricOutputSpec) -> None: - self._type = metric_type - self._output = output - - @property - def type(self) -> str: - return self._type - - def output_spec(self) -> list[MetricOutputSpec]: - return [self._output] - - async def compute_scores(self, input: MetricInput) -> MetricResult: # pragma: no cover - raise NotImplementedError - - -class _SinglePassScores(list[AgentEvalTaskScore]): - def __init__(self, scores: list[AgentEvalTaskScore]) -> None: - super().__init__(scores) - self.iterations = 0 - - def __iter__(self) -> Iterator[AgentEvalTaskScore]: - self.iterations += 1 - assert self.iterations == 1, "scores were rescanned" - return super().__iter__() - - -def _task(task_id: str, *metrics: _Metric) -> AgentEvalTask: - return AgentEvalTask(id=task_id, intent="test", inputs={}, metrics=list(metrics)) - - -def _pairs( - attempts: dict[str, dict[str, list[AgentEvalAttemptValue]]], -) -> dict[str, dict[str, list[tuple[str, float | None]]]]: - """Flatten attempt records to ``(trial_id, value)`` so assertions stay readable.""" - return { - task_id: {key: [(a.trial_id, a.value) for a in records] for key, records in by_key.items()} - for task_id, by_key in attempts.items() - } - - -def _score( - task_id: str, - trial_id: str, - metric_type: str, - output_name: str, - value: object, - *, - status: AgentEvalScoreStatus = AgentEvalScoreStatus.COMPLETED, -) -> AgentEvalTaskScore: - return AgentEvalTaskScore( - id=f"run:{task_id}:{trial_id}:{metric_type}", - run_id="run", - task_id=task_id, - trial_id=trial_id, - metric_type=metric_type, - status=status, - outputs=[MetricOutput(name=output_name, value=value)], - ) - - -def _failed_score( - task_id: str, trial_id: str, *, trial_failed: bool, metric_type: str = "reward" -) -> AgentEvalTaskScore: - details = {TRIAL_STATUS_DETAIL: "failed"} if trial_failed else {"exception_type": "TimeoutError"} - return AgentEvalTaskScore( - id=f"run:{task_id}:{trial_id}:{metric_type}", - run_id="run", - task_id=task_id, - trial_id=trial_id, - metric_type=metric_type, - status=AgentEvalScoreStatus.FAILED, - diagnostics=[ - AgentEvalDiagnostic( - severity=AgentEvalDiagnosticSeverity.ERROR, - message="failed", - details=details, - ) - ], - ) - - -def test_summary_exposes_ordered_numeric_attempt_values_per_task() -> None: - reward = _Metric("reward", MetricOutputSpec.continuous_score("score")) - retries = _Metric("retries", MetricOutputSpec.discrete_score("count")) - verdict = _Metric("verdict", MetricOutputSpec.label("label")) - complete = _Metric("complete", MetricOutputSpec.boolean("passed")) - tasks = [_task("task-a", reward, retries, verdict, complete), _task("task-b", reward)] - scores = [ - _score("task-a", "attempt-0", "reward", "score", 1.0), - _score("task-a", "attempt-0", "retries", "count", 2), - _score("task-a", "attempt-0", "verdict", "label", "good"), - _score("task-a", "attempt-0", "complete", "passed", True), - _score( - "task-a", - "attempt-1", - "reward", - "score", - 0.25, - status=AgentEvalScoreStatus.PARTIAL, - ), - _score("task-a", "attempt-1", "retries", "count", 3), - _score("task-a", "attempt-1", "complete", "passed", False), - _score("task-b", "attempt-0", "reward", "score", 0.0), - ] - - summary = AgentEvalSummary.from_scores(scores, tasks=tasks) - - assert _pairs(summary.task_metric_attempts) == { - "task-a": { - "complete.passed": [("attempt-0", 1.0), ("attempt-1", 0.0)], - "retries.count": [("attempt-0", 2.0), ("attempt-1", 3.0)], - "reward.score": [("attempt-0", 1.0), ("attempt-1", 0.25)], - }, - "task-b": {"reward.score": [("attempt-0", 0.0)]}, - } - - -def test_task_metric_attempts_scans_scores_once() -> None: - tasks = [_task("task-a", _Metric("reward", MetricOutputSpec.continuous_score("score")))] - scores = _SinglePassScores( - [ - _score("task-a", "attempt-0", "reward", "score", 1.0), - _score("task-a", "attempt-1", "reward", "score", 0.0), - ] - ) - - assert _pairs(_task_metric_attempts(scores, tasks)) == { - "task-a": {"reward.score": [("attempt-0", 1.0), ("attempt-1", 0.0)]} - } - - -def test_failed_trials_are_attempts_but_metric_failures_are_unmeasured() -> None: - tasks = [ - _task("flaky", _Metric("reward", MetricOutputSpec.continuous_score("score"))), - _task("unmeasured", _Metric("reward", MetricOutputSpec.continuous_score("score"))), - ] - scores = [ - _score("flaky", "attempt-0", "reward", "score", 1.0), - _failed_score("flaky", "attempt-1", trial_failed=True), - _failed_score("unmeasured", "attempt-0", trial_failed=False), - ] - - summary = AgentEvalSummary.from_scores(scores, tasks=tasks) - - assert _pairs(summary.task_metric_attempts) == { - # The dead trial keeps its identity, paired with its None; the unmeasured one has no entry at - # all, so it is nameable from neither -- that asymmetry is what pass@k depends on. - "flaky": {"reward.score": [("attempt-0", 1.0), ("attempt-1", None)]}, - "unmeasured": {"reward.score": []}, - } - - -def test_a_task_that_produced_no_trial_is_unmeasured_and_counted_in_pass_at_k_nan() -> None: - # from_scores can be handed a task list wider than the scores -- a caller re-aggregating a subset. - # The task still declares the metric, so it holds an empty attempt list and counts as missing - # coverage: excluding it would report pass@k over a denominator smaller than the task set asked - # for. A full run cannot reach this state; AgentEvaluator._score_trials refuses to score when a - # task produced no trial, which test_evaluator.py::test_run_rejects_tasks_without_trials pins. - reward = _Metric("reward", MetricOutputSpec.continuous_score("score")) - tasks = [_task("scored", reward), _task("never-ran", reward)] - scores = [_score("scored", "attempt-0", "reward", "score", 1.0)] - - summary = AgentEvalSummary.from_scores(scores, tasks=tasks) - by_name = {score.name: score for score in summary.scores.scores} - - assert _pairs(summary.task_metric_attempts) == { - "scored": {"reward.score": [("attempt-0", 1.0)]}, - "never-ran": {"reward.score": []}, - } - assert by_name["reward.score.pass@1"].mean == 1.0 # the one measured task passed - assert by_name["reward.score.pass@1"].count == 1 - assert by_name["reward.score.pass@1"].nan_count == 1 # ...and the unrun one is not hidden - - -def test_outputs_declared_under_an_unretained_schema_stay_out_even_when_numeric() -> None: - # Token measurements and other free models are excluded by their declared schema. Emitting a - # numeric value must not add them back: the value is a measurement, not a per-attempt score. - tasks = [ - _task( - "task-a", - _Metric("reward", MetricOutputSpec.continuous_score("score")), - _Metric("usage", MetricOutputSpec.model("prompt_tokens", _TokenCount)), - ) - ] - scores = [ - _score("task-a", "attempt-0", "reward", "score", 1.0), - _score("task-a", "attempt-0", "usage", "prompt_tokens", 1234), - ] - - assert _pairs(AgentEvalSummary.from_scores(scores, tasks=tasks).task_metric_attempts) == { - "task-a": {"reward.score": [("attempt-0", 1.0)]} - } - - -def test_one_tasks_schema_exclusion_does_not_suppress_another_tasks_output() -> None: - # The spec filter is per task: tasks in one run need not declare the same output under the same - # schema. Task-a declaring usage.prompt_tokens as a free model must not strip it from task-b, - # which never declared it and whose only evidence is the numeric value it actually emitted. - tasks = [ - _task("task-a", _Metric("usage", MetricOutputSpec.model("prompt_tokens", _TokenCount))), - _task("task-b", _Metric("reward", MetricOutputSpec.continuous_score("score"))), - ] - scores = [ - _score("task-a", "attempt-0", "usage", "prompt_tokens", 100), - _score("task-b", "attempt-0", "reward", "score", 1.0), - _score("task-b", "attempt-0", "usage", "prompt_tokens", 250), # undeclared on task-b - ] - - attempts = AgentEvalSummary.from_scores(scores, tasks=tasks).task_metric_attempts - - # task-a declared it under an unretained schema, so it is not a key there at all -- not even an - # empty one -- and the numeric value it emitted cannot add it back. - assert attempts["task-a"] == {} - # task-b never declared it, so its emitted numeric value is the only evidence and it is kept. - assert sorted(attempts["task-b"]) == ["reward.score", "usage.prompt_tokens"] - assert _pairs(attempts)["task-b"]["usage.prompt_tokens"] == [("attempt-0", 250.0)] - - -def test_nan_attempt_values_survive_json_as_a_string() -> None: - # A metric may legitimately score NaN. json.dumps would write a bare NaN token, which is not - # valid JSON, so summary.json must carry the string form -- and read it back as a float. - tasks = [_task("task-a", _Metric("reward", MetricOutputSpec.continuous_score("score")))] - summary = AgentEvalSummary.from_scores( - [_score("task-a", "attempt-0", "reward", "score", float("nan"))], tasks=tasks - ) - - payload = summary.model_dump(mode="json") - assert payload["task_metric_attempts"]["task-a"]["reward.score"][0]["value"] == "NaN" - - # Strict JSON: no bare NaN/Infinity tokens anywhere in the serialized bundle. - def _reject(constant: str) -> float: - raise AssertionError(f"summary.json contains a bare {constant} token") - - reloaded = json.loads(json.dumps(payload), parse_constant=_reject) - value = AgentEvalSummary.model_validate(reloaded).task_metric_attempts["task-a"]["reward.score"][0].value - assert value is not None and math.isnan(value) - - -def test_without_tasks_there_is_no_spec_to_filter_on() -> None: - # No tasks means no declared schemas to consult, so every numeric output observed is retained. - scores = [_score("task-a", "attempt-0", "usage", "prompt_tokens", 1234)] - - assert _pairs(AgentEvalSummary.from_scores(scores).task_metric_attempts) == { - "task-a": {"usage.prompt_tokens": [("attempt-0", 1234.0)]} - } - - -def test_attempts_align_across_keys_by_trial_id_not_position() -> None: - # A metric that raised drops its attempt entirely while a dead trial holds its slot as None, so a - # metric failure shortens its own key's list without shortening its neighbour's. Index i of two - # keys is then two different trials -- which is exactly why every attempt names its trial. - tasks = [ - _task( - "task-a", - _Metric("reward", MetricOutputSpec.continuous_score("score")), - _Metric("steps", MetricOutputSpec.discrete_score("count")), - ) - ] - scores = [ - _score("task-a", "attempt-0", "reward", "score", 1.0), - _score("task-a", "attempt-0", "steps", "count", 5), - _failed_score("task-a", "attempt-1", trial_failed=False), # the reward judge timed out - _score("task-a", "attempt-1", "steps", "count", 9), - _score("task-a", "attempt-2", "reward", "score", 0.0), - _score("task-a", "attempt-2", "steps", "count", 7), - ] - - attempts = AgentEvalSummary.from_scores(scores, tasks=tasks).task_metric_attempts["task-a"] - - # Position lies: index 1 is attempt-2 under reward.score but attempt-1 under steps.count. - assert attempts["reward.score"][1].trial_id == "attempt-2" - assert attempts["steps.count"][1].trial_id == "attempt-1" - # trial_id tells the truth, so a join across the two keys is now possible and correct. - steps_by_trial = {a.trial_id: a.value for a in attempts["steps.count"]} - assert [(a.trial_id, a.value, steps_by_trial[a.trial_id]) for a in attempts["reward.score"]] == [ - ("attempt-0", 1.0, 5.0), - ("attempt-2", 0.0, 7.0), - ] - - -def test_dead_trials_are_nameable_from_the_summary_alone() -> None: - # AALGO-428 needs to say *which* trial died to roll up exception types. Before attempts carried a - # trial id the summary could count dead attempts but not name one; now it is a join key out to - # trials.jsonl, where the error lives. - tasks = [_task("task-a", _Metric("reward", MetricOutputSpec.continuous_score("score")))] - scores = [ - _score("task-a", "attempt-0", "reward", "score", 1.0), - _failed_score("task-a", "attempt-1", trial_failed=True), - _failed_score("task-a", "attempt-2", trial_failed=False), # metric raised: unmeasured, not dead - ] - - attempts = AgentEvalSummary.from_scores(scores, tasks=tasks).task_metric_attempts["task-a"]["reward.score"] - - assert {a.trial_id for a in attempts if a.value is None} == {"attempt-1"} - - -def test_duplicate_trial_ids_are_two_attempts_not_one() -> None: - # Nothing enforces trial-id uniqueness, so the attempt list must never be re-keyed by trial id: - # collapsing two attempts into one would silently drop pass@k's n. A list cannot lose cardinality. - tasks = [_task("task-a", _Metric("reward", MetricOutputSpec.continuous_score("score")))] - scores = [ - _score("task-a", "dup", "reward", "score", 1.0), - _score("task-a", "dup", "reward", "score", 0.0), - ] - - summary = AgentEvalSummary.from_scores(scores, tasks=tasks) - by_name = {score.name: score for score in summary.scores.scores} - - assert _pairs(summary.task_metric_attempts) == {"task-a": {"reward.score": [("dup", 1.0), ("dup", 0.0)]}} - assert by_name["reward.score.pass@1"].mean == pytest.approx(0.5) # n=2, not n=1 - assert by_name["reward.score.pass@2"].mean == pytest.approx(1.0) - - -def test_attempt_values_projects_to_a_bare_value_list() -> None: - # The projection pass@k reads: order, cardinality and None-vs-absent preserved exactly. - attempts = [ - AgentEvalAttemptValue(trial_id="t0", value=1.0), - AgentEvalAttemptValue(trial_id="t1", value=None), - AgentEvalAttemptValue(trial_id="t2", value=0.0), - ] - - assert attempt_values(attempts) == [1.0, None, 0.0] - assert attempt_values([]) == [] - - -def test_pass_at_k_aggregates_are_unchanged_by_carrying_trial_ids() -> None: - """Golden table captured from the pre-change implementation, before attempts carried trial ids. - - Every branch pass@k distinguishes is present: a task that always passes, one whose attempts - include a dead trial (None counts toward ``n``), one whose metric raised on an attempt (dropped - from ``n``, so it falls out of ``k=2``), and two that yielded nothing at all (``nan_count``). - """ - reward = _Metric("reward", MetricOutputSpec.continuous_score("score")) - passed = _Metric("complete", MetricOutputSpec.boolean("passed")) - tasks = [_task(t, reward, passed) for t in ("solved", "flaky", "judged-out", "unmeasured", "never-ran")] - scores = [ - _score("solved", "s0", "reward", "score", 1.0), - _score("solved", "s0", "complete", "passed", True), - _score("solved", "s1", "reward", "score", 1.0), - _score("solved", "s1", "complete", "passed", True), - _score("solved", "s2", "reward", "score", 1.0), - _score("solved", "s2", "complete", "passed", True), - _score("flaky", "f0", "reward", "score", 1.0), - _score("flaky", "f0", "complete", "passed", True), - _failed_score("flaky", "f1", trial_failed=True), - _failed_score("flaky", "f1", trial_failed=True, metric_type="complete"), - _score("flaky", "f2", "reward", "score", 0.0), - _score("flaky", "f2", "complete", "passed", False), - _score("judged-out", "j0", "reward", "score", 1.0), - _score("judged-out", "j0", "complete", "passed", True), - _failed_score("judged-out", "j1", trial_failed=False), - _failed_score("judged-out", "j1", trial_failed=False, metric_type="complete"), - _failed_score("unmeasured", "u0", trial_failed=False), - _failed_score("unmeasured", "u0", trial_failed=False, metric_type="complete"), - ] - - summary = AgentEvalSummary.from_scores(scores, tasks=tasks) - actual = {s.name: (s.mean, s.count, s.nan_count) for s in summary.scores.scores if ".pass@" in s.name} - - assert actual == { - "complete.passed.pass@1": (pytest.approx(0.7777777777777777), 3, 2), - "complete.passed.pass@2": (pytest.approx(0.8333333333333333), 2, 2), - "complete.passed.pass@3": (pytest.approx(1.0), 2, 2), - "reward.score.pass@1": (pytest.approx(0.7777777777777777), 3, 2), - "reward.score.pass@2": (pytest.approx(0.8333333333333333), 2, 2), - "reward.score.pass@3": (pytest.approx(1.0), 2, 2), - } - - -def test_summary_without_task_metric_attempts_loads_as_empty() -> None: - assert AgentEvalSummary.model_validate({}).task_metric_attempts == {} - - -def test_vendored_summary_accepts_task_metric_attempts() -> None: - from nemo_platform.beta.evaluator.agent_eval.results import AgentEvalSummary as VendoredAgentEvalSummary - - payload = { - "task_metric_attempts": { - "task-a": {"reward.score": [{"trial_id": "t0", "value": 1.0}, {"trial_id": "t1", "value": None}]} - } - } - - attempts = VendoredAgentEvalSummary.model_validate(payload).task_metric_attempts - assert [(a.trial_id, a.value) for a in attempts["task-a"]["reward.score"]] == [("t0", 1.0), ("t1", None)] - - -def test_vendored_results_module_is_a_verbatim_copy_of_this_one() -> None: - # `make vendor` mirrors this module into the SDK, rewriting only the package root. Validating the - # field shape (above) would still pass against a stale copy carrying older filtering or docs, so - # pin the whole file: any edit here that is not mirrored is drift between two live code paths. - import nemo_evaluator_sdk.agent_eval.results as source - import nemo_platform.beta.evaluator.agent_eval.results as vendored - - expected = ( - Path(source.__file__) - .read_text(encoding="utf-8") - .replace("from nemo_evaluator_sdk.", "from nemo_platform.beta.evaluator.") - ) - - assert Path(vendored.__file__).read_text(encoding="utf-8") == expected, ( - "sdk/python/.../beta/evaluator/agent_eval/results.py is out of sync; re-run `make vendor`" - ) - - -def test_gym_example_rejects_a_bundle_written_before_task_metric_attempts(tmp_path: Path) -> None: - # The field defaults to empty, so an older bundle would load cleanly and simply show no per-task - # section -- a reader would take that as "no per-task outcomes" rather than "this script cannot - # see them". Fail with a version message instead. - from packages.nemo_evaluator_sdk.examples.gym.inspect_results import load_bundle - - (tmp_path / "summary.json").write_text(json.dumps({"task_count": 2}), encoding="utf-8") - - with pytest.raises(SystemExit, match="predates summary.task_metric_attempts"): - load_bundle(tmp_path) - - -def test_gym_example_reads_task_outcomes_from_summary() -> None: - from packages.nemo_evaluator_sdk.examples.gym.inspect_results import per_task_attempts, per_task_outcomes - - summary = AgentEvalSummary( - task_metric_attempts={ - "task-a": { - "gym_reward.reward": [ - AgentEvalAttemptValue(trial_id="task-a__aaa", value=1.0), - AgentEvalAttemptValue(trial_id="task-a__bbb", value=0.0), - ] - }, - "task-b": {"gym_reward.reward": []}, - } - ) - - # The example's headline accessor keeps its bare-value shape... - assert per_task_outcomes(summary, metric_type="gym_reward", output_name="reward") == { - "task-a": [1.0, 0.0], - "task-b": [], - } - # ...and its sibling exposes the identity that makes an attempt traceable back to a rollout. - attempts = per_task_attempts(summary, metric_type="gym_reward", output_name="reward") - assert [(a.trial_id, a.value) for a in attempts["task-a"]] == [("task-a__aaa", 1.0), ("task-a__bbb", 0.0)] diff --git a/packages/nemo_evaluator_sdk/tests/agent_eval/test_task_metric_values.py b/packages/nemo_evaluator_sdk/tests/agent_eval/test_task_metric_values.py new file mode 100644 index 0000000000..4dfe6e5c56 --- /dev/null +++ b/packages/nemo_evaluator_sdk/tests/agent_eval/test_task_metric_values.py @@ -0,0 +1,656 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import json +import math +from collections.abc import Iterator +from pathlib import Path + +import pytest +from nemo_evaluator_sdk.agent_eval.results import ( + AgentEvalSummary, + TrialMetricValue, + TrialMetricValueType, + _task_metric_values, + metric_values, + numeric_metric_values, +) +from nemo_evaluator_sdk.agent_eval.scores import ( + TRIAL_STATUS_DETAIL, + AgentEvalDiagnostic, + AgentEvalDiagnosticSeverity, + AgentEvalScoreStatus, + AgentEvalTaskScore, +) +from nemo_evaluator_sdk.agent_eval.tasks import AgentEvalTask, SemanticReducer, SemanticView, ViewSignal +from nemo_evaluator_sdk.metrics.protocol import MetricInput, MetricOutput, MetricResult +from nemo_evaluator_sdk.values.protocol import MetricOutputSpec +from pydantic import RootModel, ValidationError + + +class _TokenCount(RootModel[int]): + """A free-model output: numeric, but a measurement rather than a per-trial score.""" + + +class _Metric: + def __init__(self, metric_type: str, output: MetricOutputSpec) -> None: + self._type = metric_type + self._output = output + + @property + def type(self) -> str: + return self._type + + def output_spec(self) -> list[MetricOutputSpec]: + return [self._output] + + async def compute_scores(self, input: MetricInput) -> MetricResult: # pragma: no cover + raise NotImplementedError + + +class _SinglePassScores(list[AgentEvalTaskScore]): + def __init__(self, scores: list[AgentEvalTaskScore]) -> None: + super().__init__(scores) + self.iterations = 0 + + def __iter__(self) -> Iterator[AgentEvalTaskScore]: + self.iterations += 1 + assert self.iterations == 1, "scores were rescanned" + return super().__iter__() + + +def _task(task_id: str, *metrics: _Metric) -> AgentEvalTask: + return AgentEvalTask(id=task_id, intent="test", inputs={}, metrics=list(metrics)) + + +def _pairs( + records: dict[str, dict[str, list[TrialMetricValue]]], +) -> dict[str, dict[str, list[tuple[str, float | int | bool | str | None]]]]: + """Flatten trial-metric records to ``(trial_id, value)`` so assertions stay readable.""" + return { + task_id: {key: [(a.trial_id, a.value) for a in values] for key, values in by_key.items()} + for task_id, by_key in records.items() + } + + +def _score( + task_id: str, + trial_id: str, + metric_type: str, + output_name: str, + value: object, + *, + status: AgentEvalScoreStatus = AgentEvalScoreStatus.COMPLETED, +) -> AgentEvalTaskScore: + return AgentEvalTaskScore( + id=f"run:{task_id}:{trial_id}:{metric_type}", + run_id="run", + task_id=task_id, + trial_id=trial_id, + metric_type=metric_type, + status=status, + outputs=[MetricOutput(name=output_name, value=value)], + ) + + +def _failed_score( + task_id: str, trial_id: str, *, trial_failed: bool, metric_type: str = "reward" +) -> AgentEvalTaskScore: + details = {TRIAL_STATUS_DETAIL: "failed"} if trial_failed else {"exception_type": "TimeoutError"} + return AgentEvalTaskScore( + id=f"run:{task_id}:{trial_id}:{metric_type}", + run_id="run", + task_id=task_id, + trial_id=trial_id, + metric_type=metric_type, + status=AgentEvalScoreStatus.FAILED, + diagnostics=[ + AgentEvalDiagnostic( + severity=AgentEvalDiagnosticSeverity.ERROR, + message="failed", + details=details, + ) + ], + ) + + +def test_summary_exposes_ordered_native_metric_values_per_task() -> None: + reward = _Metric("reward", MetricOutputSpec.continuous_score("score")) + retries = _Metric("retries", MetricOutputSpec.discrete_score("count")) + verdict = _Metric("verdict", MetricOutputSpec.label("label")) + complete = _Metric("complete", MetricOutputSpec.boolean("passed")) + tasks = [_task("task-a", reward, retries, verdict, complete), _task("task-b", reward)] + scores = [ + _score("task-a", "trial-0", "reward", "score", 1.0), + _score("task-a", "trial-0", "retries", "count", 2), + _score("task-a", "trial-0", "verdict", "label", "good"), + _score("task-a", "trial-0", "complete", "passed", True), + _score( + "task-a", + "trial-1", + "reward", + "score", + 0.25, + status=AgentEvalScoreStatus.PARTIAL, + ), + _score("task-a", "trial-1", "retries", "count", 3), + _score("task-a", "trial-1", "complete", "passed", False), + _score("task-b", "trial-0", "reward", "score", 0.0), + ] + + summary = AgentEvalSummary.from_scores(scores, tasks=tasks) + + assert _pairs(summary.task_metric_values) == { + "task-a": { + "complete.passed": [("trial-0", True), ("trial-1", False)], + "retries.count": [("trial-0", 2), ("trial-1", 3)], + "reward.score": [("trial-0", 1.0), ("trial-1", 0.25)], + "verdict.label": [("trial-0", "good")], + }, + "task-b": {"reward.score": [("trial-0", 0.0)]}, + } + + # `==` cannot see the type change (2 == 2.0, True == 1.0), so assert it directly -- preserving + # the metric's own type is the whole point. + records = summary.task_metric_values["task-a"] + count = records["retries.count"][0].value + assert isinstance(count, int) and not isinstance(count, bool) + assert records["complete.passed"][0].value is True + assert isinstance(records["reward.score"][0].value, float) + assert records["verdict.label"][0].value_type is TrialMetricValueType.LABEL + + +def test_task_metric_values_scans_scores_once() -> None: + tasks = [_task("task-a", _Metric("reward", MetricOutputSpec.continuous_score("score")))] + scores = _SinglePassScores( + [ + _score("task-a", "trial-0", "reward", "score", 1.0), + _score("task-a", "trial-1", "reward", "score", 0.0), + ] + ) + + assert _pairs(_task_metric_values(scores, tasks)) == { + "task-a": {"reward.score": [("trial-0", 1.0), ("trial-1", 0.0)]} + } + + +def test_failed_trials_are_recorded_but_metric_failures_are_unmeasured() -> None: + tasks = [ + _task("flaky", _Metric("reward", MetricOutputSpec.continuous_score("score"))), + _task("unmeasured", _Metric("reward", MetricOutputSpec.continuous_score("score"))), + ] + scores = [ + _score("flaky", "trial-0", "reward", "score", 1.0), + _failed_score("flaky", "trial-1", trial_failed=True), + _failed_score("unmeasured", "trial-0", trial_failed=False), + ] + + summary = AgentEvalSummary.from_scores(scores, tasks=tasks) + + assert _pairs(summary.task_metric_values) == { + # The dead trial keeps its identity, paired with its None; the unmeasured one has no entry at + # all, so it is nameable from neither -- that asymmetry is what pass@k depends on. + "flaky": {"reward.score": [("trial-0", 1.0), ("trial-1", None)]}, + "unmeasured": {"reward.score": []}, + } + + +def test_a_task_that_produced_no_trial_is_unmeasured_and_counted_in_pass_at_k_nan() -> None: + # from_scores can be handed a task list wider than the scores -- a caller re-aggregating a subset. + # The task still declares the metric, so it holds an empty value list and counts as missing + # coverage: excluding it would report pass@k over a denominator smaller than the task set asked + # for. A full run cannot reach this state; AgentEvaluator._score_trials refuses to score when a + # task produced no trial, which test_evaluator.py::test_run_rejects_tasks_without_trials pins. + reward = _Metric("reward", MetricOutputSpec.continuous_score("score")) + tasks = [_task("scored", reward), _task("never-ran", reward)] + scores = [_score("scored", "trial-0", "reward", "score", 1.0)] + + summary = AgentEvalSummary.from_scores(scores, tasks=tasks) + by_name = {score.name: score for score in summary.scores.scores} + + assert _pairs(summary.task_metric_values) == { + "scored": {"reward.score": [("trial-0", 1.0)]}, + "never-ran": {"reward.score": []}, + } + assert by_name["reward.score.pass@1"].mean == 1.0 # the one measured task passed + assert by_name["reward.score.pass@1"].count == 1 + assert by_name["reward.score.pass@1"].nan_count == 1 # ...and the unrun one is not hidden + + +def test_outputs_declared_under_an_unretained_schema_stay_out_even_when_numeric() -> None: + # Token measurements and other free models are excluded by their declared schema. Emitting a + # numeric value must not add them back: the value is a measurement, not a per-trial score. + tasks = [ + _task( + "task-a", + _Metric("reward", MetricOutputSpec.continuous_score("score")), + _Metric("usage", MetricOutputSpec.model("prompt_tokens", _TokenCount)), + ) + ] + scores = [ + _score("task-a", "trial-0", "reward", "score", 1.0), + _score("task-a", "trial-0", "usage", "prompt_tokens", 1234), + ] + + assert _pairs(AgentEvalSummary.from_scores(scores, tasks=tasks).task_metric_values) == { + "task-a": {"reward.score": [("trial-0", 1.0)]} + } + + +def test_one_tasks_schema_exclusion_does_not_suppress_another_tasks_output() -> None: + # The spec filter is per task: tasks in one run need not declare the same output under the same + # schema. Task-a declaring usage.prompt_tokens as a free model must not strip it from task-b, + # which never declared it and whose only evidence is the numeric value it actually emitted. + tasks = [ + _task("task-a", _Metric("usage", MetricOutputSpec.model("prompt_tokens", _TokenCount))), + _task("task-b", _Metric("reward", MetricOutputSpec.continuous_score("score"))), + ] + scores = [ + _score("task-a", "trial-0", "usage", "prompt_tokens", 100), + _score("task-b", "trial-0", "reward", "score", 1.0), + _score("task-b", "trial-0", "usage", "prompt_tokens", 250), # undeclared on task-b + ] + + records = AgentEvalSummary.from_scores(scores, tasks=tasks).task_metric_values + + # task-a declared it under an unretained schema, so it is not a key there at all -- not even an + # empty one -- and the numeric value it emitted cannot add it back. + assert records["task-a"] == {} + # task-b never declared it, so its emitted numeric value is the only evidence and it is kept. + assert sorted(records["task-b"]) == ["reward.score", "usage.prompt_tokens"] + assert _pairs(records)["task-b"]["usage.prompt_tokens"] == [("trial-0", 250)] + assert isinstance(records["task-b"]["usage.prompt_tokens"][0].value, int) # not widened to float + + +def test_nan_metric_values_survive_json_as_a_string() -> None: + # A metric may legitimately score NaN. json.dumps would write a bare NaN token, which is not + # valid JSON, so summary.json must carry the string form -- and read it back as a float. + tasks = [_task("task-a", _Metric("reward", MetricOutputSpec.continuous_score("score")))] + summary = AgentEvalSummary.from_scores([_score("task-a", "trial-0", "reward", "score", float("nan"))], tasks=tasks) + + payload = summary.model_dump(mode="json") + record = payload["task_metric_values"]["task-a"]["reward.score"][0] + assert record["value"] == "NaN" + assert record["value_type"] == "number" # what tells it apart from a label reading "NaN" + + # Strict JSON: no bare NaN/Infinity tokens anywhere in the serialized bundle. + def _reject(constant: str) -> float: + raise AssertionError(f"summary.json contains a bare {constant} token") + + reloaded = json.loads(json.dumps(payload), parse_constant=_reject) + value = AgentEvalSummary.model_validate(reloaded).task_metric_values["task-a"]["reward.score"][0].value + assert isinstance(value, float) and math.isnan(value) + + +def test_a_label_and_a_real_nan_are_distinguishable_on_the_wire() -> None: + # The reason value_type is required on the wire: strict JSON has no NaN literal, so a real NaN + # travels as the string "NaN" -- the same three bytes as a judge label that happens to read "NaN". + real_nan = TrialMetricValue(trial_id="t0", value=float("nan")) + label = TrialMetricValue(trial_id="t1", value_type=TrialMetricValueType.LABEL, value="NaN") + + dumped = [a.model_dump(mode="json") for a in (real_nan, label)] + assert [d["value"] for d in dumped] == ["NaN", "NaN"] # identical payloads... + assert [d["value_type"] for d in dumped] == ["number", "label"] # ...told apart by the type + + back = [TrialMetricValue.model_validate(d) for d in json.loads(json.dumps(dumped))] + assert isinstance(back[0].value, float) and math.isnan(back[0].value) + assert back[1].value == "NaN" and isinstance(back[1].value, str) + + +def test_a_payload_without_value_type_still_loads() -> None: + # Bundles written before value_type existed, and hand-built records. The old encoding gave a + # string exactly one meaning -- the NaN escape -- so that is how a bare string is read. + legacy_nan = TrialMetricValue.model_validate({"trial_id": "t0", "value": "NaN"}) + assert legacy_nan.value_type is TrialMetricValueType.NUMBER + assert isinstance(legacy_nan.value, float) and math.isnan(legacy_nan.value) + + assert TrialMetricValue.model_validate({"trial_id": "t1", "value": 5}).value_type is TrialMetricValueType.NUMBER + assert TrialMetricValue.model_validate({"trial_id": "t2", "value": None}).value_type is TrialMetricValueType.MISSING + assert TrialMetricValue.model_validate({"trial_id": "t3", "value": "ok"}).value_type is TrialMetricValueType.LABEL + + +def test_a_record_cannot_claim_one_kind_and_carry_another() -> None: + for payload in ( + {"trial_id": "t", "value_type": "number", "value": "good"}, + {"trial_id": "t", "value_type": "label", "value": 1.0}, + {"trial_id": "t", "value_type": "label", "value": None}, + {"trial_id": "t", "value_type": "missing", "value": 1.0}, + ): + with pytest.raises(ValidationError): + TrialMetricValue.model_validate(payload) + + +def test_value_type_is_always_present_in_serialized_output() -> None: + # Optional for the caller (derived from the value), but never absent from what a reader sees -- + # they must not have to guess whether a string is an escaped float or a label. + for record in ( + TrialMetricValue(trial_id="t", value=1.0), + TrialMetricValue(trial_id="t", value="good"), + TrialMetricValue(trial_id="t", value=None), + TrialMetricValue(trial_id="t", value=float("nan")), + ): + assert "value_type" in record.model_dump(mode="json") + + +def test_without_tasks_there_is_no_spec_to_filter_on() -> None: + # No tasks means no declared schemas to consult, so every numeric output observed is retained. + scores = [_score("task-a", "trial-0", "usage", "prompt_tokens", 1234)] + + summary = AgentEvalSummary.from_scores(scores) + + assert _pairs(summary.task_metric_values) == {"task-a": {"usage.prompt_tokens": [("trial-0", 1234)]}} + assert isinstance(summary.task_metric_values["task-a"]["usage.prompt_tokens"][0].value, int) + + +def test_values_align_across_keys_by_trial_id_not_position() -> None: + # A metric that raised drops its record entirely while a dead trial holds its slot as None, so a + # metric failure shortens its own key's list without shortening its neighbour's. Index i of two + # keys is then two different trials -- which is exactly why every record names its trial. + tasks = [ + _task( + "task-a", + _Metric("reward", MetricOutputSpec.continuous_score("score")), + _Metric("steps", MetricOutputSpec.discrete_score("count")), + ) + ] + scores = [ + _score("task-a", "trial-0", "reward", "score", 1.0), + _score("task-a", "trial-0", "steps", "count", 5), + _failed_score("task-a", "trial-1", trial_failed=False), # the reward judge timed out + _score("task-a", "trial-1", "steps", "count", 9), + _score("task-a", "trial-2", "reward", "score", 0.0), + _score("task-a", "trial-2", "steps", "count", 7), + ] + + records = AgentEvalSummary.from_scores(scores, tasks=tasks).task_metric_values["task-a"] + + # Position lies: index 1 is trial-2 under reward.score but trial-1 under steps.count. + assert records["reward.score"][1].trial_id == "trial-2" + assert records["steps.count"][1].trial_id == "trial-1" + # trial_id tells the truth, so a join across the two keys is now possible and correct. + steps_by_trial = {a.trial_id: a.value for a in records["steps.count"]} + assert [(a.trial_id, a.value, steps_by_trial[a.trial_id]) for a in records["reward.score"]] == [ + ("trial-0", 1.0, 5), + ("trial-2", 0.0, 7), + ] + + +def test_an_undeclared_label_is_discovered_from_the_value_alone() -> None: + # Load-bearing: the declared branch prepopulates keys from task specs *before* the discovery + # loop, so a test using a *declared* label would still pass if that loop's gate were left as + # _semantic_value (which drops strings). Only an undeclared label exercises it. + tasks = [_task("task-a", _Metric("reward", MetricOutputSpec.continuous_score("score")))] + scores = [ + _score("task-a", "trial-0", "reward", "score", 1.0), + _score("task-a", "trial-0", "verdict", "grade", "excellent"), # no task declares this + ] + + assert _pairs(AgentEvalSummary.from_scores(scores, tasks=tasks).task_metric_values) == { + "task-a": {"reward.score": [("trial-0", 1.0)], "verdict.grade": [("trial-0", "excellent")]} + } + + # Same again with no specs at all, where the filter cannot apply. + assert _pairs(AgentEvalSummary.from_scores(scores).task_metric_values) == { + "task-a": {"reward.score": [("trial-0", 1.0)], "verdict.grade": [("trial-0", "excellent")]} + } + + +def test_a_label_under_a_scorelike_key_does_not_break_pass_at_k() -> None: + # _scorelike_outputs unions across all tasks while the spec filter is per task, and + # validate_metric_result coerces-and-discards -- so a label can reach a key pass@k reads. It must + # land in nan_count as an unmeasured task, not raise on `"good" >= 1.0`. + reward = _Metric("reward", MetricOutputSpec.continuous_score("score")) + tasks = [_task("scored", reward), _task("labelled")] # 'labelled' declares no metric at all + scores = [ + _score("scored", "trial-0", "reward", "score", 1.0), + _score("labelled", "trial-0", "reward", "score", "good"), # a label under a scorelike key + ] + + summary = AgentEvalSummary.from_scores(scores, tasks=tasks) + by_name = {score.name: score for score in summary.scores.scores} + + assert _pairs(summary.task_metric_values)["labelled"] == {"reward.score": [("trial-0", "good")]} + assert by_name["reward.score.pass@1"].mean == 1.0 # the one measurable task passed + assert by_name["reward.score.pass@1"].count == 1 + assert by_name["reward.score.pass@1"].nan_count == 1 # the labelled task is unmeasured, not failed + + +def test_a_view_over_a_label_output_reduces_to_nothing_rather_than_raising() -> None: + # Views read MetricOutput through _semantic_value, which still projects to float and drops + # strings -- so widening the value record cannot leak a label into view arithmetic. This pins + # that separation; it fails loudly if views are ever rewired onto task_metric_values. + verdict = _Metric("verdict", MetricOutputSpec.label("grade")) + task = AgentEvalTask( + id="task-a", + intent="test", + inputs={}, + metrics=[verdict], + views={ + "quality": SemanticView( + reducer=SemanticReducer.SINGLE, signals=[ViewSignal(metric="verdict", output="grade")] + ) + }, + ) + scores = [_score("task-a", "trial-0", "verdict", "grade", "excellent")] + + summary = AgentEvalSummary.from_scores(scores, tasks=[task]) + view = summary.score("view.quality") + + assert view.count == 0 and view.nan_count == 1 # reduced to nothing, no exception + assert _pairs(summary.task_metric_values) == {"task-a": {"verdict.grade": [("trial-0", "excellent")]}} + + +def test_dead_trials_are_nameable_from_the_summary_alone() -> None: + # AALGO-428 needs to say *which* trial died to roll up exception types. Before records carried a + # trial id the summary could count dead trials but not name one; now it is a lookup key out to + # trials.jsonl, where the error lives. + tasks = [_task("task-a", _Metric("reward", MetricOutputSpec.continuous_score("score")))] + scores = [ + _score("task-a", "trial-0", "reward", "score", 1.0), + _failed_score("task-a", "trial-1", trial_failed=True), + _failed_score("task-a", "trial-2", trial_failed=False), # metric raised: unmeasured, not dead + ] + + records = AgentEvalSummary.from_scores(scores, tasks=tasks).task_metric_values["task-a"]["reward.score"] + + assert {a.trial_id for a in records if a.value is None} == {"trial-1"} + + +def test_duplicate_trial_ids_are_two_records_not_one() -> None: + # Nothing enforces trial-id uniqueness, so the value list must never be re-keyed by trial id: + # collapsing two records into one would silently drop pass@k's n. A list cannot lose cardinality. + tasks = [_task("task-a", _Metric("reward", MetricOutputSpec.continuous_score("score")))] + scores = [ + _score("task-a", "dup", "reward", "score", 1.0), + _score("task-a", "dup", "reward", "score", 0.0), + ] + + summary = AgentEvalSummary.from_scores(scores, tasks=tasks) + by_name = {score.name: score for score in summary.scores.scores} + + assert _pairs(summary.task_metric_values) == {"task-a": {"reward.score": [("dup", 1.0), ("dup", 0.0)]}} + assert by_name["reward.score.pass@1"].mean == pytest.approx(0.5) # n=2, not n=1 + assert by_name["reward.score.pass@2"].mean == pytest.approx(1.0) + + +def test_metric_values_projects_to_a_bare_value_list() -> None: + # The projection pass@k reads: order, cardinality and None-vs-absent preserved exactly. + records = [ + TrialMetricValue(trial_id="t0", value=1.0), + TrialMetricValue(trial_id="t1", value=None), + TrialMetricValue(trial_id="t2", value=0.0), + ] + + assert metric_values(records) == [1.0, None, 0.0] + + # A label is preserved faithfully by metric_values and dropped by the arithmetic projection: + # a categorical verdict is an unmeasured trial, not a failed one. + with_label = [*records, TrialMetricValue(trial_id="t3", value="excellent")] + assert metric_values(with_label) == [1.0, None, 0.0, "excellent"] + assert numeric_metric_values(with_label) == [1.0, None, 0.0] + assert numeric_metric_values([]) == [] + assert metric_values([]) == [] + + +def test_pass_at_k_aggregates_are_unchanged_by_carrying_trial_ids() -> None: + """Golden table captured from the pre-change implementation, before records carried trial ids. + + Every branch pass@k distinguishes is present: a task that always passes, one whose trials + include a dead one (None counts toward ``n``), one whose metric raised on a trial (dropped + from ``n``, so it falls out of ``k=2``), and two that yielded nothing at all (``nan_count``). + """ + reward = _Metric("reward", MetricOutputSpec.continuous_score("score")) + passed = _Metric("complete", MetricOutputSpec.boolean("passed")) + tasks = [_task(t, reward, passed) for t in ("solved", "flaky", "judged-out", "unmeasured", "never-ran")] + scores = [ + _score("solved", "s0", "reward", "score", 1.0), + _score("solved", "s0", "complete", "passed", True), + _score("solved", "s1", "reward", "score", 1.0), + _score("solved", "s1", "complete", "passed", True), + _score("solved", "s2", "reward", "score", 1.0), + _score("solved", "s2", "complete", "passed", True), + _score("flaky", "f0", "reward", "score", 1.0), + _score("flaky", "f0", "complete", "passed", True), + _failed_score("flaky", "f1", trial_failed=True), + _failed_score("flaky", "f1", trial_failed=True, metric_type="complete"), + _score("flaky", "f2", "reward", "score", 0.0), + _score("flaky", "f2", "complete", "passed", False), + _score("judged-out", "j0", "reward", "score", 1.0), + _score("judged-out", "j0", "complete", "passed", True), + _failed_score("judged-out", "j1", trial_failed=False), + _failed_score("judged-out", "j1", trial_failed=False, metric_type="complete"), + _failed_score("unmeasured", "u0", trial_failed=False), + _failed_score("unmeasured", "u0", trial_failed=False, metric_type="complete"), + ] + + summary = AgentEvalSummary.from_scores(scores, tasks=tasks) + actual = {s.name: (s.mean, s.count, s.nan_count) for s in summary.scores.scores if ".pass@" in s.name} + + assert actual == { + "complete.passed.pass@1": (pytest.approx(0.7777777777777777), 3, 2), + "complete.passed.pass@2": (pytest.approx(0.8333333333333333), 2, 2), + "complete.passed.pass@3": (pytest.approx(1.0), 2, 2), + "reward.score.pass@1": (pytest.approx(0.7777777777777777), 3, 2), + "reward.score.pass@2": (pytest.approx(0.8333333333333333), 2, 2), + "reward.score.pass@3": (pytest.approx(1.0), 2, 2), + } + + +def test_summary_without_task_metric_values_loads_as_empty() -> None: + assert AgentEvalSummary.model_validate({}).task_metric_values == {} + + +def test_vendored_summary_accepts_task_metric_values() -> None: + from nemo_platform.beta.evaluator.agent_eval.results import AgentEvalSummary as VendoredAgentEvalSummary + + payload = { + "task_metric_values": { + "task-a": {"reward.score": [{"trial_id": "t0", "value": 1.0}, {"trial_id": "t1", "value": None}]} + } + } + + # The payload deliberately omits value_type, so this doubles as the legacy-derivation regression. + records = VendoredAgentEvalSummary.model_validate(payload).task_metric_values + assert [(a.trial_id, a.value) for a in records["task-a"]["reward.score"]] == [("t0", 1.0), ("t1", None)] + + +def test_vendored_module_exposes_the_public_value_api() -> None: + # The byte-copy test below proves file parity, not that the names are usable through the shipped + # package. These are the surface a consumer of nemo-platform actually imports. + from nemo_platform.beta.evaluator.agent_eval.results import ( + AgentEvalSummary as VendoredSummary, + ) + from nemo_platform.beta.evaluator.agent_eval.results import ( + TrialMetricValue as VendoredValue, + ) + from nemo_platform.beta.evaluator.agent_eval.results import ( + TrialMetricValueType as VendoredType, + ) + from nemo_platform.beta.evaluator.agent_eval.results import ( + numeric_metric_values as vendored_numeric, + ) + + records = [VendoredValue(trial_id="t0", value=1.0), VendoredValue(trial_id="t1", value="good")] + assert vendored_numeric(records) == [1.0] # the label is dropped, as in the source module + assert records[1].value_type is VendoredType.LABEL + + summary = VendoredSummary(task_metric_values={"task-a": {"reward.score": records}}) + [outcomes] = summary.task_outcomes() + assert outcomes.task_id == "task-a" and outcomes.outcomes[0].metric_name == "reward.score" + + +def test_vendored_results_module_is_a_verbatim_copy_of_this_one() -> None: + # `make vendor` mirrors this module into the SDK, rewriting only the package root. Validating the + # field shape (above) would still pass against a stale copy carrying older filtering or docs, so + # pin the whole file: any edit here that is not mirrored is drift between two live code paths. + import nemo_evaluator_sdk.agent_eval.results as source + import nemo_platform.beta.evaluator.agent_eval.results as vendored + + expected = ( + Path(source.__file__) + .read_text(encoding="utf-8") + .replace("from nemo_evaluator_sdk.", "from nemo_platform.beta.evaluator.") + ) + + assert Path(vendored.__file__).read_text(encoding="utf-8") == expected, ( + "sdk/python/.../beta/evaluator/agent_eval/results.py is out of sync; re-run `make vendor`" + ) + + +def test_gym_example_rejects_a_bundle_written_before_task_metric_values(tmp_path: Path) -> None: + # The field defaults to empty, so an older bundle would load cleanly and simply show no per-task + # section -- a reader would take that as "no per-task outcomes" rather than "this script cannot + # see them". Fail with a version message instead. + from packages.nemo_evaluator_sdk.examples.gym.inspect_results import BundleFormatError, load_bundle + + (tmp_path / "summary.json").write_text(json.dumps({"task_count": 2}), encoding="utf-8") + + with pytest.raises(BundleFormatError, match="predates summary.task_metric_values"): + load_bundle(tmp_path) + + +def test_summary_task_outcomes_name_their_own_keys() -> None: + # task_outcomes() lifts the nested dicts into models whose fields are named, for callers that + # want a typed object to pass around. It is a read-time view: the summary keeps the dict shape. + summary = AgentEvalSummary( + task_metric_values={ + "task-b": {"gym_reward.reward": [TrialMetricValue(trial_id="task-b__bbb", value=0.0)]}, + "task-a": { + "steps.count": [TrialMetricValue(trial_id="task-a__aaa", value=5)], + "gym_reward.reward": [TrialMetricValue(trial_id="task-a__aaa", value=1.0)], + }, + } + ) + + outcomes = summary.task_outcomes() + + assert [o.task_id for o in outcomes] == ["task-a", "task-b"] # sorted by task + assert [o.metric_name for o in outcomes[0].outcomes] == ["gym_reward.reward", "steps.count"] # then metric + assert [(t.trial_id, t.value) for t in outcomes[0].outcomes[0].trials] == [("task-a__aaa", 1.0)] + + +def test_gym_example_reads_task_outcomes_from_summary() -> None: + from packages.nemo_evaluator_sdk.examples.gym.inspect_results import per_task_outcomes, per_task_trial_values + + summary = AgentEvalSummary( + task_metric_values={ + "task-a": { + "gym_reward.reward": [ + TrialMetricValue(trial_id="task-a__aaa", value=1.0), + TrialMetricValue(trial_id="task-a__bbb", value=0.0), + ] + }, + "task-b": {"gym_reward.reward": []}, + } + ) + + # The example's headline accessor keeps its bare-value shape... + assert per_task_outcomes(summary, metric_type="gym_reward", output_name="reward") == { + "task-a": [1.0, 0.0], + "task-b": [], + } + # ...and its sibling exposes the identity that makes a value traceable back to a rollout. + records = per_task_trial_values(summary, metric_type="gym_reward", output_name="reward") + assert [(a.trial_id, a.value) for a in records["task-a"]] == [("task-a__aaa", 1.0), ("task-a__bbb", 0.0)] diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/results.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/results.py index 39d67c527b..562140e697 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/results.py +++ b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/results.py @@ -9,6 +9,7 @@ import math from collections.abc import Mapping, Sequence from datetime import datetime +from enum import Enum from pathlib import Path from typing import Any @@ -23,7 +24,7 @@ from nemo_platform.beta.evaluator.metrics.aggregation import compute_percentiles from nemo_platform.beta.evaluator.metrics.protocol import MetricOutput from nemo_platform.beta.evaluator.metrics.utils import metric_type_name -from nemo_platform.beta.evaluator.values.protocol import BooleanValue, ContinuousScore, DiscreteScore +from nemo_platform.beta.evaluator.values.protocol import BooleanValue, ContinuousScore, DiscreteScore, Label from nemo_platform.beta.evaluator.values.results import ( AggregatedMetricResult, AggregateRangeScore, @@ -34,16 +35,18 @@ serialize_value, summary_aggregate_record, ) -from pydantic import BaseModel, ConfigDict, Field, field_serializer +from pydantic import BaseModel, ConfigDict, Field, field_serializer, model_validator -#: Metric-output value schemas retained in the ordered per-task attempt mapping. -_TASK_METRIC_VALUE_SCHEMAS = (ContinuousScore, DiscreteScore, BooleanValue) +#: Metric-output value schemas retained in the ordered per-task value mapping. Broader than +#: :data:`_PASS_AT_K_VALUE_SCHEMAS` on purpose: a :class:`TrialMetricValue` is per-trial evidence, so a +#: count or a judge's label is worth keeping even though neither is a "did it pass?" signal. +_TASK_METRIC_VALUE_SCHEMAS = (ContinuousScore, DiscreteScore, BooleanValue, Label) -#: Metric-output value schemas eligible for pass@k (a per-attempt "did it pass?" signal). Labels, +#: Metric-output value schemas eligible for pass@k (a per-trial "did it pass?" signal). Labels, #: discrete/count outputs, and free models (e.g. token measurements) are excluded. _PASS_AT_K_VALUE_SCHEMAS = (ContinuousScore, BooleanValue) -#: Score value at or above which an attempt counts as a pass for pass@k. Full credit — pass@k answers +#: Score value at or above which a trial counts as a pass for pass@k. Full credit — pass@k answers #: "did the agent solve the task", so partial credit is not a pass. Deliberately not configurable: #: it's a reporting-time interpretation, and making it tunable would yield pass@k numbers that look #: comparable across runs but aren't. @@ -61,8 +64,53 @@ class AgentEvalMetricOutputCoverage(BaseModel): missing: int = Field(default=0, description="Scores where the output was expected but absent.") -class AgentEvalAttemptValue(BaseModel): - """One attempt at a task under one metric output: which trial made it, and what it measured. +#: Tokens :class:`TrialMetricValue` escapes non-finite floats as, and the floats they decode to. +#: Strict JSON has no literal for these, so they travel as strings -- which is the whole reason the +#: record carries ``value_type``: without it, a label that happens to read "NaN" is the same three +#: bytes as a real NaN. +_SPECIAL_FLOAT_TOKENS_MAP: dict[str, float] = { + "NaN": float("nan"), + "Infinity": float("inf"), + "-Infinity": float("-inf"), +} + + +def _escape_special_float(value: float) -> str: + """The token :data:`_SPECIAL_FLOAT_TOKENS_MAP` decodes back to ``value``. + + Looked up rather than spelled out a second time, so the encode and decode directions cannot + drift apart. NaN needs :func:`math.isnan` rather than equality: it is the one float that does + not equal itself, so a lookup keyed by value would miss it. + """ + for token, decoded in _SPECIAL_FLOAT_TOKENS_MAP.items(): + if decoded == value or (math.isnan(decoded) and math.isnan(value)): + return token + raise ValueError(f"{value!r} is a finite float and needs no escape") + + +class TrialMetricValueType(str, Enum): + """What kind of value one trial recorded under one metric output. + + Deliberately coarser than the declared value schemas: JSON already round-trips int, float and + bool distinctly, so a ``continuous``/``discrete``/``boolean`` split would restate what the payload + already says and give a reader two sources of truth for one fact. The only thing JSON cannot + carry is whether a string is a number's escape or a label, and that is exactly what this + discriminates. + """ + + NUMBER = "number" + LABEL = "label" + MISSING = "missing" + + +class TrialMetricValue(BaseModel): + """One trial's measured value under one metric output: which trial made it, and what it measured. + + Values keep the type the metric produced them in -- a count stays an int, a flag stays a bool, a + judge's verdict stays the string it was -- because this is one trial's measurement, not a mean or + other aggregate. Look up the matching trial by ``trial_id`` (in ``result.trials`` or + ``trials.jsonl``); do not assume list index lines up across metric outputs. Read it through + :func:`numeric_metric_values` when you intend to do arithmetic. Frozen because these records are handed out by reference from the summary: a consumer rescaling values in place (Gym reports reward on 0-100 where we use 0-1) would otherwise rewrite the run's @@ -73,36 +121,136 @@ class AgentEvalAttemptValue(BaseModel): trial_id: str = Field( description=( - "Identifier of the trial that made this attempt. Joins to AgentEvalTrial.id " + "Identifier of the trial that produced this value. Joins to AgentEvalTrial.id " "(trials.jsonl) and AgentEvalTaskScore.trial_id (scores.jsonl)." ) ) - value: float | None = Field( + # The default is never observed: `_derive_value_type` runs before validation and always supplies + # one. It exists so callers can write TrialMetricValue(trial_id=..., value=...) without + # restating what the value already says -- the type checker reads the signature, not the validator. + value_type: TrialMetricValueType = Field( + default=TrialMetricValueType.MISSING, + description=( + "Which kind of value this record holds: 'number' (float, int or bool), 'label' (a " + "categorical string), or 'missing' (the trial failed before it could be measured). " + "Always present in serialized output, because non-finite floats are escaped as strings " + "-- without it a genuine label reading 'NaN' and a real NaN are the same three bytes. " + "Derived from 'value' when omitted, so hand-built records and bundles written before " + "this field existed both load." + ), + ) + value: float | int | bool | str | None = Field( description=( - "What the metric output measured, or None when the trial failed before it could be " - "measured -- an attempt that did not pass. Required rather than defaulted: None is a " - "load-bearing signal pass@k counts as a failed attempt, so an omitted value must not " - "quietly become one." + "What the metric output measured, in the type the metric produced it in -- a number, a " + "label, or None when the trial failed before it could be measured: a trial that did " + "not pass. Required rather than defaulted: None is a load-bearing signal pass@k counts " + "as not passing, so an omitted value must not quietly become one. None never means " + "'no value of this kind'; that is what value_type is for." ), ) + @model_validator(mode="before") + @classmethod + def _derive_value_type(cls, data: Any) -> Any: + """Fill in ``value_type`` when absent, and decode the escaped-float form when present. + + Runs *before* the union so the escape is undone while the discriminator is still readable: + afterwards pydantic's smart mode has already committed ``"NaN"`` to ``str``, and the record + would be a label whatever the type said. + """ + if not isinstance(data, Mapping): + return data + value = data.get("value") + declared = data.get("value_type") + + if declared is None: + # No discriminator: a hand-built record, or a bundle written before this field existed. + # The old encoding gave a string exactly one meaning -- the escape -- so honour that + # rather than reading a pre-widening NaN as the label "NaN". A label that genuinely + # reads "NaN" must therefore name its value_type explicitly. + if isinstance(value, str): + if value in _SPECIAL_FLOAT_TOKENS_MAP: + return { + **data, + "value_type": TrialMetricValueType.NUMBER, + "value": _SPECIAL_FLOAT_TOKENS_MAP[value], + } + return {**data, "value_type": TrialMetricValueType.LABEL} + return { + **data, + "value_type": TrialMetricValueType.MISSING if value is None else TrialMetricValueType.NUMBER, + } + + if TrialMetricValueType(declared) is TrialMetricValueType.NUMBER and isinstance(value, str): + decoded = _SPECIAL_FLOAT_TOKENS_MAP.get(value) + if decoded is None: + raise ValueError( + f"value_type='number' but value {value!r} is not one of the escaped-float tokens " + f"{sorted(_SPECIAL_FLOAT_TOKENS_MAP)}; a categorical value must declare value_type='label'" + ) + return {**data, "value": decoded} + return data + + @model_validator(mode="after") + def _value_matches_its_type(self) -> TrialMetricValue: + """Re-narrow the union, so a record cannot claim one kind and carry another.""" + if self.value_type is TrialMetricValueType.MISSING: + if self.value is not None: + raise ValueError("value_type='missing' requires value None (a trial that died before measurement)") + elif self.value_type is TrialMetricValueType.LABEL: + if not isinstance(self.value, str): + raise ValueError(f"value_type='label' requires a string value, got {type(self.value).__name__}") + elif not isinstance(self.value, bool | int | float): + raise ValueError(f"value_type='number' requires a numeric value, got {type(self.value).__name__}") + return self + @field_serializer("value") - def serialize_nan(self, value: float | None) -> float | str | None: - """Emit NaN as the string ``"NaN"``, matching :class:`MetricOutput`. - - A metric may legitimately score an attempt NaN, and this is the first summary field to carry - a raw metric value rather than a filtered aggregate. ``json.dumps`` would write it as a bare - ``NaN`` token, which is valid Python but not valid JSON, so any strict reader of - ``summary.json`` would reject the whole bundle. Pydantic coerces the string back to a float - on load, so the round trip is lossless. + def serialize_nan(self, value: float | int | bool | str | None) -> float | int | bool | str | None: + """Escape non-finite floats as strings, so ``summary.json`` stays strict JSON. + + A metric may legitimately score a trial NaN, and this is the first summary field to carry + a raw metric value rather than a filtered aggregate. ``json.dumps`` would write a bare ``NaN`` + or ``Infinity`` token, which is valid Python but not valid JSON, so any strict reader of + ``summary.json`` would reject the whole bundle. ``value_type`` says which of these strings is + an escape and which is a label, so the round trip is lossless in both directions. + + This is deliberately *wider* than :meth:`MetricOutput.serialize_nan`, which escapes NaN only + and has no decoding validator -- an infinite value reaches ``scores.jsonl`` as ``null``. + Collapsing the SDK's several non-finite-float escapes into one pair belongs in ``values/``. """ - if isinstance(value, float) and math.isnan(value): - return "NaN" + if isinstance(value, float) and not math.isfinite(value): + return _escape_special_float(value) return value +#: One task's recorded values, keyed ``"."``. Named because the nesting is +#: otherwise spelled out at every producer, consumer and local that touches it, and because the key +#: format is the part a reader cannot infer from ``dict[str, ...]``. +TrialValuesByMetric = dict[str, list[TrialMetricValue]] + + +class PerTaskOutcome(BaseModel): + """Every trial's value at one task under one metric output.""" + + model_config = ConfigDict(extra="forbid") + + metric_name: str = Field(description="'.', e.g. 'gym_reward.reward'.") + trials: list[TrialMetricValue] = Field( + description="Values in trial order. A value of None is a trial that died before scoring." + ) + + +class PerTaskOutcomes(BaseModel): + """One task's values across every metric output that measured it.""" + + model_config = ConfigDict(extra="forbid") + + task_id: str = Field(description="The task these outcomes belong to.") + outcomes: list[PerTaskOutcome] = Field(description="One entry per metric output, sorted by metric_name.") + + class AgentEvalSummary(BaseModel): - """Aggregated scores, coverage, per-task attempt values, and run counts for an agent-eval run.""" + """Aggregated scores, coverage, per-task metric values, and run counts for an agent-eval run.""" model_config = ConfigDict(extra="forbid") @@ -193,49 +341,59 @@ class AgentEvalSummary(BaseModel): } ], ) - task_metric_attempts: dict[str, dict[str, list[AgentEvalAttemptValue]]] = Field( + task_metric_values: dict[str, TrialValuesByMetric] = Field( default_factory=dict, description=( - "Per task, the attempts each '.' measured, in trial order. Each " - "attempt names the trial that made it, so attempts join across keys -- and out to " - "trials.jsonl and scores.jsonl -- by trial_id. A failed trial has value None: an attempt " - "that did not pass. An unmeasured attempt (metric failed, output absent) has no entry at " - "all, so each key's list is independent: align by trial_id, never by position. An empty " - "list means nothing was measured, including a task that produced no trial." + "Per task, the values each '.' measured, in trial order. Each " + "record names the trial that produced it, so values join across keys -- and out to " + "trials.jsonl and scores.jsonl -- by trial_id. Values keep the type the metric produced " + "them in: a count stays an int, a flag stays a bool, a judge's verdict stays a label -- " + "read them through numeric_metric_values() before doing arithmetic. A failed trial has " + "value None: a trial that did not pass. An unmeasured trial (metric failed, output " + "absent) has no entry at all, so each key's list is independent: align by trial_id, " + "never by position. An empty list means nothing was measured, including a task that " + "produced no trial." ), examples=[ { "contract-review-msa-indemnity": { "harbor_reward.reward": [ - {"trial_id": "contract-review-msa-indemnity__k3f9wq2", "value": 1.0}, - {"trial_id": "contract-review-msa-indemnity__t7m2xb4", "value": 0.0}, - {"trial_id": "contract-review-msa-indemnity__9jr4vd1", "value": 1.0}, + {"trial_id": "contract-review-msa-indemnity__k3f9wq2", "value_type": "number", "value": 1.0}, + {"trial_id": "contract-review-msa-indemnity__t7m2xb4", "value_type": "number", "value": 0.0}, + {"trial_id": "contract-review-msa-indemnity__9jr4vd1", "value_type": "number", "value": 1.0}, + ], + # A count stays an int, and t7m2xb4's judge verdict is kept as a label -- neither + # is pass@k-eligible, but both are per-trial evidence worth recording. + "steps.count": [ + {"trial_id": "contract-review-msa-indemnity__k3f9wq2", "value_type": "number", "value": 14}, + {"trial_id": "contract-review-msa-indemnity__t7m2xb4", "value_type": "number", "value": 31}, + {"trial_id": "contract-review-msa-indemnity__9jr4vd1", "value_type": "number", "value": 12}, ], - # t7m2xb4 is absent here rather than null: its judge timed out, so that attempt + # t7m2xb4 is absent here rather than null: its judge timed out, so that trial # went unmeasured. Index 1 is therefore a different trial in each of these lists. "rubric_judge.criteria_pass_rate": [ - {"trial_id": "contract-review-msa-indemnity__k3f9wq2", "value": 0.75}, - {"trial_id": "contract-review-msa-indemnity__9jr4vd1", "value": 1.0}, + {"trial_id": "contract-review-msa-indemnity__k3f9wq2", "value_type": "number", "value": 0.75}, + {"trial_id": "contract-review-msa-indemnity__9jr4vd1", "value_type": "number", "value": 1.0}, ], }, "nda-scope-carveouts": { - # p2hn8sc died in the sandbox, so it is null in every key: an attempt that + # p2hn8sc died in the sandbox, so it is 'missing' in every key: a trial that # happened and did not pass, as opposed to one that was never measured. "harbor_reward.reward": [ - {"trial_id": "nda-scope-carveouts__p2hn8sc", "value": None}, - {"trial_id": "nda-scope-carveouts__w5db3qy", "value": 1.0}, - {"trial_id": "nda-scope-carveouts__z8kt1nf", "value": 0.0}, + {"trial_id": "nda-scope-carveouts__p2hn8sc", "value_type": "missing", "value": None}, + {"trial_id": "nda-scope-carveouts__w5db3qy", "value_type": "number", "value": 1.0}, + {"trial_id": "nda-scope-carveouts__z8kt1nf", "value_type": "number", "value": 0.0}, ], - "rubric_judge.criteria_pass_rate": [ - {"trial_id": "nda-scope-carveouts__p2hn8sc", "value": None}, - {"trial_id": "nda-scope-carveouts__w5db3qy", "value": 0.6}, - {"trial_id": "nda-scope-carveouts__z8kt1nf", "value": 0.2}, + "rubric_judge.verdict": [ + {"trial_id": "nda-scope-carveouts__p2hn8sc", "value_type": "missing", "value": None}, + {"trial_id": "nda-scope-carveouts__w5db3qy", "value_type": "label", "value": "compliant"}, + {"trial_id": "nda-scope-carveouts__z8kt1nf", "value_type": "label", "value": "overbroad"}, ], }, # Requested, but the runner returned no trial for it: keys declared, nothing measured. "merger-hsr-filing-threshold": { "harbor_reward.reward": [], - "rubric_judge.criteria_pass_rate": [], + "rubric_judge.verdict": [], }, } ], @@ -257,6 +415,24 @@ def score(self, name: str) -> AggregateScore: """ return self.scores.score(name) + def task_outcomes(self) -> list[PerTaskOutcomes]: + """:attr:`task_metric_values` as models that name their own keys, sorted by task then metric. + + A read-time *view*, not the wire format. The field itself stays a nested dict because it is + persisted per run: repeating "task_id"/"metric_name" on every row would grow ``summary.json`` + for no new information, and lookup by task and metric stays O(1). Reach for this when you + want a typed object to pass around or to hand to a template. + """ + return [ + PerTaskOutcomes( + task_id=task_id, + outcomes=[ + PerTaskOutcome(metric_name=key, trials=list(records)) for key, records in sorted(by_key.items()) + ], + ) + for task_id, by_key in sorted(self.task_metric_values.items()) + ] + @staticmethod def from_scores( scores: Sequence[AgentEvalTaskScore], @@ -270,16 +446,16 @@ def from_scores( ``runner..``), merged in so a backend's own figures are addressable the same way as ours. """ task_list = list(tasks) if tasks is not None else None - task_metric_attempts = _task_metric_attempts(scores, task_list) + task_metric_values = _task_metric_values(scores, task_list) return AgentEvalSummary( scores=_aggregate_scores( scores, task_list, extra_scores, - task_metric_attempts=task_metric_attempts, + task_metric_values=task_metric_values, ), metric_coverage=_metric_coverage(scores, task_list), - task_metric_attempts=task_metric_attempts, + task_metric_values=task_metric_values, task_count=len(task_list) if task_list is not None else len({score.task_id for score in scores}), trial_count=len({score.trial_id for score in scores}), score_count=len(scores), @@ -599,8 +775,8 @@ def _format_score_errors( ) -> list[str]: """Render the failed-score detail section, separating a failed trial from a failed metric. - Both arrive as ``FAILED``, but they mean different things to a reader: a failed trial is an - attempt the agent is answerable for, a failed metric is a measurement that never happened. The + Both arrive as ``FAILED``, but they mean different things to a reader: a failed trial is one + the agent is answerable for, a failed metric is a measurement that never happened. The dataset path has no equivalent distinction to make, so this section is agent-eval's own rather than a reuse of :func:`format_error_details`. """ @@ -628,7 +804,7 @@ def _aggregate_scores( tasks: Sequence[AgentEvalTask] | None, extra_scores: Sequence[AggregateScore] = (), *, - task_metric_attempts: dict[str, dict[str, list[AgentEvalAttemptValue]]] | None = None, + task_metric_values: dict[str, TrialValuesByMetric], ) -> AggregatedMetricResult: """Aggregate per-metric-output, per-semantic-view, and task-level pass@k values into range scores. @@ -661,29 +837,55 @@ def _aggregate_scores( for view_name, (values, total) in sorted(_semantic_view_values(scores, tasks).items()): aggregated.append(_aggregate_range_score(f"view.{view_name}", values, total)) - # if the caller already passed attempts → use them (no second scan of all scores) - # if not (None) → compute them inside _aggregate_scores (no need to pass them in) - attempts = task_metric_attempts if task_metric_attempts is not None else _task_metric_attempts(scores, tasks) - aggregated.extend(_task_pass_at_k_scores(attempts, tasks)) + # Required rather than recomputed here: the summary needs the same mapping, and deriving it + # twice is what this rewiring exists to stop. The one caller builds it once and shares it. + aggregated.extend(_task_pass_at_k_scores(task_metric_values, tasks)) aggregated.extend(extra_scores) return AggregatedMetricResult(scores=aggregated) -def attempt_values(attempts: Sequence[AgentEvalAttemptValue]) -> list[float | None]: - """The bare per-attempt values, for consumers scoring attempts without caring which trial made them. +def metric_values(records: Sequence[TrialMetricValue]) -> list[float | int | bool | str | None]: + """The bare per-trial values, for consumers reading records without caring which trial made them. + + Preserves order, cardinality, type, and the None-versus-absent distinction exactly as recorded. + Reach for :func:`numeric_metric_values` before doing arithmetic: this list may hold labels, and + ``value >= 1.0`` raises on one. + """ + return [record.value for record in records] + + +def numeric_metric_values(records: Sequence[TrialMetricValue]) -> list[float | None]: + """The values that can be compared and averaged, as floats, for consumers doing arithmetic. + + A number becomes a float (a bool becomes 1.0/0.0, matching how a pass/fail flag has always been + read). A dead trial stays ``None`` -- it is a trial that definitively did not pass, and + dropping it would let a crashed rollout flatter the agent. + + A label is **dropped**, not zeroed. A categorical verdict says nothing about whether the agent + solved the task, so it is an unmeasured trial rather than a failed one -- the same reading this + module gives a metric that raised (see :func:`is_trial_failure`). Charging it as a failure would + misattribute a measurement problem to the agent, and counting it as a pass is not defined. - Preserves order, cardinality, and the None-versus-absent distinction exactly as recorded, so - anything counting attempts (pass@k above all) reads the same sequence it would have read before - attempts carried a trial id. + A label can land under a score-like key: :func:`validate_metric_result` coerces and discards, so + a metric declaring a continuous score may still return the string ``"0.9"``, and an output one + task never declared may be score-like on another. This is where that stops being arithmetic. """ - return [attempt.value for attempt in attempts] + values: list[float | None] = [] + for record in records: + value = record.value + if value is None: + values.append(None) + elif isinstance(value, bool | int | float): + # bool first: it is a subclass of int, and False must become 0.0 rather than be dropped. + values.append(float(value)) + return values def _pass_at_k(n: int, c: int, k: int) -> float: """Unbiased pass@k estimator (Chen et al., 2021): ``1 - C(n-c, k) / C(n, k)``. - The probability that at least one of ``k`` samples drawn without replacement from ``n`` attempts + The probability that at least one of ``k`` samples drawn without replacement from ``n`` trials (``c`` of them passing) is a pass. Caller guarantees ``1 <= k <= n``. """ if n - c < k: @@ -697,7 +899,7 @@ def _pass_at_k(n: int, c: int, k: int) -> float: def _scorelike_outputs(tasks: Sequence[AgentEvalTask] | None) -> set[tuple[str, str]]: """``(metric_type, output_name)`` pairs whose declared value is a score (continuous or boolean). - pass@k is only meaningful for a per-attempt pass/fail signal, so labels, discrete/count outputs, + pass@k is only meaningful for a per-trial pass/fail signal, so labels, discrete/count outputs, and free models (e.g. token measurements) are excluded. Needs task metric specs; with no tasks the set is empty and pass@k is skipped. """ @@ -713,11 +915,11 @@ def _scorelike_outputs(tasks: Sequence[AgentEvalTask] | None) -> set[tuple[str, return scorelike -def _task_metric_attempts( +def _task_metric_values( scores: Sequence[AgentEvalTaskScore], tasks: Sequence[AgentEvalTask] | None, -) -> dict[str, dict[str, list[AgentEvalAttemptValue]]]: - """Ordered per-attempt records per task, keyed ``.``. +) -> dict[str, TrialValuesByMetric]: + """Ordered per-trial records per task, keyed ``.``. ``task-a`` declares ``reward.score`` (continuous), ``steps.count`` (discrete) and ``usage.prompt_tokens`` (a free model) and runs four trials:: @@ -728,40 +930,42 @@ def _task_metric_attempts( t3 reward 0.0 steps 7 usage 1100 out {"task-a": {"reward.score": [(t0, 1.0), (t2, None), (t3, 0.0)], - "steps.count": [(t0, 5.0), (t1, 9.0), (t2, None), (t3, 7.0)]}} + "steps.count": [(t0, 5), (t1, 9), (t2, None), (t3, 7)]}} - (shown as ``(trial_id, value)`` pairs; each is an :class:`AgentEvalAttemptValue`) + (shown as ``(trial_id, value)`` pairs; each is an :class:`TrialMetricValue`) ``usage.prompt_tokens`` is absent because its declared schema is not in :data:`_TASK_METRIC_VALUE_SCHEMAS`; t1 is missing from ``reward.score`` but present in - ``steps.count``; t2 is ``None`` in both. + ``steps.count``; t2 is ``None`` in both. ``steps.count`` keeps its ints -- values are recorded in + the type the metric produced them in, not flattened to float. Which keys a task gets: - declared by its metric spec under :data:`_TASK_METRIC_VALUE_SCHEMAS` -> kept - declared under any other schema -> dropped, even when the emitted value is numeric, so a ``MetricOutputSpec.model("prompt_tokens", TokenCount)`` measurement never becomes a key - - undeclared, but some score emitted a numeric value for it -> kept - - ``tasks is None`` -> no specs to filter against, so every numeric output observed is kept + - undeclared, but some score emitted a recordable value for it -> kept + - ``tasks is None`` -> no specs to filter against, so every recordable output observed is kept What each score contributes to its key, in trial order: - - failed trial (:func:`is_trial_failure`) -> value ``None``, an attempt that did not pass - - failed metric, or the output absent -> no entry; the attempt is unmeasured, not unsuccessful - - otherwise -> the numeric value + - failed trial (:func:`is_trial_failure`) -> value ``None``, a trial that did not pass + - failed metric, or the output absent -> no entry; the trial is unmeasured, not unsuccessful + - a value a metric can emit (number, bool or label) -> that value, in its own type + - anything else (a dict, a list, a literal null) -> no entry; see :func:`_native_value` pass@k needs that asymmetry, and it is why a list is indexed by surviving measurement rather than - by attempt: above, index 1 is t2 under ``reward.score`` but t1 under ``steps.count``. Every entry + by trial: above, index 1 is t2 under ``reward.score`` but t1 under ``steps.count``. Every entry therefore names its trial, and ``trial_id`` — not position — is what joins two keys of one task, or joins out to ``trials.jsonl`` and ``scores.jsonl``. Ids are recorded as the runner reported - them and are never deduplicated: two attempts sharing an id stay two attempts, so a runner that + them and are never deduplicated: two records sharing an id stay two records, so a runner that reuses one costs pass@k nothing. """ output_keys: dict[str, set[tuple[str, str]]] = {} - # Per task, the outputs it declared under a schema this mapping does not retain. Tracked so an - # emitted numeric value cannot add back what that task's spec filter just excluded -- and keyed by - # task because tasks in one run need not declare the same output under the same schema. - excluded: dict[str, set[tuple[str, str]]] = {} + # Outputs a task declared under a schema this mapping does not retain. Tracked so an emitted + # numeric value cannot add back what that task's spec filter just excluded, and carrying the task + # id because tasks in one run need not declare the same output under the same schema. + excluded: set[tuple[str, str, str]] = set() if tasks is not None: for task in tasks: task_keys = output_keys.setdefault(task.id, set()) @@ -771,62 +975,83 @@ def _task_metric_attempts( if issubclass(spec.value_schema, _TASK_METRIC_VALUE_SCHEMAS): task_keys.add((metric_type, spec.name)) else: - excluded.setdefault(task.id, set()).add((metric_type, spec.name)) - - scores_by_task_metric: dict[tuple[str, str], list[AgentEvalTaskScore]] = {} - for score in scores: - scores_by_task_metric.setdefault((score.task_id, score.metric_type), []).append(score) + excluded.add((task.id, metric_type, spec.name)) + + # Materialized once: the key set has to be settled before any record can be filed (a trial + # failure reaches every key of its metric, including keys only a later score reveals), and + # `scores` is walked exactly once so a one-shot sequence still works. + ordered = list(scores) + for score in ordered: + # setdefault, not add: a task whose every score failed still earns an entry, so it reads as + # measured-and-empty rather than absent. task_keys = output_keys.setdefault(score.task_id, set()) + if score.status in (AgentEvalScoreStatus.COMPLETED, AgentEvalScoreStatus.PARTIAL): + for output in score.outputs: + if (score.task_id, score.metric_type, output.name) in excluded: + continue + if _native_value(output) is not None: + task_keys.add((score.metric_type, output.name)) + + # Key set settled, so the records fill in score order -- which is what puts each key's list in + # trial order. + outputs_by_task_metric: dict[tuple[str, str], list[str]] = {} + by_task: dict[str, TrialValuesByMetric] = {} + for task_id, keys in output_keys.items(): + ordered_keys = sorted(keys) + by_task[task_id] = {f"{metric_type}.{name}": [] for metric_type, name in ordered_keys} + for metric_type, name in ordered_keys: + outputs_by_task_metric.setdefault((task_id, metric_type), []).append(name) + + for score in ordered: + output_names = outputs_by_task_metric.get((score.task_id, score.metric_type)) + if not output_names: + continue + task_values = by_task[score.task_id] + if is_trial_failure(score): + for name in output_names: + task_values[f"{score.metric_type}.{name}"].append( + TrialMetricValue(trial_id=score.trial_id, value_type=TrialMetricValueType.MISSING, value=None) + ) + continue if score.status not in (AgentEvalScoreStatus.COMPLETED, AgentEvalScoreStatus.PARTIAL): continue - task_excluded = excluded.get(score.task_id, frozenset()) + # Indexed once per score rather than rescanned per output, and first-wins on a duplicate name + # to match :func:`_score_output`. + outputs: dict[str, MetricOutput] = {} for output in score.outputs: - if (score.metric_type, output.name) in task_excluded: - continue - if _semantic_value(output) is not None: - task_keys.add((score.metric_type, output.name)) - - by_task: dict[str, dict[str, list[AgentEvalAttemptValue]]] = {} - for task_id, keys in output_keys.items(): - task_values: dict[str, list[AgentEvalAttemptValue]] = {} - for metric_type, output_name in sorted(keys): - values: list[AgentEvalAttemptValue] = [] - for score in scores_by_task_metric.get((task_id, metric_type), []): - if is_trial_failure(score): - values.append(AgentEvalAttemptValue(trial_id=score.trial_id, value=None)) - continue - if score.status not in (AgentEvalScoreStatus.COMPLETED, AgentEvalScoreStatus.PARTIAL): - continue - output = _score_output(score, output_name) - value = _semantic_value(output) if output is not None else None - if value is not None: - values.append(AgentEvalAttemptValue(trial_id=score.trial_id, value=value)) - task_values[f"{metric_type}.{output_name}"] = values - by_task[task_id] = task_values + outputs.setdefault(output.name, output) + for name in output_names: + output = outputs.get(name) + payload = _native_value(output) if output is not None else None + if payload is not None: + value_type, value = payload + task_values[f"{score.metric_type}.{name}"].append( + TrialMetricValue(trial_id=score.trial_id, value_type=value_type, value=value) + ) return by_task def _task_pass_at_k_scores( - task_metric_attempts: dict[str, dict[str, list[AgentEvalAttemptValue]]], + task_metric_values: dict[str, TrialValuesByMetric], tasks: Sequence[AgentEvalTask] | None, ) -> list[AggregateScore]: """Task-level pass@k over the R trials per task, aggregated across tasks (uniform for any runner). - For each score-like metric output, group trials by task, count attempts ``n`` and passes ``c`` + For each score-like metric output, group trials by task, count trials ``n`` and passes ``c`` (value ``>= _PASS_VALUE``), then emit ``..pass@k`` for ``k`` in ``1..max(n)`` as - the across-task mean of the unbiased per-task estimator (over tasks with at least ``k`` attempts). + the across-task mean of the unbiased per-task estimator (over tasks with at least ``k`` trials). ``pass@1`` equals the macro per-task pass rate, i.e. the task-level mean. - **A failed trial is a failed attempt.** It counts toward ``n`` and never toward ``c``: an agent that + **A failed trial did not pass.** It counts toward ``n`` and never toward ``c``: an agent that solved a task once and crashed once did not go one-for-one. A failed *metric* is different — it - leaves the attempt unmeasured rather than unsuccessful, so it stays out of ``n`` entirely rather - than being charged to the agent (see :func:`is_trial_failure`). Tasks left with no usable attempt at + leaves the trial unmeasured rather than unsuccessful, so it stays out of ``n`` entirely rather + than being charged to the agent (see :func:`is_trial_failure`). Tasks left with no usable value at all drop out of the estimate and are reported as ``nan_count``, uniform across ``k``, so a shrinking denominator is never silent. (Tasks excluded from a given ``k`` merely for having fewer than ``k`` - attempts are *not* counted there — that is the estimator working as defined, not missing data.) + trials are *not* counted there — that is the estimator working as defined, not missing data.) - "No usable attempt" includes a task that was never scored at all: it declares the metric, holds an - empty attempt list, and lands in ``nan_count`` like any other unmeasured task. That is the same + "No usable value" includes a task that was never scored at all: it declares the metric, holds an + empty value list, and lands in ``nan_count`` like any other unmeasured task. That is the same missing coverage whether the trial died or was never produced, and excluding it would report pass@k over a denominator quietly smaller than the task set asked for. @@ -841,24 +1066,23 @@ def _task_pass_at_k_scores( aggregated: list[AggregateScore] = [] for metric_type, output_name in sorted(scorelike): key = f"{metric_type}.{output_name}" - values_by_task = [attempt_values(outputs[key]) for outputs in task_metric_attempts.values() if key in outputs] + values_by_task = [ + numeric_metric_values(outputs[key]) for outputs in task_metric_values.values() if key in outputs + ] measured = [values for values in values_by_task if values] if not measured: continue - # Empty attempt lists stay in nan_count (via total); for each k, mean the unbiased + # Empty value lists stay in nan_count (via total); for each k, mean the unbiased # estimator over tasks with n >= k (None / < full credit do not count as passes). unmeasured = sum(not values for values in values_by_task) - max_n = max(len(values) for values in measured) + # (n, c) per task, counted once: neither depends on k, so counting inside the k loop would + # re-walk every task's values max_n times over. + counts = [ + (len(values), sum(value is not None and value >= _PASS_VALUE for value in values)) for values in measured + ] + max_n = max(n for n, _ in counts) for k in range(1, max_n + 1): - per_task = [ - _pass_at_k( - len(values), - sum(value is not None and value >= _PASS_VALUE for value in values), - k, - ) - for values in measured - if len(values) >= k - ] + per_task = [_pass_at_k(n, c, k) for n, c in counts if n >= k] if per_task: aggregated.append(_aggregate_range_score(f"{key}.pass@{k}", per_task, len(per_task) + unmeasured)) return aggregated @@ -1042,15 +1266,42 @@ def _numeric_value(output: MetricOutput) -> float | None: return None -def _semantic_value(output: MetricOutput) -> float | None: +def _native_value(output: MetricOutput) -> tuple[TrialMetricValueType, float | int | bool | str] | None: + """The payload for one metric output, in the type the metric produced it in. + + The *preserving* counterpart to :func:`_semantic_value`, which projects to a float because its + callers (aggregate stats, semantic views) do arithmetic. A :class:`TrialMetricValue` is not + arithmetic: it is the per-trial evidence a reader looks up by ``trial_id`` in ``result.trials`` + or ``trials.jsonl``, so a count stays an int, a flag stays a bool, and a judge's verdict stays + the string it was. + + Returns ``None`` -- "nothing a trial can record" -- rather than a value, so an output holding + a dict, a list, or a literal null stays *absent* from the value list. That is not the same as + the ``None`` a dead trial records, and conflating the two would charge pass@k a trial the + agent never made. + """ value = output.value - if isinstance(value, bool): - return 1.0 if value else 0.0 if isinstance(value, BaseModel): - root = getattr(value, "root", None) - if isinstance(root, bool): - return 1.0 if root else 0.0 - return _numeric_value(output) + value = getattr(value, "root", None) + # bool first: it is a subclass of int, and it is a pass/fail signal rather than a measurement. + if isinstance(value, bool | int | float): + return (TrialMetricValueType.NUMBER, value) + if isinstance(value, str): + return (TrialMetricValueType.LABEL, value) + return None + + +def _semantic_value(output: MetricOutput) -> float | None: + """:func:`_native_value` projected to a float, for the callers that do arithmetic. + + The two answer different questions -- preserve versus interpret -- but they must agree on what a + metric value *is*, so the RootModel unwrap and the "what counts as numeric" rule live in + :func:`_native_value` alone. A label projects to ``None``: a view or aggregate cannot average it. + """ + payload = _native_value(output) + if payload is None or payload[0] is not TrialMetricValueType.NUMBER: + return None + return float(payload[1]) def mean_numeric(values: list[float]) -> float | None: diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/scores.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/scores.py index 75313bd994..d4014a35cc 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/scores.py +++ b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/scores.py @@ -23,7 +23,7 @@ class AgentEvalScoreStatus(str, Enum): #: Diagnostic detail key stamped when a score is ``FAILED`` because the *trial* failed — the agent #: produced nothing to score — rather than because the metric itself raised. Both are reported as -#: ``FAILED``, but they mean different things to a reader: a failed trial is a failed *attempt*, a +#: ``FAILED``, but they mean different things to a reader: a failed trial did not pass, a #: failed metric is a failed *measurement*. Consumers that must tell them apart read this key via #: :func:`is_trial_failure`. TRIAL_STATUS_DETAIL = "trial_status" From df66be0118a873fa2fd100a6bc4fc12c6155675b Mon Sep 17 00:00:00 2001 From: Nick Goncharenko Date: Wed, 12 Aug 2026 21:00:34 -0700 Subject: [PATCH 06/10] test: fix sandbox test Signed-off-by: Nick Goncharenko --- .../test_sandbox_compose_provider_live.py | 38 +++++++++++-------- 1 file changed, 23 insertions(+), 15 deletions(-) diff --git a/packages/nemo_evaluator_sdk/tests/agent_eval/test_sandbox_compose_provider_live.py b/packages/nemo_evaluator_sdk/tests/agent_eval/test_sandbox_compose_provider_live.py index 2f527e54f6..a204c172f3 100644 --- a/packages/nemo_evaluator_sdk/tests/agent_eval/test_sandbox_compose_provider_live.py +++ b/packages/nemo_evaluator_sdk/tests/agent_eval/test_sandbox_compose_provider_live.py @@ -17,6 +17,11 @@ DockerComposeSandboxProvider, ) +# Keep-alive containers ignore SIGTERM; use a 1s stop grace so teardown does not wait +# the provider default of 30s on every create/close cycle. +_SHUTDOWN_TIMEOUT_SECONDS = 1.0 +_KEEPALIVE = 'command: ["sleep", "300"]' + def _docker_ready() -> bool: if shutil.which("docker") is None: @@ -71,7 +76,7 @@ async def test_no_build_runs_prebuilt_image_without_source_context(tmp_path: Pat " agent:", f" image: {image}", " build: ./missing-context", - ' command: ["sh", "-c", "sleep 300"]', + f" {_KEEPALIVE}", ] ) + "\n", @@ -82,6 +87,7 @@ async def test_no_build_runs_prebuilt_image_without_source_context(tmp_path: Pat service_topology=_topology("agent"), pull_policy="never", startup_timeout_seconds=60, + shutdown_timeout_seconds=_SHUTDOWN_TIMEOUT_SECONDS, ) try: handle = await provider.create(SandboxSpec()) @@ -184,36 +190,37 @@ async def test_build_mode_rebuilds_changed_provisioned_workspace(tmp_path: Path) " agent:", f" image: {image}", " build: ./source", - ' command: ["sh", "-c", "sleep 300"]', + f" {_KEEPALIVE}", ] ) + "\n", encoding="utf-8", ) + provider = DockerComposeSandboxProvider( + compose_files=(compose_file,), + service_topology=_topology("agent"), + build=True, + pull_policy="never", + startup_timeout_seconds=60, + shutdown_timeout_seconds=_SHUTDOWN_TIMEOUT_SECONDS, + ) async def evaluate(value: str) -> str: (context / "value.txt").write_text(value, encoding="utf-8") - provider = DockerComposeSandboxProvider( - compose_files=(compose_file,), - service_topology=_topology("agent"), - build=True, - pull_policy="never", - startup_timeout_seconds=60, - ) + handle = await provider.create(SandboxSpec()) try: - handle = await provider.create(SandboxSpec()) result = await provider.exec(handle, "cat /value.txt") - await provider.close(handle) assert result.ok assert result.stdout is not None return result.stdout.strip() finally: - await provider.aclose() + await provider.close(handle) try: assert await evaluate("candidate-one") == "candidate-one" assert await evaluate("candidate-two") == "candidate-two" finally: + await provider.aclose() subprocess.run(["docker", "image", "rm", "--force", image], capture_output=True, timeout=30) @@ -226,7 +233,7 @@ async def test_ordered_override_and_profile_activate_expected_topology(tmp_path: "services:", " agent:", " image: busybox:latest", - ' command: ["sh", "-c", "sleep 300"]', + f" {_KEEPALIVE}", ] ) + "\n", @@ -239,7 +246,7 @@ async def test_ordered_override_and_profile_activate_expected_topology(tmp_path: " worker:", " image: busybox:latest", " profiles: [extra]", - ' command: ["sh", "-c", "sleep 300"]', + f" {_KEEPALIVE}", ] ) + "\n", @@ -249,8 +256,9 @@ async def test_ordered_override_and_profile_activate_expected_topology(tmp_path: compose_files=(base, override), service_topology=_topology("agent", "worker"), profiles=("extra",), - pull_policy="missing", + pull_policy="never", startup_timeout_seconds=60, + shutdown_timeout_seconds=_SHUTDOWN_TIMEOUT_SECONDS, ) try: handle = await provider.create(SandboxSpec()) From 6c6bbe3de4fb00968c6b9c841775ef47c07eb99a Mon Sep 17 00:00:00 2001 From: Nick Goncharenko Date: Wed, 12 Aug 2026 21:34:52 -0700 Subject: [PATCH 07/10] test: fix sandbox Signed-off-by: Nick Goncharenko --- .../examples/gym/inspect_results.py | 2 +- .../examples/gym/run_gym_eval.py | 15 +++++++++++++++ .../test_sandbox_compose_provider_live.py | 2 +- 3 files changed, 17 insertions(+), 2 deletions(-) diff --git a/packages/nemo_evaluator_sdk/examples/gym/inspect_results.py b/packages/nemo_evaluator_sdk/examples/gym/inspect_results.py index f1cad972b0..4f4f35f859 100644 --- a/packages/nemo_evaluator_sdk/examples/gym/inspect_results.py +++ b/packages/nemo_evaluator_sdk/examples/gym/inspect_results.py @@ -128,7 +128,7 @@ def load_bundle(bundle: Path) -> AgentEvalSummary: try: payload = json.loads(summary_path.read_text(encoding="utf-8")) except json.JSONDecodeError as exc: - raise BundleFormatError(f"{summary_path} is not readable JSON): {exc}") from exc + raise BundleFormatError(f"{summary_path} is not readable JSON: {exc}") from exc # Checked explicitly because `model_validate` would *not* catch this: the field defaults to an # empty dict, so an older bundle loads cleanly and simply shows no per-task section. if "task_metric_values" not in payload: diff --git a/packages/nemo_evaluator_sdk/examples/gym/run_gym_eval.py b/packages/nemo_evaluator_sdk/examples/gym/run_gym_eval.py index af12a94b1a..3fa81ea4ca 100644 --- a/packages/nemo_evaluator_sdk/examples/gym/run_gym_eval.py +++ b/packages/nemo_evaluator_sdk/examples/gym/run_gym_eval.py @@ -46,6 +46,14 @@ from nemo_evaluator_sdk.agent_eval.tasks import AgentEvalRunConfig +def _non_negative_int(raw: str) -> int: + """argparse type for ``--limit``: a negative value would silently slice tasks off the END.""" + value = int(raw) + if value < 0: + raise argparse.ArgumentTypeError(f"must be non-negative, got {value}") + return value + + def _parse_args(argv: list[str] | None = None) -> argparse.Namespace: parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) parser.add_argument( @@ -68,6 +76,9 @@ def _parse_args(argv: list[str] | None = None) -> argparse.Namespace: help="`inference_provider` speaks OpenAI-compatible chat; `openai_model` uses the Responses API.", ) parser.add_argument("--num-repeats", type=int, default=2, help="Attempts per task (each becomes a trial).") + parser.add_argument( + "--limit", type=_non_negative_int, default=None, help="Run only the first N tasks (handy for smoke runs)." + ) parser.add_argument( "--output-dir", type=Path, @@ -115,6 +126,10 @@ async def _main(args: argparse.Namespace) -> int: dataset = args.dataset or _packaged_dataset(args.resources_server) tasks = discover_gym_tasks(dataset) print(f"discovered {len(tasks)} tasks from {dataset}") + if args.limit is not None: + # The runner materializes Gym's input from these tasks, so this bounds the rollout too. + tasks = tasks[: args.limit] + print(f"limited to {len(tasks)} task(s)") runner = GymAgentTaskRunner( config=GymRuntimeConfig( diff --git a/packages/nemo_evaluator_sdk/tests/agent_eval/test_sandbox_compose_provider_live.py b/packages/nemo_evaluator_sdk/tests/agent_eval/test_sandbox_compose_provider_live.py index a204c172f3..5fdb68dc48 100644 --- a/packages/nemo_evaluator_sdk/tests/agent_eval/test_sandbox_compose_provider_live.py +++ b/packages/nemo_evaluator_sdk/tests/agent_eval/test_sandbox_compose_provider_live.py @@ -256,7 +256,7 @@ async def test_ordered_override_and_profile_activate_expected_topology(tmp_path: compose_files=(base, override), service_topology=_topology("agent", "worker"), profiles=("extra",), - pull_policy="never", + pull_policy="missing", startup_timeout_seconds=60, shutdown_timeout_seconds=_SHUTDOWN_TIMEOUT_SECONDS, ) From 995cef309bf14ba215511f64faad7975977faa2f Mon Sep 17 00:00:00 2001 From: Nick Goncharenko Date: Thu, 13 Aug 2026 09:31:48 -0700 Subject: [PATCH 08/10] chore: update the return type Signed-off-by: Nick Goncharenko --- docs/evaluator/agent-eval/reading-results.mdx | 19 ++- .../nemo_evaluator_sdk/examples/gym/README.md | 8 +- .../examples/gym/inspect_results.py | 110 +++++++----------- .../nemo_evaluator_sdk/agent_eval/results.py | 31 ++++- .../agent_eval/test_task_metric_values.py | 62 +++++++--- .../beta/evaluator/agent_eval/results.py | 31 ++++- 6 files changed, 164 insertions(+), 97 deletions(-) diff --git a/docs/evaluator/agent-eval/reading-results.mdx b/docs/evaluator/agent-eval/reading-results.mdx index c5a014dc3a..ffd75dbd99 100644 --- a/docs/evaluator/agent-eval/reading-results.mdx +++ b/docs/evaluator/agent-eval/reading-results.mdx @@ -66,8 +66,23 @@ result = await AgentEvaluator().run(tasks=..., target=...) scores = numeric_metric_values(records) # [1.0, None, 0.0] ``` - For a typed object rather than nested dicts, `result.summary.task_outcomes()` returns - `list[PerTaskOutcomes]`, each naming its `task_id` and its outcomes' `metric_name`. +- **`summary.task_outcomes(metric_name=None)`** — the same data as models rather than nested dicts, + sorted by task then metric, each naming its own `task_id` and `metric_name`. Pass a + `"."` to narrow to one metric, which is what a report over a single metric + wants: + + ```python + for per_task in result.summary.task_outcomes("reward.score"): + for outcome in per_task.outcomes: + values = numeric_metric_values(outcome.trials) + print(per_task.task_id, outcome.metric_name, values) + ``` + + - When you narrow, a task the metric never measured is **dropped** — it was scored by a different + metric, so listing it would invent missing coverage. + - A task that declared the metric but produced no usable value keeps its entry with an empty + `trials` list, because there the coverage really is missing. + - Unfiltered, every task is returned. - **`summary.task_count`**, **`summary.trial_count`**, **`summary.score_count`**. ### Per-metric scores diff --git a/packages/nemo_evaluator_sdk/examples/gym/README.md b/packages/nemo_evaluator_sdk/examples/gym/README.md index 3c37df7da1..b73f0b58f0 100644 --- a/packages/nemo_evaluator_sdk/examples/gym/README.md +++ b/packages/nemo_evaluator_sdk/examples/gym/README.md @@ -58,9 +58,11 @@ Run bundle (run.json, trials.jsonl, scores.jsonl, report.html): /var/folders/... ## Read the results `inspect_results.py` reads `summary.json` and shows each result layer: run aggregates from -`summary.scores`, ordered per-task values from `summary.task_metric_values`, and runner-owned -aggregates under `runner.gym.*`. Per-task keys use `.`; a `null` value is a trial -that failed before scoring, while an empty list means the metric produced no usable measurement. +`summary.scores`, ordered per-task values from `summary.task_outcomes(".")`, and +runner-owned aggregates under `runner.gym.*`. That accessor returns models rather than nested dicts — +each row names its own `task_id` and `metric_name` — so the example needs no per-task accessor of its +own. A `null` value is a trial that failed before scoring, while an empty `trials` list means the +metric produced no usable measurement; a task the metric never measured is not returned at all. Each record names the trial that produced it, so `trial_id` — not list position — is what joins two outputs of the same task, or looks up `trials.jsonl`. A trial whose metric failed is absent diff --git a/packages/nemo_evaluator_sdk/examples/gym/inspect_results.py b/packages/nemo_evaluator_sdk/examples/gym/inspect_results.py index 4f4f35f859..62ed18469f 100644 --- a/packages/nemo_evaluator_sdk/examples/gym/inspect_results.py +++ b/packages/nemo_evaluator_sdk/examples/gym/inspect_results.py @@ -7,10 +7,11 @@ how to get at each kind of result — headline aggregates, ``pass@k``, per-task outcomes, and the runner's own aggregations. -The helpers below (:func:`aggregate`, :func:`per_task_outcomes`) are written to be lifted directly -into your own code. Everything shown here also works on the in-memory ``AgentEvalResult`` returned by -``AgentEvaluator().run(...)`` — reading from a bundle just makes the example runnable without a live -run. +Per-task results are read through the SDK's own typed view, ``summary.task_outcomes(metric_name)``, +rather than by walking the nested ``summary.task_metric_values`` dict here — that is the accessor to +lift into your own code. Everything shown here also works on the in-memory ``AgentEvalResult`` +returned by ``AgentEvaluator().run(...)`` — reading from a bundle just makes the example runnable +without a live run. There is no bundle checked into the repo — ``run_gym_eval.py`` makes one. It writes to a fresh temporary directory by default, so give it an explicit ``--output-dir`` and point this script at the @@ -33,7 +34,7 @@ from nemo_evaluator_sdk.agent_eval.results import ( AgentEvalSummary, - TrialMetricValue, + PerTaskOutcomes, numeric_metric_values, ) from nemo_evaluator_sdk.values.results import AggregateScalarScore, AggregateScore @@ -67,47 +68,13 @@ def aggregate(summary: AgentEvalSummary, name: str) -> AggregateScore: raise KeyError(f"no aggregate named {name!r}; available: {available}") -def per_task_outcomes( - summary: AgentEvalSummary, - *, - metric_type: str, - output_name: str, -) -> dict[str, list[float | None]]: - """Read ordered trial values from the summary for one metric output. - - ``None`` is a failed trial and therefore did not pass. An empty list means the task had no - usable measurement — its metric failed, omitted the output, produced no trial at all, or scored - only non-numeric labels. - - Use :func:`per_task_trial_values` when you need to know *which* trial produced a value. - """ - return { - task_id: numeric_metric_values(records) - for task_id, records in per_task_trial_values(summary, metric_type=metric_type, output_name=output_name).items() - } - - -def per_task_trial_values( - summary: AgentEvalSummary, - *, - metric_type: str, - output_name: str, -) -> dict[str, list[TrialMetricValue]]: - """The same values, each still naming the trial that produced it. - - A trial whose metric failed is absent rather than null, so lists for two different outputs of - one task need not be the same length — ``trial_id``, not position, is what lines them up. It is - also the lookup key into ``trials.jsonl``, which is where a failed trial's error lives. - """ - key = f"{metric_type}.{output_name}" - return { - task_id: list(by_metric[key]) for task_id, by_metric in summary.task_metric_values.items() if key in by_metric - } - - -# The typed view over the same data lives in the SDK: `summary.task_outcomes()` returns -# `list[PerTaskOutcomes]`, each naming its task and metric. Use it when you want an object to pass -# around rather than a dict keyed by strings. +# Per-task values need no accessor here: `summary.task_outcomes(".")` returns +# `list[PerTaskOutcomes]` already sorted by task, each naming its own `task_id` and `metric_name`, +# and each trial naming the `trial_id` that produced it — so a value can be joined back to +# `trials.jsonl` (where a failed trial's error lives) or across to another metric's outcomes. +# Records are ordered by trial, and `trial_id` rather than position is what lines two outputs up: +# a trial whose metric failed is absent rather than null, so two lists for one task need not be the +# same length. # -------------------------------------------------------------------------------------------------- # Bundle loading (see the run.json manifest for the full artifact list). @@ -166,33 +133,41 @@ def show_aggregates(summary: AgentEvalSummary) -> None: print(f"\n {summary.task_count} tasks · {summary.trial_count} trials · {summary.score_count} scores") -def show_per_task(by_task: dict[str, list[float | None]]) -> None: +def show_per_task(outcomes: list[PerTaskOutcomes]) -> None: """Per-task outcomes: which tasks were solved, and how consistently. + Takes the SDK's typed view, so every row already knows its own task and metric and no dict has + to be re-keyed here. Already sorted by task, hence no ``sorted()``. + A trial passes on full credit (``>= PASS_VALUE``), matching how the SDK computes pass@k. A ``None`` is a trial that died: it counts toward ``n`` and never as a pass, so a task that passed once and crashed once reads as flaky rather than solved. """ print("\nPer-task outcomes (trial values; a trial passes at full credit)") solved = flaky = failed = unmeasured = 0 - for task_id, values in sorted(by_task.items()): - if not values: - verdict, marker = "unmeasured", "?" - unmeasured += 1 - shown = "" - else: - passes = sum(1 for value in values if value is not None and value >= PASS_VALUE) - if passes == len(values): - verdict, marker = "solved", "+" - solved += 1 - elif passes: - verdict, marker = f"flaky ({passes}/{len(values)})", "~" - flaky += 1 + for per_task in outcomes: + for outcome in per_task.outcomes: + # Projected to floats only here, where the work is genuinely arithmetic: `>=` and `:g` + # both raise on a judge's label, and numeric_metric_values drops those while keeping a + # dead trial's None. Everything above reads the records as they were recorded. + values = numeric_metric_values(outcome.trials) + if not values: + verdict, marker = "unmeasured", "?" + unmeasured += 1 + shown = "" else: - verdict, marker = "failed", "-" - failed += 1 - shown = ", ".join("died" if value is None else f"{value:g}" for value in values) - print(f" {marker} {task_id[:16]}… [{shown}] {verdict}") + passes = sum(1 for value in values if value is not None and value >= PASS_VALUE) + if passes == len(values): + verdict, marker = "solved", "+" + solved += 1 + elif passes: + verdict, marker = f"flaky ({passes}/{len(values)})", "~" + flaky += 1 + else: + verdict, marker = "failed", "-" + failed += 1 + shown = ", ".join("died" if value is None else f"{value:g}" for value in values) + print(f" {marker} {per_task.task_id[:16]}… [{shown}] {verdict}") print(f"\n {solved} solved · {flaky} flaky · {failed} failed · {unmeasured} unmeasured") @@ -243,9 +218,10 @@ def main(argv: list[str] | None = None) -> int: raise SystemExit(str(exc)) from exc show_aggregates(summary) - by_task = per_task_outcomes(summary, metric_type=args.metric_type, output_name=args.output_name) - if by_task: - show_per_task(by_task) + # Empty when no task was measured by this metric at all -- typically a wrong --metric-type. + outcomes = summary.task_outcomes(f"{args.metric_type}.{args.output_name}") + if outcomes: + show_per_task(outcomes) show_runner_aggregations(summary) print(f"\nFull report: {args.bundle / 'report.html'}") diff --git a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/results.py b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/results.py index 26f23694b3..9680c0636c 100644 --- a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/results.py +++ b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/results.py @@ -415,23 +415,42 @@ def score(self, name: str) -> AggregateScore: """ return self.scores.score(name) - def task_outcomes(self) -> list[PerTaskOutcomes]: + def task_outcomes(self, metric_name: str | None = None) -> list[PerTaskOutcomes]: """:attr:`task_metric_values` as models that name their own keys, sorted by task then metric. A read-time *view*, not the wire format. The field itself stays a nested dict because it is persisted per run: repeating "task_id"/"metric_name" on every row would grow ``summary.json`` for no new information, and lookup by task and metric stays O(1). Reach for this when you want a typed object to pass around or to hand to a template. + + ``metric_name`` narrows to one ``"."``, which is what a report over a + single metric wants:: + + summary.task_outcomes() -> every task, every metric output + summary.task_outcomes("gym_reward.reward") -> every task that metric measured + + A task the named metric never measured is **dropped**, not returned empty: it was scored by + a different metric, so reporting it as unmeasured would invent missing coverage. A task that + declared the metric but produced no usable value is different - it keeps its entry with an + empty ``trials`` list, because there the coverage really is missing. That is the same + distinction :attr:`task_metric_values` draws by having a key at all. """ - return [ - PerTaskOutcomes( - task_id=task_id, - outcomes=[ - PerTaskOutcome(metric_name=key, trials=list(records)) for key, records in sorted(by_key.items()) + outcomes_by_task = [ + ( + task_id, + [ + PerTaskOutcome(metric_name=key, trials=list(records)) + for key, records in sorted(by_key.items()) + if metric_name is None or key == metric_name ], ) for task_id, by_key in sorted(self.task_metric_values.items()) ] + return [ + PerTaskOutcomes(task_id=task_id, outcomes=outcomes) + for task_id, outcomes in outcomes_by_task + if metric_name is None or outcomes + ] @staticmethod def from_scores( diff --git a/packages/nemo_evaluator_sdk/tests/agent_eval/test_task_metric_values.py b/packages/nemo_evaluator_sdk/tests/agent_eval/test_task_metric_values.py index 4dfe6e5c56..f56d9f01c6 100644 --- a/packages/nemo_evaluator_sdk/tests/agent_eval/test_task_metric_values.py +++ b/packages/nemo_evaluator_sdk/tests/agent_eval/test_task_metric_values.py @@ -631,26 +631,62 @@ def test_summary_task_outcomes_name_their_own_keys() -> None: assert [(t.trial_id, t.value) for t in outcomes[0].outcomes[0].trials] == [("task-a__aaa", 1.0)] -def test_gym_example_reads_task_outcomes_from_summary() -> None: - from packages.nemo_evaluator_sdk.examples.gym.inspect_results import per_task_outcomes, per_task_trial_values - - summary = AgentEvalSummary( +def _mixed_metric_summary() -> AgentEvalSummary: + """One task measured by the metric, one that declared it but yielded nothing, one scored by another.""" + return AgentEvalSummary( task_metric_values={ "task-a": { "gym_reward.reward": [ TrialMetricValue(trial_id="task-a__aaa", value=1.0), TrialMetricValue(trial_id="task-a__bbb", value=0.0), - ] + ], + "steps.count": [TrialMetricValue(trial_id="task-a__aaa", value=7)], }, "task-b": {"gym_reward.reward": []}, + "task-c": {"judge.rating": [TrialMetricValue(trial_id="task-c__ccc", value=0.5)]}, } ) - # The example's headline accessor keeps its bare-value shape... - assert per_task_outcomes(summary, metric_type="gym_reward", output_name="reward") == { - "task-a": [1.0, 0.0], - "task-b": [], - } - # ...and its sibling exposes the identity that makes a value traceable back to a rollout. - records = per_task_trial_values(summary, metric_type="gym_reward", output_name="reward") - assert [(a.trial_id, a.value) for a in records["task-a"]] == [("task-a__aaa", 1.0), ("task-a__bbb", 0.0)] + +def test_task_outcomes_narrows_to_one_metric() -> None: + # The filter is what makes the typed view usable for a single-metric report; without it a caller + # has to re-filter the nested lists by hand, which is what the dict shape already made them do. + outcomes = _mixed_metric_summary().task_outcomes("gym_reward.reward") + + assert [o.task_id for o in outcomes] == ["task-a", "task-b"] + assert [[oc.metric_name for oc in o.outcomes] for o in outcomes] == [["gym_reward.reward"], ["gym_reward.reward"]] + assert [(t.trial_id, t.value) for t in outcomes[0].outcomes[0].trials] == [ + ("task-a__aaa", 1.0), + ("task-a__bbb", 0.0), + ] + + +def test_task_outcomes_drops_a_task_the_metric_never_measured_but_keeps_an_empty_one() -> None: + # The two states are not the same and must not be flattened together. task-b declared the metric + # and yielded no usable value -- that is missing coverage, so it stays with an empty trials list + # and reports as unmeasured. task-c was scored by a different metric entirely; reporting it would + # invent coverage the run never asked for. + outcomes = _mixed_metric_summary().task_outcomes("gym_reward.reward") + + by_task = {o.task_id: o for o in outcomes} + assert "task-c" not in by_task + assert by_task["task-b"].outcomes[0].trials == [] + + # Unfiltered, every task is present including the one scored by another metric. + assert [o.task_id for o in _mixed_metric_summary().task_outcomes()] == ["task-a", "task-b", "task-c"] + + +def test_gym_example_reads_task_outcomes_from_summary() -> None: + # The example reads the SDK's typed view rather than re-deriving one: each row names its own + # task and metric, and each trial the rollout that produced it. + from packages.nemo_evaluator_sdk.examples.gym.inspect_results import show_per_task + + outcomes = _mixed_metric_summary().task_outcomes("gym_reward.reward") + + assert [(o.task_id, o.outcomes[0].metric_name) for o in outcomes] == [ + ("task-a", "gym_reward.reward"), + ("task-b", "gym_reward.reward"), + ] + # The display path is what the projection to floats exists for; it must not raise on an empty + # outcome, which is the task that reports as unmeasured. + show_per_task(outcomes) diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/results.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/results.py index 562140e697..be4d14d6fa 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/results.py +++ b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/results.py @@ -415,23 +415,42 @@ def score(self, name: str) -> AggregateScore: """ return self.scores.score(name) - def task_outcomes(self) -> list[PerTaskOutcomes]: + def task_outcomes(self, metric_name: str | None = None) -> list[PerTaskOutcomes]: """:attr:`task_metric_values` as models that name their own keys, sorted by task then metric. A read-time *view*, not the wire format. The field itself stays a nested dict because it is persisted per run: repeating "task_id"/"metric_name" on every row would grow ``summary.json`` for no new information, and lookup by task and metric stays O(1). Reach for this when you want a typed object to pass around or to hand to a template. + + ``metric_name`` narrows to one ``"."``, which is what a report over a + single metric wants:: + + summary.task_outcomes() -> every task, every metric output + summary.task_outcomes("gym_reward.reward") -> every task that metric measured + + A task the named metric never measured is **dropped**, not returned empty: it was scored by + a different metric, so reporting it as unmeasured would invent missing coverage. A task that + declared the metric but produced no usable value is different -- it keeps its entry with an + empty ``trials`` list, because there the coverage really is missing. That is the same + distinction :attr:`task_metric_values` draws by having a key at all. """ - return [ - PerTaskOutcomes( - task_id=task_id, - outcomes=[ - PerTaskOutcome(metric_name=key, trials=list(records)) for key, records in sorted(by_key.items()) + outcomes_by_task = [ + ( + task_id, + [ + PerTaskOutcome(metric_name=key, trials=list(records)) + for key, records in sorted(by_key.items()) + if metric_name is None or key == metric_name ], ) for task_id, by_key in sorted(self.task_metric_values.items()) ] + return [ + PerTaskOutcomes(task_id=task_id, outcomes=outcomes) + for task_id, outcomes in outcomes_by_task + if metric_name is None or outcomes + ] @staticmethod def from_scores( From 1dbd7ff5c6a0037ed597fec62dc1afcfa0ac3ead Mon Sep 17 00:00:00 2001 From: Nick Goncharenko Date: Thu, 13 Aug 2026 10:01:04 -0700 Subject: [PATCH 09/10] chore: rm comment Signed-off-by: Nick Goncharenko --- .../nemo_evaluator_sdk/examples/gym/inspect_results.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/packages/nemo_evaluator_sdk/examples/gym/inspect_results.py b/packages/nemo_evaluator_sdk/examples/gym/inspect_results.py index 62ed18469f..53b69ef5f1 100644 --- a/packages/nemo_evaluator_sdk/examples/gym/inspect_results.py +++ b/packages/nemo_evaluator_sdk/examples/gym/inspect_results.py @@ -67,15 +67,6 @@ def aggregate(summary: AgentEvalSummary, name: str) -> AggregateScore: available = ", ".join(sorted(score.name for score in summary.scores.scores)) raise KeyError(f"no aggregate named {name!r}; available: {available}") - -# Per-task values need no accessor here: `summary.task_outcomes(".")` returns -# `list[PerTaskOutcomes]` already sorted by task, each naming its own `task_id` and `metric_name`, -# and each trial naming the `trial_id` that produced it — so a value can be joined back to -# `trials.jsonl` (where a failed trial's error lives) or across to another metric's outcomes. -# Records are ordered by trial, and `trial_id` rather than position is what lines two outputs up: -# a trial whose metric failed is absent rather than null, so two lists for one task need not be the -# same length. - # -------------------------------------------------------------------------------------------------- # Bundle loading (see the run.json manifest for the full artifact list). # -------------------------------------------------------------------------------------------------- From 78c6a485ff0f439130291eb626ea00fb7b4e5291 Mon Sep 17 00:00:00 2001 From: Nick Goncharenko Date: Thu, 13 Aug 2026 10:07:05 -0700 Subject: [PATCH 10/10] fix: improve error handling Signed-off-by: Nick Goncharenko --- .../examples/gym/inspect_results.py | 15 ++++++++-- .../agent_eval/test_task_metric_values.py | 30 +++++++++++++++++++ .../beta/evaluator/agent_eval/results.py | 2 +- 3 files changed, 44 insertions(+), 3 deletions(-) diff --git a/packages/nemo_evaluator_sdk/examples/gym/inspect_results.py b/packages/nemo_evaluator_sdk/examples/gym/inspect_results.py index 53b69ef5f1..2e4cecbcb9 100644 --- a/packages/nemo_evaluator_sdk/examples/gym/inspect_results.py +++ b/packages/nemo_evaluator_sdk/examples/gym/inspect_results.py @@ -38,6 +38,7 @@ numeric_metric_values, ) from nemo_evaluator_sdk.values.results import AggregateScalarScore, AggregateScore +from pydantic import ValidationError #: Value at which a trial counts as a pass, matching the SDK's pass@k definition (full credit). PASS_VALUE = 1.0 @@ -67,6 +68,7 @@ def aggregate(summary: AgentEvalSummary, name: str) -> AggregateScore: available = ", ".join(sorted(score.name for score in summary.scores.scores)) raise KeyError(f"no aggregate named {name!r}; available: {available}") + # -------------------------------------------------------------------------------------------------- # Bundle loading (see the run.json manifest for the full artifact list). # -------------------------------------------------------------------------------------------------- @@ -75,7 +77,9 @@ def aggregate(summary: AgentEvalSummary, name: str) -> AggregateScore: def load_bundle(bundle: Path) -> AgentEvalSummary: """Load the persisted summary, including native and runner aggregates and per-task values. - Rejects a bundle written before ``task_metric_values`` existed rather than reading one. The + Every way a bundle can be unreadable raises :class:`BundleFormatError` and nothing else, so + :func:`main` turns all of them into an exit code rather than a traceback. In particular it + rejects a bundle written before ``task_metric_values`` existed rather than reading one: the field defaults to empty, so an older bundle would otherwise load cleanly and simply show no per-task section — the reader would conclude the run had no per-task outcomes rather than that this script cannot see them. @@ -87,6 +91,10 @@ def load_bundle(bundle: Path) -> AgentEvalSummary: payload = json.loads(summary_path.read_text(encoding="utf-8")) except json.JSONDecodeError as exc: raise BundleFormatError(f"{summary_path} is not readable JSON: {exc}") from exc + # Before the membership test below, which raises TypeError on a root that is not a container and + # matches a substring on a bare JSON string. + if not isinstance(payload, dict): + raise BundleFormatError(f"{summary_path} is not a JSON object (found {type(payload).__name__}).") # Checked explicitly because `model_validate` would *not* catch this: the field defaults to an # empty dict, so an older bundle loads cleanly and simply shows no per-task section. if "task_metric_values" not in payload: @@ -94,7 +102,10 @@ def load_bundle(bundle: Path) -> AgentEvalSummary: f"{summary_path} predates summary.task_metric_values, which this script reads " "per-task outcomes from. Re-run the eval to produce a current bundle." ) - return AgentEvalSummary.model_validate(payload) + try: + return AgentEvalSummary.model_validate(payload) + except ValidationError as exc: + raise BundleFormatError(f"{summary_path} is not a valid run summary: {exc}") from exc # -------------------------------------------------------------------------------------------------- diff --git a/packages/nemo_evaluator_sdk/tests/agent_eval/test_task_metric_values.py b/packages/nemo_evaluator_sdk/tests/agent_eval/test_task_metric_values.py index f56d9f01c6..f738d7a6d7 100644 --- a/packages/nemo_evaluator_sdk/tests/agent_eval/test_task_metric_values.py +++ b/packages/nemo_evaluator_sdk/tests/agent_eval/test_task_metric_values.py @@ -611,6 +611,36 @@ def test_gym_example_rejects_a_bundle_written_before_task_metric_values(tmp_path load_bundle(tmp_path) +@pytest.mark.parametrize( + ("label", "raw"), + [ + # A root that is not a container: `"task_metric_values" not in 1` raises TypeError. + ("null root", "null"), + ("number root", "1"), + ("bool root", "true"), + # A root that *is* a container, so the membership test passes and validation is reached -- + # a bare string matches by substring, which is the sharpest edge of the three. + ("array root", '["task_metric_values"]'), + ("string root", '"task_metric_values"'), + # A well-formed object whose field types are wrong, or that carries an extra key + # (`AgentEvalSummary` is extra="forbid"). + ("wrong field type", '{"task_metric_values": []}'), + ("unknown field", '{"task_metric_values": {}, "bogus": 1}'), + ("bad nested record", '{"task_metric_values": {"t": {"m": [{"trial_id": 5, "value": 1.0}]}}}'), + ], +) +def test_gym_example_reports_a_malformed_bundle_as_a_bundle_format_error(tmp_path: Path, label: str, raw: str) -> None: + # main() turns BundleFormatError into an exit code and lets everything else become a traceback, + # so every unreadable shape has to arrive as that one type -- not TypeError from the membership + # test, and not a raw pydantic ValidationError. + from packages.nemo_evaluator_sdk.examples.gym.inspect_results import BundleFormatError, load_bundle + + (tmp_path / "summary.json").write_text(raw, encoding="utf-8") + + with pytest.raises(BundleFormatError): + load_bundle(tmp_path) + + def test_summary_task_outcomes_name_their_own_keys() -> None: # task_outcomes() lifts the nested dicts into models whose fields are named, for callers that # want a typed object to pass around. It is a read-time view: the summary keeps the dict shape. diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/results.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/results.py index be4d14d6fa..568ff27bee 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/results.py +++ b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/results.py @@ -431,7 +431,7 @@ def task_outcomes(self, metric_name: str | None = None) -> list[PerTaskOutcomes] A task the named metric never measured is **dropped**, not returned empty: it was scored by a different metric, so reporting it as unmeasured would invent missing coverage. A task that - declared the metric but produced no usable value is different -- it keeps its entry with an + declared the metric but produced no usable value is different - it keeps its entry with an empty ``trials`` list, because there the coverage really is missing. That is the same distinction :attr:`task_metric_values` draws by having a key at all. """