From 4a090ff10eb287377a341ce21e93ab4220a2c58d Mon Sep 17 00:00:00 2001 From: shanaiabuggy <59746633+shanaiabuggy@users.noreply.github.com> Date: Tue, 11 Aug 2026 12:02:56 -0600 Subject: [PATCH 1/5] Rename evaluation span-attribute bag key Signed-off-by: shanaiabuggy <59746633+shanaiabuggy@users.noreply.github.com> --- docs/evaluator/experiments.mdx | 5 ++-- .../skills/nemo-experiments-upload/SKILL.md | 8 +++---- .../references/troubleshooting.md | 4 ++-- .../nemo-intake/references/ingest-formats.md | 9 +++---- .../record_tau_airline_traces.py | 4 +++- .../experimentalist/experiment_mirror.py | 2 +- .../experimentalist_backend.py | 6 +++-- .../nemo-experimentalist/tests/test_otlp.py | 24 +++++++++---------- plugins/nemo-insights/testbed/README.md | 2 +- plugins/nemo-insights/testbed/adapters.py | 2 +- plugins/nemo-insights/testbed/otlp_build.py | 6 +++-- .../tests/testbed/test_adapters.py | 2 +- .../tests/testbed/test_otlp_build.py | 4 ++-- .../tests/testbed/test_reingest.py | 4 ++-- .../skills/nemo-experiments-upload/SKILL.md | 8 +++---- .../references/troubleshooting.md | 4 ++-- .../nemo-intake/references/ingest-formats.md | 9 +++---- services/intake/README.md | 8 ++++--- .../scripts/spans/seed_span_type_showcase.py | 4 ++-- .../nmp/intake/spans/clickhouse_migrations.py | 24 ++++++++++++++++--- .../src/nmp/intake/spans/ingest/otlp.py | 16 ++++++------- .../intake/spans/span_attribute_catalog.py | 12 ++++++---- .../integration/spans/test_traces_read.py | 3 +++ services/intake/tests/test_atif_v17.py | 4 ++-- .../tests/test_spans_clickhouse_migrations.py | 10 ++++++-- services/intake/tests/test_spans_otlp.py | 24 +++++++++++++++---- 26 files changed, 133 insertions(+), 75 deletions(-) diff --git a/docs/evaluator/experiments.mdx b/docs/evaluator/experiments.mdx index 16c3e97649..7a323db086 100644 --- a/docs/evaluator/experiments.mdx +++ b/docs/evaluator/experiments.mdx @@ -231,8 +231,9 @@ session with the Evaluation's identity: - For Agent Trajectory Interchange Format (ATIF) and chat-completions, add a **top-level** `evaluation_context` object to the ingest payload carrying `evaluation_id` (the Evaluation's **name**) and `test_case_id`. -- For OpenTelemetry Protocol (OTLP), set the `nemo.experiment.id` and `nemo.test_case.id` root-span - attributes. +- For OpenTelemetry Protocol (OTLP), set the `nemo.evaluation.name` (the Evaluation's **name**) and + `nemo.test_case.id` root-span attributes. The legacy key `nemo.experiment.id` is still accepted for + backward compatibility. The per-evaluator scores on the leaderboard come from **evaluator results** captured on those sessions, either automatically from ATIF verifier rewards or explicitly through the evaluator-results diff --git a/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-experiments-upload/SKILL.md b/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-experiments-upload/SKILL.md index cacd767e42..f1c158157f 100644 --- a/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-experiments-upload/SKILL.md +++ b/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-experiments-upload/SKILL.md @@ -130,7 +130,7 @@ curl -sf -X POST \ Pick the ingest endpoint that matches your producer. **Read `../nemo-intake/references/ingest-formats.md` for the full schema and a copy-pasteable example for each.** How you attach evaluation identity depends on the endpoint: - **ATIF and chat-completions** (JSON body) — add an `evaluation_context = {evaluation_id: "", test_case_id: ""}` object to the payload. -- **OTLP** — there is no body field; set `nemo.experiment.id` (the Evaluation **name**) and +- **OTLP** — there is no body field; set `nemo.evaluation.name` (the Evaluation **name**) and `nemo.test_case.id` (the task ID) as **attributes on the root span**. Spans missing these still ingest but won't associate to an Evaluation. @@ -176,7 +176,7 @@ You succeeded when `GET .../evaluations/my-eval-baseline` shows: - `run_count` ≥ 1 (each ingested session counts as one run), and - non-empty `evaluator_names` / `aggregate_scores` if you logged rewards, and/or `cost_usd` if your spans carried cost. -If `run_count` is 0 after ingesting, the traces didn't associate — almost always a wrong evaluation identity: `evaluation_context.evaluation_id` for ATIF/chat-completions, or the `nemo.experiment.id` root-span attribute for OTLP (see Gotchas). +If `run_count` is 0 after ingesting, the traces didn't associate — almost always a wrong evaluation identity: `evaluation_context.evaluation_id` for ATIF/chat-completions, or the `nemo.evaluation.name` root-span attribute for OTLP (see Gotchas). ## If verification fails @@ -184,7 +184,7 @@ If `run_count` is 0 after ingesting, the traces didn't associate — almost alwa |---|---|---| | `400 "…must be created before it can be logged."` | Ingested before the Evaluation existed, or `evaluation_id` doesn't match | Create the Evaluation (step 2); ensure `evaluation_context.evaluation_id` equals its **name** | | `422 Unprocessable` on ingest | Unknown/typo'd top-level key (ATIF/chat-completions are `extra="forbid"`) or bad `schema_version` | Check the exact schema in `../nemo-intake/references/ingest-formats.md`; remove stray keys | -| Ingest 2xx but `run_count` stays 0 | Evaluation identity missing/wrong — `evaluation_context.evaluation_id` (ATIF/chat-completions) or the `nemo.experiment.id` root-span attribute (OTLP) ≠ the Evaluation's name | Attach the identity for your endpoint; use the Evaluation **name**, not its id | +| Ingest 2xx but `run_count` stays 0 | Evaluation identity missing/wrong — `evaluation_context.evaluation_id` (ATIF/chat-completions) or the `nemo.evaluation.name` root-span attribute (OTLP) ≠ the Evaluation's name | Attach the identity for your endpoint; use the Evaluation **name**, not its id | | `503` on GET evaluation / sessions | ClickHouse (telemetry store) not running | Start ClickHouse; rollups and sessions require it | | Scores don't show up | Rewards not under `extra.verifier_result.rewards`, or wrong `data_type` on `/evaluator-results` | See `references/troubleshooting.md` | @@ -192,7 +192,7 @@ If `run_count` is 0 after ingesting, the traces didn't associate — almost alwa - **Create before you log.** The Evaluation entity must exist before any ingest referencing it — otherwise `400`. - **`evaluation_id` is the Evaluation's `name`, not its entity id.** But **`experiment_ids` holds the Experiment's `id`.** Different identifiers; easy to swap. -- **OTLP uses the attribute key `nemo.experiment.id`** (and `nemo.test_case.id`) — the span-attribute key still says "experiment" even though the JSON body field is `evaluation_context`. Set `nemo.experiment.id` on your root span. +- **OTLP uses the attribute key `nemo.evaluation.name`** (and `nemo.test_case.id`) — set it to the Evaluation's **name** on your root span, matching the `evaluation_id` field the JSON `evaluation_context` carries on the other endpoints. The pre-rename key `nemo.experiment.id` is still accepted on ingest, so older exporters keep associating. - **The parent lives at `/experiments`; `/experiment-groups` is a deprecated hidden alias.** Prefer `/experiments`. Evaluations are created and logged under `/evaluations`. - **`metadata` is `dict[str, str]`** — stringify non-string values or you'll get a `422`. - **ATIF and chat-completions are `extra="forbid"`** (unknown keys → 422); `evaluation_context` itself is lenient (`extra="ignore"`). diff --git a/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-experiments-upload/references/troubleshooting.md b/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-experiments-upload/references/troubleshooting.md index 8cd5170a5c..7945cf5eb2 100644 --- a/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-experiments-upload/references/troubleshooting.md +++ b/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-experiments-upload/references/troubleshooting.md @@ -29,7 +29,7 @@ string — read it first. | Symptom | Cause | Fix | |---|---|---| -| Ingest returned 2xx but `run_count` stays 0 | `evaluation_context` missing, or `evaluation_id` ≠ the Evaluation's name | Attach `evaluation_context`; use the Evaluation **name**. For OTLP, set the span attribute `nemo.experiment.id` on the root span | +| Ingest returned 2xx but `run_count` stays 0 | `evaluation_context` missing, or `evaluation_id` ≠ the Evaluation's name | Attach `evaluation_context`; use the Evaluation **name**. For OTLP, set the span attribute `nemo.evaluation.name` on the root span (legacy `nemo.experiment.id` still accepted) | | No scores on the evaluation | Rewards not under `extra.verifier_result.rewards` (ATIF), or wrong `data_type` (`/evaluator-results`) | ATIF: `extra.verifier_result.rewards = {criterion: value}`. Explicit: `NUMERIC`/`BOOLEAN` need `value`, `CATEGORICAL`/`TEXT` need `string_value` | | No cost on the rollup | The producer never emitted cost | Cost is pass-through — set `cost_usd` (chat-completions / ATIF step `metrics`) or `llm.cost.total` / `gen_ai.usage.cost` (OTLP) | | `503` on `GET .../evaluations/{name}` or `/sessions` | ClickHouse (telemetry store) not running | Start ClickHouse; rollups, sessions, and metric sorts/filters all need it | @@ -39,5 +39,5 @@ string — read it first. - `evaluation_context.evaluation_id` → the Evaluation's **`name`**. - `experiment_ids` (on create evaluation) → a list with the Experiment's **`id`**. -- OTLP evaluation attribute key → **`nemo.experiment.id`** (test case → `nemo.test_case.id`). +- OTLP evaluation attribute key → **`nemo.evaluation.name`** (legacy `nemo.experiment.id` still accepted; test case → `nemo.test_case.id`). - Parent → **`/experiments`** (`/experiment-groups` is a deprecated hidden alias); evaluations → **`/evaluations`**. diff --git a/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-intake/references/ingest-formats.md b/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-intake/references/ingest-formats.md index a029753b8d..e4d5a248b3 100644 --- a/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-intake/references/ingest-formats.md +++ b/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-intake/references/ingest-formats.md @@ -128,11 +128,12 @@ the root span: | Meaning | Span attribute key | |---|---| -| Evaluation (by name) | **`nemo.experiment.id`** | +| Evaluation (by name) | **`nemo.evaluation.name`** | | Test case | **`nemo.test_case.id`** | -> Note the key is `nemo.experiment.id` (still "experiment"), even though the REST body field elsewhere -> is `evaluation_context`. Set `nemo.experiment.id` to the Evaluation's **name**. +> Set `nemo.evaluation.name` to the Evaluation's **name** (not its id), matching the `evaluation_id` +> field used by the JSON `evaluation_context` on the other endpoints. The pre-rename key +> `nemo.experiment.id` is still accepted on ingest, so existing exporters keep working. Cost/token/model attributes are read from standard GenAI / OpenInference keys (first match wins): @@ -158,7 +159,7 @@ export OTEL_EXPORTER_OTLP_TRACES_ENDPOINT="${NMP_BASE_URL}/apis/intake/v2/worksp export OTEL_EXPORTER_OTLP_TRACES_PROTOCOL="http/protobuf" ``` -Then set `nemo.experiment.id` (+ `nemo.test_case.id`) on the root span of each run. +Then set `nemo.evaluation.name` (+ `nemo.test_case.id`) on the root span of each run. --- diff --git a/plugins/nemo-experimentalist/examples/tau3-nooa-agent/record_tau_airline_traces.py b/plugins/nemo-experimentalist/examples/tau3-nooa-agent/record_tau_airline_traces.py index a8590c1907..0ceddda0a5 100644 --- a/plugins/nemo-experimentalist/examples/tau3-nooa-agent/record_tau_airline_traces.py +++ b/plugins/nemo-experimentalist/examples/tau3-nooa-agent/record_tau_airline_traces.py @@ -133,6 +133,8 @@ async def _upload_trials( trials: list[TrialResult], *, workspace: str, + # TODO: rename experiment_id -> evaluation_name to match the ingested attribute + # (nemo.evaluation.name carries the Evaluation's name). Plumb the rename through callers. experiment_id: str, agent_name: str, agent_version: str, @@ -149,7 +151,7 @@ async def _upload_trials( raise RuntimeError(f"Trace {trace_id} was produced by more than one trial") attrs = { - "nemo.experiment.id": experiment_id, + "nemo.evaluation.name": experiment_id, "nemo.test_case.id": trial.task_id, "nemo.trial.id": trial.id, "gen_ai.agent.name": agent_name, diff --git a/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/experiment_mirror.py b/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/experiment_mirror.py index 86f1291049..8003f0cb31 100644 --- a/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/experiment_mirror.py +++ b/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/experiment_mirror.py @@ -145,7 +145,7 @@ async def ensure_experiment(self, candidate: Candidate, split: str) -> str: name (``opt--