From 57a0f955bd52c57a2c9e8a1d339a6dabfcdd09eb Mon Sep 17 00:00:00 2001 From: Sandy Chapman Date: Mon, 10 Aug 2026 13:08:25 -0300 Subject: [PATCH] fix(evaluator): keep a Fabric trial completed when only Relay teardown failed MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit NVBug 6562846: a DeepAgents invocation finishes its turn — model calls, tool calls, workspace changes, final assistant response — and is then reported as failed because NeMo Relay's telemetry teardown raises RuntimeError: invalid argument: scope handle is not at the top of the stack Relay keeps one process-global LIFO scope stack in a ContextVar. LangGraph schedules child tasks with copy_context(), which shares that same mutable stack rather than cloning it, so overlapping chain callbacks close out of LIFO order; the Fabric DeepAgents adapter catches the resulting teardown error in the same try that guards the invocation and rewrites a successful run into an adapter failure. QA saw 31 such trials in the 2026-08-05 regression, at parallelism 1 as well as 3/5/10, so it is not an Evaluator concurrency problem. The root cause is upstream in NeMo-Relay and NeMo-Fabric, neither of which is vendored here. Until that lands, stop scoring the false negative: when a Fabric failure carries the Relay scope-stack signature in the adapter's own output.error *and* a non-empty final response, treat the trial as completed and record the fault (recovered_from_telemetry_fault, fabric_status, telemetry_error) so the recovery is auditable and the possibly-truncated trajectory is not read as complete. This mirrors the existing recovered_from_mcp_binding path. The predicate deliberately needs both halves of the evidence: a run with no final response, or a failure that is not the telemetry one, still fails. Signed-off-by: Sandy Chapman --- .../agent_eval/runtimes/fabric/runtime.py | 55 ++++++++++- .../tests/agent_eval/test_fabric_runtime.py | 95 +++++++++++++++++++ .../agent_eval/runtimes/fabric/runtime.py | 55 ++++++++++- 3 files changed, 203 insertions(+), 2 deletions(-) diff --git a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/fabric/runtime.py b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/fabric/runtime.py index 22b650efb5..ba521e0222 100644 --- a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/fabric/runtime.py +++ b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/fabric/runtime.py @@ -110,6 +110,13 @@ _ATOF_FILENAME = "events.atof.jsonl" # ``kind`` Fabric stamps on the promoted Relay ATIF artifact; used to surface it as trace evidence. _ATIF_ARTIFACT_KIND = "atif" +# Signature of a NeMo Relay telemetry-teardown fault (NVBug 6562846). Relay keeps one process-global +# LIFO scope stack in a ``ContextVar``; LangGraph schedules child tasks with ``copy_context()``, which +# shares that same mutable stack, so overlapping chain callbacks close out of LIFO order and Relay's +# scope validator rejects the pop with exactly this message. The Fabric adapter catches it in the same +# ``try`` that guards the invocation, so a purely observability-side fault is reported as an agent +# failure. The string is emitted only by Relay's scope validation, so matching it is unambiguous. +_RELAY_SCOPE_STACK_ERROR = "scope handle is not at the top of the stack" class FabricAgentRuntime: @@ -446,7 +453,26 @@ def _to_trial( evidence=self._evidence(result, result_path, workspace_dir), metadata={**base_metadata, "generated": True, "agent_ok": True}, ) - return self._failed_trial(task, evidence_dir, _result_error(result), extra_metadata=base_metadata) + telemetry_error = _telemetry_teardown_error(result) + if telemetry_error is None: + return self._failed_trial(task, evidence_dir, _result_error(result), extra_metadata=base_metadata) + # The agent finished its turn and produced a final response; only Relay's telemetry teardown + # failed, and the Fabric adapter's single try/except rewrote that into an invocation failure + # (NVBug 6562846). Treat the trial as completed rather than scoring a false negative, and + # record the fault so the recovery is auditable and the (possibly truncated) trajectory + # evidence is not read as complete. Fall through to the success path so the recovered trial + # carries the same output and evidence any other completed trial would. + logger.warning( + "Fabric task %s completed but Relay telemetry teardown failed; recovering the trial: %s", + task.id, + telemetry_error, + ) + base_metadata = { + **base_metadata, + "fabric_status": result.status, + "recovered_from_telemetry_fault": True, + "telemetry_error": telemetry_error, + } # Fabric wraps the output in a ``RunOutput`` mapping (RunOutput response contract, #52), # which is not itself a JSON value; normalize it to a plain mapping so it round-trips through the @@ -708,6 +734,33 @@ def _extract_output_text(output: object) -> str | None: return json.dumps(output, default=str) +def _telemetry_teardown_error(result: RunResult) -> str | None: + """Return the adapter error when a Fabric failure is only a Relay telemetry-teardown fault. + + ``None`` means the failure is not recoverable here and the trial must stay failed. Recovery needs + both halves of the evidence, so this deliberately stays narrow: + + * the adapter's own ``output.error`` carries the Relay scope-stack signature — a failure raised + while closing the telemetry scope, not while running the agent; and + * ``output.response`` holds a non-empty final assistant message, so the agent phase did reach a + terminal answer. A run that died mid-turn has no final response and stays failed. + + Note ``output.error`` is read rather than ``result.error``: Fabric normalizes any adapter-reported + failure to the same top-level ``adapter_reported_failure`` code, so only the adapter's own error + string distinguishes a telemetry teardown from a real agent failure. + """ + output = result.output + if not isinstance(output, Mapping): + return None + error = output.get("error") + if not isinstance(error, str) or _RELAY_SCOPE_STACK_ERROR not in error: + return None + response = output.get("response") + if not isinstance(response, str) or not response.strip(): + return None + return error + + def _result_error(result: RunResult) -> Mapping[str, Any]: error = result.error if error is None: diff --git a/packages/nemo_evaluator_sdk/tests/agent_eval/test_fabric_runtime.py b/packages/nemo_evaluator_sdk/tests/agent_eval/test_fabric_runtime.py index d13b728628..5df8a4da62 100644 --- a/packages/nemo_evaluator_sdk/tests/agent_eval/test_fabric_runtime.py +++ b/packages/nemo_evaluator_sdk/tests/agent_eval/test_fabric_runtime.py @@ -652,6 +652,101 @@ def handler(agent: Any, kwargs: dict[str, Any]) -> _FakeResult: assert "error" in trial.evidence.descriptors +# NVBug 6562846: Relay keeps one process-global LIFO scope stack, LangGraph shares it across the child +# tasks it schedules with ``copy_context()``, and overlapping chain callbacks therefore close out of +# order. The Fabric adapter catches that teardown error in the same ``try`` that guards the invocation, +# so a completed DeepAgents turn comes back as ``status=failed`` with the full response still in +# ``output``. Verbatim adapter error string from the bug's reproduction. +_RELAY_TEARDOWN_ERROR = "RuntimeError: invalid argument: scope handle is not at the top of the stack" + + +@pytest.mark.asyncio +async def test_fabric_runtime_recovers_trial_when_only_relay_teardown_failed( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + def handler(agent: Any, kwargs: dict[str, Any]) -> _FakeResult: + return _FakeResult( + status="failed", + output={ + "response": "Both files are done", + "completed": False, + "failed": True, + "error": _RELAY_TEARDOWN_ERROR, + }, + error=_FakeError(stage="invoke", code="adapter_reported_failure", message=_RELAY_TEARDOWN_ERROR), + ) + + _install_fake_fabric(monkeypatch, handler) + runtime = fabric_runtime.FabricAgentRuntime(config=_CONFIG, work_root=tmp_path / "fabric", capture_trajectory=False) + + trial = (await runtime.run_tasks([_TASK]))[0] + + assert trial.status == "completed" + assert trial.output is not None + assert trial.output.output_text == "Both files are done" + # The recovery is explicit, never silent: the original Fabric verdict and the telemetry error are + # both retained so a recovered trial can be audited (and its trajectory treated as suspect). + assert trial.metadata["recovered_from_telemetry_fault"] is True + assert trial.metadata["fabric_status"] == "failed" + assert trial.metadata["telemetry_error"] == _RELAY_TEARDOWN_ERROR + # AgentPhaseSuccessMetric reads agent_ok; the agent phase did finish cleanly. + assert trial.metadata["agent_ok"] is True + assert trial.evidence is not None + assert "result" in trial.evidence.descriptors + assert "workspace" in trial.evidence.descriptors + + +@pytest.mark.asyncio +async def test_fabric_runtime_keeps_relay_teardown_failure_without_a_final_response( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + # Half the evidence is not enough: without a final assistant message the agent never reached a + # terminal answer, so the trial stays failed even though the error is the telemetry one. + def handler(agent: Any, kwargs: dict[str, Any]) -> _FakeResult: + return _FakeResult( + status="failed", + output={"response": " ", "completed": False, "failed": True, "error": _RELAY_TEARDOWN_ERROR}, + error=_FakeError(stage="invoke", code="adapter_reported_failure", message=_RELAY_TEARDOWN_ERROR), + ) + + _install_fake_fabric(monkeypatch, handler) + runtime = fabric_runtime.FabricAgentRuntime(config=_CONFIG, work_root=tmp_path / "fabric", capture_trajectory=False) + + trial = (await runtime.run_tasks([_TASK]))[0] + + assert trial.status == "failed" + assert trial.metadata["agent_ok"] is False + assert "recovered_from_telemetry_fault" not in trial.metadata + + +@pytest.mark.asyncio +async def test_fabric_runtime_keeps_non_telemetry_failure_with_a_final_response( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + # Guard against the recovery widening into general failure suppression: a real agent failure that + # happens to carry a response must still fail. + def handler(agent: Any, kwargs: dict[str, Any]) -> _FakeResult: + return _FakeResult( + status="failed", + output={ + "response": "partial answer", + "completed": False, + "failed": True, + "error": "RuntimeError: model call failed", + }, + error=_FakeError(stage="invoke", code="adapter_reported_failure", message="model call failed"), + ) + + _install_fake_fabric(monkeypatch, handler) + runtime = fabric_runtime.FabricAgentRuntime(config=_CONFIG, work_root=tmp_path / "fabric", capture_trajectory=False) + + trial = (await runtime.run_tasks([_TASK]))[0] + + assert trial.status == "failed" + assert trial.metadata["error"] == "model call failed" + assert "recovered_from_telemetry_fault" not in trial.metadata + + @pytest.mark.asyncio async def test_fabric_runtime_maps_timeout_to_failed_trial(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: def handler(agent: Any, kwargs: dict[str, Any]) -> _FakeResult: diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/fabric/runtime.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/fabric/runtime.py index 712128bd4e..62a7dca7bf 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/fabric/runtime.py +++ b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/fabric/runtime.py @@ -110,6 +110,13 @@ _ATOF_FILENAME = "events.atof.jsonl" # ``kind`` Fabric stamps on the promoted Relay ATIF artifact; used to surface it as trace evidence. _ATIF_ARTIFACT_KIND = "atif" +# Signature of a NeMo Relay telemetry-teardown fault (NVBug 6562846). Relay keeps one process-global +# LIFO scope stack in a ``ContextVar``; LangGraph schedules child tasks with ``copy_context()``, which +# shares that same mutable stack, so overlapping chain callbacks close out of LIFO order and Relay's +# scope validator rejects the pop with exactly this message. The Fabric adapter catches it in the same +# ``try`` that guards the invocation, so a purely observability-side fault is reported as an agent +# failure. The string is emitted only by Relay's scope validation, so matching it is unambiguous. +_RELAY_SCOPE_STACK_ERROR = "scope handle is not at the top of the stack" class FabricAgentRuntime: @@ -446,7 +453,26 @@ def _to_trial( evidence=self._evidence(result, result_path, workspace_dir), metadata={**base_metadata, "generated": True, "agent_ok": True}, ) - return self._failed_trial(task, evidence_dir, _result_error(result), extra_metadata=base_metadata) + telemetry_error = _telemetry_teardown_error(result) + if telemetry_error is None: + return self._failed_trial(task, evidence_dir, _result_error(result), extra_metadata=base_metadata) + # The agent finished its turn and produced a final response; only Relay's telemetry teardown + # failed, and the Fabric adapter's single try/except rewrote that into an invocation failure + # (NVBug 6562846). Treat the trial as completed rather than scoring a false negative, and + # record the fault so the recovery is auditable and the (possibly truncated) trajectory + # evidence is not read as complete. Fall through to the success path so the recovered trial + # carries the same output and evidence any other completed trial would. + logger.warning( + "Fabric task %s completed but Relay telemetry teardown failed; recovering the trial: %s", + task.id, + telemetry_error, + ) + base_metadata = { + **base_metadata, + "fabric_status": result.status, + "recovered_from_telemetry_fault": True, + "telemetry_error": telemetry_error, + } # Fabric wraps the output in a ``RunOutput`` mapping (RunOutput response contract, #52), # which is not itself a JSON value; normalize it to a plain mapping so it round-trips through the @@ -708,6 +734,33 @@ def _extract_output_text(output: object) -> str | None: return json.dumps(output, default=str) +def _telemetry_teardown_error(result: RunResult) -> str | None: + """Return the adapter error when a Fabric failure is only a Relay telemetry-teardown fault. + + ``None`` means the failure is not recoverable here and the trial must stay failed. Recovery needs + both halves of the evidence, so this deliberately stays narrow: + + * the adapter's own ``output.error`` carries the Relay scope-stack signature — a failure raised + while closing the telemetry scope, not while running the agent; and + * ``output.response`` holds a non-empty final assistant message, so the agent phase did reach a + terminal answer. A run that died mid-turn has no final response and stays failed. + + Note ``output.error`` is read rather than ``result.error``: Fabric normalizes any adapter-reported + failure to the same top-level ``adapter_reported_failure`` code, so only the adapter's own error + string distinguishes a telemetry teardown from a real agent failure. + """ + output = result.output + if not isinstance(output, Mapping): + return None + error = output.get("error") + if not isinstance(error, str) or _RELAY_SCOPE_STACK_ERROR not in error: + return None + response = output.get("response") + if not isinstance(response, str) or not response.strip(): + return None + return error + + def _result_error(result: RunResult) -> Mapping[str, Any]: error = result.error if error is None: