diff --git a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/fabric/runtime.py b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/fabric/runtime.py index 22b650efb5..ba521e0222 100644 --- a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/fabric/runtime.py +++ b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/fabric/runtime.py @@ -110,6 +110,13 @@ _ATOF_FILENAME = "events.atof.jsonl" # ``kind`` Fabric stamps on the promoted Relay ATIF artifact; used to surface it as trace evidence. _ATIF_ARTIFACT_KIND = "atif" +# Signature of a NeMo Relay telemetry-teardown fault (NVBug 6562846). Relay keeps one process-global +# LIFO scope stack in a ``ContextVar``; LangGraph schedules child tasks with ``copy_context()``, which +# shares that same mutable stack, so overlapping chain callbacks close out of LIFO order and Relay's +# scope validator rejects the pop with exactly this message. The Fabric adapter catches it in the same +# ``try`` that guards the invocation, so a purely observability-side fault is reported as an agent +# failure. The string is emitted only by Relay's scope validation, so matching it is unambiguous. +_RELAY_SCOPE_STACK_ERROR = "scope handle is not at the top of the stack" class FabricAgentRuntime: @@ -446,7 +453,26 @@ def _to_trial( evidence=self._evidence(result, result_path, workspace_dir), metadata={**base_metadata, "generated": True, "agent_ok": True}, ) - return self._failed_trial(task, evidence_dir, _result_error(result), extra_metadata=base_metadata) + telemetry_error = _telemetry_teardown_error(result) + if telemetry_error is None: + return self._failed_trial(task, evidence_dir, _result_error(result), extra_metadata=base_metadata) + # The agent finished its turn and produced a final response; only Relay's telemetry teardown + # failed, and the Fabric adapter's single try/except rewrote that into an invocation failure + # (NVBug 6562846). Treat the trial as completed rather than scoring a false negative, and + # record the fault so the recovery is auditable and the (possibly truncated) trajectory + # evidence is not read as complete. Fall through to the success path so the recovered trial + # carries the same output and evidence any other completed trial would. + logger.warning( + "Fabric task %s completed but Relay telemetry teardown failed; recovering the trial: %s", + task.id, + telemetry_error, + ) + base_metadata = { + **base_metadata, + "fabric_status": result.status, + "recovered_from_telemetry_fault": True, + "telemetry_error": telemetry_error, + } # Fabric wraps the output in a ``RunOutput`` mapping (RunOutput response contract, #52), # which is not itself a JSON value; normalize it to a plain mapping so it round-trips through the @@ -708,6 +734,33 @@ def _extract_output_text(output: object) -> str | None: return json.dumps(output, default=str) +def _telemetry_teardown_error(result: RunResult) -> str | None: + """Return the adapter error when a Fabric failure is only a Relay telemetry-teardown fault. + + ``None`` means the failure is not recoverable here and the trial must stay failed. Recovery needs + both halves of the evidence, so this deliberately stays narrow: + + * the adapter's own ``output.error`` carries the Relay scope-stack signature — a failure raised + while closing the telemetry scope, not while running the agent; and + * ``output.response`` holds a non-empty final assistant message, so the agent phase did reach a + terminal answer. A run that died mid-turn has no final response and stays failed. + + Note ``output.error`` is read rather than ``result.error``: Fabric normalizes any adapter-reported + failure to the same top-level ``adapter_reported_failure`` code, so only the adapter's own error + string distinguishes a telemetry teardown from a real agent failure. + """ + output = result.output + if not isinstance(output, Mapping): + return None + error = output.get("error") + if not isinstance(error, str) or _RELAY_SCOPE_STACK_ERROR not in error: + return None + response = output.get("response") + if not isinstance(response, str) or not response.strip(): + return None + return error + + def _result_error(result: RunResult) -> Mapping[str, Any]: error = result.error if error is None: diff --git a/packages/nemo_evaluator_sdk/tests/agent_eval/test_fabric_runtime.py b/packages/nemo_evaluator_sdk/tests/agent_eval/test_fabric_runtime.py index d13b728628..5df8a4da62 100644 --- a/packages/nemo_evaluator_sdk/tests/agent_eval/test_fabric_runtime.py +++ b/packages/nemo_evaluator_sdk/tests/agent_eval/test_fabric_runtime.py @@ -652,6 +652,101 @@ def handler(agent: Any, kwargs: dict[str, Any]) -> _FakeResult: assert "error" in trial.evidence.descriptors +# NVBug 6562846: Relay keeps one process-global LIFO scope stack, LangGraph shares it across the child +# tasks it schedules with ``copy_context()``, and overlapping chain callbacks therefore close out of +# order. The Fabric adapter catches that teardown error in the same ``try`` that guards the invocation, +# so a completed DeepAgents turn comes back as ``status=failed`` with the full response still in +# ``output``. Verbatim adapter error string from the bug's reproduction. +_RELAY_TEARDOWN_ERROR = "RuntimeError: invalid argument: scope handle is not at the top of the stack" + + +@pytest.mark.asyncio +async def test_fabric_runtime_recovers_trial_when_only_relay_teardown_failed( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + def handler(agent: Any, kwargs: dict[str, Any]) -> _FakeResult: + return _FakeResult( + status="failed", + output={ + "response": "Both files are done", + "completed": False, + "failed": True, + "error": _RELAY_TEARDOWN_ERROR, + }, + error=_FakeError(stage="invoke", code="adapter_reported_failure", message=_RELAY_TEARDOWN_ERROR), + ) + + _install_fake_fabric(monkeypatch, handler) + runtime = fabric_runtime.FabricAgentRuntime(config=_CONFIG, work_root=tmp_path / "fabric", capture_trajectory=False) + + trial = (await runtime.run_tasks([_TASK]))[0] + + assert trial.status == "completed" + assert trial.output is not None + assert trial.output.output_text == "Both files are done" + # The recovery is explicit, never silent: the original Fabric verdict and the telemetry error are + # both retained so a recovered trial can be audited (and its trajectory treated as suspect). + assert trial.metadata["recovered_from_telemetry_fault"] is True + assert trial.metadata["fabric_status"] == "failed" + assert trial.metadata["telemetry_error"] == _RELAY_TEARDOWN_ERROR + # AgentPhaseSuccessMetric reads agent_ok; the agent phase did finish cleanly. + assert trial.metadata["agent_ok"] is True + assert trial.evidence is not None + assert "result" in trial.evidence.descriptors + assert "workspace" in trial.evidence.descriptors + + +@pytest.mark.asyncio +async def test_fabric_runtime_keeps_relay_teardown_failure_without_a_final_response( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + # Half the evidence is not enough: without a final assistant message the agent never reached a + # terminal answer, so the trial stays failed even though the error is the telemetry one. + def handler(agent: Any, kwargs: dict[str, Any]) -> _FakeResult: + return _FakeResult( + status="failed", + output={"response": " ", "completed": False, "failed": True, "error": _RELAY_TEARDOWN_ERROR}, + error=_FakeError(stage="invoke", code="adapter_reported_failure", message=_RELAY_TEARDOWN_ERROR), + ) + + _install_fake_fabric(monkeypatch, handler) + runtime = fabric_runtime.FabricAgentRuntime(config=_CONFIG, work_root=tmp_path / "fabric", capture_trajectory=False) + + trial = (await runtime.run_tasks([_TASK]))[0] + + assert trial.status == "failed" + assert trial.metadata["agent_ok"] is False + assert "recovered_from_telemetry_fault" not in trial.metadata + + +@pytest.mark.asyncio +async def test_fabric_runtime_keeps_non_telemetry_failure_with_a_final_response( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + # Guard against the recovery widening into general failure suppression: a real agent failure that + # happens to carry a response must still fail. + def handler(agent: Any, kwargs: dict[str, Any]) -> _FakeResult: + return _FakeResult( + status="failed", + output={ + "response": "partial answer", + "completed": False, + "failed": True, + "error": "RuntimeError: model call failed", + }, + error=_FakeError(stage="invoke", code="adapter_reported_failure", message="model call failed"), + ) + + _install_fake_fabric(monkeypatch, handler) + runtime = fabric_runtime.FabricAgentRuntime(config=_CONFIG, work_root=tmp_path / "fabric", capture_trajectory=False) + + trial = (await runtime.run_tasks([_TASK]))[0] + + assert trial.status == "failed" + assert trial.metadata["error"] == "model call failed" + assert "recovered_from_telemetry_fault" not in trial.metadata + + @pytest.mark.asyncio async def test_fabric_runtime_maps_timeout_to_failed_trial(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: def handler(agent: Any, kwargs: dict[str, Any]) -> _FakeResult: diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/fabric/runtime.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/fabric/runtime.py index 712128bd4e..62a7dca7bf 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/fabric/runtime.py +++ b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/fabric/runtime.py @@ -110,6 +110,13 @@ _ATOF_FILENAME = "events.atof.jsonl" # ``kind`` Fabric stamps on the promoted Relay ATIF artifact; used to surface it as trace evidence. _ATIF_ARTIFACT_KIND = "atif" +# Signature of a NeMo Relay telemetry-teardown fault (NVBug 6562846). Relay keeps one process-global +# LIFO scope stack in a ``ContextVar``; LangGraph schedules child tasks with ``copy_context()``, which +# shares that same mutable stack, so overlapping chain callbacks close out of LIFO order and Relay's +# scope validator rejects the pop with exactly this message. The Fabric adapter catches it in the same +# ``try`` that guards the invocation, so a purely observability-side fault is reported as an agent +# failure. The string is emitted only by Relay's scope validation, so matching it is unambiguous. +_RELAY_SCOPE_STACK_ERROR = "scope handle is not at the top of the stack" class FabricAgentRuntime: @@ -446,7 +453,26 @@ def _to_trial( evidence=self._evidence(result, result_path, workspace_dir), metadata={**base_metadata, "generated": True, "agent_ok": True}, ) - return self._failed_trial(task, evidence_dir, _result_error(result), extra_metadata=base_metadata) + telemetry_error = _telemetry_teardown_error(result) + if telemetry_error is None: + return self._failed_trial(task, evidence_dir, _result_error(result), extra_metadata=base_metadata) + # The agent finished its turn and produced a final response; only Relay's telemetry teardown + # failed, and the Fabric adapter's single try/except rewrote that into an invocation failure + # (NVBug 6562846). Treat the trial as completed rather than scoring a false negative, and + # record the fault so the recovery is auditable and the (possibly truncated) trajectory + # evidence is not read as complete. Fall through to the success path so the recovered trial + # carries the same output and evidence any other completed trial would. + logger.warning( + "Fabric task %s completed but Relay telemetry teardown failed; recovering the trial: %s", + task.id, + telemetry_error, + ) + base_metadata = { + **base_metadata, + "fabric_status": result.status, + "recovered_from_telemetry_fault": True, + "telemetry_error": telemetry_error, + } # Fabric wraps the output in a ``RunOutput`` mapping (RunOutput response contract, #52), # which is not itself a JSON value; normalize it to a plain mapping so it round-trips through the @@ -708,6 +734,33 @@ def _extract_output_text(output: object) -> str | None: return json.dumps(output, default=str) +def _telemetry_teardown_error(result: RunResult) -> str | None: + """Return the adapter error when a Fabric failure is only a Relay telemetry-teardown fault. + + ``None`` means the failure is not recoverable here and the trial must stay failed. Recovery needs + both halves of the evidence, so this deliberately stays narrow: + + * the adapter's own ``output.error`` carries the Relay scope-stack signature — a failure raised + while closing the telemetry scope, not while running the agent; and + * ``output.response`` holds a non-empty final assistant message, so the agent phase did reach a + terminal answer. A run that died mid-turn has no final response and stays failed. + + Note ``output.error`` is read rather than ``result.error``: Fabric normalizes any adapter-reported + failure to the same top-level ``adapter_reported_failure`` code, so only the adapter's own error + string distinguishes a telemetry teardown from a real agent failure. + """ + output = result.output + if not isinstance(output, Mapping): + return None + error = output.get("error") + if not isinstance(error, str) or _RELAY_SCOPE_STACK_ERROR not in error: + return None + response = output.get("response") + if not isinstance(response, str) or not response.strip(): + return None + return error + + def _result_error(result: RunResult) -> Mapping[str, Any]: error = result.error if error is None: