fix(evaluator): keep a Fabric trial completed when only Relay teardown failed

packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/fabric/runtime.py

@@ -110,6 +110,13 @@
 _ATOF_FILENAME = "events.atof.jsonl"
 # ``kind`` Fabric stamps on the promoted Relay ATIF artifact; used to surface it as trace evidence.
 _ATIF_ARTIFACT_KIND = "atif"
+# Signature of a NeMo Relay telemetry-teardown fault (NVBug 6562846). Relay keeps one process-global
+# LIFO scope stack in a ``ContextVar``; LangGraph schedules child tasks with ``copy_context()``, which
+# shares that same mutable stack, so overlapping chain callbacks close out of LIFO order and Relay's
+# scope validator rejects the pop with exactly this message. The Fabric adapter catches it in the same
+# ``try`` that guards the invocation, so a purely observability-side fault is reported as an agent
+# failure. The string is emitted only by Relay's scope validation, so matching it is unambiguous.
+_RELAY_SCOPE_STACK_ERROR = "scope handle is not at the top of the stack"
 
 
 class FabricAgentRuntime:
@@ -446,7 +453,26 @@ def _to_trial(
                     evidence=self._evidence(result, result_path, workspace_dir),
                     metadata={**base_metadata, "generated": True, "agent_ok": True},
                 )
-            return self._failed_trial(task, evidence_dir, _result_error(result), extra_metadata=base_metadata)
+            telemetry_error = _telemetry_teardown_error(result)
+            if telemetry_error is None:
+                return self._failed_trial(task, evidence_dir, _result_error(result), extra_metadata=base_metadata)
+            # The agent finished its turn and produced a final response; only Relay's telemetry teardown
+            # failed, and the Fabric adapter's single try/except rewrote that into an invocation failure
+            # (NVBug 6562846). Treat the trial as completed rather than scoring a false negative, and
+            # record the fault so the recovery is auditable and the (possibly truncated) trajectory
+            # evidence is not read as complete. Fall through to the success path so the recovered trial
+            # carries the same output and evidence any other completed trial would.
+            logger.warning(
+                "Fabric task %s completed but Relay telemetry teardown failed; recovering the trial: %s",
+                task.id,
+                telemetry_error,
+            )
+            base_metadata = {
+                **base_metadata,
+                "fabric_status": result.status,
+                "recovered_from_telemetry_fault": True,
+                "telemetry_error": telemetry_error,
+            }
 
         # Fabric wraps the output in a ``RunOutput`` mapping (RunOutput response contract, #52),
         # which is not itself a JSON value; normalize it to a plain mapping so it round-trips through the
@@ -708,6 +734,33 @@ def _extract_output_text(output: object) -> str | None:
     return json.dumps(output, default=str)
 
 
+def _telemetry_teardown_error(result: RunResult) -> str | None:
+    """Return the adapter error when a Fabric failure is only a Relay telemetry-teardown fault.
+
+    ``None`` means the failure is not recoverable here and the trial must stay failed. Recovery needs
+    both halves of the evidence, so this deliberately stays narrow:
+
+    * the adapter's own ``output.error`` carries the Relay scope-stack signature — a failure raised
+      while closing the telemetry scope, not while running the agent; and
+    * ``output.response`` holds a non-empty final assistant message, so the agent phase did reach a
+      terminal answer. A run that died mid-turn has no final response and stays failed.
+
+    Note ``output.error`` is read rather than ``result.error``: Fabric normalizes any adapter-reported
+    failure to the same top-level ``adapter_reported_failure`` code, so only the adapter's own error
+    string distinguishes a telemetry teardown from a real agent failure.
+    """
+    output = result.output
+    if not isinstance(output, Mapping):
+        return None
+    error = output.get("error")
+    if not isinstance(error, str) or _RELAY_SCOPE_STACK_ERROR not in error:
+        return None
+    response = output.get("response")
+    if not isinstance(response, str) or not response.strip():
+        return None
+    return error
+
+
 def _result_error(result: RunResult) -> Mapping[str, Any]:
     error = result.error
     if error is None:

packages/nemo_evaluator_sdk/tests/agent_eval/test_fabric_runtime.py

@@ -652,6 +652,101 @@ def handler(agent: Any, kwargs: dict[str, Any]) -> _FakeResult:
     assert "error" in trial.evidence.descriptors
 
 
+# NVBug 6562846: Relay keeps one process-global LIFO scope stack, LangGraph shares it across the child
+# tasks it schedules with ``copy_context()``, and overlapping chain callbacks therefore close out of
+# order. The Fabric adapter catches that teardown error in the same ``try`` that guards the invocation,
+# so a completed DeepAgents turn comes back as ``status=failed`` with the full response still in
+# ``output``. Verbatim adapter error string from the bug's reproduction.
+_RELAY_TEARDOWN_ERROR = "RuntimeError: invalid argument: scope handle is not at the top of the stack"
+
+
+@pytest.mark.asyncio
+async def test_fabric_runtime_recovers_trial_when_only_relay_teardown_failed(
+    tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    def handler(agent: Any, kwargs: dict[str, Any]) -> _FakeResult:
+        return _FakeResult(
+            status="failed",
+            output={
+                "response": "Both files are done",
+                "completed": False,
+                "failed": True,
+                "error": _RELAY_TEARDOWN_ERROR,
+            },
+            error=_FakeError(stage="invoke", code="adapter_reported_failure", message=_RELAY_TEARDOWN_ERROR),
+        )
+
+    _install_fake_fabric(monkeypatch, handler)
+    runtime = fabric_runtime.FabricAgentRuntime(config=_CONFIG, work_root=tmp_path / "fabric", capture_trajectory=False)
+
+    trial = (await runtime.run_tasks([_TASK]))[0]
+
+    assert trial.status == "completed"
+    assert trial.output is not None
+    assert trial.output.output_text == "Both files are done"
+    # The recovery is explicit, never silent: the original Fabric verdict and the telemetry error are
+    # both retained so a recovered trial can be audited (and its trajectory treated as suspect).
+    assert trial.metadata["recovered_from_telemetry_fault"] is True
+    assert trial.metadata["fabric_status"] == "failed"
+    assert trial.metadata["telemetry_error"] == _RELAY_TEARDOWN_ERROR
+    # AgentPhaseSuccessMetric reads agent_ok; the agent phase did finish cleanly.
+    assert trial.metadata["agent_ok"] is True
+    assert trial.evidence is not None
+    assert "result" in trial.evidence.descriptors
+    assert "workspace" in trial.evidence.descriptors
+
+
+@pytest.mark.asyncio
+async def test_fabric_runtime_keeps_relay_teardown_failure_without_a_final_response(
+    tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    # Half the evidence is not enough: without a final assistant message the agent never reached a
+    # terminal answer, so the trial stays failed even though the error is the telemetry one.
+    def handler(agent: Any, kwargs: dict[str, Any]) -> _FakeResult:
+        return _FakeResult(
+            status="failed",
+            output={"response": "  ", "completed": False, "failed": True, "error": _RELAY_TEARDOWN_ERROR},
+            error=_FakeError(stage="invoke", code="adapter_reported_failure", message=_RELAY_TEARDOWN_ERROR),
+        )
+
+    _install_fake_fabric(monkeypatch, handler)
+    runtime = fabric_runtime.FabricAgentRuntime(config=_CONFIG, work_root=tmp_path / "fabric", capture_trajectory=False)
+
+    trial = (await runtime.run_tasks([_TASK]))[0]
+
+    assert trial.status == "failed"
+    assert trial.metadata["agent_ok"] is False
+    assert "recovered_from_telemetry_fault" not in trial.metadata
+
+
+@pytest.mark.asyncio
+async def test_fabric_runtime_keeps_non_telemetry_failure_with_a_final_response(
+    tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    # Guard against the recovery widening into general failure suppression: a real agent failure that
+    # happens to carry a response must still fail.
+    def handler(agent: Any, kwargs: dict[str, Any]) -> _FakeResult:
+        return _FakeResult(
+            status="failed",
+            output={
+                "response": "partial answer",
+                "completed": False,
+                "failed": True,
+                "error": "RuntimeError: model call failed",
+            },
+            error=_FakeError(stage="invoke", code="adapter_reported_failure", message="model call failed"),
+        )
+
+    _install_fake_fabric(monkeypatch, handler)
+    runtime = fabric_runtime.FabricAgentRuntime(config=_CONFIG, work_root=tmp_path / "fabric", capture_trajectory=False)
+
+    trial = (await runtime.run_tasks([_TASK]))[0]
+
+    assert trial.status == "failed"
+    assert trial.metadata["error"] == "model call failed"
+    assert "recovered_from_telemetry_fault" not in trial.metadata
+
+
 @pytest.mark.asyncio
 async def test_fabric_runtime_maps_timeout_to_failed_trial(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
     def handler(agent: Any, kwargs: dict[str, Any]) -> _FakeResult: