diff --git a/packages/opentelemetry-instrumentation-anthropic/opentelemetry/instrumentation/anthropic/retry_handler.py b/packages/opentelemetry-instrumentation-anthropic/opentelemetry/instrumentation/anthropic/retry_handler.py index a94aff6d03..8b21b2f11c 100644 --- a/packages/opentelemetry-instrumentation-anthropic/opentelemetry/instrumentation/anthropic/retry_handler.py +++ b/packages/opentelemetry-instrumentation-anthropic/opentelemetry/instrumentation/anthropic/retry_handler.py @@ -41,7 +41,10 @@ from opentelemetry import trace from opentelemetry.instrumentation.anthropic.version import __version__ from opentelemetry.instrumentation.fortifyroot import ( + FR_HAS_ATTEMPT_CHILD_KEY, is_framework_owned, + llm_attempt_attributes, + next_llm_attempt, ) from opentelemetry.instrumentation.utils import unwrap from opentelemetry.semconv_ai import SUPPRESS_LANGUAGE_MODEL_INSTRUMENTATION_KEY @@ -52,10 +55,10 @@ # ST-10 §4.4 / §4.5 constants. -_FR_RETRY_ATTEMPT_SPAN_NAME = "fortifyroot.anthropic.retry_attempt" _FR_SPAN_ROLE_KEY = "fortifyroot.span.role" -_FR_SPAN_ROLE_RETRY_ATTEMPT = "retry_attempt" -_FR_HAS_RETRY_ATTEMPT_CHILD_KEY = "fortifyroot.span.has_retry_attempt_child" +_FR_LLM_ATTEMPT_SPAN_NAME_PREFIX = "fortifyroot.anthropic" +_FR_SPAN_ROLE_LLM_ATTEMPT = "llm_attempt" +_FR_HAS_ATTEMPT_CHILD_KEY = FR_HAS_ATTEMPT_CHILD_KEY _ANTHROPIC_BASE_CLIENT_MODULE = "anthropic._base_client" _SYNC_WRAPPER_CLASS = "SyncHttpxClientWrapper" @@ -179,15 +182,18 @@ def _resolve_parent_span() -> Optional["trace.Span"]: def _set_parent_marker(parent_span: "trace.Span") -> None: try: - parent_span.set_attribute(_FR_HAS_RETRY_ATTEMPT_CHILD_KEY, True) + parent_span.set_attribute(_FR_HAS_ATTEMPT_CHILD_KEY, True) except Exception: - logger.debug("failed to set has_retry_attempt_child on parent", exc_info=True) + logger.debug("failed to set has_attempt_child on parent", exc_info=True) def _start_attempt_span(request: Any, parent_span: "trace.Span") -> "trace.Span": path = _request_path(request) + span_name, attempt_number, is_retry = next_llm_attempt( + parent_span, + _FR_LLM_ATTEMPT_SPAN_NAME_PREFIX, + ) attrs: dict[str, Any] = { - _FR_SPAN_ROLE_KEY: _FR_SPAN_ROLE_RETRY_ATTEMPT, # Match the existing Anthropic instrumentor's gen_ai.system # value ("Anthropic" title-case — see # ``anthropic/__init__.py`` ``_wrap``). The earlier draft of @@ -201,6 +207,7 @@ def _start_attempt_span(request: Any, parent_span: "trace.Span") -> "trace.Span" "gen_ai.system": "Anthropic", "gen_ai.operation.name": _operation_for_path(path), } + attrs.update(llm_attempt_attributes(attempt_number, is_retry)) model = _resolve_model_from_request(request) if model: attrs["gen_ai.request.model"] = model @@ -209,7 +216,7 @@ def _start_attempt_span(request: Any, parent_span: "trace.Span") -> "trace.Span" tracer = trace.get_tracer(__name__, __version__, _tracer_provider) parent_ctx = trace.set_span_in_context(parent_span) span = tracer.start_span( - _FR_RETRY_ATTEMPT_SPAN_NAME, + span_name, kind=SpanKind.CLIENT, attributes=attrs, context=parent_ctx, @@ -272,12 +279,12 @@ def _finalize_success(span: "trace.Span", response: Any, *, is_streaming: bool = span.set_attribute("error.type", err_type) span.set_status(Status(StatusCode.ERROR, f"http {status_code}")) except Exception: - logger.debug("failed to set response attrs on anthropic retry_attempt", exc_info=True) + logger.debug("failed to set response attrs on anthropic llm_attempt", exc_info=True) def _extract_usage_from_body(span: "trace.Span", response: Any) -> None: """Parse a non-streaming Anthropic response body and copy usage / - response id / response model attrs to the retry_attempt span. + response id / response model attrs to the llm_attempt span. Anthropic Messages response: ``{"id": "...", "model": "...", "usage": {"input_tokens": N, "output_tokens": M, ...}}``. Legacy @@ -326,7 +333,7 @@ def _finalize_error(span: "trace.Span", error: BaseException) -> None: pass span.set_status(Status(StatusCode.ERROR, str(error))) except Exception: - logger.debug("failed to set error attrs on anthropic retry_attempt", exc_info=True) + logger.debug("failed to set error attrs on anthropic llm_attempt", exc_info=True) def _should_emit_for(request: Any) -> bool: @@ -552,7 +559,7 @@ def _is_installed_for_test() -> bool: __all__ = [ "instrument_retry_emitter", "uninstrument_retry_emitter", - "_FR_RETRY_ATTEMPT_SPAN_NAME", - "_FR_HAS_RETRY_ATTEMPT_CHILD_KEY", + "_FR_LLM_ATTEMPT_SPAN_NAME_PREFIX", + "_FR_HAS_ATTEMPT_CHILD_KEY", "_LLM_PATH_SUFFIXES", ] diff --git a/packages/opentelemetry-instrumentation-anthropic/tests/conftest.py b/packages/opentelemetry-instrumentation-anthropic/tests/conftest.py index b08e58a8a3..94e9aef44e 100644 --- a/packages/opentelemetry-instrumentation-anthropic/tests/conftest.py +++ b/packages/opentelemetry-instrumentation-anthropic/tests/conftest.py @@ -32,7 +32,7 @@ class _NoFortifyRootSpanExporter(InMemorySpanExporter): """ST-10.4 (2026-05-17): filter out any span whose name starts with ``fortifyroot.`` from the upstream-test span exporter. - ST-10.4 added per-attempt ``fortifyroot.anthropic.retry_attempt`` + ST-10.4 added per-attempt ``fortifyroot.anthropic.attempt_1`` sibling spans under every anthropic logical call (once the Anthropic ``_wrap`` started using ``trace.use_span`` so the retry handler can find the parent span). Upstream / legacy Anthropic @@ -54,7 +54,7 @@ def get_finished_spans(self): # type: ignore[override] # visible to tests that inspect them. return tuple( s for s in super().get_finished_spans() - if (s.attributes or {}).get("fortifyroot.span.role") != "retry_attempt" + if (s.attributes or {}).get("fortifyroot.span.role") != "llm_attempt" ) diff --git a/packages/opentelemetry-instrumentation-anthropic/tests/test_retry_attempt_emission.py b/packages/opentelemetry-instrumentation-anthropic/tests/test_retry_attempt_emission.py index a6bb6f218c..5028293ab5 100644 --- a/packages/opentelemetry-instrumentation-anthropic/tests/test_retry_attempt_emission.py +++ b/packages/opentelemetry-instrumentation-anthropic/tests/test_retry_attempt_emission.py @@ -3,8 +3,8 @@ Covers (per RETRY_LOOP.md §4.4 Anthropic row + §4.7 suppression): - Instrumentor symmetry: install/uninstall flips state cleanly and is idempotent. - - Single-attempt happy path: ONE retry_attempt span under the active - parent; parent gets has_retry_attempt_child=true; span carries + - Single-attempt happy path: ONE llm_attempt span under the active + parent; parent gets has_attempt_child=true; span carries role / gen_ai.system=anthropic / gen_ai.request.model / http.status_code. - Multi-attempt retry path: N siblings under one parent. - Error-attempt: status=ERROR, http.status_code, error.type. @@ -28,8 +28,8 @@ from opentelemetry import context as context_api from opentelemetry import trace from opentelemetry.instrumentation.anthropic.retry_handler import ( - _FR_HAS_RETRY_ATTEMPT_CHILD_KEY, - _FR_RETRY_ATTEMPT_SPAN_NAME, + _FR_HAS_ATTEMPT_CHILD_KEY, + _FR_LLM_ATTEMPT_SPAN_NAME_PREFIX, _async_send_wrapper, _has_wrappable_symbol, _is_installed_for_test, @@ -38,6 +38,7 @@ uninstrument_retry_emitter, ) from opentelemetry.instrumentation.fortifyroot import ( + clear_attempt_counters_for_test, is_framework_owned, register_framework_attempt, retry_registry, @@ -72,12 +73,14 @@ def fresh_tracer(): @pytest.fixture(autouse=True) def reset_state(): retry_registry._reset_for_test() + clear_attempt_counters_for_test() try: uninstrument_retry_emitter() except Exception: pass yield retry_registry._reset_for_test() + clear_attempt_counters_for_test() try: uninstrument_retry_emitter() except Exception: @@ -116,7 +119,18 @@ def _make_response(status_code: int = 200, request_id: str = "req-abc", def _retry_spans(exporter: InMemorySpanExporter): return [s for s in exporter.get_finished_spans() - if s.name == _FR_RETRY_ATTEMPT_SPAN_NAME] + if s.name.startswith(f"{_FR_LLM_ATTEMPT_SPAN_NAME_PREFIX}.attempt_")] + + +def _assert_attempt_sequence(spans): + by_number = sorted( + spans, + key=lambda s: int(s.attributes.get("fortifyroot.attempt.number") or -1), + ) + for expected, span in enumerate(by_number, start=1): + assert span.name == f"{_FR_LLM_ATTEMPT_SPAN_NAME_PREFIX}.attempt_{expected}" + assert span.attributes.get("fortifyroot.attempt.number") == expected + assert span.attributes.get("fortifyroot.attempt.is_retry") is (expected > 1) # --------------------------------------------------------------------------- @@ -212,8 +226,9 @@ def test_single_attempt_emits_one_span_with_marker(fresh_tracer): retry_spans = _retry_spans(exporter) assert len(retry_spans) == 1 rs = retry_spans[0] + _assert_attempt_sequence(retry_spans) assert rs.parent.span_id == parent_exported.context.span_id - assert rs.attributes.get("fortifyroot.span.role") == "retry_attempt" + assert rs.attributes.get("fortifyroot.span.role") == "llm_attempt" assert rs.attributes.get("gen_ai.system") == "Anthropic" assert rs.attributes.get("gen_ai.request.model") == "claude-haiku-4-5" assert rs.attributes.get("gen_ai.operation.name") == "chat" @@ -225,7 +240,7 @@ def test_single_attempt_emits_one_span_with_marker(fresh_tracer): assert rs.attributes.get("gen_ai.usage.output_tokens") == 4 assert rs.attributes.get("server.address") == "api.anthropic.com" assert rs.attributes.get("server.port") == 443 - assert parent_exported.attributes.get(_FR_HAS_RETRY_ATTEMPT_CHILD_KEY) is True + assert parent_exported.attributes.get(_FR_HAS_ATTEMPT_CHILD_KEY) is True # --------------------------------------------------------------------------- @@ -249,6 +264,7 @@ def test_three_attempts_share_parent(fresh_tracer): parent_exported = next(s for s in spans if s.name == "anthropic.chat") retry_spans = _retry_spans(exporter) assert len(retry_spans) == 3 + _assert_attempt_sequence(retry_spans) parent_ids = {s.parent.span_id for s in retry_spans} assert parent_ids == {parent_exported.context.span_id} @@ -299,10 +315,10 @@ def test_marker_set_AFTER_first_attempt_not_at_parent_creation(fresh_tracer): tracer, exporter, _ = fresh_tracer request = _make_request() parent = tracer.start_span("anthropic.chat") - assert _FR_HAS_RETRY_ATTEMPT_CHILD_KEY not in dict(parent.attributes or {}) + assert _FR_HAS_ATTEMPT_CHILD_KEY not in dict(parent.attributes or {}) with trace.use_span(parent, end_on_exit=False): _sync_send_wrapper(lambda *a, **kw: _make_response(200), None, (request,), {}) - assert dict(parent.attributes or {}).get(_FR_HAS_RETRY_ATTEMPT_CHILD_KEY) is True + assert dict(parent.attributes or {}).get(_FR_HAS_ATTEMPT_CHILD_KEY) is True parent.end() @@ -313,7 +329,7 @@ def test_marker_NOT_set_when_no_attempts_fire(fresh_tracer): parent_exported = next( s for s in exporter.get_finished_spans() if s.name == "anthropic.chat" ) - assert _FR_HAS_RETRY_ATTEMPT_CHILD_KEY not in (parent_exported.attributes or {}) + assert _FR_HAS_ATTEMPT_CHILD_KEY not in (parent_exported.attributes or {}) # --------------------------------------------------------------------------- @@ -531,7 +547,7 @@ def explode(): parent_exported = next( s for s in exporter.get_finished_spans() if s.name == "anthropic.chat" ) - assert _FR_HAS_RETRY_ATTEMPT_CHILD_KEY not in (parent_exported.attributes or {}) + assert _FR_HAS_ATTEMPT_CHILD_KEY not in (parent_exported.attributes or {}) def test_non_2xx_response_with_usage_in_body_populates_usage_tokens(fresh_tracer): diff --git a/packages/opentelemetry-instrumentation-bedrock/opentelemetry/instrumentation/bedrock/retry_handler.py b/packages/opentelemetry-instrumentation-bedrock/opentelemetry/instrumentation/bedrock/retry_handler.py index 7fee3d8df1..5b59ee6be8 100644 --- a/packages/opentelemetry-instrumentation-bedrock/opentelemetry/instrumentation/bedrock/retry_handler.py +++ b/packages/opentelemetry-instrumentation-bedrock/opentelemetry/instrumentation/bedrock/retry_handler.py @@ -58,7 +58,10 @@ from opentelemetry import trace from opentelemetry.instrumentation.bedrock.version import __version__ from opentelemetry.instrumentation.fortifyroot import ( + FR_HAS_ATTEMPT_CHILD_KEY, is_framework_owned, + llm_attempt_attributes, + next_llm_attempt, ) from opentelemetry.semconv_ai import SUPPRESS_LANGUAGE_MODEL_INSTRUMENTATION_KEY from opentelemetry.trace import SpanKind, Status, StatusCode @@ -67,10 +70,10 @@ # ST-10 §4.4 / §4.5 constants. -_FR_RETRY_ATTEMPT_SPAN_NAME = "fortifyroot.bedrock.retry_attempt" _FR_SPAN_ROLE_KEY = "fortifyroot.span.role" -_FR_SPAN_ROLE_RETRY_ATTEMPT = "retry_attempt" -_FR_HAS_RETRY_ATTEMPT_CHILD_KEY = "fortifyroot.span.has_retry_attempt_child" +_FR_LLM_ATTEMPT_SPAN_NAME_PREFIX = "fortifyroot.bedrock" +_FR_SPAN_ROLE_LLM_ATTEMPT = "llm_attempt" +_FR_HAS_ATTEMPT_CHILD_KEY = FR_HAS_ATTEMPT_CHILD_KEY # Event-name patterns. ``*`` matches any operation under bedrock-runtime. _BEFORE_SEND_PATTERN = "before-send.bedrock-runtime.*" @@ -120,9 +123,9 @@ def _resolve_parent_span() -> Optional["trace.Span"]: def _set_parent_marker(parent_span: "trace.Span") -> None: try: - parent_span.set_attribute(_FR_HAS_RETRY_ATTEMPT_CHILD_KEY, True) + parent_span.set_attribute(_FR_HAS_ATTEMPT_CHILD_KEY, True) except Exception: - logger.debug("failed to set has_retry_attempt_child on parent", exc_info=True) + logger.debug("failed to set has_attempt_child on parent", exc_info=True) def _operation_from_event_name(event_name: str) -> str: @@ -199,11 +202,15 @@ def _start_attempt_span( event_name: str, parent_span: "trace.Span", ) -> "trace.Span": + span_name, attempt_number, is_retry = next_llm_attempt( + parent_span, + _FR_LLM_ATTEMPT_SPAN_NAME_PREFIX, + ) attrs: dict[str, Any] = { - _FR_SPAN_ROLE_KEY: _FR_SPAN_ROLE_RETRY_ATTEMPT, "gen_ai.system": "AWS", # matches existing Bedrock instrumentor "gen_ai.operation.name": _operation_from_event_name(event_name), } + attrs.update(llm_attempt_attributes(attempt_number, is_retry)) model = _model_from_request(request) if model: attrs["gen_ai.request.model"] = model @@ -212,7 +219,7 @@ def _start_attempt_span( tracer = trace.get_tracer(__name__, __version__, _tracer_provider) parent_ctx = trace.set_span_in_context(parent_span) span = tracer.start_span( - _FR_RETRY_ATTEMPT_SPAN_NAME, + span_name, kind=SpanKind.CLIENT, attributes=attrs, context=parent_ctx, @@ -271,7 +278,7 @@ def _finalize_success(span: "trace.Span", http_response: Any, parsed: Any) -> No span.set_attribute("error.type", err_type) span.set_status(Status(StatusCode.ERROR, f"http {status_code}")) except Exception: - logger.debug("failed to set success attrs on bedrock retry_attempt", exc_info=True) + logger.debug("failed to set success attrs on bedrock llm_attempt", exc_info=True) def _finalize_error(span: "trace.Span", exception: BaseException) -> None: @@ -292,7 +299,7 @@ def _finalize_error(span: "trace.Span", exception: BaseException) -> None: pass span.set_status(Status(StatusCode.ERROR, str(exception))) except Exception: - logger.debug("failed to set error attrs on bedrock retry_attempt", exc_info=True) + logger.debug("failed to set error attrs on bedrock llm_attempt", exc_info=True) def _before_send_hook(event_name: Optional[str] = None, request: Any = None, **_kwargs) -> None: @@ -476,12 +483,12 @@ def install_event_hooks_on_client(client: Any, tracer_provider=None) -> None: events.register( _BEFORE_SEND_PATTERN, _before_send_hook, - unique_id="fortifyroot.bedrock.retry_attempt.before-send", + unique_id="fortifyroot.bedrock.llm_attempt.before-send", ) events.register( _RESPONSE_RECEIVED_PATTERN, _response_received_hook, - unique_id="fortifyroot.bedrock.retry_attempt.response-received", + unique_id="fortifyroot.bedrock.llm_attempt.response-received", ) except Exception: logger.warning( @@ -510,14 +517,14 @@ def uninstall_event_hooks_on_client(client: Any) -> None: try: events.unregister( _BEFORE_SEND_PATTERN, - unique_id="fortifyroot.bedrock.retry_attempt.before-send", + unique_id="fortifyroot.bedrock.llm_attempt.before-send", ) except Exception: pass try: events.unregister( _RESPONSE_RECEIVED_PATTERN, - unique_id="fortifyroot.bedrock.retry_attempt.response-received", + unique_id="fortifyroot.bedrock.llm_attempt.response-received", ) except Exception: pass @@ -535,8 +542,8 @@ def _reset_tracer_provider_for_test() -> None: __all__ = [ "install_event_hooks_on_client", "uninstall_event_hooks_on_client", - "_FR_RETRY_ATTEMPT_SPAN_NAME", - "_FR_HAS_RETRY_ATTEMPT_CHILD_KEY", + "_FR_LLM_ATTEMPT_SPAN_NAME_PREFIX", + "_FR_HAS_ATTEMPT_CHILD_KEY", "_before_send_hook", "_response_received_hook", ] diff --git a/packages/opentelemetry-instrumentation-bedrock/tests/test_retry_attempt_emission.py b/packages/opentelemetry-instrumentation-bedrock/tests/test_retry_attempt_emission.py index c2f30abf6f..b5810a7146 100644 --- a/packages/opentelemetry-instrumentation-bedrock/tests/test_retry_attempt_emission.py +++ b/packages/opentelemetry-instrumentation-bedrock/tests/test_retry_attempt_emission.py @@ -30,8 +30,8 @@ from opentelemetry.instrumentation.bedrock.retry_handler import ( _CTX_SPAN_KEY, _CTX_TOKEN_KEY, - _FR_HAS_RETRY_ATTEMPT_CHILD_KEY, - _FR_RETRY_ATTEMPT_SPAN_NAME, + _FR_HAS_ATTEMPT_CHILD_KEY, + _FR_LLM_ATTEMPT_SPAN_NAME_PREFIX, _BEFORE_SEND_PATTERN, _RESPONSE_RECEIVED_PATTERN, _before_send_hook, @@ -40,6 +40,7 @@ uninstall_event_hooks_on_client, ) from opentelemetry.instrumentation.fortifyroot import ( + clear_attempt_counters_for_test, is_framework_owned, register_framework_attempt, retry_registry, @@ -74,8 +75,10 @@ def fresh_tracer(): @pytest.fixture(autouse=True) def reset_registry(): retry_registry._reset_for_test() + clear_attempt_counters_for_test() yield retry_registry._reset_for_test() + clear_attempt_counters_for_test() def _make_request(model: str = "anthropic.claude-haiku-4-5", @@ -98,7 +101,18 @@ def _make_http_response(status_code: int = 200, request_id: str = "amzn-req-1") def _retry_spans(exporter: InMemorySpanExporter): return [s for s in exporter.get_finished_spans() - if s.name == _FR_RETRY_ATTEMPT_SPAN_NAME] + if s.name.startswith(f"{_FR_LLM_ATTEMPT_SPAN_NAME_PREFIX}.attempt_")] + + +def _assert_attempt_sequence(spans): + by_number = sorted( + spans, + key=lambda s: int(s.attributes.get("fortifyroot.attempt.number") or -1), + ) + for expected, span in enumerate(by_number, start=1): + assert span.name == f"{_FR_LLM_ATTEMPT_SPAN_NAME_PREFIX}.attempt_{expected}" + assert span.attributes.get("fortifyroot.attempt.number") == expected + assert span.attributes.get("fortifyroot.attempt.is_retry") is (expected > 1) # --------------------------------------------------------------------------- @@ -124,7 +138,7 @@ def test_install_event_hooks_registers_both_patterns(): # Each registration uses a stable unique_id so botocore dedups. for call in register_calls: assert "unique_id" in call.kwargs, "unique_id must be supplied for dedup" - assert call.kwargs["unique_id"].startswith("fortifyroot.bedrock.retry_attempt.") + assert call.kwargs["unique_id"].startswith("fortifyroot.bedrock.llm_attempt.") def test_uninstall_event_hooks_unregisters_both_patterns(): @@ -182,8 +196,9 @@ def test_single_attempt_emits_one_span_with_marker(fresh_tracer): retry_spans = _retry_spans(exporter) assert len(retry_spans) == 1 rs = retry_spans[0] + _assert_attempt_sequence(retry_spans) assert rs.parent.span_id == parent_exported.context.span_id - assert rs.attributes.get("fortifyroot.span.role") == "retry_attempt" + assert rs.attributes.get("fortifyroot.span.role") == "llm_attempt" assert rs.attributes.get("gen_ai.system") == "AWS" assert rs.attributes.get("gen_ai.request.model") == "anthropic.claude-haiku-4-5" assert rs.attributes.get("gen_ai.operation.name") == "chat" @@ -193,7 +208,7 @@ def test_single_attempt_emits_one_span_with_marker(fresh_tracer): assert rs.attributes.get("gen_ai.usage.output_tokens") == 7 assert rs.attributes.get("server.address") == "bedrock-runtime.us-east-1.amazonaws.com" assert rs.attributes.get("server.port") == 443 - assert parent_exported.attributes.get(_FR_HAS_RETRY_ATTEMPT_CHILD_KEY) is True + assert parent_exported.attributes.get(_FR_HAS_ATTEMPT_CHILD_KEY) is True # --------------------------------------------------------------------------- @@ -226,6 +241,7 @@ def test_three_attempts_share_parent(fresh_tracer): parent_exported = next(s for s in spans if s.name == "bedrock.converse") retry_spans = _retry_spans(exporter) assert len(retry_spans) == 3 + _assert_attempt_sequence(retry_spans) parent_ids = {s.parent.span_id for s in retry_spans} assert parent_ids == {parent_exported.context.span_id} @@ -289,7 +305,7 @@ def test_marker_NOT_set_when_no_attempts_fire(fresh_tracer): parent_exported = next( s for s in exporter.get_finished_spans() if s.name == "bedrock.completion" ) - assert _FR_HAS_RETRY_ATTEMPT_CHILD_KEY not in (parent_exported.attributes or {}) + assert _FR_HAS_ATTEMPT_CHILD_KEY not in (parent_exported.attributes or {}) # --------------------------------------------------------------------------- diff --git a/packages/opentelemetry-instrumentation-fortifyroot/opentelemetry/instrumentation/fortifyroot/__init__.py b/packages/opentelemetry-instrumentation-fortifyroot/opentelemetry/instrumentation/fortifyroot/__init__.py index d43e368805..3444b6da1a 100644 --- a/packages/opentelemetry-instrumentation-fortifyroot/opentelemetry/instrumentation/fortifyroot/__init__.py +++ b/packages/opentelemetry-instrumentation-fortifyroot/opentelemetry/instrumentation/fortifyroot/__init__.py @@ -29,6 +29,17 @@ register_framework_attempt, unregister_framework_attempt, ) +from opentelemetry.instrumentation.fortifyroot.attempt_naming import ( + FR_ATTEMPT_IS_RETRY_KEY, + FR_ATTEMPT_NUMBER_KEY, + FR_HAS_ATTEMPT_CHILD_KEY, + FR_SPAN_ROLE_KEY, + FR_SPAN_ROLE_LLM_ATTEMPT, + clear_attempt_counters_for_test, + first_llm_attempt, + llm_attempt_attributes, + next_llm_attempt, +) from opentelemetry.instrumentation.fortifyroot.text_streaming import ( CompletionTextStreamGroup, ) @@ -47,7 +58,16 @@ "SAFETY_EVENT_NAME", "HANDLER_LOCK", "clear_framework_attempts_for_thread", + "clear_attempt_counters_for_test", + "FR_ATTEMPT_IS_RETRY_KEY", + "FR_ATTEMPT_NUMBER_KEY", + "FR_HAS_ATTEMPT_CHILD_KEY", + "FR_SPAN_ROLE_KEY", + "FR_SPAN_ROLE_LLM_ATTEMPT", "is_framework_owned", + "first_llm_attempt", + "llm_attempt_attributes", + "next_llm_attempt", "register_framework_attempt", "unregister_framework_attempt", "SafetyContext", diff --git a/packages/opentelemetry-instrumentation-fortifyroot/opentelemetry/instrumentation/fortifyroot/attempt_naming.py b/packages/opentelemetry-instrumentation-fortifyroot/opentelemetry/instrumentation/fortifyroot/attempt_naming.py new file mode 100644 index 0000000000..b576049e03 --- /dev/null +++ b/packages/opentelemetry-instrumentation-fortifyroot/opentelemetry/instrumentation/fortifyroot/attempt_naming.py @@ -0,0 +1,107 @@ +"""Shared naming helpers for FR per-LLM-attempt spans.""" + +from __future__ import annotations + +import logging +import threading +import time +from typing import Any + +logger = logging.getLogger(__name__) + +FR_SPAN_ROLE_KEY = "fortifyroot.span.role" +FR_SPAN_ROLE_LLM_ATTEMPT = "llm_attempt" +FR_HAS_ATTEMPT_CHILD_KEY = "fortifyroot.span.has_attempt_child" +FR_ATTEMPT_NUMBER_KEY = "fortifyroot.attempt.number" +FR_ATTEMPT_IS_RETRY_KEY = "fortifyroot.attempt.is_retry" + +_COUNTER_LOCK = threading.Lock() +_COUNTERS: dict[str, tuple[int, float]] = {} +_COUNTER_MAX = 4096 +_COUNTER_TTL_SEC = 3600.0 +_COUNTER_EVICT_BATCH = 1024 + + +def _now() -> float: + return time.monotonic() + + +def _parent_key(parent_span: Any) -> str | None: + try: + ctx = parent_span.get_span_context() + if ctx is None or not ctx.is_valid: + return None + return f"{ctx.trace_id:032x}:{ctx.span_id:016x}" + except Exception: + return None + + +def _evict_stale_locked(now: float) -> None: + cutoff = now - _COUNTER_TTL_SEC + stale = [key for key, (_, seen_at) in _COUNTERS.items() if seen_at < cutoff] + for key in stale: + _COUNTERS.pop(key, None) + + +def _enforce_max_locked() -> None: + if len(_COUNTERS) <= _COUNTER_MAX: + return + items = sorted(_COUNTERS.items(), key=lambda item: item[1][1]) + for key, _ in items[:_COUNTER_EVICT_BATCH]: + _COUNTERS.pop(key, None) + logger.warning( + "fortifyroot attempt_naming: cap-evicted %d parent attempt counters " + "(max=%d)", + min(_COUNTER_EVICT_BATCH, len(items)), + _COUNTER_MAX, + ) + + +def next_llm_attempt(parent_span: Any, span_name_prefix: str) -> tuple[str, int, bool]: + """Return the next span name and ordinal for a parent LLM span. + + ``span_name_prefix`` should be the provider/framework prefix without + an attempt suffix, for example ``fortifyroot.openai``. The returned + span name is ``.attempt_``. + + Use this only when ``parent_span`` is the logical retry-group span. + Framework integrations that can only see a broad workflow parent should + use ``first_llm_attempt`` instead, otherwise unrelated LLM calls under the + same workflow can be mislabeled as retries. + """ + key = _parent_key(parent_span) + now = _now() + if key is None: + return f"{span_name_prefix}.attempt_1", 1, False + + with _COUNTER_LOCK: + _evict_stale_locked(now) + last, _ = _COUNTERS.get(key, (0, now)) + number = last + 1 + _COUNTERS[key] = (number, now) + _enforce_max_locked() + + return f"{span_name_prefix}.attempt_{number}", number, number > 1 + + +def first_llm_attempt(span_name_prefix: str) -> tuple[str, int, bool]: + """Return conservative attempt metadata for one observed LLM call. + + Some frameworks expose a broad workflow parent but no stable logical + retry-group key. In those cases the truthful signal is "this observed + call is an LLM attempt"; it is not safe to infer retry ordinals. + """ + return f"{span_name_prefix}.attempt_1", 1, False + + +def llm_attempt_attributes(number: int, is_retry: bool) -> dict[str, Any]: + return { + FR_SPAN_ROLE_KEY: FR_SPAN_ROLE_LLM_ATTEMPT, + FR_ATTEMPT_NUMBER_KEY: number, + FR_ATTEMPT_IS_RETRY_KEY: is_retry, + } + + +def clear_attempt_counters_for_test() -> None: + with _COUNTER_LOCK: + _COUNTERS.clear() diff --git a/packages/opentelemetry-instrumentation-fortifyroot/tests/test_attempt_naming.py b/packages/opentelemetry-instrumentation-fortifyroot/tests/test_attempt_naming.py new file mode 100644 index 0000000000..2177dcd9cb --- /dev/null +++ b/packages/opentelemetry-instrumentation-fortifyroot/tests/test_attempt_naming.py @@ -0,0 +1,142 @@ +from types import SimpleNamespace + +from opentelemetry.instrumentation.fortifyroot import attempt_naming +from opentelemetry.instrumentation.fortifyroot.attempt_naming import ( + FR_ATTEMPT_IS_RETRY_KEY, + FR_ATTEMPT_NUMBER_KEY, + FR_SPAN_ROLE_KEY, + FR_SPAN_ROLE_LLM_ATTEMPT, + clear_attempt_counters_for_test, + first_llm_attempt, + llm_attempt_attributes, + next_llm_attempt, +) + + +class _FakeSpan: + def __init__(self, trace_id: int, span_id: int, is_valid: bool = True): + self._ctx = SimpleNamespace( + trace_id=trace_id, + span_id=span_id, + is_valid=is_valid, + ) + + def get_span_context(self): + return self._ctx + + +def setup_function(): + clear_attempt_counters_for_test() + + +def teardown_function(): + clear_attempt_counters_for_test() + + +def test_next_llm_attempt_counts_per_parent(): + parent = _FakeSpan(trace_id=0xAA, span_id=0x01) + + assert next_llm_attempt(parent, "fortifyroot.openai") == ( + "fortifyroot.openai.attempt_1", + 1, + False, + ) + assert next_llm_attempt(parent, "fortifyroot.openai") == ( + "fortifyroot.openai.attempt_2", + 2, + True, + ) + assert next_llm_attempt(parent, "fortifyroot.openai") == ( + "fortifyroot.openai.attempt_3", + 3, + True, + ) + + +def test_next_llm_attempt_isolated_by_parent_context(): + parent_a = _FakeSpan(trace_id=0xAA, span_id=0x01) + parent_b = _FakeSpan(trace_id=0xAA, span_id=0x02) + + assert next_llm_attempt(parent_a, "fortifyroot.openai")[1:] == (1, False) + assert next_llm_attempt(parent_b, "fortifyroot.openai")[1:] == (1, False) + assert next_llm_attempt(parent_a, "fortifyroot.openai")[1:] == (2, True) + + +def test_invalid_parent_falls_back_to_first_attempt_without_counter(): + parent = _FakeSpan(trace_id=0, span_id=0, is_valid=False) + + assert next_llm_attempt(parent, "fortifyroot.openai") == ( + "fortifyroot.openai.attempt_1", + 1, + False, + ) + assert next_llm_attempt(parent, "fortifyroot.openai") == ( + "fortifyroot.openai.attempt_1", + 1, + False, + ) + + +def test_ttl_eviction_resets_parent_counter(monkeypatch): + parent = _FakeSpan(trace_id=0xAA, span_id=0x01) + + monkeypatch.setattr(attempt_naming, "_now", lambda: 100.0) + assert next_llm_attempt(parent, "fortifyroot.openai")[1:] == (1, False) + assert next_llm_attempt(parent, "fortifyroot.openai")[1:] == (2, True) + + monkeypatch.setattr( + attempt_naming, + "_now", + lambda: 100.0 + attempt_naming._COUNTER_TTL_SEC + 1.0, + ) + assert next_llm_attempt(parent, "fortifyroot.openai") == ( + "fortifyroot.openai.attempt_1", + 1, + False, + ) + + +def test_cap_eviction_drops_oldest_parent_counter(monkeypatch, caplog): + parent_a = _FakeSpan(trace_id=0xAA, span_id=0x01) + parent_b = _FakeSpan(trace_id=0xAA, span_id=0x02) + parent_c = _FakeSpan(trace_id=0xAA, span_id=0x03) + + monkeypatch.setattr(attempt_naming, "_COUNTER_MAX", 2) + monkeypatch.setattr(attempt_naming, "_COUNTER_EVICT_BATCH", 1) + now = 100.0 + monkeypatch.setattr(attempt_naming, "_now", lambda: now) + + assert next_llm_attempt(parent_a, "fortifyroot.openai")[1:] == (1, False) + now += 1.0 + assert next_llm_attempt(parent_a, "fortifyroot.openai")[1:] == (2, True) + now += 1.0 + assert next_llm_attempt(parent_b, "fortifyroot.openai")[1:] == (1, False) + now += 1.0 + + with caplog.at_level("WARNING", logger=attempt_naming.logger.name): + assert next_llm_attempt(parent_c, "fortifyroot.openai")[1:] == (1, False) + + assert "cap-evicted 1 parent attempt counters" in caplog.text + + now += 1.0 + assert next_llm_attempt(parent_a, "fortifyroot.openai") == ( + "fortifyroot.openai.attempt_1", + 1, + False, + ) + + +def test_first_llm_attempt_is_conservative(): + assert first_llm_attempt("fortifyroot.langchain") == ( + "fortifyroot.langchain.attempt_1", + 1, + False, + ) + + +def test_llm_attempt_attributes(): + attrs = llm_attempt_attributes(2, True) + + assert attrs[FR_SPAN_ROLE_KEY] == FR_SPAN_ROLE_LLM_ATTEMPT + assert attrs[FR_ATTEMPT_NUMBER_KEY] == 2 + assert attrs[FR_ATTEMPT_IS_RETRY_KEY] is True diff --git a/packages/opentelemetry-instrumentation-langchain/opentelemetry/instrumentation/langchain/__init__.py b/packages/opentelemetry-instrumentation-langchain/opentelemetry/instrumentation/langchain/__init__.py index d5b2050340..98925bf39c 100644 --- a/packages/opentelemetry-instrumentation-langchain/opentelemetry/instrumentation/langchain/__init__.py +++ b/packages/opentelemetry-instrumentation-langchain/opentelemetry/instrumentation/langchain/__init__.py @@ -106,7 +106,7 @@ def _instrument(self, **kwargs): # existing Traceloop handler. Per ST-10.0 C2 POC findings, this # captures per-HTTP-attempt callbacks on framework-layer retry # paths (e.g. Runnable.with_retry) and emits one - # fortifyroot.langchain.retry_attempt sibling span per attempt + # fortifyroot.langchain.attempt_ sibling span per attempt # under the parent_run_id's span. The handler gets the Traceloop # handler as a CONSTRUCTION-time reference so it can resolve the # workflow parent via Traceloop's run_id-keyed ``spans`` dict diff --git a/packages/opentelemetry-instrumentation-langchain/opentelemetry/instrumentation/langchain/retry_handler.py b/packages/opentelemetry-instrumentation-langchain/opentelemetry/instrumentation/langchain/retry_handler.py index 9a933317bd..28236fc265 100644 --- a/packages/opentelemetry-instrumentation-langchain/opentelemetry/instrumentation/langchain/retry_handler.py +++ b/packages/opentelemetry-instrumentation-langchain/opentelemetry/instrumentation/langchain/retry_handler.py @@ -11,14 +11,18 @@ retries (e.g. ``ChatOpenAI(max_retries=N)``) fire callbacks once per logical call → §4.4.2 coverage limitation applies. - Use ``run_id`` as the per-attempt correlation key. - - Emit ``fortifyroot.langchain.retry_attempt`` sibling spans + - Emit ``fortifyroot.langchain.attempt_`` sibling spans under the parent_run_id's span (NOT under the per-LLM Traceloop span — siblings need to share a parent for RetryDetectorProc's grouping to work). - §4.7.1: register/unregister framework-attempt tokens so direct-SDK wrappers (ST-10.4) suppress their own emission. - - §4.5: set has_retry_attempt_child=true on the parent AFTER - the first qualifying retry_attempt successfully starts. + - §4.5: set has_attempt_child=true on the parent AFTER + the first qualifying llm_attempt successfully starts. + - Attempt numbering is conservative for LangChain: callbacks expose + a workflow parent that can contain multiple unrelated LLM calls, so + this handler emits attempt_1 / is_retry=false for each observed LLM + call rather than inferring retry ordinals from the shared parent. This handler is registered ALONGSIDE the existing TraceloopCallbackHandler (not as a replacement) — it captures @@ -26,7 +30,7 @@ safety-critical. The existing handler continues to do prompt / completion attribute capture; the §4.5 backend dedup correctly skips its per-attempt spans because they're non-retry siblings of -the retry_attempts emitted here. +the llm_attempts emitted here. """ from __future__ import annotations @@ -41,6 +45,9 @@ from langchain_core.callbacks import BaseCallbackHandler from opentelemetry import trace from opentelemetry.instrumentation.fortifyroot import ( + FR_HAS_ATTEMPT_CHILD_KEY, + first_llm_attempt, + llm_attempt_attributes, register_framework_attempt, unregister_framework_attempt, ) @@ -58,12 +65,12 @@ logger = logging.getLogger(__name__) # ST-10 §4.4: per-attempt sibling span name + role. -_FR_RETRY_ATTEMPT_SPAN_NAME = "fortifyroot.langchain.retry_attempt" _FR_SPAN_ROLE_KEY = "fortifyroot.span.role" -_FR_SPAN_ROLE_RETRY_ATTEMPT = "retry_attempt" +_FR_LLM_ATTEMPT_SPAN_NAME_PREFIX = "fortifyroot.langchain" +_FR_SPAN_ROLE_LLM_ATTEMPT = "llm_attempt" # ST-10 §4.5 parent marker. -_FR_HAS_RETRY_ATTEMPT_CHILD_KEY = "fortifyroot.span.has_retry_attempt_child" +_FR_HAS_ATTEMPT_CHILD_KEY = FR_HAS_ATTEMPT_CHILD_KEY # ---------------------------------------------------------------------- @@ -384,11 +391,14 @@ def _start_retry_attempt( routed_provider = _resolve_routed_provider(serialized, invocation_params) model = _resolve_model(serialized, invocation_params) + span_name, attempt_number, is_retry = first_llm_attempt( + _FR_LLM_ATTEMPT_SPAN_NAME_PREFIX, + ) attrs: dict[str, Any] = { - _FR_SPAN_ROLE_KEY: _FR_SPAN_ROLE_RETRY_ATTEMPT, "gen_ai.operation.name": "chat", } + attrs.update(llm_attempt_attributes(attempt_number, is_retry)) if routed_provider: attrs["gen_ai.system"] = routed_provider if model: @@ -398,7 +408,7 @@ def _start_retry_attempt( tracer = trace.get_tracer(__name__, __version__) parent_ctx = set_span_in_context(parent_span) span = tracer.start_span( - _FR_RETRY_ATTEMPT_SPAN_NAME, + span_name, kind=SpanKind.CLIENT, attributes=attrs, context=parent_ctx, @@ -437,13 +447,13 @@ def _start_retry_attempt( "ended": False, } - # §4.5 marker timing: set AFTER the first qualifying retry_attempt + # §4.5 marker timing: set AFTER the first qualifying llm_attempt # has successfully started under this parent. Idempotent — setting # the attribute twice on the same parent is a no-op. try: - parent_span.set_attribute(_FR_HAS_RETRY_ATTEMPT_CHILD_KEY, True) + parent_span.set_attribute(_FR_HAS_ATTEMPT_CHILD_KEY, True) except Exception: - logger.debug("failed to set has_retry_attempt_child on parent", exc_info=True) + logger.debug("failed to set has_attempt_child on parent", exc_info=True) def _finalize_retry_attempt( @@ -525,7 +535,7 @@ def _finalize_retry_attempt( class _FortifyRootRetryHandler(BaseCallbackHandler): """LangChain BaseCallbackHandler that emits one - fortifyroot.langchain.retry_attempt sibling span per LLM-start + fortifyroot.langchain.attempt_ sibling span per LLM-start callback invocation. Hooks BOTH on_chat_model_start (chat models — F1 finding from ST-10.0 C2 POC) AND on_llm_start (legacy completion LLMs). @@ -648,6 +658,6 @@ def _reset_state_for_test() -> None: __all__ = [ "_FortifyRootRetryHandler", - "_FR_RETRY_ATTEMPT_SPAN_NAME", - "_FR_HAS_RETRY_ATTEMPT_CHILD_KEY", + "_FR_LLM_ATTEMPT_SPAN_NAME_PREFIX", + "_FR_HAS_ATTEMPT_CHILD_KEY", ] diff --git a/packages/opentelemetry-instrumentation-langchain/tests/test_retry_attempt_emission.py b/packages/opentelemetry-instrumentation-langchain/tests/test_retry_attempt_emission.py index 7b4878219a..8a2cf7ba4d 100644 --- a/packages/opentelemetry-instrumentation-langchain/tests/test_retry_attempt_emission.py +++ b/packages/opentelemetry-instrumentation-langchain/tests/test_retry_attempt_emission.py @@ -6,13 +6,13 @@ - on_chat_model_start path (chat models — F1 finding from ST-10.0 C2 POC). - on_llm_start path (legacy completion LLMs). - - Single-attempt happy path: 1 retry_attempt span, parent has - has_retry_attempt_child=true, retry_attempt has gen_ai.system / + - Single-attempt happy path: 1 llm_attempt span, parent has + has_attempt_child=true, llm_attempt has gen_ai.system / gen_ai.request.model / role attributes. - - Multi-attempt retry path: 3 sequential attempts sharing a - parent_run_id → 3 retry_attempt SIBLINGS under one parent - span. This is the structural shape RetryDetectorProc requires - for retry-loop detection (siblings, not nested). + - Multiple callback invocations sharing a parent_run_id → 3 + llm_attempt SIBLINGS under one parent span. LangChain numbering is + conservative: shared workflow parents can contain unrelated LLM calls, + so each emitted span remains attempt_1 / is_retry=false. - Marker timing (§4.5): parent gets the marker only AFTER the first attempt's start callback fires. - §4.7.1 token registration symmetry. @@ -33,8 +33,8 @@ from opentelemetry.instrumentation.langchain import LangchainInstrumentor from opentelemetry.instrumentation.langchain.retry_handler import ( _FortifyRootRetryHandler, - _FR_HAS_RETRY_ATTEMPT_CHILD_KEY, - _FR_RETRY_ATTEMPT_SPAN_NAME, + _FR_HAS_ATTEMPT_CHILD_KEY, + _FR_LLM_ATTEMPT_SPAN_NAME_PREFIX, _reset_state_for_test, ) from opentelemetry.sdk.trace import TracerProvider @@ -77,6 +77,13 @@ def reset_state(): _reset_state_for_test() +def _assert_conservative_attempts(spans): + for span in spans: + assert span.name == f"{_FR_LLM_ATTEMPT_SPAN_NAME_PREFIX}.attempt_1" + assert span.attributes.get("fortifyroot.attempt.number") == 1 + assert span.attributes.get("fortifyroot.attempt.is_retry") is False + + # --------------------------------------------------------------------------- # Instrumentor symmetry. # --------------------------------------------------------------------------- @@ -310,13 +317,14 @@ def test_on_chat_model_start_emits_retry_attempt(fresh_tracer): retry_spans = [ s for s in exporter.get_finished_spans() - if s.name == _FR_RETRY_ATTEMPT_SPAN_NAME + if s.name.startswith(f"{_FR_LLM_ATTEMPT_SPAN_NAME_PREFIX}.attempt_") ] assert len(retry_spans) == 1, ( f"on_chat_model_start MUST produce a retry_attempt span; got {len(retry_spans)}" ) + _assert_conservative_attempts(retry_spans) rs = retry_spans[0] - assert rs.attributes.get("fortifyroot.span.role") == "retry_attempt" + assert rs.attributes.get("fortifyroot.span.role") == "llm_attempt" assert rs.attributes.get("gen_ai.system") == "openai", ( "gen_ai.system must be the routed provider (openai), NOT 'langchain'" ) @@ -346,9 +354,10 @@ def test_on_llm_start_emits_retry_attempt(fresh_tracer): retry_spans = [ s for s in exporter.get_finished_spans() - if s.name == _FR_RETRY_ATTEMPT_SPAN_NAME + if s.name.startswith(f"{_FR_LLM_ATTEMPT_SPAN_NAME_PREFIX}.attempt_") ] assert len(retry_spans) == 1 + _assert_conservative_attempts(retry_spans) assert retry_spans[0].attributes.get("gen_ai.system") == "anthropic" assert retry_spans[0].attributes.get("gen_ai.request.model") == "claude-haiku-4-5" assert retry_spans[0].attributes.get("gen_ai.prompt.0.role") == "user" @@ -362,7 +371,7 @@ def test_on_llm_start_emits_retry_attempt(fresh_tracer): def test_single_attempt_emits_one_sibling_with_marker(fresh_tracer): """A single attempt produces ONE retry_attempt span, parent under the workflow, and the workflow span carries - has_retry_attempt_child=true.""" + has_attempt_child=true.""" tracer, exporter, _ = fresh_tracer handler = _FortifyRootRetryHandler() @@ -390,12 +399,13 @@ def test_single_attempt_emits_one_sibling_with_marker(fresh_tracer): spans = exporter.get_finished_spans() parent_exported = next(s for s in spans if s.name == "workflow") - retry_span = next(s for s in spans if s.name == _FR_RETRY_ATTEMPT_SPAN_NAME) + retry_span = next(s for s in spans if s.name.startswith(f"{_FR_LLM_ATTEMPT_SPAN_NAME_PREFIX}.attempt_")) + _assert_conservative_attempts([retry_span]) assert retry_span.parent.span_id == parent_exported.context.span_id, ( "retry_attempt MUST be a child of the workflow span (sibling-grouping requires this)" ) - assert parent_exported.attributes.get(_FR_HAS_RETRY_ATTEMPT_CHILD_KEY) is True + assert parent_exported.attributes.get(_FR_HAS_ATTEMPT_CHILD_KEY) is True assert retry_span.attributes.get("gen_ai.response.id") == "resp-abc" assert retry_span.attributes.get("gen_ai.usage.input_tokens") == 10 assert retry_span.attributes.get("gen_ai.usage.output_tokens") == 5 @@ -453,9 +463,10 @@ def __init__(self, status_code: int): spans = exporter.get_finished_spans() parent_exported = next(s for s in spans if s.name == "workflow") - retry_spans = [s for s in spans if s.name == _FR_RETRY_ATTEMPT_SPAN_NAME] + retry_spans = [s for s in spans if s.name.startswith(f"{_FR_LLM_ATTEMPT_SPAN_NAME_PREFIX}.attempt_")] assert len(retry_spans) == 3, f"expected 3 retry_attempt spans, got {len(retry_spans)}" + _assert_conservative_attempts(retry_spans) # All 3 must share the workflow as parent — this is the sibling # invariant RetryDetectorProc relies on. @@ -484,7 +495,7 @@ def __init__(self, status_code: int): # --------------------------------------------------------------------------- def test_marker_set_AFTER_first_attempt_not_at_parent_creation(fresh_tracer): - """Per §4.5 marker-timing: parent gets has_retry_attempt_child=true + """Per §4.5 marker-timing: parent gets has_attempt_child=true only AFTER the first attempt's start callback fires, NOT at parent creation.""" tracer, exporter, _ = fresh_tracer @@ -492,7 +503,7 @@ def test_marker_set_AFTER_first_attempt_not_at_parent_creation(fresh_tracer): parent = tracer.start_span("workflow") # Before any retry_attempt: no marker. - assert _FR_HAS_RETRY_ATTEMPT_CHILD_KEY not in dict(parent.attributes or {}) + assert _FR_HAS_ATTEMPT_CHILD_KEY not in dict(parent.attributes or {}) with trace.use_span(parent, end_on_exit=False): run_id = uuid4() @@ -505,7 +516,7 @@ def test_marker_set_AFTER_first_attempt_not_at_parent_creation(fresh_tracer): invocation_params={"model": "gpt-4o-mini"}, ) # After first attempt start: parent has the marker. - assert dict(parent.attributes or {}).get(_FR_HAS_RETRY_ATTEMPT_CHILD_KEY) is True + assert dict(parent.attributes or {}).get(_FR_HAS_ATTEMPT_CHILD_KEY) is True handler.on_llm_end(_FakeLLMResult(model="gpt-4o-mini"), run_id=run_id) parent.end() @@ -523,7 +534,7 @@ def test_marker_NOT_set_when_no_attempts_fire(fresh_tracer): parent_exported = next( s for s in exporter.get_finished_spans() if s.name == "workflow" ) - assert _FR_HAS_RETRY_ATTEMPT_CHILD_KEY not in (parent_exported.attributes or {}) + assert _FR_HAS_ATTEMPT_CHILD_KEY not in (parent_exported.attributes or {}) # --------------------------------------------------------------------------- @@ -576,7 +587,7 @@ def test_no_parent_does_not_emit_orphan_retry_attempt(fresh_tracer): retry_spans = [ s for s in exporter.get_finished_spans() - if s.name == _FR_RETRY_ATTEMPT_SPAN_NAME + if s.name.startswith(f"{_FR_LLM_ATTEMPT_SPAN_NAME_PREFIX}.attempt_") ] assert len(retry_spans) == 0 @@ -607,7 +618,7 @@ def test_double_finalize_does_not_double_end(fresh_tracer): retry_spans = [ s for s in exporter.get_finished_spans() - if s.name == _FR_RETRY_ATTEMPT_SPAN_NAME + if s.name.startswith(f"{_FR_LLM_ATTEMPT_SPAN_NAME_PREFIX}.attempt_") ] assert len(retry_spans) == 1, ( f"expected exactly 1 retry_attempt span (idempotent finalize); got {len(retry_spans)}" diff --git a/packages/opentelemetry-instrumentation-litellm/opentelemetry/instrumentation/litellm/__init__.py b/packages/opentelemetry-instrumentation-litellm/opentelemetry/instrumentation/litellm/__init__.py index 58d2fcd587..4e4f98e85a 100644 --- a/packages/opentelemetry-instrumentation-litellm/opentelemetry/instrumentation/litellm/__init__.py +++ b/packages/opentelemetry-instrumentation-litellm/opentelemetry/instrumentation/litellm/__init__.py @@ -10,7 +10,10 @@ from opentelemetry import context as context_api from opentelemetry import trace from opentelemetry.instrumentation.fortifyroot import ( + FR_HAS_ATTEMPT_CHILD_KEY, get_object_value, + llm_attempt_attributes, + next_llm_attempt, register_framework_attempt, unregister_framework_attempt, ) @@ -58,15 +61,15 @@ # ``buildSafetyWrapperDedupeSet`` cannot observe siblings across batches. _FR_HAS_NATIVE_OTEL_CHILD_KEY = "fortifyroot.span.has_native_otel_child" -# ST-10 §4.5: marker on parent set AFTER the first qualifying retry_attempt +# ST-10 §4.5: marker on parent set AFTER the first qualifying llm_attempt # child has started, so the FR backend's LLMUsageExtractor can dedup the -# parent + non-retry siblings cross-batch (mirrors has_native_otel_child). -_FR_HAS_RETRY_ATTEMPT_CHILD_KEY = "fortifyroot.span.has_retry_attempt_child" +# parent + non-attempt siblings cross-batch (mirrors has_native_otel_child). +_FR_HAS_ATTEMPT_CHILD_KEY = FR_HAS_ATTEMPT_CHILD_KEY # ST-10 §4.4: per-attempt sibling span emitted by _FortifyRootRetryEmitter # under the safety_wrapper parent. -_FR_RETRY_ATTEMPT_SPAN_NAME = "fortifyroot.litellm.retry_attempt" -_FR_SPAN_ROLE_RETRY_ATTEMPT = "retry_attempt" +_FR_LLM_ATTEMPT_SPAN_NAME_PREFIX = "fortifyroot.litellm" +_FR_SPAN_ROLE_LLM_ATTEMPT = "llm_attempt" _FR_COMPLETION_SAFETY_MARKER = "_fortifyroot_completion_safety_applied" _FR_COMPLETION_SAFETY_FALLBACK_MARKERS: set[tuple[int, type]] = set() _FR_COMPLETION_SAFETY_MARKERS_LOCK = threading.Lock() @@ -201,7 +204,7 @@ async def _ensure_completion_safety_applied_async( # LiteLLM CustomLogger. # ---------------------------------------------------------------------- # -# The emitter opens one ``fortifyroot.litellm.retry_attempt`` sibling span +# The emitter opens one ``fortifyroot.litellm.attempt_`` sibling span # per attempt-start callback fired by LiteLLM, and ends it on the matching # success/failure callback. Per ST-10.0 C1 source-verified findings, this # fires per-attempt only on the ``completion_with_retries(num_retries=N)`` @@ -466,11 +469,15 @@ def _start_retry_attempt_span(kwargs, *, is_text_completion: bool = False) -> No routed_provider = _resolve_routed_provider(kwargs) or "litellm" model = kwargs.get("model") operation = "text_completion" if (is_text_completion or kwargs.get("text_completion")) else "chat" + span_name, attempt_number, is_retry = next_llm_attempt( + parent, + _FR_LLM_ATTEMPT_SPAN_NAME_PREFIX, + ) attrs = { GenAIAttributes.GEN_AI_SYSTEM: routed_provider, GenAIAttributes.GEN_AI_OPERATION_NAME: operation, - _FR_SPAN_ROLE_KEY: _FR_SPAN_ROLE_RETRY_ATTEMPT, } + attrs.update(llm_attempt_attributes(attempt_number, is_retry)) if model is not None: attrs[GenAIAttributes.GEN_AI_REQUEST_MODEL] = str(model) server = _server_address(kwargs) @@ -488,7 +495,7 @@ def _start_retry_attempt_span(kwargs, *, is_text_completion: bool = False) -> No # safer for future refactoring). parent_ctx = set_span_in_context(parent) span = tracer.start_span( - _FR_RETRY_ATTEMPT_SPAN_NAME, + span_name, kind=SpanKind.CLIENT, attributes=attrs, context=parent_ctx, @@ -518,18 +525,18 @@ def _start_retry_attempt_span(kwargs, *, is_text_completion: bool = False) -> No "ended": False, } - # §4.5 marker timing: set has_retry_attempt_child=true on the - # parent ONLY AFTER the first qualifying retry_attempt has + # §4.5 marker timing: set has_attempt_child=true on the + # parent ONLY AFTER the first qualifying llm_attempt has # successfully started. We just succeeded; mark the parent now. # Idempotent: setting the attribute twice on the same parent is a # no-op (OTel deduplicates). try: - parent.set_attribute(_FR_HAS_RETRY_ATTEMPT_CHILD_KEY, True) + parent.set_attribute(_FR_HAS_ATTEMPT_CHILD_KEY, True) except Exception: # Some span impls (e.g. a NonRecordingSpan during shutdown) # may reject set_attribute. Non-fatal; the in-batch dedup # path still fires from the in-batch retry_attempt children. - logger.debug("Failed to set has_retry_attempt_child on parent", exc_info=True) + logger.debug("Failed to set has_attempt_child on parent", exc_info=True) def _finalize_retry_attempt_span( @@ -612,7 +619,7 @@ def _finalize_retry_attempt_span( class _FortifyRootRetryEmitter(_LiteLLMCustomLoggerBase): """Second LiteLLM CustomLogger. Emits per-HTTP-attempt sibling - spans (``fortifyroot.litellm.retry_attempt``) under the FR + spans (``fortifyroot.litellm.attempt_``) under the FR safety_wrapper parent. Registered AFTER ``_FortifyRootCompletionLogger`` in the LiteLLM diff --git a/packages/opentelemetry-instrumentation-litellm/tests/test_retry_attempt_emission.py b/packages/opentelemetry-instrumentation-litellm/tests/test_retry_attempt_emission.py index 9e13cf736e..3d4b962081 100644 --- a/packages/opentelemetry-instrumentation-litellm/tests/test_retry_attempt_emission.py +++ b/packages/opentelemetry-instrumentation-litellm/tests/test_retry_attempt_emission.py @@ -4,11 +4,11 @@ - Instrumentor symmetry: _FortifyRootRetryEmitter is registered at instrument() AND removed at uninstrument() (Fallback B child-emission proof requirement (ii) per RETRY_LOOP.md §4.4.1). - - Single-attempt happy path: one retry_attempt span emitted under the - safety_wrapper parent, parent carries has_retry_attempt_child=true. + - Single-attempt happy path: one llm_attempt span emitted under the + safety_wrapper parent, parent carries has_attempt_child=true. - Multi-attempt retry path: 3 retry_attempt spans emitted (2 ERROR + 1 OK) for a 429→429→200 sequence, all under one safety_wrapper. - - Marker timing (§4.5): the parent's has_retry_attempt_child marker + - Marker timing (§4.5): the parent's has_attempt_child marker is set AFTER the first retry_attempt's start, NOT at parent creation. - §4.7.1 token registration: framework-attempt tokens are registered on attempt-start AND unregistered on attempt-end. @@ -24,14 +24,17 @@ import pytest from opentelemetry import trace -from opentelemetry.instrumentation.fortifyroot import retry_registry +from opentelemetry.instrumentation.fortifyroot import ( + clear_attempt_counters_for_test, + retry_registry, +) from opentelemetry.instrumentation.litellm import ( LiteLLMInstrumentor, _FortifyRootCompletionLogger, _FortifyRootRetryEmitter, - _FR_HAS_RETRY_ATTEMPT_CHILD_KEY, + _FR_HAS_ATTEMPT_CHILD_KEY, _FR_RETRY_ATTEMPT_MAP, - _FR_RETRY_ATTEMPT_SPAN_NAME, + _FR_LLM_ATTEMPT_SPAN_NAME_PREFIX, _resolve_routed_provider, _start_retry_attempt_span, _finalize_retry_attempt_span, @@ -95,12 +98,25 @@ def fresh_tracer(): def reset_registry_and_map(): """Each test starts with empty retry-emitter state.""" retry_registry._reset_for_test() + clear_attempt_counters_for_test() _FR_RETRY_ATTEMPT_MAP.clear() yield retry_registry._reset_for_test() + clear_attempt_counters_for_test() _FR_RETRY_ATTEMPT_MAP.clear() +def _assert_attempt_sequence(spans): + by_number = sorted( + spans, + key=lambda s: int(s.attributes.get("fortifyroot.attempt.number") or -1), + ) + for expected, span in enumerate(by_number, start=1): + assert span.name == f"{_FR_LLM_ATTEMPT_SPAN_NAME_PREFIX}.attempt_{expected}" + assert span.attributes.get("fortifyroot.attempt.number") == expected + assert span.attributes.get("fortifyroot.attempt.is_retry") is (expected > 1) + + # --------------------------------------------------------------------------- # Instrumentor symmetry — Fallback B child-emission proof (ii). # --------------------------------------------------------------------------- @@ -205,8 +221,8 @@ def test_single_attempt_emits_one_retry_attempt_under_parent(fresh_tracer): directly with an ambient parent span. Verify: - exactly 1 retry_attempt span exported - retry_attempt's parent is the safety_wrapper - - parent has has_retry_attempt_child=true - - retry_attempt has fortifyroot.span.role=retry_attempt + - parent has has_attempt_child=true + - retry_attempt has fortifyroot.span.role=llm_attempt - retry_attempt has gen_ai.system, gen_ai.request.model """ tracer, exporter, _ = fresh_tracer @@ -237,16 +253,17 @@ class MockResponse: spans = exporter.get_finished_spans() span_names = [s.name for s in spans] - assert _FR_RETRY_ATTEMPT_SPAN_NAME in span_names, ( + assert f"{_FR_LLM_ATTEMPT_SPAN_NAME_PREFIX}.attempt_1" in span_names, ( f"retry_attempt span missing; got {span_names}" ) - retry_span = next(s for s in spans if s.name == _FR_RETRY_ATTEMPT_SPAN_NAME) + retry_span = next(s for s in spans if s.name.startswith(f"{_FR_LLM_ATTEMPT_SPAN_NAME_PREFIX}.attempt_")) parent_span_exported = next(s for s in spans if s.name == "fortifyroot.litellm.safety") + _assert_attempt_sequence([retry_span]) assert retry_span.parent.span_id == parent_span_exported.context.span_id, ( "retry_attempt MUST be a child of the safety_wrapper parent" ) - assert retry_span.attributes.get("fortifyroot.span.role") == "retry_attempt" + assert retry_span.attributes.get("fortifyroot.span.role") == "llm_attempt" assert retry_span.attributes.get("gen_ai.system") == "openai", ( "gen_ai.system MUST be the routed provider, NOT the framework" ) @@ -260,8 +277,8 @@ class MockResponse: # §4.5 marker: must be set on parent. assert parent_span_exported.attributes.get( - _FR_HAS_RETRY_ATTEMPT_CHILD_KEY - ) is True, "parent MUST carry has_retry_attempt_child=true" + _FR_HAS_ATTEMPT_CHILD_KEY + ) is True, "parent MUST carry has_attempt_child=true" def test_retry_attempt_accepts_anthropic_usage_token_names(fresh_tracer): @@ -292,7 +309,7 @@ class MockResponse: retry_span = next( s for s in exporter.get_finished_spans() - if s.name == _FR_RETRY_ATTEMPT_SPAN_NAME + if s.name.startswith(f"{_FR_LLM_ATTEMPT_SPAN_NAME_PREFIX}.attempt_") ) assert retry_span.attributes.get("gen_ai.usage.input_tokens") == 11 assert retry_span.attributes.get("gen_ai.usage.output_tokens") == 7 @@ -303,9 +320,15 @@ class MockResponse: # --------------------------------------------------------------------------- def test_three_attempt_retry_path_emits_three_retry_attempt_spans(fresh_tracer): - """Three sequential attempts (429 → 429 → 200) under one parent. - Each gets its own litellm_call_id. Expect exactly 3 - retry_attempt children, with IsError flags (2 ERROR + 1 OK).""" + """Shared-parent numbering mechanics for 429 → 429 → 200. + + This drives the emitter directly with one safety_wrapper parent so + the attempt counter must produce attempt_1/2/3. Some production + LiteLLM retry helpers re-enter completion() per attempt and therefore + create one safety_wrapper per attempt; those multi-trace paths can + correctly surface as separate attempt_1 spans, as documented in + RETRY_LOOP.md. + """ tracer, exporter, _ = fresh_tracer parent = tracer.start_span("fortifyroot.litellm.safety") @@ -341,9 +364,10 @@ class MockSuccess: retry_spans = [ s for s in exporter.get_finished_spans() - if s.name == _FR_RETRY_ATTEMPT_SPAN_NAME + if s.name.startswith(f"{_FR_LLM_ATTEMPT_SPAN_NAME_PREFIX}.attempt_") ] assert len(retry_spans) == 3, f"expected 3 retry_attempt spans, got {len(retry_spans)}" + _assert_attempt_sequence(retry_spans) # 2 of them must be ERROR status, 1 OK. from opentelemetry.trace import StatusCode @@ -377,7 +401,7 @@ def test_marker_set_AFTER_first_attempt_starts_not_at_parent_creation(fresh_trac # Before any retry_attempt: no marker. # (Use the live span object since it hasn't been exported yet.) parent_attrs_before = dict(parent.attributes or {}) - assert _FR_HAS_RETRY_ATTEMPT_CHILD_KEY not in parent_attrs_before, ( + assert _FR_HAS_ATTEMPT_CHILD_KEY not in parent_attrs_before, ( "parent MUST NOT have the marker before any retry_attempt starts" ) @@ -386,7 +410,7 @@ def test_marker_set_AFTER_first_attempt_starts_not_at_parent_creation(fresh_trac _start_retry_attempt_span(kwargs) # After: parent has the marker. parent_attrs_after = dict(parent.attributes or {}) - assert parent_attrs_after.get(_FR_HAS_RETRY_ATTEMPT_CHILD_KEY) is True, ( + assert parent_attrs_after.get(_FR_HAS_ATTEMPT_CHILD_KEY) is True, ( "parent MUST have the marker after first retry_attempt starts" ) @@ -410,7 +434,7 @@ def test_marker_NOT_set_when_no_retry_attempt_starts(fresh_tracer): spans = exporter.get_finished_spans() parent_exported = next(s for s in spans if s.name == "fortifyroot.litellm.safety") - assert _FR_HAS_RETRY_ATTEMPT_CHILD_KEY not in (parent_exported.attributes or {}), ( + assert _FR_HAS_ATTEMPT_CHILD_KEY not in (parent_exported.attributes or {}), ( "no retry_attempt → no marker → parent stays canonical for §4.5 dedup" ) @@ -475,7 +499,7 @@ class MockSuccess: retry_spans = [ s for s in exporter.get_finished_spans() - if s.name == _FR_RETRY_ATTEMPT_SPAN_NAME + if s.name.startswith(f"{_FR_LLM_ATTEMPT_SPAN_NAME_PREFIX}.attempt_") ] assert len(retry_spans) == 1, ( f"expected exactly 1 retry_attempt span (idempotent finalize); " @@ -500,7 +524,7 @@ def test_no_parent_span_does_not_emit_orphan_retry_attempt(fresh_tracer): _start_retry_attempt_span(kwargs) spans = exporter.get_finished_spans() - retry_spans = [s for s in spans if s.name == _FR_RETRY_ATTEMPT_SPAN_NAME] + retry_spans = [s for s in spans if s.name.startswith(f"{_FR_LLM_ATTEMPT_SPAN_NAME_PREFIX}.attempt_")] assert len(retry_spans) == 0, ( f"orphan retry_attempt MUST NOT be emitted; got {len(retry_spans)}" ) @@ -559,7 +583,7 @@ def test_no_litellm_call_id_does_not_register_map_entry(fresh_tracer): retry_spans = [ s for s in exporter.get_finished_spans() - if s.name == _FR_RETRY_ATTEMPT_SPAN_NAME + if s.name.startswith(f"{_FR_LLM_ATTEMPT_SPAN_NAME_PREFIX}.attempt_") ] assert len(retry_spans) == 0, "missing litellm_call_id → no emission" assert len(_FR_RETRY_ATTEMPT_MAP) == 0 diff --git a/packages/opentelemetry-instrumentation-llamaindex/opentelemetry/instrumentation/llamaindex/retry_handler.py b/packages/opentelemetry-instrumentation-llamaindex/opentelemetry/instrumentation/llamaindex/retry_handler.py index 9048b2154c..4682260c87 100644 --- a/packages/opentelemetry-instrumentation-llamaindex/opentelemetry/instrumentation/llamaindex/retry_handler.py +++ b/packages/opentelemetry-instrumentation-llamaindex/opentelemetry/instrumentation/llamaindex/retry_handler.py @@ -23,6 +23,11 @@ - Use ambient OTel span at first attempt as the parent for sibling-grouping (RetryDetectorProc requires retry_attempts to be siblings under one parent_span_id). + - Attempt numbering is conservative for LlamaIndex: dispatcher hooks + expose a broad ambient workflow parent that can contain unrelated LLM + calls, so this handler emits attempt_1 / is_retry=false for each + observed outer LLM call rather than inferring retry ordinals from the + shared parent. This handler is registered alongside the existing ``OpenLLMetrySpanHandler`` — they capture orthogonal data @@ -42,6 +47,9 @@ from llama_index.core.instrumentation.span_handlers.base import BaseSpanHandler from opentelemetry import trace from opentelemetry.instrumentation.fortifyroot import ( + FR_HAS_ATTEMPT_CHILD_KEY, + first_llm_attempt, + llm_attempt_attributes, register_framework_attempt, unregister_framework_attempt, ) @@ -51,12 +59,12 @@ logger = logging.getLogger(__name__) # ST-10 §4.4: per-attempt sibling span name + role. -_FR_RETRY_ATTEMPT_SPAN_NAME = "fortifyroot.llamaindex.retry_attempt" _FR_SPAN_ROLE_KEY = "fortifyroot.span.role" -_FR_SPAN_ROLE_RETRY_ATTEMPT = "retry_attempt" +_FR_LLM_ATTEMPT_SPAN_NAME_PREFIX = "fortifyroot.llamaindex" +_FR_SPAN_ROLE_LLM_ATTEMPT = "llm_attempt" # ST-10 §4.5 parent marker. -_FR_HAS_RETRY_ATTEMPT_CHILD_KEY = "fortifyroot.span.has_retry_attempt_child" +_FR_HAS_ATTEMPT_CHILD_KEY = FR_HAS_ATTEMPT_CHILD_KEY # Dispatcher span IDs follow the pattern "ClassName.method-uuid". # Capture the method name so we can filter outer (public) calls @@ -263,10 +271,13 @@ def _start_retry_attempt(id_: str, instance: Any, bound_args: Any = None) -> Non elif "predict" in method: operation = "chat" # treat as chat-shaped + span_name, attempt_number, is_retry = first_llm_attempt( + _FR_LLM_ATTEMPT_SPAN_NAME_PREFIX, + ) attrs: dict[str, Any] = { - _FR_SPAN_ROLE_KEY: _FR_SPAN_ROLE_RETRY_ATTEMPT, "gen_ai.operation.name": operation, } + attrs.update(llm_attempt_attributes(attempt_number, is_retry)) if routed_provider: attrs["gen_ai.system"] = routed_provider if model: @@ -276,7 +287,7 @@ def _start_retry_attempt(id_: str, instance: Any, bound_args: Any = None) -> Non tracer = trace.get_tracer(__name__, __version__) parent_ctx = set_span_in_context(parent_span) span = tracer.start_span( - _FR_RETRY_ATTEMPT_SPAN_NAME, + span_name, kind=SpanKind.CLIENT, attributes=attrs, context=parent_ctx, @@ -316,11 +327,11 @@ def _start_retry_attempt(id_: str, instance: Any, bound_args: Any = None) -> Non } # §4.5 marker timing: set on parent AFTER the first qualifying - # retry_attempt has successfully started under it. + # llm_attempt has successfully started under it. try: - parent_span.set_attribute(_FR_HAS_RETRY_ATTEMPT_CHILD_KEY, True) + parent_span.set_attribute(_FR_HAS_ATTEMPT_CHILD_KEY, True) except Exception: - logger.debug("failed to set has_retry_attempt_child on parent", exc_info=True) + logger.debug("failed to set has_attempt_child on parent", exc_info=True) def _finalize_retry_attempt( @@ -392,7 +403,7 @@ def _finalize_retry_attempt( class _FortifyRootRetryHandler(BaseSpanHandler): """LlamaIndex SpanHandler that emits one - ``fortifyroot.llamaindex.retry_attempt`` sibling span per OUTER + ``fortifyroot.llamaindex.attempt_`` sibling span per OUTER public LLM method invocation (chat/achat/complete/acomplete/...). De-dup vs the inner ``_chat``/``_complete`` spans is enforced by @@ -482,7 +493,7 @@ def _reset_state_for_test() -> None: __all__ = [ "_FortifyRootRetryHandler", - "_FR_RETRY_ATTEMPT_SPAN_NAME", - "_FR_HAS_RETRY_ATTEMPT_CHILD_KEY", + "_FR_LLM_ATTEMPT_SPAN_NAME_PREFIX", + "_FR_HAS_ATTEMPT_CHILD_KEY", "_OUTER_LLM_METHODS", ] diff --git a/packages/opentelemetry-instrumentation-llamaindex/tests/test_retry_attempt_emission.py b/packages/opentelemetry-instrumentation-llamaindex/tests/test_retry_attempt_emission.py index be25a165b7..0e37d1d591 100644 --- a/packages/opentelemetry-instrumentation-llamaindex/tests/test_retry_attempt_emission.py +++ b/packages/opentelemetry-instrumentation-llamaindex/tests/test_retry_attempt_emission.py @@ -9,8 +9,10 @@ - BaseLLM instance check: dispatcher spans on non-LLM classes are ignored. - Single-attempt happy path under tenacity-style wrapping. - - Multi-attempt retry path: 3 attempts share parent → 3 - retry_attempt SIBLINGS under one OTel parent. + - Multiple outer calls sharing a parent → 3 llm_attempt SIBLINGS + under one OTel parent. LlamaIndex numbering is conservative: + shared workflow parents can contain unrelated LLM calls, so each + emitted span remains attempt_1 / is_retry=false. - Marker timing (§4.5). - §4.7.1 token registration symmetry. - No-parent guard. @@ -30,9 +32,9 @@ ) from opentelemetry.instrumentation.llamaindex.retry_handler import ( _FortifyRootRetryHandler, - _FR_HAS_RETRY_ATTEMPT_CHILD_KEY, + _FR_HAS_ATTEMPT_CHILD_KEY, _FR_RETRY_ATTEMPT_MAP, - _FR_RETRY_ATTEMPT_SPAN_NAME, + _FR_LLM_ATTEMPT_SPAN_NAME_PREFIX, _is_outer_llm_method, _OUTER_LLM_METHODS, _reset_state_for_test, @@ -73,6 +75,13 @@ def reset_state(): _reset_state_for_test() +def _assert_conservative_attempts(spans): + for span in spans: + assert span.name == f"{_FR_LLM_ATTEMPT_SPAN_NAME_PREFIX}.attempt_1" + assert span.attributes.get("fortifyroot.attempt.number") == 1 + assert span.attributes.get("fortifyroot.attempt.is_retry") is False + + # --------------------------------------------------------------------------- # Helpers # --------------------------------------------------------------------------- @@ -197,12 +206,13 @@ def test_outer_method_emits_inner_method_does_not(fresh_tracer): retry_spans = [ s for s in exporter.get_finished_spans() - if s.name == _FR_RETRY_ATTEMPT_SPAN_NAME + if s.name.startswith(f"{_FR_LLM_ATTEMPT_SPAN_NAME_PREFIX}.attempt_") ] assert len(retry_spans) == 1, ( f"F4 de-dup invariant: outer chat() emits ONE retry_attempt; " f"inner _chat() must NOT emit. Got {len(retry_spans)} spans." ) + _assert_conservative_attempts(retry_spans) def test_is_outer_llm_method_filter(): @@ -269,11 +279,12 @@ def test_single_attempt_emits_one_retry_attempt(fresh_tracer): spans = exporter.get_finished_spans() parent_exported = next(s for s in spans if s.name == "workflow") - retry_span = next(s for s in spans if s.name == _FR_RETRY_ATTEMPT_SPAN_NAME) + retry_span = next(s for s in spans if s.name.startswith(f"{_FR_LLM_ATTEMPT_SPAN_NAME_PREFIX}.attempt_")) + _assert_conservative_attempts([retry_span]) assert retry_span.parent.span_id == parent_exported.context.span_id - assert parent_exported.attributes.get(_FR_HAS_RETRY_ATTEMPT_CHILD_KEY) is True - assert retry_span.attributes.get("fortifyroot.span.role") == "retry_attempt" + assert parent_exported.attributes.get(_FR_HAS_ATTEMPT_CHILD_KEY) is True + assert retry_span.attributes.get("fortifyroot.span.role") == "llm_attempt" assert retry_span.attributes.get("gen_ai.request.model") == "gpt-4o-mini" assert retry_span.attributes.get("gen_ai.response.id") == "resp-001" assert retry_span.attributes.get("gen_ai.usage.input_tokens") == 10 @@ -336,12 +347,13 @@ def __init__(self, status_code): spans = exporter.get_finished_spans() parent_exported = next(s for s in spans if s.name == "workflow") - retry_spans = [s for s in spans if s.name == _FR_RETRY_ATTEMPT_SPAN_NAME] + retry_spans = [s for s in spans if s.name.startswith(f"{_FR_LLM_ATTEMPT_SPAN_NAME_PREFIX}.attempt_")] assert len(retry_spans) == 3, ( f"3 outer-method spans should produce 3 retry_attempts (inner _chat filtered out); " f"got {len(retry_spans)}" ) + _assert_conservative_attempts(retry_spans) parent_ids = {s.parent.span_id for s in retry_spans} assert parent_ids == {parent_exported.context.span_id}, ( f"all 3 retry_attempts must be SIBLINGS under one OTel parent; " @@ -368,14 +380,14 @@ def test_marker_set_AFTER_first_attempt(fresh_tracer): instance = _FakeLLM() parent = tracer.start_span("workflow") - assert _FR_HAS_RETRY_ATTEMPT_CHILD_KEY not in dict(parent.attributes or {}) + assert _FR_HAS_ATTEMPT_CHILD_KEY not in dict(parent.attributes or {}) with trace.use_span(parent, end_on_exit=False): handler.new_span( id_=_make_id("FakeLLM", "chat"), bound_args=_empty_bound_args(), instance=instance, ) - assert dict(parent.attributes or {}).get(_FR_HAS_RETRY_ATTEMPT_CHILD_KEY) is True + assert dict(parent.attributes or {}).get(_FR_HAS_ATTEMPT_CHILD_KEY) is True handler.prepare_to_exit_span( id_=_make_id("FakeLLM", "chat"), bound_args=_empty_bound_args(), @@ -407,7 +419,7 @@ def test_marker_NOT_set_when_inner_only_fires(fresh_tracer): parent_exported = next( s for s in exporter.get_finished_spans() if s.name == "workflow" ) - assert _FR_HAS_RETRY_ATTEMPT_CHILD_KEY not in (parent_exported.attributes or {}) + assert _FR_HAS_ATTEMPT_CHILD_KEY not in (parent_exported.attributes or {}) # --------------------------------------------------------------------------- @@ -450,7 +462,7 @@ def test_no_ambient_parent_no_emission(fresh_tracer): retry_spans = [ s for s in exporter.get_finished_spans() - if s.name == _FR_RETRY_ATTEMPT_SPAN_NAME + if s.name.startswith(f"{_FR_LLM_ATTEMPT_SPAN_NAME_PREFIX}.attempt_") ] assert len(retry_spans) == 0 assert len(_FR_RETRY_ATTEMPT_MAP) == 0 @@ -482,6 +494,6 @@ def test_double_finalize_is_idempotent(fresh_tracer): retry_spans = [ s for s in exporter.get_finished_spans() - if s.name == _FR_RETRY_ATTEMPT_SPAN_NAME + if s.name.startswith(f"{_FR_LLM_ATTEMPT_SPAN_NAME_PREFIX}.attempt_") ] assert len(retry_spans) == 1 diff --git a/packages/opentelemetry-instrumentation-openai/opentelemetry/instrumentation/openai/retry_handler.py b/packages/opentelemetry-instrumentation-openai/opentelemetry/instrumentation/openai/retry_handler.py index 0afba46e5e..b30eb38198 100644 --- a/packages/opentelemetry-instrumentation-openai/opentelemetry/instrumentation/openai/retry_handler.py +++ b/packages/opentelemetry-instrumentation-openai/opentelemetry/instrumentation/openai/retry_handler.py @@ -54,7 +54,10 @@ from opentelemetry import context as context_api from opentelemetry import trace from opentelemetry.instrumentation.fortifyroot import ( + FR_HAS_ATTEMPT_CHILD_KEY, is_framework_owned, + llm_attempt_attributes, + next_llm_attempt, ) from opentelemetry.instrumentation.openai.version import __version__ from opentelemetry.instrumentation.utils import unwrap @@ -66,10 +69,10 @@ # ST-10 §4.4 / §4.5 constants. -_FR_RETRY_ATTEMPT_SPAN_NAME = "fortifyroot.openai.retry_attempt" _FR_SPAN_ROLE_KEY = "fortifyroot.span.role" -_FR_SPAN_ROLE_RETRY_ATTEMPT = "retry_attempt" -_FR_HAS_RETRY_ATTEMPT_CHILD_KEY = "fortifyroot.span.has_retry_attempt_child" +_FR_LLM_ATTEMPT_SPAN_NAME_PREFIX = "fortifyroot.openai" +_FR_SPAN_ROLE_LLM_ATTEMPT = "llm_attempt" +_FR_HAS_ATTEMPT_CHILD_KEY = FR_HAS_ATTEMPT_CHILD_KEY # Package-local context key set by the OpenAI logical-call wrappers # (currently `chat_wrappers.chat_wrapper` / `achat_wrapper`) BEFORE @@ -264,25 +267,29 @@ def _resolve_parent_span() -> Optional["trace.Span"]: def _set_parent_marker(parent_span: "trace.Span") -> None: - """§4.5: mark the parent as 'has retry_attempt child' AFTER the + """§4.5: mark the parent as 'has llm_attempt child' AFTER the first child has successfully started. Idempotent — setting twice is a no-op.""" try: - parent_span.set_attribute(_FR_HAS_RETRY_ATTEMPT_CHILD_KEY, True) + parent_span.set_attribute(_FR_HAS_ATTEMPT_CHILD_KEY, True) except Exception: - logger.debug("failed to set has_retry_attempt_child on parent", exc_info=True) + logger.debug("failed to set has_attempt_child on parent", exc_info=True) def _start_attempt_span(request: Any, parent_span: "trace.Span") -> "trace.Span": - """Open the retry_attempt sibling span under the given parent. + """Open the llm_attempt sibling span under the given parent. Caller is responsible for adding response attrs + ending it. """ path = _request_path(request) + span_name, attempt_number, is_retry = next_llm_attempt( + parent_span, + _FR_LLM_ATTEMPT_SPAN_NAME_PREFIX, + ) attrs: dict[str, Any] = { - _FR_SPAN_ROLE_KEY: _FR_SPAN_ROLE_RETRY_ATTEMPT, "gen_ai.system": "openai", "gen_ai.operation.name": _operation_for_path(path), } + attrs.update(llm_attempt_attributes(attempt_number, is_retry)) model = _resolve_model_from_request(request) if model: attrs["gen_ai.request.model"] = model @@ -291,7 +298,7 @@ def _start_attempt_span(request: Any, parent_span: "trace.Span") -> "trace.Span" tracer = trace.get_tracer(__name__, __version__, _tracer_provider) parent_ctx = trace.set_span_in_context(parent_span) span = tracer.start_span( - _FR_RETRY_ATTEMPT_SPAN_NAME, + span_name, kind=SpanKind.CLIENT, attributes=attrs, context=parent_ctx, @@ -364,12 +371,12 @@ def _finalize_success(span: "trace.Span", response: Any, *, is_streaming: bool = span.set_attribute("error.type", err_type) span.set_status(Status(StatusCode.ERROR, f"http {status_code}")) except Exception: - logger.debug("failed to set response attrs on openai retry_attempt", exc_info=True) + logger.debug("failed to set response attrs on openai llm_attempt", exc_info=True) def _extract_usage_from_body(span: "trace.Span", response: Any) -> None: """Parse a non-streaming OpenAI response body and copy usage / - response id / response model attrs to the retry_attempt span. + response id / response model attrs to the llm_attempt span. All operations are wrapped in try/except — body parsing is best-effort. If anything fails (non-JSON body, malformed schema, @@ -430,12 +437,12 @@ def _finalize_error(span: "trace.Span", error: BaseException) -> None: pass span.set_status(Status(StatusCode.ERROR, str(error))) except Exception: - logger.debug("failed to set error attrs on openai retry_attempt", exc_info=True) + logger.debug("failed to set error attrs on openai llm_attempt", exc_info=True) def _should_emit_for(request: Any) -> bool: """Combine all skip-emission guards into one decision. Returns - True iff a retry_attempt span SHOULD be emitted for this request. + True iff an llm_attempt span SHOULD be emitted for this request. """ if _is_suppressed(): return False @@ -685,7 +692,7 @@ def _is_installed_for_test() -> bool: __all__ = [ "instrument_retry_emitter", "uninstrument_retry_emitter", - "_FR_RETRY_ATTEMPT_SPAN_NAME", - "_FR_HAS_RETRY_ATTEMPT_CHILD_KEY", + "_FR_LLM_ATTEMPT_SPAN_NAME_PREFIX", + "_FR_HAS_ATTEMPT_CHILD_KEY", "_LLM_PATH_SUFFIXES", ] diff --git a/packages/opentelemetry-instrumentation-openai/tests/conftest.py b/packages/opentelemetry-instrumentation-openai/tests/conftest.py index 46c53fa9d2..f2ff4611ab 100644 --- a/packages/opentelemetry-instrumentation-openai/tests/conftest.py +++ b/packages/opentelemetry-instrumentation-openai/tests/conftest.py @@ -36,7 +36,7 @@ class _NoFortifyRootSpanExporter(InMemorySpanExporter): ``fortifyroot.`` from the upstream-test span exporter. ST-10.4 added per-attempt sibling spans (e.g. - ``fortifyroot.openai.retry_attempt``) under every openai logical + ``fortifyroot.openai.attempt_1``) under every openai logical call. Upstream/legacy OpenAI tests assert exact span-name lists (e.g. ``[span.name for span in spans] == ["openai.chat"]``); without a filter, every such assertion would now fail because the @@ -56,7 +56,7 @@ def get_finished_spans(self): # type: ignore[override] # ST-10.4 retry_attempt siblings carry role=retry_attempt. return tuple( s for s in super().get_finished_spans() - if (s.attributes or {}).get("fortifyroot.span.role") != "retry_attempt" + if (s.attributes or {}).get("fortifyroot.span.role") != "llm_attempt" ) diff --git a/packages/opentelemetry-instrumentation-openai/tests/test_retry_attempt_emission.py b/packages/opentelemetry-instrumentation-openai/tests/test_retry_attempt_emission.py index 6333e81254..046f8ac197 100644 --- a/packages/opentelemetry-instrumentation-openai/tests/test_retry_attempt_emission.py +++ b/packages/opentelemetry-instrumentation-openai/tests/test_retry_attempt_emission.py @@ -3,8 +3,8 @@ Covers (per RETRY_LOOP.md §4.4 OpenAI row + §4.7 suppression): - Instrumentor symmetry: install/uninstall flips state cleanly and is idempotent. - - Single-attempt happy path: ONE retry_attempt span under the active - parent; parent gets has_retry_attempt_child=true; span carries + - Single-attempt happy path: ONE llm_attempt span under the active + parent; parent gets has_attempt_child=true; span carries role / gen_ai.system=openai / gen_ai.request.model / http.status_code. - Multi-attempt retry path: N siblings under one parent (the structural shape RetryDetectorProc relies on). @@ -37,14 +37,15 @@ from opentelemetry import context as context_api from opentelemetry import trace from opentelemetry.instrumentation.fortifyroot import ( + clear_attempt_counters_for_test, is_framework_owned, register_framework_attempt, retry_registry, unregister_framework_attempt, ) from opentelemetry.instrumentation.openai.retry_handler import ( - _FR_HAS_RETRY_ATTEMPT_CHILD_KEY, - _FR_RETRY_ATTEMPT_SPAN_NAME, + _FR_HAS_ATTEMPT_CHILD_KEY, + _FR_LLM_ATTEMPT_SPAN_NAME_PREFIX, OPENAI_DIRECT_RETRY_PARENT_ACTIVE_KEY, _async_send_wrapper, _has_wrappable_symbol, @@ -90,6 +91,7 @@ def fresh_tracer(): def reset_registry_and_state(): """Each test starts with empty registry + emitter uninstalled.""" retry_registry._reset_for_test() + clear_attempt_counters_for_test() # Defensive uninstall in case a prior test left it installed. try: uninstrument_retry_emitter() @@ -97,6 +99,7 @@ def reset_registry_and_state(): pass yield retry_registry._reset_for_test() + clear_attempt_counters_for_test() try: uninstrument_retry_emitter() except Exception: @@ -144,7 +147,18 @@ def _make_response(status_code: int = 200, request_id: str = "req-abc", def _retry_spans(exporter: InMemorySpanExporter): return [s for s in exporter.get_finished_spans() - if s.name == _FR_RETRY_ATTEMPT_SPAN_NAME] + if s.name.startswith(f"{_FR_LLM_ATTEMPT_SPAN_NAME_PREFIX}.attempt_")] + + +def _assert_attempt_sequence(spans): + by_number = sorted( + spans, + key=lambda s: int(s.attributes.get("fortifyroot.attempt.number") or -1), + ) + for expected, span in enumerate(by_number, start=1): + assert span.name == f"{_FR_LLM_ATTEMPT_SPAN_NAME_PREFIX}.attempt_{expected}" + assert span.attributes.get("fortifyroot.attempt.number") == expected + assert span.attributes.get("fortifyroot.attempt.is_retry") is (expected > 1) # --------------------------------------------------------------------------- @@ -265,10 +279,11 @@ def test_single_attempt_emits_one_span_with_marker(fresh_tracer): retry_spans = _retry_spans(exporter) assert len(retry_spans) == 1 rs = retry_spans[0] + _assert_attempt_sequence(retry_spans) assert rs.parent.span_id == parent_exported.context.span_id, ( "retry_attempt must be a child of the active parent" ) - assert rs.attributes.get("fortifyroot.span.role") == "retry_attempt" + assert rs.attributes.get("fortifyroot.span.role") == "llm_attempt" assert rs.attributes.get("gen_ai.system") == "openai" assert rs.attributes.get("gen_ai.request.model") == "gpt-4o-mini" assert rs.attributes.get("gen_ai.operation.name") == "chat" @@ -281,7 +296,7 @@ def test_single_attempt_emits_one_span_with_marker(fresh_tracer): assert rs.attributes.get("gen_ai.usage.output_tokens") == 3 assert rs.attributes.get("server.address") == "api.openai.com" assert rs.attributes.get("server.port") == 443 - assert parent_exported.attributes.get(_FR_HAS_RETRY_ATTEMPT_CHILD_KEY) is True + assert parent_exported.attributes.get(_FR_HAS_ATTEMPT_CHILD_KEY) is True # --------------------------------------------------------------------------- @@ -311,6 +326,7 @@ def test_three_attempts_share_parent(fresh_tracer): parent_exported = next(s for s in spans if s.name == "openai.chat") retry_spans = _retry_spans(exporter) assert len(retry_spans) == 3, f"expected 3 retry_attempts, got {len(retry_spans)}" + _assert_attempt_sequence(retry_spans) parent_ids = {s.parent.span_id for s in retry_spans} assert parent_ids == {parent_exported.context.span_id}, ( @@ -400,13 +416,13 @@ def test_marker_set_AFTER_first_attempt_not_at_parent_creation(fresh_tracer): parent = tracer.start_span("openai.chat") # Before any retry_attempt: no marker. - assert _FR_HAS_RETRY_ATTEMPT_CHILD_KEY not in dict(parent.attributes or {}) + assert _FR_HAS_ATTEMPT_CHILD_KEY not in dict(parent.attributes or {}) with trace.use_span(parent, end_on_exit=False): _sync_send_wrapper(lambda *a, **kw: _make_response(200), None, (request,), {}) # After first attempt: marker present on the still-open parent. - assert dict(parent.attributes or {}).get(_FR_HAS_RETRY_ATTEMPT_CHILD_KEY) is True + assert dict(parent.attributes or {}).get(_FR_HAS_ATTEMPT_CHILD_KEY) is True parent.end() @@ -417,7 +433,7 @@ def test_marker_NOT_set_when_no_attempts_fire(fresh_tracer): parent_exported = next( s for s in exporter.get_finished_spans() if s.name == "openai.chat" ) - assert _FR_HAS_RETRY_ATTEMPT_CHILD_KEY not in (parent_exported.attributes or {}) + assert _FR_HAS_ATTEMPT_CHILD_KEY not in (parent_exported.attributes or {}) # --------------------------------------------------------------------------- @@ -567,6 +583,7 @@ async def run(): f"This is the §4.7 self-registration regression — the second task is being " f"suppressed by the first task's lingering framework token." ) + _assert_attempt_sequence(retry_spans) # --------------------------------------------------------------------------- @@ -700,12 +717,12 @@ def explode(): f"got {len(retry_spans)} span(s). The parent span stays the " f"canonical LLM event." ) - # Parent MUST NOT receive the has_retry_attempt_child marker + # Parent MUST NOT receive the has_attempt_child marker # either (no child was emitted). parent_exported = next( s for s in exporter.get_finished_spans() if s.name == "openai.chat" ) - assert _FR_HAS_RETRY_ATTEMPT_CHILD_KEY not in (parent_exported.attributes or {}) + assert _FR_HAS_ATTEMPT_CHILD_KEY not in (parent_exported.attributes or {}) def test_non_2xx_response_with_usage_in_body_populates_usage_tokens(fresh_tracer): @@ -903,7 +920,7 @@ def test_tracer_provider_plumbed_through_instrument_retry_emitter(): uninstrument_retry_emitter() spans = exporter.get_finished_spans() - retry_spans = [s for s in spans if s.name == _FR_RETRY_ATTEMPT_SPAN_NAME] + retry_spans = [s for s in spans if s.name.startswith(f"{_FR_LLM_ATTEMPT_SPAN_NAME_PREFIX}.attempt_")] assert len(retry_spans) == 1, ( f"retry_attempt MUST be exported via the explicit tracer_provider " f"passed to instrument_retry_emitter; saw {len(retry_spans)} in this exporter " diff --git a/packages/traceloop-sdk/tests/test_association_properties.py b/packages/traceloop-sdk/tests/test_association_properties.py index 4aa3f17eb2..3f5478d78c 100644 --- a/packages/traceloop-sdk/tests/test_association_properties.py +++ b/packages/traceloop-sdk/tests/test_association_properties.py @@ -9,13 +9,12 @@ from traceloop.sdk import Traceloop from traceloop.sdk.decorators import task, workflow -# ST-10.4 (review-driven 2026-05-16): generalized from name-based -# LangChain-only filter to role-based. Drops every provider's -# retry_attempt sibling (openai / anthropic / bedrock / langchain / -# llamaindex / litellm) uniformly. A new provider's retry_attempt -# automatically participates without touching this helper. +# ST-10.4 (review-driven 2026-05-16, renamed 2026-06-12): generalized +# from name-based LangChain-only filter to role-based. Drops every +# provider's FortifyRoot LLM-attempt sibling (openai / anthropic / +# bedrock / langchain / llamaindex / litellm) uniformly. _FR_SPAN_ROLE_KEY = "fortifyroot.span.role" -_FR_SPAN_ROLE_RETRY_ATTEMPT = "retry_attempt" +_FR_SPAN_ROLE_LLM_ATTEMPT = "llm_attempt" def _without_fr_langchain_retry_attempt_spans(spans): @@ -24,7 +23,7 @@ def _without_fr_langchain_retry_attempt_spans(spans): # filter (see e.g. test_workflows.py, test_prompt_management.py). return [ s for s in spans - if (s.attributes or {}).get(_FR_SPAN_ROLE_KEY) != _FR_SPAN_ROLE_RETRY_ATTEMPT + if (s.attributes or {}).get(_FR_SPAN_ROLE_KEY) != _FR_SPAN_ROLE_LLM_ATTEMPT ] diff --git a/packages/traceloop-sdk/tests/test_privacy_no_prompts.py b/packages/traceloop-sdk/tests/test_privacy_no_prompts.py index e8534eeb22..f4d844a998 100644 --- a/packages/traceloop-sdk/tests/test_privacy_no_prompts.py +++ b/packages/traceloop-sdk/tests/test_privacy_no_prompts.py @@ -12,21 +12,18 @@ from traceloop.sdk.decorators import workflow, task -# ST-10.4: ``fortifyroot.openai.retry_attempt`` (and equivalents for -# anthropic / bedrock / framework wrappers) lands in this test's +# ST-10.4: FortifyRoot LLM-attempt spans land in this test's # session-scoped exporter ahead of the logical ``openai.chat`` span. # Filter by the canonical ``fortifyroot.span.role`` attribute so every -# provider's retry_attempt is dropped uniformly. See -# fr-system-tests/docs/development/ai-logs/st_phase_10.txt addendum -# 2026-05-16 for context. +# provider is dropped uniformly. _FR_SPAN_ROLE_KEY = "fortifyroot.span.role" -_FR_SPAN_ROLE_RETRY_ATTEMPT = "retry_attempt" +_FR_SPAN_ROLE_LLM_ATTEMPT = "llm_attempt" def _without_retry_attempt_spans(spans): return [ s for s in spans - if (s.attributes or {}).get(_FR_SPAN_ROLE_KEY) != _FR_SPAN_ROLE_RETRY_ATTEMPT + if (s.attributes or {}).get(_FR_SPAN_ROLE_KEY) != _FR_SPAN_ROLE_LLM_ATTEMPT ] diff --git a/packages/traceloop-sdk/tests/test_prompt_management.py b/packages/traceloop-sdk/tests/test_prompt_management.py index d1cdaa4658..e408d9aa42 100644 --- a/packages/traceloop-sdk/tests/test_prompt_management.py +++ b/packages/traceloop-sdk/tests/test_prompt_management.py @@ -13,18 +13,17 @@ from traceloop.sdk.prompts.client import PromptRegistryClient -# ST-10.4: filter ``fortifyroot.*.retry_attempt`` sibling spans out of -# legacy exact-span-list / ``spans[0]`` assertions. Role-based filter -# so every provider's retry_attempt is dropped uniformly. See addendum -# 2026-05-16 in st_phase_10.txt for context. +# ST-10.4: filter FortifyRoot LLM-attempt sibling spans out of legacy +# exact-span-list / ``spans[0]`` assertions. Role-based filter so every +# provider is dropped uniformly. See retry-loop docs for context. _FR_SPAN_ROLE_KEY = "fortifyroot.span.role" -_FR_SPAN_ROLE_RETRY_ATTEMPT = "retry_attempt" +_FR_SPAN_ROLE_LLM_ATTEMPT = "llm_attempt" def _without_retry_attempt_spans(spans): return [ s for s in spans - if (s.attributes or {}).get(_FR_SPAN_ROLE_KEY) != _FR_SPAN_ROLE_RETRY_ATTEMPT + if (s.attributes or {}).get(_FR_SPAN_ROLE_KEY) != _FR_SPAN_ROLE_LLM_ATTEMPT ] prompts_json = """ diff --git a/packages/traceloop-sdk/tests/test_sdk_initialization.py b/packages/traceloop-sdk/tests/test_sdk_initialization.py index 5f7c51057b..6873842747 100644 --- a/packages/traceloop-sdk/tests/test_sdk_initialization.py +++ b/packages/traceloop-sdk/tests/test_sdk_initialization.py @@ -8,21 +8,20 @@ from traceloop.sdk.decorators import workflow -# ST-10.4: ``fortifyroot.openai.retry_attempt`` lands in the exporter -# ahead of the logical ``openai.chat`` span; filter by role so the -# legacy ``spans[0] == openai.chat`` assumption survives. The user's -# custom span_postprocess_callback is intentionally still invoked on -# the retry_attempt span (product-code contract: callback sees every -# exported span); only the test's assertion is filtered. See addendum -# 2026-05-16 in st_phase_10.txt for context. +# ST-10.4: FortifyRoot LLM-attempt spans land in the exporter ahead of +# the logical ``openai.chat`` span; filter by role so the legacy +# ``spans[0] == openai.chat`` assumption survives. The user's custom +# span_postprocess_callback is intentionally still invoked on the +# attempt span (product-code contract: callback sees every exported +# span); only the test's assertion is filtered. _FR_SPAN_ROLE_KEY = "fortifyroot.span.role" -_FR_SPAN_ROLE_RETRY_ATTEMPT = "retry_attempt" +_FR_SPAN_ROLE_LLM_ATTEMPT = "llm_attempt" def _without_retry_attempt_spans(spans): return [ s for s in spans - if (s.attributes or {}).get(_FR_SPAN_ROLE_KEY) != _FR_SPAN_ROLE_RETRY_ATTEMPT + if (s.attributes or {}).get(_FR_SPAN_ROLE_KEY) != _FR_SPAN_ROLE_LLM_ATTEMPT ] @@ -239,4 +238,3 @@ def test_get_default_span_processor(): assert isinstance(processor, BatchSpanProcessor) assert hasattr(processor, "_traceloop_processor") assert getattr(processor, "_traceloop_processor") is True - diff --git a/packages/traceloop-sdk/tests/test_workflows.py b/packages/traceloop-sdk/tests/test_workflows.py index 498bdcbbdf..0b88f062bf 100644 --- a/packages/traceloop-sdk/tests/test_workflows.py +++ b/packages/traceloop-sdk/tests/test_workflows.py @@ -17,19 +17,18 @@ from traceloop.sdk.decorators import workflow, task -# ST-10.4: filter ``fortifyroot.*.retry_attempt`` sibling spans out -# of legacy exact-span-list / set assertions. Role-based filter so -# every provider's retry_attempt (openai, anthropic, bedrock, -# langchain, llamaindex, litellm) is dropped uniformly. See addendum -# 2026-05-16 in st_phase_10.txt for context. +# ST-10.4: filter FortifyRoot LLM-attempt sibling spans out of legacy +# exact-span-list / set assertions. Role-based filter so every provider +# (openai, anthropic, bedrock, langchain, llamaindex, litellm) is +# dropped uniformly. See retry-loop docs for context. _FR_SPAN_ROLE_KEY = "fortifyroot.span.role" -_FR_SPAN_ROLE_RETRY_ATTEMPT = "retry_attempt" +_FR_SPAN_ROLE_LLM_ATTEMPT = "llm_attempt" def _without_retry_attempt_spans(spans): return [ s for s in spans - if (s.attributes or {}).get(_FR_SPAN_ROLE_KEY) != _FR_SPAN_ROLE_RETRY_ATTEMPT + if (s.attributes or {}).get(_FR_SPAN_ROLE_KEY) != _FR_SPAN_ROLE_LLM_ATTEMPT ]