From 8c429189690c49a64008aaea4c30a868c11d7da1 Mon Sep 17 00:00:00 2001 From: Tao Chen Date: Fri, 14 Aug 2026 21:23:44 +0000 Subject: [PATCH] Consolidate OTel GenAI Semantic Conventions versions --- .../_harness/_background_agents.py | 10 +- .../packages/core/agent_framework/_tools.py | 11 +- .../core/agent_framework/observability.py | 162 +++++++++++--- python/packages/core/tests/conftest.py | 2 + .../core/tests/core/test_observability.py | 203 ++++++++++++++++++ python/packages/core/tests/core/test_tools.py | 28 +++ .../samples/02-agents/observability/README.md | 31 ++- 7 files changed, 408 insertions(+), 39 deletions(-) diff --git a/python/packages/core/agent_framework/_harness/_background_agents.py b/python/packages/core/agent_framework/_harness/_background_agents.py index 5f6df3f07d..3ec3464d2c 100644 --- a/python/packages/core/agent_framework/_harness/_background_agents.py +++ b/python/packages/core/agent_framework/_harness/_background_agents.py @@ -364,16 +364,10 @@ async def release_session( if runtime is None or runtime.closed: return - pending = [ - task - for task in list(runtime.in_flight_tasks.values()) - if not task.done() - ] + pending = [task for task in list(runtime.in_flight_tasks.values()) if not task.done()] if pending and not cancel_running: - raise RuntimeError( - f"Cannot release session {session_id}: {len(pending)} tasks still running." - ) + raise RuntimeError(f"Cannot release session {session_id}: {len(pending)} tasks still running.") runtime.closed = True diff --git a/python/packages/core/agent_framework/_tools.py b/python/packages/core/agent_framework/_tools.py index 2d8f1ced47..d97e426b1f 100644 --- a/python/packages/core/agent_framework/_tools.py +++ b/python/packages/core/agent_framework/_tools.py @@ -745,7 +745,10 @@ async def invoke( "response_format", } } - if OBSERVABILITY_SETTINGS.SENSITIVE_DATA_ENABLED: + # gen_ai.tool.call.arguments/result were introduced above v1.36.0; only emit them + # as span attributes when that semconv version is active. + emit_tool_call_attrs = OBSERVABILITY_SETTINGS.emit_tool_call_attributes + if emit_tool_call_attrs: attributes.update({ OtelAttr.TOOL_ARGUMENTS: ( json.dumps(serializable_kwargs, default=str, ensure_ascii=False) if serializable_kwargs else "None" @@ -754,7 +757,7 @@ async def invoke( with get_function_span(attributes=attributes) as span: attributes[OtelAttr.MEASUREMENT_FUNCTION_TAG_NAME] = self.name logger.info(f"Function name: {self.name}") - if OBSERVABILITY_SETTINGS.SENSITIVE_DATA_ENABLED: + if emit_tool_call_attrs: logger.debug(f"Function arguments: {serializable_kwargs}") start_time_stamp = perf_counter() end_time_stamp: float | None = None @@ -770,7 +773,7 @@ async def invoke( else: if skip_parsing: logger.info(f"Function {self.name} succeeded.") - if OBSERVABILITY_SETTINGS.SENSITIVE_DATA_ENABLED: + if emit_tool_call_attrs: result_str = str(result) span.set_attribute(OtelAttr.TOOL_RESULT, result_str) logger.debug(f"Function result: {result_str}") @@ -783,7 +786,7 @@ async def invoke( if isinstance(parsed, str): parsed = [Content.from_text(parsed)] logger.info(f"Function {self.name} succeeded.") - if OBSERVABILITY_SETTINGS.SENSITIVE_DATA_ENABLED: + if emit_tool_call_attrs: result_str = "\n".join(c.text or "" for c in parsed if c.type == "text") or str(parsed) span.set_attribute(OtelAttr.TOOL_RESULT, result_str) logger.debug(f"Function result: {result_str}") diff --git a/python/packages/core/agent_framework/observability.py b/python/packages/core/agent_framework/observability.py index 5b40626875..a148c64efe 100644 --- a/python/packages/core/agent_framework/observability.py +++ b/python/packages/core/agent_framework/observability.py @@ -362,6 +362,7 @@ def __str__(self) -> str: "assistant": OtelAttr.ASSISTANT_MESSAGE, "tool": OtelAttr.TOOL_MESSAGE, } + FINISH_REASON_MAP = { "stop": "stop", "content_filter": "content_filter", @@ -715,12 +716,20 @@ def create_metric_views() -> list[View]: ] +# Token recognized in the OTEL_SEMCONV_STABILITY_OPT_IN env var that opts into the GenAI +# conventions above the v1.36.0 stable release (collectively "experimental"; see +# https://github.com/open-telemetry/semantic-conventions/blob/v1.37.0/docs/gen-ai). +GEN_AI_LATEST_EXPERIMENTAL_OPT_IN: Final[str] = "gen_ai_latest_experimental" + + class _ObservabilitySettingsData(TypedDict, total=False): """TypedDict schema for observability settings fields.""" enable_instrumentation: bool | None enable_sensitive_data: bool | None enable_console_exporters: bool | None + enable_message_events: bool | None + otel_semconv_stability_opt_in: str | None vs_code_extension_port: int | None @@ -754,6 +763,17 @@ class ObservabilitySettings: Can be set via environment variable ENABLE_SENSITIVE_DATA. enable_console_exporters: Enable console exporters for traces, logs, and metrics. Default is False. Can be set via environment variable ENABLE_CONSOLE_EXPORTERS. + enable_message_events: Emit the stable v1.36.0 GenAI message events (``gen_ai.system.message``, + ``gen_ai.user.message``, ``gen_ai.assistant.message``, ``gen_ai.tool.message``, ``gen_ai.choice``). + Default is True. Can be set via environment variable ENABLE_MESSAGE_EVENTS. Only takes effect + when sensitive data capture is enabled. + otel_semconv_stability_opt_in: Selects which GenAI semantic-conventions release Agent Framework emits, + following the standard OpenTelemetry comma-separated opt-in list format. v1.36.0 is the OTel-recommended + stable release; every version above it is collectively "experimental". Unset (the default, unlike + upstream OpenTelemetry which defaults to stable-only) or a list containing + ``"gen_ai_latest_experimental"`` selects the conventions above v1.36.0; a list that omits that token + (e.g. ``""``) selects the v1.36.0 conventions instead. Can be set via environment variable + OTEL_SEMCONV_STABILITY_OPT_IN. vs_code_extension_port: The port the AI Toolkit or Microsoft Foundry VS Code extensions are listening on. Default is None. Can be set via environment variable VS_CODE_EXTENSION_PORT. @@ -800,6 +820,9 @@ def __init__(self, **kwargs: Any) -> None: ) self.enable_console_exporters: bool = data.get("enable_console_exporters") or False + message_events_value = data.get("enable_message_events") + self.enable_message_events: bool = True if message_events_value is None else message_events_value + self.otel_semconv_stability_opt_in: str | None = data.get("otel_semconv_stability_opt_in") self.vs_code_extension_port: int | None = data.get("vs_code_extension_port") self.env_file_path = env_file_path self.env_file_encoding = env_file_encoding @@ -850,6 +873,23 @@ def enable_sensitive_data(self, value: bool) -> None: return self._enable_sensitive_data = value + @property + def use_latest_experimental_gen_ai_semconv(self) -> bool: + """Whether to emit the GenAI semantic conventions above the v1.36.0 stable release. + + v1.36.0 is the OTel-recommended stable release; every version above it is collectively + "experimental". + + Computed from ``otel_semconv_stability_opt_in`` (env var ``OTEL_SEMCONV_STABILITY_OPT_IN``), a + comma-separated opt-in list per the standard OpenTelemetry format. Agent Framework defaults this + to True (opted into the conventions above v1.36.0) when the setting is unset, which differs from + upstream OpenTelemetry's default of stable-only. + """ + if self.otel_semconv_stability_opt_in is None: + return True + tokens = {token.strip() for token in self.otel_semconv_stability_opt_in.split(",")} + return GEN_AI_LATEST_EXPERIMENTAL_OPT_IN in tokens + @property def ENABLED(self) -> bool: """Check if model diagnostics are enabled. @@ -866,6 +906,15 @@ def SENSITIVE_DATA_ENABLED(self) -> bool: """ return self.enable_instrumentation and self.enable_sensitive_data + @property + def emit_tool_call_attributes(self) -> bool: + """Whether to emit gen_ai.tool.call.arguments/result on execute_tool spans. + + These attributes were introduced above v1.36.0, so they require both sensitive-data + capture and the semconv version that supports them. + """ + return self.SENSITIVE_DATA_ENABLED and self.use_latest_experimental_gen_ai_semconv + @property def is_setup(self) -> bool: """Check if the setup has been executed.""" @@ -1234,6 +1283,8 @@ def configure_otel_providers( *, enable_sensitive_data: bool | None = None, enable_console_exporters: bool | None = None, + enable_message_events: bool | None = None, + otel_semconv_stability_opt_in: str | None = None, exporters: list[LogRecordExporter | SpanExporter | MetricExporter] | None = None, views: list[View] | None = None, vs_code_extension_port: int | None = None, @@ -1274,6 +1325,13 @@ def configure_otel_providers( the environment variable ENABLE_SENSITIVE_DATA if set. Default is None. enable_console_exporters: Enable console exporters for traces, logs, and metrics. Overrides the environment variable ENABLE_CONSOLE_EXPORTERS if set. Default is None. + enable_message_events: Emit the stable v1.36.0 GenAI message events (``gen_ai.system.message``, etc.). + Overrides the environment variable ENABLE_MESSAGE_EVENTS if set. Default is None, which resolves + to True (events enabled). + otel_semconv_stability_opt_in: Selects which GenAI semantic-conventions release to emit (see + ``ObservabilitySettings.otel_semconv_stability_opt_in`` for the full explanation). Overrides the + environment variable OTEL_SEMCONV_STABILITY_OPT_IN if set. Default is None, which resolves to the + conventions above the v1.36.0 stable release. exporters: A list of custom exporters for logs, metrics or spans, or any combination. These will be added in addition to exporters configured via environment variables. Default is None. @@ -1370,6 +1428,10 @@ def configure_otel_providers( settings_kwargs["enable_sensitive_data"] = enable_sensitive_data if enable_console_exporters is not None: settings_kwargs["enable_console_exporters"] = enable_console_exporters + if enable_message_events is not None: + settings_kwargs["enable_message_events"] = enable_message_events + if otel_semconv_stability_opt_in is not None: + settings_kwargs["otel_semconv_stability_opt_in"] = otel_semconv_stability_opt_in if vs_code_extension_port is not None: settings_kwargs["vs_code_extension_port"] = vs_code_extension_port @@ -1377,6 +1439,8 @@ def configure_otel_providers( OBSERVABILITY_SETTINGS.enable_instrumentation = updated_settings.enable_instrumentation OBSERVABILITY_SETTINGS.enable_sensitive_data = updated_settings.enable_sensitive_data OBSERVABILITY_SETTINGS.enable_console_exporters = updated_settings.enable_console_exporters + OBSERVABILITY_SETTINGS.enable_message_events = updated_settings.enable_message_events + OBSERVABILITY_SETTINGS.otel_semconv_stability_opt_in = updated_settings.otel_semconv_stability_opt_in OBSERVABILITY_SETTINGS.vs_code_extension_port = updated_settings.vs_code_extension_port OBSERVABILITY_SETTINGS.env_file_path = updated_settings.env_file_path OBSERVABILITY_SETTINGS.env_file_encoding = updated_settings.env_file_encoding @@ -1393,6 +1457,16 @@ def configure_otel_providers( if enable_console_exporters is not None else _read_bool_env("ENABLE_CONSOLE_EXPORTERS") ) + OBSERVABILITY_SETTINGS.enable_message_events = ( + enable_message_events + if enable_message_events is not None + else _read_bool_env("ENABLE_MESSAGE_EVENTS", default=True) + ) + OBSERVABILITY_SETTINGS.otel_semconv_stability_opt_in = ( + otel_semconv_stability_opt_in + if otel_semconv_stability_opt_in is not None + else os.getenv("OTEL_SEMCONV_STABILITY_OPT_IN") + ) OBSERVABILITY_SETTINGS.vs_code_extension_port = ( vs_code_extension_port if vs_code_extension_port is not None else _read_int_env("VS_CODE_EXTENSION_PORT") ) @@ -1880,6 +1954,8 @@ def _trace_agent_invocation( inner_response_telemetry_captured_fields: set[str] = set() inner_response_telemetry_captured_fields_token: contextvars.Token[set[str] | None] | None = None inner_accumulated_usage_token: contextvars.Token[UsageDetails | None] | None = None + # Agent Framework's agents run in-process (the actual network call happens on a nested + # chat span), so invoke_agent spans use the default INTERNAL kind. span = _start_streaming_span(attributes, OtelAttr.AGENT_NAME) if OBSERVABILITY_SETTINGS.SENSITIVE_DATA_ENABLED and messages and span.is_recording(): @@ -2020,6 +2096,8 @@ async def _run() -> AgentResponse[Any]: ) inner_accumulated_usage_token = INNER_ACCUMULATED_USAGE.set({}) try: + # Agent Framework's agents run in-process (the actual network call happens on a nested + # chat span), so invoke_agent spans use the default INTERNAL kind. with _get_span(attributes=attributes, span_name_attribute=OtelAttr.AGENT_NAME) as span: try: if OBSERVABILITY_SETTINGS.SENSITIVE_DATA_ENABLED and messages and span.is_recording(): @@ -2296,6 +2374,7 @@ def _activate_span(span: trace.Span) -> Generator[None]: def _get_span( attributes: dict[str, Any], span_name_attribute: str, + kind: trace.SpanKind = trace.SpanKind.INTERNAL, ) -> Generator[trace.Span, Any, Any]: """Start a span for a agent run. @@ -2303,7 +2382,7 @@ def _get_span( """ operation = attributes.get(OtelAttr.OPERATION, "operation") span_name = attributes.get(span_name_attribute, "unknown") - span = get_tracer().start_span(f"{operation} {span_name}") + span = get_tracer().start_span(f"{operation} {span_name}", kind=kind) span.set_attributes(attributes) with trace.use_span( span=span, @@ -2314,7 +2393,11 @@ def _get_span( yield current_span -def _start_streaming_span(attributes: dict[str, Any], span_name_attribute: str) -> trace.Span: +def _start_streaming_span( + attributes: dict[str, Any], + span_name_attribute: str, + kind: trace.SpanKind = trace.SpanKind.INTERNAL, +) -> trace.Span: """Start a non-current span for a streaming operation. Unlike :func:`_get_span`, the returned span is not attached to the current @@ -2332,7 +2415,7 @@ def _start_streaming_span(attributes: dict[str, Any], span_name_attribute: str) """ operation = attributes.get(OtelAttr.OPERATION, "operation") span_name = attributes.get(span_name_attribute, "unknown") - span = get_tracer().start_span(f"{operation} {span_name}") + span = get_tracer().start_span(f"{operation} {span_name}", kind=kind) span.set_attributes(attributes) return span @@ -2564,7 +2647,6 @@ def _otel_tool_definition(type_value: str, name_value: str, source: Mapping[str, OTEL_ATTR_MAP: dict[str | tuple[str, ...], tuple[str, Callable[[Any], Any] | None, bool, Any]] = { "choice_count": (OtelAttr.CHOICE_COUNT, None, False, 1), "operation_name": (OtelAttr.OPERATION, None, False, None), - "system_name": (OtelAttr.SYSTEM, None, False, None), "provider_name": (OtelAttr.PROVIDER_NAME, None, False, None), "service_url": (OtelAttr.ADDRESS, None, False, None), "conversation_id": (OtelAttr.CONVERSATION_ID, None, True, None), @@ -2600,6 +2682,14 @@ def _otel_tool_definition(type_value: str, name_value: str, source: Mapping[str, } +def _provider_name_attr() -> OtelAttr: + """Return the provider-identifying attribute for the active GenAI semconv version. + + ``gen_ai.system`` was renamed to ``gen_ai.provider.name`` in the conventions above v1.36.0. + """ + return OtelAttr.PROVIDER_NAME if OBSERVABILITY_SETTINGS.use_latest_experimental_gen_ai_semconv else OtelAttr.SYSTEM + + def _get_span_attributes(**kwargs: Any) -> dict[str, Any]: """Get the span attributes from a kwargs dictionary.""" attributes: dict[str, Any] = {} @@ -2634,6 +2724,11 @@ def _get_span_attributes(**kwargs: Any) -> dict[str, Any]: if result is not None: attributes[otel_key] = result + if OtelAttr.PROVIDER_NAME in attributes: + # Rename to the active semconv version's key; extend with a similar pop/rename if future + # OTel releases rename other attributes we emit. + attributes[_provider_name_attr()] = attributes.pop(OtelAttr.PROVIDER_NAME) + return attributes @@ -2732,30 +2827,44 @@ def _capture_messages( output: bool = False, finish_reason: FinishReason | None = None, ) -> None: - """Log messages with extra information.""" + """Log messages with extra information. + + Message events (``gen_ai.system.message``, etc.) are the stable v1.36.0 GenAI + semantic-conventions representation, emitted when ``enable_message_events`` is True + (the default). The ``gen_ai.input.messages``/``gen_ai.output.messages`` span + attributes are the representation used by conventions above v1.36.0, emitted only + when ``use_latest_experimental_gen_ai_semconv`` is True (also the default). + """ from ._types import normalize_messages - normalized_messages = normalize_messages(messages) - otel_messages: list[dict[str, Any]] = [] - for index, message in enumerate(normalized_messages): - # Reuse the otel message representation for logging instead of calling to_dict() - # to avoid expensive Pydantic serialization overhead - otel_message = _to_otel_message(message) - logger.info( - otel_message, - extra={ - OtelAttr.EVENT_NAME: OtelAttr.CHOICE if output else ROLE_EVENT_MAP.get(message.role), - OtelAttr.PROVIDER_NAME: provider_name, - MessageListTimestampFilter.INDEX_KEY: index, - }, - ) - otel_messages.append(otel_message) - if finish_reason: - otel_messages[-1]["finish_reason"] = FINISH_REASON_MAP[finish_reason] - span.set_attribute( - OtelAttr.OUTPUT_MESSAGES if output else OtelAttr.INPUT_MESSAGES, - json.dumps(otel_messages, ensure_ascii=False), - ) + emit_events = OBSERVABILITY_SETTINGS.enable_message_events + emit_span_attribute = OBSERVABILITY_SETTINGS.use_latest_experimental_gen_ai_semconv + + if emit_events or emit_span_attribute: + normalized_messages = normalize_messages(messages) + otel_messages: list[dict[str, Any]] = [] + for index, message in enumerate(normalized_messages): + # Reuse the otel message representation for logging instead of calling to_dict() + # to avoid expensive Pydantic serialization overhead + otel_message = _to_otel_message(message) + if emit_events: + logger.info( + otel_message, + extra={ + OtelAttr.EVENT_NAME: OtelAttr.CHOICE if output else ROLE_EVENT_MAP.get(message.role), + _provider_name_attr(): provider_name, + MessageListTimestampFilter.INDEX_KEY: index, + }, + ) + if emit_span_attribute: + otel_messages.append(otel_message) + if emit_span_attribute: + if finish_reason and otel_messages: + otel_messages[-1]["finish_reason"] = FINISH_REASON_MAP[finish_reason] + span.set_attribute( + OtelAttr.OUTPUT_MESSAGES if output else OtelAttr.INPUT_MESSAGES, + json.dumps(otel_messages, ensure_ascii=False), + ) _capture_system_instructions(span, system_instructions) @@ -2874,6 +2983,7 @@ def _get_response_attributes( GEN_AI_METRIC_ATTRIBUTES = ( OtelAttr.OPERATION, OtelAttr.PROVIDER_NAME, + OtelAttr.SYSTEM, OtelAttr.REQUEST_MODEL, OtelAttr.RESPONSE_MODEL, OtelAttr.ADDRESS, diff --git a/python/packages/core/tests/conftest.py b/python/packages/core/tests/conftest.py index 1627da85cd..83136cbf07 100644 --- a/python/packages/core/tests/conftest.py +++ b/python/packages/core/tests/conftest.py @@ -29,6 +29,8 @@ def span_exporter(monkeypatch, enable_instrumentation: bool, enable_sensitive_da "ENABLE_INSTRUMENTATION", "ENABLE_SENSITIVE_DATA", "ENABLE_CONSOLE_EXPORTERS", + "ENABLE_MESSAGE_EVENTS", + "OTEL_SEMCONV_STABILITY_OPT_IN", "OTEL_EXPORTER_OTLP_ENDPOINT", "OTEL_EXPORTER_OTLP_TRACES_ENDPOINT", "OTEL_EXPORTER_OTLP_METRICS_ENDPOINT", diff --git a/python/packages/core/tests/core/test_observability.py b/python/packages/core/tests/core/test_observability.py index 4256e62b2d..e73fa328d3 100644 --- a/python/packages/core/tests/core/test_observability.py +++ b/python/packages/core/tests/core/test_observability.py @@ -1831,6 +1831,117 @@ def test_enable_instrumentation_reads_env_sensitive_data(monkeypatch): assert observability.OBSERVABILITY_SETTINGS.enable_sensitive_data is True +# region Test GenAI semconv stability opt-in + + +def test_semconv_defaults_to_latest_experimental_when_unset(monkeypatch): + """OTEL_SEMCONV_STABILITY_OPT_IN unset → MAF defaults to the latest experimental conventions.""" + from agent_framework.observability import ObservabilitySettings + + monkeypatch.delenv("OTEL_SEMCONV_STABILITY_OPT_IN", raising=False) + settings = ObservabilitySettings() + + assert settings.otel_semconv_stability_opt_in is None + assert settings.use_latest_experimental_gen_ai_semconv is True + + +def test_semconv_explicit_empty_opts_into_stable(monkeypatch): + """Explicitly setting OTEL_SEMCONV_STABILITY_OPT_IN='' opts into the stable v1.36.0 conventions.""" + from agent_framework.observability import ObservabilitySettings + + monkeypatch.setenv("OTEL_SEMCONV_STABILITY_OPT_IN", "") + settings = ObservabilitySettings() + + assert settings.use_latest_experimental_gen_ai_semconv is False + + +def test_semconv_explicit_token_opts_into_latest_experimental(monkeypatch): + """Explicitly including 'gen_ai_latest_experimental' opts into the latest experimental conventions.""" + from agent_framework.observability import ObservabilitySettings + + monkeypatch.setenv("OTEL_SEMCONV_STABILITY_OPT_IN", "gen_ai_latest_experimental") + settings = ObservabilitySettings() + + assert settings.use_latest_experimental_gen_ai_semconv is True + + +def test_semconv_multi_value_list_checks_for_gen_ai_token(monkeypatch): + """OTEL_SEMCONV_STABILITY_OPT_IN supports the standard comma-separated multi-value list format.""" + from agent_framework.observability import ObservabilitySettings + + monkeypatch.setenv("OTEL_SEMCONV_STABILITY_OPT_IN", "database, gen_ai_latest_experimental") + settings = ObservabilitySettings() + assert settings.use_latest_experimental_gen_ai_semconv is True + + monkeypatch.setenv("OTEL_SEMCONV_STABILITY_OPT_IN", "database,messaging") + settings = ObservabilitySettings() + assert settings.use_latest_experimental_gen_ai_semconv is False + + +def test_enable_message_events_defaults_true(monkeypatch): + """ENABLE_MESSAGE_EVENTS unset → defaults to True (backward-compatible with pre-versioning behavior).""" + from agent_framework.observability import ObservabilitySettings + + monkeypatch.delenv("ENABLE_MESSAGE_EVENTS", raising=False) + settings = ObservabilitySettings() + + assert settings.enable_message_events is True + + +def test_enable_message_events_can_be_disabled(monkeypatch): + """ENABLE_MESSAGE_EVENTS=false disables the stable v1.36.0 message events.""" + from agent_framework.observability import ObservabilitySettings + + monkeypatch.setenv("ENABLE_MESSAGE_EVENTS", "false") + settings = ObservabilitySettings() + + assert settings.enable_message_events is False + + +@pytest.mark.parametrize("enable_sensitive_data", [False], indirect=True) +def test_get_span_attributes_uses_provider_name_under_latest_semconv(span_exporter: InMemorySpanExporter): + """Under the default (latest) semconv, the provider attribute is gen_ai.provider.name.""" + from agent_framework.observability import _get_span_attributes # pyright: ignore[reportPrivateUsage] + + attributes = _get_span_attributes(provider_name="test_provider") + + assert attributes[OtelAttr.PROVIDER_NAME] == "test_provider" + assert OtelAttr.SYSTEM not in attributes + + +@pytest.mark.parametrize("enable_sensitive_data", [False], indirect=True) +def test_get_span_attributes_uses_system_under_stable_semconv(span_exporter: InMemorySpanExporter): + """Under the stable v1.36.0 semconv, the provider attribute reverts to gen_ai.system.""" + import agent_framework.observability as observability + + observability.OBSERVABILITY_SETTINGS.otel_semconv_stability_opt_in = "" + attributes = observability._get_span_attributes(provider_name="test_provider") # pyright: ignore[reportPrivateUsage] + + assert attributes[OtelAttr.SYSTEM] == "test_provider" + assert OtelAttr.PROVIDER_NAME not in attributes + + +@pytest.mark.parametrize("enable_sensitive_data", [True], indirect=True) +async def test_chat_client_observability_provider_name_under_stable_semconv( + mock_chat_client, span_exporter: InMemorySpanExporter +): + """Chat spans report gen_ai.system (not gen_ai.provider.name) under the stable v1.36.0 semconv.""" + import agent_framework.observability as observability + + observability.OBSERVABILITY_SETTINGS.otel_semconv_stability_opt_in = "" + client = mock_chat_client() + + messages = [Message(role="user", contents=["Test message"])] + span_exporter.clear() + await client.get_response(messages=messages, options={"model": "Test"}) + + spans = span_exporter.get_finished_spans() + assert len(spans) == 1 + span = spans[0] + assert OtelAttr.SYSTEM in span.attributes # type: ignore[operator] + assert OtelAttr.PROVIDER_NAME not in span.attributes # type: ignore[operator] + + # region Test disable_instrumentation sticky behavior @@ -2943,6 +3054,98 @@ async def _inner_get_response(self, *, messages, options, **kwargs): assert output_messages[-1].get("finish_reason") == "stop" +# region Test _capture_messages GenAI semconv versioning + + +@pytest.mark.parametrize("enable_sensitive_data", [True], indirect=True) +def test_capture_messages_stable_semconv_emits_events_only(span_exporter: InMemorySpanExporter): + """Stable v1.36.0 conventions (opt-in list without the experimental token): events only, no span attribute.""" + from opentelemetry import trace + + import agent_framework.observability as observability + + observability.OBSERVABILITY_SETTINGS.otel_semconv_stability_opt_in = "" + tracer = trace.get_tracer("test") + span_exporter.clear() + + with ( + patch("agent_framework.observability.logger.info") as mock_logger_info, + tracer.start_as_current_span("test_span") as span, + ): + observability._capture_messages( # type: ignore[reportPrivateUsage] + span=span, + provider_name="test_provider", + messages=[Message(role="user", contents=["Test"])], + ) + + spans = span_exporter.get_finished_spans() + assert len(spans) == 1 + assert OtelAttr.INPUT_MESSAGES not in spans[0].attributes # type: ignore[operator] + assert mock_logger_info.call_count == 1 + + +@pytest.mark.parametrize("enable_sensitive_data", [True], indirect=True) +def test_capture_messages_latest_experimental_emits_span_attribute_only_when_events_disabled( + span_exporter: InMemorySpanExporter, +): + """Latest experimental conventions with ENABLE_MESSAGE_EVENTS=false: span attribute only, no events.""" + import json + + from opentelemetry import trace + + import agent_framework.observability as observability + + observability.OBSERVABILITY_SETTINGS.otel_semconv_stability_opt_in = "gen_ai_latest_experimental" + observability.OBSERVABILITY_SETTINGS.enable_message_events = False + tracer = trace.get_tracer("test") + span_exporter.clear() + + with ( + patch("agent_framework.observability.logger.info") as mock_logger_info, + tracer.start_as_current_span("test_span") as span, + ): + observability._capture_messages( # type: ignore[reportPrivateUsage] + span=span, + provider_name="test_provider", + messages=[Message(role="user", contents=["Test"])], + ) + + spans = span_exporter.get_finished_spans() + assert len(spans) == 1 + input_messages = json.loads(spans[0].attributes[OtelAttr.INPUT_MESSAGES]) # type: ignore[arg-type, index] + assert [msg.get("role") for msg in input_messages] == ["user"] + assert mock_logger_info.call_count == 0 + + +@pytest.mark.parametrize("enable_sensitive_data", [True], indirect=True) +def test_capture_messages_defaults_emit_both_events_and_span_attribute(span_exporter: InMemorySpanExporter): + """Default settings (nothing configured) preserve pre-versioning behavior: both events and span attribute.""" + import json + + from opentelemetry import trace + + import agent_framework.observability as observability + + tracer = trace.get_tracer("test") + span_exporter.clear() + + with ( + patch("agent_framework.observability.logger.info") as mock_logger_info, + tracer.start_as_current_span("test_span") as span, + ): + observability._capture_messages( # type: ignore[reportPrivateUsage] + span=span, + provider_name="test_provider", + messages=[Message(role="user", contents=["Test"])], + ) + + spans = span_exporter.get_finished_spans() + assert len(spans) == 1 + input_messages = json.loads(spans[0].attributes[OtelAttr.INPUT_MESSAGES]) # type: ignore[arg-type, index] + assert [msg.get("role") for msg in input_messages] == ["user"] + assert mock_logger_info.call_count == 1 + + # region Test agent streaming exception diff --git a/python/packages/core/tests/core/test_tools.py b/python/packages/core/tests/core/test_tools.py index a24e20cacd..24718d36f4 100644 --- a/python/packages/core/tests/core/test_tools.py +++ b/python/packages/core/tests/core/test_tools.py @@ -652,6 +652,34 @@ def telemetry_test_tool(x: int, y: int) -> int: assert attributes[OtelAttr.TOOL_CALL_ID] == "test_call_id" +@pytest.mark.parametrize("enable_sensitive_data", [True], indirect=True) +async def test_tool_invoke_telemetry_omits_tool_call_attrs_under_stable_semconv(span_exporter: InMemorySpanExporter): + """gen_ai.tool.call.arguments/result were introduced above v1.36.0; omit them under the stable semconv.""" + import agent_framework.observability as observability + + observability.OBSERVABILITY_SETTINGS.otel_semconv_stability_opt_in = "" + + @tool( + name="telemetry_test_tool", + description="A test tool for telemetry", + ) + def telemetry_test_tool(x: int, y: int) -> int: + """A function that adds two numbers for telemetry testing.""" + return x + y + + span_exporter.clear() + result = await telemetry_test_tool.invoke(x=1, y=2, tool_call_id="test_call_id") + + assert isinstance(result, list) + assert result[0].text == "3" + + spans = span_exporter.get_finished_spans() + assert len(spans) == 1 + span = spans[0] + assert OtelAttr.TOOL_ARGUMENTS not in span.attributes # type: ignore[operator] + assert OtelAttr.TOOL_RESULT not in span.attributes # type: ignore[operator] + + async def test_tool_invoke_rejects_unexpected_runtime_kwargs() -> None: """Ensure invoke() requires runtime data to flow through FunctionInvocationContext.""" diff --git a/python/samples/02-agents/observability/README.md b/python/samples/02-agents/observability/README.md index 4ed36c0c4b..4ae59a3ed4 100644 --- a/python/samples/02-agents/observability/README.md +++ b/python/samples/02-agents/observability/README.md @@ -20,7 +20,7 @@ For more information, please refer to the following resources: The Agent Framework Python SDK is **natively instrumented** to emit logs, traces, and metrics throughout agent/model invocation and tool execution, so you can monitor your AI application's performance and track token consumption. Instrumentation follows the OpenTelemetry [Semantic Conventions for GenAI](https://opentelemetry.io/docs/specs/semconv/gen-ai/), and workflows emit their own spans for end-to-end visibility. -Setting up observability is also easy: a single call to `configure_otel_providers()` from the `agent_framework.observability` module wires up the trace, log, and metric providers. It reads the standard OpenTelemetry environment variables to configure exporters automatically. +> See [GenAI semantic-conventions versioning](#genai-semantic-conventions-versioning) for details on how Agent Framework supports different versions of the conventions. ### Five patterns for configuring observability @@ -199,12 +199,41 @@ Agent Framework reads the following environment variables: | `ENABLE_INSTRUMENTATION` | `true` | Set to `false` to disable native instrumentation. See [Disabling instrumentation](#disabling-instrumentation) for the programmatic alternative with sticky semantics. | | `ENABLE_SENSITIVE_DATA` | `false` | Set to `true` to emit sensitive data (prompts, responses, etc.). | | `ENABLE_CONSOLE_EXPORTERS` | `false` | Set to `true` to add console exporters. Only used by `configure_otel_providers()`. | +| `ENABLE_MESSAGE_EVENTS` | `true` | Set to `false` to stop emitting the stable v1.36.0 GenAI message events (`gen_ai.system.message`, etc.). **Has no effect unless `ENABLE_SENSITIVE_DATA=true`.** See [GenAI semantic-conventions versioning](#genai-semantic-conventions-versioning). | +| `OTEL_SEMCONV_STABILITY_OPT_IN` | unset (conventions above v1.36.0) | Comma-separated OTel opt-in list. Include `gen_ai_latest_experimental` (the default when unset) to emit `gen_ai.input.messages`/`gen_ai.output.messages` span attributes; set to `""` or omit the token to use only the stable v1.36.0 conventions. **Only affects message content, so it has no effect unless `ENABLE_SENSITIVE_DATA=true`.** See [GenAI semantic-conventions versioning](#genai-semantic-conventions-versioning). | | `VS_CODE_EXTENSION_PORT` | unset | Port used by the [AI Toolkit for VS Code](https://marketplace.visualstudio.com/items?itemName=ms-windows-ai-studio.windows-ai-studio#tracing) tracing integration. Only used by `configure_otel_providers()`. | You can also call `enable_sensitive_telemetry()` from `agent_framework.observability` to opt in to sensitive-data capture programmatically. > **Note**: Sensitive data includes prompts, responses, and tool arguments. Only enable it in development or test environments — it may expose user or system secrets in production. +### GenAI semantic-conventions versioning + +[v1.36.0](https://github.com/open-telemetry/semantic-conventions/blob/v1.36.0/docs/gen-ai/gen-ai-spans.md) is the OpenTelemetry-recommended **stable** release of the [GenAI semantic conventions](https://opentelemetry.io/docs/specs/semconv/gen-ai/). Every release above it (v1.37.0 and later) is collectively **experimental** and, per OTel's own [stability warning](https://github.com/open-telemetry/semantic-conventions/blob/v1.37.0/docs/gen-ai/gen-ai-spans.md), keeps changing in more than one way. `OTEL_SEMCONV_STABILITY_OPT_IN` is the OTel-standard switch between these two rule sets, and Agent Framework applies it consistently across every attribute/representation it knows differs between the two: + +| Aspect | v1.36.0 (stable) | Above v1.36.0 (experimental, the default) | +|--------|------------------|--------------------------------------------| +| Input/output message representation | Log-record **events** (`gen_ai.system.message`, `gen_ai.user.message`, `gen_ai.assistant.message`, `gen_ai.tool.message`, `gen_ai.choice`) | `gen_ai.input.messages`/`gen_ai.output.messages` **span attributes** | +| Provider-identifying attribute | `gen_ai.system` | `gen_ai.provider.name` | +| Tool call arguments/results on `execute_tool` spans | Not emitted (introduced in v1.38.0) | `gen_ai.tool.call.arguments` / `gen_ai.tool.call.result` | + +`invoke_agent` spans always use `INTERNAL` span kind (the OTel default), regardless of semconv version. The v1.41.0 spec defines `CLIENT` for agents that are themselves a remote service and `INTERNAL` for agents that run in-process (no `server.address`/`server.port`/token-usage attributes, since the actual network call happens on a nested `chat` span instead). Agent Framework's own agents run in-process — `agent.run()` orchestrates a locally-running chat client, which creates its own nested `chat` span for the actual network call — so `INTERNAL` applies uniformly, without needing to classify each agent implementation across packages. What's **not yet covered** by this flag is the rest of the v1.41.0 attribute-group split: under the conventions above v1.36.0, the `invoke_agent` client span is defined to drop `gen_ai.response.id`, `gen_ai.response.model`, and `gen_ai.response.finish_reasons` and add `gen_ai.agent.version` instead. Agent Framework still emits the former three unconditionally on `invoke_agent` spans and does not emit `gen_ai.agent.version` at all under either semconv version. + +> **`ENABLE_SENSITIVE_DATA=true` is a prerequisite for the message-representation and tool-call-attribute rows above.** Chat content (prompts, responses, tool arguments/results) is only ever captured when sensitive-data capture is enabled (see [`ENABLE_SENSITIVE_DATA`](#environment-variables) above); the provider-attribute rename applies regardless, since `gen_ai.system`/`gen_ai.provider.name` is not sensitive data. If `ENABLE_SENSITIVE_DATA` is `false` (the default), `ENABLE_MESSAGE_EVENTS` has nothing to switch and is effectively ignored, and no `gen_ai.tool.call.*` attributes are emitted under either semconv version. + +Agent Framework defaults to the conventions above v1.36.0 (unlike upstream OpenTelemetry, which defaults to stable-only) because most users already depend on them, and — to avoid a breaking change for anyone consuming the older message events — also keeps emitting those events by default via `ENABLE_MESSAGE_EVENTS`. `ENABLE_MESSAGE_EVENTS` is controlled independently of `OTEL_SEMCONV_STABILITY_OPT_IN`: + +```bash +# Prerequisite for both examples below: capture message content at all. +ENABLE_SENSITIVE_DATA=true + +# Opt into the stable v1.36.0 conventions only (events, no gen_ai.input.messages/output.messages span attributes) +OTEL_SEMCONV_STABILITY_OPT_IN="" + +# Conventions above v1.36.0 only, no message events (strict spec compliance) +ENABLE_MESSAGE_EVENTS=false +``` + ### Disabling instrumentation There are two ways to turn Agent Framework's native instrumentation off, and they have **different scopes**: