diff --git a/packages/opentelemetry-instrumentation-anthropic/opentelemetry/instrumentation/anthropic/safety.py b/packages/opentelemetry-instrumentation-anthropic/opentelemetry/instrumentation/anthropic/safety.py index cb826ae87a..ee0111e137 100644 --- a/packages/opentelemetry-instrumentation-anthropic/opentelemetry/instrumentation/anthropic/safety.py +++ b/packages/opentelemetry-instrumentation-anthropic/opentelemetry/instrumentation/anthropic/safety.py @@ -3,6 +3,7 @@ from opentelemetry.instrumentation.fortifyroot import ( SafetyDecision, SafetyLocation, + build_safety_metadata, clone_value, get_object_value, run_completion_safety, @@ -17,6 +18,7 @@ def _apply_prompt_safety(span, kwargs, span_name: str): try: request_type = _request_type(span_name) + request_model = kwargs.get("model") mutated_kwargs = kwargs prompt = kwargs.get("prompt") @@ -28,6 +30,7 @@ def _apply_prompt_safety(span, kwargs, span_name: str): request_type=request_type, segment_index=0, segment_role="user", + request_model=request_model, ) if changed: mutated_kwargs = dict(kwargs) @@ -41,6 +44,7 @@ def _apply_prompt_safety(span, kwargs, span_name: str): request_type=request_type, segment_index=0, segment_role="system", + request_model=request_model, ) if system_changed: if mutated_kwargs is kwargs: @@ -62,6 +66,7 @@ def _apply_prompt_safety(span, kwargs, span_name: str): request_type=request_type, segment_index=index, segment_role=role, + request_model=request_model, ) if not changed: continue @@ -85,6 +90,7 @@ def _mask_prompt_content( request_type, segment_index, segment_role, + request_model=None, ): if isinstance(content, str): return _mask_prompt_text( @@ -94,6 +100,7 @@ def _mask_prompt_content( request_type=request_type, segment_index=segment_index, segment_role=segment_role, + request_model=request_model, ) if not isinstance(content, list): @@ -114,6 +121,7 @@ def _mask_prompt_content( segment_index=segment_index, segment_role=segment_role, metadata={"block_index": block_index}, + request_model=request_model, ) if not changed: continue @@ -127,6 +135,7 @@ def _mask_prompt_content( def _apply_completion_safety(span, response, span_name: str): try: request_type = _request_type(span_name) + response_model = get_object_value(response, "model") completion = get_object_value(response, "completion") if isinstance(completion, str): @@ -137,6 +146,7 @@ def _apply_completion_safety(span, response, span_name: str): request_type=request_type, segment_index=0, segment_role="assistant", + response_model=response_model, ) if changed: set_object_value(response, "completion", updated_completion) @@ -166,6 +176,7 @@ def _apply_completion_safety(span, response, span_name: str): request_type=request_type, segment_index=index, segment_role=role, + response_model=response_model, ) if changed: set_object_value(block, text_key, updated_text) @@ -182,7 +193,13 @@ def _mask_prompt_text( segment_index, segment_role, metadata=None, + request_model=None, ): + metadata = build_safety_metadata( + metadata, + provider=PROVIDER, + request_model=request_model, + ) result = run_prompt_safety( span=span, provider=PROVIDER, @@ -205,7 +222,14 @@ def _mask_completion_text( request_type, segment_index, segment_role, + metadata=None, + response_model=None, ): + metadata = build_safety_metadata( + metadata, + provider=PROVIDER, + response_model=response_model, + ) result = run_completion_safety( span=span, provider=PROVIDER, @@ -215,6 +239,7 @@ def _mask_completion_text( request_type=request_type, segment_index=segment_index, segment_role=segment_role, + metadata=metadata, ) return _resolve_masked_text(text, result) diff --git a/packages/opentelemetry-instrumentation-anthropic/tests/test_safety_unit.py b/packages/opentelemetry-instrumentation-anthropic/tests/test_safety_unit.py index 341e4c0238..278362f817 100644 --- a/packages/opentelemetry-instrumentation-anthropic/tests/test_safety_unit.py +++ b/packages/opentelemetry-instrumentation-anthropic/tests/test_safety_unit.py @@ -52,6 +52,21 @@ def test_apply_prompt_safety_masks_prompt_system_and_messages(monkeypatch): assert updated["messages"][0]["content"][0]["text"] == "masked:msg-secret" +def test_apply_prompt_safety_passes_model_metadata(monkeypatch): + contexts = [] + + def _prompt(**kwargs): + contexts.append(kwargs) + return None + + monkeypatch.setattr(safety, "run_prompt_safety", _prompt) + kwargs = {"model": "claude-3-5-sonnet", "messages": [{"role": "user", "content": "secret"}]} + safety._apply_prompt_safety(None, kwargs, "anthropic.chat") + + assert contexts[0]["metadata"]["gen_ai.system"] == "Anthropic" + assert contexts[0]["metadata"]["gen_ai.request.model"] == "claude-3-5-sonnet" + + def test_apply_prompt_safety_returns_partial_update_when_messages_missing(): monkeypatch = pytest.MonkeyPatch() monkeypatch.setattr(safety, "run_prompt_safety", lambda **kwargs: SafetyResult(text=f"masked:{kwargs['text']}", overall_action="MASK")) @@ -80,6 +95,21 @@ def test_apply_completion_safety_masks_completion_and_content(monkeypatch): assert response.content[1]["thinking"] == "masked:thought-secret" +def test_apply_completion_safety_passes_model_metadata(monkeypatch): + contexts = [] + + def _completion(**kwargs): + contexts.append(kwargs) + return None + + monkeypatch.setattr(safety, "run_completion_safety", _completion) + response = SimpleNamespace(model="claude-3-5-sonnet", completion="secret") + safety._apply_completion_safety(None, response, "anthropic.chat") + + assert contexts[0]["metadata"]["gen_ai.system"] == "Anthropic" + assert contexts[0]["metadata"]["gen_ai.response.model"] == "claude-3-5-sonnet" + + def test_anthropic_prompt_and_completion_helpers_cover_noop_branches(monkeypatch): monkeypatch.setattr(safety, "run_prompt_safety", lambda **kwargs: SafetyResult(text=kwargs["text"], overall_action="MASK")) updated, changed = safety._mask_prompt_content( diff --git a/packages/opentelemetry-instrumentation-bedrock/opentelemetry/instrumentation/bedrock/__init__.py b/packages/opentelemetry-instrumentation-bedrock/opentelemetry/instrumentation/bedrock/__init__.py index 0468a09641..d54bde1d8a 100644 --- a/packages/opentelemetry-instrumentation-bedrock/opentelemetry/instrumentation/bedrock/__init__.py +++ b/packages/opentelemetry-instrumentation-bedrock/opentelemetry/instrumentation/bedrock/__init__.py @@ -364,12 +364,17 @@ def stream_done(response_body): @dont_throw def _handle_call(span: Span, kwargs, response, metric_params, event_logger): request_body = json.loads(kwargs.get("body")) - response_body = _prepare_invoke_response(span, response, _BEDROCK_INVOKE_SPAN_NAME) + (provider, model_vendor, model) = _get_vendor_model(kwargs.get("modelId")) + response_body = _prepare_invoke_response( + span, + response, + _BEDROCK_INVOKE_SPAN_NAME, + response_model=model, + ) headers = {} if "ResponseMetadata" in response: headers = response.get("ResponseMetadata").get("HTTPHeaders", {}) - (provider, model_vendor, model) = _get_vendor_model(kwargs.get("modelId")) metric_params.vendor = provider metric_params.model = model metric_params.is_stream = False @@ -400,8 +405,13 @@ def _handle_call(span: Span, kwargs, response, metric_params, event_logger): @dont_throw def _handle_converse(span, kwargs, response, metric_params, event_logger): - _apply_converse_completion_safety(span, response, _BEDROCK_CONVERSE_SPAN_NAME) (provider, model_vendor, model) = _get_vendor_model(kwargs.get("modelId")) + _apply_converse_completion_safety( + span, + response, + _BEDROCK_CONVERSE_SPAN_NAME, + response_model=model, + ) guardrail_converse(span, response, provider, model, metric_params) set_converse_model_span_attributes(span, provider, model, kwargs) diff --git a/packages/opentelemetry-instrumentation-bedrock/opentelemetry/instrumentation/bedrock/safety.py b/packages/opentelemetry-instrumentation-bedrock/opentelemetry/instrumentation/bedrock/safety.py index 69400454fe..b75246826a 100644 --- a/packages/opentelemetry-instrumentation-bedrock/opentelemetry/instrumentation/bedrock/safety.py +++ b/packages/opentelemetry-instrumentation-bedrock/opentelemetry/instrumentation/bedrock/safety.py @@ -7,6 +7,7 @@ from opentelemetry.instrumentation.fortifyroot import ( SafetyDecision, SafetyLocation, + build_safety_metadata, clone_value, get_object_value, run_completion_safety, @@ -24,6 +25,7 @@ def _apply_invoke_prompt_safety(span, kwargs, span_name): try: + request_model = kwargs.get("modelId") payload, as_bytes = _decode_payload(kwargs.get("body")) if payload is None: return kwargs @@ -34,6 +36,7 @@ def _apply_invoke_prompt_safety(span, kwargs, span_name): span_name=span_name, request_type=_request_type(span_name), segment_index=0, + request_model=request_model, ) if not changed: return kwargs @@ -48,6 +51,7 @@ def _apply_invoke_prompt_safety(span, kwargs, span_name): def _apply_converse_prompt_safety(span, kwargs, span_name): try: + request_model = get_object_value(kwargs, "modelId") mutated_kwargs = kwargs system_messages = get_object_value(kwargs, "system") @@ -64,6 +68,7 @@ def _apply_converse_prompt_safety(span, kwargs, span_name): request_type=LLMRequestTypeValues.CHAT.value, segment_index=index, segment_role="system", + request_model=request_model, ) if not changed: continue @@ -88,6 +93,7 @@ def _apply_converse_prompt_safety(span, kwargs, span_name): request_type=LLMRequestTypeValues.CHAT.value, segment_index=index, segment_role=get_object_value(message, "role") or "user", + request_model=request_model, ) if not changed: continue @@ -103,7 +109,7 @@ def _apply_converse_prompt_safety(span, kwargs, span_name): return kwargs -def _apply_invoke_completion_safety(span, raw_response, span_name): +def _apply_invoke_completion_safety(span, raw_response, span_name, *, response_model=None): try: payload, as_bytes = _decode_payload(raw_response) if payload is None: @@ -115,6 +121,7 @@ def _apply_invoke_completion_safety(span, raw_response, span_name): span_name=span_name, request_type=_request_type(span_name), segment_index=0, + response_model=response_model, ) if not changed: return raw_response, False @@ -124,7 +131,7 @@ def _apply_invoke_completion_safety(span, raw_response, span_name): return raw_response, False -def _prepare_invoke_response(span, response, span_name): +def _prepare_invoke_response(span, response, span_name, *, response_model=None): body = response.get("body") if body is None: return None @@ -132,7 +139,7 @@ def _prepare_invoke_response(span, response, span_name): response["body"] = ReusableStreamingBody(body._raw_stream, body._content_length) raw_response = response["body"].read() masked_response, changed = _apply_invoke_completion_safety( - span, raw_response, span_name + span, raw_response, span_name, response_model=response_model ) if changed: raw_response = masked_response @@ -140,7 +147,7 @@ def _prepare_invoke_response(span, response, span_name): return json.loads(raw_response) -def _apply_converse_completion_safety(span, response, span_name): +def _apply_converse_completion_safety(span, response, span_name, *, response_model=None): try: output = get_object_value(response, "output") if output is None: @@ -154,6 +161,7 @@ def _apply_converse_completion_safety(span, response, span_name): span_name=span_name, request_type=LLMRequestTypeValues.CHAT.value, segment_index=0, + response_model=response_model, ) if changed: updated_message = clone_value(message) @@ -165,7 +173,7 @@ def _apply_converse_completion_safety(span, response, span_name): return -def _mask_prompt_payload(span, value, *, span_name, request_type, segment_index): +def _mask_prompt_payload(span, value, *, span_name, request_type, segment_index, request_model=None): if isinstance(value, dict): updated = value for key, item in value.items(): @@ -177,6 +185,7 @@ def _mask_prompt_payload(span, value, *, span_name, request_type, segment_index) request_type=request_type, segment_index=segment_index, segment_role="user", + request_model=request_model, ) elif key in {"messages", "content"}: updated_item, changed = _mask_prompt_content( @@ -186,6 +195,7 @@ def _mask_prompt_payload(span, value, *, span_name, request_type, segment_index) request_type=request_type, segment_index=segment_index, segment_role="user", + request_model=request_model, ) else: updated_item, changed = _mask_prompt_payload( @@ -194,6 +204,7 @@ def _mask_prompt_payload(span, value, *, span_name, request_type, segment_index) span_name=span_name, request_type=request_type, segment_index=segment_index, + request_model=request_model, ) if not changed: continue @@ -211,6 +222,7 @@ def _mask_prompt_payload(span, value, *, span_name, request_type, segment_index) span_name=span_name, request_type=request_type, segment_index=index, + request_model=request_model, ) if not changed: continue @@ -222,7 +234,15 @@ def _mask_prompt_payload(span, value, *, span_name, request_type, segment_index) return value, False -def _mask_completion_payload(span, value, *, span_name, request_type, segment_index): +def _mask_completion_payload( + span, + value, + *, + span_name, + request_type, + segment_index, + response_model=None, +): if isinstance(value, dict): updated = value for key, item in value.items(): @@ -233,6 +253,7 @@ def _mask_completion_payload(span, value, *, span_name, request_type, segment_in span_name=span_name, request_type=request_type, segment_index=segment_index, + response_model=response_model, ) elif key in {"content", "completions", "generations"}: updated_item, changed = _mask_completion_content( @@ -241,6 +262,7 @@ def _mask_completion_payload(span, value, *, span_name, request_type, segment_in span_name=span_name, request_type=request_type, segment_index=segment_index, + response_model=response_model, ) else: updated_item, changed = _mask_completion_payload( @@ -249,6 +271,7 @@ def _mask_completion_payload(span, value, *, span_name, request_type, segment_in span_name=span_name, request_type=request_type, segment_index=segment_index, + response_model=response_model, ) if not changed: continue @@ -266,6 +289,7 @@ def _mask_completion_payload(span, value, *, span_name, request_type, segment_in span_name=span_name, request_type=request_type, segment_index=index, + response_model=response_model, ) if not changed: continue @@ -285,6 +309,7 @@ def _mask_prompt_content( request_type, segment_index, segment_role, + request_model=None, ): if isinstance(content, str): return _mask_prompt_text( @@ -294,6 +319,7 @@ def _mask_prompt_content( request_type=request_type, segment_index=segment_index, segment_role=segment_role, + request_model=request_model, ) if not isinstance(content, list): @@ -310,6 +336,7 @@ def _mask_prompt_content( request_type=request_type, segment_index=segment_index, segment_role=segment_role, + request_model=request_model, ) if not changed: continue @@ -324,6 +351,7 @@ def _mask_prompt_content( span_name=span_name, request_type=request_type, segment_index=index, + request_model=request_model, ) if not changed: continue @@ -334,7 +362,15 @@ def _mask_prompt_content( return updated, updated is not content -def _mask_completion_content(span, content, *, span_name, request_type, segment_index): +def _mask_completion_content( + span, + content, + *, + span_name, + request_type, + segment_index, + response_model=None, +): if isinstance(content, str): return _mask_completion_text( span, @@ -342,6 +378,7 @@ def _mask_completion_content(span, content, *, span_name, request_type, segment_ span_name=span_name, request_type=request_type, segment_index=segment_index, + response_model=response_model, ) if not isinstance(content, list): @@ -357,6 +394,7 @@ def _mask_completion_content(span, content, *, span_name, request_type, segment_ span_name=span_name, request_type=request_type, segment_index=segment_index, + response_model=response_model, ) if not changed: continue @@ -371,6 +409,7 @@ def _mask_completion_content(span, content, *, span_name, request_type, segment_ span_name=span_name, request_type=request_type, segment_index=index, + response_model=response_model, ) if not changed: continue @@ -389,7 +428,12 @@ def _mask_prompt_text( request_type, segment_index, segment_role, + request_model=None, ): + metadata = build_safety_metadata( + provider=PROVIDER, + request_model=request_model, + ) result = run_prompt_safety( span=span, provider=PROVIDER, @@ -399,11 +443,24 @@ def _mask_prompt_text( request_type=request_type, segment_index=segment_index, segment_role=segment_role, + metadata=metadata, ) return _resolve_masked_text(text, result) -def _mask_completion_text(span, text, *, span_name, request_type, segment_index): +def _mask_completion_text( + span, + text, + *, + span_name, + request_type, + segment_index, + response_model=None, +): + metadata = build_safety_metadata( + provider=PROVIDER, + response_model=response_model, + ) result = run_completion_safety( span=span, provider=PROVIDER, @@ -413,6 +470,7 @@ def _mask_completion_text(span, text, *, span_name, request_type, segment_index) request_type=request_type, segment_index=segment_index, segment_role="assistant", + metadata=metadata, ) return _resolve_masked_text(text, result) diff --git a/packages/opentelemetry-instrumentation-bedrock/tests/test_safety_hooks.py b/packages/opentelemetry-instrumentation-bedrock/tests/test_safety_hooks.py index b72385c2d5..286823fd25 100644 --- a/packages/opentelemetry-instrumentation-bedrock/tests/test_safety_hooks.py +++ b/packages/opentelemetry-instrumentation-bedrock/tests/test_safety_hooks.py @@ -102,6 +102,24 @@ def test_invoke_prompt_safety_masks_json_body(): assert json.loads(updated_kwargs["body"])["prompt"] == "[PII.prompt]" +def test_invoke_prompt_safety_passes_model_metadata(): + _, tracer = _test_span() + contexts = [] + + def _handler(context): + contexts.append(context) + return None + + register_prompt_safety_handler(_handler) + + kwargs = {"body": json.dumps({"prompt": "secret"}), "modelId": "anthropic.claude-3-sonnet"} + with tracer.start_as_current_span("bedrock.completion") as span: + _apply_invoke_prompt_safety(span, kwargs, "bedrock.completion") + + assert contexts[0].metadata["gen_ai.system"] == "Bedrock" + assert contexts[0].metadata["gen_ai.request.model"] == "anthropic.claude-3-sonnet" + + def test_converse_prompt_safety_masks_message_content(): _, tracer = _test_span() register_prompt_safety_handler( @@ -144,6 +162,29 @@ def test_invoke_completion_safety_masks_json_response(): assert len(exporter.get_finished_spans()[0].events) == 1 +def test_invoke_completion_safety_passes_model_metadata(): + _, tracer = _test_span() + contexts = [] + + def _handler(context): + contexts.append(context) + return None + + register_completion_safety_handler(_handler) + + raw_response = json.dumps({"completion": "secret"}) + with tracer.start_as_current_span("bedrock.completion") as span: + _apply_invoke_completion_safety( + span, + raw_response, + "bedrock.completion", + response_model="claude-3-sonnet", + ) + + assert contexts[0].metadata["gen_ai.system"] == "Bedrock" + assert contexts[0].metadata["gen_ai.response.model"] == "claude-3-sonnet" + + def test_prepare_invoke_response_masks_and_rebuilds_streaming_body(): _, tracer = _test_span() register_completion_safety_handler( diff --git a/packages/opentelemetry-instrumentation-fortifyroot/opentelemetry/instrumentation/fortifyroot/__init__.py b/packages/opentelemetry-instrumentation-fortifyroot/opentelemetry/instrumentation/fortifyroot/__init__.py index 3444b6da1a..0edd6abf8d 100644 --- a/packages/opentelemetry-instrumentation-fortifyroot/opentelemetry/instrumentation/fortifyroot/__init__.py +++ b/packages/opentelemetry-instrumentation-fortifyroot/opentelemetry/instrumentation/fortifyroot/__init__.py @@ -1,11 +1,15 @@ from opentelemetry.instrumentation.fortifyroot.safety import ( HANDLER_LOCK, SAFETY_EVENT_NAME, + GEN_AI_REQUEST_MODEL, + GEN_AI_RESPONSE_MODEL, + GEN_AI_SYSTEM, SafetyContext, SafetyDecision, SafetyFinding, SafetyLocation, SafetyResult, + build_safety_metadata, clear_safety_handlers, clone_value, discard_deferred_findings, @@ -56,6 +60,9 @@ __all__ = [ "SAFETY_EVENT_NAME", + "GEN_AI_REQUEST_MODEL", + "GEN_AI_RESPONSE_MODEL", + "GEN_AI_SYSTEM", "HANDLER_LOCK", "clear_framework_attempts_for_thread", "clear_attempt_counters_for_test", @@ -75,6 +82,7 @@ "SafetyFinding", "SafetyLocation", "SafetyResult", + "build_safety_metadata", "clear_safety_handlers", "clear_completion_safety_stream_factory", "clone_value", diff --git a/packages/opentelemetry-instrumentation-fortifyroot/opentelemetry/instrumentation/fortifyroot/safety.py b/packages/opentelemetry-instrumentation-fortifyroot/opentelemetry/instrumentation/fortifyroot/safety.py index 017fab6e16..4bd3aad480 100644 --- a/packages/opentelemetry-instrumentation-fortifyroot/opentelemetry/instrumentation/fortifyroot/safety.py +++ b/packages/opentelemetry-instrumentation-fortifyroot/opentelemetry/instrumentation/fortifyroot/safety.py @@ -12,6 +12,9 @@ from opentelemetry.trace import Span SAFETY_EVENT_NAME = "fortifyroot.safety.violation" +GEN_AI_SYSTEM = "gen_ai.system" +GEN_AI_REQUEST_MODEL = "gen_ai.request.model" +GEN_AI_RESPONSE_MODEL = "gen_ai.response.model" logger = logging.getLogger(__name__) @@ -71,6 +74,35 @@ class SafetyResult: _completion_handler: CompletionSafetyHandler | None = None +def build_safety_metadata( + metadata: Mapping[str, Any] | None = None, + *, + provider: Any | None = None, + request_model: Any | None = None, + response_model: Any | None = None, +) -> dict[str, Any]: + """Build raw LLM context metadata for safety handlers. + + These are raw semantic-convention attributes. FortifyRoot backend owns + canonical provider-role enrichment, so SDK instrumentation should not emit + routing_provider or billing_provider directly. + """ + + attrs = dict(metadata or {}) + _set_metadata_if_present(attrs, GEN_AI_SYSTEM, provider) + _set_metadata_if_present(attrs, GEN_AI_REQUEST_MODEL, request_model) + _set_metadata_if_present(attrs, GEN_AI_RESPONSE_MODEL, response_model) + return attrs + + +def _set_metadata_if_present(attrs: dict[str, Any], key: str, value: Any | None) -> None: + if value is None: + return + text = str(value).strip() + if text: + attrs.setdefault(key, text) + + def register_prompt_safety_handler(handler: PromptSafetyHandler | None) -> None: """Register the global prompt safety handler.""" diff --git a/packages/opentelemetry-instrumentation-fortifyroot/tests/test_safety.py b/packages/opentelemetry-instrumentation-fortifyroot/tests/test_safety.py index 30d9a3ce1b..644c74d1f5 100644 --- a/packages/opentelemetry-instrumentation-fortifyroot/tests/test_safety.py +++ b/packages/opentelemetry-instrumentation-fortifyroot/tests/test_safety.py @@ -6,6 +6,7 @@ SafetyFinding, SafetyLocation, SafetyResult, + build_safety_metadata, clear_safety_handlers, clone_value, register_prompt_safety_handler, @@ -85,6 +86,20 @@ def test_run_prompt_safety_emits_one_event_per_finding(): clear_safety_handlers() +def test_build_safety_metadata_preserves_raw_llm_context_only(): + assert build_safety_metadata( + {"block_index": 1, "gen_ai.system": "OpenRouter"}, + provider="OpenAI", + request_model="openai/gpt-4o-mini", + response_model="gpt-4o-mini", + ) == { + "block_index": 1, + "gen_ai.system": "OpenRouter", + "gen_ai.request.model": "openai/gpt-4o-mini", + "gen_ai.response.model": "gpt-4o-mini", + } + + def test_run_prompt_safety_returns_none_without_handler(): exporter = InMemorySpanExporter() provider = TracerProvider() diff --git a/packages/opentelemetry-instrumentation-google-generativeai/opentelemetry/instrumentation/google_generativeai/__init__.py b/packages/opentelemetry-instrumentation-google-generativeai/opentelemetry/instrumentation/google_generativeai/__init__.py index c6b84e7814..a1d7149710 100644 --- a/packages/opentelemetry-instrumentation-google-generativeai/opentelemetry/instrumentation/google_generativeai/__init__.py +++ b/packages/opentelemetry-instrumentation-google-generativeai/opentelemetry/instrumentation/google_generativeai/__init__.py @@ -227,7 +227,14 @@ async def _awrap( }, ) start_time = time.perf_counter() - args, kwargs = await asyncio.to_thread(_apply_prompt_safety, span, args, kwargs, name) # FR: async safety + args, kwargs = await asyncio.to_thread( + _apply_prompt_safety, + span, + args, + kwargs, + name, + request_model=llm_model, + ) # FR: async safety _handle_request(span, args, kwargs, llm_model, event_logger) try: response = await wrapped(*args, **kwargs) @@ -257,7 +264,13 @@ async def _awrap( span, response, llm_model, event_logger, token_histogram ) else: - await asyncio.to_thread(_apply_completion_safety, span, response, name) # FR: async safety + await asyncio.to_thread( + _apply_completion_safety, + span, + response, + name, + response_model=llm_model, + ) # FR: async safety _handle_response( span, response, llm_model, event_logger, token_histogram ) @@ -307,7 +320,7 @@ def _wrap( ) start_time = time.perf_counter() - args, kwargs = _apply_prompt_safety(span, args, kwargs, name) + args, kwargs = _apply_prompt_safety(span, args, kwargs, name, request_model=llm_model) _handle_request(span, args, kwargs, llm_model, event_logger) try: response = wrapped(*args, **kwargs) @@ -337,7 +350,7 @@ def _wrap( span, response, llm_model, event_logger, token_histogram ) else: - _apply_completion_safety(span, response, name) + _apply_completion_safety(span, response, name, response_model=llm_model) _handle_response( span, response, llm_model, event_logger, token_histogram ) diff --git a/packages/opentelemetry-instrumentation-google-generativeai/opentelemetry/instrumentation/google_generativeai/safety.py b/packages/opentelemetry-instrumentation-google-generativeai/opentelemetry/instrumentation/google_generativeai/safety.py index 09bb3fa4ed..82cfaf112a 100644 --- a/packages/opentelemetry-instrumentation-google-generativeai/opentelemetry/instrumentation/google_generativeai/safety.py +++ b/packages/opentelemetry-instrumentation-google-generativeai/opentelemetry/instrumentation/google_generativeai/safety.py @@ -3,6 +3,7 @@ from opentelemetry.instrumentation.fortifyroot import ( SafetyDecision, SafetyLocation, + build_safety_metadata, clone_value, get_object_value, run_completion_safety, @@ -14,7 +15,7 @@ PROVIDER = "Google" -def _apply_prompt_safety(span, args, kwargs, span_name): +def _apply_prompt_safety(span, args, kwargs, span_name, *, request_model=None): try: updated_args = args updated_kwargs = kwargs @@ -26,6 +27,7 @@ def _apply_prompt_safety(span, args, kwargs, span_name): span_name=span_name, segment_index=0, segment_role="user", + request_model=request_model, ) if changed: updated_args = (masked_arg, *args[1:]) @@ -37,6 +39,7 @@ def _apply_prompt_safety(span, args, kwargs, span_name): span_name=span_name, segment_index=0, segment_role="user", + request_model=request_model, ) if changed: updated_kwargs = dict(kwargs) @@ -47,7 +50,15 @@ def _apply_prompt_safety(span, args, kwargs, span_name): return args, kwargs -def _mask_prompt_value(span, value, *, span_name, segment_index, segment_role): +def _mask_prompt_value( + span, + value, + *, + span_name, + segment_index, + segment_role, + request_model=None, +): if isinstance(value, str): return _mask_prompt_text( span, @@ -55,6 +66,7 @@ def _mask_prompt_value(span, value, *, span_name, segment_index, segment_role): span_name=span_name, segment_index=segment_index, segment_role=segment_role, + request_model=request_model, ) if isinstance(value, list): @@ -66,6 +78,7 @@ def _mask_prompt_value(span, value, *, span_name, segment_index, segment_role): span_name=span_name, segment_index=index, segment_role=segment_role, + request_model=request_model, ) if not changed: continue @@ -88,6 +101,7 @@ def _mask_prompt_value(span, value, *, span_name, segment_index, segment_role): segment_index=index, segment_role=segment_role, metadata={"part_index": index}, + request_model=request_model, ) if not changed: continue @@ -101,7 +115,7 @@ def _mask_prompt_value(span, value, *, span_name, segment_index, segment_role): return value, False -def _apply_completion_safety(span, response, span_name): +def _apply_completion_safety(span, response, span_name, *, response_model=None): try: text = get_object_value(response, "text") if isinstance(text, str): @@ -110,6 +124,7 @@ def _apply_completion_safety(span, response, span_name): text, span_name=span_name, segment_index=0, + response_model=response_model, ) if changed: set_object_value(response, "text", updated_text) @@ -133,6 +148,7 @@ def _apply_completion_safety(span, response, span_name): span_name=span_name, segment_index=candidate_index, metadata={"part_index": part_index}, + response_model=response_model, ) if changed: set_object_value(part, "text", updated_part_text) @@ -148,7 +164,13 @@ def _mask_prompt_text( segment_index, segment_role, metadata=None, + request_model=None, ): + metadata = build_safety_metadata( + metadata, + provider=PROVIDER, + request_model=request_model, + ) result = run_prompt_safety( span=span, provider=PROVIDER, @@ -170,7 +192,13 @@ def _mask_completion_text( span_name, segment_index, metadata=None, + response_model=None, ): + metadata = build_safety_metadata( + metadata, + provider=PROVIDER, + response_model=response_model, + ) result = run_completion_safety( span=span, provider=PROVIDER, diff --git a/packages/opentelemetry-instrumentation-google-generativeai/tests/test_safety_hooks.py b/packages/opentelemetry-instrumentation-google-generativeai/tests/test_safety_hooks.py index 91aa077855..d76a14d3af 100644 --- a/packages/opentelemetry-instrumentation-google-generativeai/tests/test_safety_hooks.py +++ b/packages/opentelemetry-instrumentation-google-generativeai/tests/test_safety_hooks.py @@ -79,6 +79,25 @@ def test_prompt_safety_masks_positional_prompt_args(): assert updated_kwargs == {} +def test_prompt_safety_passes_model_metadata(): + _, tracer = _test_span() + contexts = [] + + register_prompt_safety_handler(lambda context: contexts.append(context) or None) + + with tracer.start_as_current_span("gemini.generate_content") as span: + _apply_prompt_safety( + span, + ("secret",), + {}, + "gemini.generate_content", + request_model="gemini-2.0-flash", + ) + + assert contexts[0].metadata["gen_ai.system"] == "Google" + assert contexts[0].metadata["gen_ai.request.model"] == "gemini-2.0-flash" + + def test_prompt_safety_masks_span_prompt_attributes(): exporter, tracer = _test_span() register_prompt_safety_handler( @@ -150,6 +169,25 @@ def test_completion_safety_masks_candidate_parts(): assert len(spans[0].events) == 1 +def test_completion_safety_passes_model_metadata(): + _, tracer = _test_span() + contexts = [] + + register_completion_safety_handler(lambda context: contexts.append(context) or None) + + response = SimpleNamespace(text="secret") + with tracer.start_as_current_span("gemini.generate_content") as span: + _apply_completion_safety( + span, + response, + "gemini.generate_content", + response_model="gemini-2.0-flash", + ) + + assert contexts[0].metadata["gen_ai.system"] == "Google" + assert contexts[0].metadata["gen_ai.response.model"] == "gemini-2.0-flash" + + class _FakeStreamSession: def process_chunk(self, text): return SafetyResult(text="masked", overall_action="allow", findings=[]) diff --git a/packages/opentelemetry-instrumentation-litellm/opentelemetry/instrumentation/litellm/safety.py b/packages/opentelemetry-instrumentation-litellm/opentelemetry/instrumentation/litellm/safety.py index 59d73168b3..fb08d4251f 100644 --- a/packages/opentelemetry-instrumentation-litellm/opentelemetry/instrumentation/litellm/safety.py +++ b/packages/opentelemetry-instrumentation-litellm/opentelemetry/instrumentation/litellm/safety.py @@ -5,6 +5,7 @@ from opentelemetry.instrumentation.fortifyroot import ( SafetyDecision, SafetyLocation, + build_safety_metadata, clone_value, get_object_value, run_completion_safety, @@ -19,6 +20,7 @@ def apply_prompt_safety(span, args, kwargs, request_type, span_name): try: + request_model = _get_request_model(args, kwargs, request_type) messages, source = _get_messages(args, kwargs) if isinstance(messages, list): return _apply_messages_prompt_safety( @@ -29,12 +31,20 @@ def apply_prompt_safety(span, args, kwargs, request_type, span_name): source, request_type, span_name, + request_model, ) if request_type != LLMRequestTypeValues.COMPLETION.value: return args, kwargs - return _apply_text_prompt_safety(span, args, kwargs, request_type, span_name) + return _apply_text_prompt_safety( + span, + args, + kwargs, + request_type, + span_name, + request_model, + ) except Exception: logger.warning("safety prompt error", exc_info=True) return args, kwargs @@ -48,6 +58,7 @@ def _apply_messages_prompt_safety( source, request_type, span_name, + request_model, ): updated_messages = messages changed = False @@ -61,6 +72,7 @@ def _apply_messages_prompt_safety( request_type=request_type, segment_index=index, segment_role=get_object_value(message, "role") or "user", + request_model=request_model, ) if not content_changed: continue @@ -82,13 +94,14 @@ def _apply_messages_prompt_safety( return args, updated_kwargs -def _apply_text_prompt_safety(span, args, kwargs, request_type, span_name): +def _apply_text_prompt_safety(span, args, kwargs, request_type, span_name, request_model): prompt, source = _get_prompt(args, kwargs) updated_prompt, changed = _mask_text_prompt_value( span, prompt, span_name=span_name, request_type=request_type, + request_model=request_model, ) if not changed: return args, kwargs @@ -103,7 +116,7 @@ def _apply_text_prompt_safety(span, args, kwargs, request_type, span_name): return args, updated_kwargs -def _mask_text_prompt_value(span, value, *, span_name, request_type): +def _mask_text_prompt_value(span, value, *, span_name, request_type, request_model=None): if isinstance(value, str): return _mask_prompt_text( span, @@ -112,6 +125,7 @@ def _mask_text_prompt_value(span, value, *, span_name, request_type): request_type=request_type, segment_index=0, segment_role="user", + request_model=request_model, ) if not isinstance(value, list): @@ -125,6 +139,7 @@ def _mask_text_prompt_value(span, value, *, span_name, request_type): span_name=span_name, request_type=request_type, segment_index=index, + request_model=request_model, ) if not changed: continue @@ -143,6 +158,7 @@ def _mask_text_prompt_item( request_type, segment_index, metadata=None, + request_model=None, ): if isinstance(value, str): return _mask_prompt_text( @@ -153,6 +169,7 @@ def _mask_text_prompt_item( segment_index=segment_index, segment_role="user", metadata=metadata, + request_model=request_model, ) if not isinstance(value, list): @@ -167,6 +184,7 @@ def _mask_text_prompt_item( request_type=request_type, segment_index=segment_index, metadata={"nested_index": index, **(metadata or {})}, + request_model=request_model, ) if not changed: continue @@ -205,6 +223,7 @@ def _get_prompt(args, kwargs): def apply_completion_safety(span, response, request_type, span_name): try: + response_model = get_object_value(response, "model") choices = get_object_value(response, "choices") or [] for index, choice in enumerate(choices): message = get_object_value(choice, "message") @@ -216,6 +235,7 @@ def apply_completion_safety(span, response, request_type, span_name): span_name=span_name, request_type=request_type, segment_index=index, + response_model=response_model, ) if content_changed: set_object_value(message, "content", updated_content) @@ -232,6 +252,7 @@ def apply_completion_safety(span, response, request_type, span_name): span_name=span_name, request_type=request_type, segment_index=index, + response_model=response_model, ) if text_changed: set_object_value(choice, "text", updated_text) @@ -248,6 +269,15 @@ def _get_messages(args, kwargs): return kwargs.get("messages"), "kwargs" +def _get_request_model(args, kwargs, request_type): + model = kwargs.get("model") + if model is not None: + return model + if request_type == LLMRequestTypeValues.COMPLETION.value: + return args[1] if len(args) > 1 else None + return args[0] if args else None + + def _mask_prompt_content( span, content, @@ -257,6 +287,7 @@ def _mask_prompt_content( segment_index, segment_role, metadata=None, + request_model=None, ): if isinstance(content, str): return _mask_prompt_text( @@ -267,6 +298,7 @@ def _mask_prompt_content( segment_index=segment_index, segment_role=segment_role, metadata=metadata, + request_model=request_model, ) if not isinstance(content, list): @@ -283,6 +315,7 @@ def _mask_prompt_content( segment_index=segment_index, segment_role=segment_role, metadata={"block_index": block_index, **(metadata or {})}, + request_model=request_model, ) if not changed: continue @@ -303,6 +336,7 @@ def _mask_prompt_content( segment_index=segment_index, segment_role=segment_role, metadata={"block_index": block_index, **(metadata or {})}, + request_model=request_model, ) if not changed: continue @@ -321,6 +355,7 @@ def _mask_completion_content( request_type, segment_index, metadata=None, + response_model=None, ): if isinstance(content, str): return _mask_completion_text( @@ -330,6 +365,7 @@ def _mask_completion_content( request_type=request_type, segment_index=segment_index, metadata=metadata, + response_model=response_model, ) if not isinstance(content, list): @@ -345,6 +381,7 @@ def _mask_completion_content( request_type=request_type, segment_index=segment_index, metadata={"block_index": block_index, **(metadata or {})}, + response_model=response_model, ) if not changed: continue @@ -364,6 +401,7 @@ def _mask_completion_content( request_type=request_type, segment_index=segment_index, metadata={"block_index": block_index, **(metadata or {})}, + response_model=response_model, ) if not changed: continue @@ -383,7 +421,13 @@ def _mask_prompt_text( segment_index, segment_role, metadata=None, + request_model=None, ): + metadata = build_safety_metadata( + metadata, + provider=PROVIDER, + request_model=request_model, + ) result = run_prompt_safety( span=span, provider=PROVIDER, @@ -406,7 +450,13 @@ def _mask_completion_text( request_type, segment_index, metadata=None, + response_model=None, ): + metadata = build_safety_metadata( + metadata, + provider=PROVIDER, + response_model=response_model, + ) result = run_completion_safety( span=span, provider=PROVIDER, diff --git a/packages/opentelemetry-instrumentation-litellm/tests/test_safety_hooks.py b/packages/opentelemetry-instrumentation-litellm/tests/test_safety_hooks.py index 5436b9e458..e5607fb2c4 100644 --- a/packages/opentelemetry-instrumentation-litellm/tests/test_safety_hooks.py +++ b/packages/opentelemetry-instrumentation-litellm/tests/test_safety_hooks.py @@ -165,6 +165,39 @@ def wrapped(*args, **kwargs): assert span.attributes["fortifyroot.span.role"] == "safety_wrapper" +def test_prompt_and_completion_safety_pass_model_metadata(): + _, tracer = _test_tracer() + prompt_contexts = [] + completion_contexts = [] + + register_prompt_safety_handler( + lambda context: prompt_contexts.append(context) or None + ) + register_completion_safety_handler( + lambda context: completion_contexts.append(context) or None + ) + + with tracer.start_as_current_span("litellm.completion") as span: + apply_prompt_safety( + span, + (), + {"model": "openrouter/anthropic/claude-3", "messages": [{"role": "user", "content": "secret"}]}, + "chat", + "litellm.completion", + ) + apply_completion_safety( + span, + SimpleNamespace(model="anthropic/claude-3", choices=[SimpleNamespace(text="secret")]), + "chat", + "litellm.completion", + ) + + assert prompt_contexts[0].metadata["gen_ai.system"] == "LiteLLM" + assert prompt_contexts[0].metadata["gen_ai.request.model"] == "openrouter/anthropic/claude-3" + assert completion_contexts[0].metadata["gen_ai.system"] == "LiteLLM" + assert completion_contexts[0].metadata["gen_ai.response.model"] == "anthropic/claude-3" + + def test_sync_text_completion_masks_text_choices(): exporter, tracer = _test_tracer() register_prompt_safety_handler( diff --git a/packages/opentelemetry-instrumentation-openai/opentelemetry/instrumentation/openai/shared/chat_safety.py b/packages/opentelemetry-instrumentation-openai/opentelemetry/instrumentation/openai/shared/chat_safety.py index 561376edbc..f222160c91 100644 --- a/packages/opentelemetry-instrumentation-openai/opentelemetry/instrumentation/openai/shared/chat_safety.py +++ b/packages/opentelemetry-instrumentation-openai/opentelemetry/instrumentation/openai/shared/chat_safety.py @@ -17,6 +17,7 @@ def _apply_prompt_safety(span, kwargs): messages = kwargs.get("messages") if not isinstance(messages, list): return kwargs + request_model = kwargs.get("model") mutated_kwargs = kwargs mutated_messages = None @@ -28,6 +29,7 @@ def _apply_prompt_safety(span, kwargs): content, message_index=index, message_role=role, + request_model=request_model, ) if not changed: continue @@ -41,7 +43,7 @@ def _apply_prompt_safety(span, kwargs): return kwargs -def _mask_prompt_content(span, content, *, message_index, message_role): +def _mask_prompt_content(span, content, *, message_index, message_role, request_model=None): if isinstance(content, str): return mask_prompt_text( span, @@ -49,6 +51,7 @@ def _mask_prompt_content(span, content, *, message_index, message_role): span_name=CHAT_SPAN_NAME, segment_index=message_index, segment_role=message_role, + request_model=request_model, ) if not isinstance(content, list): @@ -67,6 +70,7 @@ def _mask_prompt_content(span, content, *, message_index, message_role): segment_index=message_index, segment_role=message_role, metadata={"block_index": block_index}, + request_model=request_model, ) if not changed: continue @@ -82,6 +86,7 @@ def _apply_completion_safety(span, response): choices = getattr(response, "choices", None) if not choices: return + response_model = get_object_value(response, "model") for choice_index, choice in enumerate(choices): message = get_object_value(choice, "message") @@ -92,6 +97,7 @@ def _apply_completion_safety(span, response): span, content, choice_index=choice_index, + response_model=response_model, ) if changed: set_object_value(message, "content", updated_content) @@ -99,13 +105,14 @@ def _apply_completion_safety(span, response): return -def _mask_completion_content(span, content, *, choice_index): +def _mask_completion_content(span, content, *, choice_index, response_model=None): if isinstance(content, str): return mask_completion_text( span, content, span_name=CHAT_SPAN_NAME, segment_index=choice_index, + response_model=response_model, ) if not isinstance(content, list): @@ -123,6 +130,7 @@ def _mask_completion_content(span, content, *, choice_index): span_name=CHAT_SPAN_NAME, segment_index=choice_index, metadata={"block_index": block_index}, + response_model=response_model, ) if not changed: continue diff --git a/packages/opentelemetry-instrumentation-openai/opentelemetry/instrumentation/openai/shared/completion_safety.py b/packages/opentelemetry-instrumentation-openai/opentelemetry/instrumentation/openai/shared/completion_safety.py index a32e91c9a8..8bbcf34ff7 100644 --- a/packages/opentelemetry-instrumentation-openai/opentelemetry/instrumentation/openai/shared/completion_safety.py +++ b/packages/opentelemetry-instrumentation-openai/opentelemetry/instrumentation/openai/shared/completion_safety.py @@ -11,6 +11,7 @@ def _apply_prompt_safety(span, kwargs): try: prompt = kwargs.get("prompt") + request_model = kwargs.get("model") if isinstance(prompt, str): updated_prompt, changed = mask_prompt_text( span, @@ -18,6 +19,7 @@ def _apply_prompt_safety(span, kwargs): span_name=COMPLETION_SPAN_NAME, segment_index=0, segment_role="user", + request_model=request_model, ) if not changed: return kwargs @@ -38,6 +40,7 @@ def _apply_prompt_safety(span, kwargs): span_name=COMPLETION_SPAN_NAME, segment_index=index, segment_role="user", + request_model=request_model, ) if not changed: continue @@ -55,6 +58,7 @@ def _apply_prompt_safety(span, kwargs): def _apply_completion_safety(span, response): try: + response_model = get_object_value(response, "model") for index, choice in enumerate(get_object_value(response, "choices", []) or []): text = get_object_value(choice, "text") if not isinstance(text, str): @@ -64,6 +68,7 @@ def _apply_completion_safety(span, response): text, span_name=COMPLETION_SPAN_NAME, segment_index=index, + response_model=response_model, ) if changed: set_object_value(choice, "text", updated_text) diff --git a/packages/opentelemetry-instrumentation-openai/opentelemetry/instrumentation/openai/shared/safety_common.py b/packages/opentelemetry-instrumentation-openai/opentelemetry/instrumentation/openai/shared/safety_common.py index 555a106e37..b810f0f43f 100644 --- a/packages/opentelemetry-instrumentation-openai/opentelemetry/instrumentation/openai/shared/safety_common.py +++ b/packages/opentelemetry-instrumentation-openai/opentelemetry/instrumentation/openai/shared/safety_common.py @@ -3,6 +3,7 @@ from opentelemetry.instrumentation.fortifyroot import ( SafetyDecision, SafetyLocation, + build_safety_metadata, run_completion_safety, run_prompt_safety, ) @@ -21,7 +22,13 @@ def mask_prompt_text( segment_index, segment_role=None, metadata=None, + request_model=None, ): + metadata = build_safety_metadata( + metadata, + provider=CHAT_PROVIDER, + request_model=request_model, + ) result = run_prompt_safety( span=span, provider=CHAT_PROVIDER, @@ -43,7 +50,15 @@ def mask_completion_text( span_name, segment_index, metadata=None, + request_model=None, + response_model=None, ): + metadata = build_safety_metadata( + metadata, + provider=CHAT_PROVIDER, + request_model=request_model, + response_model=response_model, + ) result = run_completion_safety( span=span, provider=CHAT_PROVIDER, diff --git a/packages/opentelemetry-instrumentation-openai/tests/traces/test_safety_hooks.py b/packages/opentelemetry-instrumentation-openai/tests/traces/test_safety_hooks.py index bf98a72da2..4347ebac5d 100644 --- a/packages/opentelemetry-instrumentation-openai/tests/traces/test_safety_hooks.py +++ b/packages/opentelemetry-instrumentation-openai/tests/traces/test_safety_hooks.py @@ -74,6 +74,24 @@ def test_chat_prompt_safety_masks_message_content_without_mutating_input(): assert spans[0].events[0].attributes["fortifyroot.safety.location"] == SafetyLocation.PROMPT.value +def test_chat_prompt_safety_passes_model_metadata(): + _, tracer = _test_span() + contexts = [] + + def _handler(context): + contexts.append(context) + return None + + register_prompt_safety_handler(_handler) + + kwargs = {"model": "gpt-4o-mini", "messages": [{"role": "user", "content": "secret"}]} + with tracer.start_as_current_span("openai.chat") as span: + _apply_prompt_safety(span, kwargs) + + assert contexts[0].metadata["gen_ai.system"] == "OpenAI" + assert contexts[0].metadata["gen_ai.request.model"] == "gpt-4o-mini" + + @pytest.mark.asyncio async def test_chat_prompt_safety_masks_span_prompt_attributes(): exporter, tracer = _test_span() @@ -136,3 +154,21 @@ def test_completion_response_safety_masks_choice_text(): spans = exporter.get_finished_spans() assert len(spans[0].events) == 1 assert spans[0].events[0].attributes["fortifyroot.safety.action"] == "MASK" + + +def test_completion_response_safety_passes_response_model_metadata(): + _, tracer = _test_span() + contexts = [] + + def _handler(context): + contexts.append(context) + return None + + register_completion_safety_handler(_handler) + + response = SimpleNamespace(model="gpt-3.5-turbo-instruct", choices=[SimpleNamespace(text="secret")]) + with tracer.start_as_current_span("openai.completion") as span: + _apply_completion_safety(span, response) + + assert contexts[0].metadata["gen_ai.system"] == "OpenAI" + assert contexts[0].metadata["gen_ai.response.model"] == "gpt-3.5-turbo-instruct"