From 6dfff99e335b3ad0fa8ea4ce9f97949420408e08 Mon Sep 17 00:00:00 2001 From: Manasjyoti Sharma Date: Wed, 11 Mar 2026 18:11:59 +0530 Subject: [PATCH] feat(safety): add phase 1h safety hooks --- .github/workflows/safety-pr.yml | 46 ++ .../instrumentation/anthropic/__init__.py | 10 +- .../instrumentation/anthropic/safety.py | 233 ++++++++ .../pyproject.toml | 9 + .../tests/test_safety_hooks.py | 136 +++++ .../tests/test_safety_unit.py | 112 ++++ .../instrumentation/cohere/__init__.py | 8 + .../instrumentation/cohere/safety.py | 270 ++++++++++ .../pyproject.toml | 9 + .../tests/test_safety_hooks.py | 146 +++++ .../tests/test_safety_unit.py | 148 ++++++ .../.python-version | 1 + .../README.md | 4 + .../instrumentation/fortifyroot/__init__.py | 37 ++ .../instrumentation/fortifyroot/safety.py | 255 +++++++++ .../poetry.toml | 2 + .../project.json | 77 +++ .../pyproject.toml | 66 +++ .../tests/test_safety.py | 131 +++++ .../tests/test_safety_helpers.py | 207 +++++++ .../uv.lock | 288 ++++++++++ .../google_generativeai/__init__.py | 8 + .../google_generativeai/safety.py | 193 +++++++ .../pyproject.toml | 9 + .../tests/test_safety_hooks.py | 144 +++++ .../tests/test_safety_unit.py | 89 ++++ .../openai/shared/chat_safety.py | 133 +++++ .../openai/shared/chat_wrappers.py | 8 + .../openai/shared/completion_safety.py | 71 +++ .../openai/shared/completion_wrappers.py | 8 + .../openai/shared/safety_common.py | 72 +++ .../pyproject.toml | 4 + .../pytest.ini | 2 + .../tests/traces/test_safety_helpers.py | 165 ++++++ .../tests/traces/test_safety_hooks.py | 138 +++++ scripts/run-tests.sh | 503 ++++++++++++++++++ 36 files changed, 3740 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/safety-pr.yml create mode 100644 packages/opentelemetry-instrumentation-anthropic/opentelemetry/instrumentation/anthropic/safety.py create mode 100644 packages/opentelemetry-instrumentation-anthropic/tests/test_safety_hooks.py create mode 100644 packages/opentelemetry-instrumentation-anthropic/tests/test_safety_unit.py create mode 100644 packages/opentelemetry-instrumentation-cohere/opentelemetry/instrumentation/cohere/safety.py create mode 100644 packages/opentelemetry-instrumentation-cohere/tests/test_safety_hooks.py create mode 100644 packages/opentelemetry-instrumentation-cohere/tests/test_safety_unit.py create mode 100644 packages/opentelemetry-instrumentation-fortifyroot/.python-version create mode 100644 packages/opentelemetry-instrumentation-fortifyroot/README.md create mode 100644 packages/opentelemetry-instrumentation-fortifyroot/opentelemetry/instrumentation/fortifyroot/__init__.py create mode 100644 packages/opentelemetry-instrumentation-fortifyroot/opentelemetry/instrumentation/fortifyroot/safety.py create mode 100644 packages/opentelemetry-instrumentation-fortifyroot/poetry.toml create mode 100644 packages/opentelemetry-instrumentation-fortifyroot/project.json create mode 100644 packages/opentelemetry-instrumentation-fortifyroot/pyproject.toml create mode 100644 packages/opentelemetry-instrumentation-fortifyroot/tests/test_safety.py create mode 100644 packages/opentelemetry-instrumentation-fortifyroot/tests/test_safety_helpers.py create mode 100644 packages/opentelemetry-instrumentation-fortifyroot/uv.lock create mode 100644 packages/opentelemetry-instrumentation-google-generativeai/opentelemetry/instrumentation/google_generativeai/safety.py create mode 100644 packages/opentelemetry-instrumentation-google-generativeai/tests/test_safety_hooks.py create mode 100644 packages/opentelemetry-instrumentation-google-generativeai/tests/test_safety_unit.py create mode 100644 packages/opentelemetry-instrumentation-openai/opentelemetry/instrumentation/openai/shared/chat_safety.py create mode 100644 packages/opentelemetry-instrumentation-openai/opentelemetry/instrumentation/openai/shared/completion_safety.py create mode 100644 packages/opentelemetry-instrumentation-openai/opentelemetry/instrumentation/openai/shared/safety_common.py create mode 100644 packages/opentelemetry-instrumentation-openai/tests/traces/test_safety_helpers.py create mode 100644 packages/opentelemetry-instrumentation-openai/tests/traces/test_safety_hooks.py create mode 100755 scripts/run-tests.sh diff --git a/.github/workflows/safety-pr.yml b/.github/workflows/safety-pr.yml new file mode 100644 index 0000000000..b9c8201657 --- /dev/null +++ b/.github/workflows/safety-pr.yml @@ -0,0 +1,46 @@ +name: Safety PR Tests + +on: + pull_request: + +permissions: + contents: read + +concurrency: + group: safety-pr-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + safety-tests: + name: Safety Tests + runs-on: ubuntu-latest + timeout-minutes: 45 + + steps: + - name: Check out code + uses: actions/checkout@v4 + with: + fetch-depth: 0 + ref: ${{ github.event.pull_request.head.sha }} + + - name: Set up Python 3.11 + uses: actions/setup-python@v5 + with: + python-version: "3.11" + cache: "pip" + + - name: Install Poetry + run: python -m pip install --upgrade pip poetry + + - name: Run safety test suite + env: + HAYSTACK_TELEMETRY_ENABLED: "False" + run: bash ./scripts/run-tests.sh --safety + + - name: Upload safety test reports + if: always() + uses: actions/upload-artifact@v4 + with: + name: safety-test-reports + path: reports/test-run/ + if-no-files-found: ignore diff --git a/packages/opentelemetry-instrumentation-anthropic/opentelemetry/instrumentation/anthropic/__init__.py b/packages/opentelemetry-instrumentation-anthropic/opentelemetry/instrumentation/anthropic/__init__.py index e4ac944ef5..42eec5735c 100644 --- a/packages/opentelemetry-instrumentation-anthropic/opentelemetry/instrumentation/anthropic/__init__.py +++ b/packages/opentelemetry-instrumentation-anthropic/opentelemetry/instrumentation/anthropic/__init__.py @@ -12,6 +12,10 @@ emit_input_events, emit_response_events, ) +from opentelemetry.instrumentation.anthropic.safety import ( + _apply_completion_safety, + _apply_prompt_safety, +) from opentelemetry.instrumentation.anthropic.span_utils import ( aset_input_attributes, set_response_attributes, @@ -540,8 +544,8 @@ def _wrap( }, ) + kwargs = _apply_prompt_safety(span, kwargs, name) _handle_input(span, event_logger, kwargs) - start_time = time.time() try: response = wrapped(*args, **kwargs) @@ -611,6 +615,7 @@ def _wrap( attributes=metric_attributes, ) + _apply_completion_safety(span, response, name) _handle_response(span, event_logger, response) if span.is_recording(): _set_token_usage( @@ -663,8 +668,8 @@ async def _awrap( SpanAttributes.LLM_REQUEST_TYPE: LLMRequestTypeValues.COMPLETION.value, }, ) + kwargs = _apply_prompt_safety(span, kwargs, name) await _ahandle_input(span, event_logger, kwargs) - start_time = time.time() try: response = await wrapped(*args, **kwargs) @@ -735,6 +740,7 @@ async def _awrap( attributes=metric_attributes, ) + _apply_completion_safety(span, response, name) await _ahandle_response(span, event_logger, response) if span.is_recording(): diff --git a/packages/opentelemetry-instrumentation-anthropic/opentelemetry/instrumentation/anthropic/safety.py b/packages/opentelemetry-instrumentation-anthropic/opentelemetry/instrumentation/anthropic/safety.py new file mode 100644 index 0000000000..cb826ae87a --- /dev/null +++ b/packages/opentelemetry-instrumentation-anthropic/opentelemetry/instrumentation/anthropic/safety.py @@ -0,0 +1,233 @@ +from __future__ import annotations + +from opentelemetry.instrumentation.fortifyroot import ( + SafetyDecision, + SafetyLocation, + clone_value, + get_object_value, + run_completion_safety, + run_prompt_safety, + set_object_value, +) +from opentelemetry.semconv_ai import LLMRequestTypeValues + +PROVIDER = "Anthropic" + + +def _apply_prompt_safety(span, kwargs, span_name: str): + try: + request_type = _request_type(span_name) + mutated_kwargs = kwargs + + prompt = kwargs.get("prompt") + if isinstance(prompt, str): + updated_prompt, changed = _mask_prompt_text( + span, + prompt, + span_name=span_name, + request_type=request_type, + segment_index=0, + segment_role="user", + ) + if changed: + mutated_kwargs = dict(kwargs) + mutated_kwargs["prompt"] = updated_prompt + + system = kwargs.get("system") + updated_system, system_changed = _mask_prompt_content( + span, + system, + span_name=span_name, + request_type=request_type, + segment_index=0, + segment_role="system", + ) + if system_changed: + if mutated_kwargs is kwargs: + mutated_kwargs = dict(kwargs) + mutated_kwargs["system"] = updated_system + + messages = kwargs.get("messages") + if not isinstance(messages, list): + return mutated_kwargs + + mutated_messages = None + for index, message in enumerate(messages): + role = get_object_value(message, "role") + content = get_object_value(message, "content") + updated_content, changed = _mask_prompt_content( + span, + content, + span_name=span_name, + request_type=request_type, + segment_index=index, + segment_role=role, + ) + if not changed: + continue + if mutated_messages is None: + if mutated_kwargs is kwargs: + mutated_kwargs = dict(kwargs) + mutated_messages = clone_value(messages) + mutated_kwargs["messages"] = mutated_messages + set_object_value(mutated_messages[index], "content", updated_content) + + return mutated_kwargs + except Exception: + return kwargs + + +def _mask_prompt_content( + span, + content, + *, + span_name, + request_type, + segment_index, + segment_role, +): + if isinstance(content, str): + return _mask_prompt_text( + span, + content, + span_name=span_name, + request_type=request_type, + segment_index=segment_index, + segment_role=segment_role, + ) + + if not isinstance(content, list): + return content, False + + updated_content = content + for block_index, block in enumerate(content): + if get_object_value(block, "type") != "text": + continue + text = get_object_value(block, "text") + if not isinstance(text, str): + continue + updated_text, changed = _mask_prompt_text( + span, + text, + span_name=span_name, + request_type=request_type, + segment_index=segment_index, + segment_role=segment_role, + metadata={"block_index": block_index}, + ) + if not changed: + continue + if updated_content is content: + updated_content = clone_value(content) + set_object_value(updated_content[block_index], "text", updated_text) + + return updated_content, updated_content is not content + + +def _apply_completion_safety(span, response, span_name: str): + try: + request_type = _request_type(span_name) + + completion = get_object_value(response, "completion") + if isinstance(completion, str): + updated_completion, changed = _mask_completion_text( + span, + completion, + span_name=span_name, + request_type=request_type, + segment_index=0, + segment_role="assistant", + ) + if changed: + set_object_value(response, "completion", updated_completion) + + content = get_object_value(response, "content") + if not isinstance(content, list): + return + + for index, block in enumerate(content): + block_type = get_object_value(block, "type") + text_key = None + role = "assistant" + if block_type == "text": + text_key = "text" + elif block_type == "thinking": + text_key = "thinking" + role = "thinking" + if text_key is None: + continue + text = get_object_value(block, text_key) + if not isinstance(text, str): + continue + updated_text, changed = _mask_completion_text( + span, + text, + span_name=span_name, + request_type=request_type, + segment_index=index, + segment_role=role, + ) + if changed: + set_object_value(block, text_key, updated_text) + except Exception: + return + + +def _mask_prompt_text( + span, + text, + *, + span_name, + request_type, + segment_index, + segment_role, + metadata=None, +): + result = run_prompt_safety( + span=span, + provider=PROVIDER, + span_name=span_name, + text=text, + location=SafetyLocation.PROMPT, + request_type=request_type, + segment_index=segment_index, + segment_role=segment_role, + metadata=metadata, + ) + return _resolve_masked_text(text, result) + + +def _mask_completion_text( + span, + text, + *, + span_name, + request_type, + segment_index, + segment_role, +): + result = run_completion_safety( + span=span, + provider=PROVIDER, + span_name=span_name, + text=text, + location=SafetyLocation.COMPLETION, + request_type=request_type, + segment_index=segment_index, + segment_role=segment_role, + ) + return _resolve_masked_text(text, result) + + +def _request_type(span_name: str) -> str: + if span_name.endswith("completion"): + return LLMRequestTypeValues.COMPLETION.value + return LLMRequestTypeValues.CHAT.value + + +def _resolve_masked_text(original_text, result): + if result is None or result.overall_action != SafetyDecision.MASK.value: + return original_text, False + if result.text == original_text: + return original_text, False + return result.text, True diff --git a/packages/opentelemetry-instrumentation-anthropic/pyproject.toml b/packages/opentelemetry-instrumentation-anthropic/pyproject.toml index 7eec6c6a90..34cc1f7fb6 100644 --- a/packages/opentelemetry-instrumentation-anthropic/pyproject.toml +++ b/packages/opentelemetry-instrumentation-anthropic/pyproject.toml @@ -12,6 +12,7 @@ readme = "README.md" requires-python = ">=3.10,<4" dependencies = [ "opentelemetry-api>=1.38.0,<2", + "opentelemetry-instrumentation-fortifyroot", "opentelemetry-instrumentation>=0.59b0", "opentelemetry-semantic-conventions-ai>=0.4.14,<0.5.0", "opentelemetry-semantic-conventions>=0.59b0", @@ -72,5 +73,13 @@ exclude = [ [tool.ruff.lint] select = ["E", "F", "W"] +[tool.pytest.ini_options] +markers = [ + "safety: safety-focused tests", +] + [tool.uv] constraint-dependencies = ["urllib3>=2.6.3", "pip>=25.3"] + +[tool.uv.sources] +opentelemetry-instrumentation-fortifyroot = { path = "../opentelemetry-instrumentation-fortifyroot", editable = true } diff --git a/packages/opentelemetry-instrumentation-anthropic/tests/test_safety_hooks.py b/packages/opentelemetry-instrumentation-anthropic/tests/test_safety_hooks.py new file mode 100644 index 0000000000..d2f6ef7235 --- /dev/null +++ b/packages/opentelemetry-instrumentation-anthropic/tests/test_safety_hooks.py @@ -0,0 +1,136 @@ +from types import SimpleNamespace + +import pytest + +from opentelemetry.instrumentation.anthropic import ( + _handle_input, + _apply_completion_safety, + _apply_prompt_safety, +) +from opentelemetry.instrumentation.fortifyroot import ( + SafetyFinding, + SafetyLocation, + SafetyResult, + clear_safety_handlers, + register_completion_safety_handler, + register_prompt_safety_handler, +) +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import SimpleSpanProcessor +from opentelemetry.sdk.trace.export.in_memory_span_exporter import ( + InMemorySpanExporter, +) + +pytestmark = pytest.mark.safety + + +def setup_function(): + clear_safety_handlers() + + +def teardown_function(): + clear_safety_handlers() + + +def _test_span(): + exporter = InMemorySpanExporter() + provider = TracerProvider() + provider.add_span_processor(SimpleSpanProcessor(exporter)) + tracer = provider.get_tracer(__name__) + return exporter, tracer + + +def test_prompt_safety_masks_system_message(): + _, tracer = _test_span() + register_prompt_safety_handler( + lambda context: SafetyResult( + text="[PII.system]", + overall_action="MASK", + findings=[ + SafetyFinding( + category="PII", + severity="MEDIUM", + action="MASK", + rule_name="PII.system", + start=0, + end=len(context.text), + ) + ], + ) + if context.location == SafetyLocation.PROMPT and context.text == "secret" + else None + ) + + kwargs = { + "system": "secret", + "messages": [{"role": "user", "content": "hello"}], + } + with tracer.start_as_current_span("anthropic.chat") as span: + updated_kwargs = _apply_prompt_safety(span, kwargs, "anthropic.chat") + + assert kwargs["system"] == "secret" + assert updated_kwargs["system"] == "[PII.system]" + + +def test_prompt_safety_masks_span_prompt_attributes(): + exporter, tracer = _test_span() + register_prompt_safety_handler( + lambda context: SafetyResult( + text="[PII.system]", + overall_action="MASK", + findings=[ + SafetyFinding( + category="PII", + severity="MEDIUM", + action="MASK", + rule_name="PII.system", + start=0, + end=len(context.text), + ) + ], + ) + if context.location == SafetyLocation.PROMPT and context.text == "secret" + else None + ) + + kwargs = { + "system": "secret", + "messages": [{"role": "user", "content": "hello"}], + } + with tracer.start_as_current_span("anthropic.chat") as span: + updated_kwargs = _apply_prompt_safety(span, kwargs, "anthropic.chat") + _handle_input(span, None, updated_kwargs) + + spans = exporter.get_finished_spans() + assert spans[0].attributes["gen_ai.prompt.0.content"] == "[PII.system]" + assert spans[0].attributes["gen_ai.prompt.0.role"] == "system" + + +def test_completion_safety_masks_text_blocks(): + exporter, tracer = _test_span() + register_completion_safety_handler( + lambda context: SafetyResult( + text="[SECRET.token]", + overall_action="MASK", + findings=[ + SafetyFinding( + category="SECRET", + severity="HIGH", + action="MASK", + rule_name="SECRET.token", + start=0, + end=len(context.text), + ) + ], + ) + if context.location == SafetyLocation.COMPLETION and context.text == "secret" + else None + ) + + response = SimpleNamespace(content=[SimpleNamespace(type="text", text="secret")]) + with tracer.start_as_current_span("anthropic.chat") as span: + _apply_completion_safety(span, response, "anthropic.chat") + + assert response.content[0].text == "[SECRET.token]" + spans = exporter.get_finished_spans() + assert len(spans[0].events) == 1 diff --git a/packages/opentelemetry-instrumentation-anthropic/tests/test_safety_unit.py b/packages/opentelemetry-instrumentation-anthropic/tests/test_safety_unit.py new file mode 100644 index 0000000000..fcf70ad0f5 --- /dev/null +++ b/packages/opentelemetry-instrumentation-anthropic/tests/test_safety_unit.py @@ -0,0 +1,112 @@ +from types import SimpleNamespace + +import pytest + +from opentelemetry.instrumentation.anthropic import safety +from opentelemetry.instrumentation.fortifyroot import SafetyDecision, SafetyResult + +pytestmark = pytest.mark.safety + + +def test_apply_prompt_safety_masks_prompt_system_and_messages(monkeypatch): + monkeypatch.setattr(safety, "run_prompt_safety", lambda **kwargs: SafetyResult(text=f"masked:{kwargs['text']}", overall_action="MASK")) + + kwargs = { + "prompt": "secret", + "system": [{"type": "text", "text": "sys-secret"}], + "messages": [{"role": "user", "content": [{"type": "text", "text": "msg-secret"}]}], + } + updated = safety._apply_prompt_safety(None, kwargs, "anthropic.chat") + + assert kwargs["prompt"] == "secret" + assert updated["prompt"] == "masked:secret" + assert updated["system"][0]["text"] == "masked:sys-secret" + assert updated["messages"][0]["content"][0]["text"] == "masked:msg-secret" + + +def test_apply_prompt_safety_returns_partial_update_when_messages_missing(): + monkeypatch = pytest.MonkeyPatch() + monkeypatch.setattr(safety, "run_prompt_safety", lambda **kwargs: SafetyResult(text=f"masked:{kwargs['text']}", overall_action="MASK")) + kwargs = {"prompt": "secret"} + updated = safety._apply_prompt_safety(None, kwargs, "anthropic.completion") + assert updated is not kwargs + assert updated["prompt"] == "masked:secret" + monkeypatch.undo() + + +def test_apply_completion_safety_masks_completion_and_content(monkeypatch): + monkeypatch.setattr(safety, "run_completion_safety", lambda **kwargs: SafetyResult(text=f"masked:{kwargs['text']}", overall_action="MASK")) + response = SimpleNamespace( + completion="secret", + content=[ + {"type": "text", "text": "text-secret"}, + {"type": "thinking", "thinking": "thought-secret"}, + {"type": "tool_use", "name": "ignored"}, + ], + ) + + safety._apply_completion_safety(None, response, "anthropic.chat") + + assert response.completion == "masked:secret" + assert response.content[0]["text"] == "masked:text-secret" + assert response.content[1]["thinking"] == "masked:thought-secret" + + +def test_anthropic_prompt_and_completion_helpers_cover_noop_branches(monkeypatch): + monkeypatch.setattr(safety, "run_prompt_safety", lambda **kwargs: SafetyResult(text=kwargs["text"], overall_action="MASK")) + updated, changed = safety._mask_prompt_content( + None, + [{"type": "tool_use", "text": "ignored"}, {"type": "text", "text": 1}], + span_name="anthropic.chat", + request_type="chat", + segment_index=0, + segment_role="user", + ) + assert changed is False + assert updated[0]["text"] == "ignored" + + monkeypatch.setattr(safety, "run_completion_safety", lambda **kwargs: SafetyResult(text=kwargs["text"], overall_action="MASK")) + response = SimpleNamespace(completion="keep", content=[{"type": "text", "text": 1}, {"type": "tool_use", "name": "ignored"}]) + safety._apply_completion_safety(None, response, "anthropic.chat") + assert response.completion == "keep" + + response = SimpleNamespace(completion="keep", content="not-a-list") + assert safety._apply_completion_safety(None, response, "anthropic.chat") is None + + +def test_anthropic_message_only_change_path(monkeypatch): + def _prompt(**kwargs): + return SafetyResult(text=kwargs["text"], overall_action="MASK") if kwargs["text"] == "keep" else SafetyResult(text="masked:secret", overall_action="MASK") + + monkeypatch.setattr(safety, "run_prompt_safety", _prompt) + kwargs = {"messages": [{"role": "user", "content": "secret"}]} + updated = safety._apply_prompt_safety(None, kwargs, "anthropic.chat") + assert updated["messages"][0]["content"] == "masked:secret" + + +def test_anthropic_request_type_and_resolve_masked_text(): + assert safety._request_type("anthropic.completion") == "completion" + assert safety._request_type("anthropic.chat") == "chat" + assert safety._resolve_masked_text("x", None) == ("x", False) + assert safety._resolve_masked_text( + "x", + SafetyResult(text="x", overall_action=SafetyDecision.MASK.value), + ) == ("x", False) + assert safety._resolve_masked_text( + "x", + SafetyResult(text="y", overall_action=SafetyDecision.ALLOW.value), + ) == ("x", False) + assert safety._resolve_masked_text( + "x", + SafetyResult(text="y", overall_action=SafetyDecision.MASK.value), + ) == ("y", True) + + +def test_anthropic_fail_opens_on_internal_error(monkeypatch): + kwargs = {"messages": [{"role": "user", "content": "secret"}]} + monkeypatch.setattr(safety, "_request_type", lambda *_: (_ for _ in ()).throw(RuntimeError("boom"))) + assert safety._apply_prompt_safety(None, kwargs, "anthropic.chat") is kwargs + + response = SimpleNamespace(completion="secret") + monkeypatch.setattr(safety, "_request_type", lambda *_: (_ for _ in ()).throw(RuntimeError("boom"))) + assert safety._apply_completion_safety(None, response, "anthropic.chat") is None diff --git a/packages/opentelemetry-instrumentation-cohere/opentelemetry/instrumentation/cohere/__init__.py b/packages/opentelemetry-instrumentation-cohere/opentelemetry/instrumentation/cohere/__init__.py index 5430ec9414..9f30804955 100644 --- a/packages/opentelemetry-instrumentation-cohere/opentelemetry/instrumentation/cohere/__init__.py +++ b/packages/opentelemetry-instrumentation-cohere/opentelemetry/instrumentation/cohere/__init__.py @@ -10,6 +10,10 @@ emit_input_event, emit_response_events, ) +from opentelemetry.instrumentation.cohere.safety import ( + _apply_completion_safety, + _apply_prompt_safety, +) from opentelemetry.instrumentation.cohere.span_utils import ( set_input_content_attributes, set_response_content_attributes, @@ -231,6 +235,7 @@ def _wrap( ) with use_span(span, end_on_exit=False): + kwargs = _apply_prompt_safety(span, kwargs, llm_request_type, name) set_span_request_attributes(span, kwargs) _handle_input_content(span, event_logger, llm_request_type, kwargs) @@ -246,6 +251,7 @@ def _wrap( if to_wrap.get("stream_process_func"): return to_wrap.get("stream_process_func")(span, event_logger, llm_request_type, response) + _apply_completion_safety(span, response, llm_request_type, name) set_span_response_attributes(span, response) _handle_response_content(span, event_logger, llm_request_type, response) span.end() @@ -278,6 +284,7 @@ async def _awrap( SpanAttributes.LLM_REQUEST_TYPE: llm_request_type.value, }, ) as span: + kwargs = _apply_prompt_safety(span, kwargs, llm_request_type, name) set_span_request_attributes(span, kwargs) _handle_input_content(span, event_logger, llm_request_type, kwargs) @@ -290,6 +297,7 @@ async def _awrap( span.end() raise + _apply_completion_safety(span, response, llm_request_type, name) set_span_response_attributes(span, response) _handle_response_content(span, event_logger, llm_request_type, response) diff --git a/packages/opentelemetry-instrumentation-cohere/opentelemetry/instrumentation/cohere/safety.py b/packages/opentelemetry-instrumentation-cohere/opentelemetry/instrumentation/cohere/safety.py new file mode 100644 index 0000000000..d543343bb7 --- /dev/null +++ b/packages/opentelemetry-instrumentation-cohere/opentelemetry/instrumentation/cohere/safety.py @@ -0,0 +1,270 @@ +from __future__ import annotations + +from opentelemetry.instrumentation.fortifyroot import ( + SafetyDecision, + SafetyLocation, + clone_value, + get_object_value, + run_completion_safety, + run_prompt_safety, + set_object_value, +) +from opentelemetry.semconv_ai import LLMRequestTypeValues + +PROVIDER = "Cohere" + + +def _apply_prompt_safety(span, kwargs, llm_request_type, span_name): + try: + if llm_request_type not in ( + LLMRequestTypeValues.CHAT, + LLMRequestTypeValues.COMPLETION, + ): + return kwargs + + mutated_kwargs = kwargs + + if llm_request_type == LLMRequestTypeValues.COMPLETION: + prompt = kwargs.get("prompt") + if not isinstance(prompt, str): + return kwargs + updated_prompt, changed = _mask_prompt_text( + span, + prompt, + request_type=llm_request_type.value, + span_name=span_name, + segment_index=0, + segment_role="user", + ) + if not changed: + return kwargs + mutated_kwargs = dict(kwargs) + mutated_kwargs["prompt"] = updated_prompt + return mutated_kwargs + + preamble = kwargs.get("preamble") + if isinstance(preamble, str): + updated_preamble, changed = _mask_prompt_text( + span, + preamble, + request_type=llm_request_type.value, + span_name=span_name, + segment_index=0, + segment_role="system", + ) + if changed: + mutated_kwargs = dict(kwargs) + mutated_kwargs["preamble"] = updated_preamble + + message = kwargs.get("message") + if isinstance(message, str): + updated_message, changed = _mask_prompt_text( + span, + message, + request_type=llm_request_type.value, + span_name=span_name, + segment_index=0, + segment_role="user", + ) + if changed: + if mutated_kwargs is kwargs: + mutated_kwargs = dict(kwargs) + mutated_kwargs["message"] = updated_message + + messages = kwargs.get("messages") + if not isinstance(messages, list): + return mutated_kwargs + + mutated_messages = None + for index, message_obj in enumerate(messages): + role = get_object_value(message_obj, "role") + content = get_object_value(message_obj, "content") + updated_content, changed = _mask_prompt_content( + span, + content, + request_type=llm_request_type.value, + span_name=span_name, + segment_index=index, + segment_role=role, + ) + if not changed: + continue + if mutated_messages is None: + if mutated_kwargs is kwargs: + mutated_kwargs = dict(kwargs) + mutated_messages = clone_value(messages) + mutated_kwargs["messages"] = mutated_messages + set_object_value(mutated_messages[index], "content", updated_content) + + return mutated_kwargs + except Exception: + return kwargs + + +def _mask_prompt_content( + span, + content, + *, + request_type, + span_name, + segment_index, + segment_role, +): + if isinstance(content, str): + return _mask_prompt_text( + span, + content, + request_type=request_type, + span_name=span_name, + segment_index=segment_index, + segment_role=segment_role, + ) + return content, False + + +def _apply_completion_safety(span, response, llm_request_type, span_name): + try: + if llm_request_type not in ( + LLMRequestTypeValues.CHAT, + LLMRequestTypeValues.COMPLETION, + ): + return + + if llm_request_type == LLMRequestTypeValues.CHAT: + text = get_object_value(response, "text") + if isinstance(text, str): + updated_text, changed = _mask_completion_text( + span, + text, + request_type=llm_request_type.value, + span_name=span_name, + segment_index=0, + ) + if changed: + set_object_value(response, "text", updated_text) + + message = get_object_value(response, "message") + if message is not None: + content = get_object_value(message, "content") + updated_content, changed = _mask_completion_content( + span, + content, + request_type=llm_request_type.value, + span_name=span_name, + segment_index=0, + ) + if changed: + set_object_value(message, "content", updated_content) + return + + generations = get_object_value(response, "generations") + if generations is None: + generations = response + + for index, generation in enumerate(generations or []): + text = get_object_value(generation, "text") + if not isinstance(text, str): + continue + updated_text, changed = _mask_completion_text( + span, + text, + request_type=llm_request_type.value, + span_name=span_name, + segment_index=index, + ) + if changed: + set_object_value(generation, "text", updated_text) + except Exception: + return + + +def _mask_completion_content( + span, + content, + *, + request_type, + span_name, + segment_index, +): + if isinstance(content, str): + return _mask_completion_text( + span, + content, + request_type=request_type, + span_name=span_name, + segment_index=segment_index, + ) + + if not isinstance(content, list): + return content, False + + updated_content = content + for index, block in enumerate(content): + text = get_object_value(block, "text") + if not isinstance(text, str): + continue + updated_text, changed = _mask_completion_text( + span, + text, + request_type=request_type, + span_name=span_name, + segment_index=index, + ) + if not changed: + continue + if updated_content is content: + updated_content = clone_value(content) + set_object_value(updated_content[index], "text", updated_text) + + return updated_content, updated_content is not content + + +def _mask_prompt_text( + span, + text, + *, + request_type, + span_name, + segment_index, + segment_role, +): + result = run_prompt_safety( + span=span, + provider=PROVIDER, + span_name=span_name, + text=text, + location=SafetyLocation.PROMPT, + request_type=request_type, + segment_index=segment_index, + segment_role=segment_role, + ) + return _resolve_masked_text(text, result) + + +def _mask_completion_text( + span, + text, + *, + request_type, + span_name, + segment_index, +): + result = run_completion_safety( + span=span, + provider=PROVIDER, + span_name=span_name, + text=text, + location=SafetyLocation.COMPLETION, + request_type=request_type, + segment_index=segment_index, + segment_role="assistant", + ) + return _resolve_masked_text(text, result) + + +def _resolve_masked_text(original_text, result): + if result is None or result.overall_action != SafetyDecision.MASK.value: + return original_text, False + if result.text == original_text: + return original_text, False + return result.text, True diff --git a/packages/opentelemetry-instrumentation-cohere/pyproject.toml b/packages/opentelemetry-instrumentation-cohere/pyproject.toml index 9a56e7c8e5..34661e8085 100644 --- a/packages/opentelemetry-instrumentation-cohere/pyproject.toml +++ b/packages/opentelemetry-instrumentation-cohere/pyproject.toml @@ -12,6 +12,7 @@ readme = "README.md" requires-python = ">=3.10,<4" dependencies = [ "opentelemetry-api>=1.38.0,<2", + "opentelemetry-instrumentation-fortifyroot", "opentelemetry-instrumentation>=0.59b0", "opentelemetry-semantic-conventions-ai>=0.4.13,<0.5.0", "opentelemetry-semantic-conventions>=0.59b0", @@ -72,5 +73,13 @@ exclude = [ [tool.ruff.lint] select = ["E", "F", "W"] +[tool.pytest.ini_options] +markers = [ + "safety: safety-focused tests", +] + [tool.uv] constraint-dependencies = ["urllib3>=2.6.3", "pip>=25.3"] + +[tool.uv.sources] +opentelemetry-instrumentation-fortifyroot = { path = "../opentelemetry-instrumentation-fortifyroot", editable = true } diff --git a/packages/opentelemetry-instrumentation-cohere/tests/test_safety_hooks.py b/packages/opentelemetry-instrumentation-cohere/tests/test_safety_hooks.py new file mode 100644 index 0000000000..1b1bb20e97 --- /dev/null +++ b/packages/opentelemetry-instrumentation-cohere/tests/test_safety_hooks.py @@ -0,0 +1,146 @@ +from types import SimpleNamespace + +import pytest + +from opentelemetry.instrumentation.cohere import ( + _handle_input_content, + _apply_completion_safety, + _apply_prompt_safety, +) +from opentelemetry.instrumentation.fortifyroot import ( + SafetyFinding, + SafetyLocation, + SafetyResult, + clear_safety_handlers, + register_completion_safety_handler, + register_prompt_safety_handler, +) +from opentelemetry.semconv_ai import LLMRequestTypeValues +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import SimpleSpanProcessor +from opentelemetry.sdk.trace.export.in_memory_span_exporter import ( + InMemorySpanExporter, +) + +pytestmark = pytest.mark.safety + + +def setup_function(): + clear_safety_handlers() + + +def teardown_function(): + clear_safety_handlers() + + +def _test_span(): + exporter = InMemorySpanExporter() + provider = TracerProvider() + provider.add_span_processor(SimpleSpanProcessor(exporter)) + tracer = provider.get_tracer(__name__) + return exporter, tracer + + +def test_prompt_safety_masks_chat_preamble(): + _, tracer = _test_span() + register_prompt_safety_handler( + lambda context: SafetyResult( + text="[PII.preamble]", + overall_action="MASK", + findings=[ + SafetyFinding( + category="PII", + severity="LOW", + action="MASK", + rule_name="PII.preamble", + start=0, + end=len(context.text), + ) + ], + ) + if context.location == SafetyLocation.PROMPT and context.text == "secret" + else None + ) + + kwargs = {"preamble": "secret", "message": "hello"} + with tracer.start_as_current_span("cohere.chat") as span: + updated_kwargs = _apply_prompt_safety( + span, + kwargs, + LLMRequestTypeValues.CHAT, + "cohere.chat", + ) + + assert kwargs["preamble"] == "secret" + assert updated_kwargs["preamble"] == "[PII.preamble]" + + +def test_prompt_safety_masks_span_prompt_attributes(): + exporter, tracer = _test_span() + register_prompt_safety_handler( + lambda context: SafetyResult( + text="[PII.preamble]", + overall_action="MASK", + findings=[ + SafetyFinding( + category="PII", + severity="LOW", + action="MASK", + rule_name="PII.preamble", + start=0, + end=len(context.text), + ) + ], + ) + if context.location == SafetyLocation.PROMPT and context.text == "secret" + else None + ) + + kwargs = {"preamble": "secret", "message": "hello"} + with tracer.start_as_current_span("cohere.chat") as span: + updated_kwargs = _apply_prompt_safety( + span, + kwargs, + LLMRequestTypeValues.CHAT, + "cohere.chat", + ) + _handle_input_content(span, None, LLMRequestTypeValues.CHAT, updated_kwargs) + + spans = exporter.get_finished_spans() + assert spans[0].attributes["gen_ai.prompt.0.content"] == "[PII.preamble]" + assert spans[0].attributes["gen_ai.prompt.0.role"] == "system" + + +def test_completion_safety_masks_chat_message_content(): + exporter, tracer = _test_span() + register_completion_safety_handler( + lambda context: SafetyResult( + text="[SECRET.output]", + overall_action="MASK", + findings=[ + SafetyFinding( + category="SECRET", + severity="HIGH", + action="MASK", + rule_name="SECRET.output", + start=0, + end=len(context.text), + ) + ], + ) + if context.location == SafetyLocation.COMPLETION and context.text == "secret" + else None + ) + + response = SimpleNamespace(message=SimpleNamespace(content="secret")) + with tracer.start_as_current_span("cohere.chat") as span: + _apply_completion_safety( + span, + response, + LLMRequestTypeValues.CHAT, + "cohere.chat", + ) + + assert response.message.content == "[SECRET.output]" + spans = exporter.get_finished_spans() + assert len(spans[0].events) == 1 diff --git a/packages/opentelemetry-instrumentation-cohere/tests/test_safety_unit.py b/packages/opentelemetry-instrumentation-cohere/tests/test_safety_unit.py new file mode 100644 index 0000000000..edc7ea79ca --- /dev/null +++ b/packages/opentelemetry-instrumentation-cohere/tests/test_safety_unit.py @@ -0,0 +1,148 @@ +from types import SimpleNamespace + +import pytest + +from opentelemetry.instrumentation.cohere import safety +from opentelemetry.instrumentation.fortifyroot import SafetyDecision, SafetyResult +from opentelemetry.semconv_ai import LLMRequestTypeValues + +pytestmark = pytest.mark.safety + + +def test_apply_prompt_safety_masks_chat_fields(monkeypatch): + monkeypatch.setattr(safety, "run_prompt_safety", lambda **kwargs: SafetyResult(text=f"masked:{kwargs['text']}", overall_action="MASK")) + kwargs = { + "preamble": "pre", + "message": "msg", + "messages": [{"role": "user", "content": "thread"}], + } + + updated = safety._apply_prompt_safety(None, kwargs, LLMRequestTypeValues.CHAT, "cohere.chat") + + assert kwargs["message"] == "msg" + assert updated["preamble"] == "masked:pre" + assert updated["message"] == "masked:msg" + assert updated["messages"][0]["content"] == "masked:thread" + + +def test_apply_prompt_safety_masks_completion_prompt(monkeypatch): + monkeypatch.setattr(safety, "run_prompt_safety", lambda **kwargs: SafetyResult(text=f"masked:{kwargs['text']}", overall_action="MASK")) + updated = safety._apply_prompt_safety(None, {"prompt": "secret"}, LLMRequestTypeValues.COMPLETION, "cohere.completion") + assert updated["prompt"] == "masked:secret" + + +def test_apply_prompt_safety_returns_original_for_unchanged_completion_prompt(monkeypatch): + monkeypatch.setattr(safety, "run_prompt_safety", lambda **kwargs: SafetyResult(text=kwargs["text"], overall_action="MASK")) + kwargs = {"prompt": "keep"} + assert safety._apply_prompt_safety(None, kwargs, LLMRequestTypeValues.COMPLETION, "cohere.completion") is kwargs + + +def test_apply_prompt_safety_returns_original_for_non_string_completion_prompt(): + kwargs = {"prompt": 123} + assert safety._apply_prompt_safety(None, kwargs, LLMRequestTypeValues.COMPLETION, "cohere.completion") is kwargs + + +def test_apply_prompt_safety_ignores_unsupported_type(): + kwargs = {"prompt": "secret"} + assert safety._apply_prompt_safety(None, kwargs, SimpleNamespace(value="embed"), "cohere.embed") is kwargs + + +def test_cohere_prompt_helper_and_noop_message_paths(monkeypatch): + monkeypatch.setattr(safety, "run_prompt_safety", lambda **kwargs: SafetyResult(text=kwargs["text"], overall_action="MASK")) + assert safety._mask_prompt_content(None, {"x": 1}, request_type="chat", span_name="cohere.chat", segment_index=0, segment_role="user") == ({"x": 1}, False) + + kwargs = {"preamble": "keep", "message": "keep", "messages": [{"role": "user", "content": "keep"}]} + assert safety._apply_prompt_safety(None, kwargs, LLMRequestTypeValues.CHAT, "cohere.chat") is kwargs + + +def test_cohere_message_only_and_messages_only_prompt_paths(monkeypatch): + def _prompt(**kwargs): + if kwargs["text"] == "secret": + return SafetyResult(text="masked:secret", overall_action="MASK") + return SafetyResult(text=kwargs["text"], overall_action="MASK") + + monkeypatch.setattr(safety, "run_prompt_safety", _prompt) + + updated = safety._apply_prompt_safety( + None, + {"preamble": "keep", "message": "secret"}, + LLMRequestTypeValues.CHAT, + "cohere.chat", + ) + assert updated["message"] == "masked:secret" + + updated = safety._apply_prompt_safety( + None, + {"messages": [{"role": "user", "content": "secret"}]}, + LLMRequestTypeValues.CHAT, + "cohere.chat", + ) + assert updated["messages"][0]["content"] == "masked:secret" + + +def test_apply_completion_safety_masks_chat_and_completion_shapes(monkeypatch): + monkeypatch.setattr(safety, "run_completion_safety", lambda **kwargs: SafetyResult(text=f"masked:{kwargs['text']}", overall_action="MASK")) + + chat_response = SimpleNamespace( + text="chat-secret", + message=SimpleNamespace(content=[{"text": "content-secret"}]), + ) + safety._apply_completion_safety(None, chat_response, LLMRequestTypeValues.CHAT, "cohere.chat") + assert chat_response.text == "masked:chat-secret" + assert chat_response.message.content[0]["text"] == "masked:content-secret" + + generation = SimpleNamespace(text="gen-secret") + safety._apply_completion_safety(None, [generation], LLMRequestTypeValues.COMPLETION, "cohere.completion") + assert generation.text == "masked:gen-secret" + + +def test_cohere_completion_helper_noop_branches(monkeypatch): + monkeypatch.setattr(safety, "run_completion_safety", lambda **kwargs: SafetyResult(text=kwargs["text"], overall_action="MASK")) + assert safety._apply_completion_safety(None, SimpleNamespace(text="x"), SimpleNamespace(value="embed"), "cohere.embed") is None + + chat_response = SimpleNamespace(text="keep", message=SimpleNamespace(content="keep")) + safety._apply_completion_safety(None, chat_response, LLMRequestTypeValues.CHAT, "cohere.chat") + assert chat_response.text == "keep" + assert chat_response.message.content == "keep" + + assert safety._mask_completion_content(None, {"x": 1}, request_type="chat", span_name="cohere.chat", segment_index=0) == ({"x": 1}, False) + updated, changed = safety._mask_completion_content( + None, + [{"text": "keep"}, {"binary": b"x"}], + request_type="chat", + span_name="cohere.chat", + segment_index=0, + ) + assert changed is False + assert updated[0]["text"] == "keep" + + response = SimpleNamespace(generations=[SimpleNamespace(text=None), SimpleNamespace(text="keep")]) + safety._apply_completion_safety(None, response, LLMRequestTypeValues.COMPLETION, "cohere.completion") + assert response.generations[1].text == "keep" + + monkeypatch.setattr(safety, "get_object_value", lambda *args, **kwargs: (_ for _ in ()).throw(RuntimeError("boom"))) + assert safety._apply_completion_safety(None, SimpleNamespace(text="secret"), LLMRequestTypeValues.COMPLETION, "cohere.completion") is None + + +def test_cohere_mask_completion_content_changes_list(monkeypatch): + monkeypatch.setattr(safety, "run_completion_safety", lambda **kwargs: SafetyResult(text=f"masked:{kwargs['text']}", overall_action="MASK")) + updated, changed = safety._mask_completion_content( + None, + [{"text": "secret"}], + request_type="chat", + span_name="cohere.chat", + segment_index=0, + ) + assert changed is True + assert updated[0]["text"] == "masked:secret" + + +def test_cohere_resolve_masked_text_and_fail_open(monkeypatch): + assert safety._resolve_masked_text("x", None) == ("x", False) + assert safety._resolve_masked_text("x", SafetyResult(text="x", overall_action=SafetyDecision.MASK.value)) == ("x", False) + assert safety._resolve_masked_text("x", SafetyResult(text="y", overall_action=SafetyDecision.ALLOW.value)) == ("x", False) + assert safety._resolve_masked_text("x", SafetyResult(text="y", overall_action=SafetyDecision.MASK.value)) == ("y", True) + + kwargs = {"messages": [{"role": "user", "content": "secret"}]} + monkeypatch.setattr(safety, "get_object_value", lambda *args, **kwargs: (_ for _ in ()).throw(RuntimeError("boom"))) + assert safety._apply_prompt_safety(None, kwargs, LLMRequestTypeValues.CHAT, "cohere.chat") is kwargs diff --git a/packages/opentelemetry-instrumentation-fortifyroot/.python-version b/packages/opentelemetry-instrumentation-fortifyroot/.python-version new file mode 100644 index 0000000000..c8cfe39591 --- /dev/null +++ b/packages/opentelemetry-instrumentation-fortifyroot/.python-version @@ -0,0 +1 @@ +3.10 diff --git a/packages/opentelemetry-instrumentation-fortifyroot/README.md b/packages/opentelemetry-instrumentation-fortifyroot/README.md new file mode 100644 index 0000000000..ae7e13634a --- /dev/null +++ b/packages/opentelemetry-instrumentation-fortifyroot/README.md @@ -0,0 +1,4 @@ +# OpenTelemetry FortifyRoot Safety Helpers + +Shared FortifyRoot safety hook primitives for the OpenLLMetry provider +instrumentations. diff --git a/packages/opentelemetry-instrumentation-fortifyroot/opentelemetry/instrumentation/fortifyroot/__init__.py b/packages/opentelemetry-instrumentation-fortifyroot/opentelemetry/instrumentation/fortifyroot/__init__.py new file mode 100644 index 0000000000..28bc815c5f --- /dev/null +++ b/packages/opentelemetry-instrumentation-fortifyroot/opentelemetry/instrumentation/fortifyroot/__init__.py @@ -0,0 +1,37 @@ +from opentelemetry.instrumentation.fortifyroot.safety import ( + SAFETY_EVENT_NAME, + SafetyContext, + SafetyDecision, + SafetyFinding, + SafetyLocation, + SafetyResult, + clear_safety_handlers, + clone_value, + get_completion_safety_handler, + get_object_value, + get_prompt_safety_handler, + register_completion_safety_handler, + register_prompt_safety_handler, + run_completion_safety, + run_prompt_safety, + set_object_value, +) + +__all__ = [ + "SAFETY_EVENT_NAME", + "SafetyContext", + "SafetyDecision", + "SafetyFinding", + "SafetyLocation", + "SafetyResult", + "clear_safety_handlers", + "clone_value", + "get_completion_safety_handler", + "get_object_value", + "get_prompt_safety_handler", + "register_completion_safety_handler", + "register_prompt_safety_handler", + "run_completion_safety", + "run_prompt_safety", + "set_object_value", +] diff --git a/packages/opentelemetry-instrumentation-fortifyroot/opentelemetry/instrumentation/fortifyroot/safety.py b/packages/opentelemetry-instrumentation-fortifyroot/opentelemetry/instrumentation/fortifyroot/safety.py new file mode 100644 index 0000000000..4a47bc3e91 --- /dev/null +++ b/packages/opentelemetry-instrumentation-fortifyroot/opentelemetry/instrumentation/fortifyroot/safety.py @@ -0,0 +1,255 @@ +from __future__ import annotations + +import copy +import logging +import threading +from collections.abc import Callable, Mapping, MutableMapping, Sequence +from dataclasses import dataclass, field +from enum import Enum +from typing import Any + +from opentelemetry.trace import Span + +SAFETY_EVENT_NAME = "fortifyroot.safety.violation" + +logger = logging.getLogger(__name__) + + +class SafetyDecision(str, Enum): + ALLOW = "ALLOW" + MASK = "MASK" + + +class SafetyLocation(str, Enum): + PROMPT = "PROMPT" + COMPLETION = "COMPLETION" + + +@dataclass(frozen=True, slots=True) +class SafetyFinding: + category: str + severity: str + action: str + rule_name: str + start: int + end: int + + +@dataclass(frozen=True, slots=True) +class SafetyContext: + provider: str + text: str + location: SafetyLocation + span_name: str + request_type: str | None = None + segment_index: int | None = None + segment_role: str | None = None + metadata: Mapping[str, Any] = field(default_factory=dict) + + +@dataclass(frozen=True, slots=True) +class SafetyResult: + text: str + findings: Sequence[SafetyFinding] = () + overall_action: str = SafetyDecision.ALLOW.value + + +PromptSafetyHandler = Callable[[SafetyContext], SafetyResult | None] +CompletionSafetyHandler = Callable[[SafetyContext], SafetyResult | None] + + +_handler_lock = threading.RLock() +_prompt_handler: PromptSafetyHandler | None = None +_completion_handler: CompletionSafetyHandler | None = None + + +def register_prompt_safety_handler(handler: PromptSafetyHandler | None) -> None: + global _prompt_handler + with _handler_lock: + _prompt_handler = handler + + +def register_completion_safety_handler(handler: CompletionSafetyHandler | None) -> None: + global _completion_handler + with _handler_lock: + _completion_handler = handler + + +def clear_safety_handlers() -> None: + register_prompt_safety_handler(None) + register_completion_safety_handler(None) + + +def get_prompt_safety_handler() -> PromptSafetyHandler | None: + with _handler_lock: + return _prompt_handler + + +def get_completion_safety_handler() -> CompletionSafetyHandler | None: + with _handler_lock: + return _completion_handler + + +def run_prompt_safety( + *, + span: Span | None, + provider: str, + span_name: str, + text: str | None, + location: SafetyLocation, + request_type: str | None = None, + segment_index: int | None = None, + segment_role: str | None = None, + metadata: Mapping[str, Any] | None = None, +) -> SafetyResult | None: + return _run_safety( + handler=get_prompt_safety_handler(), + span=span, + provider=provider, + span_name=span_name, + text=text, + location=location, + request_type=request_type, + segment_index=segment_index, + segment_role=segment_role, + metadata=metadata, + ) + + +def run_completion_safety( + *, + span: Span | None, + provider: str, + span_name: str, + text: str | None, + location: SafetyLocation, + request_type: str | None = None, + segment_index: int | None = None, + segment_role: str | None = None, + metadata: Mapping[str, Any] | None = None, +) -> SafetyResult | None: + return _run_safety( + handler=get_completion_safety_handler(), + span=span, + provider=provider, + span_name=span_name, + text=text, + location=location, + request_type=request_type, + segment_index=segment_index, + segment_role=segment_role, + metadata=metadata, + ) + + +def clone_value(value: Any) -> Any: + try: + return copy.deepcopy(value) + except Exception: + return value + + +def get_object_value(obj: Any, key: str, default: Any = None) -> Any: + if isinstance(obj, Mapping): + return obj.get(key, default) + return getattr(obj, key, default) + + +def set_object_value(obj: Any, key: str, value: Any) -> bool: + if isinstance(obj, MutableMapping): + obj[key] = value + return True + try: + setattr(obj, key, value) + return True + except Exception: + return False + + +def _run_safety( + *, + handler: Callable[[SafetyContext], SafetyResult | None] | None, + span: Span | None, + provider: str, + span_name: str, + text: str | None, + location: SafetyLocation, + request_type: str | None, + segment_index: int | None, + segment_role: str | None, + metadata: Mapping[str, Any] | None, +) -> SafetyResult | None: + if handler is None or text is None or text == "": + return None + + context = SafetyContext( + provider=provider, + text=text, + location=location, + span_name=span_name, + request_type=request_type, + segment_index=segment_index, + segment_role=segment_role, + metadata=metadata or {}, + ) + try: + result = handler(context) + except Exception: + logger.warning("Safety handler execution failed", exc_info=True) + return None + if result is None: + return None + + normalized = _normalize_result(text, result) + _emit_findings(span, context, normalized) + return normalized + + +def _normalize_result(original_text: str, result: SafetyResult) -> SafetyResult: + text = result.text if result.text is not None else original_text + findings = tuple(_normalize_finding(finding) for finding in result.findings) + overall_action = _normalize_decision(result.overall_action) + return SafetyResult(text=text, findings=findings, overall_action=overall_action) + + +def _normalize_finding(finding: SafetyFinding) -> SafetyFinding: + return SafetyFinding( + category=str(finding.category).upper(), + severity=str(finding.severity).upper(), + action=_normalize_decision(finding.action), + rule_name=finding.rule_name, + start=int(finding.start), + end=int(finding.end), + ) + + +def _normalize_decision(value: str) -> str: + raw = str(value).strip().upper() + if raw == SafetyDecision.MASK.value: + return SafetyDecision.MASK.value + return SafetyDecision.ALLOW.value + + +def _emit_findings( + span: Span | None, + context: SafetyContext, + result: SafetyResult, +) -> None: + if span is None or not span.is_recording(): + return + + for finding in result.findings: + attributes: dict[str, Any] = { + "fortifyroot.safety.category": finding.category, + "fortifyroot.safety.severity": finding.severity, + "fortifyroot.safety.action": finding.action, + "fortifyroot.safety.location": context.location.value, + "fortifyroot.safety.rule_name": finding.rule_name, + "fortifyroot.safety.start": finding.start, + "fortifyroot.safety.end": finding.end, + } + if context.segment_index is not None: + attributes["fortifyroot.safety.segment_index"] = context.segment_index + if context.segment_role: + attributes["fortifyroot.safety.segment_role"] = context.segment_role + span.add_event(SAFETY_EVENT_NAME, attributes=attributes) diff --git a/packages/opentelemetry-instrumentation-fortifyroot/poetry.toml b/packages/opentelemetry-instrumentation-fortifyroot/poetry.toml new file mode 100644 index 0000000000..ab1033bd37 --- /dev/null +++ b/packages/opentelemetry-instrumentation-fortifyroot/poetry.toml @@ -0,0 +1,2 @@ +[virtualenvs] +in-project = true diff --git a/packages/opentelemetry-instrumentation-fortifyroot/project.json b/packages/opentelemetry-instrumentation-fortifyroot/project.json new file mode 100644 index 0000000000..bab1c5aba5 --- /dev/null +++ b/packages/opentelemetry-instrumentation-fortifyroot/project.json @@ -0,0 +1,77 @@ +{ + "name": "opentelemetry-instrumentation-fortifyroot", + "$schema": "../../node_modules/nx/schemas/project-schema.json", + "projectType": "library", + "sourceRoot": "packages/opentelemetry-instrumentation-fortifyroot/opentelemetry/instrumentation/fortifyroot", + "targets": { + "lock": { + "executor": "nx:run-commands", + "options": { + "command": "uv lock", + "cwd": "packages/opentelemetry-instrumentation-fortifyroot" + } + }, + "add": { + "executor": "@nxlv/python:add", + "options": {} + }, + "update": { + "executor": "@nxlv/python:update", + "options": {} + }, + "remove": { + "executor": "@nxlv/python:remove", + "options": {} + }, + "build": { + "executor": "@nxlv/python:build", + "outputs": [ + "{projectRoot}/dist" + ], + "options": { + "outputPath": "packages/opentelemetry-instrumentation-fortifyroot/dist", + "publish": false, + "lockedVersions": true, + "bundleLocalDependencies": true + } + }, + "install": { + "executor": "nx:run-commands", + "options": { + "command": "uv sync --all-groups", + "cwd": "packages/opentelemetry-instrumentation-fortifyroot" + } + }, + "lint": { + "executor": "nx:run-commands", + "options": { + "command": "uv run ruff check .", + "cwd": "packages/opentelemetry-instrumentation-fortifyroot" + } + }, + "test": { + "executor": "nx:run-commands", + "outputs": [ + "{workspaceRoot}/reports/packages/opentelemetry-instrumentation-fortifyroot/unittests", + "{workspaceRoot}/coverage/packages/opentelemetry-instrumentation-fortifyroot" + ], + "options": { + "command": "uv run pytest tests/", + "cwd": "packages/opentelemetry-instrumentation-fortifyroot" + } + }, + "build-release": { + "executor": "nx:run-commands", + "options": { + "commands": [ + "chmod +x ../../scripts/build-release.sh", + "../../scripts/build-release.sh" + ], + "cwd": "packages/opentelemetry-instrumentation-fortifyroot" + } + } + }, + "tags": [ + "instrumentation" + ] +} diff --git a/packages/opentelemetry-instrumentation-fortifyroot/pyproject.toml b/packages/opentelemetry-instrumentation-fortifyroot/pyproject.toml new file mode 100644 index 0000000000..2b86eff4f5 --- /dev/null +++ b/packages/opentelemetry-instrumentation-fortifyroot/pyproject.toml @@ -0,0 +1,66 @@ +[project] +name = "opentelemetry-instrumentation-fortifyroot" +version = "0.52.6" +description = "Shared FortifyRoot safety hook helpers for OpenLLMetry" +authors = [ + { name = "Gal Kleinman", email = "gal@traceloop.com" }, + { name = "Nir Gazit", email = "nir@traceloop.com" }, + { name = "Tomer Friedman", email = "tomer@traceloop.com" }, +] +license = "Apache-2.0" +readme = "README.md" +requires-python = ">=3.10,<4" +dependencies = [ + "opentelemetry-api>=1.38.0,<2", +] + +[project.urls] +Repository = "https://github.com/traceloop/openllmetry/tree/main/packages/opentelemetry-instrumentation-fortifyroot" + +[dependency-groups] +dev = [ + "autopep8>=2.2.0,<3", + "pytest>=8.2.2,<9", + "ruff>=0.4.0", +] +test = [ + "opentelemetry-sdk>=1.38.0,<2", + "pytest>=8.2.2,<9", +] + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[tool.hatch.build.targets.wheel] +packages = ["opentelemetry"] + +[tool.coverage.run] +branch = true +source = ["opentelemetry/instrumentation/fortifyroot"] + +[tool.coverage.report] +exclude_lines = ["if TYPE_CHECKING:"] +show_missing = true + +[tool.ruff] +line-length = 120 +exclude = [ + ".git", + "__pycache__", + "build", + "dist", + ".venv", + ".pytest_cache", +] + +[tool.ruff.lint] +select = ["E", "F", "W"] + +[tool.pytest.ini_options] +markers = [ + "safety: safety-focused tests", +] + +[tool.uv] +constraint-dependencies = ["urllib3>=2.6.3", "pip>=25.3"] diff --git a/packages/opentelemetry-instrumentation-fortifyroot/tests/test_safety.py b/packages/opentelemetry-instrumentation-fortifyroot/tests/test_safety.py new file mode 100644 index 0000000000..b441fabb0f --- /dev/null +++ b/packages/opentelemetry-instrumentation-fortifyroot/tests/test_safety.py @@ -0,0 +1,131 @@ +import pytest + +from opentelemetry.instrumentation.fortifyroot import ( + SAFETY_EVENT_NAME, + SafetyDecision, + SafetyFinding, + SafetyLocation, + SafetyResult, + clear_safety_handlers, + clone_value, + register_prompt_safety_handler, + run_prompt_safety, +) +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import SimpleSpanProcessor +from opentelemetry.sdk.trace.export.in_memory_span_exporter import ( + InMemorySpanExporter, +) + +pytestmark = pytest.mark.safety + + +def test_run_prompt_safety_emits_one_event_per_finding(): + exporter = InMemorySpanExporter() + provider = TracerProvider() + provider.add_span_processor(SimpleSpanProcessor(exporter)) + tracer = provider.get_tracer(__name__) + + register_prompt_safety_handler( + lambda _: SafetyResult( + text="masked value", + overall_action="mask", + findings=[ + SafetyFinding( + category="pii", + severity="high", + action="mask", + rule_name="PII.email", + start=2, + end=7, + ), + SafetyFinding( + category="secret", + severity="medium", + action="allow", + rule_name="SECRET.api_key", + start=9, + end=13, + ), + ], + ) + ) + + with tracer.start_as_current_span("test-span") as span: + result = run_prompt_safety( + span=span, + provider="OpenAI", + span_name="openai.chat", + text="raw value", + location=SafetyLocation.PROMPT, + request_type="chat", + segment_index=1, + segment_role="user", + ) + + spans = exporter.get_finished_spans() + assert result is not None + assert result.text == "masked value" + assert result.overall_action == SafetyDecision.MASK.value + assert len(spans) == 1 + assert len(spans[0].events) == 2 + assert spans[0].events[0].name == SAFETY_EVENT_NAME + assert spans[0].events[0].attributes["fortifyroot.safety.category"] == "PII" + assert spans[0].events[0].attributes["fortifyroot.safety.segment_index"] == 1 + assert spans[0].events[0].attributes["fortifyroot.safety.segment_role"] == "user" + assert "fortifyroot.safety.request_type" not in spans[0].events[0].attributes + assert "fortifyroot.safety.provider" not in spans[0].events[0].attributes + assert spans[0].events[1].attributes["fortifyroot.safety.action"] == SafetyDecision.ALLOW.value + + clear_safety_handlers() + + +def test_run_prompt_safety_returns_none_without_handler(): + exporter = InMemorySpanExporter() + provider = TracerProvider() + provider.add_span_processor(SimpleSpanProcessor(exporter)) + tracer = provider.get_tracer(__name__) + clear_safety_handlers() + + with tracer.start_as_current_span("test-span") as span: + result = run_prompt_safety( + span=span, + provider="OpenAI", + span_name="openai.chat", + text="raw value", + location=SafetyLocation.PROMPT, + ) + + assert result is None + spans = exporter.get_finished_spans() + assert len(spans) == 1 + assert spans[0].events == () + + +def test_clone_value_falls_back_cleanly(): + class NotCloneable: + def __deepcopy__(self, memo): + raise RuntimeError("no clone") + + value = NotCloneable() + assert clone_value(value) is value + + +def test_run_prompt_safety_fail_opens_when_handler_raises(): + exporter = InMemorySpanExporter() + provider = TracerProvider() + provider.add_span_processor(SimpleSpanProcessor(exporter)) + tracer = provider.get_tracer(__name__) + + register_prompt_safety_handler(lambda _: (_ for _ in ()).throw(RuntimeError("boom"))) + + with tracer.start_as_current_span("test-span") as span: + result = run_prompt_safety( + span=span, + provider="OpenAI", + span_name="openai.chat", + text="raw value", + location=SafetyLocation.PROMPT, + ) + + assert result is None diff --git a/packages/opentelemetry-instrumentation-fortifyroot/tests/test_safety_helpers.py b/packages/opentelemetry-instrumentation-fortifyroot/tests/test_safety_helpers.py new file mode 100644 index 0000000000..44e0b99487 --- /dev/null +++ b/packages/opentelemetry-instrumentation-fortifyroot/tests/test_safety_helpers.py @@ -0,0 +1,207 @@ +from types import SimpleNamespace + +import pytest + +from opentelemetry.instrumentation.fortifyroot import safety +from opentelemetry.instrumentation.fortifyroot import ( + SAFETY_EVENT_NAME, + SafetyDecision, + SafetyFinding, + SafetyLocation, + SafetyResult, + clear_safety_handlers, + get_completion_safety_handler, + get_object_value, + get_prompt_safety_handler, + register_completion_safety_handler, + run_completion_safety, + set_object_value, +) +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import SimpleSpanProcessor +from opentelemetry.sdk.trace.export.in_memory_span_exporter import ( + InMemorySpanExporter, +) + +pytestmark = pytest.mark.safety + + +def teardown_function(): + clear_safety_handlers() + + +def _span(): + exporter = InMemorySpanExporter() + provider = TracerProvider() + provider.add_span_processor(SimpleSpanProcessor(exporter)) + tracer = provider.get_tracer(__name__) + return exporter, tracer + + +def test_get_and_set_object_value_support_mapping_and_object(): + mapping = {"value": 1} + obj = SimpleNamespace(value=2) + + assert get_object_value(mapping, "value") == 1 + assert get_object_value(obj, "value") == 2 + assert get_object_value(obj, "missing", "fallback") == "fallback" + + assert set_object_value(mapping, "value", 3) is True + assert set_object_value(obj, "value", 4) is True + assert mapping["value"] == 3 + assert obj.value == 4 + + +def test_set_object_value_returns_false_when_attribute_is_not_writable(): + class ReadOnly: + @property + def value(self): + return "x" + + assert set_object_value(ReadOnly(), "value", "y") is False + + +def test_run_completion_safety_normalizes_and_emits_event_without_overall_action(): + exporter, tracer = _span() + register_completion_safety_handler( + lambda _: SafetyResult( + text=None, + overall_action="mask", + findings=[ + SafetyFinding( + category="pii", + severity="low", + action="mask", + rule_name="PII.email", + start="1", + end="4", + ) + ], + ) + ) + + with tracer.start_as_current_span("test-span") as span: + result = run_completion_safety( + span=span, + provider="OpenAI", + span_name="openai.chat", + text="raw", + location=SafetyLocation.COMPLETION, + request_type="chat", + segment_index=2, + segment_role="assistant", + metadata={"ignored": True}, + ) + + assert result is not None + assert result.text == "raw" + assert result.overall_action == SafetyDecision.MASK.value + assert result.findings[0].category == "PII" + assert result.findings[0].severity == "LOW" + assert result.findings[0].start == 1 + assert result.findings[0].end == 4 + + span = exporter.get_finished_spans()[0] + assert span.events[0].name == SAFETY_EVENT_NAME + assert "fortifyroot.safety.overall_action" not in span.events[0].attributes + assert "fortifyroot.safety.provider" not in span.events[0].attributes + assert "fortifyroot.safety.request_type" not in span.events[0].attributes + + +def test_run_completion_safety_returns_none_for_empty_text_or_missing_handler(): + assert get_completion_safety_handler() is None + assert run_completion_safety( + span=None, + provider="OpenAI", + span_name="openai.chat", + text="", + location=SafetyLocation.COMPLETION, + ) is None + + +def test_run_completion_safety_returns_none_when_handler_returns_none(): + register_completion_safety_handler(lambda _: None) + assert run_completion_safety( + span=None, + provider="OpenAI", + span_name="openai.chat", + text="hello", + location=SafetyLocation.COMPLETION, + ) is None + + +def test_handler_getters_follow_registration(): + register_completion_safety_handler(lambda _: None) + assert get_completion_safety_handler() is not None + assert get_prompt_safety_handler() is None + clear_safety_handlers() + assert get_completion_safety_handler() is None + + +def test_emit_findings_noops_for_non_recording_span_and_none_span(): + context = safety.SafetyContext( + provider="OpenAI", + text="raw", + location=SafetyLocation.PROMPT, + span_name="openai.chat", + ) + result = SafetyResult( + text="raw", + findings=[ + SafetyFinding( + category="PII", + severity="HIGH", + action="MASK", + rule_name="PII.email", + start=0, + end=3, + ) + ], + overall_action="MASK", + ) + + safety._emit_findings(None, context, result) + + class NotRecordingSpan: + def is_recording(self): + return False + + safety._emit_findings(NotRecordingSpan(), context, result) + + +def test_emit_findings_omits_optional_attributes_when_not_present(): + exporter, tracer = _span() + context = safety.SafetyContext( + provider="OpenAI", + text="raw", + location=SafetyLocation.PROMPT, + span_name="openai.chat", + ) + result = SafetyResult( + text="raw", + findings=[ + SafetyFinding( + category="PII", + severity="HIGH", + action="ALLOW", + rule_name="PII.email", + start=0, + end=3, + ) + ], + overall_action="ALLOW", + ) + + with tracer.start_as_current_span("test-span") as span: + safety._emit_findings(span, context, result) + + attributes = exporter.get_finished_spans()[0].events[0].attributes + assert "fortifyroot.safety.request_type" not in attributes + assert "fortifyroot.safety.segment_index" not in attributes + assert "fortifyroot.safety.segment_role" not in attributes + assert "fortifyroot.safety.provider" not in attributes + + +def test_normalize_decision_defaults_unknown_to_allow(): + assert safety._normalize_decision("mask") == SafetyDecision.MASK.value + assert safety._normalize_decision("something-else") == SafetyDecision.ALLOW.value diff --git a/packages/opentelemetry-instrumentation-fortifyroot/uv.lock b/packages/opentelemetry-instrumentation-fortifyroot/uv.lock new file mode 100644 index 0000000000..25433255d9 --- /dev/null +++ b/packages/opentelemetry-instrumentation-fortifyroot/uv.lock @@ -0,0 +1,288 @@ +version = 1 +revision = 3 +requires-python = ">=3.10, <4" + +[manifest] +constraints = [ + { name = "pip", specifier = ">=25.3" }, + { name = "urllib3", specifier = ">=2.6.3" }, +] + +[[package]] +name = "autopep8" +version = "2.3.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pycodestyle" }, + { name = "tomli", marker = "python_full_version < '3.11'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/50/d8/30873d2b7b57dee9263e53d142da044c4600a46f2d28374b3e38b023df16/autopep8-2.3.2.tar.gz", hash = "sha256:89440a4f969197b69a995e4ce0661b031f455a9f776d2c5ba3dbd83466931758", size = 92210, upload-time = "2025-01-14T14:46:18.454Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9e/43/53afb8ba17218f19b77c7834128566c5bbb100a0ad9ba2e8e89d089d7079/autopep8-2.3.2-py2.py3-none-any.whl", hash = "sha256:ce8ad498672c845a0c3de2629c15b635ec2b05ef8177a6e7c91c74f3e9b51128", size = 45807, upload-time = "2025-01-14T14:46:15.466Z" }, +] + +[[package]] +name = "colorama" +version = "0.4.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d8/53/6f443c9a4a8358a93a6792e2acffb9d9d5cb0a5cfd8802644b7b1c9a02e4/colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44", size = 27697, upload-time = "2022-10-25T02:36:22.414Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" }, +] + +[[package]] +name = "exceptiongroup" +version = "1.3.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions", marker = "python_full_version < '3.13'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/50/79/66800aadf48771f6b62f7eb014e352e5d06856655206165d775e675a02c9/exceptiongroup-1.3.1.tar.gz", hash = "sha256:8b412432c6055b0b7d14c310000ae93352ed6754f70fa8f7c34141f91c4e3219", size = 30371, upload-time = "2025-11-21T23:01:54.787Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8a/0e/97c33bf5009bdbac74fd2beace167cab3f978feb69cc36f1ef79360d6c4e/exceptiongroup-1.3.1-py3-none-any.whl", hash = "sha256:a7a39a3bd276781e98394987d3a5701d0c4edffb633bb7a5144577f82c773598", size = 16740, upload-time = "2025-11-21T23:01:53.443Z" }, +] + +[[package]] +name = "importlib-metadata" +version = "8.7.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "zipp" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/f3/49/3b30cad09e7771a4982d9975a8cbf64f00d4a1ececb53297f1d9a7be1b10/importlib_metadata-8.7.1.tar.gz", hash = "sha256:49fef1ae6440c182052f407c8d34a68f72efc36db9ca90dc0113398f2fdde8bb", size = 57107, upload-time = "2025-12-21T10:00:19.278Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fa/5e/f8e9a1d23b9c20a551a8a02ea3637b4642e22c2626e3a13a9a29cdea99eb/importlib_metadata-8.7.1-py3-none-any.whl", hash = "sha256:5a1f80bf1daa489495071efbb095d75a634cf28a8bc299581244063b53176151", size = 27865, upload-time = "2025-12-21T10:00:18.329Z" }, +] + +[[package]] +name = "iniconfig" +version = "2.3.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/72/34/14ca021ce8e5dfedc35312d08ba8bf51fdd999c576889fc2c24cb97f4f10/iniconfig-2.3.0.tar.gz", hash = "sha256:c76315c77db068650d49c5b56314774a7804df16fee4402c1f19d6d15d8c4730", size = 20503, upload-time = "2025-10-18T21:55:43.219Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12", size = 7484, upload-time = "2025-10-18T21:55:41.639Z" }, +] + +[[package]] +name = "opentelemetry-api" +version = "1.40.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "importlib-metadata" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/2c/1d/4049a9e8698361cc1a1aa03a6c59e4fa4c71e0c0f94a30f988a6876a2ae6/opentelemetry_api-1.40.0.tar.gz", hash = "sha256:159be641c0b04d11e9ecd576906462773eb97ae1b657730f0ecf64d32071569f", size = 70851, upload-time = "2026-03-04T14:17:21.555Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5f/bf/93795954016c522008da367da292adceed71cca6ee1717e1d64c83089099/opentelemetry_api-1.40.0-py3-none-any.whl", hash = "sha256:82dd69331ae74b06f6a874704be0cfaa49a1650e1537d4a813b86ecef7d0ecf9", size = 68676, upload-time = "2026-03-04T14:17:01.24Z" }, +] + +[[package]] +name = "opentelemetry-instrumentation-fortifyroot" +version = "0.52.6" +source = { editable = "." } +dependencies = [ + { name = "opentelemetry-api" }, +] + +[package.dev-dependencies] +dev = [ + { name = "autopep8" }, + { name = "pytest" }, + { name = "ruff" }, +] +test = [ + { name = "opentelemetry-sdk" }, + { name = "pytest" }, +] + +[package.metadata] +requires-dist = [{ name = "opentelemetry-api", specifier = ">=1.38.0,<2" }] + +[package.metadata.requires-dev] +dev = [ + { name = "autopep8", specifier = ">=2.2.0,<3" }, + { name = "pytest", specifier = ">=8.2.2,<9" }, + { name = "ruff", specifier = ">=0.4.0" }, +] +test = [ + { name = "opentelemetry-sdk", specifier = ">=1.38.0,<2" }, + { name = "pytest", specifier = ">=8.2.2,<9" }, +] + +[[package]] +name = "opentelemetry-sdk" +version = "1.40.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "opentelemetry-api" }, + { name = "opentelemetry-semantic-conventions" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/58/fd/3c3125b20ba18ce2155ba9ea74acb0ae5d25f8cd39cfd37455601b7955cc/opentelemetry_sdk-1.40.0.tar.gz", hash = "sha256:18e9f5ec20d859d268c7cb3c5198c8d105d073714db3de50b593b8c1345a48f2", size = 184252, upload-time = "2026-03-04T14:17:31.87Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2c/c5/6a852903d8bfac758c6dc6e9a68b015d3c33f2f1be5e9591e0f4b69c7e0a/opentelemetry_sdk-1.40.0-py3-none-any.whl", hash = "sha256:787d2154a71f4b3d81f20524a8ce061b7db667d24e46753f32a7bc48f1c1f3f1", size = 141951, upload-time = "2026-03-04T14:17:17.961Z" }, +] + +[[package]] +name = "opentelemetry-semantic-conventions" +version = "0.61b0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "opentelemetry-api" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/6d/c0/4ae7973f3c2cfd2b6e321f1675626f0dab0a97027cc7a297474c9c8f3d04/opentelemetry_semantic_conventions-0.61b0.tar.gz", hash = "sha256:072f65473c5d7c6dc0355b27d6c9d1a679d63b6d4b4b16a9773062cb7e31192a", size = 145755, upload-time = "2026-03-04T14:17:32.664Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b2/37/cc6a55e448deaa9b27377d087da8615a3416d8ad523d5960b78dbeadd02a/opentelemetry_semantic_conventions-0.61b0-py3-none-any.whl", hash = "sha256:fa530a96be229795f8cef353739b618148b0fe2b4b3f005e60e262926c4d38e2", size = 231621, upload-time = "2026-03-04T14:17:19.33Z" }, +] + +[[package]] +name = "packaging" +version = "26.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/65/ee/299d360cdc32edc7d2cf530f3accf79c4fca01e96ffc950d8a52213bd8e4/packaging-26.0.tar.gz", hash = "sha256:00243ae351a257117b6a241061796684b084ed1c516a08c48a3f7e147a9d80b4", size = 143416, upload-time = "2026-01-21T20:50:39.064Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b7/b9/c538f279a4e237a006a2c98387d081e9eb060d203d8ed34467cc0f0b9b53/packaging-26.0-py3-none-any.whl", hash = "sha256:b36f1fef9334a5588b4166f8bcd26a14e521f2b55e6b9de3aaa80d3ff7a37529", size = 74366, upload-time = "2026-01-21T20:50:37.788Z" }, +] + +[[package]] +name = "pluggy" +version = "1.6.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f9/e2/3e91f31a7d2b083fe6ef3fa267035b518369d9511ffab804f839851d2779/pluggy-1.6.0.tar.gz", hash = "sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3", size = 69412, upload-time = "2025-05-15T12:30:07.975Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" }, +] + +[[package]] +name = "pycodestyle" +version = "2.14.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/11/e0/abfd2a0d2efe47670df87f3e3a0e2edda42f055053c85361f19c0e2c1ca8/pycodestyle-2.14.0.tar.gz", hash = "sha256:c4b5b517d278089ff9d0abdec919cd97262a3367449ea1c8b49b91529167b783", size = 39472, upload-time = "2025-06-20T18:49:48.75Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d7/27/a58ddaf8c588a3ef080db9d0b7e0b97215cee3a45df74f3a94dbbf5c893a/pycodestyle-2.14.0-py2.py3-none-any.whl", hash = "sha256:dd6bf7cb4ee77f8e016f9c8e74a35ddd9f67e1d5fd4184d86c3b98e07099f42d", size = 31594, upload-time = "2025-06-20T18:49:47.491Z" }, +] + +[[package]] +name = "pygments" +version = "2.19.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/b0/77/a5b8c569bf593b0140bde72ea885a803b82086995367bf2037de0159d924/pygments-2.19.2.tar.gz", hash = "sha256:636cb2477cec7f8952536970bc533bc43743542f70392ae026374600add5b887", size = 4968631, upload-time = "2025-06-21T13:39:12.283Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c7/21/705964c7812476f378728bdf590ca4b771ec72385c533964653c68e86bdc/pygments-2.19.2-py3-none-any.whl", hash = "sha256:86540386c03d588bb81d44bc3928634ff26449851e99741617ecb9037ee5ec0b", size = 1225217, upload-time = "2025-06-21T13:39:07.939Z" }, +] + +[[package]] +name = "pytest" +version = "8.4.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "exceptiongroup", marker = "python_full_version < '3.11'" }, + { name = "iniconfig" }, + { name = "packaging" }, + { name = "pluggy" }, + { name = "pygments" }, + { name = "tomli", marker = "python_full_version < '3.11'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a3/5c/00a0e072241553e1a7496d638deababa67c5058571567b92a7eaa258397c/pytest-8.4.2.tar.gz", hash = "sha256:86c0d0b93306b961d58d62a4db4879f27fe25513d4b969df351abdddb3c30e01", size = 1519618, upload-time = "2025-09-04T14:34:22.711Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a8/a4/20da314d277121d6534b3a980b29035dcd51e6744bd79075a6ce8fa4eb8d/pytest-8.4.2-py3-none-any.whl", hash = "sha256:872f880de3fc3a5bdc88a11b39c9710c3497a547cfa9320bc3c5e62fbf272e79", size = 365750, upload-time = "2025-09-04T14:34:20.226Z" }, +] + +[[package]] +name = "ruff" +version = "0.15.5" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/77/9b/840e0039e65fcf12758adf684d2289024d6140cde9268cc59887dc55189c/ruff-0.15.5.tar.gz", hash = "sha256:7c3601d3b6d76dce18c5c824fc8d06f4eef33d6df0c21ec7799510cde0f159a2", size = 4574214, upload-time = "2026-03-05T20:06:34.946Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/47/20/5369c3ce21588c708bcbe517a8fbe1a8dfdb5dfd5137e14790b1da71612c/ruff-0.15.5-py3-none-linux_armv6l.whl", hash = "sha256:4ae44c42281f42e3b06b988e442d344a5b9b72450ff3c892e30d11b29a96a57c", size = 10478185, upload-time = "2026-03-05T20:06:29.093Z" }, + { url = "https://files.pythonhosted.org/packages/44/ed/e81dd668547da281e5dce710cf0bc60193f8d3d43833e8241d006720e42b/ruff-0.15.5-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:6edd3792d408ebcf61adabc01822da687579a1a023f297618ac27a5b51ef0080", size = 10859201, upload-time = "2026-03-05T20:06:32.632Z" }, + { url = "https://files.pythonhosted.org/packages/c4/8f/533075f00aaf19b07c5cd6aa6e5d89424b06b3b3f4583bfa9c640a079059/ruff-0.15.5-py3-none-macosx_11_0_arm64.whl", hash = "sha256:89f463f7c8205a9f8dea9d658d59eff49db05f88f89cc3047fb1a02d9f344010", size = 10184752, upload-time = "2026-03-05T20:06:40.312Z" }, + { url = "https://files.pythonhosted.org/packages/66/0e/ba49e2c3fa0395b3152bad634c7432f7edfc509c133b8f4529053ff024fb/ruff-0.15.5-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ba786a8295c6574c1116704cf0b9e6563de3432ac888d8f83685654fe528fd65", size = 10534857, upload-time = "2026-03-05T20:06:19.581Z" }, + { url = "https://files.pythonhosted.org/packages/59/71/39234440f27a226475a0659561adb0d784b4d247dfe7f43ffc12dd02e288/ruff-0.15.5-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:fd4b801e57955fe9f02b31d20375ab3a5c4415f2e5105b79fb94cf2642c91440", size = 10309120, upload-time = "2026-03-05T20:06:00.435Z" }, + { url = "https://files.pythonhosted.org/packages/f5/87/4140aa86a93df032156982b726f4952aaec4a883bb98cb6ef73c347da253/ruff-0.15.5-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:391f7c73388f3d8c11b794dbbc2959a5b5afe66642c142a6effa90b45f6f5204", size = 11047428, upload-time = "2026-03-05T20:05:51.867Z" }, + { url = "https://files.pythonhosted.org/packages/5a/f7/4953e7e3287676f78fbe85e3a0ca414c5ca81237b7575bdadc00229ac240/ruff-0.15.5-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8dc18f30302e379fe1e998548b0f5e9f4dff907f52f73ad6da419ea9c19d66c8", size = 11914251, upload-time = "2026-03-05T20:06:22.887Z" }, + { url = "https://files.pythonhosted.org/packages/77/46/0f7c865c10cf896ccf5a939c3e84e1cfaeed608ff5249584799a74d33835/ruff-0.15.5-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1cc6e7f90087e2d27f98dc34ed1b3ab7c8f0d273cc5431415454e22c0bd2a681", size = 11333801, upload-time = "2026-03-05T20:05:57.168Z" }, + { url = "https://files.pythonhosted.org/packages/d3/01/a10fe54b653061585e655f5286c2662ebddb68831ed3eaebfb0eb08c0a16/ruff-0.15.5-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c1cb7169f53c1ddb06e71a9aebd7e98fc0fea936b39afb36d8e86d36ecc2636a", size = 11206821, upload-time = "2026-03-05T20:06:03.441Z" }, + { url = "https://files.pythonhosted.org/packages/7a/0d/2132ceaf20c5e8699aa83da2706ecb5c5dcdf78b453f77edca7fb70f8a93/ruff-0.15.5-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:9b037924500a31ee17389b5c8c4d88874cc6ea8e42f12e9c61a3d754ff72f1ca", size = 11133326, upload-time = "2026-03-05T20:06:25.655Z" }, + { url = "https://files.pythonhosted.org/packages/72/cb/2e5259a7eb2a0f87c08c0fe5bf5825a1e4b90883a52685524596bfc93072/ruff-0.15.5-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:65bb414e5b4eadd95a8c1e4804f6772bbe8995889f203a01f77ddf2d790929dd", size = 10510820, upload-time = "2026-03-05T20:06:37.79Z" }, + { url = "https://files.pythonhosted.org/packages/ff/20/b67ce78f9e6c59ffbdb5b4503d0090e749b5f2d31b599b554698a80d861c/ruff-0.15.5-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:d20aa469ae3b57033519c559e9bc9cd9e782842e39be05b50e852c7c981fa01d", size = 10302395, upload-time = "2026-03-05T20:05:54.504Z" }, + { url = "https://files.pythonhosted.org/packages/5f/e5/719f1acccd31b720d477751558ed74e9c88134adcc377e5e886af89d3072/ruff-0.15.5-py3-none-musllinux_1_2_i686.whl", hash = "sha256:15388dd28c9161cdb8eda68993533acc870aa4e646a0a277aa166de9ad5a8752", size = 10754069, upload-time = "2026-03-05T20:06:06.422Z" }, + { url = "https://files.pythonhosted.org/packages/c3/9c/d1db14469e32d98f3ca27079dbd30b7b44dbb5317d06ab36718dee3baf03/ruff-0.15.5-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:b30da330cbd03bed0c21420b6b953158f60c74c54c5f4c1dabbdf3a57bf355d2", size = 11304315, upload-time = "2026-03-05T20:06:10.867Z" }, + { url = "https://files.pythonhosted.org/packages/28/3a/950367aee7c69027f4f422059227b290ed780366b6aecee5de5039d50fa8/ruff-0.15.5-py3-none-win32.whl", hash = "sha256:732e5ee1f98ba5b3679029989a06ca39a950cced52143a0ea82a2102cb592b74", size = 10551676, upload-time = "2026-03-05T20:06:13.705Z" }, + { url = "https://files.pythonhosted.org/packages/b8/00/bf077a505b4e649bdd3c47ff8ec967735ce2544c8e4a43aba42ee9bf935d/ruff-0.15.5-py3-none-win_amd64.whl", hash = "sha256:821d41c5fa9e19117616c35eaa3f4b75046ec76c65e7ae20a333e9a8696bc7fe", size = 11678972, upload-time = "2026-03-05T20:06:45.379Z" }, + { url = "https://files.pythonhosted.org/packages/fe/4e/cd76eca6db6115604b7626668e891c9dd03330384082e33662fb0f113614/ruff-0.15.5-py3-none-win_arm64.whl", hash = "sha256:b498d1c60d2fe5c10c45ec3f698901065772730b411f164ae270bb6bfcc4740b", size = 10965572, upload-time = "2026-03-05T20:06:16.984Z" }, +] + +[[package]] +name = "tomli" +version = "2.4.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/82/30/31573e9457673ab10aa432461bee537ce6cef177667deca369efb79df071/tomli-2.4.0.tar.gz", hash = "sha256:aa89c3f6c277dd275d8e243ad24f3b5e701491a860d5121f2cdd399fbb31fc9c", size = 17477, upload-time = "2026-01-11T11:22:38.165Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3c/d9/3dc2289e1f3b32eb19b9785b6a006b28ee99acb37d1d47f78d4c10e28bf8/tomli-2.4.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:b5ef256a3fd497d4973c11bf142e9ed78b150d36f5773f1ca6088c230ffc5867", size = 153663, upload-time = "2026-01-11T11:21:45.27Z" }, + { url = "https://files.pythonhosted.org/packages/51/32/ef9f6845e6b9ca392cd3f64f9ec185cc6f09f0a2df3db08cbe8809d1d435/tomli-2.4.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:5572e41282d5268eb09a697c89a7bee84fae66511f87533a6f88bd2f7b652da9", size = 148469, upload-time = "2026-01-11T11:21:46.873Z" }, + { url = "https://files.pythonhosted.org/packages/d6/c2/506e44cce89a8b1b1e047d64bd495c22c9f71f21e05f380f1a950dd9c217/tomli-2.4.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:551e321c6ba03b55676970b47cb1b73f14a0a4dce6a3e1a9458fd6d921d72e95", size = 236039, upload-time = "2026-01-11T11:21:48.503Z" }, + { url = "https://files.pythonhosted.org/packages/b3/40/e1b65986dbc861b7e986e8ec394598187fa8aee85b1650b01dd925ca0be8/tomli-2.4.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5e3f639a7a8f10069d0e15408c0b96a2a828cfdec6fca05296ebcdcc28ca7c76", size = 243007, upload-time = "2026-01-11T11:21:49.456Z" }, + { url = "https://files.pythonhosted.org/packages/9c/6f/6e39ce66b58a5b7ae572a0f4352ff40c71e8573633deda43f6a379d56b3e/tomli-2.4.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1b168f2731796b045128c45982d3a4874057626da0e2ef1fdd722848b741361d", size = 240875, upload-time = "2026-01-11T11:21:50.755Z" }, + { url = "https://files.pythonhosted.org/packages/aa/ad/cb089cb190487caa80204d503c7fd0f4d443f90b95cf4ef5cf5aa0f439b0/tomli-2.4.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:133e93646ec4300d651839d382d63edff11d8978be23da4cc106f5a18b7d0576", size = 246271, upload-time = "2026-01-11T11:21:51.81Z" }, + { url = "https://files.pythonhosted.org/packages/0b/63/69125220e47fd7a3a27fd0de0c6398c89432fec41bc739823bcc66506af6/tomli-2.4.0-cp311-cp311-win32.whl", hash = "sha256:b6c78bdf37764092d369722d9946cb65b8767bfa4110f902a1b2542d8d173c8a", size = 96770, upload-time = "2026-01-11T11:21:52.647Z" }, + { url = "https://files.pythonhosted.org/packages/1e/0d/a22bb6c83f83386b0008425a6cd1fa1c14b5f3dd4bad05e98cf3dbbf4a64/tomli-2.4.0-cp311-cp311-win_amd64.whl", hash = "sha256:d3d1654e11d724760cdb37a3d7691f0be9db5fbdaef59c9f532aabf87006dbaa", size = 107626, upload-time = "2026-01-11T11:21:53.459Z" }, + { url = "https://files.pythonhosted.org/packages/2f/6d/77be674a3485e75cacbf2ddba2b146911477bd887dda9d8c9dfb2f15e871/tomli-2.4.0-cp311-cp311-win_arm64.whl", hash = "sha256:cae9c19ed12d4e8f3ebf46d1a75090e4c0dc16271c5bce1c833ac168f08fb614", size = 94842, upload-time = "2026-01-11T11:21:54.831Z" }, + { url = "https://files.pythonhosted.org/packages/3c/43/7389a1869f2f26dba52404e1ef13b4784b6b37dac93bac53457e3ff24ca3/tomli-2.4.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:920b1de295e72887bafa3ad9f7a792f811847d57ea6b1215154030cf131f16b1", size = 154894, upload-time = "2026-01-11T11:21:56.07Z" }, + { url = "https://files.pythonhosted.org/packages/e9/05/2f9bf110b5294132b2edf13fe6ca6ae456204f3d749f623307cbb7a946f2/tomli-2.4.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:7d6d9a4aee98fac3eab4952ad1d73aee87359452d1c086b5ceb43ed02ddb16b8", size = 149053, upload-time = "2026-01-11T11:21:57.467Z" }, + { url = "https://files.pythonhosted.org/packages/e8/41/1eda3ca1abc6f6154a8db4d714a4d35c4ad90adc0bcf700657291593fbf3/tomli-2.4.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:36b9d05b51e65b254ea6c2585b59d2c4cb91c8a3d91d0ed0f17591a29aaea54a", size = 243481, upload-time = "2026-01-11T11:21:58.661Z" }, + { url = "https://files.pythonhosted.org/packages/d2/6d/02ff5ab6c8868b41e7d4b987ce2b5f6a51d3335a70aa144edd999e055a01/tomli-2.4.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1c8a885b370751837c029ef9bc014f27d80840e48bac415f3412e6593bbc18c1", size = 251720, upload-time = "2026-01-11T11:22:00.178Z" }, + { url = "https://files.pythonhosted.org/packages/7b/57/0405c59a909c45d5b6f146107c6d997825aa87568b042042f7a9c0afed34/tomli-2.4.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:8768715ffc41f0008abe25d808c20c3d990f42b6e2e58305d5da280ae7d1fa3b", size = 247014, upload-time = "2026-01-11T11:22:01.238Z" }, + { url = "https://files.pythonhosted.org/packages/2c/0e/2e37568edd944b4165735687cbaf2fe3648129e440c26d02223672ee0630/tomli-2.4.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:7b438885858efd5be02a9a133caf5812b8776ee0c969fea02c45e8e3f296ba51", size = 251820, upload-time = "2026-01-11T11:22:02.727Z" }, + { url = "https://files.pythonhosted.org/packages/5a/1c/ee3b707fdac82aeeb92d1a113f803cf6d0f37bdca0849cb489553e1f417a/tomli-2.4.0-cp312-cp312-win32.whl", hash = "sha256:0408e3de5ec77cc7f81960c362543cbbd91ef883e3138e81b729fc3eea5b9729", size = 97712, upload-time = "2026-01-11T11:22:03.777Z" }, + { url = "https://files.pythonhosted.org/packages/69/13/c07a9177d0b3bab7913299b9278845fc6eaaca14a02667c6be0b0a2270c8/tomli-2.4.0-cp312-cp312-win_amd64.whl", hash = "sha256:685306e2cc7da35be4ee914fd34ab801a6acacb061b6a7abca922aaf9ad368da", size = 108296, upload-time = "2026-01-11T11:22:04.86Z" }, + { url = "https://files.pythonhosted.org/packages/18/27/e267a60bbeeee343bcc279bb9e8fbed0cbe224bc7b2a3dc2975f22809a09/tomli-2.4.0-cp312-cp312-win_arm64.whl", hash = "sha256:5aa48d7c2356055feef06a43611fc401a07337d5b006be13a30f6c58f869e3c3", size = 94553, upload-time = "2026-01-11T11:22:05.854Z" }, + { url = "https://files.pythonhosted.org/packages/34/91/7f65f9809f2936e1f4ce6268ae1903074563603b2a2bd969ebbda802744f/tomli-2.4.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:84d081fbc252d1b6a982e1870660e7330fb8f90f676f6e78b052ad4e64714bf0", size = 154915, upload-time = "2026-01-11T11:22:06.703Z" }, + { url = "https://files.pythonhosted.org/packages/20/aa/64dd73a5a849c2e8f216b755599c511badde80e91e9bc2271baa7b2cdbb1/tomli-2.4.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:9a08144fa4cba33db5255f9b74f0b89888622109bd2776148f2597447f92a94e", size = 149038, upload-time = "2026-01-11T11:22:07.56Z" }, + { url = "https://files.pythonhosted.org/packages/9e/8a/6d38870bd3d52c8d1505ce054469a73f73a0fe62c0eaf5dddf61447e32fa/tomli-2.4.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c73add4bb52a206fd0c0723432db123c0c75c280cbd67174dd9d2db228ebb1b4", size = 242245, upload-time = "2026-01-11T11:22:08.344Z" }, + { url = "https://files.pythonhosted.org/packages/59/bb/8002fadefb64ab2669e5b977df3f5e444febea60e717e755b38bb7c41029/tomli-2.4.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1fb2945cbe303b1419e2706e711b7113da57b7db31ee378d08712d678a34e51e", size = 250335, upload-time = "2026-01-11T11:22:09.951Z" }, + { url = "https://files.pythonhosted.org/packages/a5/3d/4cdb6f791682b2ea916af2de96121b3cb1284d7c203d97d92d6003e91c8d/tomli-2.4.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:bbb1b10aa643d973366dc2cb1ad94f99c1726a02343d43cbc011edbfac579e7c", size = 245962, upload-time = "2026-01-11T11:22:11.27Z" }, + { url = "https://files.pythonhosted.org/packages/f2/4a/5f25789f9a460bd858ba9756ff52d0830d825b458e13f754952dd15fb7bb/tomli-2.4.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:4cbcb367d44a1f0c2be408758b43e1ffb5308abe0ea222897d6bfc8e8281ef2f", size = 250396, upload-time = "2026-01-11T11:22:12.325Z" }, + { url = "https://files.pythonhosted.org/packages/aa/2f/b73a36fea58dfa08e8b3a268750e6853a6aac2a349241a905ebd86f3047a/tomli-2.4.0-cp313-cp313-win32.whl", hash = "sha256:7d49c66a7d5e56ac959cb6fc583aff0651094ec071ba9ad43df785abc2320d86", size = 97530, upload-time = "2026-01-11T11:22:13.865Z" }, + { url = "https://files.pythonhosted.org/packages/3b/af/ca18c134b5d75de7e8dc551c5234eaba2e8e951f6b30139599b53de9c187/tomli-2.4.0-cp313-cp313-win_amd64.whl", hash = "sha256:3cf226acb51d8f1c394c1b310e0e0e61fecdd7adcb78d01e294ac297dd2e7f87", size = 108227, upload-time = "2026-01-11T11:22:15.224Z" }, + { url = "https://files.pythonhosted.org/packages/22/c3/b386b832f209fee8073c8138ec50f27b4460db2fdae9ffe022df89a57f9b/tomli-2.4.0-cp313-cp313-win_arm64.whl", hash = "sha256:d20b797a5c1ad80c516e41bc1fb0443ddb5006e9aaa7bda2d71978346aeb9132", size = 94748, upload-time = "2026-01-11T11:22:16.009Z" }, + { url = "https://files.pythonhosted.org/packages/f3/c4/84047a97eb1004418bc10bdbcfebda209fca6338002eba2dc27cc6d13563/tomli-2.4.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:26ab906a1eb794cd4e103691daa23d95c6919cc2fa9160000ac02370cc9dd3f6", size = 154725, upload-time = "2026-01-11T11:22:17.269Z" }, + { url = "https://files.pythonhosted.org/packages/a8/5d/d39038e646060b9d76274078cddf146ced86dc2b9e8bbf737ad5983609a0/tomli-2.4.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:20cedb4ee43278bc4f2fee6cb50daec836959aadaf948db5172e776dd3d993fc", size = 148901, upload-time = "2026-01-11T11:22:18.287Z" }, + { url = "https://files.pythonhosted.org/packages/73/e5/383be1724cb30f4ce44983d249645684a48c435e1cd4f8b5cded8a816d3c/tomli-2.4.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:39b0b5d1b6dd03684b3fb276407ebed7090bbec989fa55838c98560c01113b66", size = 243375, upload-time = "2026-01-11T11:22:19.154Z" }, + { url = "https://files.pythonhosted.org/packages/31/f0/bea80c17971c8d16d3cc109dc3585b0f2ce1036b5f4a8a183789023574f2/tomli-2.4.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a26d7ff68dfdb9f87a016ecfd1e1c2bacbe3108f4e0f8bcd2228ef9a766c787d", size = 250639, upload-time = "2026-01-11T11:22:20.168Z" }, + { url = "https://files.pythonhosted.org/packages/2c/8f/2853c36abbb7608e3f945d8a74e32ed3a74ee3a1f468f1ffc7d1cb3abba6/tomli-2.4.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:20ffd184fb1df76a66e34bd1b36b4a4641bd2b82954befa32fe8163e79f1a702", size = 246897, upload-time = "2026-01-11T11:22:21.544Z" }, + { url = "https://files.pythonhosted.org/packages/49/f0/6c05e3196ed5337b9fe7ea003e95fd3819a840b7a0f2bf5a408ef1dad8ed/tomli-2.4.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:75c2f8bbddf170e8effc98f5e9084a8751f8174ea6ccf4fca5398436e0320bc8", size = 254697, upload-time = "2026-01-11T11:22:23.058Z" }, + { url = "https://files.pythonhosted.org/packages/f3/f5/2922ef29c9f2951883525def7429967fc4d8208494e5ab524234f06b688b/tomli-2.4.0-cp314-cp314-win32.whl", hash = "sha256:31d556d079d72db7c584c0627ff3a24c5d3fb4f730221d3444f3efb1b2514776", size = 98567, upload-time = "2026-01-11T11:22:24.033Z" }, + { url = "https://files.pythonhosted.org/packages/7b/31/22b52e2e06dd2a5fdbc3ee73226d763b184ff21fc24e20316a44ccc4d96b/tomli-2.4.0-cp314-cp314-win_amd64.whl", hash = "sha256:43e685b9b2341681907759cf3a04e14d7104b3580f808cfde1dfdb60ada85475", size = 108556, upload-time = "2026-01-11T11:22:25.378Z" }, + { url = "https://files.pythonhosted.org/packages/48/3d/5058dff3255a3d01b705413f64f4306a141a8fd7a251e5a495e3f192a998/tomli-2.4.0-cp314-cp314-win_arm64.whl", hash = "sha256:3d895d56bd3f82ddd6faaff993c275efc2ff38e52322ea264122d72729dca2b2", size = 96014, upload-time = "2026-01-11T11:22:26.138Z" }, + { url = "https://files.pythonhosted.org/packages/b8/4e/75dab8586e268424202d3a1997ef6014919c941b50642a1682df43204c22/tomli-2.4.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:5b5807f3999fb66776dbce568cc9a828544244a8eb84b84b9bafc080c99597b9", size = 163339, upload-time = "2026-01-11T11:22:27.143Z" }, + { url = "https://files.pythonhosted.org/packages/06/e3/b904d9ab1016829a776d97f163f183a48be6a4deb87304d1e0116a349519/tomli-2.4.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:c084ad935abe686bd9c898e62a02a19abfc9760b5a79bc29644463eaf2840cb0", size = 159490, upload-time = "2026-01-11T11:22:28.399Z" }, + { url = "https://files.pythonhosted.org/packages/e3/5a/fc3622c8b1ad823e8ea98a35e3c632ee316d48f66f80f9708ceb4f2a0322/tomli-2.4.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0f2e3955efea4d1cfbcb87bc321e00dc08d2bcb737fd1d5e398af111d86db5df", size = 269398, upload-time = "2026-01-11T11:22:29.345Z" }, + { url = "https://files.pythonhosted.org/packages/fd/33/62bd6152c8bdd4c305ad9faca48f51d3acb2df1f8791b1477d46ff86e7f8/tomli-2.4.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0e0fe8a0b8312acf3a88077a0802565cb09ee34107813bba1c7cd591fa6cfc8d", size = 276515, upload-time = "2026-01-11T11:22:30.327Z" }, + { url = "https://files.pythonhosted.org/packages/4b/ff/ae53619499f5235ee4211e62a8d7982ba9e439a0fb4f2f351a93d67c1dd2/tomli-2.4.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:413540dce94673591859c4c6f794dfeaa845e98bf35d72ed59636f869ef9f86f", size = 273806, upload-time = "2026-01-11T11:22:32.56Z" }, + { url = "https://files.pythonhosted.org/packages/47/71/cbca7787fa68d4d0a9f7072821980b39fbb1b6faeb5f5cf02f4a5559fa28/tomli-2.4.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:0dc56fef0e2c1c470aeac5b6ca8cc7b640bb93e92d9803ddaf9ea03e198f5b0b", size = 281340, upload-time = "2026-01-11T11:22:33.505Z" }, + { url = "https://files.pythonhosted.org/packages/f5/00/d595c120963ad42474cf6ee7771ad0d0e8a49d0f01e29576ee9195d9ecdf/tomli-2.4.0-cp314-cp314t-win32.whl", hash = "sha256:d878f2a6707cc9d53a1be1414bbb419e629c3d6e67f69230217bb663e76b5087", size = 108106, upload-time = "2026-01-11T11:22:34.451Z" }, + { url = "https://files.pythonhosted.org/packages/de/69/9aa0c6a505c2f80e519b43764f8b4ba93b5a0bbd2d9a9de6e2b24271b9a5/tomli-2.4.0-cp314-cp314t-win_amd64.whl", hash = "sha256:2add28aacc7425117ff6364fe9e06a183bb0251b03f986df0e78e974047571fd", size = 120504, upload-time = "2026-01-11T11:22:35.764Z" }, + { url = "https://files.pythonhosted.org/packages/b3/9f/f1668c281c58cfae01482f7114a4b88d345e4c140386241a1a24dcc9e7bc/tomli-2.4.0-cp314-cp314t-win_arm64.whl", hash = "sha256:2b1e3b80e1d5e52e40e9b924ec43d81570f0e7d09d11081b797bc4692765a3d4", size = 99561, upload-time = "2026-01-11T11:22:36.624Z" }, + { url = "https://files.pythonhosted.org/packages/23/d1/136eb2cb77520a31e1f64cbae9d33ec6df0d78bdf4160398e86eec8a8754/tomli-2.4.0-py3-none-any.whl", hash = "sha256:1f776e7d669ebceb01dee46484485f43a4048746235e683bcdffacdf1fb4785a", size = 14477, upload-time = "2026-01-11T11:22:37.446Z" }, +] + +[[package]] +name = "typing-extensions" +version = "4.15.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/72/94/1a15dd82efb362ac84269196e94cf00f187f7ed21c242792a923cdb1c61f/typing_extensions-4.15.0.tar.gz", hash = "sha256:0cea48d173cc12fa28ecabc3b837ea3cf6f38c6d1136f85cbaaf598984861466", size = 109391, upload-time = "2025-08-25T13:49:26.313Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/18/67/36e9267722cc04a6b9f15c7f3441c2363321a3ea07da7ae0c0707beb2a9c/typing_extensions-4.15.0-py3-none-any.whl", hash = "sha256:f0fa19c6845758ab08074a0cfa8b7aecb71c999ca73d62883bc25cc018c4e548", size = 44614, upload-time = "2025-08-25T13:49:24.86Z" }, +] + +[[package]] +name = "zipp" +version = "3.23.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e3/02/0f2892c661036d50ede074e376733dca2ae7c6eb617489437771209d4180/zipp-3.23.0.tar.gz", hash = "sha256:a07157588a12518c9d4034df3fbbee09c814741a33ff63c05fa29d26a2404166", size = 25547, upload-time = "2025-06-08T17:06:39.4Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2e/54/647ade08bf0db230bfea292f893923872fd20be6ac6f53b2b936ba839d75/zipp-3.23.0-py3-none-any.whl", hash = "sha256:071652d6115ed432f5ce1d34c336c0adfd6a884660d1e9712a256d3d3bd4b14e", size = 10276, upload-time = "2025-06-08T17:06:38.034Z" }, +] diff --git a/packages/opentelemetry-instrumentation-google-generativeai/opentelemetry/instrumentation/google_generativeai/__init__.py b/packages/opentelemetry-instrumentation-google-generativeai/opentelemetry/instrumentation/google_generativeai/__init__.py index 9982d66982..890d276808 100644 --- a/packages/opentelemetry-instrumentation-google-generativeai/opentelemetry/instrumentation/google_generativeai/__init__.py +++ b/packages/opentelemetry-instrumentation-google-generativeai/opentelemetry/instrumentation/google_generativeai/__init__.py @@ -14,6 +14,10 @@ emit_choice_events, emit_message_events, ) +from opentelemetry.instrumentation.google_generativeai.safety import ( + _apply_completion_safety, + _apply_prompt_safety, +) from opentelemetry.instrumentation.google_generativeai.span_utils import ( set_input_attributes_sync, set_model_request_attributes, @@ -210,6 +214,7 @@ async def _awrap( }, ) start_time = time.perf_counter() + args, kwargs = _apply_prompt_safety(span, args, kwargs, name) _handle_request(span, args, kwargs, llm_model, event_logger) try: response = await wrapped(*args, **kwargs) @@ -238,6 +243,7 @@ async def _awrap( span, response, llm_model, event_logger, token_histogram ) else: + _apply_completion_safety(span, response, name) _handle_response( span, response, llm_model, event_logger, token_histogram ) @@ -287,6 +293,7 @@ def _wrap( ) start_time = time.perf_counter() + args, kwargs = _apply_prompt_safety(span, args, kwargs, name) _handle_request(span, args, kwargs, llm_model, event_logger) try: response = wrapped(*args, **kwargs) @@ -315,6 +322,7 @@ def _wrap( span, response, llm_model, event_logger, token_histogram ) else: + _apply_completion_safety(span, response, name) _handle_response( span, response, llm_model, event_logger, token_histogram ) diff --git a/packages/opentelemetry-instrumentation-google-generativeai/opentelemetry/instrumentation/google_generativeai/safety.py b/packages/opentelemetry-instrumentation-google-generativeai/opentelemetry/instrumentation/google_generativeai/safety.py new file mode 100644 index 0000000000..09bb3fa4ed --- /dev/null +++ b/packages/opentelemetry-instrumentation-google-generativeai/opentelemetry/instrumentation/google_generativeai/safety.py @@ -0,0 +1,193 @@ +from __future__ import annotations + +from opentelemetry.instrumentation.fortifyroot import ( + SafetyDecision, + SafetyLocation, + clone_value, + get_object_value, + run_completion_safety, + run_prompt_safety, + set_object_value, +) +from opentelemetry.semconv_ai import LLMRequestTypeValues + +PROVIDER = "Google" + + +def _apply_prompt_safety(span, args, kwargs, span_name): + try: + updated_args = args + updated_kwargs = kwargs + + if args: + masked_arg, changed = _mask_prompt_value( + span, + args[0], + span_name=span_name, + segment_index=0, + segment_role="user", + ) + if changed: + updated_args = (masked_arg, *args[1:]) + + if "contents" in kwargs: + masked_contents, changed = _mask_prompt_value( + span, + kwargs.get("contents"), + span_name=span_name, + segment_index=0, + segment_role="user", + ) + if changed: + updated_kwargs = dict(kwargs) + updated_kwargs["contents"] = masked_contents + + return updated_args, updated_kwargs + except Exception: + return args, kwargs + + +def _mask_prompt_value(span, value, *, span_name, segment_index, segment_role): + if isinstance(value, str): + return _mask_prompt_text( + span, + value, + span_name=span_name, + segment_index=segment_index, + segment_role=segment_role, + ) + + if isinstance(value, list): + updated_value = value + for index, item in enumerate(value): + masked_item, changed = _mask_prompt_value( + span, + item, + span_name=span_name, + segment_index=index, + segment_role=segment_role, + ) + if not changed: + continue + if updated_value is value: + updated_value = clone_value(value) + updated_value[index] = masked_item + return updated_value, updated_value is not value + + parts = get_object_value(value, "parts") + if isinstance(parts, list): + updated_parts = parts + for index, part in enumerate(parts): + text = get_object_value(part, "text") + if not isinstance(text, str): + continue + masked_text, changed = _mask_prompt_text( + span, + text, + span_name=span_name, + segment_index=index, + segment_role=segment_role, + metadata={"part_index": index}, + ) + if not changed: + continue + if updated_parts is parts: + updated_parts = clone_value(parts) + value = clone_value(value) + set_object_value(value, "parts", updated_parts) + set_object_value(updated_parts[index], "text", masked_text) + return value, updated_parts is not parts + + return value, False + + +def _apply_completion_safety(span, response, span_name): + try: + text = get_object_value(response, "text") + if isinstance(text, str): + updated_text, changed = _mask_completion_text( + span, + text, + span_name=span_name, + segment_index=0, + ) + if changed: + set_object_value(response, "text", updated_text) + + candidates = get_object_value(response, "candidates") + if not isinstance(candidates, list): + return + + for candidate_index, candidate in enumerate(candidates): + content = get_object_value(candidate, "content") + parts = get_object_value(content, "parts") + if not isinstance(parts, list): + continue + for part_index, part in enumerate(parts): + part_text = get_object_value(part, "text") + if not isinstance(part_text, str): + continue + updated_part_text, changed = _mask_completion_text( + span, + part_text, + span_name=span_name, + segment_index=candidate_index, + metadata={"part_index": part_index}, + ) + if changed: + set_object_value(part, "text", updated_part_text) + except Exception: + return + + +def _mask_prompt_text( + span, + text, + *, + span_name, + segment_index, + segment_role, + metadata=None, +): + result = run_prompt_safety( + span=span, + provider=PROVIDER, + span_name=span_name, + text=text, + location=SafetyLocation.PROMPT, + request_type=LLMRequestTypeValues.COMPLETION.value, + segment_index=segment_index, + segment_role=segment_role, + metadata=metadata, + ) + return _resolve_masked_text(text, result) + + +def _mask_completion_text( + span, + text, + *, + span_name, + segment_index, + metadata=None, +): + result = run_completion_safety( + span=span, + provider=PROVIDER, + span_name=span_name, + text=text, + location=SafetyLocation.COMPLETION, + request_type=LLMRequestTypeValues.COMPLETION.value, + segment_index=segment_index, + segment_role="assistant", + metadata=metadata, + ) + return _resolve_masked_text(text, result) + + +def _resolve_masked_text(original_text, result): + if result is None or result.overall_action != SafetyDecision.MASK.value: + return original_text, False + if result.text == original_text: + return original_text, False + return result.text, True diff --git a/packages/opentelemetry-instrumentation-google-generativeai/pyproject.toml b/packages/opentelemetry-instrumentation-google-generativeai/pyproject.toml index 7c66563f0b..5d02ed7081 100644 --- a/packages/opentelemetry-instrumentation-google-generativeai/pyproject.toml +++ b/packages/opentelemetry-instrumentation-google-generativeai/pyproject.toml @@ -11,6 +11,7 @@ readme = "README.md" requires-python = ">=3.10,<4" dependencies = [ "opentelemetry-api>=1.38.0,<2", + "opentelemetry-instrumentation-fortifyroot", "opentelemetry-instrumentation>=0.59b0", "opentelemetry-semantic-conventions-ai>=0.4.13,<0.5.0", "opentelemetry-semantic-conventions>=0.59b0", @@ -70,5 +71,13 @@ exclude = [ [tool.ruff.lint] select = ["E", "F", "W"] +[tool.pytest.ini_options] +markers = [ + "safety: safety-focused tests", +] + [tool.uv] constraint-dependencies = ["urllib3>=2.6.3", "pip>=25.3"] + +[tool.uv.sources] +opentelemetry-instrumentation-fortifyroot = { path = "../opentelemetry-instrumentation-fortifyroot", editable = true } diff --git a/packages/opentelemetry-instrumentation-google-generativeai/tests/test_safety_hooks.py b/packages/opentelemetry-instrumentation-google-generativeai/tests/test_safety_hooks.py new file mode 100644 index 0000000000..551b4139b2 --- /dev/null +++ b/packages/opentelemetry-instrumentation-google-generativeai/tests/test_safety_hooks.py @@ -0,0 +1,144 @@ +from types import SimpleNamespace + +import pytest + +from opentelemetry.instrumentation.fortifyroot import ( + SafetyFinding, + SafetyLocation, + SafetyResult, + clear_safety_handlers, + register_completion_safety_handler, + register_prompt_safety_handler, +) +from opentelemetry.instrumentation.google_generativeai import ( + _handle_request, + _apply_completion_safety, + _apply_prompt_safety, +) +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import SimpleSpanProcessor +from opentelemetry.sdk.trace.export.in_memory_span_exporter import ( + InMemorySpanExporter, +) + +pytestmark = pytest.mark.safety + + +def setup_function(): + clear_safety_handlers() + + +def teardown_function(): + clear_safety_handlers() + + +def _test_span(): + exporter = InMemorySpanExporter() + provider = TracerProvider() + provider.add_span_processor(SimpleSpanProcessor(exporter)) + tracer = provider.get_tracer(__name__) + return exporter, tracer + + +def test_prompt_safety_masks_positional_prompt_args(): + _, tracer = _test_span() + register_prompt_safety_handler( + lambda context: SafetyResult( + text="[PII.prompt]", + overall_action="MASK", + findings=[ + SafetyFinding( + category="PII", + severity="MEDIUM", + action="MASK", + rule_name="PII.prompt", + start=0, + end=len(context.text), + ) + ], + ) + if context.location == SafetyLocation.PROMPT and context.text == "secret" + else None + ) + + with tracer.start_as_current_span("gemini.generate_content") as span: + updated_args, updated_kwargs = _apply_prompt_safety( + span, + ("secret",), + {}, + "gemini.generate_content", + ) + + assert updated_args == ("[PII.prompt]",) + assert updated_kwargs == {} + + +def test_prompt_safety_masks_span_prompt_attributes(): + exporter, tracer = _test_span() + register_prompt_safety_handler( + lambda context: SafetyResult( + text="[PII.prompt]", + overall_action="MASK", + findings=[ + SafetyFinding( + category="PII", + severity="MEDIUM", + action="MASK", + rule_name="PII.prompt", + start=0, + end=len(context.text), + ) + ], + ) + if context.location == SafetyLocation.PROMPT and context.text == "secret" + else None + ) + + with tracer.start_as_current_span("gemini.generate_content") as span: + updated_args, updated_kwargs = _apply_prompt_safety( + span, + ("secret",), + {}, + "gemini.generate_content", + ) + _handle_request(span, updated_args, updated_kwargs, "gemini-2.0", None) + + spans = exporter.get_finished_spans() + assert spans[0].attributes["gen_ai.prompt.0.content"] == '[{"type": "text", "text": "[PII.prompt]"}]' + assert spans[0].attributes["gen_ai.prompt.0.role"] == "user" + + +def test_completion_safety_masks_candidate_parts(): + exporter, tracer = _test_span() + register_completion_safety_handler( + lambda context: SafetyResult( + text="[SECRET.gemini]", + overall_action="MASK", + findings=[ + SafetyFinding( + category="SECRET", + severity="HIGH", + action="MASK", + rule_name="SECRET.gemini", + start=0, + end=len(context.text), + ) + ], + ) + if context.location == SafetyLocation.COMPLETION and context.text == "secret" + else None + ) + + response = SimpleNamespace( + candidates=[ + SimpleNamespace( + content=SimpleNamespace(parts=[SimpleNamespace(text="secret")]) + ) + ] + ) + with tracer.start_as_current_span("gemini.generate_content") as span: + _apply_completion_safety(span, response, "gemini.generate_content") + + assert response.candidates[0].content.parts[0].text == "[SECRET.gemini]" + spans = exporter.get_finished_spans() + assert len(spans[0].events) == 1 diff --git a/packages/opentelemetry-instrumentation-google-generativeai/tests/test_safety_unit.py b/packages/opentelemetry-instrumentation-google-generativeai/tests/test_safety_unit.py new file mode 100644 index 0000000000..4c17900943 --- /dev/null +++ b/packages/opentelemetry-instrumentation-google-generativeai/tests/test_safety_unit.py @@ -0,0 +1,89 @@ +from types import SimpleNamespace + +import pytest + +from opentelemetry.instrumentation.fortifyroot import SafetyDecision, SafetyResult +from opentelemetry.instrumentation.google_generativeai import safety + +pytestmark = pytest.mark.safety + + +def test_apply_prompt_safety_masks_args_kwargs_and_parts(monkeypatch): + monkeypatch.setattr(safety, "run_prompt_safety", lambda **kwargs: SafetyResult(text=f"masked:{kwargs['text']}", overall_action="MASK")) + args = ("secret",) + kwargs = { + "contents": [ + "list-secret", + SimpleNamespace(parts=[SimpleNamespace(text="part-secret")]), + ] + } + + updated_args, updated_kwargs = safety._apply_prompt_safety(None, args, kwargs, "gemini.generate_content") + + assert updated_args == ("masked:secret",) + assert updated_kwargs["contents"][0] == "masked:list-secret" + assert updated_kwargs["contents"][1].parts[0].text == "masked:part-secret" + + +def test_google_apply_prompt_safety_noop_and_exception_paths(monkeypatch): + monkeypatch.setattr(safety, "run_prompt_safety", lambda **kwargs: SafetyResult(text=kwargs["text"], overall_action="MASK")) + args = ("keep",) + kwargs = {"contents": "keep"} + updated_args, updated_kwargs = safety._apply_prompt_safety(None, args, kwargs, "gemini.generate_content") + assert updated_args == args + assert updated_kwargs is kwargs + + monkeypatch.setattr(safety, "_mask_prompt_value", lambda *args, **kwargs: (_ for _ in ()).throw(RuntimeError("boom"))) + assert safety._apply_prompt_safety(None, args, kwargs, "gemini.generate_content") == (args, kwargs) + + +def test_apply_completion_safety_masks_text_and_candidate_parts(monkeypatch): + monkeypatch.setattr(safety, "run_completion_safety", lambda **kwargs: SafetyResult(text=f"masked:{kwargs['text']}", overall_action="MASK")) + response = SimpleNamespace( + text="secret", + candidates=[ + SimpleNamespace(content=SimpleNamespace(parts=[SimpleNamespace(text="part-secret")])), + SimpleNamespace(content=SimpleNamespace(parts=[SimpleNamespace(binary=b"x")])), + ], + ) + + safety._apply_completion_safety(None, response, "gemini.generate_content") + + assert response.text == "masked:secret" + assert response.candidates[0].content.parts[0].text == "masked:part-secret" + + +def test_google_prompt_and_completion_helpers_cover_noop_branches(monkeypatch): + monkeypatch.setattr(safety, "run_prompt_safety", lambda **kwargs: SafetyResult(text=kwargs["text"], overall_action="MASK")) + assert safety._mask_prompt_value(None, {"x": 1}, span_name="gemini.generate_content", segment_index=0, segment_role="user") == ({"x": 1}, False) + updated, changed = safety._mask_prompt_value( + None, + ["keep", SimpleNamespace(parts=[SimpleNamespace(binary=b"x")])], + span_name="gemini.generate_content", + segment_index=0, + segment_role="user", + ) + assert changed is False + assert updated[0] == "keep" + + monkeypatch.setattr(safety, "run_completion_safety", lambda **kwargs: SafetyResult(text=kwargs["text"], overall_action="MASK")) + response = SimpleNamespace(text="keep", candidates="not-a-list") + assert safety._apply_completion_safety(None, response, "gemini.generate_content") is None + + response = SimpleNamespace(text="keep", candidates=[SimpleNamespace(content=SimpleNamespace(parts=[SimpleNamespace(binary=b"x")]))]) + safety._apply_completion_safety(None, response, "gemini.generate_content") + assert response.text == "keep" + + monkeypatch.setattr(safety, "get_object_value", lambda *args, **kwargs: (_ for _ in ()).throw(RuntimeError("boom"))) + assert safety._apply_completion_safety(None, SimpleNamespace(text="secret"), "gemini.generate_content") is None + + +def test_google_resolve_masked_text_and_fail_open(monkeypatch): + assert safety._resolve_masked_text("x", None) == ("x", False) + assert safety._resolve_masked_text("x", SafetyResult(text="x", overall_action=SafetyDecision.MASK.value)) == ("x", False) + assert safety._resolve_masked_text("x", SafetyResult(text="y", overall_action=SafetyDecision.ALLOW.value)) == ("x", False) + assert safety._resolve_masked_text("x", SafetyResult(text="y", overall_action=SafetyDecision.MASK.value)) == ("y", True) + + args = ("secret",) + monkeypatch.setattr(safety, "clone_value", lambda *_: (_ for _ in ()).throw(RuntimeError("boom"))) + assert safety._apply_prompt_safety(None, args, {}, "gemini.generate_content") == (args, {}) diff --git a/packages/opentelemetry-instrumentation-openai/opentelemetry/instrumentation/openai/shared/chat_safety.py b/packages/opentelemetry-instrumentation-openai/opentelemetry/instrumentation/openai/shared/chat_safety.py new file mode 100644 index 0000000000..561376edbc --- /dev/null +++ b/packages/opentelemetry-instrumentation-openai/opentelemetry/instrumentation/openai/shared/chat_safety.py @@ -0,0 +1,133 @@ +from __future__ import annotations + +from opentelemetry.instrumentation.fortifyroot import ( + clone_value, + get_object_value, + set_object_value, +) +from opentelemetry.instrumentation.openai.shared.safety_common import ( + CHAT_SPAN_NAME, + mask_completion_text, + mask_prompt_text, +) + + +def _apply_prompt_safety(span, kwargs): + try: + messages = kwargs.get("messages") + if not isinstance(messages, list): + return kwargs + + mutated_kwargs = kwargs + mutated_messages = None + for index, message in enumerate(messages): + role = get_object_value(message, "role") + content = get_object_value(message, "content") + updated_content, changed = _mask_prompt_content( + span, + content, + message_index=index, + message_role=role, + ) + if not changed: + continue + if mutated_messages is None: + mutated_kwargs = dict(kwargs) + mutated_messages = clone_value(messages) + mutated_kwargs["messages"] = mutated_messages + set_object_value(mutated_messages[index], "content", updated_content) + return mutated_kwargs + except Exception: + return kwargs + + +def _mask_prompt_content(span, content, *, message_index, message_role): + if isinstance(content, str): + return mask_prompt_text( + span, + content, + span_name=CHAT_SPAN_NAME, + segment_index=message_index, + segment_role=message_role, + ) + + if not isinstance(content, list): + return content, False + + updated_content = content + for block_index, block in enumerate(content): + block_type = get_object_value(block, "type") + block_text = get_object_value(block, "text") + if block_type not in ("text", "input_text") or not isinstance(block_text, str): + continue + resolved_text, changed = mask_prompt_text( + span, + block_text, + span_name=CHAT_SPAN_NAME, + segment_index=message_index, + segment_role=message_role, + metadata={"block_index": block_index}, + ) + if not changed: + continue + if updated_content is content: + updated_content = clone_value(content) + set_object_value(updated_content[block_index], "text", resolved_text) + + return updated_content, updated_content is not content + + +def _apply_completion_safety(span, response): + try: + choices = getattr(response, "choices", None) + if not choices: + return + + for choice_index, choice in enumerate(choices): + message = get_object_value(choice, "message") + if message is None: + continue + content = get_object_value(message, "content") + updated_content, changed = _mask_completion_content( + span, + content, + choice_index=choice_index, + ) + if changed: + set_object_value(message, "content", updated_content) + except Exception: + return + + +def _mask_completion_content(span, content, *, choice_index): + if isinstance(content, str): + return mask_completion_text( + span, + content, + span_name=CHAT_SPAN_NAME, + segment_index=choice_index, + ) + + if not isinstance(content, list): + return content, False + + updated_content = content + for block_index, block in enumerate(content): + block_type = get_object_value(block, "type") + block_text = get_object_value(block, "text") + if block_type not in (None, "text", "output_text") or not isinstance(block_text, str): + continue + resolved_text, changed = mask_completion_text( + span, + block_text, + span_name=CHAT_SPAN_NAME, + segment_index=choice_index, + metadata={"block_index": block_index}, + ) + if not changed: + continue + if updated_content is content: + updated_content = clone_value(content) + set_object_value(updated_content[block_index], "text", resolved_text) + + return updated_content, updated_content is not content diff --git a/packages/opentelemetry-instrumentation-openai/opentelemetry/instrumentation/openai/shared/chat_wrappers.py b/packages/opentelemetry-instrumentation-openai/opentelemetry/instrumentation/openai/shared/chat_wrappers.py index 515723044c..0d0a562c18 100644 --- a/packages/opentelemetry-instrumentation-openai/opentelemetry/instrumentation/openai/shared/chat_wrappers.py +++ b/packages/opentelemetry-instrumentation-openai/opentelemetry/instrumentation/openai/shared/chat_wrappers.py @@ -31,6 +31,10 @@ MessageEvent, ToolCall, ) +from opentelemetry.instrumentation.openai.shared.chat_safety import ( + _apply_completion_safety, + _apply_prompt_safety, +) from opentelemetry.instrumentation.openai.utils import ( _with_chat_telemetry_wrapper, dont_throw, @@ -92,6 +96,7 @@ def chat_wrapper( # Use the span as current context to ensure events get proper trace context with trace.use_span(span, end_on_exit=False): + kwargs = _apply_prompt_safety(span, kwargs) run_async(_handle_request(span, kwargs, instance)) try: start_time = time.time() @@ -148,6 +153,7 @@ def chat_wrapper( duration = end_time - start_time + _apply_completion_safety(span, response) _handle_response( response, span, @@ -190,6 +196,7 @@ async def achat_wrapper( # Use the span as current context to ensure events get proper trace context with trace.use_span(span, end_on_exit=False): + kwargs = _apply_prompt_safety(span, kwargs) await _handle_request(span, kwargs, instance) try: @@ -249,6 +256,7 @@ async def achat_wrapper( duration = end_time - start_time + _apply_completion_safety(span, response) _handle_response( response, span, diff --git a/packages/opentelemetry-instrumentation-openai/opentelemetry/instrumentation/openai/shared/completion_safety.py b/packages/opentelemetry-instrumentation-openai/opentelemetry/instrumentation/openai/shared/completion_safety.py new file mode 100644 index 0000000000..a32e91c9a8 --- /dev/null +++ b/packages/opentelemetry-instrumentation-openai/opentelemetry/instrumentation/openai/shared/completion_safety.py @@ -0,0 +1,71 @@ +from __future__ import annotations + +from opentelemetry.instrumentation.openai.shared.safety_common import ( + COMPLETION_SPAN_NAME, + mask_completion_text, + mask_prompt_text, +) +from opentelemetry.instrumentation.fortifyroot import get_object_value, set_object_value + + +def _apply_prompt_safety(span, kwargs): + try: + prompt = kwargs.get("prompt") + if isinstance(prompt, str): + updated_prompt, changed = mask_prompt_text( + span, + prompt, + span_name=COMPLETION_SPAN_NAME, + segment_index=0, + segment_role="user", + ) + if not changed: + return kwargs + mutated_kwargs = dict(kwargs) + mutated_kwargs["prompt"] = updated_prompt + return mutated_kwargs + + if not isinstance(prompt, list): + return kwargs + + mutated_prompt = None + for index, prompt_text in enumerate(prompt): + if not isinstance(prompt_text, str): + continue + updated_prompt, changed = mask_prompt_text( + span, + prompt_text, + span_name=COMPLETION_SPAN_NAME, + segment_index=index, + segment_role="user", + ) + if not changed: + continue + if mutated_prompt is None: + mutated_prompt = list(prompt) + mutated_prompt[index] = updated_prompt + if mutated_prompt is None: + return kwargs + mutated_kwargs = dict(kwargs) + mutated_kwargs["prompt"] = mutated_prompt + return mutated_kwargs + except Exception: + return kwargs + + +def _apply_completion_safety(span, response): + try: + for index, choice in enumerate(get_object_value(response, "choices", []) or []): + text = get_object_value(choice, "text") + if not isinstance(text, str): + continue + updated_text, changed = mask_completion_text( + span, + text, + span_name=COMPLETION_SPAN_NAME, + segment_index=index, + ) + if changed: + set_object_value(choice, "text", updated_text) + except Exception: + return diff --git a/packages/opentelemetry-instrumentation-openai/opentelemetry/instrumentation/openai/shared/completion_wrappers.py b/packages/opentelemetry-instrumentation-openai/opentelemetry/instrumentation/openai/shared/completion_wrappers.py index 08baf1ed11..50f24c81dc 100644 --- a/packages/opentelemetry-instrumentation-openai/opentelemetry/instrumentation/openai/shared/completion_wrappers.py +++ b/packages/opentelemetry-instrumentation-openai/opentelemetry/instrumentation/openai/shared/completion_wrappers.py @@ -20,6 +20,10 @@ ChoiceEvent, MessageEvent, ) +from opentelemetry.instrumentation.openai.shared.completion_safety import ( + _apply_completion_safety, + _apply_prompt_safety, +) from opentelemetry.instrumentation.openai.utils import ( _with_tracer_wrapper, dont_throw, @@ -61,6 +65,7 @@ def completion_wrapper(tracer, wrapped, instance, args, kwargs): # Use the span as current context to ensure events get proper trace context with trace.use_span(span, end_on_exit=False): + kwargs = _apply_prompt_safety(span, kwargs) _handle_request(span, kwargs, instance) try: @@ -76,6 +81,7 @@ def completion_wrapper(tracer, wrapped, instance, args, kwargs): # span will be closed after the generator is done return _build_from_streaming_response(span, kwargs, response) else: + _apply_completion_safety(span, response) _handle_response(response, span, instance) span.end() @@ -97,6 +103,7 @@ async def acompletion_wrapper(tracer, wrapped, instance, args, kwargs): # Use the span as current context to ensure events get proper trace context with trace.use_span(span, end_on_exit=False): + kwargs = _apply_prompt_safety(span, kwargs) _handle_request(span, kwargs, instance) try: @@ -112,6 +119,7 @@ async def acompletion_wrapper(tracer, wrapped, instance, args, kwargs): # span will be closed after the generator is done return _abuild_from_streaming_response(span, kwargs, response) else: + _apply_completion_safety(span, response) _handle_response(response, span, instance) span.end() diff --git a/packages/opentelemetry-instrumentation-openai/opentelemetry/instrumentation/openai/shared/safety_common.py b/packages/opentelemetry-instrumentation-openai/opentelemetry/instrumentation/openai/shared/safety_common.py new file mode 100644 index 0000000000..555a106e37 --- /dev/null +++ b/packages/opentelemetry-instrumentation-openai/opentelemetry/instrumentation/openai/shared/safety_common.py @@ -0,0 +1,72 @@ +from __future__ import annotations + +from opentelemetry.instrumentation.fortifyroot import ( + SafetyDecision, + SafetyLocation, + run_completion_safety, + run_prompt_safety, +) +from opentelemetry.semconv_ai import LLMRequestTypeValues + +CHAT_PROVIDER = "OpenAI" +CHAT_SPAN_NAME = "openai.chat" +COMPLETION_SPAN_NAME = "openai.completion" + + +def mask_prompt_text( + span, + text, + *, + span_name, + segment_index, + segment_role=None, + metadata=None, +): + result = run_prompt_safety( + span=span, + provider=CHAT_PROVIDER, + span_name=span_name, + text=text, + location=SafetyLocation.PROMPT, + request_type=request_type(span_name), + segment_index=segment_index, + segment_role=segment_role, + metadata=metadata, + ) + return resolve_masked_text(text, result) + + +def mask_completion_text( + span, + text, + *, + span_name, + segment_index, + metadata=None, +): + result = run_completion_safety( + span=span, + provider=CHAT_PROVIDER, + span_name=span_name, + text=text, + location=SafetyLocation.COMPLETION, + request_type=request_type(span_name), + segment_index=segment_index, + segment_role="assistant", + metadata=metadata, + ) + return resolve_masked_text(text, result) + + +def request_type(span_name: str) -> str: + if span_name == COMPLETION_SPAN_NAME: + return LLMRequestTypeValues.COMPLETION.value + return LLMRequestTypeValues.CHAT.value + + +def resolve_masked_text(original_text, result): + if result is None or result.overall_action != SafetyDecision.MASK.value: + return original_text, False + if result.text == original_text: + return original_text, False + return result.text, True diff --git a/packages/opentelemetry-instrumentation-openai/pyproject.toml b/packages/opentelemetry-instrumentation-openai/pyproject.toml index 1c8021fe37..c8e40064e1 100644 --- a/packages/opentelemetry-instrumentation-openai/pyproject.toml +++ b/packages/opentelemetry-instrumentation-openai/pyproject.toml @@ -12,6 +12,7 @@ readme = "README.md" requires-python = ">=3.10,<4" dependencies = [ "opentelemetry-api>=1.38.0,<2", + "opentelemetry-instrumentation-fortifyroot", "opentelemetry-instrumentation>=0.59b0", "opentelemetry-semantic-conventions-ai>=0.4.13,<0.5.0", "opentelemetry-semantic-conventions>=0.59b0", @@ -73,3 +74,6 @@ select = ["E", "F", "W"] [tool.uv] constraint-dependencies = ["urllib3>=2.6.3", "pip>=25.3"] + +[tool.uv.sources] +opentelemetry-instrumentation-fortifyroot = { path = "../opentelemetry-instrumentation-fortifyroot", editable = true } diff --git a/packages/opentelemetry-instrumentation-openai/pytest.ini b/packages/opentelemetry-instrumentation-openai/pytest.ini index 40880458c7..55e2214826 100644 --- a/packages/opentelemetry-instrumentation-openai/pytest.ini +++ b/packages/opentelemetry-instrumentation-openai/pytest.ini @@ -1,2 +1,4 @@ [pytest] asyncio_mode=auto +markers = + safety: safety-focused tests diff --git a/packages/opentelemetry-instrumentation-openai/tests/traces/test_safety_helpers.py b/packages/opentelemetry-instrumentation-openai/tests/traces/test_safety_helpers.py new file mode 100644 index 0000000000..7fe2716b15 --- /dev/null +++ b/packages/opentelemetry-instrumentation-openai/tests/traces/test_safety_helpers.py @@ -0,0 +1,165 @@ +from types import SimpleNamespace + +import pytest + +from opentelemetry.instrumentation.openai.shared import chat_safety, completion_safety, safety_common +from opentelemetry.instrumentation.fortifyroot import SafetyDecision, SafetyResult + +pytestmark = pytest.mark.safety + + +def test_chat_prompt_safety_masks_string_and_block_content(monkeypatch): + def _mask(span, text, **kwargs): + return (f"masked:{text}", True) + + monkeypatch.setattr(chat_safety, "mask_prompt_text", _mask) + + kwargs = { + "messages": [ + {"role": "user", "content": "secret"}, + { + "role": "assistant", + "content": [ + {"type": "text", "text": "secret-2"}, + {"type": "image", "image": "ignored"}, + ], + }, + ] + } + + updated = chat_safety._apply_prompt_safety(None, kwargs) + + assert kwargs["messages"][0]["content"] == "secret" + assert updated["messages"][0]["content"] == "masked:secret" + assert updated["messages"][1]["content"][0]["text"] == "masked:secret-2" + assert updated["messages"][1]["content"][1]["image"] == "ignored" + + +def test_chat_prompt_safety_returns_original_when_messages_are_not_a_list(monkeypatch): + monkeypatch.setattr(chat_safety, "mask_prompt_text", lambda *args, **kwargs: ("x", True)) + kwargs = {"messages": "secret"} + assert chat_safety._apply_prompt_safety(None, kwargs) is kwargs + + +def test_chat_prompt_safety_skips_unchanged_messages(monkeypatch): + def _mask(span, text, **kwargs): + return (text, False) if text == "keep" else (f"masked:{text}", True) + + monkeypatch.setattr(chat_safety, "mask_prompt_text", _mask) + kwargs = {"messages": [{"role": "user", "content": "keep"}, {"role": "user", "content": "secret"}]} + updated = chat_safety._apply_prompt_safety(None, kwargs) + + assert updated["messages"][0]["content"] == "keep" + assert updated["messages"][1]["content"] == "masked:secret" + + +def test_chat_completion_safety_masks_string_and_output_blocks(monkeypatch): + monkeypatch.setattr(chat_safety, "mask_completion_text", lambda span, text, **kwargs: (f"done:{text}", True)) + response = SimpleNamespace( + choices=[ + SimpleNamespace(message=SimpleNamespace(content="secret")), + SimpleNamespace( + message=SimpleNamespace( + content=[ + {"type": "output_text", "text": "secret-2"}, + {"type": "tool", "text": "ignored"}, + ] + ) + ), + ] + ) + + chat_safety._apply_completion_safety(None, response) + + assert response.choices[0].message.content == "done:secret" + assert response.choices[1].message.content[0]["text"] == "done:secret-2" + assert response.choices[1].message.content[1]["text"] == "ignored" + + +def test_chat_completion_safety_ignores_empty_choices_and_missing_message(monkeypatch): + monkeypatch.setattr(chat_safety, "mask_completion_text", lambda span, text, **kwargs: (text, False)) + assert chat_safety._apply_completion_safety(None, SimpleNamespace(choices=[])) is None + response = SimpleNamespace(choices=[SimpleNamespace(message=None)]) + assert chat_safety._apply_completion_safety(None, response) is None + + +def test_chat_and_completion_content_helpers_cover_non_list_and_unchanged(monkeypatch): + monkeypatch.setattr(chat_safety, "mask_prompt_text", lambda span, text, **kwargs: (text, False)) + assert chat_safety._mask_prompt_content(None, {"x": 1}, message_index=0, message_role="user") == ({"x": 1}, False) + + content = [{"type": "text", "text": "keep"}] + updated, changed = chat_safety._mask_prompt_content(None, content, message_index=0, message_role="user") + assert updated == content + assert changed is False + + monkeypatch.setattr(chat_safety, "mask_completion_text", lambda span, text, **kwargs: (text, False)) + assert chat_safety._mask_completion_content(None, {"x": 1}, choice_index=0) == ({"x": 1}, False) + updated, changed = chat_safety._mask_completion_content(None, [{"type": "text", "text": "keep"}], choice_index=0) + assert changed is False + assert updated[0]["text"] == "keep" + + +def test_completion_prompt_safety_masks_string_and_list(monkeypatch): + monkeypatch.setattr(completion_safety, "mask_prompt_text", lambda span, text, **kwargs: (f"masked:{text}", True)) + + assert completion_safety._apply_prompt_safety(None, {"prompt": "secret"})["prompt"] == "masked:secret" + updated = completion_safety._apply_prompt_safety(None, {"prompt": ["a", "b"]}) + assert updated["prompt"] == ["masked:a", "masked:b"] + same = {"prompt": {"unsupported": True}} + assert completion_safety._apply_prompt_safety(None, same) is same + + +def test_completion_prompt_safety_returns_original_for_unchanged_or_non_string_items(monkeypatch): + monkeypatch.setattr(completion_safety, "mask_prompt_text", lambda span, text, **kwargs: (text, False)) + kwargs = {"prompt": "keep"} + assert completion_safety._apply_prompt_safety(None, kwargs) is kwargs + kwargs = {"prompt": [1, "keep"]} + assert completion_safety._apply_prompt_safety(None, kwargs) is kwargs + + +def test_completion_safety_masks_choice_text(monkeypatch): + monkeypatch.setattr(completion_safety, "mask_completion_text", lambda span, text, **kwargs: (f"done:{text}", True)) + response = SimpleNamespace(choices=[SimpleNamespace(text="secret"), SimpleNamespace(text=None)]) + completion_safety._apply_completion_safety(None, response) + assert response.choices[0].text == "done:secret" + + +def test_completion_safety_skips_non_text_choices_and_fail_opens(): + response = SimpleNamespace(choices=[SimpleNamespace(text=None)]) + assert completion_safety._apply_completion_safety(None, response) is None + + class BrokenKwargs: + def get(self, *_args, **_kwargs): + raise RuntimeError("boom") + + broken = BrokenKwargs() + assert completion_safety._apply_prompt_safety(None, broken) is broken + + +def test_safety_common_request_type_and_resolve_masked_text(): + assert safety_common.request_type(safety_common.CHAT_SPAN_NAME) == "chat" + assert safety_common.request_type(safety_common.COMPLETION_SPAN_NAME) == "completion" + + assert safety_common.resolve_masked_text("x", None) == ("x", False) + assert safety_common.resolve_masked_text( + "x", + SafetyResult(text="x", findings=(), overall_action=SafetyDecision.MASK.value), + ) == ("x", False) + assert safety_common.resolve_masked_text( + "x", + SafetyResult(text="y", findings=(), overall_action=SafetyDecision.ALLOW.value), + ) == ("x", False) + assert safety_common.resolve_masked_text( + "x", + SafetyResult(text="y", findings=(), overall_action=SafetyDecision.MASK.value), + ) == ("y", True) + + +def test_openai_safety_helpers_fail_open_on_internal_error(monkeypatch): + monkeypatch.setattr(chat_safety, "get_object_value", lambda *args, **kwargs: (_ for _ in ()).throw(RuntimeError("boom"))) + kwargs = {"messages": [{"role": "user", "content": "secret"}]} + assert chat_safety._apply_prompt_safety(None, kwargs) is kwargs + + monkeypatch.setattr(completion_safety, "get_object_value", lambda *args, **kwargs: (_ for _ in ()).throw(RuntimeError("boom"))) + response = SimpleNamespace(choices=[SimpleNamespace(text="secret")]) + assert completion_safety._apply_completion_safety(None, response) is None diff --git a/packages/opentelemetry-instrumentation-openai/tests/traces/test_safety_hooks.py b/packages/opentelemetry-instrumentation-openai/tests/traces/test_safety_hooks.py new file mode 100644 index 0000000000..be3d240325 --- /dev/null +++ b/packages/opentelemetry-instrumentation-openai/tests/traces/test_safety_hooks.py @@ -0,0 +1,138 @@ +from types import SimpleNamespace + +import pytest + +from opentelemetry.instrumentation.fortifyroot import ( + SafetyFinding, + SafetyLocation, + SafetyResult, + clear_safety_handlers, + register_completion_safety_handler, + register_prompt_safety_handler, +) +from opentelemetry.instrumentation.openai.shared.chat_wrappers import ( + _handle_request as _handle_chat_request, + _apply_prompt_safety, +) +from opentelemetry.instrumentation.openai.shared.completion_wrappers import ( + _apply_completion_safety, +) +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import SimpleSpanProcessor +from opentelemetry.sdk.trace.export.in_memory_span_exporter import ( + InMemorySpanExporter, +) + +pytestmark = pytest.mark.safety + + +def setup_function(): + clear_safety_handlers() + + +def teardown_function(): + clear_safety_handlers() + + +def _test_span(): + exporter = InMemorySpanExporter() + provider = TracerProvider() + provider.add_span_processor(SimpleSpanProcessor(exporter)) + tracer = provider.get_tracer(__name__) + return exporter, tracer + + +def test_chat_prompt_safety_masks_message_content_without_mutating_input(): + exporter, tracer = _test_span() + register_prompt_safety_handler( + lambda context: SafetyResult( + text="[PII.email]", + overall_action="MASK", + findings=[ + SafetyFinding( + category="PII", + severity="HIGH", + action="MASK", + rule_name="PII.email", + start=0, + end=len(context.text), + ) + ], + ) + if context.location == SafetyLocation.PROMPT and context.text == "secret" + else None + ) + + kwargs = {"messages": [{"role": "user", "content": "secret"}]} + with tracer.start_as_current_span("openai.chat") as span: + updated_kwargs = _apply_prompt_safety(span, kwargs) + + assert kwargs["messages"][0]["content"] == "secret" + assert updated_kwargs["messages"][0]["content"] == "[PII.email]" + spans = exporter.get_finished_spans() + assert len(spans[0].events) == 1 + assert spans[0].events[0].attributes["fortifyroot.safety.location"] == SafetyLocation.PROMPT.value + + +@pytest.mark.asyncio +async def test_chat_prompt_safety_masks_span_prompt_attributes(): + exporter, tracer = _test_span() + register_prompt_safety_handler( + lambda context: SafetyResult( + text="[PII.email]", + overall_action="MASK", + findings=[ + SafetyFinding( + category="PII", + severity="HIGH", + action="MASK", + rule_name="PII.email", + start=0, + end=len(context.text), + ) + ], + ) + if context.location == SafetyLocation.PROMPT and context.text == "secret" + else None + ) + + kwargs = {"messages": [{"role": "user", "content": "secret"}]} + instance = SimpleNamespace(_client=None) + with tracer.start_as_current_span("openai.chat") as span: + updated_kwargs = _apply_prompt_safety(span, kwargs) + await _handle_chat_request(span, updated_kwargs, instance) + + spans = exporter.get_finished_spans() + assert spans[0].attributes["gen_ai.prompt.0.content"] == "[PII.email]" + assert spans[0].attributes["gen_ai.prompt.0.role"] == "user" + + +def test_completion_response_safety_masks_choice_text(): + exporter, tracer = _test_span() + register_completion_safety_handler( + lambda context: SafetyResult( + text="[SECRET.api_key]", + overall_action="MASK", + findings=[ + SafetyFinding( + category="SECRET", + severity="HIGH", + action="MASK", + rule_name="SECRET.api_key", + start=0, + end=len(context.text), + ) + ], + ) + if context.location == SafetyLocation.COMPLETION and context.text == "secret" + else None + ) + + response = SimpleNamespace(choices=[SimpleNamespace(text="secret")]) + with tracer.start_as_current_span("openai.completion") as span: + _apply_completion_safety(span, response) + + assert response.choices[0].text == "[SECRET.api_key]" + spans = exporter.get_finished_spans() + assert len(spans[0].events) == 1 + assert spans[0].events[0].attributes["fortifyroot.safety.action"] == "MASK" diff --git a/scripts/run-tests.sh b/scripts/run-tests.sh new file mode 100755 index 0000000000..76325b383e --- /dev/null +++ b/scripts/run-tests.sh @@ -0,0 +1,503 @@ +#!/usr/bin/env bash + +# Examples: +# Run every discovered package test suite: +# ./scripts/run-tests.sh +# Run only tests marked with `@pytest.mark.safety` across matching packages: +# ./scripts/run-tests.sh --safety +# Run all tests for packages whose basename matches a glob: +# ./scripts/run-tests.sh --package '*openai*' +# Run only tests marked with `@pytest.mark.safety` for a selected package glob: +# ./scripts/run-tests.sh --safety --package '*anthropic*' +# List all discovered packages without running tests: +# ./scripts/run-tests.sh --list +# Forward extra pytest arguments after `--`: +# ./scripts/run-tests.sh --safety -- -x + +set -euo pipefail + +ROOT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)" +PACKAGES_DIR="$ROOT_DIR/packages" +VENV_DIR="$ROOT_DIR/.venv" +REPORTS_ROOT="$ROOT_DIR/reports/test-run" +TIMESTAMP="$(date +"%Y%m%d-%H%M%S")" +REPORT_DIR="$REPORTS_ROOT/$TIMESTAMP" +STATE_DIR="$REPORT_DIR/state" + +MODE="all" +PACKAGE_FILTER="" +LIST_ONLY=0 +PYTEST_ARGS=() + +INSTALLED_PACKAGES=() +PACKAGE_NAMES=() + +usage() { + cat <<'EOF' +Usage: scripts/run-tests.sh [options] [-- ] + +Options: + --safety Run only tests marked with @pytest.mark.safety + --package Restrict packages by basename glob, e.g. "*openai*" + --list List discovered packages and exit + -h, --help Show this help + +Examples: + scripts/run-tests.sh + scripts/run-tests.sh --safety + scripts/run-tests.sh --package "*openai*" + scripts/run-tests.sh --safety -- -x +EOF +} + +log() { + printf '[run-tests] %s\n' "$*" +} + +die() { + printf '[run-tests] ERROR: %s\n' "$*" >&2 + exit 1 +} + +require_cmd() { + command -v "$1" >/dev/null 2>&1 || die "Missing required command: $1" +} + +state_file() { + local package_name="$1" + local field="$2" + local package_key + package_key="$(printf '%s' "$package_name" | tr '/ ' '__')" + printf '%s/%s.%s' "$STATE_DIR" "$package_key" "$field" +} + +set_state() { + local package_name="$1" + local field="$2" + local value="${3:-}" + printf '%s' "$value" > "$(state_file "$package_name" "$field")" +} + +get_state() { + local package_name="$1" + local field="$2" + local file_path + file_path="$(state_file "$package_name" "$field")" + if [[ -f "$file_path" ]]; then + cat "$file_path" + fi +} + +mark_package_seen() { + local package_name="$1" + local seen + for seen in "${PACKAGE_NAMES[@]:-}"; do + if [[ "$seen" == "$package_name" ]]; then + return 0 + fi + done + PACKAGE_NAMES+=("$package_name") +} + +bootstrap_venv() { + require_cmd python3 + require_cmd poetry + + if [[ ! -x "$VENV_DIR/bin/python" ]]; then + log "Creating shared virtualenv at $VENV_DIR" + python3 -m venv "$VENV_DIR" + fi + + # shellcheck disable=SC1091 + source "$VENV_DIR/bin/activate" + export POETRY_VIRTUALENVS_CREATE=false + export PIP_DISABLE_PIP_VERSION_CHECK=1 + + log "Bootstrapping shared virtualenv" + python -m pip install --upgrade pip setuptools wheel >/dev/null +} + +discover_packages() { + find "$PACKAGES_DIR" -mindepth 2 -maxdepth 2 -type f -name pyproject.toml -print \ + | sed 's#/pyproject.toml$##' \ + | sort +} + +matches_package_filter() { + local package_dir="$1" + local package_name + package_name="$(basename "$package_dir")" + + if [[ -z "$PACKAGE_FILTER" ]]; then + return 0 + fi + + [[ "$package_name" == $PACKAGE_FILTER ]] +} + +has_tests_directory() { + local package_dir="$1" + [[ -d "$package_dir/tests" ]] +} + +is_candidate_for_safety_mode() { + local package_dir="$1" + + has_tests_directory "$package_dir" || return 1 + + if command -v rg >/dev/null 2>&1; then + rg -l 'pytest\.mark\.safety' "$package_dir/tests" >/dev/null 2>&1 + return $? + fi + + grep -RIl -E 'pytest\.mark\.safety' "$package_dir/tests" >/dev/null 2>&1 +} + +python_meta() { + local package_dir="$1" + local key="$2" + + python3 - "$package_dir" "$key" <<'PY' +import json +import pathlib +import sys +import tomllib + +package_dir = pathlib.Path(sys.argv[1]).resolve() +key = sys.argv[2] +data = tomllib.loads((package_dir / "pyproject.toml").read_text()) +project = data.get("project", {}) +groups = data.get("dependency-groups", {}) +optional = project.get("optional-dependencies", {}) +uv_sources = (((data.get("tool") or {}).get("uv") or {}).get("sources") or {}) + +if key == "name": + print(project.get("name", package_dir.name)) +elif key == "install_target": + extras = [] + if "instruments" in optional: + extras.append("instruments") + suffix = f"[{','.join(extras)}]" if extras else "" + print(f".{suffix}") +elif key == "local_paths": + for source in uv_sources.values(): + if isinstance(source, dict) and "path" in source: + print((package_dir / source["path"]).resolve()) +elif key == "test_deps": + for dep in groups.get("test", []): + print(dep) +PY +} + +install_package() { + local package_dir="$1" + local owner_package_name="${2:-}" + local package_name + package_name="$(python_meta "$package_dir" name)" + if [[ -z "$owner_package_name" ]]; then + owner_package_name="$package_name" + fi + + local installed + for installed in "${INSTALLED_PACKAGES[@]:-}"; do + if [[ "$installed" == "$package_dir" ]]; then + return 0 + fi + done + + if [[ -f "$(state_file "$owner_package_name" "status")" ]] && [[ "$(get_state "$owner_package_name" "status")" == "INSTALL_FAIL" ]]; then + return 0 + fi + + local local_dep + while IFS= read -r local_dep; do + [[ -n "$local_dep" ]] || continue + install_package "$local_dep" "$owner_package_name" + done < <(python_meta "$package_dir" local_paths) + + log "Installing dependencies for $package_name" + + local install_target + install_target="$(python_meta "$package_dir" install_target)" + + local -a pip_args + pip_args=(-e "$install_target") + while IFS= read -r dep; do + [[ -n "$dep" ]] || continue + pip_args+=("$dep") + done < <(python_meta "$package_dir" test_deps) + + local install_log="$REPORT_DIR/${package_name}.install.log" + set +e + ( + cd "$package_dir" + poetry run python -m pip install "${pip_args[@]}" + ) > >(tee "$install_log") 2>&1 + local install_status=$? + set -e + + if [[ $install_status -ne 0 ]]; then + set_state "$owner_package_name" "status" "INSTALL_FAIL" + set_state "$owner_package_name" "reason" "Dependency installation failed while preparing $package_name. See $install_log" + return 1 + fi + + INSTALLED_PACKAGES+=("$package_dir") +} + +parse_junit_summary() { + local junit_xml="$1" + + python3 - "$junit_xml" <<'PY' +import json +import pathlib +import sys +import xml.etree.ElementTree as ET + +xml_path = pathlib.Path(sys.argv[1]) +summary = { + "tests": 0, + "passed": 0, + "failed": 0, + "errors": 0, + "skipped": 0, + "failing_tests": [], +} + +if not xml_path.exists(): + print(json.dumps(summary)) + raise SystemExit(0) + +root = ET.parse(xml_path).getroot() +tests = list(root.iter("testcase")) +summary["tests"] = len(tests) + +for case in tests: + is_failed = case.find("failure") is not None + is_error = case.find("error") is not None + is_skipped = case.find("skipped") is not None + if is_failed or is_error: + summary["failed"] += int(is_failed) + summary["errors"] += int(is_error) + classname = case.attrib.get("classname", "").strip() + name = case.attrib.get("name", "").strip() + if classname and name: + summary["failing_tests"].append(f"{classname}::{name}") + else: + summary["failing_tests"].append(name or classname or "") + elif is_skipped: + summary["skipped"] += 1 + +summary["passed"] = summary["tests"] - summary["failed"] - summary["errors"] - summary["skipped"] +print(json.dumps(summary)) +PY +} + +record_test_summary() { + local package_name="$1" + local junit_xml="$2" + + local summary_json + summary_json="$(parse_junit_summary "$junit_xml")" + + set_state "$package_name" "counts" "$(python3 - "$summary_json" <<'PY' +import json +import sys +data = json.loads(sys.argv[1]) +print(f"passed={data['passed']} failed={data['failed']} errors={data['errors']} skipped={data['skipped']} total={data['tests']}") +PY +)" + + set_state "$package_name" "failures" "$(python3 - "$summary_json" <<'PY' +import json +import sys +data = json.loads(sys.argv[1]) +print("\n".join(data["failing_tests"])) +PY +)" +} + +run_package_tests() { + local package_dir="$1" + local package_name + package_name="$(python_meta "$package_dir" name)" + mark_package_seen "$package_name" + + if ! has_tests_directory "$package_dir"; then + set_state "$package_name" "status" "SKIP" + set_state "$package_name" "reason" "No tests directory" + return 0 + fi + + if [[ "$MODE" == "safety" ]] && ! is_candidate_for_safety_mode "$package_dir"; then + set_state "$package_name" "status" "SKIP" + set_state "$package_name" "reason" "No safety-related tests detected" + return 0 + fi + + if ! install_package "$package_dir" "$package_name"; then + return 0 + fi + + local junit_xml="$REPORT_DIR/${package_name}.xml" + local test_log="$REPORT_DIR/${package_name}.test.log" + local -a pytest_cmd + pytest_cmd=(poetry run pytest -q --junitxml "$junit_xml" tests) + if [[ "$MODE" == "safety" ]]; then + pytest_cmd+=(-m safety) + fi + pytest_cmd+=("${PYTEST_ARGS[@]:-}") + + log "Running tests for $package_name" + set +e + ( + cd "$package_dir" + "${pytest_cmd[@]}" + ) > >(tee "$test_log") 2>&1 + local test_status=$? + set -e + + if [[ $test_status -eq 5 ]]; then + set_state "$package_name" "status" "SKIP" + set_state "$package_name" "reason" "Pytest collected no matching tests" + return 0 + fi + + record_test_summary "$package_name" "$junit_xml" + + if [[ $test_status -eq 0 ]]; then + set_state "$package_name" "status" "PASS" + else + set_state "$package_name" "status" "FAIL" + set_state "$package_name" "reason" "Test failures detected. See $test_log" + fi +} + +print_report() { + local total=0 + local passed=0 + local failed=0 + local skipped=0 + local install_failed=0 + local package_name + local -a failing_lines=() + + printf '\n=== Consolidated Test Report ===\n' + printf 'Mode: %s\n' "$MODE" + printf 'Reports: %s\n\n' "$REPORT_DIR" + + for package_name in "${PACKAGE_NAMES[@]:-}"; do + total=$((total + 1)) + case "$(get_state "$package_name" "status")" in + PASS) + passed=$((passed + 1)) + printf 'PASS %-50s %s\n' "$package_name" "$(get_state "$package_name" "counts")" + ;; + FAIL) + failed=$((failed + 1)) + printf 'FAIL %-50s %s\n' "$package_name" "$(get_state "$package_name" "counts")" + if [[ -n "$(get_state "$package_name" "failures")" ]]; then + while IFS= read -r failing_test; do + [[ -n "$failing_test" ]] || continue + failing_lines+=("$package_name :: $failing_test") + done <<< "$(get_state "$package_name" "failures")" + fi + ;; + INSTALL_FAIL) + install_failed=$((install_failed + 1)) + printf 'ERROR %-50s %s\n' "$package_name" "$(get_state "$package_name" "reason")" + failing_lines+=("$package_name :: [install] $(get_state "$package_name" "reason")") + ;; + *) + skipped=$((skipped + 1)) + printf 'SKIP %-50s %s\n' "$package_name" "$(get_state "$package_name" "reason")" + ;; + esac + done + + printf '\nPackages: total=%d passed=%d failed=%d install_failed=%d skipped=%d\n' \ + "$total" "$passed" "$failed" "$install_failed" "$skipped" + + if [[ ${#failing_lines[@]} -gt 0 ]]; then + printf '\nFailing tests:\n' + printf ' - %s\n' "${failing_lines[@]}" + else + printf '\nFailing tests:\n' + printf ' - none\n' + fi + + printf '\n' + + if (( failed > 0 || install_failed > 0 )); then + return 1 + fi + return 0 +} + +main() { + while [[ $# -gt 0 ]]; do + case "$1" in + --safety) + MODE="safety" + shift + ;; + --package) + [[ $# -ge 2 ]] || die "--package requires a glob argument" + PACKAGE_FILTER="$2" + shift 2 + ;; + --list) + LIST_ONLY=1 + shift + ;; + -h|--help) + usage + exit 0 + ;; + --) + shift + PYTEST_ARGS=("$@") + break + ;; + *) + die "Unknown argument: $1" + ;; + esac + done + + mkdir -p "$REPORT_DIR" + mkdir -p "$STATE_DIR" + bootstrap_venv + + local -a package_dirs=() + local package_dir + while IFS= read -r package_dir; do + [[ -n "$package_dir" ]] || continue + if matches_package_filter "$package_dir"; then + package_dirs+=("$package_dir") + fi + done < <(discover_packages) + + (( ${#package_dirs[@]} > 0 )) || die "No packages matched the requested filters" + + if (( LIST_ONLY == 1 )); then + printf '%s\n' "${package_dirs[@]}" + exit 0 + fi + + local package_name + for package_dir in "${package_dirs[@]}"; do + package_name="$(python_meta "$package_dir" name)" + mark_package_seen "$package_name" + set_state "$package_name" "status" "SKIP" + set_state "$package_name" "reason" "Not executed" + done + + for package_dir in "${package_dirs[@]}"; do + run_package_tests "$package_dir" + done + + print_report +} + +main "$@"