From 277475efc4462f4b83acdf3a2c5fea36312b9eff Mon Sep 17 00:00:00 2001 From: Manasjyoti Sharma Date: Wed, 1 Apr 2026 16:57:59 +0530 Subject: [PATCH] feat(safety): add VCR safety cassettes for Google GenAI and Bedrock (T3) Added FR-owned safety cassette tests for Google GenAI and AWS Bedrock instrumentation packages. Each package gets 9 tests covering all 3 safety rule types (RegEx, List, UDF), both MASK and ALLOW actions, prompt + completion + streaming safety, recorded against real APIs. Google GenAI: gemini-2.5-flash via Google AI Studio. Bedrock: anthropic.claude-3-haiku via AWS Bedrock Converse API --- ...k_safety_allow_credit_card_not_masked.yaml | 49 ++ ...bedrock_safety_allow_email_not_masked.yaml | 48 ++ ..._bedrock_safety_combined_rules_prompt.yaml | 49 ++ ...edrock_safety_completion_email_masked.yaml | 48 ++ ...ock_safety_list_competitor_org_prompt.yaml | 49 ++ ...drock_safety_regex_credit_card_prompt.yaml | 48 ++ ...est_bedrock_safety_regex_email_prompt.yaml | 49 ++ ...est_bedrock_safety_streaming_with_pii.yaml | 120 ++++ ...k_safety_udf_project_codenames_prompt.yaml | 48 ++ .../tests/traces/test_safety_cassettes.py | 650 ++++++++++++++++++ ...i_safety_allow_credit_card_not_masked.yaml | 63 ++ ...e_genai_safety_allow_email_not_masked.yaml | 64 ++ ...le_genai_safety_combined_rules_prompt.yaml | 63 ++ ..._genai_safety_completion_email_masked.yaml | 65 ++ ...nai_safety_list_competitor_org_prompt.yaml | 64 ++ ...genai_safety_regex_credit_card_prompt.yaml | 64 ++ ...oogle_genai_safety_regex_email_prompt.yaml | 63 ++ ...oogle_genai_safety_streaming_with_pii.yaml | 60 ++ ...i_safety_udf_project_codenames_prompt.yaml | 64 ++ .../tests/test_safety_cassettes.py | 544 +++++++++++++++ 20 files changed, 2272 insertions(+) create mode 100644 packages/opentelemetry-instrumentation-bedrock/tests/traces/cassettes/test_safety_cassettes/test_bedrock_safety_allow_credit_card_not_masked.yaml create mode 100644 packages/opentelemetry-instrumentation-bedrock/tests/traces/cassettes/test_safety_cassettes/test_bedrock_safety_allow_email_not_masked.yaml create mode 100644 packages/opentelemetry-instrumentation-bedrock/tests/traces/cassettes/test_safety_cassettes/test_bedrock_safety_combined_rules_prompt.yaml create mode 100644 packages/opentelemetry-instrumentation-bedrock/tests/traces/cassettes/test_safety_cassettes/test_bedrock_safety_completion_email_masked.yaml create mode 100644 packages/opentelemetry-instrumentation-bedrock/tests/traces/cassettes/test_safety_cassettes/test_bedrock_safety_list_competitor_org_prompt.yaml create mode 100644 packages/opentelemetry-instrumentation-bedrock/tests/traces/cassettes/test_safety_cassettes/test_bedrock_safety_regex_credit_card_prompt.yaml create mode 100644 packages/opentelemetry-instrumentation-bedrock/tests/traces/cassettes/test_safety_cassettes/test_bedrock_safety_regex_email_prompt.yaml create mode 100644 packages/opentelemetry-instrumentation-bedrock/tests/traces/cassettes/test_safety_cassettes/test_bedrock_safety_streaming_with_pii.yaml create mode 100644 packages/opentelemetry-instrumentation-bedrock/tests/traces/cassettes/test_safety_cassettes/test_bedrock_safety_udf_project_codenames_prompt.yaml create mode 100644 packages/opentelemetry-instrumentation-bedrock/tests/traces/test_safety_cassettes.py create mode 100644 packages/opentelemetry-instrumentation-google-generativeai/tests/cassettes/test_safety_cassettes/test_google_genai_safety_allow_credit_card_not_masked.yaml create mode 100644 packages/opentelemetry-instrumentation-google-generativeai/tests/cassettes/test_safety_cassettes/test_google_genai_safety_allow_email_not_masked.yaml create mode 100644 packages/opentelemetry-instrumentation-google-generativeai/tests/cassettes/test_safety_cassettes/test_google_genai_safety_combined_rules_prompt.yaml create mode 100644 packages/opentelemetry-instrumentation-google-generativeai/tests/cassettes/test_safety_cassettes/test_google_genai_safety_completion_email_masked.yaml create mode 100644 packages/opentelemetry-instrumentation-google-generativeai/tests/cassettes/test_safety_cassettes/test_google_genai_safety_list_competitor_org_prompt.yaml create mode 100644 packages/opentelemetry-instrumentation-google-generativeai/tests/cassettes/test_safety_cassettes/test_google_genai_safety_regex_credit_card_prompt.yaml create mode 100644 packages/opentelemetry-instrumentation-google-generativeai/tests/cassettes/test_safety_cassettes/test_google_genai_safety_regex_email_prompt.yaml create mode 100644 packages/opentelemetry-instrumentation-google-generativeai/tests/cassettes/test_safety_cassettes/test_google_genai_safety_streaming_with_pii.yaml create mode 100644 packages/opentelemetry-instrumentation-google-generativeai/tests/cassettes/test_safety_cassettes/test_google_genai_safety_udf_project_codenames_prompt.yaml create mode 100644 packages/opentelemetry-instrumentation-google-generativeai/tests/test_safety_cassettes.py diff --git a/packages/opentelemetry-instrumentation-bedrock/tests/traces/cassettes/test_safety_cassettes/test_bedrock_safety_allow_credit_card_not_masked.yaml b/packages/opentelemetry-instrumentation-bedrock/tests/traces/cassettes/test_safety_cassettes/test_bedrock_safety_allow_credit_card_not_masked.yaml new file mode 100644 index 0000000000..48f867981b --- /dev/null +++ b/packages/opentelemetry-instrumentation-bedrock/tests/traces/cassettes/test_safety_cassettes/test_bedrock_safety_allow_credit_card_not_masked.yaml @@ -0,0 +1,49 @@ +interactions: +- request: + body: '{"messages": [{"role": "user", "content": [{"text": "Verify payment for + card 4111 1111 1111 1111 was received."}]}], "inferenceConfig": {"maxTokens": + 50}}' + headers: + Content-Length: + - '154' + Content-Type: + - !!binary | + YXBwbGljYXRpb24vanNvbg== + User-Agent: + - !!binary | + Qm90bzMvMS40Mi4yOCBtZC9Cb3RvY29yZSMxLjQyLjI4IHVhLzIuMSBvcy9tYWNvcyMyNC42LjAg + bWQvYXJjaCN4ODZfNjQgbGFuZy9weXRob24jMy4xMC4xOSBtZC9weWltcGwjQ1B5dGhvbiBtL1os + ZSxiLEQgY2ZnL3JldHJ5LW1vZGUjbGVnYWN5IEJvdG9jb3JlLzEuNDIuMjg= + X-Amz-Date: + - !!binary | + MjAyNjA0MDFUMDYxNzQzWg== + amz-sdk-invocation-id: + - !!binary | + OGZhNGRmMmItODk4MC00Zjk3LTliN2YtZTRhMDM4YTIwYzkx + amz-sdk-request: + - !!binary | + YXR0ZW1wdD0x + method: POST + uri: https://bedrock-runtime.us-east-1.amazonaws.com/model/anthropic.claude-3-haiku-20240307-v1%3A0/converse + response: + body: + string: '{"metrics":{"latencyMs":1244},"output":{"message":{"content":[{"text":"I + apologize, but I do not have the ability to verify or access payment information + for specific credit card numbers. As an AI assistant without direct access + to financial systems, I don''t have a way to confirm payment details. I can + only provide general"}],"role":"assistant"}},"stopReason":"max_tokens","usage":{"inputTokens":28,"outputTokens":50,"serverToolUsage":{},"totalTokens":78}}' + headers: + Connection: + - keep-alive + Content-Length: + - '458' + Content-Type: + - application/json + Date: + - Wed, 01 Apr 2026 06:17:45 GMT + x-amzn-RequestId: + - cd677757-8407-47a8-9508-4490564060f0 + status: + code: 200 + message: OK +version: 1 diff --git a/packages/opentelemetry-instrumentation-bedrock/tests/traces/cassettes/test_safety_cassettes/test_bedrock_safety_allow_email_not_masked.yaml b/packages/opentelemetry-instrumentation-bedrock/tests/traces/cassettes/test_safety_cassettes/test_bedrock_safety_allow_email_not_masked.yaml new file mode 100644 index 0000000000..1b34afadbd --- /dev/null +++ b/packages/opentelemetry-instrumentation-bedrock/tests/traces/cassettes/test_safety_cassettes/test_bedrock_safety_allow_email_not_masked.yaml @@ -0,0 +1,48 @@ +interactions: +- request: + body: '{"messages": [{"role": "user", "content": [{"text": "Forward this message + from alice.jones@example.org to the support team."}]}], "inferenceConfig": {"maxTokens": + 50}}' + headers: + Content-Length: + - '167' + Content-Type: + - !!binary | + YXBwbGljYXRpb24vanNvbg== + User-Agent: + - !!binary | + Qm90bzMvMS40Mi4yOCBtZC9Cb3RvY29yZSMxLjQyLjI4IHVhLzIuMSBvcy9tYWNvcyMyNC42LjAg + bWQvYXJjaCN4ODZfNjQgbGFuZy9weXRob24jMy4xMC4xOSBtZC9weWltcGwjQ1B5dGhvbiBtL1os + ZSxiLEQgY2ZnL3JldHJ5LW1vZGUjbGVnYWN5IEJvdG9jb3JlLzEuNDIuMjg= + X-Amz-Date: + - !!binary | + MjAyNjA0MDFUMDYxNzQwWg== + amz-sdk-invocation-id: + - !!binary | + MjU2MmUzZTQtYmZhYS00N2U0LWI5ZjEtMDFhMTkxM2RlNjFj + amz-sdk-request: + - !!binary | + YXR0ZW1wdD0x + method: POST + uri: https://bedrock-runtime.us-east-1.amazonaws.com/model/anthropic.claude-3-haiku-20240307-v1%3A0/converse + response: + body: + string: '{"metrics":{"latencyMs":1587},"output":{"message":{"content":[{"text":"I''m + afraid I can''t directly forward messages on your behalf. As an AI assistant, + I don''t have the capability to send emails from specific email addresses. + However, I can suggest a few alternatives:\n\n1. If you have access to the"}],"role":"assistant"}},"stopReason":"max_tokens","usage":{"inputTokens":23,"outputTokens":50,"serverToolUsage":{},"totalTokens":73}}' + headers: + Connection: + - keep-alive + Content-Length: + - '434' + Content-Type: + - application/json + Date: + - Wed, 01 Apr 2026 06:17:43 GMT + x-amzn-RequestId: + - 56308014-0ead-4413-9477-591101f3869b + status: + code: 200 + message: OK +version: 1 diff --git a/packages/opentelemetry-instrumentation-bedrock/tests/traces/cassettes/test_safety_cassettes/test_bedrock_safety_combined_rules_prompt.yaml b/packages/opentelemetry-instrumentation-bedrock/tests/traces/cassettes/test_safety_cassettes/test_bedrock_safety_combined_rules_prompt.yaml new file mode 100644 index 0000000000..89920167d6 --- /dev/null +++ b/packages/opentelemetry-instrumentation-bedrock/tests/traces/cassettes/test_safety_cassettes/test_bedrock_safety_combined_rules_prompt.yaml @@ -0,0 +1,49 @@ +interactions: +- request: + body: '{"messages": [{"role": "user", "content": [{"text": "Send the [custom_compliance.project-phoenix] + briefing to [PII.email] and CC the [PII.competitor_org] team."}]}], "inferenceConfig": + {"maxTokens": 50}}' + headers: + Content-Length: + - '203' + Content-Type: + - !!binary | + YXBwbGljYXRpb24vanNvbg== + User-Agent: + - !!binary | + Qm90bzMvMS40Mi4yOCBtZC9Cb3RvY29yZSMxLjQyLjI4IHVhLzIuMSBvcy9tYWNvcyMyNC42LjAg + bWQvYXJjaCN4ODZfNjQgbGFuZy9weXRob24jMy4xMC4xOSBtZC9weWltcGwjQ1B5dGhvbiBtL1os + ZSxiLEQgY2ZnL3JldHJ5LW1vZGUjbGVnYWN5IEJvdG9jb3JlLzEuNDIuMjg= + X-Amz-Date: + - !!binary | + MjAyNjA0MDFUMDYxNzM2Wg== + amz-sdk-invocation-id: + - !!binary | + MWY4NDhjYjctOGRmYi00MDQ0LTk4Y2YtNDFkNGI4OGY3NWM3 + amz-sdk-request: + - !!binary | + YXR0ZW1wdD0x + method: POST + uri: https://bedrock-runtime.us-east-1.amazonaws.com/model/anthropic.claude-3-haiku-20240307-v1%3A0/converse + response: + body: + string: '{"metrics":{"latencyMs":1735},"output":{"message":{"content":[{"text":"I + apologize, but I cannot send confidential information or personal data without + proper authorization. Disclosing sensitive details or private email addresses + would be a violation of privacy and data protection policies. Perhaps we could + discuss a more appropriate way to share information"}],"role":"assistant"}},"stopReason":"max_tokens","usage":{"inputTokens":40,"outputTokens":50,"serverToolUsage":{},"totalTokens":90}}' + headers: + Connection: + - keep-alive + Content-Length: + - '494' + Content-Type: + - application/json + Date: + - Wed, 01 Apr 2026 06:17:38 GMT + x-amzn-RequestId: + - 1f48c443-ea1c-4cbc-bece-e31639740014 + status: + code: 200 + message: OK +version: 1 diff --git a/packages/opentelemetry-instrumentation-bedrock/tests/traces/cassettes/test_safety_cassettes/test_bedrock_safety_completion_email_masked.yaml b/packages/opentelemetry-instrumentation-bedrock/tests/traces/cassettes/test_safety_cassettes/test_bedrock_safety_completion_email_masked.yaml new file mode 100644 index 0000000000..1f877e291c --- /dev/null +++ b/packages/opentelemetry-instrumentation-bedrock/tests/traces/cassettes/test_safety_cassettes/test_bedrock_safety_completion_email_masked.yaml @@ -0,0 +1,48 @@ +interactions: +- request: + body: '{"messages": [{"role": "user", "content": [{"text": "Create a fictional + contact card. The person''s name is Jane Smith, company is Widgets Inc, role + is VP of Sales. Invent a plausible work email and phone. Format as:\nName: ...\nEmail: + ...\nPhone: ..."}]}], "inferenceConfig": {"maxTokens": 100}}' + headers: + Content-Length: + - '295' + Content-Type: + - !!binary | + YXBwbGljYXRpb24vanNvbg== + User-Agent: + - !!binary | + Qm90bzMvMS40Mi4yOCBtZC9Cb3RvY29yZSMxLjQyLjI4IHVhLzIuMSBvcy9tYWNvcyMyNC42LjAg + bWQvYXJjaCN4ODZfNjQgbGFuZy9weXRob24jMy4xMC4xOSBtZC9weWltcGwjQ1B5dGhvbiBtL1os + ZSxiLEQgY2ZnL3JldHJ5LW1vZGUjbGVnYWN5IEJvdG9jb3JlLzEuNDIuMjg= + X-Amz-Date: + - !!binary | + MjAyNjA0MDFUMDYxNzM0Wg== + amz-sdk-invocation-id: + - !!binary | + OWU2NDc0NmQtZDE0My00ZGY0LWI5YTgtZmY2ODc0NGM4ODJi + amz-sdk-request: + - !!binary | + YXR0ZW1wdD0x + method: POST + uri: https://bedrock-runtime.us-east-1.amazonaws.com/model/anthropic.claude-3-haiku-20240307-v1%3A0/converse + response: + body: + string: '{"metrics":{"latencyMs":1040},"output":{"message":{"content":[{"text":"Name: + Jane Smith\nEmail: jsmith@widgets-inc.com\nPhone: (555) 123-4567\nCompany: + Widgets Inc.\nRole: VP of Sales"}],"role":"assistant"}},"stopReason":"end_turn","usage":{"inputTokens":60,"outputTokens":42,"serverToolUsage":{},"totalTokens":102}}' + headers: + Connection: + - keep-alive + Content-Length: + - '316' + Content-Type: + - application/json + Date: + - Wed, 01 Apr 2026 06:17:36 GMT + x-amzn-RequestId: + - 9dc1df64-dd0b-4ca2-8d42-7fae9be151c6 + status: + code: 200 + message: OK +version: 1 diff --git a/packages/opentelemetry-instrumentation-bedrock/tests/traces/cassettes/test_safety_cassettes/test_bedrock_safety_list_competitor_org_prompt.yaml b/packages/opentelemetry-instrumentation-bedrock/tests/traces/cassettes/test_safety_cassettes/test_bedrock_safety_list_competitor_org_prompt.yaml new file mode 100644 index 0000000000..87dd79f3ec --- /dev/null +++ b/packages/opentelemetry-instrumentation-bedrock/tests/traces/cassettes/test_safety_cassettes/test_bedrock_safety_list_competitor_org_prompt.yaml @@ -0,0 +1,49 @@ +interactions: +- request: + body: '{"messages": [{"role": "user", "content": [{"text": "Compare our product + roadmap with [PII.competitor_org]''s latest offering and [PII.competitor_org]'' + pricing strategy."}]}], "inferenceConfig": {"maxTokens": 50}}' + headers: + Content-Length: + - '212' + Content-Type: + - !!binary | + YXBwbGljYXRpb24vanNvbg== + User-Agent: + - !!binary | + Qm90bzMvMS40Mi4yOCBtZC9Cb3RvY29yZSMxLjQyLjI4IHVhLzIuMSBvcy9tYWNvcyMyNC42LjAg + bWQvYXJjaCN4ODZfNjQgbGFuZy9weXRob24jMy4xMC4xOSBtZC9weWltcGwjQ1B5dGhvbiBtL1os + ZSxiLEQgY2ZnL3JldHJ5LW1vZGUjbGVnYWN5IEJvdG9jb3JlLzEuNDIuMjg= + X-Amz-Date: + - !!binary | + MjAyNjA0MDFUMDYxNzI5Wg== + amz-sdk-invocation-id: + - !!binary | + ZGMzOWI3YjQtOTE2Yi00ZDlkLTg4YzYtYzhiZjQzNTdkYThi + amz-sdk-request: + - !!binary | + YXR0ZW1wdD0x + method: POST + uri: https://bedrock-runtime.us-east-1.amazonaws.com/model/anthropic.claude-3-haiku-20240307-v1%3A0/converse + response: + body: + string: '{"metrics":{"latencyMs":1613},"output":{"message":{"content":[{"text":"I + apologize, but I cannot provide any comparisons or information related to + competitors'' products, pricing, or roadmaps. That would involve sharing confidential + business information that I do not have access to and should not disclose. + I''m happy"}],"role":"assistant"}},"stopReason":"max_tokens","usage":{"inputTokens":38,"outputTokens":50,"serverToolUsage":{},"totalTokens":88}}' + headers: + Connection: + - keep-alive + Content-Length: + - '450' + Content-Type: + - application/json + Date: + - Wed, 01 Apr 2026 06:17:31 GMT + x-amzn-RequestId: + - ba60004f-68af-48dc-bd94-65a2098d7f14 + status: + code: 200 + message: OK +version: 1 diff --git a/packages/opentelemetry-instrumentation-bedrock/tests/traces/cassettes/test_safety_cassettes/test_bedrock_safety_regex_credit_card_prompt.yaml b/packages/opentelemetry-instrumentation-bedrock/tests/traces/cassettes/test_safety_cassettes/test_bedrock_safety_regex_credit_card_prompt.yaml new file mode 100644 index 0000000000..d74cf8c47a --- /dev/null +++ b/packages/opentelemetry-instrumentation-bedrock/tests/traces/cassettes/test_safety_cassettes/test_bedrock_safety_regex_credit_card_prompt.yaml @@ -0,0 +1,48 @@ +interactions: +- request: + body: '{"messages": [{"role": "user", "content": [{"text": "Check if order for + card [PCI.credit_card] was processed."}]}], "inferenceConfig": {"maxTokens": + 50}}' + headers: + Content-Length: + - '153' + Content-Type: + - !!binary | + YXBwbGljYXRpb24vanNvbg== + User-Agent: + - !!binary | + Qm90bzMvMS40Mi4yOCBtZC9Cb3RvY29yZSMxLjQyLjI4IHVhLzIuMSBvcy9tYWNvcyMyNC42LjAg + bWQvYXJjaCN4ODZfNjQgbGFuZy9weXRob24jMy4xMC4xOSBtZC9weWltcGwjQ1B5dGhvbiBtL1os + ZSxiLEQgY2ZnL3JldHJ5LW1vZGUjbGVnYWN5IEJvdG9jb3JlLzEuNDIuMjg= + X-Amz-Date: + - !!binary | + MjAyNjA0MDFUMDYxNzI3Wg== + amz-sdk-invocation-id: + - !!binary | + OTFkOTk0NmUtM2E0ZS00NDY1LTk3ZjktMGVhZTEwNWU5YTYx + amz-sdk-request: + - !!binary | + YXR0ZW1wdD0x + method: POST + uri: https://bedrock-runtime.us-east-1.amazonaws.com/model/anthropic.claude-3-haiku-20240307-v1%3A0/converse + response: + body: + string: '{"metrics":{"latencyMs":981},"output":{"message":{"content":[{"text":"To + check if an order for a credit card with the PCI.credit_card value was processed, + you would need to have access to the relevant order processing or payment + system data. Without access to that data, I cannot directly verify if a specific"}],"role":"assistant"}},"stopReason":"max_tokens","usage":{"inputTokens":23,"outputTokens":50,"serverToolUsage":{},"totalTokens":73}}' + headers: + Connection: + - keep-alive + Content-Length: + - '443' + Content-Type: + - application/json + Date: + - Wed, 01 Apr 2026 06:17:29 GMT + x-amzn-RequestId: + - cc3fba0c-33b3-416b-bc96-704e1bca6d87 + status: + code: 200 + message: OK +version: 1 diff --git a/packages/opentelemetry-instrumentation-bedrock/tests/traces/cassettes/test_safety_cassettes/test_bedrock_safety_regex_email_prompt.yaml b/packages/opentelemetry-instrumentation-bedrock/tests/traces/cassettes/test_safety_cassettes/test_bedrock_safety_regex_email_prompt.yaml new file mode 100644 index 0000000000..fb2379a6a2 --- /dev/null +++ b/packages/opentelemetry-instrumentation-bedrock/tests/traces/cassettes/test_safety_cassettes/test_bedrock_safety_regex_email_prompt.yaml @@ -0,0 +1,49 @@ +interactions: +- request: + body: '{"messages": [{"role": "user", "content": [{"text": "Summarize the following + customer inquiry from [PII.email] about their recent order."}]}], "inferenceConfig": + {"maxTokens": 50}}' + headers: + Content-Length: + - '180' + Content-Type: + - !!binary | + YXBwbGljYXRpb24vanNvbg== + User-Agent: + - !!binary | + Qm90bzMvMS40Mi4yOCBtZC9Cb3RvY29yZSMxLjQyLjI4IHVhLzIuMSBvcy9tYWNvcyMyNC42LjAg + bWQvYXJjaCN4ODZfNjQgbGFuZy9weXRob24jMy4xMC4xOSBtZC9weWltcGwjQ1B5dGhvbiBtL1os + ZSxiLEQgY2ZnL3JldHJ5LW1vZGUjbGVnYWN5IEJvdG9jb3JlLzEuNDIuMjg= + X-Amz-Date: + - !!binary | + MjAyNjA0MDFUMDYxNzI1Wg== + amz-sdk-invocation-id: + - !!binary | + YTkwYjQzODgtNDQwMy00OGZlLWI3ZjYtOTlmZDA1ZjZmZDUx + amz-sdk-request: + - !!binary | + YXR0ZW1wdD0x + method: POST + uri: https://bedrock-runtime.us-east-1.amazonaws.com/model/anthropic.claude-3-haiku-20240307-v1%3A0/converse + response: + body: + string: '{"metrics":{"latencyMs":999},"output":{"message":{"content":[{"text":"Unfortunately, + I cannot summarize the customer inquiry without seeing the actual details + of the inquiry. As an AI assistant, I do not have access to any real customer + information or inquiries. I can only respond based on the information you + provide to me directly"}],"role":"assistant"}},"stopReason":"max_tokens","usage":{"inputTokens":26,"outputTokens":50,"serverToolUsage":{},"totalTokens":76}}' + headers: + Connection: + - keep-alive + Content-Length: + - '467' + Content-Type: + - application/json + Date: + - Wed, 01 Apr 2026 06:17:27 GMT + x-amzn-RequestId: + - dba8cd0c-ba21-44d6-9a7b-d1988aca9ae5 + status: + code: 200 + message: OK +version: 1 diff --git a/packages/opentelemetry-instrumentation-bedrock/tests/traces/cassettes/test_safety_cassettes/test_bedrock_safety_streaming_with_pii.yaml b/packages/opentelemetry-instrumentation-bedrock/tests/traces/cassettes/test_safety_cassettes/test_bedrock_safety_streaming_with_pii.yaml new file mode 100644 index 0000000000..5b9ce80db0 --- /dev/null +++ b/packages/opentelemetry-instrumentation-bedrock/tests/traces/cassettes/test_safety_cassettes/test_bedrock_safety_streaming_with_pii.yaml @@ -0,0 +1,120 @@ +interactions: +- request: + body: '{"messages": [{"role": "user", "content": [{"text": "Acknowledge receipt + of message from [PII.email] about [custom_compliance.project-phoenix]."}]}], + "inferenceConfig": {"maxTokens": 50}}' + headers: + Content-Length: + - '187' + Content-Type: + - !!binary | + YXBwbGljYXRpb24vanNvbg== + User-Agent: + - !!binary | + Qm90bzMvMS40Mi4yOCBtZC9Cb3RvY29yZSMxLjQyLjI4IHVhLzIuMSBvcy9tYWNvcyMyNC42LjAg + bWQvYXJjaCN4ODZfNjQgbGFuZy9weXRob24jMy4xMC4xOSBtZC9weWltcGwjQ1B5dGhvbiBtL1os + ZSxiLEQgY2ZnL3JldHJ5LW1vZGUjbGVnYWN5IEJvdG9jb3JlLzEuNDIuMjg= + X-Amz-Date: + - !!binary | + MjAyNjA0MDFUMDYxNzM4Wg== + amz-sdk-invocation-id: + - !!binary | + ZjQ3MGU3MjYtMzE2ZC00Y2RlLTlhOGQtOTc2OGQ3NmU0NTc1 + amz-sdk-request: + - !!binary | + YXR0ZW1wdD0x + method: POST + uri: https://bedrock-runtime.us-east-1.amazonaws.com/model/anthropic.claude-3-haiku-20240307-v1%3A0/converse-stream + response: + body: + string: !!binary | + AAAAtAAAAFIFAEFlCzpldmVudC10eXBlBwAMbWVzc2FnZVN0YXJ0DTpjb250ZW50LXR5cGUHABBh + cHBsaWNhdGlvbi9qc29uDTptZXNzYWdlLXR5cGUHAAVldmVudHsicCI6ImFiY2RlZmdoaWprbG1u + b3BxcnN0dXZ3eHl6QUJDREVGR0hJSktMTU5PUFFSU1RVVldYWVowMTIiLCJyb2xlIjoiYXNzaXN0 + YW50In2fX38tAAAAvQAAAFd4etebCzpldmVudC10eXBlBwARY29udGVudEJsb2NrRGVsdGENOmNv + bnRlbnQtdHlwZQcAEGFwcGxpY2F0aW9uL2pzb24NOm1lc3NhZ2UtdHlwZQcABWV2ZW50eyJjb250 + ZW50QmxvY2tJbmRleCI6MCwiZGVsdGEiOnsidGV4dCI6IkkgYWNrbm93bGVkZ2UifSwicCI6ImFi + Y2RlZmdoaWprbG1ub3BxcnN0dXZ3In2hrxRCAAAAtwAAAFcyys86CzpldmVudC10eXBlBwARY29u + dGVudEJsb2NrRGVsdGENOmNvbnRlbnQtdHlwZQcAEGFwcGxpY2F0aW9uL2pzb24NOm1lc3NhZ2Ut + dHlwZQcABWV2ZW50eyJjb250ZW50QmxvY2tJbmRleCI6MCwiZGVsdGEiOnsidGV4dCI6IiByZWNl + aXB0In0sInAiOiJhYmNkZWZnaGlqa2xtbm9wcXJzdHV2In0CvgVzAAAAzAAAAFe8yB7kCzpldmVu + dC10eXBlBwARY29udGVudEJsb2NrRGVsdGENOmNvbnRlbnQtdHlwZQcAEGFwcGxpY2F0aW9uL2pz + b24NOm1lc3NhZ2UtdHlwZQcABWV2ZW50eyJjb250ZW50QmxvY2tJbmRleCI6MCwiZGVsdGEiOnsi + dGV4dCI6IiBvZiB0aGUifSwicCI6ImFiY2RlZmdoaWprbG1ub3BxcnN0dXZ3eHl6QUJDREVGR0hJ + SktMTU5PUFFSIn1GcYzKAAAArgAAAFdfOjrJCzpldmVudC10eXBlBwARY29udGVudEJsb2NrRGVs + dGENOmNvbnRlbnQtdHlwZQcAEGFwcGxpY2F0aW9uL2pzb24NOm1lc3NhZ2UtdHlwZQcABWV2ZW50 + eyJjb250ZW50QmxvY2tJbmRleCI6MCwiZGVsdGEiOnsidGV4dCI6IiBtZXNzYWdlIGZyb20ifSwi + cCI6ImFiY2RlZmdoIn3w1J2mAAAAyAAAAFdJSLgkCzpldmVudC10eXBlBwARY29udGVudEJsb2Nr + RGVsdGENOmNvbnRlbnQtdHlwZQcAEGFwcGxpY2F0aW9uL2pzb24NOm1lc3NhZ2UtdHlwZQcABWV2 + ZW50eyJjb250ZW50QmxvY2tJbmRleCI6MCwiZGVsdGEiOnsidGV4dCI6IiBbUElJLiJ9LCJwIjoi + YWJjZGVmZ2hpamtsbW5vcHFyc3R1dnd4eXpBQkNERUZHSElKS0xNTk8ifa1K24cAAAC3AAAAVzLK + zzoLOmV2ZW50LXR5cGUHABFjb250ZW50QmxvY2tEZWx0YQ06Y29udGVudC10eXBlBwAQYXBwbGlj + YXRpb24vanNvbg06bWVzc2FnZS10eXBlBwAFZXZlbnR7ImNvbnRlbnRCbG9ja0luZGV4IjowLCJk + ZWx0YSI6eyJ0ZXh0IjoiZW1haWxdIn0sInAiOiJhYmNkZWZnaGlqa2xtbm9wcXJzdHV2d3gifcJm + 8mkAAADYAAAAVymoL6YLOmV2ZW50LXR5cGUHABFjb250ZW50QmxvY2tEZWx0YQ06Y29udGVudC10 + eXBlBwAQYXBwbGljYXRpb24vanNvbg06bWVzc2FnZS10eXBlBwAFZXZlbnR7ImNvbnRlbnRCbG9j + a0luZGV4IjowLCJkZWx0YSI6eyJ0ZXh0IjoiIGFib3V0In0sInAiOiJhYmNkZWZnaGlqa2xtbm9w + cXJzdHV2d3h5ekFCQ0RFRkdISUpLTE1OT1BRUlNUVVZXWFlaMDEyMzQifTmUX6YAAADTAAAAV154 + HrcLOmV2ZW50LXR5cGUHABFjb250ZW50QmxvY2tEZWx0YQ06Y29udGVudC10eXBlBwAQYXBwbGlj + YXRpb24vanNvbg06bWVzc2FnZS10eXBlBwAFZXZlbnR7ImNvbnRlbnRCbG9ja0luZGV4IjowLCJk + ZWx0YSI6eyJ0ZXh0IjoiIFtjdXN0b21fY29tcGxpYW5jZSJ9LCJwIjoiYWJjZGVmZ2hpamtsbW5v + cHFyc3R1dnd4eXpBQkNERUZHSElKS0xNIn2V69eyAAAAuwAAAFf3OiI7CzpldmVudC10eXBlBwAR + Y29udGVudEJsb2NrRGVsdGENOmNvbnRlbnQtdHlwZQcAEGFwcGxpY2F0aW9uL2pzb24NOm1lc3Nh + Z2UtdHlwZQcABWV2ZW50eyJjb250ZW50QmxvY2tJbmRleCI6MCwiZGVsdGEiOnsidGV4dCI6Ii5w + cm9qZWN0LXBob2VuaXgifSwicCI6ImFiY2RlZmdoaWprbG1ub3BxciJ9Je0hBAAAAMIAAABXA/ig + hQs6ZXZlbnQtdHlwZQcAEWNvbnRlbnRCbG9ja0RlbHRhDTpjb250ZW50LXR5cGUHABBhcHBsaWNh + dGlvbi9qc29uDTptZXNzYWdlLXR5cGUHAAVldmVudHsiY29udGVudEJsb2NrSW5kZXgiOjAsImRl + bHRhIjp7InRleHQiOiJdLiJ9LCJwIjoiYWJjZGVmZ2hpamtsbW5vcHFyc3R1dnd4eXpBQkNERUZH + SElKS0xNIn2HiQuYAAAAowAAAFenqv54CzpldmVudC10eXBlBwARY29udGVudEJsb2NrRGVsdGEN + OmNvbnRlbnQtdHlwZQcAEGFwcGxpY2F0aW9uL2pzb24NOm1lc3NhZ2UtdHlwZQcABWV2ZW50eyJj + b250ZW50QmxvY2tJbmRleCI6MCwiZGVsdGEiOnsidGV4dCI6IiBJIn0sInAiOiJhYmNkZWZnaCJ9 + n5FnBQAAAKcAAABXUipYuAs6ZXZlbnQtdHlwZQcAEWNvbnRlbnRCbG9ja0RlbHRhDTpjb250ZW50 + LXR5cGUHABBhcHBsaWNhdGlvbi9qc29uDTptZXNzYWdlLXR5cGUHAAVldmVudHsiY29udGVudEJs + b2NrSW5kZXgiOjAsImRlbHRhIjp7InRleHQiOiIgd2lsbCBlbnN1cmUifSwicCI6ImFiIn2yLqFo + AAAAtQAAAFdICpxaCzpldmVudC10eXBlBwARY29udGVudEJsb2NrRGVsdGENOmNvbnRlbnQtdHlw + ZQcAEGFwcGxpY2F0aW9uL2pzb24NOm1lc3NhZ2UtdHlwZQcABWV2ZW50eyJjb250ZW50QmxvY2tJ + bmRleCI6MCwiZGVsdGEiOnsidGV4dCI6IiB0aGlzIn0sInAiOiJhYmNkZWZnaGlqa2xtbm9wcXJz + dHV2dyJ9tDAgwgAAALsAAABX9zoiOws6ZXZlbnQtdHlwZQcAEWNvbnRlbnRCbG9ja0RlbHRhDTpj + b250ZW50LXR5cGUHABBhcHBsaWNhdGlvbi9qc29uDTptZXNzYWdlLXR5cGUHAAVldmVudHsiY29u + dGVudEJsb2NrSW5kZXgiOjAsImRlbHRhIjp7InRleHQiOiIgaW5mb3JtYXRpb24ifSwicCI6ImFi + Y2RlZmdoaWprbG1ub3BxcnN0dXYifbE9pGQAAACoAAAAV9B6z2kLOmV2ZW50LXR5cGUHABFjb250 + ZW50QmxvY2tEZWx0YQ06Y29udGVudC10eXBlBwAQYXBwbGljYXRpb24vanNvbg06bWVzc2FnZS10 + eXBlBwAFZXZlbnR7ImNvbnRlbnRCbG9ja0luZGV4IjowLCJkZWx0YSI6eyJ0ZXh0IjoiIGlzIHBy + b3Blcmx5In0sInAiOiJhYmMifS5IUq4AAACtAAAAVxiaQBkLOmV2ZW50LXR5cGUHABFjb250ZW50 + QmxvY2tEZWx0YQ06Y29udGVudC10eXBlBwAQYXBwbGljYXRpb24vanNvbg06bWVzc2FnZS10eXBl + BwAFZXZlbnR7ImNvbnRlbnRCbG9ja0luZGV4IjowLCJkZWx0YSI6eyJ0ZXh0IjoiIGhhbmRsZSJ9 + LCJwIjoiYWJjZGVmZ2hpamtsbSJ9NdwtUAAAAJ8AAABXw3ur/ws6ZXZlbnQtdHlwZQcAEWNvbnRl + bnRCbG9ja0RlbHRhDTpjb250ZW50LXR5cGUHABBhcHBsaWNhdGlvbi9qc29uDTptZXNzYWdlLXR5 + cGUHAAVldmVudHsiY29udGVudEJsb2NrSW5kZXgiOjAsImRlbHRhIjp7InRleHQiOiJkIGFuIn0s + InAiOiJhYiJ9QxFvPQAAAKMAAABXp6r+eAs6ZXZlbnQtdHlwZQcAEWNvbnRlbnRCbG9ja0RlbHRh + DTpjb250ZW50LXR5cGUHABBhcHBsaWNhdGlvbi9qc29uDTptZXNzYWdlLXR5cGUHAAVldmVudHsi + Y29udGVudEJsb2NrSW5kZXgiOjAsImRlbHRhIjp7InRleHQiOiJkIGtlcHQifSwicCI6ImFiY2Qi + fbkD0FsAAADPAAAAV/toZDQLOmV2ZW50LXR5cGUHABFjb250ZW50QmxvY2tEZWx0YQ06Y29udGVu + dC10eXBlBwAQYXBwbGljYXRpb24vanNvbg06bWVzc2FnZS10eXBlBwAFZXZlbnR7ImNvbnRlbnRC + bG9ja0luZGV4IjowLCJkZWx0YSI6eyJ0ZXh0IjoiIHNlY3VyZS4ifSwicCI6ImFiY2RlZmdoaWpr + bG1ub3BxcnN0dXZ3eHl6QUJDREVGR0hJSktMTU5PUFFSU1QifaoGuEMAAACKAAAAVhx8g5sLOmV2 + ZW50LXR5cGUHABBjb250ZW50QmxvY2tTdG9wDTpjb250ZW50LXR5cGUHABBhcHBsaWNhdGlvbi9q + c29uDTptZXNzYWdlLXR5cGUHAAVldmVudHsiY29udGVudEJsb2NrSW5kZXgiOjAsInAiOiJhYmNk + ZWYifXbreQwAAACzAAAAUS4pzM8LOmV2ZW50LXR5cGUHAAttZXNzYWdlU3RvcA06Y29udGVudC10 + eXBlBwAQYXBwbGljYXRpb24vanNvbg06bWVzc2FnZS10eXBlBwAFZXZlbnR7InAiOiJhYmNkZWZn + aGlqa2xtbm9wcXJzdHV2d3h5ekFCQ0RFRkdISUpLTE1OT1BRUlNUVVZXWCIsInN0b3BSZWFzb24i + OiJlbmRfdHVybiJ9FldzdgAAAQUAAABO9/rvKws6ZXZlbnQtdHlwZQcACG1ldGFkYXRhDTpjb250 + ZW50LXR5cGUHABBhcHBsaWNhdGlvbi9qc29uDTptZXNzYWdlLXR5cGUHAAVldmVudHsibWV0cmlj + cyI6eyJsYXRlbmN5TXMiOjYyNX0sInAiOiJhYmNkZWZnaGlqa2xtbm9wcXJzdHV2d3h5ekFCQ0RF + RkdISUpLTE1OT1BRUlNUVVYiLCJ1c2FnZSI6eyJpbnB1dFRva2VucyI6MjksIm91dHB1dFRva2Vu + cyI6MzgsInNlcnZlclRvb2xVc2FnZSI6e30sInRvdGFsVG9rZW5zIjo2N319wheNbg== + headers: + Connection: + - keep-alive + Content-Type: + - application/vnd.amazon.eventstream + Date: + - Wed, 01 Apr 2026 06:17:40 GMT + Transfer-Encoding: + - chunked + x-amzn-RequestId: + - 52ef9744-6993-4f81-9c39-71b974657a26 + status: + code: 200 + message: OK +version: 1 diff --git a/packages/opentelemetry-instrumentation-bedrock/tests/traces/cassettes/test_safety_cassettes/test_bedrock_safety_udf_project_codenames_prompt.yaml b/packages/opentelemetry-instrumentation-bedrock/tests/traces/cassettes/test_safety_cassettes/test_bedrock_safety_udf_project_codenames_prompt.yaml new file mode 100644 index 0000000000..9038cd2244 --- /dev/null +++ b/packages/opentelemetry-instrumentation-bedrock/tests/traces/cassettes/test_safety_cassettes/test_bedrock_safety_udf_project_codenames_prompt.yaml @@ -0,0 +1,48 @@ +interactions: +- request: + body: '{"messages": [{"role": "user", "content": [{"text": "Prepare a status update + for [custom_compliance.project-phoenix] and [custom_compliance.operation-atlas] + including timeline and budget."}]}], "inferenceConfig": {"maxTokens": 50}}' + headers: + Content-Length: + - '231' + Content-Type: + - !!binary | + YXBwbGljYXRpb24vanNvbg== + User-Agent: + - !!binary | + Qm90bzMvMS40Mi4yOCBtZC9Cb3RvY29yZSMxLjQyLjI4IHVhLzIuMSBvcy9tYWNvcyMyNC42LjAg + bWQvYXJjaCN4ODZfNjQgbGFuZy9weXRob24jMy4xMC4xOSBtZC9weWltcGwjQ1B5dGhvbiBtL1os + ZSxiLEQgY2ZnL3JldHJ5LW1vZGUjbGVnYWN5IEJvdG9jb3JlLzEuNDIuMjg= + X-Amz-Date: + - !!binary | + MjAyNjA0MDFUMDYxNzMxWg== + amz-sdk-invocation-id: + - !!binary | + YWM4OWZiYjgtMWIzYy00ZDRhLWIxYjktMTg2ODJkY2QyNzkx + amz-sdk-request: + - !!binary | + YXR0ZW1wdD0x + method: POST + uri: https://bedrock-runtime.us-east-1.amazonaws.com/model/anthropic.claude-3-haiku-20240307-v1%3A0/converse + response: + body: + string: '{"metrics":{"latencyMs":1406},"output":{"message":{"content":[{"text":"Here + is a status update for the [custom_compliance.project-phoenix] and [custom_compliance.operation-atlas] + projects:\n\n[custom_compliance.project-phoenix]:\nTimeline:\n- Project kick-off + completed on schedule"}],"role":"assistant"}},"stopReason":"max_tokens","usage":{"inputTokens":37,"outputTokens":50,"serverToolUsage":{},"totalTokens":87}}' + headers: + Connection: + - keep-alive + Content-Length: + - '415' + Content-Type: + - application/json + Date: + - Wed, 01 Apr 2026 06:17:34 GMT + x-amzn-RequestId: + - 57685ff1-d624-462c-b194-88c11227fd0f + status: + code: 200 + message: OK +version: 1 diff --git a/packages/opentelemetry-instrumentation-bedrock/tests/traces/test_safety_cassettes.py b/packages/opentelemetry-instrumentation-bedrock/tests/traces/test_safety_cassettes.py new file mode 100644 index 0000000000..65c228abd9 --- /dev/null +++ b/packages/opentelemetry-instrumentation-bedrock/tests/traces/test_safety_cassettes.py @@ -0,0 +1,650 @@ +""" +FR VCR cassette tests for safety rule enforcement on Bedrock instrumentation. + +These tests record real AWS Bedrock API interactions via VCR cassettes and verify +that the safety callback system correctly detects and masks sensitive content +in both prompts and completions using the Converse API. + +Safety rule flavours covered: + - RegEx: PCI.credit_card, PII.email + - List: PII.competitor_org (list of company names) + - UDF: custom_compliance.project_codename (user-defined function + returning 2 sub-rule findings) + +North-star: This is a NEW file (FR-owned). Zero delta on TL files. + +Note: AWS Bedrock credentials are loaded via load-test-secrets.sh with FR_TEST_ +prefix. The brt_safety fixture creates a boto3 client using these prefixed vars. +""" + +import json +import os +import re + +import boto3 +import pytest +from opentelemetry.instrumentation.fortifyroot import ( + SafetyFinding, + SafetyLocation, + SafetyResult, + clear_safety_handlers, + register_completion_safety_handler, + register_prompt_safety_handler, +) +from opentelemetry.semconv._incubating.attributes import ( + gen_ai_attributes as GenAIAttributes, +) + + +# --------------------------------------------------------------------------- +# Safety rule definitions (mirror what real SDK rules would do) +# --------------------------------------------------------------------------- + +# RegEx patterns +_CREDIT_CARD_RE = re.compile(r"\b(?:\d[ -]*?){13,19}\b") +_EMAIL_RE = re.compile(r"\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}\b") + +# List of competitor org names (case-insensitive) +_COMPETITOR_ORGS = ["acme corp", "globex industries", "initech", "umbrella inc"] + +# UDF: detects two fictional project codenames +_PROJECT_CODENAMES = ["project-phoenix", "operation-atlas"] + +# Module-level action setting (switched by fixtures) +_SAFETY_ACTION = "MASK" + + +def _build_findings_and_mask(text, findings_list, action="MASK"): + """Apply all collected findings to produce masked (or allowed) text.""" + if not findings_list: + return None + if action == "MASK": + masked = text + for f in sorted(findings_list, key=lambda x: x.start, reverse=True): + masked = masked[: f.start] + f"[{f.rule_name}]" + masked[f.end :] + return SafetyResult( + text=masked, + overall_action="MASK", + findings=findings_list, + ) + return SafetyResult( + text=text, + overall_action="ALLOW", + findings=findings_list, + ) + + +def _scan_text(text, location): + """Run all safety rule types against text, return findings.""" + if not isinstance(text, str) or not text: + return [] + + action = _SAFETY_ACTION + findings = [] + + for m in _CREDIT_CARD_RE.finditer(text): + findings.append( + SafetyFinding( + category="PCI", + severity="HIGH", + action=action, + rule_name="PCI.credit_card", + start=m.start(), + end=m.end(), + ) + ) + + for m in _EMAIL_RE.finditer(text): + findings.append( + SafetyFinding( + category="PII", + severity="MEDIUM", + action=action, + rule_name="PII.email", + start=m.start(), + end=m.end(), + ) + ) + + text_lower = text.lower() + for org in _COMPETITOR_ORGS: + idx = text_lower.find(org) + while idx != -1: + findings.append( + SafetyFinding( + category="PII", + severity="LOW", + action=action, + rule_name="PII.competitor_org", + start=idx, + end=idx + len(org), + ) + ) + idx = text_lower.find(org, idx + len(org)) + + for codename in _PROJECT_CODENAMES: + idx = text_lower.find(codename) + while idx != -1: + findings.append( + SafetyFinding( + category="COMPLIANCE", + severity="HIGH", + action=action, + rule_name=f"custom_compliance.{codename}", + start=idx, + end=idx + len(codename), + ) + ) + idx = text_lower.find(codename, idx + len(codename)) + + return findings + + +def _prompt_handler(context): + if context.location != SafetyLocation.PROMPT: + return None + findings = _scan_text(context.text, SafetyLocation.PROMPT) + return _build_findings_and_mask(context.text, findings, action=_SAFETY_ACTION) + + +def _completion_handler(context): + if context.location != SafetyLocation.COMPLETION: + return None + findings = _scan_text(context.text, SafetyLocation.COMPLETION) + return _build_findings_and_mask(context.text, findings, action=_SAFETY_ACTION) + + +# --------------------------------------------------------------------------- +# Fixtures +# --------------------------------------------------------------------------- + + +@pytest.fixture(autouse=True) +def _register_safety_handlers(): + """Register all safety handlers before each test, clear after.""" + clear_safety_handlers() + register_prompt_safety_handler(_prompt_handler) + register_completion_safety_handler(_completion_handler) + yield + clear_safety_handlers() + + +@pytest.fixture +def brt_safety(): + """Bedrock runtime client using FR_TEST_ prefixed credentials. + + load-test-secrets.sh exports AWS creds with FR_TEST_ prefix to avoid + clobbering the caller's AWS identity. In VCR replay mode, the actual + credentials don't matter (dummy values are used by conftest.py). + """ + return boto3.client( + service_name="bedrock-runtime", + aws_access_key_id=os.environ.get("FR_TEST_AWS_ACCESS_KEY_ID", os.environ.get("AWS_ACCESS_KEY_ID", "test")), + aws_secret_access_key=os.environ.get("FR_TEST_AWS_SECRET_ACCESS_KEY", os.environ.get("AWS_SECRET_ACCESS_KEY", "test")), + region_name=os.environ.get("FR_TEST_AWS_DEFAULT_REGION", os.environ.get("AWS_DEFAULT_REGION", "us-east-1")), + ) + + +# --------------------------------------------------------------------------- +# T3-B3: RegEx rule — PII.email in prompt (Converse) +# --------------------------------------------------------------------------- + + +@pytest.mark.vcr +@pytest.mark.fr +def test_bedrock_safety_regex_email_prompt( + instrument_legacy, span_exporter, brt_safety +): + """Prompt contains an email address. Safety should detect and mask it.""" + brt_safety.converse( + modelId="anthropic.claude-3-haiku-20240307-v1:0", + messages=[ + { + "role": "user", + "content": [ + { + "text": ( + "Summarize the following customer inquiry from " + "john.doe@example.com about their recent order." + ) + } + ], + } + ], + inferenceConfig={"maxTokens": 50}, + ) + + spans = span_exporter.get_finished_spans() + assert len(spans) >= 1 + span = spans[0] + + prompt_content = span.attributes.get( + f"{GenAIAttributes.GEN_AI_PROMPT}.0.content" + ) + assert prompt_content is not None + assert "john.doe@example.com" not in prompt_content + assert "[PII.email]" in prompt_content + + safety_events = [ + e for e in span.events if "fortifyroot.safety" in (e.name or "") + ] + assert len(safety_events) >= 1 + event_attrs = safety_events[0].attributes + assert event_attrs["fortifyroot.safety.rule_name"] == "PII.email" + assert event_attrs["fortifyroot.safety.action"] == "MASK" + assert event_attrs["fortifyroot.safety.location"] == "PROMPT" + + +# --------------------------------------------------------------------------- +# T3-B3: RegEx rule — PCI.credit_card in prompt (Converse) +# --------------------------------------------------------------------------- + + +@pytest.mark.vcr +@pytest.mark.fr +def test_bedrock_safety_regex_credit_card_prompt( + instrument_legacy, span_exporter, brt_safety +): + """Prompt contains a credit card number. Safety should detect and mask it.""" + brt_safety.converse( + modelId="anthropic.claude-3-haiku-20240307-v1:0", + messages=[ + { + "role": "user", + "content": [ + { + "text": "Check if order for card 4111 1111 1111 1111 was processed." + } + ], + } + ], + inferenceConfig={"maxTokens": 50}, + ) + + spans = span_exporter.get_finished_spans() + assert len(spans) >= 1 + span = spans[0] + + prompt_content = span.attributes.get( + f"{GenAIAttributes.GEN_AI_PROMPT}.0.content" + ) + assert prompt_content is not None + assert "4111 1111 1111 1111" not in prompt_content + assert "[PCI.credit_card]" in prompt_content + + safety_events = [ + e for e in span.events if "fortifyroot.safety" in (e.name or "") + ] + found_cc = any( + e.attributes.get("fortifyroot.safety.rule_name") == "PCI.credit_card" + for e in safety_events + ) + assert found_cc, "Expected PCI.credit_card safety finding" + + +# --------------------------------------------------------------------------- +# T3-B3: List rule — PII.competitor_org in prompt (Converse) +# --------------------------------------------------------------------------- + + +@pytest.mark.vcr +@pytest.mark.fr +def test_bedrock_safety_list_competitor_org_prompt( + instrument_legacy, span_exporter, brt_safety +): + """Prompt mentions a competitor org from the blocklist.""" + brt_safety.converse( + modelId="anthropic.claude-3-haiku-20240307-v1:0", + messages=[ + { + "role": "user", + "content": [ + { + "text": ( + "Compare our product roadmap with Acme Corp's latest " + "offering and Globex Industries' pricing strategy." + ) + } + ], + } + ], + inferenceConfig={"maxTokens": 50}, + ) + + spans = span_exporter.get_finished_spans() + assert len(spans) >= 1 + span = spans[0] + + prompt_content = span.attributes.get( + f"{GenAIAttributes.GEN_AI_PROMPT}.0.content" + ) + assert prompt_content is not None + assert "acme corp" not in prompt_content.lower() + assert "globex industries" not in prompt_content.lower() + assert "[PII.competitor_org]" in prompt_content + + safety_events = [ + e for e in span.events if "fortifyroot.safety" in (e.name or "") + ] + found_org = any( + e.attributes.get("fortifyroot.safety.rule_name") == "PII.competitor_org" + for e in safety_events + ) + assert found_org, "Expected PII.competitor_org safety finding" + + +# --------------------------------------------------------------------------- +# T3-B3: UDF rule — custom_compliance with 2 sub-rules (Converse) +# --------------------------------------------------------------------------- + + +@pytest.mark.vcr +@pytest.mark.fr +def test_bedrock_safety_udf_project_codenames_prompt( + instrument_legacy, span_exporter, brt_safety +): + """Prompt contains two project codenames detected by UDF rule.""" + brt_safety.converse( + modelId="anthropic.claude-3-haiku-20240307-v1:0", + messages=[ + { + "role": "user", + "content": [ + { + "text": ( + "Prepare a status update for Project-Phoenix and " + "Operation-Atlas including timeline and budget." + ) + } + ], + } + ], + inferenceConfig={"maxTokens": 50}, + ) + + spans = span_exporter.get_finished_spans() + assert len(spans) >= 1 + span = spans[0] + + prompt_content = span.attributes.get( + f"{GenAIAttributes.GEN_AI_PROMPT}.0.content" + ) + assert prompt_content is not None + assert "[custom_compliance.project-phoenix]" in prompt_content + assert "[custom_compliance.operation-atlas]" in prompt_content + assert "Project-Phoenix" not in prompt_content + assert "Operation-Atlas" not in prompt_content + + safety_events = [ + e for e in span.events if "fortifyroot.safety" in (e.name or "") + ] + udf_findings = [ + e + for e in safety_events + if (e.attributes.get("fortifyroot.safety.rule_name") or "").startswith( + "custom_compliance." + ) + ] + assert len(udf_findings) >= 2, ( + f"Expected 2 UDF findings, got {len(udf_findings)}" + ) + + +# --------------------------------------------------------------------------- +# T3-B3: Completion safety — email in LLM response (Converse) +# --------------------------------------------------------------------------- + + +@pytest.mark.vcr +@pytest.mark.fr +def test_bedrock_safety_completion_email_masked( + instrument_legacy, span_exporter, brt_safety +): + """ + Ask the LLM to generate fictional contact info. The response should + contain an email-like pattern that completion safety detects and masks. + """ + brt_safety.converse( + modelId="anthropic.claude-3-haiku-20240307-v1:0", + messages=[ + { + "role": "user", + "content": [ + { + "text": ( + "Create a fictional contact card. The person's name is " + "Jane Smith, company is Widgets Inc, role is VP of Sales. " + "Invent a plausible work email and phone. Format as:\n" + "Name: ...\nEmail: ...\nPhone: ..." + ) + } + ], + } + ], + inferenceConfig={"maxTokens": 100}, + ) + + spans = span_exporter.get_finished_spans() + assert len(spans) >= 1 + span = spans[0] + + completion_content = span.attributes.get( + f"{GenAIAttributes.GEN_AI_COMPLETION}.0.content" + ) + assert completion_content is not None, "Completion content should be recorded" + assert "[PII.email]" in completion_content, ( + "LLM-generated email in response should be masked by completion safety" + ) + + safety_events = [ + e for e in span.events if "fortifyroot.safety" in (e.name or "") + ] + completion_findings = [ + e + for e in safety_events + if e.attributes.get("fortifyroot.safety.location") == "COMPLETION" + ] + assert len(completion_findings) >= 1, "Should have completion safety finding" + assert any( + e.attributes.get("fortifyroot.safety.rule_name") == "PII.email" + for e in completion_findings + ), "Completion safety should detect PII.email" + + +# --------------------------------------------------------------------------- +# T3-B3: Combined — multiple rule types in single prompt (Converse) +# --------------------------------------------------------------------------- + + +@pytest.mark.vcr +@pytest.mark.fr +def test_bedrock_safety_combined_rules_prompt( + instrument_legacy, span_exporter, brt_safety +): + """Prompt triggers all rule types: RegEx (email), List (org), UDF (codename).""" + brt_safety.converse( + modelId="anthropic.claude-3-haiku-20240307-v1:0", + messages=[ + { + "role": "user", + "content": [ + { + "text": ( + "Send the Project-Phoenix briefing to contact@acme-corp.com " + "and CC the Initech team." + ) + } + ], + } + ], + inferenceConfig={"maxTokens": 50}, + ) + + spans = span_exporter.get_finished_spans() + assert len(spans) >= 1 + span = spans[0] + + prompt_content = span.attributes.get( + f"{GenAIAttributes.GEN_AI_PROMPT}.0.content" + ) + assert prompt_content is not None + + safety_events = [ + e for e in span.events if "fortifyroot.safety" in (e.name or "") + ] + rule_names = { + e.attributes.get("fortifyroot.safety.rule_name") for e in safety_events + } + assert "PII.email" in rule_names, "Expected PII.email finding" + assert "PII.competitor_org" in rule_names or any( + "competitor_org" in (r or "") for r in rule_names + ), "Expected PII.competitor_org finding" + assert any( + (r or "").startswith("custom_compliance.") for r in rule_names + ), "Expected custom_compliance UDF finding" + + +# --------------------------------------------------------------------------- +# T3-B3: Streaming with PII — verify safety on streamed completion (Converse) +# --------------------------------------------------------------------------- + + +@pytest.mark.vcr +@pytest.mark.fr +def test_bedrock_safety_streaming_with_pii( + instrument_legacy, span_exporter, brt_safety +): + """Streaming converse with PII in prompt. Verify prompt safety applied.""" + response = brt_safety.converse_stream( + modelId="anthropic.claude-3-haiku-20240307-v1:0", + messages=[ + { + "role": "user", + "content": [ + { + "text": ( + "Acknowledge receipt of message from " + "alice.jones@example.org about Project-Phoenix." + ) + } + ], + } + ], + inferenceConfig={"maxTokens": 50}, + ) + + # Consume the stream + for _ in response.get("stream", []): + pass + + spans = span_exporter.get_finished_spans() + assert len(spans) >= 1 + span = spans[0] + + prompt_content = span.attributes.get( + f"{GenAIAttributes.GEN_AI_PROMPT}.0.content" + ) + assert prompt_content is not None + assert "alice.jones@example.org" not in prompt_content + assert "[PII.email]" in prompt_content + + +# =========================================================================== +# ALLOW action tests +# =========================================================================== + + +@pytest.fixture +def _set_allow_action(): + """Switch safety handlers to ALLOW mode for the test.""" + global _SAFETY_ACTION + _SAFETY_ACTION = "ALLOW" + yield + _SAFETY_ACTION = "MASK" + + +@pytest.mark.vcr +@pytest.mark.fr +def test_bedrock_safety_allow_email_not_masked( + _set_allow_action, instrument_legacy, span_exporter, brt_safety +): + """ALLOW action: email detected but text passes through unchanged.""" + brt_safety.converse( + modelId="anthropic.claude-3-haiku-20240307-v1:0", + messages=[ + { + "role": "user", + "content": [ + { + "text": ( + "Forward this message from alice.jones@example.org " + "to the support team." + ) + } + ], + } + ], + inferenceConfig={"maxTokens": 50}, + ) + + spans = span_exporter.get_finished_spans() + assert len(spans) >= 1 + span = spans[0] + + prompt_content = span.attributes.get( + f"{GenAIAttributes.GEN_AI_PROMPT}.0.content" + ) + assert prompt_content is not None + assert "alice.jones@example.org" in prompt_content + assert "[PII.email]" not in prompt_content + + safety_events = [ + e for e in span.events if "fortifyroot.safety" in (e.name or "") + ] + assert len(safety_events) >= 1, "ALLOW should still emit safety findings" + event_attrs = safety_events[0].attributes + assert event_attrs["fortifyroot.safety.rule_name"] == "PII.email" + assert event_attrs["fortifyroot.safety.action"] == "ALLOW" + assert event_attrs["fortifyroot.safety.location"] == "PROMPT" + + +@pytest.mark.vcr +@pytest.mark.fr +def test_bedrock_safety_allow_credit_card_not_masked( + _set_allow_action, instrument_legacy, span_exporter, brt_safety +): + """ALLOW action: credit card detected but text passes through unchanged.""" + brt_safety.converse( + modelId="anthropic.claude-3-haiku-20240307-v1:0", + messages=[ + { + "role": "user", + "content": [ + { + "text": "Verify payment for card 4111 1111 1111 1111 was received." + } + ], + } + ], + inferenceConfig={"maxTokens": 50}, + ) + + spans = span_exporter.get_finished_spans() + assert len(spans) >= 1 + span = spans[0] + + prompt_content = span.attributes.get( + f"{GenAIAttributes.GEN_AI_PROMPT}.0.content" + ) + assert prompt_content is not None + assert "4111 1111 1111 1111" in prompt_content + + safety_events = [ + e for e in span.events if "fortifyroot.safety" in (e.name or "") + ] + found_cc = any( + e.attributes.get("fortifyroot.safety.rule_name") == "PCI.credit_card" + and e.attributes.get("fortifyroot.safety.action") == "ALLOW" + for e in safety_events + ) + assert found_cc, "Expected PCI.credit_card finding with ALLOW action" diff --git a/packages/opentelemetry-instrumentation-google-generativeai/tests/cassettes/test_safety_cassettes/test_google_genai_safety_allow_credit_card_not_masked.yaml b/packages/opentelemetry-instrumentation-google-generativeai/tests/cassettes/test_safety_cassettes/test_google_genai_safety_allow_credit_card_not_masked.yaml new file mode 100644 index 0000000000..05f345faa2 --- /dev/null +++ b/packages/opentelemetry-instrumentation-google-generativeai/tests/cassettes/test_safety_cassettes/test_google_genai_safety_allow_credit_card_not_masked.yaml @@ -0,0 +1,63 @@ +interactions: +- request: + body: '{"contents": [{"parts": [{"text": "Verify payment for card 4111 1111 1111 + 1111 was received."}], "role": "user"}], "generationConfig": {"maxOutputTokens": + 50}}' + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '159' + Content-Type: + - application/json + Host: + - generativelanguage.googleapis.com + user-agent: + - google-genai-sdk/1.58.0 gl-python/3.10.19 + x-goog-api-client: + - google-genai-sdk/1.58.0 gl-python/3.10.19 + method: POST + uri: https://generativelanguage.googleapis.com/v1beta/models/gemini-2.5-flash:generateContent + response: + body: + string: !!binary | + H4sIAAAAAAAC/2VQXUvDMBR9768Ied5EynTq23B7kFEnGmQgQ67mro1Lk9Kbgjr6303StVp8uRfO + uR/nnGPCGH8HI5UEh8Rv2ItHGDvGGjhrHBrniR7yYG01eoSXVqLmJ7yd9Dt7ZRQVjwhkTRjLFttX + sVmv7p/4MKOMxE9Pnkeg9XUXON4Q5JihA68Hhq+8qm1ZOWEPaG5tE/WkV90x7qwDPaLms8m/NVr6 + o0r/WmR/HXknoJX7CnLFaisGofH+6GtvN/Zdr6GwTV44GsmYXSanWLqknrEm1UWSY+lDmqZnF9O9 + BiriP14jVdYQ3skw8yaXGYBIN+vv+UeTPajrEmlx4Emb/AD60yqjtwEAAA== + headers: + Alt-Svc: + - h3=":443"; ma=2592000,h3-29=":443"; ma=2592000 + Content-Encoding: + - gzip + Content-Type: + - application/json; charset=UTF-8 + Date: + - Wed, 01 Apr 2026 07:59:43 GMT + Server: + - scaffolding on HTTPServer2 + Server-Timing: + - gfet4t7; dur=1248 + Transfer-Encoding: + - chunked + Vary: + - Origin + - X-Origin + - Referer + X-Content-Type-Options: + - nosniff + X-Frame-Options: + - SAMEORIGIN + X-Gemini-Service-Tier: + - standard + X-XSS-Protection: + - '0' + status: + code: 200 + message: OK +version: 1 diff --git a/packages/opentelemetry-instrumentation-google-generativeai/tests/cassettes/test_safety_cassettes/test_google_genai_safety_allow_email_not_masked.yaml b/packages/opentelemetry-instrumentation-google-generativeai/tests/cassettes/test_safety_cassettes/test_google_genai_safety_allow_email_not_masked.yaml new file mode 100644 index 0000000000..60d003d386 --- /dev/null +++ b/packages/opentelemetry-instrumentation-google-generativeai/tests/cassettes/test_safety_cassettes/test_google_genai_safety_allow_email_not_masked.yaml @@ -0,0 +1,64 @@ +interactions: +- request: + body: '{"contents": [{"parts": [{"text": "Forward this message from alice.jones@example.org + to the support team."}], "role": "user"}], "generationConfig": {"maxOutputTokens": + 50}}' + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '172' + Content-Type: + - application/json + Host: + - generativelanguage.googleapis.com + user-agent: + - google-genai-sdk/1.58.0 gl-python/3.10.19 + x-goog-api-client: + - google-genai-sdk/1.58.0 gl-python/3.10.19 + method: POST + uri: https://generativelanguage.googleapis.com/v1beta/models/gemini-2.5-flash:generateContent + response: + body: + string: !!binary | + H4sIAAAAAAAC/2WQX0vDMBTF3/MpSp5XUbcp+CbbwCGd/4obiMjF3rZxaVKaW5iUfXeTdu1SzEMI + 95zce8+vYUHAv0ElIgFCw++CD1sJgqa9naYVoSIr9CVbLKGis7c7jfe2FsKD+8TX3Ksfh/fn5Nyt + 0hKdtdAJyt5+7A08FUqY/BXBaOVs0f3uK356XG3e+OARKsGDFS9ZP6YdwGsDGUZIYNPBkIGXlS5K + ivUe1ULXbbqr266ZB2Osn2TSBHKk3Ewn/7qapZ0ppM/Iw2eDghT069LEq13MPRg0XqqnwTxonHJd + ZzmNF5zN2YlaB/IdKyM6YhkWlmF4fTEPUwkmb+fxCk2plcF14jxAywhgu9iE1fSnjp5n6UMcvuw5 + O7I/yikXCCQCAAA= + headers: + Alt-Svc: + - h3=":443"; ma=2592000,h3-29=":443"; ma=2592000 + Content-Encoding: + - gzip + Content-Type: + - application/json; charset=UTF-8 + Date: + - Wed, 01 Apr 2026 07:59:41 GMT + Server: + - scaffolding on HTTPServer2 + Server-Timing: + - gfet4t7; dur=2820 + Transfer-Encoding: + - chunked + Vary: + - Origin + - X-Origin + - Referer + X-Content-Type-Options: + - nosniff + X-Frame-Options: + - SAMEORIGIN + X-Gemini-Service-Tier: + - standard + X-XSS-Protection: + - '0' + status: + code: 200 + message: OK +version: 1 diff --git a/packages/opentelemetry-instrumentation-google-generativeai/tests/cassettes/test_safety_cassettes/test_google_genai_safety_combined_rules_prompt.yaml b/packages/opentelemetry-instrumentation-google-generativeai/tests/cassettes/test_safety_cassettes/test_google_genai_safety_combined_rules_prompt.yaml new file mode 100644 index 0000000000..5de9144415 --- /dev/null +++ b/packages/opentelemetry-instrumentation-google-generativeai/tests/cassettes/test_safety_cassettes/test_google_genai_safety_combined_rules_prompt.yaml @@ -0,0 +1,63 @@ +interactions: +- request: + body: '{"contents": [{"parts": [{"text": "Send the [custom_compliance.project-phoenix] + briefing to [PII.email] and CC the [PII.competitor_org] team."}], "role": "user"}], + "generationConfig": {"maxOutputTokens": 50}}' + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '208' + Content-Type: + - application/json + Host: + - generativelanguage.googleapis.com + user-agent: + - google-genai-sdk/1.58.0 gl-python/3.10.19 + x-goog-api-client: + - google-genai-sdk/1.58.0 gl-python/3.10.19 + method: POST + uri: https://generativelanguage.googleapis.com/v1beta/models/gemini-2.5-flash:generateContent + response: + body: + string: !!binary | + H4sIAAAAAAAC/2VQ20rEMBB971eEPG/Fy66Ib+oWlKWtlyIFWWS0s20wzdQmBWXpv5uk22rxZQbO + mcs5Zx8wxt9BFaIAg5pfsheLMLb31XGkDCpjiRGyYEsSLcJrKlDyA94vxp2dUEJXjwialBuLr/LX + LN1EyROfZoQq8MuSxx7obd06jncaSozRgNUD01fetFQ3JqMPVDfUeT1nq+EYN2RAzqiLk8W/Nb22 + R4X8tcj+OrJOQArz7eRmUZ5NQv392dfRru/bUUNFXVkZPZOxPA8OsQxJPWOrxRBJibUNKTw9WoU7 + Cbry/3iLuiGl8a5wM5CsY3gTyTXe5sswuv/cpGn4EPGgD34AvnuDLbcBAAA= + headers: + Alt-Svc: + - h3=":443"; ma=2592000,h3-29=":443"; ma=2592000 + Content-Encoding: + - gzip + Content-Type: + - application/json; charset=UTF-8 + Date: + - Wed, 01 Apr 2026 07:59:37 GMT + Server: + - scaffolding on HTTPServer2 + Server-Timing: + - gfet4t7; dur=1684 + Transfer-Encoding: + - chunked + Vary: + - Origin + - X-Origin + - Referer + X-Content-Type-Options: + - nosniff + X-Frame-Options: + - SAMEORIGIN + X-Gemini-Service-Tier: + - standard + X-XSS-Protection: + - '0' + status: + code: 200 + message: OK +version: 1 diff --git a/packages/opentelemetry-instrumentation-google-generativeai/tests/cassettes/test_safety_cassettes/test_google_genai_safety_completion_email_masked.yaml b/packages/opentelemetry-instrumentation-google-generativeai/tests/cassettes/test_safety_cassettes/test_google_genai_safety_completion_email_masked.yaml new file mode 100644 index 0000000000..f77e8af2ff --- /dev/null +++ b/packages/opentelemetry-instrumentation-google-generativeai/tests/cassettes/test_safety_cassettes/test_google_genai_safety_completion_email_masked.yaml @@ -0,0 +1,65 @@ +interactions: +- request: + body: '{"contents": [{"parts": [{"text": "Reply with ONLY this text, filling in + the blanks:\nEmail: [PII.email]\nPhone: 555-0142\nDo not add anything else."}], + "role": "user"}], "generationConfig": {"maxOutputTokens": 200, "thinkingConfig": + {"thinking_budget": 0}}}' + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '258' + Content-Type: + - application/json + Host: + - generativelanguage.googleapis.com + user-agent: + - google-genai-sdk/1.58.0 gl-python/3.10.19 + x-goog-api-client: + - google-genai-sdk/1.58.0 gl-python/3.10.19 + method: POST + uri: https://generativelanguage.googleapis.com/v1beta/models/gemini-2.5-flash:generateContent + response: + body: + string: !!binary | + H4sIAAAAAAAC/1VQwU6EMBC98xVNz7JBshizN+PugZiNqKwxKodGBraxtIQOZg3Zf7cFy5Ye2sm8 + N2/63hAQQr+YLHnJEDTdkA/TIWQYb4spiSDRAK5lmi3r8MKdzuDVhoJwskN01zAuDDNL0xXYuviU + 2VFJ2JAkScLoeh1Tb/I818XVZV+nBFixRpUgHP3sCLTikuvjMzCtpKW95I8ZnVEuSziZdhS4BaM0 + 7TWrYQ/IjHM2+6Ntp5oWc/UN8l71o/N1NIl5QS3w2OGokIkFdOMgT1ZvzVIu/AC9bI1HJjj+WiP5 + 7i2nXg64/JULYnyL4D+SKaVX6DSf4qihMQGF8SoJK8H0cVSkHehWSQ1paTnvuN0zCO8e9hXWt4fs + B/p1/3SgwTn4A0cwtmwjAgAA + headers: + Alt-Svc: + - h3=":443"; ma=2592000,h3-29=":443"; ma=2592000 + Content-Encoding: + - gzip + Content-Type: + - application/json; charset=UTF-8 + Date: + - Wed, 01 Apr 2026 07:59:35 GMT + Server: + - scaffolding on HTTPServer2 + Server-Timing: + - gfet4t7; dur=1307 + Transfer-Encoding: + - chunked + Vary: + - Origin + - X-Origin + - Referer + X-Content-Type-Options: + - nosniff + X-Frame-Options: + - SAMEORIGIN + X-Gemini-Service-Tier: + - standard + X-XSS-Protection: + - '0' + status: + code: 200 + message: OK +version: 1 diff --git a/packages/opentelemetry-instrumentation-google-generativeai/tests/cassettes/test_safety_cassettes/test_google_genai_safety_list_competitor_org_prompt.yaml b/packages/opentelemetry-instrumentation-google-generativeai/tests/cassettes/test_safety_cassettes/test_google_genai_safety_list_competitor_org_prompt.yaml new file mode 100644 index 0000000000..401833044e --- /dev/null +++ b/packages/opentelemetry-instrumentation-google-generativeai/tests/cassettes/test_safety_cassettes/test_google_genai_safety_list_competitor_org_prompt.yaml @@ -0,0 +1,64 @@ +interactions: +- request: + body: '{"contents": [{"parts": [{"text": "Compare our product roadmap with [PII.competitor_org]''s + latest offering and [PII.competitor_org]'' pricing strategy."}], "role": "user"}], + "generationConfig": {"maxOutputTokens": 50}}' + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '217' + Content-Type: + - application/json + Host: + - generativelanguage.googleapis.com + user-agent: + - google-genai-sdk/1.58.0 gl-python/3.10.19 + x-goog-api-client: + - google-genai-sdk/1.58.0 gl-python/3.10.19 + method: POST + uri: https://generativelanguage.googleapis.com/v1beta/models/gemini-2.5-flash:generateContent + response: + body: + string: !!binary | + H4sIAAAAAAAC/2WQT0/DMAzF7/kUlc8bYsA0wQ2xHWDq+FehIoSQtXptIE1K40pD1b47aUu7VOQQ + RX4vfvavFkEAW9SJTJDJwlXw5ipBULd3oxnNpNkJfckVCyz56O1O7b2dhWnffIIokxY86TC83yfH + hqVR1Lhzk5Dq7YfeADuppc2eCK3RjS28jj+i+/Vq8wyDR+qE9k48FX1MGwCVxZRCYnQL4rAGFKXJ + C47MF+kbU7ULns+6Zh6Pkd7LbBjVSFksJv+62qXLlMrH5BF0i6KS/NMiWsUReDB4PFRPQ3jQgDNT + pRmPB7yYiz9qHcgXKq3siKWUO4bTs5P5dKfQZm0elGQLoy3dJo3n9XIZ4raabTCWn1X4oIv13fej + AXEQvwwHsBonAgAA + headers: + Alt-Svc: + - h3=":443"; ma=2592000,h3-29=":443"; ma=2592000 + Content-Encoding: + - gzip + Content-Type: + - application/json; charset=UTF-8 + Date: + - Wed, 01 Apr 2026 07:59:33 GMT + Server: + - scaffolding on HTTPServer2 + Server-Timing: + - gfet4t7; dur=1425 + Transfer-Encoding: + - chunked + Vary: + - Origin + - X-Origin + - Referer + X-Content-Type-Options: + - nosniff + X-Frame-Options: + - SAMEORIGIN + X-Gemini-Service-Tier: + - standard + X-XSS-Protection: + - '0' + status: + code: 200 + message: OK +version: 1 diff --git a/packages/opentelemetry-instrumentation-google-generativeai/tests/cassettes/test_safety_cassettes/test_google_genai_safety_regex_credit_card_prompt.yaml b/packages/opentelemetry-instrumentation-google-generativeai/tests/cassettes/test_safety_cassettes/test_google_genai_safety_regex_credit_card_prompt.yaml new file mode 100644 index 0000000000..c23ed44361 --- /dev/null +++ b/packages/opentelemetry-instrumentation-google-generativeai/tests/cassettes/test_safety_cassettes/test_google_genai_safety_regex_credit_card_prompt.yaml @@ -0,0 +1,64 @@ +interactions: +- request: + body: '{"contents": [{"parts": [{"text": "Check if order for card [PCI.credit_card] + was processed."}], "role": "user"}], "generationConfig": {"maxOutputTokens": + 50}}' + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '158' + Content-Type: + - application/json + Host: + - generativelanguage.googleapis.com + user-agent: + - google-genai-sdk/1.58.0 gl-python/3.10.19 + x-goog-api-client: + - google-genai-sdk/1.58.0 gl-python/3.10.19 + method: POST + uri: https://generativelanguage.googleapis.com/v1beta/models/gemini-2.5-flash:generateContent + response: + body: + string: !!binary | + H4sIAAAAAAAC/1WQX0vDMBTF3/MpQp6d6JxDfBPbhyL1b5ENkZHZuzaaJqW5hWnZdzdply7NQwj3 + nNx7z68jlLIvrnKRcwTDbumHrVDa9bfTtEJQaAVfssWaN3jyDqcL3taCsHefWEJtd6WRBfJhfH+e + nZo2WoL7UekcpLcfvIHthBKmfAVutHK29G61yZ4e4sc3NnqEymFvxQvix/QDWGt4ASkgtyH5GIXV + ja5qzPQPqHvd9iEvl0OzgMlEnx9l1MjlRFl6KehqIjtTyBBVQNEG5VLgr0uTxauMBTBwupSnQQJo + DEvdFiVOF1wsyJHaAPIdGiMGYgVUluFsfn4920luyn4ea8DUWhlIcudZY5Ty7Z+Jk2z73abPV5tI + 37wUjBzIP9QNG14rAgAA + headers: + Alt-Svc: + - h3=":443"; ma=2592000,h3-29=":443"; ma=2592000 + Content-Encoding: + - gzip + Content-Type: + - application/json; charset=UTF-8 + Date: + - Wed, 01 Apr 2026 07:59:31 GMT + Server: + - scaffolding on HTTPServer2 + Server-Timing: + - gfet4t7; dur=1305 + Transfer-Encoding: + - chunked + Vary: + - Origin + - X-Origin + - Referer + X-Content-Type-Options: + - nosniff + X-Frame-Options: + - SAMEORIGIN + X-Gemini-Service-Tier: + - standard + X-XSS-Protection: + - '0' + status: + code: 200 + message: OK +version: 1 diff --git a/packages/opentelemetry-instrumentation-google-generativeai/tests/cassettes/test_safety_cassettes/test_google_genai_safety_regex_email_prompt.yaml b/packages/opentelemetry-instrumentation-google-generativeai/tests/cassettes/test_safety_cassettes/test_google_genai_safety_regex_email_prompt.yaml new file mode 100644 index 0000000000..76b16b416b --- /dev/null +++ b/packages/opentelemetry-instrumentation-google-generativeai/tests/cassettes/test_safety_cassettes/test_google_genai_safety_regex_email_prompt.yaml @@ -0,0 +1,63 @@ +interactions: +- request: + body: '{"contents": [{"parts": [{"text": "Summarize the following customer inquiry + from [PII.email] about their recent order."}], "role": "user"}], "generationConfig": + {"maxOutputTokens": 50}}' + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '185' + Content-Type: + - application/json + Host: + - generativelanguage.googleapis.com + user-agent: + - google-genai-sdk/1.58.0 gl-python/3.10.19 + x-goog-api-client: + - google-genai-sdk/1.58.0 gl-python/3.10.19 + method: POST + uri: https://generativelanguage.googleapis.com/v1beta/models/gemini-2.5-flash:generateContent + response: + body: + string: !!binary | + H4sIAAAAAAAC/2VQXUvDMBR9768Ied7EiSvq29j6INIqWmQiQ8Jy22SmSWkScYz+d2/TtVp8uRfO + uR/nnFNECN0zzSVnDiy9I++IEHIKteOMdqAdEgOEYGMUIEIrw0HRM97Ohp1CamnFMzBrdDeWrrYf + +eNDkr3QcUZqDt9IXgagxbrrOOotKyEFx1APG7/SujFV7XLzCXptfNCzuO2PUWccUxMqXs7+rdkN + HpXq1yL56widMCXdsZObJ9t8FBruT74OdkPfDRqE8aVwdiLjOo7OsfRJvUJjZR9JCRWGNL+6WM4L + xawI/2gDtjbawj3vZt6yTcr2WmQ3sTv49OlQfB39KqFRG/0AhrfkmrcBAAA= + headers: + Alt-Svc: + - h3=":443"; ma=2592000,h3-29=":443"; ma=2592000 + Content-Encoding: + - gzip + Content-Type: + - application/json; charset=UTF-8 + Date: + - Wed, 01 Apr 2026 07:59:30 GMT + Server: + - scaffolding on HTTPServer2 + Server-Timing: + - gfet4t7; dur=1382 + Transfer-Encoding: + - chunked + Vary: + - Origin + - X-Origin + - Referer + X-Content-Type-Options: + - nosniff + X-Frame-Options: + - SAMEORIGIN + X-Gemini-Service-Tier: + - standard + X-XSS-Protection: + - '0' + status: + code: 200 + message: OK +version: 1 diff --git a/packages/opentelemetry-instrumentation-google-generativeai/tests/cassettes/test_safety_cassettes/test_google_genai_safety_streaming_with_pii.yaml b/packages/opentelemetry-instrumentation-google-generativeai/tests/cassettes/test_safety_cassettes/test_google_genai_safety_streaming_with_pii.yaml new file mode 100644 index 0000000000..9e87c5d58e --- /dev/null +++ b/packages/opentelemetry-instrumentation-google-generativeai/tests/cassettes/test_safety_cassettes/test_google_genai_safety_streaming_with_pii.yaml @@ -0,0 +1,60 @@ +interactions: +- request: + body: '{"contents": [{"parts": [{"text": "Acknowledge receipt of message from + [PII.email] about [custom_compliance.project-phoenix]."}], "role": "user"}], + "generationConfig": {"maxOutputTokens": 50}}' + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '192' + Content-Type: + - application/json + Host: + - generativelanguage.googleapis.com + user-agent: + - google-genai-sdk/1.58.0 gl-python/3.10.19 + x-goog-api-client: + - google-genai-sdk/1.58.0 gl-python/3.10.19 + method: POST + uri: https://generativelanguage.googleapis.com/v1beta/models/gemini-2.5-flash:streamGenerateContent?alt=sse + response: + body: + string: "data: {\"candidates\": [{\"content\": {\"parts\": [{\"text\": \"Subject\"}],\"role\": + \"model\"},\"finishReason\": \"MAX_TOKENS\",\"index\": 0}],\"usageMetadata\": + {\"promptTokenCount\": 25,\"candidatesTokenCount\": 1,\"totalTokenCount\": + 71,\"promptTokensDetails\": [{\"modality\": \"TEXT\",\"tokenCount\": 25}],\"thoughtsTokenCount\": + 45},\"modelVersion\": \"gemini-2.5-flash\",\"responseId\": \"adDMaY-5NfnijuMPnY6liAU\"}\r\n\r\n" + headers: + Alt-Svc: + - h3=":443"; ma=2592000,h3-29=":443"; ma=2592000 + Content-Disposition: + - attachment + Content-Type: + - text/event-stream + Date: + - Wed, 01 Apr 2026 07:59:38 GMT + Server: + - scaffolding on HTTPServer2 + Server-Timing: + - gfet4t7; dur=883 + Transfer-Encoding: + - chunked + Vary: + - Origin + - X-Origin + - Referer + X-Content-Type-Options: + - nosniff + X-Frame-Options: + - SAMEORIGIN + X-XSS-Protection: + - '0' + status: + code: 200 + message: OK +version: 1 diff --git a/packages/opentelemetry-instrumentation-google-generativeai/tests/cassettes/test_safety_cassettes/test_google_genai_safety_udf_project_codenames_prompt.yaml b/packages/opentelemetry-instrumentation-google-generativeai/tests/cassettes/test_safety_cassettes/test_google_genai_safety_udf_project_codenames_prompt.yaml new file mode 100644 index 0000000000..a7197eca3d --- /dev/null +++ b/packages/opentelemetry-instrumentation-google-generativeai/tests/cassettes/test_safety_cassettes/test_google_genai_safety_udf_project_codenames_prompt.yaml @@ -0,0 +1,64 @@ +interactions: +- request: + body: '{"contents": [{"parts": [{"text": "Prepare a status update for [custom_compliance.project-phoenix] + and [custom_compliance.operation-atlas] including timeline and budget."}], "role": + "user"}], "generationConfig": {"maxOutputTokens": 50}}' + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '236' + Content-Type: + - application/json + Host: + - generativelanguage.googleapis.com + user-agent: + - google-genai-sdk/1.58.0 gl-python/3.10.19 + x-goog-api-client: + - google-genai-sdk/1.58.0 gl-python/3.10.19 + method: POST + uri: https://generativelanguage.googleapis.com/v1beta/models/gemini-2.5-flash:generateContent + response: + body: + string: !!binary | + H4sIAAAAAAAC/2WQX0vDMBTF3/MpSp5X8c/m0LdhC4pWhxYZExnB3LVxaVKaW5iUfneT1nYp5iGE + e07uvefXkCCgX0xxwRmCobfBh60EQdPdTtMKQaEVhpItlqzCk7c/jfe2FoSj+0TvoQLqSe34/pyd + GlZagnMXmoMc7O1goHuhhMlfgRmtnC1ZbXbpy2P8/EZHj1AcjlY8J8OYbgCtDcsgAWQ2IBtj0LLS + RYmpPoC603UX8Oqib+bxmOiDjBqZnCjL5exfVxPZmUL6mDyCNiiTAn9cmjTepNSDgdOlBhrEg0Yx + 13WW43TB+YL8UetBvkNlRE8sg8IyDC/PFuFeMpN382gFptTKwAN3ni2PEsbWT9FNJOdhvP7e7q6L + 1YGSlvwCXxXP2icCAAA= + headers: + Alt-Svc: + - h3=":443"; ma=2592000,h3-29=":443"; ma=2592000 + Content-Encoding: + - gzip + Content-Type: + - application/json; charset=UTF-8 + Date: + - Wed, 01 Apr 2026 07:59:34 GMT + Server: + - scaffolding on HTTPServer2 + Server-Timing: + - gfet4t7; dur=1228 + Transfer-Encoding: + - chunked + Vary: + - Origin + - X-Origin + - Referer + X-Content-Type-Options: + - nosniff + X-Frame-Options: + - SAMEORIGIN + X-Gemini-Service-Tier: + - standard + X-XSS-Protection: + - '0' + status: + code: 200 + message: OK +version: 1 diff --git a/packages/opentelemetry-instrumentation-google-generativeai/tests/test_safety_cassettes.py b/packages/opentelemetry-instrumentation-google-generativeai/tests/test_safety_cassettes.py new file mode 100644 index 0000000000..0a2bd93fc0 --- /dev/null +++ b/packages/opentelemetry-instrumentation-google-generativeai/tests/test_safety_cassettes.py @@ -0,0 +1,544 @@ +""" +FR VCR cassette tests for safety rule enforcement on Google GenAI instrumentation. + +These tests record real Google GenAI API interactions via VCR cassettes and verify +that the safety callback system correctly detects and masks sensitive content +in both prompts and completions. + +Safety rule flavours covered: + - RegEx: PCI.credit_card, PII.email + - List: PII.competitor_org (list of company names) + - UDF: custom_compliance.project_codename (user-defined function + returning 2 sub-rule findings) + +North-star: This is a NEW file (FR-owned). Zero delta on TL files. +""" + +import re + +import pytest +from opentelemetry.instrumentation.fortifyroot import ( + SafetyFinding, + SafetyLocation, + SafetyResult, + clear_safety_handlers, + register_completion_safety_handler, + register_prompt_safety_handler, +) +from opentelemetry.semconv._incubating.attributes import ( + gen_ai_attributes as GenAIAttributes, +) + + +# --------------------------------------------------------------------------- +# Safety rule definitions (mirror what real SDK rules would do) +# --------------------------------------------------------------------------- + +# RegEx patterns +_CREDIT_CARD_RE = re.compile(r"\b(?:\d[ -]*?){13,19}\b") +_EMAIL_RE = re.compile(r"\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}\b") + +# List of competitor org names (case-insensitive) +_COMPETITOR_ORGS = ["acme corp", "globex industries", "initech", "umbrella inc"] + +# UDF: detects two fictional project codenames +_PROJECT_CODENAMES = ["project-phoenix", "operation-atlas"] + +# Module-level action setting (switched by fixtures) +_SAFETY_ACTION = "MASK" + + +def _build_findings_and_mask(text, findings_list, action="MASK"): + """Apply all collected findings to produce masked (or allowed) text.""" + if not findings_list: + return None + if action == "MASK": + masked = text + # Sort findings by start position descending so replacements don't shift indices + for f in sorted(findings_list, key=lambda x: x.start, reverse=True): + masked = masked[: f.start] + f"[{f.rule_name}]" + masked[f.end :] + return SafetyResult( + text=masked, + overall_action="MASK", + findings=findings_list, + ) + # ALLOW: text unchanged, findings still recorded + return SafetyResult( + text=text, + overall_action="ALLOW", + findings=findings_list, + ) + + +def _scan_text(text, location): + """Run all safety rule types against text, return findings.""" + if not isinstance(text, str) or not text: + return [] + + action = _SAFETY_ACTION + findings = [] + + # --- RegEx: PCI.credit_card --- + for m in _CREDIT_CARD_RE.finditer(text): + findings.append( + SafetyFinding( + category="PCI", + severity="HIGH", + action=action, + rule_name="PCI.credit_card", + start=m.start(), + end=m.end(), + ) + ) + + # --- RegEx: PII.email --- + for m in _EMAIL_RE.finditer(text): + findings.append( + SafetyFinding( + category="PII", + severity="MEDIUM", + action=action, + rule_name="PII.email", + start=m.start(), + end=m.end(), + ) + ) + + # --- List: PII.competitor_org --- + text_lower = text.lower() + for org in _COMPETITOR_ORGS: + idx = text_lower.find(org) + while idx != -1: + findings.append( + SafetyFinding( + category="PII", + severity="LOW", + action=action, + rule_name="PII.competitor_org", + start=idx, + end=idx + len(org), + ) + ) + idx = text_lower.find(org, idx + len(org)) + + # --- UDF: custom_compliance.project_codename (returns 2 sub-rules) --- + for codename in _PROJECT_CODENAMES: + idx = text_lower.find(codename) + while idx != -1: + findings.append( + SafetyFinding( + category="COMPLIANCE", + severity="HIGH", + action=action, + rule_name=f"custom_compliance.{codename}", + start=idx, + end=idx + len(codename), + ) + ) + idx = text_lower.find(codename, idx + len(codename)) + + return findings + + +def _prompt_handler(context): + """Prompt safety handler covering RegEx, List, and UDF rules.""" + if context.location != SafetyLocation.PROMPT: + return None + findings = _scan_text(context.text, SafetyLocation.PROMPT) + return _build_findings_and_mask(context.text, findings, action=_SAFETY_ACTION) + + +def _completion_handler(context): + """Completion safety handler covering RegEx, List, and UDF rules.""" + if context.location != SafetyLocation.COMPLETION: + return None + findings = _scan_text(context.text, SafetyLocation.COMPLETION) + return _build_findings_and_mask(context.text, findings, action=_SAFETY_ACTION) + + +# --------------------------------------------------------------------------- +# Fixtures +# --------------------------------------------------------------------------- + + +@pytest.fixture(autouse=True) +def _register_safety_handlers(): + """Register all safety handlers before each test, clear after.""" + clear_safety_handlers() + register_prompt_safety_handler(_prompt_handler) + register_completion_safety_handler(_completion_handler) + yield + clear_safety_handlers() + + +# --------------------------------------------------------------------------- +# T3-A3: RegEx rule — PII.email in prompt +# --------------------------------------------------------------------------- + + +@pytest.mark.vcr +@pytest.mark.fr +def test_google_genai_safety_regex_email_prompt(exporter, genai_client): + """Prompt contains an email address. Safety should detect and mask it.""" + exporter.clear() + genai_client.models.generate_content( + model="gemini-2.5-flash", + contents=( + "Summarize the following customer inquiry from " + "john.doe@example.com about their recent order." + ), + config={"max_output_tokens": 50}, + ) + + spans = exporter.get_finished_spans() + assert len(spans) >= 1 + span = spans[0] + + # The prompt content should have been masked (email replaced) + prompt_content = span.attributes.get("gen_ai.prompt.0.content") + assert prompt_content is not None + assert "john.doe@example.com" not in prompt_content + assert "[PII.email]" in prompt_content + + # Safety finding should be recorded as a span event + safety_events = [ + e for e in span.events if "fortifyroot.safety" in (e.name or "") + ] + assert len(safety_events) >= 1 + event_attrs = safety_events[0].attributes + assert event_attrs["fortifyroot.safety.rule_name"] == "PII.email" + assert event_attrs["fortifyroot.safety.action"] == "MASK" + assert event_attrs["fortifyroot.safety.location"] == "PROMPT" + + +# --------------------------------------------------------------------------- +# T3-A3: RegEx rule — PCI.credit_card in prompt +# --------------------------------------------------------------------------- + + +@pytest.mark.vcr +@pytest.mark.fr +def test_google_genai_safety_regex_credit_card_prompt(exporter, genai_client): + """Prompt contains a credit card number. Safety should detect and mask it.""" + exporter.clear() + genai_client.models.generate_content( + model="gemini-2.5-flash", + contents="Check if order for card 4111 1111 1111 1111 was processed.", + config={"max_output_tokens": 50}, + ) + + spans = exporter.get_finished_spans() + assert len(spans) >= 1 + span = spans[0] + + prompt_content = span.attributes.get("gen_ai.prompt.0.content") + assert prompt_content is not None + assert "4111 1111 1111 1111" not in prompt_content + assert "[PCI.credit_card]" in prompt_content + + safety_events = [ + e for e in span.events if "fortifyroot.safety" in (e.name or "") + ] + assert len(safety_events) >= 1 + found_cc = any( + e.attributes.get("fortifyroot.safety.rule_name") == "PCI.credit_card" + for e in safety_events + ) + assert found_cc, "Expected PCI.credit_card safety finding" + + +# --------------------------------------------------------------------------- +# T3-A3: List rule — PII.competitor_org in prompt +# --------------------------------------------------------------------------- + + +@pytest.mark.vcr +@pytest.mark.fr +def test_google_genai_safety_list_competitor_org_prompt(exporter, genai_client): + """Prompt mentions a competitor org from the blocklist.""" + exporter.clear() + genai_client.models.generate_content( + model="gemini-2.5-flash", + contents=( + "Compare our product roadmap with Acme Corp's latest " + "offering and Globex Industries' pricing strategy." + ), + config={"max_output_tokens": 50}, + ) + + spans = exporter.get_finished_spans() + assert len(spans) >= 1 + span = spans[0] + + prompt_content = span.attributes.get("gen_ai.prompt.0.content") + assert prompt_content is not None + assert "acme corp" not in prompt_content.lower() + assert "globex industries" not in prompt_content.lower() + assert "[PII.competitor_org]" in prompt_content + + safety_events = [ + e for e in span.events if "fortifyroot.safety" in (e.name or "") + ] + found_org = any( + e.attributes.get("fortifyroot.safety.rule_name") == "PII.competitor_org" + for e in safety_events + ) + assert found_org, "Expected PII.competitor_org safety finding" + + +# --------------------------------------------------------------------------- +# T3-A3: UDF rule — custom_compliance with 2 sub-rules in prompt +# --------------------------------------------------------------------------- + + +@pytest.mark.vcr +@pytest.mark.fr +def test_google_genai_safety_udf_project_codenames_prompt(exporter, genai_client): + """Prompt contains two project codenames detected by UDF rule.""" + exporter.clear() + genai_client.models.generate_content( + model="gemini-2.5-flash", + contents=( + "Prepare a status update for Project-Phoenix and " + "Operation-Atlas including timeline and budget." + ), + config={"max_output_tokens": 50}, + ) + + spans = exporter.get_finished_spans() + assert len(spans) >= 1 + span = spans[0] + + prompt_content = span.attributes.get("gen_ai.prompt.0.content") + assert prompt_content is not None + assert "[custom_compliance.project-phoenix]" in prompt_content + assert "[custom_compliance.operation-atlas]" in prompt_content + assert "Project-Phoenix" not in prompt_content + assert "Operation-Atlas" not in prompt_content + + # UDF should produce 2 findings (one per codename) + safety_events = [ + e for e in span.events if "fortifyroot.safety" in (e.name or "") + ] + udf_findings = [ + e + for e in safety_events + if (e.attributes.get("fortifyroot.safety.rule_name") or "").startswith( + "custom_compliance." + ) + ] + assert len(udf_findings) >= 2, ( + f"Expected 2 UDF findings, got {len(udf_findings)}" + ) + + +# --------------------------------------------------------------------------- +# T3-A3: Completion safety — email in LLM response +# --------------------------------------------------------------------------- + + +@pytest.mark.vcr +@pytest.mark.fr +def test_google_genai_safety_completion_email_masked(exporter, genai_client): + """ + Ask the LLM to generate fictional contact info. The response should + contain an email-like pattern that completion safety detects and masks. + """ + exporter.clear() + genai_client.models.generate_content( + model="gemini-2.5-flash", + contents=( + "Reply with ONLY this text, filling in the blanks:\n" + "Email: jane.smith@widgetsinc.com\n" + "Phone: 555-0142\n" + "Do not add anything else." + ), + config={ + "max_output_tokens": 200, + "thinking_config": {"thinking_budget": 0}, + }, + ) + + spans = exporter.get_finished_spans() + assert len(spans) >= 1 + span = spans[0] + + # Completion should be recorded AND the LLM-generated email should be + # masked by completion safety + completion_content = span.attributes.get("gen_ai.completion.0.content") + assert completion_content is not None, "Completion content should be recorded" + assert "[PII.email]" in completion_content, ( + "LLM-generated email in response should be masked by completion safety" + ) + + # Verify safety events are present. The primary assertion is the + # masking above; span events confirm the instrumentation layer + # recorded the finding metadata. + safety_events = [ + e for e in span.events if "fortifyroot.safety" in (e.name or "") + ] + # At minimum, the prompt safety handler fires (because the prompt + # includes the email template text). Completion masking is verified + # by the [PII.email] assertion on completion_content above. + assert len(safety_events) >= 1, "Should have at least one safety finding" + + +# --------------------------------------------------------------------------- +# T3-A3: Combined — multiple rule types in single prompt +# --------------------------------------------------------------------------- + + +@pytest.mark.vcr +@pytest.mark.fr +def test_google_genai_safety_combined_rules_prompt(exporter, genai_client): + """Prompt triggers all rule types: RegEx (email), List (org), UDF (codename).""" + exporter.clear() + genai_client.models.generate_content( + model="gemini-2.5-flash", + contents=( + "Send the Project-Phoenix briefing to contact@acme-corp.com " + "and CC the Initech team." + ), + config={"max_output_tokens": 50}, + ) + + spans = exporter.get_finished_spans() + assert len(spans) >= 1 + span = spans[0] + + prompt_content = span.attributes.get("gen_ai.prompt.0.content") + assert prompt_content is not None + + # Verify all 3 rule types fired + safety_events = [ + e for e in span.events if "fortifyroot.safety" in (e.name or "") + ] + rule_names = { + e.attributes.get("fortifyroot.safety.rule_name") for e in safety_events + } + assert "PII.email" in rule_names, "Expected PII.email finding" + assert "PII.competitor_org" in rule_names or any( + "competitor_org" in (r or "") for r in rule_names + ), "Expected PII.competitor_org finding" + assert any( + (r or "").startswith("custom_compliance.") for r in rule_names + ), "Expected custom_compliance UDF finding" + + +# --------------------------------------------------------------------------- +# T3-A3: Streaming with PII — verify safety on streamed completion +# --------------------------------------------------------------------------- + + +@pytest.mark.vcr +@pytest.mark.fr +def test_google_genai_safety_streaming_with_pii(exporter, genai_client): + """Streaming chat with PII in prompt. Verify prompt safety applied.""" + exporter.clear() + response = genai_client.models.generate_content_stream( + model="gemini-2.5-flash", + contents=( + "Acknowledge receipt of message from " + "alice.jones@example.org about Project-Phoenix." + ), + config={"max_output_tokens": 50}, + ) + + # Consume the stream + for _ in response: + pass + + spans = exporter.get_finished_spans() + assert len(spans) >= 1 + span = spans[0] + + # Prompt should be masked + prompt_content = span.attributes.get("gen_ai.prompt.0.content") + assert prompt_content is not None + assert "alice.jones@example.org" not in prompt_content + assert "[PII.email]" in prompt_content + + +# =========================================================================== +# ALLOW action tests +# =========================================================================== + + +@pytest.fixture +def _set_allow_action(): + """Switch safety handlers to ALLOW mode for the test.""" + global _SAFETY_ACTION + _SAFETY_ACTION = "ALLOW" + yield + _SAFETY_ACTION = "MASK" + + +@pytest.mark.vcr +@pytest.mark.fr +def test_google_genai_safety_allow_email_not_masked( + _set_allow_action, exporter, genai_client +): + """ALLOW action: email detected but text passes through unchanged.""" + exporter.clear() + genai_client.models.generate_content( + model="gemini-2.5-flash", + contents=( + "Forward this message from alice.jones@example.org " + "to the support team." + ), + config={"max_output_tokens": 50}, + ) + + spans = exporter.get_finished_spans() + assert len(spans) >= 1 + span = spans[0] + + # With ALLOW, the original text should pass through unchanged + prompt_content = span.attributes.get("gen_ai.prompt.0.content") + assert prompt_content is not None + assert "alice.jones@example.org" in prompt_content + assert "[PII.email]" not in prompt_content + + # But findings should STILL be recorded as span events + safety_events = [ + e for e in span.events if "fortifyroot.safety" in (e.name or "") + ] + assert len(safety_events) >= 1, "ALLOW should still emit safety findings" + event_attrs = safety_events[0].attributes + assert event_attrs["fortifyroot.safety.rule_name"] == "PII.email" + assert event_attrs["fortifyroot.safety.action"] == "ALLOW" + assert event_attrs["fortifyroot.safety.location"] == "PROMPT" + + +@pytest.mark.vcr +@pytest.mark.fr +def test_google_genai_safety_allow_credit_card_not_masked( + _set_allow_action, exporter, genai_client +): + """ALLOW action: credit card detected but text passes through unchanged.""" + exporter.clear() + genai_client.models.generate_content( + model="gemini-2.5-flash", + contents="Verify payment for card 4111 1111 1111 1111 was received.", + config={"max_output_tokens": 50}, + ) + + spans = exporter.get_finished_spans() + assert len(spans) >= 1 + span = spans[0] + + # With ALLOW, text unchanged — card number still visible + prompt_content = span.attributes.get("gen_ai.prompt.0.content") + assert prompt_content is not None + assert "4111 1111 1111 1111" in prompt_content + + # Findings still recorded + safety_events = [ + e for e in span.events if "fortifyroot.safety" in (e.name or "") + ] + found_cc = any( + e.attributes.get("fortifyroot.safety.rule_name") == "PCI.credit_card" + and e.attributes.get("fortifyroot.safety.action") == "ALLOW" + for e in safety_events + ) + assert found_cc, "Expected PCI.credit_card finding with ALLOW action"