diff --git a/packages/opentelemetry-instrumentation-anthropic/tests/cassettes/test_safety_cassettes/test_anthropic_safety_allow_credit_card_not_masked.yaml b/packages/opentelemetry-instrumentation-anthropic/tests/cassettes/test_safety_cassettes/test_anthropic_safety_allow_credit_card_not_masked.yaml new file mode 100644 index 0000000000..b25c181a1d --- /dev/null +++ b/packages/opentelemetry-instrumentation-anthropic/tests/cassettes/test_safety_cassettes/test_anthropic_safety_allow_credit_card_not_masked.yaml @@ -0,0 +1,118 @@ +interactions: +- request: + body: '{"max_tokens":50,"messages":[{"role":"user","content":"Verify payment for + card 4111 1111 1111 1111 was received."}],"model":"claude-3-haiku-20240307"}' + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '150' + Content-Type: + - application/json + Host: + - api.anthropic.com + User-Agent: + - Anthropic/Python 0.76.0 + X-Stainless-Arch: + - x64 + X-Stainless-Async: + - 'false' + X-Stainless-Lang: + - python + X-Stainless-OS: + - MacOS + X-Stainless-Package-Version: + - 0.76.0 + X-Stainless-Runtime: + - CPython + X-Stainless-Runtime-Version: + - 3.10.19 + anthropic-version: + - '2023-06-01' + x-stainless-read-timeout: + - '600' + x-stainless-retry-count: + - '0' + x-stainless-timeout: + - '600' + method: POST + uri: https://api.anthropic.com/v1/messages + response: + body: + string: !!binary | + H4sIAAAAAAAAA3RS0W7bMAz8FUIve3EKN23QIW8Z0gFB9wfDYNAybQuRJVWijHpB/n1UMGNbhz5J + 4B2PxwMvavIdWbVX2mLuaPOwGdGc82Zbbx/rh/pJVcp0Ak9paOr7x5f5+Ly44dvz05fz4ejx2LZf + d8LhJVBhUUo4kBSit6WAKZnE6FhK2jsm+e2/X1Y+01tBbs9enQCDt34wP6mCNjOcoPPgPMOIMwFq + LfLAHtAtEAktBFwmkYQQfcGMGyAtiWlK4CPonNhPFKE3Dp020mBc7+OEbLy7g9OnSaTgIHNXm6BF + l6mDdoGD4zH6YHQZ2RKMZEOfbSVu4mRlXCXdHYzeUeK7d2Z5JNAYsDXW8FIUZoqmX8SXuv6olDgL + jcxK3pXc8K1hfyaX1G8o0WsmpyUkl62tVL7lur8o40LmlbzffpZcUY/U3IzLWs2/hHrFBe4+wtbe + ok9hJMkMbbOb/uf/Qe/H9+i1Uj7z36WddCSKs9HUsKEom5aUO4xduSvXUyw7NgN5gSS7Bmc0Flu5 + nev1FwAAAP//AwDi6o2fnAIAAA== + headers: + CF-RAY: + - 9e4d34aaee677fa8-BLR + Connection: + - keep-alive + Content-Encoding: + - gzip + Content-Security-Policy: + - default-src 'none'; frame-ancestors 'none' + Content-Type: + - application/json + Date: + - Tue, 31 Mar 2026 06:13:02 GMT + Server: + - cloudflare + Transfer-Encoding: + - chunked + X-Robots-Tag: + - none + anthropic-organization-id: + - 887534b2-e0d2-4865-a85b-8a07958e7c46 + anthropic-ratelimit-input-tokens-limit: + - '50000' + anthropic-ratelimit-input-tokens-remaining: + - '50000' + anthropic-ratelimit-input-tokens-reset: + - '2026-03-31T06:13:01Z' + anthropic-ratelimit-output-tokens-limit: + - '10000' + anthropic-ratelimit-output-tokens-remaining: + - '10000' + anthropic-ratelimit-output-tokens-reset: + - '2026-03-31T06:13:02Z' + anthropic-ratelimit-requests-limit: + - '50' + anthropic-ratelimit-requests-remaining: + - '48' + anthropic-ratelimit-requests-reset: + - '2026-03-31T06:13:02Z' + anthropic-ratelimit-tokens-limit: + - '60000' + anthropic-ratelimit-tokens-remaining: + - '60000' + anthropic-ratelimit-tokens-reset: + - '2026-03-31T06:13:01Z' + cf-cache-status: + - DYNAMIC + request-id: + - req_011CZafVUKwN9gFxw7UDxLfb + server-timing: + - x-originResponse;dur=664 + set-cookie: + - _cfuvid=_LqbATMrwrdPtwuB3diEWIw1_Db_X3VgftFJ_J8o95A-1774937581.2661583-1.0.1.1-C.nMTHV_w6cG3L3L878I5YnBW_8X60tZ.yiogrCyu0U; + HttpOnly; SameSite=None; Secure; Path=/; Domain=api.anthropic.com + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + vary: + - Accept-Encoding + x-envoy-upstream-service-time: + - '661' + status: + code: 200 + message: OK +version: 1 diff --git a/packages/opentelemetry-instrumentation-anthropic/tests/cassettes/test_safety_cassettes/test_anthropic_safety_allow_email_not_masked.yaml b/packages/opentelemetry-instrumentation-anthropic/tests/cassettes/test_safety_cassettes/test_anthropic_safety_allow_email_not_masked.yaml new file mode 100644 index 0000000000..c97d749949 --- /dev/null +++ b/packages/opentelemetry-instrumentation-anthropic/tests/cassettes/test_safety_cassettes/test_anthropic_safety_allow_email_not_masked.yaml @@ -0,0 +1,118 @@ +interactions: +- request: + body: '{"max_tokens":50,"messages":[{"role":"user","content":"Forward this message + from alice.jones@example.org to the support team."}],"model":"claude-3-haiku-20240307"}' + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '163' + Content-Type: + - application/json + Host: + - api.anthropic.com + User-Agent: + - Anthropic/Python 0.76.0 + X-Stainless-Arch: + - x64 + X-Stainless-Async: + - 'false' + X-Stainless-Lang: + - python + X-Stainless-OS: + - MacOS + X-Stainless-Package-Version: + - 0.76.0 + X-Stainless-Runtime: + - CPython + X-Stainless-Runtime-Version: + - 3.10.19 + anthropic-version: + - '2023-06-01' + x-stainless-read-timeout: + - '600' + x-stainless-retry-count: + - '0' + x-stainless-timeout: + - '600' + method: POST + uri: https://api.anthropic.com/v1/messages + response: + body: + string: !!binary | + H4sIAAAAAAAAA3RSy27bMBD8lQXPciC/UFS3HHrwrWjr5lAEwopaW2worsKHEsHwv2cpVK2boieS + M7OzL15Uzy1ZVSltMbW02q46NE9ptSk3u3JbflCFMq3QfTjX5frzw8C7n/v1x+PX3fOX8K359P3Y + NKKJ00BZRSHgmQTwbDOAIZgQ0UWBNLtIcqt+XBZ9pNfMzEelju7EPiaHkexUwAFaBscROhwJYkeg + ccDGWBMniAwifkHfAvVobLiD+wDo4P4Av5O+90CtpcAcO8fkNycXA7Cf/W/MA7nFGdjBxMlDQx3a + 052YaknEzk4weB5NS2By6T1GI1qUyDmdNDySDzMq9R1O2QYcUXtbPjp1fSxUiDzUnjCwy4PE1zry + E7mgflGBnhM5LVNzydpCpXnQ1UUZN6S4iKvNVgaNuqNai1nOXP8tKBde6PZ/3BKb/WnoqCePtt73 + /+r/sOvuPXstFKd4C+0lIpAfjaY6GvLSaV5UK3PIH82dyOce6zOxULK3GkdZATbyma7XNwAAAP// + AwCtDvlKrQIAAA== + headers: + CF-RAY: + - 9e4d34a469da8a2a-BLR + Connection: + - keep-alive + Content-Encoding: + - gzip + Content-Security-Policy: + - default-src 'none'; frame-ancestors 'none' + Content-Type: + - application/json + Date: + - Tue, 31 Mar 2026 06:13:01 GMT + Server: + - cloudflare + Transfer-Encoding: + - chunked + X-Robots-Tag: + - none + anthropic-organization-id: + - 887534b2-e0d2-4865-a85b-8a07958e7c46 + anthropic-ratelimit-input-tokens-limit: + - '50000' + anthropic-ratelimit-input-tokens-remaining: + - '50000' + anthropic-ratelimit-input-tokens-reset: + - '2026-03-31T06:13:00Z' + anthropic-ratelimit-output-tokens-limit: + - '10000' + anthropic-ratelimit-output-tokens-remaining: + - '10000' + anthropic-ratelimit-output-tokens-reset: + - '2026-03-31T06:13:01Z' + anthropic-ratelimit-requests-limit: + - '50' + anthropic-ratelimit-requests-remaining: + - '49' + anthropic-ratelimit-requests-reset: + - '2026-03-31T06:13:01Z' + anthropic-ratelimit-tokens-limit: + - '60000' + anthropic-ratelimit-tokens-remaining: + - '60000' + anthropic-ratelimit-tokens-reset: + - '2026-03-31T06:13:00Z' + cf-cache-status: + - DYNAMIC + request-id: + - req_011CZafVPtHpCohauZwKR2uf + server-timing: + - x-originResponse;dur=699 + set-cookie: + - _cfuvid=hwpmmxZs304c9UBNXrEWKwcX9h4RIW2qkqWTb.X9sko-1774937580.228889-1.0.1.1-77QFivims_jEhtOeiEwtZj9DiydwPgeaY70epwhRg44; + HttpOnly; SameSite=None; Secure; Path=/; Domain=api.anthropic.com + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + vary: + - Accept-Encoding + x-envoy-upstream-service-time: + - '697' + status: + code: 200 + message: OK +version: 1 diff --git a/packages/opentelemetry-instrumentation-anthropic/tests/cassettes/test_safety_cassettes/test_anthropic_safety_combined_rules_prompt.yaml b/packages/opentelemetry-instrumentation-anthropic/tests/cassettes/test_safety_cassettes/test_anthropic_safety_combined_rules_prompt.yaml new file mode 100644 index 0000000000..8ed3d0ef89 --- /dev/null +++ b/packages/opentelemetry-instrumentation-anthropic/tests/cassettes/test_safety_cassettes/test_anthropic_safety_combined_rules_prompt.yaml @@ -0,0 +1,118 @@ +interactions: +- request: + body: '{"max_tokens":50,"messages":[{"role":"user","content":"Send the [custom_compliance.project-phoenix] + briefing to [PII.email] and CC the [PII.competitor_org] team."}],"model":"claude-3-haiku-20240307"}' + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '199' + Content-Type: + - application/json + Host: + - api.anthropic.com + User-Agent: + - Anthropic/Python 0.76.0 + X-Stainless-Arch: + - x64 + X-Stainless-Async: + - 'false' + X-Stainless-Lang: + - python + X-Stainless-OS: + - MacOS + X-Stainless-Package-Version: + - 0.76.0 + X-Stainless-Runtime: + - CPython + X-Stainless-Runtime-Version: + - 3.10.19 + anthropic-version: + - '2023-06-01' + x-stainless-read-timeout: + - '600' + x-stainless-retry-count: + - '0' + x-stainless-timeout: + - '600' + method: POST + uri: https://api.anthropic.com/v1/messages + response: + body: + string: !!binary | + H4sIAAAAAAAAA3RS227bMAz9FULPTuCmDQb4dcWA7GlAh2HdMBiMRMdEZMmV6KRpkH8f5S3rLtiT + ZZ4LySOdzRAdedMY63FytLhd9Mj7abGqV3f1bf3GVIadwkPetfXN2/efnh6m1ecO84M8Dvf3776s + H1fKkdNIhUU54460kKIvBcyZs2AQLdkYhPTUfD1f+ULPBZk/jdkAjtHHHb9QBdtJYAMWQ4gCmYID + 1Xfs1IHRA4cupgGFY4CYYKSUY0DvT/CD0jFuPYFDQTiy9FHtxhSVBzjpb+KXWbyEjz0KHOPkHWwJ + ELaJ0PYQO+XzAe0JUJvPRmogZOeeOihbprx8HdLGYfQkBNJzhkRPE2UBzKABCLklfPCEmUA5MBDs + QzwCd8qmRKAKVBM9wxFPsDGXb5XJEsdWx9HVSrj43ErcU8jmJ5RLj2A1yTB5X5lpDr85Gw7jJFdy + c1dr+LoTtVbNyvjtn4RfuMLuf9hVW/xp7GmghL5dD//yX9Gb/m/0Uhm9it9La1VkSge21ApT0k3L + i3GYXHl8odN4dMd2R1EhzajFA7Iv12sul+8AAAD//wMAMNgwMcECAAA= + headers: + CF-RAY: + - 9e4cd943ebe1b22e-BLR + Connection: + - keep-alive + Content-Encoding: + - gzip + Content-Security-Policy: + - default-src 'none'; frame-ancestors 'none' + Content-Type: + - application/json + Date: + - Tue, 31 Mar 2026 05:10:38 GMT + Server: + - cloudflare + Transfer-Encoding: + - chunked + X-Robots-Tag: + - none + anthropic-organization-id: + - 887534b2-e0d2-4865-a85b-8a07958e7c46 + anthropic-ratelimit-input-tokens-limit: + - '50000' + anthropic-ratelimit-input-tokens-remaining: + - '50000' + anthropic-ratelimit-input-tokens-reset: + - '2026-03-31T05:10:37Z' + anthropic-ratelimit-output-tokens-limit: + - '10000' + anthropic-ratelimit-output-tokens-remaining: + - '10000' + anthropic-ratelimit-output-tokens-reset: + - '2026-03-31T05:10:38Z' + anthropic-ratelimit-requests-limit: + - '50' + anthropic-ratelimit-requests-remaining: + - '48' + anthropic-ratelimit-requests-reset: + - '2026-03-31T05:10:39Z' + anthropic-ratelimit-tokens-limit: + - '60000' + anthropic-ratelimit-tokens-remaining: + - '60000' + anthropic-ratelimit-tokens-reset: + - '2026-03-31T05:10:37Z' + cf-cache-status: + - DYNAMIC + request-id: + - req_011CZaajW6f3QeKp6ztiZgNR + server-timing: + - x-originResponse;dur=608 + set-cookie: + - _cfuvid=mMY_0NE_wfM8aY65GDzEg8HhNDcPGaFMTL3U79Tvw24-1774933837.429631-1.0.1.1-l6yg4p8u2vaUAmw96B7q2K11Xo_0PQF9ouobZdpgmZU; + HttpOnly; SameSite=None; Secure; Path=/; Domain=api.anthropic.com + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + vary: + - Accept-Encoding + x-envoy-upstream-service-time: + - '607' + status: + code: 200 + message: OK +version: 1 diff --git a/packages/opentelemetry-instrumentation-anthropic/tests/cassettes/test_safety_cassettes/test_anthropic_safety_completion_email_masked.yaml b/packages/opentelemetry-instrumentation-anthropic/tests/cassettes/test_safety_cassettes/test_anthropic_safety_completion_email_masked.yaml new file mode 100644 index 0000000000..6dde54c4af --- /dev/null +++ b/packages/opentelemetry-instrumentation-anthropic/tests/cassettes/test_safety_cassettes/test_anthropic_safety_completion_email_masked.yaml @@ -0,0 +1,119 @@ +interactions: +- request: + body: '{"max_tokens":100,"messages":[{"role":"user","content":"Create a fictional + contact card. The person''s name is Jane Smith, company is Widgets Inc, role + is VP of Sales. Invent a plausible work email and phone. Format as:\nName: ...\nEmail: + ...\nPhone: ..."}],"model":"claude-3-haiku-20240307"}' + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '291' + Content-Type: + - application/json + Host: + - api.anthropic.com + User-Agent: + - Anthropic/Python 0.76.0 + X-Stainless-Arch: + - x64 + X-Stainless-Async: + - 'false' + X-Stainless-Lang: + - python + X-Stainless-OS: + - MacOS + X-Stainless-Package-Version: + - 0.76.0 + X-Stainless-Runtime: + - CPython + X-Stainless-Runtime-Version: + - 3.10.19 + anthropic-version: + - '2023-06-01' + x-stainless-read-timeout: + - '600' + x-stainless-retry-count: + - '0' + x-stainless-timeout: + - '600' + method: POST + uri: https://api.anthropic.com/v1/messages + response: + body: + string: !!binary | + H4sIAAAAAAAAA3SRX0/CMBTFv0rTJ002MhjD2CcTMEYSDYrRB2easl1YZb2dawcSwnf3DsW/8anN + +Z3T3NO75cbmUHLBs1I1OYRxWCi9bMJe1OtHcXTCA65zwsYtZNQdVr2r5Wx8cQqjZTK+GSHq5G5N + Hr+poHWBc2oBJNS2bAXlnHZeoScps+iBbuJxe/B7eG3J/hD8WhkQbKwQ2NRoX6R4bpQuBXsmqeNa + 6Wyt8wV4F2rMOpk1KU4Ki5Q6SpLkmHV7cdhPBicpDq2pFG4Ee3gPsEsKpHhLYwl2P2F2zqaqBMd3 + TwF33layBuUs0hiAufRNjfwDOHhpADOaF5uyDHizryi2XGPVeOntEtBxMYioosoKkBk95bVF+dPw + yQnn/7FDtn0fqgIM1KqUifnr/6Ld4jfdBdw2/rvUj6kN1CudgfQaaurZ7iVXdd6uGOdQtx3lAiwh + tF6qFf29mtEad7s3AAAA//8DACGJtMonAgAA + headers: + CF-RAY: + - 9e4d64bb2e0c343c-BLR + Connection: + - keep-alive + Content-Encoding: + - gzip + Content-Security-Policy: + - default-src 'none'; frame-ancestors 'none' + Content-Type: + - application/json + Date: + - Tue, 31 Mar 2026 06:45:52 GMT + Server: + - cloudflare + Transfer-Encoding: + - chunked + X-Robots-Tag: + - none + anthropic-organization-id: + - 887534b2-e0d2-4865-a85b-8a07958e7c46 + anthropic-ratelimit-input-tokens-limit: + - '50000' + anthropic-ratelimit-input-tokens-remaining: + - '50000' + anthropic-ratelimit-input-tokens-reset: + - '2026-03-31T06:45:51Z' + anthropic-ratelimit-output-tokens-limit: + - '10000' + anthropic-ratelimit-output-tokens-remaining: + - '10000' + anthropic-ratelimit-output-tokens-reset: + - '2026-03-31T06:45:52Z' + anthropic-ratelimit-requests-limit: + - '50' + anthropic-ratelimit-requests-remaining: + - '49' + anthropic-ratelimit-requests-reset: + - '2026-03-31T06:45:52Z' + anthropic-ratelimit-tokens-limit: + - '60000' + anthropic-ratelimit-tokens-remaining: + - '60000' + anthropic-ratelimit-tokens-reset: + - '2026-03-31T06:45:51Z' + cf-cache-status: + - DYNAMIC + request-id: + - req_011CZahzbVb1AiRvw4Vt4RMN + server-timing: + - x-originResponse;dur=2117 + set-cookie: + - _cfuvid=tTEqPPU6i6l3LWpP9vF5khnfa52Kpuj5ZM_qs1a9r_Y-1774939549.9445589-1.0.1.1-NmWwuMGTfSoZLFOOIxF0Ms9CS0TjqCfjcUuY20Im7l4; + HttpOnly; SameSite=None; Secure; Path=/; Domain=api.anthropic.com + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + vary: + - Accept-Encoding + x-envoy-upstream-service-time: + - '2115' + status: + code: 200 + message: OK +version: 1 diff --git a/packages/opentelemetry-instrumentation-anthropic/tests/cassettes/test_safety_cassettes/test_anthropic_safety_list_competitor_org_prompt.yaml b/packages/opentelemetry-instrumentation-anthropic/tests/cassettes/test_safety_cassettes/test_anthropic_safety_list_competitor_org_prompt.yaml new file mode 100644 index 0000000000..b39b1a1a33 --- /dev/null +++ b/packages/opentelemetry-instrumentation-anthropic/tests/cassettes/test_safety_cassettes/test_anthropic_safety_list_competitor_org_prompt.yaml @@ -0,0 +1,119 @@ +interactions: +- request: + body: '{"max_tokens":50,"messages":[{"role":"user","content":"Compare our product + roadmap with [PII.competitor_org]''s latest offering and [PII.competitor_org]'' + pricing strategy."}],"model":"claude-3-haiku-20240307"}' + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '208' + Content-Type: + - application/json + Host: + - api.anthropic.com + User-Agent: + - Anthropic/Python 0.76.0 + X-Stainless-Arch: + - x64 + X-Stainless-Async: + - 'false' + X-Stainless-Lang: + - python + X-Stainless-OS: + - MacOS + X-Stainless-Package-Version: + - 0.76.0 + X-Stainless-Runtime: + - CPython + X-Stainless-Runtime-Version: + - 3.10.19 + anthropic-version: + - '2023-06-01' + x-stainless-read-timeout: + - '600' + x-stainless-retry-count: + - '0' + x-stainless-timeout: + - '600' + method: POST + uri: https://api.anthropic.com/v1/messages + response: + body: + string: !!binary | + H4sIAAAAAAAAA3SSQWsbMRCF/8qgSy7rYMcNDT72UHBvhYRSSlnG0mRXRKvZaEaOXeP/3lGoadPS + k8S8T09Pozm5iQMlt3E+YQ20WC9GjE91cbO8ebdcL9+7zsVg8iRDv1w93B3984cXLJ+/fnnwHwf/ + 6XC/qsbocaZGkQgOZIXCqRVQJIpiVit5zkq223w7XXilQ1Nel43bAs6ceIg/qINdVdiCx5xZYS68 + j4EA8xFifuQyoUbOgDs27Mi1gOdpJo3KRa4aH6pX6WwXfcxDB1ygMIYJZ7mG+xEVXrimYHZ7TnuC + EMUnFmNh5pYzYkpHEMoSNRpgBmZrfqRY3sbQZreFwPlKYUSD0XtrBShb5AAytquaKCMWujZW6jCQ + KDyyr+1Sd/7eOVGe+0IonFsz8dArP1kA90sSeq6UvXUu15Q6V1+bvTm5mOeqF3izvrNmox+p92bW + EvZvgeVFNzn8T7ucbf40jzRRwdTfTv/yv9XV+Ld67px90Z+lWzshVPbRU6+Rir20TUjAEtqw5Ucq + 7Y39QGyS/X6Pe4wJdzZQ5/NPAAAA//8DAPpjy4qxAgAA + headers: + CF-RAY: + - 9e4cd92b3aefb155-BLR + Connection: + - keep-alive + Content-Encoding: + - gzip + Content-Security-Policy: + - default-src 'none'; frame-ancestors 'none' + Content-Type: + - application/json + Date: + - Tue, 31 Mar 2026 05:10:35 GMT + Server: + - cloudflare + Transfer-Encoding: + - chunked + X-Robots-Tag: + - none + anthropic-organization-id: + - 887534b2-e0d2-4865-a85b-8a07958e7c46 + anthropic-ratelimit-input-tokens-limit: + - '50000' + anthropic-ratelimit-input-tokens-remaining: + - '50000' + anthropic-ratelimit-input-tokens-reset: + - '2026-03-31T05:10:34Z' + anthropic-ratelimit-output-tokens-limit: + - '10000' + anthropic-ratelimit-output-tokens-remaining: + - '10000' + anthropic-ratelimit-output-tokens-reset: + - '2026-03-31T05:10:35Z' + anthropic-ratelimit-requests-limit: + - '50' + anthropic-ratelimit-requests-remaining: + - '48' + anthropic-ratelimit-requests-reset: + - '2026-03-31T05:10:35Z' + anthropic-ratelimit-tokens-limit: + - '60000' + anthropic-ratelimit-tokens-remaining: + - '60000' + anthropic-ratelimit-tokens-reset: + - '2026-03-31T05:10:34Z' + cf-cache-status: + - DYNAMIC + request-id: + - req_011CZaajDgviwhp3rrv2tmTs + server-timing: + - x-originResponse;dur=1325 + set-cookie: + - _cfuvid=JOFb40v9vDB20B5W9NlBFhSgA1lP5YIPHGWr.uxf6gA-1774933833.4787397-1.0.1.1-rOKcKvyrplc6PkaY8cRnHdpbHigbf1AcRHoCLpYErjk; + HttpOnly; SameSite=None; Secure; Path=/; Domain=api.anthropic.com + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + vary: + - Accept-Encoding + x-envoy-upstream-service-time: + - '1323' + status: + code: 200 + message: OK +version: 1 diff --git a/packages/opentelemetry-instrumentation-anthropic/tests/cassettes/test_safety_cassettes/test_anthropic_safety_regex_credit_card_prompt.yaml b/packages/opentelemetry-instrumentation-anthropic/tests/cassettes/test_safety_cassettes/test_anthropic_safety_regex_credit_card_prompt.yaml new file mode 100644 index 0000000000..0a1b021f94 --- /dev/null +++ b/packages/opentelemetry-instrumentation-anthropic/tests/cassettes/test_safety_cassettes/test_anthropic_safety_regex_credit_card_prompt.yaml @@ -0,0 +1,118 @@ +interactions: +- request: + body: '{"max_tokens":50,"messages":[{"role":"user","content":"Check if order for + card [PCI.credit_card] was processed."}],"model":"claude-3-haiku-20240307"}' + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '149' + Content-Type: + - application/json + Host: + - api.anthropic.com + User-Agent: + - Anthropic/Python 0.76.0 + X-Stainless-Arch: + - x64 + X-Stainless-Async: + - 'false' + X-Stainless-Lang: + - python + X-Stainless-OS: + - MacOS + X-Stainless-Package-Version: + - 0.76.0 + X-Stainless-Runtime: + - CPython + X-Stainless-Runtime-Version: + - 3.10.19 + anthropic-version: + - '2023-06-01' + x-stainless-read-timeout: + - '600' + x-stainless-retry-count: + - '0' + x-stainless-timeout: + - '600' + method: POST + uri: https://api.anthropic.com/v1/messages + response: + body: + string: !!binary | + H4sIAAAAAAAAA3SSQW/bMAyF/wqhyy7O4CYNBviWHVa06A5t0VMxGKxE20JkyZOotEaQ/z7KWLa2 + w04S+D2RT086qjEYcqpR2mE2tNqsBrT7vFrX68t6U39RlbJG8Jj6tr64ub//mujbNN/13x+ucLt7 + vL3ajaLheaKiopSwJynE4EoBU7KJ0bOUdPBMsmuejmc902shy9KoR9+FyNkjk5sruAYT/CeGAQ8E + 5EPuB7BFMiLb4IED6IH0HmwHCGkibTurIURDEUQmRR3JWAaN0cALJphi0GKRzGfYJUAPu2v4YxFe + LA8hM6AuotIf/QyR0EFnPXptZZfmxDQmGQMc0SfUixmDjB8s80AyecJn6yzPpZ0LYQ95WoiM5Cxt + 3npXpx+VShymVoam4Eui+Npy2JNP6jdK9DOT1xKfz85VKi+JN0dl/ZT5LG7WG0kcJaBWQlgCa98L + 6jMXbP7HzmdLf5oGGimia7fjv/q/9GL4SE+VklzflrZyIlE8WE0tW4py0/IGRh6q/DjfUSx3bHsK + gnzgFg9oHT7LrzqdfgEAAP//AwBt9B/otgIAAA== + headers: + CF-RAY: + - 9e4cd925bbf1c1b3-BLR + Connection: + - keep-alive + Content-Encoding: + - gzip + Content-Security-Policy: + - default-src 'none'; frame-ancestors 'none' + Content-Type: + - application/json + Date: + - Tue, 31 Mar 2026 05:10:33 GMT + Server: + - cloudflare + Transfer-Encoding: + - chunked + X-Robots-Tag: + - none + anthropic-organization-id: + - 887534b2-e0d2-4865-a85b-8a07958e7c46 + anthropic-ratelimit-input-tokens-limit: + - '50000' + anthropic-ratelimit-input-tokens-remaining: + - '50000' + anthropic-ratelimit-input-tokens-reset: + - '2026-03-31T05:10:32Z' + anthropic-ratelimit-output-tokens-limit: + - '10000' + anthropic-ratelimit-output-tokens-remaining: + - '10000' + anthropic-ratelimit-output-tokens-reset: + - '2026-03-31T05:10:33Z' + anthropic-ratelimit-requests-limit: + - '50' + anthropic-ratelimit-requests-remaining: + - '48' + anthropic-ratelimit-requests-reset: + - '2026-03-31T05:10:34Z' + anthropic-ratelimit-tokens-limit: + - '60000' + anthropic-ratelimit-tokens-remaining: + - '60000' + anthropic-ratelimit-tokens-reset: + - '2026-03-31T05:10:32Z' + cf-cache-status: + - DYNAMIC + request-id: + - req_011CZaaj9C4FJth2DFerJksK + server-timing: + - x-originResponse;dur=557 + set-cookie: + - _cfuvid=DO9tcL5br6vu.85g7pf5WSxRDfSwUwTTS10X1H6b98M-1774933832.5998373-1.0.1.1-svoaUKaHdp4WNDK4jfCkcy1C6bmFxKQbEWMlr3Pvdl8; + HttpOnly; SameSite=None; Secure; Path=/; Domain=api.anthropic.com + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + vary: + - Accept-Encoding + x-envoy-upstream-service-time: + - '554' + status: + code: 200 + message: OK +version: 1 diff --git a/packages/opentelemetry-instrumentation-anthropic/tests/cassettes/test_safety_cassettes/test_anthropic_safety_regex_email_prompt.yaml b/packages/opentelemetry-instrumentation-anthropic/tests/cassettes/test_safety_cassettes/test_anthropic_safety_regex_email_prompt.yaml new file mode 100644 index 0000000000..e2c6d2b568 --- /dev/null +++ b/packages/opentelemetry-instrumentation-anthropic/tests/cassettes/test_safety_cassettes/test_anthropic_safety_regex_email_prompt.yaml @@ -0,0 +1,118 @@ +interactions: +- request: + body: '{"max_tokens":50,"messages":[{"role":"user","content":"Summarize the following + customer inquiry from [PII.email] about their recent order."}],"model":"claude-3-haiku-20240307"}' + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '176' + Content-Type: + - application/json + Host: + - api.anthropic.com + User-Agent: + - Anthropic/Python 0.76.0 + X-Stainless-Arch: + - x64 + X-Stainless-Async: + - 'false' + X-Stainless-Lang: + - python + X-Stainless-OS: + - MacOS + X-Stainless-Package-Version: + - 0.76.0 + X-Stainless-Runtime: + - CPython + X-Stainless-Runtime-Version: + - 3.10.19 + anthropic-version: + - '2023-06-01' + x-stainless-read-timeout: + - '600' + x-stainless-retry-count: + - '0' + x-stainless-timeout: + - '600' + method: POST + uri: https://api.anthropic.com/v1/messages + response: + body: + string: !!binary | + H4sIAAAAAAAAA3SSW2vcMBCF/8qgZ29wNpeC3/IYSp9CQ6AUM5ZmLWFZ4+iyxF32v3e0rWma0ifb + 8505ZzTWSc1syKtOaY/F0O5mZ9FNZbdv97ftTftJNcoZwXMa+/b6+f7zlxLsy9fbp0McIo/D9DRN + osnrQlVFKeFIUojsawFTciljyFLSHDLJW/fttOkzvVVyeXTqEXBhz6P7QQ0MJcMjGIbAGSweCVBr + sYfMgGGVr1zQgy4p80wRXHgtLjpKwBEWiomDYBcOHGfMjsMVPCTphAeJ2aZqPkRkS6BxwcF5l9dL + 1K9Q8UxlnjHKbBDpfbDBjJVzNJJ6JY5aUjj4FZbIR2dkchgpUETfgF0XzladvzdK+pdevGTSujp8 + 6zNPFJL6jRK9Fgpa9hSK940ql9V2J+XCUvIm7vb3slrUlnotZvWk/d+CduOCzf/Y1lv9abE013H7 + u/lf/R96bT/Sc6O45PelO+lIFI9OU58dRTlp3bzBaOrVCgeK9Yz9SCxIfkSPR3QeB7k+5/NPAAAA + //8DAJwxGiqfAgAA + headers: + CF-RAY: + - 9e4cd91f490f5ace-BLR + Connection: + - keep-alive + Content-Encoding: + - gzip + Content-Security-Policy: + - default-src 'none'; frame-ancestors 'none' + Content-Type: + - application/json + Date: + - Tue, 31 Mar 2026 05:10:32 GMT + Server: + - cloudflare + Transfer-Encoding: + - chunked + X-Robots-Tag: + - none + anthropic-organization-id: + - 887534b2-e0d2-4865-a85b-8a07958e7c46 + anthropic-ratelimit-input-tokens-limit: + - '50000' + anthropic-ratelimit-input-tokens-remaining: + - '50000' + anthropic-ratelimit-input-tokens-reset: + - '2026-03-31T05:10:31Z' + anthropic-ratelimit-output-tokens-limit: + - '10000' + anthropic-ratelimit-output-tokens-remaining: + - '10000' + anthropic-ratelimit-output-tokens-reset: + - '2026-03-31T05:10:32Z' + anthropic-ratelimit-requests-limit: + - '50' + anthropic-ratelimit-requests-remaining: + - '49' + anthropic-ratelimit-requests-reset: + - '2026-03-31T05:10:33Z' + anthropic-ratelimit-tokens-limit: + - '60000' + anthropic-ratelimit-tokens-remaining: + - '60000' + anthropic-ratelimit-tokens-reset: + - '2026-03-31T05:10:31Z' + cf-cache-status: + - DYNAMIC + request-id: + - req_011CZaaj4kfPCkjjkM7ddL2d + server-timing: + - x-originResponse;dur=690 + set-cookie: + - _cfuvid=oL7Ja2Al0aIPQBhb5XJ6IR6ADkZHWep5YhwgrcrSnIg-1774933831.5663667-1.0.1.1-QjEuRtk21ycP9PKPUJWc6_s_9G6HiODmniGnkx80y1E; + HttpOnly; SameSite=None; Secure; Path=/; Domain=api.anthropic.com + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + vary: + - Accept-Encoding + x-envoy-upstream-service-time: + - '688' + status: + code: 200 + message: OK +version: 1 diff --git a/packages/opentelemetry-instrumentation-anthropic/tests/cassettes/test_safety_cassettes/test_anthropic_safety_streaming_with_pii.yaml b/packages/opentelemetry-instrumentation-anthropic/tests/cassettes/test_safety_cassettes/test_anthropic_safety_streaming_with_pii.yaml new file mode 100644 index 0000000000..3b081d889c --- /dev/null +++ b/packages/opentelemetry-instrumentation-anthropic/tests/cassettes/test_safety_cassettes/test_anthropic_safety_streaming_with_pii.yaml @@ -0,0 +1,127 @@ +interactions: +- request: + body: '{"max_tokens":50,"messages":[{"role":"user","content":"Acknowledge receipt + of message from [PII.email] about [custom_compliance.project-phoenix]."}],"model":"claude-3-haiku-20240307","stream":true}' + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '197' + Content-Type: + - application/json + Host: + - api.anthropic.com + User-Agent: + - Anthropic/Python 0.76.0 + X-Stainless-Arch: + - x64 + X-Stainless-Async: + - 'false' + X-Stainless-Helper-Method: + - stream + X-Stainless-Lang: + - python + X-Stainless-OS: + - MacOS + X-Stainless-Package-Version: + - 0.76.0 + X-Stainless-Runtime: + - CPython + X-Stainless-Runtime-Version: + - 3.10.19 + X-Stainless-Stream-Helper: + - messages + anthropic-version: + - '2023-06-01' + x-stainless-read-timeout: + - '600' + x-stainless-retry-count: + - '0' + x-stainless-timeout: + - NOT_GIVEN + method: POST + uri: https://api.anthropic.com/v1/messages + response: + body: + string: !!binary | + H4sIAAAAAAAAA3SQT0vEMBDF7/splpy70HZdFnMVPPgHBBFBkTDbjG3YNKnJZEGXfncnYtEqm1ve + 7+XN5OEBHclljzFCiyoSBFpoIJDLo6D3AYUUMyiK6S7kUfReo2VLYyFpXK1XHZh9WtVlfVauyy2b + jc4JsVVlddGmcLfbbLsnf3/7eGWuPy7x4YY98zksBG+zADEaHury0MY74lWFfH4pRCQ/qIAQvRPS + JWu/pYhvCV2Dk5imPY0bEinye3RRyPqc86DpUDUcQsY7NTeUE2esT7Hpbc7HocMeA1i16f/7f2jV + /aVjIXyi31LFn8FwMA0qMhi4h9yBhqBzne4VQ/6iatEzcp4UHMBY2HFl47j8OuNi8QkAAP//SoVE + LDTY4pNy8pOzsUcvFiVgi1JSKyC+RZYHeRaqryS1AqQQTFkpKUGtVoC7AAAAAP//grqgIDMvHc1K + BSWQoBJQDQAAAP//wupKYJIqScTvSrASFFdCRFBdB1cGdaMnpiMBAAAA//+8k8ENhDAQA1txA6QP + fjzuhyIUlg0JkGwUBd1d9ywPWqAByyOP3y0AR3uW78HLyqhMHEuDeLTAzwXhqySMQ98bTrqrhZtV + EIx0quJpIknliE4NMKXKxtS6EoRz/FmDT3B5x19OeKl36g2slBcAAAD//6KzL4GJNL8oF5w59KBB + DnQEAAAA///CkSLzCwglyPwCJCcooMcgAAAA//9KRS3BsPoKRVIJyRcoBYlSal5KfElpUZ4S1uKk + lmblCVoBYGyEmkzB3gQAAAD//0pFL6gxww1ZTgmqDQAAAP//AwAmCvmW4AUAAA== + headers: + CF-RAY: + - 9e4cd94a38ef999b-BLR + Cache-Control: + - no-cache + Connection: + - keep-alive + Content-Encoding: + - gzip + Content-Security-Policy: + - default-src 'none'; frame-ancestors 'none' + Content-Type: + - text/event-stream; charset=utf-8 + Date: + - Tue, 31 Mar 2026 05:10:38 GMT + Server: + - cloudflare + Transfer-Encoding: + - chunked + X-Robots-Tag: + - none + anthropic-organization-id: + - 887534b2-e0d2-4865-a85b-8a07958e7c46 + anthropic-ratelimit-input-tokens-limit: + - '50000' + anthropic-ratelimit-input-tokens-remaining: + - '50000' + anthropic-ratelimit-input-tokens-reset: + - '2026-03-31T05:10:38Z' + anthropic-ratelimit-output-tokens-limit: + - '10000' + anthropic-ratelimit-output-tokens-remaining: + - '10000' + anthropic-ratelimit-output-tokens-reset: + - '2026-03-31T05:10:38Z' + anthropic-ratelimit-requests-limit: + - '50' + anthropic-ratelimit-requests-remaining: + - '48' + anthropic-ratelimit-requests-reset: + - '2026-03-31T05:10:40Z' + anthropic-ratelimit-tokens-limit: + - '60000' + anthropic-ratelimit-tokens-remaining: + - '60000' + anthropic-ratelimit-tokens-reset: + - '2026-03-31T05:10:38Z' + cf-cache-status: + - DYNAMIC + request-id: + - req_011CZaaja71Qm9fe1u7NJ3KC + server-timing: + - x-originResponse;dur=260 + set-cookie: + - _cfuvid=RcXbDBQ0OAeDCOBAt7cckh4gamv3vIT5Qa6tuSR9dmY-1774933838.43259-1.0.1.1-D6CyQ3lqIQSwq2EQYFxxyr4pHcA4UsO1FIT.2Pq0w3k; + HttpOnly; SameSite=None; Secure; Path=/; Domain=api.anthropic.com + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + vary: + - Accept-Encoding + x-envoy-upstream-service-time: + - '258' + status: + code: 200 + message: OK +version: 1 diff --git a/packages/opentelemetry-instrumentation-anthropic/tests/cassettes/test_safety_cassettes/test_anthropic_safety_udf_project_codenames_prompt.yaml b/packages/opentelemetry-instrumentation-anthropic/tests/cassettes/test_safety_cassettes/test_anthropic_safety_udf_project_codenames_prompt.yaml new file mode 100644 index 0000000000..b07800743d --- /dev/null +++ b/packages/opentelemetry-instrumentation-anthropic/tests/cassettes/test_safety_cassettes/test_anthropic_safety_udf_project_codenames_prompt.yaml @@ -0,0 +1,118 @@ +interactions: +- request: + body: '{"max_tokens":50,"messages":[{"role":"user","content":"Prepare a status + update for [custom_compliance.project-phoenix] and [custom_compliance.operation-atlas] + including timeline and budget."}],"model":"claude-3-haiku-20240307"}' + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '227' + Content-Type: + - application/json + Host: + - api.anthropic.com + User-Agent: + - Anthropic/Python 0.76.0 + X-Stainless-Arch: + - x64 + X-Stainless-Async: + - 'false' + X-Stainless-Lang: + - python + X-Stainless-OS: + - MacOS + X-Stainless-Package-Version: + - 0.76.0 + X-Stainless-Runtime: + - CPython + X-Stainless-Runtime-Version: + - 3.10.19 + anthropic-version: + - '2023-06-01' + x-stainless-read-timeout: + - '600' + x-stainless-retry-count: + - '0' + x-stainless-timeout: + - '600' + method: POST + uri: https://api.anthropic.com/v1/messages + response: + body: + string: !!binary | + H4sIAAAAAAAAA4yST2/bMAzFv4qgc1w4TYOiPu+w01AU3akJBFZmba0ypYlUkSLIdy/tNdjfAjtJ + 4O+9R5DS0U6px2g76yPUHptNM0J4rs1le3nVbtpru7KhVzzx4Nr13dfC8nrDuaw/Sf1SHrdUhzvV + yGvGWYXMMKAWSopzAZgDC5BoyScS1Fv3cDzrBQ8zWY7OfsaCJrABoxapbGruQdA8pWJkRPPgK0ua + nE9TjgHI40Uu6Rt6afKYkMJhb4D6f+lSxgISEjUgEXhv3o3c7WhH/xG8o/swYQyE6mjM7Q9sAgUJ + S/DSOUcgCjSYPAKjWfJQ0J72K6sdsisInGjeFBycpGcktu+I8XtFbW07qjGubF022R1toFzlLO42 + 17pJ8CM6r2FzZ/e7oD1zxf1H7Oyd8zGPOOl6ottOf+t/0vX4Jz2tbKrya2mrDsbyEjw6CVh00vn5 + eyj9/JPoSZ9YZ3QDJkWUxMELhAiP+ltOpzcAAAD//wMADxG7QI4CAAA= + headers: + CF-RAY: + - 9e4cd93588e27fa8-BLR + Connection: + - keep-alive + Content-Encoding: + - gzip + Content-Security-Policy: + - default-src 'none'; frame-ancestors 'none' + Content-Type: + - application/json + Date: + - Tue, 31 Mar 2026 05:10:36 GMT + Server: + - cloudflare + Transfer-Encoding: + - chunked + X-Robots-Tag: + - none + anthropic-organization-id: + - 887534b2-e0d2-4865-a85b-8a07958e7c46 + anthropic-ratelimit-input-tokens-limit: + - '50000' + anthropic-ratelimit-input-tokens-remaining: + - '50000' + anthropic-ratelimit-input-tokens-reset: + - '2026-03-31T05:10:35Z' + anthropic-ratelimit-output-tokens-limit: + - '10000' + anthropic-ratelimit-output-tokens-remaining: + - '10000' + anthropic-ratelimit-output-tokens-reset: + - '2026-03-31T05:10:36Z' + anthropic-ratelimit-requests-limit: + - '50' + anthropic-ratelimit-requests-remaining: + - '48' + anthropic-ratelimit-requests-reset: + - '2026-03-31T05:10:36Z' + anthropic-ratelimit-tokens-limit: + - '60000' + anthropic-ratelimit-tokens-remaining: + - '60000' + anthropic-ratelimit-tokens-reset: + - '2026-03-31T05:10:35Z' + cf-cache-status: + - DYNAMIC + request-id: + - req_011CZaajKvyRoNZkgSUm1NZt + server-timing: + - x-originResponse;dur=697 + set-cookie: + - _cfuvid=ZmzZrh1LBv9qFcZ1HDx.ZazJV..AQi24OgDWGWz9VY4-1774933835.120807-1.0.1.1-fr24Jt7.cekmAA75REg.lkGunynUI6oDE0PQkmCeJnQ; + HttpOnly; SameSite=None; Secure; Path=/; Domain=api.anthropic.com + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + vary: + - Accept-Encoding + x-envoy-upstream-service-time: + - '696' + status: + code: 200 + message: OK +version: 1 diff --git a/packages/opentelemetry-instrumentation-anthropic/tests/test_safety_cassettes.py b/packages/opentelemetry-instrumentation-anthropic/tests/test_safety_cassettes.py new file mode 100644 index 0000000000..6d25fc1128 --- /dev/null +++ b/packages/opentelemetry-instrumentation-anthropic/tests/test_safety_cassettes.py @@ -0,0 +1,600 @@ +""" +FR VCR cassette tests for safety rule enforcement on Anthropic instrumentation. + +These tests record real Anthropic API interactions via VCR cassettes and verify +that the safety callback system correctly detects and masks sensitive content +in both prompts and completions. + +Safety rule flavours covered: + - RegEx: PCI.credit_card, PII.email + - List: PII.competitor_org (list of company names) + - UDF: custom_compliance.project_codename (user-defined function + returning 2 sub-rule findings) + +North-star: This is a NEW file (FR-owned). Zero delta on TL files. +""" + +import re + +import pytest +from opentelemetry.instrumentation.fortifyroot import ( + SafetyFinding, + SafetyLocation, + SafetyResult, + clear_safety_handlers, + register_completion_safety_handler, + register_prompt_safety_handler, +) +from opentelemetry.semconv_ai import SpanAttributes + + +# --------------------------------------------------------------------------- +# Safety rule definitions (mirror what real SDK rules would do) +# --------------------------------------------------------------------------- + +# RegEx patterns +_CREDIT_CARD_RE = re.compile(r"\b(?:\d[ -]*?){13,19}\b") +_EMAIL_RE = re.compile(r"\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}\b") + +# List of competitor org names (case-insensitive) +_COMPETITOR_ORGS = ["acme corp", "globex industries", "initech", "umbrella inc"] + +# UDF: detects two fictional project codenames +_PROJECT_CODENAMES = ["project-phoenix", "operation-atlas"] + +# Module-level action setting (switched by fixtures) +_SAFETY_ACTION = "MASK" + + +def _build_findings_and_mask(text, findings_list, action="MASK"): + """Apply all collected findings to produce masked (or allowed) text.""" + if not findings_list: + return None + if action == "MASK": + masked = text + # Sort findings by start position descending so replacements don't shift indices + for f in sorted(findings_list, key=lambda x: x.start, reverse=True): + masked = masked[:f.start] + f"[{f.rule_name}]" + masked[f.end:] + return SafetyResult( + text=masked, + overall_action="MASK", + findings=findings_list, + ) + # ALLOW: text unchanged, findings still recorded + return SafetyResult( + text=text, + overall_action="ALLOW", + findings=findings_list, + ) + + +def _scan_text(text, location): + """Run all safety rule types against text, return findings.""" + if not isinstance(text, str) or not text: + return [] + + action = _SAFETY_ACTION + findings = [] + + # --- RegEx: PCI.credit_card --- + for m in _CREDIT_CARD_RE.finditer(text): + findings.append(SafetyFinding( + category="PCI", + severity="HIGH", + action=action, + rule_name="PCI.credit_card", + start=m.start(), + end=m.end(), + )) + + # --- RegEx: PII.email --- + for m in _EMAIL_RE.finditer(text): + findings.append(SafetyFinding( + category="PII", + severity="MEDIUM", + action=action, + rule_name="PII.email", + start=m.start(), + end=m.end(), + )) + + # --- List: PII.competitor_org --- + text_lower = text.lower() + for org in _COMPETITOR_ORGS: + idx = text_lower.find(org) + while idx != -1: + findings.append(SafetyFinding( + category="PII", + severity="LOW", + action=action, + rule_name="PII.competitor_org", + start=idx, + end=idx + len(org), + )) + idx = text_lower.find(org, idx + len(org)) + + # --- UDF: custom_compliance.project_codename (returns 2 sub-rules) --- + for codename in _PROJECT_CODENAMES: + idx = text_lower.find(codename) + while idx != -1: + findings.append(SafetyFinding( + category="COMPLIANCE", + severity="HIGH", + action=action, + rule_name=f"custom_compliance.{codename}", + start=idx, + end=idx + len(codename), + )) + idx = text_lower.find(codename, idx + len(codename)) + + return findings + + +def _prompt_handler(context): + """Prompt safety handler covering RegEx, List, and UDF rules.""" + if context.location != SafetyLocation.PROMPT: + return None + findings = _scan_text(context.text, SafetyLocation.PROMPT) + return _build_findings_and_mask(context.text, findings, action=_SAFETY_ACTION) + + +def _completion_handler(context): + """Completion safety handler covering RegEx, List, and UDF rules.""" + if context.location != SafetyLocation.COMPLETION: + return None + findings = _scan_text(context.text, SafetyLocation.COMPLETION) + return _build_findings_and_mask(context.text, findings, action=_SAFETY_ACTION) + + +# --------------------------------------------------------------------------- +# Fixtures +# --------------------------------------------------------------------------- + +@pytest.fixture(autouse=True) +def _register_safety_handlers(): + """Register all safety handlers before each test, clear after.""" + clear_safety_handlers() + register_prompt_safety_handler(_prompt_handler) + register_completion_safety_handler(_completion_handler) + yield + clear_safety_handlers() + + +# --------------------------------------------------------------------------- +# T2-B3: RegEx rule — PII.email in prompt +# --------------------------------------------------------------------------- + +@pytest.mark.vcr +@pytest.mark.fr +def test_anthropic_safety_regex_email_prompt( + instrument_legacy, span_exporter, anthropic_client +): + """Prompt contains an email address. Safety should detect and mask it.""" + anthropic_client.messages.create( + model="claude-3-haiku-20240307", + max_tokens=50, + messages=[ + { + "role": "user", + "content": ( + "Summarize the following customer inquiry from " + "john.doe@example.com about their recent order." + ), + } + ], + ) + + spans = span_exporter.get_finished_spans() + assert len(spans) >= 1 + span = spans[0] + + # The prompt content should have been masked (email replaced) + prompt_content = span.attributes.get("gen_ai.prompt.0.content") + assert prompt_content is not None + assert "john.doe@example.com" not in prompt_content + assert "[PII.email]" in prompt_content + + # Safety finding should be recorded as a span event + safety_events = [ + e for e in span.events + if "fortifyroot.safety" in (e.name or "") + ] + assert len(safety_events) >= 1 + event_attrs = safety_events[0].attributes + assert event_attrs["fortifyroot.safety.rule_name"] == "PII.email" + assert event_attrs["fortifyroot.safety.action"] == "MASK" + assert event_attrs["fortifyroot.safety.location"] == "PROMPT" + + +# --------------------------------------------------------------------------- +# T2-B3: RegEx rule — PCI.credit_card in prompt +# --------------------------------------------------------------------------- + +@pytest.mark.vcr +@pytest.mark.fr +def test_anthropic_safety_regex_credit_card_prompt( + instrument_legacy, span_exporter, anthropic_client +): + """Prompt contains a credit card number. Safety should detect and mask it.""" + anthropic_client.messages.create( + model="claude-3-haiku-20240307", + max_tokens=50, + messages=[ + { + "role": "user", + "content": ( + "Check if order for card 4111 1111 1111 1111 was processed." + ), + } + ], + ) + + spans = span_exporter.get_finished_spans() + assert len(spans) >= 1 + span = spans[0] + + prompt_content = span.attributes.get("gen_ai.prompt.0.content") + assert prompt_content is not None + assert "4111 1111 1111 1111" not in prompt_content + assert "[PCI.credit_card]" in prompt_content + + safety_events = [ + e for e in span.events + if "fortifyroot.safety" in (e.name or "") + ] + assert len(safety_events) >= 1 + found_cc = any( + e.attributes.get("fortifyroot.safety.rule_name") == "PCI.credit_card" + for e in safety_events + ) + assert found_cc, "Expected PCI.credit_card safety finding" + + +# --------------------------------------------------------------------------- +# T2-B3: List rule — PII.competitor_org in prompt +# --------------------------------------------------------------------------- + +@pytest.mark.vcr +@pytest.mark.fr +def test_anthropic_safety_list_competitor_org_prompt( + instrument_legacy, span_exporter, anthropic_client +): + """Prompt mentions a competitor org from the blocklist.""" + anthropic_client.messages.create( + model="claude-3-haiku-20240307", + max_tokens=50, + messages=[ + { + "role": "user", + "content": ( + "Compare our product roadmap with Acme Corp's latest " + "offering and Globex Industries' pricing strategy." + ), + } + ], + ) + + spans = span_exporter.get_finished_spans() + assert len(spans) >= 1 + span = spans[0] + + prompt_content = span.attributes.get("gen_ai.prompt.0.content") + assert prompt_content is not None + assert "acme corp" not in prompt_content.lower() + assert "globex industries" not in prompt_content.lower() + assert "[PII.competitor_org]" in prompt_content + + safety_events = [ + e for e in span.events + if "fortifyroot.safety" in (e.name or "") + ] + found_org = any( + e.attributes.get("fortifyroot.safety.rule_name") == "PII.competitor_org" + for e in safety_events + ) + assert found_org, "Expected PII.competitor_org safety finding" + + +# --------------------------------------------------------------------------- +# T2-B3: UDF rule — custom_compliance with 2 sub-rules in prompt +# --------------------------------------------------------------------------- + +@pytest.mark.vcr +@pytest.mark.fr +def test_anthropic_safety_udf_project_codenames_prompt( + instrument_legacy, span_exporter, anthropic_client +): + """Prompt contains two project codenames detected by UDF rule.""" + anthropic_client.messages.create( + model="claude-3-haiku-20240307", + max_tokens=50, + messages=[ + { + "role": "user", + "content": ( + "Prepare a status update for Project-Phoenix and " + "Operation-Atlas including timeline and budget." + ), + } + ], + ) + + spans = span_exporter.get_finished_spans() + assert len(spans) >= 1 + span = spans[0] + + prompt_content = span.attributes.get("gen_ai.prompt.0.content") + assert prompt_content is not None + # Masking replaces "Project-Phoenix" with "[custom_compliance.project-phoenix]" + # so we verify the mask tokens are present (original casing is gone) + assert "[custom_compliance.project-phoenix]" in prompt_content + assert "[custom_compliance.operation-atlas]" in prompt_content + # Verify original text (with capital letters) is replaced + assert "Project-Phoenix" not in prompt_content + assert "Operation-Atlas" not in prompt_content + + # UDF should produce 2 findings (one per codename) + safety_events = [ + e for e in span.events + if "fortifyroot.safety" in (e.name or "") + ] + udf_findings = [ + e for e in safety_events + if (e.attributes.get("fortifyroot.safety.rule_name") or "").startswith( + "custom_compliance." + ) + ] + assert len(udf_findings) >= 2, ( + f"Expected 2 UDF findings, got {len(udf_findings)}" + ) + + +# --------------------------------------------------------------------------- +# T2-B3: Completion safety — email in LLM response +# --------------------------------------------------------------------------- + +@pytest.mark.vcr +@pytest.mark.fr +def test_anthropic_safety_completion_email_masked( + instrument_legacy, span_exporter, anthropic_client +): + """ + Ask the LLM to generate fictional contact info. The response should + contain an email-like pattern that completion safety detects and masks. + + The prompt does NOT contain PII — it asks the LLM to invent an email. + This ensures completion safety (not just prompt echo-back) is tested. + """ + anthropic_client.messages.create( + model="claude-3-haiku-20240307", + max_tokens=100, + messages=[ + { + "role": "user", + "content": ( + "Create a fictional contact card. The person's name is " + "Jane Smith, company is Widgets Inc, role is VP of Sales. " + "Invent a plausible work email and phone. Format as:\n" + "Name: ...\nEmail: ...\nPhone: ..." + ), + } + ], + ) + + spans = span_exporter.get_finished_spans() + assert len(spans) >= 1 + span = spans[0] + + # Prompt has no PII, so no prompt masking expected + prompt_content = span.attributes.get("gen_ai.prompt.0.content") + assert prompt_content is not None + + # Completion should be recorded AND the LLM-generated email should be + # masked by completion safety + completion_content = span.attributes.get("gen_ai.completion.0.content") + assert completion_content is not None, "Completion content should be recorded" + assert "[PII.email]" in completion_content, ( + "LLM-generated email in response should be masked by completion safety" + ) + + # Verify completion safety event was emitted + safety_events = [ + e for e in span.events + if "fortifyroot.safety" in (e.name or "") + ] + completion_findings = [ + e for e in safety_events + if e.attributes.get("fortifyroot.safety.location") == "COMPLETION" + ] + assert len(completion_findings) >= 1, "Should have completion safety finding" + assert any( + e.attributes.get("fortifyroot.safety.rule_name") == "PII.email" + for e in completion_findings + ), "Completion safety should detect PII.email" + + +# --------------------------------------------------------------------------- +# T2-B3: Combined — multiple rule types in single prompt +# --------------------------------------------------------------------------- + +@pytest.mark.vcr +@pytest.mark.fr +def test_anthropic_safety_combined_rules_prompt( + instrument_legacy, span_exporter, anthropic_client +): + """Prompt triggers all rule types: RegEx (email), List (org), UDF (codename).""" + anthropic_client.messages.create( + model="claude-3-haiku-20240307", + max_tokens=50, + messages=[ + { + "role": "user", + "content": ( + "Send the Project-Phoenix briefing to contact@acme-corp.com " + "and CC the Initech team." + ), + } + ], + ) + + spans = span_exporter.get_finished_spans() + assert len(spans) >= 1 + span = spans[0] + + prompt_content = span.attributes.get("gen_ai.prompt.0.content") + assert prompt_content is not None + + # Verify all 3 rule types fired + safety_events = [ + e for e in span.events + if "fortifyroot.safety" in (e.name or "") + ] + rule_names = { + e.attributes.get("fortifyroot.safety.rule_name") + for e in safety_events + } + assert "PII.email" in rule_names, "Expected PII.email finding" + assert "PII.competitor_org" in rule_names or any( + "competitor_org" in (r or "") for r in rule_names + ), "Expected PII.competitor_org finding" + assert any( + (r or "").startswith("custom_compliance.") for r in rule_names + ), "Expected custom_compliance UDF finding" + + +# --------------------------------------------------------------------------- +# T2-B3: Streaming with PII — verify safety on streamed completion +# --------------------------------------------------------------------------- + +@pytest.mark.vcr +@pytest.mark.fr +def test_anthropic_safety_streaming_with_pii( + instrument_legacy, span_exporter, anthropic_client +): + """Streaming chat with PII in prompt. Verify prompt safety applied.""" + with anthropic_client.messages.stream( + model="claude-3-haiku-20240307", + max_tokens=50, + messages=[ + { + "role": "user", + "content": ( + "Acknowledge receipt of message from " + "alice.jones@example.org about Project-Phoenix." + ), + } + ], + ) as stream: + # Consume the stream + for _ in stream.text_stream: + pass + + spans = span_exporter.get_finished_spans() + assert len(spans) >= 1 + span = spans[0] + + # Prompt should be masked + prompt_content = span.attributes.get("gen_ai.prompt.0.content") + assert prompt_content is not None + assert "alice.jones@example.org" not in prompt_content + assert "[PII.email]" in prompt_content + + +# =========================================================================== +# ALLOW action tests +# =========================================================================== +# When action=ALLOW, text passes through UNCHANGED but findings are still +# recorded as span events for audit/observability. +# =========================================================================== + + +@pytest.fixture +def _set_allow_action(): + """Switch safety handlers to ALLOW mode for the test.""" + global _SAFETY_ACTION + _SAFETY_ACTION = "ALLOW" + yield + _SAFETY_ACTION = "MASK" + + +@pytest.mark.vcr +@pytest.mark.fr +def test_anthropic_safety_allow_email_not_masked( + _set_allow_action, instrument_legacy, span_exporter, anthropic_client +): + """ALLOW action: email detected but text passes through unchanged.""" + anthropic_client.messages.create( + model="claude-3-haiku-20240307", + max_tokens=50, + messages=[ + { + "role": "user", + "content": ( + "Forward this message from alice.jones@example.org " + "to the support team." + ), + } + ], + ) + + spans = span_exporter.get_finished_spans() + assert len(spans) >= 1 + span = spans[0] + + # With ALLOW, the original text should pass through unchanged + prompt_content = span.attributes.get("gen_ai.prompt.0.content") + assert prompt_content is not None + assert "alice.jones@example.org" in prompt_content + assert "[PII.email]" not in prompt_content + + # But findings should STILL be recorded as span events + safety_events = [ + e for e in span.events + if "fortifyroot.safety" in (e.name or "") + ] + assert len(safety_events) >= 1, "ALLOW should still emit safety findings" + event_attrs = safety_events[0].attributes + assert event_attrs["fortifyroot.safety.rule_name"] == "PII.email" + assert event_attrs["fortifyroot.safety.action"] == "ALLOW" + assert event_attrs["fortifyroot.safety.location"] == "PROMPT" + + +@pytest.mark.vcr +@pytest.mark.fr +def test_anthropic_safety_allow_credit_card_not_masked( + _set_allow_action, instrument_legacy, span_exporter, anthropic_client +): + """ALLOW action: credit card detected but text passes through unchanged.""" + anthropic_client.messages.create( + model="claude-3-haiku-20240307", + max_tokens=50, + messages=[ + { + "role": "user", + "content": ( + "Verify payment for card 4111 1111 1111 1111 was received." + ), + } + ], + ) + + spans = span_exporter.get_finished_spans() + assert len(spans) >= 1 + span = spans[0] + + # With ALLOW, text unchanged — card number still visible + prompt_content = span.attributes.get("gen_ai.prompt.0.content") + assert prompt_content is not None + assert "4111 1111 1111 1111" in prompt_content + + # Findings still recorded + safety_events = [ + e for e in span.events + if "fortifyroot.safety" in (e.name or "") + ] + found_cc = any( + e.attributes.get("fortifyroot.safety.rule_name") == "PCI.credit_card" + and e.attributes.get("fortifyroot.safety.action") == "ALLOW" + for e in safety_events + ) + assert found_cc, "Expected PCI.credit_card finding with ALLOW action" diff --git a/packages/opentelemetry-instrumentation-openai/tests/traces/cassettes/test_safety_cassettes/test_openai_safety_allow_credit_card_not_masked.yaml b/packages/opentelemetry-instrumentation-openai/tests/traces/cassettes/test_safety_cassettes/test_openai_safety_allow_credit_card_not_masked.yaml new file mode 100644 index 0000000000..f3da4a3cb6 --- /dev/null +++ b/packages/opentelemetry-instrumentation-openai/tests/traces/cassettes/test_safety_cassettes/test_openai_safety_allow_credit_card_not_masked.yaml @@ -0,0 +1,111 @@ +interactions: +- request: + body: '{"messages":[{"role":"user","content":"Verify payment for card 4111 1111 + 1111 1111 was received."}],"model":"gpt-4o-mini","max_tokens":50}' + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '138' + Content-Type: + - application/json + Host: + - api.openai.com + User-Agent: + - OpenAI/Python 1.99.7 + X-Stainless-Arch: + - x64 + X-Stainless-Async: + - 'false' + X-Stainless-Lang: + - python + X-Stainless-OS: + - MacOS + X-Stainless-Package-Version: + - 1.99.7 + X-Stainless-Runtime: + - CPython + X-Stainless-Runtime-Version: + - 3.10.19 + traceparent: + - 00-284c4c1582461da2ec3b43482d63ad38-21bb1d4b92c9b4e0-01 + x-stainless-read-timeout: + - '600' + x-stainless-retry-count: + - '0' + method: POST + uri: https://api.openai.com/v1/chat/completions + response: + body: + string: !!binary | + H4sIAAAAAAAA/6rmUlBQykxRslJQSs5ILEnOLcjRdQnwTa6ycAYAAAD//4xTTW/UMBC9768Y+cIl + WW33K+0VcWg5FQkhBKoirz3Jmjp2sJ1tI7T/nbGzaVIoEpdImTfv5b2Zifh4+vzt+bZ9et98va/d + 7acP4sv+J8siwx5+oAgjayks8TAoawZYOOQBo+pVUWxvNsVut0lAYyXqSKvbkG9t3iij8vVqvc1X + RX51fWEfrRLoqe07vQL8Ss/o00h8pvIqGysNes9rpNrYREVndaww7r3ygZvAsgkU1gQ0yfrduwa8 + da7P4NAFuAPBjbEBBh48qXCEEzpV9crU0PK+ISIoU1nX8BgWrIPWWfLqYwc3PVTKcCMU1xAcN56L + 2OeXcK+Re4T4dapBbzsHB24ek8RF2aM7UfAoeVISHUjlaMqaVKlrTEMNydnISg6VSIaW86QOq87z + OG3TaT0DUsrUn2b8cEHOL1PVtiYPB/8HlVE45Y8lLdfTpmmCPtiWJfRMz4e0ve7VQhgJNW0og33E + 9Ln1fpBj081M4GZ9AQMZ1FN9d529oVZKDFxpP9s+E1wcUU7M6VR4J5WdAYtZ5r/NvKU95KZN/4/8 + BAiBLf0MZetQKvE68NTmMP5R/2p7mXEyzC6XUgaFLu5BYsU7Pdw5870P2JS0rBpd69Rw7FVbipvN + fruTVbFni/PiNwAAAP//AwCrZkY6+gMAAA== + headers: + CF-Cache-Status: + - DYNAMIC + CF-Ray: + - 9e4d33fadc0b4e3d-BLR + Connection: + - keep-alive + Content-Encoding: + - gzip + Content-Type: + - application/json + Date: + - Tue, 31 Mar 2026 06:12:34 GMT + Server: + - cloudflare + Strict-Transport-Security: + - max-age=31536000; includeSubDomains; preload + Transfer-Encoding: + - chunked + X-Content-Type-Options: + - nosniff + access-control-expose-headers: + - X-Request-ID + alt-svc: + - h3=":443"; ma=86400 + openai-organization: + - personal-test-clou2a + openai-processing-ms: + - '804' + openai-project: + - proj_5zhcB7xbj7xhoXUDTAPm4WiB + openai-version: + - '2020-10-01' + set-cookie: + - __cf_bm=NPsNiSjFoWw5af2gsmgXGrKNxXa66OEg2g2nAfrFIaY-1774937553.0998738-1.0.1.1-k_ra7BRw7wOfqUxyt2zQ8Q0tx15ES84atnyID5C6xWSNmXhqxUcbJIEi3BSYVG31tF8g2pAxn124GlnS7dH1FeSkKgsFXEKvyA018Qx6.4CJvExNeJ32_..lJ6wKIPwG; + HttpOnly; Secure; Path=/; Domain=api.openai.com; Expires=Tue, 31 Mar 2026 + 06:42:34 GMT + x-openai-proxy-wasm: + - v0.1 + x-ratelimit-limit-requests: + - '10000' + x-ratelimit-limit-tokens: + - '200000' + x-ratelimit-remaining-requests: + - '9998' + x-ratelimit-remaining-tokens: + - '199983' + x-ratelimit-reset-requests: + - 15.342s + x-ratelimit-reset-tokens: + - 5ms + x-request-id: + - req_d742906a500c4ce7a228d1abe6bccd14 + status: + code: 200 + message: OK +version: 1 diff --git a/packages/opentelemetry-instrumentation-openai/tests/traces/cassettes/test_safety_cassettes/test_openai_safety_allow_email_not_masked.yaml b/packages/opentelemetry-instrumentation-openai/tests/traces/cassettes/test_safety_cassettes/test_openai_safety_allow_email_not_masked.yaml new file mode 100644 index 0000000000..fef303b558 --- /dev/null +++ b/packages/opentelemetry-instrumentation-openai/tests/traces/cassettes/test_safety_cassettes/test_openai_safety_allow_email_not_masked.yaml @@ -0,0 +1,111 @@ +interactions: +- request: + body: '{"messages":[{"role":"user","content":"Forward this message from alice.jones@example.org + to the support team."}],"model":"gpt-4o-mini","max_tokens":50}' + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '151' + Content-Type: + - application/json + Host: + - api.openai.com + User-Agent: + - OpenAI/Python 1.99.7 + X-Stainless-Arch: + - x64 + X-Stainless-Async: + - 'false' + X-Stainless-Lang: + - python + X-Stainless-OS: + - MacOS + X-Stainless-Package-Version: + - 1.99.7 + X-Stainless-Runtime: + - CPython + X-Stainless-Runtime-Version: + - 3.10.19 + traceparent: + - 00-e1cc0c28e70762695382837b31562916-f1b408fa79bd6bf6-01 + x-stainless-read-timeout: + - '600' + x-stainless-retry-count: + - '0' + method: POST + uri: https://api.openai.com/v1/chat/completions + response: + body: + string: !!binary | + H4sIAAAAAAAA/6rmUlBQykxRslJQSs5ILEnOLcjRdQnwTa7I8gMAAAD//41TTY/aMBC98yumvnAB + xDflWrVqF1XaQ2+tVpGxJ+DFsS17siyq+O8dJ7CBfki9RMq8eS8z702+u/Lj6+bbY/y03nz+OkvP + H+KhfBSDzPDbZ1R0ZY2UZx6S8a6FVURJmFUnq9V8PVstFpMGqLxGm2m7QMO5H1bGmeF0PJ0Px6vh + 5P2FvfdGYeK2H/wK8LN55jmdxlcujwfXSoUpyR1y7drExehtrgiZkkkkHYlBByrvCF0z+kO/guRj + PA1gWxM8gJKuT1D6eJRRw0U7gY+AlTQ2gTaR17anEXzxR3zBOGhZsEcb4ORr0FGWBPJKBvJAe4RU + h+AjAaGswJS5ta/BmgOOYFMnAnaPOXBw/ghHthSM4zkqmT1thI+8R1YzTtla47vbnSKWdZLZV1db + ewNI5zw1Go2bTxfk/Oaf9bsQ/Tb9RhUl55L2BceYOFP2KpEPokHP/HxqcqrvrBcsVAUqyB+w+dx0 + 0sqJ7jo6cH7JUBAPaLv68kq6Uys0Ug7gJmehpNqj7pjdUchaG38D9G52/nOYv2m3exu3+x/5DlAK + A599ESJqo+4X7toi5n/nX21vHjcDi4TxhX+GggzGnIPGUta2vWiRTomwKjisHcYQTXvWZSjUerac + L3S5WoreufcLYzN7KOQDAAA= + headers: + CF-Cache-Status: + - DYNAMIC + CF-Ray: + - 9e4d33ed7d087fa8-BLR + Connection: + - keep-alive + Content-Encoding: + - gzip + Content-Type: + - application/json + Date: + - Tue, 31 Mar 2026 06:12:33 GMT + Server: + - cloudflare + Strict-Transport-Security: + - max-age=31536000; includeSubDomains; preload + Transfer-Encoding: + - chunked + X-Content-Type-Options: + - nosniff + access-control-expose-headers: + - X-Request-ID + alt-svc: + - h3=":443"; ma=86400 + openai-organization: + - personal-test-clou2a + openai-processing-ms: + - '981' + openai-project: + - proj_5zhcB7xbj7xhoXUDTAPm4WiB + openai-version: + - '2020-10-01' + set-cookie: + - __cf_bm=Rop7npDdlIJp42ZBYZ9r_JiCgxuDYtznJ8x1sZKrySs-1774937550.955291-1.0.1.1-mUt2jfqsdP0khy9i_GfFnfZuq261aMqxxC3Bpe3GkvioNgHv0jCUHFJ1Q4fABSCiNwky4OFKBtQ9oJCBF5VsZEeemCuZzLnwrZxpR7YKMEBwjDUlvXf92nuQWkZQaVoc; + HttpOnly; Secure; Path=/; Domain=api.openai.com; Expires=Tue, 31 Mar 2026 + 06:42:33 GMT + x-openai-proxy-wasm: + - v0.1 + x-ratelimit-limit-requests: + - '10000' + x-ratelimit-limit-tokens: + - '200000' + x-ratelimit-remaining-requests: + - '9999' + x-ratelimit-remaining-tokens: + - '199980' + x-ratelimit-reset-requests: + - 8.64s + x-ratelimit-reset-tokens: + - 6ms + x-request-id: + - req_3d86319d350f44868ab355fe03d924f0 + status: + code: 200 + message: OK +version: 1 diff --git a/packages/opentelemetry-instrumentation-openai/tests/traces/cassettes/test_safety_cassettes/test_openai_safety_combined_rules_prompt.yaml b/packages/opentelemetry-instrumentation-openai/tests/traces/cassettes/test_safety_cassettes/test_openai_safety_combined_rules_prompt.yaml new file mode 100644 index 0000000000..24964b2621 --- /dev/null +++ b/packages/opentelemetry-instrumentation-openai/tests/traces/cassettes/test_safety_cassettes/test_openai_safety_combined_rules_prompt.yaml @@ -0,0 +1,110 @@ +interactions: +- request: + body: '{"messages":[{"role":"user","content":"Send the [custom_compliance.project-phoenix] + briefing to [PII.email] and CC the [PII.competitor_org] team."}],"model":"gpt-4o-mini","max_tokens":50}' + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '187' + Content-Type: + - application/json + Host: + - api.openai.com + User-Agent: + - OpenAI/Python 1.99.7 + X-Stainless-Arch: + - x64 + X-Stainless-Async: + - 'false' + X-Stainless-Lang: + - python + X-Stainless-OS: + - MacOS + X-Stainless-Package-Version: + - 1.99.7 + X-Stainless-Runtime: + - CPython + X-Stainless-Runtime-Version: + - 3.10.19 + traceparent: + - 00-b4db49a13f1181d7a26d37911abf0f82-786030964c953201-01 + x-stainless-read-timeout: + - '600' + x-stainless-retry-count: + - '0' + method: POST + uri: https://api.openai.com/v1/chat/completions + response: + body: + string: !!binary | + H4sIAAAAAAAA/6rmUlBQykxRslJQSs5ILEnOLcjRdQnwSfHPzAQAAAD//4xTwW7bMAy95ys4XXqx + i7TO4vY8oECw61Z0GApDlqhYqywKkrwtGPLvk+ykdtYN2MWw+fjIRz76of3gdtp+rB++PD3asH18 + +vS5OrAiM6j9hiKeWdeCEg+jJjvBwiOPmKve1PXmvqrqajMCPUk0mbZ3sdxQ2Wury9v17aZc1+XN + 3YndkRYYUtrX9Anwa3xmnVbizxReF+dIjyHwPabYOSkFPZkcYTwEHSK3kRUzKMhGtKP03VUPgbw/ + FNAOEXYguL2KMNHgh44dBLRS2z1gz7UJQB46bqXJIYc+kOUGtFXke56nv4adggMNYBEldGgcSM9V + zPkcWq9R5ddU5iS8AIVoQHlEiJRav7xbivWohsDzwuxgzALg1lIcW45rej4hx9fFGNo7T234g8pS + fx26JvmTxOclhEiOjegxPZ9HA4aLnbJUqHexifSCY7vqbirHZtsX4MkcFpNAM8e3Z9JFtUZizJtd + GMgEFx3KmTm7zQepaQGsFjO/FfO32tPcyYP/KT8DQqBL99w4j1KLy4HnNI/5p/hX2uuOR8EsoP+e + rryJGn32QaLig5lOlYVDiNg3+VjQO6+ne1WuEffVdvNeqnrLVsfVbwAAAP//AwB2wqnWvQMAAA== + headers: + CF-Cache-Status: + - DYNAMIC + CF-Ray: + - 9e4cd6b83fc67fe6-MAA + Connection: + - keep-alive + Content-Encoding: + - gzip + Content-Type: + - application/json + Date: + - Tue, 31 Mar 2026 05:08:55 GMT + Server: + - cloudflare + Strict-Transport-Security: + - max-age=31536000; includeSubDomains; preload + Transfer-Encoding: + - chunked + X-Content-Type-Options: + - nosniff + access-control-expose-headers: + - X-Request-ID + alt-svc: + - h3=":443"; ma=86400 + openai-organization: + - personal-test-clou2a + openai-processing-ms: + - '1111' + openai-project: + - proj_5zhcB7xbj7xhoXUDTAPm4WiB + openai-version: + - '2020-10-01' + set-cookie: + - __cf_bm=W9hu_sz9gE8N3F8t9Yl57FEHbCdYWWyCzSTxZ1ZQyCE-1774933733.152509-1.0.1.1-kbI.q2Ef6RgXz20ycAirNUfEai.Y1qPG1fRSnRjOg7R.NC.PIeYPP2zsaCVZuGswj7dEaWp0kekk5v625qga7.bW7yifDOWI8tLZnwbAZyyCA4s59hnnUVsXCN.H5Mul; + HttpOnly; Secure; Path=/; Domain=api.openai.com; Expires=Tue, 31 Mar 2026 + 05:38:55 GMT + x-openai-proxy-wasm: + - v0.1 + x-ratelimit-limit-requests: + - '10000' + x-ratelimit-limit-tokens: + - '200000' + x-ratelimit-remaining-requests: + - '9995' + x-ratelimit-remaining-tokens: + - '199971' + x-ratelimit-reset-requests: + - 40.223s + x-ratelimit-reset-tokens: + - 8ms + x-request-id: + - req_dc5fd9874c3b4755b2154cfc1d1697ce + status: + code: 200 + message: OK +version: 1 diff --git a/packages/opentelemetry-instrumentation-openai/tests/traces/cassettes/test_safety_cassettes/test_openai_safety_completion_email_masked.yaml b/packages/opentelemetry-instrumentation-openai/tests/traces/cassettes/test_safety_cassettes/test_openai_safety_completion_email_masked.yaml new file mode 100644 index 0000000000..d083251cee --- /dev/null +++ b/packages/opentelemetry-instrumentation-openai/tests/traces/cassettes/test_safety_cassettes/test_openai_safety_completion_email_masked.yaml @@ -0,0 +1,112 @@ +interactions: +- request: + body: '{"messages":[{"role":"user","content":"Create a fictional contact card. + The person''s name is Jane Smith, company is Widgets Inc, role is VP of Sales. + Invent a plausible work email and phone. Format as:\nName: ...\nEmail: ...\nPhone: + ..."}],"model":"gpt-4o-mini","max_tokens":100}' + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '279' + Content-Type: + - application/json + Host: + - api.openai.com + User-Agent: + - OpenAI/Python 1.99.7 + X-Stainless-Arch: + - x64 + X-Stainless-Async: + - 'false' + X-Stainless-Lang: + - python + X-Stainless-OS: + - MacOS + X-Stainless-Package-Version: + - 1.99.7 + X-Stainless-Runtime: + - CPython + X-Stainless-Runtime-Version: + - 3.10.19 + traceparent: + - 00-95b49cbafdc4a15926f5363bfa7e5e4c-d65547368ba7f4a7-01 + x-stainless-read-timeout: + - '600' + x-stainless-retry-count: + - '0' + method: POST + uri: https://api.openai.com/v1/chat/completions + response: + body: + string: !!binary | + H4sIAAAAAAAA/6rmUlBQykxRslJQSs5ILEnOLcjRdQnws8x0KwEAAAD//4xSy27bMBC8+ysEnhrA + CmxZsuKcChRJ2hz6QI9tIDDkSqIjPsCl+oT/PUvJjpQ2BXoRoJmd5c7s7sG4tw+9fnPz7tOHlex/ + XRe3jWfLqLD3exDhpDoXlnQQlDUjLTzwALHruizz3WZXXGQDoa2ELsoaF9LcploZlWarLE9XZbq+ + OKpbqwQglX2h3yT5PXzjnEbCD4JXyxOiAZE3QNipiEBvu4gwjqgwcBPYciKFNQHMMPp7ruEyueUG + ks9ahTZJvporzVV3mewJPMcIvv6uZAMBlRHRZqz52FpDwldFUZwl62yT5sW2pNazVzzUPfLo1PRd + NyO4MTbwmNTg7+7IHJ4cdbZx3t7jH1JWU1LYVhQsUso0PQbr2MAe6Hs3JNc/C4NRI+1CFewDDM8V + 67Edm/Y1kdn2SAYasJvwsly+0K2SECgpnCXPBBctyEk5rYn3UtkZsZh5/nuYl3qPvpVp/qf9RAgB + jg6xch6kEs8NT2Ue4jX/q+wp42FghuC/0XlWQYGPe5BQ874bb4zhTwygK1pWA955NR5a7Sqx22zz + Qtblli0Oi0cAAAD//wMAx1X4PnYDAAA= + headers: + CF-Cache-Status: + - DYNAMIC + CF-Ray: + - 9e4d657fddb3c1b5-BLR + Connection: + - keep-alive + Content-Encoding: + - gzip + Content-Type: + - application/json + Date: + - Tue, 31 Mar 2026 06:46:23 GMT + Server: + - cloudflare + Strict-Transport-Security: + - max-age=31536000; includeSubDomains; preload + Transfer-Encoding: + - chunked + X-Content-Type-Options: + - nosniff + access-control-expose-headers: + - X-Request-ID + alt-svc: + - h3=":443"; ma=86400 + openai-organization: + - personal-test-clou2a + openai-processing-ms: + - '1115' + openai-project: + - proj_5zhcB7xbj7xhoXUDTAPm4WiB + openai-version: + - '2020-10-01' + set-cookie: + - __cf_bm=iVvj.fJv97wtGDGu5di1NWjjTmGbtT_K_fgH63h.FGk-1774939581.4195058-1.0.1.1-j1ViAC6ONhI3xrUPjafVeeF895TkPjbhaqihLjhnGFETDPnYM6ocbXoV863A05pFIm.p3AWvONBwA.8wtIbRKuClGkyNg2byujZMjKRaOmIFmBEkXo3winQ1Q5rbdPMq; + HttpOnly; Secure; Path=/; Domain=api.openai.com; Expires=Tue, 31 Mar 2026 + 07:16:23 GMT + x-openai-proxy-wasm: + - v0.1 + x-ratelimit-limit-requests: + - '10000' + x-ratelimit-limit-tokens: + - '200000' + x-ratelimit-remaining-requests: + - '9999' + x-ratelimit-remaining-tokens: + - '199949' + x-ratelimit-reset-requests: + - 8.64s + x-ratelimit-reset-tokens: + - 15ms + x-request-id: + - req_c2799badaf904b2daf8294ec157c46f1 + status: + code: 200 + message: OK +version: 1 diff --git a/packages/opentelemetry-instrumentation-openai/tests/traces/cassettes/test_safety_cassettes/test_openai_safety_list_competitor_org_prompt.yaml b/packages/opentelemetry-instrumentation-openai/tests/traces/cassettes/test_safety_cassettes/test_openai_safety_list_competitor_org_prompt.yaml new file mode 100644 index 0000000000..1eff76936e --- /dev/null +++ b/packages/opentelemetry-instrumentation-openai/tests/traces/cassettes/test_safety_cassettes/test_openai_safety_list_competitor_org_prompt.yaml @@ -0,0 +1,112 @@ +interactions: +- request: + body: '{"messages":[{"role":"user","content":"Compare our product roadmap with + [PII.competitor_org]''s latest offering and [PII.competitor_org]'' pricing strategy."}],"model":"gpt-4o-mini","max_tokens":50}' + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '196' + Content-Type: + - application/json + Host: + - api.openai.com + User-Agent: + - OpenAI/Python 1.99.7 + X-Stainless-Arch: + - x64 + X-Stainless-Async: + - 'false' + X-Stainless-Lang: + - python + X-Stainless-OS: + - MacOS + X-Stainless-Package-Version: + - 1.99.7 + X-Stainless-Runtime: + - CPython + X-Stainless-Runtime-Version: + - 3.10.19 + traceparent: + - 00-ab6557f39c4a2fdcf345ab51c6cbf4e1-89510baec81c0f20-01 + x-stainless-read-timeout: + - '600' + x-stainless-retry-count: + - '0' + method: POST + uri: https://api.openai.com/v1/chat/completions + response: + body: + string: !!binary | + H4sIAAAAAAAA/6rmUlBQykxRslJQSs5ILEnOLcjRdQnwSXEPDQUAAAD//4xTzW7bMAy+9ykI57BL + UqRN0rS7DijW3YYdNmAtDFqibXW2aEh0024osNfY6+1JRjlJnewH2EWQSH6fSH7kenPN/f18U9sP + /PHN9df3n24270I2TQgu7snIHnVqWHEkjv3WbQKhUGI9W6+XV4vF+vxicLRsqUmwqpPZkmet8252 + Pj9fzubr2dnlDl2zMxQ17LM+Ab4NZ8rTW3pU83y6t7QUI1aktn2QGgM3yZJhjC4Kesmmo9OwF/JD + 6jc/v/9oofdYNATCgEZ/jUCPQsFjAxsqohOKwAEsChYY9aGBXAg6D7Ej40pnwJK+mwhYcC/wxH2A + LrDtjUBgtC12gN6C1ARclhScrwbSLjijd4gStF2VS1+VkJqpvRQO8RTe8oYeKEzhBgx6qKnp0gcJ + ovR9IMABgMFF9pBStKCXijwFraFEk4hS1lp5dJaCklKgVxFq3gxcratqAew0ZzQ1OHl962/9ZDI5 + 7Fugso+YtPN90xw40HsWTNoPit3tPM8vGjVcKXMRf4NmpWof61xHRTNPejTkK6mzwf+s590wDf2R + wJlStZ3kwl9o+HCx2hJm4wyOztVuUjLRFJvRfrkHHbHlOx0Ppikz2hGyI3IcPeyt4wPHyUHVfybz + N+5t5ToA/0M/OnRKO12uvAtknTkueAwLlDb0X2EvPR4SziKFB125XByFpISlEvtmuzdZfIpCba5y + VRR0ZLfLU3a5uVpcLFe2XF9kJ88nvwAAAP//AwAK9gPASgQAAA== + headers: + CF-Cache-Status: + - DYNAMIC + CF-Ray: + - 9e4cd68cec17b0da-MAA + Connection: + - keep-alive + Content-Encoding: + - gzip + Content-Type: + - application/json + Date: + - Tue, 31 Mar 2026 05:08:48 GMT + Server: + - cloudflare + Strict-Transport-Security: + - max-age=31536000; includeSubDomains; preload + Transfer-Encoding: + - chunked + X-Content-Type-Options: + - nosniff + access-control-expose-headers: + - X-Request-ID + alt-svc: + - h3=":443"; ma=86400 + openai-organization: + - personal-test-clou2a + openai-processing-ms: + - '1952' + openai-project: + - proj_5zhcB7xbj7xhoXUDTAPm4WiB + openai-version: + - '2020-10-01' + set-cookie: + - __cf_bm=FvJoymSabHcDWsipGgvh06W2zURml50pdhWKYlbPB.s-1774933726.2225685-1.0.1.1-9VMbPqra0Y2q78PITIJotwk97yCMvLZposOSZ9KuCU3jdhxjWJ_EVScuVJndV9LsePrcOvrGw9EVWsS5YAmQN955r6fF2O3uLSbF7JRwcNltQBv9eFOessKUFbDNS53e; + HttpOnly; Secure; Path=/; Domain=api.openai.com; Expires=Tue, 31 Mar 2026 + 05:38:48 GMT + x-openai-proxy-wasm: + - v0.1 + x-ratelimit-limit-requests: + - '10000' + x-ratelimit-limit-tokens: + - '200000' + x-ratelimit-remaining-requests: + - '9997' + x-ratelimit-remaining-tokens: + - '199969' + x-ratelimit-reset-requests: + - 21.886s + x-ratelimit-reset-tokens: + - 9ms + x-request-id: + - req_2d42157fbc2c456585c469386c1577d6 + status: + code: 200 + message: OK +version: 1 diff --git a/packages/opentelemetry-instrumentation-openai/tests/traces/cassettes/test_safety_cassettes/test_openai_safety_regex_credit_card_prompt.yaml b/packages/opentelemetry-instrumentation-openai/tests/traces/cassettes/test_safety_cassettes/test_openai_safety_regex_credit_card_prompt.yaml new file mode 100644 index 0000000000..2bd6c8c192 --- /dev/null +++ b/packages/opentelemetry-instrumentation-openai/tests/traces/cassettes/test_safety_cassettes/test_openai_safety_regex_credit_card_prompt.yaml @@ -0,0 +1,111 @@ +interactions: +- request: + body: '{"messages":[{"role":"user","content":"Check if order for card [PCI.credit_card] + was processed."}],"model":"gpt-4o-mini","max_tokens":50}' + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '137' + Content-Type: + - application/json + Host: + - api.openai.com + User-Agent: + - OpenAI/Python 1.99.7 + X-Stainless-Arch: + - x64 + X-Stainless-Async: + - 'false' + X-Stainless-Lang: + - python + X-Stainless-OS: + - MacOS + X-Stainless-Package-Version: + - 1.99.7 + X-Stainless-Runtime: + - CPython + X-Stainless-Runtime-Version: + - 3.10.19 + traceparent: + - 00-fcd8adb6a798c87b9f90d09b43390d47-e2eef6526fe91e6b-01 + x-stainless-read-timeout: + - '600' + x-stainless-retry-count: + - '0' + method: POST + uri: https://api.openai.com/v1/chat/completions + response: + body: + string: !!binary | + H4sIAAAAAAAA/4xTTYvbMBC951cMuuzFDvlcd3srW1pCS7tQClvKYhRpbKuRJSPJ3TpL/vuO7GSd + 9AN6EXjezPOb96SnCQBTkr0GJioeRN3o9O3dR/nutvg0u52/33zdf/m2v3+8/7DeLz7XbwRL4oTd + /kARTlNTYWkOg7JmgIVDHjCyzrNsdbNcZot1D9RWoo5jZRPSlU1rZVS6mC1W6SxL56+O05VVAj21 + fadPgKf+jDqNxF9UniWnSo3e8xKpdmqiorM6Vhj3XvnATWDJCAprAppe+uaqBm+d6xLYtgE2ILi5 + CiAqFDsIFQINh9aDLYAWkoog7iRYJ9FR1QE3XHd76mtQqEIJCI4bz0U0wk/hTiP3eOSTypFjuoNH + FaqeveFdTUqgcZa2JSEJ1OjITxOSyN7Z1sGWmx0U9KUMnTWP1OCwJCHKlENPL2h6vqTDovU8Gm1a + rc8AbowNPUlv78MRObwYqm1Jerb+t1FWUFC+yilXTyGTeT7YhvXogc6HPrj2IgtGRHUT8mB32P9u + ccyNjddlBJfZEQwkUI/19al+wZZLDFxpfxY8E5ycluPkeEt4K5U9AyZnO/8p5m/cw97k+P/Qj4AQ + 2NA7yJt4fcTlwmObw/iY/tX24nEvmHl0P+l15EGhizlILHirhyvOfOcD1jmFVaJrnBruedHk4mZ5 + vVrLIrtmk8PkGQAA//8DAO2LJUz1AwAA + headers: + CF-Cache-Status: + - DYNAMIC + CF-Ray: + - 9e4cd67f4b1758e1-BLR + Connection: + - keep-alive + Content-Encoding: + - gzip + Content-Type: + - application/json + Date: + - Tue, 31 Mar 2026 05:08:46 GMT + Server: + - cloudflare + Strict-Transport-Security: + - max-age=31536000; includeSubDomains; preload + Transfer-Encoding: + - chunked + X-Content-Type-Options: + - nosniff + access-control-expose-headers: + - X-Request-ID + alt-svc: + - h3=":443"; ma=86400 + openai-organization: + - personal-test-clou2a + openai-processing-ms: + - '1008' + openai-project: + - proj_5zhcB7xbj7xhoXUDTAPm4WiB + openai-version: + - '2020-10-01' + set-cookie: + - __cf_bm=Lh2Fmmb_rloCnnDm.p3o7LvEQp2XS6BV_O7aZdc6xVo-1774933724.0455327-1.0.1.1-kqCU4xmdNs77Po_CxUNy5ENOD_CBM6xEHlRwWf1Xg3pj.6qFdnt1MYdHzOvjn0WzOrw20Ini6jH2zjdz7K2Ir_d5IvOc0gRbeA4SSlCGOPA25jNkEik0VfNbfBM2sZEu; + HttpOnly; Secure; Path=/; Domain=api.openai.com; Expires=Tue, 31 Mar 2026 + 05:38:46 GMT + x-openai-proxy-wasm: + - v0.1 + x-ratelimit-limit-requests: + - '10000' + x-ratelimit-limit-tokens: + - '200000' + x-ratelimit-remaining-requests: + - '9998' + x-ratelimit-remaining-tokens: + - '199984' + x-ratelimit-reset-requests: + - 14.617s + x-ratelimit-reset-tokens: + - 4ms + x-request-id: + - req_56bc539967a44407b31e4a2dce4e04fb + status: + code: 200 + message: OK +version: 1 diff --git a/packages/opentelemetry-instrumentation-openai/tests/traces/cassettes/test_safety_cassettes/test_openai_safety_regex_email_prompt.yaml b/packages/opentelemetry-instrumentation-openai/tests/traces/cassettes/test_safety_cassettes/test_openai_safety_regex_email_prompt.yaml new file mode 100644 index 0000000000..eec987b174 --- /dev/null +++ b/packages/opentelemetry-instrumentation-openai/tests/traces/cassettes/test_safety_cassettes/test_openai_safety_regex_email_prompt.yaml @@ -0,0 +1,111 @@ +interactions: +- request: + body: '{"messages":[{"role":"user","content":"Summarize the following customer + inquiry from [PII.email] about their recent order."}],"model":"gpt-4o-mini","max_tokens":50}' + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '164' + Content-Type: + - application/json + Host: + - api.openai.com + User-Agent: + - OpenAI/Python 1.99.7 + X-Stainless-Arch: + - x64 + X-Stainless-Async: + - 'false' + X-Stainless-Lang: + - python + X-Stainless-OS: + - MacOS + X-Stainless-Package-Version: + - 1.99.7 + X-Stainless-Runtime: + - CPython + X-Stainless-Runtime-Version: + - 3.10.19 + traceparent: + - 00-5a1362be4fd117f74a3440f4d1b03861-ce68b1867e29fdcb-01 + x-stainless-read-timeout: + - '600' + x-stainless-retry-count: + - '0' + method: POST + uri: https://api.openai.com/v1/chat/completions + response: + body: + string: !!binary | + H4sIAAAAAAAA/6rmUlBQykxRslJQSs5ILEnOLcjRdQnwSXGOKAEAAAD//41TTY+bMBC951dMuaSV + yCqbsEtz7K1pD6uq2lO1QsYM4C14XNuQ0ir/vWNIlqQfUi9IvDfvzac/POTl4/Pj+mGH6+Rz+end + xy3lIoqDgvJnlP6supHEOvSK9ERLi8JjcL1N02S33aabzUi0VGATZJXxq4RWrdJqtVlvktU6Xd2+ + PalrUhIdh33hX4Cf4zfUqQv8zvA6PiMtOicqZOwcxKClJiCRcE45L7SP4pmUpD3qsfT9sgVH1g4x + 5J2HPUihlx4mGRyUr8FzdwyA8qB0T02PDgxaR1o0jJRkWxHavoH3dMAebQyqhIG64AXGUq8KBAEV + arQsIQ7pFR6ALBTohWociJw4u6+RDb91yg7wOuQOoNDDnI4lDrVTXvV4mftNDPtlATlCLYwZwBPU + 2BhwXdsKq35wsH91OQKLZedEWIPumuaCEFqTHz3H4T+dmOPLuBuquKfc/SaNSl6jqzPeOpcaRus8 + mWhkj/x9GtfaXW0qYqPW+MzTVxzTbe4mu2g+pplMzqTnApsZT0+ncO2WnUZ7cRaRFLLGYlbONyS6 + QtEFsbjo+c9i/uY99a109T/2MyElGn4lmbFYKHnd8BxmMTy1f4W9zHgsOHLhvCRmXqENeyiwFF0z + PYDIDc5jm/GyKrTGqukVlCaTu+19cleU6X20OC5+ATLQrvQTBAAA + headers: + CF-Cache-Status: + - DYNAMIC + CF-Ray: + - 9e4cd66d98c19135-MAA + Connection: + - keep-alive + Content-Encoding: + - gzip + Content-Type: + - application/json + Date: + - Tue, 31 Mar 2026 05:08:43 GMT + Server: + - cloudflare + Strict-Transport-Security: + - max-age=31536000; includeSubDomains; preload + Transfer-Encoding: + - chunked + X-Content-Type-Options: + - nosniff + access-control-expose-headers: + - X-Request-ID + alt-svc: + - h3=":443"; ma=86400 + openai-organization: + - personal-test-clou2a + openai-processing-ms: + - '1500' + openai-project: + - proj_5zhcB7xbj7xhoXUDTAPm4WiB + openai-version: + - '2020-10-01' + set-cookie: + - __cf_bm=yW_h_X8AdmgDGCDS9eJJqFgp6fCq7LUYQWd1I.Q0onY-1774933721.2115366-1.0.1.1-Cup6GSrVQ9zEL4jBVjl_aML3eZ9dOhqIjIcY_VQp00EueLKtuRhBSDuVjhl22qcsE3GN0QAXjp3Nlsm4lOJAuPgRauOCoirKC2CP0ybHnyrAWwGoajeTYuJeBimGsFXU; + HttpOnly; Secure; Path=/; Domain=api.openai.com; Expires=Tue, 31 Mar 2026 + 05:38:43 GMT + x-openai-proxy-wasm: + - v0.1 + x-ratelimit-limit-requests: + - '10000' + x-ratelimit-limit-tokens: + - '200000' + x-ratelimit-remaining-requests: + - '9999' + x-ratelimit-remaining-tokens: + - '199977' + x-ratelimit-reset-requests: + - 8.64s + x-ratelimit-reset-tokens: + - 6ms + x-request-id: + - req_c6a72f2ac40445f486f2902914351c69 + status: + code: 200 + message: OK +version: 1 diff --git a/packages/opentelemetry-instrumentation-openai/tests/traces/cassettes/test_safety_cassettes/test_openai_safety_streaming_with_pii.yaml b/packages/opentelemetry-instrumentation-openai/tests/traces/cassettes/test_safety_cassettes/test_openai_safety_streaming_with_pii.yaml new file mode 100644 index 0000000000..92834bd57a --- /dev/null +++ b/packages/opentelemetry-instrumentation-openai/tests/traces/cassettes/test_safety_cassettes/test_openai_safety_streaming_with_pii.yaml @@ -0,0 +1,243 @@ +interactions: +- request: + body: '{"messages":[{"role":"user","content":"Acknowledge receipt of message from + [PII.email] about [custom_compliance.project-phoenix]."}],"model":"gpt-4o-mini","max_tokens":50,"stream":true}' + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '185' + Content-Type: + - application/json + Host: + - api.openai.com + User-Agent: + - OpenAI/Python 1.99.7 + X-Stainless-Arch: + - x64 + X-Stainless-Async: + - 'false' + X-Stainless-Lang: + - python + X-Stainless-OS: + - MacOS + X-Stainless-Package-Version: + - 1.99.7 + X-Stainless-Runtime: + - CPython + X-Stainless-Runtime-Version: + - 3.10.19 + traceparent: + - 00-845f4b5c2e8c53f66748b6be8c89fb98-ab56057423c892b1-01 + x-stainless-read-timeout: + - '600' + x-stainless-retry-count: + - '0' + method: POST + uri: https://api.openai.com/v1/chat/completions + response: + body: + string: 'data: {"id":"chatcmpl-DPLdPTyvRW3K2pfOmlyL62VUg6FpW","object":"chat.completion.chunk","created":1774933735,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_c93645df76","choices":[{"index":0,"delta":{"role":"assistant","content":"","refusal":null},"logprobs":null,"finish_reason":null}],"obfuscation":"MU7RNg"} + + + data: {"id":"chatcmpl-DPLdPTyvRW3K2pfOmlyL62VUg6FpW","object":"chat.completion.chunk","created":1774933735,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_c93645df76","choices":[{"index":0,"delta":{"content":"I"},"logprobs":null,"finish_reason":null}],"obfuscation":"g5CiZAe"} + + + data: {"id":"chatcmpl-DPLdPTyvRW3K2pfOmlyL62VUg6FpW","object":"chat.completion.chunk","created":1774933735,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_c93645df76","choices":[{"index":0,"delta":{"content":" + acknowledge"},"logprobs":null,"finish_reason":null}],"obfuscation":"vfwK0KSwFIcw"} + + + data: {"id":"chatcmpl-DPLdPTyvRW3K2pfOmlyL62VUg6FpW","object":"chat.completion.chunk","created":1774933735,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_c93645df76","choices":[{"index":0,"delta":{"content":" + receipt"},"logprobs":null,"finish_reason":null}],"obfuscation":""} + + + data: {"id":"chatcmpl-DPLdPTyvRW3K2pfOmlyL62VUg6FpW","object":"chat.completion.chunk","created":1774933735,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_c93645df76","choices":[{"index":0,"delta":{"content":" + of"},"logprobs":null,"finish_reason":null}],"obfuscation":"8jcLm"} + + + data: {"id":"chatcmpl-DPLdPTyvRW3K2pfOmlyL62VUg6FpW","object":"chat.completion.chunk","created":1774933735,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_c93645df76","choices":[{"index":0,"delta":{"content":" + your"},"logprobs":null,"finish_reason":null}],"obfuscation":"DKc"} + + + data: {"id":"chatcmpl-DPLdPTyvRW3K2pfOmlyL62VUg6FpW","object":"chat.completion.chunk","created":1774933735,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_c93645df76","choices":[{"index":0,"delta":{"content":" + message"},"logprobs":null,"finish_reason":null}],"obfuscation":""} + + + data: {"id":"chatcmpl-DPLdPTyvRW3K2pfOmlyL62VUg6FpW","object":"chat.completion.chunk","created":1774933735,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_c93645df76","choices":[{"index":0,"delta":{"content":" + regarding"},"logprobs":null,"finish_reason":null}],"obfuscation":"4B5uKdTmwrwaVf"} + + + data: {"id":"chatcmpl-DPLdPTyvRW3K2pfOmlyL62VUg6FpW","object":"chat.completion.chunk","created":1774933735,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_c93645df76","choices":[{"index":0,"delta":{"content":" + the"},"logprobs":null,"finish_reason":null}],"obfuscation":"2hhv"} + + + data: {"id":"chatcmpl-DPLdPTyvRW3K2pfOmlyL62VUg6FpW","object":"chat.completion.chunk","created":1774933735,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_c93645df76","choices":[{"index":0,"delta":{"content":" + custom"},"logprobs":null,"finish_reason":null}],"obfuscation":"w"} + + + data: {"id":"chatcmpl-DPLdPTyvRW3K2pfOmlyL62VUg6FpW","object":"chat.completion.chunk","created":1774933735,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_c93645df76","choices":[{"index":0,"delta":{"content":" + compliance"},"logprobs":null,"finish_reason":null}],"obfuscation":"HpOAcK3Ou3N4h"} + + + data: {"id":"chatcmpl-DPLdPTyvRW3K2pfOmlyL62VUg6FpW","object":"chat.completion.chunk","created":1774933735,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_c93645df76","choices":[{"index":0,"delta":{"content":" + project"},"logprobs":null,"finish_reason":null}],"obfuscation":""} + + + data: {"id":"chatcmpl-DPLdPTyvRW3K2pfOmlyL62VUg6FpW","object":"chat.completion.chunk","created":1774933735,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_c93645df76","choices":[{"index":0,"delta":{"content":" + \""},"logprobs":null,"finish_reason":null}],"obfuscation":"2vCOL"} + + + data: {"id":"chatcmpl-DPLdPTyvRW3K2pfOmlyL62VUg6FpW","object":"chat.completion.chunk","created":1774933735,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_c93645df76","choices":[{"index":0,"delta":{"content":"Project"},"logprobs":null,"finish_reason":null}],"obfuscation":"w"} + + + data: {"id":"chatcmpl-DPLdPTyvRW3K2pfOmlyL62VUg6FpW","object":"chat.completion.chunk","created":1774933735,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_c93645df76","choices":[{"index":0,"delta":{"content":" + Phoenix"},"logprobs":null,"finish_reason":null}],"obfuscation":""} + + + data: {"id":"chatcmpl-DPLdPTyvRW3K2pfOmlyL62VUg6FpW","object":"chat.completion.chunk","created":1774933735,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_c93645df76","choices":[{"index":0,"delta":{"content":".\""},"logprobs":null,"finish_reason":null}],"obfuscation":"LHTao"} + + + data: {"id":"chatcmpl-DPLdPTyvRW3K2pfOmlyL62VUg6FpW","object":"chat.completion.chunk","created":1774933735,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_c93645df76","choices":[{"index":0,"delta":{"content":" + If"},"logprobs":null,"finish_reason":null}],"obfuscation":"ccp8b"} + + + data: {"id":"chatcmpl-DPLdPTyvRW3K2pfOmlyL62VUg6FpW","object":"chat.completion.chunk","created":1774933735,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_c93645df76","choices":[{"index":0,"delta":{"content":" + you"},"logprobs":null,"finish_reason":null}],"obfuscation":"PBsW"} + + + data: {"id":"chatcmpl-DPLdPTyvRW3K2pfOmlyL62VUg6FpW","object":"chat.completion.chunk","created":1774933735,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_c93645df76","choices":[{"index":0,"delta":{"content":" + have"},"logprobs":null,"finish_reason":null}],"obfuscation":"aOv"} + + + data: {"id":"chatcmpl-DPLdPTyvRW3K2pfOmlyL62VUg6FpW","object":"chat.completion.chunk","created":1774933735,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_c93645df76","choices":[{"index":0,"delta":{"content":" + any"},"logprobs":null,"finish_reason":null}],"obfuscation":"won6"} + + + data: {"id":"chatcmpl-DPLdPTyvRW3K2pfOmlyL62VUg6FpW","object":"chat.completion.chunk","created":1774933735,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_c93645df76","choices":[{"index":0,"delta":{"content":" + specific"},"logprobs":null,"finish_reason":null}],"obfuscation":"C64ccDPH3UGQuH3"} + + + data: {"id":"chatcmpl-DPLdPTyvRW3K2pfOmlyL62VUg6FpW","object":"chat.completion.chunk","created":1774933735,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_c93645df76","choices":[{"index":0,"delta":{"content":" + questions"},"logprobs":null,"finish_reason":null}],"obfuscation":"3BKTQxApbW6N2U"} + + + data: {"id":"chatcmpl-DPLdPTyvRW3K2pfOmlyL62VUg6FpW","object":"chat.completion.chunk","created":1774933735,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_c93645df76","choices":[{"index":0,"delta":{"content":" + or"},"logprobs":null,"finish_reason":null}],"obfuscation":"rwj3p"} + + + data: {"id":"chatcmpl-DPLdPTyvRW3K2pfOmlyL62VUg6FpW","object":"chat.completion.chunk","created":1774933735,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_c93645df76","choices":[{"index":0,"delta":{"content":" + need"},"logprobs":null,"finish_reason":null}],"obfuscation":"UUT"} + + + data: {"id":"chatcmpl-DPLdPTyvRW3K2pfOmlyL62VUg6FpW","object":"chat.completion.chunk","created":1774933735,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_c93645df76","choices":[{"index":0,"delta":{"content":" + further"},"logprobs":null,"finish_reason":null}],"obfuscation":""} + + + data: {"id":"chatcmpl-DPLdPTyvRW3K2pfOmlyL62VUg6FpW","object":"chat.completion.chunk","created":1774933735,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_c93645df76","choices":[{"index":0,"delta":{"content":" + assistance"},"logprobs":null,"finish_reason":null}],"obfuscation":"IjyboOvB1cOfQ"} + + + data: {"id":"chatcmpl-DPLdPTyvRW3K2pfOmlyL62VUg6FpW","object":"chat.completion.chunk","created":1774933735,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_c93645df76","choices":[{"index":0,"delta":{"content":" + related"},"logprobs":null,"finish_reason":null}],"obfuscation":""} + + + data: {"id":"chatcmpl-DPLdPTyvRW3K2pfOmlyL62VUg6FpW","object":"chat.completion.chunk","created":1774933735,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_c93645df76","choices":[{"index":0,"delta":{"content":" + to"},"logprobs":null,"finish_reason":null}],"obfuscation":"COkNX"} + + + data: {"id":"chatcmpl-DPLdPTyvRW3K2pfOmlyL62VUg6FpW","object":"chat.completion.chunk","created":1774933735,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_c93645df76","choices":[{"index":0,"delta":{"content":" + this"},"logprobs":null,"finish_reason":null}],"obfuscation":"fFU"} + + + data: {"id":"chatcmpl-DPLdPTyvRW3K2pfOmlyL62VUg6FpW","object":"chat.completion.chunk","created":1774933735,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_c93645df76","choices":[{"index":0,"delta":{"content":" + topic"},"logprobs":null,"finish_reason":null}],"obfuscation":"TC"} + + + data: {"id":"chatcmpl-DPLdPTyvRW3K2pfOmlyL62VUg6FpW","object":"chat.completion.chunk","created":1774933735,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_c93645df76","choices":[{"index":0,"delta":{"content":","},"logprobs":null,"finish_reason":null}],"obfuscation":"Qfi0yTH"} + + + data: {"id":"chatcmpl-DPLdPTyvRW3K2pfOmlyL62VUg6FpW","object":"chat.completion.chunk","created":1774933735,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_c93645df76","choices":[{"index":0,"delta":{"content":" + feel"},"logprobs":null,"finish_reason":null}],"obfuscation":"wnt"} + + + data: {"id":"chatcmpl-DPLdPTyvRW3K2pfOmlyL62VUg6FpW","object":"chat.completion.chunk","created":1774933735,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_c93645df76","choices":[{"index":0,"delta":{"content":" + free"},"logprobs":null,"finish_reason":null}],"obfuscation":"cLC"} + + + data: {"id":"chatcmpl-DPLdPTyvRW3K2pfOmlyL62VUg6FpW","object":"chat.completion.chunk","created":1774933735,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_c93645df76","choices":[{"index":0,"delta":{"content":" + to"},"logprobs":null,"finish_reason":null}],"obfuscation":"XMcPN"} + + + data: {"id":"chatcmpl-DPLdPTyvRW3K2pfOmlyL62VUg6FpW","object":"chat.completion.chunk","created":1774933735,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_c93645df76","choices":[{"index":0,"delta":{"content":" + ask"},"logprobs":null,"finish_reason":null}],"obfuscation":"fL0q"} + + + data: {"id":"chatcmpl-DPLdPTyvRW3K2pfOmlyL62VUg6FpW","object":"chat.completion.chunk","created":1774933735,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_c93645df76","choices":[{"index":0,"delta":{"content":"!"},"logprobs":null,"finish_reason":null}],"obfuscation":"nHqAKxc"} + + + data: {"id":"chatcmpl-DPLdPTyvRW3K2pfOmlyL62VUg6FpW","object":"chat.completion.chunk","created":1774933735,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_c93645df76","choices":[{"index":0,"delta":{},"logprobs":null,"finish_reason":"stop"}],"obfuscation":"hK"} + + + data: [DONE] + + + ' + headers: + CF-Cache-Status: + - DYNAMIC + CF-Ray: + - 9e4cd6c5aeb37f87-MAA + Connection: + - keep-alive + Content-Type: + - text/event-stream; charset=utf-8 + Date: + - Tue, 31 Mar 2026 05:08:55 GMT + Server: + - cloudflare + Strict-Transport-Security: + - max-age=31536000; includeSubDomains; preload + Transfer-Encoding: + - chunked + X-Content-Type-Options: + - nosniff + access-control-expose-headers: + - X-Request-ID + alt-svc: + - h3=":443"; ma=86400 + openai-organization: + - personal-test-clou2a + openai-processing-ms: + - '281' + openai-project: + - proj_5zhcB7xbj7xhoXUDTAPm4WiB + openai-version: + - '2020-10-01' + set-cookie: + - __cf_bm=A_o71N6qboDVDequIrvFo7o1_VJ6wB9Q9qdjthkG86o-1774933735.303754-1.0.1.1-Rq5hplLUIb_4Lx4wViiviljySSZ1kd8GP.K9QoWNyx0dr1YvZLhK.J.W3N3Kpmx52GmUtq5xLzc4DT1xbeDMuW.ATMpKIL_UwOjILnIdK_yuh5oTegFvVJKMpnMIyTLf; + HttpOnly; Secure; Path=/; Domain=api.openai.com; Expires=Tue, 31 Mar 2026 + 05:38:55 GMT + x-openai-proxy-wasm: + - v0.1 + x-ratelimit-limit-requests: + - '10000' + x-ratelimit-limit-tokens: + - '200000' + x-ratelimit-remaining-requests: + - '9994' + x-ratelimit-remaining-tokens: + - '199975' + x-ratelimit-reset-requests: + - 47.362s + x-ratelimit-reset-tokens: + - 7ms + x-request-id: + - req_24dde50d108349eaac96f1bdd1678e3f + status: + code: 200 + message: OK +version: 1 diff --git a/packages/opentelemetry-instrumentation-openai/tests/traces/cassettes/test_safety_cassettes/test_openai_safety_udf_project_codenames_prompt.yaml b/packages/opentelemetry-instrumentation-openai/tests/traces/cassettes/test_safety_cassettes/test_openai_safety_udf_project_codenames_prompt.yaml new file mode 100644 index 0000000000..625460d5f0 --- /dev/null +++ b/packages/opentelemetry-instrumentation-openai/tests/traces/cassettes/test_safety_cassettes/test_openai_safety_udf_project_codenames_prompt.yaml @@ -0,0 +1,112 @@ +interactions: +- request: + body: '{"messages":[{"role":"user","content":"Prepare a status update for [custom_compliance.project-phoenix] + and [custom_compliance.operation-atlas] including timeline and budget."}],"model":"gpt-4o-mini","max_tokens":50}' + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '215' + Content-Type: + - application/json + Host: + - api.openai.com + User-Agent: + - OpenAI/Python 1.99.7 + X-Stainless-Arch: + - x64 + X-Stainless-Async: + - 'false' + X-Stainless-Lang: + - python + X-Stainless-OS: + - MacOS + X-Stainless-Package-Version: + - 1.99.7 + X-Stainless-Runtime: + - CPython + X-Stainless-Runtime-Version: + - 3.10.19 + traceparent: + - 00-86531b0dfbffe521bdb08a6b450ce1ea-2c27ef59fdd0a667-01 + x-stainless-read-timeout: + - '600' + x-stainless-retry-count: + - '0' + method: POST + uri: https://api.openai.com/v1/chat/completions + response: + body: + string: !!binary | + H4sIAAAAAAAA/6rmUlBQykxRslJQSs5ILEnOLcjRdQnwSfHycgcAAAD//4xTyW7bMBC9+ysI+iZY + htc4zq3wrUCboAsCNDYEhhpJjCWSJSnHQeF/z5CSLbkL0IsAvTdv5s3Co/j08Zg9Hu30i2aPP1Jw + +5+fX+jIK9TzC3B3Vo25Qh04oWRDcwPMgc86Xa0W6/l8NVsHolIplF6WaxcvVFwJKeLZZLaIJ6t4 + etuqCyU4WAx7wl9CfoWv9ylTOCI8GZ2RCqxlOSB2DkLQqNIjlFkrrGPS0VFHciUdyGB9OBySr465 + 2pLvOkXDW7mVQ48+GOX7s+T+AOYg4HUrvxXCkjqEEW3UQaRgCSO20RvQyjiSKUPcqyJ7eCPYmRPM + iQPYOxJFbUryUCiQ4hhFhMkU8XsNhvnJkQ+uZBbxGrvENAWQTW2dqsjGT1cwyUPl3LBq7J3Gcdwa + Jk88RCb8EjnWTb1YN/V2PjSKNrUx2H3b9h1WI1vZH4+BrLbMr0jWZdkjmJTKBaNhMbuWOV1WUaoc + az7b36Q0w0HYIsGLsHgeOPYSZO4KGvgTfndh6fXVHimmqrRLnNpDKDhfNglpd2oduWwPgjq0WHb4 + 7Vl0lS3BS2aitL2joZzxAtJO2V0Yq1OhesSg1/WfZv6Wu+lcyPx/0ncE56DxDSXaQCr4dcNdmAG/ + 5X+FXWYcDFPrj5lD4gQYv4kUMlaXzfOg9s06qBJcVw5GG9G8kUwnfD2/WSzTbHVDB6fBOwAAAP// + AwDhn8n2MQQAAA== + headers: + CF-Cache-Status: + - DYNAMIC + CF-Ray: + - 9e4cd69b9aaf7edf-MAA + Connection: + - keep-alive + Content-Encoding: + - gzip + Content-Type: + - application/json + Date: + - Tue, 31 Mar 2026 05:08:51 GMT + Server: + - cloudflare + Strict-Transport-Security: + - max-age=31536000; includeSubDomains; preload + Transfer-Encoding: + - chunked + X-Content-Type-Options: + - nosniff + access-control-expose-headers: + - X-Request-ID + alt-svc: + - h3=":443"; ma=86400 + openai-organization: + - personal-test-clou2a + openai-processing-ms: + - '1810' + openai-project: + - proj_5zhcB7xbj7xhoXUDTAPm4WiB + openai-version: + - '2020-10-01' + set-cookie: + - __cf_bm=a2XvgdY50Q8OoUV1mlHQsEPMVQoSCNhTrT2WqTGgbIM-1774933728.5755851-1.0.1.1-huykiYuPIB7SySZRUFq4.00QYdA_EOj.yunqv8e_JY_tgpesd3oZqsCAYPo_ylbe8QyKrdIxidV2ZMDGeB2rw_6vHAvSpmEEoSmlOPGacZ6HuC8shcs6fsC2llgkKPbF; + HttpOnly; Secure; Path=/; Domain=api.openai.com; Expires=Tue, 31 Mar 2026 + 05:38:51 GMT + x-openai-proxy-wasm: + - v0.1 + x-ratelimit-limit-requests: + - '10000' + x-ratelimit-limit-tokens: + - '200000' + x-ratelimit-remaining-requests: + - '9996' + x-ratelimit-remaining-tokens: + - '199964' + x-ratelimit-reset-requests: + - 27.502s + x-ratelimit-reset-tokens: + - 10ms + x-request-id: + - req_22db4a9ab1b34137a2092cba8c2ae841 + status: + code: 200 + message: OK +version: 1 diff --git a/packages/opentelemetry-instrumentation-openai/tests/traces/test_safety_cassettes.py b/packages/opentelemetry-instrumentation-openai/tests/traces/test_safety_cassettes.py new file mode 100644 index 0000000000..3fe8a7cfa9 --- /dev/null +++ b/packages/opentelemetry-instrumentation-openai/tests/traces/test_safety_cassettes.py @@ -0,0 +1,624 @@ +""" +FR VCR cassette tests for safety rule enforcement on OpenAI instrumentation. + +These tests record real OpenAI API interactions via VCR cassettes and verify +that the safety callback system correctly detects and masks sensitive content +in both prompts and completions. + +Safety rule flavours covered: + - RegEx: PCI.credit_card, PII.email + - List: PII.competitor_org (list of company names) + - UDF: custom_compliance.project_codename (user-defined function + returning 2 sub-rule findings) + +North-star: This is a NEW file (FR-owned). Zero delta on TL files. +""" + +import re + +import pytest +from opentelemetry.instrumentation.fortifyroot import ( + SafetyFinding, + SafetyLocation, + SafetyResult, + clear_safety_handlers, + register_completion_safety_handler, + register_prompt_safety_handler, +) +from opentelemetry.semconv._incubating.attributes import ( + gen_ai_attributes as GenAIAttributes, +) + + +# --------------------------------------------------------------------------- +# Safety rule definitions (mirror what real SDK rules would do) +# --------------------------------------------------------------------------- + +# RegEx patterns +_CREDIT_CARD_RE = re.compile(r"\b(?:\d[ -]*?){13,19}\b") +_EMAIL_RE = re.compile(r"\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}\b") + +# List of competitor org names (case-insensitive) +_COMPETITOR_ORGS = ["acme corp", "globex industries", "initech", "umbrella inc"] + +# UDF: detects two fictional project codenames +_PROJECT_CODENAMES = ["project-phoenix", "operation-atlas"] + +# Module-level action setting (switched by fixtures) +_SAFETY_ACTION = "MASK" + + +def _build_findings_and_mask(text, findings_list, action="MASK"): + """Apply all collected findings to produce masked (or allowed) text.""" + if not findings_list: + return None + if action == "MASK": + masked = text + # Sort findings by start position descending so replacements don't shift indices + for f in sorted(findings_list, key=lambda x: x.start, reverse=True): + masked = masked[:f.start] + f"[{f.rule_name}]" + masked[f.end:] + return SafetyResult( + text=masked, + overall_action="MASK", + findings=findings_list, + ) + # ALLOW: text unchanged, findings still recorded + return SafetyResult( + text=text, + overall_action="ALLOW", + findings=findings_list, + ) + + +def _scan_text(text, location): + """Run all safety rule types against text, return findings.""" + if not isinstance(text, str) or not text: + return [] + + action = _SAFETY_ACTION + findings = [] + + # --- RegEx: PCI.credit_card --- + for m in _CREDIT_CARD_RE.finditer(text): + findings.append(SafetyFinding( + category="PCI", + severity="HIGH", + action=action, + rule_name="PCI.credit_card", + start=m.start(), + end=m.end(), + )) + + # --- RegEx: PII.email --- + for m in _EMAIL_RE.finditer(text): + findings.append(SafetyFinding( + category="PII", + severity="MEDIUM", + action=action, + rule_name="PII.email", + start=m.start(), + end=m.end(), + )) + + # --- List: PII.competitor_org --- + text_lower = text.lower() + for org in _COMPETITOR_ORGS: + idx = text_lower.find(org) + while idx != -1: + findings.append(SafetyFinding( + category="PII", + severity="LOW", + action=action, + rule_name="PII.competitor_org", + start=idx, + end=idx + len(org), + )) + idx = text_lower.find(org, idx + len(org)) + + # --- UDF: custom_compliance.project_codename (returns 2 sub-rules) --- + for codename in _PROJECT_CODENAMES: + idx = text_lower.find(codename) + while idx != -1: + findings.append(SafetyFinding( + category="COMPLIANCE", + severity="HIGH", + action=action, + rule_name=f"custom_compliance.{codename}", + start=idx, + end=idx + len(codename), + )) + idx = text_lower.find(codename, idx + len(codename)) + + return findings + + +def _prompt_handler(context): + """Prompt safety handler covering RegEx, List, and UDF rules.""" + if context.location != SafetyLocation.PROMPT: + return None + findings = _scan_text(context.text, SafetyLocation.PROMPT) + return _build_findings_and_mask(context.text, findings, action=_SAFETY_ACTION) + + +def _completion_handler(context): + """Completion safety handler covering RegEx, List, and UDF rules.""" + if context.location != SafetyLocation.COMPLETION: + return None + findings = _scan_text(context.text, SafetyLocation.COMPLETION) + return _build_findings_and_mask(context.text, findings, action=_SAFETY_ACTION) + + +# --------------------------------------------------------------------------- +# Fixtures +# --------------------------------------------------------------------------- + +@pytest.fixture(autouse=True) +def _register_safety_handlers(): + """Register all safety handlers before each test, clear after.""" + clear_safety_handlers() + register_prompt_safety_handler(_prompt_handler) + register_completion_safety_handler(_completion_handler) + yield + clear_safety_handlers() + + +# --------------------------------------------------------------------------- +# T2-A3: RegEx rule — PII.email in prompt +# --------------------------------------------------------------------------- + +@pytest.mark.vcr +@pytest.mark.fr +def test_openai_safety_regex_email_prompt( + instrument_legacy, span_exporter, openai_client +): + """Prompt contains an email address. Safety should detect and mask it.""" + openai_client.chat.completions.create( + model="gpt-4o-mini", + messages=[ + { + "role": "user", + "content": ( + "Summarize the following customer inquiry from " + "john.doe@example.com about their recent order." + ), + } + ], + max_tokens=50, + ) + + spans = span_exporter.get_finished_spans() + assert len(spans) >= 1 + span = spans[0] + + # The prompt content should have been masked (email replaced) + prompt_content = span.attributes.get( + f"{GenAIAttributes.GEN_AI_PROMPT}.0.content" + ) + assert prompt_content is not None + assert "john.doe@example.com" not in prompt_content + assert "[PII.email]" in prompt_content + + # Safety finding should be recorded as a span event + safety_events = [ + e for e in span.events + if "fortifyroot.safety" in (e.name or "") + ] + assert len(safety_events) >= 1 + event_attrs = safety_events[0].attributes + assert event_attrs["fortifyroot.safety.rule_name"] == "PII.email" + assert event_attrs["fortifyroot.safety.action"] == "MASK" + assert event_attrs["fortifyroot.safety.location"] == "PROMPT" + + +# --------------------------------------------------------------------------- +# T2-A3: RegEx rule — PCI.credit_card in prompt +# --------------------------------------------------------------------------- + +@pytest.mark.vcr +@pytest.mark.fr +def test_openai_safety_regex_credit_card_prompt( + instrument_legacy, span_exporter, openai_client +): + """Prompt contains a credit card number. Safety should detect and mask it.""" + openai_client.chat.completions.create( + model="gpt-4o-mini", + messages=[ + { + "role": "user", + "content": ( + "Check if order for card 4111 1111 1111 1111 was processed." + ), + } + ], + max_tokens=50, + ) + + spans = span_exporter.get_finished_spans() + assert len(spans) >= 1 + span = spans[0] + + prompt_content = span.attributes.get( + f"{GenAIAttributes.GEN_AI_PROMPT}.0.content" + ) + assert prompt_content is not None + assert "4111 1111 1111 1111" not in prompt_content + assert "[PCI.credit_card]" in prompt_content + + safety_events = [ + e for e in span.events + if "fortifyroot.safety" in (e.name or "") + ] + assert len(safety_events) >= 1 + found_cc = any( + e.attributes.get("fortifyroot.safety.rule_name") == "PCI.credit_card" + for e in safety_events + ) + assert found_cc, "Expected PCI.credit_card safety finding" + + +# --------------------------------------------------------------------------- +# T2-A3: List rule — PII.competitor_org in prompt +# --------------------------------------------------------------------------- + +@pytest.mark.vcr +@pytest.mark.fr +def test_openai_safety_list_competitor_org_prompt( + instrument_legacy, span_exporter, openai_client +): + """Prompt mentions a competitor org from the blocklist.""" + openai_client.chat.completions.create( + model="gpt-4o-mini", + messages=[ + { + "role": "user", + "content": ( + "Compare our product roadmap with Acme Corp's latest " + "offering and Globex Industries' pricing strategy." + ), + } + ], + max_tokens=50, + ) + + spans = span_exporter.get_finished_spans() + assert len(spans) >= 1 + span = spans[0] + + prompt_content = span.attributes.get( + f"{GenAIAttributes.GEN_AI_PROMPT}.0.content" + ) + assert prompt_content is not None + assert "acme corp" not in prompt_content.lower() + assert "globex industries" not in prompt_content.lower() + assert "[PII.competitor_org]" in prompt_content + + safety_events = [ + e for e in span.events + if "fortifyroot.safety" in (e.name or "") + ] + found_org = any( + e.attributes.get("fortifyroot.safety.rule_name") == "PII.competitor_org" + for e in safety_events + ) + assert found_org, "Expected PII.competitor_org safety finding" + + +# --------------------------------------------------------------------------- +# T2-A3: UDF rule — custom_compliance with 2 sub-rules in prompt +# --------------------------------------------------------------------------- + +@pytest.mark.vcr +@pytest.mark.fr +def test_openai_safety_udf_project_codenames_prompt( + instrument_legacy, span_exporter, openai_client +): + """Prompt contains two project codenames detected by UDF rule.""" + openai_client.chat.completions.create( + model="gpt-4o-mini", + messages=[ + { + "role": "user", + "content": ( + "Prepare a status update for Project-Phoenix and " + "Operation-Atlas including timeline and budget." + ), + } + ], + max_tokens=50, + ) + + spans = span_exporter.get_finished_spans() + assert len(spans) >= 1 + span = spans[0] + + prompt_content = span.attributes.get( + f"{GenAIAttributes.GEN_AI_PROMPT}.0.content" + ) + assert prompt_content is not None + # Masking replaces "Project-Phoenix" with "[custom_compliance.project-phoenix]" + # so we verify the mask tokens are present (original casing is gone) + assert "[custom_compliance.project-phoenix]" in prompt_content + assert "[custom_compliance.operation-atlas]" in prompt_content + # Verify original text (with capital letters) is replaced + assert "Project-Phoenix" not in prompt_content + assert "Operation-Atlas" not in prompt_content + + # UDF should produce 2 findings (one per codename) + safety_events = [ + e for e in span.events + if "fortifyroot.safety" in (e.name or "") + ] + udf_findings = [ + e for e in safety_events + if (e.attributes.get("fortifyroot.safety.rule_name") or "").startswith( + "custom_compliance." + ) + ] + assert len(udf_findings) >= 2, ( + f"Expected 2 UDF findings, got {len(udf_findings)}" + ) + + +# --------------------------------------------------------------------------- +# T2-A3: Completion safety — email in LLM response +# --------------------------------------------------------------------------- + +@pytest.mark.vcr +@pytest.mark.fr +def test_openai_safety_completion_email_masked( + instrument_legacy, span_exporter, openai_client +): + """ + Ask the LLM to generate fictional contact info. The response should + contain an email-like pattern that completion safety detects and masks. + + The prompt does NOT contain PII — it asks the LLM to invent an email. + This ensures completion safety (not just prompt echo-back) is tested. + """ + openai_client.chat.completions.create( + model="gpt-4o-mini", + messages=[ + { + "role": "user", + "content": ( + "Create a fictional contact card. The person's name is " + "Jane Smith, company is Widgets Inc, role is VP of Sales. " + "Invent a plausible work email and phone. Format as:\n" + "Name: ...\nEmail: ...\nPhone: ..." + ), + } + ], + max_tokens=100, + ) + + spans = span_exporter.get_finished_spans() + assert len(spans) >= 1 + span = spans[0] + + # Prompt has no PII, so no prompt masking expected + prompt_content = span.attributes.get( + f"{GenAIAttributes.GEN_AI_PROMPT}.0.content" + ) + assert prompt_content is not None + + # Completion should be recorded AND the LLM-generated email should be + # masked by completion safety + completion_content = span.attributes.get( + f"{GenAIAttributes.GEN_AI_COMPLETION}.0.content" + ) + assert completion_content is not None, "Completion content should be recorded" + assert "[PII.email]" in completion_content, ( + "LLM-generated email in response should be masked by completion safety" + ) + + # Verify completion safety event was emitted + safety_events = [ + e for e in span.events + if "fortifyroot.safety" in (e.name or "") + ] + completion_findings = [ + e for e in safety_events + if e.attributes.get("fortifyroot.safety.location") == "COMPLETION" + ] + assert len(completion_findings) >= 1, "Should have completion safety finding" + assert any( + e.attributes.get("fortifyroot.safety.rule_name") == "PII.email" + for e in completion_findings + ), "Completion safety should detect PII.email" + + +# --------------------------------------------------------------------------- +# T2-A3: Combined — multiple rule types in single prompt +# --------------------------------------------------------------------------- + +@pytest.mark.vcr +@pytest.mark.fr +def test_openai_safety_combined_rules_prompt( + instrument_legacy, span_exporter, openai_client +): + """Prompt triggers all rule types: RegEx (email), List (org), UDF (codename).""" + openai_client.chat.completions.create( + model="gpt-4o-mini", + messages=[ + { + "role": "user", + "content": ( + "Send the Project-Phoenix briefing to contact@acme-corp.com " + "and CC the Initech team." + ), + } + ], + max_tokens=50, + ) + + spans = span_exporter.get_finished_spans() + assert len(spans) >= 1 + span = spans[0] + + prompt_content = span.attributes.get( + f"{GenAIAttributes.GEN_AI_PROMPT}.0.content" + ) + assert prompt_content is not None + + # Verify all 3 rule types fired + safety_events = [ + e for e in span.events + if "fortifyroot.safety" in (e.name or "") + ] + rule_names = { + e.attributes.get("fortifyroot.safety.rule_name") + for e in safety_events + } + assert "PII.email" in rule_names, "Expected PII.email finding" + assert "PII.competitor_org" in rule_names or any( + "competitor_org" in (r or "") for r in rule_names + ), "Expected PII.competitor_org finding" + assert any( + (r or "").startswith("custom_compliance.") for r in rule_names + ), "Expected custom_compliance UDF finding" + + +# --------------------------------------------------------------------------- +# T2-A3: Streaming with PII — verify safety on streamed completion +# --------------------------------------------------------------------------- + +@pytest.mark.vcr +@pytest.mark.fr +def test_openai_safety_streaming_with_pii( + instrument_legacy, span_exporter, openai_client +): + """Streaming chat with PII in prompt. Verify prompt safety applied.""" + response = openai_client.chat.completions.create( + model="gpt-4o-mini", + messages=[ + { + "role": "user", + "content": ( + "Acknowledge receipt of message from " + "alice.jones@example.org about Project-Phoenix." + ), + } + ], + max_tokens=50, + stream=True, + ) + + # Consume the stream + for _ in response: + pass + + spans = span_exporter.get_finished_spans() + assert len(spans) >= 1 + span = spans[0] + + # Prompt should be masked + prompt_content = span.attributes.get( + f"{GenAIAttributes.GEN_AI_PROMPT}.0.content" + ) + assert prompt_content is not None + assert "alice.jones@example.org" not in prompt_content + assert "[PII.email]" in prompt_content + + +# =========================================================================== +# ALLOW action tests +# =========================================================================== +# When action=ALLOW, text passes through UNCHANGED but findings are still +# recorded as span events for audit/observability. +# =========================================================================== + + +@pytest.fixture +def _set_allow_action(): + """Switch safety handlers to ALLOW mode for the test.""" + global _SAFETY_ACTION + _SAFETY_ACTION = "ALLOW" + yield + _SAFETY_ACTION = "MASK" + + +@pytest.mark.vcr +@pytest.mark.fr +def test_openai_safety_allow_email_not_masked( + _set_allow_action, instrument_legacy, span_exporter, openai_client +): + """ALLOW action: email detected but text passes through unchanged.""" + openai_client.chat.completions.create( + model="gpt-4o-mini", + messages=[ + { + "role": "user", + "content": ( + "Forward this message from alice.jones@example.org " + "to the support team." + ), + } + ], + max_tokens=50, + ) + + spans = span_exporter.get_finished_spans() + assert len(spans) >= 1 + span = spans[0] + + # With ALLOW, the original text should pass through unchanged + prompt_content = span.attributes.get( + f"{GenAIAttributes.GEN_AI_PROMPT}.0.content" + ) + assert prompt_content is not None + assert "alice.jones@example.org" in prompt_content + assert "[PII.email]" not in prompt_content + + # But findings should STILL be recorded as span events + safety_events = [ + e for e in span.events + if "fortifyroot.safety" in (e.name or "") + ] + assert len(safety_events) >= 1, "ALLOW should still emit safety findings" + event_attrs = safety_events[0].attributes + assert event_attrs["fortifyroot.safety.rule_name"] == "PII.email" + assert event_attrs["fortifyroot.safety.action"] == "ALLOW" + assert event_attrs["fortifyroot.safety.location"] == "PROMPT" + + +@pytest.mark.vcr +@pytest.mark.fr +def test_openai_safety_allow_credit_card_not_masked( + _set_allow_action, instrument_legacy, span_exporter, openai_client +): + """ALLOW action: credit card detected but text passes through unchanged.""" + openai_client.chat.completions.create( + model="gpt-4o-mini", + messages=[ + { + "role": "user", + "content": ( + "Verify payment for card 4111 1111 1111 1111 was received." + ), + } + ], + max_tokens=50, + ) + + spans = span_exporter.get_finished_spans() + assert len(spans) >= 1 + span = spans[0] + + # With ALLOW, text unchanged — card number still visible + prompt_content = span.attributes.get( + f"{GenAIAttributes.GEN_AI_PROMPT}.0.content" + ) + assert prompt_content is not None + assert "4111 1111 1111 1111" in prompt_content + + # Findings still recorded + safety_events = [ + e for e in span.events + if "fortifyroot.safety" in (e.name or "") + ] + found_cc = any( + e.attributes.get("fortifyroot.safety.rule_name") == "PCI.credit_card" + and e.attributes.get("fortifyroot.safety.action") == "ALLOW" + for e in safety_events + ) + assert found_cc, "Expected PCI.credit_card finding with ALLOW action"