ref(integrations): route HTTP header filtering through data collection config

sentry_sdk/data_collection.py

@@ -23,7 +23,9 @@
 """
 
 import warnings
-from typing import TYPE_CHECKING, cast
+from typing import TYPE_CHECKING, List, Mapping, Optional, cast
+
+from sentry_sdk._types import SENSITIVE_DATA_SUBSTITUTE
 
 if TYPE_CHECKING:
     from typing import Any, Dict, Literal
@@ -77,6 +79,62 @@
 ]
 
 
+def _is_sensitive_key(key: str, extra_terms: "Optional[List[str]]" = None) -> bool:
+    """
+    Return whether ``key`` matches the sensitive denylist using a partial,
+    case-insensitive substring match.
+
+    :param extra_terms: additional deny terms (e.g. user-provided) to consider
+        alongside the built-in `_SENSITIVE_DENYLIST`.
+    """
+    lowered = key.lower()
+    for term in _SENSITIVE_DENYLIST:
+        if term in lowered:
+            return True
+    if extra_terms:
+        for term in extra_terms:
+            if term and term.lower() in lowered:
+                return True
+    return False
+
+
+def _apply_key_value_collection_filtering(
+    items: "Mapping[str, Any]",
+    behaviour: "KeyValueCollectionBehaviour",
+    substitute: "Any" = SENSITIVE_DATA_SUBSTITUTE,
+) -> "Dict[str, Any]":
+
+    if behaviour["mode"] == "off":
+        return {}
+
+    result: "Dict[str, Any]" = {}
+
+    if behaviour["mode"] == "allowlist":
+        for key, value in items.items():
+            is_allowed = False
+            if isinstance(key, str):
+                lowered = key.lower()
+                is_allowed = any(
+                    term and term.lower() in lowered
+                    for term in behaviour.get("terms", [])
+                )
+            if is_allowed and not _is_sensitive_key(key):
+                result[key] = value
+            else:
+                result[key] = substitute
+        return result
+
+    # denylist behaviour
+    for key, value in items.items():
+        if isinstance(key, str) and _is_sensitive_key(
+            key=key, extra_terms=behaviour.get("terms", [])
+        ):
+            result[key] = substitute
+        else:
+            result[key] = value
+    return result
+
+
 def _map_from_send_default_pii(
     *,
     send_default_pii: bool,

sentry_sdk/integrations/_asgi_common.py

@@ -148,7 +148,7 @@ def _get_request_attributes(
         if asgi_scope.get("method"):
             attributes["http.request.method"] = asgi_scope["method"].upper()
 
-        headers = _filter_headers(_get_headers(asgi_scope), use_annotated_value=False)
+        headers = _filter_headers(_get_headers(asgi_scope))
         for header, value in headers.items():
             attributes[f"http.request.header.{header.lower()}"] = value
 

sentry_sdk/integrations/_wsgi_common.py

@@ -4,6 +4,7 @@
 
 import sentry_sdk
 from sentry_sdk._types import SENSITIVE_DATA_SUBSTITUTE
+from sentry_sdk.data_collection import _apply_key_value_collection_filtering
 from sentry_sdk.scope import should_send_default_pii
 from sentry_sdk.utils import AnnotatedValue, logger
 
@@ -23,22 +24,6 @@
     from sentry_sdk._types import Event, HttpStatusCodeRange
 
 
-SENSITIVE_ENV_KEYS = (
-    "REMOTE_ADDR",
-    "HTTP_X_FORWARDED_FOR",
-    "HTTP_SET_COOKIE",
-    "HTTP_COOKIE",
-    "HTTP_AUTHORIZATION",
-    "HTTP_PROXY_AUTHORIZATION",
-    "HTTP_X_API_KEY",
-    "HTTP_X_FORWARDED_FOR",
-    "HTTP_X_REAL_IP",
-)
-
-SENSITIVE_HEADERS = tuple(
-    x[len("HTTP_") :] for x in SENSITIVE_ENV_KEYS if x.startswith("HTTP_")
-)
-
 DEFAULT_HTTP_METHODS_TO_CAPTURE = (
     "CONNECT",
     "DELETE",
@@ -211,21 +196,19 @@ def _is_json_content_type(ct: "Optional[str]") -> bool:
 
 def _filter_headers(
     headers: "Mapping[str, str]",
-    use_annotated_value: bool = True,
-) -> "Mapping[str, Union[AnnotatedValue, str]]":
-    if should_send_default_pii():
-        return headers
-
-    substitute: "Union[AnnotatedValue, str]" = (
-        SENSITIVE_DATA_SUBSTITUTE
-        if not use_annotated_value
-        else AnnotatedValue.removed_because_over_size_limit()
+) -> "Mapping[str, str]":
+    data_collection_configuration = sentry_sdk.get_client().options["data_collection"]
+
+    filtered = _apply_key_value_collection_filtering(
+        items=headers,
+        behaviour=data_collection_configuration["http_headers"]["request"],
     )
 
-    return {
-        k: (v if k.upper().replace("-", "_") not in SENSITIVE_HEADERS else substitute)
-        for k, v in headers.items()
-    }
+    for key in filtered:
+        if isinstance(key, str) and key.lower() in ("cookie", "set-cookie"):
+            filtered[key] = SENSITIVE_DATA_SUBSTITUTE
+
+    return filtered
 
 
 def _in_http_status_code_range(

sentry_sdk/integrations/aiohttp.py

@@ -150,7 +150,7 @@ async def sentry_app_handle(
 
                         header_attributes: "dict[str, Any]" = {}
                         for header, header_value in _filter_headers(
-                            headers, use_annotated_value=False
+                            headers
                         ).items():
                             header_attributes[
                                 f"http.request.header.{header.lower()}"

sentry_sdk/integrations/aws_lambda.py

@@ -152,7 +152,7 @@ def sentry_handler(
 
             header_attributes: "dict[str, Any]" = {}
             for header, header_value in _filter_headers(
-                headers, use_annotated_value=False
+                headers
             ).items():
                 header_attributes[f"http.request.header.{header.lower()}"] = (
                     header_value

sentry_sdk/integrations/chalice.py

@@ -92,7 +92,7 @@ def wrapped_view_function(**function_args: "Any") -> "Any":
 
                     header_attrs: "Dict[str, Any]" = {}
                     for header, value in _filter_headers(
-                        headers, use_annotated_value=False
+                        headers
                     ).items():
                         header_attrs[f"http.request.header.{header.lower()}"] = value
 

sentry_sdk/integrations/gcp.py

@@ -89,7 +89,7 @@ def sentry_func(
             if hasattr(gcp_event, "headers"):
                 headers = gcp_event.headers
                 for header, header_value in _filter_headers(
-                    headers, use_annotated_value=False
+                    headers
                 ).items():
                     header_attributes[f"http.request.header.{header.lower()}"] = (
                         # header_value will always be a string because we set `use_annotated_value` to false above

sentry_sdk/integrations/quart.py

@@ -195,7 +195,7 @@ async def _request_websocket_started(app: "Quart", **kwargs: "Any") -> None:
             header_attributes: "dict[str, Any]" = {}
 
             for header, header_value in _filter_headers(
-                dict(request_websocket.headers), use_annotated_value=False
+                dict(request_websocket.headers)
             ).items():
                 header_attributes[f"http.request.header.{header.lower()}"] = (
                     header_value

sentry_sdk/integrations/sanic.py

@@ -374,7 +374,7 @@ def _get_request_attributes(request: "Request") -> "Dict[str, Any]":
     if request.method:
         attributes[SPANDATA.HTTP_REQUEST_METHOD] = request.method.upper()
 
-    headers = _filter_headers(dict(request.headers), use_annotated_value=False)
+    headers = _filter_headers(dict(request.headers))
     for header, value in headers.items():
         attributes[f"{SPANDATA.HTTP_REQUEST_HEADER}.{header.lower()}"] = value
 

sentry_sdk/integrations/tornado.py

@@ -193,7 +193,7 @@ def _get_request_attributes(request: "Any") -> "Dict[str, Any]":
     if request.method:
         attributes[SPANDATA.HTTP_REQUEST_METHOD] = request.method.upper()
 
-    headers = _filter_headers(dict(request.headers), use_annotated_value=False)
+    headers = _filter_headers(dict(request.headers))
     for header, value in headers.items():
         attributes[f"{SPANDATA.HTTP_REQUEST_HEADER}.{header.lower()}"] = value
 

sentry_sdk/integrations/wsgi.py

@@ -390,7 +390,7 @@ def _get_request_attributes(
     if method:
         attributes["http.request.method"] = method.upper()
 
-    headers = _filter_headers(dict(_get_headers(environ)), use_annotated_value=False)
+    headers = _filter_headers(dict(_get_headers(environ)))
     for header, value in headers.items():
         attributes[f"http.request.header.{header.lower()}"] = value