feat: Remove string truncation for events

Author: alexander-alderman-webb

sentry_sdk/consts.py

@@ -2,11 +2,7 @@
 from enum import Enum
 from typing import TYPE_CHECKING
 
-# up top to prevent circular import due to integration import
-# This is more or less an arbitrary large-ish value for now, so that we allow
-# pretty long strings (like LLM prompts), but still have *some* upper limit
-# until we verify that removing the trimming completely is safe.
-DEFAULT_MAX_VALUE_LENGTH = 100_000
+DEFAULT_MAX_VALUE_LENGTH = None
 
 DEFAULT_MAX_STACK_FRAMES = 100
 DEFAULT_ADD_FULL_STACK = False
@@ -1233,7 +1229,7 @@ def __init__(
         ],
         functions_to_trace: "Sequence[Dict[str, str]]" = [],  # noqa: B006
         event_scrubber: "Optional[sentry_sdk.scrubber.EventScrubber]" = None,
-        max_value_length: int = DEFAULT_MAX_VALUE_LENGTH,
+        max_value_length: "Optional[int]" = DEFAULT_MAX_VALUE_LENGTH,
         enable_backpressure_handling: bool = True,
         error_sampler: "Optional[Callable[[Event, Hint], Union[float, bool]]]" = None,
         enable_db_query_source: bool = True,

sentry_sdk/serializer.py

@@ -103,7 +103,7 @@ def serialize(event: "Dict[str, Any]", **kwargs: "Any") -> "Dict[str, Any]":
     * Annotating the payload with the _meta field whenever trimming happens.
 
     :param max_request_body_size: If set to "always", will never trim request bodies.
-    :param max_value_length: The max length to strip strings to, defaults to sentry_sdk.consts.DEFAULT_MAX_VALUE_LENGTH
+    :param max_value_length: The max length to strip strings to, or None to disable string truncation. Defaults to None.
     :param is_vars: If we're serializing vars early, we want to repr() things that are JSON-serializable to make their type more apparent. For example, it's useful to see the difference between a unicode-string and a bytestring when viewing a stacktrace.
     :param custom_repr: A custom repr function that runs before safe_repr on the object to be serialized. If it returns None or throws internally, we will fallback to safe_repr.
 

sentry_sdk/utils.py

@@ -40,7 +40,6 @@
 from sentry_sdk.consts import (
     DEFAULT_ADD_FULL_STACK,
     DEFAULT_MAX_STACK_FRAMES,
-    DEFAULT_MAX_VALUE_LENGTH,
     EndpointType,
 )
 
@@ -754,7 +753,7 @@ def single_exception_from_error_tuple(
     if client_options is None:
         include_local_variables = True
         include_source_context = True
-        max_value_length = DEFAULT_MAX_VALUE_LENGTH  # fallback
+        max_value_length = None  # fallback
         custom_repr = None
     else:
         include_local_variables = client_options["include_local_variables"]
@@ -1268,12 +1267,9 @@ def _get_size_in_bytes(value: str) -> "Optional[int]":
 def strip_string(
     value: str, max_length: "Optional[int]" = None
 ) -> "Union[AnnotatedValue, str]":
-    if not value:
+    if not value or max_length is None:
         return value
 
-    if max_length is None:
-        max_length = DEFAULT_MAX_VALUE_LENGTH
-
     byte_size = _get_size_in_bytes(value)
     text_size = len(value)
 

tests/integrations/bottle/test_bottle.py

@@ -9,7 +9,6 @@
 from werkzeug.wrappers import Response
 
 from sentry_sdk import capture_message
-from sentry_sdk.consts import DEFAULT_MAX_VALUE_LENGTH
 from sentry_sdk.integrations.bottle import BottleIntegration
 from sentry_sdk.integrations.logging import LoggingIntegration
 from sentry_sdk.serializer import MAX_DATABAG_BREADTH
@@ -122,10 +121,17 @@ def index():
     assert event["exception"]["values"][0]["mechanism"]["handled"] is False
 
 
-def test_large_json_request(sentry_init, capture_events, app, get_client):
-    sentry_init(integrations=[BottleIntegration()], max_request_body_size="always")
+@pytest.mark.parametrize("max_value_length", [1024, None])
+def test_large_json_request(
+    sentry_init, capture_events, app, get_client, max_value_length
+):
+    sentry_init(
+        integrations=[BottleIntegration()],
+        max_request_body_size="always",
+        max_value_length=max_value_length,
+    )
 
-    data = {"foo": {"bar": "a" * (DEFAULT_MAX_VALUE_LENGTH + 10)}}
+    data = {"foo": {"bar": "a" * (1034)}}
 
     @app.route("/", method="POST")
     def index():
@@ -145,15 +151,17 @@ def index():
     assert response[1] == "200 OK"
 
     (event,) = events
-    assert event["_meta"]["request"]["data"]["foo"]["bar"] == {
-        "": {
-            "len": DEFAULT_MAX_VALUE_LENGTH + 10,
-            "rem": [
-                ["!limit", "x", DEFAULT_MAX_VALUE_LENGTH - 3, DEFAULT_MAX_VALUE_LENGTH]
-            ],
+
+    if max_value_length:
+        assert event["_meta"]["request"]["data"]["foo"]["bar"] == {
+            "": {
+                "len": 1034,
+                "rem": [["!limit", "x", 1021, 1024]],
+            }
         }
-    }
-    assert len(event["request"]["data"]["foo"]["bar"]) == DEFAULT_MAX_VALUE_LENGTH
+        assert len(event["request"]["data"]["foo"]["bar"]) == 1024
+    else:
+        assert len(event["request"]["data"]["foo"]["bar"]) == 1034
 
 
 @pytest.mark.parametrize("data", [{}, []], ids=["empty-dict", "empty-list"])
@@ -180,10 +188,17 @@ def index():
     assert event["request"]["data"] == data
 
 
-def test_medium_formdata_request(sentry_init, capture_events, app, get_client):
-    sentry_init(integrations=[BottleIntegration()], max_request_body_size="always")
+@pytest.mark.parametrize("max_value_length", [1024, None])
+def test_medium_formdata_request(
+    sentry_init, capture_events, app, get_client, max_value_length
+):
+    sentry_init(
+        integrations=[BottleIntegration()],
+        max_request_body_size="always",
+        max_value_length=max_value_length,
+    )
 
-    data = {"foo": "a" * (DEFAULT_MAX_VALUE_LENGTH + 10)}
+    data = {"foo": "a" * (1034)}
 
     @app.route("/", method="POST")
     def index():
@@ -200,15 +215,17 @@ def index():
     assert response[1] == "200 OK"
 
     (event,) = events
-    assert event["_meta"]["request"]["data"]["foo"] == {
-        "": {
-            "len": DEFAULT_MAX_VALUE_LENGTH + 10,
-            "rem": [
-                ["!limit", "x", DEFAULT_MAX_VALUE_LENGTH - 3, DEFAULT_MAX_VALUE_LENGTH]
-            ],
+
+    if max_value_length:
+        assert event["_meta"]["request"]["data"]["foo"] == {
+            "": {
+                "len": 1034,
+                "rem": [["!limit", "x", 1021, 1024]],
+            }
         }
-    }
-    assert len(event["request"]["data"]["foo"]) == DEFAULT_MAX_VALUE_LENGTH
+        assert len(event["request"]["data"]["foo"]) == 1024
+    else:
+        assert len(event["request"]["data"]["foo"]) == 1034
 
 
 @pytest.mark.parametrize("input_char", ["a", b"a"])
@@ -242,11 +259,16 @@ def index():
     assert not event["request"]["data"]
 
 
-def test_files_and_form(sentry_init, capture_events, app, get_client):
-    sentry_init(integrations=[BottleIntegration()], max_request_body_size="always")
+@pytest.mark.parametrize("max_value_length", [1024, None])
+def test_files_and_form(sentry_init, capture_events, app, get_client, max_value_length):
+    sentry_init(
+        integrations=[BottleIntegration()],
+        max_request_body_size="always",
+        max_value_length=max_value_length,
+    )
 
     data = {
-        "foo": "a" * (DEFAULT_MAX_VALUE_LENGTH + 10),
+        "foo": "a" * (1034),
         "file": (BytesIO(b"hello"), "hello.txt"),
     }
 
@@ -267,15 +289,16 @@ def index():
     assert response[1] == "200 OK"
 
     (event,) = events
-    assert event["_meta"]["request"]["data"]["foo"] == {
-        "": {
-            "len": DEFAULT_MAX_VALUE_LENGTH + 10,
-            "rem": [
-                ["!limit", "x", DEFAULT_MAX_VALUE_LENGTH - 3, DEFAULT_MAX_VALUE_LENGTH]
-            ],
+    if max_value_length:
+        assert event["_meta"]["request"]["data"]["foo"] == {
+            "": {
+                "len": 1034,
+                "rem": [["!limit", "x", 1021, 1024]],
+            }
         }
-    }
-    assert len(event["request"]["data"]["foo"]) == DEFAULT_MAX_VALUE_LENGTH
+        assert len(event["request"]["data"]["foo"]) == 1024
+    else:
+        assert len(event["request"]["data"]["foo"]) == 1034
 
     assert event["_meta"]["request"]["data"]["file"] == {
         "": {

tests/integrations/falcon/test_falcon.py

@@ -5,7 +5,6 @@
 import pytest
 
 import sentry_sdk
-from sentry_sdk.consts import DEFAULT_MAX_VALUE_LENGTH
 from sentry_sdk.integrations.falcon import FalconIntegration
 from sentry_sdk.integrations.logging import LoggingIntegration
 from sentry_sdk.utils import parse_version
@@ -206,10 +205,15 @@ def on_get(self, req, resp):
     assert len(events) == 0
 
 
-def test_falcon_large_json_request(sentry_init, capture_events):
-    sentry_init(integrations=[FalconIntegration()], max_request_body_size="always")
+@pytest.mark.parametrize("max_value_length", [1024, None])
+def test_falcon_large_json_request(sentry_init, capture_events, max_value_length):
+    sentry_init(
+        integrations=[FalconIntegration()],
+        max_request_body_size="always",
+        max_value_length=max_value_length,
+    )
 
-    data = {"foo": {"bar": "a" * (DEFAULT_MAX_VALUE_LENGTH + 10)}}
+    data = {"foo": {"bar": "a" * (1034)}}
 
     class Resource:
         def on_post(self, req, resp):
@@ -227,15 +231,16 @@ def on_post(self, req, resp):
     assert response.status == falcon.HTTP_200
 
     (event,) = events
-    assert event["_meta"]["request"]["data"]["foo"]["bar"] == {
-        "": {
-            "len": DEFAULT_MAX_VALUE_LENGTH + 10,
-            "rem": [
-                ["!limit", "x", DEFAULT_MAX_VALUE_LENGTH - 3, DEFAULT_MAX_VALUE_LENGTH]
-            ],
+    if max_value_length:
+        assert event["_meta"]["request"]["data"]["foo"]["bar"] == {
+            "": {
+                "len": 1034,
+                "rem": [["!limit", "x", 1021, 1024]],
+            }
         }
-    }
-    assert len(event["request"]["data"]["foo"]["bar"]) == DEFAULT_MAX_VALUE_LENGTH
+        assert len(event["request"]["data"]["foo"]["bar"]) == 1024
+    else:
+        assert len(event["request"]["data"]["foo"]["bar"]) == 1034
 
 
 @pytest.mark.parametrize("data", [{}, []], ids=["empty-dict", "empty-list"])