diff --git a/autoresearch.ideas.md b/autoresearch.ideas.md new file mode 100644 index 00000000..977fb122 --- /dev/null +++ b/autoresearch.ideas.md @@ -0,0 +1 @@ +- Extract a shared recursive container sanitizer for `_to_bt_safe` model outputs (especially non-Pydantic `model_dump`/`dict` adapters) so nested `dict`/`list` fields can avoid `bt_dumps()`/`bt_loads()` roundtrips without regressing the already-optimized `bt_safe_deep_copy()` hot path. diff --git a/autoresearch.jsonl b/autoresearch.jsonl new file mode 100644 index 00000000..1cdabf1a --- /dev/null +++ b/autoresearch.jsonl @@ -0,0 +1,80 @@ +{"type":"config","name":"Optimize bt_json benchmark aggregate in Braintrust SDK","metricName":"bt_json_total_us","metricUnit":"µs","bestDirection":"lower"} +{"run":1,"commit":"716f9e4","metric":0,"metrics":{},"status":"crash","description":"Baseline bt_json benchmark command failed; need to debug benchmark invocation/path handling before starting optimization loop.","timestamp":1774398454476,"segment":0,"confidence":null} +{"run":2,"commit":"716f9e4","metric":0,"metrics":{},"status":"crash","description":"Second baseline attempt also failed; mktemp pre-created the output path and pyperf requires a non-existent JSON destination.","timestamp":1774398471101,"segment":0,"confidence":null} +{"run":3,"commit":"da9dd28","metric":6806.403259418478,"metrics":{},"status":"keep","description":"Established the first successful bt_json benchmark baseline using the existing benchmark suite aggregate across all 11 cases.","timestamp":1774398754782,"segment":0,"confidence":null} +{"run":4,"commit":"124ddb1","metric":1459.11131989952,"metrics":{},"status":"keep","description":"Move primitive scalar and float fast paths to the top of _to_bt_safe so deep-copy traversal avoids logger imports and pydantic/dataclass probing for the overwhelmingly common leaf values.","timestamp":1774398923198,"segment":0,"confidence":null} +{"run":5,"commit":"79a8239","metric":1052.0784615771965,"metrics":{},"status":"keep","description":"Inline primitive scalar handling inside bt_safe_deep_copy and iterate mappings with .items() so recursive traversal avoids calling _to_bt_safe for common leaf values and avoids extra dict lookups.","timestamp":1774399092247,"segment":0,"confidence":2.5847506904321493} +{"run":6,"commit":"8408611","metric":808.4420663400282,"metrics":{},"status":"keep","description":"Use collections.abc.Mapping instead of typing.Mapping in bt_json hot paths; typing.Mapping incurs expensive runtime __instancecheck__ logic during every recursive container visit.","timestamp":1774399243162,"segment":0,"confidence":2.4849554882682376} +{"run":7,"commit":"ea5eea6","metric":597.1877789749944,"metrics":{},"status":"keep","description":"Split bt_safe_deep_copy container handling into dict/list/tuple/set fast paths and add a string-key fast path, avoiding expensive ABC checks and unnecessary str() calls on the common recursive cases.","timestamp":1774399396913,"segment":0,"confidence":1.4671733909553075} +{"run":8,"commit":"e77be99","metric":583.086666156607,"metrics":{},"status":"keep","description":"Add exact-type fast paths for the common built-in scalar and container cases in _to_bt_safe and bt_safe_deep_copy, falling back to isinstance only for subclasses and uncommon container implementations.","timestamp":1774399717444,"segment":0,"confidence":1.714340386258025} +{"run":9,"commit":"e77be99","metric":583.0733557876912,"metrics":{},"status":"discard","description":"Confirmation rerun of the exact-type fast paths produced essentially the same aggregate result; no additional code change beyond the already-kept optimization.","timestamp":1774399719558,"segment":0,"confidence":2.587256521668733} +{"run":10,"commit":"e77be99","metric":601.3881805183818,"metrics":{},"status":"discard","description":"Tried separating exact-type and subclass fallback container checks more aggressively, but the aggregate benchmark regressed slightly versus the previous fast-path structure.","timestamp":1774399906968,"segment":0,"confidence":4.785867180630146} +{"run":11,"commit":"e77be99","metric":612.3751781224904,"metrics":{},"status":"discard","description":"Tried routing dataclass field sanitization through bt_safe_deep_copy to avoid container roundtrips inside _to_bt_safe. It improved the standalone dataclass case but regressed the aggregate benchmark overall, so not keeping it.","timestamp":1774400090293,"segment":0,"confidence":19.89933115743882} +{"run":12,"commit":"63cdf7a","metric":576.2479212949469,"metrics":{},"status":"keep","description":"Replace dataclasses.is_dataclass/dataclasses.fields and exception-driven pydantic probing with direct attribute checks (__dataclass_fields__, model_dump, dict), cutting overhead for dataclass objects and especially pydantic-v1-like models.","timestamp":1774400426606,"segment":0,"confidence":21.16920972900574} +{"run":13,"commit":"63cdf7a","metric":581.3545543372375,"metrics":{},"status":"discard","description":"Confirmation rerun of the attribute-check optimization stayed in the same range as the first result but did not clearly beat it further; keeping the earlier commit and discarding this no-code rerun.","timestamp":1774400428634,"segment":0,"confidence":28.76403483246146} +{"run":14,"commit":"63cdf7a","metric":578.8566004577588,"metrics":{},"status":"discard","description":"Tried an exact-type fast path for string dict keys to shave a few more isinstance calls in recursive copies, but the aggregate result was slightly worse than the current best.","timestamp":1774400595952,"segment":0,"confidence":30.04044596735723} +{"run":15,"commit":"c65c6e6","metric":603.2877908944215,"metrics":{},"status":"discard","description":"Post-rebase baseline rerun on current main branch to refresh noise and confirm the bt_json aggregate still sits around the previously improved range (~0.60 ms total).","timestamp":1774400834227,"segment":0,"confidence":28.76403483246146} +{"run":16,"commit":"fc1271c","metric":489.8326024747584,"metrics":{},"status":"keep","description":"Reordered bt_safe_deep_copy to check exact built-in container types before subclass scalar fallbacks, so common dict/list/tuple/set recursion avoids extra isinstance probes on every container node.","timestamp":1774401042128,"segment":0,"confidence":25.53551914708137} +{"run":17,"commit":"90dbe0d","metric":482.174736600974,"metrics":{},"status":"keep","description":"Cache the lazily imported braintrust.logger special-case types so _to_bt_safe avoids repeating the in-function import path for non-primitive values while preserving the circular-import-safe initialization behavior.","timestamp":1774401371510,"segment":0,"confidence":26.303531774981874} +{"run":18,"commit":"90dbe0d","metric":475.3576222431998,"metrics":{},"status":"discard","description":"Confirmation rerun of the cached special-type import path came in slightly better but reflects the same kept code change; discarding the no-code rerun after validating the improvement is stable.","timestamp":1774401374557,"segment":0,"confidence":26.693127513260983} +{"run":19,"commit":"90dbe0d","metric":482.40646235414977,"metrics":{},"status":"discard","description":"Tried prebinding visited.add/discard in bt_safe_deep_copy to reduce method lookup overhead, but it did not beat the current best aggregate benchmark.","timestamp":1774401580974,"segment":0,"confidence":16.46293048833045} +{"run":20,"commit":"8de12d7","metric":464.24171432099206,"metrics":{},"status":"keep","description":"Hoist depth+1 out of the inner dict/list/tuple/set recursion loops in bt_safe_deep_copy so the hot container traversal avoids repeated integer addition on every element/key visit.","timestamp":1774402493998,"segment":0,"confidence":7.576823187649135} +{"run":21,"commit":"0f895af","metric":458.9441213200044,"metrics":{},"status":"keep","description":"Look up dataclass and pydantic methods on the class instead of the instance in _to_bt_safe, avoiding bound-method creation and redundant instance attribute lookups on dataclass/model inputs.","timestamp":1774402734658,"segment":0,"confidence":4.922140855937802} +{"run":22,"commit":"84f6e0d","metric":445.26971205827834,"metrics":{},"status":"keep","description":"Use exact-type checks for dict keys in bt_safe_deep_copy so the common all-built-in-string-key path avoids isinstance overhead while still coercing string subclasses and non-string keys safely.","timestamp":1774402932765,"segment":0,"confidence":4.6336686332908466} +{"run":23,"commit":"84f6e0d","metric":486.5291488976704,"metrics":{},"status":"discard","description":"Tried prebinding id/math/set-method/_to_bt_safe lookups into bt_safe_deep_copy locals to reduce global and attribute lookups, but the aggregate bt_json benchmark regressed versus the current best fast-path structure.","timestamp":1774403157676,"segment":0,"confidence":4.695679496377708} +{"run":24,"commit":"84f6e0d","metric":455.39023340549636,"metrics":{},"status":"discard","description":"Tried skipping the model_json_schema getattr on non-type values in _to_bt_safe so normal instances avoid an unused attribute lookup, but the aggregate benchmark still regressed versus the current best.","timestamp":1774403328000,"segment":0,"confidence":4.6557934570215345} +{"run":25,"commit":"84f6e0d","metric":452.2752495124428,"metrics":{},"status":"discard","description":"Tried inlining exact scalar child handling inside the exact dict/list recursion loops to avoid recursive calls for common leaves, but the extra branch work regressed the aggregate benchmark relative to the current best implementation.","timestamp":1774403507629,"segment":0,"confidence":4.616579310440792} +{"run":26,"commit":"84f6e0d","metric":456.32574799191985,"metrics":{},"status":"discard","description":"Tried reading dataclass field values from the instance __dict__ before falling back to getattr/Field iteration so common non-slotted dataclasses avoid repeated attribute lookups, but the aggregate bt_json benchmark regressed overall.","timestamp":1774403682269,"segment":0,"confidence":4.674175376747412} +{"run":27,"commit":"4267cd1","metric":443.3354319026354,"metrics":{},"status":"keep","description":"Only enter the warning-suppression context for model_dump on actual Pydantic v2 models (detected via __pydantic_serializer__), so model_dump-compatible non-Pydantic objects avoid the catch_warnings/filterwarnings overhead.","timestamp":1774403848390,"segment":0,"confidence":4.712665286551514} +{"run":28,"commit":"4267cd1","metric":445.665584844845,"metrics":{},"status":"discard","description":"Tried replacing math.isnan/math.isinf with direct NaN/infinity comparisons using cached +/-inf constants to reduce float-special-case overhead, but the aggregate benchmark came in slightly worse than the current best.","timestamp":1774404066080,"segment":0,"confidence":6.376813252526581} +{"run":29,"commit":"4267cd1","metric":454.39122985763623,"metrics":{},"status":"discard","description":"Tried dropping callable() checks for model_dump/dict and relying on the existing TypeError fallback to shave method-probing overhead in _to_bt_safe, but the aggregate benchmark regressed.","timestamp":1774404246776,"segment":0,"confidence":9.534675474821974} +{"run":30,"commit":"bfaaa44","metric":432.30320549698376,"metrics":{},"status":"keep","description":"Add an exact-dict scalar-prefix fast path in bt_safe_deep_copy: scalar-only dicts now finish without recursive calls, and mixed dicts recurse only once they hit the first non-scalar value while preserving key coercion and depth handling.","timestamp":1774404454721,"segment":0,"confidence":8.583992063900737} +{"run":31,"commit":"d91864c","metric":430.73053226363567,"metrics":{},"status":"keep","description":"Reorder the hottest exact-type scalar checks around the observed bt_json payload mix so str/int hits happen before the rarely-taken None case in _to_bt_safe, bt_safe_deep_copy, and the new dict scalar-prefix fast path.","timestamp":1774404881626,"segment":0,"confidence":7.943255667879845} +{"run":32,"commit":"b920671","metric":416.8542630214215,"metrics":{},"status":"keep","description":"Make the exact-dict scalar-prefix fast path optimistic for exact-string keys: the common all-string-key case now writes keys directly and only falls back to stringification once a non-string key is actually encountered.","timestamp":1774405315868,"segment":0,"confidence":7.872462483981603} +{"run":33,"commit":"82ef812","metric":409.41510602223036,"metrics":{},"status":"keep","description":"Delay exact-dict visited-set insertion until the first child that actually needs recursion, so scalar-only dicts skip add/discard churn entirely while mixed dicts still detect cycles before descending.","timestamp":1774405522848,"segment":0,"confidence":7.922742857363388} +{"run":34,"commit":"82ef812","metric":412.76050610079136,"metrics":{},"status":"discard","description":"Tried delaying exact-dict id()/visited-membership checks until the first recursive child instead of doing the membership probe up front, but the aggregate benchmark regressed slightly versus the current best despite the promising microbenchmark.","timestamp":1774405763016,"segment":0,"confidence":6.9987639479379835} +{"run":35,"commit":"45f1df5","metric":400.99191405921846,"metrics":{},"status":"keep","description":"Route common non-slotted dataclass instances through bt_safe_deep_copy on their __dict__ when it exactly matches the dataclass field set, letting the heavily optimized dict fast path sanitize fields while preserving the existing fallback for slotted or dynamically-extended dataclasses.","timestamp":1774405964974,"segment":0,"confidence":6.138839663660102} +{"run":36,"commit":"c69d1ee","metric":395.8065035184231,"metrics":{},"status":"keep","description":"Use math.isfinite as the hot finite-float fast path in _to_bt_safe, bt_safe_deep_copy, and the exact-dict scalar-prefix loop, only falling back to NaN/+/-Infinity handling for the uncommon non-finite cases.","timestamp":1774406180059,"segment":0,"confidence":6.1884609361189495} +{"run":37,"commit":"f7ff50a","metric":393.92183346204956,"metrics":{},"status":"keep","description":"Move dataclass/model-dump/dict probing ahead of the Braintrust special-object isinstance chain while keeping model_json_schema safely type-gated, so dataclasses and model-like objects avoid the cached special-type tuple fetch plus six unused isinstance checks.","timestamp":1774406424693,"segment":0,"confidence":6.292970950384959} +{"run":38,"commit":"f7ff50a","metric":400.74319437738706,"metrics":{},"status":"discard","description":"Tried extending the exact-dict scalar-prefix fast path so non-string-key entries also handle scalar and finite-float values inline before falling back to recursion, but the aggregate benchmark regressed versus the current best.","timestamp":1774406714527,"segment":0,"confidence":6.709407921384854} +{"run":39,"commit":"f7ff50a","metric":395.7300176388153,"metrics":{},"status":"discard","description":"Tried reading dataclass __dict__ via direct attribute access with an AttributeError fallback instead of getattr(..., None), but the aggregate bt_json benchmark stayed slightly worse than the current best.","timestamp":1774406888722,"segment":0,"confidence":7.18486599272176} +{"run":40,"commit":"f7ff50a","metric":401.75335352757253,"metrics":{},"status":"discard","description":"Tried prebinding only id/visited.add/visited.discard into bt_safe_deep_copy locals for the hot exact built-in container branches, but the aggregate benchmark regressed despite a small standalone microbenchmark win.","timestamp":1774407127817,"segment":0,"confidence":7.033227293975485} +{"run":41,"commit":"f7ff50a","metric":404.93684831821935,"metrics":{},"status":"discard","description":"Tried extracting the exact-dict copier into a dedicated helper and dispatching exact-list dict elements straight to that helper to avoid some full _deep_copy_object type-dispatch work, but the extra helper-call structure regressed the aggregate benchmark.","timestamp":1774407362462,"segment":0,"confidence":7.2938663918289235} +{"run":42,"commit":"f7ff50a","metric":394.2353693876712,"metrics":{},"status":"discard","description":"Confirmation rerun of the current best bt_json fast-path set stayed in the same ~394µs range, reinforcing that the latest exact-dict/dataclass/model-path improvements are stable but without adding any new code change.","timestamp":1774407585552,"segment":0,"confidence":7.255904215859193} +{"run":43,"commit":"4385aeb","metric":370.0572772262796,"metrics":{},"status":"keep","description":"Special-case exact dict elements inside the exact list fast path so the common list-of-dicts workload avoids an extra _deep_copy_object dispatch per element while preserving circular detection, key coercion, and depth handling.","timestamp":1774407859241,"segment":0,"confidence":6.781034273070822} +{"run":44,"commit":"7c45cc6","metric":366.36490207095034,"metrics":{},"status":"keep","description":"Delay exact-dict visited-set membership/insertion until the first child that actually needs recursive descent, including the new list-of-dicts fast path, so scalar-prefix dicts skip upfront id/in-set work while still catching cycles before recursing.","timestamp":1774408217375,"segment":0,"confidence":6.724098416160464} +{"run":45,"commit":"7c45cc6","metric":365.5854441248045,"metrics":{},"status":"discard","description":"Confirmation rerun of the delayed exact-dict visited-set check stayed in the same ~366µs range, validating the new best path without adding any further code change.","timestamp":1774408364097,"segment":0,"confidence":6.734857004291221} +{"run":46,"commit":"450d2be","metric":364.93806625247976,"metrics":{},"status":"keep","description":"Flip the exact-list branch to make exact dict elements the direct hot path and send the rare non-dict elements to the fallback branch, matching the benchmark's overwhelmingly list-of-dicts container mix.","timestamp":1774408698459,"segment":0,"confidence":6.7552129239048035} +{"run":47,"commit":"450d2be","metric":365.04957787212527,"metrics":{},"status":"discard","description":"Tried hoisting the exact-list next_depth>=max_depth handling out of the per-dict-element loop so the common non-max-depth case avoids that repeated branch, but the aggregate bt_json benchmark was effectively flat/slightly worse than the current best.","timestamp":1774408912141,"segment":0,"confidence":6.8024497632692205} +{"run":48,"commit":"450d2be","metric":368.83125307103586,"metrics":{},"status":"discard","description":"Tried restructuring the exact-dict and list-of-dicts key handling to make exact string keys the direct branch and move non-string-key coercion into the rarer fallback branch, but the aggregate benchmark regressed.","timestamp":1774409093818,"segment":0,"confidence":6.6282007404612555} +{"run":49,"commit":"450d2be","metric":367.33230196280397,"metrics":{},"status":"discard","description":"Revisited prebinding visited.add/visited.discard inside bt_safe_deep_copy after the newer delayed-check and list-of-dicts optimizations, but it still regressed the aggregate benchmark versus the current best.","timestamp":1774409465199,"segment":0,"confidence":6.462655754585521} +{"run":50,"commit":"450d2be","metric":382.3847063273385,"metrics":{},"status":"discard","description":"Tried extracting the exact dict/list fast paths into dedicated helpers that recursively dispatch exact containers directly, hoping to skip generic _deep_copy_object type checks on nested container children, but the helper-call structure regressed the aggregate benchmark badly.","timestamp":1774409790196,"segment":0,"confidence":6.859460021444134} +{"run":51,"commit":"450d2be","metric":365.13756809688266,"metrics":{},"status":"discard","description":"Tried a tiny exact-list fast path that returns list(v) immediately for homogeneous exact-string lists before any visited-set work, but the aggregate benchmark was effectively flat/slightly worse than the current best.","timestamp":1774410035381,"segment":0,"confidence":7.308179052953935} +{"run":52,"commit":"89a4583","metric":352.578193876466,"metrics":{},"status":"keep","description":"Move the exact built-in container branches ahead of the exact scalar/float branches in bt_safe_deep_copy so the recursively dominant dict/list/tuple/set nodes hit their hot path sooner, while still preserving the existing scalar fast paths and fallback semantics.","timestamp":1774410495880,"segment":0,"confidence":6.983827049211779} +{"run":53,"commit":"89a4583","metric":350.9943682331325,"metrics":{},"status":"discard","description":"Confirmation rerun of the container-first bt_safe_deep_copy ordering came in slightly better but reflects the same kept code change; the new best remains stable in the ~351–353µs range.","timestamp":1774410659907,"segment":0,"confidence":6.90864573591404} +{"run":54,"commit":"89a4583","metric":354.3889459143506,"metrics":{},"status":"discard","description":"Tried moving the exact tuple/set branch below the exact scalar/float fast paths so the far more common scalar leaves skip that rare-container check, but the aggregate bt_json benchmark regressed.","timestamp":1774410945338,"segment":0,"confidence":7.02011788347503} +{"run":55,"commit":"0c76d33","metric":350.61670127282156,"metrics":{},"status":"keep","description":"Move dataclass/model-dump/dict probing ahead of the subclass scalar fallback checks in _to_bt_safe, so dataclass and model-like objects avoid failed isinstance(str/bool/int/float) probes while exact scalar fast paths and special-object handling stay intact.","timestamp":1774411192454,"segment":0,"confidence":7.09555086732649} +{"run":56,"commit":"0c76d33","metric":352.3806919243555,"metrics":{},"status":"discard","description":"Confirmation rerun of the reordered _to_bt_safe subclass-fallback checks stayed in the same ~351–352µs band, validating the latest dataclass/model-path improvement without introducing any further code change.","timestamp":1774411341788,"segment":0,"confidence":6.5262646146513825} +{"run":57,"commit":"0c76d33","metric":354.50636387850363,"metrics":{},"status":"discard","description":"Tried splitting the exact-list fast path into a dedicated first-item-is-dict branch so the common list-of-dicts case could skip the per-element dict test after the first element, but the aggregate benchmark regressed despite promising microbenchmarks.","timestamp":1774411635487,"segment":0,"confidence":6.094565903554933} +{"run":58,"commit":"0c76d33","metric":353.5577450363009,"metrics":{},"status":"discard","description":"Tried delaying exact-list visited-set insertion until the first child that actually needs recursive descent, while also inlining scalar/float handling for scalar-only lists, but the aggregate bt_json benchmark regressed versus the current best.","timestamp":1774412040522,"segment":0,"confidence":5.854573144267451} +{"run":59,"commit":"78d9836","metric":350.21564039386595,"metrics":{},"status":"keep","description":"Switch visited-set cleanup from discard() to remove() in bt_safe_deep_copy branches where membership is guaranteed, trimming a bit of defensive overhead from the hot exact-container and fallback container teardown paths.","timestamp":1774412370397,"segment":0,"confidence":5.772158427601105} +{"run":60,"commit":"78d9836","metric":352.7949322872183,"metrics":{},"status":"discard","description":"Confirmation rerun of the remove()-based visited cleanup landed back in the broader ~350–353µs band; keeping the earlier winning commit but discarding this no-code rerun.","timestamp":1774412665469,"segment":0,"confidence":6.007681030664957} +{"run":61,"commit":"78d9836","metric":351.2608287342308,"metrics":{},"status":"discard","description":"Tried moving the model_json_schema class-path check below model_dump/dict probing so ordinary model instances skip an unused isinstance(v, type) branch earlier, but the aggregate bt_json benchmark still regressed slightly versus the current best.","timestamp":1774412996598,"segment":0,"confidence":6.760308249815194} +{"run":62,"commit":"78d9836","metric":351.7327037242361,"metrics":{},"status":"discard","description":"Retried dropping callable() checks for model_dump/dict in _to_bt_safe after the newer bt_json fast-path changes, but the aggregate benchmark still regressed slightly versus the current best.","timestamp":1774413211117,"segment":0,"confidence":7.030527888911275} +{"run":63,"commit":"aaadddc","metric":349.94741905749265,"metrics":{},"status":"keep","description":"Add an early isinstance(v, str) fast path in _to_bt_safe ahead of dataclass/model probing so common string subclasses (like enum-style span attributes) bypass the richer object-detection work without affecting exact scalar or model/dataclass handling.","timestamp":1774413440515,"segment":0,"confidence":6.797575875854772} +{"run":64,"commit":"aaadddc","metric":352.7529096491738,"metrics":{},"status":"discard","description":"Confirmation rerun of the early string-subclass fast path landed back in the broader ~350–353µs range, validating the kept improvement without introducing any further code change.","timestamp":1774413591455,"segment":0,"confidence":6.378025269822195} +{"run":65,"commit":"78d9836","metric":353.22677356757947,"metrics":{},"status":"discard","description":"Tried narrowing the post-model fallback scalar subclass check in _to_bt_safe from (str,bool,int) to only (bool,int) because string subclasses now have their own earlier fast path, but the aggregate benchmark regressed.","timestamp":1774413822761,"segment":0,"confidence":6.373255066406103} +{"run":66,"commit":"ea3166e","metric":344.83290970465237,"metrics":{},"status":"keep","description":"Inline exact scalar and finite-float handling for non-dict elements inside the exact-list fast path so scalar/string list items avoid a full _deep_copy_object redispatch while the dominant list-of-dicts path stays unchanged.","timestamp":1774413992248,"segment":0,"confidence":6.481500984428061} +{"run":67,"commit":"ea3166e","metric":346.82634252922696,"metrics":{},"status":"discard","description":"Confirmation rerun of the exact-list scalar-inline optimization stayed in the improved mid-340µs range, validating the latest kept change without introducing any further code change.","timestamp":1774414140888,"segment":0,"confidence":6.668582696264335} +{"run":68,"commit":"8ccb4a2","metric":348.1037284222322,"metrics":{},"status":"keep","description":"Compute type(value) once per exact-list element and reuse it for the dict/scalar/float branches, avoiding a redundant type() call on every non-dict list item while preserving the optimized list-of-dicts fast path.","timestamp":1774414601782,"segment":0,"confidence":6.779287305242741} +{"run":69,"commit":"8ccb4a2","metric":348.56693704142793,"metrics":{},"status":"discard","description":"Confirmation rerun of the exact-list cached value_type optimization stayed in the improved high-340µs range, validating the kept change without introducing any further code change.","timestamp":1774414759894,"segment":0,"confidence":6.893811077233381} +{"run":70,"commit":"faab97c","metric":335.9028560562187,"metrics":{},"status":"keep","description":"Split the exact-dict and list-of-dicts teardown into a true two-phase path: scalar-only prefixes now return without paying a try/finally cleanup frame, and visited removal is only wrapped around the branch that actually recurses after the first non-scalar child.","timestamp":1774415380027,"segment":0,"confidence":6.734777207115278} +{"run":71,"commit":"faab97c","metric":337.0230954503598,"metrics":{},"status":"discard","description":"Confirmation rerun of the two-phase exact-dict/list-of-dicts teardown stayed in the improved mid-330µs range, validating the kept change without introducing any further code change.","timestamp":1774415529071,"segment":0,"confidence":6.754384181666804} +{"run":72,"commit":"2f5b82f","metric":338.7206501108379,"metrics":{},"status":"keep","description":"Apply the same two-phase visited-set teardown to exact lists: delay adding the list itself to the visited set until the first child that actually needs recursive descent, so scalar-only and scalar-prefix lists skip unconditional add/remove overhead.","timestamp":1774415752252,"segment":0,"confidence":6.753179633861559} +{"run":73,"commit":"2f5b82f","metric":345.6859002454439,"metrics":{},"status":"discard","description":"Confirmation rerun of the delayed exact-list visited-set insertion came back in the broader high-330s to mid-340s band; keeping the earlier winning commit and discarding this no-code rerun.","timestamp":1774415901564,"segment":0,"confidence":6.751975515607452} +{"run":74,"commit":"2f5b82f","metric":344.11972941192465,"metrics":{},"status":"discard","description":"Tried flipping the exact-dict and list-of-dicts key branches to make exact string keys the direct hot path after the newer two-phase teardown changes, but the aggregate bt_json benchmark still regressed versus the current best.","timestamp":1774416211345,"segment":0,"confidence":7.068002136010104} +{"run":75,"commit":"2f5b82f","metric":353.6480895464782,"metrics":{},"status":"discard","description":"Tried delaying exact-list id(v) computation itself until the first recursive child, but the aggregate bt_json benchmark regressed versus the current best despite the small list-focused microbenchmark win.","timestamp":1774416473298,"segment":0,"confidence":7.32467427129712} +{"run":76,"commit":"2f5b82f","metric":338.78047035625866,"metrics":{},"status":"discard","description":"Tried hoisting the exact-list next_depth>=max_depth handling out of the per-dict-element loop after the newer two-phase list teardown changes, but the aggregate bt_json benchmark was effectively flat/slightly worse than the current best.","timestamp":1774416755085,"segment":0,"confidence":7.221996087612733} +{"run":77,"commit":"2f5b82f","metric":341.49193656110333,"metrics":{},"status":"discard","description":"Tried splitting the exact-list fast path into a dedicated first-item-is-dict branch after the newer two-phase list teardown changes, but the aggregate bt_json benchmark still regressed versus the current best despite passing tests.","timestamp":1774417194985,"segment":0,"confidence":7.122156818459104} +{"run":78,"commit":"2f5b82f","metric":342.714444557262,"metrics":{},"status":"discard","description":"Tried flipping the exact-dict and list-of-dicts key branches to make exact string keys the direct hot path after the latest list two-phase cleanup, but the aggregate bt_json benchmark still regressed.","timestamp":1774417614316,"segment":0,"confidence":7.200897337584435} +{"run":79,"commit":"2f5b82f","metric":337.812004597463,"metrics":{},"status":"discard","description":"Confirmation rerun of the two-phase exact-list visited-set teardown landed essentially on top of the kept result, reinforcing that the current best bt_json implementation is stable in the high-330µs range without any additional code change.","timestamp":1774417862198,"segment":0,"confidence":7.281398385716894} diff --git a/py/src/braintrust/bt_json.py b/py/src/braintrust/bt_json.py index e0c7be13..f91b596a 100644 --- a/py/src/braintrust/bt_json.py +++ b/py/src/braintrust/bt_json.py @@ -2,7 +2,8 @@ import json import math import warnings -from typing import Any, Callable, Mapping, NamedTuple, cast, overload +from collections.abc import Mapping +from typing import Any, Callable, NamedTuple, cast, overload # Try to import orjson for better performance @@ -15,75 +16,116 @@ _HAS_ORJSON = False +_BT_SAFE_SPECIAL_TYPES: tuple[type[Any], type[Any], type[Any], type[Any], type[Any], type[Any]] | None = None + + +def _get_bt_safe_special_types() -> tuple[type[Any], type[Any], type[Any], type[Any], type[Any], type[Any]]: + global _BT_SAFE_SPECIAL_TYPES + if _BT_SAFE_SPECIAL_TYPES is None: + # avoid circular imports + from braintrust.logger import BaseAttachment, Dataset, Experiment, Logger, ReadonlyAttachment, Span + + _BT_SAFE_SPECIAL_TYPES = (Span, Experiment, Dataset, Logger, BaseAttachment, ReadonlyAttachment) + + return _BT_SAFE_SPECIAL_TYPES + + def _to_bt_safe(v: Any) -> Any: """ Converts the object to a Braintrust-safe representation (i.e. Attachment objects are safe (specially handled by background logger)). """ - # avoid circular imports - from braintrust.logger import BaseAttachment, Dataset, Experiment, Logger, ReadonlyAttachment, Span - - if isinstance(v, Span): - return "" + v_type = type(v) + if v_type is str or v_type is int or v_type is bool or v is None: + # Skip all richer object checks for primitive scalar values. + return v - if isinstance(v, Experiment): - return "" + if v_type is float: + # Handle NaN and Infinity for JSON compatibility + if math.isfinite(v): + return v - if isinstance(v, Dataset): - return "" + if math.isnan(v): + return "NaN" - if isinstance(v, Logger): - return "" + return "Infinity" if v > 0 else "-Infinity" - if isinstance(v, BaseAttachment): + if isinstance(v, str): return v - if isinstance(v, ReadonlyAttachment): - return v.reference - - if dataclasses.is_dataclass(v) and not isinstance(v, type): + dataclass_fields = getattr(v_type, "__dataclass_fields__", None) + if dataclass_fields is not None: # Use manual field iteration instead of dataclasses.asdict() because # asdict() deep-copies values, which breaks objects like Attachment # that contain non-copyable items (thread locks, file handles, etc.) - return {f.name: _to_bt_safe(getattr(v, f.name)) for f in dataclasses.fields(v)} + instance_dict = getattr(v, "__dict__", None) + if instance_dict is not None and len(instance_dict) == len(dataclass_fields): + return bt_safe_deep_copy(instance_dict) + return {f.name: _to_bt_safe(getattr(v, f.name)) for f in dataclass_fields.values()} # Pydantic model classes (not instances) with model_json_schema - if isinstance(v, type) and hasattr(v, "model_json_schema") and callable(cast(Any, v).model_json_schema): - try: - return cast(Any, v).model_json_schema() - except Exception: - pass + if isinstance(v, type): + model_json_schema = getattr(v, "model_json_schema", None) + if callable(model_json_schema): + try: + return model_json_schema() + except Exception: + pass # Attempt to dump a Pydantic v2 `BaseModel`. # Suppress Pydantic serializer warnings that arise from generic/discriminated-union # models (e.g. OpenAI's ParsedResponse[T]). See # https://github.com/braintrustdata/braintrust-sdk-python/issues/60 - try: - with warnings.catch_warnings(): - warnings.filterwarnings("ignore", message="Pydantic serializer warnings", category=UserWarning) - return cast(Any, v).model_dump(exclude_none=True) - except (AttributeError, TypeError): - pass + model_dump = getattr(v_type, "model_dump", None) + if callable(model_dump): + try: + if hasattr(v_type, "__pydantic_serializer__"): + with warnings.catch_warnings(): + warnings.filterwarnings("ignore", message="Pydantic serializer warnings", category=UserWarning) + return model_dump(v, exclude_none=True) + return model_dump(v, exclude_none=True) + except TypeError: + pass # Attempt to dump a Pydantic v1 `BaseModel`. - try: - return cast(Any, v).dict(exclude_none=True) - except (AttributeError, TypeError): - pass + dict_method = getattr(v_type, "dict", None) + if callable(dict_method): + try: + return dict_method(v, exclude_none=True) + except TypeError: + pass + + if isinstance(v, (str, bool, int)): + return v if isinstance(v, float): - # Handle NaN and Infinity for JSON compatibility + if math.isfinite(v): + return v + if math.isnan(v): return "NaN" - if math.isinf(v): - return "Infinity" if v > 0 else "-Infinity" + return "Infinity" if v > 0 else "-Infinity" - return v + Span, Experiment, Dataset, Logger, BaseAttachment, ReadonlyAttachment = _get_bt_safe_special_types() - if isinstance(v, (int, str, bool)) or v is None: - # Skip roundtrip for primitive types. + if isinstance(v, Span): + return "" + + if isinstance(v, Experiment): + return "" + + if isinstance(v, Dataset): + return "" + + if isinstance(v, Logger): + return "" + + if isinstance(v, BaseAttachment): return v + if isinstance(v, ReadonlyAttachment): + return v.reference + # Note: we avoid using copy.deepcopy, because it's difficult to # guarantee the independence of such copied types from their origin. # E.g. the original type could have a `__del__` method that alters @@ -133,35 +175,307 @@ def _deep_copy_object(v: Any, depth: int = 0) -> Any: if depth >= max_depth: return "" - # Check for circular references in mutable containers - # Use id() to track object identity - if isinstance(v, (Mapping, list, tuple, set)): + v_type = type(v) + + # Check for circular references in mutable containers. + # Fast-path the built-in container types we expect most often. + if v_type is dict: + # Prevent dict keys from holding references to user data. Note that + # `bt_json` already coerces keys to string, a behavior that comes from + # `json.dumps`. However, that runs at log upload time, while we want to + # cut out all the references to user objects synchronously in this + # function. + result = {} + next_depth = depth + 1 + if next_depth >= max_depth: + for k in v: + if type(k) is str: + key_str = k + else: + try: + key_str = str(k) + except Exception: + # If str() fails on the key, use a fallback representation + key_str = f"" + result[key_str] = "" + return result + + items = iter(v.items()) + for k, value in items: + if type(k) is not str: + try: + key_str = str(k) + except Exception: + # If str() fails on the key, use a fallback representation + key_str = f"" + obj_id = id(v) + if obj_id in visited: + return "" + visited.add(obj_id) + try: + result[key_str] = _deep_copy_object(value, next_depth) + for k, value in items: + if type(k) is str: + key_str = k + else: + try: + key_str = str(k) + except Exception: + # If str() fails on the key, use a fallback representation + key_str = f"" + result[key_str] = _deep_copy_object(value, next_depth) + return result + finally: + visited.remove(obj_id) + + value_type = type(value) + if value_type is str or value_type is int or value_type is bool or value is None: + result[k] = value + continue + + if value_type is float: + if math.isfinite(value): + result[k] = value + elif math.isnan(value): + result[k] = "NaN" + else: + result[k] = "Infinity" if value > 0 else "-Infinity" + continue + + obj_id = id(v) + if obj_id in visited: + return "" + visited.add(obj_id) + try: + result[k] = _deep_copy_object(value, next_depth) + for k, value in items: + if type(k) is str: + key_str = k + else: + try: + key_str = str(k) + except Exception: + # If str() fails on the key, use a fallback representation + key_str = f"" + result[key_str] = _deep_copy_object(value, next_depth) + return result + finally: + visited.remove(obj_id) + + return result + + if v_type is list: + obj_id = id(v) + added_to_visited = False + try: + next_depth = depth + 1 + result = [] + for value in v: + value_type = type(value) + if value_type is dict: + if not added_to_visited: + if obj_id in visited: + return "" + visited.add(obj_id) + added_to_visited = True + nested_result = {} + if next_depth >= max_depth: + for k in value: + if type(k) is str: + key_str = k + else: + try: + key_str = str(k) + except Exception: + key_str = f"" + nested_result[key_str] = "" + result.append(nested_result) + continue + + items = iter(value.items()) + for k, nested_value in items: + if type(k) is not str: + try: + key_str = str(k) + except Exception: + key_str = f"" + value_id = id(value) + if value_id in visited: + result.append("") + break + visited.add(value_id) + try: + nested_result[key_str] = _deep_copy_object(nested_value, next_depth) + for k, nested_value in items: + if type(k) is str: + key_str = k + else: + try: + key_str = str(k) + except Exception: + key_str = f"" + nested_result[key_str] = _deep_copy_object(nested_value, next_depth) + result.append(nested_result) + finally: + visited.remove(value_id) + break + + nested_value_type = type(nested_value) + if nested_value_type is str or nested_value_type is int or nested_value_type is bool or nested_value is None: + nested_result[k] = nested_value + continue + + if nested_value_type is float: + if math.isfinite(nested_value): + nested_result[k] = nested_value + elif math.isnan(nested_value): + nested_result[k] = "NaN" + else: + nested_result[k] = "Infinity" if nested_value > 0 else "-Infinity" + continue + + value_id = id(value) + if value_id in visited: + result.append("") + break + visited.add(value_id) + try: + nested_result[k] = _deep_copy_object(nested_value, next_depth) + for k, nested_value in items: + if type(k) is str: + key_str = k + else: + try: + key_str = str(k) + except Exception: + key_str = f"" + nested_result[key_str] = _deep_copy_object(nested_value, next_depth) + result.append(nested_result) + finally: + visited.remove(value_id) + break + else: + result.append(nested_result) + continue + + if value_type is str or value_type is int or value_type is bool or value is None: + result.append(value) + continue + + if value_type is float: + if math.isfinite(value): + result.append(value) + elif math.isnan(value): + result.append("NaN") + else: + result.append("Infinity" if value > 0 else "-Infinity") + continue + + if not added_to_visited: + if obj_id in visited: + return "" + visited.add(obj_id) + added_to_visited = True + result.append(_deep_copy_object(value, next_depth)) + return result + finally: + if added_to_visited: + visited.remove(obj_id) + + if v_type is tuple or v_type is set: + obj_id = id(v) + if obj_id in visited: + return "" + visited.add(obj_id) + try: + next_depth = depth + 1 + return [_deep_copy_object(x, next_depth) for x in v] + finally: + visited.remove(obj_id) + + if v_type is str or v_type is int or v_type is bool or v is None: + return v + + if v_type is float: + if math.isfinite(v): + return v + if math.isnan(v): + return "NaN" + return "Infinity" if v > 0 else "-Infinity" + + if isinstance(v, (str, bool, int)): + return v + + if isinstance(v, float): + if math.isfinite(v): + return v + if math.isnan(v): + return "NaN" + return "Infinity" if v > 0 else "-Infinity" + + if isinstance(v, dict): obj_id = id(v) if obj_id in visited: return "" visited.add(obj_id) try: - if isinstance(v, Mapping): - # Prevent dict keys from holding references to user data. Note that - # `bt_json` already coerces keys to string, a behavior that comes from - # `json.dumps`. However, that runs at log upload time, while we want to - # cut out all the references to user objects synchronously in this - # function. - result = {} - for k in v: + result = {} + next_depth = depth + 1 + for k, value in v.items(): + if type(k) is str: + key_str = k + else: try: key_str = str(k) except Exception: - # If str() fails on the key, use a fallback representation key_str = f"" - result[key_str] = _deep_copy_object(v[k], depth + 1) - return result - elif isinstance(v, (list, tuple, set)): - return [_deep_copy_object(x, depth + 1) for x in v] + result[key_str] = _deep_copy_object(value, next_depth) + return result + finally: + visited.remove(obj_id) + + if isinstance(v, list): + obj_id = id(v) + if obj_id in visited: + return "" + visited.add(obj_id) + try: + next_depth = depth + 1 + return [_deep_copy_object(x, next_depth) for x in v] + finally: + visited.remove(obj_id) + + if isinstance(v, (tuple, set)): + obj_id = id(v) + if obj_id in visited: + return "" + visited.add(obj_id) + try: + next_depth = depth + 1 + return [_deep_copy_object(x, next_depth) for x in v] + finally: + visited.remove(obj_id) + + if isinstance(v, Mapping): + obj_id = id(v) + if obj_id in visited: + return "" + visited.add(obj_id) + try: + result = {} + next_depth = depth + 1 + for k, value in v.items(): + if type(k) is str: + key_str = k + else: + try: + key_str = str(k) + except Exception: + key_str = f"" + result[key_str] = _deep_copy_object(value, next_depth) + return result finally: - # Remove from visited set after processing to allow the same object - # to appear in different branches of the tree - visited.discard(obj_id) + visited.remove(obj_id) try: return _to_bt_safe(v)