diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..7db4b78 --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +.coverage +*.py[cod] +__pycache__/ diff --git a/README.md b/README.md index a078908..a6a4179 100644 --- a/README.md +++ b/README.md @@ -31,6 +31,22 @@ - Example: [`examples/route-manifest-v0.1.example.json`](examples/route-manifest-v0.1.example.json) - Fixtures: [`fixtures/valid/route-manifest-v0.1.valid.json`](fixtures/valid/route-manifest-v0.1.valid.json), [`fixtures/invalid/route-manifest-v0.1.invalid.json`](fixtures/invalid/route-manifest-v0.1.invalid.json) +## Governed evaluation packets + +- GPT-5.6 Sol Ultra evidence packet (search-bounded, checked 2026-07-11): + [`docs/evaluations/2026-07-11-gpt-5.6-sol-ultra-evidence-packet.md`](docs/evaluations/2026-07-11-gpt-5.6-sol-ultra-evidence-packet.md) +- Provisional reasoning-effort route policy: + [`docs/route-policy/reasoning-effort-route-policy.v0.1.md`](docs/route-policy/reasoning-effort-route-policy.v0.1.md) +- Controlled xhigh–Max–Ultra protocol: + [`docs/evaluations/ultra-evaluation-protocol.v0.1.md`](docs/evaluations/ultra-evaluation-protocol.v0.1.md) +- Machine benchmark inventory and raw provenance: + [`data/gpt-5.6-sol-ultra/benchmark_dataset.v0.1.json`](data/gpt-5.6-sol-ultra/benchmark_dataset.v0.1.json) +- Governed execution-receipt contract: + [`schemas/ultra_evaluation_receipt.v0.1.schema.json`](schemas/ultra_evaluation_receipt.v0.1.schema.json) + +These artifacts are evidence and experiment scaffolding, not adopted universal +routing policy. + ## Status Public seed repository. Initial executable packet in progress as `seed` -> `v0.1-draft`. diff --git a/data/gpt-5.6-sol-ultra/archive_manifest.json b/data/gpt-5.6-sol-ultra/archive_manifest.json new file mode 100644 index 0000000..300c503 --- /dev/null +++ b/data/gpt-5.6-sol-ultra/archive_manifest.json @@ -0,0 +1,297 @@ +{ + "archive": { + "capture_timestamp": "20260710150348", + "capture_timestamp_utc": "2026-07-10T15:03:48Z", + "content_sha256": "242b04c63bd0da245bc2dcf601801ee958deb9ebc08def9193f5c6846fdbf892", + "hash_scope": "decompressed HTTP response entity bytes from the id_ capture after HTTP content decoding", + "original_url": "https://openai.com/index/gpt-5-6/", + "retrieval_date": "2026-07-11", + "wayback_capture_url": "https://web.archive.org/web/20260710150348id_/https://openai.com/index/gpt-5-6/", + "wayback_ui_url": "https://web.archive.org/web/20260710150348/https://openai.com/index/gpt-5-6/" + }, + "availability": { + "BrowseComp": { + "expected_agent_counts": [ + 1, + 4, + 16 + ], + "missing_agent_counts": [], + "observed_agent_counts": [ + 1, + 4, + 16 + ], + "observed_efforts": [ + "low", + "medium", + "high", + "xhigh", + "max" + ], + "observed_metrics": [ + "api_cost_usd", + "latency_s", + "output_tokens" + ], + "record_count": 45 + }, + "SEC-Bench Pro": { + "expected_agent_counts": [ + 1, + 4, + 16 + ], + "missing_agent_counts": [], + "observed_agent_counts": [ + 1, + 4, + 16 + ], + "observed_efforts": [ + "low", + "medium", + "high", + "xhigh", + "max" + ], + "observed_metrics": [ + "api_cost_usd", + "latency_s", + "output_tokens" + ], + "record_count": 45 + }, + "Terminal-Bench 2.1": { + "expected_agent_counts": [ + 1, + 4, + 16 + ], + "missing_agent_counts": [ + 16 + ], + "observed_agent_counts": [ + 1, + 4 + ], + "observed_efforts": [ + "low", + "medium", + "high", + "xhigh", + "max" + ], + "observed_metrics": [ + "api_cost_usd", + "latency_s", + "output_tokens" + ], + "record_count": 30 + } + }, + "evidence_class": "archived_chart_data", + "extraction": { + "deduplication": "exact semantic JSON object; first occurrence retained", + "parser_input": "escaped flat eval JSON objects embedded in archive HTML", + "script": "tools/extract_gpt56_ultra_archive.py", + "selected_record_count": 120, + "selected_records": "data/gpt-5.6-sol-ultra/archived_chart_records.json", + "selected_records_sha256": "b2970b62c76a688f34b68f3a8739777f677268e50d0657db82d32a3f62345e11" + }, + "limitations": [ + "The capture is a provider page and the selected records are provider-reported.", + "Terminal-Bench 2.1 has no 16-agent flat chart records in this capture.", + "Cost and latency are provider estimates or simulations, not service guarantees.", + "Raw selected records intentionally preserve two distinct inaccessible internal OpenAI Slack source URLs embedded by the provider; no credentials or tokens were found. This accepts a provenance/privacy tradeoff in favor of raw-source fidelity." + ], + "provenance_privacy": { + "credentials_or_tokens_found": false, + "decision": "Preserve provider-embedded source_url values for raw provenance fidelity despite inaccessible internal workspace destinations.", + "preserved_internal_source_url_count": 2, + "preserved_internal_source_urls": [ + "https://openai-corpws.slack.com/archives/C0B95RM2XS5/p1782526942178749?thread_ts=1782423237.343579&cid=C0B95RM2XS5", + "https://openai-corpws.slack.com/archives/C0B95RM2XS5/p1783576882966699?thread_ts=1783574276.164169&cid=C0B95RM2XS5" + ] + }, + "schema_version": "gpt-5.6-sol-ultra.archive-manifest.v0.1", + "score_reconciliation": { + "BrowseComp": { + "archived_max": { + "1_agent": { + "availability": "available", + "effort": "max", + "score": 0.9083728278041074, + "score_label": "90.84%", + "score_percent": 90.83728278041075 + }, + "4_agents": { + "availability": "available", + "effort": "max", + "score": 0.9218009478672986, + "score_label": "92.18%", + "score_percent": 92.18009478672985 + } + }, + "comparison_rule": "Current one-decimal headline scores are reported separately from archived max chart scores; operational comparisons use archived max records only.", + "current_headline": { + "1_agent": { + "agrees_after_rounding_to_one_decimal": false, + "archived_score_percent": 90.83728278041075, + "difference_archived_minus_current_percentage_points": 0.437283, + "score_percent": 90.4 + }, + "4_agents": { + "agrees_after_rounding_to_one_decimal": true, + "archived_score_percent": 92.18009478672985, + "difference_archived_minus_current_percentage_points": -0.019905, + "score_percent": 92.2 + } + } + }, + "SEC-Bench Pro": { + "archived_max": { + "1_agent": { + "availability": "available", + "effort": "max", + "score": 0.7144808743169399, + "score_label": "71.45%", + "score_percent": 71.44808743169399 + }, + "4_agents": { + "availability": "available", + "effort": "max", + "score": 0.7431693989071039, + "score_label": "74.32%", + "score_percent": 74.31693989071039 + } + }, + "comparison_rule": "Current one-decimal headline scores are reported separately from archived max chart scores; operational comparisons use archived max records only.", + "current_headline": { + "1_agent": { + "agrees_after_rounding_to_one_decimal": false, + "archived_score_percent": 71.44808743169399, + "difference_archived_minus_current_percentage_points": 0.248087, + "score_percent": 71.2 + }, + "4_agents": { + "agrees_after_rounding_to_one_decimal": true, + "archived_score_percent": 74.31693989071039, + "difference_archived_minus_current_percentage_points": 0.01694, + "score_percent": 74.3 + } + } + }, + "Terminal-Bench 2.1": { + "archived_max": { + "1_agent": { + "availability": "available", + "effort": "max", + "score": 0.887640449438, + "score_label": "88.76%", + "score_percent": 88.7640449438 + }, + "4_agents": { + "availability": "available", + "effort": "max", + "score": 0.919101123596, + "score_label": "91.91%", + "score_percent": 91.9101123596 + } + }, + "comparison_rule": "Current one-decimal headline scores are reported separately from archived max chart scores; operational comparisons use archived max records only.", + "current_headline": { + "1_agent": { + "agrees_after_rounding_to_one_decimal": true, + "archived_score_percent": 88.7640449438, + "difference_archived_minus_current_percentage_points": -0.035955, + "score_percent": 88.8 + }, + "4_agents": { + "agrees_after_rounding_to_one_decimal": true, + "archived_score_percent": 91.9101123596, + "difference_archived_minus_current_percentage_points": 0.010112, + "score_percent": 91.9 + } + } + } + }, + "unit_interpretation": { + "api_cost_usd": { + "interpretation": "Provider-estimated API cost including all agents; not an observed public-service billing guarantee.", + "raw_metric_name": "api_cost_usd", + "raw_value_field": "x_value", + "x_value_unit": "estimated USD" + }, + "latency_s": { + "companion_source_evidence": [ + { + "canonical_record_sha256": "cd5bc06b88020e861daac3bf97dbe134e6fde9485e26e0a4bc935b9df940ffb5", + "record": { + "category": "Coding", + "code_mode": "true", + "cost": 1.7200842764057498, + "cost_label": "$$1.72", + "eval": "Terminal-Bench 2.1", + "eval_id": "terminal_bench_2_1", + "eval_variant": "", + "juice_index": 5, + "juice_level": "max", + "latency_label": "4.24 minutes", + "latency_minutes": 4.240172912070197, + "model": "GPT-5.6 Sol", + "output_tokens": 12795.8044944, + "output_tokens_label": "12.8K", + "score": 0.887640449438, + "score_label": "88.76%", + "score_metric": "", + "score_unit": "accuracy" + } + }, + { + "canonical_record_sha256": "4a231f9d3ae0298d46190115db127640db34afd0aaa041483bb83c4a4204b310", + "record": { + "category": "Coding", + "code_mode": "true", + "cost": 5.14084233483, + "cost_label": "$$5.14", + "eval": "Terminal-Bench 2.1", + "eval_id": "terminal_bench_2_1", + "eval_variant": "", + "juice_index": 5, + "juice_level": "max", + "latency_label": "4.28 minutes", + "latency_minutes": 4.278585916301308, + "model": "GPT-5.6 Sol Ultra", + "output_tokens": 39275.1280899, + "output_tokens_label": "39.27K", + "score": 0.919101123596, + "score_label": "91.91%", + "score_metric": "", + "score_unit": "accuracy" + } + } + ], + "interpretation": "Provider-simulated root-agent latency; not an observed public-service latency guarantee.", + "interpretation_basis": [ + "Every selected latency display label renders x_value in minutes.", + "Preserved companion source evidence names the same quantity latency_minutes." + ], + "raw_metric_name": "latency_s", + "raw_value_field": "x_value", + "warning": "The raw metric name suggests seconds, but treating x_value as seconds conflicts with the displayed labels; values are preserved unchanged and interpreted as minutes.", + "x_value_unit": "minutes" + }, + "output_tokens": { + "interpretation": "Aggregate output tokens include all agents.", + "raw_metric_name": "output_tokens", + "raw_value_field": "x_value", + "x_value_unit": "tokens" + }, + "score": { + "display_unit": "percent", + "raw_value_field": "score", + "stored_unit": "fraction" + } + } +} diff --git a/data/gpt-5.6-sol-ultra/archived_chart_records.json b/data/gpt-5.6-sol-ultra/archived_chart_records.json new file mode 100644 index 0000000..60dc0f2 --- /dev/null +++ b/data/gpt-5.6-sol-ultra/archived_chart_records.json @@ -0,0 +1,2399 @@ +{ + "availability": { + "BrowseComp": { + "expected_agent_counts": [ + 1, + 4, + 16 + ], + "missing_agent_counts": [], + "observed_agent_counts": [ + 1, + 4, + 16 + ], + "observed_efforts": [ + "low", + "medium", + "high", + "xhigh", + "max" + ], + "observed_metrics": [ + "api_cost_usd", + "latency_s", + "output_tokens" + ], + "record_count": 45 + }, + "SEC-Bench Pro": { + "expected_agent_counts": [ + 1, + 4, + 16 + ], + "missing_agent_counts": [], + "observed_agent_counts": [ + 1, + 4, + 16 + ], + "observed_efforts": [ + "low", + "medium", + "high", + "xhigh", + "max" + ], + "observed_metrics": [ + "api_cost_usd", + "latency_s", + "output_tokens" + ], + "record_count": 45 + }, + "Terminal-Bench 2.1": { + "expected_agent_counts": [ + 1, + 4, + 16 + ], + "missing_agent_counts": [ + 16 + ], + "observed_agent_counts": [ + 1, + 4 + ], + "observed_efforts": [ + "low", + "medium", + "high", + "xhigh", + "max" + ], + "observed_metrics": [ + "api_cost_usd", + "latency_s", + "output_tokens" + ], + "record_count": 30 + } + }, + "evidence_class": "archived_chart_data", + "record_count": 120, + "records": [ + { + "category": "Computer Use", + "code_mode": "true", + "eval": "BrowseComp", + "eval_id": "truffle_hunter_oss", + "eval_variant": "1 agent", + "juice_index": 1, + "juice_level": "low", + "model": "GPT-5.6 Sol · 1 agent", + "model_slug": "gpt-5.6-sol-ma", + "score": 0.6903633491311216, + "score_label": "69.04%", + "score_metric": "", + "score_unit": "score", + "source_url": "https://openai-corpws.slack.com/archives/C0B95RM2XS5/p1783576882966699?thread_ts=1783574276.164169&cid=C0B95RM2XS5", + "x_label": "$$0.27", + "x_metric": "api_cost_usd", + "x_value": 0.26650899842 + }, + { + "category": "Computer Use", + "code_mode": "true", + "eval": "BrowseComp", + "eval_id": "truffle_hunter_oss", + "eval_variant": "1 agent", + "juice_index": 1, + "juice_level": "low", + "model": "GPT-5.6 Sol · 1 agent", + "model_slug": "gpt-5.6-sol-ma", + "score": 0.6903633491311216, + "score_label": "69.04%", + "score_metric": "", + "score_unit": "score", + "source_url": "https://openai-corpws.slack.com/archives/C0B95RM2XS5/p1783576882966699?thread_ts=1783574276.164169&cid=C0B95RM2XS5", + "x_label": "0.88 minutes", + "x_metric": "latency_s", + "x_value": 0.8753079567977966 + }, + { + "category": "Computer Use", + "code_mode": "true", + "eval": "BrowseComp", + "eval_id": "truffle_hunter_oss", + "eval_variant": "1 agent", + "juice_index": 1, + "juice_level": "low", + "model": "GPT-5.6 Sol · 1 agent", + "model_slug": "gpt-5.6-sol-ma", + "score": 0.6903633491311216, + "score_label": "69.04%", + "score_metric": "", + "score_unit": "score", + "source_url": "https://openai-corpws.slack.com/archives/C0B95RM2XS5/p1783576882966699?thread_ts=1783574276.164169&cid=C0B95RM2XS5", + "x_label": "1,957", + "x_metric": "output_tokens", + "x_value": 1957.11374407583 + }, + { + "category": "Computer Use", + "code_mode": "true", + "eval": "BrowseComp", + "eval_id": "truffle_hunter_oss", + "eval_variant": "1 agent", + "juice_index": 2, + "juice_level": "medium", + "model": "GPT-5.6 Sol · 1 agent", + "model_slug": "gpt-5.6-sol-ma", + "score": 0.8459715639810427, + "score_label": "84.6%", + "score_metric": "", + "score_unit": "score", + "source_url": "https://openai-corpws.slack.com/archives/C0B95RM2XS5/p1783576882966699?thread_ts=1783574276.164169&cid=C0B95RM2XS5", + "x_label": "$$1.14", + "x_metric": "api_cost_usd", + "x_value": 1.144912364534 + }, + { + "category": "Computer Use", + "code_mode": "true", + "eval": "BrowseComp", + "eval_id": "truffle_hunter_oss", + "eval_variant": "1 agent", + "juice_index": 2, + "juice_level": "medium", + "model": "GPT-5.6 Sol · 1 agent", + "model_slug": "gpt-5.6-sol-ma", + "score": 0.8459715639810427, + "score_label": "84.6%", + "score_metric": "", + "score_unit": "score", + "source_url": "https://openai-corpws.slack.com/archives/C0B95RM2XS5/p1783576882966699?thread_ts=1783574276.164169&cid=C0B95RM2XS5", + "x_label": "2.53 minutes", + "x_metric": "latency_s", + "x_value": 2.5334598233108 + }, + { + "category": "Computer Use", + "code_mode": "true", + "eval": "BrowseComp", + "eval_id": "truffle_hunter_oss", + "eval_variant": "1 agent", + "juice_index": 2, + "juice_level": "medium", + "model": "GPT-5.6 Sol · 1 agent", + "model_slug": "gpt-5.6-sol-ma", + "score": 0.8459715639810427, + "score_label": "84.6%", + "score_metric": "", + "score_unit": "score", + "source_url": "https://openai-corpws.slack.com/archives/C0B95RM2XS5/p1783576882966699?thread_ts=1783574276.164169&cid=C0B95RM2XS5", + "x_label": "6,095", + "x_metric": "output_tokens", + "x_value": 6094.73617693523 + }, + { + "category": "Computer Use", + "code_mode": "true", + "eval": "BrowseComp", + "eval_id": "truffle_hunter_oss", + "eval_variant": "1 agent", + "juice_index": 3, + "juice_level": "high", + "model": "GPT-5.6 Sol · 1 agent", + "model_slug": "gpt-5.6-sol-ma", + "score": 0.882306477093207, + "score_label": "88.23%", + "score_metric": "", + "score_unit": "score", + "source_url": "https://openai-corpws.slack.com/archives/C0B95RM2XS5/p1783576882966699?thread_ts=1783574276.164169&cid=C0B95RM2XS5", + "x_label": "$$2.04", + "x_metric": "api_cost_usd", + "x_value": 2.04333285466 + }, + { + "category": "Computer Use", + "code_mode": "true", + "eval": "BrowseComp", + "eval_id": "truffle_hunter_oss", + "eval_variant": "1 agent", + "juice_index": 3, + "juice_level": "high", + "model": "GPT-5.6 Sol · 1 agent", + "model_slug": "gpt-5.6-sol-ma", + "score": 0.882306477093207, + "score_label": "88.23%", + "score_metric": "", + "score_unit": "score", + "source_url": "https://openai-corpws.slack.com/archives/C0B95RM2XS5/p1783576882966699?thread_ts=1783574276.164169&cid=C0B95RM2XS5", + "x_label": "4.33 minutes", + "x_metric": "latency_s", + "x_value": 4.334339348751284 + }, + { + "category": "Computer Use", + "code_mode": "true", + "eval": "BrowseComp", + "eval_id": "truffle_hunter_oss", + "eval_variant": "1 agent", + "juice_index": 3, + "juice_level": "high", + "model": "GPT-5.6 Sol · 1 agent", + "model_slug": "gpt-5.6-sol-ma", + "score": 0.882306477093207, + "score_label": "88.23%", + "score_metric": "", + "score_unit": "score", + "source_url": "https://openai-corpws.slack.com/archives/C0B95RM2XS5/p1783576882966699?thread_ts=1783574276.164169&cid=C0B95RM2XS5", + "x_label": "10.89K", + "x_metric": "output_tokens", + "x_value": 10886.8475513428 + }, + { + "category": "Computer Use", + "code_mode": "true", + "eval": "BrowseComp", + "eval_id": "truffle_hunter_oss", + "eval_variant": "1 agent", + "juice_index": 4, + "juice_level": "xhigh", + "model": "GPT-5.6 Sol · 1 agent", + "model_slug": "gpt-5.6-sol-ma", + "score": 0.8941548183254344, + "score_label": "89.42%", + "score_metric": "", + "score_unit": "score", + "source_url": "https://openai-corpws.slack.com/archives/C0B95RM2XS5/p1783576882966699?thread_ts=1783574276.164169&cid=C0B95RM2XS5", + "x_label": "$$2.7", + "x_metric": "api_cost_usd", + "x_value": 2.697721325829 + }, + { + "category": "Computer Use", + "code_mode": "true", + "eval": "BrowseComp", + "eval_id": "truffle_hunter_oss", + "eval_variant": "1 agent", + "juice_index": 4, + "juice_level": "xhigh", + "model": "GPT-5.6 Sol · 1 agent", + "model_slug": "gpt-5.6-sol-ma", + "score": 0.8941548183254344, + "score_label": "89.42%", + "score_metric": "", + "score_unit": "score", + "source_url": "https://openai-corpws.slack.com/archives/C0B95RM2XS5/p1783576882966699?thread_ts=1783574276.164169&cid=C0B95RM2XS5", + "x_label": "5.63 minutes", + "x_metric": "latency_s", + "x_value": 5.6263142407134 + }, + { + "category": "Computer Use", + "code_mode": "true", + "eval": "BrowseComp", + "eval_id": "truffle_hunter_oss", + "eval_variant": "1 agent", + "juice_index": 4, + "juice_level": "xhigh", + "model": "GPT-5.6 Sol · 1 agent", + "model_slug": "gpt-5.6-sol-ma", + "score": 0.8941548183254344, + "score_label": "89.42%", + "score_metric": "", + "score_unit": "score", + "source_url": "https://openai-corpws.slack.com/archives/C0B95RM2XS5/p1783576882966699?thread_ts=1783574276.164169&cid=C0B95RM2XS5", + "x_label": "14.84K", + "x_metric": "output_tokens", + "x_value": 14838.7314375987 + }, + { + "category": "Computer Use", + "code_mode": "true", + "eval": "BrowseComp", + "eval_id": "truffle_hunter_oss", + "eval_variant": "1 agent", + "juice_index": 5, + "juice_level": "max", + "model": "GPT-5.6 Sol · 1 agent", + "model_slug": "gpt-5.6-sol-ma", + "score": 0.9083728278041074, + "score_label": "90.84%", + "score_metric": "", + "score_unit": "score", + "source_url": "https://openai-corpws.slack.com/archives/C0B95RM2XS5/p1783576882966699?thread_ts=1783574276.164169&cid=C0B95RM2XS5", + "x_label": "$$4.15", + "x_metric": "api_cost_usd", + "x_value": 4.153737759084 + }, + { + "category": "Computer Use", + "code_mode": "true", + "eval": "BrowseComp", + "eval_id": "truffle_hunter_oss", + "eval_variant": "1 agent", + "juice_index": 5, + "juice_level": "max", + "model": "GPT-5.6 Sol · 1 agent", + "model_slug": "gpt-5.6-sol-ma", + "score": 0.9083728278041074, + "score_label": "90.84%", + "score_metric": "", + "score_unit": "score", + "source_url": "https://openai-corpws.slack.com/archives/C0B95RM2XS5/p1783576882966699?thread_ts=1783574276.164169&cid=C0B95RM2XS5", + "x_label": "7.99 minutes", + "x_metric": "latency_s", + "x_value": 7.99102880342545 + }, + { + "category": "Computer Use", + "code_mode": "true", + "eval": "BrowseComp", + "eval_id": "truffle_hunter_oss", + "eval_variant": "1 agent", + "juice_index": 5, + "juice_level": "max", + "model": "GPT-5.6 Sol · 1 agent", + "model_slug": "gpt-5.6-sol-ma", + "score": 0.9083728278041074, + "score_label": "90.84%", + "score_metric": "", + "score_unit": "score", + "source_url": "https://openai-corpws.slack.com/archives/C0B95RM2XS5/p1783576882966699?thread_ts=1783574276.164169&cid=C0B95RM2XS5", + "x_label": "22K", + "x_metric": "output_tokens", + "x_value": 22000.9273301738 + }, + { + "category": "Computer Use", + "code_mode": "true", + "eval": "BrowseComp", + "eval_id": "truffle_hunter_oss", + "eval_variant": "4 agents", + "juice_index": 1, + "juice_level": "low", + "model": "GPT-5.6 Sol · 4 agents", + "model_slug": "gpt-5.6-sol-ma", + "score": 0.8175355450236966, + "score_label": "81.75%", + "score_metric": "", + "score_unit": "score", + "source_url": "https://openai-corpws.slack.com/archives/C0B95RM2XS5/p1783576882966699?thread_ts=1783574276.164169&cid=C0B95RM2XS5", + "x_label": "$$1.11", + "x_metric": "api_cost_usd", + "x_value": 1.109023554107 + }, + { + "category": "Computer Use", + "code_mode": "true", + "eval": "BrowseComp", + "eval_id": "truffle_hunter_oss", + "eval_variant": "4 agents", + "juice_index": 1, + "juice_level": "low", + "model": "GPT-5.6 Sol · 4 agents", + "model_slug": "gpt-5.6-sol-ma", + "score": 0.8175355450236966, + "score_label": "81.75%", + "score_metric": "", + "score_unit": "score", + "source_url": "https://openai-corpws.slack.com/archives/C0B95RM2XS5/p1783576882966699?thread_ts=1783574276.164169&cid=C0B95RM2XS5", + "x_label": "1 minutes", + "x_metric": "latency_s", + "x_value": 1.0031166317695284 + }, + { + "category": "Computer Use", + "code_mode": "true", + "eval": "BrowseComp", + "eval_id": "truffle_hunter_oss", + "eval_variant": "4 agents", + "juice_index": 1, + "juice_level": "low", + "model": "GPT-5.6 Sol · 4 agents", + "model_slug": "gpt-5.6-sol-ma", + "score": 0.8175355450236966, + "score_label": "81.75%", + "score_metric": "", + "score_unit": "score", + "source_url": "https://openai-corpws.slack.com/archives/C0B95RM2XS5/p1783576882966699?thread_ts=1783574276.164169&cid=C0B95RM2XS5", + "x_label": "7,418", + "x_metric": "output_tokens", + "x_value": 7417.793839 + }, + { + "category": "Computer Use", + "code_mode": "true", + "eval": "BrowseComp", + "eval_id": "truffle_hunter_oss", + "eval_variant": "4 agents", + "juice_index": 2, + "juice_level": "medium", + "model": "GPT-5.6 Sol · 4 agents", + "model_slug": "gpt-5.6-sol-ma", + "score": 0.891785150078989, + "score_label": "89.18%", + "score_metric": "", + "score_unit": "score", + "source_url": "https://openai-corpws.slack.com/archives/C0B95RM2XS5/p1783576882966699?thread_ts=1783574276.164169&cid=C0B95RM2XS5", + "x_label": "$$2.74", + "x_metric": "api_cost_usd", + "x_value": 2.744410505134 + }, + { + "category": "Computer Use", + "code_mode": "true", + "eval": "BrowseComp", + "eval_id": "truffle_hunter_oss", + "eval_variant": "4 agents", + "juice_index": 2, + "juice_level": "medium", + "model": "GPT-5.6 Sol · 4 agents", + "model_slug": "gpt-5.6-sol-ma", + "score": 0.891785150078989, + "score_label": "89.18%", + "score_metric": "", + "score_unit": "score", + "source_url": "https://openai-corpws.slack.com/archives/C0B95RM2XS5/p1783576882966699?thread_ts=1783574276.164169&cid=C0B95RM2XS5", + "x_label": "1.83 minutes", + "x_metric": "latency_s", + "x_value": 1.8287385864166834 + }, + { + "category": "Computer Use", + "code_mode": "true", + "eval": "BrowseComp", + "eval_id": "truffle_hunter_oss", + "eval_variant": "4 agents", + "juice_index": 2, + "juice_level": "medium", + "model": "GPT-5.6 Sol · 4 agents", + "model_slug": "gpt-5.6-sol-ma", + "score": 0.891785150078989, + "score_label": "89.18%", + "score_metric": "", + "score_unit": "score", + "source_url": "https://openai-corpws.slack.com/archives/C0B95RM2XS5/p1783576882966699?thread_ts=1783574276.164169&cid=C0B95RM2XS5", + "x_label": "14.79K", + "x_metric": "output_tokens", + "x_value": 14790.652449 + }, + { + "category": "Computer Use", + "code_mode": "true", + "eval": "BrowseComp", + "eval_id": "truffle_hunter_oss", + "eval_variant": "4 agents", + "juice_index": 3, + "juice_level": "high", + "model": "GPT-5.6 Sol · 4 agents", + "model_slug": "gpt-5.6-sol-ma", + "score": 0.9075829383886256, + "score_label": "90.76%", + "score_metric": "", + "score_unit": "score", + "source_url": "https://openai-corpws.slack.com/archives/C0B95RM2XS5/p1783576882966699?thread_ts=1783574276.164169&cid=C0B95RM2XS5", + "x_label": "$$5.04", + "x_metric": "api_cost_usd", + "x_value": 5.042890102291 + }, + { + "category": "Computer Use", + "code_mode": "true", + "eval": "BrowseComp", + "eval_id": "truffle_hunter_oss", + "eval_variant": "4 agents", + "juice_index": 3, + "juice_level": "high", + "model": "GPT-5.6 Sol · 4 agents", + "model_slug": "gpt-5.6-sol-ma", + "score": 0.9075829383886256, + "score_label": "90.76%", + "score_metric": "", + "score_unit": "score", + "source_url": "https://openai-corpws.slack.com/archives/C0B95RM2XS5/p1783576882966699?thread_ts=1783574276.164169&cid=C0B95RM2XS5", + "x_label": "2.99 minutes", + "x_metric": "latency_s", + "x_value": 2.9944808313166664 + }, + { + "category": "Computer Use", + "code_mode": "true", + "eval": "BrowseComp", + "eval_id": "truffle_hunter_oss", + "eval_variant": "4 agents", + "juice_index": 3, + "juice_level": "high", + "model": "GPT-5.6 Sol · 4 agents", + "model_slug": "gpt-5.6-sol-ma", + "score": 0.9075829383886256, + "score_label": "90.76%", + "score_metric": "", + "score_unit": "score", + "source_url": "https://openai-corpws.slack.com/archives/C0B95RM2XS5/p1783576882966699?thread_ts=1783574276.164169&cid=C0B95RM2XS5", + "x_label": "24.36K", + "x_metric": "output_tokens", + "x_value": 24354.684834 + }, + { + "category": "Computer Use", + "code_mode": "true", + "eval": "BrowseComp", + "eval_id": "truffle_hunter_oss", + "eval_variant": "4 agents", + "juice_index": 4, + "juice_level": "xhigh", + "model": "GPT-5.6 Sol · 4 agents", + "model_slug": "gpt-5.6-sol-ma", + "score": 0.9115323854660348, + "score_label": "91.15%", + "score_metric": "", + "score_unit": "score", + "source_url": "https://openai-corpws.slack.com/archives/C0B95RM2XS5/p1783576882966699?thread_ts=1783574276.164169&cid=C0B95RM2XS5", + "x_label": "$$6.98", + "x_metric": "api_cost_usd", + "x_value": 6.984249310821 + }, + { + "category": "Computer Use", + "code_mode": "true", + "eval": "BrowseComp", + "eval_id": "truffle_hunter_oss", + "eval_variant": "4 agents", + "juice_index": 4, + "juice_level": "xhigh", + "model": "GPT-5.6 Sol · 4 agents", + "model_slug": "gpt-5.6-sol-ma", + "score": 0.9115323854660348, + "score_label": "91.15%", + "score_metric": "", + "score_unit": "score", + "source_url": "https://openai-corpws.slack.com/archives/C0B95RM2XS5/p1783576882966699?thread_ts=1783574276.164169&cid=C0B95RM2XS5", + "x_label": "4.07 minutes", + "x_metric": "latency_s", + "x_value": 4.0714196387973836 + }, + { + "category": "Computer Use", + "code_mode": "true", + "eval": "BrowseComp", + "eval_id": "truffle_hunter_oss", + "eval_variant": "4 agents", + "juice_index": 4, + "juice_level": "xhigh", + "model": "GPT-5.6 Sol · 4 agents", + "model_slug": "gpt-5.6-sol-ma", + "score": 0.9115323854660348, + "score_label": "91.15%", + "score_metric": "", + "score_unit": "score", + "source_url": "https://openai-corpws.slack.com/archives/C0B95RM2XS5/p1783576882966699?thread_ts=1783574276.164169&cid=C0B95RM2XS5", + "x_label": "33.56K", + "x_metric": "output_tokens", + "x_value": 33558.408373 + }, + { + "category": "Computer Use", + "code_mode": "true", + "eval": "BrowseComp", + "eval_id": "truffle_hunter_oss", + "eval_variant": "4 agents", + "juice_index": 5, + "juice_level": "max", + "model": "GPT-5.6 Sol · 4 agents", + "model_slug": "gpt-5.6-sol-ma", + "score": 0.9218009478672986, + "score_label": "92.18%", + "score_metric": "", + "score_unit": "score", + "source_url": "https://openai-corpws.slack.com/archives/C0B95RM2XS5/p1783576882966699?thread_ts=1783574276.164169&cid=C0B95RM2XS5", + "x_label": "$$12.17", + "x_metric": "api_cost_usd", + "x_value": 12.173256025671 + }, + { + "category": "Computer Use", + "code_mode": "true", + "eval": "BrowseComp", + "eval_id": "truffle_hunter_oss", + "eval_variant": "4 agents", + "juice_index": 5, + "juice_level": "max", + "model": "GPT-5.6 Sol · 4 agents", + "model_slug": "gpt-5.6-sol-ma", + "score": 0.9218009478672986, + "score_label": "92.18%", + "score_metric": "", + "score_unit": "score", + "source_url": "https://openai-corpws.slack.com/archives/C0B95RM2XS5/p1783576882966699?thread_ts=1783574276.164169&cid=C0B95RM2XS5", + "x_label": "6.58 minutes", + "x_metric": "latency_s", + "x_value": 6.577935191698133 + }, + { + "category": "Computer Use", + "code_mode": "true", + "eval": "BrowseComp", + "eval_id": "truffle_hunter_oss", + "eval_variant": "4 agents", + "juice_index": 5, + "juice_level": "max", + "model": "GPT-5.6 Sol · 4 agents", + "model_slug": "gpt-5.6-sol-ma", + "score": 0.9218009478672986, + "score_label": "92.18%", + "score_metric": "", + "score_unit": "score", + "source_url": "https://openai-corpws.slack.com/archives/C0B95RM2XS5/p1783576882966699?thread_ts=1783574276.164169&cid=C0B95RM2XS5", + "x_label": "54.14K", + "x_metric": "output_tokens", + "x_value": 54138.374408 + }, + { + "category": "Computer Use", + "code_mode": "true", + "eval": "BrowseComp", + "eval_id": "truffle_hunter_oss", + "eval_variant": "16 agents", + "juice_index": 1, + "juice_level": "low", + "model": "GPT-5.6 Sol · 16 agents", + "model_slug": "gpt-5.6-sol-ma", + "score": 0.8641390205371248, + "score_label": "86.41%", + "score_metric": "", + "score_unit": "score", + "source_url": "https://openai-corpws.slack.com/archives/C0B95RM2XS5/p1783576882966699?thread_ts=1783574276.164169&cid=C0B95RM2XS5", + "x_label": "$$3.25", + "x_metric": "api_cost_usd", + "x_value": 3.247865451027 + }, + { + "category": "Computer Use", + "code_mode": "true", + "eval": "BrowseComp", + "eval_id": "truffle_hunter_oss", + "eval_variant": "16 agents", + "juice_index": 1, + "juice_level": "low", + "model": "GPT-5.6 Sol · 16 agents", + "model_slug": "gpt-5.6-sol-ma", + "score": 0.8641390205371248, + "score_label": "86.41%", + "score_metric": "", + "score_unit": "score", + "source_url": "https://openai-corpws.slack.com/archives/C0B95RM2XS5/p1783576882966699?thread_ts=1783574276.164169&cid=C0B95RM2XS5", + "x_label": "0.89 minutes", + "x_metric": "latency_s", + "x_value": 0.8918668368613983 + }, + { + "category": "Computer Use", + "code_mode": "true", + "eval": "BrowseComp", + "eval_id": "truffle_hunter_oss", + "eval_variant": "16 agents", + "juice_index": 1, + "juice_level": "low", + "model": "GPT-5.6 Sol · 16 agents", + "model_slug": "gpt-5.6-sol-ma", + "score": 0.8641390205371248, + "score_label": "86.41%", + "score_metric": "", + "score_unit": "score", + "source_url": "https://openai-corpws.slack.com/archives/C0B95RM2XS5/p1783576882966699?thread_ts=1783574276.164169&cid=C0B95RM2XS5", + "x_label": "22.85K", + "x_metric": "output_tokens", + "x_value": 22851.28831 + }, + { + "category": "Computer Use", + "code_mode": "true", + "eval": "BrowseComp", + "eval_id": "truffle_hunter_oss", + "eval_variant": "16 agents", + "juice_index": 2, + "juice_level": "medium", + "model": "GPT-5.6 Sol · 16 agents", + "model_slug": "gpt-5.6-sol-ma", + "score": 0.9107424960505529, + "score_label": "91.07%", + "score_metric": "", + "score_unit": "score", + "source_url": "https://openai-corpws.slack.com/archives/C0B95RM2XS5/p1783576882966699?thread_ts=1783574276.164169&cid=C0B95RM2XS5", + "x_label": "$$7.54", + "x_metric": "api_cost_usd", + "x_value": 7.543719900869 + }, + { + "category": "Computer Use", + "code_mode": "true", + "eval": "BrowseComp", + "eval_id": "truffle_hunter_oss", + "eval_variant": "16 agents", + "juice_index": 2, + "juice_level": "medium", + "model": "GPT-5.6 Sol · 16 agents", + "model_slug": "gpt-5.6-sol-ma", + "score": 0.9107424960505529, + "score_label": "91.07%", + "score_metric": "", + "score_unit": "score", + "source_url": "https://openai-corpws.slack.com/archives/C0B95RM2XS5/p1783576882966699?thread_ts=1783574276.164169&cid=C0B95RM2XS5", + "x_label": "1.63 minutes", + "x_metric": "latency_s", + "x_value": 1.6325888919769467 + }, + { + "category": "Computer Use", + "code_mode": "true", + "eval": "BrowseComp", + "eval_id": "truffle_hunter_oss", + "eval_variant": "16 agents", + "juice_index": 2, + "juice_level": "medium", + "model": "GPT-5.6 Sol · 16 agents", + "model_slug": "gpt-5.6-sol-ma", + "score": 0.9107424960505529, + "score_label": "91.07%", + "score_metric": "", + "score_unit": "score", + "source_url": "https://openai-corpws.slack.com/archives/C0B95RM2XS5/p1783576882966699?thread_ts=1783574276.164169&cid=C0B95RM2XS5", + "x_label": "45.37K", + "x_metric": "output_tokens", + "x_value": 45366.191943 + }, + { + "category": "Computer Use", + "code_mode": "true", + "eval": "BrowseComp", + "eval_id": "truffle_hunter_oss", + "eval_variant": "16 agents", + "juice_index": 3, + "juice_level": "high", + "model": "GPT-5.6 Sol · 16 agents", + "model_slug": "gpt-5.6-sol-ma", + "score": 0.9218009478672986, + "score_label": "92.18%", + "score_metric": "", + "score_unit": "score", + "source_url": "https://openai-corpws.slack.com/archives/C0B95RM2XS5/p1783576882966699?thread_ts=1783574276.164169&cid=C0B95RM2XS5", + "x_label": "$$13.42", + "x_metric": "api_cost_usd", + "x_value": 13.423979772907 + }, + { + "category": "Computer Use", + "code_mode": "true", + "eval": "BrowseComp", + "eval_id": "truffle_hunter_oss", + "eval_variant": "16 agents", + "juice_index": 3, + "juice_level": "high", + "model": "GPT-5.6 Sol · 16 agents", + "model_slug": "gpt-5.6-sol-ma", + "score": 0.9218009478672986, + "score_label": "92.18%", + "score_metric": "", + "score_unit": "score", + "source_url": "https://openai-corpws.slack.com/archives/C0B95RM2XS5/p1783576882966699?thread_ts=1783574276.164169&cid=C0B95RM2XS5", + "x_label": "2.79 minutes", + "x_metric": "latency_s", + "x_value": 2.787323899602467 + }, + { + "category": "Computer Use", + "code_mode": "true", + "eval": "BrowseComp", + "eval_id": "truffle_hunter_oss", + "eval_variant": "16 agents", + "juice_index": 3, + "juice_level": "high", + "model": "GPT-5.6 Sol · 16 agents", + "model_slug": "gpt-5.6-sol-ma", + "score": 0.9218009478672986, + "score_label": "92.18%", + "score_metric": "", + "score_unit": "score", + "source_url": "https://openai-corpws.slack.com/archives/C0B95RM2XS5/p1783576882966699?thread_ts=1783574276.164169&cid=C0B95RM2XS5", + "x_label": "71.46K", + "x_metric": "output_tokens", + "x_value": 71458.00158 + }, + { + "category": "Computer Use", + "code_mode": "true", + "eval": "BrowseComp", + "eval_id": "truffle_hunter_oss", + "eval_variant": "16 agents", + "juice_index": 4, + "juice_level": "xhigh", + "model": "GPT-5.6 Sol · 16 agents", + "model_slug": "gpt-5.6-sol-ma", + "score": 0.933649289099526, + "score_label": "93.36%", + "score_metric": "", + "score_unit": "score", + "source_url": "https://openai-corpws.slack.com/archives/C0B95RM2XS5/p1783576882966699?thread_ts=1783574276.164169&cid=C0B95RM2XS5", + "x_label": "$$18.41", + "x_metric": "api_cost_usd", + "x_value": 18.413747080569 + }, + { + "category": "Computer Use", + "code_mode": "true", + "eval": "BrowseComp", + "eval_id": "truffle_hunter_oss", + "eval_variant": "16 agents", + "juice_index": 4, + "juice_level": "xhigh", + "model": "GPT-5.6 Sol · 16 agents", + "model_slug": "gpt-5.6-sol-ma", + "score": 0.933649289099526, + "score_label": "93.36%", + "score_metric": "", + "score_unit": "score", + "source_url": "https://openai-corpws.slack.com/archives/C0B95RM2XS5/p1783576882966699?thread_ts=1783574276.164169&cid=C0B95RM2XS5", + "x_label": "4.02 minutes", + "x_metric": "latency_s", + "x_value": 4.019723311876049 + }, + { + "category": "Computer Use", + "code_mode": "true", + "eval": "BrowseComp", + "eval_id": "truffle_hunter_oss", + "eval_variant": "16 agents", + "juice_index": 4, + "juice_level": "xhigh", + "model": "GPT-5.6 Sol · 16 agents", + "model_slug": "gpt-5.6-sol-ma", + "score": 0.933649289099526, + "score_label": "93.36%", + "score_metric": "", + "score_unit": "score", + "source_url": "https://openai-corpws.slack.com/archives/C0B95RM2XS5/p1783576882966699?thread_ts=1783574276.164169&cid=C0B95RM2XS5", + "x_label": "97.73K", + "x_metric": "output_tokens", + "x_value": 97725.923381 + }, + { + "category": "Computer Use", + "code_mode": "true", + "eval": "BrowseComp", + "eval_id": "truffle_hunter_oss", + "eval_variant": "16 agents", + "juice_index": 5, + "juice_level": "max", + "model": "GPT-5.6 Sol · 16 agents", + "model_slug": "gpt-5.6-sol-ma", + "score": 0.9328593996840442, + "score_label": "93.29%", + "score_metric": "", + "score_unit": "score", + "source_url": "https://openai-corpws.slack.com/archives/C0B95RM2XS5/p1783576882966699?thread_ts=1783574276.164169&cid=C0B95RM2XS5", + "x_label": "$$30.75", + "x_metric": "api_cost_usd", + "x_value": 30.749903566746 + }, + { + "category": "Computer Use", + "code_mode": "true", + "eval": "BrowseComp", + "eval_id": "truffle_hunter_oss", + "eval_variant": "16 agents", + "juice_index": 5, + "juice_level": "max", + "model": "GPT-5.6 Sol · 16 agents", + "model_slug": "gpt-5.6-sol-ma", + "score": 0.9328593996840442, + "score_label": "93.29%", + "score_metric": "", + "score_unit": "score", + "source_url": "https://openai-corpws.slack.com/archives/C0B95RM2XS5/p1783576882966699?thread_ts=1783574276.164169&cid=C0B95RM2XS5", + "x_label": "6.69 minutes", + "x_metric": "latency_s", + "x_value": 6.692417839998717 + }, + { + "category": "Computer Use", + "code_mode": "true", + "eval": "BrowseComp", + "eval_id": "truffle_hunter_oss", + "eval_variant": "16 agents", + "juice_index": 5, + "juice_level": "max", + "model": "GPT-5.6 Sol · 16 agents", + "model_slug": "gpt-5.6-sol-ma", + "score": 0.9328593996840442, + "score_label": "93.29%", + "score_metric": "", + "score_unit": "score", + "source_url": "https://openai-corpws.slack.com/archives/C0B95RM2XS5/p1783576882966699?thread_ts=1783574276.164169&cid=C0B95RM2XS5", + "x_label": "144.15K", + "x_metric": "output_tokens", + "x_value": 144146.381517 + }, + { + "category": "Cybersecurity", + "code_mode": "true", + "eval": "SEC-Bench Pro", + "eval_id": "secbench_pro", + "eval_variant": "1 agent", + "juice_index": 1, + "juice_level": "low", + "model": "GPT-5.6 Sol · 1 agent", + "model_slug": "gpt-5.6-sol-ma", + "score": 0.17076502732240437, + "score_label": "17.08%", + "score_metric": "", + "score_unit": "score", + "source_url": "https://openai-corpws.slack.com/archives/C0B95RM2XS5/p1783576882966699?thread_ts=1783574276.164169&cid=C0B95RM2XS5", + "x_label": "$$0.46", + "x_metric": "api_cost_usd", + "x_value": 0.455842161202 + }, + { + "category": "Cybersecurity", + "code_mode": "true", + "eval": "SEC-Bench Pro", + "eval_id": "secbench_pro", + "eval_variant": "1 agent", + "juice_index": 1, + "juice_level": "low", + "model": "GPT-5.6 Sol · 1 agent", + "model_slug": "gpt-5.6-sol-ma", + "score": 0.17076502732240437, + "score_label": "17.08%", + "score_metric": "", + "score_unit": "score", + "source_url": "https://openai-corpws.slack.com/archives/C0B95RM2XS5/p1783576882966699?thread_ts=1783574276.164169&cid=C0B95RM2XS5", + "x_label": "1.58 minutes", + "x_metric": "latency_s", + "x_value": 1.5846478110722935 + }, + { + "category": "Cybersecurity", + "code_mode": "true", + "eval": "SEC-Bench Pro", + "eval_id": "secbench_pro", + "eval_variant": "1 agent", + "juice_index": 1, + "juice_level": "low", + "model": "GPT-5.6 Sol · 1 agent", + "model_slug": "gpt-5.6-sol-ma", + "score": 0.17076502732240437, + "score_label": "17.08%", + "score_metric": "", + "score_unit": "score", + "source_url": "https://openai-corpws.slack.com/archives/C0B95RM2XS5/p1783576882966699?thread_ts=1783574276.164169&cid=C0B95RM2XS5", + "x_label": "4,791", + "x_metric": "output_tokens", + "x_value": 4791.01639344262 + }, + { + "category": "Cybersecurity", + "code_mode": "true", + "eval": "SEC-Bench Pro", + "eval_id": "secbench_pro", + "eval_variant": "1 agent", + "juice_index": 2, + "juice_level": "medium", + "model": "GPT-5.6 Sol · 1 agent", + "model_slug": "gpt-5.6-sol-ma", + "score": 0.3729508196721312, + "score_label": "37.3%", + "score_metric": "", + "score_unit": "score", + "source_url": "https://openai-corpws.slack.com/archives/C0B95RM2XS5/p1783576882966699?thread_ts=1783574276.164169&cid=C0B95RM2XS5", + "x_label": "$$2.09", + "x_metric": "api_cost_usd", + "x_value": 2.092052260246 + }, + { + "category": "Cybersecurity", + "code_mode": "true", + "eval": "SEC-Bench Pro", + "eval_id": "secbench_pro", + "eval_variant": "1 agent", + "juice_index": 2, + "juice_level": "medium", + "model": "GPT-5.6 Sol · 1 agent", + "model_slug": "gpt-5.6-sol-ma", + "score": 0.3729508196721312, + "score_label": "37.3%", + "score_metric": "", + "score_unit": "score", + "source_url": "https://openai-corpws.slack.com/archives/C0B95RM2XS5/p1783576882966699?thread_ts=1783574276.164169&cid=C0B95RM2XS5", + "x_label": "4.46 minutes", + "x_metric": "latency_s", + "x_value": 4.455248633095933 + }, + { + "category": "Cybersecurity", + "code_mode": "true", + "eval": "SEC-Bench Pro", + "eval_id": "secbench_pro", + "eval_variant": "1 agent", + "juice_index": 2, + "juice_level": "medium", + "model": "GPT-5.6 Sol · 1 agent", + "model_slug": "gpt-5.6-sol-ma", + "score": 0.3729508196721312, + "score_label": "37.3%", + "score_metric": "", + "score_unit": "score", + "source_url": "https://openai-corpws.slack.com/archives/C0B95RM2XS5/p1783576882966699?thread_ts=1783574276.164169&cid=C0B95RM2XS5", + "x_label": "15.32K", + "x_metric": "output_tokens", + "x_value": 15323.8005464481 + }, + { + "category": "Cybersecurity", + "code_mode": "true", + "eval": "SEC-Bench Pro", + "eval_id": "secbench_pro", + "eval_variant": "1 agent", + "juice_index": 3, + "juice_level": "high", + "model": "GPT-5.6 Sol · 1 agent", + "model_slug": "gpt-5.6-sol-ma", + "score": 0.5314207650273224, + "score_label": "53.14%", + "score_metric": "", + "score_unit": "score", + "source_url": "https://openai-corpws.slack.com/archives/C0B95RM2XS5/p1783576882966699?thread_ts=1783574276.164169&cid=C0B95RM2XS5", + "x_label": "$$4.77", + "x_metric": "api_cost_usd", + "x_value": 4.770598243169 + }, + { + "category": "Cybersecurity", + "code_mode": "true", + "eval": "SEC-Bench Pro", + "eval_id": "secbench_pro", + "eval_variant": "1 agent", + "juice_index": 3, + "juice_level": "high", + "model": "GPT-5.6 Sol · 1 agent", + "model_slug": "gpt-5.6-sol-ma", + "score": 0.5314207650273224, + "score_label": "53.14%", + "score_metric": "", + "score_unit": "score", + "source_url": "https://openai-corpws.slack.com/archives/C0B95RM2XS5/p1783576882966699?thread_ts=1783574276.164169&cid=C0B95RM2XS5", + "x_label": "7.42 minutes", + "x_metric": "latency_s", + "x_value": 7.423437562072617 + }, + { + "category": "Cybersecurity", + "code_mode": "true", + "eval": "SEC-Bench Pro", + "eval_id": "secbench_pro", + "eval_variant": "1 agent", + "juice_index": 3, + "juice_level": "high", + "model": "GPT-5.6 Sol · 1 agent", + "model_slug": "gpt-5.6-sol-ma", + "score": 0.5314207650273224, + "score_label": "53.14%", + "score_metric": "", + "score_unit": "score", + "source_url": "https://openai-corpws.slack.com/archives/C0B95RM2XS5/p1783576882966699?thread_ts=1783574276.164169&cid=C0B95RM2XS5", + "x_label": "27.03K", + "x_metric": "output_tokens", + "x_value": 27030.0478142076 + }, + { + "category": "Cybersecurity", + "code_mode": "true", + "eval": "SEC-Bench Pro", + "eval_id": "secbench_pro", + "eval_variant": "1 agent", + "juice_index": 4, + "juice_level": "xhigh", + "model": "GPT-5.6 Sol · 1 agent", + "model_slug": "gpt-5.6-sol-ma", + "score": 0.6407103825136612, + "score_label": "64.07%", + "score_metric": "", + "score_unit": "score", + "source_url": "https://openai-corpws.slack.com/archives/C0B95RM2XS5/p1783576882966699?thread_ts=1783574276.164169&cid=C0B95RM2XS5", + "x_label": "$$10.59", + "x_metric": "api_cost_usd", + "x_value": 10.594469718579 + }, + { + "category": "Cybersecurity", + "code_mode": "true", + "eval": "SEC-Bench Pro", + "eval_id": "secbench_pro", + "eval_variant": "1 agent", + "juice_index": 4, + "juice_level": "xhigh", + "model": "GPT-5.6 Sol · 1 agent", + "model_slug": "gpt-5.6-sol-ma", + "score": 0.6407103825136612, + "score_label": "64.07%", + "score_metric": "", + "score_unit": "score", + "source_url": "https://openai-corpws.slack.com/archives/C0B95RM2XS5/p1783576882966699?thread_ts=1783574276.164169&cid=C0B95RM2XS5", + "x_label": "14.25 minutes", + "x_metric": "latency_s", + "x_value": 14.2528992291872 + }, + { + "category": "Cybersecurity", + "code_mode": "true", + "eval": "SEC-Bench Pro", + "eval_id": "secbench_pro", + "eval_variant": "1 agent", + "juice_index": 4, + "juice_level": "xhigh", + "model": "GPT-5.6 Sol · 1 agent", + "model_slug": "gpt-5.6-sol-ma", + "score": 0.6407103825136612, + "score_label": "64.07%", + "score_metric": "", + "score_unit": "score", + "source_url": "https://openai-corpws.slack.com/archives/C0B95RM2XS5/p1783576882966699?thread_ts=1783574276.164169&cid=C0B95RM2XS5", + "x_label": "52.91K", + "x_metric": "output_tokens", + "x_value": 52908.3538251366 + }, + { + "category": "Cybersecurity", + "code_mode": "true", + "eval": "SEC-Bench Pro", + "eval_id": "secbench_pro", + "eval_variant": "1 agent", + "juice_index": 5, + "juice_level": "max", + "model": "GPT-5.6 Sol · 1 agent", + "model_slug": "gpt-5.6-sol-ma", + "score": 0.7144808743169399, + "score_label": "71.45%", + "score_metric": "", + "score_unit": "score", + "source_url": "https://openai-corpws.slack.com/archives/C0B95RM2XS5/p1783576882966699?thread_ts=1783574276.164169&cid=C0B95RM2XS5", + "x_label": "$$15.47", + "x_metric": "api_cost_usd", + "x_value": 15.467434829918 + }, + { + "category": "Cybersecurity", + "code_mode": "true", + "eval": "SEC-Bench Pro", + "eval_id": "secbench_pro", + "eval_variant": "1 agent", + "juice_index": 5, + "juice_level": "max", + "model": "GPT-5.6 Sol · 1 agent", + "model_slug": "gpt-5.6-sol-ma", + "score": 0.7144808743169399, + "score_label": "71.45%", + "score_metric": "", + "score_unit": "score", + "source_url": "https://openai-corpws.slack.com/archives/C0B95RM2XS5/p1783576882966699?thread_ts=1783574276.164169&cid=C0B95RM2XS5", + "x_label": "20.57 minutes", + "x_metric": "latency_s", + "x_value": 20.570648385029497 + }, + { + "category": "Cybersecurity", + "code_mode": "true", + "eval": "SEC-Bench Pro", + "eval_id": "secbench_pro", + "eval_variant": "1 agent", + "juice_index": 5, + "juice_level": "max", + "model": "GPT-5.6 Sol · 1 agent", + "model_slug": "gpt-5.6-sol-ma", + "score": 0.7144808743169399, + "score_label": "71.45%", + "score_metric": "", + "score_unit": "score", + "source_url": "https://openai-corpws.slack.com/archives/C0B95RM2XS5/p1783576882966699?thread_ts=1783574276.164169&cid=C0B95RM2XS5", + "x_label": "76.56K", + "x_metric": "output_tokens", + "x_value": 76557.25 + }, + { + "category": "Cybersecurity", + "code_mode": "true", + "eval": "SEC-Bench Pro", + "eval_id": "secbench_pro", + "eval_variant": "4 agents", + "juice_index": 1, + "juice_level": "low", + "model": "GPT-5.6 Sol · 4 agents", + "model_slug": "gpt-5.6-sol-ma", + "score": 0.20355191256830601, + "score_label": "20.36%", + "score_metric": "", + "score_unit": "score", + "source_url": "https://openai-corpws.slack.com/archives/C0B95RM2XS5/p1783576882966699?thread_ts=1783574276.164169&cid=C0B95RM2XS5", + "x_label": "$$1.1", + "x_metric": "api_cost_usd", + "x_value": 1.097616343579 + }, + { + "category": "Cybersecurity", + "code_mode": "true", + "eval": "SEC-Bench Pro", + "eval_id": "secbench_pro", + "eval_variant": "4 agents", + "juice_index": 1, + "juice_level": "low", + "model": "GPT-5.6 Sol · 4 agents", + "model_slug": "gpt-5.6-sol-ma", + "score": 0.20355191256830601, + "score_label": "20.36%", + "score_metric": "", + "score_unit": "score", + "source_url": "https://openai-corpws.slack.com/archives/C0B95RM2XS5/p1783576882966699?thread_ts=1783574276.164169&cid=C0B95RM2XS5", + "x_label": "1.55 minutes", + "x_metric": "latency_s", + "x_value": 1.5452711294177384 + }, + { + "category": "Cybersecurity", + "code_mode": "true", + "eval": "SEC-Bench Pro", + "eval_id": "secbench_pro", + "eval_variant": "4 agents", + "juice_index": 1, + "juice_level": "low", + "model": "GPT-5.6 Sol · 4 agents", + "model_slug": "gpt-5.6-sol-ma", + "score": 0.20355191256830601, + "score_label": "20.36%", + "score_metric": "", + "score_unit": "score", + "source_url": "https://openai-corpws.slack.com/archives/C0B95RM2XS5/p1783576882966699?thread_ts=1783574276.164169&cid=C0B95RM2XS5", + "x_label": "11.06K", + "x_metric": "output_tokens", + "x_value": 11061.062842 + }, + { + "category": "Cybersecurity", + "code_mode": "true", + "eval": "SEC-Bench Pro", + "eval_id": "secbench_pro", + "eval_variant": "4 agents", + "juice_index": 2, + "juice_level": "medium", + "model": "GPT-5.6 Sol · 4 agents", + "model_slug": "gpt-5.6-sol-ma", + "score": 0.38934426229508196, + "score_label": "38.93%", + "score_metric": "", + "score_unit": "score", + "source_url": "https://openai-corpws.slack.com/archives/C0B95RM2XS5/p1783576882966699?thread_ts=1783574276.164169&cid=C0B95RM2XS5", + "x_label": "$$3.56", + "x_metric": "api_cost_usd", + "x_value": 3.557768159153 + }, + { + "category": "Cybersecurity", + "code_mode": "true", + "eval": "SEC-Bench Pro", + "eval_id": "secbench_pro", + "eval_variant": "4 agents", + "juice_index": 2, + "juice_level": "medium", + "model": "GPT-5.6 Sol · 4 agents", + "model_slug": "gpt-5.6-sol-ma", + "score": 0.38934426229508196, + "score_label": "38.93%", + "score_metric": "", + "score_unit": "score", + "source_url": "https://openai-corpws.slack.com/archives/C0B95RM2XS5/p1783576882966699?thread_ts=1783574276.164169&cid=C0B95RM2XS5", + "x_label": "3.19 minutes", + "x_metric": "latency_s", + "x_value": 3.1857971765955333 + }, + { + "category": "Cybersecurity", + "code_mode": "true", + "eval": "SEC-Bench Pro", + "eval_id": "secbench_pro", + "eval_variant": "4 agents", + "juice_index": 2, + "juice_level": "medium", + "model": "GPT-5.6 Sol · 4 agents", + "model_slug": "gpt-5.6-sol-ma", + "score": 0.38934426229508196, + "score_label": "38.93%", + "score_metric": "", + "score_unit": "score", + "source_url": "https://openai-corpws.slack.com/archives/C0B95RM2XS5/p1783576882966699?thread_ts=1783574276.164169&cid=C0B95RM2XS5", + "x_label": "29.58K", + "x_metric": "output_tokens", + "x_value": 29576.539617 + }, + { + "category": "Cybersecurity", + "code_mode": "true", + "eval": "SEC-Bench Pro", + "eval_id": "secbench_pro", + "eval_variant": "4 agents", + "juice_index": 3, + "juice_level": "high", + "model": "GPT-5.6 Sol · 4 agents", + "model_slug": "gpt-5.6-sol-ma", + "score": 0.5013661202185792, + "score_label": "50.14%", + "score_metric": "", + "score_unit": "score", + "source_url": "https://openai-corpws.slack.com/archives/C0B95RM2XS5/p1783576882966699?thread_ts=1783574276.164169&cid=C0B95RM2XS5", + "x_label": "$$8.05", + "x_metric": "api_cost_usd", + "x_value": 8.050742817623 + }, + { + "category": "Cybersecurity", + "code_mode": "true", + "eval": "SEC-Bench Pro", + "eval_id": "secbench_pro", + "eval_variant": "4 agents", + "juice_index": 3, + "juice_level": "high", + "model": "GPT-5.6 Sol · 4 agents", + "model_slug": "gpt-5.6-sol-ma", + "score": 0.5013661202185792, + "score_label": "50.14%", + "score_metric": "", + "score_unit": "score", + "source_url": "https://openai-corpws.slack.com/archives/C0B95RM2XS5/p1783576882966699?thread_ts=1783574276.164169&cid=C0B95RM2XS5", + "x_label": "5 minutes", + "x_metric": "latency_s", + "x_value": 4.99500248185535 + }, + { + "category": "Cybersecurity", + "code_mode": "true", + "eval": "SEC-Bench Pro", + "eval_id": "secbench_pro", + "eval_variant": "4 agents", + "juice_index": 3, + "juice_level": "high", + "model": "GPT-5.6 Sol · 4 agents", + "model_slug": "gpt-5.6-sol-ma", + "score": 0.5013661202185792, + "score_label": "50.14%", + "score_metric": "", + "score_unit": "score", + "source_url": "https://openai-corpws.slack.com/archives/C0B95RM2XS5/p1783576882966699?thread_ts=1783574276.164169&cid=C0B95RM2XS5", + "x_label": "54.21K", + "x_metric": "output_tokens", + "x_value": 54213.102459 + }, + { + "category": "Cybersecurity", + "code_mode": "true", + "eval": "SEC-Bench Pro", + "eval_id": "secbench_pro", + "eval_variant": "4 agents", + "juice_index": 4, + "juice_level": "xhigh", + "model": "GPT-5.6 Sol · 4 agents", + "model_slug": "gpt-5.6-sol-ma", + "score": 0.6530054644808743, + "score_label": "65.3%", + "score_metric": "", + "score_unit": "score", + "source_url": "https://openai-corpws.slack.com/archives/C0B95RM2XS5/p1783576882966699?thread_ts=1783574276.164169&cid=C0B95RM2XS5", + "x_label": "$$15.98", + "x_metric": "api_cost_usd", + "x_value": 15.983231145492 + }, + { + "category": "Cybersecurity", + "code_mode": "true", + "eval": "SEC-Bench Pro", + "eval_id": "secbench_pro", + "eval_variant": "4 agents", + "juice_index": 4, + "juice_level": "xhigh", + "model": "GPT-5.6 Sol · 4 agents", + "model_slug": "gpt-5.6-sol-ma", + "score": 0.6530054644808743, + "score_label": "65.3%", + "score_metric": "", + "score_unit": "score", + "source_url": "https://openai-corpws.slack.com/archives/C0B95RM2XS5/p1783576882966699?thread_ts=1783574276.164169&cid=C0B95RM2XS5", + "x_label": "7.43 minutes", + "x_metric": "latency_s", + "x_value": 7.426303248391166 + }, + { + "category": "Cybersecurity", + "code_mode": "true", + "eval": "SEC-Bench Pro", + "eval_id": "secbench_pro", + "eval_variant": "4 agents", + "juice_index": 4, + "juice_level": "xhigh", + "model": "GPT-5.6 Sol · 4 agents", + "model_slug": "gpt-5.6-sol-ma", + "score": 0.6530054644808743, + "score_label": "65.3%", + "score_metric": "", + "score_unit": "score", + "source_url": "https://openai-corpws.slack.com/archives/C0B95RM2XS5/p1783576882966699?thread_ts=1783574276.164169&cid=C0B95RM2XS5", + "x_label": "89.93K", + "x_metric": "output_tokens", + "x_value": 89934.051913 + }, + { + "category": "Cybersecurity", + "code_mode": "true", + "eval": "SEC-Bench Pro", + "eval_id": "secbench_pro", + "eval_variant": "4 agents", + "juice_index": 5, + "juice_level": "max", + "model": "GPT-5.6 Sol · 4 agents", + "model_slug": "gpt-5.6-sol-ma", + "score": 0.7431693989071039, + "score_label": "74.32%", + "score_metric": "", + "score_unit": "score", + "source_url": "https://openai-corpws.slack.com/archives/C0B95RM2XS5/p1783576882966699?thread_ts=1783574276.164169&cid=C0B95RM2XS5", + "x_label": "$$32.41", + "x_metric": "api_cost_usd", + "x_value": 32.413453601093 + }, + { + "category": "Cybersecurity", + "code_mode": "true", + "eval": "SEC-Bench Pro", + "eval_id": "secbench_pro", + "eval_variant": "4 agents", + "juice_index": 5, + "juice_level": "max", + "model": "GPT-5.6 Sol · 4 agents", + "model_slug": "gpt-5.6-sol-ma", + "score": 0.7431693989071039, + "score_label": "74.32%", + "score_metric": "", + "score_unit": "score", + "source_url": "https://openai-corpws.slack.com/archives/C0B95RM2XS5/p1783576882966699?thread_ts=1783574276.164169&cid=C0B95RM2XS5", + "x_label": "11.85 minutes", + "x_metric": "latency_s", + "x_value": 11.852198645430615 + }, + { + "category": "Cybersecurity", + "code_mode": "true", + "eval": "SEC-Bench Pro", + "eval_id": "secbench_pro", + "eval_variant": "4 agents", + "juice_index": 5, + "juice_level": "max", + "model": "GPT-5.6 Sol · 4 agents", + "model_slug": "gpt-5.6-sol-ma", + "score": 0.7431693989071039, + "score_label": "74.32%", + "score_metric": "", + "score_unit": "score", + "source_url": "https://openai-corpws.slack.com/archives/C0B95RM2XS5/p1783576882966699?thread_ts=1783574276.164169&cid=C0B95RM2XS5", + "x_label": "159.01K", + "x_metric": "output_tokens", + "x_value": 159006.422131 + }, + { + "category": "Cybersecurity", + "code_mode": "true", + "eval": "SEC-Bench Pro", + "eval_id": "secbench_pro", + "eval_variant": "16 agents", + "juice_index": 1, + "juice_level": "low", + "model": "GPT-5.6 Sol · 16 agents", + "model_slug": "gpt-5.6-sol-ma", + "score": 0.22404371584699453, + "score_label": "22.4%", + "score_metric": "", + "score_unit": "score", + "source_url": "https://openai-corpws.slack.com/archives/C0B95RM2XS5/p1783576882966699?thread_ts=1783574276.164169&cid=C0B95RM2XS5", + "x_label": "$$1.72", + "x_metric": "api_cost_usd", + "x_value": 1.718291409153 + }, + { + "category": "Cybersecurity", + "code_mode": "true", + "eval": "SEC-Bench Pro", + "eval_id": "secbench_pro", + "eval_variant": "16 agents", + "juice_index": 1, + "juice_level": "low", + "model": "GPT-5.6 Sol · 16 agents", + "model_slug": "gpt-5.6-sol-ma", + "score": 0.22404371584699453, + "score_label": "22.4%", + "score_metric": "", + "score_unit": "score", + "source_url": "https://openai-corpws.slack.com/archives/C0B95RM2XS5/p1783576882966699?thread_ts=1783574276.164169&cid=C0B95RM2XS5", + "x_label": "1.61 minutes", + "x_metric": "latency_s", + "x_value": 1.6147197231377899 + }, + { + "category": "Cybersecurity", + "code_mode": "true", + "eval": "SEC-Bench Pro", + "eval_id": "secbench_pro", + "eval_variant": "16 agents", + "juice_index": 1, + "juice_level": "low", + "model": "GPT-5.6 Sol · 16 agents", + "model_slug": "gpt-5.6-sol-ma", + "score": 0.22404371584699453, + "score_label": "22.4%", + "score_metric": "", + "score_unit": "score", + "source_url": "https://openai-corpws.slack.com/archives/C0B95RM2XS5/p1783576882966699?thread_ts=1783574276.164169&cid=C0B95RM2XS5", + "x_label": "17.48K", + "x_metric": "output_tokens", + "x_value": 17484.590164 + }, + { + "category": "Cybersecurity", + "code_mode": "true", + "eval": "SEC-Bench Pro", + "eval_id": "secbench_pro", + "eval_variant": "16 agents", + "juice_index": 2, + "juice_level": "medium", + "model": "GPT-5.6 Sol · 16 agents", + "model_slug": "gpt-5.6-sol-ma", + "score": 0.4562841530054645, + "score_label": "45.63%", + "score_metric": "", + "score_unit": "score", + "source_url": "https://openai-corpws.slack.com/archives/C0B95RM2XS5/p1783576882966699?thread_ts=1783574276.164169&cid=C0B95RM2XS5", + "x_label": "$$7.9", + "x_metric": "api_cost_usd", + "x_value": 7.901404681011 + }, + { + "category": "Cybersecurity", + "code_mode": "true", + "eval": "SEC-Bench Pro", + "eval_id": "secbench_pro", + "eval_variant": "16 agents", + "juice_index": 2, + "juice_level": "medium", + "model": "GPT-5.6 Sol · 16 agents", + "model_slug": "gpt-5.6-sol-ma", + "score": 0.4562841530054645, + "score_label": "45.63%", + "score_metric": "", + "score_unit": "score", + "source_url": "https://openai-corpws.slack.com/archives/C0B95RM2XS5/p1783576882966699?thread_ts=1783574276.164169&cid=C0B95RM2XS5", + "x_label": "2.87 minutes", + "x_metric": "latency_s", + "x_value": 2.8703074750363 + }, + { + "category": "Cybersecurity", + "code_mode": "true", + "eval": "SEC-Bench Pro", + "eval_id": "secbench_pro", + "eval_variant": "16 agents", + "juice_index": 2, + "juice_level": "medium", + "model": "GPT-5.6 Sol · 16 agents", + "model_slug": "gpt-5.6-sol-ma", + "score": 0.4562841530054645, + "score_label": "45.63%", + "score_metric": "", + "score_unit": "score", + "source_url": "https://openai-corpws.slack.com/archives/C0B95RM2XS5/p1783576882966699?thread_ts=1783574276.164169&cid=C0B95RM2XS5", + "x_label": "68.57K", + "x_metric": "output_tokens", + "x_value": 68568.016393 + }, + { + "category": "Cybersecurity", + "code_mode": "true", + "eval": "SEC-Bench Pro", + "eval_id": "secbench_pro", + "eval_variant": "16 agents", + "juice_index": 3, + "juice_level": "high", + "model": "GPT-5.6 Sol · 16 agents", + "model_slug": "gpt-5.6-sol-ma", + "score": 0.6147540983606558, + "score_label": "61.48%", + "score_metric": "", + "score_unit": "score", + "source_url": "https://openai-corpws.slack.com/archives/C0B95RM2XS5/p1783576882966699?thread_ts=1783574276.164169&cid=C0B95RM2XS5", + "x_label": "$$20.84", + "x_metric": "api_cost_usd", + "x_value": 20.84144789276 + }, + { + "category": "Cybersecurity", + "code_mode": "true", + "eval": "SEC-Bench Pro", + "eval_id": "secbench_pro", + "eval_variant": "16 agents", + "juice_index": 3, + "juice_level": "high", + "model": "GPT-5.6 Sol · 16 agents", + "model_slug": "gpt-5.6-sol-ma", + "score": 0.6147540983606558, + "score_label": "61.48%", + "score_metric": "", + "score_unit": "score", + "source_url": "https://openai-corpws.slack.com/archives/C0B95RM2XS5/p1783576882966699?thread_ts=1783574276.164169&cid=C0B95RM2XS5", + "x_label": "4.34 minutes", + "x_metric": "latency_s", + "x_value": 4.335489756029566 + }, + { + "category": "Cybersecurity", + "code_mode": "true", + "eval": "SEC-Bench Pro", + "eval_id": "secbench_pro", + "eval_variant": "16 agents", + "juice_index": 3, + "juice_level": "high", + "model": "GPT-5.6 Sol · 16 agents", + "model_slug": "gpt-5.6-sol-ma", + "score": 0.6147540983606558, + "score_label": "61.48%", + "score_metric": "", + "score_unit": "score", + "source_url": "https://openai-corpws.slack.com/archives/C0B95RM2XS5/p1783576882966699?thread_ts=1783574276.164169&cid=C0B95RM2XS5", + "x_label": "153.12K", + "x_metric": "output_tokens", + "x_value": 153116.976776 + }, + { + "category": "Cybersecurity", + "code_mode": "true", + "eval": "SEC-Bench Pro", + "eval_id": "secbench_pro", + "eval_variant": "16 agents", + "juice_index": 4, + "juice_level": "xhigh", + "model": "GPT-5.6 Sol · 16 agents", + "model_slug": "gpt-5.6-sol-ma", + "score": 0.6666666666666666, + "score_label": "66.67%", + "score_metric": "", + "score_unit": "score", + "source_url": "https://openai-corpws.slack.com/archives/C0B95RM2XS5/p1783576882966699?thread_ts=1783574276.164169&cid=C0B95RM2XS5", + "x_label": "$$35.87", + "x_metric": "api_cost_usd", + "x_value": 35.871672088798 + }, + { + "category": "Cybersecurity", + "code_mode": "true", + "eval": "SEC-Bench Pro", + "eval_id": "secbench_pro", + "eval_variant": "16 agents", + "juice_index": 4, + "juice_level": "xhigh", + "model": "GPT-5.6 Sol · 16 agents", + "model_slug": "gpt-5.6-sol-ma", + "score": 0.6666666666666666, + "score_label": "66.67%", + "score_metric": "", + "score_unit": "score", + "source_url": "https://openai-corpws.slack.com/archives/C0B95RM2XS5/p1783576882966699?thread_ts=1783574276.164169&cid=C0B95RM2XS5", + "x_label": "6.15 minutes", + "x_metric": "latency_s", + "x_value": 6.149322832829617 + }, + { + "category": "Cybersecurity", + "code_mode": "true", + "eval": "SEC-Bench Pro", + "eval_id": "secbench_pro", + "eval_variant": "16 agents", + "juice_index": 4, + "juice_level": "xhigh", + "model": "GPT-5.6 Sol · 16 agents", + "model_slug": "gpt-5.6-sol-ma", + "score": 0.6666666666666666, + "score_label": "66.67%", + "score_metric": "", + "score_unit": "score", + "source_url": "https://openai-corpws.slack.com/archives/C0B95RM2XS5/p1783576882966699?thread_ts=1783574276.164169&cid=C0B95RM2XS5", + "x_label": "232.58K", + "x_metric": "output_tokens", + "x_value": 232576.512295 + }, + { + "category": "Cybersecurity", + "code_mode": "true", + "eval": "SEC-Bench Pro", + "eval_id": "secbench_pro", + "eval_variant": "16 agents", + "juice_index": 5, + "juice_level": "max", + "model": "GPT-5.6 Sol · 16 agents", + "model_slug": "gpt-5.6-sol-ma", + "score": 0.7622950819672131, + "score_label": "76.23%", + "score_metric": "", + "score_unit": "score", + "source_url": "https://openai-corpws.slack.com/archives/C0B95RM2XS5/p1783576882966699?thread_ts=1783574276.164169&cid=C0B95RM2XS5", + "x_label": "$$65.86", + "x_metric": "api_cost_usd", + "x_value": 65.857745166667 + }, + { + "category": "Cybersecurity", + "code_mode": "true", + "eval": "SEC-Bench Pro", + "eval_id": "secbench_pro", + "eval_variant": "16 agents", + "juice_index": 5, + "juice_level": "max", + "model": "GPT-5.6 Sol · 16 agents", + "model_slug": "gpt-5.6-sol-ma", + "score": 0.7622950819672131, + "score_label": "76.23%", + "score_metric": "", + "score_unit": "score", + "source_url": "https://openai-corpws.slack.com/archives/C0B95RM2XS5/p1783576882966699?thread_ts=1783574276.164169&cid=C0B95RM2XS5", + "x_label": "7.97 minutes", + "x_metric": "latency_s", + "x_value": 7.968403900718267 + }, + { + "category": "Cybersecurity", + "code_mode": "true", + "eval": "SEC-Bench Pro", + "eval_id": "secbench_pro", + "eval_variant": "16 agents", + "juice_index": 5, + "juice_level": "max", + "model": "GPT-5.6 Sol · 16 agents", + "model_slug": "gpt-5.6-sol-ma", + "score": 0.7622950819672131, + "score_label": "76.23%", + "score_metric": "", + "score_unit": "score", + "source_url": "https://openai-corpws.slack.com/archives/C0B95RM2XS5/p1783576882966699?thread_ts=1783574276.164169&cid=C0B95RM2XS5", + "x_label": "357.36K", + "x_metric": "output_tokens", + "x_value": 357363.177596 + }, + { + "category": "Coding", + "code_mode": "true", + "eval": "Terminal-Bench 2.1", + "eval_id": "terminal_bench_2_1", + "eval_variant": "1 agent", + "juice_index": 1, + "juice_level": "low", + "model": "GPT-5.6 Sol · 1 agent", + "model_slug": "gpt-5.6", + "score": 0.76404494382, + "score_label": "76.4%", + "score_metric": "", + "score_unit": "accuracy", + "source_url": "https://openai-corpws.slack.com/archives/C0B95RM2XS5/p1782526942178749?thread_ts=1782423237.343579&cid=C0B95RM2XS5", + "x_label": "$$0.66", + "x_metric": "api_cost_usd", + "x_value": 0.6628902410108 + }, + { + "category": "Coding", + "code_mode": "true", + "eval": "Terminal-Bench 2.1", + "eval_id": "terminal_bench_2_1", + "eval_variant": "1 agent", + "juice_index": 1, + "juice_level": "low", + "model": "GPT-5.6 Sol · 1 agent", + "model_slug": "gpt-5.6", + "score": 0.76404494382, + "score_label": "76.4%", + "score_metric": "", + "score_unit": "accuracy", + "source_url": "https://openai-corpws.slack.com/archives/C0B95RM2XS5/p1782526942178749?thread_ts=1782423237.343579&cid=C0B95RM2XS5", + "x_label": "2.14 minutes", + "x_metric": "latency_s", + "x_value": 2.136913573614477 + }, + { + "category": "Coding", + "code_mode": "true", + "eval": "Terminal-Bench 2.1", + "eval_id": "terminal_bench_2_1", + "eval_variant": "1 agent", + "juice_index": 1, + "juice_level": "low", + "model": "GPT-5.6 Sol · 1 agent", + "model_slug": "gpt-5.6", + "score": 0.76404494382, + "score_label": "76.4%", + "score_metric": "", + "score_unit": "accuracy", + "source_url": "https://openai-corpws.slack.com/archives/C0B95RM2XS5/p1782526942178749?thread_ts=1782423237.343579&cid=C0B95RM2XS5", + "x_label": "3,828", + "x_metric": "output_tokens", + "x_value": 3828.31011236 + }, + { + "category": "Coding", + "code_mode": "true", + "eval": "Terminal-Bench 2.1", + "eval_id": "terminal_bench_2_1", + "eval_variant": "1 agent", + "juice_index": 2, + "juice_level": "medium", + "model": "GPT-5.6 Sol · 1 agent", + "model_slug": "gpt-5.6", + "score": 0.81797752809, + "score_label": "81.8%", + "score_metric": "", + "score_unit": "accuracy", + "source_url": "https://openai-corpws.slack.com/archives/C0B95RM2XS5/p1782526942178749?thread_ts=1782423237.343579&cid=C0B95RM2XS5", + "x_label": "$$0.89", + "x_metric": "api_cost_usd", + "x_value": 0.8932330932582 + }, + { + "category": "Coding", + "code_mode": "true", + "eval": "Terminal-Bench 2.1", + "eval_id": "terminal_bench_2_1", + "eval_variant": "1 agent", + "juice_index": 2, + "juice_level": "medium", + "model": "GPT-5.6 Sol · 1 agent", + "model_slug": "gpt-5.6", + "score": 0.81797752809, + "score_label": "81.8%", + "score_metric": "", + "score_unit": "accuracy", + "source_url": "https://openai-corpws.slack.com/archives/C0B95RM2XS5/p1782526942178749?thread_ts=1782423237.343579&cid=C0B95RM2XS5", + "x_label": "2.65 minutes", + "x_metric": "latency_s", + "x_value": 2.651167399244263 + }, + { + "category": "Coding", + "code_mode": "true", + "eval": "Terminal-Bench 2.1", + "eval_id": "terminal_bench_2_1", + "eval_variant": "1 agent", + "juice_index": 2, + "juice_level": "medium", + "model": "GPT-5.6 Sol · 1 agent", + "model_slug": "gpt-5.6", + "score": 0.81797752809, + "score_label": "81.8%", + "score_metric": "", + "score_unit": "accuracy", + "source_url": "https://openai-corpws.slack.com/archives/C0B95RM2XS5/p1782526942178749?thread_ts=1782423237.343579&cid=C0B95RM2XS5", + "x_label": "5,553", + "x_metric": "output_tokens", + "x_value": 5552.91460674 + }, + { + "category": "Coding", + "code_mode": "true", + "eval": "Terminal-Bench 2.1", + "eval_id": "terminal_bench_2_1", + "eval_variant": "1 agent", + "juice_index": 3, + "juice_level": "high", + "model": "GPT-5.6 Sol · 1 agent", + "model_slug": "gpt-5.6", + "score": 0.847191011236, + "score_label": "84.72%", + "score_metric": "", + "score_unit": "accuracy", + "source_url": "https://openai-corpws.slack.com/archives/C0B95RM2XS5/p1782526942178749?thread_ts=1782423237.343579&cid=C0B95RM2XS5", + "x_label": "$$1.09", + "x_metric": "api_cost_usd", + "x_value": 1.0930691702249251 + }, + { + "category": "Coding", + "code_mode": "true", + "eval": "Terminal-Bench 2.1", + "eval_id": "terminal_bench_2_1", + "eval_variant": "1 agent", + "juice_index": 3, + "juice_level": "high", + "model": "GPT-5.6 Sol · 1 agent", + "model_slug": "gpt-5.6", + "score": 0.847191011236, + "score_label": "84.72%", + "score_metric": "", + "score_unit": "accuracy", + "source_url": "https://openai-corpws.slack.com/archives/C0B95RM2XS5/p1782526942178749?thread_ts=1782423237.343579&cid=C0B95RM2XS5", + "x_label": "2.94 minutes", + "x_metric": "latency_s", + "x_value": 2.9418347448676534 + }, + { + "category": "Coding", + "code_mode": "true", + "eval": "Terminal-Bench 2.1", + "eval_id": "terminal_bench_2_1", + "eval_variant": "1 agent", + "juice_index": 3, + "juice_level": "high", + "model": "GPT-5.6 Sol · 1 agent", + "model_slug": "gpt-5.6", + "score": 0.847191011236, + "score_label": "84.72%", + "score_metric": "", + "score_unit": "accuracy", + "source_url": "https://openai-corpws.slack.com/archives/C0B95RM2XS5/p1782526942178749?thread_ts=1782423237.343579&cid=C0B95RM2XS5", + "x_label": "7,229", + "x_metric": "output_tokens", + "x_value": 7229.32359551 + }, + { + "category": "Coding", + "code_mode": "true", + "eval": "Terminal-Bench 2.1", + "eval_id": "terminal_bench_2_1", + "eval_variant": "1 agent", + "juice_index": 4, + "juice_level": "xhigh", + "model": "GPT-5.6 Sol · 1 agent", + "model_slug": "gpt-5.6", + "score": 0.849438202247, + "score_label": "84.94%", + "score_metric": "", + "score_unit": "accuracy", + "source_url": "https://openai-corpws.slack.com/archives/C0B95RM2XS5/p1782526942178749?thread_ts=1782423237.343579&cid=C0B95RM2XS5", + "x_label": "$$1.33", + "x_metric": "api_cost_usd", + "x_value": 1.33106034943805 + }, + { + "category": "Coding", + "code_mode": "true", + "eval": "Terminal-Bench 2.1", + "eval_id": "terminal_bench_2_1", + "eval_variant": "1 agent", + "juice_index": 4, + "juice_level": "xhigh", + "model": "GPT-5.6 Sol · 1 agent", + "model_slug": "gpt-5.6", + "score": 0.849438202247, + "score_label": "84.94%", + "score_metric": "", + "score_unit": "accuracy", + "source_url": "https://openai-corpws.slack.com/archives/C0B95RM2XS5/p1782526942178749?thread_ts=1782423237.343579&cid=C0B95RM2XS5", + "x_label": "3.46 minutes", + "x_metric": "latency_s", + "x_value": 3.4643631837188047 + }, + { + "category": "Coding", + "code_mode": "true", + "eval": "Terminal-Bench 2.1", + "eval_id": "terminal_bench_2_1", + "eval_variant": "1 agent", + "juice_index": 4, + "juice_level": "xhigh", + "model": "GPT-5.6 Sol · 1 agent", + "model_slug": "gpt-5.6", + "score": 0.849438202247, + "score_label": "84.94%", + "score_metric": "", + "score_unit": "accuracy", + "source_url": "https://openai-corpws.slack.com/archives/C0B95RM2XS5/p1782526942178749?thread_ts=1782423237.343579&cid=C0B95RM2XS5", + "x_label": "9,515", + "x_metric": "output_tokens", + "x_value": 9515.28539326 + }, + { + "category": "Coding", + "code_mode": "true", + "eval": "Terminal-Bench 2.1", + "eval_id": "terminal_bench_2_1", + "eval_variant": "1 agent", + "juice_index": 5, + "juice_level": "max", + "model": "GPT-5.6 Sol · 1 agent", + "model_slug": "gpt-5.6", + "score": 0.887640449438, + "score_label": "88.76%", + "score_metric": "", + "score_unit": "accuracy", + "source_url": "https://openai-corpws.slack.com/archives/C0B95RM2XS5/p1782526942178749?thread_ts=1782423237.343579&cid=C0B95RM2XS5", + "x_label": "$$1.72", + "x_metric": "api_cost_usd", + "x_value": 1.7200842764057498 + }, + { + "category": "Coding", + "code_mode": "true", + "eval": "Terminal-Bench 2.1", + "eval_id": "terminal_bench_2_1", + "eval_variant": "1 agent", + "juice_index": 5, + "juice_level": "max", + "model": "GPT-5.6 Sol · 1 agent", + "model_slug": "gpt-5.6", + "score": 0.887640449438, + "score_label": "88.76%", + "score_metric": "", + "score_unit": "accuracy", + "source_url": "https://openai-corpws.slack.com/archives/C0B95RM2XS5/p1782526942178749?thread_ts=1782423237.343579&cid=C0B95RM2XS5", + "x_label": "4.24 minutes", + "x_metric": "latency_s", + "x_value": 4.240172912070197 + }, + { + "category": "Coding", + "code_mode": "true", + "eval": "Terminal-Bench 2.1", + "eval_id": "terminal_bench_2_1", + "eval_variant": "1 agent", + "juice_index": 5, + "juice_level": "max", + "model": "GPT-5.6 Sol · 1 agent", + "model_slug": "gpt-5.6", + "score": 0.887640449438, + "score_label": "88.76%", + "score_metric": "", + "score_unit": "accuracy", + "source_url": "https://openai-corpws.slack.com/archives/C0B95RM2XS5/p1782526942178749?thread_ts=1782423237.343579&cid=C0B95RM2XS5", + "x_label": "12.8K", + "x_metric": "output_tokens", + "x_value": 12795.8044944 + }, + { + "category": "Coding", + "code_mode": "true", + "eval": "Terminal-Bench 2.1", + "eval_id": "terminal_bench_2_1", + "eval_variant": "4 agents", + "juice_index": 1, + "juice_level": "low", + "model": "GPT-5.6 Sol · 4 agents", + "model_slug": "gpt-5.6-sol-ma", + "score": 0.797752808989, + "score_label": "79.78%", + "score_metric": "", + "score_unit": "accuracy", + "source_url": "https://openai-corpws.slack.com/archives/C0B95RM2XS5/p1782526942178749?thread_ts=1782423237.343579&cid=C0B95RM2XS5", + "x_label": "$$1.44", + "x_metric": "api_cost_usd", + "x_value": 1.44022222472 + }, + { + "category": "Coding", + "code_mode": "true", + "eval": "Terminal-Bench 2.1", + "eval_id": "terminal_bench_2_1", + "eval_variant": "4 agents", + "juice_index": 1, + "juice_level": "low", + "model": "GPT-5.6 Sol · 4 agents", + "model_slug": "gpt-5.6-sol-ma", + "score": 0.797752808989, + "score_label": "79.78%", + "score_metric": "", + "score_unit": "accuracy", + "source_url": "https://openai-corpws.slack.com/archives/C0B95RM2XS5/p1782526942178749?thread_ts=1782423237.343579&cid=C0B95RM2XS5", + "x_label": "2.43 minutes", + "x_metric": "latency_s", + "x_value": 2.4298159761451856 + }, + { + "category": "Coding", + "code_mode": "true", + "eval": "Terminal-Bench 2.1", + "eval_id": "terminal_bench_2_1", + "eval_variant": "4 agents", + "juice_index": 1, + "juice_level": "low", + "model": "GPT-5.6 Sol · 4 agents", + "model_slug": "gpt-5.6-sol-ma", + "score": 0.797752808989, + "score_label": "79.78%", + "score_metric": "", + "score_unit": "accuracy", + "source_url": "https://openai-corpws.slack.com/archives/C0B95RM2XS5/p1782526942178749?thread_ts=1782423237.343579&cid=C0B95RM2XS5", + "x_label": "10.45K", + "x_metric": "output_tokens", + "x_value": 10447.9595506 + }, + { + "category": "Coding", + "code_mode": "true", + "eval": "Terminal-Bench 2.1", + "eval_id": "terminal_bench_2_1", + "eval_variant": "4 agents", + "juice_index": 2, + "juice_level": "medium", + "model": "GPT-5.6 Sol · 4 agents", + "model_slug": "gpt-5.6-sol-ma", + "score": 0.85393258427, + "score_label": "85.39%", + "score_metric": "", + "score_unit": "accuracy", + "source_url": "https://openai-corpws.slack.com/archives/C0B95RM2XS5/p1782526942178749?thread_ts=1782423237.343579&cid=C0B95RM2XS5", + "x_label": "$$2.23", + "x_metric": "api_cost_usd", + "x_value": 2.2289924764 + }, + { + "category": "Coding", + "code_mode": "true", + "eval": "Terminal-Bench 2.1", + "eval_id": "terminal_bench_2_1", + "eval_variant": "4 agents", + "juice_index": 2, + "juice_level": "medium", + "model": "GPT-5.6 Sol · 4 agents", + "model_slug": "gpt-5.6-sol-ma", + "score": 0.85393258427, + "score_label": "85.39%", + "score_metric": "", + "score_unit": "accuracy", + "source_url": "https://openai-corpws.slack.com/archives/C0B95RM2XS5/p1782526942178749?thread_ts=1782423237.343579&cid=C0B95RM2XS5", + "x_label": "2.54 minutes", + "x_metric": "latency_s", + "x_value": 2.541658441422345 + }, + { + "category": "Coding", + "code_mode": "true", + "eval": "Terminal-Bench 2.1", + "eval_id": "terminal_bench_2_1", + "eval_variant": "4 agents", + "juice_index": 2, + "juice_level": "medium", + "model": "GPT-5.6 Sol · 4 agents", + "model_slug": "gpt-5.6-sol-ma", + "score": 0.85393258427, + "score_label": "85.39%", + "score_metric": "", + "score_unit": "accuracy", + "source_url": "https://openai-corpws.slack.com/archives/C0B95RM2XS5/p1782526942178749?thread_ts=1782423237.343579&cid=C0B95RM2XS5", + "x_label": "16.39K", + "x_metric": "output_tokens", + "x_value": 16386.7213483 + }, + { + "category": "Coding", + "code_mode": "true", + "eval": "Terminal-Bench 2.1", + "eval_id": "terminal_bench_2_1", + "eval_variant": "4 agents", + "juice_index": 3, + "juice_level": "high", + "model": "GPT-5.6 Sol · 4 agents", + "model_slug": "gpt-5.6-sol-ma", + "score": 0.867415730337, + "score_label": "86.74%", + "score_metric": "", + "score_unit": "accuracy", + "source_url": "https://openai-corpws.slack.com/archives/C0B95RM2XS5/p1782526942178749?thread_ts=1782423237.343579&cid=C0B95RM2XS5", + "x_label": "$$2.92", + "x_metric": "api_cost_usd", + "x_value": 2.91810692809 + }, + { + "category": "Coding", + "code_mode": "true", + "eval": "Terminal-Bench 2.1", + "eval_id": "terminal_bench_2_1", + "eval_variant": "4 agents", + "juice_index": 3, + "juice_level": "high", + "model": "GPT-5.6 Sol · 4 agents", + "model_slug": "gpt-5.6-sol-ma", + "score": 0.867415730337, + "score_label": "86.74%", + "score_metric": "", + "score_unit": "accuracy", + "source_url": "https://openai-corpws.slack.com/archives/C0B95RM2XS5/p1782526942178749?thread_ts=1782423237.343579&cid=C0B95RM2XS5", + "x_label": "2.89 minutes", + "x_metric": "latency_s", + "x_value": 2.8921123790932315 + }, + { + "category": "Coding", + "code_mode": "true", + "eval": "Terminal-Bench 2.1", + "eval_id": "terminal_bench_2_1", + "eval_variant": "4 agents", + "juice_index": 3, + "juice_level": "high", + "model": "GPT-5.6 Sol · 4 agents", + "model_slug": "gpt-5.6-sol-ma", + "score": 0.867415730337, + "score_label": "86.74%", + "score_metric": "", + "score_unit": "accuracy", + "source_url": "https://openai-corpws.slack.com/archives/C0B95RM2XS5/p1782526942178749?thread_ts=1782423237.343579&cid=C0B95RM2XS5", + "x_label": "21.62K", + "x_metric": "output_tokens", + "x_value": 21617.5235955 + }, + { + "category": "Coding", + "code_mode": "true", + "eval": "Terminal-Bench 2.1", + "eval_id": "terminal_bench_2_1", + "eval_variant": "4 agents", + "juice_index": 4, + "juice_level": "xhigh", + "model": "GPT-5.6 Sol · 4 agents", + "model_slug": "gpt-5.6-sol-ma", + "score": 0.892134831461, + "score_label": "89.21%", + "score_metric": "", + "score_unit": "accuracy", + "source_url": "https://openai-corpws.slack.com/archives/C0B95RM2XS5/p1782526942178749?thread_ts=1782423237.343579&cid=C0B95RM2XS5", + "x_label": "$$3.64", + "x_metric": "api_cost_usd", + "x_value": 3.64038341348 + }, + { + "category": "Coding", + "code_mode": "true", + "eval": "Terminal-Bench 2.1", + "eval_id": "terminal_bench_2_1", + "eval_variant": "4 agents", + "juice_index": 4, + "juice_level": "xhigh", + "model": "GPT-5.6 Sol · 4 agents", + "model_slug": "gpt-5.6-sol-ma", + "score": 0.892134831461, + "score_label": "89.21%", + "score_metric": "", + "score_unit": "accuracy", + "source_url": "https://openai-corpws.slack.com/archives/C0B95RM2XS5/p1782526942178749?thread_ts=1782423237.343579&cid=C0B95RM2XS5", + "x_label": "3.54 minutes", + "x_metric": "latency_s", + "x_value": 3.540027139734408 + }, + { + "category": "Coding", + "code_mode": "true", + "eval": "Terminal-Bench 2.1", + "eval_id": "terminal_bench_2_1", + "eval_variant": "4 agents", + "juice_index": 4, + "juice_level": "xhigh", + "model": "GPT-5.6 Sol · 4 agents", + "model_slug": "gpt-5.6-sol-ma", + "score": 0.892134831461, + "score_label": "89.21%", + "score_metric": "", + "score_unit": "accuracy", + "source_url": "https://openai-corpws.slack.com/archives/C0B95RM2XS5/p1782526942178749?thread_ts=1782423237.343579&cid=C0B95RM2XS5", + "x_label": "27.71K", + "x_metric": "output_tokens", + "x_value": 27709.9910112 + }, + { + "category": "Coding", + "code_mode": "true", + "eval": "Terminal-Bench 2.1", + "eval_id": "terminal_bench_2_1", + "eval_variant": "4 agents", + "juice_index": 5, + "juice_level": "max", + "model": "GPT-5.6 Sol · 4 agents", + "model_slug": "gpt-5.6-sol-ma", + "score": 0.919101123596, + "score_label": "91.91%", + "score_metric": "", + "score_unit": "accuracy", + "source_url": "https://openai-corpws.slack.com/archives/C0B95RM2XS5/p1782526942178749?thread_ts=1782423237.343579&cid=C0B95RM2XS5", + "x_label": "$$5.14", + "x_metric": "api_cost_usd", + "x_value": 5.14084233483 + }, + { + "category": "Coding", + "code_mode": "true", + "eval": "Terminal-Bench 2.1", + "eval_id": "terminal_bench_2_1", + "eval_variant": "4 agents", + "juice_index": 5, + "juice_level": "max", + "model": "GPT-5.6 Sol · 4 agents", + "model_slug": "gpt-5.6-sol-ma", + "score": 0.919101123596, + "score_label": "91.91%", + "score_metric": "", + "score_unit": "accuracy", + "source_url": "https://openai-corpws.slack.com/archives/C0B95RM2XS5/p1782526942178749?thread_ts=1782423237.343579&cid=C0B95RM2XS5", + "x_label": "4.28 minutes", + "x_metric": "latency_s", + "x_value": 4.278585916301308 + }, + { + "category": "Coding", + "code_mode": "true", + "eval": "Terminal-Bench 2.1", + "eval_id": "terminal_bench_2_1", + "eval_variant": "4 agents", + "juice_index": 5, + "juice_level": "max", + "model": "GPT-5.6 Sol · 4 agents", + "model_slug": "gpt-5.6-sol-ma", + "score": 0.919101123596, + "score_label": "91.91%", + "score_metric": "", + "score_unit": "accuracy", + "source_url": "https://openai-corpws.slack.com/archives/C0B95RM2XS5/p1782526942178749?thread_ts=1782423237.343579&cid=C0B95RM2XS5", + "x_label": "39.27K", + "x_metric": "output_tokens", + "x_value": 39275.1280899 + } + ], + "schema_version": "gpt-5.6-sol-ultra.archived-chart-records.v0.1", + "selection": { + "agent_counts": [ + 1, + 4, + 16 + ], + "benchmarks": [ + "BrowseComp", + "SEC-Bench Pro", + "Terminal-Bench 2.1" + ], + "deduplication": "exact semantic JSON object; first occurrence retained", + "efforts": [ + "low", + "medium", + "high", + "xhigh", + "max" + ], + "metrics": [ + "api_cost_usd", + "latency_s", + "output_tokens" + ], + "model_identity_rule": "exact GPT-5.6 Sol · chart label" + }, + "source": { + "original_url": "https://openai.com/index/gpt-5-6/", + "wayback_capture_timestamp": "20260710150348", + "wayback_capture_url": "https://web.archive.org/web/20260710150348id_/https://openai.com/index/gpt-5-6/" + } +} diff --git a/data/gpt-5.6-sol-ultra/benchmark_dataset.v0.1.json b/data/gpt-5.6-sol-ultra/benchmark_dataset.v0.1.json new file mode 100644 index 0000000..7eb1382 --- /dev/null +++ b/data/gpt-5.6-sol-ultra/benchmark_dataset.v0.1.json @@ -0,0 +1,597 @@ +{ + "schema_version": "gpt_5_6_sol_ultra_benchmark_dataset.v0.1", + "dataset_id": "gpt-5.6-sol-ultra-public-inventory-2026-07-11", + "status": "search-bounded-draft", + "inventory": { + "scope": "search_bounded_public_inventory", + "checked_on": "2026-07-11", + "claim": "Three OpenAI-run Ultra benchmark families were found in the named public sources during the dated search window; the inventory is neither exhaustive nor universal.", + "named_sources_checked": [ + "openai_gpt_5_6_release", + "openai_gpt_5_6_sol_model_page", + "openai_codex_model_guidance", + "openai_gpt_5_6_release_wayback", + "terminal_bench_2_1_leaderboard", + "sec_bench_pro_leaderboard", + "artificial_analysis_terminal_bench_2_1", + "vals_benchmarks", + "livebench_checked_2026_07_11", + "lmarena_leaderboard", + "swe_bench_verified", + "aider_leaderboards", + "arc_prize_gpt_5_6", + "openrouter_gpt_5_6_sol", + "hugging_face_and_papers_with_code_surfaces", + "metr_gpt_5_6_sol" + ] + }, + "model_identity": { + "provider": "OpenAI", + "base_model_id": "gpt-5.6-sol", + "display_label": "GPT-5.6 Sol Ultra", + "orchestration_setting": "ultra", + "default_agent_count": 4, + "is_distinct_base_model": false, + "identity_boundary": "Ultra is an orchestration and effort setting over GPT-5.6 Sol, not an independently listed API model identity." + }, + "evidence_classes": [ + "official_provider_result", + "independent_base_model_evidence", + "independent_ultra_replication", + "archived_chart_data", + "local_execution_receipt", + "inference" + ], + "sources": [ + { + "source_id": "openai_gpt_5_6_release", + "url": "https://openai.com/index/gpt-5-6/", + "retrieved_on": "2026-07-11", + "evidence_class": "official_provider_result", + "role": "Current headline scores, Ultra definition, four-agent default, and methodology footnotes" + }, + { + "source_id": "openai_gpt_5_6_sol_model_page", + "url": "https://developers.openai.com/api/docs/models/gpt-5.6-sol", + "retrieved_on": "2026-07-11", + "evidence_class": "official_provider_result", + "role": "Underlying API model identity" + }, + { + "source_id": "openai_codex_model_guidance", + "url": "https://learn.chatgpt.com/docs/models", + "retrieved_on": "2026-07-11", + "evidence_class": "official_provider_result", + "role": "Ultra user-facing routing and automatic delegation guidance" + }, + { + "source_id": "openai_gpt_5_6_release_wayback", + "url": "https://web.archive.org/web/20260710150348/https://openai.com/index/gpt-5-6/", + "original_url": "https://openai.com/index/gpt-5-6/", + "capture_timestamp_utc": "2026-07-10T15:03:48Z", + "retrieved_on": "2026-07-11", + "evidence_class": "archived_chart_data", + "role": "Embedded chart records for apples-to-apples operational comparisons", + "content_hash": { + "algorithm": "sha256", + "value": "242b04c63bd0da245bc2dcf601801ee958deb9ebc08def9193f5c6846fdbf892", + "scope": "decompressed HTTP response entity bytes after HTTP content decoding", + "manifest_ref": "data/gpt-5.6-sol-ultra/archive_manifest.json" + } + }, + { + "source_id": "terminal_bench_2_1_leaderboard", + "url": "https://www.tbench.ai/leaderboard/terminal-bench/2.1", + "retrieved_on": "2026-07-11", + "evidence_class": "inference", + "search_finding": "negative_search_evidence_only", + "role": "Named leaderboard check; no GPT-5.6 Sol Ultra row was found" + }, + { + "source_id": "sec_bench_pro_leaderboard", + "url": "https://sec-bench.github.io/", + "retrieved_on": "2026-07-11", + "evidence_class": "inference", + "search_finding": "negative_search_evidence_only", + "role": "Named leaderboard check; no GPT-5.6 Sol Ultra row was found" + }, + { + "source_id": "artificial_analysis_terminal_bench_2_1", + "url": "https://artificialanalysis.ai/evaluations/terminalbench-v2-1", + "retrieved_on": "2026-07-11", + "evidence_class": "independent_base_model_evidence", + "search_finding": "base_model_variants_found_no_ultra_identifier", + "role": "Independent Sol effort-variant results; not an Ultra replication" + }, + { + "source_id": "vals_benchmarks", + "url": "https://www.vals.ai/benchmarks", + "retrieved_on": "2026-07-11", + "evidence_class": "independent_base_model_evidence", + "search_finding": "base_model_results_found_no_ultra_identifier", + "role": "Independent Sol benchmark results; not an Ultra replication" + }, + { + "source_id": "livebench_checked_2026_07_11", + "url": "https://livebench.ai/", + "retrieved_on": "2026-07-11", + "evidence_class": "inference", + "search_finding": "insufficient_for_durable_absence_claim", + "role": "Named search surface; current client shell did not expose a durable exact result" + }, + { + "source_id": "lmarena_leaderboard", + "url": "https://arena.ai/leaderboard", + "retrieved_on": "2026-07-11", + "evidence_class": "independent_base_model_evidence", + "search_finding": "base_model_variants_found_no_ultra_identifier", + "role": "Base-model and harness context; not an Ultra replication" + }, + { + "source_id": "swe_bench_verified", + "url": "https://www.swebench.com/", + "retrieved_on": "2026-07-11", + "evidence_class": "inference", + "search_finding": "negative_search_evidence_only", + "role": "Named leaderboard check; no Ultra identifier was found" + }, + { + "source_id": "aider_leaderboards", + "url": "https://aider.chat/docs/leaderboards/", + "retrieved_on": "2026-07-11", + "evidence_class": "inference", + "search_finding": "negative_search_evidence_only", + "role": "Named leaderboard check; no GPT-5.6 entry was found" + }, + { + "source_id": "arc_prize_gpt_5_6", + "url": "https://arcprize.org/results/openai-gpt-5-6", + "retrieved_on": "2026-07-11", + "evidence_class": "independent_base_model_evidence", + "search_finding": "base_model_effort_variants_found_no_ultra_identifier", + "role": "Independent Sol effort-variant results through Max; not an Ultra replication" + }, + { + "source_id": "openrouter_gpt_5_6_sol", + "url": "https://openrouter.ai/openai/gpt-5.6-sol-20260709", + "retrieved_on": "2026-07-11", + "evidence_class": "inference", + "search_finding": "model_directory_entry_no_ultra_replication", + "role": "Model-directory evidence, not benchmark validation" + }, + { + "source_id": "hugging_face_and_papers_with_code_surfaces", + "url": "https://huggingface.co/models", + "retrieved_on": "2026-07-11", + "evidence_class": "inference", + "search_finding": "negative_api_search_and_redirected_surface", + "role": "Exact-phrase model/dataset searches were negative; Papers With Code redirected and was rejected as an operative absence surface" + }, + { + "source_id": "metr_gpt_5_6_sol", + "url": "https://metr.org/blog/2026-06-26-gpt-5-6-sol/", + "retrieved_on": "2026-07-11", + "evidence_class": "independent_base_model_evidence", + "role": "Independent base-Sol capability evidence and benchmark-gaming caveat; not an Ultra replication" + } + ], + "headline_results": [ + { + "benchmark_id": "browsecomp", + "benchmark_name": "BrowseComp", + "configuration": { + "model": "gpt-5.6-sol", + "orchestration": "ultra", + "agent_count": 4, + "effort": { + "availability": "unavailable", + "reason": "The current headline table identifies Ultra but does not expose the archived chart's per-agent juice_level field." + } + }, + "ultra_score": { + "value": 0.922, + "unit": "proportion", + "display": "92.2%" + }, + "single_agent_score": { + "value": 0.904, + "unit": "proportion", + "display": "90.4%" + }, + "score_gain": { + "value": 1.8, + "unit": "percentage_points", + "comparison_scope": "current_headline_to_current_headline" + }, + "cost": { + "availability": "unavailable", + "unit": "usd", + "reason": "The current headline table does not publish a benchmark-level cost value." + }, + "output_tokens": { + "availability": "unavailable", + "unit": "tokens", + "reason": "The current headline table does not publish a benchmark-level output-token value." + }, + "latency": { + "availability": "unavailable", + "unit": "minutes", + "reason": "The current headline table does not publish a benchmark-level latency value." + }, + "evidence_class": "official_provider_result", + "source_id": "openai_gpt_5_6_release" + }, + { + "benchmark_id": "sec_bench_pro", + "benchmark_name": "SEC-Bench Pro", + "configuration": { + "model": "gpt-5.6-sol", + "orchestration": "ultra", + "agent_count": 4, + "effort": { + "availability": "unavailable", + "reason": "The current headline table identifies Ultra but does not expose the archived chart's per-agent juice_level field." + } + }, + "ultra_score": { + "value": 0.743, + "unit": "proportion", + "display": "74.3%" + }, + "single_agent_score": { + "value": 0.712, + "unit": "proportion", + "display": "71.2%" + }, + "score_gain": { + "value": 3.1, + "unit": "percentage_points", + "comparison_scope": "current_headline_to_current_headline" + }, + "cost": { + "availability": "unavailable", + "unit": "usd", + "reason": "The current headline table does not publish a benchmark-level cost value." + }, + "output_tokens": { + "availability": "unavailable", + "unit": "tokens", + "reason": "The current headline table does not publish a benchmark-level output-token value." + }, + "latency": { + "availability": "unavailable", + "unit": "minutes", + "reason": "The current headline table does not publish a benchmark-level latency value." + }, + "evidence_class": "official_provider_result", + "source_id": "openai_gpt_5_6_release" + }, + { + "benchmark_id": "terminal_bench_2_1", + "benchmark_name": "Terminal-Bench 2.1", + "configuration": { + "model": "gpt-5.6-sol", + "orchestration": "ultra", + "agent_count": 4, + "effort": { + "availability": "unavailable", + "reason": "The current headline table identifies Ultra but does not expose the archived chart's per-agent juice_level field." + } + }, + "ultra_score": { + "value": 0.919, + "unit": "proportion", + "display": "91.9%" + }, + "single_agent_score": { + "value": 0.888, + "unit": "proportion", + "display": "88.8%" + }, + "score_gain": { + "value": 3.1, + "unit": "percentage_points", + "comparison_scope": "current_headline_to_current_headline" + }, + "cost": { + "availability": "unavailable", + "unit": "usd", + "reason": "The current headline table does not publish a benchmark-level cost value." + }, + "output_tokens": { + "availability": "unavailable", + "unit": "tokens", + "reason": "The current headline table does not publish a benchmark-level output-token value." + }, + "latency": { + "availability": "unavailable", + "unit": "minutes", + "reason": "The current headline table does not publish a benchmark-level latency value." + }, + "evidence_class": "official_provider_result", + "source_id": "openai_gpt_5_6_release" + } + ], + "archived_max_comparisons": [ + { + "benchmark_id": "browsecomp", + "benchmark_name": "BrowseComp", + "comparison_scope": "archived_max_to_archived_max", + "evidence_class": "archived_chart_data", + "source_id": "openai_gpt_5_6_release_wayback", + "single_agent": { + "agent_count": 1, + "effort": "max", + "score": { + "value": 0.9083728278041074, + "unit": "proportion" + }, + "cost": { + "value": 4.153737759084, + "unit": "usd", + "is_provider_estimate": true + }, + "output_tokens": { + "value": 22000.9273301738, + "unit": "tokens", + "aggregation": "all_agents" + }, + "latency": { + "value": 7.99102880342545, + "unit": "minutes", + "is_provider_estimate": true, + "attribution": "root_agent" + } + }, + "ultra_four_agent": { + "agent_count": 4, + "effort": "max", + "score": { + "value": 0.9218009478672986, + "unit": "proportion" + }, + "cost": { + "value": 12.173256025671, + "unit": "usd", + "is_provider_estimate": true + }, + "output_tokens": { + "value": 54138.374408, + "unit": "tokens", + "aggregation": "all_agents" + }, + "latency": { + "value": 6.577935191698133, + "unit": "minutes", + "is_provider_estimate": true, + "attribution": "root_agent" + } + }, + "derived": { + "evidence_class": "inference", + "score_gain_percentage_points": 1.34, + "cost_multiplier": 2.93, + "output_token_multiplier": 2.46, + "latency_change_percent": -17.7, + "latency_interpretation": "faster" + }, + "reconciliation": { + "current_ultra_score": 0.922, + "archived_ultra_score": 0.9218009478672986, + "current_single_agent_score": 0.904, + "archived_single_agent_score": 0.9083728278041074, + "status": "operational_baseline_differs_do_not_mix", + "note": "Ultra agrees after rounding; the archived and current single-agent values differ, so current headline gains must not be combined with archived operational measurements." + } + }, + { + "benchmark_id": "sec_bench_pro", + "benchmark_name": "SEC-Bench Pro", + "comparison_scope": "archived_max_to_archived_max", + "evidence_class": "archived_chart_data", + "source_id": "openai_gpt_5_6_release_wayback", + "single_agent": { + "agent_count": 1, + "effort": "max", + "score": { + "value": 0.7144808743169399, + "unit": "proportion" + }, + "cost": { + "value": 15.467434829918, + "unit": "usd", + "is_provider_estimate": true + }, + "output_tokens": { + "value": 76557.25, + "unit": "tokens", + "aggregation": "all_agents" + }, + "latency": { + "value": 20.570648385029497, + "unit": "minutes", + "is_provider_estimate": true, + "attribution": "root_agent" + } + }, + "ultra_four_agent": { + "agent_count": 4, + "effort": "max", + "score": { + "value": 0.7431693989071039, + "unit": "proportion" + }, + "cost": { + "value": 32.413453601093, + "unit": "usd", + "is_provider_estimate": true + }, + "output_tokens": { + "value": 159006.422131, + "unit": "tokens", + "aggregation": "all_agents" + }, + "latency": { + "value": 11.852198645430615, + "unit": "minutes", + "is_provider_estimate": true, + "attribution": "root_agent" + } + }, + "derived": { + "evidence_class": "inference", + "score_gain_percentage_points": 2.87, + "cost_multiplier": 2.1, + "output_token_multiplier": 2.08, + "latency_change_percent": -42.4, + "latency_interpretation": "faster" + }, + "reconciliation": { + "current_ultra_score": 0.743, + "archived_ultra_score": 0.7431693989071039, + "current_single_agent_score": 0.712, + "archived_single_agent_score": 0.7144808743169399, + "status": "operational_baseline_differs_do_not_mix", + "note": "Ultra agrees after rounding; the archived and current single-agent values differ, so current headline gains must not be combined with archived operational measurements." + } + }, + { + "benchmark_id": "terminal_bench_2_1", + "benchmark_name": "Terminal-Bench 2.1", + "comparison_scope": "archived_max_to_archived_max", + "evidence_class": "archived_chart_data", + "source_id": "openai_gpt_5_6_release_wayback", + "single_agent": { + "agent_count": 1, + "effort": "max", + "score": { + "value": 0.887640449438, + "unit": "proportion" + }, + "cost": { + "value": 1.7200842764057498, + "unit": "usd", + "is_provider_estimate": true + }, + "output_tokens": { + "value": 12795.8044944, + "unit": "tokens", + "aggregation": "all_agents" + }, + "latency": { + "value": 4.240172912070197, + "unit": "minutes", + "is_provider_estimate": true, + "attribution": "root_agent" + } + }, + "ultra_four_agent": { + "agent_count": 4, + "effort": "max", + "score": { + "value": 0.919101123596, + "unit": "proportion" + }, + "cost": { + "value": 5.14084233483, + "unit": "usd", + "is_provider_estimate": true + }, + "output_tokens": { + "value": 39275.1280899, + "unit": "tokens", + "aggregation": "all_agents" + }, + "latency": { + "value": 4.278585916301308, + "unit": "minutes", + "is_provider_estimate": true, + "attribution": "root_agent" + } + }, + "derived": { + "evidence_class": "inference", + "score_gain_percentage_points": 3.15, + "cost_multiplier": 2.99, + "output_token_multiplier": 3.07, + "latency_change_percent": 0.9, + "latency_interpretation": "slower" + }, + "reconciliation": { + "current_ultra_score": 0.919, + "archived_ultra_score": 0.919101123596, + "current_single_agent_score": 0.888, + "archived_single_agent_score": 0.887640449438, + "status": "agrees_after_rounding", + "note": "Current headline and archived operational scores agree at one decimal percentage precision." + } + } + ], + "independent_verification": { + "search_evidence_class": "inference", + "target_evidence_class": "independent_ultra_replication", + "status": "not_found_in_named_sources", + "checked_on": "2026-07-11", + "claim_boundary": "No independent replication of the four-agent Ultra configuration was found in the named sources during the dated search window; this is not an exhaustive absence claim.", + "independent_base_model_evidence_is_ultra_validation": false + }, + "independent_base_model_context": [ + { + "source_url": "https://artificialanalysis.ai/evaluations/terminalbench-v2-1", + "retrieved_on": "2026-07-11", + "evidence_class": "independent_base_model_evidence", + "configuration": "gpt-5.6-sol-xhigh", + "benchmark": "Terminal-Bench 2.1", + "score": 0.895131, + "unit": "proportion", + "ultra_replication": false + }, + { + "source_url": "https://artificialanalysis.ai/evaluations/terminalbench-v2-1", + "retrieved_on": "2026-07-11", + "evidence_class": "independent_base_model_evidence", + "configuration": "gpt-5.6-sol-max", + "benchmark": "Terminal-Bench 2.1", + "score": 0.88015, + "unit": "proportion", + "ultra_replication": false + }, + { + "source_url": "https://metr.org/blog/2026-06-26-gpt-5-6-sol/", + "retrieved_on": "2026-07-11", + "evidence_class": "independent_base_model_evidence", + "configuration": "gpt-5.6-sol-final-checkpoint", + "benchmark": "METR Time Horizon 1.1", + "result_status": "not_robust", + "ultra_replication": false, + "note": "Reported estimates changed materially with the treatment of detected evaluation-rule violations; METR did not treat any estimate as robust." + } + ], + "missing_or_unavailable_fields": [ + { + "field": "independent_ultra_replication_score", + "availability": "unavailable", + "reason": "No independent four-agent Ultra replication was found in the named sources during the dated search window." + }, + { + "field": "observed_public_service_latency", + "availability": "unavailable", + "reason": "OpenAI publishes simulated latency, not an observed public-service guarantee." + }, + { + "field": "observed_public_service_cost", + "availability": "unavailable", + "reason": "OpenAI publishes estimated API cost derived from modeled production behavior." + }, + { + "field": "ultra_headline_confidence_intervals", + "availability": "unavailable", + "reason": "No confidence interval is provided in the current headline table." + }, + { + "field": "universal_absence_proof", + "availability": "not_applicable", + "reason": "The inventory is intentionally bounded to named sources and a dated search window." + } + ] +} diff --git a/docs/evaluations/2026-07-11-gpt-5.6-sol-ultra-evidence-packet.md b/docs/evaluations/2026-07-11-gpt-5.6-sol-ultra-evidence-packet.md new file mode 100644 index 0000000..041c435 --- /dev/null +++ b/docs/evaluations/2026-07-11-gpt-5.6-sol-ultra-evidence-packet.md @@ -0,0 +1,213 @@ +# GPT-5.6 Sol Ultra evidence packet — 2026-07-11 + +## Status and claim boundary + +This packet is a **search-bounded public inventory** checked on 2026-07-11. It is +not a complete universal record, an independent validation of Ultra, or a +production-service guarantee. + +Three OpenAI-run Ultra benchmark families were found in the named public sources. +No independent replication of the four-agent Ultra configuration was found in +those sources during the dated search window. Single-agent GPT-5.6 Sol evidence +is retained as context and is not classified as Ultra validation. + +The durable finding is: + +> Ultra buys modest score improvements with roughly 2–3× aggregate compute. Its +> value depends on whether parallelism materially reduces elapsed time or changes +> the outcome. + +In that statement, “aggregate compute” is routing shorthand, not a measured FLOP +count. The public chart directly exposes aggregate output tokens and +provider-estimated API cost; those are the packet's compute proxies. + +## Repository placement + +This packet lives in `hummbl-dev/model-routing-as-code` because that repository's +declared scope explicitly covers versioned model-selection rules, evaluation +thresholds, schemas, fixtures, and auditable routing receipts. The operational +`founder-mode` repository was inspected but not modified: promoting this +search-bounded evidence directly into its runtime policy would overstate a +provisional, unvalidated route rule. This placement keeps the implementation +executable and reviewable while preserving the repository's existing non-canon +boundary. + +## Identity and naming + +`GPT-5.6 Sol Ultra` is represented here as the `ultra` orchestration setting over +the API model `gpt-5.6-sol`, not as a separate base-model identity. OpenAI's +release describes an Ultra run as coordinating four agents by default, while the +API model catalog lists the underlying model as `gpt-5.6-sol`. The Codex model +guidance describes Ultra as maximum reasoning with automatic task delegation. + +This distinction prevents results for Sol, Sol Max, Sol xhigh, Ultra, and +experimental agent counts from being silently combined. + +## Evidence classes + +Every material record uses one of these classes: + +| Class | Meaning in this packet | +| --- | --- | +| `official_provider_result` | A current result or configuration statement published by OpenAI. | +| `archived_chart_data` | A raw chart record extracted from a dated archive of OpenAI's release page. | +| `independent_base_model_evidence` | Third-party evidence about GPT-5.6 Sol that did not reproduce the four-agent Ultra setting. | +| `independent_ultra_replication` | A third party independently reproducing the four-agent Ultra configuration. None was found in the named sources. | +| `local_execution_receipt` | Observed facts about the repository work performed for this packet. | +| `inference` | A transparent calculation or interpretation derived from cited records. | + +## Current official headline results + +The current OpenAI release page reports the following four-agent Ultra results. +This table compares current headline values only. + +| Benchmark | Sol | Sol Ultra | Gain | +| --- | ---: | ---: | ---: | +| BrowseComp | 90.4% | 92.2% | +1.8 percentage points | +| SEC-Bench Pro | 71.2% | 74.3% | +3.1 percentage points | +| Terminal-Bench 2.1 | 88.8% | 91.9% | +3.1 percentage points | + +These are provider-run evaluations. They do not establish independent +replicability, general autonomous reliability, or a guaranteed service-level +performance improvement. + +## Archived max-to-max economics + +The operational comparison below uses the archived chart's one-agent `max` and +four-agent `max` records on both sides. It does not mix current headline +baselines with archived operational measurements. + +| Benchmark | Score gain | Cost multiplier | Output-token multiplier | Root-agent latency change | +| --- | ---: | ---: | ---: | ---: | +| BrowseComp | +1.34 pp | 2.93× | 2.46× | 17.7% faster | +| SEC-Bench Pro | +2.87 pp | 2.10× | 2.08× | 42.4% faster | +| Terminal-Bench 2.1 | +3.15 pp | 2.99× | 3.07× | 0.9% slower | + +The unrounded selected records are preserved in +`data/gpt-5.6-sol-ultra/archived_chart_records.json`; the calculations and their +rounded display values are also recorded in +`data/gpt-5.6-sol-ultra/benchmark_dataset.v0.1.json`. + +OpenAI attributes latency to the root agent and aggregates cost and output tokens +across all agents. The provider describes latency and cost as simulated estimates: +latency models the fast API, while cost uses regular API pricing. They are not +observed public-service guarantees. The chart data's internal metric key is +`latency_s`, but its displayed label and axis values identify minutes. This +packet preserves that conflict and interprets the displayed values as minutes; +it does not silently convert seconds to minutes. + +### Current-versus-archived reconciliation + +- BrowseComp: archived Ultra rounds to the current 92.2%, but the archived + one-agent 90.84% does not equal the current 90.4% headline baseline. +- SEC-Bench Pro: archived Ultra rounds to the current 74.3%, but the archived + one-agent 71.45% does not equal the current 71.2% headline baseline. +- Terminal-Bench 2.1: both archived values agree with the current headline values + at one-decimal percentage precision. + +Consequently, the +1.34, +2.87, and +3.15 percentage-point gains are derived only +from internally consistent archived max-to-max pairs. The +1.8, +3.1, and +3.1 +headline gains remain a separate current-to-current series. + +## Method and provenance + +1. Resolve the model/configuration identity before collecting scores. +2. Record the current OpenAI release table separately from archived chart data. +3. Retrieve the 2026-07-10T15:03:48Z Wayback capture of the OpenAI release page. +4. Hash the retrieved archive and extract matching one-agent and four-agent `max` + chart records deterministically. +5. Preserve the selected raw records, labels, units, and the `latency_s` naming + conflict without normalizing away contradictory source metadata. +6. Compute score-point gains and cost/token/latency ratios only within each + archived benchmark pair. +7. Search the named third-party surfaces for an independently reproduced + four-agent Ultra configuration; classify ordinary Sol results separately. +8. Validate the dataset, extraction output, receipt schema, fixtures, and this + packet locally. No GitHub Actions minutes are required. + +The reproducible extraction entry point is +`tools/extract_gpt56_ultra_archive.py`. Its archive timestamp, retrieval URL, +content hash, unit interpretation, and selected-record digest live in +`data/gpt-5.6-sol-ultra/archive_manifest.json`. + +## Sources checked + +Primary source records: + +- [OpenAI GPT-5.6 release](https://openai.com/index/gpt-5-6/) +- [OpenAI GPT-5.6 Sol API model](https://developers.openai.com/api/docs/models/gpt-5.6-sol) +- [OpenAI Codex model guidance](https://learn.chatgpt.com/docs/models) +- [Archived OpenAI release capture, 2026-07-10T15:03:48Z](https://web.archive.org/web/20260710150348/https://openai.com/index/gpt-5-6/) + +The dated independent-replication search also named Terminal-Bench 2.1, +SEC-Bench Pro, Artificial Analysis, Vals, LiveBench, LMArena, SWE-bench, Aider, +ARC Prize, OpenRouter, Hugging Face/Papers with Code surfaces, and METR. Presence +or absence can change after the search date; this packet does not claim to prove +absence outside that bounded set. + +## Independent evidence status + +No independent replication of the four-agent Ultra configuration was found in +the named sources during the 2026-07-11 search window. + +Artificial Analysis and other ordinary model leaderboards can provide +`independent_base_model_evidence`, but a Sol xhigh or Sol Max row is not an +`independent_ultra_replication`. METR's evaluation also concerns a Sol checkpoint, +not four-agent Ultra. METR reported that its time-horizon estimate was not robust +to the treatment of detected evaluation-rule violations. That caveat is relevant +counter-evidence to broad autonomy claims, but it does not directly estimate +Ultra's four-agent performance. + +## Interpretation and provisional routing implication + +The strongest observed benefit is benchmark-specific parallel wall-clock progress, +not compute efficiency. SEC-Bench Pro combines the largest archived latency +reduction with roughly twice the aggregate cost and tokens. Terminal-Bench 2.1 is +the counterexample: about triple the cost and tokens produced a modest score gain +and no latency advantage. + +This supports a provisional route rule—not universal canon: + +- escalate ordinary work through medium → high → xhigh; +- prefer Max for difficult, tightly coupled work; +- consider Ultra for decomposable, consequential work where independent lanes can + improve coverage, elapsed time, or the outcome enough to justify duplication + and synthesis overhead. + +The local route policy and the future controlled-comparison protocol are separate +artifacts. Adoption requires local empirical evidence and authorized review. + +## Limitations + +- The three Ultra benchmark families are OpenAI-run; no independent Ultra + reproduction was found in the bounded search. +- Provider cost and latency are simulations, not observed public API telemetry. +- Confidence intervals and run-level variance are unavailable for the headline + table. +- Benchmark tasks may be unusually decomposable and may not transfer to local + operational work. +- The archived page can change or disappear; this packet preserves a hash and + selected records, not the archive service itself. +- The raw provider records repeat two inaccessible internal OpenAI Slack URLs + exposed by the public archive. No credentials were detected. They are retained + locally for byte-level provenance, but their inclusion is an explicit + provenance/privacy review item before public publication. +- The internal `latency_s` field conflicts with the display unit. Minutes are an + evidenced interpretation of the chart labels, not a corrected source field. +- METR's rule-violation caveat is base-model evidence and cannot be mechanically + projected onto Ultra. +- This exact local Ultra run exposes delegation state and repository outcomes, but + not model-token, model-cost, or authoritative route-level latency telemetry. + +## Packet artifacts + +- Machine dataset: `data/gpt-5.6-sol-ultra/benchmark_dataset.v0.1.json` +- Archive manifest: `data/gpt-5.6-sol-ultra/archive_manifest.json` +- Selected raw records: `data/gpt-5.6-sol-ultra/archived_chart_records.json` +- Extractor: `tools/extract_gpt56_ultra_archive.py` +- Receipt contract: `schemas/ultra_evaluation_receipt.v0.1.schema.json` +- Local run receipt: `receipts/2026-07-11-gpt-5.6-sol-ultra-run.json` +- Claim verification: `receipts/2026-07-11-gpt-5.6-sol-ultra-claim-verification.md` +- Publication content review: `receipts/2026-07-11-gpt-5.6-sol-ultra-content-review.md` +- Route policy: `docs/route-policy/reasoning-effort-route-policy.v0.1.md` +- Future protocol: `docs/evaluations/ultra-evaluation-protocol.v0.1.md` diff --git a/docs/evaluations/ultra-evaluation-protocol.v0.1.md b/docs/evaluations/ultra-evaluation-protocol.v0.1.md new file mode 100644 index 0000000..f2bd9a9 --- /dev/null +++ b/docs/evaluations/ultra-evaluation-protocol.v0.1.md @@ -0,0 +1,180 @@ +# Governed xhigh–Max–Ultra comparison protocol v0.1 + +## Status + +This protocol is a prospective experiment design, not evidence that any route is +superior. It compares GPT-5.6 Sol xhigh, Max, and Ultra on each identical task +while holding the harness, success criteria, and evidence standard fixed. + +## Research question + +For consequential tasks representative of local work, when does four-agent Ultra +produce enough marginal correctness, coverage, or elapsed-time benefit over +xhigh or Max to justify aggregate token use, estimated cost, duplication, and +synthesis overhead? + +## Invariants + +For each task, freeze before the first run: + +- the exact task prompt and all attachments; +- repository commit, dependency state, permissions, and network policy; +- tool availability and external-source snapshot where practical; +- wall-clock deadline and a preregistered resource-budget regime; +- deterministic harness and validation commands; +- success criteria, scoring rubric, evidence standard, and failure taxonomy; +- allowed clarification policy and stopping conditions; +- receipt schema version; +- the identity of any blinded human adjudicators. + +Do not revise the task, harness, success criteria, or evidence standard between +routes. If a defect requires a change, invalidate the affected block and rerun all +three routes under a new protocol revision. + +## Task set + +Use a preregistered set containing both task shapes: + +1. **Tightly coupled tasks**: debugging, architecture, or proof work where one + decision constrains most later decisions. +2. **Decomposable tasks**: multi-source research plus implementation and review, + with lane boundaries that do not require overlapping writes. + +Tasks must be consequential enough to discriminate routes, locally reproducible, +free of secrets in published fixtures, and not selected after observing model +performance. Include negative controls where medium/high should already suffice +to detect needless escalation. + +## Experimental design + +1. Run xhigh, Max, and Ultra in separate clean worktrees from the same frozen + commit. +2. Randomize route order within each task. Counterbalance order across tasks to + reduce learning, cache, and temporal effects. +3. Use at least three independent repetitions per task/route when budget permits; + preregister any smaller pilot as exploratory. +4. Prevent one route from seeing another route's output until adjudication is + complete. +5. Give all routes the same tool, network, time, and source-access envelope. Ultra + alone may delegate because delegation is the treatment under test. +6. Preserve raw outputs, diffs, validation logs, and a receipt for every run. +7. Perform all validation locally. **No GitHub Actions** may be invoked while the + organization has no remaining Actions minutes. + +### Resource-budget regimes + +Do not hide a budget choice inside “same harness.” Preregister one of these +regimes for an entire comparison block: + +1. **Route-native capacity**: give every arm the same wall-clock deadline and + safety policy, set aggregate spend/token ceilings high enough that they are + expected not to bind, and measure actual aggregate use. This estimates the + product routes as offered. +2. **Equal aggregate budget**: give every arm the same total dollar/token ceiling. + This estimates outcome per constrained budget, but may prevent Ultra from + expressing its default four-agent capacity. + +Never give each Ultra agent the full single-agent budget while describing the +result as equal-budget. Record both aggregate and per-agent ceilings, identify any +run where a ceiling bound, and analyze the two regimes separately. A bound primary +run is a protocol deviation and must be rerun or reported as invalid. + +## Route configurations + +| Arm | Base model | Reasoning/orchestration | Delegation | +| --- | --- | --- | --- | +| xhigh | `gpt-5.6-sol` | xhigh | none unless the product intrinsically adds it, in which case record and invalidate route comparability if unequal | +| Max | `gpt-5.6-sol` | Max | no four-agent Ultra orchestration | +| Ultra | `gpt-5.6-sol` | Ultra | four agents by default; record actual lanes and count | + +Record product version and configuration identifiers. Do not infer them from a +display label when the runtime exposes authoritative metadata. + +## Measurements + +### Primary outcomes + +- task success against the frozen rubric; +- critical correctness failures; +- evidence/receipt completeness and provenance quality; +- elapsed wall-clock time to a validated deliverable. + +### Secondary outcomes + +- aggregate model cost and input/output tokens, only when exposed; +- validation pass rate and number of repair cycles; +- unique useful findings per lane; +- duplicated work, measured as repeated investigation or overlapping artifacts; +- synthesis loss, measured as valid lane findings omitted, distorted, or left + unresolved in the final result; +- human review time; +- unsafe or out-of-scope actions; +- route-selection confidence and counterfactual judgment. + +Unavailable telemetry must be encoded as unavailable with a reason. Do not impute +tokens, cost, latency, duplication, or outcome deltas. + +## Scoring and adjudication + +Before execution, assign weights only if a composite score is necessary. Always +retain the component measures so a composite cannot hide safety or correctness +failures. + +Use blinded adjudication where feasible. Two reviewers independently score the +deliverable; resolve disagreements with a documented third review or consensus +rule. Automated tests count as evidence for the behavior they cover, not for +unmeasured document or policy quality. + +A route is dominated for a task class when another route meets or exceeds its +success/evidence threshold with lower median cost and no material wall-clock or +safety disadvantage. Ultra is justified only when its marginal outcome or time +benefit crosses a preregistered value threshold. + +## Ultra lane governance + +Before an Ultra run, record each lane's scope, assigned agent, anticipated artifact, +and non-overlapping write boundary. At completion, record final state, findings, +artifacts, contribution, duplication, rejected or superseded work, and unresolved +limitations. The root agent owns reconciliation and must record synthesis loss. + +Cosmetic lanes count as a protocol failure, not evidence of successful +orchestration. + +## Validation receipt + +Each run must validate against +`schemas/ultra_evaluation_receipt.v0.1.schema.json` or a route-neutral successor. +Attach: + +- route configuration and counterfactual; +- frozen task/harness identifiers and content hashes; +- local test and lint command results; +- outcome rubric and adjudication result; +- actual telemetry or explicit unavailable values; +- lane receipt for Ultra; +- deviations, incidents, and invalidation status. + +## Analysis and decision rule + +Report per-task results and distributions; do not rely only on an aggregate mean. +Separate tightly coupled from decomposable tasks. Compare: + +- marginal success and correctness improvement per dollar; +- marginal elapsed-time improvement per dollar; +- receipt/evidence quality; +- duplication and synthesis loss; +- route failures and variance. + +Adopt no universal route from a single run. Promote a provisional route rule only +after repeated local results show a stable benefit within a task class and an +authorized reviewer approves the policy change. Preserve contrary cases, especially +when Ultra consumes more compute without reducing elapsed time or changing the +validated outcome. + +## Pilot exit criteria + +The pilot is complete when all preregistered task/route cells have valid receipts, +all deviations are resolved or marked invalid, adjudication is complete, and the +decision report includes both positive and negative Ultra cases. If telemetry is +unavailable for a primary decision variable, report the experiment as +inconclusive rather than substituting an estimate. diff --git a/docs/route-policy/reasoning-effort-route-policy.v0.1.md b/docs/route-policy/reasoning-effort-route-policy.v0.1.md new file mode 100644 index 0000000..c93d4fb --- /dev/null +++ b/docs/route-policy/reasoning-effort-route-policy.v0.1.md @@ -0,0 +1,112 @@ +# Reasoning-effort route policy v0.1 + +## Non-canon status + +This is a provisional, reviewable route hypothesis for `model-routing-as-code`. +It is not universal policy, a production default, or adopted HUMMBL canon. Local +controlled evaluations must precede any operational adoption. + +## Purpose + +Choose the least-expensive route that can satisfy the task's success, safety, +evidence, and time requirements. A higher reasoning setting is justified by +expected outcome improvement, not task prestige or model naming. + +## Route ladder + +| Route | Select when | Reject or escalate when | +| --- | --- | --- | +| medium | The task is routine, bounded, reversible, and locally verifiable. | Important ambiguity, repeated validation failure, or consequential tradeoffs remain. | +| high | The task needs deeper analysis or moderate implementation judgment but remains bounded. | Cross-domain evidence, difficult debugging, or unresolved correctness risk persists. | +| xhigh | The task is hard and benefits from sustained single-context reasoning, with a credible local validation path. | The work is tightly coupled enough to need Max, or decomposes into genuinely independent consequential lanes that may justify Ultra. | +| Max | The task is difficult, tightly coupled, and synthesis depends on one coherent line of reasoning. | The route cannot meet a hard deadline, or independent parallel lanes are likely to change coverage or outcome enough to justify Ultra. | +| Ultra | The task is decomposable, high-value, and independent lanes can materially improve coverage, elapsed time, or outcome. | Decomposition is cosmetic, telemetry/governance is unavailable, synthesis risk dominates, or the same result is likely from xhigh/Max. | + +The ordinary escalation order is medium → high → xhigh. Max and Ultra are +different routing branches, not simply two more rungs: Max favors coherent depth; +Ultra buys governed parallelism. + +## Minimum gates for Ultra + +Select Ultra only when all of these conditions are documented before execution: + +1. The task has at least two meaningfully independent lanes with distinct evidence + or implementation boundaries. +2. The outcome is consequential enough that marginal correctness, coverage, or + elapsed-time improvement could justify additional aggregate token use and + estimated cost. +3. Each lane has a scope, owner, expected artifact, completion state, and + reconciliation plan. +4. One root owner is accountable for conflict resolution, validation, and final + synthesis. +5. The run can produce an `ultra_evaluation_receipt.v0.1`, explicitly marking + unavailable telemetry instead of estimating it. +6. A cheaper counterfactual route—normally xhigh or Max—is stated in advance. + +## Reject Ultra + +Reject Ultra and use another route when any of the following holds: + +- The work is sequential or tightly coupled, so parallel agents mostly wait on or + duplicate one another. +- The task is small, low-value, reversible, or already has an obvious deterministic + solution. +- Agents would edit overlapping files without a safe ownership boundary. +- Additional lanes cannot access distinct evidence or perform distinct validation. +- The expected synthesis burden is greater than the expected discovery benefit. +- Required data cannot be shared across lanes safely or within policy. +- A complete receipt cannot be produced, or unavailable telemetry would be + replaced with guesses. +- Aggregate token or spend capacity is constrained and no plausible wall-clock or + outcome benefit compensates for it. + +Terminal-Bench 2.1 is the motivating failure case in the dated public packet: +four-agent Ultra used about 2.99× estimated cost and 3.07× output tokens while +root-agent latency was 0.9% slower than the archived one-agent Max comparison. +That is benchmark evidence, not a universal forecast, but it makes the rejection +gate concrete. + +## Decision procedure + +1. Define the task, fixed success criteria, safety constraints, and required + evidence. +2. Assess consequence, reversibility, ambiguity, coupling, and decomposability. +3. Start at the lowest plausible route in medium → high → xhigh unless prior + governed evidence supports a higher initial route. +4. For tightly coupled difficult work, compare xhigh with Max. +5. For decomposable difficult work, compare the best single-agent route with Ultra + using expected outcome change, wall-clock value, aggregate cost, duplication, + and synthesis risk. +6. Record the chosen route, rejected routes, counterfactual route, and uncertainty. +7. After execution, record observed outcome and whether escalation changed it. + +## Required route receipt + +An Ultra receipt must record: + +- model identity and orchestration setting; +- task and decomposition rationale; +- every lane's owner, scope, state, findings, artifacts, contribution, duplication, + rejected findings, and limitations; +- evidence classes; +- aggregate cost, tokens, and elapsed time when actually exposed; +- explicit `unavailable` states otherwise; +- validation performed, duplicate work, and synthesis loss; +- outcome improvement or its unavailability; +- selected route and counterfactual route. + +The receipt supports later evaluation; it does not itself prove that Ultra was the +correct route. + +## Current evidence basis + +The 2026-07-11 search-bounded public inventory found three OpenAI-run Ultra +benchmark families and no independent replication of the four-agent setting in +the named sources. Archived max-to-max comparisons show +1.34 to +3.15 percentage +points with approximately 2.1–3.0× estimated cost and 2.1–3.1× output tokens. +Latency varied from 42.4% faster to 0.9% slower. + +This supports experimentation with the route doctrine; it does not justify +canonization. The controlled protocol in +`docs/evaluations/ultra-evaluation-protocol.v0.1.md` defines how to gather the +missing local evidence. diff --git a/docs/routing-policy-v0.1.md b/docs/routing-policy-v0.1.md new file mode 100644 index 0000000..02d20a6 --- /dev/null +++ b/docs/routing-policy-v0.1.md @@ -0,0 +1,298 @@ +# Whole-Codebase and Corpus Model Routing Policy — Draft v0.1 + +## Status + +**CANDIDATE ROUTING POLICY — EVIDENCE REQUIRED — NO DEFAULT ROUTE CHANGE AUTHORIZED** + +## Governing doctrine + +> Cheapest correct model wins only after correctness, safety, privacy, +> authority, latency, reliability, reproducibility, and governance gates +> pass. + +A larger context window does not itself justify routing. + +--- + +## 1. Workload classes + +| Class | Description | +|------|-------------| +| `REPO_INVENTORY` | Catalog files, modules, dependencies | +| `REPO_ARCHITECTURE_RECONSTRUCTION` | Reconstruct architecture from code | +| `CROSS_FILE_TRACE` | Trace data/control flow across files | +| `CLAIM_AND_REFERENCE_AUDIT` | Verify claims and references | +| `ISSUE_ROOT_CAUSE` | Find root cause of an issue | +| `BOUNDED_PATCH_IMPLEMENTATION` | Implement a bounded patch | +| `INDEPENDENT_CODE_REVIEW` | Review code independently | +| `BULK_SOURCE_INGESTION` | Ingest large source corpus | +| `MULTI_DOCUMENT_SYNTHESIS` | Synthesize across documents | +| `CONTRADICTION_AND_VERSION_ANALYSIS` | Find contradictions and version drift | +| `OCR_HISTORICAL_CORPUS` | OCR and analyze historical documents | +| `MULTIMODAL_SOURCE_PACKET` | Process multimodal sources | +| `FINAL_SCHOLARLY_CLAIM_REVIEW` | Final review of scholarly claims | + +--- + +## 2. Route dimensions + +Each route must consider: + +| Dimension | Description | +|-----------|-------------| +| `source_size_tokens` | Estimated token count | +| `source_type` | Text, code, image, multimodal | +| `source_modality` | File, API, database, web | +| `private_public` | Private or public classification | +| `required_tool_surface` | Tools required (file_search, code_exec, etc.) | +| `single_context_feasibility` | Can it fit in one context window? | +| `retrieval_memory_requirement` | Retrieval/memory needed | +| `reasoning_depth` | shallow | moderate | deep | +| `citation_precision` | low | medium | high | +| `implementation_authority` | none | propose | implement | +| `latency_target` | Target latency | +| `cost_ceiling` | Maximum cost | +| `reliability_target` | Target reliability | +| `independent_review` | Required or not | +| `provider_constraints` | Approved providers | +| `regional_constraints` | Approved regions | +| `model_availability` | GA, preview, deprecated | +| `benchmark_evidence_date` | Date of benchmark evidence | + +--- + +## 3. Route dispositions + +| Disposition | Description | +|-------------|-------------| +| `REJECTED` | Model is rejected for this workload | +| `DISCOVERY_ONLY` | Model may be used for discovery only | +| `BULK_EXTRACTOR` | Model may extract bulk content | +| `BOUNDED_READER` | Model may read bounded sections | +| `PRIMARY_READER_WITH_REVIEW` | Model may be primary reader with review | +| `IMPLEMENTER_WITH_GATES` | Model may implement with gates | +| `INDEPENDENT_REVIEWER` | Model may serve as independent reviewer | +| `FINAL_SYNTHESIS_WITH_HUMAN_REVIEW` | Model may synthesize with human review | +| `PROMOTED_WITH_SCOPE` | Model is promoted with explicit scope | + +No model should be labeled universally best. + +--- + +## 4. Hard gates + +A route must pass ALL hard gates before soft ranking: + +| Gate | Description | +|------|-------------| +| `correctness` | Model can produce correct results for this workload | +| `safety` | Model meets safety requirements | +| `privacy` | Model meets privacy requirements (private data stays in approved boundary) | +| `authority` | Model has authority for the requested action | +| `latency` | Model meets latency target | +| `reliability` | Model meets reliability target | +| `reproducibility` | Model results are reproducible | +| `governance` | Model meets governance requirements | +| `model_version_verified` | Model version is verified and not stale | +| `budget_metadata` | Budget metadata is present | +| `privacy_metadata` | Privacy metadata is present | +| `authority_metadata` | Authority metadata is present | + +**Fail closed**: If any hard gate fails, the route is rejected. No silent +fallback to a cheaper or less-governed model. + +--- + +## 5. Soft ranking + +After hard gates pass, routes are ranked by: + +1. **Cost** — lowest cost wins +2. **Latency** — lower latency wins (tiebreaker) +3. **Reliability** — higher reliability wins (tiebreaker) +4. **Evidence recency** — more recent benchmark evidence wins (tiebreaker) + +--- + +## 6. Required policy behavior + +1. Select the lowest-cost route meeting all hard gates +2. Prefer tool navigation or structured memory when source size exceeds effective context +3. Require a different-provider reviewer for high-stakes claims where feasible +4. Keep private repositories and sensitive corpora within approved provider/environment boundaries +5. Escalate when benchmark coverage is missing, stale, or materially mismatched +6. Fail closed on missing model version, budget, privacy, or authority metadata +7. Record why a cheaper route was rejected +8. Include reevaluation and expiry dates for every promoted route + +--- + +## 7. Initial candidate lanes + +Subject to exact model/version verification at run time: + +| Lane | Model | Status | +|------|-------|--------| +| GPT-5.6 Sol | GPT-5.6 Sol / Codex-compatible | GA | +| Claude Fable 5 | Claude Fable 5 | GA | +| Claude Opus 4.8 | Claude Opus 4.8 | GA | +| Claude Sonnet 5 | Claude Sonnet 5 | GA | +| Gemini 3.1 Pro | Gemini 3.1 Pro Preview custom-tools | Preview | +| Gemini 3.5 Flash | Gemini 3.5 Flash | GA | +| Grok 4.5 | Grok 4.5 | GA | +| Open/local models | Declared serving and quantization | Variable | + +Preview models carry stability and deprecation risk. Aliases carry +resolution risk unless pinned or receipted. + +--- + +## 8. Example route card + +```yaml +workload: REPO_ARCHITECTURE_RECONSTRUCTION +source_classification: private +source_estimated_tokens: 640000 +required_capabilities: + - file_search + - cross_file_reasoning + - architecture_synthesis +hard_gates: + correctness: verified + safety: pass + privacy: private_within_boundary + authority: implementer_with_gates + latency: PT5M + reliability: 0.95 + reproducibility: best_effort + governance: pass +soft_rank: + - cost: lowest + - latency: PT3M +selected_route: + model: claude-opus-4.8 + provider: anthropic + disposition: PRIMARY_READER_WITH_REVIEW + review_route: + model: gpt-5.6-sol + provider: openai + disposition: INDEPENDENT_REVIEWER +benchmark_evidence_date: 2026-07-01 +reevaluation_date: 2026-08-01 +expiry_date: 2026-09-01 +rejected_cheaper_routes: + - model: gemini-3.5-flash + reason: "cross_file_reasoning benchmark below threshold" +receipt_id: route-001 +``` + +--- + +## 9. Machine-readable workload schema + +```yaml +workload_id: string +workload_class: enum +source: + size_tokens: integer + type: string + modality: string + classification: private|public +required_capabilities: [] +hard_gates: + correctness: verified|unverified|unknown + safety: pass|fail|unknown + privacy: private_within_boundary|public|unknown + authority: string + latency: string + reliability: float + reproducibility: deterministic|best_effort|unknown + governance: pass|fail|unknown +soft_rank: + - cost: lowest + - latency: string + - reliability: float +constraints: + provider: [] + region: [] + model_availability: GA|preview|deprecated + benchmark_evidence_date: string + reevaluation_date: string + expiry_date: string +``` + +--- + +## 10. Route decision schema + +```yaml +route_id: string +workload_id: string +selected_route: + model: string + provider: string + disposition: enum + review_route: + model: string + provider: string + disposition: enum +hard_gate_results: + - gate: string + result: pass|fail + evidence: string +soft_rank_results: + - criterion: string + value: string +rejected_routes: + - model: string + reason: string +benchmark_evidence_date: string +reevaluation_date: string +expiry_date: string +receipt_id: string +timestamp: string +``` + +--- + +## 11. Fixtures + +| Fixture | Type | Description | +|---------|------|-------------| +| `valid-minimal-route.json` | valid | Minimal valid route card | +| `invalid-missing-hard-gate.json` | invalid | Missing required hard gate | +| `stale-evidence.json` | stale | Benchmark evidence is stale | +| `privacy-conflict.json` | conflict | Private source routed to public provider | +| `missing-model-version.json` | invalid | Model version not verified | +| `missing-budget-metadata.json` | invalid | Budget metadata missing | +| `cheaper-route-rejected.json` | valid | Cheaper route rejected with reason | +| `preview-model-risk.json` | valid | Preview model with stability risk noted | + +--- + +## Non-goals + +- No default route change authorized +- No provider-specific implementation +- No declaration of universally best model +- No migration into Founder Mode runtime without approval + +## Unresolved questions + +1. Should the workload schema be JSON Schema, YAML, or both? +2. How should multi-model routes be represented? +3. What is the minimum benchmark evidence required? +4. How should preview model risk be quantified? + +## Rollback instructions + +This is a specification document. Rollback = revert the commit. No runtime +impact. + +## Related + +- `hummbl-dev/model-routing-as-code#8` — this issue +- `hummbl-dev/hummbl-dev#155` — parent benchmark +- `hummbl-dev/autoresearch-pipeline#30` — harness +- `hummbl-dev/hummbl-bibliography#78` — corpus pack +- `hummbl-dev/hummbl-dev#153` — master index diff --git a/docs/schemas/routing-policy/v0.1/fixtures/cheaper-route-rejected.json b/docs/schemas/routing-policy/v0.1/fixtures/cheaper-route-rejected.json new file mode 100644 index 0000000..0955ef7 --- /dev/null +++ b/docs/schemas/routing-policy/v0.1/fixtures/cheaper-route-rejected.json @@ -0,0 +1,30 @@ +{ + "fixture_id": "cheaper-route-rejected", + "type": "valid", + "description": "Cheaper route rejected with documented reason", + "route": { + "route_id": "route-007", + "workload_id": "wl-007", + "workload_class": "REPO_ARCHITECTURE_RECONSTRUCTION", + "source": {"size_tokens": 640000, "type": "code", "classification": "private"}, + "selected_route": {"model": "claude-opus-4.8", "provider": "anthropic", "disposition": "PRIMARY_READER_WITH_REVIEW"}, + "hard_gate_results": [ + {"gate": "correctness", "result": "pass", "evidence": "benchmark-007"}, + {"gate": "safety", "result": "pass", "evidence": "policy-001"}, + {"gate": "privacy", "result": "pass", "evidence": "private_within_boundary"}, + {"gate": "model_version_verified", "result": "pass", "evidence": "claude-opus-4.8@2026-07-01"}, + {"gate": "budget_metadata", "result": "pass", "evidence": "budget-007"}, + {"gate": "authority_metadata", "result": "pass", "evidence": "auth-007"} + ], + "rejected_routes": [ + {"model": "gemini-3.5-flash", "reason": "cross_file_reasoning benchmark below threshold for REPO_ARCHITECTURE_RECONSTRUCTION"}, + {"model": "claude-sonnet-5", "reason": "source_size_tokens exceeds effective single-context feasibility for this model"} + ], + "benchmark_evidence_date": "2026-07-01", + "reevaluation_date": "2026-08-01", + "expiry_date": "2026-09-01", + "receipt_id": "route-007", + "timestamp": "2026-07-11T12:00:00Z" + }, + "expected_validation": "valid" +} diff --git a/docs/schemas/routing-policy/v0.1/fixtures/invalid-missing-hard-gate.json b/docs/schemas/routing-policy/v0.1/fixtures/invalid-missing-hard-gate.json new file mode 100644 index 0000000..43202b6 --- /dev/null +++ b/docs/schemas/routing-policy/v0.1/fixtures/invalid-missing-hard-gate.json @@ -0,0 +1,20 @@ +{ + "fixture_id": "invalid-missing-hard-gate", + "type": "invalid", + "description": "Route missing required hard gate (safety)", + "route": { + "route_id": "route-002", + "workload_id": "wl-002", + "workload_class": "BOUNDED_PATCH_IMPLEMENTATION", + "selected_route": {"model": "gpt-5.6-sol", "provider": "openai", "disposition": "IMPLEMENTER_WITH_GATES"}, + "hard_gate_results": [ + {"gate": "correctness", "result": "pass", "evidence": "benchmark-002"}, + {"gate": "privacy", "result": "pass", "evidence": "private_within_boundary"} + ], + "benchmark_evidence_date": "2026-07-01", + "receipt_id": "route-002", + "timestamp": "2026-07-11T12:00:00Z" + }, + "expected_validation": "invalid", + "expected_errors": ["missing_hard_gate:safety", "missing_hard_gate:model_version_verified", "missing_hard_gate:budget_metadata", "missing_hard_gate:authority_metadata"] +} diff --git a/docs/schemas/routing-policy/v0.1/fixtures/missing-budget-metadata.json b/docs/schemas/routing-policy/v0.1/fixtures/missing-budget-metadata.json new file mode 100644 index 0000000..0c1e251 --- /dev/null +++ b/docs/schemas/routing-policy/v0.1/fixtures/missing-budget-metadata.json @@ -0,0 +1,23 @@ +{ + "fixture_id": "missing-budget-metadata", + "type": "invalid", + "description": "Budget metadata missing", + "route": { + "route_id": "route-006", + "workload_id": "wl-006", + "workload_class": "MULTI_DOCUMENT_SYNTHESIS", + "selected_route": {"model": "grok-4.5", "provider": "xai", "disposition": "FINAL_SYNTHESIS_WITH_HUMAN_REVIEW"}, + "hard_gate_results": [ + {"gate": "correctness", "result": "pass", "evidence": "benchmark-006"}, + {"gate": "safety", "result": "pass", "evidence": "policy-001"}, + {"gate": "privacy", "result": "pass", "evidence": "public"}, + {"gate": "model_version_verified", "result": "pass", "evidence": "grok-4.5@2026-07-01"}, + {"gate": "budget_metadata", "result": "fail", "evidence": "budget_not_specified"}, + {"gate": "authority_metadata", "result": "pass", "evidence": "auth-006"} + ], + "receipt_id": "route-006", + "timestamp": "2026-07-11T12:00:00Z" + }, + "expected_validation": "invalid", + "expected_errors": ["budget_metadata_missing"] +} diff --git a/docs/schemas/routing-policy/v0.1/fixtures/missing-model-version.json b/docs/schemas/routing-policy/v0.1/fixtures/missing-model-version.json new file mode 100644 index 0000000..9b6e1a4 --- /dev/null +++ b/docs/schemas/routing-policy/v0.1/fixtures/missing-model-version.json @@ -0,0 +1,23 @@ +{ + "fixture_id": "missing-model-version", + "type": "invalid", + "description": "Model version not verified", + "route": { + "route_id": "route-005", + "workload_id": "wl-005", + "workload_class": "INDEPENDENT_CODE_REVIEW", + "selected_route": {"model": "claude-opus-4.8", "provider": "anthropic", "disposition": "INDEPENDENT_REVIEWER"}, + "hard_gate_results": [ + {"gate": "correctness", "result": "pass", "evidence": "benchmark-005"}, + {"gate": "safety", "result": "pass", "evidence": "policy-001"}, + {"gate": "privacy", "result": "pass", "evidence": "private_within_boundary"}, + {"gate": "model_version_verified", "result": "fail", "evidence": "version_not_pinned"}, + {"gate": "budget_metadata", "result": "pass", "evidence": "budget-005"}, + {"gate": "authority_metadata", "result": "pass", "evidence": "auth-005"} + ], + "receipt_id": "route-005", + "timestamp": "2026-07-11T12:00:00Z" + }, + "expected_validation": "invalid", + "expected_errors": ["model_version_not_verified"] +} diff --git a/docs/schemas/routing-policy/v0.1/fixtures/preview-model-risk.json b/docs/schemas/routing-policy/v0.1/fixtures/preview-model-risk.json new file mode 100644 index 0000000..d71c526 --- /dev/null +++ b/docs/schemas/routing-policy/v0.1/fixtures/preview-model-risk.json @@ -0,0 +1,27 @@ +{ + "fixture_id": "preview-model-risk", + "type": "valid", + "description": "Preview model with stability and deprecation risk noted", + "route": { + "route_id": "route-008", + "workload_id": "wl-008", + "workload_class": "MULTIMODAL_SOURCE_PACKET", + "selected_route": {"model": "gemini-3.1-pro-preview", "provider": "google", "disposition": "DISCOVERY_ONLY"}, + "hard_gate_results": [ + {"gate": "correctness", "result": "pass", "evidence": "benchmark-008"}, + {"gate": "safety", "result": "pass", "evidence": "policy-001"}, + {"gate": "privacy", "result": "pass", "evidence": "public"}, + {"gate": "model_version_verified", "result": "pass", "evidence": "gemini-3.1-pro-preview@2026-07-01"}, + {"gate": "budget_metadata", "result": "pass", "evidence": "budget-008"}, + {"gate": "authority_metadata", "result": "pass", "evidence": "auth-008"} + ], + "risk_notes": ["preview_model_stability_risk", "deprecation_risk", "disposition_limited_to_discovery_only"], + "benchmark_evidence_date": "2026-07-01", + "reevaluation_date": "2026-07-15", + "expiry_date": "2026-08-01", + "receipt_id": "route-008", + "timestamp": "2026-07-11T12:00:00Z" + }, + "expected_validation": "valid", + "expected_warnings": ["preview_model_stability_risk"] +} diff --git a/docs/schemas/routing-policy/v0.1/fixtures/privacy-conflict.json b/docs/schemas/routing-policy/v0.1/fixtures/privacy-conflict.json new file mode 100644 index 0000000..e8e573e --- /dev/null +++ b/docs/schemas/routing-policy/v0.1/fixtures/privacy-conflict.json @@ -0,0 +1,24 @@ +{ + "fixture_id": "privacy-conflict", + "type": "conflict", + "description": "Private source routed to public-only provider", + "route": { + "route_id": "route-004", + "workload_id": "wl-004", + "workload_class": "BULK_SOURCE_INGESTION", + "source": {"size_tokens": 500000, "type": "code", "classification": "private"}, + "selected_route": {"model": "gemini-3.5-flash", "provider": "google", "disposition": "BULK_EXTRACTOR"}, + "hard_gate_results": [ + {"gate": "correctness", "result": "pass", "evidence": "benchmark-004"}, + {"gate": "safety", "result": "pass", "evidence": "policy-001"}, + {"gate": "privacy", "result": "fail", "evidence": "private_source_to_public_provider"}, + {"gate": "model_version_verified", "result": "pass", "evidence": "gemini-3.5-flash@2026-07-01"}, + {"gate": "budget_metadata", "result": "pass", "evidence": "budget-004"}, + {"gate": "authority_metadata", "result": "pass", "evidence": "auth-004"} + ], + "receipt_id": "route-004", + "timestamp": "2026-07-11T12:00:00Z" + }, + "expected_validation": "invalid", + "expected_errors": ["privacy_gate_failed"] +} diff --git a/docs/schemas/routing-policy/v0.1/fixtures/stale-evidence.json b/docs/schemas/routing-policy/v0.1/fixtures/stale-evidence.json new file mode 100644 index 0000000..06b2125 --- /dev/null +++ b/docs/schemas/routing-policy/v0.1/fixtures/stale-evidence.json @@ -0,0 +1,26 @@ +{ + "fixture_id": "stale-evidence", + "type": "stale", + "description": "Benchmark evidence is stale (>6 months old)", + "route": { + "route_id": "route-003", + "workload_id": "wl-003", + "workload_class": "CROSS_FILE_TRACE", + "selected_route": {"model": "claude-opus-4.8", "provider": "anthropic", "disposition": "PRIMARY_READER_WITH_REVIEW"}, + "hard_gate_results": [ + {"gate": "correctness", "result": "pass", "evidence": "benchmark-003"}, + {"gate": "safety", "result": "pass", "evidence": "policy-001"}, + {"gate": "privacy", "result": "pass", "evidence": "private_within_boundary"}, + {"gate": "model_version_verified", "result": "pass", "evidence": "claude-opus-4.8@2026-01-01"}, + {"gate": "budget_metadata", "result": "pass", "evidence": "budget-003"}, + {"gate": "authority_metadata", "result": "pass", "evidence": "auth-003"} + ], + "benchmark_evidence_date": "2025-12-01", + "reevaluation_date": "2026-01-01", + "expiry_date": "2026-02-01", + "receipt_id": "route-003", + "timestamp": "2026-07-11T12:00:00Z" + }, + "expected_validation": "stale", + "expected_warnings": ["benchmark_evidence_stale", "route_expired"] +} diff --git a/docs/schemas/routing-policy/v0.1/fixtures/valid-minimal-route.json b/docs/schemas/routing-policy/v0.1/fixtures/valid-minimal-route.json new file mode 100644 index 0000000..372987f --- /dev/null +++ b/docs/schemas/routing-policy/v0.1/fixtures/valid-minimal-route.json @@ -0,0 +1,31 @@ +{ + "fixture_id": "valid-minimal-route", + "type": "valid", + "description": "Minimal valid route card for REPO_INVENTORY", + "route": { + "route_id": "route-001", + "workload_id": "wl-001", + "workload_class": "REPO_INVENTORY", + "source": {"size_tokens": 50000, "type": "code", "classification": "private"}, + "selected_route": { + "model": "claude-sonnet-5", + "provider": "anthropic", + "disposition": "BOUNDED_READER" + }, + "hard_gate_results": [ + {"gate": "correctness", "result": "pass", "evidence": "benchmark-001"}, + {"gate": "safety", "result": "pass", "evidence": "policy-001"}, + {"gate": "privacy", "result": "pass", "evidence": "private_within_boundary"}, + {"gate": "model_version_verified", "result": "pass", "evidence": "claude-sonnet-5@2026-07-01"}, + {"gate": "budget_metadata", "result": "pass", "evidence": "budget-001"}, + {"gate": "authority_metadata", "result": "pass", "evidence": "auth-001"} + ], + "rejected_routes": [], + "benchmark_evidence_date": "2026-07-01", + "reevaluation_date": "2026-08-01", + "expiry_date": "2026-09-01", + "receipt_id": "route-001", + "timestamp": "2026-07-11T12:00:00Z" + }, + "expected_validation": "valid" +} diff --git a/fixtures/invalid/ultra_evaluation_receipt.v0.1.duplicate-lane.invalid.json b/fixtures/invalid/ultra_evaluation_receipt.v0.1.duplicate-lane.invalid.json new file mode 100644 index 0000000..a9026cb --- /dev/null +++ b/fixtures/invalid/ultra_evaluation_receipt.v0.1.duplicate-lane.invalid.json @@ -0,0 +1,65 @@ +{ + "schema_id": "ultra_evaluation_receipt.v0.1", + "schema_version": "0.1", + "receipt_id": "ultra-eval-invalid-duplicate-lane", + "recorded_at": "2026-07-11T05:30:00Z", + "model": { + "provider": "OpenAI", + "base_model_id": "gpt-5.6-sol", + "orchestration_setting": "ultra", + "orchestration_agent_count": {"availability": "available", "value": 4, "measurement_kind": "configured"}, + "configuration_source": "Evaluation harness configuration" + }, + "task": { + "task_id": "invalid-duplicate-lane", + "title": "Duplicate delegation lane identifiers", + "objective": "Prove that lane receipts cannot collide by identifier.", + "decomposable": true, + "decomposition_rationale": "The case isolates lane identity uniqueness.", + "success_criteria": ["The validator rejects the receipt."] + }, + "delegation_lanes": [ + { + "lane_id": "same-lane", + "scope": "First lane.", + "assigned_agent": "codex", + "start_state": "started", + "started_at": "2026-07-11T05:04:36Z", + "final_state": "completed_incorporated", + "completed_at": {"availability": "available", "value": "2026-07-11T05:20:00Z"}, + "findings": [], + "artifacts": [], + "contribution": "First duplicate identifier.", + "duplicate_work": [], + "rejected_or_superseded_findings": [], + "unresolved_limitations": [] + }, + { + "lane_id": "same-lane", + "scope": "Second lane.", + "assigned_agent": "opencode", + "start_state": "started", + "started_at": "2026-07-11T05:05:00Z", + "final_state": "completed_not_incorporated", + "completed_at": {"availability": "available", "value": "2026-07-11T05:21:00Z"}, + "findings": [], + "artifacts": [], + "contribution": "Second duplicate identifier.", + "duplicate_work": ["Repeated the first lane's identity."], + "rejected_or_superseded_findings": ["Superseded by the first lane."], + "unresolved_limitations": [] + } + ], + "evidence_classes": [{"evidence_class": "local_execution_receipt", "scope": "Local negative test.", "source_ids": ["duplicate-lane-fixture"]}], + "telemetry": { + "aggregate_cost": {"availability": "unavailable", "reason": "Not exposed."}, + "aggregate_tokens": {"availability": "unavailable", "reason": "Not exposed."}, + "elapsed_wall_clock_time": {"availability": "unavailable", "reason": "Not exposed."} + }, + "duplicate_work": {"availability": "available", "assessment": "observed", "details": ["The lane identifier was duplicated."]}, + "synthesis_loss": {"availability": "available", "assessment": "none_observed", "details": []}, + "validation_performed": [{"name": "Negative test", "status": "passed", "command": "unittest", "result": "Expected rejection.", "artifact_refs": []}], + "outcome_improvement": {"assessment": "unavailable", "basis": "Negative contract fixture.", "observed_improvements": [], "unavailable_measurements": ["All outcome measurements"]}, + "routing_decision": {"route_used": "ultra", "recommendation": "conditional", "recommended_route": "max", "rationale": "Fixture only.", "counterfactual_route": "max", "counterfactual_rationale": "Fixture only.", "policy_status": "provisional"}, + "limitations": ["Intentionally duplicates a lane identifier."] +} diff --git a/fixtures/invalid/ultra_evaluation_receipt.v0.1.empirical-policy.invalid.json b/fixtures/invalid/ultra_evaluation_receipt.v0.1.empirical-policy.invalid.json new file mode 100644 index 0000000..542801a --- /dev/null +++ b/fixtures/invalid/ultra_evaluation_receipt.v0.1.empirical-policy.invalid.json @@ -0,0 +1,17 @@ +{ + "schema_id": "ultra_evaluation_receipt.v0.1", + "schema_version": "0.1", + "receipt_id": "ultra-eval-invalid-empirical-policy", + "recorded_at": "2026-07-11T05:30:00Z", + "model": {"provider": "OpenAI", "base_model_id": "gpt-5.6-sol", "orchestration_setting": "ultra", "orchestration_agent_count": {"availability": "available", "value": 4, "measurement_kind": "configured"}, "configuration_source": "Evaluation harness configuration"}, + "task": {"task_id": "invalid-empirical-policy", "title": "Unsupported empirical status", "objective": "Prove v0.1 cannot assert empirical policy validation.", "decomposable": true, "decomposition_rationale": "The case isolates policy status.", "success_criteria": ["The validator rejects the receipt."]}, + "delegation_lanes": [{"lane_id": "policy-check", "scope": "Check policy status.", "assigned_agent": "codex", "start_state": "started", "started_at": "2026-07-11T05:04:36Z", "final_state": "completed_incorporated", "completed_at": {"availability": "available", "value": "2026-07-11T05:28:00Z"}, "findings": [], "artifacts": [], "contribution": "Negative policy fixture.", "duplicate_work": [], "rejected_or_superseded_findings": [], "unresolved_limitations": []}], + "evidence_classes": [{"evidence_class": "local_execution_receipt", "scope": "Local negative test.", "source_ids": ["empirical-policy-fixture"]}], + "telemetry": {"aggregate_cost": {"availability": "unavailable", "reason": "Not exposed."}, "aggregate_tokens": {"availability": "unavailable", "reason": "Not exposed."}, "elapsed_wall_clock_time": {"availability": "unavailable", "reason": "Not exposed."}}, + "duplicate_work": {"availability": "available", "assessment": "none_observed", "details": []}, + "synthesis_loss": {"availability": "available", "assessment": "none_observed", "details": []}, + "validation_performed": [{"name": "Negative test", "status": "passed", "command": "unittest", "result": "Expected rejection.", "artifact_refs": []}], + "outcome_improvement": {"assessment": "unavailable", "basis": "Negative contract fixture.", "observed_improvements": [], "unavailable_measurements": ["All outcome measurements"]}, + "routing_decision": {"route_used": "ultra", "recommendation": "conditional", "recommended_route": "max", "rationale": "Fixture only.", "counterfactual_route": "max", "counterfactual_rationale": "Fixture only.", "policy_status": "empirically_validated"}, + "limitations": ["Intentionally claims an unsupported policy status."] +} diff --git a/fixtures/invalid/ultra_evaluation_receipt.v0.1.evidence-class.invalid.json b/fixtures/invalid/ultra_evaluation_receipt.v0.1.evidence-class.invalid.json new file mode 100644 index 0000000..092ebcb --- /dev/null +++ b/fixtures/invalid/ultra_evaluation_receipt.v0.1.evidence-class.invalid.json @@ -0,0 +1,48 @@ +{ + "schema_id": "ultra_evaluation_receipt.v0.1", + "schema_version": "0.1", + "receipt_id": "ultra-eval-invalid-evidence-class", + "recorded_at": "2026-07-11T05:30:00Z", + "model": { + "provider": "OpenAI", + "base_model_id": "gpt-5.6-sol", + "orchestration_setting": "ultra", + "orchestration_agent_count": {"availability": "available", "value": 4, "measurement_kind": "configured"}, + "configuration_source": "Evaluation harness configuration" + }, + "task": { + "task_id": "invalid-evidence-class", + "title": "Conflated independent evidence", + "objective": "Prove that an invented Ultra-validation evidence class is rejected.", + "decomposable": true, + "decomposition_rationale": "The case isolates evidence classification.", + "success_criteria": ["The validator rejects the receipt."] + }, + "delegation_lanes": [{ + "lane_id": "evidence-check", + "scope": "Check evidence-class vocabulary.", + "assigned_agent": "codex", + "start_state": "started", + "started_at": "2026-07-11T05:04:36Z", + "final_state": "completed_incorporated", + "completed_at": {"availability": "available", "value": "2026-07-11T05:28:00Z"}, + "findings": [], + "artifacts": [], + "contribution": "Negative evidence-class fixture.", + "duplicate_work": [], + "rejected_or_superseded_findings": [], + "unresolved_limitations": [] + }], + "evidence_classes": [{"evidence_class": "independent_ultra_validation", "scope": "Invalid conflated class.", "source_ids": ["not-a-valid-class"]}], + "telemetry": { + "aggregate_cost": {"availability": "unavailable", "reason": "Not exposed."}, + "aggregate_tokens": {"availability": "unavailable", "reason": "Not exposed."}, + "elapsed_wall_clock_time": {"availability": "unavailable", "reason": "Not exposed."} + }, + "duplicate_work": {"availability": "available", "assessment": "none_observed", "details": []}, + "synthesis_loss": {"availability": "available", "assessment": "none_observed", "details": []}, + "validation_performed": [{"name": "Negative test", "status": "passed", "command": "unittest", "result": "Expected rejection.", "artifact_refs": []}], + "outcome_improvement": {"assessment": "unavailable", "basis": "Negative contract fixture.", "observed_improvements": [], "unavailable_measurements": ["All outcome measurements"]}, + "routing_decision": {"route_used": "ultra", "recommendation": "conditional", "recommended_route": "max", "rationale": "Fixture only.", "counterfactual_route": "max", "counterfactual_rationale": "Fixture only.", "policy_status": "provisional"}, + "limitations": ["Intentionally uses a non-canonical evidence class."] +} diff --git a/fixtures/invalid/ultra_evaluation_receipt.v0.1.lane-state.invalid.json b/fixtures/invalid/ultra_evaluation_receipt.v0.1.lane-state.invalid.json new file mode 100644 index 0000000..488a98d --- /dev/null +++ b/fixtures/invalid/ultra_evaluation_receipt.v0.1.lane-state.invalid.json @@ -0,0 +1,48 @@ +{ + "schema_id": "ultra_evaluation_receipt.v0.1", + "schema_version": "0.1", + "receipt_id": "ultra-eval-invalid-lane-state", + "recorded_at": "2026-07-11T05:30:00Z", + "model": { + "provider": "OpenAI", + "base_model_id": "gpt-5.6-sol", + "orchestration_setting": "ultra", + "orchestration_agent_count": {"availability": "available", "value": 4, "measurement_kind": "configured"}, + "configuration_source": "Evaluation harness configuration" + }, + "task": { + "task_id": "invalid-lane-state", + "title": "Invalid delegation final state", + "objective": "Prove that ambiguous lane completion labels are rejected.", + "decomposable": true, + "decomposition_rationale": "The case isolates lane state validation.", + "success_criteria": ["The validator rejects the receipt."] + }, + "delegation_lanes": [{ + "lane_id": "lane-state-check", + "scope": "Check final-state vocabulary.", + "assigned_agent": "codex", + "start_state": "started", + "started_at": "2026-07-11T05:04:36Z", + "final_state": "done", + "completed_at": {"availability": "available", "value": "2026-07-11T05:28:00Z"}, + "findings": [], + "artifacts": [], + "contribution": "Negative lane-state fixture.", + "duplicate_work": [], + "rejected_or_superseded_findings": [], + "unresolved_limitations": [] + }], + "evidence_classes": [{"evidence_class": "local_execution_receipt", "scope": "Local negative test.", "source_ids": ["lane-state-fixture"]}], + "telemetry": { + "aggregate_cost": {"availability": "unavailable", "reason": "Not exposed."}, + "aggregate_tokens": {"availability": "unavailable", "reason": "Not exposed."}, + "elapsed_wall_clock_time": {"availability": "unavailable", "reason": "Not exposed."} + }, + "duplicate_work": {"availability": "available", "assessment": "none_observed", "details": []}, + "synthesis_loss": {"availability": "available", "assessment": "none_observed", "details": []}, + "validation_performed": [{"name": "Negative test", "status": "passed", "command": "unittest", "result": "Expected rejection.", "artifact_refs": []}], + "outcome_improvement": {"assessment": "unavailable", "basis": "Negative contract fixture.", "observed_improvements": [], "unavailable_measurements": ["All outcome measurements"]}, + "routing_decision": {"route_used": "ultra", "recommendation": "conditional", "recommended_route": "max", "rationale": "Fixture only.", "counterfactual_route": "max", "counterfactual_rationale": "Fixture only.", "policy_status": "provisional"}, + "limitations": ["Intentionally uses an invalid final state."] +} diff --git a/fixtures/invalid/ultra_evaluation_receipt.v0.1.none-observed-details.invalid.json b/fixtures/invalid/ultra_evaluation_receipt.v0.1.none-observed-details.invalid.json new file mode 100644 index 0000000..788b2b1 --- /dev/null +++ b/fixtures/invalid/ultra_evaluation_receipt.v0.1.none-observed-details.invalid.json @@ -0,0 +1,17 @@ +{ + "schema_id": "ultra_evaluation_receipt.v0.1", + "schema_version": "0.1", + "receipt_id": "ultra-eval-invalid-none-observed-details", + "recorded_at": "2026-07-11T05:30:00Z", + "model": {"provider": "OpenAI", "base_model_id": "gpt-5.6-sol", "orchestration_setting": "ultra", "orchestration_agent_count": {"availability": "available", "value": 4, "measurement_kind": "configured"}, "configuration_source": "Evaluation harness configuration"}, + "task": {"task_id": "invalid-none-observed-details", "title": "Contradictory duplicate assessment", "objective": "Prove none_observed cannot contain contradictory details.", "decomposable": true, "decomposition_rationale": "The case isolates assessment consistency.", "success_criteria": ["The validator rejects the receipt."]}, + "delegation_lanes": [{"lane_id": "assessment-check", "scope": "Check assessment details.", "assigned_agent": "codex", "start_state": "started", "started_at": "2026-07-11T05:04:36Z", "final_state": "completed_incorporated", "completed_at": {"availability": "available", "value": "2026-07-11T05:28:00Z"}, "findings": [], "artifacts": [], "contribution": "Negative assessment fixture.", "duplicate_work": [], "rejected_or_superseded_findings": [], "unresolved_limitations": []}], + "evidence_classes": [{"evidence_class": "local_execution_receipt", "scope": "Local negative test.", "source_ids": ["none-observed-details-fixture"]}], + "telemetry": {"aggregate_cost": {"availability": "unavailable", "reason": "Not exposed."}, "aggregate_tokens": {"availability": "unavailable", "reason": "Not exposed."}, "elapsed_wall_clock_time": {"availability": "unavailable", "reason": "Not exposed."}}, + "duplicate_work": {"availability": "available", "assessment": "none_observed", "details": ["This contradicts none_observed."]}, + "synthesis_loss": {"availability": "available", "assessment": "none_observed", "details": []}, + "validation_performed": [{"name": "Negative test", "status": "passed", "command": "unittest", "result": "Expected rejection.", "artifact_refs": []}], + "outcome_improvement": {"assessment": "unavailable", "basis": "Negative contract fixture.", "observed_improvements": [], "unavailable_measurements": ["All outcome measurements"]}, + "routing_decision": {"route_used": "ultra", "recommendation": "conditional", "recommended_route": "max", "rationale": "Fixture only.", "counterfactual_route": "max", "counterfactual_rationale": "Fixture only.", "policy_status": "provisional"}, + "limitations": ["Intentionally contradicts its duplicate-work assessment."] +} diff --git a/fixtures/invalid/ultra_evaluation_receipt.v0.1.numeric-overflow.invalid.json b/fixtures/invalid/ultra_evaluation_receipt.v0.1.numeric-overflow.invalid.json new file mode 100644 index 0000000..118f22e --- /dev/null +++ b/fixtures/invalid/ultra_evaluation_receipt.v0.1.numeric-overflow.invalid.json @@ -0,0 +1,48 @@ +{ + "schema_id": "ultra_evaluation_receipt.v0.1", + "schema_version": "0.1", + "receipt_id": "ultra-eval-invalid-numeric-overflow", + "recorded_at": "2026-07-11T05:30:00Z", + "model": { + "provider": "OpenAI", + "base_model_id": "gpt-5.6-sol", + "orchestration_setting": "ultra", + "orchestration_agent_count": {"availability": "available", "value": 4, "measurement_kind": "configured"}, + "configuration_source": "Evaluation harness configuration" + }, + "task": { + "task_id": "invalid-numeric-overflow", + "title": "Overflowed aggregate cost", + "objective": "Prove that a standards-parseable number which overflows a host float is rejected.", + "decomposable": true, + "decomposition_rationale": "The case isolates finite-number enforcement.", + "success_criteria": ["The validator rejects the receipt."] + }, + "delegation_lanes": [{ + "lane_id": "finite-number-check", + "scope": "Check numeric finiteness.", + "assigned_agent": "codex", + "start_state": "started", + "started_at": "2026-07-11T05:04:36Z", + "final_state": "completed_incorporated", + "completed_at": {"availability": "available", "value": "2026-07-11T05:28:00Z"}, + "findings": [], + "artifacts": [], + "contribution": "Negative numeric-overflow fixture.", + "duplicate_work": [], + "rejected_or_superseded_findings": [], + "unresolved_limitations": [] + }], + "evidence_classes": [{"evidence_class": "local_execution_receipt", "scope": "Local negative test.", "source_ids": ["numeric-overflow-fixture"]}], + "telemetry": { + "aggregate_cost": {"availability": "available", "value": 1e999, "unit": "USD", "measurement_kind": "observed"}, + "aggregate_tokens": {"availability": "unavailable", "reason": "Not exposed."}, + "elapsed_wall_clock_time": {"availability": "unavailable", "reason": "Not exposed."} + }, + "duplicate_work": {"availability": "available", "assessment": "none_observed", "details": []}, + "synthesis_loss": {"availability": "available", "assessment": "none_observed", "details": []}, + "validation_performed": [{"name": "Negative test", "status": "passed", "command": "unittest", "result": "Expected rejection.", "artifact_refs": []}], + "outcome_improvement": {"assessment": "unavailable", "basis": "Negative contract fixture.", "observed_improvements": [], "unavailable_measurements": ["All outcome measurements"]}, + "routing_decision": {"route_used": "ultra", "recommendation": "conditional", "recommended_route": "max", "rationale": "Fixture only.", "counterfactual_route": "max", "counterfactual_rationale": "Fixture only.", "policy_status": "provisional"}, + "limitations": ["Intentionally contains a number that overflows binary floating point."] +} diff --git a/fixtures/invalid/ultra_evaluation_receipt.v0.1.route-mismatch.invalid.json b/fixtures/invalid/ultra_evaluation_receipt.v0.1.route-mismatch.invalid.json new file mode 100644 index 0000000..f01bae5 --- /dev/null +++ b/fixtures/invalid/ultra_evaluation_receipt.v0.1.route-mismatch.invalid.json @@ -0,0 +1,23 @@ +{ + "schema_id": "ultra_evaluation_receipt.v0.1", + "schema_version": "0.1", + "receipt_id": "ultra-eval-invalid-route-mismatch", + "recorded_at": "2026-07-11T05:30:00Z", + "model": { + "provider": "OpenAI", + "base_model_id": "gpt-5.6-sol", + "orchestration_setting": "xhigh", + "orchestration_agent_count": {"availability": "available", "value": 1, "measurement_kind": "configured"}, + "configuration_source": "Evaluation harness configuration" + }, + "task": {"task_id": "invalid-route-mismatch", "title": "Mismatched route", "objective": "Prove model and routed settings cannot disagree.", "decomposable": true, "decomposition_rationale": "The case isolates route identity.", "success_criteria": ["The validator rejects the receipt."]}, + "delegation_lanes": [{"lane_id": "route-check", "scope": "Check route agreement.", "assigned_agent": "codex", "start_state": "started", "started_at": "2026-07-11T05:04:36Z", "final_state": "completed_incorporated", "completed_at": {"availability": "available", "value": "2026-07-11T05:28:00Z"}, "findings": [], "artifacts": [], "contribution": "Negative route fixture.", "duplicate_work": [], "rejected_or_superseded_findings": [], "unresolved_limitations": []}], + "evidence_classes": [{"evidence_class": "local_execution_receipt", "scope": "Local negative test.", "source_ids": ["route-mismatch-fixture"]}], + "telemetry": {"aggregate_cost": {"availability": "unavailable", "reason": "Not exposed."}, "aggregate_tokens": {"availability": "unavailable", "reason": "Not exposed."}, "elapsed_wall_clock_time": {"availability": "unavailable", "reason": "Not exposed."}}, + "duplicate_work": {"availability": "available", "assessment": "none_observed", "details": []}, + "synthesis_loss": {"availability": "available", "assessment": "none_observed", "details": []}, + "validation_performed": [{"name": "Negative test", "status": "passed", "command": "unittest", "result": "Expected rejection.", "artifact_refs": []}], + "outcome_improvement": {"assessment": "unavailable", "basis": "Negative contract fixture.", "observed_improvements": [], "unavailable_measurements": ["All outcome measurements"]}, + "routing_decision": {"route_used": "ultra", "recommendation": "conditional", "recommended_route": "max", "rationale": "Fixture only.", "counterfactual_route": "max", "counterfactual_rationale": "Fixture only.", "policy_status": "provisional"}, + "limitations": ["Intentionally disagrees about the executed route."] +} diff --git a/fixtures/invalid/ultra_evaluation_receipt.v0.1.schema-identity.invalid.json b/fixtures/invalid/ultra_evaluation_receipt.v0.1.schema-identity.invalid.json new file mode 100644 index 0000000..46bcdc3 --- /dev/null +++ b/fixtures/invalid/ultra_evaluation_receipt.v0.1.schema-identity.invalid.json @@ -0,0 +1,48 @@ +{ + "schema_id": "ultra_evaluation_receipt.v9.9", + "schema_version": "0.1", + "receipt_id": "ultra-eval-invalid-schema-id", + "recorded_at": "2026-07-11T05:30:00Z", + "model": { + "provider": "OpenAI", + "base_model_id": "gpt-5.6-sol", + "orchestration_setting": "ultra", + "orchestration_agent_count": {"availability": "available", "value": 4, "measurement_kind": "configured"}, + "configuration_source": "Evaluation harness configuration" + }, + "task": { + "task_id": "invalid-schema-id", + "title": "Invalid schema identity", + "objective": "Prove that a mismatched schema identity is rejected.", + "decomposable": true, + "decomposition_rationale": "The case isolates identity validation.", + "success_criteria": ["The validator rejects the receipt."] + }, + "delegation_lanes": [{ + "lane_id": "identity-check", + "scope": "Check schema identity.", + "assigned_agent": "codex", + "start_state": "started", + "started_at": "2026-07-11T05:04:36Z", + "final_state": "completed_incorporated", + "completed_at": {"availability": "available", "value": "2026-07-11T05:28:00Z"}, + "findings": [], + "artifacts": [], + "contribution": "Negative identity fixture.", + "duplicate_work": [], + "rejected_or_superseded_findings": [], + "unresolved_limitations": [] + }], + "evidence_classes": [{"evidence_class": "local_execution_receipt", "scope": "Local negative test.", "source_ids": ["schema-identity-fixture"]}], + "telemetry": { + "aggregate_cost": {"availability": "unavailable", "reason": "Not exposed."}, + "aggregate_tokens": {"availability": "unavailable", "reason": "Not exposed."}, + "elapsed_wall_clock_time": {"availability": "unavailable", "reason": "Not exposed."} + }, + "duplicate_work": {"availability": "available", "assessment": "none_observed", "details": []}, + "synthesis_loss": {"availability": "available", "assessment": "none_observed", "details": []}, + "validation_performed": [{"name": "Negative test", "status": "passed", "command": "unittest", "result": "Expected rejection.", "artifact_refs": []}], + "outcome_improvement": {"assessment": "unavailable", "basis": "Negative contract fixture.", "observed_improvements": [], "unavailable_measurements": ["All outcome measurements"]}, + "routing_decision": {"route_used": "ultra", "recommendation": "conditional", "recommended_route": "max", "rationale": "Fixture only.", "counterfactual_route": "max", "counterfactual_rationale": "Fixture only.", "policy_status": "provisional"}, + "limitations": ["Intentionally invalid schema identity."] +} diff --git a/fixtures/invalid/ultra_evaluation_receipt.v0.1.unavailable-value.invalid.json b/fixtures/invalid/ultra_evaluation_receipt.v0.1.unavailable-value.invalid.json new file mode 100644 index 0000000..bea781a --- /dev/null +++ b/fixtures/invalid/ultra_evaluation_receipt.v0.1.unavailable-value.invalid.json @@ -0,0 +1,48 @@ +{ + "schema_id": "ultra_evaluation_receipt.v0.1", + "schema_version": "0.1", + "receipt_id": "ultra-eval-invalid-unavailable-value", + "recorded_at": "2026-07-11T05:30:00Z", + "model": { + "provider": "OpenAI", + "base_model_id": "gpt-5.6-sol", + "orchestration_setting": "ultra", + "orchestration_agent_count": {"availability": "available", "value": 4, "measurement_kind": "configured"}, + "configuration_source": "Evaluation harness configuration" + }, + "task": { + "task_id": "invalid-unavailable-value", + "title": "Unavailable telemetry with a fabricated value", + "objective": "Prove that unavailable telemetry cannot carry a numeric estimate.", + "decomposable": true, + "decomposition_rationale": "The case isolates telemetry honesty.", + "success_criteria": ["The validator rejects the receipt."] + }, + "delegation_lanes": [{ + "lane_id": "telemetry-check", + "scope": "Check unavailable telemetry.", + "assigned_agent": "codex", + "start_state": "started", + "started_at": "2026-07-11T05:04:36Z", + "final_state": "completed_incorporated", + "completed_at": {"availability": "available", "value": "2026-07-11T05:28:00Z"}, + "findings": [], + "artifacts": [], + "contribution": "Negative telemetry fixture.", + "duplicate_work": [], + "rejected_or_superseded_findings": [], + "unresolved_limitations": [] + }], + "evidence_classes": [{"evidence_class": "local_execution_receipt", "scope": "Local negative test.", "source_ids": ["unavailable-value-fixture"]}], + "telemetry": { + "aggregate_cost": {"availability": "unavailable", "reason": "Not exposed.", "value": 9.99}, + "aggregate_tokens": {"availability": "unavailable", "reason": "Not exposed."}, + "elapsed_wall_clock_time": {"availability": "unavailable", "reason": "Not exposed."} + }, + "duplicate_work": {"availability": "available", "assessment": "none_observed", "details": []}, + "synthesis_loss": {"availability": "available", "assessment": "none_observed", "details": []}, + "validation_performed": [{"name": "Negative test", "status": "passed", "command": "unittest", "result": "Expected rejection.", "artifact_refs": []}], + "outcome_improvement": {"assessment": "unavailable", "basis": "Negative contract fixture.", "observed_improvements": [], "unavailable_measurements": ["All outcome measurements"]}, + "routing_decision": {"route_used": "ultra", "recommendation": "conditional", "recommended_route": "max", "rationale": "Fixture only.", "counterfactual_route": "max", "counterfactual_rationale": "Fixture only.", "policy_status": "provisional"}, + "limitations": ["Intentionally fabricates a value while declaring it unavailable."] +} diff --git a/fixtures/valid/ultra_evaluation_receipt.v0.1.valid.json b/fixtures/valid/ultra_evaluation_receipt.v0.1.valid.json new file mode 100644 index 0000000..5134a0f --- /dev/null +++ b/fixtures/valid/ultra_evaluation_receipt.v0.1.valid.json @@ -0,0 +1,129 @@ +{ + "schema_id": "ultra_evaluation_receipt.v0.1", + "schema_version": "0.1", + "receipt_id": "ultra-eval-valid-fixture", + "recorded_at": "2026-07-11T05:30:00Z", + "model": { + "provider": "OpenAI", + "base_model_id": "gpt-5.6-sol", + "orchestration_setting": "ultra", + "orchestration_agent_count": { + "availability": "available", + "value": 4, + "measurement_kind": "configured" + }, + "configuration_source": "Evaluation harness configuration" + }, + "task": { + "task_id": "receipt-validator-fixture", + "title": "Validate an Ultra execution receipt", + "objective": "Exercise the v0.1 receipt contract without inventing runtime telemetry.", + "decomposable": true, + "decomposition_rationale": "Schema, validation, and adversarial review can be checked independently.", + "success_criteria": [ + "The valid fixture passes deterministically.", + "Invalid evidence classes and fabricated telemetry fail with paths." + ] + }, + "delegation_lanes": [ + { + "lane_id": "receipt-contract", + "scope": "Define and validate the receipt contract.", + "assigned_agent": "codex", + "start_state": "started", + "started_at": "2026-07-11T05:04:36Z", + "final_state": "completed_incorporated", + "completed_at": { + "availability": "available", + "value": "2026-07-11T05:28:00Z" + }, + "findings": [ + "Unavailable telemetry needs a reason and must omit numeric values." + ], + "artifacts": [ + "schemas/ultra_evaluation_receipt.v0.1.schema.json", + "tools/validate_ultra_evaluation_receipt.py" + ], + "contribution": "Provided the machine-checkable receipt boundary.", + "duplicate_work": [], + "rejected_or_superseded_findings": [], + "unresolved_limitations": [ + "This fixture is not an independent Ultra benchmark replication." + ] + } + ], + "evidence_classes": [ + { + "evidence_class": "local_execution_receipt", + "scope": "Local schema and validator execution only.", + "source_ids": [ + "tests/test_ultra_evaluation_receipt.py" + ] + }, + { + "evidence_class": "inference", + "scope": "Routing interpretation remains provisional pending controlled local comparisons.", + "source_ids": [ + "routing-policy-provisional" + ] + } + ], + "telemetry": { + "aggregate_cost": { + "availability": "unavailable", + "reason": "The execution environment did not expose aggregate cost." + }, + "aggregate_tokens": { + "availability": "unavailable", + "reason": "The execution environment did not expose aggregate token usage." + }, + "elapsed_wall_clock_time": { + "availability": "unavailable", + "reason": "A harness-level wall-clock timer was not exposed to the receipt." + } + }, + "duplicate_work": { + "availability": "available", + "assessment": "none_observed", + "details": [] + }, + "synthesis_loss": { + "availability": "available", + "assessment": "none_observed", + "details": [] + }, + "validation_performed": [ + { + "name": "Receipt validator unit tests", + "status": "passed", + "command": "python3 -m unittest discover -s tests -p test_ultra_evaluation_receipt.py -v", + "result": "The valid fixture passed and negative fixtures were rejected.", + "artifact_refs": [ + "tests/test_ultra_evaluation_receipt.py" + ] + } + ], + "outcome_improvement": { + "assessment": "inconclusive", + "basis": "No identical xhigh or Max counterfactual run was performed.", + "observed_improvements": [], + "unavailable_measurements": [ + "Counterfactual task success", + "Counterfactual elapsed wall-clock time", + "Counterfactual aggregate cost and tokens" + ] + }, + "routing_decision": { + "route_used": "ultra", + "recommendation": "conditional", + "recommended_route": "max", + "rationale": "Use Ultra only when independent lanes can materially alter coverage or elapsed time.", + "counterfactual_route": "max", + "counterfactual_rationale": "Max is the lower-coordination counterfactual for a tightly coupled implementation.", + "policy_status": "provisional" + }, + "limitations": [ + "This is a contract fixture, not evidence of model quality.", + "Cost, tokens, and elapsed time are explicitly unavailable." + ] +} diff --git a/receipts/2026-07-11-gpt-5.6-sol-ultra-claim-verification.md b/receipts/2026-07-11-gpt-5.6-sol-ultra-claim-verification.md new file mode 100644 index 0000000..ee79bf8 --- /dev/null +++ b/receipts/2026-07-11-gpt-5.6-sol-ultra-claim-verification.md @@ -0,0 +1,93 @@ +# GPT-5.6 Sol Ultra claim-verification receipt — 2026-07-11 + +## Receipt status + +- Mode: official-first public claim verification +- Search date: 2026-07-11 +- Claim boundary: named-source, dated inventory; not an exhaustive absence proof +- Privacy: public claims and repository artifacts only +- Result: central claims supported after the precision corrections below + +## Atomic claims + +| # | Claim | Verdict | Evidence class | Basis | +| ---: | --- | --- | --- | --- | +| 1 | The API model identity is `gpt-5.6-sol`. | Supported | `official_provider_result` | OpenAI API model page | +| 2 | Ultra is a Sol reasoning/orchestration selection, not a separately listed API model. | Supported | `official_provider_result` | OpenAI API model page and Codex model guidance | +| 3 | Ultra coordinates four agents by default. | Supported | `official_provider_result` | OpenAI release, multi-agent footnote | +| 4 | The checked OpenAI release reports Ultra values for BrowseComp, SEC-Bench Pro, and Terminal-Bench 2.1. | Supported within source | `official_provider_result` | OpenAI release tables; other Ultra cells in that table are dashes | +| 5 | Current BrowseComp values are 90.4% Sol and 92.2% Ultra. | Supported | `official_provider_result` | OpenAI release | +| 6 | Current SEC-Bench Pro values are 71.2% Sol and 74.3% Ultra. | Supported | `official_provider_result` | OpenAI release | +| 7 | Current Terminal-Bench 2.1 values are 88.8% Sol and 91.9% Ultra. | Supported | `official_provider_result` | OpenAI release | +| 8 | Provider cost and latency values are offline simulations, not service guarantees. | Supported | `official_provider_result` | OpenAI release methodology footnote | +| 9 | Multi-agent latency is attributed to the root agent; cost and output tokens include all agents. | Supported | `official_provider_result` | OpenAI release multi-agent footnote | +| 10 | Archived BrowseComp max-to-max is +1.34 pp, 2.93× cost, 2.46× tokens, and 17.7% lower latency. | Supported | `archived_chart_data` + `inference` | Hashed archive records and reproduced arithmetic | +| 11 | Archived SEC-Bench Pro max-to-max is +2.87 pp, 2.10× cost, 2.08× tokens, and 42.4% lower latency. | Supported | `archived_chart_data` + `inference` | Hashed archive records and reproduced arithmetic | +| 12 | Archived Terminal-Bench 2.1 max-to-max is +3.15 pp, 2.99× cost, 3.07× tokens, and 0.9% higher latency. | Supported | `archived_chart_data` + `inference` | Hashed archive records and reproduced arithmetic | +| 13 | Ultra is universally faster. | Contradicted | `inference` | Terminal-Bench max-to-max root-agent latency is slightly higher | +| 14 | Ultra directly measures roughly 2–3× physical compute. | Partially supported | `inference` | The source measures tokens and estimated API cost, not FLOPs; “compute” must be labeled shorthand | +| 15 | No independent four-agent Ultra replication was found in the named sources during the dated window. | Supported as bounded search finding | `inference` | Named-surface audit; universal absence remains unverified | +| 16 | Single-agent Sol evaluations independently validate Ultra. | Contradicted | `independent_base_model_evidence` | They do not reproduce four-agent orchestration | +| 17 | The proposed route doctrine is empirically established. | Unverified | `inference` | No controlled local xhigh–Max–Ultra comparison exists yet | + +## Independent-evidence boundary + +| Surface | Finding on 2026-07-11 | Classification | +| --- | --- | --- | +| [Terminal-Bench 2.1](https://www.tbench.ai/leaderboard/terminal-bench/2.1) | No GPT-5.6/Ultra row in the displayed entries. | Negative search evidence only | +| [SEC-Bench Pro](https://sec-bench.github.io/) | No GPT-5.6/Ultra row in the rendered leaderboards. | Negative search evidence only | +| [Artificial Analysis](https://artificialanalysis.ai/evaluations/terminalbench-v2-1) | Sol effort variants; no Ultra row found. | `independent_base_model_evidence` | +| [Vals](https://www.vals.ai/benchmarks) | Sol benchmark rows; no Ultra identifier found. | `independent_base_model_evidence` | +| [ARC Prize](https://arcprize.org/results/openai-gpt-5-6) | Sol effort variants through Max; no Ultra. | `independent_base_model_evidence` | +| [METR](https://metr.org/blog/2026-06-26-gpt-5-6-sol/) | Sol checkpoint, not Ultra; time-horizon estimate was not robust to treatment of detected rule violations. | `independent_base_model_evidence` | +| [Arena](https://arena.ai/leaderboard) | Sol variants found; no exact Ultra identifier found. | Base/harness context only | +| [OpenRouter](https://openrouter.ai/openai/gpt-5.6-sol-20260709) | Sol model directory entry; no Ultra replication. | Directory evidence only | +| [Aider](https://aider.chat/docs/leaderboards/) | No GPT-5.6 entry found. | Negative search evidence only | +| [SWE-bench](https://www.swebench.com/) | No Ultra entry found. | Negative search evidence only | +| [LiveBench](https://livebench.ai/) | Current client shell did not yield a durable exact result. | Insufficient for absence claim | +| [Hugging Face](https://huggingface.co/models) | Exact-phrase model/dataset searches were negative. | Negative search evidence only | +| Papers With Code | Surface redirected to Hugging Face trending papers. | Rejected as operative absence evidence | + +Zero `independent_ultra_replication` records were found in this named, dated +search. This is not proof that none exists elsewhere or after the retrieval date. + +## Archive cross-check + +- Capture: `2026-07-10T15:03:48Z` +- Retrieval date: `2026-07-11` +- Decompressed response SHA-256: + `242b04c63bd0da245bc2dcf601801ee958deb9ebc08def9193f5c6846fdbf892` +- All three archived Ultra scores round to the current headline Ultra scores. +- BrowseComp and SEC-Bench Pro archived one-agent scores do not round to their + current headline baselines, so operational economics stay archive-to-archive. +- The raw metric name `latency_s` conflicts with labels displaying minutes; both + are preserved and minutes are explicitly an interpretation of the labels. + +## Corrections applied to the packet + +- “Complete public record” was rejected in favor of a dated, named-source + inventory. +- “2–3× aggregate compute” is retained only as disclosed routing shorthand; the + measured proxies are output tokens and provider-estimated cost. +- Base-model evidence is not labeled Ultra replication. +- The provider's general faster characterization does not override the slightly + slower Terminal-Bench max-to-max result. +- Papers With Code was rejected as a durable negative surface because it + redirected. + +## Verification-lane accounting + +- Assigned agent: `/root/official_claims` +- Started: `2026-07-11T05:04:36Z` +- Completed: `2026-07-11T05:13:35Z` +- Final state: completed and incorporated +- Contribution: model identity, four-agent default, headline scores, provider + methodology, independent-evidence boundary, arithmetic cross-check, and + counter-evidence +- Intentional overlap: independently reproduced archived max values to cross-check + the archive-provenance lane +- Rejected findings: universal completeness, universal speedup, physical-compute + measurement, and single-agent evidence as Ultra validation +- Remaining limitations: negative searches are time-sensitive; two archived + one-agent baselines differ from current headlines; provider simulations are not + observed service telemetry; no controlled local route comparison exists diff --git a/receipts/2026-07-11-gpt-5.6-sol-ultra-content-review.md b/receipts/2026-07-11-gpt-5.6-sol-ultra-content-review.md new file mode 100644 index 0000000..a1c6866 --- /dev/null +++ b/receipts/2026-07-11-gpt-5.6-sol-ultra-content-review.md @@ -0,0 +1,71 @@ +# Content review — GPT-5.6 Sol Ultra evaluation packet + +## Verdict: HOLD for public publication + +The local packet is technically ready for governed review, but it should not be +pushed to the public repository until an authorized reviewer explicitly accepts +or resolves the archived source-locator tradeoff below. This hold does not block +the local signed commit. + +## Factual accuracy + +| Claim | Status | Evidence | +| --- | --- | --- | +| OpenAI reports Ultra results for three benchmark families. | Verified within the named release source | Current OpenAI release table and dated claim-verification receipt | +| Current scores are 92.2% BrowseComp, 74.3% SEC-Bench Pro, and 91.9% Terminal-Bench 2.1. | Verified provider results | Current OpenAI release table | +| Archived max-to-max economics match the packet table. | Verified | Raw max records, manifest, independent arithmetic review, and cross-artifact validator | +| The archive selection contains 120 semantic records. | Verified | Exact production-matrix validator and manifest | +| Archive entity SHA-256 is `242b04c63bd0da245bc2dcf601801ee958deb9ebc08def9193f5c6846fdbf892`. | Verified | Fresh network-backed archive verification | +| No independent four-agent Ultra replication was found. | Verified only as a named-source, dated search outcome | Claim-verification receipt; packet consistently rejects universal absence | +| This run benefited from Ultra. | Qualified observational judgment | Three implementation/research lanes contributed and adversarial review found material defects; no counterfactual, cost, token, or wall-clock comparison exists | +| The route policy is established. | Not claimed | Every route artifact is labeled provisional/non-canon | + +## Tone and brand + +- Direct, technical, and appropriately qualified. +- No prohibited buzzwords or unsupported superlatives were found. +- Model, orchestration, benchmark, and evidence-class naming is consistent. + +## Compliance and publication risk + +- Secret-pattern scan found no credentials, tokens, or private keys. +- No absolute local filesystem paths are present in the outbound artifacts. +- The 120 selected raw records repeat two distinct + `openai-corpws.slack.com` channel/message locators exposed by the public archive. + They contain no detected credentials and are not accessible as public evidence, + but republishing internal provider locators in a public repository is an + explicit provenance/privacy decision. +- The user requested selected raw records, so silently deleting or changing those + fields would weaken the requested provenance. No unilateral redaction was made. +- GitHub Actions was not invoked because the organization has no remaining + Actions minutes. + +## Audience and structure + +- The evidence packet leads with the claim boundary and decision-relevant result. +- Technical detail is layered into dataset, manifest, validators, receipts, and + protocol artifacts. +- Tables are used for quantitative comparisons and claim verdicts. + +## Publication resolution options + +An authorized reviewer must choose one of these before a public push: + +1. Accept retention of the two distinct locator values because they are already + exposed in the hashed public archive and contain no detected credential; or +2. Approve a documented redaction profile that replaces each locator with a + deterministic hash while preserving the unmodified archive entity hash and + reproducible extraction procedure. + +The second option changes what “raw selected records” means and therefore needs +explicit approval. Until one option is chosen, branch publication and draft PR +creation remain on hold. + +## Residual risk + +- Public-source contents and negative-search results can change after 2026-07-11. +- Provider cost and latency remain simulations. +- No independent Ultra replication or controlled local xhigh/Max comparison + exists. +- Retaining or redacting the locators each has a provenance tradeoff that tooling + alone cannot authorize. diff --git a/receipts/2026-07-11-gpt-5.6-sol-ultra-run.json b/receipts/2026-07-11-gpt-5.6-sol-ultra-run.json new file mode 100644 index 0000000..3cbd3eb --- /dev/null +++ b/receipts/2026-07-11-gpt-5.6-sol-ultra-run.json @@ -0,0 +1,413 @@ +{ + "schema_id": "ultra_evaluation_receipt.v0.1", + "schema_version": "0.1", + "receipt_id": "ultra-eval-gpt-5.6-sol-ultra-packet-2026-07-11", + "recorded_at": "2026-07-11T05:36:02Z", + "model": { + "provider": "OpenAI", + "base_model_id": "gpt-5.6-sol", + "orchestration_setting": "ultra", + "orchestration_agent_count": { + "availability": "available", + "value": 4, + "measurement_kind": "observed" + }, + "configuration_source": "The task identified the session as GPT-5.6 Sol Ultra, and the collaboration runtime exposed one root plus three delegated agent identities; no authoritative API model-response metadata was exposed." + }, + "task": { + "task_id": "gpt-5.6-sol-ultra-evidence-packet-2026-07-11", + "title": "Implement a governed GPT-5.6 Sol Ultra evaluation packet", + "objective": "Turn the reconciled public Ultra research into a durable evidence packet, machine dataset, reproducible archive extraction, validated execution-receipt contract, provisional route policy, and controlled future benchmark protocol.", + "decomposable": true, + "decomposition_rationale": "Official-source verification, archive provenance, receipt validation, and adversarial policy review had distinct evidence and file boundaries and could proceed independently before root synthesis.", + "success_criteria": [ + "Preserve a dated search-bounded claim and the model-versus-orchestration naming distinction.", + "Keep current headline results separate from archived max-to-max economics.", + "Provide source URLs, retrieval dates, raw records, archive timestamp and hash, units, reconciliation, and unavailable fields.", + "Provide a stdlib-only receipt schema, deterministic validator, valid and invalid fixtures, and tests.", + "Provide a provisional medium/high/xhigh/Max/Ultra route policy and a fixed-harness future comparison protocol.", + "Record every lane, actual exposed telemetry, duplication, synthesis loss, limitations, and counterfactual route.", + "Run local validation without invoking GitHub Actions." + ] + }, + "delegation_lanes": [ + { + "lane_id": "official-source-and-claim-verification", + "scope": "Verify model identity, Ultra configuration, official headline scores, provider methodology, independent-evidence classifications, and the bounded absence claim against public sources.", + "assigned_agent": "/root/official_claims", + "start_state": "started", + "started_at": "2026-07-11T05:04:36Z", + "final_state": "completed_incorporated", + "completed_at": { + "availability": "available", + "value": "2026-07-11T05:13:35Z" + }, + "findings": [ + "OpenAI identifies the underlying API model as gpt-5.6-sol and describes Ultra as four-agent orchestration by default.", + "The current release reports Ultra results for BrowseComp, SEC-Bench Pro, and Terminal-Bench 2.1.", + "Provider cost and latency values are offline simulations; latency is attributed to the root agent while cost and output tokens aggregate all agents.", + "No independent four-agent Ultra replication was found in the named sources during the dated search window.", + "Output tokens and estimated API cost are measured proxies; physical aggregate compute is not directly measured." + ], + "artifacts": [ + "receipts/2026-07-11-gpt-5.6-sol-ultra-claim-verification.md", + "data/gpt-5.6-sol-ultra/benchmark_dataset.v0.1.json", + "docs/evaluations/2026-07-11-gpt-5.6-sol-ultra-evidence-packet.md" + ], + "contribution": "Established the public claim boundary, verified all current scores and methodology footnotes, separated base-model evidence from Ultra replication, and corrected the compute-proxy language.", + "duplicate_work": [ + "Independently reproduced the six archived max rows and their arithmetic as an intentional cross-check of the archive-provenance lane." + ], + "rejected_or_superseded_findings": [ + "Rejected a complete-public-record claim in favor of a named-source dated inventory.", + "Rejected single-agent Sol evidence as independent Ultra validation.", + "Rejected universal faster and directly measured physical-compute claims." + ], + "unresolved_limitations": [ + "Negative searches remain time-sensitive and do not prove universal absence.", + "The two current-versus-archived single-agent baseline discrepancies remain unexplained." + ] + }, + { + "lane_id": "archive-extraction-and-data-provenance", + "scope": "Retrieve, hash, extract, preserve, reconcile, and validate the archived chart data, including production matrix completeness and cross-artifact economics.", + "assigned_agent": "/root/archive_provenance", + "start_state": "started", + "started_at": "2026-07-11T05:04:36Z", + "final_state": "completed_incorporated", + "completed_at": { + "availability": "available", + "value": "2026-07-11T05:30:25Z" + }, + "findings": [ + "The dated capture contains 120 target flat records across the three benchmark families, expected agent counts, five efforts, and three operational metrics.", + "Terminal-Bench 2.1 has no 16-agent flat chart records in the capture, and that absence is explicit.", + "The decompressed response entity SHA-256 is 242b04c63bd0da245bc2dcf601801ee958deb9ebc08def9193f5c6846fdbf892.", + "The raw field latency_s conflicts with displayed minute labels; preserved companion records also name latency_minutes.", + "Cross-artifact reconstruction matches all archived max values and derived economics." + ], + "artifacts": [ + "tools/extract_gpt56_ultra_archive.py", + "tools/validate_ultra_benchmark_dataset.py", + "data/gpt-5.6-sol-ultra/archive_manifest.json", + "data/gpt-5.6-sol-ultra/archived_chart_records.json", + "tests/test_extract_gpt56_ultra_archive.py", + "tests/test_ultra_benchmark_dataset.py", + "tests/fixtures/gpt56-ultra-archive-sample.html" + ], + "contribution": "Made the archived operational series reproducible, hash-addressed, unit-explicit, complete-matrix checked, and mechanically tied to the normalized dataset.", + "duplicate_work": [ + "Economic reconstruction intentionally duplicates extraction-side expectations so one-sided drift cannot self-validate." + ], + "rejected_or_superseded_findings": [ + "Rejected model-slug-only filtering because a Terminal-Bench one-agent row uses a different slug.", + "Rejected treating latency_s values as seconds because that contradicts the source labels.", + "Superseded permissive partial-matrix production verification with exact semantic-identity checks.", + "Superseded an unpreserved latency_minutes reference with two hashed companion source records." + ], + "unresolved_limitations": [ + "The evidence is provider-reported and archive-byte verification needs the dated capture or an equivalent local copy.", + "The archive hash covers the decompressed HTTP entity rather than a WARC or compressed transport body.", + "Two inaccessible internal provider source locators are retained from the public archive as an explicit publication-review issue." + ] + }, + { + "lane_id": "receipt-schema-validator-and-fixtures", + "scope": "Define the ultra_evaluation_receipt.v0.1 contract and implement deterministic stdlib validation, valid and invalid fixtures, and adversarial semantic checks.", + "assigned_agent": "/root/receipt_schema", + "start_state": "started", + "started_at": "2026-07-11T05:04:36Z", + "final_state": "completed_incorporated", + "completed_at": { + "availability": "available", + "value": "2026-07-11T05:26:33Z" + }, + "findings": [ + "Tagged unavailable objects prevent absent telemetry from carrying fabricated numeric values.", + "Canonical evidence classes prevent base-model evidence from being mislabeled as Ultra replication.", + "The validator rejects non-finite numbers, route-orchestration mismatch, same-route counterfactuals, contradictory observational states, and unsupported empirical policy status.", + "v0.1 policy status remains provisional until a future schema defines machine evidence for empirical validation." + ], + "artifacts": [ + "schemas/ultra_evaluation_receipt.v0.1.schema.json", + "tools/validate_ultra_evaluation_receipt.py", + "fixtures/valid/ultra_evaluation_receipt.v0.1.valid.json", + "fixtures/invalid/ultra_evaluation_receipt.v0.1.*.invalid.json", + "tests/test_ultra_evaluation_receipt.py" + ], + "contribution": "Provided a dependency-free, machine-checkable receipt boundary with honesty and cross-field invariants and stable JSON-path errors.", + "duplicate_work": [], + "rejected_or_superseded_findings": [ + "Rejected ambiguous nullable telemetry in favor of explicit availability objects.", + "Rejected a third-party JSON Schema runtime dependency.", + "Rejected NaN, infinity, numeric overflow, unsupported empirical status, and narrative-only same-route counterfactuals." + ], + "unresolved_limitations": [ + "The validator implements only the Draft 2020-12 keywords used by this schema.", + "It validates structure and internal consistency, not the truth of external claims." + ] + }, + { + "lane_id": "adversarial-review-and-routing-policy", + "scope": "Red-team claims, arithmetic, evidence typing, archive reproducibility, receipt honesty, publication risk, test blind spots, and the provisional route policy.", + "assigned_agent": "/root/official_claims", + "start_state": "started", + "started_at": "2026-07-11T05:15:32Z", + "final_state": "completed_incorporated", + "completed_at": { + "availability": "available", + "value": "2026-07-11T05:26:32Z" + }, + "findings": [ + "Current headline effort was incorrectly inferred from archived max rows and was changed to unavailable.", + "A not-found replication search was incorrectly typed as replication evidence and was changed to inference targeting the replication class.", + "Non-finite receipt telemetry, route mismatch, and contradictory none_observed details originally passed and were blocked.", + "Production archive matrices and normalized economics needed independent completeness and drift checks.", + "Budget comparisons must distinguish route-native capacity from equal aggregate-budget regimes.", + "Raw public archive records expose inaccessible internal provider locators that require an explicit publication decision." + ], + "artifacts": [ + "docs/route-policy/reasoning-effort-route-policy.v0.1.md", + "docs/evaluations/ultra-evaluation-protocol.v0.1.md", + "receipts/2026-07-11-gpt-5.6-sol-ultra-claim-verification.md" + ], + "contribution": "Converted plausible but weakly enforced claims into tested invariants, corrected evidence typing, tightened the budget protocol, and surfaced the only remaining public-publication review issue.", + "duplicate_work": [ + "Repeated archive extraction, arithmetic, and source checks as intentional independent verification.", + "Re-read the official-source lane evidence for a different adversarial purpose." + ], + "rejected_or_superseded_findings": [ + "Superseded the inferred current headline effort and mislabeled negative replication evidence.", + "Superseded permissive numeric, route, observation, matrix, and cross-artifact validation.", + "Rejected an unqualified equal-budget comparison design." + ], + "unresolved_limitations": [ + "Semgrep was unavailable in the local environment.", + "No xhigh or Max counterfactual run exists, so route superiority remains unmeasured.", + "Negative independent-replication searches remain time-sensitive." + ] + } + ], + "evidence_classes": [ + { + "evidence_class": "official_provider_result", + "scope": "Model identity, Ultra definition, current headline scores, four-agent default, and methodology footnotes.", + "source_ids": [ + "openai_gpt_5_6_release", + "openai_gpt_5_6_sol_model_page", + "openai_codex_model_guidance" + ] + }, + { + "evidence_class": "archived_chart_data", + "scope": "Hashed provider chart records and companion unit evidence from the dated capture.", + "source_ids": [ + "openai_gpt_5_6_release_wayback", + "data/gpt-5.6-sol-ultra/archive_manifest.json", + "data/gpt-5.6-sol-ultra/archived_chart_records.json" + ] + }, + { + "evidence_class": "independent_base_model_evidence", + "scope": "Third-party Sol results and capability caveats that do not reproduce four-agent Ultra.", + "source_ids": [ + "artificial_analysis_terminal_bench_2_1", + "vals_benchmarks", + "arc_prize_gpt_5_6", + "metr_gpt_5_6_sol" + ] + }, + { + "evidence_class": "local_execution_receipt", + "scope": "Observed collaboration topology, lane states, repository artifacts, and local validation results for this run.", + "source_ids": [ + "receipts/2026-07-11-gpt-5.6-sol-ultra-run.json", + "tests/", + "local-git-worktree" + ] + }, + { + "evidence_class": "inference", + "scope": "Derived max-to-max economics, bounded negative-search conclusion, provisional routing interpretation, and qualitative run assessment.", + "source_ids": [ + "data/gpt-5.6-sol-ultra/benchmark_dataset.v0.1.json", + "docs/route-policy/reasoning-effort-route-policy.v0.1.md", + "docs/evaluations/ultra-evaluation-protocol.v0.1.md" + ] + } + ], + "telemetry": { + "aggregate_cost": { + "availability": "unavailable", + "reason": "The collaboration runtime did not expose aggregate model cost for this run." + }, + "aggregate_tokens": { + "availability": "unavailable", + "reason": "The collaboration runtime did not expose aggregate model token usage for this run." + }, + "elapsed_wall_clock_time": { + "availability": "unavailable", + "reason": "No authoritative session-level start/end timer was exposed; lane timestamps do not bound the complete user-visible run." + } + }, + "duplicate_work": { + "availability": "available", + "assessment": "observed", + "details": [ + "The official-source and archive-provenance lanes independently reconstructed the archived max rows and arithmetic as a deliberate cross-check.", + "The adversarial lane repeated source, archive, and arithmetic checks and re-read earlier evidence for a distinct review purpose.", + "No quantitative duplication rate or cost was estimated because the runtime exposed no such telemetry." + ] + }, + "synthesis_loss": { + "availability": "available", + "assessment": "none_observed", + "details": [] + }, + "validation_performed": [ + { + "name": "Local unit and contract suite", + "status": "passed", + "command": "python3 -m unittest discover -s tests -v", + "result": "All 41 tests passed, including archive matrix, cross-artifact economics, packet, receipt-contract, adversarial invalid-fixture, and exact-run receipt checks.", + "artifact_refs": [ + "tests/test_extract_gpt56_ultra_archive.py", + "tests/test_ultra_benchmark_dataset.py", + "tests/test_ultra_evaluation_packet.py", + "tests/test_ultra_evaluation_receipt.py" + ] + }, + { + "name": "Cross-artifact benchmark reconciliation", + "status": "passed", + "command": "python3 tools/validate_ultra_benchmark_dataset.py", + "result": "The checked-in 120-record archive, manifest hashes and sources, three normalized comparisons, and derived economics reconciled.", + "artifact_refs": [ + "tools/validate_ultra_benchmark_dataset.py", + "data/gpt-5.6-sol-ultra/benchmark_dataset.v0.1.json" + ] + }, + { + "name": "Fresh archive reproduction", + "status": "passed", + "command": "curl --compressed --fail --location https://web.archive.org/web/20260710150348id_/https://openai.com/index/gpt-5-6/ | python3 tools/extract_gpt56_ultra_archive.py --archive-html - --retrieval-date 2026-07-11 --verify", + "result": "The fresh dated capture reproduced and verified the checked-in records and manifest.", + "artifact_refs": [ + "tools/extract_gpt56_ultra_archive.py", + "data/gpt-5.6-sol-ultra/archive_manifest.json" + ] + }, + { + "name": "Official-source and adversarial claim review", + "status": "passed", + "command": "claim-verify plus read-only adversarial review", + "result": "Official claims and arithmetic were supported; overclaims and validation gaps were corrected or preserved as explicit limitations.", + "artifact_refs": [ + "receipts/2026-07-11-gpt-5.6-sol-ultra-claim-verification.md" + ] + }, + { + "name": "Exact-run receipt validation", + "status": "passed", + "command": "python3 tools/validate_ultra_evaluation_receipt.py receipts/2026-07-11-gpt-5.6-sol-ultra-run.json", + "result": "The exact receipt validated with four completed lanes and explicitly unavailable cost, token, and wall-clock telemetry.", + "artifact_refs": [ + "receipts/2026-07-11-gpt-5.6-sol-ultra-run.json", + "schemas/ultra_evaluation_receipt.v0.1.schema.json" + ] + }, + { + "name": "Python compatibility and formatting", + "status": "passed", + "command": "python3.11 -m unittest discover -s tests -v; python3.13 -m unittest discover -s tests -v; ruff check tools tests; ruff format --check tools tests", + "result": "All 41 tests passed independently on Python 3.11 and 3.13; Ruff lint and format checks passed for seven Python files.", + "artifact_refs": [ + "tools/", + "tests/" + ] + }, + { + "name": "Branch coverage", + "status": "passed", + "command": "python3.13 -m coverage run --branch --source=tools -m unittest discover -s tests && python3.13 -m coverage report -m", + "result": "The 41-test suite passed under coverage; combined tool coverage was 77%, including 77% extractor, 86% cross-artifact validator, and 73% receipt validator coverage.", + "artifact_refs": [ + "tools/extract_gpt56_ultra_archive.py", + "tools/validate_ultra_benchmark_dataset.py", + "tools/validate_ultra_evaluation_receipt.py" + ] + }, + { + "name": "Strict data and security checks", + "status": "passed", + "command": "strict Decimal JSON parse; python3 -m compileall -q tools tests; bandit -q -r tools -lll; credential-pattern scan; git diff --check", + "result": "Nineteen JSON files parsed strictly, compilation and whitespace checks passed, Bandit reported no high-severity finding, and no credential pattern was detected.", + "artifact_refs": [ + "data/", + "fixtures/", + "receipts/", + "schemas/", + "tools/" + ] + }, + { + "name": "Outbound content review", + "status": "passed", + "command": "content-review over the complete packet", + "result": "Factual, tone, and secret checks passed; public publication is held pending an authorized decision on two distinct internal provider locators repeated by the raw archived records.", + "artifact_refs": [ + "receipts/2026-07-11-gpt-5.6-sol-ultra-content-review.md" + ] + }, + { + "name": "GitHub Actions", + "status": "not_run", + "command": "not invoked", + "result": "The organization has no remaining GitHub Actions minutes; all validation was local.", + "artifact_refs": [] + }, + { + "name": "Semgrep", + "status": "not_run", + "command": "semgrep --config auto tools tests", + "result": "Semgrep was unavailable in the local environment.", + "artifact_refs": [] + } + ], + "outcome_improvement": { + "assessment": "benefited", + "basis": "Three independent research and implementation lanes produced incorporated artifacts in parallel, and a separate adversarial pass found material evidence-model, validator, matrix, and protocol defects before handoff. This is an observed coverage and defect-detection benefit, not proof that Ultra was faster or more cost-effective than xhigh or Max.", + "observed_improvements": [ + "Official claims, archive provenance, and receipt validation progressed under non-overlapping ownership boundaries.", + "Independent arithmetic reconstruction agreed across lanes and now has a cross-artifact drift validator.", + "Adversarial review converted multiple silent honesty and completeness failures into deterministic tests." + ], + "unavailable_measurements": [ + "Counterfactual xhigh task success", + "Counterfactual Max task success", + "Counterfactual elapsed wall-clock time", + "Counterfactual and actual aggregate cost", + "Counterfactual and actual aggregate token usage" + ] + }, + "routing_decision": { + "route_used": "ultra", + "recommendation": "conditional", + "recommended_route": "ultra", + "rationale": "For a fresh implementation of this same decomposable, consequential packet, Ultra is conditionally appropriate because all independent lanes contributed and the adversarial lane materially changed the result. This does not establish a universal default or measured efficiency advantage.", + "counterfactual_route": "max", + "counterfactual_rationale": "Max is the strongest single-agent counterfactual and could preserve coherent synthesis, but would serialize distinct source, archive, and schema work; no controlled counterfactual was run.", + "policy_status": "provisional" + }, + "limitations": [ + "The session route and base model were identified by the task context, not authoritative API response metadata.", + "Aggregate cost, tokens, and authoritative session wall-clock time were unavailable and were not estimated.", + "No identical xhigh or Max counterfactual was run, so route superiority and marginal improvement per dollar remain unmeasured.", + "No independent four-agent Ultra replication was found in the named sources during the dated search window; this is not a universal absence proof.", + "Provider benchmark cost and latency are simulations rather than observed public-service guarantees.", + "The archive contains inaccessible internal provider source locators with no detected credentials; public publication requires an explicit provenance/privacy decision.", + "Outbound content review placed public publication on hold pending that provenance/privacy decision.", + "The archive lane's adversarial follow-up dispatch timestamp was not exposed, although its initial start and final completion timestamps were available.", + "Semgrep was unavailable, and GitHub Actions was not invoked." + ] +} diff --git a/schemas/ultra_evaluation_receipt.v0.1.schema.json b/schemas/ultra_evaluation_receipt.v0.1.schema.json new file mode 100644 index 0000000..0ecca7c --- /dev/null +++ b/schemas/ultra_evaluation_receipt.v0.1.schema.json @@ -0,0 +1,612 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://raw.githubusercontent.com/hummbl-dev/model-routing-as-code/main/schemas/ultra_evaluation_receipt.v0.1.schema.json", + "title": "Ultra Evaluation Receipt v0.1", + "description": "Governed execution receipt for an evaluated model route, including explicit unavailable telemetry and multi-agent delegation outcomes.", + "type": "object", + "required": [ + "schema_id", + "schema_version", + "receipt_id", + "recorded_at", + "model", + "task", + "delegation_lanes", + "evidence_classes", + "telemetry", + "duplicate_work", + "synthesis_loss", + "validation_performed", + "outcome_improvement", + "routing_decision", + "limitations" + ], + "properties": { + "schema_id": { + "type": "string", + "const": "ultra_evaluation_receipt.v0.1" + }, + "schema_version": { + "type": "string", + "const": "0.1" + }, + "receipt_id": { + "type": "string", + "pattern": "^ultra-eval-[a-z0-9][a-z0-9._-]*$" + }, + "recorded_at": { + "type": "string", + "format": "date-time" + }, + "model": { + "$ref": "#/$defs/model" + }, + "task": { + "$ref": "#/$defs/task" + }, + "delegation_lanes": { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/$defs/delegationLane" + } + }, + "evidence_classes": { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/$defs/evidenceEntry" + } + }, + "telemetry": { + "type": "object", + "required": [ + "aggregate_cost", + "aggregate_tokens", + "elapsed_wall_clock_time" + ], + "properties": { + "aggregate_cost": { + "$ref": "#/$defs/numericTelemetry" + }, + "aggregate_tokens": { + "$ref": "#/$defs/numericTelemetry" + }, + "elapsed_wall_clock_time": { + "$ref": "#/$defs/numericTelemetry" + } + }, + "additionalProperties": false + }, + "duplicate_work": { + "$ref": "#/$defs/observationalAssessment" + }, + "synthesis_loss": { + "$ref": "#/$defs/observationalAssessment" + }, + "validation_performed": { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/$defs/validationRecord" + } + }, + "outcome_improvement": { + "$ref": "#/$defs/outcomeImprovement" + }, + "routing_decision": { + "$ref": "#/$defs/routingDecision" + }, + "limitations": { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/$defs/nonEmptyString" + } + } + }, + "additionalProperties": false, + "$defs": { + "nonEmptyString": { + "type": "string", + "minLength": 1 + }, + "route": { + "type": "string", + "enum": [ + "medium", + "high", + "xhigh", + "max", + "ultra" + ] + }, + "evidenceClass": { + "type": "string", + "enum": [ + "official_provider_result", + "independent_base_model_evidence", + "independent_ultra_replication", + "archived_chart_data", + "local_execution_receipt", + "inference" + ] + }, + "timestampAvailability": { + "oneOf": [ + { + "type": "object", + "required": [ + "availability", + "value" + ], + "properties": { + "availability": { + "const": "available" + }, + "value": { + "type": "string", + "format": "date-time" + } + }, + "additionalProperties": false + }, + { + "type": "object", + "required": [ + "availability", + "reason" + ], + "properties": { + "availability": { + "const": "unavailable" + }, + "reason": { + "$ref": "#/$defs/nonEmptyString" + } + }, + "additionalProperties": false + } + ] + }, + "agentCountAvailability": { + "oneOf": [ + { + "type": "object", + "required": [ + "availability", + "value", + "measurement_kind" + ], + "properties": { + "availability": { + "const": "available" + }, + "value": { + "type": "integer", + "minimum": 1 + }, + "measurement_kind": { + "type": "string", + "enum": [ + "configured", + "observed" + ] + } + }, + "additionalProperties": false + }, + { + "type": "object", + "required": [ + "availability", + "reason" + ], + "properties": { + "availability": { + "const": "unavailable" + }, + "reason": { + "$ref": "#/$defs/nonEmptyString" + } + }, + "additionalProperties": false + } + ] + }, + "numericTelemetry": { + "oneOf": [ + { + "type": "object", + "required": [ + "availability", + "value", + "unit", + "measurement_kind" + ], + "properties": { + "availability": { + "const": "available" + }, + "value": { + "type": "number", + "minimum": 0, + "description": "A finite JSON number. NaN, Infinity, and host-number overflow are invalid." + }, + "unit": { + "type": "string", + "enum": [ + "USD", + "tokens", + "seconds" + ] + }, + "measurement_kind": { + "type": "string", + "enum": [ + "observed", + "provider_estimate" + ] + } + }, + "additionalProperties": false + }, + { + "type": "object", + "required": [ + "availability", + "reason" + ], + "properties": { + "availability": { + "const": "unavailable" + }, + "reason": { + "$ref": "#/$defs/nonEmptyString" + } + }, + "additionalProperties": false + } + ] + }, + "observationalAssessment": { + "oneOf": [ + { + "type": "object", + "required": [ + "availability", + "assessment", + "details" + ], + "properties": { + "availability": { + "const": "available" + }, + "assessment": { + "type": "string", + "enum": [ + "none_observed", + "observed" + ] + }, + "details": { + "type": "array", + "items": { + "$ref": "#/$defs/nonEmptyString" + } + } + }, + "additionalProperties": false + }, + { + "type": "object", + "required": [ + "availability", + "reason" + ], + "properties": { + "availability": { + "const": "unavailable" + }, + "reason": { + "$ref": "#/$defs/nonEmptyString" + } + }, + "additionalProperties": false + } + ] + }, + "model": { + "type": "object", + "required": [ + "provider", + "base_model_id", + "orchestration_setting", + "orchestration_agent_count", + "configuration_source" + ], + "properties": { + "provider": { + "$ref": "#/$defs/nonEmptyString" + }, + "base_model_id": { + "$ref": "#/$defs/nonEmptyString" + }, + "orchestration_setting": { + "$ref": "#/$defs/route" + }, + "orchestration_agent_count": { + "$ref": "#/$defs/agentCountAvailability" + }, + "configuration_source": { + "$ref": "#/$defs/nonEmptyString" + } + }, + "additionalProperties": false + }, + "task": { + "type": "object", + "required": [ + "task_id", + "title", + "objective", + "decomposable", + "decomposition_rationale", + "success_criteria" + ], + "properties": { + "task_id": { + "type": "string", + "pattern": "^[a-z0-9][a-z0-9._-]*$" + }, + "title": { + "$ref": "#/$defs/nonEmptyString" + }, + "objective": { + "$ref": "#/$defs/nonEmptyString" + }, + "decomposable": { + "type": "boolean" + }, + "decomposition_rationale": { + "$ref": "#/$defs/nonEmptyString" + }, + "success_criteria": { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/$defs/nonEmptyString" + } + } + }, + "additionalProperties": false + }, + "delegationLane": { + "type": "object", + "required": [ + "lane_id", + "scope", + "assigned_agent", + "start_state", + "started_at", + "final_state", + "completed_at", + "findings", + "artifacts", + "contribution", + "duplicate_work", + "rejected_or_superseded_findings", + "unresolved_limitations" + ], + "properties": { + "lane_id": { + "type": "string", + "pattern": "^[a-z0-9][a-z0-9._-]*$" + }, + "scope": { + "$ref": "#/$defs/nonEmptyString" + }, + "assigned_agent": { + "$ref": "#/$defs/nonEmptyString" + }, + "start_state": { + "type": "string", + "enum": [ + "assigned", + "started" + ] + }, + "started_at": { + "type": "string", + "format": "date-time" + }, + "final_state": { + "type": "string", + "enum": [ + "completed_incorporated", + "completed_not_incorporated", + "interrupted", + "failed", + "outstanding" + ] + }, + "completed_at": { + "$ref": "#/$defs/timestampAvailability" + }, + "findings": { + "type": "array", + "items": { + "$ref": "#/$defs/nonEmptyString" + } + }, + "artifacts": { + "type": "array", + "items": { + "$ref": "#/$defs/nonEmptyString" + } + }, + "contribution": { + "$ref": "#/$defs/nonEmptyString" + }, + "duplicate_work": { + "type": "array", + "items": { + "$ref": "#/$defs/nonEmptyString" + } + }, + "rejected_or_superseded_findings": { + "type": "array", + "items": { + "$ref": "#/$defs/nonEmptyString" + } + }, + "unresolved_limitations": { + "type": "array", + "items": { + "$ref": "#/$defs/nonEmptyString" + } + } + }, + "additionalProperties": false + }, + "evidenceEntry": { + "type": "object", + "required": [ + "evidence_class", + "scope", + "source_ids" + ], + "properties": { + "evidence_class": { + "$ref": "#/$defs/evidenceClass" + }, + "scope": { + "$ref": "#/$defs/nonEmptyString" + }, + "source_ids": { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/$defs/nonEmptyString" + } + } + }, + "additionalProperties": false + }, + "validationRecord": { + "type": "object", + "required": [ + "name", + "status", + "command", + "result", + "artifact_refs" + ], + "properties": { + "name": { + "$ref": "#/$defs/nonEmptyString" + }, + "status": { + "type": "string", + "enum": [ + "passed", + "failed", + "not_run" + ] + }, + "command": { + "$ref": "#/$defs/nonEmptyString" + }, + "result": { + "$ref": "#/$defs/nonEmptyString" + }, + "artifact_refs": { + "type": "array", + "items": { + "$ref": "#/$defs/nonEmptyString" + } + } + }, + "additionalProperties": false + }, + "outcomeImprovement": { + "type": "object", + "required": [ + "assessment", + "basis", + "observed_improvements", + "unavailable_measurements" + ], + "properties": { + "assessment": { + "type": "string", + "enum": [ + "benefited", + "did_not_benefit", + "inconclusive", + "unavailable" + ] + }, + "basis": { + "$ref": "#/$defs/nonEmptyString" + }, + "observed_improvements": { + "type": "array", + "items": { + "$ref": "#/$defs/nonEmptyString" + } + }, + "unavailable_measurements": { + "type": "array", + "items": { + "$ref": "#/$defs/nonEmptyString" + } + } + }, + "additionalProperties": false + }, + "routingDecision": { + "type": "object", + "required": [ + "route_used", + "recommendation", + "recommended_route", + "rationale", + "counterfactual_route", + "counterfactual_rationale", + "policy_status" + ], + "properties": { + "route_used": { + "$ref": "#/$defs/route" + }, + "recommendation": { + "type": "string", + "enum": [ + "select", + "reject", + "conditional" + ] + }, + "recommended_route": { + "$ref": "#/$defs/route" + }, + "rationale": { + "$ref": "#/$defs/nonEmptyString" + }, + "counterfactual_route": { + "$ref": "#/$defs/route" + }, + "counterfactual_rationale": { + "$ref": "#/$defs/nonEmptyString" + }, + "policy_status": { + "type": "string", + "const": "provisional", + "description": "v0.1 remains provisional. A future schema revision may define machine-evidence requirements for empirical validation." + } + }, + "additionalProperties": false + } + } +} diff --git a/tests/fixtures/gpt56-ultra-archive-sample.html b/tests/fixtures/gpt56-ultra-archive-sample.html new file mode 100644 index 0000000..17e4ae4 --- /dev/null +++ b/tests/fixtures/gpt56-ultra-archive-sample.html @@ -0,0 +1,18 @@ + + + + + + diff --git a/tests/test_extract_gpt56_ultra_archive.py b/tests/test_extract_gpt56_ultra_archive.py new file mode 100644 index 0000000..d094941 --- /dev/null +++ b/tests/test_extract_gpt56_ultra_archive.py @@ -0,0 +1,222 @@ +from __future__ import annotations + +import copy +import hashlib +import importlib.util +import json +from datetime import date +from pathlib import Path +import tempfile +import unittest + + +ROOT = Path(__file__).resolve().parents[1] +SCRIPT = ROOT / "tools" / "extract_gpt56_ultra_archive.py" +FIXTURE = ROOT / "tests" / "fixtures" / "gpt56-ultra-archive-sample.html" + + +def load_extractor(): + spec = importlib.util.spec_from_file_location("extract_gpt56_ultra_archive", SCRIPT) + if spec is None or spec.loader is None: + raise RuntimeError(f"could not import {SCRIPT}") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +class ExtractArchiveTests(unittest.TestCase): + @classmethod + def setUpClass(cls) -> None: + cls.extractor = load_extractor() + cls.archive_bytes = FIXTURE.read_bytes() + cls.archive_text = cls.archive_bytes.decode("utf-8") + + def test_selects_target_records_and_deduplicates_exact_repeats(self) -> None: + records = self.extractor.extract_selected_records(self.archive_text) + + self.assertEqual(len(records), 8) + identity = [ + ( + row["eval"], + row["eval_variant"], + row["juice_level"], + row["x_metric"], + ) + for row in records + ] + self.assertEqual( + identity, + [ + ("BrowseComp", "1 agent", "max", "api_cost_usd"), + ("BrowseComp", "4 agents", "max", "latency_s"), + ("BrowseComp", "16 agents", "xhigh", "output_tokens"), + ("SEC-Bench Pro", "1 agent", "max", "api_cost_usd"), + ("SEC-Bench Pro", "4 agents", "max", "output_tokens"), + ("SEC-Bench Pro", "16 agents", "max", "latency_s"), + ("Terminal-Bench 2.1", "1 agent", "max", "api_cost_usd"), + ("Terminal-Bench 2.1", "4 agents", "max", "latency_s"), + ], + ) + self.assertEqual( + records[0]["source_url"], + "https://example.test/source?thread=1&record=browse-1", + ) + self.assertNotIn("GPT-5.6 Terra", {row["model"] for row in records}) + + def test_records_document_is_deterministic_and_preserves_raw_rows(self) -> None: + records = self.extractor.extract_selected_records(self.archive_text) + + first = self.extractor.build_records_document(records) + second = self.extractor.build_records_document(reversed(records)) + + self.assertEqual(first, second) + self.assertEqual(first["evidence_class"], "archived_chart_data") + self.assertEqual(first["record_count"], 8) + self.assertEqual(first["records"][1]["x_label"], "6.58 minutes") + self.assertEqual( + first["availability"]["Terminal-Bench 2.1"]["missing_agent_counts"], + [16], + ) + + def test_manifest_hash_units_and_reconciliation_are_explicit(self) -> None: + records = self.extractor.extract_selected_records(self.archive_text) + document = self.extractor.build_records_document(records) + + manifest = self.extractor.build_manifest( + self.archive_bytes, + document, + retrieval_date=date(2026, 7, 11), + ) + + self.assertEqual( + manifest["archive"]["content_sha256"], + hashlib.sha256(self.archive_bytes).hexdigest(), + ) + self.assertEqual(manifest["archive"]["capture_timestamp"], "20260710150348") + self.assertEqual(manifest["archive"]["retrieval_date"], "2026-07-11") + latency = manifest["unit_interpretation"]["latency_s"] + self.assertEqual(latency["raw_metric_name"], "latency_s") + self.assertEqual(latency["x_value_unit"], "minutes") + self.assertIn("minutes", latency["interpretation_basis"][0]) + companion = latency["companion_source_evidence"] + self.assertEqual(len(companion), 1) + self.assertEqual(companion[0]["record"]["latency_minutes"], 6.581) + expected_hash = hashlib.sha256( + self.extractor.canonical_json_bytes(companion[0]["record"]) + ).hexdigest() + self.assertEqual(companion[0]["canonical_record_sha256"], expected_hash) + browse = manifest["score_reconciliation"]["BrowseComp"] + self.assertAlmostEqual( + browse["archived_max"]["1_agent"]["score_percent"], + 90.83728278041074, + ) + self.assertFalse( + browse["current_headline"]["1_agent"][ + "agrees_after_rounding_to_one_decimal" + ] + ) + self.assertTrue( + browse["current_headline"]["4_agents"][ + "agrees_after_rounding_to_one_decimal" + ] + ) + + def test_production_records_manifest_detects_hash_tampering(self) -> None: + records_path = ( + ROOT / "data" / "gpt-5.6-sol-ultra" / "archived_chart_records.json" + ) + manifest_path = ROOT / "data" / "gpt-5.6-sol-ultra" / "archive_manifest.json" + document = json.loads(records_path.read_text(encoding="utf-8")) + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + + altered = copy.deepcopy(manifest) + altered["extraction"]["selected_records_sha256"] = "0" * 64 + with self.assertRaisesRegex(ValueError, "records hash"): + self.extractor.verify_records_manifest(document, altered) + + def test_cli_rejects_partial_fixture_but_function_extraction_remains_usable( + self, + ) -> None: + self.assertEqual( + len(self.extractor.extract_selected_records(self.archive_text)), 8 + ) + with tempfile.TemporaryDirectory() as directory: + records_path = Path(directory) / "records.json" + manifest_path = Path(directory) / "manifest.json" + with self.assertRaisesRegex(ValueError, "incomplete production matrix"): + self.extractor.main( + [ + "--archive-html", + str(FIXTURE), + "--retrieval-date", + "2026-07-11", + "--records-output", + str(records_path), + "--manifest-output", + str(manifest_path), + ] + ) + + def test_production_verification_rejects_a_removed_matrix_row(self) -> None: + records_path = ( + ROOT / "data" / "gpt-5.6-sol-ultra" / "archived_chart_records.json" + ) + document = json.loads(records_path.read_text(encoding="utf-8")) + incomplete = self.extractor.build_records_document(document["records"][:-1]) + manifest = self.extractor.build_manifest( + b"archive", incomplete, retrieval_date=date(2026, 7, 11) + ) + + with self.assertRaisesRegex(ValueError, "incomplete production matrix"): + self.extractor.verify_records_manifest(incomplete, manifest) + + def test_production_verification_rejects_conflicting_semantic_identity( + self, + ) -> None: + records_path = ( + ROOT / "data" / "gpt-5.6-sol-ultra" / "archived_chart_records.json" + ) + document = json.loads(records_path.read_text(encoding="utf-8")) + conflict = copy.deepcopy(document["records"][0]) + conflict["x_value"] += 1 + conflict["x_label"] = "conflicting" + conflicting = self.extractor.build_records_document( + [*document["records"], conflict] + ) + manifest = self.extractor.build_manifest( + b"archive", conflicting, retrieval_date=date(2026, 7, 11) + ) + + with self.assertRaisesRegex(ValueError, "conflicting semantic identity"): + self.extractor.verify_records_manifest(conflicting, manifest) + + def test_checked_in_archive_outputs_are_internally_consistent(self) -> None: + records_path = ( + ROOT / "data" / "gpt-5.6-sol-ultra" / "archived_chart_records.json" + ) + manifest_path = ROOT / "data" / "gpt-5.6-sol-ultra" / "archive_manifest.json" + records_document = json.loads(records_path.read_text(encoding="utf-8")) + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + + self.extractor.verify_records_manifest(records_document, manifest) + self.assertEqual(records_document["record_count"], 120) + self.assertEqual( + records_document["availability"]["Terminal-Bench 2.1"][ + "missing_agent_counts" + ], + [16], + ) + self.assertEqual( + manifest["archive"]["content_sha256"], + "242b04c63bd0da245bc2dcf601801ee958deb9ebc08def9193f5c6846fdbf892", + ) + self.assertTrue( + any( + "two distinct inaccessible internal OpenAI Slack source URLs" in item + for item in manifest["limitations"] + ) + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_ultra_benchmark_dataset.py b/tests/test_ultra_benchmark_dataset.py new file mode 100644 index 0000000..8813f1a --- /dev/null +++ b/tests/test_ultra_benchmark_dataset.py @@ -0,0 +1,144 @@ +from __future__ import annotations + +import copy +from contextlib import redirect_stdout +import hashlib +import importlib.util +import io +import json +from pathlib import Path +import tempfile +import unittest + + +ROOT = Path(__file__).resolve().parents[1] +SCRIPT = ROOT / "tools" / "validate_ultra_benchmark_dataset.py" +DATA_DIR = ROOT / "data" / "gpt-5.6-sol-ultra" +RECORDS_PATH = DATA_DIR / "archived_chart_records.json" +MANIFEST_PATH = DATA_DIR / "archive_manifest.json" +DATASET_PATH = DATA_DIR / "benchmark_dataset.v0.1.json" + + +def load_validator(): + spec = importlib.util.spec_from_file_location( + "validate_ultra_benchmark_dataset", SCRIPT + ) + if spec is None or spec.loader is None: + raise RuntimeError(f"could not import {SCRIPT}") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +class UltraBenchmarkDatasetValidatorTests(unittest.TestCase): + @classmethod + def setUpClass(cls) -> None: + cls.validator = load_validator() + cls.records_bytes = RECORDS_PATH.read_bytes() + cls.records = json.loads(cls.records_bytes) + cls.manifest = json.loads(MANIFEST_PATH.read_text(encoding="utf-8")) + cls.dataset = json.loads(DATASET_PATH.read_text(encoding="utf-8")) + + def test_checked_in_cross_artifact_packet_validates(self) -> None: + report = self.validator.validate_paths( + RECORDS_PATH, MANIFEST_PATH, DATASET_PATH + ) + + self.assertEqual(report["record_count"], 120) + self.assertEqual(report["comparisons_checked"], 3) + self.assertEqual( + report["records_sha256"], hashlib.sha256(self.records_bytes).hexdigest() + ) + + def test_cli_validates_explicit_local_paths(self) -> None: + output = io.StringIO() + with redirect_stdout(output): + result = self.validator.main( + [ + "--records", + str(RECORDS_PATH), + "--manifest", + str(MANIFEST_PATH), + "--dataset", + str(DATASET_PATH), + ] + ) + + self.assertEqual(result, 0) + self.assertEqual(json.loads(output.getvalue())["status"], "valid") + + def test_raw_record_hash_drift_is_rejected(self) -> None: + altered = copy.deepcopy(self.records) + altered["records"][0]["x_value"] += 1 + + with tempfile.TemporaryDirectory() as directory: + records_path = Path(directory) / "records.json" + records_path.write_text(json.dumps(altered), encoding="utf-8") + with self.assertRaisesRegex( + self.validator.DatasetValidationError, "records SHA-256" + ): + self.validator.validate_paths(records_path, MANIFEST_PATH, DATASET_PATH) + + def test_source_reference_drift_is_rejected(self) -> None: + altered = copy.deepcopy(self.dataset) + source = next( + item + for item in altered["sources"] + if item["source_id"] == "openai_gpt_5_6_release_wayback" + ) + source["url"] = "https://example.test/drifted" + + with self.assertRaisesRegex( + self.validator.DatasetValidationError, "Wayback source URL" + ): + self.validator.validate_documents( + self.records, + self.manifest, + altered, + records_sha256=hashlib.sha256(self.records_bytes).hexdigest(), + ) + + def test_raw_max_value_drift_is_caught_even_with_updated_hash(self) -> None: + altered_records = copy.deepcopy(self.records) + row = next( + item + for item in altered_records["records"] + if item["eval"] == "BrowseComp" + and item["eval_variant"] == "1 agent" + and item["juice_level"] == "max" + and item["x_metric"] == "api_cost_usd" + ) + row["x_value"] += 1 + altered_bytes = self.validator.canonical_json_bytes(altered_records) + altered_manifest = copy.deepcopy(self.manifest) + altered_manifest["extraction"]["selected_records_sha256"] = hashlib.sha256( + altered_bytes + ).hexdigest() + + with self.assertRaisesRegex( + self.validator.DatasetValidationError, "single_agent.cost" + ): + self.validator.validate_documents( + altered_records, + altered_manifest, + self.dataset, + records_sha256=hashlib.sha256(altered_bytes).hexdigest(), + ) + + def test_derived_economics_drift_is_rejected(self) -> None: + altered = copy.deepcopy(self.dataset) + altered["archived_max_comparisons"][0]["derived"]["cost_multiplier"] = 9.99 + + with self.assertRaisesRegex( + self.validator.DatasetValidationError, "derived.cost_multiplier" + ): + self.validator.validate_documents( + self.records, + self.manifest, + altered, + records_sha256=hashlib.sha256(self.records_bytes).hexdigest(), + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_ultra_evaluation_packet.py b/tests/test_ultra_evaluation_packet.py new file mode 100644 index 0000000..cd72978 --- /dev/null +++ b/tests/test_ultra_evaluation_packet.py @@ -0,0 +1,183 @@ +"""Contract tests for the bounded GPT-5.6 Sol Ultra evaluation packet.""" + +from __future__ import annotations + +import json +import unittest +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] +DATASET_PATH = ROOT / "data" / "gpt-5.6-sol-ultra" / "benchmark_dataset.v0.1.json" +EVIDENCE_PACKET_PATH = ( + ROOT / "docs" / "evaluations" / "2026-07-11-gpt-5.6-sol-ultra-evidence-packet.md" +) +ROUTE_POLICY_PATH = ( + ROOT / "docs" / "route-policy" / "reasoning-effort-route-policy.v0.1.md" +) +PROTOCOL_PATH = ROOT / "docs" / "evaluations" / "ultra-evaluation-protocol.v0.1.md" + + +class UltraEvaluationDatasetTests(unittest.TestCase): + @classmethod + def setUpClass(cls) -> None: + cls.dataset = json.loads(DATASET_PATH.read_text(encoding="utf-8")) + + def test_inventory_is_explicitly_search_bounded(self) -> None: + inventory = self.dataset["inventory"] + self.assertEqual(inventory["scope"], "search_bounded_public_inventory") + self.assertEqual(inventory["checked_on"], "2026-07-11") + self.assertGreaterEqual(len(inventory["named_sources_checked"]), 3) + self.assertNotIn("complete universal record", inventory["claim"].lower()) + + def test_evidence_classes_are_canonical_and_complete(self) -> None: + self.assertEqual( + set(self.dataset["evidence_classes"]), + { + "official_provider_result", + "independent_base_model_evidence", + "independent_ultra_replication", + "archived_chart_data", + "local_execution_receipt", + "inference", + }, + ) + + def test_model_identity_does_not_invent_an_ultra_model(self) -> None: + identity = self.dataset["model_identity"] + self.assertEqual(identity["base_model_id"], "gpt-5.6-sol") + self.assertEqual(identity["orchestration_setting"], "ultra") + self.assertEqual(identity["default_agent_count"], 4) + self.assertFalse(identity["is_distinct_base_model"]) + + def test_three_current_headline_results_are_kept_separate(self) -> None: + results = { + item["benchmark_id"]: item for item in self.dataset["headline_results"] + } + self.assertEqual( + set(results), {"browsecomp", "sec_bench_pro", "terminal_bench_2_1"} + ) + expected = { + "browsecomp": (0.922, 0.904, 1.8), + "sec_bench_pro": (0.743, 0.712, 3.1), + "terminal_bench_2_1": (0.919, 0.888, 3.1), + } + for benchmark_id, values in expected.items(): + result = results[benchmark_id] + self.assertEqual(result["evidence_class"], "official_provider_result") + self.assertEqual(result["configuration"]["agent_count"], 4) + effort = result["configuration"]["effort"] + self.assertEqual(effort["availability"], "unavailable") + self.assertTrue(effort["reason"]) + self.assertEqual(result["ultra_score"]["value"], values[0]) + self.assertEqual(result["single_agent_score"]["value"], values[1]) + self.assertEqual(result["score_gain"]["value"], values[2]) + self.assertEqual(result["score_gain"]["unit"], "percentage_points") + + def test_every_named_search_surface_has_dated_source_metadata(self) -> None: + sources = {item["source_id"]: item for item in self.dataset["sources"]} + self.assertEqual( + set(self.dataset["inventory"]["named_sources_checked"]), set(sources) + ) + for source_id in self.dataset["inventory"]["named_sources_checked"]: + with self.subTest(source_id=source_id): + self.assertIn(source_id, sources) + self.assertTrue(sources[source_id]["url"].startswith("https://")) + self.assertEqual(sources[source_id]["retrieved_on"], "2026-07-11") + self.assertIn( + sources[source_id]["evidence_class"], + self.dataset["evidence_classes"], + ) + + def test_headline_operational_fields_are_explicitly_unavailable(self) -> None: + expected_units = { + "cost": "usd", + "output_tokens": "tokens", + "latency": "minutes", + } + for result in self.dataset["headline_results"]: + for field, unit in expected_units.items(): + with self.subTest(benchmark=result["benchmark_id"], field=field): + metric = result[field] + self.assertEqual(metric["availability"], "unavailable") + self.assertEqual(metric["unit"], unit) + self.assertTrue(metric["reason"]) + self.assertNotIn("value", metric) + + def test_archived_economics_are_max_to_max_and_reconciled(self) -> None: + comparisons = { + item["benchmark_id"]: item + for item in self.dataset["archived_max_comparisons"] + } + expected = { + "browsecomp": (1.34, 2.93, 2.46, -17.7), + "sec_bench_pro": (2.87, 2.10, 2.08, -42.4), + "terminal_bench_2_1": (3.15, 2.99, 3.07, 0.9), + } + for benchmark_id, values in expected.items(): + comparison = comparisons[benchmark_id] + self.assertEqual( + comparison["comparison_scope"], "archived_max_to_archived_max" + ) + self.assertEqual(comparison["evidence_class"], "archived_chart_data") + derived = comparison["derived"] + self.assertEqual(derived["score_gain_percentage_points"], values[0]) + self.assertEqual(derived["cost_multiplier"], values[1]) + self.assertEqual(derived["output_token_multiplier"], values[2]) + self.assertEqual(derived["latency_change_percent"], values[3]) + self.assertIn( + comparison["reconciliation"]["status"], + {"agrees_after_rounding", "operational_baseline_differs_do_not_mix"}, + ) + + def test_simulated_metrics_and_missing_values_are_honest(self) -> None: + for comparison in self.dataset["archived_max_comparisons"]: + for route in ("single_agent", "ultra_four_agent"): + metrics = comparison[route] + self.assertEqual(metrics["latency"]["unit"], "minutes") + self.assertEqual(metrics["cost"]["unit"], "usd") + self.assertEqual(metrics["output_tokens"]["unit"], "tokens") + self.assertTrue(metrics["latency"]["is_provider_estimate"]) + self.assertTrue(metrics["cost"]["is_provider_estimate"]) + self.assertTrue(self.dataset["missing_or_unavailable_fields"]) + + def test_no_independent_ultra_replication_is_overclaimed(self) -> None: + status = self.dataset["independent_verification"] + self.assertEqual(status["search_evidence_class"], "inference") + self.assertEqual( + status["target_evidence_class"], "independent_ultra_replication" + ) + self.assertEqual(status["status"], "not_found_in_named_sources") + self.assertEqual(status["checked_on"], "2026-07-11") + self.assertIn("search window", status["claim_boundary"].lower()) + + +class UltraEvaluationDocumentationTests(unittest.TestCase): + def test_evidence_packet_preserves_bounded_language(self) -> None: + text = EVIDENCE_PACKET_PATH.read_text(encoding="utf-8") + self.assertIn("search-bounded public inventory", text) + self.assertIn("No independent replication", text) + self.assertIn("simulated", text.lower()) + self.assertIn("2–3× aggregate compute", text) + + def test_route_policy_is_provisional_and_rejects_bad_ultra_routes(self) -> None: + text = ROUTE_POLICY_PATH.read_text(encoding="utf-8") + self.assertIn("Non-canon status", text) + self.assertIn("medium → high → xhigh", text) + self.assertIn("tightly coupled", text) + self.assertIn("decomposable", text) + self.assertIn("Reject Ultra", text) + + def test_future_protocol_holds_task_and_evidence_constant(self) -> None: + text = PROTOCOL_PATH.read_text(encoding="utf-8") + self.assertIn("xhigh", text) + self.assertIn("Max", text) + self.assertIn("Ultra", text) + self.assertIn("identical task", text) + self.assertIn("success criteria", text) + self.assertIn("evidence standard", text) + self.assertIn("No GitHub Actions", text) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_ultra_evaluation_receipt.py b/tests/test_ultra_evaluation_receipt.py new file mode 100644 index 0000000..46522a7 --- /dev/null +++ b/tests/test_ultra_evaluation_receipt.py @@ -0,0 +1,307 @@ +"""Tests for the governed Ultra evaluation receipt contract.""" + +from __future__ import annotations + +import importlib.util +import json +from pathlib import Path +import subprocess +import sys +import tempfile +import unittest + + +ROOT = Path(__file__).resolve().parents[1] +SCHEMA_PATH = ROOT / "schemas" / "ultra_evaluation_receipt.v0.1.schema.json" +VALID_PATH = ROOT / "fixtures" / "valid" / "ultra_evaluation_receipt.v0.1.valid.json" +INVALID_DIR = ROOT / "fixtures" / "invalid" +VALIDATOR_PATH = ROOT / "tools" / "validate_ultra_evaluation_receipt.py" +RUN_RECEIPT_PATH = ROOT / "receipts" / "2026-07-11-gpt-5.6-sol-ultra-run.json" + +SPEC = importlib.util.spec_from_file_location("ultra_receipt_validator", VALIDATOR_PATH) +assert SPEC is not None and SPEC.loader is not None +VALIDATOR = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(VALIDATOR) + + +def load_json(path: Path) -> object: + with path.open(encoding="utf-8") as handle: + return json.load(handle) + + +class UltraEvaluationReceiptTests(unittest.TestCase): + @classmethod + def setUpClass(cls) -> None: + cls.schema = load_json(SCHEMA_PATH) + cls.valid_receipt = load_json(VALID_PATH) + + def test_valid_fixture_passes(self) -> None: + self.assertEqual( + [], VALIDATOR.validate_receipt(self.valid_receipt, self.schema) + ) + + def test_exact_run_receipt_passes_without_fabricated_telemetry(self) -> None: + receipt = load_json(RUN_RECEIPT_PATH) + + self.assertEqual([], VALIDATOR.validate_receipt(receipt, self.schema)) + self.assertEqual(len(receipt["delegation_lanes"]), 4) + self.assertTrue( + all( + lane["final_state"] == "completed_incorporated" + for lane in receipt["delegation_lanes"] + ) + ) + self.assertEqual( + receipt["model"]["orchestration_agent_count"], + { + "availability": "available", + "value": 4, + "measurement_kind": "observed", + }, + ) + for metric in receipt["telemetry"].values(): + self.assertEqual(metric["availability"], "unavailable") + self.assertNotIn("value", metric) + self.assertNotIn( + "independent_ultra_replication", + {item["evidence_class"] for item in receipt["evidence_classes"]}, + ) + + def test_validation_is_deterministic(self) -> None: + fixture = load_json( + INVALID_DIR / "ultra_evaluation_receipt.v0.1.unavailable-value.invalid.json" + ) + first = VALIDATOR.validate_receipt(fixture, self.schema) + second = VALIDATOR.validate_receipt(fixture, self.schema) + self.assertEqual(first, second) + self.assertEqual(sorted(first), first) + + def test_all_invalid_fixtures_fail_with_clear_paths(self) -> None: + expectations = { + "ultra_evaluation_receipt.v0.1.schema-identity.invalid.json": ( + "$.schema_id", + ), + "ultra_evaluation_receipt.v0.1.unavailable-value.invalid.json": ( + "$.telemetry.aggregate_cost", + "unavailable", + ), + "ultra_evaluation_receipt.v0.1.lane-state.invalid.json": ( + "$.delegation_lanes[0].final_state", + ), + "ultra_evaluation_receipt.v0.1.evidence-class.invalid.json": ( + "$.evidence_classes[0].evidence_class", + ), + "ultra_evaluation_receipt.v0.1.duplicate-lane.invalid.json": ( + "$.delegation_lanes[1].lane_id", + "duplicate", + ), + "ultra_evaluation_receipt.v0.1.numeric-overflow.invalid.json": ( + "$.telemetry.aggregate_cost.value", + "finite", + ), + "ultra_evaluation_receipt.v0.1.route-mismatch.invalid.json": ( + "$.routing_decision.route_used", + "orchestration_setting", + ), + "ultra_evaluation_receipt.v0.1.empirical-policy.invalid.json": ( + "$.routing_decision.policy_status", + "provisional", + ), + "ultra_evaluation_receipt.v0.1.none-observed-details.invalid.json": ( + "$.duplicate_work.details", + "empty", + ), + } + fixture_names = { + path.name + for path in INVALID_DIR.glob("ultra_evaluation_receipt.v0.1.*.invalid.json") + } + self.assertEqual(set(expectations), fixture_names) + + for name, fragments in expectations.items(): + with self.subTest(fixture=name): + errors = VALIDATOR.validate_receipt( + load_json(INVALID_DIR / name), self.schema + ) + rendered = "\n".join(errors) + self.assertTrue(errors) + for fragment in fragments: + self.assertIn(fragment, rendered) + + def test_unavailable_telemetry_rejects_numeric_value(self) -> None: + receipt = json.loads(json.dumps(self.valid_receipt)) + receipt["telemetry"]["aggregate_tokens"] = { + "availability": "unavailable", + "reason": "Runtime did not expose aggregate token usage.", + "value": 12345, + } + errors = VALIDATOR.validate_receipt(receipt, self.schema) + rendered = "\n".join(errors) + self.assertIn("$.telemetry.aggregate_tokens", rendered) + self.assertIn("unavailable", rendered) + + def test_available_telemetry_requires_value_and_unit(self) -> None: + receipt = json.loads(json.dumps(self.valid_receipt)) + receipt["telemetry"]["elapsed_wall_clock_time"] = { + "availability": "available", + "measurement_kind": "observed", + } + errors = VALIDATOR.validate_receipt(receipt, self.schema) + rendered = "\n".join(errors) + self.assertIn("$.telemetry.elapsed_wall_clock_time.value", rendered) + self.assertIn("$.telemetry.elapsed_wall_clock_time.unit", rendered) + + def test_in_memory_validation_rejects_all_non_finite_numbers(self) -> None: + metrics = ( + ("aggregate_cost", "USD"), + ("aggregate_tokens", "tokens"), + ("elapsed_wall_clock_time", "seconds"), + ) + for metric, unit in metrics: + for value in (float("nan"), float("inf"), float("-inf")): + with self.subTest(metric=metric, value=repr(value)): + receipt = json.loads(json.dumps(self.valid_receipt)) + receipt["telemetry"][metric] = { + "availability": "available", + "value": value, + "unit": unit, + "measurement_kind": "observed", + } + errors = VALIDATOR.validate_receipt(receipt, self.schema) + self.assertIn( + f"$.telemetry.{metric}.value: number must be finite", + errors, + ) + + for value in (float("nan"), float("inf"), float("-inf")): + with self.subTest(metric="orchestration_agent_count", value=repr(value)): + receipt = json.loads(json.dumps(self.valid_receipt)) + receipt["model"]["orchestration_agent_count"] = { + "availability": "available", + "value": value, + "measurement_kind": "configured", + } + errors = VALIDATOR.validate_receipt(receipt, self.schema) + self.assertIn( + "$.model.orchestration_agent_count.value: expected integer", + errors, + ) + + def test_completed_lane_requires_available_completion_timestamp(self) -> None: + receipt = json.loads(json.dumps(self.valid_receipt)) + receipt["delegation_lanes"][0]["completed_at"] = { + "availability": "unavailable", + "reason": "Timestamp was not captured.", + } + errors = VALIDATOR.validate_receipt(receipt, self.schema) + self.assertIn( + "$.delegation_lanes[0].completed_at: must be available when final_state " + "is terminal", + errors, + ) + + def test_route_used_must_match_orchestration_setting(self) -> None: + receipt = json.loads(json.dumps(self.valid_receipt)) + receipt["model"]["orchestration_setting"] = "xhigh" + errors = VALIDATOR.validate_receipt(receipt, self.schema) + self.assertIn( + "$.routing_decision.route_used: must equal " + "$.model.orchestration_setting ('xhigh')", + errors, + ) + + def test_counterfactual_route_must_differ_from_route_used(self) -> None: + receipt = json.loads(json.dumps(self.valid_receipt)) + receipt["routing_decision"]["counterfactual_route"] = "ultra" + errors = VALIDATOR.validate_receipt(receipt, self.schema) + self.assertIn( + "$.routing_decision.counterfactual_route: must differ from route_used", + errors, + ) + + def test_none_observed_requires_empty_details(self) -> None: + receipt = json.loads(json.dumps(self.valid_receipt)) + receipt["synthesis_loss"]["details"] = ["Contradictory detail."] + errors = VALIDATOR.validate_receipt(receipt, self.schema) + self.assertIn( + "$.synthesis_loss.details: none_observed requires an empty array", + errors, + ) + + def test_v01_policy_status_cannot_claim_empirical_validation(self) -> None: + receipt = json.loads(json.dumps(self.valid_receipt)) + receipt["routing_decision"]["policy_status"] = "empirically_validated" + errors = VALIDATOR.validate_receipt(receipt, self.schema) + self.assertIn("$.routing_decision.policy_status", "\n".join(errors)) + self.assertIn("provisional", "\n".join(errors)) + + def test_cli_exit_codes_and_messages(self) -> None: + valid = subprocess.run( + [sys.executable, str(VALIDATOR_PATH), str(VALID_PATH)], + cwd=ROOT, + check=False, + capture_output=True, + text=True, + ) + self.assertEqual(0, valid.returncode, valid.stderr) + self.assertIn("VALID", valid.stdout) + + invalid_path = ( + INVALID_DIR / "ultra_evaluation_receipt.v0.1.evidence-class.invalid.json" + ) + invalid = subprocess.run( + [sys.executable, str(VALIDATOR_PATH), str(invalid_path)], + cwd=ROOT, + check=False, + capture_output=True, + text=True, + ) + self.assertEqual(1, invalid.returncode) + self.assertIn("INVALID", invalid.stderr) + self.assertIn("$.evidence_classes[0].evidence_class", invalid.stderr) + + def test_cli_rejects_non_standard_numeric_constants(self) -> None: + values = (float("nan"), float("inf"), float("-inf")) + with tempfile.TemporaryDirectory() as directory: + for index, value in enumerate(values): + with self.subTest(value=repr(value)): + receipt = json.loads(json.dumps(self.valid_receipt)) + receipt["telemetry"]["aggregate_cost"] = { + "availability": "available", + "value": value, + "unit": "USD", + "measurement_kind": "observed", + } + path = Path(directory) / f"non-finite-{index}.json" + path.write_text(json.dumps(receipt), encoding="utf-8") + result = subprocess.run( + [sys.executable, str(VALIDATOR_PATH), str(path)], + cwd=ROOT, + check=False, + capture_output=True, + text=True, + ) + self.assertEqual(2, result.returncode) + self.assertIn("non-standard JSON numeric constant", result.stderr) + + def test_cli_rejects_overflowed_json_number(self) -> None: + overflow = subprocess.run( + [ + sys.executable, + str(VALIDATOR_PATH), + str( + INVALID_DIR + / "ultra_evaluation_receipt.v0.1.numeric-overflow.invalid.json" + ), + ], + cwd=ROOT, + check=False, + capture_output=True, + text=True, + ) + self.assertEqual(2, overflow.returncode) + self.assertIn("overflowed JSON number", overflow.stderr) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/extract_gpt56_ultra_archive.py b/tools/extract_gpt56_ultra_archive.py new file mode 100644 index 0000000..8dcd038 --- /dev/null +++ b/tools/extract_gpt56_ultra_archive.py @@ -0,0 +1,639 @@ +#!/usr/bin/env python3 +"""Extract GPT-5.6 Sol multi-agent chart records from an archived page. + +The Wayback capture stores each Vega-Lite point as escaped, flat JSON inside +the rendered HTML. This script deliberately does not fetch the network. A +reproducible retrieval and extraction command is: + + curl --compressed --fail --location \ + 'https://web.archive.org/web/20260710150348id_/https://openai.com/index/gpt-5-6/' \ + | python3 tools/extract_gpt56_ultra_archive.py \ + --archive-html - --retrieval-date 2026-07-11 + +``--compressed`` matters: the recorded SHA-256 is over the decompressed HTTP +response entity consumed by this script, not the transport-compressed bytes. +""" + +from __future__ import annotations + +import argparse +from collections.abc import Iterable, Sequence +from datetime import date +import hashlib +import json +from pathlib import Path +import re +import sys +from typing import Any + + +ORIGINAL_URL = "https://openai.com/index/gpt-5-6/" +CAPTURE_TIMESTAMP = "20260710150348" +CAPTURE_TIMESTAMP_UTC = "2026-07-10T15:03:48Z" +CAPTURE_URL = ( + "https://web.archive.org/web/20260710150348id_/https://openai.com/index/gpt-5-6/" +) +CAPTURE_UI_URL = ( + "https://web.archive.org/web/20260710150348/https://openai.com/index/gpt-5-6/" +) + +TARGET_BENCHMARKS = ( + "BrowseComp", + "SEC-Bench Pro", + "Terminal-Bench 2.1", +) +TARGET_AGENT_COUNTS = (1, 4, 16) +EXPECTED_AGENT_COUNTS_BY_BENCHMARK = { + "BrowseComp": (1, 4, 16), + "SEC-Bench Pro": (1, 4, 16), + "Terminal-Bench 2.1": (1, 4), +} +TARGET_METRICS = ( + "api_cost_usd", + "latency_s", + "output_tokens", +) +EFFORT_ORDER = ("low", "medium", "high", "xhigh", "max") + +CURRENT_HEADLINE_SCORES_PERCENT = { + "BrowseComp": {1: 90.4, 4: 92.2}, + "SEC-Bench Pro": {1: 71.2, 4: 74.3}, + "Terminal-Bench 2.1": {1: 88.8, 4: 91.9}, +} + +_BENCHMARK_INDEX = {value: index for index, value in enumerate(TARGET_BENCHMARKS)} +_AGENT_INDEX = {value: index for index, value in enumerate(TARGET_AGENT_COUNTS)} +_METRIC_INDEX = {value: index for index, value in enumerate(TARGET_METRICS)} +_EFFORT_INDEX = {value: index for index, value in enumerate(EFFORT_ORDER)} +_ESCAPED_RECORD = re.compile(r'\{\\"eval\\":.*?\}') +_REQUIRED_SELECTED_FIELDS = { + "eval", + "eval_id", + "eval_variant", + "model", + "juice_level", + "juice_index", + "x_metric", + "x_value", + "x_label", + "score", + "score_label", + "score_unit", +} + + +def canonical_json_bytes(value: Any) -> bytes: + """Return the repository's deterministic JSON representation.""" + + return ( + json.dumps(value, ensure_ascii=False, indent=2, sort_keys=True) + "\n" + ).encode("utf-8") + + +def _canonical_record_key(record: dict[str, Any]) -> str: + return json.dumps(record, ensure_ascii=False, separators=(",", ":"), sort_keys=True) + + +def _agent_count(record: dict[str, Any]) -> int: + return int(record["eval_variant"].split(" ", 1)[0]) + + +def _record_sort_key(record: dict[str, Any]) -> tuple[Any, ...]: + agent_count = _agent_count(record) + return ( + _BENCHMARK_INDEX[record["eval"]], + _AGENT_INDEX[agent_count], + _EFFORT_INDEX[record["juice_level"]], + _METRIC_INDEX[record["x_metric"]], + _canonical_record_key(record), + ) + + +def _decode_flat_records(archive_html: str) -> list[dict[str, Any]]: + records: list[dict[str, Any]] = [] + for match in _ESCAPED_RECORD.finditer(archive_html): + escaped_object = match.group(0) + try: + decoded_object = json.loads(f'"{escaped_object}"') + record = json.loads(decoded_object) + except json.JSONDecodeError: + continue + if isinstance(record, dict): + records.append(record) + return records + + +def _is_target_record(record: dict[str, Any]) -> bool: + benchmark = record.get("eval") + variant = record.get("eval_variant") + metric = record.get("x_metric") + if benchmark not in TARGET_BENCHMARKS: + return False + target_variants = { + f"{count} agent" if count == 1 else f"{count} agents" + for count in TARGET_AGENT_COUNTS + } + if variant not in target_variants: + return False + if metric not in TARGET_METRICS: + return False + return record.get("model") == f"GPT-5.6 Sol · {variant}" + + +def _normalise_selected( + records: Iterable[dict[str, Any]], +) -> list[dict[str, Any]]: + unique: dict[str, dict[str, Any]] = {} + for record in records: + if not _is_target_record(record): + continue + missing = sorted(_REQUIRED_SELECTED_FIELDS - record.keys()) + if missing: + raise ValueError( + f"selected record for {record.get('eval')!r} is missing fields: " + + ", ".join(missing) + ) + if record["juice_level"] not in _EFFORT_INDEX: + raise ValueError( + f"unsupported effort in selected record: {record['juice_level']!r}" + ) + key = _canonical_record_key(record) + unique.setdefault(key, record) + return sorted(unique.values(), key=_record_sort_key) + + +def extract_selected_records(archive_html: str) -> list[dict[str, Any]]: + """Extract, select, exactly deduplicate, and deterministically sort points.""" + + records = _normalise_selected(_decode_flat_records(archive_html)) + if not records: + raise ValueError("no GPT-5.6 Sol multi-agent chart records found") + return records + + +def _availability(records: Sequence[dict[str, Any]]) -> dict[str, Any]: + result: dict[str, Any] = {} + for benchmark in TARGET_BENCHMARKS: + benchmark_records = [row for row in records if row["eval"] == benchmark] + observed_counts = sorted({_agent_count(row) for row in benchmark_records}) + result[benchmark] = { + "expected_agent_counts": list(TARGET_AGENT_COUNTS), + "observed_agent_counts": observed_counts, + "missing_agent_counts": [ + count for count in TARGET_AGENT_COUNTS if count not in observed_counts + ], + "observed_efforts": [ + effort + for effort in EFFORT_ORDER + if any(row["juice_level"] == effort for row in benchmark_records) + ], + "observed_metrics": [ + metric + for metric in TARGET_METRICS + if any(row["x_metric"] == metric for row in benchmark_records) + ], + "record_count": len(benchmark_records), + } + return result + + +def _semantic_identity(record: dict[str, Any]) -> tuple[str, int, str, str]: + return ( + record["eval"], + _agent_count(record), + record["juice_level"], + record["x_metric"], + ) + + +def _expected_production_identities() -> set[tuple[str, int, str, str]]: + return { + (benchmark, agent_count, effort, metric) + for benchmark, agent_counts in EXPECTED_AGENT_COUNTS_BY_BENCHMARK.items() + for agent_count in agent_counts + for effort in EFFORT_ORDER + for metric in TARGET_METRICS + } + + +def validate_production_matrix(records_document: dict[str, Any]) -> None: + """Require the exact checked-in matrix while leaving extraction composable.""" + + records = records_document.get("records", []) + identities: dict[tuple[str, int, str, str], dict[str, Any]] = {} + for record in records: + identity = _semantic_identity(record) + previous = identities.get(identity) + if previous is not None: + if previous != record: + raise ValueError(f"conflicting semantic identity: {identity!r}") + raise ValueError(f"duplicate semantic identity: {identity!r}") + identities[identity] = record + expected_juice_index = _EFFORT_INDEX[record["juice_level"]] + 1 + if record["juice_index"] != expected_juice_index: + raise ValueError( + f"juice_index does not match effort for semantic identity {identity!r}" + ) + + expected = _expected_production_identities() + actual = set(identities) + if actual != expected: + missing = sorted(expected - actual) + unexpected = sorted(actual - expected) + raise ValueError( + "incomplete production matrix: " + f"missing={missing!r}; unexpected={unexpected!r}" + ) + + for benchmark, agent_counts in EXPECTED_AGENT_COUNTS_BY_BENCHMARK.items(): + for agent_count in agent_counts: + for effort in EFFORT_ORDER: + rows = [ + identities[(benchmark, agent_count, effort, metric)] + for metric in TARGET_METRICS + ] + if len({row["score"] for row in rows}) != 1: + raise ValueError( + "conflicting scores across metric rows for " + f"{(benchmark, agent_count, effort)!r}" + ) + + terminal = records_document.get("availability", {}).get("Terminal-Bench 2.1", {}) + if terminal.get("observed_agent_counts") != [1, 4] or terminal.get( + "missing_agent_counts" + ) != [16]: + raise ValueError("Terminal-Bench 16-agent unavailability is not explicit") + + +def build_records_document( + records: Iterable[dict[str, Any]], +) -> dict[str, Any]: + """Build the checked-in raw-record document without changing row fields.""" + + selected = _normalise_selected(records) + if not selected: + raise ValueError("records document cannot be empty") + return { + "schema_version": "gpt-5.6-sol-ultra.archived-chart-records.v0.1", + "evidence_class": "archived_chart_data", + "source": { + "original_url": ORIGINAL_URL, + "wayback_capture_url": CAPTURE_URL, + "wayback_capture_timestamp": CAPTURE_TIMESTAMP, + }, + "selection": { + "benchmarks": list(TARGET_BENCHMARKS), + "agent_counts": list(TARGET_AGENT_COUNTS), + "efforts": list(EFFORT_ORDER), + "metrics": list(TARGET_METRICS), + "model_identity_rule": "exact GPT-5.6 Sol · chart label", + "deduplication": "exact semantic JSON object; first occurrence retained", + }, + "availability": _availability(selected), + "record_count": len(selected), + "records": selected, + } + + +def _score_reconciliation(records_document: dict[str, Any]) -> dict[str, Any]: + records = records_document["records"] + reconciliation: dict[str, Any] = {} + for benchmark in TARGET_BENCHMARKS: + archived_max: dict[str, Any] = {} + current_headline: dict[str, Any] = {} + for agent_count in (1, 4): + scores = { + float(row["score"]) + for row in records + if row["eval"] == benchmark + and _agent_count(row) == agent_count + and row["juice_level"] == "max" + } + labels = { + row["score_label"] + for row in records + if row["eval"] == benchmark + and _agent_count(row) == agent_count + and row["juice_level"] == "max" + } + key = "1_agent" if agent_count == 1 else "4_agents" + if not scores: + archived_max[key] = { + "availability": "unavailable", + "score": None, + "score_percent": None, + "score_label": None, + } + current_headline[key] = { + "score_percent": CURRENT_HEADLINE_SCORES_PERCENT[benchmark][ + agent_count + ], + "archived_score_percent": None, + "difference_archived_minus_current_percentage_points": None, + "agrees_after_rounding_to_one_decimal": None, + } + continue + if len(scores) != 1 or len(labels) != 1: + raise ValueError( + f"inconsistent max scores for {benchmark}, {agent_count} agent(s)" + ) + score = scores.pop() + score_percent = score * 100 + current = CURRENT_HEADLINE_SCORES_PERCENT[benchmark][agent_count] + archived_max[key] = { + "availability": "available", + "effort": "max", + "score": score, + "score_percent": score_percent, + "score_label": labels.pop(), + } + current_headline[key] = { + "score_percent": current, + "archived_score_percent": score_percent, + "difference_archived_minus_current_percentage_points": round( + score_percent - current, 6 + ), + "agrees_after_rounding_to_one_decimal": round(score_percent, 1) + == current, + } + reconciliation[benchmark] = { + "archived_max": archived_max, + "current_headline": current_headline, + "comparison_rule": ( + "Current one-decimal headline scores are reported separately from " + "archived max chart scores; operational comparisons use archived " + "max records only." + ), + } + return reconciliation + + +def _latency_companion_evidence(archive_bytes: bytes) -> list[dict[str, Any]]: + archive_html = archive_bytes.decode("utf-8") + candidates = [ + record + for record in _decode_flat_records(archive_html) + if record.get("eval") in TARGET_BENCHMARKS + and record.get("juice_level") == "max" + and record.get("model") in {"GPT-5.6 Sol", "GPT-5.6 Sol Ultra"} + and "latency_minutes" in record + and str(record.get("latency_label", "")).endswith(" minutes") + ] + terminal_candidates = [ + record for record in candidates if record["eval"] == "Terminal-Bench 2.1" + ] + selected = terminal_candidates or candidates[:2] + return [ + { + "record": record, + "canonical_record_sha256": hashlib.sha256( + canonical_json_bytes(record) + ).hexdigest(), + } + for record in sorted(selected, key=_canonical_record_key) + ] + + +def _internal_slack_source_urls(records_document: dict[str, Any]) -> list[str]: + return sorted( + { + record["source_url"] + for record in records_document["records"] + if str(record.get("source_url", "")).startswith( + "https://openai-corpws.slack.com/" + ) + } + ) + + +def build_manifest( + archive_bytes: bytes, + records_document: dict[str, Any], + *, + retrieval_date: date, +) -> dict[str, Any]: + """Build provenance, unit, and score-reconciliation metadata.""" + + records_sha256 = hashlib.sha256(canonical_json_bytes(records_document)).hexdigest() + companion_evidence = _latency_companion_evidence(archive_bytes) + internal_slack_urls = _internal_slack_source_urls(records_document) + internal_url_count = len(internal_slack_urls) + internal_url_count_label = ( + "two" if internal_url_count == 2 else str(internal_url_count) + ) + return { + "schema_version": "gpt-5.6-sol-ultra.archive-manifest.v0.1", + "evidence_class": "archived_chart_data", + "archive": { + "original_url": ORIGINAL_URL, + "wayback_capture_url": CAPTURE_URL, + "wayback_ui_url": CAPTURE_UI_URL, + "capture_timestamp": CAPTURE_TIMESTAMP, + "capture_timestamp_utc": CAPTURE_TIMESTAMP_UTC, + "retrieval_date": retrieval_date.isoformat(), + "content_sha256": hashlib.sha256(archive_bytes).hexdigest(), + "hash_scope": ( + "decompressed HTTP response entity bytes from the id_ capture " + "after HTTP content decoding" + ), + }, + "extraction": { + "script": "tools/extract_gpt56_ultra_archive.py", + "selected_records": "data/gpt-5.6-sol-ultra/archived_chart_records.json", + "selected_record_count": records_document["record_count"], + "selected_records_sha256": records_sha256, + "parser_input": "escaped flat eval JSON objects embedded in archive HTML", + "deduplication": "exact semantic JSON object; first occurrence retained", + }, + "unit_interpretation": { + "api_cost_usd": { + "raw_metric_name": "api_cost_usd", + "raw_value_field": "x_value", + "x_value_unit": "estimated USD", + "interpretation": ( + "Provider-estimated API cost including all agents; not an " + "observed public-service billing guarantee." + ), + }, + "latency_s": { + "raw_metric_name": "latency_s", + "raw_value_field": "x_value", + "x_value_unit": "minutes", + "interpretation_basis": [ + "Every selected latency display label renders x_value in minutes.", + "Preserved companion source evidence names the same quantity latency_minutes.", + ], + "companion_source_evidence": companion_evidence, + "warning": ( + "The raw metric name suggests seconds, but treating x_value as " + "seconds conflicts with the displayed labels; values are preserved " + "unchanged and interpreted as minutes." + ), + "interpretation": ( + "Provider-simulated root-agent latency; not an observed " + "public-service latency guarantee." + ), + }, + "output_tokens": { + "raw_metric_name": "output_tokens", + "raw_value_field": "x_value", + "x_value_unit": "tokens", + "interpretation": "Aggregate output tokens include all agents.", + }, + "score": { + "raw_value_field": "score", + "stored_unit": "fraction", + "display_unit": "percent", + }, + }, + "score_reconciliation": _score_reconciliation(records_document), + "availability": records_document["availability"], + "provenance_privacy": { + "preserved_internal_source_url_count": internal_url_count, + "preserved_internal_source_urls": internal_slack_urls, + "credentials_or_tokens_found": False, + "decision": ( + "Preserve provider-embedded source_url values for raw provenance " + "fidelity despite inaccessible internal workspace destinations." + ), + }, + "limitations": [ + "The capture is a provider page and the selected records are provider-reported.", + "Terminal-Bench 2.1 has no 16-agent flat chart records in this capture.", + "Cost and latency are provider estimates or simulations, not service guarantees.", + ( + "Raw selected records intentionally preserve " + f"{internal_url_count_label} distinct inaccessible internal OpenAI " + "Slack source URLs embedded by the provider; no credentials or tokens " + "were found. This accepts a provenance/privacy tradeoff in favor of " + "raw-source fidelity." + ), + ], + } + + +def write_json(path: Path, value: Any) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(canonical_json_bytes(value)) + + +def verify_records_manifest( + records_document: dict[str, Any], manifest: dict[str, Any] +) -> None: + """Verify deterministic record structure and its manifest-level receipt.""" + + rebuilt_document = build_records_document(records_document.get("records", [])) + if rebuilt_document != records_document: + raise ValueError( + "records document is not deterministic or internally consistent" + ) + validate_production_matrix(records_document) + records_hash = hashlib.sha256(canonical_json_bytes(records_document)).hexdigest() + extraction = manifest.get("extraction", {}) + if extraction.get("selected_record_count") != records_document["record_count"]: + raise ValueError( + "manifest selected record count does not match records document" + ) + if extraction.get("selected_records_sha256") != records_hash: + raise ValueError( + "manifest selected records hash does not match records document" + ) + if manifest.get("availability") != records_document["availability"]: + raise ValueError("manifest availability does not match records document") + if manifest.get("score_reconciliation") != _score_reconciliation(records_document): + raise ValueError( + "manifest score reconciliation does not match records document" + ) + + +def verify_artifacts( + archive_bytes: bytes, + records_path: Path, + manifest_path: Path, + *, + retrieval_date: date, +) -> None: + """Re-extract from the supplied bytes and compare both checked-in artifacts.""" + + archive_text = archive_bytes.decode("utf-8") + expected_records = build_records_document(extract_selected_records(archive_text)) + validate_production_matrix(expected_records) + actual_records = json.loads(records_path.read_text(encoding="utf-8")) + if actual_records != expected_records: + raise ValueError("records document does not match deterministic extraction") + expected_manifest = build_manifest( + archive_bytes, expected_records, retrieval_date=retrieval_date + ) + actual_manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + if actual_manifest != expected_manifest: + raise ValueError("manifest does not match deterministic extraction") + verify_records_manifest(actual_records, actual_manifest) + + +def _read_archive(path: str) -> bytes: + if path == "-": + return sys.stdin.buffer.read() + return Path(path).read_bytes() + + +def _parse_args(argv: Sequence[str] | None) -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--archive-html", + required=True, + help="path to decompressed archived HTML, or '-' for stdin", + ) + parser.add_argument( + "--retrieval-date", + required=True, + type=date.fromisoformat, + help="archive retrieval date in YYYY-MM-DD form", + ) + parser.add_argument( + "--records-output", + type=Path, + default=Path("data/gpt-5.6-sol-ultra/archived_chart_records.json"), + ) + parser.add_argument( + "--manifest-output", + type=Path, + default=Path("data/gpt-5.6-sol-ultra/archive_manifest.json"), + ) + parser.add_argument( + "--verify", + action="store_true", + help="verify existing outputs instead of replacing them", + ) + return parser.parse_args(argv) + + +def main(argv: Sequence[str] | None = None) -> int: + args = _parse_args(argv) + archive_bytes = _read_archive(args.archive_html) + if args.verify: + verify_artifacts( + archive_bytes, + args.records_output, + args.manifest_output, + retrieval_date=args.retrieval_date, + ) + print(f"verified {args.records_output} and {args.manifest_output}") + return 0 + + archive_text = archive_bytes.decode("utf-8") + records_document = build_records_document(extract_selected_records(archive_text)) + validate_production_matrix(records_document) + manifest = build_manifest( + archive_bytes, records_document, retrieval_date=args.retrieval_date + ) + write_json(args.records_output, records_document) + write_json(args.manifest_output, manifest) + print( + f"wrote {records_document['record_count']} records to {args.records_output} " + f"and provenance to {args.manifest_output}" + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/validate_ultra_benchmark_dataset.py b/tools/validate_ultra_benchmark_dataset.py new file mode 100644 index 0000000..088d7c9 --- /dev/null +++ b/tools/validate_ultra_benchmark_dataset.py @@ -0,0 +1,458 @@ +#!/usr/bin/env python3 +"""Validate GPT-5.6 Sol Ultra raw, provenance, and normalized artifacts.""" + +from __future__ import annotations + +import argparse +from collections.abc import Sequence +import hashlib +import json +import math +from pathlib import Path +from typing import Any + + +BENCHMARK_IDS = { + "BrowseComp": "browsecomp", + "SEC-Bench Pro": "sec_bench_pro", + "Terminal-Bench 2.1": "terminal_bench_2_1", +} +EXPECTED_AGENT_COUNTS = { + "BrowseComp": (1, 4, 16), + "SEC-Bench Pro": (1, 4, 16), + "Terminal-Bench 2.1": (1, 4), +} +EFFORTS = ("low", "medium", "high", "xhigh", "max") +METRICS = ("api_cost_usd", "latency_s", "output_tokens") +EXPECTED_RECORDS_PATH = "data/gpt-5.6-sol-ultra/archived_chart_records.json" +WAYBACK_SOURCE_ID = "openai_gpt_5_6_release_wayback" + + +class DatasetValidationError(ValueError): + """Raised when the three-artifact evidence chain does not reconcile.""" + + +def canonical_json_bytes(value: Any) -> bytes: + return ( + json.dumps(value, ensure_ascii=False, indent=2, sort_keys=True) + "\n" + ).encode("utf-8") + + +def _agent_count(record: dict[str, Any]) -> int: + try: + return int(record["eval_variant"].split(" ", 1)[0]) + except (KeyError, AttributeError, ValueError) as error: + raise DatasetValidationError("raw record has invalid eval_variant") from error + + +def _identity(record: dict[str, Any]) -> tuple[str, int, str, str]: + try: + return ( + record["eval"], + _agent_count(record), + record["juice_level"], + record["x_metric"], + ) + except KeyError as error: + raise DatasetValidationError( + f"raw record is missing identity field {error.args[0]!r}" + ) from error + + +def _expected_identities() -> set[tuple[str, int, str, str]]: + return { + (benchmark, agent_count, effort, metric) + for benchmark, agent_counts in EXPECTED_AGENT_COUNTS.items() + for agent_count in agent_counts + for effort in EFFORTS + for metric in METRICS + } + + +def _require_equal(path: str, actual: Any, expected: Any) -> None: + if actual != expected: + raise DatasetValidationError( + f"{path} drift: expected {expected!r}, found {actual!r}" + ) + + +def _require_float(path: str, actual: Any, expected: float) -> None: + if not isinstance(actual, (int, float)) or not math.isclose( + float(actual), expected, rel_tol=0, abs_tol=1e-12 + ): + raise DatasetValidationError( + f"{path} drift: expected {expected!r}, found {actual!r}" + ) + + +def _index_records( + records_document: dict[str, Any], +) -> dict[tuple[str, int, str, str], dict[str, Any]]: + records = records_document.get("records") + if not isinstance(records, list): + raise DatasetValidationError("raw records must be a list") + _require_equal( + "raw record_count", records_document.get("record_count"), len(records) + ) + identities: dict[tuple[str, int, str, str], dict[str, Any]] = {} + for record in records: + if not isinstance(record, dict): + raise DatasetValidationError("raw record must be an object") + identity = _identity(record) + if identity in identities: + qualifier = "conflicting" if identities[identity] != record else "duplicate" + raise DatasetValidationError( + f"{qualifier} raw semantic identity {identity!r}" + ) + identities[identity] = record + expected = _expected_identities() + if set(identities) != expected: + raise DatasetValidationError( + "raw production matrix drift: " + f"missing={sorted(expected - set(identities))!r}; " + f"unexpected={sorted(set(identities) - expected)!r}" + ) + _require_equal("raw record count", len(records), len(expected)) + terminal = records_document.get("availability", {}).get("Terminal-Bench 2.1", {}) + _require_equal( + "Terminal-Bench observed agent counts", + terminal.get("observed_agent_counts"), + [1, 4], + ) + _require_equal( + "Terminal-Bench missing agent counts", + terminal.get("missing_agent_counts"), + [16], + ) + return identities + + +def _source_by_id(dataset: dict[str, Any], source_id: str) -> dict[str, Any]: + matches = [ + source + for source in dataset.get("sources", []) + if source.get("source_id") == source_id + ] + if len(matches) != 1: + raise DatasetValidationError( + f"dataset must contain exactly one source {source_id!r}" + ) + return matches[0] + + +def _validate_hashes_and_sources( + records_document: dict[str, Any], + manifest: dict[str, Any], + dataset: dict[str, Any], + records_sha256: str, +) -> None: + extraction = manifest.get("extraction", {}) + _require_equal( + "manifest records SHA-256", + extraction.get("selected_records_sha256"), + records_sha256, + ) + _require_equal( + "manifest selected record count", + extraction.get("selected_record_count"), + records_document.get("record_count"), + ) + _require_equal( + "manifest selected records path", + extraction.get("selected_records"), + EXPECTED_RECORDS_PATH, + ) + + raw_source = records_document.get("source", {}) + archive = manifest.get("archive", {}) + _require_equal( + "raw original source URL", + raw_source.get("original_url"), + archive.get("original_url"), + ) + _require_equal( + "raw Wayback capture URL", + raw_source.get("wayback_capture_url"), + archive.get("wayback_capture_url"), + ) + _require_equal( + "raw Wayback capture timestamp", + raw_source.get("wayback_capture_timestamp"), + archive.get("capture_timestamp"), + ) + + wayback_source = _source_by_id(dataset, WAYBACK_SOURCE_ID) + _require_equal( + "dataset Wayback source URL", + wayback_source.get("url"), + archive.get("wayback_ui_url"), + ) + _require_equal( + "dataset Wayback original URL", + wayback_source.get("original_url"), + archive.get("original_url"), + ) + _require_equal( + "dataset Wayback capture timestamp", + wayback_source.get("capture_timestamp_utc"), + archive.get("capture_timestamp_utc"), + ) + _require_equal( + "dataset Wayback retrieval date", + wayback_source.get("retrieved_on"), + archive.get("retrieval_date"), + ) + content_hash = wayback_source.get("content_hash", {}) + _require_equal( + "dataset archive hash algorithm", content_hash.get("algorithm"), "sha256" + ) + _require_equal( + "dataset archive content hash", + content_hash.get("value"), + archive.get("content_sha256"), + ) + _require_equal( + "dataset archive hash manifest reference", + content_hash.get("manifest_ref"), + "data/gpt-5.6-sol-ultra/archive_manifest.json", + ) + archive_hash = archive.get("content_sha256") + if not isinstance(archive_hash, str) or len(archive_hash) != 64: + raise DatasetValidationError("manifest archive content SHA-256 is invalid") + + +def _max_route( + index: dict[tuple[str, int, str, str], dict[str, Any]], + benchmark: str, + agent_count: int, +) -> dict[str, float]: + rows = { + metric: index[(benchmark, agent_count, "max", metric)] for metric in METRICS + } + scores = {float(row["score"]) for row in rows.values()} + if len(scores) != 1: + raise DatasetValidationError( + f"raw max score conflicts across metrics for {benchmark}, {agent_count} agent(s)" + ) + return { + "score": scores.pop(), + "cost": float(rows["api_cost_usd"]["x_value"]), + "output_tokens": float(rows["output_tokens"]["x_value"]), + "latency": float(rows["latency_s"]["x_value"]), + } + + +def _comparison_by_id(dataset: dict[str, Any]) -> dict[str, dict[str, Any]]: + comparisons = dataset.get("archived_max_comparisons", []) + indexed = {item.get("benchmark_id"): item for item in comparisons} + if len(indexed) != len(comparisons): + raise DatasetValidationError("dataset has duplicate archived comparison ids") + _require_equal("archived comparison ids", set(indexed), set(BENCHMARK_IDS.values())) + return indexed + + +def _headline_by_id(dataset: dict[str, Any]) -> dict[str, dict[str, Any]]: + headlines = dataset.get("headline_results", []) + indexed = {item.get("benchmark_id"): item for item in headlines} + _require_equal("headline benchmark ids", set(indexed), set(BENCHMARK_IDS.values())) + return indexed + + +def _validate_route( + path: str, route: dict[str, Any], expected: dict[str, float], agent_count: int +) -> None: + _require_equal(f"{path}.agent_count", route.get("agent_count"), agent_count) + _require_equal(f"{path}.effort", route.get("effort"), "max") + _require_float( + f"{path}.score", route.get("score", {}).get("value"), expected["score"] + ) + _require_float(f"{path}.cost", route.get("cost", {}).get("value"), expected["cost"]) + _require_float( + f"{path}.output_tokens", + route.get("output_tokens", {}).get("value"), + expected["output_tokens"], + ) + _require_float( + f"{path}.latency", route.get("latency", {}).get("value"), expected["latency"] + ) + + +def _validate_comparisons( + index: dict[tuple[str, int, str, str], dict[str, Any]], + dataset: dict[str, Any], +) -> None: + comparisons = _comparison_by_id(dataset) + headlines = _headline_by_id(dataset) + for benchmark, benchmark_id in BENCHMARK_IDS.items(): + comparison = comparisons[benchmark_id] + headline = headlines[benchmark_id] + _require_equal( + f"{benchmark_id}.comparison_scope", + comparison.get("comparison_scope"), + "archived_max_to_archived_max", + ) + _require_equal( + f"{benchmark_id}.source_id", comparison.get("source_id"), WAYBACK_SOURCE_ID + ) + single = _max_route(index, benchmark, 1) + ultra = _max_route(index, benchmark, 4) + _validate_route( + f"{benchmark_id}.single_agent", + comparison.get("single_agent", {}), + single, + 1, + ) + _validate_route( + f"{benchmark_id}.ultra_four_agent", + comparison.get("ultra_four_agent", {}), + ultra, + 4, + ) + + expected_derived = { + "score_gain_percentage_points": round( + (ultra["score"] - single["score"]) * 100, 2 + ), + "cost_multiplier": round(ultra["cost"] / single["cost"], 2), + "output_token_multiplier": round( + ultra["output_tokens"] / single["output_tokens"], 2 + ), + "latency_change_percent": round( + (ultra["latency"] - single["latency"]) / single["latency"] * 100, + 1, + ), + } + derived = comparison.get("derived", {}) + for field, expected in expected_derived.items(): + _require_float( + f"{benchmark_id}.derived.{field}", derived.get(field), expected + ) + latency_change = expected_derived["latency_change_percent"] + interpretation = ( + "faster" + if latency_change < 0 + else "slower" + if latency_change > 0 + else "unchanged" + ) + _require_equal( + f"{benchmark_id}.derived.latency_interpretation", + derived.get("latency_interpretation"), + interpretation, + ) + + reconciliation = comparison.get("reconciliation", {}) + _require_float( + f"{benchmark_id}.reconciliation.archived_ultra_score", + reconciliation.get("archived_ultra_score"), + ultra["score"], + ) + _require_float( + f"{benchmark_id}.reconciliation.archived_single_agent_score", + reconciliation.get("archived_single_agent_score"), + single["score"], + ) + current_ultra = float(headline["ultra_score"]["value"]) + current_single = float(headline["single_agent_score"]["value"]) + _require_float( + f"{benchmark_id}.reconciliation.current_ultra_score", + reconciliation.get("current_ultra_score"), + current_ultra, + ) + _require_float( + f"{benchmark_id}.reconciliation.current_single_agent_score", + reconciliation.get("current_single_agent_score"), + current_single, + ) + scores_round_together = round(ultra["score"] * 100, 1) == round( + current_ultra * 100, 1 + ) and round(single["score"] * 100, 1) == round(current_single * 100, 1) + expected_status = ( + "agrees_after_rounding" + if scores_round_together + else "operational_baseline_differs_do_not_mix" + ) + _require_equal( + f"{benchmark_id}.reconciliation.status", + reconciliation.get("status"), + expected_status, + ) + + +def validate_documents( + records_document: dict[str, Any], + manifest: dict[str, Any], + dataset: dict[str, Any], + *, + records_sha256: str | None = None, +) -> dict[str, Any]: + """Validate in-memory artifacts and return a compact deterministic report.""" + + if records_sha256 is None: + records_sha256 = hashlib.sha256( + canonical_json_bytes(records_document) + ).hexdigest() + index = _index_records(records_document) + _validate_hashes_and_sources(records_document, manifest, dataset, records_sha256) + _validate_comparisons(index, dataset) + return { + "record_count": len(index), + "comparisons_checked": len(BENCHMARK_IDS), + "records_sha256": records_sha256, + "status": "valid", + } + + +def _read_json(path: Path) -> dict[str, Any]: + try: + value = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as error: + raise DatasetValidationError( + f"could not read JSON artifact {path}: {error}" + ) from error + if not isinstance(value, dict): + raise DatasetValidationError(f"JSON artifact must be an object: {path}") + return value + + +def validate_paths( + records_path: Path, manifest_path: Path, dataset_path: Path +) -> dict[str, Any]: + records_bytes = records_path.read_bytes() + return validate_documents( + _read_json(records_path), + _read_json(manifest_path), + _read_json(dataset_path), + records_sha256=hashlib.sha256(records_bytes).hexdigest(), + ) + + +def _parse_args(argv: Sequence[str] | None) -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--records", + type=Path, + default=Path(EXPECTED_RECORDS_PATH), + ) + parser.add_argument( + "--manifest", + type=Path, + default=Path("data/gpt-5.6-sol-ultra/archive_manifest.json"), + ) + parser.add_argument( + "--dataset", + type=Path, + default=Path("data/gpt-5.6-sol-ultra/benchmark_dataset.v0.1.json"), + ) + return parser.parse_args(argv) + + +def main(argv: Sequence[str] | None = None) -> int: + args = _parse_args(argv) + report = validate_paths(args.records, args.manifest, args.dataset) + print(json.dumps(report, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/validate_ultra_evaluation_receipt.py b/tools/validate_ultra_evaluation_receipt.py new file mode 100644 index 0000000..ff56c33 --- /dev/null +++ b/tools/validate_ultra_evaluation_receipt.py @@ -0,0 +1,494 @@ +"""Deterministically validate an ``ultra_evaluation_receipt.v0.1`` document. + +The repository has no runtime dependency on a JSON Schema package. This module +therefore implements only the Draft 2020-12 keywords used by the adjacent +schema, followed by receipt-specific honesty and consistency checks. +""" + +from __future__ import annotations + +import argparse +from datetime import datetime +import json +import math +from pathlib import Path +import re +import sys +from typing import Any + + +EXPECTED_META_SCHEMA = "https://json-schema.org/draft/2020-12/schema" +EXPECTED_SCHEMA_URI = ( + "https://raw.githubusercontent.com/hummbl-dev/model-routing-as-code/main/" + "schemas/ultra_evaluation_receipt.v0.1.schema.json" +) +EXPECTED_RECEIPT_SCHEMA_ID = "ultra_evaluation_receipt.v0.1" +EXPECTED_RECEIPT_SCHEMA_VERSION = "0.1" +DEFAULT_SCHEMA_PATH = ( + Path(__file__).resolve().parents[1] + / "schemas" + / "ultra_evaluation_receipt.v0.1.schema.json" +) + + +def _json_type_matches(value: Any, expected: str) -> bool: + """Return whether *value* has the requested JSON type.""" + + if expected == "object": + return isinstance(value, dict) + if expected == "array": + return isinstance(value, list) + if expected == "string": + return isinstance(value, str) + if expected == "boolean": + return isinstance(value, bool) + if expected == "null": + return value is None + if expected == "integer": + return isinstance(value, int) and not isinstance(value, bool) + if expected == "number": + return isinstance(value, (int, float)) and not isinstance(value, bool) + return False + + +def _display_json(value: Any) -> str: + return json.dumps(value, ensure_ascii=False, sort_keys=True) + + +def _property_path(path: str, key: str) -> str: + if re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]*", key): + return f"{path}.{key}" + return f"{path}[{json.dumps(key, ensure_ascii=False)}]" + + +def _resolve_local_ref(root_schema: dict[str, Any], ref: str) -> Any: + if not ref.startswith("#/"): + raise ValueError(f"only local JSON Pointer references are supported: {ref}") + current: Any = root_schema + for raw_part in ref[2:].split("/"): + part = raw_part.replace("~1", "/").replace("~0", "~") + if not isinstance(current, dict) or part not in current: + raise ValueError(f"unresolvable schema reference: {ref}") + current = current[part] + return current + + +def _parse_date_time(value: Any) -> datetime | None: + if not isinstance(value, str): + return None + normalized = value[:-1] + "+00:00" if value.endswith("Z") else value + try: + parsed = datetime.fromisoformat(normalized) + except ValueError: + return None + if parsed.tzinfo is None or parsed.utcoffset() is None: + return None + return parsed + + +def _validate_schema_subset( + value: Any, + schema: Any, + root_schema: dict[str, Any], + path: str, + errors: list[str], +) -> None: + """Validate the Draft 2020-12 subset used by the receipt schema.""" + + if not isinstance(schema, dict): + errors.append(f"{path}: internal schema node must be an object") + return + + ref = schema.get("$ref") + if ref is not None: + if not isinstance(ref, str): + errors.append(f"{path}: internal schema $ref must be a string") + return + try: + target = _resolve_local_ref(root_schema, ref) + except ValueError as exc: + errors.append(f"{path}: internal schema error: {exc}") + return + _validate_schema_subset(value, target, root_schema, path, errors) + return + + branches = schema.get("oneOf") + if branches is not None: + if not isinstance(branches, list) or not branches: + errors.append(f"{path}: internal schema oneOf must be a non-empty array") + return + branch_errors: list[list[str]] = [] + matching_branches = 0 + for branch in branches: + candidate_errors: list[str] = [] + _validate_schema_subset(value, branch, root_schema, path, candidate_errors) + branch_errors.append(candidate_errors) + if not candidate_errors: + matching_branches += 1 + if matching_branches == 1: + return + if matching_branches > 1: + errors.append(f"{path}: matches more than one permitted shape") + return + # The closest branch yields concrete property paths instead of a vague + # oneOf failure. Index breaks ties deterministically. + _, closest = min( + enumerate(branch_errors), key=lambda item: (len(item[1]), item[0]) + ) + errors.extend(closest) + return + + expected_type = schema.get("type") + if expected_type is not None: + expected_types = ( + expected_type if isinstance(expected_type, list) else [expected_type] + ) + if not all(isinstance(item, str) for item in expected_types): + errors.append(f"{path}: internal schema type must contain strings") + return + if not any(_json_type_matches(value, item) for item in expected_types): + rendered = " or ".join(expected_types) + errors.append(f"{path}: expected {rendered}") + return + + if "const" in schema and value != schema["const"]: + errors.append(f"{path}: must equal {_display_json(schema['const'])}") + + enum = schema.get("enum") + if enum is not None and value not in enum: + errors.append( + f"{path}: must be one of " + ", ".join(_display_json(item) for item in enum) + ) + + if isinstance(value, dict): + required = schema.get("required", []) + if isinstance(required, list): + for key in sorted(item for item in required if isinstance(item, str)): + if key not in value: + errors.append( + f"{_property_path(path, key)}: required property missing" + ) + + properties = schema.get("properties", {}) + if isinstance(properties, dict): + for key in sorted(value): + if key in properties: + _validate_schema_subset( + value[key], + properties[key], + root_schema, + _property_path(path, key), + errors, + ) + elif schema.get("additionalProperties") is False: + errors.append( + f"{_property_path(path, key)}: additional property not allowed" + ) + + if isinstance(value, list): + minimum_items = schema.get("minItems") + if isinstance(minimum_items, int) and len(value) < minimum_items: + errors.append(f"{path}: must contain at least {minimum_items} item(s)") + maximum_items = schema.get("maxItems") + if isinstance(maximum_items, int) and len(value) > maximum_items: + errors.append(f"{path}: must contain at most {maximum_items} item(s)") + if schema.get("uniqueItems") is True: + serialized = [ + json.dumps(item, sort_keys=True, separators=(",", ":")) + for item in value + ] + if len(serialized) != len(set(serialized)): + errors.append(f"{path}: array items must be unique") + item_schema = schema.get("items") + if isinstance(item_schema, dict): + for index, item in enumerate(value): + _validate_schema_subset( + item, item_schema, root_schema, f"{path}[{index}]", errors + ) + + if isinstance(value, str): + minimum_length = schema.get("minLength") + if isinstance(minimum_length, int) and len(value) < minimum_length: + errors.append( + f"{path}: must contain at least {minimum_length} character(s)" + ) + pattern = schema.get("pattern") + if isinstance(pattern, str) and re.search(pattern, value) is None: + errors.append(f"{path}: must match pattern {pattern!r}") + if schema.get("format") == "date-time" and _parse_date_time(value) is None: + errors.append(f"{path}: must be an RFC 3339 date-time with timezone") + + if isinstance(value, (int, float)) and not isinstance(value, bool): + if isinstance(value, float) and not math.isfinite(value): + errors.append(f"{path}: number must be finite") + return + minimum = schema.get("minimum") + if isinstance(minimum, (int, float)) and value < minimum: + errors.append(f"{path}: must be greater than or equal to {minimum}") + maximum = schema.get("maximum") + if isinstance(maximum, (int, float)) and value > maximum: + errors.append(f"{path}: must be less than or equal to {maximum}") + + +def _validate_unavailable_shape(value: Any, path: str, errors: list[str]) -> None: + if not isinstance(value, dict) or value.get("availability") != "unavailable": + return + forbidden = sorted( + field + for field in ("value", "unit", "measurement_kind", "assessment", "details") + if field in value + ) + if forbidden: + errors.append( + f"{path}: availability is unavailable; omit " + ", ".join(forbidden) + ) + + +def _validate_receipt_semantics(receipt: Any, errors: list[str]) -> None: + if not isinstance(receipt, dict): + return + + if receipt.get("schema_id") != EXPECTED_RECEIPT_SCHEMA_ID: + errors.append( + f"$.schema_id: expected {EXPECTED_RECEIPT_SCHEMA_ID!r} for this validator" + ) + if receipt.get("schema_version") != EXPECTED_RECEIPT_SCHEMA_VERSION: + errors.append( + "$.schema_version: expected " + f"{EXPECTED_RECEIPT_SCHEMA_VERSION!r} for this validator" + ) + + model = receipt.get("model") + if isinstance(model, dict): + _validate_unavailable_shape( + model.get("orchestration_agent_count"), + "$.model.orchestration_agent_count", + errors, + ) + + telemetry = receipt.get("telemetry") + expected_units = { + "aggregate_cost": "USD", + "aggregate_tokens": "tokens", + "elapsed_wall_clock_time": "seconds", + } + if isinstance(telemetry, dict): + for name, expected_unit in expected_units.items(): + metric = telemetry.get(name) + path = f"$.telemetry.{name}" + _validate_unavailable_shape(metric, path, errors) + if ( + not isinstance(metric, dict) + or metric.get("availability") != "available" + ): + continue + if metric.get("unit") != expected_unit: + errors.append(f"{path}.unit: must be {expected_unit!r}") + value = metric.get("value") + if name == "aggregate_tokens" and not ( + isinstance(value, int) and not isinstance(value, bool) + ): + errors.append(f"{path}.value: aggregate tokens must be an integer") + + for name in ("duplicate_work", "synthesis_loss"): + assessment = receipt.get(name) + path = f"$.{name}" + _validate_unavailable_shape(assessment, path, errors) + if ( + isinstance(assessment, dict) + and assessment.get("availability") == "available" + and assessment.get("assessment") == "observed" + and assessment.get("details") == [] + ): + errors.append(f"{path}.details: observed work requires at least one detail") + if ( + isinstance(assessment, dict) + and assessment.get("availability") == "available" + and assessment.get("assessment") == "none_observed" + and isinstance(assessment.get("details"), list) + and assessment["details"] + ): + errors.append(f"{path}.details: none_observed requires an empty array") + + lanes = receipt.get("delegation_lanes") + seen_lane_ids: set[str] = set() + terminal_states = { + "completed_incorporated", + "completed_not_incorporated", + "interrupted", + "failed", + } + recorded_at = _parse_date_time(receipt.get("recorded_at")) + if isinstance(lanes, list): + for index, lane in enumerate(lanes): + if not isinstance(lane, dict): + continue + lane_path = f"$.delegation_lanes[{index}]" + lane_id = lane.get("lane_id") + if isinstance(lane_id, str): + if lane_id in seen_lane_ids: + errors.append(f"{lane_path}.lane_id: duplicate lane_id {lane_id!r}") + seen_lane_ids.add(lane_id) + + final_state = lane.get("final_state") + completed_at = lane.get("completed_at") + completion_available = ( + isinstance(completed_at, dict) + and completed_at.get("availability") == "available" + ) + if final_state in terminal_states and not completion_available: + errors.append( + f"{lane_path}.completed_at: must be available when final_state " + "is terminal" + ) + if final_state == "outstanding" and completion_available: + errors.append( + f"{lane_path}.completed_at: must be unavailable while final_state " + "is outstanding" + ) + + started = _parse_date_time(lane.get("started_at")) + completed = ( + _parse_date_time(completed_at.get("value")) + if completion_available and isinstance(completed_at, dict) + else None + ) + if started is not None and completed is not None and completed < started: + errors.append( + f"{lane_path}.completed_at.value: must not precede started_at" + ) + if ( + recorded_at is not None + and completed is not None + and completed > recorded_at + ): + errors.append( + f"{lane_path}.completed_at.value: must not be later than recorded_at" + ) + + outcome = receipt.get("outcome_improvement") + if isinstance(outcome, dict): + assessment = outcome.get("assessment") + observed = outcome.get("observed_improvements") + unavailable = outcome.get("unavailable_measurements") + if assessment == "benefited" and observed == []: + errors.append( + "$.outcome_improvement.observed_improvements: benefited requires " + "at least one observed improvement" + ) + if assessment == "unavailable": + if observed not in (None, []): + errors.append( + "$.outcome_improvement.observed_improvements: must be empty when " + "assessment is unavailable" + ) + if unavailable == []: + errors.append( + "$.outcome_improvement.unavailable_measurements: unavailable " + "assessment requires at least one unavailable measurement" + ) + + routing = receipt.get("routing_decision") + if isinstance(model, dict) and isinstance(routing, dict): + orchestration_setting = model.get("orchestration_setting") + route_used = routing.get("route_used") + if ( + isinstance(orchestration_setting, str) + and isinstance(route_used, str) + and route_used != orchestration_setting + ): + errors.append( + "$.routing_decision.route_used: must equal " + "$.model.orchestration_setting " + f"({orchestration_setting!r})" + ) + counterfactual_route = routing.get("counterfactual_route") + if ( + isinstance(route_used, str) + and isinstance(counterfactual_route, str) + and counterfactual_route == route_used + ): + errors.append( + "$.routing_decision.counterfactual_route: must differ from route_used" + ) + if routing.get("policy_status") != "provisional": + errors.append( + "$.routing_decision.policy_status: v0.1 permits only 'provisional'; " + "machine-evidence requirements require a future schema revision" + ) + + +def validate_receipt(receipt: Any, schema: Any) -> list[str]: + """Return sorted validation errors; an empty list means the receipt is valid.""" + + errors: list[str] = [] + if not isinstance(schema, dict): + return ["$schema: schema document must be a JSON object"] + if schema.get("$schema") != EXPECTED_META_SCHEMA: + errors.append( + f"$schema.$schema: expected Draft 2020-12 URI {EXPECTED_META_SCHEMA!r}" + ) + if schema.get("$id") != EXPECTED_SCHEMA_URI: + errors.append(f"$schema.$id: expected schema identity {EXPECTED_SCHEMA_URI!r}") + + _validate_schema_subset(receipt, schema, schema, "$", errors) + _validate_receipt_semantics(receipt, errors) + return sorted(set(errors)) + + +def _reject_non_standard_constant(token: str) -> Any: + raise ValueError(f"non-standard JSON numeric constant {token!r} is not permitted") + + +def _parse_finite_float(token: str) -> float: + value = float(token) + if not math.isfinite(value): + raise ValueError(f"overflowed JSON number {token!r} is not permitted") + return value + + +def _load_json(path: Path) -> Any: + with path.open(encoding="utf-8") as handle: + return json.load( + handle, + parse_constant=_reject_non_standard_constant, + parse_float=_parse_finite_float, + ) + + +def _build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + description="Validate an ultra_evaluation_receipt.v0.1 JSON document." + ) + parser.add_argument("receipt", type=Path, help="Receipt JSON path") + parser.add_argument( + "--schema", + type=Path, + default=DEFAULT_SCHEMA_PATH, + help=f"Schema JSON path (default: {DEFAULT_SCHEMA_PATH})", + ) + return parser + + +def main(argv: list[str] | None = None) -> int: + args = _build_parser().parse_args(argv) + try: + schema = _load_json(args.schema) + receipt = _load_json(args.receipt) + except (OSError, ValueError) as exc: + print(f"ERROR: {exc}", file=sys.stderr) + return 2 + + errors = validate_receipt(receipt, schema) + if errors: + print(f"INVALID: {args.receipt}", file=sys.stderr) + for error in errors: + print(f"- {error}", file=sys.stderr) + return 1 + + print(f"VALID: {args.receipt}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main())