feat: whole-codebase and corpus model routing policy v0.1

.gitignore

@@ -0,0 +1,3 @@
+.coverage
+*.py[cod]
+__pycache__/

README.md

@@ -31,6 +31,22 @@
 - Example: [`examples/route-manifest-v0.1.example.json`](examples/route-manifest-v0.1.example.json)
 - Fixtures: [`fixtures/valid/route-manifest-v0.1.valid.json`](fixtures/valid/route-manifest-v0.1.valid.json), [`fixtures/invalid/route-manifest-v0.1.invalid.json`](fixtures/invalid/route-manifest-v0.1.invalid.json)
 
+## Governed evaluation packets
+
+- GPT-5.6 Sol Ultra evidence packet (search-bounded, checked 2026-07-11):
+  [`docs/evaluations/2026-07-11-gpt-5.6-sol-ultra-evidence-packet.md`](docs/evaluations/2026-07-11-gpt-5.6-sol-ultra-evidence-packet.md)
+- Provisional reasoning-effort route policy:
+  [`docs/route-policy/reasoning-effort-route-policy.v0.1.md`](docs/route-policy/reasoning-effort-route-policy.v0.1.md)
+- Controlled xhigh–Max–Ultra protocol:
+  [`docs/evaluations/ultra-evaluation-protocol.v0.1.md`](docs/evaluations/ultra-evaluation-protocol.v0.1.md)
+- Machine benchmark inventory and raw provenance:
+  [`data/gpt-5.6-sol-ultra/benchmark_dataset.v0.1.json`](data/gpt-5.6-sol-ultra/benchmark_dataset.v0.1.json)
+- Governed execution-receipt contract:
+  [`schemas/ultra_evaluation_receipt.v0.1.schema.json`](schemas/ultra_evaluation_receipt.v0.1.schema.json)
+
+These artifacts are evidence and experiment scaffolding, not adopted universal
+routing policy.
+
 ## Status
 
 Public seed repository. Initial executable packet in progress as `seed` -> `v0.1-draft`.

data/gpt-5.6-sol-ultra/archive_manifest.json

@@ -0,0 +1,297 @@
+{
+  "archive": {
+    "capture_timestamp": "20260710150348",
+    "capture_timestamp_utc": "2026-07-10T15:03:48Z",
+    "content_sha256": "242b04c63bd0da245bc2dcf601801ee958deb9ebc08def9193f5c6846fdbf892",
+    "hash_scope": "decompressed HTTP response entity bytes from the id_ capture after HTTP content decoding",
+    "original_url": "https://openai.com/index/gpt-5-6/",
+    "retrieval_date": "2026-07-11",
+    "wayback_capture_url": "https://web.archive.org/web/20260710150348id_/https://openai.com/index/gpt-5-6/",
+    "wayback_ui_url": "https://web.archive.org/web/20260710150348/https://openai.com/index/gpt-5-6/"
+  },
+  "availability": {
+    "BrowseComp": {
+      "expected_agent_counts": [
+        1,
+        4,
+        16
+      ],
+      "missing_agent_counts": [],
+      "observed_agent_counts": [
+        1,
+        4,
+        16
+      ],
+      "observed_efforts": [
+        "low",
+        "medium",
+        "high",
+        "xhigh",
+        "max"
+      ],
+      "observed_metrics": [
+        "api_cost_usd",
+        "latency_s",
+        "output_tokens"
+      ],
+      "record_count": 45
+    },
+    "SEC-Bench Pro": {
+      "expected_agent_counts": [
+        1,
+        4,
+        16
+      ],
+      "missing_agent_counts": [],
+      "observed_agent_counts": [
+        1,
+        4,
+        16
+      ],
+      "observed_efforts": [
+        "low",
+        "medium",
+        "high",
+        "xhigh",
+        "max"
+      ],
+      "observed_metrics": [
+        "api_cost_usd",
+        "latency_s",
+        "output_tokens"
+      ],
+      "record_count": 45
+    },
+    "Terminal-Bench 2.1": {
+      "expected_agent_counts": [
+        1,
+        4,
+        16
+      ],
+      "missing_agent_counts": [
+        16
+      ],
+      "observed_agent_counts": [
+        1,
+        4
+      ],
+      "observed_efforts": [
+        "low",
+        "medium",
+        "high",
+        "xhigh",
+        "max"
+      ],
+      "observed_metrics": [
+        "api_cost_usd",
+        "latency_s",
+        "output_tokens"
+      ],
+      "record_count": 30
+    }
+  },
+  "evidence_class": "archived_chart_data",
+  "extraction": {
+    "deduplication": "exact semantic JSON object; first occurrence retained",
+    "parser_input": "escaped flat eval JSON objects embedded in archive HTML",
+    "script": "tools/extract_gpt56_ultra_archive.py",
+    "selected_record_count": 120,
+    "selected_records": "data/gpt-5.6-sol-ultra/archived_chart_records.json",
+    "selected_records_sha256": "b2970b62c76a688f34b68f3a8739777f677268e50d0657db82d32a3f62345e11"
+  },
+  "limitations": [
+    "The capture is a provider page and the selected records are provider-reported.",
+    "Terminal-Bench 2.1 has no 16-agent flat chart records in this capture.",
+    "Cost and latency are provider estimates or simulations, not service guarantees.",
+    "Raw selected records intentionally preserve two distinct inaccessible internal OpenAI Slack source URLs embedded by the provider; no credentials or tokens were found. This accepts a provenance/privacy tradeoff in favor of raw-source fidelity."
+  ],
+  "provenance_privacy": {
+    "credentials_or_tokens_found": false,
+    "decision": "Preserve provider-embedded source_url values for raw provenance fidelity despite inaccessible internal workspace destinations.",
+    "preserved_internal_source_url_count": 2,
+    "preserved_internal_source_urls": [
+      "https://openai-corpws.slack.com/archives/C0B95RM2XS5/p1782526942178749?thread_ts=1782423237.343579&cid=C0B95RM2XS5",
+      "https://openai-corpws.slack.com/archives/C0B95RM2XS5/p1783576882966699?thread_ts=1783574276.164169&cid=C0B95RM2XS5"
+    ]
+  },
+  "schema_version": "gpt-5.6-sol-ultra.archive-manifest.v0.1",
+  "score_reconciliation": {
+    "BrowseComp": {
+      "archived_max": {
+        "1_agent": {
+          "availability": "available",
+          "effort": "max",
+          "score": 0.9083728278041074,
+          "score_label": "90.84%",
+          "score_percent": 90.83728278041075
+        },
+        "4_agents": {
+          "availability": "available",
+          "effort": "max",
+          "score": 0.9218009478672986,
+          "score_label": "92.18%",
+          "score_percent": 92.18009478672985
+        }
+      },
+      "comparison_rule": "Current one-decimal headline scores are reported separately from archived max chart scores; operational comparisons use archived max records only.",
+      "current_headline": {
+        "1_agent": {
+          "agrees_after_rounding_to_one_decimal": false,
+          "archived_score_percent": 90.83728278041075,
+          "difference_archived_minus_current_percentage_points": 0.437283,
+          "score_percent": 90.4
+        },
+        "4_agents": {
+          "agrees_after_rounding_to_one_decimal": true,
+          "archived_score_percent": 92.18009478672985,
+          "difference_archived_minus_current_percentage_points": -0.019905,
+          "score_percent": 92.2
+        }
+      }
+    },
+    "SEC-Bench Pro": {
+      "archived_max": {
+        "1_agent": {
+          "availability": "available",
+          "effort": "max",
+          "score": 0.7144808743169399,
+          "score_label": "71.45%",
+          "score_percent": 71.44808743169399
+        },
+        "4_agents": {
+          "availability": "available",
+          "effort": "max",
+          "score": 0.7431693989071039,
+          "score_label": "74.32%",
+          "score_percent": 74.31693989071039
+        }
+      },
+      "comparison_rule": "Current one-decimal headline scores are reported separately from archived max chart scores; operational comparisons use archived max records only.",
+      "current_headline": {
+        "1_agent": {
+          "agrees_after_rounding_to_one_decimal": false,
+          "archived_score_percent": 71.44808743169399,
+          "difference_archived_minus_current_percentage_points": 0.248087,
+          "score_percent": 71.2
+        },
+        "4_agents": {
+          "agrees_after_rounding_to_one_decimal": true,
+          "archived_score_percent": 74.31693989071039,
+          "difference_archived_minus_current_percentage_points": 0.01694,
+          "score_percent": 74.3
+        }
+      }
+    },
+    "Terminal-Bench 2.1": {
+      "archived_max": {
+        "1_agent": {
+          "availability": "available",
+          "effort": "max",
+          "score": 0.887640449438,
+          "score_label": "88.76%",
+          "score_percent": 88.7640449438
+        },
+        "4_agents": {
+          "availability": "available",
+          "effort": "max",
+          "score": 0.919101123596,
+          "score_label": "91.91%",
+          "score_percent": 91.9101123596
+        }
+      },
+      "comparison_rule": "Current one-decimal headline scores are reported separately from archived max chart scores; operational comparisons use archived max records only.",
+      "current_headline": {
+        "1_agent": {
+          "agrees_after_rounding_to_one_decimal": true,
+          "archived_score_percent": 88.7640449438,
+          "difference_archived_minus_current_percentage_points": -0.035955,
+          "score_percent": 88.8
+        },
+        "4_agents": {
+          "agrees_after_rounding_to_one_decimal": true,
+          "archived_score_percent": 91.9101123596,
+          "difference_archived_minus_current_percentage_points": 0.010112,
+          "score_percent": 91.9
+        }
+      }
+    }
+  },
+  "unit_interpretation": {
+    "api_cost_usd": {
+      "interpretation": "Provider-estimated API cost including all agents; not an observed public-service billing guarantee.",
+      "raw_metric_name": "api_cost_usd",
+      "raw_value_field": "x_value",
+      "x_value_unit": "estimated USD"
+    },
+    "latency_s": {
+      "companion_source_evidence": [
+        {
+          "canonical_record_sha256": "cd5bc06b88020e861daac3bf97dbe134e6fde9485e26e0a4bc935b9df940ffb5",
+          "record": {
+            "category": "Coding",
+            "code_mode": "true",
+            "cost": 1.7200842764057498,
+            "cost_label": "$$1.72",
+            "eval": "Terminal-Bench 2.1",
+            "eval_id": "terminal_bench_2_1",
+            "eval_variant": "",
+            "juice_index": 5,
+            "juice_level": "max",
+            "latency_label": "4.24 minutes",
+            "latency_minutes": 4.240172912070197,
+            "model": "GPT-5.6 Sol",
+            "output_tokens": 12795.8044944,
+            "output_tokens_label": "12.8K",
+            "score": 0.887640449438,
+            "score_label": "88.76%",
+            "score_metric": "",
+            "score_unit": "accuracy"
+          }
+        },
+        {
+          "canonical_record_sha256": "4a231f9d3ae0298d46190115db127640db34afd0aaa041483bb83c4a4204b310",
+          "record": {
+            "category": "Coding",
+            "code_mode": "true",
+            "cost": 5.14084233483,
+            "cost_label": "$$5.14",
+            "eval": "Terminal-Bench 2.1",
+            "eval_id": "terminal_bench_2_1",
+            "eval_variant": "",
+            "juice_index": 5,
+            "juice_level": "max",
+            "latency_label": "4.28 minutes",
+            "latency_minutes": 4.278585916301308,
+            "model": "GPT-5.6 Sol Ultra",
+            "output_tokens": 39275.1280899,
+            "output_tokens_label": "39.27K",
+            "score": 0.919101123596,
+            "score_label": "91.91%",
+            "score_metric": "",
+            "score_unit": "accuracy"
+          }
+        }
+      ],
+      "interpretation": "Provider-simulated root-agent latency; not an observed public-service latency guarantee.",
+      "interpretation_basis": [
+        "Every selected latency display label renders x_value in minutes.",
+        "Preserved companion source evidence names the same quantity latency_minutes."
+      ],
+      "raw_metric_name": "latency_s",
+      "raw_value_field": "x_value",
+      "warning": "The raw metric name suggests seconds, but treating x_value as seconds conflicts with the displayed labels; values are preserved unchanged and interpreted as minutes.",
+      "x_value_unit": "minutes"
+    },
+    "output_tokens": {
+      "interpretation": "Aggregate output tokens include all agents.",
+      "raw_metric_name": "output_tokens",
+      "raw_value_field": "x_value",
+      "x_value_unit": "tokens"
+    },
+    "score": {
+      "display_unit": "percent",
+      "raw_value_field": "score",
+      "stored_unit": "fraction"
+    }
+  }
+}