diff --git a/reports/mhs/run_study.py b/reports/mhs/run_study.py index f7ebfaa..3f9f729 100644 --- a/reports/mhs/run_study.py +++ b/reports/mhs/run_study.py @@ -26,6 +26,7 @@ SOURCE_REVISION = "5468f6e" SOURCE_SHA256 = "6819525ce61bc24344df9fc3f7bf48270b31038273cc27c67fc225b51433b0e1" PROTOCOL_PATH = "reports/part-3b-dataset-selection-audit.md" +ADDENDUM_PATH = "reports/part-3b-reliability-corpus-addendum-2026-07-19.md" RELIABILITY_MIN_CONFIDENT = 20 RELIABILITY_MIN_PER_COHORT = 30 PARQUET_COLUMNS = ( @@ -307,6 +308,7 @@ def render_report(results, counts, tool_commit): ground truth, or variation-versus-error attribution. Protocol: `{protocol}`
+Reliability addendum: `{addendum}`
Tool commit: `{commit}` ## Structural counts @@ -328,6 +330,7 @@ def render_report(results, counts, tool_commit): """.format( revision=SOURCE_REVISION, protocol=PROTOCOL_PATH, + addendum=ADDENDUM_PATH, commit=tool_commit, primary_items=counts["primary_items"], reliability_items=counts["reliability_items"], @@ -444,6 +447,22 @@ def render_report(results, counts, tool_commit): ) +def build_manifest(source_hash, counts, results, tool_commit, generated_at): + """Build the aggregate evidence object without reading source data.""" + return { + "evidence_tier": "Tier 2", + "generated_at": generated_at, + "source_revision": SOURCE_REVISION, + "source_sha256": source_hash, + "protocol": PROTOCOL_PATH, + "addendum": ADDENDUM_PATH, + "tool_commit": tool_commit, + "counts": counts, + "metrics": results, + "contains_source_rows_or_ids": False, + } + + def main(argv=None): args = _parse_args(argv) source = Path(args.source).resolve() @@ -468,20 +487,19 @@ def main(argv=None): ), } commit = _tool_commit() - manifest = { - "evidence_tier": "Tier 2", - "generated_at": datetime.datetime.now(datetime.timezone.utc) + generated_at = ( + datetime.datetime.now(datetime.timezone.utc) .replace(microsecond=0) .isoformat() - .replace("+00:00", "Z"), - "source_revision": SOURCE_REVISION, - "source_sha256": source_hash, - "protocol": PROTOCOL_PATH, - "tool_commit": commit, - "counts": converted["counts"], - "metrics": results, - "contains_source_rows_or_ids": False, - } + .replace("+00:00", "Z") + ) + manifest = build_manifest( + source_hash, + converted["counts"], + results, + commit, + generated_at, + ) report_dir = Path(args.report_dir).resolve() report_dir.mkdir(parents=True, exist_ok=True) _write_json(report_dir / "manifest.json", manifest) diff --git a/test_claims.py b/test_claims.py index acb84d4..399c2a4 100644 --- a/test_claims.py +++ b/test_claims.py @@ -1619,15 +1619,44 @@ def test_powered_reliability_mirror_and_production_bootstrap(self): self.assertAlmostEqual(interval["upper"], 1 / 24) primary = mhs_study.aggregate_primary(dataset, triage) + results = {"primary": primary, "reliability": reliability} + counts = { + "primary_items": 24, + "reliability_items": 24, + "conservative_annotators": 30, + "liberal_annotators": 30, + } report = mhs_study.render_report( - {"primary": primary, "reliability": reliability}, + results, + counts, + "synthetic-tool-commit", + ) + manifest = mhs_study.build_manifest( + "synthetic-source-sha256", + counts, + results, + "synthetic-tool-commit", + "2026-07-19T00:00:00Z", + ) + self.assertEqual( + manifest, { - "primary_items": 24, - "reliability_items": 24, - "conservative_annotators": 30, - "liberal_annotators": 30, + "evidence_tier": "Tier 2", + "generated_at": "2026-07-19T00:00:00Z", + "source_revision": mhs_study.SOURCE_REVISION, + "source_sha256": "synthetic-source-sha256", + "protocol": mhs_study.PROTOCOL_PATH, + "addendum": mhs_study.ADDENDUM_PATH, + "tool_commit": "synthetic-tool-commit", + "counts": counts, + "metrics": results, + "contains_source_rows_or_ids": False, }, - "synthetic-tool-commit", + ) + self.assertIn("Protocol: `{}`".format(mhs_study.PROTOCOL_PATH), report) + self.assertIn( + "Reliability addendum: `{}`".format(mhs_study.ADDENDUM_PATH), + report, ) self.assertEqual( report.count( @@ -1637,7 +1666,7 @@ def test_powered_reliability_mirror_and_production_bootstrap(self): 2, ) result_keys = json.dumps( - {"primary": primary, "reliability": reliability}, sort_keys=True + results, sort_keys=True ) self.assertNotIn("p_value", result_keys) self.assertNotIn("pvalue", result_keys)