diff --git a/reports/mhs/run_study.py b/reports/mhs/run_study.py
index f7ebfaa..3f9f729 100644
--- a/reports/mhs/run_study.py
+++ b/reports/mhs/run_study.py
@@ -26,6 +26,7 @@
SOURCE_REVISION = "5468f6e"
SOURCE_SHA256 = "6819525ce61bc24344df9fc3f7bf48270b31038273cc27c67fc225b51433b0e1"
PROTOCOL_PATH = "reports/part-3b-dataset-selection-audit.md"
+ADDENDUM_PATH = "reports/part-3b-reliability-corpus-addendum-2026-07-19.md"
RELIABILITY_MIN_CONFIDENT = 20
RELIABILITY_MIN_PER_COHORT = 30
PARQUET_COLUMNS = (
@@ -307,6 +308,7 @@ def render_report(results, counts, tool_commit):
ground truth, or variation-versus-error attribution.
Protocol: `{protocol}`
+Reliability addendum: `{addendum}`
Tool commit: `{commit}`
## Structural counts
@@ -328,6 +330,7 @@ def render_report(results, counts, tool_commit):
""".format(
revision=SOURCE_REVISION,
protocol=PROTOCOL_PATH,
+ addendum=ADDENDUM_PATH,
commit=tool_commit,
primary_items=counts["primary_items"],
reliability_items=counts["reliability_items"],
@@ -444,6 +447,22 @@ def render_report(results, counts, tool_commit):
)
+def build_manifest(source_hash, counts, results, tool_commit, generated_at):
+ """Build the aggregate evidence object without reading source data."""
+ return {
+ "evidence_tier": "Tier 2",
+ "generated_at": generated_at,
+ "source_revision": SOURCE_REVISION,
+ "source_sha256": source_hash,
+ "protocol": PROTOCOL_PATH,
+ "addendum": ADDENDUM_PATH,
+ "tool_commit": tool_commit,
+ "counts": counts,
+ "metrics": results,
+ "contains_source_rows_or_ids": False,
+ }
+
+
def main(argv=None):
args = _parse_args(argv)
source = Path(args.source).resolve()
@@ -468,20 +487,19 @@ def main(argv=None):
),
}
commit = _tool_commit()
- manifest = {
- "evidence_tier": "Tier 2",
- "generated_at": datetime.datetime.now(datetime.timezone.utc)
+ generated_at = (
+ datetime.datetime.now(datetime.timezone.utc)
.replace(microsecond=0)
.isoformat()
- .replace("+00:00", "Z"),
- "source_revision": SOURCE_REVISION,
- "source_sha256": source_hash,
- "protocol": PROTOCOL_PATH,
- "tool_commit": commit,
- "counts": converted["counts"],
- "metrics": results,
- "contains_source_rows_or_ids": False,
- }
+ .replace("+00:00", "Z")
+ )
+ manifest = build_manifest(
+ source_hash,
+ converted["counts"],
+ results,
+ commit,
+ generated_at,
+ )
report_dir = Path(args.report_dir).resolve()
report_dir.mkdir(parents=True, exist_ok=True)
_write_json(report_dir / "manifest.json", manifest)
diff --git a/test_claims.py b/test_claims.py
index acb84d4..399c2a4 100644
--- a/test_claims.py
+++ b/test_claims.py
@@ -1619,15 +1619,44 @@ def test_powered_reliability_mirror_and_production_bootstrap(self):
self.assertAlmostEqual(interval["upper"], 1 / 24)
primary = mhs_study.aggregate_primary(dataset, triage)
+ results = {"primary": primary, "reliability": reliability}
+ counts = {
+ "primary_items": 24,
+ "reliability_items": 24,
+ "conservative_annotators": 30,
+ "liberal_annotators": 30,
+ }
report = mhs_study.render_report(
- {"primary": primary, "reliability": reliability},
+ results,
+ counts,
+ "synthetic-tool-commit",
+ )
+ manifest = mhs_study.build_manifest(
+ "synthetic-source-sha256",
+ counts,
+ results,
+ "synthetic-tool-commit",
+ "2026-07-19T00:00:00Z",
+ )
+ self.assertEqual(
+ manifest,
{
- "primary_items": 24,
- "reliability_items": 24,
- "conservative_annotators": 30,
- "liberal_annotators": 30,
+ "evidence_tier": "Tier 2",
+ "generated_at": "2026-07-19T00:00:00Z",
+ "source_revision": mhs_study.SOURCE_REVISION,
+ "source_sha256": "synthetic-source-sha256",
+ "protocol": mhs_study.PROTOCOL_PATH,
+ "addendum": mhs_study.ADDENDUM_PATH,
+ "tool_commit": "synthetic-tool-commit",
+ "counts": counts,
+ "metrics": results,
+ "contains_source_rows_or_ids": False,
},
- "synthetic-tool-commit",
+ )
+ self.assertIn("Protocol: `{}`".format(mhs_study.PROTOCOL_PATH), report)
+ self.assertIn(
+ "Reliability addendum: `{}`".format(mhs_study.ADDENDUM_PATH),
+ report,
)
self.assertEqual(
report.count(
@@ -1637,7 +1666,7 @@ def test_powered_reliability_mirror_and_production_bootstrap(self):
2,
)
result_keys = json.dumps(
- {"primary": primary, "reliability": reliability}, sort_keys=True
+ results, sort_keys=True
)
self.assertNotIn("p_value", result_keys)
self.assertNotIn("pvalue", result_keys)