Phase 2a: harden MHS reliability bootstrap

reports/mhs/metrics.py

@@ -76,7 +76,13 @@ def bootstrap_statistic(
     iterations=BOOTSTRAP_ITERATIONS,
     seed=BOOTSTRAP_SEED,
 ):
-    """Item-level percentile-bootstrap interval for an arbitrary statistic."""
+    """Item-level percentile bootstrap with explicit degenerate-draw handling.
+
+    ``iterations`` is always the total number of draws.  A statistic may
+    return ``None`` when it is undefined for a resample.  Such draws are never
+    replaced: any degeneracy makes the interval non-applicable rather than
+    conditioning it on the surviving estimates.
+    """
     observations = list(observations)
     if not observations:
         raise ValueError("observations must not be empty")
@@ -87,15 +93,23 @@ def bootstrap_statistic(
     estimates = []
     for _ in range(iterations):
         sample = [observations[rng.randrange(size)] for _ in range(size)]
-        estimate = float(statistic(sample))
+        estimate = statistic(sample)
+        if estimate is None:
+            continue
+        estimate = float(estimate)
         if not math.isfinite(estimate):
             raise ValueError("bootstrap statistic must be finite")
         estimates.append(estimate)
+    degenerate_resamples = iterations - len(estimates)
+    applicable = degenerate_resamples == 0
     return {
-        "lower": _percentile(estimates, 0.025),
-        "upper": _percentile(estimates, 0.975),
+        "lower": _percentile(estimates, 0.025) if applicable else None,
+        "upper": _percentile(estimates, 0.975) if applicable else None,
         "iterations": iterations,
         "seed": seed,
+        "valid_estimates": len(estimates),
+        "degenerate_resamples": degenerate_resamples,
+        "status": "ok" if applicable else "degenerate/non-applicable",
     }
 
 

reports/mhs/run_study.py

@@ -197,14 +197,30 @@ def _confident_contributions(dataset, triage):
     return contributions
 
 
-def _reliability_values(contributions, annotators):
+def _reliability_by_annotator(contributions, annotators):
     hits, totals = Counter(), Counter()
     for contribution in contributions:
         for annotator, (hit, total) in contribution.items():
             if annotator in annotators:
                 hits[annotator] += hit
                 totals[annotator] += total
-    return [hits[annotator] / totals[annotator] for annotator in annotators if totals[annotator]]
+    return {
+        annotator: hits[annotator] / totals[annotator]
+        for annotator in sorted(annotators)
+        if totals[annotator]
+    }
+
+
+def _reliability_values(contributions, annotators):
+    return list(_reliability_by_annotator(contributions, annotators).values())
+
+
+def _reliability_median_difference(sample, fixed_cohorts):
+    conservative = _reliability_values(sample, fixed_cohorts["Conservative"])
+    liberal = _reliability_values(sample, fixed_cohorts["Liberal"])
+    if not conservative or not liberal:
+        return None
+    return statistics.median(conservative) - statistics.median(liberal)
 
 
 def aggregate_reliability(dataset, triage):
@@ -240,13 +256,10 @@ def aggregate_reliability(dataset, triage):
     interval = None
     if powered:
         fixed = {cohort: set(qualifying[cohort]) for cohort in mhs.COHORTS}
-
-        def statistic(sample):
-            conservative = _reliability_values(sample, fixed["Conservative"])
-            liberal = _reliability_values(sample, fixed["Liberal"])
-            return statistics.median(conservative) - statistics.median(liberal)
-
-        interval = metrics.bootstrap_statistic(contributions, statistic)
+        interval = metrics.bootstrap_statistic(
+            contributions,
+            lambda sample: _reliability_median_difference(sample, fixed),
+        )
     return {
         "status": "descriptive" if powered else "underpowered/non-applicable",
         "minimum_confident_cells": RELIABILITY_MIN_CONFIDENT,
@@ -275,6 +288,13 @@ def _fmt(number):
     return "not applicable" if number is None else "{:.6f}".format(number)
 
 
+def _bootstrap_field(interval, key):
+    if interval is None:
+        return "not applicable"
+    value = interval[key]
+    return "not applicable" if value is None else str(value)
+
+
 def render_report(results, counts, tool_commit):
     primary = results["primary"]
     reliability = results["reliability"]
@@ -300,8 +320,8 @@ def render_report(results, counts, tool_commit):
 
 - Value forks: {fork_count}/{fork_total} ({fork_rate:.6f}); Wilson 95% [{fork_lo:.6f}, {fork_hi:.6f}]
 - Manufactured consensus: {manufactured_count}/{manufactured_total} ({manufactured_rate:.6f}); Wilson 95% [{manufactured_lo:.6f}, {manufactured_hi:.6f}]
-- Geometry: {undefined} undefined/disjoint-support items ({undefined_share:.6f}); defined-gap median {gap_median}, Q1 {gap_q1}, Q3 {gap_q3}, IQR {gap_iqr}, item-bootstrap 95% [{gap_boot_lo}, {gap_boot_hi}]
-- Reliability: {reliability_status}; coverage Conservative {conservative_qualifying}/{conservative_eligible}, Liberal {liberal_qualifying}/{liberal_eligible}; Conservative median {conservative_median} (IQR {conservative_iqr}), Liberal median {liberal_median} (IQR {liberal_iqr}); Conservative-minus-Liberal median difference {reliability_difference}, item-bootstrap 95% [{reliability_boot_lo}, {reliability_boot_hi}]
+- Geometry: {undefined} undefined/disjoint-support items ({undefined_share:.6f}); defined-gap median {gap_median}, Q1 {gap_q1}, Q3 {gap_q3}, IQR {gap_iqr}, item-bootstrap 95% [{gap_boot_lo}, {gap_boot_hi}] (status {gap_boot_status}; total draws {gap_boot_iterations}; valid estimates {gap_boot_valid}; degenerate resamples {gap_boot_degenerate})
+- Reliability: {reliability_status}; coverage Conservative {conservative_qualifying}/{conservative_eligible}, Liberal {liberal_qualifying}/{liberal_eligible}; Conservative median {conservative_median} (IQR {conservative_iqr}), Liberal median {liberal_median} (IQR {liberal_iqr}); Conservative-minus-Liberal median difference {reliability_difference}, item-bootstrap 95% [{reliability_boot_lo}, {reliability_boot_hi}] (status {reliability_boot_status}; total draws {reliability_boot_iterations}; valid estimates {reliability_boot_valid}; degenerate resamples {reliability_boot_degenerate})
 
 No source rows or identifiers are published. The source checksum and aggregate
 results are recorded in `manifest.json`. No null-hypothesis p-values are used.
@@ -347,6 +367,30 @@ def render_report(results, counts, tool_commit):
             if geometry_result["defined"]
             else None
         ),
+        gap_boot_status=_bootstrap_field(
+            geometry_result["defined"]["bootstrap_95"]
+            if geometry_result["defined"]
+            else None,
+            "status",
+        ),
+        gap_boot_iterations=_bootstrap_field(
+            geometry_result["defined"]["bootstrap_95"]
+            if geometry_result["defined"]
+            else None,
+            "iterations",
+        ),
+        gap_boot_valid=_bootstrap_field(
+            geometry_result["defined"]["bootstrap_95"]
+            if geometry_result["defined"]
+            else None,
+            "valid_estimates",
+        ),
+        gap_boot_degenerate=_bootstrap_field(
+            geometry_result["defined"]["bootstrap_95"]
+            if geometry_result["defined"]
+            else None,
+            "degenerate_resamples",
+        ),
         reliability_status=reliability["status"],
         conservative_qualifying=reliability["coverage"]["Conservative"]["qualifying"],
         conservative_eligible=reliability["coverage"]["Conservative"]["eligible"],
@@ -385,6 +429,18 @@ def render_report(results, counts, tool_commit):
             if reliability["bootstrap_95"]
             else None
         ),
+        reliability_boot_status=_bootstrap_field(
+            reliability["bootstrap_95"], "status"
+        ),
+        reliability_boot_iterations=_bootstrap_field(
+            reliability["bootstrap_95"], "iterations"
+        ),
+        reliability_boot_valid=_bootstrap_field(
+            reliability["bootstrap_95"], "valid_estimates"
+        ),
+        reliability_boot_degenerate=_bootstrap_field(
+            reliability["bootstrap_95"], "degenerate_resamples"
+        ),
     )
 
 

test_claims.py

@@ -17,6 +17,7 @@
 import math
 import random
 import shutil
+import statistics
 import subprocess
 import sys
 import tempfile
@@ -1486,6 +1487,200 @@ def test_synthetic_tool_wiring_and_frozen_metric_results(self):
                 },
             )
             self.assertIsNone(reliability["bootstrap_95"])
+            underpowered_report = mhs_study.render_report(
+                {"primary": primary_metrics, "reliability": reliability},
+                self.converted["counts"],
+                "synthetic-tool-commit",
+            )
+            self.assertIn(
+                "item-bootstrap 95% [not applicable, not applicable] "
+                "(status not applicable; total draws not applicable; valid "
+                "estimates not applicable; degenerate resamples not applicable)",
+                underpowered_report,
+            )
+
+    def test_powered_reliability_mirror_and_production_bootstrap(self):
+        conservative = ["fake-c-{:02d}".format(index) for index in range(30)]
+        liberal = ["fake-l-{:02d}".format(index) for index in range(30)]
+        annotators = conservative + liberal
+        items = []
+        for index in range(24):
+            items.append(
+                {
+                    "id": "fake-powered-{:02d}".format(index),
+                    "desc": "Synthetic powered reliability cell {}".format(index),
+                    "labels": {
+                        "hatespeech": {annotator: "0" for annotator in annotators}
+                    },
+                    "reasons": {},
+                }
+            )
+
+        for index, annotator in enumerate(conservative[:16]):
+            items[index]["labels"]["hatespeech"][annotator] = "1"
+        for index, annotator in enumerate(liberal[:16]):
+            distinct_cells = ((16 + index) % 24, (8 + index) % 24)
+            self.assertNotEqual(*distinct_cells)
+            for cell in distinct_cells:
+                items[cell]["labels"]["hatespeech"][annotator] = "1"
+
+        self.assertLessEqual(
+            max(
+                sum(label != "0" for label in item["labels"]["hatespeech"].values())
+                for item in items
+            ),
+            3,
+        )
+
+        dataset = {
+            "questions": {
+                "hatespeech": {
+                    "type": "categorical",
+                    "labels": ["0", "1", "2"],
+                }
+            },
+            "annotators": annotators,
+            "cohorts": {
+                "Conservative": conservative,
+                "Liberal": liberal,
+            },
+            "items": items,
+        }
+
+        with tempfile.TemporaryDirectory(prefix="groundless-mhs-powered-") as temp:
+            temp_path = Path(temp)
+            data_path = temp_path / "powered-labels.json"
+            data_path.write_text(
+                json.dumps(dataset, indent=2, sort_keys=True) + "\n",
+                encoding="utf-8",
+            )
+            subprocess.run(
+                [
+                    sys.executable,
+                    str(ROOT / "disagreement.py"),
+                    "--data",
+                    str(data_path),
+                    "--out",
+                    str(temp_path / "out"),
+                ],
+                check=True,
+                stdout=subprocess.PIPE,
+                stderr=subprocess.PIPE,
+                universal_newlines=True,
+            )
+            triage = _load_json(temp_path / "out" / "triage.json")
+
+        self.assertEqual(len(triage["cells"]), 24)
+        self.assertTrue(all(cell["verdict"] == "CONFIDENT" for cell in triage["cells"]))
+        contributions = mhs_study._confident_contributions(dataset, triage)
+        self.assertEqual(len(contributions), 24)
+        scored_counts = Counter(
+            annotator
+            for contribution in contributions
+            for annotator in contribution
+        )
+        self.assertEqual(set(scored_counts.values()), {24})
+        self.assertEqual(set(scored_counts), set(annotators))
+        mirror = mhs_study._reliability_by_annotator(contributions, annotators)
+        self.assertEqual(mirror, triage["reliability"])
+        self.assertEqual(set(mirror), set(annotators))
+
+        conservative_median = statistics.median(
+            mirror[annotator] for annotator in conservative
+        )
+        liberal_median = statistics.median(mirror[annotator] for annotator in liberal)
+        self.assertEqual(conservative_median, 23 / 24)
+        self.assertEqual(liberal_median, 22 / 24)
+        self.assertAlmostEqual(conservative_median - liberal_median, 1 / 24)
+
+        reliability = mhs_study.aggregate_reliability(dataset, triage)
+        self.assertEqual(reliability["status"], "descriptive")
+        self.assertEqual(
+            reliability["coverage"],
+            {
+                "Conservative": {"eligible": 30, "qualifying": 30},
+                "Liberal": {"eligible": 30, "qualifying": 30},
+            },
+        )
+        self.assertEqual(
+            reliability["summaries"]["Conservative"]["median"], 23 / 24
+        )
+        self.assertEqual(reliability["summaries"]["Liberal"]["median"], 22 / 24)
+        self.assertAlmostEqual(
+            reliability["median_difference_conservative_minus_liberal"], 1 / 24
+        )
+        interval = reliability["bootstrap_95"]
+        self.assertEqual(interval["iterations"], 10000)
+        self.assertEqual(interval["seed"], 20260718)
+        self.assertEqual(interval["valid_estimates"], 10000)
+        self.assertEqual(interval["degenerate_resamples"], 0)
+        self.assertEqual(interval["status"], "ok")
+        self.assertEqual(interval["lower"], 0.0)
+        self.assertAlmostEqual(interval["upper"], 1 / 24)
+
+        primary = mhs_study.aggregate_primary(dataset, triage)
+        report = mhs_study.render_report(
+            {"primary": primary, "reliability": reliability},
+            {
+                "primary_items": 24,
+                "reliability_items": 24,
+                "conservative_annotators": 30,
+                "liberal_annotators": 30,
+            },
+            "synthetic-tool-commit",
+        )
+        self.assertEqual(
+            report.count(
+                "status ok; total draws 10000; valid estimates 10000; "
+                "degenerate resamples 0"
+            ),
+            2,
+        )
+        result_keys = json.dumps(
+            {"primary": primary, "reliability": reliability}, sort_keys=True
+        )
+        self.assertNotIn("p_value", result_keys)
+        self.assertNotIn("pvalue", result_keys)
+
+    def test_degenerate_bootstrap_is_disclosed_without_redraws(self):
+        fixed = {
+            "Conservative": {"fake-c"},
+            "Liberal": {"fake-l"},
+        }
+        contributions = [
+            {"fake-c": (1, 1)},
+            {"fake-l": (1, 1)},
+        ]
+
+        def statistic(sample):
+            return mhs_study._reliability_median_difference(sample, fixed)
+
+        first = mhs_metrics.bootstrap_statistic(
+            contributions, statistic
+        )
+        second = mhs_metrics.bootstrap_statistic(
+            contributions, statistic
+        )
+        self.assertEqual(first, second)
+        self.assertEqual(first["iterations"], 10000)
+        self.assertGreater(first["valid_estimates"], 0)
+        self.assertGreater(first["degenerate_resamples"], 0)
+        self.assertEqual(
+            first["valid_estimates"] + first["degenerate_resamples"], 10000
+        )
+        self.assertIsNone(first["lower"])
+        self.assertIsNone(first["upper"])
+        self.assertEqual(first["status"], "degenerate/non-applicable")
+
+        always = mhs_metrics.bootstrap_statistic(
+            [{"fake-c": (1, 1)}], statistic
+        )
+        self.assertEqual(always["iterations"], 10000)
+        self.assertEqual(always["valid_estimates"], 0)
+        self.assertEqual(always["degenerate_resamples"], 10000)
+        self.assertIsNone(always["lower"])
+        self.assertIsNone(always["upper"])
+        self.assertEqual(always["status"], "degenerate/non-applicable")
 
     def test_frozen_descriptive_statistics(self):
         interval = mhs_metrics.wilson_interval(5, 10)
@@ -1502,6 +1697,9 @@ def test_frozen_descriptive_statistics(self):
         self.assertEqual(first, second)
         self.assertEqual(first["iterations"], 10000)
         self.assertEqual(first["seed"], 20260718)
+        self.assertEqual(first["valid_estimates"], 10000)
+        self.assertEqual(first["degenerate_resamples"], 0)
+        self.assertEqual(first["status"], "ok")
         self.assertNotIn("p_value", first)
         self.assertNotIn("pvalue", first)