diff --git a/reports/mhs/metrics.py b/reports/mhs/metrics.py index e1026ec..dd4e78a 100644 --- a/reports/mhs/metrics.py +++ b/reports/mhs/metrics.py @@ -76,7 +76,13 @@ def bootstrap_statistic( iterations=BOOTSTRAP_ITERATIONS, seed=BOOTSTRAP_SEED, ): - """Item-level percentile-bootstrap interval for an arbitrary statistic.""" + """Item-level percentile bootstrap with explicit degenerate-draw handling. + + ``iterations`` is always the total number of draws. A statistic may + return ``None`` when it is undefined for a resample. Such draws are never + replaced: any degeneracy makes the interval non-applicable rather than + conditioning it on the surviving estimates. + """ observations = list(observations) if not observations: raise ValueError("observations must not be empty") @@ -87,15 +93,23 @@ def bootstrap_statistic( estimates = [] for _ in range(iterations): sample = [observations[rng.randrange(size)] for _ in range(size)] - estimate = float(statistic(sample)) + estimate = statistic(sample) + if estimate is None: + continue + estimate = float(estimate) if not math.isfinite(estimate): raise ValueError("bootstrap statistic must be finite") estimates.append(estimate) + degenerate_resamples = iterations - len(estimates) + applicable = degenerate_resamples == 0 return { - "lower": _percentile(estimates, 0.025), - "upper": _percentile(estimates, 0.975), + "lower": _percentile(estimates, 0.025) if applicable else None, + "upper": _percentile(estimates, 0.975) if applicable else None, "iterations": iterations, "seed": seed, + "valid_estimates": len(estimates), + "degenerate_resamples": degenerate_resamples, + "status": "ok" if applicable else "degenerate/non-applicable", } diff --git a/reports/mhs/run_study.py b/reports/mhs/run_study.py index 491626d..f7ebfaa 100644 --- a/reports/mhs/run_study.py +++ b/reports/mhs/run_study.py @@ -197,14 +197,30 @@ def _confident_contributions(dataset, triage): return contributions -def _reliability_values(contributions, annotators): +def _reliability_by_annotator(contributions, annotators): hits, totals = Counter(), Counter() for contribution in contributions: for annotator, (hit, total) in contribution.items(): if annotator in annotators: hits[annotator] += hit totals[annotator] += total - return [hits[annotator] / totals[annotator] for annotator in annotators if totals[annotator]] + return { + annotator: hits[annotator] / totals[annotator] + for annotator in sorted(annotators) + if totals[annotator] + } + + +def _reliability_values(contributions, annotators): + return list(_reliability_by_annotator(contributions, annotators).values()) + + +def _reliability_median_difference(sample, fixed_cohorts): + conservative = _reliability_values(sample, fixed_cohorts["Conservative"]) + liberal = _reliability_values(sample, fixed_cohorts["Liberal"]) + if not conservative or not liberal: + return None + return statistics.median(conservative) - statistics.median(liberal) def aggregate_reliability(dataset, triage): @@ -240,13 +256,10 @@ def aggregate_reliability(dataset, triage): interval = None if powered: fixed = {cohort: set(qualifying[cohort]) for cohort in mhs.COHORTS} - - def statistic(sample): - conservative = _reliability_values(sample, fixed["Conservative"]) - liberal = _reliability_values(sample, fixed["Liberal"]) - return statistics.median(conservative) - statistics.median(liberal) - - interval = metrics.bootstrap_statistic(contributions, statistic) + interval = metrics.bootstrap_statistic( + contributions, + lambda sample: _reliability_median_difference(sample, fixed), + ) return { "status": "descriptive" if powered else "underpowered/non-applicable", "minimum_confident_cells": RELIABILITY_MIN_CONFIDENT, @@ -275,6 +288,13 @@ def _fmt(number): return "not applicable" if number is None else "{:.6f}".format(number) +def _bootstrap_field(interval, key): + if interval is None: + return "not applicable" + value = interval[key] + return "not applicable" if value is None else str(value) + + def render_report(results, counts, tool_commit): primary = results["primary"] reliability = results["reliability"] @@ -300,8 +320,8 @@ def render_report(results, counts, tool_commit): - Value forks: {fork_count}/{fork_total} ({fork_rate:.6f}); Wilson 95% [{fork_lo:.6f}, {fork_hi:.6f}] - Manufactured consensus: {manufactured_count}/{manufactured_total} ({manufactured_rate:.6f}); Wilson 95% [{manufactured_lo:.6f}, {manufactured_hi:.6f}] -- Geometry: {undefined} undefined/disjoint-support items ({undefined_share:.6f}); defined-gap median {gap_median}, Q1 {gap_q1}, Q3 {gap_q3}, IQR {gap_iqr}, item-bootstrap 95% [{gap_boot_lo}, {gap_boot_hi}] -- Reliability: {reliability_status}; coverage Conservative {conservative_qualifying}/{conservative_eligible}, Liberal {liberal_qualifying}/{liberal_eligible}; Conservative median {conservative_median} (IQR {conservative_iqr}), Liberal median {liberal_median} (IQR {liberal_iqr}); Conservative-minus-Liberal median difference {reliability_difference}, item-bootstrap 95% [{reliability_boot_lo}, {reliability_boot_hi}] +- Geometry: {undefined} undefined/disjoint-support items ({undefined_share:.6f}); defined-gap median {gap_median}, Q1 {gap_q1}, Q3 {gap_q3}, IQR {gap_iqr}, item-bootstrap 95% [{gap_boot_lo}, {gap_boot_hi}] (status {gap_boot_status}; total draws {gap_boot_iterations}; valid estimates {gap_boot_valid}; degenerate resamples {gap_boot_degenerate}) +- Reliability: {reliability_status}; coverage Conservative {conservative_qualifying}/{conservative_eligible}, Liberal {liberal_qualifying}/{liberal_eligible}; Conservative median {conservative_median} (IQR {conservative_iqr}), Liberal median {liberal_median} (IQR {liberal_iqr}); Conservative-minus-Liberal median difference {reliability_difference}, item-bootstrap 95% [{reliability_boot_lo}, {reliability_boot_hi}] (status {reliability_boot_status}; total draws {reliability_boot_iterations}; valid estimates {reliability_boot_valid}; degenerate resamples {reliability_boot_degenerate}) No source rows or identifiers are published. The source checksum and aggregate results are recorded in `manifest.json`. No null-hypothesis p-values are used. @@ -347,6 +367,30 @@ def render_report(results, counts, tool_commit): if geometry_result["defined"] else None ), + gap_boot_status=_bootstrap_field( + geometry_result["defined"]["bootstrap_95"] + if geometry_result["defined"] + else None, + "status", + ), + gap_boot_iterations=_bootstrap_field( + geometry_result["defined"]["bootstrap_95"] + if geometry_result["defined"] + else None, + "iterations", + ), + gap_boot_valid=_bootstrap_field( + geometry_result["defined"]["bootstrap_95"] + if geometry_result["defined"] + else None, + "valid_estimates", + ), + gap_boot_degenerate=_bootstrap_field( + geometry_result["defined"]["bootstrap_95"] + if geometry_result["defined"] + else None, + "degenerate_resamples", + ), reliability_status=reliability["status"], conservative_qualifying=reliability["coverage"]["Conservative"]["qualifying"], conservative_eligible=reliability["coverage"]["Conservative"]["eligible"], @@ -385,6 +429,18 @@ def render_report(results, counts, tool_commit): if reliability["bootstrap_95"] else None ), + reliability_boot_status=_bootstrap_field( + reliability["bootstrap_95"], "status" + ), + reliability_boot_iterations=_bootstrap_field( + reliability["bootstrap_95"], "iterations" + ), + reliability_boot_valid=_bootstrap_field( + reliability["bootstrap_95"], "valid_estimates" + ), + reliability_boot_degenerate=_bootstrap_field( + reliability["bootstrap_95"], "degenerate_resamples" + ), ) diff --git a/test_claims.py b/test_claims.py index 6921c4a..acb84d4 100644 --- a/test_claims.py +++ b/test_claims.py @@ -17,6 +17,7 @@ import math import random import shutil +import statistics import subprocess import sys import tempfile @@ -1486,6 +1487,200 @@ def test_synthetic_tool_wiring_and_frozen_metric_results(self): }, ) self.assertIsNone(reliability["bootstrap_95"]) + underpowered_report = mhs_study.render_report( + {"primary": primary_metrics, "reliability": reliability}, + self.converted["counts"], + "synthetic-tool-commit", + ) + self.assertIn( + "item-bootstrap 95% [not applicable, not applicable] " + "(status not applicable; total draws not applicable; valid " + "estimates not applicable; degenerate resamples not applicable)", + underpowered_report, + ) + + def test_powered_reliability_mirror_and_production_bootstrap(self): + conservative = ["fake-c-{:02d}".format(index) for index in range(30)] + liberal = ["fake-l-{:02d}".format(index) for index in range(30)] + annotators = conservative + liberal + items = [] + for index in range(24): + items.append( + { + "id": "fake-powered-{:02d}".format(index), + "desc": "Synthetic powered reliability cell {}".format(index), + "labels": { + "hatespeech": {annotator: "0" for annotator in annotators} + }, + "reasons": {}, + } + ) + + for index, annotator in enumerate(conservative[:16]): + items[index]["labels"]["hatespeech"][annotator] = "1" + for index, annotator in enumerate(liberal[:16]): + distinct_cells = ((16 + index) % 24, (8 + index) % 24) + self.assertNotEqual(*distinct_cells) + for cell in distinct_cells: + items[cell]["labels"]["hatespeech"][annotator] = "1" + + self.assertLessEqual( + max( + sum(label != "0" for label in item["labels"]["hatespeech"].values()) + for item in items + ), + 3, + ) + + dataset = { + "questions": { + "hatespeech": { + "type": "categorical", + "labels": ["0", "1", "2"], + } + }, + "annotators": annotators, + "cohorts": { + "Conservative": conservative, + "Liberal": liberal, + }, + "items": items, + } + + with tempfile.TemporaryDirectory(prefix="groundless-mhs-powered-") as temp: + temp_path = Path(temp) + data_path = temp_path / "powered-labels.json" + data_path.write_text( + json.dumps(dataset, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + subprocess.run( + [ + sys.executable, + str(ROOT / "disagreement.py"), + "--data", + str(data_path), + "--out", + str(temp_path / "out"), + ], + check=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + ) + triage = _load_json(temp_path / "out" / "triage.json") + + self.assertEqual(len(triage["cells"]), 24) + self.assertTrue(all(cell["verdict"] == "CONFIDENT" for cell in triage["cells"])) + contributions = mhs_study._confident_contributions(dataset, triage) + self.assertEqual(len(contributions), 24) + scored_counts = Counter( + annotator + for contribution in contributions + for annotator in contribution + ) + self.assertEqual(set(scored_counts.values()), {24}) + self.assertEqual(set(scored_counts), set(annotators)) + mirror = mhs_study._reliability_by_annotator(contributions, annotators) + self.assertEqual(mirror, triage["reliability"]) + self.assertEqual(set(mirror), set(annotators)) + + conservative_median = statistics.median( + mirror[annotator] for annotator in conservative + ) + liberal_median = statistics.median(mirror[annotator] for annotator in liberal) + self.assertEqual(conservative_median, 23 / 24) + self.assertEqual(liberal_median, 22 / 24) + self.assertAlmostEqual(conservative_median - liberal_median, 1 / 24) + + reliability = mhs_study.aggregate_reliability(dataset, triage) + self.assertEqual(reliability["status"], "descriptive") + self.assertEqual( + reliability["coverage"], + { + "Conservative": {"eligible": 30, "qualifying": 30}, + "Liberal": {"eligible": 30, "qualifying": 30}, + }, + ) + self.assertEqual( + reliability["summaries"]["Conservative"]["median"], 23 / 24 + ) + self.assertEqual(reliability["summaries"]["Liberal"]["median"], 22 / 24) + self.assertAlmostEqual( + reliability["median_difference_conservative_minus_liberal"], 1 / 24 + ) + interval = reliability["bootstrap_95"] + self.assertEqual(interval["iterations"], 10000) + self.assertEqual(interval["seed"], 20260718) + self.assertEqual(interval["valid_estimates"], 10000) + self.assertEqual(interval["degenerate_resamples"], 0) + self.assertEqual(interval["status"], "ok") + self.assertEqual(interval["lower"], 0.0) + self.assertAlmostEqual(interval["upper"], 1 / 24) + + primary = mhs_study.aggregate_primary(dataset, triage) + report = mhs_study.render_report( + {"primary": primary, "reliability": reliability}, + { + "primary_items": 24, + "reliability_items": 24, + "conservative_annotators": 30, + "liberal_annotators": 30, + }, + "synthetic-tool-commit", + ) + self.assertEqual( + report.count( + "status ok; total draws 10000; valid estimates 10000; " + "degenerate resamples 0" + ), + 2, + ) + result_keys = json.dumps( + {"primary": primary, "reliability": reliability}, sort_keys=True + ) + self.assertNotIn("p_value", result_keys) + self.assertNotIn("pvalue", result_keys) + + def test_degenerate_bootstrap_is_disclosed_without_redraws(self): + fixed = { + "Conservative": {"fake-c"}, + "Liberal": {"fake-l"}, + } + contributions = [ + {"fake-c": (1, 1)}, + {"fake-l": (1, 1)}, + ] + + def statistic(sample): + return mhs_study._reliability_median_difference(sample, fixed) + + first = mhs_metrics.bootstrap_statistic( + contributions, statistic + ) + second = mhs_metrics.bootstrap_statistic( + contributions, statistic + ) + self.assertEqual(first, second) + self.assertEqual(first["iterations"], 10000) + self.assertGreater(first["valid_estimates"], 0) + self.assertGreater(first["degenerate_resamples"], 0) + self.assertEqual( + first["valid_estimates"] + first["degenerate_resamples"], 10000 + ) + self.assertIsNone(first["lower"]) + self.assertIsNone(first["upper"]) + self.assertEqual(first["status"], "degenerate/non-applicable") + + always = mhs_metrics.bootstrap_statistic( + [{"fake-c": (1, 1)}], statistic + ) + self.assertEqual(always["iterations"], 10000) + self.assertEqual(always["valid_estimates"], 0) + self.assertEqual(always["degenerate_resamples"], 10000) + self.assertIsNone(always["lower"]) + self.assertIsNone(always["upper"]) + self.assertEqual(always["status"], "degenerate/non-applicable") def test_frozen_descriptive_statistics(self): interval = mhs_metrics.wilson_interval(5, 10) @@ -1502,6 +1697,9 @@ def test_frozen_descriptive_statistics(self): self.assertEqual(first, second) self.assertEqual(first["iterations"], 10000) self.assertEqual(first["seed"], 20260718) + self.assertEqual(first["valid_estimates"], 10000) + self.assertEqual(first["degenerate_resamples"], 0) + self.assertEqual(first["status"], "ok") self.assertNotIn("p_value", first) self.assertNotIn("pvalue", first)