diff --git a/.gitignore b/.gitignore index 6a2c7cf..3ffda47 100644 --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,6 @@ reports/chaosnli/work/ reports/chaosnli/source/ reports/chaosnli/*.zip reports/chaosnli/chaosNLI_*.jsonl +reports/mhs/source/ +reports/mhs/work/ +reports/mhs/*.parquet diff --git a/adapters/mhs.py b/adapters/mhs.py new file mode 100644 index 0000000..4073a90 --- /dev/null +++ b/adapters/mhs.py @@ -0,0 +1,266 @@ +#!/usr/bin/env python3 +"""Convert already-parsed Measuring Hate Speech rows under the frozen protocol. + +This module is deliberately standard-library only. It does not read parquet, +perform network access, or compute study outcomes. ``reports/mhs/run_study.py`` +owns the separately authorized parquet-loading and orchestration boundary. +""" + +import json +from collections import Counter, OrderedDict + + +QUESTION = "hatespeech" +LABELS = ("0", "1", "2") +ELIGIBILITY_FLOOR = 20 +PRIMARY_MIN_PER_COHORT = 2 +PRIMARY_MIN_ITEMS = 50 +CONSERVATIVE_IDEOLOGIES = frozenset( + ("extremely_conservative", "conservative", "slightly_conservative") +) +LIBERAL_IDEOLOGIES = frozenset( + ("extremely_liberal", "liberal", "slightly_liberal") +) +COHORTS = ("Conservative", "Liberal") +ABOUT = ( + "Measuring Hate Speech conversion under the frozen Part-3b protocol: " + "annotators need at least 20 non-null hatespeech judgments; Conservative " + "and Liberal cohorts use only the six exact author-supplied ideology " + "values; primary items need at least two members of each cohort; raw " + "ordinal values remain categorical strings 0/1/2. Real-data evidence is " + "Tier 2 and source rows are never committed; see " + "reports/part-3b-dataset-selection-audit.md." +) + + +class MHSInputError(ValueError): + """The parsed rows cannot be converted without violating the protocol.""" + + +def cohort_for_ideology(value): + """Return a cohort only for one of the six exact pre-registered strings.""" + if value in CONSERVATIVE_IDEOLOGIES: + return "Conservative" + if value in LIBERAL_IDEOLOGIES: + return "Liberal" + return None + + +def _identifier(value, field, index): + if isinstance(value, bool) or not isinstance(value, (str, int)): + raise MHSInputError( + "record {} field {} must be a non-empty string or integer".format( + index, field + ) + ) + normalized = str(value) + if not normalized: + raise MHSInputError( + "record {} field {} must not be empty".format(index, field) + ) + return normalized + + +def _label(value, index): + if value is None: + return None + if isinstance(value, bool): + raise MHSInputError( + "record {} hatespeech must be null or one of 0, 1, 2".format(index) + ) + normalized = str(value) + if normalized not in LABELS: + raise MHSInputError( + "record {} hatespeech must be null or one of 0, 1, 2".format(index) + ) + return normalized + + +def _normalize(records): + if not isinstance(records, list) or not records: + raise MHSInputError("records must be a non-empty list") + normalized = [] + ideologies = {} + seen_pairs = set() + for index, record in enumerate(records, 1): + if not isinstance(record, dict): + raise MHSInputError("record {} must be an object".format(index)) + missing = [ + key + for key in ( + "annotator_id", + "comment_id", + "hatespeech", + "annotator_ideology", + ) + if key not in record + ] + if missing: + raise MHSInputError( + "record {} is missing {}".format(index, ", ".join(missing)) + ) + annotator = _identifier(record["annotator_id"], "annotator_id", index) + comment = _identifier(record["comment_id"], "comment_id", index) + label = _label(record["hatespeech"], index) + ideology = record["annotator_ideology"] + if ideology is not None and not isinstance(ideology, str): + raise MHSInputError( + "record {} annotator_ideology must be a string or null".format(index) + ) + if annotator in ideologies and ideologies[annotator] != ideology: + raise MHSInputError( + "annotator {!r} has inconsistent author-supplied ideology values".format( + annotator + ) + ) + ideologies[annotator] = ideology + pair = (annotator, comment) + if pair in seen_pairs: + raise MHSInputError( + "duplicate judgment for annotator {!r}, comment {!r}".format( + annotator, comment + ) + ) + seen_pairs.add(pair) + normalized.append( + { + "annotator_id": annotator, + "comment_id": comment, + "hatespeech": label, + "annotator_ideology": ideology, + } + ) + return normalized, ideologies + + +def _dataset(items, cohort_members, purpose): + ordered_items = [] + for comment, votes in items.items(): + if votes: + ordered_items.append( + { + "id": comment, + "labels": { + QUESTION: { + annotator: votes[annotator] + for annotator in sorted(votes) + } + }, + } + ) + annotators = sorted( + set(cohort_members["Conservative"]) | set(cohort_members["Liberal"]) + ) + return { + "_about": ABOUT + " This object is the {} corpus.".format(purpose), + "questions": { + QUESTION: { + "type": "categorical", + "labels": list(LABELS), + "note": "Raw MHS ordinal hatespeech values; not binarized or replaced by an IRT score.", + } + }, + "annotators": annotators, + "cohorts": { + "Conservative": sorted(cohort_members["Conservative"]), + "Liberal": sorted(cohort_members["Liberal"]), + }, + "items": ordered_items, + } + + +def convert_records(records): + """Return primary/reliability datasets and a frozen-filter halt signal. + + The function performs structural conversion only. It never examines a + label to decide whether an item enters the primary set; selection depends + exclusively on repeated-identity, ideology, and per-cohort coverage. + """ + normalized, ideologies = _normalize(records) + judgment_counts = Counter( + row["annotator_id"] + for row in normalized + if row["hatespeech"] is not None + ) + eligible = { + annotator + for annotator, count in judgment_counts.items() + if count >= ELIGIBILITY_FLOOR + } + cohort_members = {name: [] for name in COHORTS} + cohort_by_annotator = {} + for annotator in sorted(eligible): + cohort = cohort_for_ideology(ideologies.get(annotator)) + if cohort is not None: + cohort_members[cohort].append(annotator) + cohort_by_annotator[annotator] = cohort + + if not all(cohort_members.values()): + raise MHSInputError( + "conversion needs at least one eligible Conservative and Liberal annotator" + ) + + items = OrderedDict() + for row in normalized: + annotator = row["annotator_id"] + label = row["hatespeech"] + if label is None or annotator not in cohort_by_annotator: + continue + items.setdefault(row["comment_id"], {})[annotator] = label + + primary_items = OrderedDict() + for comment, votes in items.items(): + coverage = Counter(cohort_by_annotator[annotator] for annotator in votes) + if all(coverage[name] >= PRIMARY_MIN_PER_COHORT for name in COHORTS): + primary_items[comment] = votes + + primary = _dataset(primary_items, cohort_members, "primary") + reliability = _dataset(items, cohort_members, "full reliability") + primary_count = len(primary["items"]) + status = "ready" if primary_count >= PRIMARY_MIN_ITEMS else "halt" + halt_reason = None + if status == "halt": + halt_reason = ( + "primary item count {} is below the frozen minimum {}; halt and " + "return to the owner without weakening the filter" + ).format(primary_count, PRIMARY_MIN_ITEMS) + return { + "status": status, + "halt_reason": halt_reason, + "primary": primary, + "reliability": reliability, + "counts": { + "source_records": len(normalized), + "non_null_judgments": sum(judgment_counts.values()), + "eligible_annotators": len(eligible), + "conservative_annotators": len(cohort_members["Conservative"]), + "liberal_annotators": len(cohort_members["Liberal"]), + "excluded_eligible_annotators": len(eligible) + - len(cohort_by_annotator), + "primary_items": primary_count, + "reliability_items": len(reliability["items"]), + }, + } + + +def load_jsonl(path): + """Load already-parsed JSONL records; used only for synthetic fixtures.""" + records = [] + with open(path, encoding="utf-8") as handle: + for line_number, line in enumerate(handle, 1): + if not line.strip(): + continue + try: + record = json.loads(line) + except json.JSONDecodeError as exc: + raise MHSInputError( + "line {} is invalid JSON: {}".format(line_number, exc.msg) + ) + records.append(record) + return records + + +def write_dataset(dataset, path): + with open(path, "w", encoding="utf-8") as handle: + json.dump(dataset, handle, ensure_ascii=False, indent=2) + handle.write("\n") diff --git a/reports/mhs/README.md b/reports/mhs/README.md new file mode 100644 index 0000000..9b0411b --- /dev/null +++ b/reports/mhs/README.md @@ -0,0 +1,61 @@ +# Measuring Hate Speech: Phase 1 build only + +This directory contains the audited-to-be **Phase 1** adapter logic and dormant +Phase-2 harness for the bounded Measuring Hate Speech (MHS) pilot. No real MHS +row, identifier, or study outcome is committed or computed by CI. + +The binding protocol is +[`reports/part-3b-dataset-selection-audit.md`](../part-3b-dataset-selection-audit.md). +Its frozen parameters are implemented verbatim: + +- annotators need at least **20** non-null `hatespeech` judgments; +- Conservative is exactly `extremely_conservative`, `conservative`, or + `slightly_conservative`; +- Liberal is exactly `extremely_liberal`, `liberal`, or `slightly_liberal`; +- neutral, no-opinion, null, and every other ideology value are excluded with + no inference or imputation; +- primary items need at least **2 Conservative + 2 Liberal** judgments; +- fewer than **50** primary items halts the study and returns to the owner; +- labels remain categorical strings `"0"`, `"1"`, and `"2"`; +- intervals use an item-level bootstrap with **10,000** resamples and seed + **`20260718`**; there are no null-hypothesis p-values; +- reliability requires at least 20 actually scored CONFIDENT cells per + annotator and at least **30 qualifying annotators per cohort**, or it is + labelled underpowered/non-applicable; +- disjoint cohort support is reported as undefined, never replaced by a maximum + geometry gap. + +## Evidence tiers and execution boundary + +Tier 1 is the synthetic fixture and standard-library logic exercised in CI. +Tier 2 begins only after Claude audits this Phase-1 PR and the owner separately +authorizes the real run. `run_study.py` is therefore present but never executed +in CI. + +The adapter accepts already-parsed records and has no parquet or network +dependency. Only the dormant loader needs `pyarrow`; its import is guarded and +occurs only when Phase 2 is explicitly run. `disagreement.py` and +`soft_labels.py` are called unchanged with `--data/--out`. `geometry.py` is +intentionally demo-pinned and has no BYOD CLI, so the harness calls its existing +`arithmetic_mean`, `geometric_mean`, and `tv` functions directly without +modifying it. + +## Phase-2 reproduction (not yet authorized) + +After separate authorization, obtain the official MHS parquet at revision +`5468f6e` outside the repository and verify: + +```text +SHA-256 6819525ce61bc24344df9fc3f7bf48270b31038273cc27c67fc225b51433b0e1 +``` + +Install `pyarrow` in a separate local environment and run: + +```text +python3 reports/mhs/run_study.py /absolute/path/to/measuring-hate-speech.parquet +``` + +The harness has no network path. A checksum mismatch or fewer than 50 primary +items halts before outcomes. Its local source/work directories are ignored. +Only a separately reviewed aggregate `report.md` and `manifest.json` may be +committed in Phase 2; no source rows or identifiers may be published. diff --git a/reports/mhs/fixtures/mhs_sample.jsonl b/reports/mhs/fixtures/mhs_sample.jsonl new file mode 100644 index 0000000..e8d9e0a --- /dev/null +++ b/reports/mhs/fixtures/mhs_sample.jsonl @@ -0,0 +1,245 @@ +{"annotator_id":"c-extreme","comment_id":"synthetic-001","hatespeech":0,"annotator_ideology":"extremely_conservative"} +{"annotator_id":"c-slight","comment_id":"synthetic-001","hatespeech":0,"annotator_ideology":"slightly_conservative"} +{"annotator_id":"l-extreme","comment_id":"synthetic-001","hatespeech":2,"annotator_ideology":"extremely_liberal"} +{"annotator_id":"l-slight","comment_id":"synthetic-001","hatespeech":2,"annotator_ideology":"slightly_liberal"} +{"annotator_id":"c-extreme","comment_id":"synthetic-002","hatespeech":1,"annotator_ideology":"extremely_conservative"} +{"annotator_id":"c-slight","comment_id":"synthetic-002","hatespeech":1,"annotator_ideology":"slightly_conservative"} +{"annotator_id":"l-extreme","comment_id":"synthetic-002","hatespeech":1,"annotator_ideology":"extremely_liberal"} +{"annotator_id":"l-slight","comment_id":"synthetic-002","hatespeech":1,"annotator_ideology":"slightly_liberal"} +{"annotator_id":"c-extreme","comment_id":"synthetic-003","hatespeech":0,"annotator_ideology":"extremely_conservative"} +{"annotator_id":"c-slight","comment_id":"synthetic-003","hatespeech":1,"annotator_ideology":"slightly_conservative"} +{"annotator_id":"l-extreme","comment_id":"synthetic-003","hatespeech":2,"annotator_ideology":"extremely_liberal"} +{"annotator_id":"l-slight","comment_id":"synthetic-003","hatespeech":2,"annotator_ideology":"slightly_liberal"} +{"annotator_id":"c-extreme","comment_id":"synthetic-004","hatespeech":1,"annotator_ideology":"extremely_conservative"} +{"annotator_id":"c-slight","comment_id":"synthetic-004","hatespeech":1,"annotator_ideology":"slightly_conservative"} +{"annotator_id":"l-extreme","comment_id":"synthetic-004","hatespeech":1,"annotator_ideology":"extremely_liberal"} +{"annotator_id":"l-slight","comment_id":"synthetic-004","hatespeech":1,"annotator_ideology":"slightly_liberal"} +{"annotator_id":"c-extreme","comment_id":"synthetic-005","hatespeech":1,"annotator_ideology":"extremely_conservative"} +{"annotator_id":"c-slight","comment_id":"synthetic-005","hatespeech":1,"annotator_ideology":"slightly_conservative"} +{"annotator_id":"l-extreme","comment_id":"synthetic-005","hatespeech":1,"annotator_ideology":"extremely_liberal"} +{"annotator_id":"l-slight","comment_id":"synthetic-005","hatespeech":1,"annotator_ideology":"slightly_liberal"} +{"annotator_id":"c-extreme","comment_id":"synthetic-006","hatespeech":1,"annotator_ideology":"extremely_conservative"} +{"annotator_id":"c-slight","comment_id":"synthetic-006","hatespeech":1,"annotator_ideology":"slightly_conservative"} +{"annotator_id":"l-extreme","comment_id":"synthetic-006","hatespeech":1,"annotator_ideology":"extremely_liberal"} +{"annotator_id":"l-slight","comment_id":"synthetic-006","hatespeech":1,"annotator_ideology":"slightly_liberal"} +{"annotator_id":"c-extreme","comment_id":"synthetic-007","hatespeech":1,"annotator_ideology":"extremely_conservative"} +{"annotator_id":"c-slight","comment_id":"synthetic-007","hatespeech":1,"annotator_ideology":"slightly_conservative"} +{"annotator_id":"l-extreme","comment_id":"synthetic-007","hatespeech":1,"annotator_ideology":"extremely_liberal"} +{"annotator_id":"l-slight","comment_id":"synthetic-007","hatespeech":1,"annotator_ideology":"slightly_liberal"} +{"annotator_id":"c-extreme","comment_id":"synthetic-008","hatespeech":1,"annotator_ideology":"extremely_conservative"} +{"annotator_id":"c-slight","comment_id":"synthetic-008","hatespeech":1,"annotator_ideology":"slightly_conservative"} +{"annotator_id":"l-extreme","comment_id":"synthetic-008","hatespeech":1,"annotator_ideology":"extremely_liberal"} +{"annotator_id":"l-slight","comment_id":"synthetic-008","hatespeech":1,"annotator_ideology":"slightly_liberal"} +{"annotator_id":"c-extreme","comment_id":"synthetic-009","hatespeech":1,"annotator_ideology":"extremely_conservative"} +{"annotator_id":"c-slight","comment_id":"synthetic-009","hatespeech":1,"annotator_ideology":"slightly_conservative"} +{"annotator_id":"l-extreme","comment_id":"synthetic-009","hatespeech":1,"annotator_ideology":"extremely_liberal"} +{"annotator_id":"l-slight","comment_id":"synthetic-009","hatespeech":1,"annotator_ideology":"slightly_liberal"} +{"annotator_id":"c-extreme","comment_id":"synthetic-010","hatespeech":1,"annotator_ideology":"extremely_conservative"} +{"annotator_id":"c-slight","comment_id":"synthetic-010","hatespeech":1,"annotator_ideology":"slightly_conservative"} +{"annotator_id":"l-extreme","comment_id":"synthetic-010","hatespeech":1,"annotator_ideology":"extremely_liberal"} +{"annotator_id":"l-slight","comment_id":"synthetic-010","hatespeech":1,"annotator_ideology":"slightly_liberal"} +{"annotator_id":"c-extreme","comment_id":"synthetic-011","hatespeech":1,"annotator_ideology":"extremely_conservative"} +{"annotator_id":"c-slight","comment_id":"synthetic-011","hatespeech":1,"annotator_ideology":"slightly_conservative"} +{"annotator_id":"l-extreme","comment_id":"synthetic-011","hatespeech":1,"annotator_ideology":"extremely_liberal"} +{"annotator_id":"l-slight","comment_id":"synthetic-011","hatespeech":1,"annotator_ideology":"slightly_liberal"} +{"annotator_id":"c-extreme","comment_id":"synthetic-012","hatespeech":1,"annotator_ideology":"extremely_conservative"} +{"annotator_id":"c-slight","comment_id":"synthetic-012","hatespeech":1,"annotator_ideology":"slightly_conservative"} +{"annotator_id":"l-extreme","comment_id":"synthetic-012","hatespeech":1,"annotator_ideology":"extremely_liberal"} +{"annotator_id":"l-slight","comment_id":"synthetic-012","hatespeech":1,"annotator_ideology":"slightly_liberal"} +{"annotator_id":"c-extreme","comment_id":"synthetic-013","hatespeech":1,"annotator_ideology":"extremely_conservative"} +{"annotator_id":"c-slight","comment_id":"synthetic-013","hatespeech":1,"annotator_ideology":"slightly_conservative"} +{"annotator_id":"l-extreme","comment_id":"synthetic-013","hatespeech":1,"annotator_ideology":"extremely_liberal"} +{"annotator_id":"l-slight","comment_id":"synthetic-013","hatespeech":1,"annotator_ideology":"slightly_liberal"} +{"annotator_id":"c-extreme","comment_id":"synthetic-014","hatespeech":1,"annotator_ideology":"extremely_conservative"} +{"annotator_id":"c-slight","comment_id":"synthetic-014","hatespeech":1,"annotator_ideology":"slightly_conservative"} +{"annotator_id":"l-extreme","comment_id":"synthetic-014","hatespeech":1,"annotator_ideology":"extremely_liberal"} +{"annotator_id":"l-slight","comment_id":"synthetic-014","hatespeech":1,"annotator_ideology":"slightly_liberal"} +{"annotator_id":"c-extreme","comment_id":"synthetic-015","hatespeech":1,"annotator_ideology":"extremely_conservative"} +{"annotator_id":"c-slight","comment_id":"synthetic-015","hatespeech":1,"annotator_ideology":"slightly_conservative"} +{"annotator_id":"l-extreme","comment_id":"synthetic-015","hatespeech":1,"annotator_ideology":"extremely_liberal"} +{"annotator_id":"l-slight","comment_id":"synthetic-015","hatespeech":1,"annotator_ideology":"slightly_liberal"} +{"annotator_id":"c-extreme","comment_id":"synthetic-016","hatespeech":1,"annotator_ideology":"extremely_conservative"} +{"annotator_id":"c-slight","comment_id":"synthetic-016","hatespeech":1,"annotator_ideology":"slightly_conservative"} +{"annotator_id":"l-extreme","comment_id":"synthetic-016","hatespeech":1,"annotator_ideology":"extremely_liberal"} +{"annotator_id":"l-slight","comment_id":"synthetic-016","hatespeech":1,"annotator_ideology":"slightly_liberal"} +{"annotator_id":"c-extreme","comment_id":"synthetic-017","hatespeech":1,"annotator_ideology":"extremely_conservative"} +{"annotator_id":"c-slight","comment_id":"synthetic-017","hatespeech":1,"annotator_ideology":"slightly_conservative"} +{"annotator_id":"l-extreme","comment_id":"synthetic-017","hatespeech":1,"annotator_ideology":"extremely_liberal"} +{"annotator_id":"l-slight","comment_id":"synthetic-017","hatespeech":1,"annotator_ideology":"slightly_liberal"} +{"annotator_id":"c-extreme","comment_id":"synthetic-018","hatespeech":1,"annotator_ideology":"extremely_conservative"} +{"annotator_id":"c-slight","comment_id":"synthetic-018","hatespeech":1,"annotator_ideology":"slightly_conservative"} +{"annotator_id":"l-extreme","comment_id":"synthetic-018","hatespeech":1,"annotator_ideology":"extremely_liberal"} +{"annotator_id":"l-slight","comment_id":"synthetic-018","hatespeech":1,"annotator_ideology":"slightly_liberal"} +{"annotator_id":"c-extreme","comment_id":"synthetic-019","hatespeech":1,"annotator_ideology":"extremely_conservative"} +{"annotator_id":"c-slight","comment_id":"synthetic-019","hatespeech":1,"annotator_ideology":"slightly_conservative"} +{"annotator_id":"l-extreme","comment_id":"synthetic-019","hatespeech":1,"annotator_ideology":"extremely_liberal"} +{"annotator_id":"l-slight","comment_id":"synthetic-019","hatespeech":1,"annotator_ideology":"slightly_liberal"} +{"annotator_id":"c-extreme","comment_id":"synthetic-020","hatespeech":1,"annotator_ideology":"extremely_conservative"} +{"annotator_id":"c-slight","comment_id":"synthetic-020","hatespeech":1,"annotator_ideology":"slightly_conservative"} +{"annotator_id":"l-extreme","comment_id":"synthetic-020","hatespeech":1,"annotator_ideology":"extremely_liberal"} +{"annotator_id":"l-slight","comment_id":"synthetic-020","hatespeech":1,"annotator_ideology":"slightly_liberal"} +{"annotator_id":"c-extreme","comment_id":"synthetic-021","hatespeech":1,"annotator_ideology":"extremely_conservative"} +{"annotator_id":"c-slight","comment_id":"synthetic-021","hatespeech":1,"annotator_ideology":"slightly_conservative"} +{"annotator_id":"l-extreme","comment_id":"synthetic-021","hatespeech":1,"annotator_ideology":"extremely_liberal"} +{"annotator_id":"l-slight","comment_id":"synthetic-021","hatespeech":1,"annotator_ideology":"slightly_liberal"} +{"annotator_id":"c-extreme","comment_id":"synthetic-022","hatespeech":1,"annotator_ideology":"extremely_conservative"} +{"annotator_id":"c-slight","comment_id":"synthetic-022","hatespeech":1,"annotator_ideology":"slightly_conservative"} +{"annotator_id":"l-extreme","comment_id":"synthetic-022","hatespeech":1,"annotator_ideology":"extremely_liberal"} +{"annotator_id":"l-slight","comment_id":"synthetic-022","hatespeech":1,"annotator_ideology":"slightly_liberal"} +{"annotator_id":"c-extreme","comment_id":"synthetic-023","hatespeech":1,"annotator_ideology":"extremely_conservative"} +{"annotator_id":"c-slight","comment_id":"synthetic-023","hatespeech":1,"annotator_ideology":"slightly_conservative"} +{"annotator_id":"l-extreme","comment_id":"synthetic-023","hatespeech":1,"annotator_ideology":"extremely_liberal"} +{"annotator_id":"l-slight","comment_id":"synthetic-023","hatespeech":1,"annotator_ideology":"slightly_liberal"} +{"annotator_id":"c-extreme","comment_id":"synthetic-024","hatespeech":1,"annotator_ideology":"extremely_conservative"} +{"annotator_id":"c-slight","comment_id":"synthetic-024","hatespeech":1,"annotator_ideology":"slightly_conservative"} +{"annotator_id":"l-extreme","comment_id":"synthetic-024","hatespeech":1,"annotator_ideology":"extremely_liberal"} +{"annotator_id":"l-slight","comment_id":"synthetic-024","hatespeech":1,"annotator_ideology":"slightly_liberal"} +{"annotator_id":"c-extreme","comment_id":"synthetic-025","hatespeech":1,"annotator_ideology":"extremely_conservative"} +{"annotator_id":"c-slight","comment_id":"synthetic-025","hatespeech":1,"annotator_ideology":"slightly_conservative"} +{"annotator_id":"l-extreme","comment_id":"synthetic-025","hatespeech":1,"annotator_ideology":"extremely_liberal"} +{"annotator_id":"l-slight","comment_id":"synthetic-025","hatespeech":1,"annotator_ideology":"slightly_liberal"} +{"annotator_id":"c-extreme","comment_id":"synthetic-026","hatespeech":1,"annotator_ideology":"extremely_conservative"} +{"annotator_id":"c-slight","comment_id":"synthetic-026","hatespeech":1,"annotator_ideology":"slightly_conservative"} +{"annotator_id":"l-extreme","comment_id":"synthetic-026","hatespeech":1,"annotator_ideology":"extremely_liberal"} +{"annotator_id":"l-slight","comment_id":"synthetic-026","hatespeech":1,"annotator_ideology":"slightly_liberal"} +{"annotator_id":"c-extreme","comment_id":"synthetic-027","hatespeech":1,"annotator_ideology":"extremely_conservative"} +{"annotator_id":"c-slight","comment_id":"synthetic-027","hatespeech":1,"annotator_ideology":"slightly_conservative"} +{"annotator_id":"l-extreme","comment_id":"synthetic-027","hatespeech":1,"annotator_ideology":"extremely_liberal"} +{"annotator_id":"l-slight","comment_id":"synthetic-027","hatespeech":1,"annotator_ideology":"slightly_liberal"} +{"annotator_id":"c-extreme","comment_id":"synthetic-028","hatespeech":1,"annotator_ideology":"extremely_conservative"} +{"annotator_id":"c-slight","comment_id":"synthetic-028","hatespeech":1,"annotator_ideology":"slightly_conservative"} +{"annotator_id":"l-extreme","comment_id":"synthetic-028","hatespeech":1,"annotator_ideology":"extremely_liberal"} +{"annotator_id":"l-slight","comment_id":"synthetic-028","hatespeech":1,"annotator_ideology":"slightly_liberal"} +{"annotator_id":"c-extreme","comment_id":"synthetic-029","hatespeech":1,"annotator_ideology":"extremely_conservative"} +{"annotator_id":"c-slight","comment_id":"synthetic-029","hatespeech":1,"annotator_ideology":"slightly_conservative"} +{"annotator_id":"l-extreme","comment_id":"synthetic-029","hatespeech":1,"annotator_ideology":"extremely_liberal"} +{"annotator_id":"l-slight","comment_id":"synthetic-029","hatespeech":1,"annotator_ideology":"slightly_liberal"} +{"annotator_id":"c-extreme","comment_id":"synthetic-030","hatespeech":1,"annotator_ideology":"extremely_conservative"} +{"annotator_id":"c-slight","comment_id":"synthetic-030","hatespeech":1,"annotator_ideology":"slightly_conservative"} +{"annotator_id":"l-extreme","comment_id":"synthetic-030","hatespeech":1,"annotator_ideology":"extremely_liberal"} +{"annotator_id":"l-slight","comment_id":"synthetic-030","hatespeech":1,"annotator_ideology":"slightly_liberal"} +{"annotator_id":"c-extreme","comment_id":"synthetic-031","hatespeech":1,"annotator_ideology":"extremely_conservative"} +{"annotator_id":"c-slight","comment_id":"synthetic-031","hatespeech":1,"annotator_ideology":"slightly_conservative"} +{"annotator_id":"l-extreme","comment_id":"synthetic-031","hatespeech":1,"annotator_ideology":"extremely_liberal"} +{"annotator_id":"l-slight","comment_id":"synthetic-031","hatespeech":1,"annotator_ideology":"slightly_liberal"} +{"annotator_id":"c-extreme","comment_id":"synthetic-032","hatespeech":1,"annotator_ideology":"extremely_conservative"} +{"annotator_id":"c-slight","comment_id":"synthetic-032","hatespeech":1,"annotator_ideology":"slightly_conservative"} +{"annotator_id":"l-extreme","comment_id":"synthetic-032","hatespeech":1,"annotator_ideology":"extremely_liberal"} +{"annotator_id":"l-slight","comment_id":"synthetic-032","hatespeech":1,"annotator_ideology":"slightly_liberal"} +{"annotator_id":"c-extreme","comment_id":"synthetic-033","hatespeech":1,"annotator_ideology":"extremely_conservative"} +{"annotator_id":"c-slight","comment_id":"synthetic-033","hatespeech":1,"annotator_ideology":"slightly_conservative"} +{"annotator_id":"l-extreme","comment_id":"synthetic-033","hatespeech":1,"annotator_ideology":"extremely_liberal"} +{"annotator_id":"l-slight","comment_id":"synthetic-033","hatespeech":1,"annotator_ideology":"slightly_liberal"} +{"annotator_id":"c-extreme","comment_id":"synthetic-034","hatespeech":1,"annotator_ideology":"extremely_conservative"} +{"annotator_id":"c-slight","comment_id":"synthetic-034","hatespeech":1,"annotator_ideology":"slightly_conservative"} +{"annotator_id":"l-extreme","comment_id":"synthetic-034","hatespeech":1,"annotator_ideology":"extremely_liberal"} +{"annotator_id":"l-slight","comment_id":"synthetic-034","hatespeech":1,"annotator_ideology":"slightly_liberal"} +{"annotator_id":"c-extreme","comment_id":"synthetic-035","hatespeech":1,"annotator_ideology":"extremely_conservative"} +{"annotator_id":"c-slight","comment_id":"synthetic-035","hatespeech":1,"annotator_ideology":"slightly_conservative"} +{"annotator_id":"l-extreme","comment_id":"synthetic-035","hatespeech":1,"annotator_ideology":"extremely_liberal"} +{"annotator_id":"l-slight","comment_id":"synthetic-035","hatespeech":1,"annotator_ideology":"slightly_liberal"} +{"annotator_id":"c-extreme","comment_id":"synthetic-036","hatespeech":1,"annotator_ideology":"extremely_conservative"} +{"annotator_id":"c-slight","comment_id":"synthetic-036","hatespeech":1,"annotator_ideology":"slightly_conservative"} +{"annotator_id":"l-extreme","comment_id":"synthetic-036","hatespeech":1,"annotator_ideology":"extremely_liberal"} +{"annotator_id":"l-slight","comment_id":"synthetic-036","hatespeech":1,"annotator_ideology":"slightly_liberal"} +{"annotator_id":"c-extreme","comment_id":"synthetic-037","hatespeech":1,"annotator_ideology":"extremely_conservative"} +{"annotator_id":"c-slight","comment_id":"synthetic-037","hatespeech":1,"annotator_ideology":"slightly_conservative"} +{"annotator_id":"l-extreme","comment_id":"synthetic-037","hatespeech":1,"annotator_ideology":"extremely_liberal"} +{"annotator_id":"l-slight","comment_id":"synthetic-037","hatespeech":1,"annotator_ideology":"slightly_liberal"} +{"annotator_id":"c-extreme","comment_id":"synthetic-038","hatespeech":1,"annotator_ideology":"extremely_conservative"} +{"annotator_id":"c-slight","comment_id":"synthetic-038","hatespeech":1,"annotator_ideology":"slightly_conservative"} +{"annotator_id":"l-extreme","comment_id":"synthetic-038","hatespeech":1,"annotator_ideology":"extremely_liberal"} +{"annotator_id":"l-slight","comment_id":"synthetic-038","hatespeech":1,"annotator_ideology":"slightly_liberal"} +{"annotator_id":"c-extreme","comment_id":"synthetic-039","hatespeech":1,"annotator_ideology":"extremely_conservative"} +{"annotator_id":"c-slight","comment_id":"synthetic-039","hatespeech":1,"annotator_ideology":"slightly_conservative"} +{"annotator_id":"l-extreme","comment_id":"synthetic-039","hatespeech":1,"annotator_ideology":"extremely_liberal"} +{"annotator_id":"l-slight","comment_id":"synthetic-039","hatespeech":1,"annotator_ideology":"slightly_liberal"} +{"annotator_id":"c-extreme","comment_id":"synthetic-040","hatespeech":1,"annotator_ideology":"extremely_conservative"} +{"annotator_id":"c-slight","comment_id":"synthetic-040","hatespeech":1,"annotator_ideology":"slightly_conservative"} +{"annotator_id":"l-extreme","comment_id":"synthetic-040","hatespeech":1,"annotator_ideology":"extremely_liberal"} +{"annotator_id":"l-slight","comment_id":"synthetic-040","hatespeech":1,"annotator_ideology":"slightly_liberal"} +{"annotator_id":"c-extreme","comment_id":"synthetic-041","hatespeech":1,"annotator_ideology":"extremely_conservative"} +{"annotator_id":"c-slight","comment_id":"synthetic-041","hatespeech":1,"annotator_ideology":"slightly_conservative"} +{"annotator_id":"l-extreme","comment_id":"synthetic-041","hatespeech":1,"annotator_ideology":"extremely_liberal"} +{"annotator_id":"l-slight","comment_id":"synthetic-041","hatespeech":1,"annotator_ideology":"slightly_liberal"} +{"annotator_id":"c-extreme","comment_id":"synthetic-042","hatespeech":1,"annotator_ideology":"extremely_conservative"} +{"annotator_id":"c-slight","comment_id":"synthetic-042","hatespeech":1,"annotator_ideology":"slightly_conservative"} +{"annotator_id":"l-extreme","comment_id":"synthetic-042","hatespeech":1,"annotator_ideology":"extremely_liberal"} +{"annotator_id":"l-slight","comment_id":"synthetic-042","hatespeech":1,"annotator_ideology":"slightly_liberal"} +{"annotator_id":"c-extreme","comment_id":"synthetic-043","hatespeech":1,"annotator_ideology":"extremely_conservative"} +{"annotator_id":"c-slight","comment_id":"synthetic-043","hatespeech":1,"annotator_ideology":"slightly_conservative"} +{"annotator_id":"l-extreme","comment_id":"synthetic-043","hatespeech":1,"annotator_ideology":"extremely_liberal"} +{"annotator_id":"l-slight","comment_id":"synthetic-043","hatespeech":1,"annotator_ideology":"slightly_liberal"} +{"annotator_id":"c-extreme","comment_id":"synthetic-044","hatespeech":1,"annotator_ideology":"extremely_conservative"} +{"annotator_id":"c-slight","comment_id":"synthetic-044","hatespeech":1,"annotator_ideology":"slightly_conservative"} +{"annotator_id":"l-extreme","comment_id":"synthetic-044","hatespeech":1,"annotator_ideology":"extremely_liberal"} +{"annotator_id":"l-slight","comment_id":"synthetic-044","hatespeech":1,"annotator_ideology":"slightly_liberal"} +{"annotator_id":"c-extreme","comment_id":"synthetic-045","hatespeech":1,"annotator_ideology":"extremely_conservative"} +{"annotator_id":"c-slight","comment_id":"synthetic-045","hatespeech":1,"annotator_ideology":"slightly_conservative"} +{"annotator_id":"l-extreme","comment_id":"synthetic-045","hatespeech":1,"annotator_ideology":"extremely_liberal"} +{"annotator_id":"l-slight","comment_id":"synthetic-045","hatespeech":1,"annotator_ideology":"slightly_liberal"} +{"annotator_id":"c-extreme","comment_id":"synthetic-046","hatespeech":1,"annotator_ideology":"extremely_conservative"} +{"annotator_id":"c-slight","comment_id":"synthetic-046","hatespeech":1,"annotator_ideology":"slightly_conservative"} +{"annotator_id":"l-extreme","comment_id":"synthetic-046","hatespeech":1,"annotator_ideology":"extremely_liberal"} +{"annotator_id":"l-slight","comment_id":"synthetic-046","hatespeech":1,"annotator_ideology":"slightly_liberal"} +{"annotator_id":"c-extreme","comment_id":"synthetic-047","hatespeech":1,"annotator_ideology":"extremely_conservative"} +{"annotator_id":"c-slight","comment_id":"synthetic-047","hatespeech":1,"annotator_ideology":"slightly_conservative"} +{"annotator_id":"l-extreme","comment_id":"synthetic-047","hatespeech":1,"annotator_ideology":"extremely_liberal"} +{"annotator_id":"l-slight","comment_id":"synthetic-047","hatespeech":1,"annotator_ideology":"slightly_liberal"} +{"annotator_id":"c-extreme","comment_id":"synthetic-048","hatespeech":1,"annotator_ideology":"extremely_conservative"} +{"annotator_id":"c-slight","comment_id":"synthetic-048","hatespeech":1,"annotator_ideology":"slightly_conservative"} +{"annotator_id":"l-extreme","comment_id":"synthetic-048","hatespeech":1,"annotator_ideology":"extremely_liberal"} +{"annotator_id":"l-slight","comment_id":"synthetic-048","hatespeech":1,"annotator_ideology":"slightly_liberal"} +{"annotator_id":"c-extreme","comment_id":"synthetic-049","hatespeech":1,"annotator_ideology":"extremely_conservative"} +{"annotator_id":"c-slight","comment_id":"synthetic-049","hatespeech":1,"annotator_ideology":"slightly_conservative"} +{"annotator_id":"l-extreme","comment_id":"synthetic-049","hatespeech":1,"annotator_ideology":"extremely_liberal"} +{"annotator_id":"l-slight","comment_id":"synthetic-049","hatespeech":1,"annotator_ideology":"slightly_liberal"} +{"annotator_id":"c-extreme","comment_id":"synthetic-050","hatespeech":1,"annotator_ideology":"extremely_conservative"} +{"annotator_id":"c-slight","comment_id":"synthetic-050","hatespeech":1,"annotator_ideology":"slightly_conservative"} +{"annotator_id":"l-extreme","comment_id":"synthetic-050","hatespeech":1,"annotator_ideology":"extremely_liberal"} +{"annotator_id":"l-slight","comment_id":"synthetic-050","hatespeech":1,"annotator_ideology":"slightly_liberal"} +{"annotator_id":"c-extreme","comment_id":"synthetic-filter-fail","hatespeech":0,"annotator_ideology":"extremely_conservative"} +{"annotator_id":"c-slight","comment_id":"synthetic-filter-fail","hatespeech":0,"annotator_ideology":"slightly_conservative"} +{"annotator_id":"l-extreme","comment_id":"synthetic-filter-fail","hatespeech":2,"annotator_ideology":"extremely_liberal"} +{"annotator_id":"c-below-floor","comment_id":"synthetic-001","hatespeech":0,"annotator_ideology":"conservative"} +{"annotator_id":"c-below-floor","comment_id":"synthetic-002","hatespeech":0,"annotator_ideology":"conservative"} +{"annotator_id":"c-below-floor","comment_id":"synthetic-003","hatespeech":0,"annotator_ideology":"conservative"} +{"annotator_id":"c-below-floor","comment_id":"synthetic-004","hatespeech":0,"annotator_ideology":"conservative"} +{"annotator_id":"c-below-floor","comment_id":"synthetic-005","hatespeech":0,"annotator_ideology":"conservative"} +{"annotator_id":"c-below-floor","comment_id":"synthetic-006","hatespeech":0,"annotator_ideology":"conservative"} +{"annotator_id":"c-below-floor","comment_id":"synthetic-007","hatespeech":0,"annotator_ideology":"conservative"} +{"annotator_id":"c-below-floor","comment_id":"synthetic-008","hatespeech":0,"annotator_ideology":"conservative"} +{"annotator_id":"c-below-floor","comment_id":"synthetic-009","hatespeech":0,"annotator_ideology":"conservative"} +{"annotator_id":"c-below-floor","comment_id":"synthetic-010","hatespeech":0,"annotator_ideology":"conservative"} +{"annotator_id":"c-below-floor","comment_id":"synthetic-011","hatespeech":0,"annotator_ideology":"conservative"} +{"annotator_id":"c-below-floor","comment_id":"synthetic-012","hatespeech":0,"annotator_ideology":"conservative"} +{"annotator_id":"c-below-floor","comment_id":"synthetic-013","hatespeech":0,"annotator_ideology":"conservative"} +{"annotator_id":"c-below-floor","comment_id":"synthetic-014","hatespeech":0,"annotator_ideology":"conservative"} +{"annotator_id":"c-below-floor","comment_id":"synthetic-015","hatespeech":0,"annotator_ideology":"conservative"} +{"annotator_id":"c-below-floor","comment_id":"synthetic-016","hatespeech":0,"annotator_ideology":"conservative"} +{"annotator_id":"c-below-floor","comment_id":"synthetic-017","hatespeech":0,"annotator_ideology":"conservative"} +{"annotator_id":"c-below-floor","comment_id":"synthetic-018","hatespeech":0,"annotator_ideology":"conservative"} +{"annotator_id":"c-below-floor","comment_id":"synthetic-019","hatespeech":0,"annotator_ideology":"conservative"} +{"annotator_id":"excluded-neutral","comment_id":"synthetic-001","hatespeech":2,"annotator_ideology":"neutral"} +{"annotator_id":"excluded-neutral","comment_id":"synthetic-002","hatespeech":2,"annotator_ideology":"neutral"} +{"annotator_id":"excluded-neutral","comment_id":"synthetic-003","hatespeech":2,"annotator_ideology":"neutral"} +{"annotator_id":"excluded-neutral","comment_id":"synthetic-004","hatespeech":2,"annotator_ideology":"neutral"} +{"annotator_id":"excluded-neutral","comment_id":"synthetic-005","hatespeech":2,"annotator_ideology":"neutral"} +{"annotator_id":"excluded-neutral","comment_id":"synthetic-006","hatespeech":2,"annotator_ideology":"neutral"} +{"annotator_id":"excluded-neutral","comment_id":"synthetic-007","hatespeech":2,"annotator_ideology":"neutral"} +{"annotator_id":"excluded-neutral","comment_id":"synthetic-008","hatespeech":2,"annotator_ideology":"neutral"} +{"annotator_id":"excluded-neutral","comment_id":"synthetic-009","hatespeech":2,"annotator_ideology":"neutral"} +{"annotator_id":"excluded-neutral","comment_id":"synthetic-010","hatespeech":2,"annotator_ideology":"neutral"} +{"annotator_id":"excluded-neutral","comment_id":"synthetic-011","hatespeech":2,"annotator_ideology":"neutral"} +{"annotator_id":"excluded-neutral","comment_id":"synthetic-012","hatespeech":2,"annotator_ideology":"neutral"} +{"annotator_id":"excluded-neutral","comment_id":"synthetic-013","hatespeech":2,"annotator_ideology":"neutral"} +{"annotator_id":"excluded-neutral","comment_id":"synthetic-014","hatespeech":2,"annotator_ideology":"neutral"} +{"annotator_id":"excluded-neutral","comment_id":"synthetic-015","hatespeech":2,"annotator_ideology":"neutral"} +{"annotator_id":"excluded-neutral","comment_id":"synthetic-016","hatespeech":2,"annotator_ideology":"neutral"} +{"annotator_id":"excluded-neutral","comment_id":"synthetic-017","hatespeech":2,"annotator_ideology":"neutral"} +{"annotator_id":"excluded-neutral","comment_id":"synthetic-018","hatespeech":2,"annotator_ideology":"neutral"} +{"annotator_id":"excluded-neutral","comment_id":"synthetic-019","hatespeech":2,"annotator_ideology":"neutral"} +{"annotator_id":"excluded-neutral","comment_id":"synthetic-020","hatespeech":2,"annotator_ideology":"neutral"} +{"annotator_id":"excluded-no-opinion","comment_id":"synthetic-001","hatespeech":1,"annotator_ideology":"no_opinion"} +{"annotator_id":"excluded-null","comment_id":"synthetic-001","hatespeech":1,"annotator_ideology":null} +{"annotator_id":"null-judgment","comment_id":"synthetic-001","hatespeech":null,"annotator_ideology":"conservative"} diff --git a/reports/mhs/metrics.py b/reports/mhs/metrics.py new file mode 100644 index 0000000..e1026ec --- /dev/null +++ b/reports/mhs/metrics.py @@ -0,0 +1,110 @@ +"""Frozen, standard-library descriptive statistics for the MHS pilot. + +There are deliberately no null-hypothesis tests or p-values here. Bootstrap +resampling is item-level, uses 10,000 iterations, and is fixed at seed 20260718. +""" + +import math +import random +import statistics + + +CONFIDENCE_Z = 1.959963984540054 +BOOTSTRAP_ITERATIONS = 10000 +BOOTSTRAP_SEED = 20260718 + + +def _finite(values): + normalized = [float(value) for value in values] + if not normalized or any(not math.isfinite(value) for value in normalized): + raise ValueError("values must be a non-empty sequence of finite numbers") + return normalized + + +def _percentile(values, probability): + ordered = sorted(_finite(values)) + if not 0.0 <= probability <= 1.0: + raise ValueError("probability must lie in [0, 1]") + if len(ordered) == 1: + return ordered[0] + position = (len(ordered) - 1) * probability + lower = int(math.floor(position)) + upper = int(math.ceil(position)) + if lower == upper: + return ordered[lower] + weight = position - lower + return ordered[lower] * (1.0 - weight) + ordered[upper] * weight + + +def median_iqr(values): + """Return median, Q1, Q3, and IQR using linear interpolated quartiles.""" + normalized = _finite(values) + q1 = _percentile(normalized, 0.25) + q3 = _percentile(normalized, 0.75) + return { + "median": statistics.median(normalized), + "q1": q1, + "q3": q3, + "iqr": q3 - q1, + } + + +def wilson_interval(successes, total, z=CONFIDENCE_Z): + """Wilson score interval for a binomial proportion (95% by default).""" + if type(successes) is not int or type(total) is not int: + raise ValueError("successes and total must be integers") + if total <= 0 or successes < 0 or successes > total: + raise ValueError("require 0 <= successes <= total and total > 0") + proportion = successes / total + z2 = z * z + denominator = 1.0 + z2 / total + centre = (proportion + z2 / (2.0 * total)) / denominator + radius = ( + z + * math.sqrt( + proportion * (1.0 - proportion) / total + + z2 / (4.0 * total * total) + ) + / denominator + ) + return {"lower": max(0.0, centre - radius), "upper": min(1.0, centre + radius)} + + +def bootstrap_statistic( + observations, + statistic, + iterations=BOOTSTRAP_ITERATIONS, + seed=BOOTSTRAP_SEED, +): + """Item-level percentile-bootstrap interval for an arbitrary statistic.""" + observations = list(observations) + if not observations: + raise ValueError("observations must not be empty") + if type(iterations) is not int or iterations <= 0: + raise ValueError("iterations must be a positive integer") + rng = random.Random(seed) + size = len(observations) + estimates = [] + for _ in range(iterations): + sample = [observations[rng.randrange(size)] for _ in range(size)] + estimate = float(statistic(sample)) + if not math.isfinite(estimate): + raise ValueError("bootstrap statistic must be finite") + estimates.append(estimate) + return { + "lower": _percentile(estimates, 0.025), + "upper": _percentile(estimates, 0.975), + "iterations": iterations, + "seed": seed, + } + + +def bootstrap_median( + values, iterations=BOOTSTRAP_ITERATIONS, seed=BOOTSTRAP_SEED +): + return bootstrap_statistic( + _finite(values), + statistics.median, + iterations=iterations, + seed=seed, + ) diff --git a/reports/mhs/run_study.py b/reports/mhs/run_study.py new file mode 100644 index 0000000..491626d --- /dev/null +++ b/reports/mhs/run_study.py @@ -0,0 +1,440 @@ +#!/usr/bin/env python3 +"""Phase-2 MHS study harness. Present for audit; do not run in Phase 1.""" + +import argparse +import datetime +import hashlib +import json +import os +import statistics +import subprocess +import sys +from collections import Counter +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[2] +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + +import disagreement # noqa: E402 - imported from the repository root +import geometry # noqa: E402 - pure frozen centre/TV functions +from adapters import mhs # noqa: E402 +from reports.mhs import metrics # noqa: E402 + + +SOURCE_REVISION = "5468f6e" +SOURCE_SHA256 = "6819525ce61bc24344df9fc3f7bf48270b31038273cc27c67fc225b51433b0e1" +PROTOCOL_PATH = "reports/part-3b-dataset-selection-audit.md" +RELIABILITY_MIN_CONFIDENT = 20 +RELIABILITY_MIN_PER_COHORT = 30 +PARQUET_COLUMNS = ( + "annotator_id", + "comment_id", + "hatespeech", + "annotator_ideology", +) + + +def _parse_args(argv): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("source", help="local measuring-hate-speech.parquet") + parser.add_argument( + "--work-dir", + default=str(ROOT / "reports" / "mhs" / "work"), + help="git-ignored generated-data directory", + ) + parser.add_argument( + "--report-dir", + default=str(ROOT / "reports" / "mhs"), + help="directory for Phase-2 report.md and manifest.json", + ) + return parser.parse_args(argv) + + +def _sha256(path): + digest = hashlib.sha256() + with open(path, "rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def verify_source(path): + observed = _sha256(path) + if observed != SOURCE_SHA256: + raise SystemExit( + "MHS source SHA-256 mismatch: expected {}, observed {}. Halt: " + "freeze the new revision/hash, repeat the structural gate, and " + "obtain owner approval before outcomes.".format(SOURCE_SHA256, observed) + ) + return observed + + +def load_parquet_records(path): + """Thin dependency boundary; pyarrow is never imported by CI/module import.""" + try: + import pyarrow.parquet as parquet + except ImportError: + raise SystemExit( + "Phase 2 requires pyarrow to read the local parquet. Install it in " + "a separate environment; CI does not need it. Expected source " + "SHA-256: {}".format(SOURCE_SHA256) + ) + table = parquet.read_table(path, columns=list(PARQUET_COLUMNS)) + return table.to_pylist() + + +def _write_json(path, value): + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("w", encoding="utf-8") as handle: + json.dump(value, handle, indent=2, sort_keys=True) + handle.write("\n") + + +def _load_json(path): + with path.open(encoding="utf-8") as handle: + return json.load(handle) + + +def _run_instruments(data_path, out_dir): + out_dir.mkdir(parents=True, exist_ok=True) + for tool in ("disagreement.py", "soft_labels.py"): + subprocess.run( + [ + sys.executable, + str(ROOT / tool), + "--data", + str(data_path), + "--out", + str(out_dir), + ], + cwd=str(ROOT), + check=True, + ) + return _load_json(out_dir / "triage.json") + + +def _cohort_distribution(item, members): + votes = item["labels"][mhs.QUESTION] + present = [member for member in members if member in votes] + if not present: + raise ValueError("primary item has no votes for a frozen cohort") + counts = Counter(votes[member] for member in present) + total = sum(counts.values()) + return {label: count / total for label, count in counts.items()} + + +def aggregate_primary(dataset, triage): + """Aggregate the three primary-item metrics without changing instruments.""" + cells = triage["cells"] + total = len(dataset["items"]) + if total != len(cells): + raise ValueError("primary dataset and triage cell counts differ") + fork_count = sum(bool(cell["value_fork"]) for cell in cells) + manufactured_count = sum( + bool(cell["manufactured_consensus"]) for cell in cells + ) + gaps = [] + undefined = 0 + for item in dataset["items"]: + conservative = _cohort_distribution( + item, dataset["cohorts"]["Conservative"] + ) + liberal = _cohort_distribution(item, dataset["cohorts"]["Liberal"]) + arithmetic = geometry.arithmetic_mean([conservative, liberal]) + geometric = geometry.geometric_mean([conservative, liberal]) + if geometric is None: + undefined += 1 + else: + gaps.append(geometry.tv(arithmetic, geometric)) + gap_summary = None + if gaps: + gap_summary = metrics.median_iqr(gaps) + gap_summary["bootstrap_95"] = metrics.bootstrap_median(gaps) + return { + "value_fork": { + "count": fork_count, + "total": total, + "rate": fork_count / total, + "wilson_95": metrics.wilson_interval(fork_count, total), + }, + "manufactured_consensus": { + "count": manufactured_count, + "total": total, + "rate": manufactured_count / total, + "wilson_95": metrics.wilson_interval(manufactured_count, total), + }, + "geometry_gap": { + "defined_count": len(gaps), + "undefined_count": undefined, + "undefined_share": undefined / total, + "defined": gap_summary, + }, + } + + +def _confident_contributions(dataset, triage): + items = {item["id"]: item for item in dataset["items"]} + contributions = [] + for cell in triage["cells"]: + if cell["verdict"] != "CONFIDENT": + continue + votes = items[cell["item"]]["labels"][cell["question"]] + full = Counter(votes.values()) + contribution = {} + for annotator, label in votes.items(): + others = full.copy() + others[label] -= 1 + if not others[label]: + del others[label] + if not others: + continue + top = max(others.values()) + modes = {candidate for candidate, count in others.items() if count == top} + contribution[annotator] = (1 if label in modes else 0, 1) + contributions.append(contribution) + return contributions + + +def _reliability_values(contributions, annotators): + hits, totals = Counter(), Counter() + for contribution in contributions: + for annotator, (hit, total) in contribution.items(): + if annotator in annotators: + hits[annotator] += hit + totals[annotator] += total + return [hits[annotator] / totals[annotator] for annotator in annotators if totals[annotator]] + + +def aggregate_reliability(dataset, triage): + contributions = _confident_contributions(dataset, triage) + scored_counts = Counter() + for contribution in contributions: + scored_counts.update(contribution.keys()) + qualifying = { + cohort: [ + annotator + for annotator in dataset["cohorts"][cohort] + if scored_counts[annotator] >= RELIABILITY_MIN_CONFIDENT + ] + for cohort in mhs.COHORTS + } + values = { + cohort: [triage["reliability"][annotator] for annotator in qualifying[cohort]] + for cohort in mhs.COHORTS + } + summaries = { + cohort: (metrics.median_iqr(values[cohort]) if values[cohort] else None) + for cohort in mhs.COHORTS + } + powered = all( + len(qualifying[cohort]) >= RELIABILITY_MIN_PER_COHORT + for cohort in mhs.COHORTS + ) + difference = None + if all(values.values()): + difference = statistics.median(values["Conservative"]) - statistics.median( + values["Liberal"] + ) + interval = None + if powered: + fixed = {cohort: set(qualifying[cohort]) for cohort in mhs.COHORTS} + + def statistic(sample): + conservative = _reliability_values(sample, fixed["Conservative"]) + liberal = _reliability_values(sample, fixed["Liberal"]) + return statistics.median(conservative) - statistics.median(liberal) + + interval = metrics.bootstrap_statistic(contributions, statistic) + return { + "status": "descriptive" if powered else "underpowered/non-applicable", + "minimum_confident_cells": RELIABILITY_MIN_CONFIDENT, + "minimum_qualifying_annotators_per_cohort": RELIABILITY_MIN_PER_COHORT, + "coverage": { + cohort: { + "eligible": len(dataset["cohorts"][cohort]), + "qualifying": len(qualifying[cohort]), + } + for cohort in mhs.COHORTS + }, + "summaries": summaries, + "median_difference_conservative_minus_liberal": difference, + "bootstrap_95": interval, + } + + +def _tool_commit(): + return subprocess.check_output( + ["git", "-C", str(ROOT), "rev-parse", "HEAD"], + universal_newlines=True, + ).strip() + + +def _fmt(number): + return "not applicable" if number is None else "{:.6f}".format(number) + + +def render_report(results, counts, tool_commit): + primary = results["primary"] + reliability = results["reliability"] + geometry_result = primary["geometry_gap"] + return """# MHS bounded Tier-2 pilot + +**Evidence boundary:** dataset-specific, topology-qualified descriptive results +for MHS revision `{revision}` and the frozen Conservative/Liberal contrast. This +is not universal or production prevalence, a causal ideology effect, a correct +ground truth, or variation-versus-error attribution. + +Protocol: `{protocol}`
+Tool commit: `{commit}` + +## Structural counts + +- Primary items: {primary_items} +- Reliability-corpus items: {reliability_items} +- Conservative annotators: {conservative} +- Liberal annotators: {liberal} + +## Frozen metrics + +- Value forks: {fork_count}/{fork_total} ({fork_rate:.6f}); Wilson 95% [{fork_lo:.6f}, {fork_hi:.6f}] +- Manufactured consensus: {manufactured_count}/{manufactured_total} ({manufactured_rate:.6f}); Wilson 95% [{manufactured_lo:.6f}, {manufactured_hi:.6f}] +- Geometry: {undefined} undefined/disjoint-support items ({undefined_share:.6f}); defined-gap median {gap_median}, Q1 {gap_q1}, Q3 {gap_q3}, IQR {gap_iqr}, item-bootstrap 95% [{gap_boot_lo}, {gap_boot_hi}] +- Reliability: {reliability_status}; coverage Conservative {conservative_qualifying}/{conservative_eligible}, Liberal {liberal_qualifying}/{liberal_eligible}; Conservative median {conservative_median} (IQR {conservative_iqr}), Liberal median {liberal_median} (IQR {liberal_iqr}); Conservative-minus-Liberal median difference {reliability_difference}, item-bootstrap 95% [{reliability_boot_lo}, {reliability_boot_hi}] + +No source rows or identifiers are published. The source checksum and aggregate +results are recorded in `manifest.json`. No null-hypothesis p-values are used. +""".format( + revision=SOURCE_REVISION, + protocol=PROTOCOL_PATH, + commit=tool_commit, + primary_items=counts["primary_items"], + reliability_items=counts["reliability_items"], + conservative=counts["conservative_annotators"], + liberal=counts["liberal_annotators"], + fork_count=primary["value_fork"]["count"], + fork_total=primary["value_fork"]["total"], + fork_rate=primary["value_fork"]["rate"], + fork_lo=primary["value_fork"]["wilson_95"]["lower"], + fork_hi=primary["value_fork"]["wilson_95"]["upper"], + manufactured_count=primary["manufactured_consensus"]["count"], + manufactured_total=primary["manufactured_consensus"]["total"], + manufactured_rate=primary["manufactured_consensus"]["rate"], + manufactured_lo=primary["manufactured_consensus"]["wilson_95"]["lower"], + manufactured_hi=primary["manufactured_consensus"]["wilson_95"]["upper"], + undefined=geometry_result["undefined_count"], + undefined_share=geometry_result["undefined_share"], + gap_median=_fmt( + geometry_result["defined"]["median"] if geometry_result["defined"] else None + ), + gap_q1=_fmt( + geometry_result["defined"]["q1"] if geometry_result["defined"] else None + ), + gap_q3=_fmt( + geometry_result["defined"]["q3"] if geometry_result["defined"] else None + ), + gap_iqr=_fmt( + geometry_result["defined"]["iqr"] if geometry_result["defined"] else None + ), + gap_boot_lo=_fmt( + geometry_result["defined"]["bootstrap_95"]["lower"] + if geometry_result["defined"] + else None + ), + gap_boot_hi=_fmt( + geometry_result["defined"]["bootstrap_95"]["upper"] + if geometry_result["defined"] + else None + ), + reliability_status=reliability["status"], + conservative_qualifying=reliability["coverage"]["Conservative"]["qualifying"], + conservative_eligible=reliability["coverage"]["Conservative"]["eligible"], + liberal_qualifying=reliability["coverage"]["Liberal"]["qualifying"], + liberal_eligible=reliability["coverage"]["Liberal"]["eligible"], + conservative_median=_fmt( + reliability["summaries"]["Conservative"]["median"] + if reliability["summaries"]["Conservative"] + else None + ), + conservative_iqr=_fmt( + reliability["summaries"]["Conservative"]["iqr"] + if reliability["summaries"]["Conservative"] + else None + ), + liberal_median=_fmt( + reliability["summaries"]["Liberal"]["median"] + if reliability["summaries"]["Liberal"] + else None + ), + liberal_iqr=_fmt( + reliability["summaries"]["Liberal"]["iqr"] + if reliability["summaries"]["Liberal"] + else None + ), + reliability_difference=_fmt( + reliability["median_difference_conservative_minus_liberal"] + ), + reliability_boot_lo=_fmt( + reliability["bootstrap_95"]["lower"] + if reliability["bootstrap_95"] + else None + ), + reliability_boot_hi=_fmt( + reliability["bootstrap_95"]["upper"] + if reliability["bootstrap_95"] + else None + ), + ) + + +def main(argv=None): + args = _parse_args(argv) + source = Path(args.source).resolve() + source_hash = verify_source(source) + records = load_parquet_records(source) + converted = mhs.convert_records(records) + if converted["status"] == "halt": + raise SystemExit("MHS protocol halt: " + converted["halt_reason"]) + + work = Path(args.work_dir).resolve() + work.mkdir(parents=True, exist_ok=True) + primary_path = work / "primary-labels.json" + reliability_path = work / "reliability-labels.json" + mhs.write_dataset(converted["primary"], primary_path) + mhs.write_dataset(converted["reliability"], reliability_path) + primary_triage = _run_instruments(primary_path, work / "primary") + reliability_triage = _run_instruments(reliability_path, work / "reliability") + results = { + "primary": aggregate_primary(converted["primary"], primary_triage), + "reliability": aggregate_reliability( + converted["reliability"], reliability_triage + ), + } + commit = _tool_commit() + manifest = { + "evidence_tier": "Tier 2", + "generated_at": datetime.datetime.now(datetime.timezone.utc) + .replace(microsecond=0) + .isoformat() + .replace("+00:00", "Z"), + "source_revision": SOURCE_REVISION, + "source_sha256": source_hash, + "protocol": PROTOCOL_PATH, + "tool_commit": commit, + "counts": converted["counts"], + "metrics": results, + "contains_source_rows_or_ids": False, + } + report_dir = Path(args.report_dir).resolve() + report_dir.mkdir(parents=True, exist_ok=True) + _write_json(report_dir / "manifest.json", manifest) + (report_dir / "report.md").write_text( + render_report(results, converted["counts"], commit), encoding="utf-8" + ) + print("Tier-2 report -> {}".format(report_dir / "report.md")) + print("aggregate manifest -> {}".format(report_dir / "manifest.json")) + + +if __name__ == "__main__": + main() diff --git a/test_claims.py b/test_claims.py index 3ebe21b..6921c4a 100644 --- a/test_claims.py +++ b/test_claims.py @@ -32,6 +32,9 @@ import resolution import topology from adapters import chaosnli as chaosnli_adapter +from adapters import mhs as mhs_adapter +from reports.mhs import metrics as mhs_metrics +from reports.mhs import run_study as mhs_study try: from jsonschema import Draft202012Validator, FormatChecker @@ -1328,6 +1331,256 @@ def test_converted_fixture_matches_the_input_schema(self): Draft202012Validator(schema).validate(_load_json(labels_path)) +class MHSPhaseOneClaims(unittest.TestCase): + FIXTURE = ROOT / "reports" / "mhs" / "fixtures" / "mhs_sample.jsonl" + + @classmethod + def setUpClass(cls): + cls.records = mhs_adapter.load_jsonl(str(cls.FIXTURE)) + cls.converted = mhs_adapter.convert_records(cls.records) + + def test_frozen_eligibility_cohorts_primary_filter_and_halt(self): + converted = self.converted + self.assertEqual(converted["status"], "ready") + self.assertIsNone(converted["halt_reason"]) + self.assertEqual( + converted["counts"], + { + "source_records": 245, + "non_null_judgments": 244, + "eligible_annotators": 5, + "conservative_annotators": 2, + "liberal_annotators": 2, + "excluded_eligible_annotators": 1, + "primary_items": 50, + "reliability_items": 51, + }, + ) + self.assertEqual( + converted["primary"]["cohorts"], + { + "Conservative": ["c-extreme", "c-slight"], + "Liberal": ["l-extreme", "l-slight"], + }, + ) + for ideology in mhs_adapter.CONSERVATIVE_IDEOLOGIES: + self.assertEqual( + mhs_adapter.cohort_for_ideology(ideology), "Conservative" + ) + for ideology in mhs_adapter.LIBERAL_IDEOLOGIES: + self.assertEqual(mhs_adapter.cohort_for_ideology(ideology), "Liberal") + for ideology in ("neutral", "no_opinion", None, "Conservative"): + self.assertIsNone(mhs_adapter.cohort_for_ideology(ideology)) + + primary_ids = [item["id"] for item in converted["primary"]["items"]] + reliability_ids = [ + item["id"] for item in converted["reliability"]["items"] + ] + self.assertNotIn("synthetic-filter-fail", primary_ids) + self.assertIn("synthetic-filter-fail", reliability_ids) + self.assertEqual( + converted["primary"]["questions"]["hatespeech"]["labels"], + ["0", "1", "2"], + ) + observed_labels = { + label + for item in converted["primary"]["items"] + for label in item["labels"]["hatespeech"].values() + } + self.assertEqual(observed_labels, {"0", "1", "2"}) + self.assertFalse( + { + "c-below-floor", + "null-judgment", + "excluded-neutral", + "excluded-no-opinion", + "excluded-null", + } + & set(converted["reliability"]["annotators"]) + ) + + below_fifty = [ + record + for record in self.records + if record["comment_id"] + not in ("synthetic-050", "synthetic-filter-fail") + ] + halted = mhs_adapter.convert_records(below_fifty) + self.assertEqual(halted["status"], "halt") + self.assertEqual(halted["counts"]["primary_items"], 49) + self.assertIn("below the frozen minimum 50", halted["halt_reason"]) + + def test_synthetic_tool_wiring_and_frozen_metric_results(self): + with tempfile.TemporaryDirectory(prefix="groundless-mhs-") as temp: + temp_path = Path(temp) + primary_path = temp_path / "primary-labels.json" + reliability_path = temp_path / "reliability-labels.json" + mhs_adapter.write_dataset(self.converted["primary"], primary_path) + mhs_adapter.write_dataset( + self.converted["reliability"], reliability_path + ) + primary_out = temp_path / "primary" + reliability_out = temp_path / "reliability" + for data_path, out_dir in ( + (primary_path, primary_out), + (reliability_path, reliability_out), + ): + for tool in ("disagreement.py", "soft_labels.py"): + subprocess.run( + [ + sys.executable, + str(ROOT / tool), + "--data", + str(data_path), + "--out", + str(out_dir), + ], + check=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + ) + + triage = _load_json(primary_out / "triage.json") + by_item = {cell["item"]: cell for cell in triage["cells"]} + self.assertTrue(by_item["synthetic-001"]["value_fork"]) + self.assertEqual( + by_item["synthetic-001"]["cohort_majorities"], + {"Conservative": "0", "Liberal": "2"}, + ) + self.assertFalse(by_item["synthetic-002"]["value_fork"]) + self.assertFalse(by_item["synthetic-003"]["value_fork"]) + self.assertEqual( + by_item["synthetic-003"]["cohort_majorities"], + {"Conservative": None, "Liberal": "2"}, + ) + + primary_metrics = mhs_study.aggregate_primary( + self.converted["primary"], triage + ) + self.assertEqual(primary_metrics["value_fork"]["count"], 1) + self.assertEqual(primary_metrics["value_fork"]["total"], 50) + self.assertEqual( + primary_metrics["manufactured_consensus"]["count"], 1 + ) + self.assertEqual( + primary_metrics["geometry_gap"]["undefined_count"], 2 + ) + self.assertEqual( + primary_metrics["geometry_gap"]["defined_count"], 48 + ) + self.assertEqual( + primary_metrics["geometry_gap"]["defined"]["median"], 0.0 + ) + + reliability = mhs_study.aggregate_reliability( + self.converted["reliability"], + _load_json(reliability_out / "triage.json"), + ) + self.assertEqual(reliability["status"], "underpowered/non-applicable") + self.assertEqual( + reliability["coverage"], + { + "Conservative": {"eligible": 2, "qualifying": 2}, + "Liberal": {"eligible": 2, "qualifying": 2}, + }, + ) + self.assertIsNone(reliability["bootstrap_95"]) + + def test_frozen_descriptive_statistics(self): + interval = mhs_metrics.wilson_interval(5, 10) + self.assertAlmostEqual(interval["lower"], 0.236593090512564, places=12) + self.assertAlmostEqual(interval["upper"], 0.763406909487436, places=12) + self.assertEqual( + mhs_metrics.median_iqr([1, 2, 3, 4]), + {"median": 2.5, "q1": 1.75, "q3": 3.25, "iqr": 1.5}, + ) + self.assertEqual(mhs_metrics.BOOTSTRAP_ITERATIONS, 10000) + self.assertEqual(mhs_metrics.BOOTSTRAP_SEED, 20260718) + first = mhs_metrics.bootstrap_median([0, 1, 2, 3]) + second = mhs_metrics.bootstrap_median([0, 1, 2, 3]) + self.assertEqual(first, second) + self.assertEqual(first["iterations"], 10000) + self.assertEqual(first["seed"], 20260718) + self.assertNotIn("p_value", first) + self.assertNotIn("pvalue", first) + + def test_adapter_rejects_ambiguous_rows_and_harness_halts_on_hash(self): + invalid_label = copy.deepcopy(self.records) + invalid_label[0]["hatespeech"] = 3 + with self.assertRaisesRegex(mhs_adapter.MHSInputError, "one of 0, 1, 2"): + mhs_adapter.convert_records(invalid_label) + + inconsistent_ideology = copy.deepcopy(self.records) + inconsistent_ideology[4]["annotator_ideology"] = "liberal" + with self.assertRaisesRegex( + mhs_adapter.MHSInputError, "inconsistent author-supplied ideology" + ): + mhs_adapter.convert_records(inconsistent_ideology) + + duplicate = copy.deepcopy(self.records) + duplicate.append(copy.deepcopy(duplicate[0])) + with self.assertRaisesRegex(mhs_adapter.MHSInputError, "duplicate judgment"): + mhs_adapter.convert_records(duplicate) + + with tempfile.TemporaryDirectory(prefix="groundless-mhs-hash-") as temp: + fake_source = Path(temp) / "not-mhs.parquet" + fake_source.write_bytes(b"synthetic, not parquet") + with self.assertRaisesRegex(SystemExit, "MHS source SHA-256 mismatch"): + mhs_study.verify_source(fake_source) + self.assertEqual(list(Path(temp).iterdir()), [fake_source]) + + def test_phase_boundary_and_frozen_constants_are_explicit(self): + adapter_source = (ROOT / "adapters" / "mhs.py").read_text(encoding="utf-8") + harness_source = ( + ROOT / "reports" / "mhs" / "run_study.py" + ).read_text(encoding="utf-8") + readme = (ROOT / "reports" / "mhs" / "README.md").read_text( + encoding="utf-8" + ) + self.assertLess( + harness_source.index("def load_parquet_records"), + harness_source.index("import pyarrow.parquet as parquet"), + ) + self.assertNotIn("pyarrow", adapter_source) + for token in ( + "20", + "extremely_conservative", + "conservative", + "slightly_conservative", + "extremely_liberal", + "liberal", + "slightly_liberal", + "50", + ): + self.assertIn(token, adapter_source) + for token in ("10,000", "20260718", "30", "no null-hypothesis p-values"): + self.assertIn(token, readme) + tracked_parquet = subprocess.run( + ["git", "ls-files", "*.parquet"], + cwd=str(ROOT), + check=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + ).stdout.strip() + self.assertEqual(tracked_parquet, "") + self.assertTrue( + all( + record["comment_id"].startswith("synthetic-") + for record in self.records + ) + ) + + @unittest.skipUnless(_HAS_JSONSCHEMA, "jsonschema is not installed") + def test_both_mhs_outputs_match_the_input_schema(self): + schema = _load_json(ROOT / "schema" / "labels.schema.json") + Draft202012Validator.check_schema(schema) + validator = Draft202012Validator(schema) + validator.validate(self.converted["primary"]) + validator.validate(self.converted["reliability"]) + + class GovernanceClaims(unittest.TestCase): def _prepare_queue(self, out_dir): for tool in ("disagreement.py", "soft_labels.py"):