From 8f40405f0aa9881d7c77d38879a2e4b99913727d Mon Sep 17 00:00:00 2001 From: ABrandes <79119643+AMBRA7592@users.noreply.github.com> Date: Sat, 18 Jul 2026 07:16:29 +0200 Subject: [PATCH 1/3] ChaosNLI: add deterministic sharding and report harness --- .gitignore | 4 + adapters/README.md | 31 +- adapters/chaosnli.py | 299 +++++++++++++++++- adapters/fixtures/chaosnli_sample.jsonl | 9 +- reports/chaosnli/README.md | 35 +++ reports/chaosnli/run_report.py | 387 ++++++++++++++++++++++++ test_claims.py | 151 ++++++++- 7 files changed, 904 insertions(+), 12 deletions(-) create mode 100644 reports/chaosnli/README.md create mode 100644 reports/chaosnli/run_report.py diff --git a/.gitignore b/.gitignore index 3c0482f..6a2c7cf 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,7 @@ governance.jsonl resolution_records.jsonl __pycache__/ *.pyc +reports/chaosnli/work/ +reports/chaosnli/source/ +reports/chaosnli/*.zip +reports/chaosnli/chaosNLI_*.jsonl diff --git a/adapters/README.md b/adapters/README.md index f34d23d..6ab8410 100644 --- a/adapters/README.md +++ b/adapters/README.md @@ -50,10 +50,30 @@ recomputed from `label_counter`. The input contract represents individual votes, so each published count is expanded into that many virtual voters. Full ChaosNLI splits therefore produce -large `labels.json` files, and this repository's explanatory tools are not -optimized as a high-throughput corpus engine. Trial the workflow on a JSONL -shard before committing resources to a full split; sharding does not change any -per-item distribution. +large `labels.json` files. Use deterministic, contiguous shards to bound each +pipeline run without dropping or reordering records: + +```bash +for shard in 0 1 2 3; do + python3 adapters/chaosnli.py path/to/chaosNLI_snli.jsonl \ + --shard-index "$shard" --shard-count 4 \ + --out "/tmp/chaos-shard-$shard.json" +done +``` + +For `M` source records, shard `K` contains +`[floor(K*M/N):floor((K+1)*M/N)]`. Each selected conversion writes an ordered, +deterministic `.manifest.json` containing source/output hashes, record and +vote counts, UID coverage, and the tool commit. The alternative +`--offset O [--limit L]` mode selects the same contiguous ranges directly; it +cannot be combined with shard mode. Omitting all selection flags preserves the +whole-file behavior. + +The external-data harness in [`reports/chaosnli/`](../reports/chaosnli/) +constructs every shard, proves that their manifests cover every source UID once +and in order, validates the schema, checks every converted counter and soft +label, and runs the operational pipeline. Only its report and hash/count +manifest are committed—not the licensed rows or converted outputs. To exercise the conversion without downloading anything: @@ -61,4 +81,5 @@ To exercise the conversion without downloading anything: python3 adapters/chaosnli.py adapters/fixtures/chaosnli_sample.jsonl --out /tmp/chaos-labels.json ``` -The synthetic fixture includes one `e/n/c` example and one αNLI `1/2` example. +The seven-row synthetic fixture includes both `e/n/c` and αNLI `1/2` examples +and exercises uneven full-coverage sharding in CI. diff --git a/adapters/chaosnli.py b/adapters/chaosnli.py index 11ce66a..fbc4775 100644 --- a/adapters/chaosnli.py +++ b/adapters/chaosnli.py @@ -7,8 +7,11 @@ """ import argparse +import hashlib import json import os +import subprocess +from collections import Counter from urllib.parse import quote @@ -22,6 +25,12 @@ "value-fork results are therefore not meaningful for this conversion. The " "faithful outputs are the per-item distribution, entropy, and soft labels." ) +ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +MANIFEST_VERSION = 1 + + +class ManifestError(ValueError): + """A shard manifest set does not prove complete, ordered coverage.""" def _parse_args(argv): @@ -32,6 +41,30 @@ def _parse_args(argv): default="labels.json", help="output labels.json path (default: ./labels.json)", ) + parser.add_argument( + "--shard-index", + type=int, + default=None, + help="zero-based contiguous shard index (requires --shard-count)", + ) + parser.add_argument( + "--shard-count", + type=int, + default=None, + help="number of contiguous shards (requires --shard-index)", + ) + parser.add_argument( + "--offset", + type=int, + default=None, + help="zero-based record offset (mutually exclusive with shard mode)", + ) + parser.add_argument( + "--limit", + type=int, + default=None, + help="maximum records from --offset (default: through end)", + ) return parser.parse_args(argv) @@ -107,7 +140,247 @@ def load_records(path): return records +def _record_uids(records): + uids = [] + seen = set() + for line_number, record in records: + uid = record.get("uid") if isinstance(record, dict) else None + if not isinstance(uid, str) or not uid.strip(): + _fail("line {} needs a non-empty string uid".format(line_number)) + if uid in seen: + _fail("line {} repeats uid {!r}".format(line_number, uid)) + seen.add(uid) + uids.append(uid) + return uids + + +def select_records(records, shard_index=None, shard_count=None, offset=None, limit=None): + """Return one contiguous selection and deterministic selection metadata.""" + has_shard = shard_index is not None or shard_count is not None + has_range = offset is not None or limit is not None + if has_shard and has_range: + _fail("shard mode and offset/limit mode are mutually exclusive") + if has_shard: + if shard_index is None or shard_count is None: + _fail("--shard-index and --shard-count must be supplied together") + if shard_count <= 0: + _fail("--shard-count must be greater than zero") + if shard_index < 0 or shard_index >= shard_count: + _fail("--shard-index must satisfy 0 <= index < shard-count") + total = len(records) + start = shard_index * total // shard_count + stop = (shard_index + 1) * total // shard_count + selected = records[start:stop] + metadata = { + "mode": "shard", + "shard_index": shard_index, + "shard_count": shard_count, + "offset": start, + "limit": stop - start, + } + elif has_range: + if offset is None: + _fail("--limit requires --offset") + if offset < 0: + _fail("--offset must be non-negative") + if limit is not None and limit < 0: + _fail("--limit must be non-negative") + stop = len(records) if limit is None else offset + limit + selected = records[offset:stop] + metadata = { + "mode": "range", + "shard_index": None, + "shard_count": None, + "offset": offset, + "limit": limit, + } + else: + return records, None + if not selected: + _fail("selection contains no records") + return selected, metadata + + +def _sha256_bytes(content): + return hashlib.sha256(content).hexdigest() + + +def _sha256_file(path): + digest = hashlib.sha256() + with open(path, "rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def _uids_sha256(uids): + payload = json.dumps( + list(uids), ensure_ascii=False, separators=(",", ":") + ).encode("utf-8") + return _sha256_bytes(payload) + + +def _tool_commit(): + try: + return subprocess.check_output( + ["git", "-C", ROOT, "rev-parse", "HEAD"], + stderr=subprocess.DEVNULL, + universal_newlines=True, + ).strip() + except (OSError, subprocess.CalledProcessError): + return "unknown" + + +def build_manifest(input_path, output_path, source_uids, dataset, selection): + selected_uids = [item["id"] for item in dataset["items"]] + votes = sum( + len(item["labels"][QUESTION]) for item in dataset["items"] + ) + return { + "manifest_version": MANIFEST_VERSION, + "source_path": os.path.abspath(input_path), + "source_sha256": _sha256_file(input_path), + "total_records": len(source_uids), + "mode": selection["mode"], + "shard_index": selection["shard_index"], + "shard_count": selection["shard_count"], + "offset": selection["offset"], + "limit": selection["limit"], + "record_count": len(selected_uids), + "uid_first": selected_uids[0], + "uid_last": selected_uids[-1], + "uids": selected_uids, + "uids_sha256": _uids_sha256(selected_uids), + "annotator_count": len(dataset["annotators"]), + "vote_count": votes, + "label_set": sorted( + { + label + for item in dataset["items"] + for label in item["labels"][QUESTION].values() + } + ), + "tool_commit": _tool_commit(), + "output_sha256": _sha256_file(output_path), + } + + +def aggregate_shard_manifests(manifests, source_uids): + """Validate shard manifests and return a whole-source coverage manifest.""" + if not manifests: + raise ManifestError("no shard manifests supplied") + if len(set(source_uids)) != len(source_uids): + raise ManifestError("source uid list contains a duplicate") + if any(manifest.get("mode") != "shard" for manifest in manifests): + raise ManifestError("all manifests must use shard mode") + shard_counts = {manifest.get("shard_count") for manifest in manifests} + if len(shard_counts) != 1: + raise ManifestError("shard_count disagrees across manifests") + shard_count = next(iter(shard_counts)) + if type(shard_count) is not int or shard_count <= 0: + raise ManifestError("invalid shard_count") + by_index = {} + for manifest in manifests: + index = manifest.get("shard_index") + if type(index) is not int: + raise ManifestError("invalid shard index") + if index in by_index: + raise ManifestError("duplicate shard index {}".format(index)) + by_index[index] = manifest + expected_indices = list(range(shard_count)) + if sorted(by_index) != expected_indices: + raise ManifestError( + "shard indices must cover 0..{} exactly".format(shard_count - 1) + ) + + totals = {manifest.get("total_records") for manifest in manifests} + source_hashes = {manifest.get("source_sha256") for manifest in manifests} + if totals != {len(source_uids)}: + raise ManifestError("total_records does not match the source") + if len(source_hashes) != 1: + raise ManifestError("source_sha256 disagrees across manifests") + if not all( + isinstance(source_hash, str) and len(source_hash) == 64 + for source_hash in source_hashes + ): + raise ManifestError("invalid source_sha256") + + ordered = [by_index[index] for index in expected_indices] + concatenated = [] + for manifest in ordered: + uids = manifest.get("uids") + if not isinstance(uids, list): + raise ManifestError("each shard manifest needs an ordered uids list") + if manifest.get("record_count") != len(uids): + raise ManifestError("record_count does not match a shard's uids") + if not uids or manifest.get("uid_first") != uids[0] or manifest.get("uid_last") != uids[-1]: + raise ManifestError("uid boundary metadata does not match a shard") + if manifest.get("uids_sha256") != _uids_sha256(uids): + raise ManifestError("uids_sha256 does not match a shard's uids") + concatenated.extend(uids) + if sum(manifest["record_count"] for manifest in ordered) != len(source_uids): + raise ManifestError("shard record counts do not sum to total_records") + if len(set(concatenated)) != len(concatenated): + raise ManifestError("shard coverage contains a duplicate uid") + if concatenated != list(source_uids): + raise ManifestError("shard coverage has a gap, overlap, or order change") + + return { + "manifest_version": MANIFEST_VERSION, + "source_path": ordered[0].get("source_path"), + "source_sha256": next(iter(source_hashes)), + "total_records": len(source_uids), + "shard_count": shard_count, + "record_count": sum(manifest["record_count"] for manifest in ordered), + "uids": concatenated, + "uids_sha256": _uids_sha256(concatenated), + "annotator_count": sum(manifest["annotator_count"] for manifest in ordered), + "vote_count": sum(manifest["vote_count"] for manifest in ordered), + "label_set": sorted( + {label for manifest in ordered for label in manifest["label_set"]} + ), + "tool_commits": sorted({manifest["tool_commit"] for manifest in ordered}), + "shards": [ + { + "shard_index": manifest["shard_index"], + "record_count": manifest["record_count"], + "uid_first": manifest["uid_first"], + "uid_last": manifest["uid_last"], + "uids_sha256": manifest["uids_sha256"], + "output_sha256": manifest["output_sha256"], + } + for manifest in ordered + ], + } + + +def verify_converted_distributions(records, dataset): + """Prove that each converted vote multiset equals its source counter.""" + expected_uids = _record_uids(records) + items = dataset.get("items", []) + actual_uids = [item.get("id") for item in items] + if actual_uids != expected_uids: + raise ValueError("converted item order/uids do not match the source selection") + vote_count = 0 + for (line_number, source), item in zip(records, items): + counter, _ = _counter(source, line_number) + observed = Counter(item["labels"][QUESTION].values()) + labels = set(counter) | set(observed) + expected = {label: counter.get(label, 0) for label in labels} + actual = {label: observed.get(label, 0) for label in labels} + if actual != expected: + raise ValueError( + "uid {!r} distribution mismatch: expected {}, observed {}".format( + item["id"], expected, actual + ) + ) + vote_count += sum(actual.values()) + return {"record_count": len(records), "vote_count": vote_count} + + def convert(records): + if not records: + _fail("selection contains no records") annotators = [] items = [] seen_uids = set() @@ -167,7 +440,17 @@ def convert(records): def main(argv=None): args = _parse_args(argv) - dataset = convert(load_records(args.input)) + records = load_records(args.input) + source_uids = _record_uids(records) + selected, selection = select_records( + records, + shard_index=args.shard_index, + shard_count=args.shard_count, + offset=args.offset, + limit=args.limit, + ) + dataset = convert(selected) + verify_converted_distributions(selected, dataset) output = os.path.abspath(args.out) directory = os.path.dirname(output) if directory: @@ -175,11 +458,21 @@ def main(argv=None): with open(output, "w", encoding="utf-8") as handle: json.dump(dataset, handle, ensure_ascii=False, indent=2) handle.write("\n") + if selection is not None: + manifest = build_manifest( + args.input, output, source_uids, dataset, selection + ) + manifest_path = output + ".manifest.json" + with open(manifest_path, "w", encoding="utf-8") as handle: + json.dump(manifest, handle, ensure_ascii=False, indent=2, sort_keys=True) + handle.write("\n") print( - "converted {} item(s), {} anonymous vote(s) -> {}".format( - len(dataset["items"]), len(dataset["annotators"]), output + "converted {} of {} item(s), {} anonymous vote(s) -> {}".format( + len(dataset["items"]), len(records), len(dataset["annotators"]), output ) ) + if selection is not None: + print("manifest -> {}".format(manifest_path)) print( "caveat: reliability and value forks are not meaningful; " "use distributions, entropy, and soft labels" diff --git a/adapters/fixtures/chaosnli_sample.jsonl b/adapters/fixtures/chaosnli_sample.jsonl index c6902e1..2772d60 100644 --- a/adapters/fixtures/chaosnli_sample.jsonl +++ b/adapters/fixtures/chaosnli_sample.jsonl @@ -1,2 +1,7 @@ -{"uid":"synthetic-snli-1","label_counter":{"e":5,"n":3,"c":2},"majority_label":"e","label_dist":[0.5,0.3,0.2],"label_count":[5,3,2],"entropy":1.4854752972273344,"example":{"uid":"synthetic-snli-1","premise":"A synthetic researcher opens a notebook.","hypothesis":"A person is using written notes.","source":"synthetic_snli"},"old_label":"e","old_labels":["entailment","neutral","entailment","contradiction","entailment"]} -{"uid":"synthetic-alpha-1","label_counter":{"1":7,"2":3},"majority_label":1,"label_dist":[0.7,0.3],"label_count":[7,3],"entropy":0.8812908992306927,"example":{"uid":"synthetic-alpha-1","obs1":"A synthetic traveler packed an umbrella.","obs2":"The traveler arrived dry.","hyp1":"Rain was forecast before departure.","hyp2":"The umbrella was left at home.","source":"synthetic_abdnli"},"old_label":1} +{"uid":"synthetic-snli-1","label_counter":{"e":5,"n":3,"c":2},"majority_label":"e","example":{"uid":"synthetic-snli-1","premise":"A synthetic researcher opens a notebook.","hypothesis":"A person is using written notes.","source":"synthetic_snli"}} +{"uid":"synthetic-snli-2","label_counter":{"e":2,"n":5,"c":3},"majority_label":"n","example":{"uid":"synthetic-snli-2","premise":"A synthetic cyclist stops beside a bridge.","hypothesis":"A person pauses near a crossing.","source":"synthetic_snli"}} +{"uid":"synthetic-snli-3","label_counter":{"e":1,"n":3,"c":6},"majority_label":"c","example":{"uid":"synthetic-snli-3","premise":"A synthetic chef closes the empty kitchen.","hypothesis":"Dinner service is beginning.","source":"synthetic_snli"}} +{"uid":"synthetic-snli-4","label_counter":{"e":8,"n":1,"c":1},"majority_label":"e","example":{"uid":"synthetic-snli-4","premise":"A synthetic child reads a book.","hypothesis":"Someone is reading.","source":"synthetic_snli"}} +{"uid":"synthetic-alpha-1","label_counter":{"1":7,"2":3},"majority_label":1,"example":{"uid":"synthetic-alpha-1","obs1":"A synthetic traveler packed an umbrella.","obs2":"The traveler arrived dry.","hyp1":"Rain was forecast before departure.","hyp2":"The umbrella was left at home.","source":"synthetic_abdnli"}} +{"uid":"synthetic-alpha-2","label_counter":{"1":4,"2":6},"majority_label":2,"example":{"uid":"synthetic-alpha-2","obs1":"A synthetic office light was left on.","obs2":"The room was occupied at dawn.","hyp1":"A cleaner arrived overnight.","hyp2":"An employee worked late.","source":"synthetic_abdnli"}} +{"uid":"synthetic-snli-5","label_counter":{"e":3,"n":3,"c":4},"majority_label":"c","example":{"uid":"synthetic-snli-5","premise":"A synthetic gardener watches dark clouds.","hypothesis":"The weather will remain dry.","source":"synthetic_snli"}} diff --git a/reports/chaosnli/README.md b/reports/chaosnli/README.md new file mode 100644 index 0000000..f7bbfd4 --- /dev/null +++ b/reports/chaosnli/README.md @@ -0,0 +1,35 @@ +# ChaosNLI reproducibility evidence + +This directory is deliberately split from the CI-pinned core. The adapter, +sharding, complete-coverage proof, distribution verifier, and manifest +aggregation are tested in CI using synthetic rows. The numbers in `report.md` +and `manifest.json` are **Tier 2 external empirical evidence** from an official +ChaosNLI split downloaded separately under CC BY-NC 4.0. + +No official row, converted `labels.json`, or expanded virtual-voter record is +committed. The manifest contains only hashes, aggregate counts, and per-shard +output/UID-list hashes. + +## Reproduce + +1. Download `chaosNLI_v1.0.zip` from the + [official ChaosNLI repository](https://github.com/easonnie/ChaosNLI) and + extract it outside this repository. +2. Confirm the split's SHA-256 equals the value in `manifest.json`. +3. From the repository root, with `jsonschema` installed, run: + + ```bash + python3 reports/chaosnli/run_report.py \ + /path/to/chaosNLI_snli.jsonl \ + --expected-source-sha256 SHA256_FROM_MANIFEST \ + --shard-count 16 + ``` + +4. Compare the regenerated deterministic `manifest.json` with the committed + one. Runtime and memory belong only to the human-readable report because + they vary by machine. + +The report cannot measure annotator reliability, distinguish variation from +error, or estimate cohort value-fork prevalence. ChaosNLI's counts do not carry +stable annotator identities across items, and the conversion faithfully uses a +single crowd cohort. Those quantities are undefined for this dataset, not zero. diff --git a/reports/chaosnli/run_report.py b/reports/chaosnli/run_report.py new file mode 100644 index 0000000..d482d08 --- /dev/null +++ b/reports/chaosnli/run_report.py @@ -0,0 +1,387 @@ +#!/usr/bin/env python3 +"""Run one official ChaosNLI split and emit reproducibility evidence, not data.""" + +import argparse +import hashlib +import json +import math +import os +import platform +import resource +import subprocess +import sys +import time +import tracemalloc +from datetime import datetime, timezone +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[2] +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + +from adapters import chaosnli as adapter # noqa: E402 + + +DEFAULT_SOURCE_URL = "https://www.dropbox.com/s/h4j7dqszmpt2679/chaosNLI_v1.0.zip" +DEFAULT_SOURCE_VERSION = "ChaosNLI v1.0" +DEFAULT_LICENSE = "CC BY-NC 4.0" + + +def parse_args(argv=None): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("input", help="locally downloaded official ChaosNLI JSONL split") + parser.add_argument("--source-url", default=DEFAULT_SOURCE_URL) + parser.add_argument("--source-version", default=DEFAULT_SOURCE_VERSION) + parser.add_argument("--license", default=DEFAULT_LICENSE) + parser.add_argument("--expected-source-sha256", default=None) + parser.add_argument("--shard-count", type=int, default=16) + parser.add_argument( + "--work-dir", + default=str(ROOT / "reports" / "chaosnli" / "work"), + help="ignored directory for converted data and pipeline artifacts", + ) + parser.add_argument( + "--manifest-out", + default=str(ROOT / "reports" / "chaosnli" / "manifest.json"), + ) + parser.add_argument( + "--report-out", + default=str(ROOT / "reports" / "chaosnli" / "report.md"), + ) + return parser.parse_args(argv) + + +def sha256_file(path): + digest = hashlib.sha256() + with Path(path).open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def git_commit(): + return subprocess.check_output( + ["git", "-C", str(ROOT), "rev-parse", "HEAD"], + universal_newlines=True, + ).strip() + + +def load_json(path): + with Path(path).open(encoding="utf-8") as handle: + return json.load(handle) + + +def load_jsonl(path): + with Path(path).open(encoding="utf-8") as handle: + return [json.loads(line) for line in handle if line.strip()] + + +def run(command): + started = time.perf_counter() + result = subprocess.run( + [str(part) for part in command], + cwd=str(ROOT), + check=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + ) + return result, time.perf_counter() - started + + +def validate_schema(dataset, schema): + try: + from jsonschema import Draft202012Validator + except ImportError: + raise SystemExit( + "run_report.py requires jsonschema for the explicit schema-validation gate" + ) + Draft202012Validator.check_schema(schema) + Draft202012Validator(schema).validate(dataset) + + +def expected_soft_label(record): + counter = {str(label): count for label, count in record["label_counter"].items()} + total = sum(counter.values()) + return { + label: round(count / total, 3) + for label, count in counter.items() + if count + } + + +def expected_entropy(record): + probabilities = expected_soft_label(record).values() + return -sum(probability * math.log2(probability) for probability in probabilities) + + +def directory_bytes(path): + return sum(file.stat().st_size for file in Path(path).rglob("*") if file.is_file()) + + +def peak_rss_bytes(): + value = resource.getrusage(resource.RUSAGE_CHILDREN).ru_maxrss + return int(value if sys.platform == "darwin" else value * 1024) + + +def format_bytes(value): + units = ("B", "KiB", "MiB", "GiB") + amount = float(value) + for unit in units: + if amount < 1024 or unit == units[-1]: + return "{:.2f} {}".format(amount, unit) + amount /= 1024 + + +def write_json(path, value): + path = Path(path) + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("w", encoding="utf-8") as handle: + json.dump(value, handle, ensure_ascii=False, indent=2, sort_keys=True) + handle.write("\n") + + +def main(argv=None): + args = parse_args(argv) + if args.shard_count <= 0: + raise SystemExit("--shard-count must be greater than zero") + source_path = Path(args.input).resolve() + work_dir = Path(args.work_dir).resolve() + work_dir.mkdir(parents=True, exist_ok=True) + if any(work_dir.iterdir()): + raise SystemExit("work directory must be empty: {}".format(work_dir)) + + source_sha256 = sha256_file(source_path) + if ( + args.expected_source_sha256 + and source_sha256 != args.expected_source_sha256 + ): + raise SystemExit( + "source SHA-256 mismatch: expected {}, observed {}".format( + args.expected_source_sha256, source_sha256 + ) + ) + + records = adapter.load_records(str(source_path)) + source_uids = [record["uid"] for _, record in records] + schema = load_json(ROOT / "schema" / "labels.schema.json") + tool_commit = git_commit() + manifests = [] + verified_distributions = 0 + verified_soft_labels = 0 + verified_entropies = 0 + total_pipeline_bytes = 0 + total_runtime = 0.0 + + tracemalloc.start() + started_all = time.perf_counter() + for shard_index in range(args.shard_count): + shard_dir = work_dir / "shard-{:03d}".format(shard_index) + pipeline_dir = shard_dir / "pipeline" + labels_path = shard_dir / "labels.json" + shard_dir.mkdir(parents=True) + _, adapter_runtime = run( + [ + sys.executable, + ROOT / "adapters" / "chaosnli.py", + source_path, + "--shard-index", + shard_index, + "--shard-count", + args.shard_count, + "--out", + labels_path, + ] + ) + manifest = load_json(str(labels_path) + ".manifest.json") + manifests.append(manifest) + start = shard_index * len(records) // args.shard_count + stop = (shard_index + 1) * len(records) // args.shard_count + selected = records[start:stop] + dataset = load_json(labels_path) + verification = adapter.verify_converted_distributions(selected, dataset) + verified_distributions += verification["record_count"] + validate_schema(dataset, schema) + + pipeline_runtime = 0.0 + for tool in ("disagreement.py", "soft_labels.py", "resolution.py"): + _, elapsed = run( + [ + sys.executable, + ROOT / tool, + "--data", + labels_path, + "--out", + pipeline_dir, + ] + ) + pipeline_runtime += elapsed + + soft_by_uid = { + row["item_id"]: row + for row in load_jsonl(pipeline_dir / "soft_labels.jsonl") + } + triage_by_uid = { + row["item"]: row for row in load_json(pipeline_dir / "triage.json")["cells"] + } + for _, source in selected: + uid = source["uid"] + if soft_by_uid[uid]["soft_label"] != expected_soft_label(source): + raise ValueError("uid {!r} soft-label mismatch".format(uid)) + verified_soft_labels += 1 + if triage_by_uid[uid]["entropy_bits"] != round(expected_entropy(source), 3): + raise ValueError("uid {!r} entropy mismatch".format(uid)) + verified_entropies += 1 + + total_pipeline_bytes += directory_bytes(pipeline_dir) + shard_runtime = adapter_runtime + pipeline_runtime + total_runtime += shard_runtime + + wall_seconds = time.perf_counter() - started_all + _, harness_peak = tracemalloc.get_traced_memory() + tracemalloc.stop() + aggregate = adapter.aggregate_shard_manifests(manifests, source_uids) + if aggregate["tool_commits"] != [tool_commit]: + raise ValueError("shard manifests do not name the executing tool commit") + + deterministic_manifest = { + "evidence_tier": "Tier 2 - manifest-reproducible external empirical run", + "dataset": source_path.name, + "source_url": args.source_url, + "source_version": args.source_version, + "source_license": args.license, + "source_sha256": source_sha256, + "tool_commit": tool_commit, + "total_records": len(records), + "vote_count": aggregate["vote_count"], + "label_set": aggregate["label_set"], + "shard_count": args.shard_count, + "coverage": { + "record_count": aggregate["record_count"], + "uids_sha256": aggregate["uids_sha256"], + "ordered_complete_unique": True, + }, + "verification": { + "distribution_matches": verified_distributions, + "soft_label_matches": verified_soft_labels, + "entropy_matches": verified_entropies, + "schema_valid_shards": args.shard_count, + "pipeline_complete_shards": args.shard_count, + }, + "shards": [ + { + "shard_index": shard["shard_index"], + "record_count": shard["record_count"], + "vote_count": manifests[shard["shard_index"]]["vote_count"], + "uids_sha256": shard["uids_sha256"], + "output_sha256": shard["output_sha256"], + } + for shard in aggregate["shards"] + ], + } + write_json(args.manifest_out, deterministic_manifest) + manifest_sha256 = sha256_file(args.manifest_out) + + entropies = [expected_entropy(record) for _, record in records] + generated_at = datetime.now(timezone.utc).isoformat(timespec="seconds").replace( + "+00:00", "Z" + ) + report = """# ChaosNLI real-split reproducibility report + +**Evidence tier:** Tier 2 — manifest-reproducible external empirical evidence. +**Generated:** {generated_at} +**Tool commit:** `{tool_commit}` + +This report records one run on a separately downloaded official split. No +ChaosNLI rows, converted labels, or virtual-voter records are committed here. +The CI-pinned Tier-1 tests exercise the same sharding, coverage, distribution, +and manifest logic on a synthetic fixture. + +## Source and coverage + +- Source: `{source_name}` ({source_version}) +- Official download: {source_url} +- Dataset license: {license_name} +- Source SHA-256: `{source_sha256}` +- Committed manifest SHA-256: `{manifest_sha256}` +- Records: {record_count:,} +- Anonymous votes represented: {vote_count:,} +- Shards: {shard_count} +- Coverage: every source UID appears exactly once, in source order; no gaps, + overlaps, duplicates, or missing shard indices + +## Verification results + +- Converted vote distributions exactly matching source `label_counter`: + **{distribution_matches:,}/{record_count:,}** +- Trainer soft labels exactly matching `label_counter / sum(label_counter)`: + **{soft_label_matches:,}/{record_count:,}** +- Recomputed entropy matching triage output at its documented 3-decimal + precision: **{entropy_matches:,}/{record_count:,}** +- Input-schema validation: **{schema_shards}/{shard_count} shards** +- Operational pipeline completion (`disagreement` → `soft_labels` → + `resolution`): **{pipeline_shards}/{shard_count} shards** +- Source entropy range: {entropy_min:.6f} to {entropy_max:.6f} bits; mean + {entropy_mean:.6f} bits + +## Runtime and resources + +- Platform: {platform_name}; Python {python_version} +- End-to-end wall time: {wall_seconds:.3f} seconds +- Sum of measured adapter + pipeline subprocess times: {total_runtime:.3f} seconds +- Peak child-process resident memory: {peak_rss} +- Peak harness-tracked Python memory: {harness_peak} +- Converted and pipeline work-directory size: {work_bytes} +- Pipeline artifacts alone: {pipeline_bytes} + +Runtime and memory are observations from this machine, not deterministic claims; +the committed JSON manifest intentionally contains only reproducibility checks, +hashes, and counts. + +## Non-applicable findings + +**Reliability, error attribution, variation-vs-error classification, and +cohort value-fork prevalence are NON-APPLICABLE — undefined here, not zero.** +ChaosNLI exposes anonymous per-item counts rather than stable cross-item +annotator identities, and this faithful conversion uses one crowd cohort. +Accordingly, this report makes claims only about distributions, entropy, soft +labels, coverage, and reproducibility. +""".format( + generated_at=generated_at, + tool_commit=tool_commit, + source_name=source_path.name, + source_version=args.source_version, + source_url=args.source_url, + license_name=args.license, + source_sha256=source_sha256, + manifest_sha256=manifest_sha256, + record_count=len(records), + vote_count=aggregate["vote_count"], + shard_count=args.shard_count, + distribution_matches=verified_distributions, + soft_label_matches=verified_soft_labels, + entropy_matches=verified_entropies, + schema_shards=args.shard_count, + pipeline_shards=args.shard_count, + entropy_min=min(entropies), + entropy_max=max(entropies), + entropy_mean=sum(entropies) / len(entropies), + platform_name=platform.platform(), + python_version=platform.python_version(), + wall_seconds=wall_seconds, + total_runtime=total_runtime, + peak_rss=format_bytes(peak_rss_bytes()), + harness_peak=format_bytes(harness_peak), + work_bytes=format_bytes(directory_bytes(work_dir)), + pipeline_bytes=format_bytes(total_pipeline_bytes), + ) + report_path = Path(args.report_out) + report_path.parent.mkdir(parents=True, exist_ok=True) + report_path.write_text(report, encoding="utf-8") + print("manifest -> {}".format(Path(args.manifest_out).resolve())) + print("report -> {}".format(report_path.resolve())) + + +if __name__ == "__main__": + main() diff --git a/test_claims.py b/test_claims.py index e51b449..fc90dd4 100644 --- a/test_claims.py +++ b/test_claims.py @@ -29,6 +29,7 @@ import geometry import resolution import topology +from adapters import chaosnli as chaosnli_adapter try: from jsonschema import Draft202012Validator, FormatChecker @@ -940,14 +941,21 @@ def test_anonymous_counts_round_trip_to_exact_soft_labels(self): self.assertIn(caveat, about) self.assertEqual(len(dataset["items"]), len(source)) - self.assertEqual(len(dataset["annotators"]), 20) + self.assertEqual(len(dataset["annotators"]), 70) self.assertEqual(dataset["cohorts"], {"crowd": dataset["annotators"]}) annotators_by_item = [] for item in dataset["items"]: votes = item["labels"]["nli_label"] annotators_by_item.append(set(votes)) self.assertEqual(Counter(votes.values()), Counter(source_counts[item["id"]])) - self.assertTrue(annotators_by_item[0].isdisjoint(annotators_by_item[1])) + for index, annotators in enumerate(annotators_by_item): + self.assertTrue( + annotators.isdisjoint( + set().union(*annotators_by_item[:index]) + if index + else set() + ) + ) out_dir = temp_path / "run" for tool in ("disagreement.py", "soft_labels.py", "resolution.py"): @@ -992,6 +1000,145 @@ def test_anonymous_counts_round_trip_to_exact_soft_labels(self): all(record["measures"]["geometry_gap"] is None for record in records) ) + def test_shards_cover_fixture_exactly_and_reject_corruption(self): + source = _load_jsonl(self.FIXTURE) + source_uids = [record["uid"] for record in source] + with tempfile.TemporaryDirectory(prefix="groundless-chaosnli-shards-") as temp: + temp_path = Path(temp) + manifests = [] + shard_uids = [] + for index in range(3): + output = temp_path / "shard-{}.json".format(index) + subprocess.run( + [ + sys.executable, + str(self.ADAPTER), + str(self.FIXTURE), + "--shard-index", + str(index), + "--shard-count", + "3", + "--out", + str(output), + ], + check=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + ) + manifest = _load_json(Path(str(output) + ".manifest.json")) + manifests.append(manifest) + shard_uids.append(manifest["uids"]) + + self.assertEqual([len(uids) for uids in shard_uids], [2, 2, 3]) + self.assertEqual( + [uid for uids in shard_uids for uid in uids], source_uids + ) + self.assertEqual( + len({uid for uids in shard_uids for uid in uids}), len(source_uids) + ) + aggregate = chaosnli_adapter.aggregate_shard_manifests( + manifests, source_uids + ) + self.assertEqual(aggregate["record_count"], 7) + self.assertEqual(aggregate["uids"], source_uids) + + duplicate = copy.deepcopy(manifests) + duplicate[1]["uids"][0] = duplicate[0]["uids"][-1] + duplicate[1]["uid_first"] = duplicate[1]["uids"][0] + duplicate[1]["uids_sha256"] = chaosnli_adapter._uids_sha256( + duplicate[1]["uids"] + ) + with self.assertRaisesRegex( + chaosnli_adapter.ManifestError, "duplicate uid" + ): + chaosnli_adapter.aggregate_shard_manifests(duplicate, source_uids) + + gap = copy.deepcopy(manifests) + gap[1]["uids"][0] = "synthetic-missing-uid" + gap[1]["uid_first"] = gap[1]["uids"][0] + gap[1]["uids_sha256"] = chaosnli_adapter._uids_sha256(gap[1]["uids"]) + with self.assertRaisesRegex( + chaosnli_adapter.ManifestError, "gap, overlap, or order change" + ): + chaosnli_adapter.aggregate_shard_manifests(gap, source_uids) + + def test_offset_limit_matches_shard_math_and_manifests_are_deterministic(self): + with tempfile.TemporaryDirectory(prefix="groundless-chaosnli-ranges-") as temp: + temp_path = Path(temp) + for index, (offset, limit) in enumerate(((0, 2), (2, 2), (4, 3))): + shard = temp_path / "shard-{}.json".format(index) + ranged = temp_path / "range-{}.json".format(index) + subprocess.run( + [ + sys.executable, + str(self.ADAPTER), + str(self.FIXTURE), + "--shard-index", + str(index), + "--shard-count", + "3", + "--out", + str(shard), + ], + check=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + ) + subprocess.run( + [ + sys.executable, + str(self.ADAPTER), + str(self.FIXTURE), + "--offset", + str(offset), + "--limit", + str(limit), + "--out", + str(ranged), + ], + check=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + ) + self.assertEqual(shard.read_bytes(), ranged.read_bytes()) + + first = temp_path / "first.json" + second = temp_path / "second.json" + for output in (first, second): + subprocess.run( + [ + sys.executable, + str(self.ADAPTER), + str(self.FIXTURE), + "--shard-index", + "2", + "--shard-count", + "3", + "--out", + str(output), + ], + check=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + ) + self.assertEqual( + Path(str(first) + ".manifest.json").read_bytes(), + Path(str(second) + ".manifest.json").read_bytes(), + ) + + def test_distribution_verifier_rejects_a_broken_converted_record(self): + records = chaosnli_adapter.load_records(str(self.FIXTURE)) + dataset = chaosnli_adapter.convert(records) + first_votes = dataset["items"][0]["labels"]["nli_label"] + first_annotator = next(iter(first_votes)) + first_votes[first_annotator] = "n" + with self.assertRaisesRegex(ValueError, "distribution mismatch"): + chaosnli_adapter.verify_converted_distributions(records, dataset) + def test_bad_counter_and_duplicate_uid_are_rejected(self): source = _load_jsonl(self.FIXTURE) cases = [] From b6b9914552d7200c6dc8ffdec193a6cc390e3e9b Mon Sep 17 00:00:00 2001 From: ABrandes <79119643+AMBRA7592@users.noreply.github.com> Date: Sat, 18 Jul 2026 07:18:46 +0200 Subject: [PATCH 2/3] Performance: scale reliability and cohort lookup --- disagreement.py | 28 ++++++++-- test_claims.py | 144 ++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 166 insertions(+), 6 deletions(-) diff --git a/disagreement.py b/disagreement.py index d231b40..eb13438 100644 --- a/disagreement.py +++ b/disagreement.py @@ -270,8 +270,17 @@ def unique_plurality(counts): return winners[0] if len(winners) == 1 else None -def cohort_of(ann, cohorts): - return next((name for name, m in cohorts.items() if ann in m), None) +def cohort_index(cohorts): + """Annotator -> first declared cohort, preserving the prior scan semantics.""" + by_annotator = {} + for name, members in cohorts.items(): + for annotator in members: + by_annotator.setdefault(annotator, name) + return by_annotator + + +def cohort_of(ann, by_annotator): + return by_annotator.get(ann) # --------------------------------------------------------------------------- # @@ -332,8 +341,12 @@ def reliability(items, struct_by_cell, cohorts): for q, votes in it["labels"].items(): if (it["id"], q) not in confident_cells: continue + full = Counter(votes.values()) for ann, lab in votes.items(): - others = Counter(v for a, v in votes.items() if a != ann) + others = full.copy() + others[lab] -= 1 + if not others[lab]: + del others[lab] if not others: continue top = max(others.values()) @@ -346,14 +359,14 @@ def reliability(items, struct_by_cell, cohorts): # --------------------------------------------------------------------------- # # pass 3: classify dissents + final verdict # --------------------------------------------------------------------------- # -def classify(s, votes, rel, cohorts): +def classify(s, votes, rel, by_annotator): maj = s["majority_vote"] dissents = [] for ann, lab in votes.items(): if str(lab) == maj and not s["tie"]: continue shared = s["counts"][lab] - coh = cohort_of(ann, cohorts) + coh = cohort_of(ann, by_annotator) cohort_backed = s["cohort_majorities"].get(coh) == str(lab) and s["value_fork"] if shared >= 2 or cohort_backed: kind = "variation" # coherent bloc -> signal @@ -387,6 +400,7 @@ def main(argv=None): os.makedirs(out_dir, exist_ok=True) ds = load_dataset(data_path) items, cohorts, questions = ds["items"], ds["cohorts"], ds["questions"] + by_annotator = cohort_index(cohorts) struct_by_cell = {(it["id"], q): structure(it, q, it["labels"][q], cohorts) for it in items for q in questions} @@ -396,7 +410,9 @@ def main(argv=None): for it in items: for q in questions: s = struct_by_cell[(it["id"], q)] - verdict, manufactured, dissents = classify(s, it["labels"][q], rel, cohorts) + verdict, manufactured, dissents = classify( + s, it["labels"][q], rel, by_annotator + ) s.update(verdict=verdict, manufactured_consensus=manufactured, dissents=dissents) cells.append(s) diff --git a/test_claims.py b/test_claims.py index fc90dd4..3ebe21b 100644 --- a/test_claims.py +++ b/test_claims.py @@ -15,6 +15,7 @@ import hashlib import json import math +import random import shutil import subprocess import sys @@ -25,6 +26,7 @@ import aggregation import bayes_optimal +import disagreement import frustration import geometry import resolution @@ -208,6 +210,148 @@ def test_bill_and_confident_only_reliability(self): self.assertAlmostEqual(triage["reliability"][annotator], expected, places=12) self.assertIn("CONFIDENT cells only", pipeline["output"]["disagreement.py"]) + def test_optimized_reliability_and_cohort_lookup_match_naive_references(self): + def naive_cohort_of(annotator, cohorts): + return next( + (name for name, members in cohorts.items() if annotator in members), + None, + ) + + def naive_reliability(items, struct_by_cell, cohorts): + confident_cells = { + (cell["item"], cell["question"]) + for cell in struct_by_cell.values() + if not cell["value_fork"] + and not cell["no_ground"] + and not cell["structured"] + and cell["majority_share"] >= disagreement.NEAR_CONSENSUS + } + hits, total = Counter(), Counter() + for item in items: + for question, votes in item["labels"].items(): + if (item["id"], question) not in confident_cells: + continue + for annotator, label in votes.items(): + others = Counter( + vote + for other_annotator, vote in votes.items() + if other_annotator != annotator + ) + if not others: + continue + top = max(others.values()) + modes = { + candidate + for candidate, count in others.items() + if count == top + } + total[annotator] += 1 + hits[annotator] += label in modes + return { + annotator: ( + hits[annotator] / total[annotator] + if total[annotator] + else 1.0 + ) + for annotator in total + } + + forced_cohorts = { + "first": ["ann-0", "ann-1"], + "second": ["ann-0", "ann-2"], + } + forced_index = disagreement.cohort_index(forced_cohorts) + self.assertEqual(forced_index["ann-0"], "first") + for annotator in ("ann-0", "ann-1", "ann-2", "unknown"): + self.assertEqual( + disagreement.cohort_of(annotator, forced_index), + naive_cohort_of(annotator, forced_cohorts), + ) + + forced_items = [ + { + "id": "forced", + "labels": { + "three_way_tie": { + "ann-0": "x", + "ann-1": "y", + "ann-2": "z", + }, + "singleton": {"ann-0": "x"}, + "repeated": { + "ann-0": "x", + "ann-1": "x", + "ann-2": "y", + }, + }, + } + ] + forced_structures = { + ("forced", question): { + "item": "forced", + "question": question, + "value_fork": False, + "no_ground": False, + "structured": False, + "majority_share": 1.0, + } + for question in forced_items[0]["labels"] + } + self.assertEqual( + disagreement.reliability( + forced_items, forced_structures, forced_cohorts + ), + naive_reliability(forced_items, forced_structures, forced_cohorts), + ) + + rng = random.Random(8675309) + labels = ("x", "y", "z", "w") + for trial in range(250): + annotators = [ + "trial-{}-ann-{}".format(trial, index) + for index in range(rng.randint(1, 10)) + ] + cohorts = {} + for cohort_number in range(rng.randint(1, 4)): + members = [ + annotator + for annotator in annotators + if rng.random() < 0.65 + ] + if not members: + members = [rng.choice(annotators)] + cohorts["cohort-{}".format(cohort_number)] = members + optimized_index = disagreement.cohort_index(cohorts) + for annotator in annotators + ["unknown"]: + self.assertEqual( + disagreement.cohort_of(annotator, optimized_index), + naive_cohort_of(annotator, cohorts), + ) + + item = {"id": "trial-{}".format(trial), "labels": {}} + structures = {} + for question_number in range(rng.randint(1, 5)): + question = "q{}".format(question_number) + voters = rng.sample( + annotators, rng.randint(1, len(annotators)) + ) + item["labels"][question] = { + annotator: rng.choice(labels) for annotator in voters + } + confident = rng.random() < 0.75 + structures[(item["id"], question)] = { + "item": item["id"], + "question": question, + "value_fork": False, + "no_ground": False, + "structured": False, + "majority_share": 1.0 if confident else 0.0, + } + self.assertEqual( + disagreement.reliability([item], structures, cohorts), + naive_reliability([item], structures, cohorts), + ) + def test_bill_and_reliability_are_pinned_in_the_essay(self): triage = _pipeline()["triage"] cells = triage["cells"] From cf3909a68a760764b55bf78519b85e51f27acada Mon Sep 17 00:00:00 2001 From: ABrandes <79119643+AMBRA7592@users.noreply.github.com> Date: Sat, 18 Jul 2026 07:21:25 +0200 Subject: [PATCH 3/3] Reports: record official ChaosNLI SNLI run --- reports/chaosnli/README.md | 9 ++- reports/chaosnli/manifest.json | 143 +++++++++++++++++++++++++++++++++ reports/chaosnli/report.md | 62 ++++++++++++++ reports/chaosnli/run_report.py | 6 +- 4 files changed, 215 insertions(+), 5 deletions(-) create mode 100644 reports/chaosnli/manifest.json create mode 100644 reports/chaosnli/report.md diff --git a/reports/chaosnli/README.md b/reports/chaosnli/README.md index f7bbfd4..b9eca9c 100644 --- a/reports/chaosnli/README.md +++ b/reports/chaosnli/README.md @@ -16,16 +16,19 @@ output/UID-list hashes. [official ChaosNLI repository](https://github.com/easonnie/ChaosNLI) and extract it outside this repository. 2. Confirm the split's SHA-256 equals the value in `manifest.json`. -3. From the repository root, with `jsonschema` installed, run: +3. Run from the exact tool commit recorded in the manifest + (`b6b9914552d7200c6dc8ffdec193a6cc390e3e9b`), preferably in a detached + worktree so your current checkout is untouched. +4. From that worktree's root, with `jsonschema` installed, run: ```bash python3 reports/chaosnli/run_report.py \ /path/to/chaosNLI_snli.jsonl \ - --expected-source-sha256 SHA256_FROM_MANIFEST \ + --expected-source-sha256 99f9015ddda7d85f66a087452bc30d53974314fe27e7d589e2f41ad44bd509c1 \ --shard-count 16 ``` -4. Compare the regenerated deterministic `manifest.json` with the committed +5. Compare the regenerated deterministic `manifest.json` with the committed one. Runtime and memory belong only to the human-readable report because they vary by machine. diff --git a/reports/chaosnli/manifest.json b/reports/chaosnli/manifest.json new file mode 100644 index 0000000..26c7220 --- /dev/null +++ b/reports/chaosnli/manifest.json @@ -0,0 +1,143 @@ +{ + "coverage": { + "ordered_complete_unique": true, + "record_count": 1514, + "uids_sha256": "3adeec4e4753665a6655e3e8e1175caa2c56f838f2b02e106331309fb60407c6" + }, + "dataset": "chaosNLI_snli.jsonl", + "evidence_tier": "Tier 2 - manifest-reproducible external empirical run", + "label_set": [ + "c", + "e", + "n" + ], + "shard_count": 16, + "shards": [ + { + "output_sha256": "4e6518f1566f2bd335f3c42fdf8bc101da174eb2efc3086b0b3d6a44ded3f158", + "record_count": 94, + "shard_index": 0, + "uids_sha256": "3cfba32a86652349bbecd14c242ea7004d01b7344e3be7f62b161a45da0cbcc9", + "vote_count": 9400 + }, + { + "output_sha256": "028198a06ddc2d79415902b1ccc6d706ad74d15efb2a170aa5779668fc0aa433", + "record_count": 95, + "shard_index": 1, + "uids_sha256": "721f8072ea16efb53d7afadb3938315f2eb6599654fad1cb98b69fad0756a3d2", + "vote_count": 9500 + }, + { + "output_sha256": "cd440dbfd38a01a8c3cbdb13730b0ba3f234afc0395041e87dfde7ae84352793", + "record_count": 94, + "shard_index": 2, + "uids_sha256": "48b0b73d902b23b2e9d42f2bd032fd7f0903192faca53d654cd70c273672ee36", + "vote_count": 9400 + }, + { + "output_sha256": "f97bf99333e3dfa3f672ac906e1cf38b60be6e5913a52357077d3bfa07d872f8", + "record_count": 95, + "shard_index": 3, + "uids_sha256": "ac36c339b3e4418dd5c7067b519183ddda6ff3a63b567afedc90cfbd02b7267d", + "vote_count": 9500 + }, + { + "output_sha256": "2e8d7b99f7e6f444107b4098eefc2090ae4b7817e872b063e9b1e4436f5cfc9d", + "record_count": 95, + "shard_index": 4, + "uids_sha256": "756a009fbdbf043ac1c043ceecdd9c4ff2ae88306a0615c9e96b3c9405cf795d", + "vote_count": 9500 + }, + { + "output_sha256": "41aeec4e248c3fdb51ec17159b8477a56db0239fe0e7531bf20204fb510e4305", + "record_count": 94, + "shard_index": 5, + "uids_sha256": "18b2b1d869a8e0ce414214317baf47b23ecf62c944f8c7d451363a6e90273d6a", + "vote_count": 9400 + }, + { + "output_sha256": "7989bf873a0a8b72442007edf2ecf0e14c8ddf5e1b82f14a8998f9ea72707690", + "record_count": 95, + "shard_index": 6, + "uids_sha256": "dd10e82d5a71f014c4149a52568f5b5a9388582b561d0bf2ce8ae09767c3c58b", + "vote_count": 9500 + }, + { + "output_sha256": "94126c8d58afb1939e1f360809c4053c5344e7055cbf9d8162b253494d664c89", + "record_count": 95, + "shard_index": 7, + "uids_sha256": "6c2f54785eff99df7b7b051101060fe29c8ce0d71ed72b3ce2ccf7c1627ab16a", + "vote_count": 9500 + }, + { + "output_sha256": "874cd6ac8aa3d18578ef25938726b6a8b142ff60a43c10ae697bb67a1e64a144", + "record_count": 94, + "shard_index": 8, + "uids_sha256": "fd2c15edfdcdf63355bd243cde3398691204d9c5b2010ea48a506dd228a10f19", + "vote_count": 9400 + }, + { + "output_sha256": "f2109912e3ec2c982cacd41e15f0307191f60b3b5dd56290263b01cce448b233", + "record_count": 95, + "shard_index": 9, + "uids_sha256": "413196a94029e9b64b311fc514a1396c3c837d125102c95d95ca1b50ce906437", + "vote_count": 9500 + }, + { + "output_sha256": "6e83b5b0a3d9c824042f4035d3fa1efa5cebf62a8705bb305d1efd809e990cfa", + "record_count": 94, + "shard_index": 10, + "uids_sha256": "6c545958cf154e9e7131bdb982a89de447b67385f8b4403b50389bc1d57e1edb", + "vote_count": 9400 + }, + { + "output_sha256": "ae3cc87986ad55a4979701af6d4d8715a2ddc80735adeea089fcddd1a26f5917", + "record_count": 95, + "shard_index": 11, + "uids_sha256": "274aab5c10cb9a4cf4df1d4cc9ab1de5aeef7adc22cca68307f6ed9c0089ea45", + "vote_count": 9500 + }, + { + "output_sha256": "921fcf99bf3cea51c852195b0cf250a53c385a290cefab1fdd6ec218b85420c5", + "record_count": 95, + "shard_index": 12, + "uids_sha256": "cc213c2cd086d5140a91d3cc9c564239940580be0ac2512fd949625494987d6e", + "vote_count": 9500 + }, + { + "output_sha256": "1fd28c3ac4f1b446fd173e2dc001588318515d400f155a6cbe3a6b2c2a22368e", + "record_count": 94, + "shard_index": 13, + "uids_sha256": "4139cc5e78d4cd4534e5fb0fe94901a4f5a564c47cbf39405a208296c7340fc7", + "vote_count": 9400 + }, + { + "output_sha256": "ac0604423ea7bd757f4fc677104945e49d736b99ce7f60d08994dabbf61f030d", + "record_count": 95, + "shard_index": 14, + "uids_sha256": "586bc4badbaf8889f93eb64bca2da392bc109218f6840b2385ae2f8a871d108a", + "vote_count": 9500 + }, + { + "output_sha256": "c7adc7ced562e04aa823fe6ef3fc3e92cbc49ee41f8c50cb89efed5fe3d32da9", + "record_count": 95, + "shard_index": 15, + "uids_sha256": "6cdce5a80fc6e3deb144aa209f5276f9155e77fd338dd328bd15efbeb0b1b1ac", + "vote_count": 9500 + } + ], + "source_license": "CC BY-NC 4.0", + "source_sha256": "99f9015ddda7d85f66a087452bc30d53974314fe27e7d589e2f41ad44bd509c1", + "source_url": "https://www.dropbox.com/s/h4j7dqszmpt2679/chaosNLI_v1.0.zip", + "source_version": "ChaosNLI v1.0", + "tool_commit": "b6b9914552d7200c6dc8ffdec193a6cc390e3e9b", + "total_records": 1514, + "verification": { + "distribution_matches": 1514, + "entropy_matches": 1514, + "pipeline_complete_shards": 16, + "schema_valid_shards": 16, + "soft_label_matches": 1514 + }, + "vote_count": 151400 +} diff --git a/reports/chaosnli/report.md b/reports/chaosnli/report.md new file mode 100644 index 0000000..9949f6d --- /dev/null +++ b/reports/chaosnli/report.md @@ -0,0 +1,62 @@ +# ChaosNLI real-split reproducibility report + +**Evidence tier:** Tier 2 — manifest-reproducible external empirical evidence. + +**Generated:** 2026-07-18T05:19:46Z + +**Tool commit:** `b6b9914552d7200c6dc8ffdec193a6cc390e3e9b` + +This report records one run on a separately downloaded official split. No +ChaosNLI rows, converted labels, or virtual-voter records are committed here. +The CI-pinned Tier-1 tests exercise the same sharding, coverage, distribution, +and manifest logic on a synthetic fixture. + +## Source and coverage + +- Source: `chaosNLI_snli.jsonl` (ChaosNLI v1.0) +- Official download: https://www.dropbox.com/s/h4j7dqszmpt2679/chaosNLI_v1.0.zip +- Dataset license: CC BY-NC 4.0 +- Source SHA-256: `99f9015ddda7d85f66a087452bc30d53974314fe27e7d589e2f41ad44bd509c1` +- Committed manifest SHA-256: `1406173ca39cdf9721d0048bcbafd52f334319280fe6d7fac513d251d23672f3` +- Records: 1,514 +- Anonymous votes represented: 151,400 +- Shards: 16 +- Coverage: every source UID appears exactly once, in source order; no gaps, + overlaps, duplicates, or missing shard indices + +## Verification results + +- Converted vote distributions exactly matching source `label_counter`: + **1,514/1,514** +- Trainer soft labels exactly matching `label_counter / sum(label_counter)`: + **1,514/1,514** +- Recomputed entropy matching triage output at its documented 3-decimal + precision: **1,514/1,514** +- Input-schema validation: **16/16 shards** +- Operational pipeline completion (`disagreement` → `soft_labels` → + `resolution`): **16/16 shards** +- Source entropy range: -0.000000 to 1.583069 bits; mean + 0.798014 bits + +## Runtime and resources + +- Platform: macOS-15.6.1-arm64-arm-64bit-Mach-O; Python 3.14.2 +- End-to-end wall time: 15.362 seconds +- Sum of measured adapter + pipeline subprocess times: 8.176 seconds +- Peak child-process resident memory: 40.50 MiB +- Peak harness-tracked Python memory: 14.09 MiB +- Converted and pipeline work-directory size: 629.12 MiB +- Pipeline artifacts alone: 606.54 MiB + +Runtime and memory are observations from this machine, not deterministic claims; +the committed JSON manifest intentionally contains only reproducibility checks, +hashes, and counts. + +## Non-applicable findings + +**Reliability, error attribution, variation-vs-error classification, and +cohort value-fork prevalence are NON-APPLICABLE — undefined here, not zero.** +ChaosNLI exposes anonymous per-item counts rather than stable cross-item +annotator identities, and this faithful conversion uses one crowd cohort. +Accordingly, this report makes claims only about distributions, entropy, soft +labels, coverage, and reproducibility. diff --git a/reports/chaosnli/run_report.py b/reports/chaosnli/run_report.py index d482d08..2d3c9df 100644 --- a/reports/chaosnli/run_report.py +++ b/reports/chaosnli/run_report.py @@ -289,8 +289,10 @@ def main(argv=None): ) report = """# ChaosNLI real-split reproducibility report -**Evidence tier:** Tier 2 — manifest-reproducible external empirical evidence. -**Generated:** {generated_at} +**Evidence tier:** Tier 2 — manifest-reproducible external empirical evidence. + +**Generated:** {generated_at} + **Tool commit:** `{tool_commit}` This report records one run on a separately downloaded official split. No