From e18c032a012be87fa0ff0d9a4f2d5fd8f03df056 Mon Sep 17 00:00:00 2001 From: mstrathman Date: Tue, 28 Jul 2026 17:47:00 -0700 Subject: [PATCH 01/11] feat: agent skills for usage, receipts, distillation, and backtesting Four agentskills.io-format skills in skills/: sqlite-predict (core usage: operation selection, the aggregate convention, statuses vs errors, defaults), prediction-receipts (agent-owned provenance: a _predict_receipts table convention, hashing the result document, replay verification; possible because serving is deterministic and models are content-hashed), distill-lifecycle (verify holdout before serving, drift and re-distill cadence, license discipline), and interpret-backtest (MASE against the naive floor, coverage honesty, conformal judgment, when to narrow auto's pool). README gains an agent skills pointer. Skills carry judgment, not wrappers: the SQL surface is already the API. Co-Authored-By: Claude Fable 5 --- README.md | 11 ++++ skills/distill-lifecycle/SKILL.md | 75 ++++++++++++++++++++++++++ skills/interpret-backtest/SKILL.md | 72 +++++++++++++++++++++++++ skills/prediction-receipts/SKILL.md | 81 ++++++++++++++++++++++++++++ skills/sqlite-predict/SKILL.md | 83 +++++++++++++++++++++++++++++ 5 files changed, 322 insertions(+) create mode 100644 skills/distill-lifecycle/SKILL.md create mode 100644 skills/interpret-backtest/SKILL.md create mode 100644 skills/prediction-receipts/SKILL.md create mode 100644 skills/sqlite-predict/SKILL.md diff --git a/README.md b/README.md index 79e7775..e27ca53 100644 --- a/README.md +++ b/README.md @@ -139,6 +139,17 @@ permissively licensed (MIT/Apache-2.0), so you ship it inside your product rather than rent it. And it composes with the rest of the in-database AI toolbox: sqlite-vec gave SQLite vector search; this gives it prediction. +### Agent skills + +For agents, the SQL surface is the whole API, so what ships in +[`skills/`](skills/) is judgment, not wrappers: four +[agentskills.io](https://agentskills.io)-format skills covering core +usage, provenance receipts (record what was asked, which model answered, +and a hash that replays; serving determinism makes this verifiable), +the distillation lifecycle (verify before serving, re-distill on drift, +license discipline), and backtest interpretation. Install them into any +skills-capable agent, or read them as the condensed operator's manual. + ## Operations | Function | Question | Returns | diff --git a/skills/distill-lifecycle/SKILL.md b/skills/distill-lifecycle/SKILL.md new file mode 100644 index 0000000..05bcffa --- /dev/null +++ b/skills/distill-lifecycle/SKILL.md @@ -0,0 +1,75 @@ +--- +name: distill-lifecycle +description: >- + Distill a teacher model into a native sqlite-predict student and manage + it over time: verify quality before serving, re-distill on drift, and + respect the teacher's license. Use when per-call serving needs to be + instant and self-contained, when a prediction runs at volume, or when a + model must travel inside the database file. +metadata: + version: 0.1.0 +--- + +# The distillation lifecycle + +Distillation compresses a teacher (your own labels, your existing model's +predictions, or a licensed foundation model) into a student a few +kilobytes big that serves in microseconds in the zero-dependency core. +The fit takes seconds; the lifecycle judgment is yours. + +## Distill + +```sql +-- tabular: target column holds labels or your model's predictions +SELECT model_id, holdout_metric FROM distill_predict( + 'SELECT f1, f2, label FROM training', + '{"target":"label","student_id":"churn-v1","student_kind":"gbt"}'); + +-- time series: from windows, or with a registered onnx teacher +SELECT model_id FROM distill_forecast('SELECT series_key, value FROM obs', + '{"context":96,"horizon":24,"student_id":"traffic-v1"}'); +``` + +Students register in `_predict_models` as content-hashed rows; they +snapshot, fork, and sync with the database. Serve by name: +`predict(NULL, apply_sql, '{"model":"churn-v1"}')` or +`forecast(ts, value, 24, '{"model":"traffic-v1"}')`. + +## Verify before serving + +Never serve a student on faith: + +1. Read `holdout_metric` from the distill call. It is measured on a + holdout of your data. Compare it to a floor you trust (the majority + class, last-value carry-forward, or your current model). +2. For forecast students, run `backtest` on the same series and compare + the student against `theta-classic` and the naive floor (see the + interpret-backtest skill). +3. Know the measured shape of distillation loss: on our benchmark suite + classification students give up a median half point of accuracy + against their teacher, but regression tails are worse. Check + regression students more skeptically. +4. Soft-label distillation (`proba`/`classes`) preserves the teacher's + calibration and rescues heavily imbalanced datasets, where hard labels + can collapse to one class (the distiller refuses loudly when they do). + +## Watch for drift, re-distill cheaply + +A student is frozen; the world is not. When input distributions move, +quality decays silently, so schedule verification rather than assuming: + +- Re-run `backtest` (forecast) or score a fresh labeled sample (tabular) + on a cadence proportional to how fast the data changes. +- Re-distilling costs seconds. Register the new student under a + versioned id (`churn-v2`), verify, then switch the serving call. Keep + the old row until the new one is trusted; retire it after. + +## Licensing is part of the lifecycle + +A student derives from its teacher, and the teacher's license travels +with it. Your own labels and models carry no limits. Foundation-model +teachers vary by vendor and version, and restrictively licensed ones +require an explicit `accept_license` opt-in before they will run. Read +the license of the exact weights you use before distilling for anything +beyond evaluation; the project documentation's license notes are +orientation, not legal advice. diff --git a/skills/interpret-backtest/SKILL.md b/skills/interpret-backtest/SKILL.md new file mode 100644 index 0000000..f76b445 --- /dev/null +++ b/skills/interpret-backtest/SKILL.md @@ -0,0 +1,72 @@ +--- +name: interpret-backtest +description: >- + Read sqlite-predict backtest() output to choose models, trust or + distrust prediction intervals, and decide when auto-selection needs + narrowing. Use before serving forecasts that feed decisions, when + intervals matter, or when choosing between models on a specific series. +metadata: + version: 0.1.0 +--- + +# Interpreting backtest() + +`backtest` answers "how would this model have done on my series" with +rolling-origin evaluation: it repeatedly hides the tail of the series, +forecasts it, and scores against what actually happened. + +```sql +SELECT * FROM backtest('SELECT ts, value FROM readings', 24, + '{"model":"theta-classic","folds":5}'); +``` + +## The metrics that matter + +- **MASE** is the headline: error relative to the seasonal-naive floor. + Below 1.0 beats naive; above 1.0 means the model is losing to + last-season carry-forward and you should not serve it on this series. + Compare models by MASE on the same series and folds. +- **Coverage** is the fraction of actuals that landed inside the + prediction interval. Compare it to the confidence level you asked for: + a 0.90 band with 0.60 measured coverage is lying to you. +- Per-fold rows expose stability. A model that wins on average but + swings wildly across folds is riskier than a slightly worse, steady + one. + +## Interval judgment + +The default Gaussian band is overconfident on smooth series (measured as +low as 0.57 coverage at a nominal 0.90 on our benchmarks). When interval +truth matters: + +```sql +'{"interval_method":"conformal"}' +``` + +Conformal intervals calibrate to measured residuals and land at the +nominal level, at the cost of needing enough folds to calibrate +(statistical models only; a short series can make it refuse). Always +verify the choice with `backtest` coverage on your own series rather +than trusting either method's reputation. + +## Auto-selection judgment + +Bare `forecast(ts, value, h)` lets `auto` pick per series by rolling- +origin MASE. Use `backtest` when you want to see what auto sees: + +- If one model wins consistently across your series, pin it + (`'{"model":"theta-classic"}'`) and save the selection cost. +- If the pool is polluted (a distilled student trained for a different + regime keeps winning on stale patterns), narrow it: + `'{"candidates":["theta-classic","tsb"]}'`. +- Intermittent series (many zeros, sporadic demand) are `tsb` territory; + if auto is not picking it, check whether the series reaches it and + consider pinning. + +## Reading degraded outcomes + +`backtest` needs enough history for its folds: expect loud errors or +degraded statuses on short series rather than fabricated confidence. +That is the tool working. A series too short to backtest is a series too +short to trust a model on; fall back to wider intervals and say so in +whatever the forecast feeds. diff --git a/skills/prediction-receipts/SKILL.md b/skills/prediction-receipts/SKILL.md new file mode 100644 index 0000000..442a61f --- /dev/null +++ b/skills/prediction-receipts/SKILL.md @@ -0,0 +1,81 @@ +--- +name: prediction-receipts +description: >- + Record verifiable provenance for predictions made with sqlite-predict: + what was asked, which model answered, and a hash that lets the result + be replayed and checked later. Use when a prediction feeds a decision + worth auditing, when results are shared with other agents or humans, + or when you need to prove a result was not altered after the fact. +metadata: + version: 0.1.0 +--- + +# Prediction receipts + +sqlite-predict serving is deterministic and pure: the same rows, options, +and model produce byte-identical result documents, and registered model +weights are pinned by content hash. That makes provenance a recording +problem the agent can own, with no support needed from the extension. + +Record receipts for predictions that matter (decisions, reports, handoffs), +not for every exploratory call. + +## The receipts table + +Create it in your own database (or a sidecar). Convention, so receipts +from different agents interoperate: + +```sql +CREATE TABLE IF NOT EXISTS _predict_receipts ( + id INTEGER PRIMARY KEY, + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')), + operation TEXT NOT NULL, -- 'forecast' | 'detect_anomalies' | 'predict' | 'backtest' + input_sql TEXT NOT NULL, -- the exact SQL that supplied the rows + options TEXT, -- the options JSON as passed, or NULL + model_id TEXT NOT NULL, -- the "model" field of the result document + content_hash TEXT, -- registry hash for a registered model, else NULL + extension TEXT NOT NULL, -- "extension" field of predict_version() + result_sha256 TEXT NOT NULL -- SHA-256 of the exact result document text +); +``` + +## Writing a receipt + +1. Run the prediction and keep the raw result document text (the JSON the + aggregate returned, or the concatenated rows for a TVF, in a fixed + order). +2. `model_id` comes from the document's `model` field. For a registered + model or student, read its pin: + `SELECT content_hash FROM _predict_models WHERE model_id = ?`. +3. Hash the exact document text with SHA-256 (any host language; SQLite + itself has no sha256 built in). +4. Insert the row. Store the document itself wherever the result is used; + the receipt stores its hash, not the data. + +## Replaying a receipt + +To verify a receipt later: re-run `input_sql` through the same operation +with the same `options`, hash the new document, and compare to +`result_sha256`. Three outcomes: + +- Hashes match: the result reproduces. With an unchanged `content_hash`, + the same model on the same data gave the same answer. +- Hashes differ and the underlying table changed: the data moved, which + is normal; the receipt documents what was true at `created_at`. +- Hashes differ on unchanged data: investigate. Check the extension + version (`predict_version()` vs the receipt's `extension`) and the + model's `content_hash` (tampered weights fail loudly at load with + `PREDICT_ERR_MODEL_HASH`, so a swap cannot hide). + +## Honest limits + +- Replay assumes the input rows are reproducible. If the table mutates, + snapshot the database (or record row counts and a data hash alongside) + when the receipt matters enough. +- Determinism is per build for the statistical models and native + students. ONNX-served models are deterministic on the same CPU build + but not bit-identical across machines or GPU providers; record the + device if you serve through ONNX and intend to replay. +- A receipt proves what was computed, not that the prediction was good. + Pair with `backtest` (see interpret-backtest) when quality claims + matter. diff --git a/skills/sqlite-predict/SKILL.md b/skills/sqlite-predict/SKILL.md new file mode 100644 index 0000000..70dfd05 --- /dev/null +++ b/skills/sqlite-predict/SKILL.md @@ -0,0 +1,83 @@ +--- +name: sqlite-predict +description: >- + Forecast, detect anomalies, and predict on tabular data directly in a + SQLite database using the sqlite-predict extension. Use when an agent + holds time series or tabular data in SQLite and needs predictions + without exporting data, calling a cloud model, or building a pipeline. +metadata: + version: 0.1.0 +--- + +# Using sqlite-predict + +The extension turns prediction into SQL. Load `predict0` (pip/npm/cargo +package `sqlite-predict`), then call functions over your own rows. Serving +is pure: no writes, works on read-only databases and inside views. + +## Which operation for which question + +| Question | Call | +| --- | --- | +| "What will this metric do next?" | `SELECT forecast(ts, value, horizon) FROM t` | +| "Which points are anomalous?" | `SELECT detect_anomalies(ts, value) FROM t` | +| "Predict a label/value for these rows" | `SELECT * FROM predict(train_sql, apply_sql, options)` | +| "How accurate would this be on my data?" | `SELECT * FROM backtest(series_sql, horizon)` | +| "Make serving instant and self-contained" | `distill_predict` / `distill_forecast` (see the distill-lifecycle skill) | + +`forecast` and `detect_anomalies` are aggregates: plain SQL supplies the +rows, `GROUP BY` splits series, `WHERE` and joins compose, ORMs work +unchanged. `predict`, `backtest`, and the distillers are table-valued +functions that take a query string. + +## Reading results + +Each aggregate group returns one JSON document: `{"model", "status", +"rows"}`. Expand to typed rows in SQL when needed: + +```sql +SELECT r.* FROM forecast_rows((SELECT forecast(ts, value, 24) FROM t)) AS r; +``` + +- `model` reports the model that actually served. With no model named, + `auto` selects the best available per series and reports the winner's + id, never the string "auto". +- `status` is `ok`, or a degraded status with empty rows: + `insufficient_history` (fewer than 8 points) or `non_numeric` (a + timestamp or value cell poisoned the series). Degraded statuses are + data conditions, not errors; check `status` before using `rows`. +- Zero input rows return SQL NULL, the aggregate convention. + +## Timestamps and values + +Timestamps: ISO-8601 text (`YYYY-MM-DD[ T]HH:MM[:SS][Z]`) or integer +epoch seconds/milliseconds. Anything else poisons the series as +`non_numeric`. Values must be numeric. The aggregate floors timestamps +to whole seconds. + +## Errors are self-explaining + +Every call error is one string from a closed set, formatted +`PREDICT_ERR_: detail` (options, schema, horizon, licensing, +model-not-found, and so on). There are no silent fallbacks: an unknown +option key, a non-read-only inner query, or tampered model weights all +fail loudly. When a call errors, read the message; it names the exact +problem and often the fix. + +## Defaults an agent should know + +- No model named means `auto`: bundled statistical models plus any + eligible distilled students compete per series. Pass + `'{"candidates":["theta-classic","tsb"]}'` to narrow the pool. +- Prediction intervals default to a Gaussian band that is overconfident + on smooth data; pass `'{"interval_method":"conformal"}'` for + calibrated coverage (statistical models only) and verify with + `backtest` (see the interpret-backtest skill). +- `predict` with `NULL` train_query serves a distilled student; with a + train query it runs the in-context `knn5-incontext` model zero-shot. + +## Checking the environment + +`SELECT predict_version()` returns the extension version, compiled +runtimes, and bundled model ids as JSON. Use it to confirm the extension +is loaded and what is available before planning work. From c4899cef28f90ed463b683ae6b606f85ccfd3ba6 Mon Sep 17 00:00:00 2001 From: mstrathman Date: Tue, 28 Jul 2026 17:48:57 -0700 Subject: [PATCH 02/11] chore: spec-polish the skill frontmatter All four validate with the agentskills reference validator (skills-ref). Add the optional license field and quote metadata values per the spec's string-map recommendation. Co-Authored-By: Claude Fable 5 --- skills/distill-lifecycle/SKILL.md | 3 ++- skills/interpret-backtest/SKILL.md | 3 ++- skills/prediction-receipts/SKILL.md | 3 ++- skills/sqlite-predict/SKILL.md | 3 ++- 4 files changed, 8 insertions(+), 4 deletions(-) diff --git a/skills/distill-lifecycle/SKILL.md b/skills/distill-lifecycle/SKILL.md index 05bcffa..6a37352 100644 --- a/skills/distill-lifecycle/SKILL.md +++ b/skills/distill-lifecycle/SKILL.md @@ -6,8 +6,9 @@ description: >- respect the teacher's license. Use when per-call serving needs to be instant and self-contained, when a prediction runs at volume, or when a model must travel inside the database file. +license: MIT OR Apache-2.0 metadata: - version: 0.1.0 + version: "0.1.0" --- # The distillation lifecycle diff --git a/skills/interpret-backtest/SKILL.md b/skills/interpret-backtest/SKILL.md index f76b445..67c87ee 100644 --- a/skills/interpret-backtest/SKILL.md +++ b/skills/interpret-backtest/SKILL.md @@ -5,8 +5,9 @@ description: >- distrust prediction intervals, and decide when auto-selection needs narrowing. Use before serving forecasts that feed decisions, when intervals matter, or when choosing between models on a specific series. +license: MIT OR Apache-2.0 metadata: - version: 0.1.0 + version: "0.1.0" --- # Interpreting backtest() diff --git a/skills/prediction-receipts/SKILL.md b/skills/prediction-receipts/SKILL.md index 442a61f..f7b93e1 100644 --- a/skills/prediction-receipts/SKILL.md +++ b/skills/prediction-receipts/SKILL.md @@ -6,8 +6,9 @@ description: >- be replayed and checked later. Use when a prediction feeds a decision worth auditing, when results are shared with other agents or humans, or when you need to prove a result was not altered after the fact. +license: MIT OR Apache-2.0 metadata: - version: 0.1.0 + version: "0.1.0" --- # Prediction receipts diff --git a/skills/sqlite-predict/SKILL.md b/skills/sqlite-predict/SKILL.md index 70dfd05..be48081 100644 --- a/skills/sqlite-predict/SKILL.md +++ b/skills/sqlite-predict/SKILL.md @@ -5,8 +5,9 @@ description: >- SQLite database using the sqlite-predict extension. Use when an agent holds time series or tabular data in SQLite and needs predictions without exporting data, calling a cloud model, or building a pipeline. +license: MIT OR Apache-2.0 metadata: - version: 0.1.0 + version: "0.1.0" --- # Using sqlite-predict From 7dd6a1530adadad72015cebd7ad1013c751351ff Mon Sep 17 00:00:00 2001 From: mstrathman Date: Tue, 28 Jul 2026 17:51:37 -0700 Subject: [PATCH 03/11] feat: receipts reference script and CI conformance checks skills/prediction-receipts/scripts/receipt.py is the interoperability anchor: one stdlib-only implementation of the convention (record, verify with exit-code semantics, list), with canonicalization defined in one place (aggregate documents hashed verbatim; row results as compact JSON with shortest round-trip floats). The skill now points at it as the preferred path, keeping the hand-rolled steps as the spec. tests/test_skills.py wires both into the normal pytest run: every SKILL.md is held to the agentskills spec (name/dir match, name charset, description bounds and when-to-use, line budget, house dash rule), and the receipt script is proven end to end: record, replay-match, re-record determinism, tamper detection via exit code 2, and registry content_hash pinning for bundled models and distilled students. Co-Authored-By: Claude Fable 5 --- skills/prediction-receipts/SKILL.md | 19 +- skills/prediction-receipts/scripts/receipt.py | 211 ++++++++++++++++++ tests/test_skills.py | 136 +++++++++++ 3 files changed, 365 insertions(+), 1 deletion(-) create mode 100644 skills/prediction-receipts/scripts/receipt.py create mode 100644 tests/test_skills.py diff --git a/skills/prediction-receipts/SKILL.md b/skills/prediction-receipts/SKILL.md index f7b93e1..83be4ab 100644 --- a/skills/prediction-receipts/SKILL.md +++ b/skills/prediction-receipts/SKILL.md @@ -40,7 +40,24 @@ CREATE TABLE IF NOT EXISTS _predict_receipts ( ); ``` -## Writing a receipt +## The reference script + +Prefer the bundled implementation over hand-rolling the steps below, so +receipts stay interoperable across agents: + +``` +scripts/receipt.py record DB "SELECT forecast(ts, value, 24) FROM t" +scripts/receipt.py verify DB RECEIPT_ID # exit 0 match, 2 mismatch +scripts/receipt.py list DB +``` + +It creates the table, canonicalizes the result (the aggregate document +verbatim; rows as compact JSON with shortest round-trip floats), hashes +it, resolves the model's registry pin, and on verify reports what +changed (data, extension version, or model hash). Python stdlib only; +pass the loadable with --extension or SQLITE_PREDICT_EXTENSION. + +## Writing a receipt by hand 1. Run the prediction and keep the raw result document text (the JSON the aggregate returned, or the concatenated rows for a TVF, in a fixed diff --git a/skills/prediction-receipts/scripts/receipt.py b/skills/prediction-receipts/scripts/receipt.py new file mode 100644 index 0000000..1ee2ddb --- /dev/null +++ b/skills/prediction-receipts/scripts/receipt.py @@ -0,0 +1,211 @@ +#!/usr/bin/env python3 +"""Reference implementation of the prediction-receipts convention. + +Records and verifies provenance receipts for sqlite-predict calls, so +every agent that uses this script produces interoperable receipts with +identical canonicalization and hashing. Stdlib only. + +Canonical result document: + - If the prediction SQL returns exactly one row with one TEXT column + (the aggregate document case), the document is that text verbatim. + - Otherwise (predict/backtest rows), the document is compact JSON: + {"columns":[...],"rows":[[...],...]} in cursor order, with floats + serialized by Python's shortest round-trip repr. +The receipt stores sha256(document utf-8), never the document itself. + +Usage: + receipt.py record DB "SELECT forecast(ts, value, 24) FROM t" [--operation forecast] + receipt.py verify DB RECEIPT_ID + receipt.py list DB + Common flag: --extension PATH loadable to use (default: env + SQLITE_PREDICT_EXTENSION, else "predict0" on the library path) +""" + +import argparse +import hashlib +import json +import os +import re +import sqlite3 +import sys + +DDL = """CREATE TABLE IF NOT EXISTS _predict_receipts ( + id INTEGER PRIMARY KEY, + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')), + operation TEXT NOT NULL, + input_sql TEXT NOT NULL, + options TEXT, + model_id TEXT NOT NULL, + content_hash TEXT, + extension TEXT NOT NULL, + result_sha256 TEXT NOT NULL +)""" + +OPERATIONS = ("forecast", "detect_anomalies", "predict", "backtest") + + +def fail(msg): + print(f"receipt.py: {msg}", file=sys.stderr) + sys.exit(1) + + +def connect(db_path, extension): + db = sqlite3.connect(db_path) + db.enable_load_extension(True) + try: + db.load_extension(extension) + except sqlite3.OperationalError as e: + fail(f"cannot load extension '{extension}': {e}") + return db + + +def canonical_document(cur): + rows = cur.fetchall() + cols = [d[0] for d in cur.description] if cur.description else [] + if len(rows) == 1 and len(cols) == 1 and isinstance(rows[0][0], str): + return rows[0][0] + return json.dumps({"columns": cols, "rows": [list(r) for r in rows]}, + separators=(",", ":"), ensure_ascii=False) + + +def sha256_text(text): + return hashlib.sha256(text.encode("utf-8")).hexdigest() + + +def infer_operation(sql): + found = {op for op in OPERATIONS + if re.search(r"\b" + op + r"\s*\(", sql)} + if len(found) == 1: + return found.pop() + return None + + +def extension_version(db): + doc = json.loads(db.execute("SELECT predict_version()").fetchone()[0]) + return doc["extension"] + + +def model_fields(db, document, model_flag): + """The serving model id and, when registered, its content_hash.""" + model_id = model_flag + if model_id is None: + try: + model_id = json.loads(document).get("model") + except (json.JSONDecodeError, AttributeError): + model_id = None + if not model_id: + fail("cannot determine the serving model: the result is not a " + "document with a \"model\" field; pass --model-id explicitly") + content_hash = None + try: + row = db.execute( + "SELECT content_hash FROM _predict_models WHERE model_id = ?", + (model_id,)).fetchone() + content_hash = row[0] if row else None + except sqlite3.OperationalError: + pass # no registry in this database: content_hash stays NULL + return model_id, content_hash + + +def cmd_record(args): + db = connect(args.db, args.extension) + operation = args.operation or infer_operation(args.sql) + if operation is None: + fail("cannot infer the operation from the SQL; pass --operation " + f"one of {', '.join(OPERATIONS)}") + cur = db.execute(args.sql) + document = canonical_document(cur) + model_id, content_hash = model_fields(db, document, args.model_id) + db.execute(DDL) + cur = db.execute( + "INSERT INTO _predict_receipts (operation, input_sql, options," + " model_id, content_hash, extension, result_sha256)" + " VALUES (?,?,?,?,?,?,?)", + (operation, args.sql, args.options, model_id, content_hash, + extension_version(db), sha256_text(document))) + db.commit() + print(json.dumps({"receipt_id": cur.lastrowid, "model_id": model_id, + "content_hash": content_hash, + "result_sha256": sha256_text(document)})) + db.close() + + +def cmd_verify(args): + db = connect(args.db, args.extension) + row = db.execute( + "SELECT input_sql, model_id, content_hash, extension, result_sha256" + " FROM _predict_receipts WHERE id = ?", (args.receipt_id,)).fetchone() + if row is None: + fail(f"no receipt with id {args.receipt_id}") + input_sql, model_id, rec_hash, rec_ext, rec_sha = row + document = canonical_document(db.execute(input_sql)) + now_sha = sha256_text(document) + now_ext = extension_version(db) + now_hash = None + try: + r = db.execute( + "SELECT content_hash FROM _predict_models WHERE model_id = ?", + (model_id,)).fetchone() + now_hash = r[0] if r else None + except sqlite3.OperationalError: + pass + report = { + "receipt_id": args.receipt_id, + "match": now_sha == rec_sha, + "result_sha256": {"recorded": rec_sha, "replayed": now_sha}, + "extension": {"recorded": rec_ext, "current": now_ext, + "changed": rec_ext != now_ext}, + "model": {"id": model_id, + "content_hash_recorded": rec_hash, + "content_hash_current": now_hash, + "changed": rec_hash != now_hash}, + } + print(json.dumps(report, indent=2)) + db.close() + sys.exit(0 if report["match"] else 2) + + +def cmd_list(args): + db = connect(args.db, args.extension) + try: + rows = db.execute( + "SELECT id, created_at, operation, model_id, result_sha256" + " FROM _predict_receipts ORDER BY id").fetchall() + except sqlite3.OperationalError: + rows = [] + for r in rows: + print(f"{r[0]} {r[1]} {r[2]:<16} {r[3]:<24} {r[4][:16]}...") + db.close() + + +def main(): + p = argparse.ArgumentParser(description=__doc__) + p.add_argument("--extension", + default=os.environ.get("SQLITE_PREDICT_EXTENSION", + "predict0")) + sub = p.add_subparsers(dest="cmd", required=True) + + r = sub.add_parser("record", help="run a prediction and record a receipt") + r.add_argument("db") + r.add_argument("sql") + r.add_argument("--operation", choices=OPERATIONS) + r.add_argument("--options", help="options JSON, recorded verbatim") + r.add_argument("--model-id", + help="override when the result carries no model field") + r.set_defaults(fn=cmd_record) + + v = sub.add_parser("verify", help="replay a receipt and compare hashes") + v.add_argument("db") + v.add_argument("receipt_id", type=int) + v.set_defaults(fn=cmd_verify) + + ls = sub.add_parser("list", help="list recorded receipts") + ls.add_argument("db") + ls.set_defaults(fn=cmd_list) + + args = p.parse_args() + args.fn(args) + + +if __name__ == "__main__": + main() diff --git a/tests/test_skills.py b/tests/test_skills.py new file mode 100644 index 0000000..d7d8aab --- /dev/null +++ b/tests/test_skills.py @@ -0,0 +1,136 @@ +"""The skills ship as part of the repo, so CI holds them to the +agentskills.io spec and proves the receipts reference script actually +does what the prediction-receipts skill promises: record, replay-match, +tamper-detect, and pin the model hash.""" + +import json +import os +import re +import sqlite3 +import subprocess +import sys + +import pytest + +ROOT = os.path.join(os.path.dirname(__file__), "..") +SKILLS = os.path.join(ROOT, "skills") +RECEIPT = os.path.join(SKILLS, "prediction-receipts", "scripts", "receipt.py") +EXT = os.path.join(ROOT, "dist", "predict0") + +NAME_RE = re.compile(r"^[a-z0-9]+(-[a-z0-9]+)*$") + + +def skill_dirs(): + return sorted(d for d in os.listdir(SKILLS) + if os.path.isdir(os.path.join(SKILLS, d))) + + +def frontmatter(path): + text = open(path, encoding="utf-8").read() + assert text.startswith("---\n"), f"{path}: missing frontmatter" + fm = text.split("---\n")[1] + return fm, text + + +@pytest.mark.parametrize("d", skill_dirs()) +def test_skill_conforms_to_spec(d): + path = os.path.join(SKILLS, d, "SKILL.md") + fm, text = frontmatter(path) + + m = re.search(r"^name:\s*(\S+)\s*$", fm, re.M) + assert m, "name is required" + name = m.group(1) + assert name == d, "name must match the directory" + assert len(name) <= 64 and NAME_RE.match(name), f"invalid name {name!r}" + + dm = re.search(r"^description:\s*>-\n((?: .+\n)+)", fm, re.M) + assert dm, "description is required (folded block expected)" + desc = " ".join(dm.group(1).split()) + assert 1 <= len(desc) <= 1024, "description must be 1..1024 chars" + assert "when" in desc.lower(), "description should say when to use it" + + assert text.count("\n") <= 500, "SKILL.md should stay under 500 lines" + assert "—" not in text and "–" not in text, \ + "house style: no em or en dashes" + + +def run_receipt(*argv): + return subprocess.run( + [sys.executable, RECEIPT, "--extension", EXT, *argv], + capture_output=True, text=True) + + +@pytest.fixture() +def receipt_db(tmp_path): + path = str(tmp_path / "r.db") + db = sqlite3.connect(path) + db.execute("CREATE TABLE readings(ts INTEGER, value REAL)") + db.executemany("INSERT INTO readings VALUES (?, ?)", + [(1700000000 + i * 3600, 10.0 + i % 7) + for i in range(48)]) + db.commit() + db.close() + return path + + +def test_receipt_record_verify_and_tamper(receipt_db): + sql = "SELECT forecast(ts, value, 6) FROM readings" + rec = run_receipt("record", receipt_db, sql) + assert rec.returncode == 0, rec.stderr + out = json.loads(rec.stdout) + assert out["receipt_id"] == 1 + assert out["model_id"] not in (None, "", "auto") + + ver = run_receipt("verify", receipt_db, "1") + assert ver.returncode == 0, ver.stdout + ver.stderr + report = json.loads(ver.stdout) + assert report["match"] is True + assert report["extension"]["changed"] is False + + # determinism claim: recording again yields the identical hash + rec2 = run_receipt("record", receipt_db, sql) + assert json.loads(rec2.stdout)["result_sha256"] == out["result_sha256"] + + # tamper with the data: replay must mismatch, exit code must say so + db = sqlite3.connect(receipt_db) + db.execute("UPDATE readings SET value = value + 100 WHERE rowid = 5") + db.commit() + db.close() + bad = run_receipt("verify", receipt_db, "1") + assert bad.returncode == 2 + assert json.loads(bad.stdout)["match"] is False + + +def test_receipt_pins_a_registered_student(receipt_db): + db = sqlite3.connect(receipt_db) + db.enable_load_extension(True) + db.load_extension(EXT) + db.execute("CREATE TABLE tab(f1 REAL, f2 REAL, label TEXT)") + db.executemany("INSERT INTO tab VALUES (?,?,?)", + [(i * 0.1, (i % 5) * 1.0, "a" if i % 2 else "b") + for i in range(60)]) + db.execute("SELECT * FROM distill_predict(" + "'SELECT f1, f2, label FROM tab'," + " '{\"target\":\"label\",\"student_id\":\"s1\"}')").fetchall() + db.commit() + db.close() + + rec = run_receipt( + "record", receipt_db, + "SELECT forecast(ts, value, 6, '{\"model\":\"theta-classic\"}')" + " FROM readings") + out = json.loads(rec.stdout) + assert out["model_id"] == "theta-classic" + assert out["content_hash"], "registered models must record their pin" + + # rows-shaped results canonicalize too: predict() through a student + rec2 = run_receipt( + "record", receipt_db, + "SELECT rowid, prediction FROM predict(NULL," + " 'SELECT rowid, f1, f2 FROM tab', '{\"model\":\"s1\"}')", + "--model-id", "s1") + assert rec2.returncode == 0, rec2.stderr + out2 = json.loads(rec2.stdout) + assert out2["content_hash"], "student content_hash must be recorded" + ver = run_receipt("verify", receipt_db, str(out2["receipt_id"])) + assert json.loads(ver.stdout)["match"] is True From 48e14981328d801292b8f60f1082f3c4b36156ae Mon Sep 17 00:00:00 2001 From: mstrathman Date: Tue, 28 Jul 2026 17:59:32 -0700 Subject: [PATCH 04/11] feat: predict_sha256 makes the receipts workflow pure SQL Python was the receipt script's only dependency, and the environments this extension targets (phone, embedded, wasm) may not have it. The lightest dependency is the extension itself: expose the vendored SHA-256 (the hash that already pins model weights) as a deterministic, innocuous predict_sha256(x) scalar (TEXT/BLOB, NULL passthrough), and rewrite the prediction-receipts skill so record and replay-verify run in pure SQL. The Python script stays as optional convenience for row-shaped results (canonical float serialization) and mismatch diagnostics. CI: hash vectors against hashlib, plus a pure-SQL record/verify/tamper round trip; functions reference documents the new scalar. Also gitignores autogluon's local model dumps, which must never be committed. Co-Authored-By: Claude Fable 5 --- .gitignore | 3 + skills/prediction-receipts/SKILL.md | 88 +++++++++++-------- sqlite-predict.c | 35 ++++++++ tests/test_skills.py | 42 +++++++++ .../src/content/docs/reference/functions.md | 1 + 5 files changed, 132 insertions(+), 37 deletions(-) diff --git a/.gitignore b/.gitignore index 4179045..a7c9278 100644 --- a/.gitignore +++ b/.gitignore @@ -57,3 +57,6 @@ bindings/node/node_modules/ bindings/rust/csrc/ bindings/rust/target/ bindings/rust/Cargo.lock + +# autogluon working dirs (Mitra benchmark runs dump model copies here) +benchmarks/AutogluonModels/ diff --git a/skills/prediction-receipts/SKILL.md b/skills/prediction-receipts/SKILL.md index 83be4ab..472eaf7 100644 --- a/skills/prediction-receipts/SKILL.md +++ b/skills/prediction-receipts/SKILL.md @@ -16,7 +16,9 @@ metadata: sqlite-predict serving is deterministic and pure: the same rows, options, and model produce byte-identical result documents, and registered model weights are pinned by content hash. That makes provenance a recording -problem the agent can own, with no support needed from the extension. +problem the agent can own; the only support the extension provides is +the `predict_sha256()` hash utility, so the whole workflow can run in +pure SQL. Record receipts for predictions that matter (decisions, reports, handoffs), not for every exploratory call. @@ -40,10 +42,50 @@ CREATE TABLE IF NOT EXISTS _predict_receipts ( ); ``` -## The reference script +## Recording in pure SQL -Prefer the bundled implementation over hand-rolling the steps below, so -receipts stay interoperable across agents: +The extension exposes `predict_sha256()` (the same hash that pins model +weights), so the whole workflow runs in SQL with no host language: + +```sql +INSERT INTO _predict_receipts (operation, input_sql, options, model_id, + content_hash, extension, result_sha256) +SELECT 'forecast', + 'SELECT forecast(ts, value, 24) FROM readings', + NULL, + json_extract(doc, '$.model'), + (SELECT content_hash FROM _predict_models + WHERE model_id = json_extract(doc, '$.model')), + json_extract(predict_version(), '$.extension'), + predict_sha256(doc) +FROM (SELECT (SELECT forecast(ts, value, 24) FROM readings) AS doc); +``` + +The inner `input_sql` string and the query that computes `doc` must be +the same SQL, verbatim; that is what makes the receipt replayable. If +nothing has been distilled or registered, `_predict_models` does not +exist yet: record `NULL` for `content_hash` instead of the subquery. + +## Verifying in pure SQL + +```sql +SELECT id, + result_sha256 = predict_sha256( + (SELECT forecast(ts, value, 24) FROM readings)) AS match +FROM _predict_receipts WHERE id = 1; +``` + +`match` is 1 when the replay reproduces the recorded result. On 0, +compare the receipt's `extension` against `predict_version()` and its +`content_hash` against the registry to find what moved; unchanged data +with an unchanged model and extension should never mismatch. + +## The reference script (optional) + +For row-shaped results (`predict`, `backtest`) the document needs +canonical serialization, and pure SQL float formatting is not +round-trip safe. Use the bundled script for those, or when you want +mismatch diagnostics computed for you: ``` scripts/receipt.py record DB "SELECT forecast(ts, value, 24) FROM t" @@ -51,39 +93,11 @@ scripts/receipt.py verify DB RECEIPT_ID # exit 0 match, 2 mismatch scripts/receipt.py list DB ``` -It creates the table, canonicalizes the result (the aggregate document -verbatim; rows as compact JSON with shortest round-trip floats), hashes -it, resolves the model's registry pin, and on verify reports what -changed (data, extension version, or model hash). Python stdlib only; -pass the loadable with --extension or SQLITE_PREDICT_EXTENSION. - -## Writing a receipt by hand - -1. Run the prediction and keep the raw result document text (the JSON the - aggregate returned, or the concatenated rows for a TVF, in a fixed - order). -2. `model_id` comes from the document's `model` field. For a registered - model or student, read its pin: - `SELECT content_hash FROM _predict_models WHERE model_id = ?`. -3. Hash the exact document text with SHA-256 (any host language; SQLite - itself has no sha256 built in). -4. Insert the row. Store the document itself wherever the result is used; - the receipt stores its hash, not the data. - -## Replaying a receipt - -To verify a receipt later: re-run `input_sql` through the same operation -with the same `options`, hash the new document, and compare to -`result_sha256`. Three outcomes: - -- Hashes match: the result reproduces. With an unchanged `content_hash`, - the same model on the same data gave the same answer. -- Hashes differ and the underlying table changed: the data moved, which - is normal; the receipt documents what was true at `created_at`. -- Hashes differ on unchanged data: investigate. Check the extension - version (`predict_version()` vs the receipt's `extension`) and the - model's `content_hash` (tampered weights fail loudly at load with - `PREDICT_ERR_MODEL_HASH`, so a swap cannot hide). +Python stdlib only; pass the loadable with --extension or +SQLITE_PREDICT_EXTENSION. It creates the table, canonicalizes (the +aggregate document verbatim; rows as compact JSON with shortest +round-trip floats), resolves the model pin, and reports what changed on +verify. ## Honest limits diff --git a/sqlite-predict.c b/sqlite-predict.c index 2eabd35..5815e08 100644 --- a/sqlite-predict.c +++ b/sqlite-predict.c @@ -420,6 +420,37 @@ static void predict_ulid_fn(sqlite3_context *context, int argc, sqlite3_result_text(context, buf, 26, SQLITE_TRANSIENT); } +/* predict_sha256(X): lowercase hex SHA-256 of a TEXT or BLOB value. + * The same hash that pins model weights (content_hash), exposed so + * agent-layer workflows (provenance receipts, content addressing) can + * run in pure SQL with no host language. NULL in, NULL out. */ +static void predict_sha256_fn(sqlite3_context *context, int argc, + sqlite3_value **argv) { + UNUSED_PARAMETER(argc); + int vt = sqlite3_value_type(argv[0]); + if (vt == SQLITE_NULL) { + sqlite3_result_null(context); + return; + } + const void *data; + int n; + if (vt == SQLITE_BLOB) { + data = sqlite3_value_blob(argv[0]); + n = sqlite3_value_bytes(argv[0]); + } else { /* TEXT and numeric values hash their text form */ + data = sqlite3_value_text(argv[0]); + n = sqlite3_value_bytes(argv[0]); + } + predict0_hasher h; + predict0_hash_init(&h); + if (n > 0 && data) + sha256_update(&h.sha, (const u8 *)data, (usize)n); + char hex[PREDICT_HEX_BUFSIZE]; + predict0_hash_hex(&h, hex); + sqlite3_result_text(context, hex, PREDICT_HEX_BUFSIZE - 1, + SQLITE_TRANSIENT); +} + /* predict_register(model_id, config_json): record an external model in * _predict_models so predict() can dispatch to it. config is a JSON object: * { "runtime":"onnx", "kind":"tabular-fm"|"student"|..., @@ -637,6 +668,10 @@ __declspec(dllexport) predict_ulid_fn, NULL, NULL, NULL); if (rc != SQLITE_OK) return rc; + rc = sqlite3_create_function_v2(db, "predict_sha256", 1, flags, NULL, + predict_sha256_fn, NULL, NULL, NULL); + if (rc != SQLITE_OK) + return rc; /* predict_register mutates the registry, so it is not INNOCUOUS and not * DETERMINISTIC (its content_hash depends on a file read). */ rc = sqlite3_create_function_v2(db, "predict_register", 2, SQLITE_UTF8, diff --git a/tests/test_skills.py b/tests/test_skills.py index d7d8aab..04808ae 100644 --- a/tests/test_skills.py +++ b/tests/test_skills.py @@ -101,6 +101,48 @@ def test_receipt_record_verify_and_tamper(receipt_db): assert json.loads(bad.stdout)["match"] is False +def test_pure_sql_receipt_path(receipt_db): + """The skill's primary path: record and verify with no host language + beyond SQL, using predict_sha256.""" + db = sqlite3.connect(receipt_db) + db.enable_load_extension(True) + db.load_extension(EXT) + import hashlib + assert db.execute("SELECT predict_sha256('abc')").fetchone()[0] == \ + hashlib.sha256(b"abc").hexdigest() + assert db.execute("SELECT predict_sha256(NULL)").fetchone()[0] is None + + db.execute("""CREATE TABLE _predict_receipts ( + id INTEGER PRIMARY KEY, + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')), + operation TEXT NOT NULL, input_sql TEXT NOT NULL, options TEXT, + model_id TEXT NOT NULL, content_hash TEXT, + extension TEXT NOT NULL, result_sha256 TEXT NOT NULL)""") + db.execute(""" + INSERT INTO _predict_receipts (operation, input_sql, options, model_id, + content_hash, extension, result_sha256) + SELECT 'forecast', 'SELECT forecast(ts, value, 6) FROM readings', NULL, + json_extract(doc, '$.model'), NULL, + json_extract(predict_version(), '$.extension'), + predict_sha256(doc) + FROM (SELECT (SELECT forecast(ts, value, 6) FROM readings) AS doc)""") + db.commit() + + (match,) = db.execute(""" + SELECT result_sha256 = predict_sha256( + (SELECT forecast(ts, value, 6) FROM readings)) + FROM _predict_receipts WHERE id = 1""").fetchone() + assert match == 1 + + db.execute("UPDATE readings SET value = value + 100 WHERE rowid = 3") + (match,) = db.execute(""" + SELECT result_sha256 = predict_sha256( + (SELECT forecast(ts, value, 6) FROM readings)) + FROM _predict_receipts WHERE id = 1""").fetchone() + assert match == 0 + db.close() + + def test_receipt_pins_a_registered_student(receipt_db): db = sqlite3.connect(receipt_db) db.enable_load_extension(True) diff --git a/website/src/content/docs/reference/functions.md b/website/src/content/docs/reference/functions.md index 9b983a7..92b156b 100644 --- a/website/src/content/docs/reference/functions.md +++ b/website/src/content/docs/reference/functions.md @@ -123,6 +123,7 @@ window laid out by position as `context` input columns followed by the | --- | --- | | `predict_version()` | the extension version string | | `predict_ulid(ts)` | the smallest ULID for an ISO-8601 UTC timestamp: deterministic (the time component is set, the entropy bits are zero), useful as a range key | +| `predict_sha256(x)` | lowercase hex SHA-256 of a TEXT or BLOB value (NULL passes through). The same hash that pins model weights, exposed so provenance workflows like the prediction-receipts skill can record and replay-verify results in pure SQL | | `predict_debug()` | build info: profile, compiled features, available runtimes | | `predict_register(model_id, config)` | registers an external model in `_predict_models` and returns its content hash; `config` is a bare weights path or a JSON object (`runtime`, `kind`, `license`, `weights_uri`, `io_spec`, `target`). Registration is metadata only: executing an `onnx` model still needs the ONNX build | From 7c3c6f62234393eae8419a6821773ba1faff7518 Mon Sep 17 00:00:00 2001 From: mstrathman Date: Wed, 29 Jul 2026 17:40:16 -0700 Subject: [PATCH 05/11] fix: address all nine CodeRabbit findings on the skills PR Security (critical): receipt.py locks extension loading immediately after loading predict0, so a tampered receipt's replayed input_sql can no longer call load_extension() on an arbitrary library; a regression test replays exactly that attack and asserts sqlite refuses it. Interoperability (major): canonicalization now follows the recorded operation instead of the result shape, so a one-text-cell backtest or predict result hashes as {"columns","rows"} rather than masquerading as an aggregate document; verify() reads the stored operation. Claims discipline (the reviewer enforcing our own path instructions): size/latency claims in distill-lifecycle now cite the benchmarks and state measured ranges; the soft-label rescue is scoped and qualified; the licensing section says obligations 'may' apply, tells users to record the exact license with each student, and states that accept_license is an acknowledgment, not compliance; the 0.57 coverage figure carries its model/dataset scope and source; the conformal fragment became a complete backtest() call; and conformal coverage is 'substantially improves, reached nominal in our benchmarks, no finite-sample guarantee' instead of a promise. Tests: adversarial cases for the one-cell TVF canonicalization, unknown option keys asserting the exact PREDICT_ERR_OPTIONS code, missing receipt ids, and the load_extension replay attack. Eight pass. Co-Authored-By: Claude Fable 5 --- skills/distill-lifecycle/SKILL.md | 30 +++++++---- skills/interpret-backtest/SKILL.md | 24 +++++---- skills/prediction-receipts/scripts/receipt.py | 28 +++++++--- tests/test_skills.py | 53 +++++++++++++++++++ 4 files changed, 108 insertions(+), 27 deletions(-) diff --git a/skills/distill-lifecycle/SKILL.md b/skills/distill-lifecycle/SKILL.md index 6a37352..69df48e 100644 --- a/skills/distill-lifecycle/SKILL.md +++ b/skills/distill-lifecycle/SKILL.md @@ -14,9 +14,12 @@ metadata: # The distillation lifecycle Distillation compresses a teacher (your own labels, your existing model's -predictions, or a licensed foundation model) into a student a few -kilobytes big that serves in microseconds in the zero-dependency core. -The fit takes seconds; the lifecycle judgment is yours. +predictions, or a licensed foundation model) into a native student the +serving core executes with no runtime. Measured on the repo's benchmark +suite: student blobs run tens to hundreds of kilobytes, single-row +serving is sub-millisecond on CPU, and fits complete in seconds at +benchmark scale (see `benchmarks/results/` in the sqlite-predict repo). +The lifecycle judgment is yours. ## Distill @@ -51,8 +54,10 @@ Never serve a student on faith: against their teacher, but regression tails are worse. Check regression students more skeptically. 4. Soft-label distillation (`proba`/`classes`) preserves the teacher's - calibration and rescues heavily imbalanced datasets, where hard labels - can collapse to one class (the distiller refuses loudly when they do). + calibration when the teacher emits probabilities, and usually rescues + imbalanced datasets where hard labels collapse to one class (the + distiller refuses loudly on collapse rather than fitting a + constant). ## Watch for drift, re-distill cheaply @@ -67,10 +72,13 @@ quality decays silently, so schedule verification rather than assuming: ## Licensing is part of the lifecycle -A student derives from its teacher, and the teacher's license travels -with it. Your own labels and models carry no limits. Foundation-model -teachers vary by vendor and version, and restrictively licensed ones -require an explicit `accept_license` opt-in before they will run. Read -the license of the exact weights you use before distilling for anything -beyond evaluation; the project documentation's license notes are +A student derives from its teacher, and the teacher's license may +impose obligations on what you distill and distribute; what applies +depends on the exact weights, license version, and how the student is +used. Record the teacher's license alongside each student you register. +Your own labels and models carry no such limits. Restrictively licensed +teachers require an explicit `accept_license` opt-in before they will +run, but that gate is an acknowledgment, not compliance: read the +license of the exact weights you use before distilling for anything +beyond evaluation. The project documentation's license notes are orientation, not legal advice. diff --git a/skills/interpret-backtest/SKILL.md b/skills/interpret-backtest/SKILL.md index 67c87ee..798c24e 100644 --- a/skills/interpret-backtest/SKILL.md +++ b/skills/interpret-backtest/SKILL.md @@ -36,19 +36,25 @@ SELECT * FROM backtest('SELECT ts, value FROM readings', 24, ## Interval judgment -The default Gaussian band is overconfident on smooth series (measured as -low as 0.57 coverage at a nominal 0.90 on our benchmarks). When interval -truth matters: +The default Gaussian band can be overconfident on smooth series: the +sqlite-predict benchmarks measured 0.57 coverage at a nominal 0.90 for +theta-class models on smooth gluonts series (`benchmarks/results/` in +the repo). When interval truth matters, request conformal intervals in +the same call: ```sql -'{"interval_method":"conformal"}' +SELECT * FROM backtest('SELECT ts, value FROM readings', 24, + '{"model":"theta-classic", + "interval_method":"conformal"}'); ``` -Conformal intervals calibrate to measured residuals and land at the -nominal level, at the cost of needing enough folds to calibrate -(statistical models only; a short series can make it refuse). Always -verify the choice with `backtest` coverage on your own series rather -than trusting either method's reputation. +Conformal intervals calibrate to measured out-of-sample residuals and +substantially improve empirical coverage (they reached the nominal level +in our benchmarks, but finite folds carry no guarantee). They need +enough history to calibrate and apply to the statistical models only; a +short series makes the call refuse rather than fabricate. Always verify +with `backtest` coverage on your own series rather than trusting either +method's reputation, ours included. ## Auto-selection judgment diff --git a/skills/prediction-receipts/scripts/receipt.py b/skills/prediction-receipts/scripts/receipt.py index 1ee2ddb..25c61ea 100644 --- a/skills/prediction-receipts/scripts/receipt.py +++ b/skills/prediction-receipts/scripts/receipt.py @@ -56,13 +56,26 @@ def connect(db_path, extension): db.load_extension(extension) except sqlite3.OperationalError as e: fail(f"cannot load extension '{extension}': {e}") + finally: + # verify() replays stored SQL; with loading left enabled, a + # tampered receipt could call load_extension() on an arbitrary + # local library. One load, then locked. + db.enable_load_extension(False) return db -def canonical_document(cur): +AGGREGATE_OPS = ("forecast", "detect_anomalies") + + +def canonical_document(cur, operation): + """Aggregate operations hash their single JSON document verbatim; + row-shaped operations always hash the {"columns","rows"} form, even + when the result happens to be one text cell. Shape must follow the + operation or receipts stop being interoperable.""" rows = cur.fetchall() cols = [d[0] for d in cur.description] if cur.description else [] - if len(rows) == 1 and len(cols) == 1 and isinstance(rows[0][0], str): + if (operation in AGGREGATE_OPS and len(rows) == 1 and len(cols) == 1 + and isinstance(rows[0][0], str)): return rows[0][0] return json.dumps({"columns": cols, "rows": [list(r) for r in rows]}, separators=(",", ":"), ensure_ascii=False) @@ -114,7 +127,7 @@ def cmd_record(args): fail("cannot infer the operation from the SQL; pass --operation " f"one of {', '.join(OPERATIONS)}") cur = db.execute(args.sql) - document = canonical_document(cur) + document = canonical_document(cur, operation) model_id, content_hash = model_fields(db, document, args.model_id) db.execute(DDL) cur = db.execute( @@ -133,12 +146,13 @@ def cmd_record(args): def cmd_verify(args): db = connect(args.db, args.extension) row = db.execute( - "SELECT input_sql, model_id, content_hash, extension, result_sha256" - " FROM _predict_receipts WHERE id = ?", (args.receipt_id,)).fetchone() + "SELECT operation, input_sql, model_id, content_hash, extension," + " result_sha256 FROM _predict_receipts WHERE id = ?", + (args.receipt_id,)).fetchone() if row is None: fail(f"no receipt with id {args.receipt_id}") - input_sql, model_id, rec_hash, rec_ext, rec_sha = row - document = canonical_document(db.execute(input_sql)) + operation, input_sql, model_id, rec_hash, rec_ext, rec_sha = row + document = canonical_document(db.execute(input_sql), operation) now_sha = sha256_text(document) now_ext = extension_version(db) now_hash = None diff --git a/tests/test_skills.py b/tests/test_skills.py index 04808ae..b05dd83 100644 --- a/tests/test_skills.py +++ b/tests/test_skills.py @@ -176,3 +176,56 @@ def test_receipt_pins_a_registered_student(receipt_db): assert out2["content_hash"], "student content_hash must be recorded" ver = run_receipt("verify", receipt_db, str(out2["receipt_id"])) assert json.loads(ver.stdout)["match"] is True + + +def test_receipt_adversarial_paths(receipt_db): + """Break the receipt tool the ways a hostile or careless caller + would: single-text-cell TVF results must canonicalize as rows (not + as an aggregate document), extension errors must surface their + PREDICT_ERR_* code verbatim, missing receipts must fail loudly, and + a replayed receipt must not be able to re-enable extension + loading.""" + import hashlib + + # a backtest projected to one text column: operation-based + # canonicalization must hash the {"columns","rows"} form, so the + # receipt hash must NOT equal the raw-cell hash + one_cell_sql = ("SELECT model FROM backtest(" + "'SELECT ts, value FROM readings', 4," + " '{\"model\":\"theta-classic\"}') LIMIT 1") + rec = run_receipt("record", receipt_db, one_cell_sql, + "--model-id", "theta-classic") + assert rec.returncode == 0, rec.stderr + out = json.loads(rec.stdout) + db = sqlite3.connect(receipt_db) + db.enable_load_extension(True) + db.load_extension(EXT) + raw_cell = db.execute(one_cell_sql).fetchone()[0] + db.close() + assert out["result_sha256"] != hashlib.sha256( + raw_cell.encode()).hexdigest(), "TVF result canonicalized as an aggregate document" + ver = run_receipt("verify", receipt_db, str(out["receipt_id"])) + assert json.loads(ver.stdout)["match"] is True + + # unknown option key: the extension's exact error code must surface + bad = run_receipt( + "record", receipt_db, + "SELECT forecast(ts, value, 6, '{\"bogus_key\":1}') FROM readings") + assert bad.returncode != 0 + assert "PREDICT_ERR_OPTIONS" in (bad.stderr + bad.stdout) + + # missing receipt id fails loudly + gone = run_receipt("verify", receipt_db, "9999") + assert gone.returncode != 0 + assert "no receipt" in gone.stderr + + # a tampered receipt cannot load an arbitrary extension on replay + db = sqlite3.connect(receipt_db) + db.execute( + "UPDATE _predict_receipts SET input_sql =" + " 'SELECT load_extension(''/tmp/evil'')' WHERE id = 1") + db.commit() + db.close() + evil = run_receipt("verify", receipt_db, "1") + assert evil.returncode != 0 + assert "not authorized" in (evil.stderr + evil.stdout).lower() or "no receipt" not in evil.stderr From 0db3468185d0029284eed63671fc7e2a5ae8aa13 Mon Sep 17 00:00:00 2001 From: mstrathman Date: Wed, 29 Jul 2026 17:40:58 -0700 Subject: [PATCH 06/11] test: drop the vacuous or-clause from the replay-attack assertion Co-Authored-By: Claude Fable 5 --- tests/test_skills.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_skills.py b/tests/test_skills.py index b05dd83..0cceb56 100644 --- a/tests/test_skills.py +++ b/tests/test_skills.py @@ -228,4 +228,4 @@ def test_receipt_adversarial_paths(receipt_db): db.close() evil = run_receipt("verify", receipt_db, "1") assert evil.returncode != 0 - assert "not authorized" in (evil.stderr + evil.stdout).lower() or "no receipt" not in evil.stderr + assert "not authorized" in (evil.stderr + evil.stdout).lower() From 18ceff020986c752f7eeee411865fd8c4df605ce Mon Sep 17 00:00:00 2001 From: mstrathman Date: Wed, 29 Jul 2026 17:54:36 -0700 Subject: [PATCH 07/11] fix: address round-two CodeRabbit findings predict_sha256 rejects INTEGER/REAL with PREDICT_ERR_SCHEMA instead of hashing SQLite's 15-digit number-to-text coercion, which could collapse distinct doubles into one hash; the functions reference documents the contract. The receipt CLI gains stable RECEIPT_ERR_* failure codes so agents branch on outcomes the way they do on PREDICT_ERR_* (the test asserts the code, not prose). The receipts skill now distinguishes result hashing from weight pinning explicitly, ships an executable no-registry recording variant, labels the pure-SQL verify as a spot check of one query's hash and routes full replay to the reference script, and tags the command block for MD040. Co-Authored-By: Claude Fable 5 --- benchmarks/results/scale-arena.jsonl | 13 + benchmarks/results/scale.md | 25 ++ benchmarks/scale_arena.py | 323 ++++++++++++++++++ skills/prediction-receipts/SKILL.md | 42 ++- skills/prediction-receipts/scripts/receipt.py | 20 +- sqlite-predict.c | 11 +- tests/test_skills.py | 13 +- .../src/content/docs/reference/functions.md | 2 +- 8 files changed, 428 insertions(+), 21 deletions(-) create mode 100644 benchmarks/results/scale-arena.jsonl create mode 100644 benchmarks/results/scale.md create mode 100644 benchmarks/scale_arena.py diff --git a/benchmarks/results/scale-arena.jsonl b/benchmarks/results/scale-arena.jsonl new file mode 100644 index 0000000..d211654 --- /dev/null +++ b/benchmarks/results/scale-arena.jsonl @@ -0,0 +1,13 @@ +{"dataset": "Diabetes130US", "n_train": 1500, "n_test": 500, "d": 40, "xgboost": 0.906, "xgb_s": 0.1, "tabicl": 0.916, "tabicl_s": 2.2, "tabicl_device": "mps", "leak_gap": 0.0007, "ic_maxp": 0.9251, "collapsed": 1, "gbt<-tabicl soft": 0.916, "oof_s": 7.6, "oof_gap": 0.0007, "gbt<-tabicl soft-oof": 0.916} +{"dataset": "Diabetes130US", "n_train": 9999, "n_test": 3334, "d": 40, "xgboost": 0.9070185962807439, "xgb_s": 0.1, "tabicl": 0.9130173965206959, "tabicl_s": 101.9, "tabicl_device": "mps", "leak_gap": -0.0, "ic_maxp": 0.922, "collapsed": 1, "gbt<-tabicl soft": 0.9130173965206959, "oof_s": 116.4, "oof_gap": -0.0, "gbt<-tabicl soft-oof": 0.9130173965206959} +{"dataset": "airline_satisfaction", "n_train": 1500, "n_test": 500, "d": 21, "xgboost": 0.91, "xgb_s": 0.0, "tabicl": 0.928, "tabicl_s": 1.6, "tabicl_device": "mps", "leak_gap": 0.0693, "ic_maxp": 0.8787, "gbt<-tabicl": 0.908, "distill_s": 0.5, "blob_kb": 103.4, "gbt<-tabicl soft": 0.904, "oof_s": 5.5, "oof_gap": -0.0073, "gbt<-tabicl soft-oof": 0.906} +{"dataset": "GiveMeSomeCredit", "n_train": 1500, "n_test": 500, "d": 10, "xgboost": 0.922, "xgb_s": 0.0, "tabicl": 0.936, "tabicl_s": 1.2, "tabicl_device": "mps", "leak_gap": 0.0067, "ic_maxp": 0.9407, "gbt<-tabicl": 0.932, "distill_s": 0.3, "blob_kb": 97.1, "gbt<-tabicl soft": 0.926, "oof_s": 4.1, "oof_gap": -0.0113, "gbt<-tabicl soft-oof": 0.93} +{"dataset": "GiveMeSomeCredit", "n_train": 9999, "n_test": 3334, "d": 10, "xgboost": 0.9289142171565686, "xgb_s": 0.1, "tabicl": 0.9322135572885423, "tabicl_s": 8.4, "tabicl_device": "mps", "leak_gap": 0.021, "ic_maxp": 0.9452, "gbt<-tabicl": 0.9334133173365327, "distill_s": 2.6, "blob_kb": 104.3, "gbt<-tabicl soft": 0.9334133173365327, "oof_s": 29.5, "oof_gap": 0.0009, "gbt<-tabicl soft-oof": 0.9322135572885423} +{"dataset": "APSFailure", "n_train": 1500, "n_test": 500, "d": 40, "xgboost": 0.994, "xgb_s": 0.1, "tabicl": 0.996, "tabicl_s": 2.6, "tabicl_device": "mps", "leak_gap": -0.004, "ic_maxp": 0.9902, "gbt<-tabicl": 0.99, "distill_s": 1.4, "blob_kb": 76.6, "gbt<-tabicl soft": 0.99, "oof_s": 8.2, "oof_gap": -0.0093, "gbt<-tabicl soft-oof": 0.996} +{"dataset": "SDSS17", "n_train": 1500, "n_test": 500, "d": 11, "xgboost": 0.97, "xgb_s": 0.1, "tabicl": 0.972, "tabicl_s": 1.4, "tabicl_device": "mps", "leak_gap": 0.0167, "ic_maxp": 0.9892, "gbt<-tabicl": 0.968, "distill_s": 1.1, "blob_kb": 145.8, "gbt<-tabicl soft": 0.968, "oof_s": 4.3, "oof_gap": 0.0007, "gbt<-tabicl soft-oof": 0.964} +{"dataset": "SDSS17", "n_train": 9999, "n_test": 3334, "d": 11, "xgboost": 0.9679064187162567, "xgb_s": 0.3, "tabicl": 0.9718056388722256, "tabicl_s": 8.7, "tabicl_device": "mps", "leak_gap": 0.0205, "ic_maxp": 0.9906, "gbt<-tabicl": 0.9658068386322736, "distill_s": 9.2, "blob_kb": 159.2, "gbt<-tabicl soft": 0.9661067786442712, "oof_s": 30.3, "oof_gap": 0.0052, "gbt<-tabicl soft-oof": 0.9676064787042592} +{"dataset": "taiwanese_bankruptcy", "n_train": 1500, "n_test": 500, "d": 40, "xgboost": 0.966, "xgb_s": 0.1, "tabicl": 0.968, "tabicl_s": 2.4, "tabicl_device": "mps", "leak_gap": 0.004, "ic_maxp": 0.9721, "gbt<-tabicl": 0.968, "distill_s": 2.9, "blob_kb": 93.8, "gbt<-tabicl soft": 0.966, "oof_s": 8.2, "oof_gap": 0.0007, "gbt<-tabicl soft-oof": 0.968} +{"dataset": "online_shoppers_intention", "n_train": 1500, "n_test": 500, "d": 17, "xgboost": 0.882, "xgb_s": 0.1, "tabicl": 0.89, "tabicl_s": 1.5, "tabicl_device": "mps", "leak_gap": 0.0507, "ic_maxp": 0.8863, "gbt<-tabicl": 0.89, "distill_s": 0.4, "blob_kb": 100.9, "gbt<-tabicl soft": 0.888, "oof_s": 4.9, "oof_gap": 0.0153, "gbt<-tabicl soft-oof": 0.886} +{"dataset": "credit_card_default", "n_train": 1500, "n_test": 500, "d": 23, "xgboost": 0.81, "xgb_s": 0.1, "tabicl": 0.836, "tabicl_s": 1.7, "tabicl_device": "mps", "leak_gap": -0.0247, "ic_maxp": 0.8101, "gbt<-tabicl": 0.798, "distill_s": 0.9, "blob_kb": 93.5, "gbt<-tabicl soft": 0.81, "oof_s": 5.8, "oof_gap": -0.0153, "gbt<-tabicl soft-oof": 0.838} +{"dataset": "airline_satisfaction", "n_train": 9999, "n_test": 3334, "d": 21, "xgboost": 0.9448110377924415, "xgb_s": 0.3, "tabicl": 0.9511097780443911, "tabicl_s": 139.6, "tabicl_device": "cpu", "leak_gap": 0.0465, "ic_maxp": 0.9842, "gbt<-tabicl": 0.9337132573485303, "distill_s": 3.5, "blob_kb": 105.7, "gbt<-tabicl soft": 0.9340131973605279, "oof_s": 486.7, "oof_gap": 0.0043, "gbt<-tabicl soft-oof": 0.9337132573485303} +{"dataset": "credit_card_default", "n_train": 9999, "n_test": 3334, "d": 23, "xgboost": 0.8047390521895621, "xgb_s": 0.2, "tabicl": 0.8191361727654469, "tabicl_s": 167.5, "tabicl_device": "cpu", "leak_gap": 0.0205, "ic_maxp": 0.8387, "gbt<-tabicl": 0.8197360527894421, "distill_s": 7.4, "blob_kb": 96.8, "gbt<-tabicl soft": 0.8182363527294542, "oof_s": 571.6, "oof_gap": -0.001, "gbt<-tabicl soft-oof": 0.8194361127774445} diff --git a/benchmarks/results/scale.md b/benchmarks/results/scale.md new file mode 100644 index 0000000..3863b44 --- /dev/null +++ b/benchmarks/results/scale.md @@ -0,0 +1,25 @@ +# Scale study: distillation beyond 1500 rows + +TabICL v2 as the teacher on the naturally large TabArena +datasets, at increasing training sizes. Questions: does student +retention hold, where does tuned xgboost catch the zero-shot +teacher, and does the in-context label leak grow with context +(the gap column; see permissive-teachers.md). + +Device: mps with cpu fallback (per-cell `dev`). + +| dataset | n_train | xgboost | TabICL | gbt<-TabICL | soft | soft-oof | leak gap | oof gap | teacher s | distill s | blob KB | +|---|---|---|---|---|---|---|---|---|---|---|---| +| APSFailure | 1500 | 0.994 | 0.996 | 0.990 | 0.990 | 0.996 | -0.0040 | -0.0093 | 2.6 | 1.4 | 76.6 | +| Diabetes130US | 1500 | 0.906 | 0.916 | - | 0.916 | 0.916 | 0.0007 | 0.0007 | 2.2 | - | - | +| Diabetes130US | 9999 | 0.907 | 0.913 | - | 0.913 | 0.913 | -0.0000 | -0.0000 | 101.9 | - | - | +| GiveMeSomeCredit | 1500 | 0.922 | 0.936 | 0.932 | 0.926 | 0.930 | 0.0067 | -0.0113 | 1.2 | 0.3 | 97.1 | +| GiveMeSomeCredit | 9999 | 0.929 | 0.932 | 0.933 | 0.933 | 0.932 | 0.0210 | 0.0009 | 8.4 | 2.6 | 104.3 | +| SDSS17 | 1500 | 0.970 | 0.972 | 0.968 | 0.968 | 0.964 | 0.0167 | 0.0007 | 1.4 | 1.1 | 145.8 | +| SDSS17 | 9999 | 0.968 | 0.972 | 0.966 | 0.966 | 0.968 | 0.0205 | 0.0052 | 8.7 | 9.2 | 159.2 | +| SDSS17 | 49999 | 0.975 | 0.610 | - | 0.610 | - | -0.0000 | - | 118.9 | - | - | +| airline_satisfaction | 1500 | 0.910 | 0.928 | 0.908 | 0.904 | 0.906 | 0.0693 | -0.0073 | 1.6 | 0.5 | 103.4 | +| credit_card_default | 1500 | 0.810 | 0.836 | 0.798 | 0.810 | 0.838 | -0.0247 | -0.0153 | 1.7 | 0.9 | 93.5 | +| credit_card_default | 9999 | 0.805 | 0.771 | 0.776 | 0.776 | 0.776 | 0.0019 | 0.0054 | 11.7 | 7.4 | 96.2 | +| online_shoppers_intention | 1500 | 0.882 | 0.890 | 0.890 | 0.888 | 0.886 | 0.0507 | 0.0153 | 1.5 | 0.4 | 100.9 | +| taiwanese_bankruptcy | 1500 | 0.966 | 0.968 | 0.968 | 0.966 | 0.968 | 0.0040 | 0.0007 | 2.4 | 2.9 | 93.8 | diff --git a/benchmarks/scale_arena.py b/benchmarks/scale_arena.py new file mode 100644 index 0000000..1e38f23 --- /dev/null +++ b/benchmarks/scale_arena.py @@ -0,0 +1,323 @@ +"""Scale study: does the distillation story survive larger data? + +Three questions, one overnight run over the naturally large TabArena +datasets at n in {1500, 10k, 50k, 100k}: + + 1. Retention: does the distilled gbt student keep tracking its TabICL + teacher as rows grow (our published numbers stop at 1500)? + 2. The boundary: where does tuned xgboost catch the zero-shot teacher? + The TFM edge concentrates in small data; publish the crossover + instead of letting readers find it. + 3. The leak: does the in-context label leak (teacher train-agreement + minus held-out accuracy; see permissive-teachers.md and + arXiv:2605.18654) grow with context size? Out-of-fold labels are + compared at 10k, where five fold-fits stay affordable. + +Checkpointed per (dataset, n) cell into results/scale-arena.jsonl. +Teacher outputs are not cached across runs (contexts differ per n); +each cell records wall times instead. + +Run overnight: + + caffeinate -i env TEACHERS=tabicl uv run --with tabicl \ + --with scikit-learn --with xgboost --with pandas \ + python benchmarks/scale_arena.py +""" + +import os +import sys + +# Single-threaded math pools: xgboost's OpenMP and torch's ATen pool +# deadlock each other in one process on macOS (see tabarena.py, which +# learned this first; this file re-learned it the hard way). TabICL's +# heavy path runs on MPS, so the CPU pools cost little here. +for _v in ("OMP_NUM_THREADS", "MKL_NUM_THREADS", "OPENBLAS_NUM_THREADS", + "VECLIB_MAXIMUM_THREADS", "NUMEXPR_NUM_THREADS"): + os.environ[_v] = "1" +os.environ.setdefault("KMP_DUPLICATE_LIB_OK", "TRUE") +os.environ["HF_HUB_OFFLINE"] = "0" + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +import tabarena as TA # noqa: E402 + +import json # noqa: E402 +import time # noqa: E402 + +import numpy as np # noqa: E402 +import pandas as pd # noqa: E402 + +JSONL = os.path.join(os.path.dirname(__file__), "results", + "scale-arena.jsonl") +RESULTS = os.path.join(os.path.dirname(__file__), "results", "scale.md") +DEVICE = os.environ.get("SCALE_DEVICE", "mps") +CPU_AT = int(os.environ.get("SCALE_CPU_AT", 50_000)) # mps died natively at 50k +# Metal's 2**32-byte single-tensor cap binds on roughly rows x features: +# 50k x 10 survived with batch_size=1, 50k x 40 did not. Cells over this +# budget are skipped as measured hardware scope, not attempted. +MPS_CELL_BUDGET = int(os.environ.get("SCALE_MPS_BUDGET", 1_000_000)) +CELL_TIMEOUT_S = int(os.environ.get("SCALE_CELL_TIMEOUT", 2700)) +ATTEMPTS = os.path.join(os.path.dirname(__file__), "results", + "scale-attempts") +SIZES = (1500, 10_000, 50_000, 100_000) +OOF_AT = 10_000 +SEED = 0 + +# TabArena datasets whose full size supports the ladder (classification +# only: the leak diagnostic needs probabilities). +LARGE = [ + (46922, "Diabetes130US"), (46920, "airline_satisfaction"), + (46929, "GiveMeSomeCredit"), (46908, "APSFailure"), + (46955, "SDSS17"), (46962, "taiwanese_bankruptcy"), + (46947, "online_shoppers_intention"), (46919, "credit_card_default"), +] + + +def load_full(did, name): + from sklearn.datasets import fetch_openml + b = fetch_openml(data_id=did, as_frame=True) + X, y = TA._prep(b.data, b.target, "cls") + return X, y + + +def make_teacher(): + from tabicl import TabICLClassifier + return TabICLClassifier(device=DEVICE) + + +def teacher_with_fallback(Xtr, ytr, Xte): + """Fit + predict with an MPS-to-CPU fallback on failure or + non-finite output (seen from TabICL on MPS). Native-level MPS + crashes cannot be caught here; the cell-subprocess driver absorbs + those.""" + first = DEVICE # batch_size=1 makes mps survive big cells; cpu is + # still the fallback if the guard or a crash rejects the attempt + for attempt_device in dict.fromkeys((first, "cpu")): + try: + from tabicl import TabICLClassifier + kwargs = {"device": attempt_device} + bs = os.environ.get("SCALE_BATCH") + if bs: + kwargs["batch_size"] = int(bs) + elif attempt_device == "mps" and len(Xtr) >= 20_000: + # Metal caps one NDArray at 2**32 bytes; batch_size=1 + # divides the big tensor and survives 50k (probed) + kwargs["batch_size"] = 1 + if len(Xtr) >= CPU_AT: + # offload_mode="auto" engages a large-data path whose + # workers allocate MPS tensors past Metal's 2**32-byte + # cap regardless of device=. Pin everything to cpu, no + # worker processes. + kwargs.update(n_jobs=1) + for om in ("cpu", "none", None): + try: + est = TabICLClassifier(**kwargs, offload_mode=om) if om is not None else TabICLClassifier(**kwargs) + break + except (ValueError, TypeError): + continue + else: + est = TabICLClassifier(**kwargs) + t0 = time.time() + est.fit(Xtr.values, ytr.values) + pred_te = est.predict(Xte.values) + fit_s = time.time() - t0 + proba_tr = est.predict_proba(Xtr.values) + pred_tr = est.predict(Xtr.values) + if not (np.isfinite(proba_tr).all()): + raise ValueError("non-finite probabilities") + # wrong-but-finite guard (seen from TabICL on MPS at 10k: a + # 0.93-accuracy teacher suddenly at chance): a teacher that + # cannot beat the majority class ON ITS OWN CONTEXT is + # malfunctioning, not weak. Retry on the next device. + majority = float(ytr.value_counts(normalize=True).max()) + agree = float((np.asarray(pred_tr).astype(str) + == ytr.astype(str).values).mean()) + # below the mode = malfunction (a healthy teacher on heavily + # imbalanced data may only match the majority rate; a broken + # one falls under it, as TabICL-on-MPS did on airline) + if agree < majority - 0.01: + raise ValueError( + f"teacher below majority on its own context" + f" (agree={agree:.3f}, majority={majority:.3f})") + return est, pred_te, pred_tr, proba_tr, fit_s, attempt_device + except Exception as e: # noqa: BLE001 + last = e + continue + raise last + + +def one_cell(name, X, y, n): + if len(X) < n * 4 // 3: # need n plus a test split + return None + rng = np.random.default_rng(SEED) + idx = rng.choice(len(X), min(len(X), n + n // 3), replace=False) + Xs = X.iloc[idx].reset_index(drop=True) + ys = y.iloc[idx].reset_index(drop=True) + Xtr, Xte, ytr, yte = TA.split(Xs, ys, "cls") + + row = {"dataset": name, "n_train": len(Xtr), "n_test": len(Xte), + "d": Xs.shape[1]} + + t0 = time.time() + row["xgboost"] = TA.score(yte, TA.run_xgb(Xtr, ytr, Xte, "cls"), "cls") + row["xgb_s"] = round(time.time() - t0, 1) + + est, pred_te, pred_tr, proba_tr, fit_s, dev = \ + teacher_with_fallback(Xtr, ytr, Xte) + row["tabicl"] = TA.score(yte, pred_te, "cls") + row["tabicl_s"] = round(fit_s, 1) + row["tabicl_device"] = dev + + # the leak gap: agreement with own training labels minus held-out acc + agree = float((np.asarray(pred_tr).astype(str) + == ytr.astype(str).values).mean()) + row["leak_gap"] = round(agree - row["tabicl"], 4) + row["ic_maxp"] = round(float(np.asarray(proba_tr).max(axis=1).mean()), 4) + + t0 = time.time() + try: + preds, blob, hold = TA.run_ours_distill_teacher( + Xtr, np.asarray(pred_tr), Xte, "cls", kind="gbt") + row["gbt<-tabicl"] = TA.score(yte, preds, "cls") + row["distill_s"] = round(time.time() - t0, 1) + row["blob_kb"] = round(blob / 1024, 1) + except Exception as e: # noqa: BLE001 + if "single class" not in str(e): + raise + row["collapsed"] = 1 + + classes = [str(c) for c in est.classes_] + preds, _, _ = TA.run_ours_distill_soft( + Xtr, ytr, np.asarray(proba_tr), classes, Xte, kind="gbt") + row["gbt<-tabicl soft"] = TA.score(yte, preds, "cls") + + if len(Xtr) <= OOF_AT: + t0 = time.time() + proba_oof, classes_oof = TA.oof_proba(make_teacher, Xtr, ytr) + row["oof_s"] = round(time.time() - t0, 1) + oof_agree = float( + (np.array(classes_oof)[np.asarray(proba_oof).argmax(1)] + == ytr.astype(str).values).mean()) + row["oof_gap"] = round(oof_agree - row["tabicl"], 4) + preds, _, _ = TA.run_ours_distill_soft( + Xtr, ytr, np.asarray(proba_oof), classes_oof, Xte, kind="gbt") + row["gbt<-tabicl soft-oof"] = TA.score(yte, preds, "cls") + return row + + +def run_single_cell(name, n): + """Child-process entry: one (dataset, n) cell, result to the jsonl.""" + if n >= CPU_AT: + # Metal caps a single NDArray at 2**32 bytes and TabICL touches + # MPS internally even with device="cpu" (native SIGABRT, not + # catchable). Make torch report MPS unavailable in this child. + import torch + torch.backends.mps.is_available = lambda: False # noqa: E731 + torch.backends.mps.is_built = lambda: False # noqa: E731 + did = dict((nm, d) for d, nm in LARGE)[name] + X, y = load_full(did, name) + row = one_cell(name, X, y, n) + if row is not None: + with open(JSONL, "a") as f: + f.write(json.dumps(row) + "\n") + print(f" n={row['n_train']}: tabicl={row['tabicl']:.4f}" + f" xgb={row['xgboost']:.4f}" + f" student={row.get('gbt<-tabicl', float('nan')):.4f}" + f" leak={row['leak_gap']:+.4f}") + + +def main(): + """Driver: every cell runs in its own subprocess so a native crash + (seen from torch/MPS at 50k rows) or a hang costs one cell, not the + campaign. A cell that crashed once is skipped, not retried.""" + import subprocess + os.makedirs(ATTEMPTS, exist_ok=True) + done = set() + if os.path.exists(JSONL): + with open(JSONL) as f: + done = {(json.loads(ln)["dataset"], json.loads(ln)["n_train"]) + for ln in f if ln.strip()} + for did, name in LARGE: + for n in SIZES: + if any(d == name and abs(k - n) <= n // 3 for d, k in done): + print(f"{name} n~{n}: cached") + continue + d_est = None + try: + d_est = load_full(did, name)[0].shape[1] + except Exception: # noqa: BLE001 + pass + if d_est and n * d_est > MPS_CELL_BUDGET: + print(f"{name} n={n}: skipped, over mps budget" + f" ({n}x{d_est} > {MPS_CELL_BUDGET})") + continue + marker = os.path.join(ATTEMPTS, f"{name}-{n}") + if os.path.exists(marker): + print(f"{name} n={n}: crashed before, skipping") + continue + open(marker, "w").write("attempting\n") + t0 = time.time() + try: + r = subprocess.run( + [sys.executable, "-u", os.path.abspath(__file__), + "--cell", name, str(n)], + timeout=None if CELL_TIMEOUT_S == 0 else CELL_TIMEOUT_S, + cwd=os.path.dirname(os.path.abspath(__file__)) or ".", + capture_output=True, text=True) + except subprocess.TimeoutExpired as te: + sys.stdout.write((te.stdout or b"").decode(errors="ignore") + if isinstance(te.stdout, bytes) + else (te.stdout or "")) + print(f"{name} n={n}: CELL TIMEOUT after {CELL_TIMEOUT_S}s") + continue + sys.stdout.write(r.stdout) + if r.returncode == 0: + os.remove(marker) + # a completed run may still have skipped (too few rows) + else: + tailerr = (r.stderr or "").strip().splitlines()[-2:] + print(f"{name} n={n}: CELL DIED rc={r.returncode}" + f" after {time.time()-t0:.0f}s :: {' | '.join(tailerr)}") + print(f"{name} n={n}: {time.time()-t0:.0f}s") + report() + + +def report(): + if not os.path.exists(JSONL): + return + with open(JSONL) as f: + rows = [json.loads(ln) for ln in f if ln.strip()] + rows.sort(key=lambda r: (r["dataset"], r["n_train"])) + lines = [ + "# Scale study: distillation beyond 1500 rows", "", + "TabICL v2 as the teacher on the naturally large TabArena", + "datasets, at increasing training sizes. Questions: does student", + "retention hold, where does tuned xgboost catch the zero-shot", + "teacher, and does the in-context label leak grow with context", + "(the gap column; see permissive-teachers.md).", "", + f"Device: {DEVICE} with cpu fallback (per-cell `dev`).", "", + "| dataset | n_train | xgboost | TabICL | gbt<-TabICL | soft |" + " soft-oof | leak gap | oof gap | teacher s | distill s | blob KB |", + "|---|---|---|---|---|---|---|---|---|---|---|---|", + ] + for r in rows: + def fmt(v, nd=3): + return f"{v:.{nd}f}" if isinstance(v, (int, float)) else "-" + lines.append( + f"| {r['dataset']} | {r['n_train']} | {fmt(r.get('xgboost'))} |" + f" {fmt(r.get('tabicl'))} | {fmt(r.get('gbt<-tabicl'))} |" + f" {fmt(r.get('gbt<-tabicl soft'))} |" + f" {fmt(r.get('gbt<-tabicl soft-oof'))} |" + f" {fmt(r.get('leak_gap'), 4)} | {fmt(r.get('oof_gap'), 4)} |" + f" {r.get('tabicl_s', '-')} | {r.get('distill_s', '-')} |" + f" {r.get('blob_kb', '-')} |") + lines.append("") + with open(RESULTS, "w") as f: + f.write("\n".join(lines)) + print(f"wrote {RESULTS}") + + +if __name__ == "__main__": + if len(sys.argv) >= 4 and sys.argv[1] == "--cell": + run_single_cell(sys.argv[2], int(sys.argv[3])) + else: + main() diff --git a/skills/prediction-receipts/SKILL.md b/skills/prediction-receipts/SKILL.md index 472eaf7..18fd5cc 100644 --- a/skills/prediction-receipts/SKILL.md +++ b/skills/prediction-receipts/SKILL.md @@ -44,8 +44,11 @@ CREATE TABLE IF NOT EXISTS _predict_receipts ( ## Recording in pure SQL -The extension exposes `predict_sha256()` (the same hash that pins model -weights), so the whole workflow runs in SQL with no host language: +The extension exposes `predict_sha256()`, a plain SHA-256 over TEXT or +BLOB. Two different provenance values use it: `result_sha256` hashes the +result document (computed here), while `content_hash` pins the model's +weights (computed by the registry at registration; you only copy it). +Never conflate the two. The whole workflow runs in SQL: ```sql INSERT INTO _predict_receipts (operation, input_sql, options, model_id, @@ -62,9 +65,24 @@ FROM (SELECT (SELECT forecast(ts, value, 24) FROM readings) AS doc); ``` The inner `input_sql` string and the query that computes `doc` must be -the same SQL, verbatim; that is what makes the receipt replayable. If -nothing has been distilled or registered, `_predict_models` does not -exist yet: record `NULL` for `content_hash` instead of the subquery. +the same SQL, verbatim; that is what makes the receipt replayable. + +If nothing has been distilled or registered, `_predict_models` does not +exist and the statement above fails on the subquery. Use this variant, +which records `NULL` for `content_hash`: + +```sql +INSERT INTO _predict_receipts (operation, input_sql, options, model_id, + content_hash, extension, result_sha256) +SELECT 'forecast', + 'SELECT forecast(ts, value, 24) FROM readings', + NULL, + json_extract(doc, '$.model'), + NULL, + json_extract(predict_version(), '$.extension'), + predict_sha256(doc) +FROM (SELECT (SELECT forecast(ts, value, 24) FROM readings) AS doc); +``` ## Verifying in pure SQL @@ -75,10 +93,14 @@ SELECT id, FROM _predict_receipts WHERE id = 1; ``` -`match` is 1 when the replay reproduces the recorded result. On 0, -compare the receipt's `extension` against `predict_version()` and its -`content_hash` against the registry to find what moved; unchanged data -with an unchanged model and extension should never mismatch. +`match` is 1 when the replay reproduces the recorded result. This is a +spot check of one known query's result hash, not full verification: it +trusts the receipt row's own metadata, so tampered `input_sql`, +`options`, or `model_id` fields are not detected here. For full replay +that re-derives everything from the stored fields and diagnoses what +moved (data, extension version, or model hash), use the reference +script below. On a mismatch, compare the receipt's `extension` against +`predict_version()` and its `content_hash` against the registry. ## The reference script (optional) @@ -87,7 +109,7 @@ canonical serialization, and pure SQL float formatting is not round-trip safe. Use the bundled script for those, or when you want mismatch diagnostics computed for you: -``` +```text scripts/receipt.py record DB "SELECT forecast(ts, value, 24) FROM t" scripts/receipt.py verify DB RECEIPT_ID # exit 0 match, 2 mismatch scripts/receipt.py list DB diff --git a/skills/prediction-receipts/scripts/receipt.py b/skills/prediction-receipts/scripts/receipt.py index 25c61ea..15e81f5 100644 --- a/skills/prediction-receipts/scripts/receipt.py +++ b/skills/prediction-receipts/scripts/receipt.py @@ -44,8 +44,10 @@ OPERATIONS = ("forecast", "detect_anomalies", "predict", "backtest") -def fail(msg): - print(f"receipt.py: {msg}", file=sys.stderr) +def fail(code, msg): + """Stable, grep-able failure codes so agents can branch on outcomes + the way they branch on the extension's PREDICT_ERR_* codes.""" + print(f"receipt.py: {code}: {msg}", file=sys.stderr) sys.exit(1) @@ -55,7 +57,7 @@ def connect(db_path, extension): try: db.load_extension(extension) except sqlite3.OperationalError as e: - fail(f"cannot load extension '{extension}': {e}") + fail("RECEIPT_ERR_EXTENSION", f"cannot load '{extension}': {e}") finally: # verify() replays stored SQL; with loading left enabled, a # tampered receipt could call load_extension() on an arbitrary @@ -107,8 +109,9 @@ def model_fields(db, document, model_flag): except (json.JSONDecodeError, AttributeError): model_id = None if not model_id: - fail("cannot determine the serving model: the result is not a " - "document with a \"model\" field; pass --model-id explicitly") + fail("RECEIPT_ERR_MODEL_UNKNOWN", + "the result is not a document with a \"model\" field;" + " pass --model-id explicitly") content_hash = None try: row = db.execute( @@ -124,8 +127,9 @@ def cmd_record(args): db = connect(args.db, args.extension) operation = args.operation or infer_operation(args.sql) if operation is None: - fail("cannot infer the operation from the SQL; pass --operation " - f"one of {', '.join(OPERATIONS)}") + fail("RECEIPT_ERR_OPERATION_UNKNOWN", + f"cannot infer the operation; pass --operation one of" + f" {', '.join(OPERATIONS)}") cur = db.execute(args.sql) document = canonical_document(cur, operation) model_id, content_hash = model_fields(db, document, args.model_id) @@ -150,7 +154,7 @@ def cmd_verify(args): " result_sha256 FROM _predict_receipts WHERE id = ?", (args.receipt_id,)).fetchone() if row is None: - fail(f"no receipt with id {args.receipt_id}") + fail("RECEIPT_ERR_NOT_FOUND", f"no receipt with id {args.receipt_id}") operation, input_sql, model_id, rec_hash, rec_ext, rec_sha = row document = canonical_document(db.execute(input_sql), operation) now_sha = sha256_text(document) diff --git a/sqlite-predict.c b/sqlite-predict.c index 5815e08..c19217f 100644 --- a/sqlite-predict.c +++ b/sqlite-predict.c @@ -437,9 +437,18 @@ static void predict_sha256_fn(sqlite3_context *context, int argc, if (vt == SQLITE_BLOB) { data = sqlite3_value_blob(argv[0]); n = sqlite3_value_bytes(argv[0]); - } else { /* TEXT and numeric values hash their text form */ + } else if (vt == SQLITE_TEXT) { data = sqlite3_value_text(argv[0]); n = sqlite3_value_bytes(argv[0]); + } else { + /* numbers are rejected, not coerced: SQLite's REAL->TEXT keeps 15 + * significant digits, so distinct doubles could collapse to one + * hash. A provenance primitive must not do that silently. */ + sqlite3_result_error(context, + PREDICT_ERR_SCHEMA + ": predict_sha256 takes TEXT or BLOB", + -1); + return; } predict0_hasher h; predict0_hash_init(&h); diff --git a/tests/test_skills.py b/tests/test_skills.py index 0cceb56..934a75d 100644 --- a/tests/test_skills.py +++ b/tests/test_skills.py @@ -214,10 +214,21 @@ def test_receipt_adversarial_paths(receipt_db): assert bad.returncode != 0 assert "PREDICT_ERR_OPTIONS" in (bad.stderr + bad.stdout) + # numeric input to the hash primitive is rejected, never coerced + db = sqlite3.connect(receipt_db) + db.enable_load_extension(True) + db.load_extension(EXT) + import pytest as _pytest + for bad_val in ("1.5", "42"): + with _pytest.raises(sqlite3.OperationalError, match="PREDICT_ERR_SCHEMA"): + db.execute(f"SELECT predict_sha256({bad_val})").fetchone() + assert db.execute("SELECT predict_sha256(NULL)").fetchone()[0] is None + db.close() + # missing receipt id fails loudly gone = run_receipt("verify", receipt_db, "9999") assert gone.returncode != 0 - assert "no receipt" in gone.stderr + assert "RECEIPT_ERR_NOT_FOUND" in gone.stderr # a tampered receipt cannot load an arbitrary extension on replay db = sqlite3.connect(receipt_db) diff --git a/website/src/content/docs/reference/functions.md b/website/src/content/docs/reference/functions.md index 92b156b..352e439 100644 --- a/website/src/content/docs/reference/functions.md +++ b/website/src/content/docs/reference/functions.md @@ -123,7 +123,7 @@ window laid out by position as `context` input columns followed by the | --- | --- | | `predict_version()` | the extension version string | | `predict_ulid(ts)` | the smallest ULID for an ISO-8601 UTC timestamp: deterministic (the time component is set, the entropy bits are zero), useful as a range key | -| `predict_sha256(x)` | lowercase hex SHA-256 of a TEXT or BLOB value (NULL passes through). The same hash that pins model weights, exposed so provenance workflows like the prediction-receipts skill can record and replay-verify results in pure SQL | +| `predict_sha256(x)` | lowercase hex SHA-256 of a TEXT or BLOB value. NULL passes through; numeric arguments are rejected with `PREDICT_ERR_SCHEMA` rather than hashed through SQLite's lossy number-to-text conversion. Used by provenance workflows like the prediction-receipts skill to hash result documents in pure SQL | | `predict_debug()` | build info: profile, compiled features, available runtimes | | `predict_register(model_id, config)` | registers an external model in `_predict_models` and returns its content hash; `config` is a bare weights path or a JSON object (`runtime`, `kind`, `license`, `weights_uri`, `io_spec`, `target`). Registration is metadata only: executing an `onnx` model still needs the ONNX build | From 3aaef195b2ee7f21e531708128fd410c282e4cdc Mon Sep 17 00:00:00 2001 From: mstrathman Date: Wed, 29 Jul 2026 17:54:57 -0700 Subject: [PATCH 08/11] chore: keep the scale-study harness out of the skills PR It lands on main with the scale results. Co-Authored-By: Claude Fable 5 --- benchmarks/scale_arena.py | 323 -------------------------------------- 1 file changed, 323 deletions(-) delete mode 100644 benchmarks/scale_arena.py diff --git a/benchmarks/scale_arena.py b/benchmarks/scale_arena.py deleted file mode 100644 index 1e38f23..0000000 --- a/benchmarks/scale_arena.py +++ /dev/null @@ -1,323 +0,0 @@ -"""Scale study: does the distillation story survive larger data? - -Three questions, one overnight run over the naturally large TabArena -datasets at n in {1500, 10k, 50k, 100k}: - - 1. Retention: does the distilled gbt student keep tracking its TabICL - teacher as rows grow (our published numbers stop at 1500)? - 2. The boundary: where does tuned xgboost catch the zero-shot teacher? - The TFM edge concentrates in small data; publish the crossover - instead of letting readers find it. - 3. The leak: does the in-context label leak (teacher train-agreement - minus held-out accuracy; see permissive-teachers.md and - arXiv:2605.18654) grow with context size? Out-of-fold labels are - compared at 10k, where five fold-fits stay affordable. - -Checkpointed per (dataset, n) cell into results/scale-arena.jsonl. -Teacher outputs are not cached across runs (contexts differ per n); -each cell records wall times instead. - -Run overnight: - - caffeinate -i env TEACHERS=tabicl uv run --with tabicl \ - --with scikit-learn --with xgboost --with pandas \ - python benchmarks/scale_arena.py -""" - -import os -import sys - -# Single-threaded math pools: xgboost's OpenMP and torch's ATen pool -# deadlock each other in one process on macOS (see tabarena.py, which -# learned this first; this file re-learned it the hard way). TabICL's -# heavy path runs on MPS, so the CPU pools cost little here. -for _v in ("OMP_NUM_THREADS", "MKL_NUM_THREADS", "OPENBLAS_NUM_THREADS", - "VECLIB_MAXIMUM_THREADS", "NUMEXPR_NUM_THREADS"): - os.environ[_v] = "1" -os.environ.setdefault("KMP_DUPLICATE_LIB_OK", "TRUE") -os.environ["HF_HUB_OFFLINE"] = "0" - -sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) -import tabarena as TA # noqa: E402 - -import json # noqa: E402 -import time # noqa: E402 - -import numpy as np # noqa: E402 -import pandas as pd # noqa: E402 - -JSONL = os.path.join(os.path.dirname(__file__), "results", - "scale-arena.jsonl") -RESULTS = os.path.join(os.path.dirname(__file__), "results", "scale.md") -DEVICE = os.environ.get("SCALE_DEVICE", "mps") -CPU_AT = int(os.environ.get("SCALE_CPU_AT", 50_000)) # mps died natively at 50k -# Metal's 2**32-byte single-tensor cap binds on roughly rows x features: -# 50k x 10 survived with batch_size=1, 50k x 40 did not. Cells over this -# budget are skipped as measured hardware scope, not attempted. -MPS_CELL_BUDGET = int(os.environ.get("SCALE_MPS_BUDGET", 1_000_000)) -CELL_TIMEOUT_S = int(os.environ.get("SCALE_CELL_TIMEOUT", 2700)) -ATTEMPTS = os.path.join(os.path.dirname(__file__), "results", - "scale-attempts") -SIZES = (1500, 10_000, 50_000, 100_000) -OOF_AT = 10_000 -SEED = 0 - -# TabArena datasets whose full size supports the ladder (classification -# only: the leak diagnostic needs probabilities). -LARGE = [ - (46922, "Diabetes130US"), (46920, "airline_satisfaction"), - (46929, "GiveMeSomeCredit"), (46908, "APSFailure"), - (46955, "SDSS17"), (46962, "taiwanese_bankruptcy"), - (46947, "online_shoppers_intention"), (46919, "credit_card_default"), -] - - -def load_full(did, name): - from sklearn.datasets import fetch_openml - b = fetch_openml(data_id=did, as_frame=True) - X, y = TA._prep(b.data, b.target, "cls") - return X, y - - -def make_teacher(): - from tabicl import TabICLClassifier - return TabICLClassifier(device=DEVICE) - - -def teacher_with_fallback(Xtr, ytr, Xte): - """Fit + predict with an MPS-to-CPU fallback on failure or - non-finite output (seen from TabICL on MPS). Native-level MPS - crashes cannot be caught here; the cell-subprocess driver absorbs - those.""" - first = DEVICE # batch_size=1 makes mps survive big cells; cpu is - # still the fallback if the guard or a crash rejects the attempt - for attempt_device in dict.fromkeys((first, "cpu")): - try: - from tabicl import TabICLClassifier - kwargs = {"device": attempt_device} - bs = os.environ.get("SCALE_BATCH") - if bs: - kwargs["batch_size"] = int(bs) - elif attempt_device == "mps" and len(Xtr) >= 20_000: - # Metal caps one NDArray at 2**32 bytes; batch_size=1 - # divides the big tensor and survives 50k (probed) - kwargs["batch_size"] = 1 - if len(Xtr) >= CPU_AT: - # offload_mode="auto" engages a large-data path whose - # workers allocate MPS tensors past Metal's 2**32-byte - # cap regardless of device=. Pin everything to cpu, no - # worker processes. - kwargs.update(n_jobs=1) - for om in ("cpu", "none", None): - try: - est = TabICLClassifier(**kwargs, offload_mode=om) if om is not None else TabICLClassifier(**kwargs) - break - except (ValueError, TypeError): - continue - else: - est = TabICLClassifier(**kwargs) - t0 = time.time() - est.fit(Xtr.values, ytr.values) - pred_te = est.predict(Xte.values) - fit_s = time.time() - t0 - proba_tr = est.predict_proba(Xtr.values) - pred_tr = est.predict(Xtr.values) - if not (np.isfinite(proba_tr).all()): - raise ValueError("non-finite probabilities") - # wrong-but-finite guard (seen from TabICL on MPS at 10k: a - # 0.93-accuracy teacher suddenly at chance): a teacher that - # cannot beat the majority class ON ITS OWN CONTEXT is - # malfunctioning, not weak. Retry on the next device. - majority = float(ytr.value_counts(normalize=True).max()) - agree = float((np.asarray(pred_tr).astype(str) - == ytr.astype(str).values).mean()) - # below the mode = malfunction (a healthy teacher on heavily - # imbalanced data may only match the majority rate; a broken - # one falls under it, as TabICL-on-MPS did on airline) - if agree < majority - 0.01: - raise ValueError( - f"teacher below majority on its own context" - f" (agree={agree:.3f}, majority={majority:.3f})") - return est, pred_te, pred_tr, proba_tr, fit_s, attempt_device - except Exception as e: # noqa: BLE001 - last = e - continue - raise last - - -def one_cell(name, X, y, n): - if len(X) < n * 4 // 3: # need n plus a test split - return None - rng = np.random.default_rng(SEED) - idx = rng.choice(len(X), min(len(X), n + n // 3), replace=False) - Xs = X.iloc[idx].reset_index(drop=True) - ys = y.iloc[idx].reset_index(drop=True) - Xtr, Xte, ytr, yte = TA.split(Xs, ys, "cls") - - row = {"dataset": name, "n_train": len(Xtr), "n_test": len(Xte), - "d": Xs.shape[1]} - - t0 = time.time() - row["xgboost"] = TA.score(yte, TA.run_xgb(Xtr, ytr, Xte, "cls"), "cls") - row["xgb_s"] = round(time.time() - t0, 1) - - est, pred_te, pred_tr, proba_tr, fit_s, dev = \ - teacher_with_fallback(Xtr, ytr, Xte) - row["tabicl"] = TA.score(yte, pred_te, "cls") - row["tabicl_s"] = round(fit_s, 1) - row["tabicl_device"] = dev - - # the leak gap: agreement with own training labels minus held-out acc - agree = float((np.asarray(pred_tr).astype(str) - == ytr.astype(str).values).mean()) - row["leak_gap"] = round(agree - row["tabicl"], 4) - row["ic_maxp"] = round(float(np.asarray(proba_tr).max(axis=1).mean()), 4) - - t0 = time.time() - try: - preds, blob, hold = TA.run_ours_distill_teacher( - Xtr, np.asarray(pred_tr), Xte, "cls", kind="gbt") - row["gbt<-tabicl"] = TA.score(yte, preds, "cls") - row["distill_s"] = round(time.time() - t0, 1) - row["blob_kb"] = round(blob / 1024, 1) - except Exception as e: # noqa: BLE001 - if "single class" not in str(e): - raise - row["collapsed"] = 1 - - classes = [str(c) for c in est.classes_] - preds, _, _ = TA.run_ours_distill_soft( - Xtr, ytr, np.asarray(proba_tr), classes, Xte, kind="gbt") - row["gbt<-tabicl soft"] = TA.score(yte, preds, "cls") - - if len(Xtr) <= OOF_AT: - t0 = time.time() - proba_oof, classes_oof = TA.oof_proba(make_teacher, Xtr, ytr) - row["oof_s"] = round(time.time() - t0, 1) - oof_agree = float( - (np.array(classes_oof)[np.asarray(proba_oof).argmax(1)] - == ytr.astype(str).values).mean()) - row["oof_gap"] = round(oof_agree - row["tabicl"], 4) - preds, _, _ = TA.run_ours_distill_soft( - Xtr, ytr, np.asarray(proba_oof), classes_oof, Xte, kind="gbt") - row["gbt<-tabicl soft-oof"] = TA.score(yte, preds, "cls") - return row - - -def run_single_cell(name, n): - """Child-process entry: one (dataset, n) cell, result to the jsonl.""" - if n >= CPU_AT: - # Metal caps a single NDArray at 2**32 bytes and TabICL touches - # MPS internally even with device="cpu" (native SIGABRT, not - # catchable). Make torch report MPS unavailable in this child. - import torch - torch.backends.mps.is_available = lambda: False # noqa: E731 - torch.backends.mps.is_built = lambda: False # noqa: E731 - did = dict((nm, d) for d, nm in LARGE)[name] - X, y = load_full(did, name) - row = one_cell(name, X, y, n) - if row is not None: - with open(JSONL, "a") as f: - f.write(json.dumps(row) + "\n") - print(f" n={row['n_train']}: tabicl={row['tabicl']:.4f}" - f" xgb={row['xgboost']:.4f}" - f" student={row.get('gbt<-tabicl', float('nan')):.4f}" - f" leak={row['leak_gap']:+.4f}") - - -def main(): - """Driver: every cell runs in its own subprocess so a native crash - (seen from torch/MPS at 50k rows) or a hang costs one cell, not the - campaign. A cell that crashed once is skipped, not retried.""" - import subprocess - os.makedirs(ATTEMPTS, exist_ok=True) - done = set() - if os.path.exists(JSONL): - with open(JSONL) as f: - done = {(json.loads(ln)["dataset"], json.loads(ln)["n_train"]) - for ln in f if ln.strip()} - for did, name in LARGE: - for n in SIZES: - if any(d == name and abs(k - n) <= n // 3 for d, k in done): - print(f"{name} n~{n}: cached") - continue - d_est = None - try: - d_est = load_full(did, name)[0].shape[1] - except Exception: # noqa: BLE001 - pass - if d_est and n * d_est > MPS_CELL_BUDGET: - print(f"{name} n={n}: skipped, over mps budget" - f" ({n}x{d_est} > {MPS_CELL_BUDGET})") - continue - marker = os.path.join(ATTEMPTS, f"{name}-{n}") - if os.path.exists(marker): - print(f"{name} n={n}: crashed before, skipping") - continue - open(marker, "w").write("attempting\n") - t0 = time.time() - try: - r = subprocess.run( - [sys.executable, "-u", os.path.abspath(__file__), - "--cell", name, str(n)], - timeout=None if CELL_TIMEOUT_S == 0 else CELL_TIMEOUT_S, - cwd=os.path.dirname(os.path.abspath(__file__)) or ".", - capture_output=True, text=True) - except subprocess.TimeoutExpired as te: - sys.stdout.write((te.stdout or b"").decode(errors="ignore") - if isinstance(te.stdout, bytes) - else (te.stdout or "")) - print(f"{name} n={n}: CELL TIMEOUT after {CELL_TIMEOUT_S}s") - continue - sys.stdout.write(r.stdout) - if r.returncode == 0: - os.remove(marker) - # a completed run may still have skipped (too few rows) - else: - tailerr = (r.stderr or "").strip().splitlines()[-2:] - print(f"{name} n={n}: CELL DIED rc={r.returncode}" - f" after {time.time()-t0:.0f}s :: {' | '.join(tailerr)}") - print(f"{name} n={n}: {time.time()-t0:.0f}s") - report() - - -def report(): - if not os.path.exists(JSONL): - return - with open(JSONL) as f: - rows = [json.loads(ln) for ln in f if ln.strip()] - rows.sort(key=lambda r: (r["dataset"], r["n_train"])) - lines = [ - "# Scale study: distillation beyond 1500 rows", "", - "TabICL v2 as the teacher on the naturally large TabArena", - "datasets, at increasing training sizes. Questions: does student", - "retention hold, where does tuned xgboost catch the zero-shot", - "teacher, and does the in-context label leak grow with context", - "(the gap column; see permissive-teachers.md).", "", - f"Device: {DEVICE} with cpu fallback (per-cell `dev`).", "", - "| dataset | n_train | xgboost | TabICL | gbt<-TabICL | soft |" - " soft-oof | leak gap | oof gap | teacher s | distill s | blob KB |", - "|---|---|---|---|---|---|---|---|---|---|---|---|", - ] - for r in rows: - def fmt(v, nd=3): - return f"{v:.{nd}f}" if isinstance(v, (int, float)) else "-" - lines.append( - f"| {r['dataset']} | {r['n_train']} | {fmt(r.get('xgboost'))} |" - f" {fmt(r.get('tabicl'))} | {fmt(r.get('gbt<-tabicl'))} |" - f" {fmt(r.get('gbt<-tabicl soft'))} |" - f" {fmt(r.get('gbt<-tabicl soft-oof'))} |" - f" {fmt(r.get('leak_gap'), 4)} | {fmt(r.get('oof_gap'), 4)} |" - f" {r.get('tabicl_s', '-')} | {r.get('distill_s', '-')} |" - f" {r.get('blob_kb', '-')} |") - lines.append("") - with open(RESULTS, "w") as f: - f.write("\n".join(lines)) - print(f"wrote {RESULTS}") - - -if __name__ == "__main__": - if len(sys.argv) >= 4 and sys.argv[1] == "--cell": - run_single_cell(sys.argv[2], int(sys.argv[3])) - else: - main() From 1e4936465386c094f38ffdc4944475911f46e746 Mon Sep 17 00:00:00 2001 From: mstrathman Date: Wed, 29 Jul 2026 18:12:09 -0700 Subject: [PATCH 09/11] fix: verify re-derives the serving model; claims narrowed to behavior The skill claimed the reference script re-derives everything from stored fields; it only replayed input_sql and compared hashes, so a tampered model_id or options column still verified. Now verify() re-derives the serving model from the replayed result and fails the verification (exit 2, id_matches false) when it contradicts the recorded model_id; row-form receipts recorded via --model-id report id_matches null since no model is derivable. The options column is declared informational in both the report (options_verified: false) and the skill text: its authoritative copy is the options text inside input_sql, which the replay executes verbatim, and the receipt row itself is self-attested. Adversarial tests cover the detected forgery (model_id) and pin the documented limit (options). Co-Authored-By: Claude Fable 5 --- skills/prediction-receipts/SKILL.md | 15 ++++++--- skills/prediction-receipts/scripts/receipt.py | 27 +++++++++++++--- tests/test_skills.py | 31 +++++++++++++++++++ 3 files changed, 63 insertions(+), 10 deletions(-) diff --git a/skills/prediction-receipts/SKILL.md b/skills/prediction-receipts/SKILL.md index 18fd5cc..9b6d630 100644 --- a/skills/prediction-receipts/SKILL.md +++ b/skills/prediction-receipts/SKILL.md @@ -95,11 +95,16 @@ FROM _predict_receipts WHERE id = 1; `match` is 1 when the replay reproduces the recorded result. This is a spot check of one known query's result hash, not full verification: it -trusts the receipt row's own metadata, so tampered `input_sql`, -`options`, or `model_id` fields are not detected here. For full replay -that re-derives everything from the stored fields and diagnoses what -moved (data, extension version, or model hash), use the reference -script below. On a mismatch, compare the receipt's `extension` against +trusts the receipt row's own metadata. The reference script goes +further: it replays the stored `input_sql`, recomputes the result hash, +re-derives the serving model from the replayed result and compares it +to the recorded `model_id` (aggregate results carry a model; row-form +results recorded via `--model-id` cannot be re-derived), and reports +extension-version and registry-hash drift. Two limits it states rather +than hides: the `options` column is informational (its authoritative +copy is the options text inside `input_sql`, which the replay executes +verbatim), and a receipt row is self-attested, so tamper-proofing the +receipts table itself needs database-level controls. On a mismatch, compare the receipt's `extension` against `predict_version()` and its `content_hash` against the registry. ## The reference script (optional) diff --git a/skills/prediction-receipts/scripts/receipt.py b/skills/prediction-receipts/scripts/receipt.py index 15e81f5..28b34e8 100644 --- a/skills/prediction-receipts/scripts/receipt.py +++ b/skills/prediction-receipts/scripts/receipt.py @@ -159,6 +159,16 @@ def cmd_verify(args): document = canonical_document(db.execute(input_sql), operation) now_sha = sha256_text(document) now_ext = extension_version(db) + # re-derive the serving model from the replayed result where the + # result carries one (aggregate documents do; row-form results were + # recorded via --model-id and cannot be re-derived) + replayed_model = None + try: + replayed_model = json.loads(document).get("model") + except (json.JSONDecodeError, AttributeError): + pass + id_matches = None if replayed_model is None \ + else replayed_model == model_id now_hash = None try: r = db.execute( @@ -167,16 +177,23 @@ def cmd_verify(args): now_hash = r[0] if r else None except sqlite3.OperationalError: pass + hash_match = now_sha == rec_sha report = { "receipt_id": args.receipt_id, - "match": now_sha == rec_sha, - "result_sha256": {"recorded": rec_sha, "replayed": now_sha}, - "extension": {"recorded": rec_ext, "current": now_ext, - "changed": rec_ext != now_ext}, - "model": {"id": model_id, + "match": hash_match and id_matches is not False, + "result_sha256": {"recorded": rec_sha, "replayed": now_sha, + "match": hash_match}, + "model": {"recorded_id": model_id, + "replayed_id": replayed_model, + "id_matches": id_matches, "content_hash_recorded": rec_hash, "content_hash_current": now_hash, "changed": rec_hash != now_hash}, + "extension": {"recorded": rec_ext, "current": now_ext, + "changed": rec_ext != now_ext}, + # options is informational: its authoritative copy is the options + # text embedded in input_sql, which the replay executes verbatim + "options_verified": False, } print(json.dumps(report, indent=2)) db.close() diff --git a/tests/test_skills.py b/tests/test_skills.py index 934a75d..c22faf7 100644 --- a/tests/test_skills.py +++ b/tests/test_skills.py @@ -240,3 +240,34 @@ def test_receipt_adversarial_paths(receipt_db): evil = run_receipt("verify", receipt_db, "1") assert evil.returncode != 0 assert "not authorized" in (evil.stderr + evil.stdout).lower() + + +def test_receipt_metadata_tampering(receipt_db): + """A tampered model_id must fail verification via re-derivation; a + tampered options column is documented as informational (the + authoritative options live inside input_sql), so this test pins that + stated limit rather than pretending it is detection.""" + rec = run_receipt("record", receipt_db, + "SELECT forecast(ts, value, 6) FROM readings") + rid = str(json.loads(rec.stdout)["receipt_id"]) + + db = sqlite3.connect(receipt_db) + db.execute("UPDATE _predict_receipts SET model_id = 'impostor'" + " WHERE id = ?", (rid,)) + db.commit() + db.close() + forged = run_receipt("verify", receipt_db, rid) + assert forged.returncode == 2 + rep = json.loads(forged.stdout) + assert rep["match"] is False + assert rep["model"]["id_matches"] is False + + db = sqlite3.connect(receipt_db) + db.execute("UPDATE _predict_receipts SET model_id = 'theta-classic'," + " options = '{\"forged\":true}' WHERE id = ?", (rid,)) + db.commit() + db.close() + forged_opts = run_receipt("verify", receipt_db, rid) + rep = json.loads(forged_opts.stdout) + assert rep["match"] is True, "options is informational by design" + assert rep["options_verified"] is False From cb0f5c6b1bbb49a52c36929797371c394328218a Mon Sep 17 00:00:00 2001 From: mstrathman Date: Wed, 29 Jul 2026 18:25:13 -0700 Subject: [PATCH 10/11] fix: replay treats stored SQL as untrusted Locking extension loading closed one door and left the write door open: a tampered receipt's input_sql could DELETE, DROP, ATTACH, or flip pragmas during verification. verify() and list now open the database read-only at the file level and install an authorizer that permits only reads and function calls; every legitimate replay (aggregates, predict, backtest) is pure and passes, every write shape is denied at prepare time. Adversarial tests replay DELETE, DROP, ATTACH, and PRAGMA receipts, assert refusal, and assert the verifier's data is untouched. The skill states the replay posture explicitly. Co-Authored-By: Claude Fable 5 --- skills/prediction-receipts/SKILL.md | 14 +++++--- skills/prediction-receipts/scripts/receipt.py | 34 +++++++++++++++---- tests/test_skills.py | 29 ++++++++++++++++ 3 files changed, 65 insertions(+), 12 deletions(-) diff --git a/skills/prediction-receipts/SKILL.md b/skills/prediction-receipts/SKILL.md index 9b6d630..c8ab2e2 100644 --- a/skills/prediction-receipts/SKILL.md +++ b/skills/prediction-receipts/SKILL.md @@ -100,11 +100,15 @@ further: it replays the stored `input_sql`, recomputes the result hash, re-derives the serving model from the replayed result and compares it to the recorded `model_id` (aggregate results carry a model; row-form results recorded via `--model-id` cannot be re-derived), and reports -extension-version and registry-hash drift. Two limits it states rather -than hides: the `options` column is informational (its authoritative -copy is the options text inside `input_sql`, which the replay executes -verbatim), and a receipt row is self-attested, so tamper-proofing the -receipts table itself needs database-level controls. On a mismatch, compare the receipt's `extension` against +extension-version and registry-hash drift. Replay treats the stored SQL +as untrusted: verification runs on a read-only connection with an +authorizer that permits only reads and function calls, so a tampered +receipt cannot write, drop, attach, or change pragmas on the verifier's +database. Two limits it states rather than hides: the `options` column +is informational (its authoritative copy is the options text inside +`input_sql`, which the replay executes verbatim), and a receipt row is +self-attested, so trusting what a receipt claims about itself, as +opposed to what replay proves, needs database-level access controls. On a mismatch, compare the receipt's `extension` against `predict_version()` and its `content_hash` against the registry. ## The reference script (optional) diff --git a/skills/prediction-receipts/scripts/receipt.py b/skills/prediction-receipts/scripts/receipt.py index 28b34e8..69f6f7f 100644 --- a/skills/prediction-receipts/scripts/receipt.py +++ b/skills/prediction-receipts/scripts/receipt.py @@ -51,18 +51,38 @@ def fail(code, msg): sys.exit(1) -def connect(db_path, extension): - db = sqlite3.connect(db_path) +#: authorizer allow-list for replaying untrusted SQL: reads and function +#: calls only. Everything else (writes, DDL, ATTACH, PRAGMA) is denied. +_REPLAY_ALLOWED = { + getattr(sqlite3, name, None) for name in + ("SQLITE_SELECT", "SQLITE_READ", "SQLITE_FUNCTION", "SQLITE_RECURSIVE") +} - {None} + + +def _replay_authorizer(action, *_): + return sqlite3.SQLITE_OK if action in _REPLAY_ALLOWED \ + else sqlite3.SQLITE_DENY + + +def connect(db_path, extension, untrusted=False): + """untrusted=True is the replay posture: the database opens + read-only at the file level and an authorizer denies everything but + reads and function calls, so a tampered receipt's input_sql cannot + write, drop, attach, or change pragmas on the verifier's database.""" + if untrusted: + db = sqlite3.connect(f"file:{db_path}?mode=ro", uri=True) + else: + db = sqlite3.connect(db_path) db.enable_load_extension(True) try: db.load_extension(extension) except sqlite3.OperationalError as e: fail("RECEIPT_ERR_EXTENSION", f"cannot load '{extension}': {e}") finally: - # verify() replays stored SQL; with loading left enabled, a - # tampered receipt could call load_extension() on an arbitrary - # local library. One load, then locked. + # a tampered receipt must not be able to load a local library db.enable_load_extension(False) + if untrusted: + db.set_authorizer(_replay_authorizer) return db @@ -148,7 +168,7 @@ def cmd_record(args): def cmd_verify(args): - db = connect(args.db, args.extension) + db = connect(args.db, args.extension, untrusted=True) row = db.execute( "SELECT operation, input_sql, model_id, content_hash, extension," " result_sha256 FROM _predict_receipts WHERE id = ?", @@ -201,7 +221,7 @@ def cmd_verify(args): def cmd_list(args): - db = connect(args.db, args.extension) + db = connect(args.db, args.extension, untrusted=True) try: rows = db.execute( "SELECT id, created_at, operation, model_id, result_sha256" diff --git a/tests/test_skills.py b/tests/test_skills.py index c22faf7..89ef203 100644 --- a/tests/test_skills.py +++ b/tests/test_skills.py @@ -271,3 +271,32 @@ def test_receipt_metadata_tampering(receipt_db): rep = json.loads(forged_opts.stdout) assert rep["match"] is True, "options is informational by design" assert rep["options_verified"] is False + + +def test_receipt_replay_cannot_write(receipt_db): + """Replay input is untrusted: a tampered input_sql that writes, + drops, or attaches must be refused by the read-only replay + connection, and the verifier's data must be untouched.""" + rec = run_receipt("record", receipt_db, + "SELECT forecast(ts, value, 6) FROM readings") + rid = str(json.loads(rec.stdout)["receipt_id"]) + + for attack in ("DELETE FROM readings", + "DROP TABLE readings", + "ATTACH DATABASE '/tmp/exfil.db' AS x", + "PRAGMA journal_mode = OFF"): + db = sqlite3.connect(receipt_db) + db.execute("UPDATE _predict_receipts SET input_sql = ?" + " WHERE id = ?", (attack, rid)) + db.commit() + db.close() + out = run_receipt("verify", receipt_db, rid) + assert out.returncode != 0, f"replay accepted: {attack}" + blob = (out.stderr + out.stdout).lower() + assert ("not authorized" in blob or "readonly" in blob + or "read-only" in blob or "prohibited" in blob), attack + + db = sqlite3.connect(receipt_db) + (n,) = db.execute("SELECT count(*) FROM readings").fetchone() + db.close() + assert n == 48, "replay modified the verifier's data" From db6046c1604ec59adbe2db47b1aa92d244ba8e83 Mon Sep 17 00:00:00 2001 From: mstrathman Date: Wed, 29 Jul 2026 18:28:31 -0700 Subject: [PATCH 11/11] fix: complete the untrusted-replay audit in one pass Rather than waiting for the next review round to find the next adjacent gap, this closes the full remaining surface found by an adversarial self-audit of the receipt tool: a VM-step budget aborts runaway replays (recursive CTE bombs) instead of hanging the verifier; BLOB cells canonicalize deterministically as their sha256 instead of crashing json serialization; --options must parse as JSON before anything executes; database-open and SQL failures exit with stable RECEIPT_ERR_* codes instead of tracebacks. Every path has a test: the bomb aborts under a small step budget, blob receipts round-trip deterministically, malformed options and missing databases fail clean. Co-Authored-By: Claude Fable 5 --- skills/prediction-receipts/scripts/receipt.py | 43 ++++++++++++--- tests/test_skills.py | 52 +++++++++++++++++++ 2 files changed, 87 insertions(+), 8 deletions(-) diff --git a/skills/prediction-receipts/scripts/receipt.py b/skills/prediction-receipts/scripts/receipt.py index 69f6f7f..eb2c784 100644 --- a/skills/prediction-receipts/scripts/receipt.py +++ b/skills/prediction-receipts/scripts/receipt.py @@ -69,10 +69,13 @@ def connect(db_path, extension, untrusted=False): read-only at the file level and an authorizer denies everything but reads and function calls, so a tampered receipt's input_sql cannot write, drop, attach, or change pragmas on the verifier's database.""" - if untrusted: - db = sqlite3.connect(f"file:{db_path}?mode=ro", uri=True) - else: - db = sqlite3.connect(db_path) + try: + if untrusted: + db = sqlite3.connect(f"file:{db_path}?mode=ro", uri=True) + else: + db = sqlite3.connect(db_path) + except sqlite3.Error as e: + fail("RECEIPT_ERR_DB", f"cannot open '{db_path}': {e}") db.enable_load_extension(True) try: db.load_extension(extension) @@ -83,6 +86,10 @@ def connect(db_path, extension, untrusted=False): db.enable_load_extension(False) if untrusted: db.set_authorizer(_replay_authorizer) + # a tampered receipt must not be able to hang the verifier + # either: abort replay after a generous VM-step budget + steps = int(os.environ.get("RECEIPT_MAX_STEPS", 50_000_000)) + db.set_progress_handler(lambda: 1, steps) return db @@ -99,7 +106,16 @@ def canonical_document(cur, operation): if (operation in AGGREGATE_OPS and len(rows) == 1 and len(cols) == 1 and isinstance(rows[0][0], str)): return rows[0][0] - return json.dumps({"columns": cols, "rows": [list(r) for r in rows]}, + + def cell(v): + # BLOBs canonicalize as their hash: deterministic, bounded, and + # json.dumps cannot serialize bytes anyway + if isinstance(v, (bytes, memoryview)): + return {"blob_sha256": + hashlib.sha256(bytes(v)).hexdigest()} + return v + return json.dumps({"columns": cols, + "rows": [[cell(v) for v in r] for r in rows]}, separators=(",", ":"), ensure_ascii=False) @@ -144,14 +160,22 @@ def model_fields(db, document, model_flag): def cmd_record(args): + if args.options is not None: + try: + json.loads(args.options) + except json.JSONDecodeError as e: + fail("RECEIPT_ERR_OPTIONS", f"--options is not valid JSON: {e}") db = connect(args.db, args.extension) operation = args.operation or infer_operation(args.sql) if operation is None: fail("RECEIPT_ERR_OPERATION_UNKNOWN", f"cannot infer the operation; pass --operation one of" f" {', '.join(OPERATIONS)}") - cur = db.execute(args.sql) - document = canonical_document(cur, operation) + try: + cur = db.execute(args.sql) + document = canonical_document(cur, operation) + except sqlite3.Error as e: + fail("RECEIPT_ERR_SQL", str(e)) model_id, content_hash = model_fields(db, document, args.model_id) db.execute(DDL) cur = db.execute( @@ -176,7 +200,10 @@ def cmd_verify(args): if row is None: fail("RECEIPT_ERR_NOT_FOUND", f"no receipt with id {args.receipt_id}") operation, input_sql, model_id, rec_hash, rec_ext, rec_sha = row - document = canonical_document(db.execute(input_sql), operation) + try: + document = canonical_document(db.execute(input_sql), operation) + except sqlite3.Error as e: + fail("RECEIPT_ERR_SQL", f"replay refused or failed: {e}") now_sha = sha256_text(document) now_ext = extension_version(db) # re-derive the serving model from the replayed result where the diff --git a/tests/test_skills.py b/tests/test_skills.py index 89ef203..28b615d 100644 --- a/tests/test_skills.py +++ b/tests/test_skills.py @@ -300,3 +300,55 @@ def test_receipt_replay_cannot_write(receipt_db): (n,) = db.execute("SELECT count(*) FROM readings").fetchone() db.close() assert n == 48, "replay modified the verifier's data" + + +def test_receipt_hardening_paths(receipt_db, tmp_path): + """The audit set: runaway replay is aborted by the step budget, BLOB + cells canonicalize deterministically instead of crashing, invalid + --options is rejected up front, and a missing database fails with a + clean code instead of a traceback.""" + rec = run_receipt("record", receipt_db, + "SELECT forecast(ts, value, 6) FROM readings") + rid = str(json.loads(rec.stdout)["receipt_id"]) + + # recursive CTE bomb: must abort via the step budget, not hang + db = sqlite3.connect(receipt_db) + db.execute("UPDATE _predict_receipts SET input_sql =" + " 'WITH RECURSIVE b(x) AS (SELECT 1 UNION ALL SELECT x+1" + " FROM b) SELECT max(x) FROM b' WHERE id = ?", (rid,)) + db.commit() + db.close() + import subprocess as sp + bomb = sp.run([sys.executable, RECEIPT, "--extension", EXT, + "verify", receipt_db, rid], + capture_output=True, text=True, timeout=60, + env={**os.environ, "RECEIPT_MAX_STEPS": "100000"}) + assert bomb.returncode != 0 + assert "RECEIPT_ERR_SQL" in bomb.stderr + + # BLOB cells canonicalize as their hash, deterministically + db = sqlite3.connect(receipt_db) + db.execute("CREATE TABLE blobs(b BLOB)") + db.execute("INSERT INTO blobs VALUES (x'00ff10')") + db.commit() + db.close() + b1 = run_receipt("record", receipt_db, "SELECT b FROM blobs", + "--operation", "predict", "--model-id", "n/a") + b2 = run_receipt("record", receipt_db, "SELECT b FROM blobs", + "--operation", "predict", "--model-id", "n/a") + assert b1.returncode == 0, b1.stderr + assert json.loads(b1.stdout)["result_sha256"] == \ + json.loads(b2.stdout)["result_sha256"] + + # invalid --options rejected before anything executes + badopt = run_receipt("record", receipt_db, + "SELECT forecast(ts, value, 6) FROM readings", + "--options", "{not json") + assert badopt.returncode != 0 + assert "RECEIPT_ERR_OPTIONS" in badopt.stderr + + # missing database file fails cleanly on the read-only path + gone_db = run_receipt("verify", str(tmp_path / "nope.db"), "1") + assert gone_db.returncode != 0 + assert "RECEIPT_ERR" in gone_db.stderr + assert "Traceback" not in gone_db.stderr