diff --git a/.gitignore b/.gitignore index 4179045..a7c9278 100644 --- a/.gitignore +++ b/.gitignore @@ -57,3 +57,6 @@ bindings/node/node_modules/ bindings/rust/csrc/ bindings/rust/target/ bindings/rust/Cargo.lock + +# autogluon working dirs (Mitra benchmark runs dump model copies here) +benchmarks/AutogluonModels/ diff --git a/README.md b/README.md index 79e7775..e27ca53 100644 --- a/README.md +++ b/README.md @@ -139,6 +139,17 @@ permissively licensed (MIT/Apache-2.0), so you ship it inside your product rather than rent it. And it composes with the rest of the in-database AI toolbox: sqlite-vec gave SQLite vector search; this gives it prediction. +### Agent skills + +For agents, the SQL surface is the whole API, so what ships in +[`skills/`](skills/) is judgment, not wrappers: four +[agentskills.io](https://agentskills.io)-format skills covering core +usage, provenance receipts (record what was asked, which model answered, +and a hash that replays; serving determinism makes this verifiable), +the distillation lifecycle (verify before serving, re-distill on drift, +license discipline), and backtest interpretation. Install them into any +skills-capable agent, or read them as the condensed operator's manual. + ## Operations | Function | Question | Returns | diff --git a/benchmarks/results/scale-arena.jsonl b/benchmarks/results/scale-arena.jsonl new file mode 100644 index 0000000..d211654 --- /dev/null +++ b/benchmarks/results/scale-arena.jsonl @@ -0,0 +1,13 @@ +{"dataset": "Diabetes130US", "n_train": 1500, "n_test": 500, "d": 40, "xgboost": 0.906, "xgb_s": 0.1, "tabicl": 0.916, "tabicl_s": 2.2, "tabicl_device": "mps", "leak_gap": 0.0007, "ic_maxp": 0.9251, "collapsed": 1, "gbt<-tabicl soft": 0.916, "oof_s": 7.6, "oof_gap": 0.0007, "gbt<-tabicl soft-oof": 0.916} +{"dataset": "Diabetes130US", "n_train": 9999, "n_test": 3334, "d": 40, "xgboost": 0.9070185962807439, "xgb_s": 0.1, "tabicl": 0.9130173965206959, "tabicl_s": 101.9, "tabicl_device": "mps", "leak_gap": -0.0, "ic_maxp": 0.922, "collapsed": 1, "gbt<-tabicl soft": 0.9130173965206959, "oof_s": 116.4, "oof_gap": -0.0, "gbt<-tabicl soft-oof": 0.9130173965206959} +{"dataset": "airline_satisfaction", "n_train": 1500, "n_test": 500, "d": 21, "xgboost": 0.91, "xgb_s": 0.0, "tabicl": 0.928, "tabicl_s": 1.6, "tabicl_device": "mps", "leak_gap": 0.0693, "ic_maxp": 0.8787, "gbt<-tabicl": 0.908, "distill_s": 0.5, "blob_kb": 103.4, "gbt<-tabicl soft": 0.904, "oof_s": 5.5, "oof_gap": -0.0073, "gbt<-tabicl soft-oof": 0.906} +{"dataset": "GiveMeSomeCredit", "n_train": 1500, "n_test": 500, "d": 10, "xgboost": 0.922, "xgb_s": 0.0, "tabicl": 0.936, "tabicl_s": 1.2, "tabicl_device": "mps", "leak_gap": 0.0067, "ic_maxp": 0.9407, "gbt<-tabicl": 0.932, "distill_s": 0.3, "blob_kb": 97.1, "gbt<-tabicl soft": 0.926, "oof_s": 4.1, "oof_gap": -0.0113, "gbt<-tabicl soft-oof": 0.93} +{"dataset": "GiveMeSomeCredit", "n_train": 9999, "n_test": 3334, "d": 10, "xgboost": 0.9289142171565686, "xgb_s": 0.1, "tabicl": 0.9322135572885423, "tabicl_s": 8.4, "tabicl_device": "mps", "leak_gap": 0.021, "ic_maxp": 0.9452, "gbt<-tabicl": 0.9334133173365327, "distill_s": 2.6, "blob_kb": 104.3, "gbt<-tabicl soft": 0.9334133173365327, "oof_s": 29.5, "oof_gap": 0.0009, "gbt<-tabicl soft-oof": 0.9322135572885423} +{"dataset": "APSFailure", "n_train": 1500, "n_test": 500, "d": 40, "xgboost": 0.994, "xgb_s": 0.1, "tabicl": 0.996, "tabicl_s": 2.6, "tabicl_device": "mps", "leak_gap": -0.004, "ic_maxp": 0.9902, "gbt<-tabicl": 0.99, "distill_s": 1.4, "blob_kb": 76.6, "gbt<-tabicl soft": 0.99, "oof_s": 8.2, "oof_gap": -0.0093, "gbt<-tabicl soft-oof": 0.996} +{"dataset": "SDSS17", "n_train": 1500, "n_test": 500, "d": 11, "xgboost": 0.97, "xgb_s": 0.1, "tabicl": 0.972, "tabicl_s": 1.4, "tabicl_device": "mps", "leak_gap": 0.0167, "ic_maxp": 0.9892, "gbt<-tabicl": 0.968, "distill_s": 1.1, "blob_kb": 145.8, "gbt<-tabicl soft": 0.968, "oof_s": 4.3, "oof_gap": 0.0007, "gbt<-tabicl soft-oof": 0.964} +{"dataset": "SDSS17", "n_train": 9999, "n_test": 3334, "d": 11, "xgboost": 0.9679064187162567, "xgb_s": 0.3, "tabicl": 0.9718056388722256, "tabicl_s": 8.7, "tabicl_device": "mps", "leak_gap": 0.0205, "ic_maxp": 0.9906, "gbt<-tabicl": 0.9658068386322736, "distill_s": 9.2, "blob_kb": 159.2, "gbt<-tabicl soft": 0.9661067786442712, "oof_s": 30.3, "oof_gap": 0.0052, "gbt<-tabicl soft-oof": 0.9676064787042592} +{"dataset": "taiwanese_bankruptcy", "n_train": 1500, "n_test": 500, "d": 40, "xgboost": 0.966, "xgb_s": 0.1, "tabicl": 0.968, "tabicl_s": 2.4, "tabicl_device": "mps", "leak_gap": 0.004, "ic_maxp": 0.9721, "gbt<-tabicl": 0.968, "distill_s": 2.9, "blob_kb": 93.8, "gbt<-tabicl soft": 0.966, "oof_s": 8.2, "oof_gap": 0.0007, "gbt<-tabicl soft-oof": 0.968} +{"dataset": "online_shoppers_intention", "n_train": 1500, "n_test": 500, "d": 17, "xgboost": 0.882, "xgb_s": 0.1, "tabicl": 0.89, "tabicl_s": 1.5, "tabicl_device": "mps", "leak_gap": 0.0507, "ic_maxp": 0.8863, "gbt<-tabicl": 0.89, "distill_s": 0.4, "blob_kb": 100.9, "gbt<-tabicl soft": 0.888, "oof_s": 4.9, "oof_gap": 0.0153, "gbt<-tabicl soft-oof": 0.886} +{"dataset": "credit_card_default", "n_train": 1500, "n_test": 500, "d": 23, "xgboost": 0.81, "xgb_s": 0.1, "tabicl": 0.836, "tabicl_s": 1.7, "tabicl_device": "mps", "leak_gap": -0.0247, "ic_maxp": 0.8101, "gbt<-tabicl": 0.798, "distill_s": 0.9, "blob_kb": 93.5, "gbt<-tabicl soft": 0.81, "oof_s": 5.8, "oof_gap": -0.0153, "gbt<-tabicl soft-oof": 0.838} +{"dataset": "airline_satisfaction", "n_train": 9999, "n_test": 3334, "d": 21, "xgboost": 0.9448110377924415, "xgb_s": 0.3, "tabicl": 0.9511097780443911, "tabicl_s": 139.6, "tabicl_device": "cpu", "leak_gap": 0.0465, "ic_maxp": 0.9842, "gbt<-tabicl": 0.9337132573485303, "distill_s": 3.5, "blob_kb": 105.7, "gbt<-tabicl soft": 0.9340131973605279, "oof_s": 486.7, "oof_gap": 0.0043, "gbt<-tabicl soft-oof": 0.9337132573485303} +{"dataset": "credit_card_default", "n_train": 9999, "n_test": 3334, "d": 23, "xgboost": 0.8047390521895621, "xgb_s": 0.2, "tabicl": 0.8191361727654469, "tabicl_s": 167.5, "tabicl_device": "cpu", "leak_gap": 0.0205, "ic_maxp": 0.8387, "gbt<-tabicl": 0.8197360527894421, "distill_s": 7.4, "blob_kb": 96.8, "gbt<-tabicl soft": 0.8182363527294542, "oof_s": 571.6, "oof_gap": -0.001, "gbt<-tabicl soft-oof": 0.8194361127774445} diff --git a/benchmarks/results/scale.md b/benchmarks/results/scale.md new file mode 100644 index 0000000..3863b44 --- /dev/null +++ b/benchmarks/results/scale.md @@ -0,0 +1,25 @@ +# Scale study: distillation beyond 1500 rows + +TabICL v2 as the teacher on the naturally large TabArena +datasets, at increasing training sizes. Questions: does student +retention hold, where does tuned xgboost catch the zero-shot +teacher, and does the in-context label leak grow with context +(the gap column; see permissive-teachers.md). + +Device: mps with cpu fallback (per-cell `dev`). + +| dataset | n_train | xgboost | TabICL | gbt<-TabICL | soft | soft-oof | leak gap | oof gap | teacher s | distill s | blob KB | +|---|---|---|---|---|---|---|---|---|---|---|---| +| APSFailure | 1500 | 0.994 | 0.996 | 0.990 | 0.990 | 0.996 | -0.0040 | -0.0093 | 2.6 | 1.4 | 76.6 | +| Diabetes130US | 1500 | 0.906 | 0.916 | - | 0.916 | 0.916 | 0.0007 | 0.0007 | 2.2 | - | - | +| Diabetes130US | 9999 | 0.907 | 0.913 | - | 0.913 | 0.913 | -0.0000 | -0.0000 | 101.9 | - | - | +| GiveMeSomeCredit | 1500 | 0.922 | 0.936 | 0.932 | 0.926 | 0.930 | 0.0067 | -0.0113 | 1.2 | 0.3 | 97.1 | +| GiveMeSomeCredit | 9999 | 0.929 | 0.932 | 0.933 | 0.933 | 0.932 | 0.0210 | 0.0009 | 8.4 | 2.6 | 104.3 | +| SDSS17 | 1500 | 0.970 | 0.972 | 0.968 | 0.968 | 0.964 | 0.0167 | 0.0007 | 1.4 | 1.1 | 145.8 | +| SDSS17 | 9999 | 0.968 | 0.972 | 0.966 | 0.966 | 0.968 | 0.0205 | 0.0052 | 8.7 | 9.2 | 159.2 | +| SDSS17 | 49999 | 0.975 | 0.610 | - | 0.610 | - | -0.0000 | - | 118.9 | - | - | +| airline_satisfaction | 1500 | 0.910 | 0.928 | 0.908 | 0.904 | 0.906 | 0.0693 | -0.0073 | 1.6 | 0.5 | 103.4 | +| credit_card_default | 1500 | 0.810 | 0.836 | 0.798 | 0.810 | 0.838 | -0.0247 | -0.0153 | 1.7 | 0.9 | 93.5 | +| credit_card_default | 9999 | 0.805 | 0.771 | 0.776 | 0.776 | 0.776 | 0.0019 | 0.0054 | 11.7 | 7.4 | 96.2 | +| online_shoppers_intention | 1500 | 0.882 | 0.890 | 0.890 | 0.888 | 0.886 | 0.0507 | 0.0153 | 1.5 | 0.4 | 100.9 | +| taiwanese_bankruptcy | 1500 | 0.966 | 0.968 | 0.968 | 0.966 | 0.968 | 0.0040 | 0.0007 | 2.4 | 2.9 | 93.8 | diff --git a/skills/distill-lifecycle/SKILL.md b/skills/distill-lifecycle/SKILL.md new file mode 100644 index 0000000..69df48e --- /dev/null +++ b/skills/distill-lifecycle/SKILL.md @@ -0,0 +1,84 @@ +--- +name: distill-lifecycle +description: >- + Distill a teacher model into a native sqlite-predict student and manage + it over time: verify quality before serving, re-distill on drift, and + respect the teacher's license. Use when per-call serving needs to be + instant and self-contained, when a prediction runs at volume, or when a + model must travel inside the database file. +license: MIT OR Apache-2.0 +metadata: + version: "0.1.0" +--- + +# The distillation lifecycle + +Distillation compresses a teacher (your own labels, your existing model's +predictions, or a licensed foundation model) into a native student the +serving core executes with no runtime. Measured on the repo's benchmark +suite: student blobs run tens to hundreds of kilobytes, single-row +serving is sub-millisecond on CPU, and fits complete in seconds at +benchmark scale (see `benchmarks/results/` in the sqlite-predict repo). +The lifecycle judgment is yours. + +## Distill + +```sql +-- tabular: target column holds labels or your model's predictions +SELECT model_id, holdout_metric FROM distill_predict( + 'SELECT f1, f2, label FROM training', + '{"target":"label","student_id":"churn-v1","student_kind":"gbt"}'); + +-- time series: from windows, or with a registered onnx teacher +SELECT model_id FROM distill_forecast('SELECT series_key, value FROM obs', + '{"context":96,"horizon":24,"student_id":"traffic-v1"}'); +``` + +Students register in `_predict_models` as content-hashed rows; they +snapshot, fork, and sync with the database. Serve by name: +`predict(NULL, apply_sql, '{"model":"churn-v1"}')` or +`forecast(ts, value, 24, '{"model":"traffic-v1"}')`. + +## Verify before serving + +Never serve a student on faith: + +1. Read `holdout_metric` from the distill call. It is measured on a + holdout of your data. Compare it to a floor you trust (the majority + class, last-value carry-forward, or your current model). +2. For forecast students, run `backtest` on the same series and compare + the student against `theta-classic` and the naive floor (see the + interpret-backtest skill). +3. Know the measured shape of distillation loss: on our benchmark suite + classification students give up a median half point of accuracy + against their teacher, but regression tails are worse. Check + regression students more skeptically. +4. Soft-label distillation (`proba`/`classes`) preserves the teacher's + calibration when the teacher emits probabilities, and usually rescues + imbalanced datasets where hard labels collapse to one class (the + distiller refuses loudly on collapse rather than fitting a + constant). + +## Watch for drift, re-distill cheaply + +A student is frozen; the world is not. When input distributions move, +quality decays silently, so schedule verification rather than assuming: + +- Re-run `backtest` (forecast) or score a fresh labeled sample (tabular) + on a cadence proportional to how fast the data changes. +- Re-distilling costs seconds. Register the new student under a + versioned id (`churn-v2`), verify, then switch the serving call. Keep + the old row until the new one is trusted; retire it after. + +## Licensing is part of the lifecycle + +A student derives from its teacher, and the teacher's license may +impose obligations on what you distill and distribute; what applies +depends on the exact weights, license version, and how the student is +used. Record the teacher's license alongside each student you register. +Your own labels and models carry no such limits. Restrictively licensed +teachers require an explicit `accept_license` opt-in before they will +run, but that gate is an acknowledgment, not compliance: read the +license of the exact weights you use before distilling for anything +beyond evaluation. The project documentation's license notes are +orientation, not legal advice. diff --git a/skills/interpret-backtest/SKILL.md b/skills/interpret-backtest/SKILL.md new file mode 100644 index 0000000..798c24e --- /dev/null +++ b/skills/interpret-backtest/SKILL.md @@ -0,0 +1,79 @@ +--- +name: interpret-backtest +description: >- + Read sqlite-predict backtest() output to choose models, trust or + distrust prediction intervals, and decide when auto-selection needs + narrowing. Use before serving forecasts that feed decisions, when + intervals matter, or when choosing between models on a specific series. +license: MIT OR Apache-2.0 +metadata: + version: "0.1.0" +--- + +# Interpreting backtest() + +`backtest` answers "how would this model have done on my series" with +rolling-origin evaluation: it repeatedly hides the tail of the series, +forecasts it, and scores against what actually happened. + +```sql +SELECT * FROM backtest('SELECT ts, value FROM readings', 24, + '{"model":"theta-classic","folds":5}'); +``` + +## The metrics that matter + +- **MASE** is the headline: error relative to the seasonal-naive floor. + Below 1.0 beats naive; above 1.0 means the model is losing to + last-season carry-forward and you should not serve it on this series. + Compare models by MASE on the same series and folds. +- **Coverage** is the fraction of actuals that landed inside the + prediction interval. Compare it to the confidence level you asked for: + a 0.90 band with 0.60 measured coverage is lying to you. +- Per-fold rows expose stability. A model that wins on average but + swings wildly across folds is riskier than a slightly worse, steady + one. + +## Interval judgment + +The default Gaussian band can be overconfident on smooth series: the +sqlite-predict benchmarks measured 0.57 coverage at a nominal 0.90 for +theta-class models on smooth gluonts series (`benchmarks/results/` in +the repo). When interval truth matters, request conformal intervals in +the same call: + +```sql +SELECT * FROM backtest('SELECT ts, value FROM readings', 24, + '{"model":"theta-classic", + "interval_method":"conformal"}'); +``` + +Conformal intervals calibrate to measured out-of-sample residuals and +substantially improve empirical coverage (they reached the nominal level +in our benchmarks, but finite folds carry no guarantee). They need +enough history to calibrate and apply to the statistical models only; a +short series makes the call refuse rather than fabricate. Always verify +with `backtest` coverage on your own series rather than trusting either +method's reputation, ours included. + +## Auto-selection judgment + +Bare `forecast(ts, value, h)` lets `auto` pick per series by rolling- +origin MASE. Use `backtest` when you want to see what auto sees: + +- If one model wins consistently across your series, pin it + (`'{"model":"theta-classic"}'`) and save the selection cost. +- If the pool is polluted (a distilled student trained for a different + regime keeps winning on stale patterns), narrow it: + `'{"candidates":["theta-classic","tsb"]}'`. +- Intermittent series (many zeros, sporadic demand) are `tsb` territory; + if auto is not picking it, check whether the series reaches it and + consider pinning. + +## Reading degraded outcomes + +`backtest` needs enough history for its folds: expect loud errors or +degraded statuses on short series rather than fabricated confidence. +That is the tool working. A series too short to backtest is a series too +short to trust a model on; fall back to wider intervals and say so in +whatever the forecast feeds. diff --git a/skills/prediction-receipts/SKILL.md b/skills/prediction-receipts/SKILL.md new file mode 100644 index 0000000..c8ab2e2 --- /dev/null +++ b/skills/prediction-receipts/SKILL.md @@ -0,0 +1,144 @@ +--- +name: prediction-receipts +description: >- + Record verifiable provenance for predictions made with sqlite-predict: + what was asked, which model answered, and a hash that lets the result + be replayed and checked later. Use when a prediction feeds a decision + worth auditing, when results are shared with other agents or humans, + or when you need to prove a result was not altered after the fact. +license: MIT OR Apache-2.0 +metadata: + version: "0.1.0" +--- + +# Prediction receipts + +sqlite-predict serving is deterministic and pure: the same rows, options, +and model produce byte-identical result documents, and registered model +weights are pinned by content hash. That makes provenance a recording +problem the agent can own; the only support the extension provides is +the `predict_sha256()` hash utility, so the whole workflow can run in +pure SQL. + +Record receipts for predictions that matter (decisions, reports, handoffs), +not for every exploratory call. + +## The receipts table + +Create it in your own database (or a sidecar). Convention, so receipts +from different agents interoperate: + +```sql +CREATE TABLE IF NOT EXISTS _predict_receipts ( + id INTEGER PRIMARY KEY, + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')), + operation TEXT NOT NULL, -- 'forecast' | 'detect_anomalies' | 'predict' | 'backtest' + input_sql TEXT NOT NULL, -- the exact SQL that supplied the rows + options TEXT, -- the options JSON as passed, or NULL + model_id TEXT NOT NULL, -- the "model" field of the result document + content_hash TEXT, -- registry hash for a registered model, else NULL + extension TEXT NOT NULL, -- "extension" field of predict_version() + result_sha256 TEXT NOT NULL -- SHA-256 of the exact result document text +); +``` + +## Recording in pure SQL + +The extension exposes `predict_sha256()`, a plain SHA-256 over TEXT or +BLOB. Two different provenance values use it: `result_sha256` hashes the +result document (computed here), while `content_hash` pins the model's +weights (computed by the registry at registration; you only copy it). +Never conflate the two. The whole workflow runs in SQL: + +```sql +INSERT INTO _predict_receipts (operation, input_sql, options, model_id, + content_hash, extension, result_sha256) +SELECT 'forecast', + 'SELECT forecast(ts, value, 24) FROM readings', + NULL, + json_extract(doc, '$.model'), + (SELECT content_hash FROM _predict_models + WHERE model_id = json_extract(doc, '$.model')), + json_extract(predict_version(), '$.extension'), + predict_sha256(doc) +FROM (SELECT (SELECT forecast(ts, value, 24) FROM readings) AS doc); +``` + +The inner `input_sql` string and the query that computes `doc` must be +the same SQL, verbatim; that is what makes the receipt replayable. + +If nothing has been distilled or registered, `_predict_models` does not +exist and the statement above fails on the subquery. Use this variant, +which records `NULL` for `content_hash`: + +```sql +INSERT INTO _predict_receipts (operation, input_sql, options, model_id, + content_hash, extension, result_sha256) +SELECT 'forecast', + 'SELECT forecast(ts, value, 24) FROM readings', + NULL, + json_extract(doc, '$.model'), + NULL, + json_extract(predict_version(), '$.extension'), + predict_sha256(doc) +FROM (SELECT (SELECT forecast(ts, value, 24) FROM readings) AS doc); +``` + +## Verifying in pure SQL + +```sql +SELECT id, + result_sha256 = predict_sha256( + (SELECT forecast(ts, value, 24) FROM readings)) AS match +FROM _predict_receipts WHERE id = 1; +``` + +`match` is 1 when the replay reproduces the recorded result. This is a +spot check of one known query's result hash, not full verification: it +trusts the receipt row's own metadata. The reference script goes +further: it replays the stored `input_sql`, recomputes the result hash, +re-derives the serving model from the replayed result and compares it +to the recorded `model_id` (aggregate results carry a model; row-form +results recorded via `--model-id` cannot be re-derived), and reports +extension-version and registry-hash drift. Replay treats the stored SQL +as untrusted: verification runs on a read-only connection with an +authorizer that permits only reads and function calls, so a tampered +receipt cannot write, drop, attach, or change pragmas on the verifier's +database. Two limits it states rather than hides: the `options` column +is informational (its authoritative copy is the options text inside +`input_sql`, which the replay executes verbatim), and a receipt row is +self-attested, so trusting what a receipt claims about itself, as +opposed to what replay proves, needs database-level access controls. On a mismatch, compare the receipt's `extension` against +`predict_version()` and its `content_hash` against the registry. + +## The reference script (optional) + +For row-shaped results (`predict`, `backtest`) the document needs +canonical serialization, and pure SQL float formatting is not +round-trip safe. Use the bundled script for those, or when you want +mismatch diagnostics computed for you: + +```text +scripts/receipt.py record DB "SELECT forecast(ts, value, 24) FROM t" +scripts/receipt.py verify DB RECEIPT_ID # exit 0 match, 2 mismatch +scripts/receipt.py list DB +``` + +Python stdlib only; pass the loadable with --extension or +SQLITE_PREDICT_EXTENSION. It creates the table, canonicalizes (the +aggregate document verbatim; rows as compact JSON with shortest +round-trip floats), resolves the model pin, and reports what changed on +verify. + +## Honest limits + +- Replay assumes the input rows are reproducible. If the table mutates, + snapshot the database (or record row counts and a data hash alongside) + when the receipt matters enough. +- Determinism is per build for the statistical models and native + students. ONNX-served models are deterministic on the same CPU build + but not bit-identical across machines or GPU providers; record the + device if you serve through ONNX and intend to replay. +- A receipt proves what was computed, not that the prediction was good. + Pair with `backtest` (see interpret-backtest) when quality claims + matter. diff --git a/skills/prediction-receipts/scripts/receipt.py b/skills/prediction-receipts/scripts/receipt.py new file mode 100644 index 0000000..eb2c784 --- /dev/null +++ b/skills/prediction-receipts/scripts/receipt.py @@ -0,0 +1,293 @@ +#!/usr/bin/env python3 +"""Reference implementation of the prediction-receipts convention. + +Records and verifies provenance receipts for sqlite-predict calls, so +every agent that uses this script produces interoperable receipts with +identical canonicalization and hashing. Stdlib only. + +Canonical result document: + - If the prediction SQL returns exactly one row with one TEXT column + (the aggregate document case), the document is that text verbatim. + - Otherwise (predict/backtest rows), the document is compact JSON: + {"columns":[...],"rows":[[...],...]} in cursor order, with floats + serialized by Python's shortest round-trip repr. +The receipt stores sha256(document utf-8), never the document itself. + +Usage: + receipt.py record DB "SELECT forecast(ts, value, 24) FROM t" [--operation forecast] + receipt.py verify DB RECEIPT_ID + receipt.py list DB + Common flag: --extension PATH loadable to use (default: env + SQLITE_PREDICT_EXTENSION, else "predict0" on the library path) +""" + +import argparse +import hashlib +import json +import os +import re +import sqlite3 +import sys + +DDL = """CREATE TABLE IF NOT EXISTS _predict_receipts ( + id INTEGER PRIMARY KEY, + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')), + operation TEXT NOT NULL, + input_sql TEXT NOT NULL, + options TEXT, + model_id TEXT NOT NULL, + content_hash TEXT, + extension TEXT NOT NULL, + result_sha256 TEXT NOT NULL +)""" + +OPERATIONS = ("forecast", "detect_anomalies", "predict", "backtest") + + +def fail(code, msg): + """Stable, grep-able failure codes so agents can branch on outcomes + the way they branch on the extension's PREDICT_ERR_* codes.""" + print(f"receipt.py: {code}: {msg}", file=sys.stderr) + sys.exit(1) + + +#: authorizer allow-list for replaying untrusted SQL: reads and function +#: calls only. Everything else (writes, DDL, ATTACH, PRAGMA) is denied. +_REPLAY_ALLOWED = { + getattr(sqlite3, name, None) for name in + ("SQLITE_SELECT", "SQLITE_READ", "SQLITE_FUNCTION", "SQLITE_RECURSIVE") +} - {None} + + +def _replay_authorizer(action, *_): + return sqlite3.SQLITE_OK if action in _REPLAY_ALLOWED \ + else sqlite3.SQLITE_DENY + + +def connect(db_path, extension, untrusted=False): + """untrusted=True is the replay posture: the database opens + read-only at the file level and an authorizer denies everything but + reads and function calls, so a tampered receipt's input_sql cannot + write, drop, attach, or change pragmas on the verifier's database.""" + try: + if untrusted: + db = sqlite3.connect(f"file:{db_path}?mode=ro", uri=True) + else: + db = sqlite3.connect(db_path) + except sqlite3.Error as e: + fail("RECEIPT_ERR_DB", f"cannot open '{db_path}': {e}") + db.enable_load_extension(True) + try: + db.load_extension(extension) + except sqlite3.OperationalError as e: + fail("RECEIPT_ERR_EXTENSION", f"cannot load '{extension}': {e}") + finally: + # a tampered receipt must not be able to load a local library + db.enable_load_extension(False) + if untrusted: + db.set_authorizer(_replay_authorizer) + # a tampered receipt must not be able to hang the verifier + # either: abort replay after a generous VM-step budget + steps = int(os.environ.get("RECEIPT_MAX_STEPS", 50_000_000)) + db.set_progress_handler(lambda: 1, steps) + return db + + +AGGREGATE_OPS = ("forecast", "detect_anomalies") + + +def canonical_document(cur, operation): + """Aggregate operations hash their single JSON document verbatim; + row-shaped operations always hash the {"columns","rows"} form, even + when the result happens to be one text cell. Shape must follow the + operation or receipts stop being interoperable.""" + rows = cur.fetchall() + cols = [d[0] for d in cur.description] if cur.description else [] + if (operation in AGGREGATE_OPS and len(rows) == 1 and len(cols) == 1 + and isinstance(rows[0][0], str)): + return rows[0][0] + + def cell(v): + # BLOBs canonicalize as their hash: deterministic, bounded, and + # json.dumps cannot serialize bytes anyway + if isinstance(v, (bytes, memoryview)): + return {"blob_sha256": + hashlib.sha256(bytes(v)).hexdigest()} + return v + return json.dumps({"columns": cols, + "rows": [[cell(v) for v in r] for r in rows]}, + separators=(",", ":"), ensure_ascii=False) + + +def sha256_text(text): + return hashlib.sha256(text.encode("utf-8")).hexdigest() + + +def infer_operation(sql): + found = {op for op in OPERATIONS + if re.search(r"\b" + op + r"\s*\(", sql)} + if len(found) == 1: + return found.pop() + return None + + +def extension_version(db): + doc = json.loads(db.execute("SELECT predict_version()").fetchone()[0]) + return doc["extension"] + + +def model_fields(db, document, model_flag): + """The serving model id and, when registered, its content_hash.""" + model_id = model_flag + if model_id is None: + try: + model_id = json.loads(document).get("model") + except (json.JSONDecodeError, AttributeError): + model_id = None + if not model_id: + fail("RECEIPT_ERR_MODEL_UNKNOWN", + "the result is not a document with a \"model\" field;" + " pass --model-id explicitly") + content_hash = None + try: + row = db.execute( + "SELECT content_hash FROM _predict_models WHERE model_id = ?", + (model_id,)).fetchone() + content_hash = row[0] if row else None + except sqlite3.OperationalError: + pass # no registry in this database: content_hash stays NULL + return model_id, content_hash + + +def cmd_record(args): + if args.options is not None: + try: + json.loads(args.options) + except json.JSONDecodeError as e: + fail("RECEIPT_ERR_OPTIONS", f"--options is not valid JSON: {e}") + db = connect(args.db, args.extension) + operation = args.operation or infer_operation(args.sql) + if operation is None: + fail("RECEIPT_ERR_OPERATION_UNKNOWN", + f"cannot infer the operation; pass --operation one of" + f" {', '.join(OPERATIONS)}") + try: + cur = db.execute(args.sql) + document = canonical_document(cur, operation) + except sqlite3.Error as e: + fail("RECEIPT_ERR_SQL", str(e)) + model_id, content_hash = model_fields(db, document, args.model_id) + db.execute(DDL) + cur = db.execute( + "INSERT INTO _predict_receipts (operation, input_sql, options," + " model_id, content_hash, extension, result_sha256)" + " VALUES (?,?,?,?,?,?,?)", + (operation, args.sql, args.options, model_id, content_hash, + extension_version(db), sha256_text(document))) + db.commit() + print(json.dumps({"receipt_id": cur.lastrowid, "model_id": model_id, + "content_hash": content_hash, + "result_sha256": sha256_text(document)})) + db.close() + + +def cmd_verify(args): + db = connect(args.db, args.extension, untrusted=True) + row = db.execute( + "SELECT operation, input_sql, model_id, content_hash, extension," + " result_sha256 FROM _predict_receipts WHERE id = ?", + (args.receipt_id,)).fetchone() + if row is None: + fail("RECEIPT_ERR_NOT_FOUND", f"no receipt with id {args.receipt_id}") + operation, input_sql, model_id, rec_hash, rec_ext, rec_sha = row + try: + document = canonical_document(db.execute(input_sql), operation) + except sqlite3.Error as e: + fail("RECEIPT_ERR_SQL", f"replay refused or failed: {e}") + now_sha = sha256_text(document) + now_ext = extension_version(db) + # re-derive the serving model from the replayed result where the + # result carries one (aggregate documents do; row-form results were + # recorded via --model-id and cannot be re-derived) + replayed_model = None + try: + replayed_model = json.loads(document).get("model") + except (json.JSONDecodeError, AttributeError): + pass + id_matches = None if replayed_model is None \ + else replayed_model == model_id + now_hash = None + try: + r = db.execute( + "SELECT content_hash FROM _predict_models WHERE model_id = ?", + (model_id,)).fetchone() + now_hash = r[0] if r else None + except sqlite3.OperationalError: + pass + hash_match = now_sha == rec_sha + report = { + "receipt_id": args.receipt_id, + "match": hash_match and id_matches is not False, + "result_sha256": {"recorded": rec_sha, "replayed": now_sha, + "match": hash_match}, + "model": {"recorded_id": model_id, + "replayed_id": replayed_model, + "id_matches": id_matches, + "content_hash_recorded": rec_hash, + "content_hash_current": now_hash, + "changed": rec_hash != now_hash}, + "extension": {"recorded": rec_ext, "current": now_ext, + "changed": rec_ext != now_ext}, + # options is informational: its authoritative copy is the options + # text embedded in input_sql, which the replay executes verbatim + "options_verified": False, + } + print(json.dumps(report, indent=2)) + db.close() + sys.exit(0 if report["match"] else 2) + + +def cmd_list(args): + db = connect(args.db, args.extension, untrusted=True) + try: + rows = db.execute( + "SELECT id, created_at, operation, model_id, result_sha256" + " FROM _predict_receipts ORDER BY id").fetchall() + except sqlite3.OperationalError: + rows = [] + for r in rows: + print(f"{r[0]} {r[1]} {r[2]:<16} {r[3]:<24} {r[4][:16]}...") + db.close() + + +def main(): + p = argparse.ArgumentParser(description=__doc__) + p.add_argument("--extension", + default=os.environ.get("SQLITE_PREDICT_EXTENSION", + "predict0")) + sub = p.add_subparsers(dest="cmd", required=True) + + r = sub.add_parser("record", help="run a prediction and record a receipt") + r.add_argument("db") + r.add_argument("sql") + r.add_argument("--operation", choices=OPERATIONS) + r.add_argument("--options", help="options JSON, recorded verbatim") + r.add_argument("--model-id", + help="override when the result carries no model field") + r.set_defaults(fn=cmd_record) + + v = sub.add_parser("verify", help="replay a receipt and compare hashes") + v.add_argument("db") + v.add_argument("receipt_id", type=int) + v.set_defaults(fn=cmd_verify) + + ls = sub.add_parser("list", help="list recorded receipts") + ls.add_argument("db") + ls.set_defaults(fn=cmd_list) + + args = p.parse_args() + args.fn(args) + + +if __name__ == "__main__": + main() diff --git a/skills/sqlite-predict/SKILL.md b/skills/sqlite-predict/SKILL.md new file mode 100644 index 0000000..be48081 --- /dev/null +++ b/skills/sqlite-predict/SKILL.md @@ -0,0 +1,84 @@ +--- +name: sqlite-predict +description: >- + Forecast, detect anomalies, and predict on tabular data directly in a + SQLite database using the sqlite-predict extension. Use when an agent + holds time series or tabular data in SQLite and needs predictions + without exporting data, calling a cloud model, or building a pipeline. +license: MIT OR Apache-2.0 +metadata: + version: "0.1.0" +--- + +# Using sqlite-predict + +The extension turns prediction into SQL. Load `predict0` (pip/npm/cargo +package `sqlite-predict`), then call functions over your own rows. Serving +is pure: no writes, works on read-only databases and inside views. + +## Which operation for which question + +| Question | Call | +| --- | --- | +| "What will this metric do next?" | `SELECT forecast(ts, value, horizon) FROM t` | +| "Which points are anomalous?" | `SELECT detect_anomalies(ts, value) FROM t` | +| "Predict a label/value for these rows" | `SELECT * FROM predict(train_sql, apply_sql, options)` | +| "How accurate would this be on my data?" | `SELECT * FROM backtest(series_sql, horizon)` | +| "Make serving instant and self-contained" | `distill_predict` / `distill_forecast` (see the distill-lifecycle skill) | + +`forecast` and `detect_anomalies` are aggregates: plain SQL supplies the +rows, `GROUP BY` splits series, `WHERE` and joins compose, ORMs work +unchanged. `predict`, `backtest`, and the distillers are table-valued +functions that take a query string. + +## Reading results + +Each aggregate group returns one JSON document: `{"model", "status", +"rows"}`. Expand to typed rows in SQL when needed: + +```sql +SELECT r.* FROM forecast_rows((SELECT forecast(ts, value, 24) FROM t)) AS r; +``` + +- `model` reports the model that actually served. With no model named, + `auto` selects the best available per series and reports the winner's + id, never the string "auto". +- `status` is `ok`, or a degraded status with empty rows: + `insufficient_history` (fewer than 8 points) or `non_numeric` (a + timestamp or value cell poisoned the series). Degraded statuses are + data conditions, not errors; check `status` before using `rows`. +- Zero input rows return SQL NULL, the aggregate convention. + +## Timestamps and values + +Timestamps: ISO-8601 text (`YYYY-MM-DD[ T]HH:MM[:SS][Z]`) or integer +epoch seconds/milliseconds. Anything else poisons the series as +`non_numeric`. Values must be numeric. The aggregate floors timestamps +to whole seconds. + +## Errors are self-explaining + +Every call error is one string from a closed set, formatted +`PREDICT_ERR_: detail` (options, schema, horizon, licensing, +model-not-found, and so on). There are no silent fallbacks: an unknown +option key, a non-read-only inner query, or tampered model weights all +fail loudly. When a call errors, read the message; it names the exact +problem and often the fix. + +## Defaults an agent should know + +- No model named means `auto`: bundled statistical models plus any + eligible distilled students compete per series. Pass + `'{"candidates":["theta-classic","tsb"]}'` to narrow the pool. +- Prediction intervals default to a Gaussian band that is overconfident + on smooth data; pass `'{"interval_method":"conformal"}'` for + calibrated coverage (statistical models only) and verify with + `backtest` (see the interpret-backtest skill). +- `predict` with `NULL` train_query serves a distilled student; with a + train query it runs the in-context `knn5-incontext` model zero-shot. + +## Checking the environment + +`SELECT predict_version()` returns the extension version, compiled +runtimes, and bundled model ids as JSON. Use it to confirm the extension +is loaded and what is available before planning work. diff --git a/sqlite-predict.c b/sqlite-predict.c index 2eabd35..c19217f 100644 --- a/sqlite-predict.c +++ b/sqlite-predict.c @@ -420,6 +420,46 @@ static void predict_ulid_fn(sqlite3_context *context, int argc, sqlite3_result_text(context, buf, 26, SQLITE_TRANSIENT); } +/* predict_sha256(X): lowercase hex SHA-256 of a TEXT or BLOB value. + * The same hash that pins model weights (content_hash), exposed so + * agent-layer workflows (provenance receipts, content addressing) can + * run in pure SQL with no host language. NULL in, NULL out. */ +static void predict_sha256_fn(sqlite3_context *context, int argc, + sqlite3_value **argv) { + UNUSED_PARAMETER(argc); + int vt = sqlite3_value_type(argv[0]); + if (vt == SQLITE_NULL) { + sqlite3_result_null(context); + return; + } + const void *data; + int n; + if (vt == SQLITE_BLOB) { + data = sqlite3_value_blob(argv[0]); + n = sqlite3_value_bytes(argv[0]); + } else if (vt == SQLITE_TEXT) { + data = sqlite3_value_text(argv[0]); + n = sqlite3_value_bytes(argv[0]); + } else { + /* numbers are rejected, not coerced: SQLite's REAL->TEXT keeps 15 + * significant digits, so distinct doubles could collapse to one + * hash. A provenance primitive must not do that silently. */ + sqlite3_result_error(context, + PREDICT_ERR_SCHEMA + ": predict_sha256 takes TEXT or BLOB", + -1); + return; + } + predict0_hasher h; + predict0_hash_init(&h); + if (n > 0 && data) + sha256_update(&h.sha, (const u8 *)data, (usize)n); + char hex[PREDICT_HEX_BUFSIZE]; + predict0_hash_hex(&h, hex); + sqlite3_result_text(context, hex, PREDICT_HEX_BUFSIZE - 1, + SQLITE_TRANSIENT); +} + /* predict_register(model_id, config_json): record an external model in * _predict_models so predict() can dispatch to it. config is a JSON object: * { "runtime":"onnx", "kind":"tabular-fm"|"student"|..., @@ -637,6 +677,10 @@ __declspec(dllexport) predict_ulid_fn, NULL, NULL, NULL); if (rc != SQLITE_OK) return rc; + rc = sqlite3_create_function_v2(db, "predict_sha256", 1, flags, NULL, + predict_sha256_fn, NULL, NULL, NULL); + if (rc != SQLITE_OK) + return rc; /* predict_register mutates the registry, so it is not INNOCUOUS and not * DETERMINISTIC (its content_hash depends on a file read). */ rc = sqlite3_create_function_v2(db, "predict_register", 2, SQLITE_UTF8, diff --git a/tests/test_skills.py b/tests/test_skills.py new file mode 100644 index 0000000..28b615d --- /dev/null +++ b/tests/test_skills.py @@ -0,0 +1,354 @@ +"""The skills ship as part of the repo, so CI holds them to the +agentskills.io spec and proves the receipts reference script actually +does what the prediction-receipts skill promises: record, replay-match, +tamper-detect, and pin the model hash.""" + +import json +import os +import re +import sqlite3 +import subprocess +import sys + +import pytest + +ROOT = os.path.join(os.path.dirname(__file__), "..") +SKILLS = os.path.join(ROOT, "skills") +RECEIPT = os.path.join(SKILLS, "prediction-receipts", "scripts", "receipt.py") +EXT = os.path.join(ROOT, "dist", "predict0") + +NAME_RE = re.compile(r"^[a-z0-9]+(-[a-z0-9]+)*$") + + +def skill_dirs(): + return sorted(d for d in os.listdir(SKILLS) + if os.path.isdir(os.path.join(SKILLS, d))) + + +def frontmatter(path): + text = open(path, encoding="utf-8").read() + assert text.startswith("---\n"), f"{path}: missing frontmatter" + fm = text.split("---\n")[1] + return fm, text + + +@pytest.mark.parametrize("d", skill_dirs()) +def test_skill_conforms_to_spec(d): + path = os.path.join(SKILLS, d, "SKILL.md") + fm, text = frontmatter(path) + + m = re.search(r"^name:\s*(\S+)\s*$", fm, re.M) + assert m, "name is required" + name = m.group(1) + assert name == d, "name must match the directory" + assert len(name) <= 64 and NAME_RE.match(name), f"invalid name {name!r}" + + dm = re.search(r"^description:\s*>-\n((?: .+\n)+)", fm, re.M) + assert dm, "description is required (folded block expected)" + desc = " ".join(dm.group(1).split()) + assert 1 <= len(desc) <= 1024, "description must be 1..1024 chars" + assert "when" in desc.lower(), "description should say when to use it" + + assert text.count("\n") <= 500, "SKILL.md should stay under 500 lines" + assert "—" not in text and "–" not in text, \ + "house style: no em or en dashes" + + +def run_receipt(*argv): + return subprocess.run( + [sys.executable, RECEIPT, "--extension", EXT, *argv], + capture_output=True, text=True) + + +@pytest.fixture() +def receipt_db(tmp_path): + path = str(tmp_path / "r.db") + db = sqlite3.connect(path) + db.execute("CREATE TABLE readings(ts INTEGER, value REAL)") + db.executemany("INSERT INTO readings VALUES (?, ?)", + [(1700000000 + i * 3600, 10.0 + i % 7) + for i in range(48)]) + db.commit() + db.close() + return path + + +def test_receipt_record_verify_and_tamper(receipt_db): + sql = "SELECT forecast(ts, value, 6) FROM readings" + rec = run_receipt("record", receipt_db, sql) + assert rec.returncode == 0, rec.stderr + out = json.loads(rec.stdout) + assert out["receipt_id"] == 1 + assert out["model_id"] not in (None, "", "auto") + + ver = run_receipt("verify", receipt_db, "1") + assert ver.returncode == 0, ver.stdout + ver.stderr + report = json.loads(ver.stdout) + assert report["match"] is True + assert report["extension"]["changed"] is False + + # determinism claim: recording again yields the identical hash + rec2 = run_receipt("record", receipt_db, sql) + assert json.loads(rec2.stdout)["result_sha256"] == out["result_sha256"] + + # tamper with the data: replay must mismatch, exit code must say so + db = sqlite3.connect(receipt_db) + db.execute("UPDATE readings SET value = value + 100 WHERE rowid = 5") + db.commit() + db.close() + bad = run_receipt("verify", receipt_db, "1") + assert bad.returncode == 2 + assert json.loads(bad.stdout)["match"] is False + + +def test_pure_sql_receipt_path(receipt_db): + """The skill's primary path: record and verify with no host language + beyond SQL, using predict_sha256.""" + db = sqlite3.connect(receipt_db) + db.enable_load_extension(True) + db.load_extension(EXT) + import hashlib + assert db.execute("SELECT predict_sha256('abc')").fetchone()[0] == \ + hashlib.sha256(b"abc").hexdigest() + assert db.execute("SELECT predict_sha256(NULL)").fetchone()[0] is None + + db.execute("""CREATE TABLE _predict_receipts ( + id INTEGER PRIMARY KEY, + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')), + operation TEXT NOT NULL, input_sql TEXT NOT NULL, options TEXT, + model_id TEXT NOT NULL, content_hash TEXT, + extension TEXT NOT NULL, result_sha256 TEXT NOT NULL)""") + db.execute(""" + INSERT INTO _predict_receipts (operation, input_sql, options, model_id, + content_hash, extension, result_sha256) + SELECT 'forecast', 'SELECT forecast(ts, value, 6) FROM readings', NULL, + json_extract(doc, '$.model'), NULL, + json_extract(predict_version(), '$.extension'), + predict_sha256(doc) + FROM (SELECT (SELECT forecast(ts, value, 6) FROM readings) AS doc)""") + db.commit() + + (match,) = db.execute(""" + SELECT result_sha256 = predict_sha256( + (SELECT forecast(ts, value, 6) FROM readings)) + FROM _predict_receipts WHERE id = 1""").fetchone() + assert match == 1 + + db.execute("UPDATE readings SET value = value + 100 WHERE rowid = 3") + (match,) = db.execute(""" + SELECT result_sha256 = predict_sha256( + (SELECT forecast(ts, value, 6) FROM readings)) + FROM _predict_receipts WHERE id = 1""").fetchone() + assert match == 0 + db.close() + + +def test_receipt_pins_a_registered_student(receipt_db): + db = sqlite3.connect(receipt_db) + db.enable_load_extension(True) + db.load_extension(EXT) + db.execute("CREATE TABLE tab(f1 REAL, f2 REAL, label TEXT)") + db.executemany("INSERT INTO tab VALUES (?,?,?)", + [(i * 0.1, (i % 5) * 1.0, "a" if i % 2 else "b") + for i in range(60)]) + db.execute("SELECT * FROM distill_predict(" + "'SELECT f1, f2, label FROM tab'," + " '{\"target\":\"label\",\"student_id\":\"s1\"}')").fetchall() + db.commit() + db.close() + + rec = run_receipt( + "record", receipt_db, + "SELECT forecast(ts, value, 6, '{\"model\":\"theta-classic\"}')" + " FROM readings") + out = json.loads(rec.stdout) + assert out["model_id"] == "theta-classic" + assert out["content_hash"], "registered models must record their pin" + + # rows-shaped results canonicalize too: predict() through a student + rec2 = run_receipt( + "record", receipt_db, + "SELECT rowid, prediction FROM predict(NULL," + " 'SELECT rowid, f1, f2 FROM tab', '{\"model\":\"s1\"}')", + "--model-id", "s1") + assert rec2.returncode == 0, rec2.stderr + out2 = json.loads(rec2.stdout) + assert out2["content_hash"], "student content_hash must be recorded" + ver = run_receipt("verify", receipt_db, str(out2["receipt_id"])) + assert json.loads(ver.stdout)["match"] is True + + +def test_receipt_adversarial_paths(receipt_db): + """Break the receipt tool the ways a hostile or careless caller + would: single-text-cell TVF results must canonicalize as rows (not + as an aggregate document), extension errors must surface their + PREDICT_ERR_* code verbatim, missing receipts must fail loudly, and + a replayed receipt must not be able to re-enable extension + loading.""" + import hashlib + + # a backtest projected to one text column: operation-based + # canonicalization must hash the {"columns","rows"} form, so the + # receipt hash must NOT equal the raw-cell hash + one_cell_sql = ("SELECT model FROM backtest(" + "'SELECT ts, value FROM readings', 4," + " '{\"model\":\"theta-classic\"}') LIMIT 1") + rec = run_receipt("record", receipt_db, one_cell_sql, + "--model-id", "theta-classic") + assert rec.returncode == 0, rec.stderr + out = json.loads(rec.stdout) + db = sqlite3.connect(receipt_db) + db.enable_load_extension(True) + db.load_extension(EXT) + raw_cell = db.execute(one_cell_sql).fetchone()[0] + db.close() + assert out["result_sha256"] != hashlib.sha256( + raw_cell.encode()).hexdigest(), "TVF result canonicalized as an aggregate document" + ver = run_receipt("verify", receipt_db, str(out["receipt_id"])) + assert json.loads(ver.stdout)["match"] is True + + # unknown option key: the extension's exact error code must surface + bad = run_receipt( + "record", receipt_db, + "SELECT forecast(ts, value, 6, '{\"bogus_key\":1}') FROM readings") + assert bad.returncode != 0 + assert "PREDICT_ERR_OPTIONS" in (bad.stderr + bad.stdout) + + # numeric input to the hash primitive is rejected, never coerced + db = sqlite3.connect(receipt_db) + db.enable_load_extension(True) + db.load_extension(EXT) + import pytest as _pytest + for bad_val in ("1.5", "42"): + with _pytest.raises(sqlite3.OperationalError, match="PREDICT_ERR_SCHEMA"): + db.execute(f"SELECT predict_sha256({bad_val})").fetchone() + assert db.execute("SELECT predict_sha256(NULL)").fetchone()[0] is None + db.close() + + # missing receipt id fails loudly + gone = run_receipt("verify", receipt_db, "9999") + assert gone.returncode != 0 + assert "RECEIPT_ERR_NOT_FOUND" in gone.stderr + + # a tampered receipt cannot load an arbitrary extension on replay + db = sqlite3.connect(receipt_db) + db.execute( + "UPDATE _predict_receipts SET input_sql =" + " 'SELECT load_extension(''/tmp/evil'')' WHERE id = 1") + db.commit() + db.close() + evil = run_receipt("verify", receipt_db, "1") + assert evil.returncode != 0 + assert "not authorized" in (evil.stderr + evil.stdout).lower() + + +def test_receipt_metadata_tampering(receipt_db): + """A tampered model_id must fail verification via re-derivation; a + tampered options column is documented as informational (the + authoritative options live inside input_sql), so this test pins that + stated limit rather than pretending it is detection.""" + rec = run_receipt("record", receipt_db, + "SELECT forecast(ts, value, 6) FROM readings") + rid = str(json.loads(rec.stdout)["receipt_id"]) + + db = sqlite3.connect(receipt_db) + db.execute("UPDATE _predict_receipts SET model_id = 'impostor'" + " WHERE id = ?", (rid,)) + db.commit() + db.close() + forged = run_receipt("verify", receipt_db, rid) + assert forged.returncode == 2 + rep = json.loads(forged.stdout) + assert rep["match"] is False + assert rep["model"]["id_matches"] is False + + db = sqlite3.connect(receipt_db) + db.execute("UPDATE _predict_receipts SET model_id = 'theta-classic'," + " options = '{\"forged\":true}' WHERE id = ?", (rid,)) + db.commit() + db.close() + forged_opts = run_receipt("verify", receipt_db, rid) + rep = json.loads(forged_opts.stdout) + assert rep["match"] is True, "options is informational by design" + assert rep["options_verified"] is False + + +def test_receipt_replay_cannot_write(receipt_db): + """Replay input is untrusted: a tampered input_sql that writes, + drops, or attaches must be refused by the read-only replay + connection, and the verifier's data must be untouched.""" + rec = run_receipt("record", receipt_db, + "SELECT forecast(ts, value, 6) FROM readings") + rid = str(json.loads(rec.stdout)["receipt_id"]) + + for attack in ("DELETE FROM readings", + "DROP TABLE readings", + "ATTACH DATABASE '/tmp/exfil.db' AS x", + "PRAGMA journal_mode = OFF"): + db = sqlite3.connect(receipt_db) + db.execute("UPDATE _predict_receipts SET input_sql = ?" + " WHERE id = ?", (attack, rid)) + db.commit() + db.close() + out = run_receipt("verify", receipt_db, rid) + assert out.returncode != 0, f"replay accepted: {attack}" + blob = (out.stderr + out.stdout).lower() + assert ("not authorized" in blob or "readonly" in blob + or "read-only" in blob or "prohibited" in blob), attack + + db = sqlite3.connect(receipt_db) + (n,) = db.execute("SELECT count(*) FROM readings").fetchone() + db.close() + assert n == 48, "replay modified the verifier's data" + + +def test_receipt_hardening_paths(receipt_db, tmp_path): + """The audit set: runaway replay is aborted by the step budget, BLOB + cells canonicalize deterministically instead of crashing, invalid + --options is rejected up front, and a missing database fails with a + clean code instead of a traceback.""" + rec = run_receipt("record", receipt_db, + "SELECT forecast(ts, value, 6) FROM readings") + rid = str(json.loads(rec.stdout)["receipt_id"]) + + # recursive CTE bomb: must abort via the step budget, not hang + db = sqlite3.connect(receipt_db) + db.execute("UPDATE _predict_receipts SET input_sql =" + " 'WITH RECURSIVE b(x) AS (SELECT 1 UNION ALL SELECT x+1" + " FROM b) SELECT max(x) FROM b' WHERE id = ?", (rid,)) + db.commit() + db.close() + import subprocess as sp + bomb = sp.run([sys.executable, RECEIPT, "--extension", EXT, + "verify", receipt_db, rid], + capture_output=True, text=True, timeout=60, + env={**os.environ, "RECEIPT_MAX_STEPS": "100000"}) + assert bomb.returncode != 0 + assert "RECEIPT_ERR_SQL" in bomb.stderr + + # BLOB cells canonicalize as their hash, deterministically + db = sqlite3.connect(receipt_db) + db.execute("CREATE TABLE blobs(b BLOB)") + db.execute("INSERT INTO blobs VALUES (x'00ff10')") + db.commit() + db.close() + b1 = run_receipt("record", receipt_db, "SELECT b FROM blobs", + "--operation", "predict", "--model-id", "n/a") + b2 = run_receipt("record", receipt_db, "SELECT b FROM blobs", + "--operation", "predict", "--model-id", "n/a") + assert b1.returncode == 0, b1.stderr + assert json.loads(b1.stdout)["result_sha256"] == \ + json.loads(b2.stdout)["result_sha256"] + + # invalid --options rejected before anything executes + badopt = run_receipt("record", receipt_db, + "SELECT forecast(ts, value, 6) FROM readings", + "--options", "{not json") + assert badopt.returncode != 0 + assert "RECEIPT_ERR_OPTIONS" in badopt.stderr + + # missing database file fails cleanly on the read-only path + gone_db = run_receipt("verify", str(tmp_path / "nope.db"), "1") + assert gone_db.returncode != 0 + assert "RECEIPT_ERR" in gone_db.stderr + assert "Traceback" not in gone_db.stderr diff --git a/website/src/content/docs/reference/functions.md b/website/src/content/docs/reference/functions.md index 9b983a7..352e439 100644 --- a/website/src/content/docs/reference/functions.md +++ b/website/src/content/docs/reference/functions.md @@ -123,6 +123,7 @@ window laid out by position as `context` input columns followed by the | --- | --- | | `predict_version()` | the extension version string | | `predict_ulid(ts)` | the smallest ULID for an ISO-8601 UTC timestamp: deterministic (the time component is set, the entropy bits are zero), useful as a range key | +| `predict_sha256(x)` | lowercase hex SHA-256 of a TEXT or BLOB value. NULL passes through; numeric arguments are rejected with `PREDICT_ERR_SCHEMA` rather than hashed through SQLite's lossy number-to-text conversion. Used by provenance workflows like the prediction-receipts skill to hash result documents in pure SQL | | `predict_debug()` | build info: profile, compiled features, available runtimes | | `predict_register(model_id, config)` | registers an external model in `_predict_models` and returns its content hash; `config` is a bare weights path or a JSON object (`runtime`, `kind`, `license`, `weights_uri`, `io_spec`, `target`). Registration is metadata only: executing an `onnx` model still needs the ONNX build |