Extract the corpus from every pinned test script, and assert error offsets

PLAN.md milestone 2 named a hand-picked starter set of 69 scripts. Taking
the whole pinned test suite instead costs 4.4 MB and raises the corpus
from 4,685 to 20,971 cases across 943 scripts, with no curation to redo
when the pin advances. Scripts that yield no literal-SQL block (the
C-level tokenizer tests, the VFS and WAL harnesses) get no corpus file.

The harness now also checks the error byte offset against
sqlite3_error_offset wherever SQLite reported one, not just the message.

Two fixes this shook out of the tooling:

- An error message can span lines, because the offending token can:
  "SELECT X'0102, 1" reports the rest of the input, newline included, as
  an unrecognized token. runOracle joined nothing and failed the whole
  run; it now folds continuation lines back into the message so
  testfile.CheckCase can drop the case the way it already documents.
- regenerate-parse wrote an empty corpus file for a script with nothing
  extractable, and left a stale one behind if a script stopped yielding
  cases.

Corpus: 20971/20971 cases passing, and the parser needed no changes to
get there.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01JzBeCg7rjweVW3uGPg5G7T

Author: claude

CLAUDE.md

@@ -22,9 +22,11 @@ https://sqlite.org/lang.html.
 ## The corpus
 
 Each `parser/testdata/<name>.test` holds cases extracted from SQLite's
-`test/<name>.test` TCL scripts. For every case, the raw oracle results (one
-line per statement: prepared OK, or the exact error message and offset) are
-stored; the harness derives the expectation from them:
+`test/<name>.test` TCL script; there is one for every script in the pinned
+source tree that yields at least one literal-SQL block. For every case, the
+raw oracle results (one line per statement: prepared OK, or the exact error
+message, offset and parse tail) are stored; the harness derives the
+expectation from them:
 
 - If any statement failed with a **syntax-family** message (`near "…":
   syntax error`, `unrecognized token: "…"`, `incomplete input`), meyer must
@@ -64,7 +66,7 @@ the fix is in `cmd/regenerate-parse` (or the classification in
 ## Regenerating the corpus
 
 ```sh
-go run ./cmd/regenerate-parse                  # the full starter set
+go run ./cmd/regenerate-parse                  # every script in the pinned tree
 go run ./cmd/regenerate-parse -files select1   # one file
 ```
 

cmd/regenerate-parse/main.go

@@ -22,7 +22,10 @@
 //
 //	go run ./cmd/regenerate-parse [-files select1,expr,...] [flags]
 //
-// With no -files, the starter set below is regenerated.
+// With no -files, every test script in the pinned source tree is
+// regenerated. Scripts that yield no literal-SQL cases (the tokenizer
+// tests written in C, the VFS and WAL harnesses, and so on) produce no
+// corpus file, and a stale one left over from an earlier run is removed.
 package main
 
 import (
@@ -37,6 +40,7 @@ import (
 	"os/exec"
 	"path/filepath"
 	"runtime"
+	"sort"
 	"strconv"
 	"strings"
 	"sync"
@@ -60,24 +64,21 @@ const (
 	srcSHA          = "d18fa15aec74d8c17e1463f861095adc01b5ad190256acb4f91d22f0368d232b"
 )
 
-// starterSet is the corpus from PLAN.md milestone 2: the dedicated
-// parser/tokenizer tests, the grammar-heavy statement suites, and the
-// evidence files that map 1:1 to the documented grammar.
-var starterSet = []string{
-	"parser1", "tokenize", "keyword1",
-	"select1", "select2", "select3", "select4", "select5", "select6",
-	"select7", "select8", "select9", "selectA", "selectB", "selectC",
-	"selectD", "selectE", "selectF", "selectG", "selectH",
-	"expr", "expr2", "in", "in2", "in3", "in4", "in5", "between",
-	"with1", "with2", "with3", "withM",
-	"window1", "window2", "window3", "window4", "window5",
-	"window6", "window7", "window8", "window9", "windowA",
-	"windowB", "windowC", "windowD", "windowE", "windowerr",
-	"alter", "alter2", "alter3", "alter4", "altertab", "altertab2", "altertab3",
-	"alterdropcol", "alterdropcol2",
-	"trigger1", "upsert1", "upsert2", "upsert3", "upsert4", "upsert5",
-	"returning1",
-	"e_select", "e_expr", "e_createtable", "e_insert", "e_update", "e_delete",
+// allScripts lists every test/*.test script of the pinned source tree, in
+// sorted order. PLAN.md milestone 2 named a hand-picked starter set; taking
+// the whole suite instead costs a few megabytes and quadruples the case
+// count, and needs no curation when the pin advances.
+func allScripts(testDir string) ([]string, error) {
+	paths, err := filepath.Glob(filepath.Join(testDir, "*.test"))
+	if err != nil {
+		return nil, err
+	}
+	names := make([]string, len(paths))
+	for i, path := range paths {
+		names[i] = strings.TrimSuffix(filepath.Base(path), ".test")
+	}
+	sort.Strings(names)
+	return names, nil
 }
 
 func main() {
@@ -89,11 +90,6 @@ func main() {
 	)
 	flag.Parse()
 
-	names := starterSet
-	if *files != "" {
-		names = strings.Split(*files, ",")
-	}
-
 	oracle, err := ensureOracle(*cacheDir)
 	if err != nil {
 		fatal(err)
@@ -102,11 +98,18 @@ func main() {
 	if err != nil {
 		fatal(err)
 	}
+
+	var names []string
+	if *files != "" {
+		names = strings.Split(*files, ",")
+	} else if names, err = allScripts(testDir); err != nil {
+		fatal(err)
+	}
 	if err := os.MkdirAll(*testdata, 0o755); err != nil {
 		fatal(err)
 	}
 
-	var totalCases, totalTodo int
+	var totalCases, totalTodo, usedScripts, emptyScripts int
 	var totals tclextract.Stats
 	for _, name := range names {
 		// Snapshot the previous corpus state before regenerating, so the
@@ -127,6 +130,17 @@ func main() {
 		if err != nil {
 			fatal(fmt.Errorf("%s: %w", name, err))
 		}
+		totals.Found += stats.Found
+		totals.SkippedSubst += stats.SkippedSubst
+		totals.SkippedForm += stats.SkippedForm
+		if len(cases) == 0 {
+			// Nothing extractable: leave no corpus file behind, and clear
+			// away one from an earlier run of a differently pinned tree.
+			os.Remove(testPath)
+			os.Remove(testfile.MetadataPath(testPath))
+			emptyScripts++
+			continue
+		}
 		todo, err := mergeMetadata(testPath, prevCases, prevMeta.Todo, cases)
 		if err != nil {
 			fatal(fmt.Errorf("%s: %w", name, err))
@@ -135,12 +149,11 @@ func main() {
 			name, len(cases), todo, stats.SkippedSubst, stats.SkippedForm)
 		totalCases += len(cases)
 		totalTodo += todo
-		totals.Found += stats.Found
-		totals.SkippedSubst += stats.SkippedSubst
-		totals.SkippedForm += stats.SkippedForm
+		usedScripts++
 	}
-	fmt.Printf("\ntotal: %d cases (%d todo) from %d files; %d of %d found blocks skipped (%d substitution, %d malformed/non-literal)\n",
-		totalCases, totalTodo, len(names),
+	fmt.Printf("\ntotal: %d cases (%d todo) from %d of %d scripts (%d yielded nothing); "+
+		"%d of %d found blocks skipped (%d substitution, %d malformed/non-literal)\n",
+		totalCases, totalTodo, usedScripts, len(names), emptyScripts,
 		totals.SkippedSubst+totals.SkippedForm, totals.Found,
 		totals.SkippedSubst, totals.SkippedForm)
 }
@@ -196,6 +209,9 @@ func regenerateFile(oracle, testDir, outDir, name string, jobs int) ([]testfile.
 		}
 		kept = append(kept, c)
 	}
+	if len(kept) == 0 {
+		return nil, stats, nil
+	}
 	if err := testfile.Write(filepath.Join(outDir, name+".test"), kept); err != nil {
 		return nil, stats, err
 	}
@@ -214,6 +230,17 @@ func runOracle(oracle, sql string) ([]testfile.StmtResult, error) {
 	}
 	var results []testfile.StmtResult
 	for _, line := range strings.Split(strings.TrimRight(out.String(), "\n"), "\n") {
+		// An error message can itself contain a newline, because the
+		// offending token can: "SELECT X'0102, 1" reports the rest of the
+		// input, newline included, as an unrecognized token. Such a line is
+		// a continuation of the message before it. The corpus file format
+		// cannot hold a multi-line message, so testfile.CheckCase drops the
+		// case later; joining here keeps the observation faithful until
+		// then instead of failing the whole run.
+		if !strings.HasPrefix(line, "stmt ") && len(results) > 0 {
+			results[len(results)-1].Message += "\n" + line
+			continue
+		}
 		if line == "" {
 			continue
 		}

parser/parser_test.go

@@ -76,9 +76,8 @@ func runFile(t *testing.T, path string) {
 
 // runCase checks one corpus case against the oracle-derived expectation:
 // meyer must accept exactly when SQLite's parser accepted, and must produce
-// the identical message on rejection. (Error offsets are recorded in the
-// corpus but not asserted yet; they become part of this check once the
-// parser tracks positions.)
+// the identical message and byte offset on rejection. SQLite reports -1 from
+// sqlite3_error_offset for errors it cannot place, and those are not checked.
 func runCase(c testfile.Case) error {
 	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
 	defer cancel()
@@ -108,5 +107,9 @@ func runCase(c testfile.Case) error {
 	if pe.Message != exp.Message {
 		return fmt.Errorf("error message mismatch:\n  got:  %s\n  want: %s", pe.Message, exp.Message)
 	}
+	if exp.Offset >= 0 && pe.Offset != exp.Offset {
+		return fmt.Errorf("error offset mismatch for %s: got %d, want %d",
+			exp.Message, pe.Offset, exp.Offset)
+	}
 	return nil
 }

parser/testdata/README.md

@@ -3,10 +3,14 @@
 The `*.test` files in this directory are generated by
 `go run ./cmd/regenerate-parse`. **Do not edit them by hand.**
 
-- **Source of the SQL**: SQLite's own test suite (`test/*.test` TCL scripts)
-  from the pinned release below. SQL blocks are extracted verbatim from
-  `do_execsql_test` / `do_catchsql_test` invocations whose SQL argument is a
-  literal TCL brace block (cases using TCL substitution are skipped).
+- **Source of the SQL**: SQLite's own test suite from the pinned release
+  below — every `test/*.test` TCL script, not a curated subset. SQL blocks
+  are extracted verbatim from `do_execsql_test` / `do_catchsql_test`
+  invocations whose SQL argument is a literal TCL brace block (cases using
+  TCL substitution are skipped, as are the few whose oracle message spans
+  more than one line, which the file format cannot hold). Scripts that yield
+  no such block — the C-level tokenizer tests, the VFS and WAL harnesses —
+  have no corpus file here.
 - **Source of the expectations**: a real SQLite build. Every statement of
   every case is prepared independently with `sqlite3_prepare_v2` against an
   empty in-memory database (nothing is executed) using the oracle program in

parser/testdata/keyword1.metadata.json parser/testdata/affinity2.metadata.jsonparser/testdata/keyword1.metadata.json renamed to parser/testdata/affinity2.metadata.json

@@ -0,0 +1,165 @@
+==== affinity2-100
+
+  CREATE TABLE t1(
+    xi INTEGER,
+    xr REAL,
+    xb BLOB,
+    xn NUMERIC,
+    xt TEXT
+  );
+  INSERT INTO t1(rowid,xi,xr,xb,xn,xt) VALUES(1,1,1,1,1,1);
+  INSERT INTO t1(rowid,xi,xr,xb,xn,xt) VALUES(2,'2','2','2','2','2');
+  INSERT INTO t1(rowid,xi,xr,xb,xn,xt) VALUES(3,'03','03','03','03','03');
+----
+stmt 0 ok
+stmt 94 err 1 -1 154 no such table: t1
+stmt 154 err 1 -1 224 no such table: t1
+stmt 224 err 1 -1 299 no such table: t1
+==== affinity2-110
+
+  SELECT xi, typeof(xi) FROM t1 ORDER BY rowid;
+----
+stmt 0 err 1 -1 48 no such table: t1
+==== affinity2-120
+
+  SELECT xr, typeof(xr) FROM t1 ORDER BY rowid;
+----
+stmt 0 err 1 -1 48 no such table: t1
+==== affinity2-130
+
+  SELECT xb, typeof(xb) FROM t1 ORDER BY rowid;
+----
+stmt 0 err 1 -1 48 no such table: t1
+==== affinity2-140
+
+  SELECT xn, typeof(xn) FROM t1 ORDER BY rowid;
+----
+stmt 0 err 1 -1 48 no such table: t1
+==== affinity2-150
+
+  SELECT xt, typeof(xt) FROM t1 ORDER BY rowid;
+----
+stmt 0 err 1 -1 48 no such table: t1
+==== affinity2-200
+
+  SELECT rowid, xi==xt, xi==xb, xi==+xt FROM t1 ORDER BY rowid;
+----
+stmt 0 err 1 -1 64 no such table: t1
+==== affinity2-210
+
+  SELECT rowid, xr==xt, xr==xb, xr==+xt FROM t1 ORDER BY rowid;
+----
+stmt 0 err 1 -1 64 no such table: t1
+==== affinity2-220
+
+  SELECT rowid, xn==xt, xn==xb, xn==+xt FROM t1 ORDER BY rowid;
+----
+stmt 0 err 1 -1 64 no such table: t1
+==== affinity2-300
+
+  SELECT rowid, xt==+xi, xt==xi, xt==xb FROM t1 ORDER BY rowid;
+----
+stmt 0 err 1 -1 64 no such table: t1
+==== 400
+
+  CREATE TABLE ttt(c0, c1);
+  CREATE INDEX ii ON ttt(CAST(c0 AS NUMERIC)); 
+  INSERT INTO ttt VALUES('abc', '-1');
+----
+stmt 0 ok
+stmt 28 err 1 -1 75 no such table: main.ttt
+stmt 75 err 1 -1 115 no such table: ttt
+==== 410
+
+  SELECT * FROM ttt WHERE CAST(c0 AS NUMERIC) > c1 GROUP BY rowid;
+----
+stmt 0 err 1 -1 67 no such table: ttt
+==== 420
+
+  SELECT * FROM ttt INDEXED BY ii WHERE CAST(c0 AS NUMERIC) > c1 GROUP BY rowid;
+----
+stmt 0 err 1 -1 81 no such table: ttt
+==== 430
+
+  CREATE TABLE t3(a, b, c INTEGER);
+  CREATE INDEX t3ac ON t3(a, c-1);
+  INSERT INTO t3 VALUES(1, 1, 1);
+  INSERT INTO t3 VALUES(2, 1, 0);
+  INSERT INTO t3 VALUES(3, 1, 1);
+  INSERT INTO t3 VALUES(4, 1, 0);
+  INSERT INTO t3 VALUES(5, 1, 1);
+----
+stmt 0 ok
+stmt 36 err 1 -1 71 no such table: main.t3
+stmt 71 err 1 -1 105 no such table: t3
+stmt 105 err 1 -1 139 no such table: t3
+stmt 139 err 1 -1 173 no such table: t3
+stmt 173 err 1 -1 207 no such table: t3
+stmt 207 err 1 -1 241 no such table: t3
+==== 440
+
+  SELECT * FROM t3 WHERE c='0' ORDER BY a;
+----
+stmt 0 err 1 -1 43 no such table: t3
+==== 500
+
+  DROP TABLE IF EXISTS t0;
+  CREATE TABLE t0(c0 TEXT UNIQUE, c1);
+  INSERT INTO t0(c0) VALUES (-1);
+  SELECT quote(- x'ce'), quote(t0.c0), quote(- x'ce' >= t0.c0) FROM t0;
+----
+stmt 0 ok
+stmt 27 ok
+stmt 66 err 1 -1 100 no such table: t0
+stmt 100 err 1 -1 172 no such table: t0
+==== 501
+
+  SELECT * FROM t0 WHERE - x'ce' >= t0.c0;
+----
+stmt 0 err 1 -1 43 no such table: t0
+==== 502
+
+  SELECT quote(+-+x'ce'), quote(t0.c0), quote(+-+x'ce' >= t0.c0) FROM t0;
+----
+stmt 0 err 1 -1 74 no such table: t0
+==== 503
+
+  SELECT * FROM t0 WHERE +-+x'ce' >= t0.c0;
+----
+stmt 0 err 1 -1 44 no such table: t0
+==== 504
+
+  SELECT quote(- 'ce'), quote(t0.c0), quote(- 'ce' >= t0.c0) FROM t0;
+----
+stmt 0 err 1 -1 70 no such table: t0
+==== 505
+
+  SELECT * FROM t0 WHERE - 'ce' >= t0.c0;
+----
+stmt 0 err 1 -1 42 no such table: t0
+==== 506
+
+  SELECT quote(+-+'ce'), quote(t0.c0), quote(+-+'ce' >= t0.c0) FROM t0;
+----
+stmt 0 err 1 -1 72 no such table: t0
+==== 507
+
+  SELECT * FROM t0 WHERE +-+'ce' >= t0.c0;
+----
+stmt 0 err 1 -1 43 no such table: t0
+==== 600
+
+    DROP TABLE IF EXISTS t0;
+    CREATE TABLE t0(c0 REAL UNIQUE);
+    INSERT INTO t0(c0) VALUES (3175546974276630385);
+    SELECT 3175546974276630385 < c0 FROM t0;
+----
+stmt 0 ok
+stmt 29 ok
+stmt 66 err 1 -1 119 no such table: t0
+stmt 119 err 1 -1 164 no such table: t0
+==== 601
+
+    SELECT 1 FROM t0 WHERE 3175546974276630385 < c0;
+----
+stmt 0 err 1 -1 53 no such table: t0