Record how far SQLite's parser got, so truncated parses do not lie

The corpus derived "meyer must accept the whole case" from a statement
that failed with a semantic message, on the reasoning that it must have
parsed to get that far. That is wrong when the message comes from a
grammar action in the middle of a statement: sqlite3BeginTrigger raises
"no such table" at the trigger_decl reduce, sqlite3RunParser sees
pParse->rc set and abandons the loop, and the trigger body is never
parsed at all. Three such cases have a deliberate syntax error in the
body that SQLite never saw, and the corpus was demanding meyer accept
them.

sqlite3_prepare_v2's pzTail says exactly how far the parser got: the end
of the statement for an ordinary end-of-parse failure, and just past
BEGIN for these. The oracle now records it on every failing statement,
(*Case).Expected turns short tails into unverified byte ranges, and the
harness lets the parser under test fail inside one. 192 of the 4685
cases have such a range; 4 of them were failing.

Regenerated the corpus for the added field: the diff is confined to err
lines, which gain one integer each.

Corpus: 4685/4685 cases passing.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01JzBeCg7rjweVW3uGPg5G7T

Author: claude

CLAUDE.md

@@ -32,6 +32,12 @@ stored; the harness derives the expectation from them:
 - Otherwise meyer must accept the whole case. This includes statements that
   failed **semantically** in SQLite (`no such table: …`) — those parsed
   successfully; meyer does no semantic analysis.
+- …except in text SQLite's parser never reached. A grammar action can fail
+  in the middle of a statement — `sqlite3BeginTrigger` raising `no such
+  table` at the `trigger_decl` reduce, before the trigger body is looked at
+  — and `sqlite3RunParser` then abandons the rest of the statement. The
+  oracle records `pzTail` on every failing statement, so the harness knows
+  which byte ranges are unverified and lets meyer fail inside them.
 
 Known looseness: messages produced by grammar *actions* (e.g. `ORDER BY
 clause should come after UNION not before`) are currently classified as

cmd/regenerate-parse/main.go

@@ -238,16 +238,19 @@ func parseOracleLine(line string) (testfile.StmtResult, error) {
 	if fields[2] == "ok" {
 		return testfile.StmtResult{Offset: off, OK: true}, nil
 	}
-	rest := strings.SplitN(strings.TrimPrefix(fields[2], "err "), " ", 3)
-	if !strings.HasPrefix(fields[2], "err ") || len(rest) != 3 {
+	rest := strings.SplitN(strings.TrimPrefix(fields[2], "err "), " ", 4)
+	if !strings.HasPrefix(fields[2], "err ") || len(rest) != 4 {
 		return testfile.StmtResult{}, fmt.Errorf("malformed oracle line %q", line)
 	}
 	rc, err1 := strconv.Atoi(rest[0])
 	erroff, err2 := strconv.Atoi(rest[1])
-	if err1 != nil || err2 != nil {
+	tail, err3 := strconv.Atoi(rest[2])
+	if err1 != nil || err2 != nil || err3 != nil {
 		return testfile.StmtResult{}, fmt.Errorf("malformed oracle line %q", line)
 	}
-	return testfile.StmtResult{Offset: off, RC: rc, ErrOffset: erroff, Message: rest[2]}, nil
+	return testfile.StmtResult{
+		Offset: off, RC: rc, ErrOffset: erroff, Tail: tail, Message: rest[3],
+	}, nil
 }
 
 // mergeMetadata reconciles the sidecar with the regenerated case list:

cmd/regenerate-parse/oracle/oracle.c

@@ -11,11 +11,20 @@
 ** Output: one line per non-empty statement:
 **
 **   stmt <offset> ok
-**   stmt <offset> err <rc> <erroff> <message to end of line>
+**   stmt <offset> err <rc> <erroff> <tail> <message to end of line>
 **
 ** <offset>  byte offset of the statement within the input script
 ** <rc>      sqlite3_prepare_v2 result code
 ** <erroff>  sqlite3_error_offset relative to the whole script (-1 if unknown)
+** <tail>    pzTail relative to the whole script: how far the parser got
+**
+** <tail> matters because a grammar action can fail in the middle of a
+** statement -- sqlite3BeginTrigger raising "no such table" at the
+** trigger_decl reduce, say -- and sqlite3RunParser then abandons the rest
+** of the statement unparsed. When that happens <tail> stops short of the
+** end of the statement, marking the text SQLite never looked at. A
+** successful prepare always consumes the whole statement, so "ok" lines
+** carry no <tail>.
 **
 ** SQLite error messages never contain newlines, so the line format is safe.
 */
@@ -105,13 +114,17 @@ int main(int argc, char **argv){
     stmtText[len] = 0;
 
     sqlite3_stmt *pStmt = 0;
-    int rc = sqlite3_prepare_v2(db, stmtText, (int)len, &pStmt, 0);
+    const char *zTail = 0;
+    int rc = sqlite3_prepare_v2(db, stmtText, (int)len, &pStmt, &zTail);
     if( rc==SQLITE_OK ){
       printf("stmt %zu ok\n", pos);
     }else{
       int erroff = sqlite3_error_offset(db);
+      size_t tail = zTail ? (size_t)(zTail-stmtText) : len;
       if( erroff>=0 ) erroff += (int)pos;
-      printf("stmt %zu err %d %d %s\n", pos, rc, erroff, sqlite3_errmsg(db));
+      if( tail>len ) tail = len;
+      printf("stmt %zu err %d %d %zu %s\n", pos, rc, erroff, pos+tail,
+             sqlite3_errmsg(db));
     }
     sqlite3_finalize(pStmt);
     free(stmtText);

internal/testfile/testfile.go

@@ -6,14 +6,16 @@
 //	<SQL, verbatim, one or more lines>
 //	----
 //	stmt <offset> ok
-//	stmt <offset> err <rc> <erroff> <message>
+//	stmt <offset> err <rc> <erroff> <tail> <message>
 //
 // The result lines are the raw per-statement output of the SQLite oracle
 // (see cmd/regenerate-parse): every statement in the case was prepared
 // independently with sqlite3_prepare_v2 against an empty in-memory database.
 // Offsets are byte offsets into the case SQL; erroff is -1 when SQLite did
-// not report an error position. The corpus stores this raw truth; the
-// pass/fail policy for the parser under test is derived from it by
+// not report an error position, and tail is how far sqlite3_prepare_v2
+// reported having got (a successful prepare always reaches the end of its
+// statement, so ok lines carry no tail). The corpus stores this raw truth;
+// the pass/fail policy for the parser under test is derived from it by
 // (*Case).Expected, so the policy can evolve without regenerating.
 //
 // Each corpus file has a sidecar <name>.metadata.json:
@@ -37,9 +39,11 @@ import (
 // StmtResult is one raw oracle observation for a single statement.
 type StmtResult struct {
 	Offset    int    // byte offset of the statement within the case SQL
+	End       int    // byte offset one past the statement, within the case SQL
 	OK        bool   // sqlite3_prepare_v2 returned SQLITE_OK
 	RC        int    // prepare result code when !OK
 	ErrOffset int    // sqlite3_error_offset within the case SQL; -1 if unknown
+	Tail      int    // pzTail within the case SQL: how far the parser got
 	Message   string // sqlite3_errmsg when !OK
 }
 
@@ -55,6 +59,30 @@ type Expectation struct {
 	OK      bool
 	Message string // expected parse error message when !OK
 	Offset  int    // expected error offset when !OK; -1 if unknown
+
+	// Unreached lists byte ranges of the case SQL that SQLite's parser
+	// never looked at, because a grammar action failed part-way through a
+	// statement and sqlite3RunParser abandoned the rest of it. The corpus
+	// says nothing about whether that text is valid SQL, so a parser under
+	// test may fail inside one of these ranges without being wrong.
+	Unreached []Range
+}
+
+// Range is a half-open byte range of a case's SQL.
+type Range struct{ Start, End int }
+
+// IsUnreached reports whether offset falls in text SQLite's parser never
+// reached. An offset of -1 (unknown) is never unreached.
+func (e Expectation) IsUnreached(offset int) bool {
+	if offset < 0 {
+		return false
+	}
+	for _, r := range e.Unreached {
+		if offset >= r.Start && offset < r.End {
+			return true
+		}
+	}
+	return false
 }
 
 // syntaxFamily matches error messages produced by SQLite's parser/tokenizer
@@ -79,14 +107,24 @@ func IsSyntaxError(msg string) bool {
 
 // Expected derives the harness expectation: the first syntax-family error in
 // statement order wins (meyer is fail-fast); if there is none, every
-// statement must parse.
+// statement must parse, except within the ranges SQLite's own parser never
+// reached.
 func (c *Case) Expected() Expectation {
+	var unreached []Range
 	for _, r := range c.Results {
-		if !r.OK && IsSyntaxError(r.Message) {
+		if r.OK {
+			continue
+		}
+		if IsSyntaxError(r.Message) {
 			return Expectation{OK: false, Message: r.Message, Offset: r.ErrOffset}
 		}
+		// A semantic failure means the statement parsed -- but only as far
+		// as the parser had got when the grammar action raised it.
+		if r.Tail < r.End {
+			unreached = append(unreached, Range{Start: r.Tail, End: r.End})
+		}
 	}
-	return Expectation{OK: true, Offset: -1}
+	return Expectation{OK: true, Offset: -1, Unreached: unreached}
 }
 
 const (
@@ -132,6 +170,15 @@ func Read(path string) ([]Case, error) {
 			c.Results = append(c.Results, r)
 			i++
 		}
+		// Statement ends are implied by the next statement's offset, and by
+		// the end of the case SQL for the last one.
+		for j := range c.Results {
+			if j+1 < len(c.Results) {
+				c.Results[j].End = c.Results[j+1].Offset
+			} else {
+				c.Results[j].End = len(c.SQL)
+			}
+		}
 		cases = append(cases, c)
 	}
 	return cases, nil
@@ -150,16 +197,19 @@ func parseResultLine(line string) (StmtResult, error) {
 	if parts[1] == "ok" {
 		return StmtResult{Offset: off, OK: true}, nil
 	}
-	fields := strings.SplitN(strings.TrimPrefix(parts[1], "err "), " ", 3)
-	if !strings.HasPrefix(parts[1], "err ") || len(fields) != 3 {
+	fields := strings.SplitN(strings.TrimPrefix(parts[1], "err "), " ", 4)
+	if !strings.HasPrefix(parts[1], "err ") || len(fields) != 4 {
 		return StmtResult{}, fmt.Errorf("malformed result line %q", line)
 	}
 	rc, err1 := strconv.Atoi(fields[0])
 	erroff, err2 := strconv.Atoi(fields[1])
-	if err1 != nil || err2 != nil {
+	tail, err3 := strconv.Atoi(fields[2])
+	if err1 != nil || err2 != nil || err3 != nil {
 		return StmtResult{}, fmt.Errorf("malformed result line %q", line)
 	}
-	return StmtResult{Offset: off, RC: rc, ErrOffset: erroff, Message: fields[2]}, nil
+	return StmtResult{
+		Offset: off, RC: rc, ErrOffset: erroff, Tail: tail, Message: fields[3],
+	}, nil
 }
 
 // CheckCase reports whether a case can be represented in the file format
@@ -186,7 +236,8 @@ func Write(path string, cases []Case) error {
 			if r.OK {
 				fmt.Fprintf(&b, "stmt %d ok\n", r.Offset)
 			} else {
-				fmt.Fprintf(&b, "stmt %d err %d %d %s\n", r.Offset, r.RC, r.ErrOffset, r.Message)
+				fmt.Fprintf(&b, "stmt %d err %d %d %d %s\n",
+					r.Offset, r.RC, r.ErrOffset, r.Tail, r.Message)
 			}
 		}
 	}

internal/testfile/testfile_test.go

@@ -7,20 +7,23 @@ import (
 )
 
 func TestRoundTrip(t *testing.T) {
+	// Statement ends are derived on read from the following statement's
+	// offset (and the length of the SQL for the last one), so they are
+	// spelled out here even though Write does not emit them.
 	cases := []Case{
 		{
 			Name: "a-1.1",
 			SQL:  "CREATE TABLE t1(a int);\nSELECT * FROM t1;\n",
 			Results: []StmtResult{
-				{Offset: 0, OK: true},
-				{Offset: 24, RC: 1, ErrOffset: -1, Message: "no such table: t1"},
+				{Offset: 0, End: 24, OK: true},
+				{Offset: 24, End: 42, RC: 1, ErrOffset: -1, Tail: 42, Message: "no such table: t1"},
 			},
 		},
 		{
 			Name: "a-1.2",
 			SQL:  "SELEC 1;\n",
 			Results: []StmtResult{
-				{Offset: 0, RC: 1, ErrOffset: 0, Message: `near "SELEC": syntax error`},
+				{Offset: 0, End: 9, RC: 1, ErrOffset: 0, Tail: 5, Message: `near "SELEC": syntax error`},
 			},
 		},
 	}
@@ -39,23 +42,43 @@ func TestRoundTrip(t *testing.T) {
 
 func TestExpected(t *testing.T) {
 	tests := []struct {
-		name    string
-		results []StmtResult
-		wantOK  bool
-		wantMsg string
+		name          string
+		results       []StmtResult
+		wantOK        bool
+		wantMsg       string
+		wantUnreached []Range
 	}{
-		{"all ok", []StmtResult{{OK: true}, {OK: true}}, true, ""},
-		{"semantic only", []StmtResult{{RC: 1, ErrOffset: -1, Message: "no such table: t1"}}, true, ""},
+		{"all ok", []StmtResult{{OK: true}, {OK: true}}, true, "", nil},
+		{
+			"semantic only",
+			[]StmtResult{{End: 30, RC: 1, ErrOffset: -1, Tail: 30, Message: "no such table: t1"}},
+			true, "", nil,
+		},
 		{
 			"syntax after semantic",
 			[]StmtResult{
-				{Offset: 0, RC: 1, ErrOffset: -1, Message: "no such table: t1"},
-				{Offset: 20, RC: 1, ErrOffset: 21, Message: `near "WHERE": syntax error`},
+				{Offset: 0, End: 20, RC: 1, ErrOffset: -1, Tail: 20, Message: "no such table: t1"},
+				{Offset: 20, End: 40, RC: 1, ErrOffset: 21, Tail: 21, Message: `near "WHERE": syntax error`},
 			},
-			false, `near "WHERE": syntax error`,
+			false, `near "WHERE": syntax error`, nil,
+		},
+		{
+			"unrecognized token",
+			[]StmtResult{{End: 10, RC: 1, ErrOffset: 3, Tail: 3, Message: `unrecognized token: "0x"`}},
+			false, `unrecognized token: "0x"`, nil,
+		},
+		{
+			"incomplete",
+			[]StmtResult{{End: 10, RC: 1, ErrOffset: -1, Tail: 10, Message: "incomplete input"}},
+			false, "incomplete input", nil,
+		},
+		{
+			// sqlite3BeginTrigger fails at the trigger_decl reduce, so
+			// everything after BEGIN is never parsed.
+			"semantic failure part-way through a statement",
+			[]StmtResult{{Offset: 0, End: 62, RC: 1, ErrOffset: -1, Tail: 42, Message: "no such table: main.t1"}},
+			true, "", []Range{{Start: 42, End: 62}},
 		},
-		{"unrecognized token", []StmtResult{{RC: 1, ErrOffset: 3, Message: `unrecognized token: "0x"`}}, false, `unrecognized token: "0x"`},
-		{"incomplete", []StmtResult{{RC: 1, ErrOffset: -1, Message: "incomplete input"}}, false, "incomplete input"},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
@@ -64,6 +87,17 @@ func TestExpected(t *testing.T) {
 			if exp.OK != tt.wantOK || exp.Message != tt.wantMsg {
 				t.Fatalf("Expected() = %+v, want ok=%v msg=%q", exp, tt.wantOK, tt.wantMsg)
 			}
+			if !reflect.DeepEqual(exp.Unreached, tt.wantUnreached) {
+				t.Fatalf("Unreached = %+v, want %+v", exp.Unreached, tt.wantUnreached)
+			}
+			for _, r := range tt.wantUnreached {
+				if !exp.IsUnreached(r.Start) || exp.IsUnreached(r.End) {
+					t.Fatalf("IsUnreached disagrees with %+v", r)
+				}
+			}
+			if exp.IsUnreached(-1) {
+				t.Fatal("IsUnreached(-1) should be false")
+			}
 		})
 	}
 }

parser/parser_test.go

@@ -86,6 +86,14 @@ func runCase(c testfile.Case) error {
 	exp := c.Expected()
 	if exp.OK {
 		if err != nil {
+			// A grammar action can fail part-way through a statement, and
+			// SQLite then leaves the rest of it unparsed. The corpus records
+			// how far it got; failing inside text SQLite never reached is
+			// not a conformance failure, because nothing verified it.
+			var pe *parser.Error
+			if errors.As(err, &pe) && exp.IsUnreached(pe.Offset) {
+				return nil
+			}
 			return fmt.Errorf("expected successful parse, got error: %v", err)
 		}
 		return nil

parser/testdata/README.md

@@ -11,8 +11,11 @@ The `*.test` files in this directory are generated by
   every case is prepared independently with `sqlite3_prepare_v2` against an
   empty in-memory database (nothing is executed) using the oracle program in
   `cmd/regenerate-parse/oracle.c`, compiled against the pinned amalgamation.
-  The raw per-statement outcomes (OK, or result code + error offset + exact
-  error message) are stored in the corpus files.
+  The raw per-statement outcomes (OK, or result code + error offset +
+  `pzTail` + exact error message) are stored in the corpus files. `pzTail`
+  records how far SQLite's parser got: a grammar action can fail part-way
+  through a statement and leave the rest of it unparsed, and the harness
+  needs to know which text was never verified.
 
 ## Pinned SQLite release