release(v0.49.0): finalize parser recovery and performance ratchet

CHANGELOG.md

@@ -77,6 +77,14 @@ for tags and release notes while still in `0.x`.
 
 ### Fixed
 
+- C enum lists with three or more enumerators no longer publish `ERROR` or
+  `MISSING` nodes. A clean forest result can replace a recovered tree only
+  after it covers the full source and contains no recovery nodes. This fixes
+  [issue #667](https://github.com/odvcencio/gotreesitter/issues/667).
+
+- `Node.HasErrorOrMissing` reports both recovery node forms. The
+  `grammargen parse -strict` command now rejects either form.
+
 - JavaScript, TypeScript, and TSX scanners now bind external results through
   each language's positional symbol table. Regenerated blobs no longer mistype
   shifted external symbols.
@@ -186,6 +194,30 @@ for tags and release notes while still in `0.x`.
 
 ### Changed
 
+- Parser stop checks now skip inactive callbacks and keep the common callback
+  direct. Result materialization reads the wall clock every 64 checkpoints.
+  Cancellation and sticky stop checks still run at every checkpoint.
+
+- GLR recovery now computes C-compatible error cost and visible counts in one
+  tree walk. Memo indexing uses pointer-bit folds and checks the primary way
+  first. Graph-structured stack (GSS) nodes store clean-zero merge results
+  without a larger node layout. Extra-link mutations invalidate the result.
+
+- C-recovery promotes an error stack to the graph-structured stack before
+  reduction forks. Deep recovery branches now share their immutable prefix.
+  The Swift recovery witness reduced time by 9.96%, bytes by 59.65%, and mean
+  peak resident memory by 22.09%. The 20-seed combined suite reduced KDL
+  recovery time by 1.20%, bytes by 13.14%, and allocations by 1.58%.
+  Other parser timings stayed neutral.
+
+- The randomized benchmark suite now accepts an exact recovery corpus file and
+  language. The 20-seed comparison against the release boundary reduced the
+  timing geomean by 1.77%. Elixir recovery improved by 15.21%, KDL recovery by
+  9.12%, full parse by 1.16%, and incremental no-edit by 6.51%.
+  `FactProgram` parse and extraction improved by 1.23%.
+  The parser-core control stayed neutral. No timing, byte, or allocation metric
+  had a significant regression.
+
 - The guarded parser-core bytecode experiment now supports `REDUCE_CHAIN` and
   `REDUCE_SHIFT`. The corridor remains off by default. Each superinstruction
   also requires its own experiment gate.

README.md

@@ -773,11 +773,9 @@ Test suite covers: smoke tests (206 grammars), golden S-expression snapshots, hi
 
 ## Roadmap
 
-The current release is **v0.49.0**. Publication remains pending the exact
-commit CI run and the governed soak. The latest immutable published release is
-**v0.48.1**.
+The current release is **v0.49.0**.
 
-This candidate consolidates parser correctness, recovery bounds, parser-core
+This release consolidates parser correctness, recovery bounds, parser-core
 bytecode, fact extraction bytecode, replay caches, randomized benchmarks, and
 V10 fleet controls. It also adds opt-in Lean 4 support and scanner corrections
 for JavaScript, TypeScript, and TSX.

arena.go

@@ -131,6 +131,7 @@ type nodeArena struct {
 	fieldSourceSlabs                   []fieldSourceSliceSlab
 	externalScannerNodeCheckpoints     externalScannerCheckpointSet
 	externalScannerNodeCheckpointSlabs []externalScannerCheckpointSlab
+	hiddenFieldRepeatScratch           hiddenFieldRepeatScratch
 	childSlabCursor                    int
 	fieldSlabCursor                    int
 	fieldSourceSlabCursor              int
@@ -576,6 +577,7 @@ func (a *nodeArena) reset() {
 	// Drop any subtree pointer left in the compare scratch so a pooled arena
 	// sitting idle between parses doesn't pin the previous parse's tree.
 	a.forestResultLinkCompareScratch = [2]stackEntry{}
+	a.hiddenFieldRepeatScratch.reset()
 }
 
 func (a *nodeArena) resetPrimaryNodes() {

cgo_harness/c_issue667_parity_test.go

@@ -0,0 +1,30 @@
+//go:build cgo && treesitter_c_parity
+
+package cgoharness
+
+import "testing"
+
+func TestIssue667CEnumListsMatchCReference(t *testing.T) {
+	cases := []struct {
+		name   string
+		source string
+	}{
+		{name: "one", source: "enum E { A };\n"},
+		{name: "two", source: "enum E { A, B };\n"},
+		{name: "three", source: "enum E { A, B, C };\n"},
+		{name: "four", source: "enum E { A, B, C, D };\n"},
+		{name: "five", source: "enum E { A, B, C, D, E };\n"},
+		{name: "trailing comma", source: "enum E { A, B, C, };\n"},
+		{name: "explicit values", source: "enum E { A = 1, B = 2, C = 3 };\n"},
+		{name: "typedef", source: "typedef enum { RED, GREEN, BLUE } Colour;\n"},
+		{name: "comment before close", source: "enum E { A, B, C /* close */\n};\n"},
+		{name: "neighboring declarations", source: "enum First { A, B, C };\nenum Second { D, E, F };\n"},
+	}
+
+	for _, test := range cases {
+		test := test
+		t.Run(test.name, func(t *testing.T) {
+			runParityCase(t, parityCase{name: "c"}, "issue667-"+test.name, []byte(test.source))
+		})
+	}
+}

cmd/grammargen/commands.go

@@ -128,7 +128,7 @@ func registerSampleFlags(fs *flag.FlagSet, sample *sampleFlags) {
 func registerParseOptionFlags(fs *flag.FlagSet, opts *parseOptions) {
 	fs.StringVar(&opts.format, "format", "text", "output format: text, sexpr, json")
 	fs.BoolVar(&opts.runtime, "runtime", false, "print parser runtime summary")
-	fs.BoolVar(&opts.strict, "strict", false, "exit non-zero if the parse has ERROR nodes or stops early")
+	fs.BoolVar(&opts.strict, "strict", false, "exit non-zero if the parse has ERROR or MISSING nodes, or stops early")
 	fs.StringVar(&opts.expectPath, "expect", "", "path to expected S-expression file")
 	fs.StringVar(&opts.writeExpectPath, "write-expect", "", "write actual S-expression to this file")
 }
@@ -386,6 +386,7 @@ func printParseResult(result parseResult, lang *gotreesitter.Language, runtime b
 	}
 	fmt.Printf("Root:    %s [%d:%d]\n", root.Type(lang), root.StartByte(), root.EndByte())
 	fmt.Printf("Error:   %v\n", root.HasError())
+	fmt.Printf("Error or missing: %v\n", root.HasErrorOrMissing())
 	fmt.Printf("Stop:    %s\n", result.tree.ParseStopReason())
 	if runtime {
 		fmt.Printf("Runtime: %s\n", result.tree.ParseRuntime().Summary())
@@ -398,7 +399,7 @@ func parseResultFailed(result parseResult) bool {
 	if result.err != nil || result.tree == nil || result.root == nil {
 		return true
 	}
-	return result.root.HasError() || result.tree.ParseStoppedEarly()
+	return result.root.HasErrorOrMissing() || result.tree.ParseStoppedEarly()
 }
 
 func validateParseOptions(opts parseOptions) {
@@ -494,16 +495,17 @@ type parseJSON struct {
 }
 
 type parseStatus struct {
-	OK           bool   `json:"ok"`
-	Root         string `json:"root,omitempty"`
-	StartByte    uint32 `json:"start_byte"`
-	EndByte      uint32 `json:"end_byte"`
-	HasError     bool   `json:"has_error"`
-	StoppedEarly bool   `json:"stopped_early"`
-	StopReason   string `json:"stop_reason,omitempty"`
-	SExpr        string `json:"sexpr,omitempty"`
-	Runtime      string `json:"runtime,omitempty"`
-	Error        string `json:"error,omitempty"`
+	OK                bool   `json:"ok"`
+	Root              string `json:"root,omitempty"`
+	StartByte         uint32 `json:"start_byte"`
+	EndByte           uint32 `json:"end_byte"`
+	HasError          bool   `json:"has_error"`
+	HasErrorOrMissing bool   `json:"has_error_or_missing"`
+	StoppedEarly      bool   `json:"stopped_early"`
+	StopReason        string `json:"stop_reason,omitempty"`
+	SExpr             string `json:"sexpr,omitempty"`
+	Runtime           string `json:"runtime,omitempty"`
+	Error             string `json:"error,omitempty"`
 }
 
 func parseJSONReport(name, sampleName string, sampleBytes int, result parseResult, lang *gotreesitter.Language, runtime bool, golden goldenResult) *parseJSON {
@@ -525,14 +527,15 @@ func parseJSONReport(name, sampleName string, sampleBytes int, result parseResul
 	}
 	root := result.root
 	out.Parse = parseStatus{
-		OK:           !parseResultFailed(result),
-		Root:         root.Type(lang),
-		StartByte:    root.StartByte(),
-		EndByte:      root.EndByte(),
-		HasError:     root.HasError(),
-		StoppedEarly: result.tree.ParseStoppedEarly(),
-		StopReason:   string(result.tree.ParseStopReason()),
-		SExpr:        resultSExpr(result, lang),
+		OK:                !parseResultFailed(result),
+		Root:              root.Type(lang),
+		StartByte:         root.StartByte(),
+		EndByte:           root.EndByte(),
+		HasError:          root.HasError(),
+		HasErrorOrMissing: root.HasErrorOrMissing(),
+		StoppedEarly:      result.tree.ParseStoppedEarly(),
+		StopReason:        string(result.tree.ParseStopReason()),
+		SExpr:             resultSExpr(result, lang),
 	}
 	if runtime {
 		out.Parse.Runtime = result.tree.ParseRuntime().Summary()

cmd/grammargen/commands_test.go

@@ -3,6 +3,9 @@ package main
 import (
 	"reflect"
 	"testing"
+
+	gotreesitter "github.com/odvcencio/gotreesitter"
+	"github.com/odvcencio/gotreesitter/grammars"
 )
 
 func TestNormalizeSubcommandArgsAllowsGrammarBeforeFlags(t *testing.T) {
@@ -44,3 +47,19 @@ func TestNormalizeSubcommandArgsHandlesAuthoringValueFlags(t *testing.T) {
 		t.Fatalf("normalizeSubcommandArgs() = %#v, want %#v", got, want)
 	}
 }
+
+func TestParseResultFailedRejectsMissingNode(t *testing.T) {
+	lang := grammars.CLanguage()
+	tree, err := gotreesitter.NewParser(lang).Parse([]byte("int value"))
+	if err != nil {
+		t.Fatalf("Parse: %v", err)
+	}
+	defer tree.Release()
+	root := tree.RootNode()
+	if root == nil || !root.HasErrorOrMissing() {
+		t.Fatalf("expected recovery node, got %v", root)
+	}
+	if !parseResultFailed(parseResult{tree: tree, root: root}) {
+		t.Fatal("parseResultFailed accepted a tree with a missing node")
+	}
+}

glr.go

@@ -406,6 +406,13 @@ type gssCleanZeroErrorCacheEntry struct {
 	clean       bool
 }
 
+const (
+	gssCleanZeroUnknown uint8 = iota
+	gssCleanZeroClean
+	gssCleanZeroDirty
+	gssCleanZeroVisiting
+)
+
 type glrEntryScratch struct {
 	slabs          []stackEntrySlab
 	slabCursor     int
@@ -1154,10 +1161,9 @@ func (s *glrMergeScratch) bumpGSSPointerEpoch() {
 	s.gssPointerEpoch++
 }
 
-// invalidateGSSPointersForReuse drops every merge-scratch reference whose
-// identity is tied to a gssNode address, then advances the epochs guarding the
-// uintptr-keyed caches. Callers may recycle GSS slab slots only after this and
-// after clearing any live glrStack slices that used the old graph.
+// invalidateGSSPointersForReuse invalidates merge-scratch state whose identity
+// is tied to a gssNode address. Callers may recycle GSS slab slots only after
+// this and after clearing live glrStack slices that used the old graph.
 func (s *glrMergeScratch) invalidateGSSPointersForReuse() {
 	if s == nil {
 		return
@@ -1400,6 +1406,38 @@ func (s *glrMergeScratch) beginCleanZeroEpoch() {
 	s.cleanZeroEpoch++
 }
 
+// GSS node cleanliness remains stable between recovery-relevant node changes.
+// Merge paths add only clean links. aggGen invalidates payload mutations, and
+// every allocation or slab recycle resets the state.
+func lookupCleanZeroNodeState(n *gssNode, gen uint64) (bool, bool) {
+	if n == nil || n.aggGen != gen {
+		return false, false
+	}
+	switch n.cleanZeroState {
+	case gssCleanZeroClean:
+		return true, true
+	case gssCleanZeroDirty:
+		return false, true
+	default:
+		return false, false
+	}
+}
+
+func storeCleanZeroNodeState(n *gssNode, gen uint64, clean bool) {
+	if n == nil {
+		return
+	}
+	if n.aggGen != gen {
+		n.aggGen = gen
+		n.aggValid = 0
+	}
+	if clean {
+		n.cleanZeroState = gssCleanZeroClean
+	} else {
+		n.cleanZeroState = gssCleanZeroDirty
+	}
+}
+
 // ensureMergeHotCaches provisions the fixed-size merge-attempt caches. Called
 // only for persistent (pooled, per-parse) scratches so their cost amortizes
 // across the whole parse; one-shot local scratches never allocate these.
@@ -1411,10 +1449,6 @@ func (s *glrMergeScratch) ensureMergeHotCaches() {
 		s.shapePrefixCache = make([]glrShapePrefixCacheEntry, glrShapePrefixCacheSize)
 		s.shapePrefixBytes = int64(cap(s.shapePrefixCache)) * int64(unsafe.Sizeof(glrShapePrefixCacheEntry{}))
 	}
-	if len(s.cleanZeroFront) == 0 {
-		s.cleanZeroFront = make([]glrCleanZeroFrontCacheEntry, glrCleanZeroFrontCacheSize)
-		s.cleanZeroBytes = int64(cap(s.cleanZeroFront)) * int64(unsafe.Sizeof(glrCleanZeroFrontCacheEntry{}))
-	}
 	if len(s.spineEquivCache) == 0 {
 		s.spineEquivCache = make([]glrSpineEquivCacheEntry, glrSpineEquivCacheSize)
 		s.spineEquivBytes = glrSpineEquivCacheBytesForCap(cap(s.spineEquivCache))
@@ -3530,35 +3564,16 @@ func gssNodeCleanZeroErrorAllLinksWithScratch(scratch *glrMergeScratch, n *gssNo
 	if scratch.cleanZeroEpoch == 0 {
 		scratch.beginCleanZeroEpoch()
 	}
-	if clean, ok := lookupCleanZeroFrontCache(scratch, n); ok {
+	cleanGen := gssPrefixAggGen.Load()
+	if clean, ok := lookupCleanZeroNodeState(n, cleanGen); ok {
 		return clean
 	}
-	if entry, ok := scratch.cleanZeroCache[n]; ok && entry.resultEpoch == scratch.cleanZeroEpoch {
-		storeCleanZeroFrontCache(scratch, n, entry.clean)
-		return entry.clean
-	}
-	if scratch.cleanZeroCache == nil {
-		scratch.cleanZeroCache = make(map[*gssNode]gssCleanZeroErrorCacheEntry, 64)
-	}
-	if scratch.cleanZeroScan == ^uint32(0) {
-		for node, entry := range scratch.cleanZeroCache {
-			entry.scanEpoch = 0
-			scratch.cleanZeroCache[node] = entry
-		}
-		scratch.cleanZeroScan = 0
-	}
-	scratch.cleanZeroScan++
-	scanEpoch := scratch.cleanZeroScan
 	frames := scratch.cleanZeroFrames[:0]
 	frames = append(frames, gssCleanZeroFrame{node: n})
 	cacheFailure := func() bool {
 		for _, frame := range frames {
-			scratch.cleanZeroCache[frame.node] = gssCleanZeroErrorCacheEntry{
-				resultEpoch: scratch.cleanZeroEpoch,
-				clean:       false,
-			}
+			storeCleanZeroNodeState(frame.node, cleanGen, false)
 		}
-		storeCleanZeroFrontCache(scratch, n, false)
 		scratch.cleanZeroFrames = frames[:0]
 		return false
 	}
@@ -3567,26 +3582,26 @@ func gssNodeCleanZeroErrorAllLinksWithScratch(scratch *glrMergeScratch, n *gssNo
 		frame := &frames[last]
 		cur := frame.node
 		if frame.nextLink == 0 {
-			entry, ok := scratch.cleanZeroCache[cur]
-			if ok && entry.resultEpoch == scratch.cleanZeroEpoch {
-				if !entry.clean {
-					return cacheFailure()
-				}
-				frames = frames[:last]
-				continue
+			state := gssCleanZeroUnknown
+			if cur.aggGen == cleanGen {
+				state = cur.cleanZeroState
 			}
-			if ok && entry.scanEpoch == scanEpoch {
+			switch state {
+			case gssCleanZeroDirty:
+				return cacheFailure()
+			case gssCleanZeroClean, gssCleanZeroVisiting:
 				frames = frames[:last]
 				continue
+			default:
+				if cur.aggGen != cleanGen {
+					cur.aggGen = cleanGen
+					cur.aggValid = 0
+				}
+				cur.cleanZeroState = gssCleanZeroVisiting
 			}
-			entry.scanEpoch = scanEpoch
-			scratch.cleanZeroCache[cur] = entry
 		}
 		if frame.nextLink == cur.linkCount() {
-			scratch.cleanZeroCache[cur] = gssCleanZeroErrorCacheEntry{
-				resultEpoch: scratch.cleanZeroEpoch,
-				clean:       true,
-			}
+			storeCleanZeroNodeState(cur, cleanGen, true)
 			frames = frames[:last]
 			continue
 		}
@@ -3600,7 +3615,6 @@ func gssNodeCleanZeroErrorAllLinksWithScratch(scratch *glrMergeScratch, n *gssNo
 			frames = append(frames, gssCleanZeroFrame{node: prev})
 		}
 	}
-	storeCleanZeroFrontCache(scratch, n, true)
 	scratch.cleanZeroFrames = frames[:0]
 	return true
 }