diff --git a/CHANGELOG.md b/CHANGELOG.md index 3bdb4449..49154251 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -77,6 +77,14 @@ for tags and release notes while still in `0.x`. ### Fixed +- C enum lists with three or more enumerators no longer publish `ERROR` or + `MISSING` nodes. A clean forest result can replace a recovered tree only + after it covers the full source and contains no recovery nodes. This fixes + [issue #667](https://github.com/odvcencio/gotreesitter/issues/667). + +- `Node.HasErrorOrMissing` reports both recovery node forms. The + `grammargen parse -strict` command now rejects either form. + - JavaScript, TypeScript, and TSX scanners now bind external results through each language's positional symbol table. Regenerated blobs no longer mistype shifted external symbols. @@ -186,6 +194,30 @@ for tags and release notes while still in `0.x`. ### Changed +- Parser stop checks now skip inactive callbacks and keep the common callback + direct. Result materialization reads the wall clock every 64 checkpoints. + Cancellation and sticky stop checks still run at every checkpoint. + +- GLR recovery now computes C-compatible error cost and visible counts in one + tree walk. Memo indexing uses pointer-bit folds and checks the primary way + first. Graph-structured stack (GSS) nodes store clean-zero merge results + without a larger node layout. Extra-link mutations invalidate the result. + +- C-recovery promotes an error stack to the graph-structured stack before + reduction forks. Deep recovery branches now share their immutable prefix. + The Swift recovery witness reduced time by 9.96%, bytes by 59.65%, and mean + peak resident memory by 22.09%. The 20-seed combined suite reduced KDL + recovery time by 1.20%, bytes by 13.14%, and allocations by 1.58%. + Other parser timings stayed neutral. + +- The randomized benchmark suite now accepts an exact recovery corpus file and + language. The 20-seed comparison against the release boundary reduced the + timing geomean by 1.77%. Elixir recovery improved by 15.21%, KDL recovery by + 9.12%, full parse by 1.16%, and incremental no-edit by 6.51%. + `FactProgram` parse and extraction improved by 1.23%. + The parser-core control stayed neutral. No timing, byte, or allocation metric + had a significant regression. + - The guarded parser-core bytecode experiment now supports `REDUCE_CHAIN` and `REDUCE_SHIFT`. The corridor remains off by default. Each superinstruction also requires its own experiment gate. diff --git a/README.md b/README.md index b6cde9e5..3d802eac 100644 --- a/README.md +++ b/README.md @@ -773,11 +773,9 @@ Test suite covers: smoke tests (206 grammars), golden S-expression snapshots, hi ## Roadmap -The current release is **v0.49.0**. Publication remains pending the exact -commit CI run and the governed soak. The latest immutable published release is -**v0.48.1**. +The current release is **v0.49.0**. -This candidate consolidates parser correctness, recovery bounds, parser-core +This release consolidates parser correctness, recovery bounds, parser-core bytecode, fact extraction bytecode, replay caches, randomized benchmarks, and V10 fleet controls. It also adds opt-in Lean 4 support and scanner corrections for JavaScript, TypeScript, and TSX. diff --git a/arena.go b/arena.go index 2598e715..3c623254 100644 --- a/arena.go +++ b/arena.go @@ -131,6 +131,7 @@ type nodeArena struct { fieldSourceSlabs []fieldSourceSliceSlab externalScannerNodeCheckpoints externalScannerCheckpointSet externalScannerNodeCheckpointSlabs []externalScannerCheckpointSlab + hiddenFieldRepeatScratch hiddenFieldRepeatScratch childSlabCursor int fieldSlabCursor int fieldSourceSlabCursor int @@ -576,6 +577,7 @@ func (a *nodeArena) reset() { // Drop any subtree pointer left in the compare scratch so a pooled arena // sitting idle between parses doesn't pin the previous parse's tree. a.forestResultLinkCompareScratch = [2]stackEntry{} + a.hiddenFieldRepeatScratch.reset() } func (a *nodeArena) resetPrimaryNodes() { diff --git a/cgo_harness/c_issue667_parity_test.go b/cgo_harness/c_issue667_parity_test.go new file mode 100644 index 00000000..f1a3b8c7 --- /dev/null +++ b/cgo_harness/c_issue667_parity_test.go @@ -0,0 +1,30 @@ +//go:build cgo && treesitter_c_parity + +package cgoharness + +import "testing" + +func TestIssue667CEnumListsMatchCReference(t *testing.T) { + cases := []struct { + name string + source string + }{ + {name: "one", source: "enum E { A };\n"}, + {name: "two", source: "enum E { A, B };\n"}, + {name: "three", source: "enum E { A, B, C };\n"}, + {name: "four", source: "enum E { A, B, C, D };\n"}, + {name: "five", source: "enum E { A, B, C, D, E };\n"}, + {name: "trailing comma", source: "enum E { A, B, C, };\n"}, + {name: "explicit values", source: "enum E { A = 1, B = 2, C = 3 };\n"}, + {name: "typedef", source: "typedef enum { RED, GREEN, BLUE } Colour;\n"}, + {name: "comment before close", source: "enum E { A, B, C /* close */\n};\n"}, + {name: "neighboring declarations", source: "enum First { A, B, C };\nenum Second { D, E, F };\n"}, + } + + for _, test := range cases { + test := test + t.Run(test.name, func(t *testing.T) { + runParityCase(t, parityCase{name: "c"}, "issue667-"+test.name, []byte(test.source)) + }) + } +} diff --git a/cmd/grammargen/commands.go b/cmd/grammargen/commands.go index 8e02ef6e..8b1cadf7 100644 --- a/cmd/grammargen/commands.go +++ b/cmd/grammargen/commands.go @@ -128,7 +128,7 @@ func registerSampleFlags(fs *flag.FlagSet, sample *sampleFlags) { func registerParseOptionFlags(fs *flag.FlagSet, opts *parseOptions) { fs.StringVar(&opts.format, "format", "text", "output format: text, sexpr, json") fs.BoolVar(&opts.runtime, "runtime", false, "print parser runtime summary") - fs.BoolVar(&opts.strict, "strict", false, "exit non-zero if the parse has ERROR nodes or stops early") + fs.BoolVar(&opts.strict, "strict", false, "exit non-zero if the parse has ERROR or MISSING nodes, or stops early") fs.StringVar(&opts.expectPath, "expect", "", "path to expected S-expression file") fs.StringVar(&opts.writeExpectPath, "write-expect", "", "write actual S-expression to this file") } @@ -386,6 +386,7 @@ func printParseResult(result parseResult, lang *gotreesitter.Language, runtime b } fmt.Printf("Root: %s [%d:%d]\n", root.Type(lang), root.StartByte(), root.EndByte()) fmt.Printf("Error: %v\n", root.HasError()) + fmt.Printf("Error or missing: %v\n", root.HasErrorOrMissing()) fmt.Printf("Stop: %s\n", result.tree.ParseStopReason()) if runtime { fmt.Printf("Runtime: %s\n", result.tree.ParseRuntime().Summary()) @@ -398,7 +399,7 @@ func parseResultFailed(result parseResult) bool { if result.err != nil || result.tree == nil || result.root == nil { return true } - return result.root.HasError() || result.tree.ParseStoppedEarly() + return result.root.HasErrorOrMissing() || result.tree.ParseStoppedEarly() } func validateParseOptions(opts parseOptions) { @@ -494,16 +495,17 @@ type parseJSON struct { } type parseStatus struct { - OK bool `json:"ok"` - Root string `json:"root,omitempty"` - StartByte uint32 `json:"start_byte"` - EndByte uint32 `json:"end_byte"` - HasError bool `json:"has_error"` - StoppedEarly bool `json:"stopped_early"` - StopReason string `json:"stop_reason,omitempty"` - SExpr string `json:"sexpr,omitempty"` - Runtime string `json:"runtime,omitempty"` - Error string `json:"error,omitempty"` + OK bool `json:"ok"` + Root string `json:"root,omitempty"` + StartByte uint32 `json:"start_byte"` + EndByte uint32 `json:"end_byte"` + HasError bool `json:"has_error"` + HasErrorOrMissing bool `json:"has_error_or_missing"` + StoppedEarly bool `json:"stopped_early"` + StopReason string `json:"stop_reason,omitempty"` + SExpr string `json:"sexpr,omitempty"` + Runtime string `json:"runtime,omitempty"` + Error string `json:"error,omitempty"` } func parseJSONReport(name, sampleName string, sampleBytes int, result parseResult, lang *gotreesitter.Language, runtime bool, golden goldenResult) *parseJSON { @@ -525,14 +527,15 @@ func parseJSONReport(name, sampleName string, sampleBytes int, result parseResul } root := result.root out.Parse = parseStatus{ - OK: !parseResultFailed(result), - Root: root.Type(lang), - StartByte: root.StartByte(), - EndByte: root.EndByte(), - HasError: root.HasError(), - StoppedEarly: result.tree.ParseStoppedEarly(), - StopReason: string(result.tree.ParseStopReason()), - SExpr: resultSExpr(result, lang), + OK: !parseResultFailed(result), + Root: root.Type(lang), + StartByte: root.StartByte(), + EndByte: root.EndByte(), + HasError: root.HasError(), + HasErrorOrMissing: root.HasErrorOrMissing(), + StoppedEarly: result.tree.ParseStoppedEarly(), + StopReason: string(result.tree.ParseStopReason()), + SExpr: resultSExpr(result, lang), } if runtime { out.Parse.Runtime = result.tree.ParseRuntime().Summary() diff --git a/cmd/grammargen/commands_test.go b/cmd/grammargen/commands_test.go index b955bb34..6aba9fa8 100644 --- a/cmd/grammargen/commands_test.go +++ b/cmd/grammargen/commands_test.go @@ -3,6 +3,9 @@ package main import ( "reflect" "testing" + + gotreesitter "github.com/odvcencio/gotreesitter" + "github.com/odvcencio/gotreesitter/grammars" ) func TestNormalizeSubcommandArgsAllowsGrammarBeforeFlags(t *testing.T) { @@ -44,3 +47,19 @@ func TestNormalizeSubcommandArgsHandlesAuthoringValueFlags(t *testing.T) { t.Fatalf("normalizeSubcommandArgs() = %#v, want %#v", got, want) } } + +func TestParseResultFailedRejectsMissingNode(t *testing.T) { + lang := grammars.CLanguage() + tree, err := gotreesitter.NewParser(lang).Parse([]byte("int value")) + if err != nil { + t.Fatalf("Parse: %v", err) + } + defer tree.Release() + root := tree.RootNode() + if root == nil || !root.HasErrorOrMissing() { + t.Fatalf("expected recovery node, got %v", root) + } + if !parseResultFailed(parseResult{tree: tree, root: root}) { + t.Fatal("parseResultFailed accepted a tree with a missing node") + } +} diff --git a/glr.go b/glr.go index 070e4897..f0b1b556 100644 --- a/glr.go +++ b/glr.go @@ -406,6 +406,13 @@ type gssCleanZeroErrorCacheEntry struct { clean bool } +const ( + gssCleanZeroUnknown uint8 = iota + gssCleanZeroClean + gssCleanZeroDirty + gssCleanZeroVisiting +) + type glrEntryScratch struct { slabs []stackEntrySlab slabCursor int @@ -1154,10 +1161,9 @@ func (s *glrMergeScratch) bumpGSSPointerEpoch() { s.gssPointerEpoch++ } -// invalidateGSSPointersForReuse drops every merge-scratch reference whose -// identity is tied to a gssNode address, then advances the epochs guarding the -// uintptr-keyed caches. Callers may recycle GSS slab slots only after this and -// after clearing any live glrStack slices that used the old graph. +// invalidateGSSPointersForReuse invalidates merge-scratch state whose identity +// is tied to a gssNode address. Callers may recycle GSS slab slots only after +// this and after clearing live glrStack slices that used the old graph. func (s *glrMergeScratch) invalidateGSSPointersForReuse() { if s == nil { return @@ -1400,6 +1406,38 @@ func (s *glrMergeScratch) beginCleanZeroEpoch() { s.cleanZeroEpoch++ } +// GSS node cleanliness remains stable between recovery-relevant node changes. +// Merge paths add only clean links. aggGen invalidates payload mutations, and +// every allocation or slab recycle resets the state. +func lookupCleanZeroNodeState(n *gssNode, gen uint64) (bool, bool) { + if n == nil || n.aggGen != gen { + return false, false + } + switch n.cleanZeroState { + case gssCleanZeroClean: + return true, true + case gssCleanZeroDirty: + return false, true + default: + return false, false + } +} + +func storeCleanZeroNodeState(n *gssNode, gen uint64, clean bool) { + if n == nil { + return + } + if n.aggGen != gen { + n.aggGen = gen + n.aggValid = 0 + } + if clean { + n.cleanZeroState = gssCleanZeroClean + } else { + n.cleanZeroState = gssCleanZeroDirty + } +} + // ensureMergeHotCaches provisions the fixed-size merge-attempt caches. Called // only for persistent (pooled, per-parse) scratches so their cost amortizes // across the whole parse; one-shot local scratches never allocate these. @@ -1411,10 +1449,6 @@ func (s *glrMergeScratch) ensureMergeHotCaches() { s.shapePrefixCache = make([]glrShapePrefixCacheEntry, glrShapePrefixCacheSize) s.shapePrefixBytes = int64(cap(s.shapePrefixCache)) * int64(unsafe.Sizeof(glrShapePrefixCacheEntry{})) } - if len(s.cleanZeroFront) == 0 { - s.cleanZeroFront = make([]glrCleanZeroFrontCacheEntry, glrCleanZeroFrontCacheSize) - s.cleanZeroBytes = int64(cap(s.cleanZeroFront)) * int64(unsafe.Sizeof(glrCleanZeroFrontCacheEntry{})) - } if len(s.spineEquivCache) == 0 { s.spineEquivCache = make([]glrSpineEquivCacheEntry, glrSpineEquivCacheSize) s.spineEquivBytes = glrSpineEquivCacheBytesForCap(cap(s.spineEquivCache)) @@ -3530,35 +3564,16 @@ func gssNodeCleanZeroErrorAllLinksWithScratch(scratch *glrMergeScratch, n *gssNo if scratch.cleanZeroEpoch == 0 { scratch.beginCleanZeroEpoch() } - if clean, ok := lookupCleanZeroFrontCache(scratch, n); ok { + cleanGen := gssPrefixAggGen.Load() + if clean, ok := lookupCleanZeroNodeState(n, cleanGen); ok { return clean } - if entry, ok := scratch.cleanZeroCache[n]; ok && entry.resultEpoch == scratch.cleanZeroEpoch { - storeCleanZeroFrontCache(scratch, n, entry.clean) - return entry.clean - } - if scratch.cleanZeroCache == nil { - scratch.cleanZeroCache = make(map[*gssNode]gssCleanZeroErrorCacheEntry, 64) - } - if scratch.cleanZeroScan == ^uint32(0) { - for node, entry := range scratch.cleanZeroCache { - entry.scanEpoch = 0 - scratch.cleanZeroCache[node] = entry - } - scratch.cleanZeroScan = 0 - } - scratch.cleanZeroScan++ - scanEpoch := scratch.cleanZeroScan frames := scratch.cleanZeroFrames[:0] frames = append(frames, gssCleanZeroFrame{node: n}) cacheFailure := func() bool { for _, frame := range frames { - scratch.cleanZeroCache[frame.node] = gssCleanZeroErrorCacheEntry{ - resultEpoch: scratch.cleanZeroEpoch, - clean: false, - } + storeCleanZeroNodeState(frame.node, cleanGen, false) } - storeCleanZeroFrontCache(scratch, n, false) scratch.cleanZeroFrames = frames[:0] return false } @@ -3567,26 +3582,26 @@ func gssNodeCleanZeroErrorAllLinksWithScratch(scratch *glrMergeScratch, n *gssNo frame := &frames[last] cur := frame.node if frame.nextLink == 0 { - entry, ok := scratch.cleanZeroCache[cur] - if ok && entry.resultEpoch == scratch.cleanZeroEpoch { - if !entry.clean { - return cacheFailure() - } - frames = frames[:last] - continue + state := gssCleanZeroUnknown + if cur.aggGen == cleanGen { + state = cur.cleanZeroState } - if ok && entry.scanEpoch == scanEpoch { + switch state { + case gssCleanZeroDirty: + return cacheFailure() + case gssCleanZeroClean, gssCleanZeroVisiting: frames = frames[:last] continue + default: + if cur.aggGen != cleanGen { + cur.aggGen = cleanGen + cur.aggValid = 0 + } + cur.cleanZeroState = gssCleanZeroVisiting } - entry.scanEpoch = scanEpoch - scratch.cleanZeroCache[cur] = entry } if frame.nextLink == cur.linkCount() { - scratch.cleanZeroCache[cur] = gssCleanZeroErrorCacheEntry{ - resultEpoch: scratch.cleanZeroEpoch, - clean: true, - } + storeCleanZeroNodeState(cur, cleanGen, true) frames = frames[:last] continue } @@ -3600,7 +3615,6 @@ func gssNodeCleanZeroErrorAllLinksWithScratch(scratch *glrMergeScratch, n *gssNo frames = append(frames, gssCleanZeroFrame{node: prev}) } } - storeCleanZeroFrontCache(scratch, n, true) scratch.cleanZeroFrames = frames[:0] return true } diff --git a/glr_forest.go b/glr_forest.go index f79ed314..600389ab 100644 --- a/glr_forest.go +++ b/glr_forest.go @@ -417,6 +417,42 @@ func (p *Parser) ParseForestExperimental(source []byte) (*Tree, bool) { return tree, true } +// maybeReplaceRecoveredTreeWithForest replaces a recovered DFA result only +// when the forest produces a complete tree without ERROR or MISSING nodes. +func (p *Parser) maybeReplaceRecoveredTreeWithForest(source []byte, tree *Tree) (*Tree, bool) { + if p == nil || tree == nil || tree.rawParseStoppedEarly() || tree.UsedForestFastPath() || p.recoveryInitialOnly { + return tree, false + } + if p.language == nil || p.language.ExternalScanner != nil || len(p.language.ExternalSymbols) != 0 || len(p.included) != 0 { + return tree, false + } + root := rawRootOrNil(tree) + if root == nil || !root.HasErrorOrMissing() { + return tree, false + } + + candidate, ok := p.ParseForestExperimental(source) + if !ok || candidate == nil { + return tree, false + } + p.normalizeReturnedTreeForParse(candidate, source) + candidateRoot := rawRootOrNil(candidate) + if candidate.rawParseStoppedEarly() || candidateRoot == nil || candidateRoot.StartByte() != 0 || candidateRoot.EndByte() != uint32(len(source)) || candidateRoot.HasErrorOrMissing() { + candidate.Release() + return tree, false + } + tree.Release() + return candidate, true +} + +func (p *Parser) maybeReplaceRecoveredTokenSourceTreeWithForest(source []byte, tree *Tree, ts TokenSource) (*Tree, bool) { + eligible, ok := ts.(forestRecoveryFallbackEligible) + if !ok || !eligible.SupportsForestRecoveryFallback() { + return tree, false + } + return p.maybeReplaceRecoveredTreeWithForest(source, tree) +} + // ForestDeclineInfo returns where/why the forest fast path last declined: the // byte offset and lookahead symbol at the decline, a short reason code, and (for // reason "dead_end") the surviving GLR states. The normal Parse path may then @@ -2204,7 +2240,7 @@ func (p *Parser) forestEOFRecoveryCouldCompete(idx *gssForestIndex, arena *nodeA if p.lookupActionIndex(entry.state, 0) == 0 { continue } - fork, ok := p.cRecoverToState(&stack, entry.depth, entry.state, arena, &entryScratch, &gssScratch, &trackChildErrors) + fork, ok := p.cRecoverToState(&stack, int(entry.depth), entry.state, arena, &entryScratch, &gssScratch, &trackChildErrors) if !ok { continue } diff --git a/glr_gss.go b/glr_gss.go index f0f3ccbb..1873ad08 100644 --- a/glr_gss.go +++ b/glr_gss.go @@ -47,6 +47,9 @@ const ( fullParseGSSNodeSlabCap = 32 * 1024 maxRetainedGSSNodes = 256 * 1024 maxRetainedGSSStackEntries = 4 * 1024 + + gssAggCostValid uint8 = 1 << iota + gssAggVisValid ) type gssNode struct { @@ -62,7 +65,7 @@ type gssNode struct { // field equals gssPrefixAggGen (parser_recover_c.go); allocNode resets the // generation to 0 (never valid — the counter starts at 1). aggCost is filled // by both the parser-side and merge-side walks (identical math); aggVis only - // by the parser side, hence aggVisValid. + // by the parser side. aggValid records which values are current. aggGen uint64 // extraLinks points at the backing array for links 1..k after a gated @@ -75,7 +78,11 @@ type gssNode struct { depth uint32 extraLinkCount uint8 extraLinkCap uint8 - aggVisValid bool + aggValid uint8 + // cleanZeroState fills the final layout byte. Allocation and recycling + // reset it. aggGen validates it against recovery-relevant node mutations. + // These fields do not increase the 64-bit or 32-bit node size. + cleanZeroState uint8 } type gssMainLink struct { @@ -117,12 +124,16 @@ func (n *gssNode) setExtraLink(i int, link gssMainLink) { panic("gssNode.setExtraLink: index out of range") } unsafe.Slice(n.extraLinks, int(n.extraLinkCount))[i] = link + // An extra-link rewrite can change the all-links cleanliness result. + // Clear the local result before another merge check reads it. + n.cleanZeroState = gssCleanZeroUnknown } func (n *gssNode) appendExtraLink(link gssMainLink) { if n == nil || int(n.extraLinkCount) >= maxMainLinkCount-1 { panic("gssNode.appendExtraLink: link limit exceeded") } + n.cleanZeroState = gssCleanZeroUnknown count := int(n.extraLinkCount) capacity := int(n.extraLinkCap) if count < capacity && n.extraLinks != nil { @@ -599,7 +610,8 @@ func (s *gssScratch) allocNode(entry stackEntry, prev *gssNode, depth uint32) *g n.extraLinkCount = 0 n.extraLinkCap = 0 n.aggGen = 0 - n.aggVisValid = false + n.aggValid = 0 + n.cleanZeroState = 0 return n } } @@ -654,7 +666,8 @@ func (s *gssScratch) allocNodeSlow(entry stackEntry, prev *gssNode, depth uint32 n.extraLinkCount = 0 n.extraLinkCap = 0 n.aggGen = 0 - n.aggVisValid = false + n.aggValid = 0 + n.cleanZeroState = 0 if s.audit != nil { s.audit.recordGSSAlloc(n) } diff --git a/glr_gss_test.go b/glr_gss_test.go index dd53e02c..07f8c2e3 100644 --- a/glr_gss_test.go +++ b/glr_gss_test.go @@ -1045,7 +1045,8 @@ func TestGSSScratchRecycleForParseReusesClearedSlots(t *testing.T) { first := scratch.allocNode(newStackEntryNode(2, payload), nil, 1) first.appendExtraLink(gssMainLink{entry: stackEntry{state: 9}}) first.aggGen = 12 - first.aggVisValid = true + first.aggValid = gssAggCostValid | gssAggVisValid + first.cleanZeroState = gssCleanZeroDirty scratch.recycleForParse() @@ -1053,7 +1054,7 @@ func TestGSSScratchRecycleForParseReusesClearedSlots(t *testing.T) { t.Fatalf("used nodes after recycle = %d, want 0", scratch.usedTotal) } if first.prev != nil || first.entry.node != nil || first.extraLinks != nil || first.extraLinkCount != 0 || - first.extraLinkCap != 0 || first.aggGen != 0 || first.aggVisValid { + first.extraLinkCap != 0 || first.aggGen != 0 || first.aggValid != 0 || first.cleanZeroState != gssCleanZeroUnknown { t.Fatalf("recycled slot retained state: %+v", *first) } second := scratch.allocNode(stackEntry{state: 4}, nil, 1) @@ -1081,7 +1082,10 @@ func TestParserRecycleDemotedGSSInvalidatesPointerHolders(t *testing.T) { scratch.merge.result = append(scratch.merge.result, stack, stale) stacks := scratch.merge.result[:1] scratch.merge.cPrefixPath = append(scratch.merge.cPrefixPath, oldHead) - scratch.merge.cleanZeroCache = map[*gssNode]gssCleanZeroErrorCacheEntry{oldHead: {clean: true}} + scratch.merge.cleanZeroCache = map[*gssNode]gssCleanZeroErrorCacheEntry{ + oldHead: {resultEpoch: scratch.merge.cleanZeroEpoch, clean: false}, + } + storeCleanZeroNodeState(oldHead, gssPrefixAggGen.Load(), false) scratch.merge.cleanZeroFrames = append(scratch.merge.cleanZeroFrames, gssCleanZeroFrame{node: oldHead}) scratch.merge.spineVisit = append(scratch.merge.spineVisit, spinePairKey{a: oldHead, b: oldHead.prev}) scratch.merge.mergeSeen = map[gssMergePair]bool{{a: oldHead, b: oldHead.prev}: true} @@ -1117,8 +1121,11 @@ func TestParserRecycleDemotedGSSInvalidatesPointerHolders(t *testing.T) { if scratch.merge.gssPointerEpoch == gssPointerEpochBefore { t.Fatalf("GSS pointer epoch did not advance: %d", scratch.merge.gssPointerEpoch) } - if len(scratch.merge.result) != 0 || len(scratch.merge.cPrefixPath) != 0 || len(scratch.merge.cleanZeroCache) != 0 || len(scratch.merge.mergeSeen) != 0 { - t.Fatalf("merge pointer holders not reset: result=%d prefix=%d clean=%d seen=%d", len(scratch.merge.result), len(scratch.merge.cPrefixPath), len(scratch.merge.cleanZeroCache), len(scratch.merge.mergeSeen)) + if len(scratch.merge.result) != 0 || len(scratch.merge.cPrefixPath) != 0 || len(scratch.merge.mergeSeen) != 0 { + t.Fatalf("merge pointer holders not reset: result=%d prefix=%d seen=%d", len(scratch.merge.result), len(scratch.merge.cPrefixPath), len(scratch.merge.mergeSeen)) + } + if len(scratch.merge.cleanZeroCache) != 0 { + t.Fatalf("clean-zero cache len after invalidation = %d, want 0", len(scratch.merge.cleanZeroCache)) } if scratch.merge.preflight == nil || len(scratch.merge.preflight.virtualLink) != 0 || len(scratch.merge.preflight.reachCache) != 0 { t.Fatal("preflight pointer holders not reset") @@ -1143,6 +1150,12 @@ func TestParserRecycleDemotedGSSInvalidatesPointerHolders(t *testing.T) { if got := stacks[0].gss.materialize(nil); len(got) != 2 || stackEntryNode(got[1]) != payload { t.Fatalf("reforked stack entries = %+v", got) } + if !gssNodeCleanZeroErrorAllLinksWithScratch(&scratch.merge, stacks[0].gss.head) { + t.Fatal("stale clean-zero entry survived recycled-address lookup") + } + if clean, ok := lookupCleanZeroNodeState(stacks[0].gss.head, gssPrefixAggGen.Load()); !ok || !clean { + t.Fatalf("refreshed clean-zero state = %v, %v; want true, true", clean, ok) + } } func TestGSSReuseRetainsOnlyFingerprintedSpineCache(t *testing.T) { diff --git a/glr_test.go b/glr_test.go index 67790314..4297b1d6 100644 --- a/glr_test.go +++ b/glr_test.go @@ -4019,7 +4019,7 @@ func TestGSSCleanZeroAllLinksRejectsErrorBearingPackedPredecessor(t *testing.T) } } -func TestGSSCleanZeroAllLinksCachesFailureForEveryActiveAncestor(t *testing.T) { +func TestGSSCleanZeroAllLinksStoresFailureForEveryActiveAncestor(t *testing.T) { var nodes gssScratch var scratch glrMergeScratch scratch.beginEquivEpoch() @@ -4037,14 +4037,14 @@ func TestGSSCleanZeroAllLinksCachesFailureForEveryActiveAncestor(t *testing.T) { if gssNodeCleanZeroErrorAllLinksWithScratch(&scratch, head) { t.Fatal("clean-zero scan accepted an error-bearing ancestor path") } + gen := gssPrefixAggGen.Load() for _, node := range []*gssNode{head, middle, bad} { - entry := scratch.cleanZeroCache[node] - if entry.resultEpoch != scratch.cleanZeroEpoch || entry.clean { - t.Fatalf("ancestor cache = %#v, want current-epoch failure", entry) + if clean, ok := lookupCleanZeroNodeState(node, gen); !ok || clean { + t.Fatalf("ancestor state = %v, %v; want false, true", clean, ok) } } - if entry := scratch.cleanZeroCache[cleanSibling]; entry.resultEpoch != scratch.cleanZeroEpoch || !entry.clean { - t.Fatalf("clean sibling cache = %#v, want current-epoch success", entry) + if clean, ok := lookupCleanZeroNodeState(cleanSibling, gen); !ok || !clean { + t.Fatalf("clean sibling state = %v, %v; want true, true", clean, ok) } if gssNodeCleanZeroErrorAllLinksWithScratch(&scratch, middle) { t.Fatal("cached ancestor failure returned clean") @@ -4054,6 +4054,23 @@ func TestGSSCleanZeroAllLinksCachesFailureForEveryActiveAncestor(t *testing.T) { } } +func TestGSSCleanZeroAllLinksTerminatesOnCycle(t *testing.T) { + a := &gssNode{entry: stackEntry{state: 1}} + b := &gssNode{entry: stackEntry{state: 2}, prev: a} + a.prev = b + var scratch glrMergeScratch + + if !gssNodeCleanZeroErrorAllLinksWithScratch(&scratch, a) { + t.Fatal("clean-zero scan rejected a clean cycle") + } + gen := gssPrefixAggGen.Load() + for _, node := range []*gssNode{a, b} { + if clean, ok := lookupCleanZeroNodeState(node, gen); !ok || !clean { + t.Fatalf("cycle node state = %v, %v; want true, true", clean, ok) + } + } +} + func TestGSSCleanZeroAllLinksFreshParseProofFallsBackAfterError(t *testing.T) { var gssScratch gssScratch childErrors := false @@ -4075,6 +4092,46 @@ func TestGSSCleanZeroAllLinksFreshParseProofFallsBackAfterError(t *testing.T) { } } +func TestGSSCleanZeroNodeStateSurvivesScratchEpochForStableNode(t *testing.T) { + var nodes gssScratch + payload := NewLeafNode(11, true, 0, 1, Point{}, Point{Column: 1}) + head := nodes.allocNode(newStackEntryNode(2, payload), nil, 1) + var scratch glrMergeScratch + scratch.beginEquivEpoch() + + if !gssNodeCleanZeroErrorAllLinksWithScratch(&scratch, head) { + t.Fatal("clean-zero scan rejected the initial clean node") + } + if clean, ok := lookupCleanZeroNodeState(head, gssPrefixAggGen.Load()); !ok || !clean { + t.Fatalf("clean-zero node state = %v, %v; want true, true", clean, ok) + } + + scratch.cleanZeroCache = nil + clear(scratch.cleanZeroFront) + scratch.beginCleanZeroEpoch() + if !gssNodeCleanZeroErrorAllLinksWithScratch(&scratch, head) { + t.Fatal("stable clean-zero node state did not survive a scratch epoch") + } +} + +func TestGSSCleanZeroNodeStateRejectsPublishedPayloadMutation(t *testing.T) { + var nodes gssScratch + payload := NewLeafNode(11, true, 0, 1, Point{}, Point{Column: 1}) + head := nodes.allocNode(newStackEntryNode(2, payload), nil, 1) + var scratch glrMergeScratch + scratch.beginEquivEpoch() + + if !gssNodeCleanZeroErrorAllLinksWithScratch(&scratch, head) { + t.Fatal("clean-zero scan rejected the initial clean node") + } + payload.setHasError(true) + nodeBumpEquivVersion(payload) + scratch.beginCleanZeroEpoch() + if gssNodeCleanZeroErrorAllLinksWithScratch(&scratch, head) { + t.Fatal("clean-zero state survived a recovery-relevant payload mutation") + } +} + func BenchmarkGSSCleanZeroAllLinksNegativeAncestorCache(b *testing.B) { var nodes gssScratch errorEntry := newStackEntryNode(10, NewLeafNode(errorSymbol, true, 0, 1, Point{}, Point{Column: 1})) @@ -4140,6 +4197,9 @@ func TestGSSNodesCanMergeAllowsCleanPackedPredecessorLinks(t *testing.T) { errorEntry := newStackEntryNode(2, NewLeafNode(99, true, 0, 1, Point{}, Point{Column: 1})) stackEntryNode(errorEntry).setHasError(true) withExtra.setExtraLink(0, gssMainLink{prev: extraPrev, entry: errorEntry}) + if _, ok := lookupCleanZeroNodeState(withExtra, gssPrefixAggGen.Load()); ok { + t.Fatal("extra-link rewrite retained the clean-zero node result") + } if gssNodesCanMerge(withExtra, candidate) { t.Fatal("gssNodesCanMerge = true for error-bearing packed predecessor") } diff --git a/grammars/c_issue667_regression_test.go b/grammars/c_issue667_regression_test.go new file mode 100644 index 00000000..911a051b --- /dev/null +++ b/grammars/c_issue667_regression_test.go @@ -0,0 +1,300 @@ +package grammars_test + +import ( + "reflect" + "testing" + + gotreesitter "github.com/odvcencio/gotreesitter" + "github.com/odvcencio/gotreesitter/grammars" +) + +func TestIssue667CEnumListsRemainHealthy(t *testing.T) { + lang := grammars.CLanguage() + for _, test := range issue667CEnumCases() { + test := test + t.Run(test.name, func(t *testing.T) { + for run := 1; run <= 2; run++ { + tree, err := gotreesitter.NewParser(lang).Parse([]byte(test.source)) + if err != nil { + t.Fatalf("Parse run %d: %v", run, err) + } + defer tree.Release() + issue667RequireHealthyTree(t, lang, tree, []byte(test.source)) + if got := issue667Enumerators(tree.RootNode(), lang, []byte(test.source)); !reflect.DeepEqual(got, test.enumerators) { + t.Fatalf("enumerators = %q, want %q; tree=%s", got, test.enumerators, tree.RootNode().SExpr(lang)) + } + } + }) + } +} + +func TestIssue667CEnumTokenSourceRouteRemainsHealthy(t *testing.T) { + lang := grammars.CLanguage() + for _, test := range issue667CEnumCases() { + test := test + t.Run(test.name, func(t *testing.T) { + source := []byte(test.source) + tree, err := gotreesitter.NewParser(lang).ParseWithTokenSource(source, grammars.NewCTokenSourceOrEOF(source, lang)) + if err != nil { + t.Fatalf("ParseWithTokenSource: %v", err) + } + defer tree.Release() + issue667RequireHealthyTree(t, lang, tree, source) + if got := issue667Enumerators(tree.RootNode(), lang, source); !reflect.DeepEqual(got, test.enumerators) { + t.Fatalf("enumerators = %q, want %q; tree=%s", got, test.enumerators, tree.RootNode().SExpr(lang)) + } + }) + } +} + +func TestIssue667CEnumForestCandidateRemainsHealthy(t *testing.T) { + lang := grammars.CLanguage() + for _, test := range issue667CEnumCases() { + test := test + t.Run(test.name, func(t *testing.T) { + tree, ok := gotreesitter.NewParser(lang).ParseForestExperimental([]byte(test.source)) + if !ok || tree == nil { + t.Fatal("forest candidate declined") + } + defer tree.Release() + issue667RequireHealthyTree(t, lang, tree, []byte(test.source)) + if got := issue667Enumerators(tree.RootNode(), lang, []byte(test.source)); !reflect.DeepEqual(got, test.enumerators) { + t.Fatalf("enumerators = %q, want %q; tree=%s", got, test.enumerators, tree.RootNode().SExpr(lang)) + } + }) + } +} + +func TestIssue667CEnumIncrementalMatchesFresh(t *testing.T) { + lang := grammars.CLanguage() + cases := []struct { + name string + before string + after string + enumerators []string + }{ + { + name: "two to three", + before: "enum E { A, B };\n", + after: "enum E { A, B, C };\n", + enumerators: []string{"A", "B", "C"}, + }, + { + name: "three to four", + before: "enum E { A, B, C };\n", + after: "enum E { A, B, C, D };\n", + enumerators: []string{"A", "B", "C", "D"}, + }, + { + name: "add trailing comma", + before: "enum E { A, B, C };\n", + after: "enum E { A, B, C, };\n", + enumerators: []string{"A", "B", "C"}, + }, + { + name: "remove trailing comma", + before: "enum E { A, B, C, };\n", + after: "enum E { A, B, C };\n", + enumerators: []string{"A", "B", "C"}, + }, + { + name: "add explicit values", + before: "enum E { A = 1, B = 2 };\n", + after: "enum E { A = 1, B = 2, C = 3 };\n", + enumerators: []string{"A", "B", "C"}, + }, + } + + for _, test := range cases { + test := test + t.Run(test.name, func(t *testing.T) { + oldTree, err := gotreesitter.NewParser(lang).Parse([]byte(test.before)) + if err != nil { + t.Fatalf("base Parse: %v", err) + } + defer oldTree.Release() + issue667ApplyEdit(t, oldTree, []byte(test.before), []byte(test.after)) + + incremental, _, err := gotreesitter.NewParser(lang).ParseIncrementalProfiled([]byte(test.after), oldTree) + if err != nil { + t.Fatalf("ParseIncrementalProfiled: %v", err) + } + defer incremental.Release() + issue667RequireHealthyTree(t, lang, incremental, []byte(test.after)) + if got := issue667Enumerators(incremental.RootNode(), lang, []byte(test.after)); !reflect.DeepEqual(got, test.enumerators) { + t.Fatalf("incremental enumerators = %q, want %q", got, test.enumerators) + } + fresh, err := gotreesitter.NewParser(lang).Parse([]byte(test.after)) + if err != nil { + t.Fatalf("fresh Parse: %v", err) + } + defer fresh.Release() + issue667RequireHealthyTree(t, lang, fresh, []byte(test.after)) + if got, want := incremental.RootNode().SExpr(lang), fresh.RootNode().SExpr(lang); got != want { + t.Fatalf("incremental tree = %s, want fresh tree = %s", got, want) + } + }) + } +} + +func TestIssue667CEnumTokenSourceIncrementalMatchesFresh(t *testing.T) { + lang := grammars.CLanguage() + before := []byte("enum E { A, B };\n") + after := []byte("enum E { A, B, C };\n") + oldTree, err := gotreesitter.NewParser(lang).ParseWithTokenSource(before, grammars.NewCTokenSourceOrEOF(before, lang)) + if err != nil { + t.Fatalf("base ParseWithTokenSource: %v", err) + } + defer oldTree.Release() + issue667ApplyEdit(t, oldTree, before, after) + + incremental, profile, err := gotreesitter.NewParser(lang).ParseIncrementalWithTokenSourceProfiled(after, oldTree, grammars.NewCTokenSourceOrEOF(after, lang)) + if err != nil { + t.Fatalf("ParseIncrementalWithTokenSourceProfiled: %v", err) + } + defer incremental.Release() + issue667RequireHealthyTree(t, lang, incremental, after) + if incremental.UsedForestFastPath() { + if got, want := profile.ReuseUnsupportedReason, "forest_recovery_fallback"; got != want { + t.Fatalf("reuse unsupported reason = %q, want %q", got, want) + } + } + + fresh, err := gotreesitter.NewParser(lang).ParseWithTokenSource(after, grammars.NewCTokenSourceOrEOF(after, lang)) + if err != nil { + t.Fatalf("fresh ParseWithTokenSource: %v", err) + } + defer fresh.Release() + issue667RequireHealthyTree(t, lang, fresh, after) + if got, want := incremental.RootNode().SExpr(lang), fresh.RootNode().SExpr(lang); got != want { + t.Fatalf("incremental tree = %s, want fresh tree = %s", got, want) + } +} + +func TestIssue667ForestFallbackRejectsIncompleteInput(t *testing.T) { + lang := grammars.CLanguage() + source := []byte("enum E { A, B, C\n") + tree, err := gotreesitter.NewParser(lang).Parse(source) + if err != nil { + t.Fatalf("Parse: %v", err) + } + defer tree.Release() + if !tree.RootNode().HasErrorOrMissing() { + t.Fatalf("incomplete input returned a healthy tree: %s", tree.RootNode().SExpr(lang)) + } + if tree.UsedForestFastPath() { + t.Fatal("forest fallback accepted incomplete input") + } +} + +type issue667CEnumCase struct { + name string + source string + enumerators []string +} + +func issue667CEnumCases() []issue667CEnumCase { + return []issue667CEnumCase{ + {name: "one", source: "enum E { A };\n", enumerators: []string{"A"}}, + {name: "two", source: "enum E { A, B };\n", enumerators: []string{"A", "B"}}, + {name: "three", source: "enum E { A, B, C };\n", enumerators: []string{"A", "B", "C"}}, + {name: "four", source: "enum E { A, B, C, D };\n", enumerators: []string{"A", "B", "C", "D"}}, + {name: "five", source: "enum E { A, B, C, D, E };\n", enumerators: []string{"A", "B", "C", "D", "E"}}, + {name: "trailing comma", source: "enum E { A, B, C, };\n", enumerators: []string{"A", "B", "C"}}, + {name: "explicit values", source: "enum E { A = 1, B = 2, C = 3 };\n", enumerators: []string{"A", "B", "C"}}, + {name: "typedef", source: "typedef enum { RED, GREEN, BLUE } Colour;\n", enumerators: []string{"RED", "GREEN", "BLUE"}}, + {name: "comment before close", source: "enum E { A, B, C /* close */\n};\n", enumerators: []string{"A", "B", "C"}}, + {name: "neighboring declarations", source: "enum First { A, B, C };\nenum Second { D, E, F };\n", enumerators: []string{"A", "B", "C", "D", "E", "F"}}, + } +} + +func issue667RequireHealthyTree(t *testing.T, lang *gotreesitter.Language, tree *gotreesitter.Tree, source []byte) { + t.Helper() + if tree == nil || tree.RootNode() == nil { + t.Fatal("parse returned no root") + } + if tree.ParseStoppedEarly() { + t.Fatalf("parse stopped early: %s", tree.ParseRuntime().Summary()) + } + root := tree.RootNode() + if got, want := root.EndByte(), uint32(len(source)); got != want { + t.Fatalf("root end = %d, want %d; tree=%s", got, want, root.SExpr(lang)) + } + var errors, missing int + issue667Walk(root, func(node *gotreesitter.Node) { + if node.IsError() { + errors++ + } + if node.IsMissing() { + missing++ + } + }) + if errors != 0 || missing != 0 { + t.Fatalf("recovery nodes: errors=%d missing=%d; rootHasError=%t runtime=%s tree=%s", errors, missing, root.HasError(), tree.ParseRuntime().Summary(), root.SExpr(lang)) + } +} + +func issue667Enumerators(root *gotreesitter.Node, lang *gotreesitter.Language, source []byte) []string { + var names []string + issue667Walk(root, func(node *gotreesitter.Node) { + if node.Type(lang) != "enumerator" { + return + } + for i := 0; i < node.ChildCount(); i++ { + child := node.Child(i) + if child.Type(lang) == "identifier" { + names = append(names, string(source[child.StartByte():child.EndByte()])) + return + } + } + }) + return names +} + +func issue667Walk(node *gotreesitter.Node, visit func(*gotreesitter.Node)) { + if node == nil { + return + } + visit(node) + for i := 0; i < node.ChildCount(); i++ { + issue667Walk(node.Child(i), visit) + } +} + +func issue667ApplyEdit(t *testing.T, tree *gotreesitter.Tree, before, after []byte) { + t.Helper() + start := 0 + for start < len(before) && start < len(after) && before[start] == after[start] { + start++ + } + oldEnd := len(before) + newEnd := len(after) + for oldEnd > start && newEnd > start && before[oldEnd-1] == after[newEnd-1] { + oldEnd-- + newEnd-- + } + if start == oldEnd && start == newEnd { + t.Fatal("edit has no changed bytes") + } + tree.Edit(gotreesitter.InputEdit{ + StartByte: uint32(start), + OldEndByte: uint32(oldEnd), + NewEndByte: uint32(newEnd), + StartPoint: issue667PointAt(before, start), + OldEndPoint: issue667PointAt(before, oldEnd), + NewEndPoint: issue667PointAt(after, newEnd), + }) +} + +func issue667PointAt(source []byte, offset int) gotreesitter.Point { + point := gotreesitter.Point{} + for _, b := range source[:offset] { + if b == '\n' { + point.Row++ + point.Column = 0 + continue + } + point.Column++ + } + return point +} diff --git a/grammars/c_lexer.go b/grammars/c_lexer.go index 0aeb193b..ca1bdfab 100644 --- a/grammars/c_lexer.go +++ b/grammars/c_lexer.go @@ -238,6 +238,20 @@ func (ts *CTokenSource) SupportsIncrementalReuse() bool { return true } +// SupportsForestRecoveryFallback permits forest confirmation when the source +// has no preprocessor directive. The parser DFA does not model directives. +func (ts *CTokenSource) SupportsForestRecoveryFallback() bool { + if ts == nil { + return false + } + for _, b := range ts.src { + if b == '#' { + return false + } + } + return true +} + func (ts *CTokenSource) SetParserState(state gotreesitter.StateID) { ts.parserState = state } diff --git a/grammars/c_lexer_test.go b/grammars/c_lexer_test.go index c74ba208..1b04caf9 100644 --- a/grammars/c_lexer_test.go +++ b/grammars/c_lexer_test.go @@ -32,6 +32,30 @@ func TestNewCTokenSourceOrEOFFallsBack(t *testing.T) { } } +func TestCTokenSourceForestRecoveryFallbackEligibility(t *testing.T) { + lang := CLanguage() + for _, test := range []struct { + name string + src []byte + want bool + }{ + {name: "plain source", src: []byte("enum E { A, B, C };\n"), want: true}, + {name: "preprocessor directive", src: []byte("#define FLAG 1\nenum E { A, B, C };\n"), want: false}, + } { + test := test + t.Run(test.name, func(t *testing.T) { + ts, err := NewCTokenSource(test.src, lang) + if err != nil { + t.Fatalf("NewCTokenSource: %v", err) + } + defer ts.Close() + if got := ts.SupportsForestRecoveryFallback(); got != test.want { + t.Fatalf("SupportsForestRecoveryFallback = %t, want %t", got, test.want) + } + }) + } +} + func TestCTokenSourceSkipToByte(t *testing.T) { lang := CLanguage() src := []byte("int main(void) {\n int x = 1;\n return x;\n}\n") diff --git a/parser_api.go b/parser_api.go index 12964352..1214dc90 100644 --- a/parser_api.go +++ b/parser_api.go @@ -177,6 +177,8 @@ func finalizeDeferredReturnedTreeTruncation(tree *Tree, _ []byte) { const forestIncrementalReuseUnsupportedReason = "old tree was built by GSS forest fast path" +const forestRecoveryFallbackReuseReason = "forest_recovery_fallback" + func oldTreeDisablesIncrementalReuse(oldTree *Tree) bool { return oldTree != nil && oldTree.incrementalReuseDisabled } @@ -365,6 +367,22 @@ func profileFreshParseFallback(start time.Time, tree *Tree, reason string) Incre return profile } +func profileForestRecoveryFallback(profile IncrementalParseProfile, tree *Tree, elapsed time.Duration) IncrementalParseProfile { + if tree == nil { + return profile + } + profile.ReparseNanos += elapsed.Nanoseconds() + profile.ReusedSubtrees = 0 + profile.ReusedBytes = 0 + profile.ReuseUnsupported = true + profile.ReuseUnsupportedReason = forestRecoveryFallbackReuseReason + profile.OldTreeReuseRoute = false + profile.StopReason = tree.ParseStopReason() + profile.ExpectedEOFByte = tree.ParseRuntime().ExpectedEOFByte + profile.LastTokenEndByte = tree.ParseRuntime().LastTokenEndByte + return profile +} + func (p *Parser) normalizeReturnedTree(root *Node, source []byte, incrementalRanges []Range) ParseStopReason { if p == nil || p.language == nil || root == nil || p.noResultCompatibilityBenchmarkOnly { return ParseStopNone @@ -725,6 +743,9 @@ func (p *Parser) parseWithTokenSource(source []byte, ts TokenSource, reparseFact } } p.normalizeReturnedTreeForParse(tree, source) + if !p.recoveryInitialOnly { + tree, _ = p.maybeReplaceRecoveredTokenSourceTreeWithForest(source, tree, ts) + } return tree, nil } @@ -766,6 +787,7 @@ func (p *Parser) parseIncrementalWithTokenSourceChanged(source []byte, oldTree * tree = p.retryIncrementalMemoryBudgetAsPlainFullWithTokenSource(source, ts, tree, nil) } p.normalizeReturnedIncrementalTree(tree, oldTree, source) + tree, _ = p.maybeReplaceRecoveredTokenSourceTreeWithForest(source, tree, ts) return tree, nil } @@ -989,6 +1011,12 @@ type incrementalReuseUnsupportedReasoner interface { IncrementalReuseUnsupportedReason() string } +// forestRecoveryFallbackEligible marks a token source whose clean forest +// result can replace its recovered result. +type forestRecoveryFallbackEligible interface { + SupportsForestRecoveryFallback() bool +} + type parserStateTokenSource interface { SetParserState(state StateID) // SetGLRStates provides all active GLR stack states so the token source @@ -1117,6 +1145,7 @@ func (p *Parser) Parse(source []byte) (*Tree, error) { p.normalizeReturnedTreeForParse(tree, source) if !p.recoveryInitialOnly { tree = p.resolveCRecoverySwallowedError(source, tree) + tree, _ = p.maybeReplaceRecoveredTreeWithForest(source, tree) } tree = p.maybeCompactReturnedFullTree(tree, source) } @@ -1521,6 +1550,7 @@ func (p *Parser) parseIncrementalChanged(source []byte, oldTree *Tree) (*Tree, e tree = p.retryIncrementalMemoryBudgetAsPlainFullWithDFA(source, tree, nil) } p.normalizeReturnedIncrementalTree(tree, oldTree, source) + tree, _ = p.maybeReplaceRecoveredTreeWithForest(source, tree) return tree, nil } @@ -1695,7 +1725,13 @@ func (p *Parser) parseIncrementalChangedProfiled(source []byte, oldTree *Tree) ( tree = p.retryIncrementalMemoryBudgetAsPlainFullWithDFA(source, tree, timing) } p.normalizeReturnedIncrementalTree(tree, oldTree, source) - return tree, timing.toProfile(), nil + forestStart := time.Now() + tree, forestRepaired := p.maybeReplaceRecoveredTreeWithForest(source, tree) + profile := timing.toProfile() + if forestRepaired { + profile = profileForestRecoveryFallback(profile, tree, time.Since(forestStart)) + } + return tree, profile, nil } // ParseIncrementalWithTokenSourceProfiled is like ParseIncrementalWithTokenSource @@ -1736,7 +1772,13 @@ func (p *Parser) parseIncrementalWithTokenSourceChangedProfiled(source []byte, o tree = p.retryIncrementalMemoryBudgetAsPlainFullWithTokenSource(source, ts, tree, timing) } p.normalizeReturnedIncrementalTree(tree, oldTree, source) - return tree, timing.toProfile(), nil + forestStart := time.Now() + tree, forestRepaired := p.maybeReplaceRecoveredTokenSourceTreeWithForest(source, tree, ts) + profile := timing.toProfile() + if forestRepaired { + profile = profileForestRecoveryFallback(profile, tree, time.Since(forestStart)) + } + return tree, profile, nil } // ParseWith parses source using option-based configuration. diff --git a/parser_recover_c.go b/parser_recover_c.go index 02269625..e24d3397 100644 --- a/parser_recover_c.go +++ b/parser_recover_c.go @@ -1027,10 +1027,10 @@ func cStackPosPoint(s *glrStack) Point { // pair with the stack position at that depth, recorded when entering the // error state and consulted by ts_parser__recover strategy 1. type cStackSummaryEntry struct { - depth int state StateID posBytes uint32 posRow uint32 + depth uint8 // cRecoverMaxSummaryDepth is 16. } // cRecoverElectionScratch owns the reusable cursors and generation-stamped @@ -1152,10 +1152,10 @@ func (it *cRecoverElectionDepthIter) next(stacks []glrStack) (int, cStackSummary } summary := stacks[mi].cRec.summary cursor := s.cursors[memberOrder] - for cursor < len(summary) && summary[cursor].depth < it.depth { + for cursor < len(summary) && int(summary[cursor].depth) < it.depth { cursor++ } - for cursor < len(summary) && summary[cursor].depth == it.depth { + for cursor < len(summary) && int(summary[cursor].depth) == it.depth { entry := summary[cursor] cursor++ s.cursors[memberOrder] = cursor @@ -1321,7 +1321,11 @@ func (p *Parser) cNodeVisibleSubtreeCount(n *Node) int { return 0 } if p != nil && len(p.cNodeMemoCache) != 0 { - if slot := p.cNodeMemoSlot(n); slot.hasVis && slot.ver == n.equivVersion { + slot := p.cNodeMemoPrimaryHit(n) + if slot == nil { + slot = p.cNodeMemoSlot(n) + } + if slot.hasVis && slot.ver == n.equivVersion { return int(slot.visCount) } } @@ -1336,7 +1340,10 @@ func (p *Parser) cNodeVisibleSubtreeCount(n *Node) int { // Re-fetch the slot: the recursive calls above may have evicted n's // slot (a child's pointer hashing into the same 2-way set), so the // pointer captured before recursing could now be stale. - slot := p.cNodeMemoSlot(n) + slot := p.cNodeMemoPrimaryHit(n) + if slot == nil { + slot = p.cNodeMemoSlot(n) + } if slot.ver != n.equivVersion { *slot = cNodeMemoCacheEntry{ node: uintptr(unsafe.Pointer(n)), @@ -1561,12 +1568,11 @@ const ( ) func cNodeMemoCacheIndex(p uintptr, setCount int) int { - h := uint64(p) - h ^= h >> 33 - h *= 0xff51afd7ed558ccd - h ^= h >> 33 - h *= 0xc4ceb9fe1a85ec53 - h ^= h >> 33 + // Nodes come from aligned slabs. Remove the alignment zeros, then fold + // slab-address bits into the set index without multiplication. + h := uint64(p / unsafe.Alignof(Node{})) + h ^= h >> 17 + h ^= h >> 9 return int(h&uint64(setCount-1)) << 1 } @@ -1697,6 +1703,18 @@ func (p *Parser) beginCNodeMemoEpoch() { p.cNodeMemoEpoch++ } +// cNodeMemoPrimaryHit checks the primary cache way. Callers must provide a +// non-nil parser and node with a provisioned cache. +func (p *Parser) cNodeMemoPrimaryHit(n *Node) *cNodeMemoCacheEntry { + ptr := uintptr(unsafe.Pointer(n)) + idx := cNodeMemoCacheIndex(ptr, len(p.cNodeMemoCache)>>1) + primary := &p.cNodeMemoCache[idx] + if primary.epoch == p.cNodeMemoEpoch && primary.node == ptr { + return primary + } + return nil +} + // cNodeMemoSlot returns the writable 2-way set-associative slot for node n. // A current-epoch miss evicts the primary into the victim half; stale-epoch // occupants are ignored. This mirrors map[*Node]cNodeMemoEntry lookup @@ -1856,10 +1874,11 @@ func (p *Parser) cErrRegionPreAbsorb(n *Node) cErrRegionAbsorbPre { } // Route through the standard memoized walks so the delta base is exactly // the full-walk answer at the pre-absorb version (O(1) once warm). + cost, vis := p.cNodeErrorCostAndVisibleSubtreeCount(n) return cErrRegionAbsorbPre{ node: n, - cost: p.cNodeErrorCost(n), - vis: p.cNodeVisibleSubtreeCount(n), + cost: cost, + vis: vis, spanCost: cErrRegionSpanCost(n), valid: true, } @@ -1883,11 +1902,12 @@ func (p *Parser) cErrRegionPostAbsorb(pre cErrRegionAbsorbPre, added ...*Node) { if c == nil { continue } - vis += p.cNodeVisibleSubtreeCount(c) + childCost, childVis := p.cNodeErrorCostAndVisibleSubtreeCount(c) + vis += childVis if !(c.symbol == errorSymbol && len(c.children) == 0) { // C ERROR leaf children keep subtree error_cost 0 (see // cNodeErrorCostLang); everything else contributes its own cost. - cost += p.cNodeErrorCost(c) + cost += childCost } if !c.isExtra() { if cSymbolVisibleLang(lang, c.symbol) { @@ -1900,7 +1920,11 @@ func (p *Parser) cErrRegionPostAbsorb(pre cErrRegionAbsorbPre, added ...*Node) { if debugRecoveryIncrementalCost { p.debugCheckErrRegionIncremental(n, cost, vis) } - if slot := p.cNodeMemoSlot(n); slot != nil { + slot := p.cNodeMemoPrimaryHit(n) + if slot == nil { + slot = p.cNodeMemoSlot(n) + } + if slot != nil { entry := cNodeMemoCacheEntry{ node: uintptr(unsafe.Pointer(n)), ver: n.equivVersion, @@ -1929,8 +1953,7 @@ func (p *Parser) cErrRegionPrime(n *Node) { if p == nil || len(p.cNodeMemoCache) == 0 || p.language == nil || n == nil { return } - cost := p.cNodeErrorCost(n) - p.cNodeVisibleSubtreeCount(n) + cost, _ := p.cNodeErrorCostAndVisibleSubtreeCount(n) if ms := p.mergeScratch; ms != nil && ms.cErrorCostParser != p { if ms.cErrorCost == nil { ms.cErrorCost = make(map[*Node]glrCErrorCostEntry) @@ -1982,7 +2005,11 @@ func (p *Parser) cNodeErrorCost(n *Node) uint32 { if len(p.cNodeMemoCache) == 0 { return cNodeErrorCostLang(p.language, n) } - if slot := p.cNodeMemoSlot(n); slot.hasCost && slot.ver == n.equivVersion { + slot := p.cNodeMemoPrimaryHit(n) + if slot == nil { + slot = p.cNodeMemoSlot(n) + } + if slot.hasCost && slot.ver == n.equivVersion { return slot.cost } if n.isMissing() && len(n.children) == 0 { @@ -2021,7 +2048,10 @@ func (p *Parser) cNodeErrorCost(n *Node) uint32 { // Re-fetch the slot: the recursive p.cNodeErrorCost(c) calls above may // have evicted n's slot (a child's pointer hashing into the same 2-way // set), so the pointer captured before recursing could now be stale. - slot := p.cNodeMemoSlot(n) + slot = p.cNodeMemoPrimaryHit(n) + if slot == nil { + slot = p.cNodeMemoSlot(n) + } if slot.ver != n.equivVersion { *slot = cNodeMemoCacheEntry{ node: uintptr(unsafe.Pointer(n)), @@ -2034,6 +2064,98 @@ func (p *Parser) cNodeErrorCost(n *Node) uint32 { return cost } +// cNodeErrorCostAndVisibleSubtreeCount computes both C subtree aggregates in +// one walk. The recovery stack needs both values at the same call sites. +func (p *Parser) cNodeErrorCostAndVisibleSubtreeCount(n *Node) (uint32, int) { + if p == nil || n == nil { + return 0, 0 + } + if len(p.cNodeMemoCache) == 0 { + return cNodeErrorCostLang(p.language, n), cNodeVisibleSubtreeCountUncachedLang(p.language, n) + } + + version := n.equivVersion + slot := p.cNodeMemoPrimaryHit(n) + if slot == nil { + slot = p.cNodeMemoSlot(n) + } + if slot.ver == version { + switch { + case slot.hasCost && slot.hasVis: + return slot.cost, int(slot.visCount) + case slot.hasCost: + cost := slot.cost + return cost, p.cNodeVisibleSubtreeCount(n) + case slot.hasVis: + visible := int(slot.visCount) + return p.cNodeErrorCost(n), visible + } + } + + var cost uint32 + visible := 0 + if p.cSymbolVisible(n.symbol) { + visible++ + } + if n.isMissing() && len(n.children) == 0 { + cost = cErrCostPerMissingTree + cErrCostPerRecovery + } else { + for _, child := range n.children { + if child == nil { + continue + } + childCost, childVisible := p.cNodeErrorCostAndVisibleSubtreeCount(child) + visible += childVisible + if child.symbol != errorSymbol || len(child.children) != 0 { + cost += childCost + } + } + if n.symbol == errorSymbol { + lang := p.language + for _, child := range n.children { + if child == nil || child.isExtra() { + continue + } + if cSymbolVisibleLang(lang, child.symbol) { + cost += cErrCostPerSkippedTree + } else if len(child.children) > 0 { + cost += cErrCostPerSkippedTree * uint32(cNodeVisibleChildCountLang(lang, child)) + } + } + bytes := uint32(0) + rows := uint32(0) + if n.endByte > n.startByte { + bytes = n.endByte - n.startByte + } + if n.endPoint.Row > n.startPoint.Row { + rows = n.endPoint.Row - n.startPoint.Row + } + cost += cErrCostPerRecovery + cErrCostPerSkippedChar*bytes + cErrCostPerSkippedLine*rows + } + } + + // Child recursion can evict the original slot. Resolve it again before the + // write and preserve any matching partial entry. + slot = p.cNodeMemoPrimaryHit(n) + if slot == nil { + slot = p.cNodeMemoSlot(n) + } + if slot.ver != version { + *slot = cNodeMemoCacheEntry{ + node: uintptr(unsafe.Pointer(n)), + ver: version, + epoch: p.cNodeMemoEpoch, + } + } + slot.cost = cost + slot.hasCost = true + if uint64(visible) <= uint64(^uint32(0)) { + slot.visCount = uint32(visible) + slot.hasVis = true + } + return cost, visible +} + // --------------------------------------------------------------------------- // GSS prefix aggregates: O(1) ts_stack_error_cost / node_count reads // @@ -2046,7 +2168,7 @@ func (p *Parser) cNodeErrorCost(n *Node) uint32 { // merge / competition paths issue hundreds of such calls per token, which // dominates error-region parses even with warm per-node memos. // -// The on-node aggregates below (gssNode.aggGen/aggCost/aggVis/aggVisValid) +// The on-node aggregates below (gssNode.aggGen/aggCost/aggVis/aggValid) // restore C's shape: per gssNode, the cumulative aggregates of the prev-chain // prefix root..node inclusive. gssNode prev/entry links are write-once at // allocation except setGSSMainLink (link-0 rewrite), and node payload @@ -2060,7 +2182,7 @@ func (p *Parser) cNodeErrorCost(n *Node) uint32 { // --------------------------------------------------------------------------- // gssPrefixAggGen is the global invalidation generation for the GSS prefix -// aggregates stored on gssNode (aggGen/aggCost/aggVis/aggVisValid). Bumped by +// aggregates stored on gssNode (aggGen/aggCost/aggVis/aggValid). Bumped by // recovery-relevant nodeBumpEquivVersion mutations (tree.go) and link-0 // rewrites that change the predecessor or full-Node payload (glr.go). Global // rather than per-parser because nodeBumpEquivVersion has no parser in scope; @@ -2097,7 +2219,7 @@ func (p *Parser) cStackPrefixAgg(head *gssNode) (uint32, int) { path := p.cPrefixPath[:0] gn := head for gn != nil { - if gn.aggGen == gen && gn.aggVisValid { + if gn.aggGen == gen && gn.aggValid&(gssAggCostValid|gssAggVisValid) == (gssAggCostValid|gssAggVisValid) { cost, vis = gn.aggCost, gn.aggVis break } @@ -2107,11 +2229,15 @@ func (p *Parser) cStackPrefixAgg(head *gssNode) (uint32, int) { for i := len(path) - 1; i >= 0; i-- { gn := path[i] if n := stackEntryNode(gn.entry); n != nil { - cost += p.cNodeErrorCost(n) - vis += int32(p.cNodeVisibleSubtreeCount(n)) + nodeCost, nodeVisible := p.cNodeErrorCostAndVisibleSubtreeCount(n) + cost += nodeCost + vis += int32(nodeVisible) + } + if gn.aggGen != gen { + gn.cleanZeroState = gssCleanZeroUnknown } gn.aggGen = gen - gn.aggVisValid = true + gn.aggValid = gssAggCostValid | gssAggVisValid gn.aggCost = cost gn.aggVis = vis } @@ -2129,7 +2255,7 @@ func cStackPrefixCostForMerge(scratch *glrMergeScratch, lang *Language, head *gs path := scratch.cPrefixPath[:0] gn := head for gn != nil { - if gn.aggGen == gen { + if gn.aggGen == gen && gn.aggValid&gssAggCostValid != 0 { cost = gn.aggCost break } @@ -2141,8 +2267,11 @@ func cStackPrefixCostForMerge(scratch *glrMergeScratch, lang *Language, head *gs if n := stackEntryNode(gn.entry); n != nil { cost += cNodeErrorCostLangWithScratch(scratch, lang, n) } + if gn.aggGen != gen { + gn.cleanZeroState = gssCleanZeroUnknown + } gn.aggGen = gen - gn.aggVisValid = false + gn.aggValid = gssAggCostValid gn.aggCost = cost } scratch.cPrefixPath = path @@ -2275,8 +2404,9 @@ func (p *Parser) cStackEntryAgg(s *glrStack) (uint32, int) { var vis int32 for i := range s.entries { if n := stackEntryNode(s.entries[i]); n != nil { - cost += p.cNodeErrorCost(n) - vis += int32(p.cNodeVisibleSubtreeCount(n)) + nodeCost, nodeVisible := p.cNodeErrorCostAndVisibleSubtreeCount(n) + cost += nodeCost + vis += int32(nodeVisible) } } if p != nil && len(p.cNodeMemoCache) != 0 && s.cRec != nil { @@ -2651,14 +2781,14 @@ func (p *Parser) cRecordSummary(entries []stackEntry) []cStackSummaryEntry { depth := 0 record := func(d int, st StateID, posBytes, posRow uint32) { for j := len(summary) - 1; j >= 0; j-- { - if summary[j].depth < d { + if int(summary[j].depth) < d { break } - if summary[j].depth == d && summary[j].state == st { + if int(summary[j].depth) == d && summary[j].state == st { return } } - summary = append(summary, cStackSummaryEntry{depth: d, state: st, posBytes: posBytes, posRow: posRow}) + summary = append(summary, cStackSummaryEntry{depth: uint8(d), state: st, posBytes: posBytes, posRow: posRow}) } // A node-bearing entry owns its position. Node-less discontinuities use the // next payload below them. The cached index advances monotonically, so this @@ -3142,7 +3272,9 @@ func (p *Parser) cHandleError(stacks *[]glrStack, si int, source []byte, tok Tok p.crecoveryHandleErrorSingleStack = len(*stacks) == 1 // 1. Close in-progress productions: reductions reachable on any symbol. - versions, _, reason := p.cDoAllPotentialReductions(source, s.clone(), 0, true, tok, nodeCount, arena, entryScratch, gssScratch, trackChildErrors) + // Promote the error stack to the graph-structured stack before reductions. + // Recovery forks then share the immutable prefix instead of copying each deep linear stack. + versions, _, reason := p.cDoAllPotentialReductions(source, s.cloneWithScratch(gssScratch), 0, true, tok, nodeCount, arena, entryScratch, gssScratch, trackChildErrors) if reason != ParseStopNone { return cRecHalted, false, reason } @@ -3666,7 +3798,7 @@ func (p *Parser) cRecoverStrategy1Election(stacks *[]glrStack, group *cRecGroup, if entry.posBytes == pos { continue } - depth := entry.depth + depthBump + depth := int(entry.depth) + depthBump // Do not recover in ways that create redundant stack versions. wouldMerge := false for i := range *stacks { diff --git a/parser_recover_c_test.go b/parser_recover_c_test.go index 90a46b5e..ec0bf97b 100644 --- a/parser_recover_c_test.go +++ b/parser_recover_c_test.go @@ -540,17 +540,17 @@ func cRecordSummaryPositionTableReference(entries []stackEntry) []cStackSummaryE for i := range entries { duplicate := false for j := len(summary) - 1; j >= 0; j-- { - if summary[j].depth < depth { + if int(summary[j].depth) < depth { break } - if summary[j].depth == depth && summary[j].state == entries[i].state { + if int(summary[j].depth) == depth && summary[j].state == entries[i].state { duplicate = true break } } if !duplicate { summary = append(summary, cStackSummaryEntry{ - depth: depth, + depth: uint8(depth), state: entries[i].state, posBytes: posBytesAt[i], posRow: posRowAt[i], @@ -798,7 +798,7 @@ func TestCRecoverElectionDepthIteratorPreservesMergedSummaryOrder(t *testing.T) if status == cRecoverElectionIterPoll { continue } - got = append(got, tuple{depth: entry.depth, state: entry.state, owner: stacks[mi].cRec.groupOrder}) + got = append(got, tuple{depth: int(entry.depth), state: entry.state, owner: stacks[mi].cRec.groupOrder}) } } want := []tuple{ @@ -1597,6 +1597,12 @@ func TestCNodeMemoCacheEntrySize(t *testing.T) { } } +func TestCStackSummaryEntrySize(t *testing.T) { + if got, want := unsafe.Sizeof(cStackSummaryEntry{}), uintptr(16); got != want { + t.Fatalf("cStackSummaryEntry size = %d, want %d", got, want) + } +} + func TestRecoveryMemoTelemetryPreservesAMD64HotLayouts(t *testing.T) { if unsafe.Sizeof(uintptr(0)) != 8 { t.Skip("amd64 layout ratchet") @@ -1886,17 +1892,17 @@ func frozenCRecoveryStrategy1AttemptTrace(p *Parser, stacks []glrStack, group *c for _, member := range members { rec := stacks[member].cRec for _, entry := range rec.summary { - if entry.depth != depth || entry.state == cErrorState { + if int(entry.depth) != depth || entry.state == cErrorState { continue } - key := seenKey{depth: entry.depth, state: entry.state} + key := seenKey{depth: int(entry.depth), state: entry.state} if _, ok := seen[key]; ok { continue } seen[key] = struct{}{} trace = append(trace, electionTraceEntry{ - EntryDepth: entry.depth, - RecoverDepth: entry.depth + depthBump, + EntryDepth: int(entry.depth), + RecoverDepth: int(entry.depth) + depthBump, State: entry.state, OwnerIndex: member, OwnerOrder: rec.groupOrder, @@ -1955,7 +1961,7 @@ func linearCRecoveryStrategy1AttemptTrace(p *Parser, stacks []glrStack, group *c } rec := stacks[member].cRec trace = append(trace, electionTraceEntry{ - EntryDepth: entry.depth, RecoverDepth: entry.depth + depthBump, + EntryDepth: int(entry.depth), RecoverDepth: int(entry.depth) + depthBump, State: entry.state, OwnerIndex: member, OwnerOrder: rec.groupOrder, PosBytes: entry.posBytes, PosRow: entry.posRow, }) @@ -2402,13 +2408,13 @@ func TestCRecordSummaryDepthsAreMonotonic(t *testing.T) { summary := parser.cRecordSummary(entries) if len(summary) > 0 { summaries++ - if summary[len(summary)-1].depth > maxDepth { - maxDepth = summary[len(summary)-1].depth + if int(summary[len(summary)-1].depth) > maxDepth { + maxDepth = int(summary[len(summary)-1].depth) } } for i := 1; i < len(summary); i++ { comparisons++ - if summary[i].depth < summary[i-1].depth { + if int(summary[i].depth) < int(summary[i-1].depth) { t.Fatalf("fixture %d summary depth fell at %d: %d -> %d", fixture, i, summary[i-1].depth, summary[i].depth) } if summary[i].depth == summary[i-1].depth { diff --git a/parser_recover_prefix_agg_test.go b/parser_recover_prefix_agg_test.go index 2ad527f5..b24dd76e 100644 --- a/parser_recover_prefix_agg_test.go +++ b/parser_recover_prefix_agg_test.go @@ -2,6 +2,34 @@ package gotreesitter import "testing" +func TestCNodeErrorCostAndVisibleSubtreeCountMatchesIndependentWalks(t *testing.T) { + lang := &Language{SymbolMetadata: []SymbolMetadata{{}, {Visible: true}}} + missing := &Node{symbol: 1, equivVersion: 1} + missing.setMissing(true) + plain := &Node{symbol: 1, equivVersion: 1} + root := &Node{ + symbol: errorSymbol, + equivVersion: 1, + endByte: 4, + children: []*Node{plain, missing}, + } + p := &Parser{ + language: lang, + cNodeMemoCache: make([]cNodeMemoCacheEntry, cNodeMemoCacheSize), + } + + wantCost := cNodeErrorCostLang(lang, root) + wantVisible := cNodeVisibleSubtreeCountUncachedLang(lang, root) + gotCost, gotVisible := p.cNodeErrorCostAndVisibleSubtreeCount(root) + if gotCost != wantCost || gotVisible != wantVisible { + t.Fatalf("combined aggregate = (%d, %d), want (%d, %d)", gotCost, gotVisible, wantCost, wantVisible) + } + slot := p.cNodeMemoSlot(root) + if slot.ver != root.equivVersion || !slot.hasCost || !slot.hasVis { + t.Fatalf("combined aggregate did not cache both values: %#v", *slot) + } +} + func TestBeforePublicationVersionBumpDoesNotInvalidateLivePrefixes(t *testing.T) { lang := &Language{SymbolMetadata: []SymbolMetadata{{}, {Visible: true}}} p := &Parser{ @@ -56,8 +84,9 @@ func TestMetadataVersionBumpKeepsRecoveryPrefixAggregate(t *testing.T) { t.Fatalf("initial aggregate = (%d, %d), want (0, 1)", cost, visible) } gen := gssPrefixAggGen.Load() - if head.aggGen != gen || !head.aggVisValid { - t.Fatalf("initial cache generation = %d valid=%v, want %d valid", head.aggGen, head.aggVisValid, gen) + valid := head.aggValid&(gssAggCostValid|gssAggVisValid) == (gssAggCostValid | gssAggVisValid) + if head.aggGen != gen || !valid { + t.Fatalf("initial cache generation = %d valid=%v, want %d valid", head.aggGen, valid, gen) } payload.parseState = 2 @@ -96,7 +125,7 @@ func TestMergeCostFillInvalidatesAndParserRefillsVisibility(t *testing.T) { if cost, visible := p.cStackPrefixAgg(head); cost != 0 || visible != 2 { t.Fatalf("initial aggregate = (%d, %d), want (0, 2)", cost, visible) } - if !base.aggVisValid || !head.aggVisValid { + if base.aggValid&gssAggVisValid == 0 || head.aggValid&gssAggVisValid == 0 { t.Fatal("parser aggregate did not validate visibility") } @@ -105,13 +134,13 @@ func TestMergeCostFillInvalidatesAndParserRefillsVisibility(t *testing.T) { if cost := cStackPrefixCostForMerge(&merge, lang, head); cost != 0 { t.Fatalf("merge-side cost = %d, want 0", cost) } - if base.aggVisValid || head.aggVisValid { + if base.aggValid&gssAggVisValid != 0 || head.aggValid&gssAggVisValid != 0 { t.Fatal("cost-only refill left stale visibility valid") } if cost, visible := p.cStackPrefixAgg(head); cost != 0 || visible != 2 { t.Fatalf("parser refill = (%d, %d), want (0, 2)", cost, visible) } - if !base.aggVisValid || !head.aggVisValid { + if base.aggValid&gssAggVisValid == 0 || head.aggValid&gssAggVisValid == 0 { t.Fatal("parser refill did not restore visibility validity") } } @@ -221,11 +250,11 @@ func TestSetGSSMainLinkInvalidatesOnlyChangedRecoveryContribution(t *testing.T) prev := &gssNode{depth: 1} payload := &Node{symbol: 1, equivVersion: 1} head := &gssNode{ - prev: prev, - entry: newStackEntryNode(1, payload), - depth: 2, - aggGen: gssPrefixAggGen.Load(), - aggVisValid: true, + prev: prev, + entry: newStackEntryNode(1, payload), + depth: 2, + aggGen: gssPrefixAggGen.Load(), + aggValid: gssAggCostValid | gssAggVisValid, } gen := gssPrefixAggGen.Load() diff --git a/parser_reduce.go b/parser_reduce.go index eda5221d..6e72d018 100644 --- a/parser_reduce.go +++ b/parser_reduce.go @@ -6627,7 +6627,61 @@ func appendFlattenedHiddenChildren(dst []*Node, out int, n *Node, symbolMeta []S return appendFlattenedHiddenChildrenWithFields(dst, nil, nil, out, n, symbolMeta, preservedHidden) } +type hiddenFieldSpan struct { + fieldID FieldID + count int + source uint8 +} + +type hiddenFieldRepeatScratch struct { + inline [8]hiddenFieldSpan + spans []hiddenFieldSpan +} + +func (s *hiddenFieldRepeatScratch) reset() { + if s == nil { + return + } + if cap(s.spans) > 1024 { + s.spans = s.inline[:0] + return + } + if s.spans == nil { + s.spans = s.inline[:0] + return + } + s.spans = s.spans[:0] +} + +func (s *hiddenFieldRepeatScratch) begin() int { + if s.spans == nil { + s.spans = s.inline[:0] + } + return len(s.spans) +} + +func (s *hiddenFieldRepeatScratch) record(start int, fieldID FieldID, source uint8) { + for i := start; i < len(s.spans); i++ { + if s.spans[i].fieldID != fieldID { + continue + } + s.spans[i].count++ + s.spans[i].source = source + return + } + s.spans = append(s.spans, hiddenFieldSpan{fieldID: fieldID, count: 1, source: source}) +} + +func (s *hiddenFieldRepeatScratch) end(start int) { + s.spans = s.spans[:start] +} + func appendFlattenedHiddenChildrenWithFields(dst []*Node, fieldDst []FieldID, fieldSrcDst []uint8, out int, n *Node, symbolMeta []SymbolMetadata, preservedHidden []bool) int { + var scratch hiddenFieldRepeatScratch + return appendFlattenedHiddenChildrenWithFieldsScratch(&scratch, dst, fieldDst, fieldSrcDst, out, n, symbolMeta, preservedHidden) +} + +func appendFlattenedHiddenChildrenWithFieldsScratch(scratch *hiddenFieldRepeatScratch, dst []*Node, fieldDst []FieldID, fieldSrcDst []uint8, out int, n *Node, symbolMeta []SymbolMetadata, preservedHidden []bool) int { if n == nil { return out } @@ -6636,18 +6690,14 @@ func appendFlattenedHiddenChildrenWithFields(dst []*Node, fieldDst []FieldID, fi return out + 1 } nodeStart := out + repeatedStart := scratch.begin() paddingStartByte := n.startByte paddingStartPoint := n.startPoint fieldIDs := n.fieldIDs() fieldSources := n.fieldSources() - type hiddenFieldSpan struct { - count int - source uint8 - } - var repeated map[FieldID]hiddenFieldSpan for i, child := range n.children { spanStart := out - out = appendFlattenedHiddenChildrenWithFields(dst, fieldDst, fieldSrcDst, out, child, symbolMeta, preservedHidden) + out = appendFlattenedHiddenChildrenWithFieldsScratch(scratch, dst, fieldDst, fieldSrcDst, out, child, symbolMeta, preservedHidden) paddingStartByte, paddingStartPoint = absorbFlattenedHiddenPaddingNodes(dst, spanStart, out, paddingStartByte, paddingStartPoint, child, nil, symbolMeta) if fieldDst != nil && i < len(fieldIDs) && fieldIDs[i] != 0 { source := fieldSourceAt(fieldSources, i) @@ -6656,13 +6706,7 @@ func appendFlattenedHiddenChildrenWithFields(dst []*Node, fieldDst []FieldID, fi spanStart, out, fieldIDs[i], source, ); deferred { if direct && spanStart < out { - if repeated == nil { - repeated = make(map[FieldID]hiddenFieldSpan) - } - span := repeated[fieldIDs[i]] - span.count++ - span.source = fieldSourceDirect - repeated[fieldIDs[i]] = span + scratch.record(repeatedStart, fieldIDs[i], fieldSourceDirect) } continue } @@ -6671,23 +6715,18 @@ func appendFlattenedHiddenChildrenWithFields(dst []*Node, fieldDst []FieldID, fi } applyFieldToFlattenedSpan(dst, fieldDst, fieldSrcDst, spanStart, out, fieldIDs[i], source, false) if fieldSourceIsDirect(source) && spanStart < out { - if repeated == nil { - repeated = make(map[FieldID]hiddenFieldSpan) - } - span := repeated[fieldIDs[i]] - span.count++ - span.source = source - repeated[fieldIDs[i]] = span + scratch.record(repeatedStart, fieldIDs[i], source) } } } - for fid, span := range repeated { + for _, span := range scratch.spans[repeatedStart:] { if span.count < 2 { continue } - applyFieldToFlattenedSpan(dst, fieldDst, fieldSrcDst, nodeStart, out, fid, span.source, false) + applyFieldToFlattenedSpan(dst, fieldDst, fieldSrcDst, nodeStart, out, span.fieldID, span.source, false) } normalizeMixedSourceFieldSpan(fieldDst, fieldSrcDst, nodeStart, out) + scratch.end(repeatedStart) return out } @@ -8701,7 +8740,10 @@ func materializeHiddenNodeForAlias(arena *nodeArena, lang *Language, n *Node) *N fieldIDs = arena.allocFieldIDSlice(normalizedCount) fieldSources = arena.allocFieldSourceSlice(normalizedCount) } - out := appendFlattenedHiddenChildrenWithFields(children, fieldIDs, fieldSources, 0, n, symbolMeta, nil) + repeatScratch := &arena.hiddenFieldRepeatScratch + repeatScratch.reset() + out := appendFlattenedHiddenChildrenWithFieldsScratch(repeatScratch, children, fieldIDs, fieldSources, 0, n, symbolMeta, nil) + repeatScratch.reset() cloned.children = children[:out] if len(fieldIDs) > 0 { fieldIDs = fieldIDs[:out] diff --git a/parser_result.go b/parser_result.go index 13c3eb1f..051bb091 100644 --- a/parser_result.go +++ b/parser_result.go @@ -147,7 +147,7 @@ func (p *Parser) currentMaterializationTiming() *parseMaterializationTiming { func (p *Parser) resultMaterializationStopReason(arena *nodeArena) ParseStopReason { if p != nil { - if reason := p.parseStopReasonNow(); parseStopReasonIsActive(reason) { + if reason := p.materializationParseStopReason(); parseStopReasonIsActive(reason) { return reason } // compatMemoryBudgetTripped is compat normalization's own sticky diff --git a/parser_scratch.go b/parser_scratch.go index 86a9eb8d..a16dc225 100644 --- a/parser_scratch.go +++ b/parser_scratch.go @@ -26,6 +26,7 @@ type parserScratch struct { relexSnapshotBuffer []byte relexSnapshotInUse bool trackChildErrors bool + materializeStopPollCount uint32 budgetBytes int64 budgetBaselineBytes int64 gssBaselineBytes int64 @@ -187,6 +188,7 @@ func releaseParserScratch(s *parserScratch, skipGSSClear bool) { } s.relexSnapshotInUse = false s.trackChildErrors = false + s.materializeStopPollCount = 0 s.entries.reset() s.gss.skipClear = skipGSSClear s.gss.audit = nil diff --git a/parser_scratch_reset_test.go b/parser_scratch_reset_test.go index 30a9b792..b5b95761 100644 --- a/parser_scratch_reset_test.go +++ b/parser_scratch_reset_test.go @@ -32,50 +32,19 @@ func TestGLRMergeScratchResetInvalidatesEquivCacheByEpoch(t *testing.T) { } } -func TestGLRMergeScratchResetPreservesCleanZeroEpochAndRejectsReusedAddress(t *testing.T) { +func TestGLRMergeScratchResetClearsCleanZeroFrames(t *testing.T) { var nodes gssScratch old := nodes.allocNode(stackEntry{state: 1}, nil, 1) - var pooled parserScratch - scratch := &pooled.merge - scratch.ensureMergeHotCaches() - scratch.beginCleanZeroEpoch() - storeCleanZeroFrontCache(scratch, old, true) - scratch.cleanZeroCache = map[*gssNode]gssCleanZeroErrorCacheEntry{ - old: {resultEpoch: scratch.cleanZeroEpoch, clean: true}, - } + var scratch glrMergeScratch scratch.cleanZeroFrames = append(scratch.cleanZeroFrames, gssCleanZeroFrame{node: old}) - epochBefore := scratch.cleanZeroEpoch scratch.reset() - if scratch.cleanZeroEpoch != epochBefore { - t.Fatalf("clean-zero epoch after reset = %d, want preserved %d", scratch.cleanZeroEpoch, epochBefore) - } - if got, ok := lookupCleanZeroFrontCache(scratch, old); !ok || !got { - t.Fatalf("clean-zero front after reset = %v, %v; want retained true, true", got, ok) - } - if len(scratch.cleanZeroCache) != 0 { - t.Fatalf("clean-zero map retained %d GSS pointers", len(scratch.cleanZeroCache)) - } if len(scratch.cleanZeroFrames) != 0 { t.Fatalf("clean-zero traversal scratch not reset: frames=%d", len(scratch.cleanZeroFrames)) } if cap(scratch.cleanZeroFrames) > 0 && scratch.cleanZeroFrames[:cap(scratch.cleanZeroFrames)][0].node != nil { t.Fatal("clean-zero frame backing retained a GSS pointer") } - - nodes.recycleForParse() - reused := nodes.allocNode(stackEntry{state: 2}, nil, 1) - if reused != old { - t.Fatalf("recycled GSS address = %p, want %p", reused, old) - } - parser := NewParser(buildArithmeticLanguage()) - parser.configureParseScratch(&pooled, nil, nil, nil, arenaClassFull, true) - if scratch.cleanZeroEpoch != epochBefore+1 { - t.Fatalf("clean-zero epoch after acquisition = %d, want %d", scratch.cleanZeroEpoch, epochBefore+1) - } - if _, ok := lookupCleanZeroFrontCache(scratch, reused); ok { - t.Fatal("clean-zero front hit after a GSS address was reused in a new parse epoch") - } } func TestGLRMergeScratchCleanZeroEpochWrapClearsFront(t *testing.T) { @@ -106,6 +75,21 @@ func TestGLRMergeScratchCleanZeroEpochWrapClearsFront(t *testing.T) { } } +func TestGSSCleanZeroNodeStateStoresBothResults(t *testing.T) { + cleanNode := &gssNode{} + dirtyNode := &gssNode{} + gen := gssPrefixAggGen.Load() + storeCleanZeroNodeState(cleanNode, gen, true) + storeCleanZeroNodeState(dirtyNode, gen, false) + + if clean, ok := lookupCleanZeroNodeState(cleanNode, gen); !ok || !clean { + t.Fatalf("clean node state = %v, %v; want true, true", clean, ok) + } + if clean, ok := lookupCleanZeroNodeState(dirtyNode, gen); !ok || clean { + t.Fatalf("dirty node state = %v, %v; want false, true", clean, ok) + } +} + func TestGLREntryScratchResetClearsReservedWrittenRange(t *testing.T) { var scratch glrEntryScratch entries := scratch.allocWithCap(1, 8) diff --git a/parser_stop.go b/parser_stop.go index 6dfabaf1..95e94cc7 100644 --- a/parser_stop.go +++ b/parser_stop.go @@ -36,6 +36,10 @@ func (p *Parser) endParseOperationBudget(state parseOperationBudgetState) { } func (p *Parser) parseStopReasonNow() ParseStopReason { + // Parse loops arm parseBudgetDepth. Retry work can poll cancellation between budgets. + if p == nil || (p.parseBudgetDepth == 0 && p.cancellationFlag == nil) { + return ParseStopNone + } return p.activeParseStopReason() } diff --git a/parser_timeout.go b/parser_timeout.go index 2a512b32..d83bd9b3 100644 --- a/parser_timeout.go +++ b/parser_timeout.go @@ -27,6 +27,11 @@ type parseStopPoller struct { const parseStopPollMask = 1023 +// Materialization checks can run many times inside one parser iteration. +// Poll the deadline every 64 checks. Check sticky stops and cancellation on +// every call. +const materializationDeadlinePollMask = 63 + func (p *parseStopPoller) poll() ParseStopReason { if p == nil { return ParseStopNone @@ -113,7 +118,9 @@ func (p *Parser) needsParseBudget() bool { } func (p *Parser) activeParseStopCheck() parseStopCheck { - if p == nil { + // Long walks treat nil as an unbudgeted check. + // Avoid callback polls when no timeout or cancellation is active. + if p == nil || !p.needsParseBudget() { return nil } if p.activeParseStopCheckFn == nil { @@ -122,6 +129,8 @@ func (p *Parser) activeParseStopCheck() parseStopCheck { return p.activeParseStopCheckFn } +// Keep the common stop path in this function. The parser loop stores this method as a callback. +// A wrapper adds one call to each poll. func (p *Parser) activeParseStopReason() ParseStopReason { if p == nil { return ParseStopNone @@ -141,6 +150,38 @@ func (p *Parser) activeParseStopReason() ParseStopReason { return ParseStopNone } +// Repeat the cheap stop checks here. This keeps the parser callback direct. +// Only materialization throttles wall-clock reads. +func (p *Parser) materializationParseStopReason() ParseStopReason { + if p == nil { + return ParseStopNone + } + if !p.needsParseBudget() { + return ParseStopNone + } + if parseStopReasonIsActive(p.parseStoppedReason) { + return p.parseStoppedReason + } + if flag := p.cancellationFlag; flag != nil && atomic.LoadUint32(flag) != 0 { + return p.markActiveParseStopped(ParseStopCancelled) + } + if p.parseDeadline.IsZero() { + return ParseStopNone + } + scratch := p.budgetScratch + if scratch != nil { + count := scratch.materializeStopPollCount + scratch.materializeStopPollCount++ + if count&materializationDeadlinePollMask != 0 { + return ParseStopNone + } + } + if !time.Now().Before(p.parseDeadline) { + return p.markActiveParseStopped(ParseStopTimeout) + } + return ParseStopNone +} + func (p *Parser) markActiveParseStopped(reason ParseStopReason) ParseStopReason { if p == nil || !parseStopReasonIsActive(reason) { return ParseStopNone diff --git a/parser_timeout_cache_test.go b/parser_timeout_cache_test.go index d2bf2352..ca598a1e 100644 --- a/parser_timeout_cache_test.go +++ b/parser_timeout_cache_test.go @@ -1,6 +1,10 @@ package gotreesitter -import "testing" +import ( + "sync/atomic" + "testing" + "time" +) func TestActiveParseStopCheckReusesBoundMethod(t *testing.T) { parser := &Parser{parseBudgetDepth: 1} @@ -24,9 +28,67 @@ func TestActiveParseStopCheckReusesBoundMethod(t *testing.T) { } } +func TestActiveParseStopCheckSkipsUnbudgetedParser(t *testing.T) { + parser := &Parser{} + if check := parser.activeParseStopCheck(); check != nil { + t.Fatal("unbudgeted active parse stop check is not nil") + } + + var cancelled uint32 + parser.cancellationFlag = &cancelled + if check := parser.activeParseStopCheck(); check == nil { + t.Fatal("cancellable active parse stop check is nil") + } +} + +func TestParseStopReasonNowChecksCancellationBetweenBudgets(t *testing.T) { + var cancelled uint32 = 1 + parser := &Parser{cancellationFlag: &cancelled} + if got := parser.parseStopReasonNow(); got != ParseStopCancelled { + t.Fatalf("parse stop reason = %q, want %q", got, ParseStopCancelled) + } +} + func TestNewParserBindsActiveParseStopCheck(t *testing.T) { parser := NewParser(nil) if parser.activeParseStopCheckFn == nil { t.Fatal("new parser did not bind its active parse stop check") } } + +func TestMaterializationStopPollsDeadlineAtBoundedCadence(t *testing.T) { + parser := &Parser{ + parseBudgetDepth: 1, + parseDeadline: time.Now().Add(time.Hour), + budgetScratch: &parserScratch{}, + } + if got := parser.materializationParseStopReason(); got != ParseStopNone { + t.Fatalf("initial materialization stop = %q, want %q", got, ParseStopNone) + } + parser.parseDeadline = time.Now().Add(-time.Hour) + for i := 0; i < materializationDeadlinePollMask; i++ { + if got := parser.materializationParseStopReason(); got != ParseStopNone { + t.Fatalf("materialization stop before deadline poll %d = %q, want %q", i, got, ParseStopNone) + } + } + if got := parser.materializationParseStopReason(); got != ParseStopTimeout { + t.Fatalf("bounded materialization deadline poll = %q, want %q", got, ParseStopTimeout) + } +} + +func TestMaterializationStopChecksCancellationEveryCall(t *testing.T) { + var cancelled uint32 + parser := &Parser{ + parseBudgetDepth: 1, + parseDeadline: time.Now().Add(time.Hour), + cancellationFlag: &cancelled, + budgetScratch: &parserScratch{}, + } + if got := parser.materializationParseStopReason(); got != ParseStopNone { + t.Fatalf("initial materialization stop = %q, want %q", got, ParseStopNone) + } + atomic.StoreUint32(&cancelled, 1) + if got := parser.materializationParseStopReason(); got != ParseStopCancelled { + t.Fatalf("materialization cancellation stop = %q, want %q", got, ParseStopCancelled) + } +} diff --git a/scripts/README.md b/scripts/README.md index 4b3b2858..251ab855 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -34,6 +34,15 @@ The default set includes: - the fresh full and selected-store canonical fixtures; - the tags, legacy fact, and compiled `FactProgram` extraction lanes. +Set `GTS_RECOVERY_CORPUS_FILE` and `GTS_RECOVERY_CORPUS_LANG` to add one exact +corpus file. The script skips `BenchmarkRecoveryCorpusFile` when either value +is absent. Use the same file and language for both comparison runs. + +```sh +export GTS_RECOVERY_CORPUS_FILE=/absolute/path/to/corpus-file +export GTS_RECOVERY_CORPUS_LANG=elixir +``` + Run both checkouts with the same seed range. Alternate checkout order between seed batches when the host cannot stay thermally stable. diff --git a/scripts/run_randomized_benchmarks.sh b/scripts/run_randomized_benchmarks.sh index 9e15971a..84663ce9 100755 --- a/scripts/run_randomized_benchmarks.sh +++ b/scripts/run_randomized_benchmarks.sh @@ -103,7 +103,7 @@ output_dir=$(dirname -- "$output_path") mkdir -p -- "$output_dir" # Keep this set aligned across baseline and candidate runs. -benchmark_re='^(BenchmarkGoParse(FullDFA|CoreDFA|IncrementalSingleByteEditDFA|IncrementalNoEditDFA|IncrementalRandomSingleByteEdit)|BenchmarkKDLRecoveryGarbageSuffix|BenchmarkExpectedRootCanFrameLongRepeat|BenchmarkDiagnosticParserCore(CorridorSchedulerOnly|WarmSchedulerOnlyQueryCompile|WarmMaterializationOnlyQueryCompile)|BenchmarkParserCoreFreshFull(Canonical|SelectedStoreCanonical)|Benchmark(TaggerTag(Tree)?Go|ExtractCodeUnderstanding(Tree)?Go|ExtractAllFactsTreeGo|FactProgram(All)?(Tree)?Go))$' +benchmark_re='^(BenchmarkGoParse(FullDFA|CoreDFA|IncrementalSingleByteEditDFA|IncrementalNoEditDFA|IncrementalRandomSingleByteEdit)|Benchmark(KDLRecoveryGarbageSuffix|RecoveryCorpusFile)|BenchmarkExpectedRootCanFrameLongRepeat|BenchmarkDiagnosticParserCore(CorridorSchedulerOnly|WarmSchedulerOnlyQueryCompile|WarmMaterializationOnlyQueryCompile)|BenchmarkParserCoreFreshFull(Canonical|SelectedStoreCanonical)|Benchmark(TaggerTag(Tree)?Go|ExtractCodeUnderstanding(Tree)?Go|ExtractAllFactsTreeGo|FactProgram(All)?(Tree)?Go))$' printf 'randomized benchmark output: %s\n' "$output_path" >&2 printf 'seeds: %s..%s\n' "$seed_start" "$((seed_start + runs - 1))" >&2 diff --git a/tree.go b/tree.go index 108ab480..2ffb6be0 100644 --- a/tree.go +++ b/tree.go @@ -1539,6 +1539,36 @@ func (n *Node) IsError() bool { return n.symbol == errorSymbol } // HasError reports whether this node or any descendant contains a parse error. func (n *Node) HasError() bool { return n.hasError() } +// HasErrorOrMissing reports whether this node or a descendant contains an +// ERROR or MISSING node. Use it for strict parse-health checks. +func (n *Node) HasErrorOrMissing() bool { + if n == nil { + return false + } + if n.hasError() || n.IsError() || n.IsMissing() { + return true + } + + var local [64]*Node + stack := local[:0] + stack = append(stack, n) + for len(stack) > 0 { + last := len(stack) - 1 + current := stack[last] + stack = stack[:last] + for _, child := range current.children { + if child == nil { + continue + } + if child.hasError() || child.IsError() || child.IsMissing() { + return true + } + stack = append(stack, child) + } + } + return false +} + // HasChanges reports whether this node was marked dirty by Tree.Edit. func (n *Node) HasChanges() bool { return n.dirty() } @@ -2182,8 +2212,17 @@ func wireParentLinksWithScratchUntil( stack = local[:0] } stack = append(stack, root) + if reason := p.parseStopReasonNow(); parseStopReasonIsTerminal(reason) { + if scratch != nil { + *scratch = stack[:0] + } + if errorSummary != nil && *errorSummary != resultErrorSummaryPresent { + *errorSummary = resultErrorSummaryUnknown + } + return false + } for len(stack) > 0 { - if reason := p.parseStopReasonNow(); parseStopReasonIsTerminal(reason) { + if reason := p.materializationParseStopReason(); parseStopReasonIsTerminal(reason) { if scratch != nil { *scratch = stack[:0] } @@ -2210,13 +2249,11 @@ func wireParentLinksWithScratchUntil( if scratch != nil { *scratch = stack[:0] } - if errorSummary != nil { - if reason := p.parseStopReasonNow(); parseStopReasonIsTerminal(reason) { - if *errorSummary != resultErrorSummaryPresent { - *errorSummary = resultErrorSummaryUnknown - } - return false + if reason := p.parseStopReasonNow(); parseStopReasonIsTerminal(reason) { + if errorSummary != nil && *errorSummary != resultErrorSummaryPresent { + *errorSummary = resultErrorSummaryUnknown } + return false } return true } diff --git a/tree_health_test.go b/tree_health_test.go new file mode 100644 index 00000000..376e92bf --- /dev/null +++ b/tree_health_test.go @@ -0,0 +1,23 @@ +package gotreesitter + +import "testing" + +func TestNodeHasErrorOrMissingFindsDescendantMissingNode(t *testing.T) { + missing := &Node{} + missing.setMissing(true) + root := &Node{children: []*Node{missing}} + + if root.HasError() { + t.Fatal("HasError reports a missing child") + } + if !root.HasErrorOrMissing() { + t.Fatal("HasErrorOrMissing missed a descendant missing node") + } +} + +func TestNodeHasErrorOrMissingLeavesHealthyTreeClean(t *testing.T) { + root := &Node{children: []*Node{{}, {children: []*Node{{}}}}} + if root.HasErrorOrMissing() { + t.Fatal("HasErrorOrMissing reports a healthy tree") + } +} diff --git a/work_count_convergence_test.go b/work_count_convergence_test.go index e0c89b91..46b6fd1f 100644 --- a/work_count_convergence_test.go +++ b/work_count_convergence_test.go @@ -544,7 +544,7 @@ func TestWorkCountConvergenceHeadErrorCostIsReadOnly(t *testing.T) { missing.setMissing(true) headNode := &gssNode{ entry: newStackEntryNode(1, missing), depth: 1, - aggGen: 77, aggCost: 13, aggVis: 9, aggVisValid: true, + aggGen: 77, aggCost: 13, aggVis: 9, aggValid: gssAggCostValid | gssAggVisValid, } stack := glrStack{gss: gssStack{head: headNode}} sentinel := &gssNode{depth: 99} @@ -560,13 +560,13 @@ func TestWorkCountConvergenceHeadErrorCostIsReadOnly(t *testing.T) { prefixBefore := append([]*gssNode(nil), parser.cPrefixPath...) mergePrefixBefore := append([]*gssNode(nil), mergeScratch.cPrefixPath...) memoBefore := append([]cNodeMemoCacheEntry(nil), parser.cNodeMemoCache...) - aggGen, aggCost, aggVis, aggValid := headNode.aggGen, headNode.aggCost, headNode.aggVis, headNode.aggVisValid + aggGen, aggCost, aggVis, aggValid := headNode.aggGen, headNode.aggCost, headNode.aggVis, headNode.aggValid head := workCountConvergenceHead(parser, &stack) if !head.ErrorCostComparable || head.ErrorCost == 0 { t.Fatalf("read-only head cost=%+v", head) } - if headNode.aggGen != aggGen || headNode.aggCost != aggCost || headNode.aggVis != aggVis || headNode.aggVisValid != aggValid { + if headNode.aggGen != aggGen || headNode.aggCost != aggCost || headNode.aggVis != aggVis || headNode.aggValid != aggValid { t.Fatalf("head aggregate cache mutated: %+v", headNode) } if !reflect.DeepEqual(parser.cPrefixPath, prefixBefore) || !reflect.DeepEqual(mergeScratch.cPrefixPath, mergePrefixBefore) || !reflect.DeepEqual(parser.cNodeMemoCache, memoBefore) || parser.cNodeMemoEpoch != 4 {