diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index 79db3cc4d4..3303e8c2e3 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -493,6 +493,10 @@ Container ownership follows the same contract. If references repeatedly resolve an extracted declaration by name and source range, index candidates by name once and scan only that name's ordered range list. Preserve first-candidate behavior for duplicate names and keep the index local to one extraction call. +Symbol container assignment keeps one reusable path buffer for the sorted +per-symbol walk. Enumerate the active stack into that buffer and reverse it to +outer-to-inner order; do not materialize both a stack array and a fresh path list +for every member in a deeply nested generated file. Duplicate detection in hot extraction loops should use a `HashSet` or another constant-time structure keyed by the full emitted record identity. Do not add @@ -508,6 +512,106 @@ consumer needs only one item at a time or validation can stay on the source string. `string.Split` creates an array and substrings for every import, dependency, path segment, or declaration item. Preserve the original empty-item, trimming, quote, and first-separator semantics when replacing it. +`DelimitedSpanEnumerable` is the shared allocation-free walker for single +delimiters; repository metadata, application manifests, VHDL declarations and +package paths, and CUDA parameter headers use it instead of split arrays. +Likewise, exclusion-range checks inside dense match loops must use indexed +helpers rather than capturing LINQ predicates. Erlang, OCaml, and Raku share +`ContainsFunctionalSpan` / `OverlapsFunctionalSpan` for remote, qualified, +quoted-atom, and type-reference suppression. +The same rule applies to hardware languages: Verilog / SystemVerilog / VHDL +shadow scopes and CUDA / GLSL / HLSL / Metal / WGSL binding and resource scopes +use direct indexed loops so every identifier does not allocate a predicate +closure. +Repository metadata character validation uses `SpanCharacterSearch` instead of +predicate-based enumeration, and application-manifest dependency ownership is +tracked as XML depth rather than rescanning an ancestor stack for every +`assemblyIdentity`. +State-machine sentinel checks must also stay on spans. Erlang specification and +callable terminators plus Raku heredoc terminators trim views of the original +line without materializing padded copies. +`SpanCharacterSearch.EndsWithAfterTrim` is the shared suffix primitive for +these sentinels and for CSS selector continuations plus C# / Java body-less +declaration termination. +Functional-language exclusion span lists are lazy. Erlang quoted/remote calls, +OCAML type/qualified calls, and Raku qualified/method calls must not allocate an +empty list on every source line when no corresponding match exists. +Functional-language regex loops enumerate matches on demand and stop as soon as +the bounded reference list is full. Keep this contract across Clojure, Elixir, +Erlang, OCAML, and Raku so one dense line cannot force unused match objects. +JVM-family reference scanners follow the same rule. Java, Kotlin, Scala, and +Gradle/Groovy multi-match loops use `BoundedRegex.EnumerateMatches`; loops that +own the bounded reference list stop immediately at its cap. +Python reference scanners stream decorators, annotations, runtime type checks, +typing factories, dataclass/attrs integrations, and dynamic imports. The +start-offset overload of `BoundedRegex.EnumerateMatches` keeps decorator +argument scans demand-driven without rescanning the decorator prefix. The +no-offset overload preserves the regex instance's default start position, +including reverse source order for `RegexOptions.RightToLeft`. +Dynamic-language scanners for PHP, Ruby, R, and Perl stream multi-match +attributes/types, DSL targets, namespace/member/resource references, and arrow +calls. Any loop that writes directly to a bounded reference list must exit at +the cap instead of walking the rest of a dense line. +Prolog goal scanning keeps its per-line call list lazy. When calls are present, +update directive metadata in that same list before storing it; do not allocate +an empty list for call-free rules or project a populated list into a second one. +Secondary reference scanners also keep Fortran, Visual Basic, F#, Pascal, +Objective-C, Haskell, Elixir, Smalltalk, Lua, Dart, Razor, JSON, JavaScript, +GitHub Actions, and C++ compound-requirement matches demand-driven. Static +pattern enumeration accepts the extraction timeout explicitly so these paths +do not trade streaming for a different timeout contract. Regex loops that own +a reference list use `ReferenceExtractor.EnumerateReferenceMatches` so bounded +lists stop before requesting the next match, and the shared per-line pipeline + checks the same cap between type, infrastructure, SQL, call, member, metadata, + Razor, Python, and R phases. +Symbol and dependency extractors follow the same streaming rule across +scientific/native, Pascal/Ada, SQL, Python, Swift, GraphQL, markup/XAML, shell, +Ruby, Perl, Elixir, CSS, HDL, C++, and manifest parsing. When only a total is +needed, use `BoundedRegex.CountMatches`; it preserves the prior all-or-nothing +timeout result without retaining a `MatchCollection`. A scanner that needs the +first match for classification and the rest for parsing must keep one enumerator +instead of materializing or rescanning the input. +XAML supplemental symbol phases share the structured-data symbol budget. They +retain at most one overflow marker for diagnostic replacement, then return +through `TrimStructuredDataSymbols` immediately; do not accumulate an +unbounded temporary symbol list and trim it only after every XAML phase. +JSON symbol and reference byte-offset mapping shares `Utf8LineStarts`. It counts +newlines first and allocates the final offset array at exact capacity; do not +grow a `List` and retain it beside a copied array for dense JSON files. +Systems-language scanners stream C/C++ construction and template groups, Rust +calls and value/signature types, Swift property wrappers, Go concurrency and +composite/signature types, plus shared scientific/native call groups. Preserve +source-order emission and stop owned bounded lists at capacity. +SQL reference scanners stream statement, source, target, generated-column, +window-clause, procedure-call, and temporary-object matches. Helpers that accept +multiple SQL matches must keep the sequence demand-driven, and loops that emit +references must stop consuming it when the bounded list reaches capacity. +Infrastructure and markup scanners stream CSS, XAML, HTML/GraphQL/Markdown, +HDL, MSBuild, Dockerfile, shell, and PowerShell match groups. Keep state-only +scans demand-driven as well, while applying bounded-list exits only where the +scanner owns the reference list. +Core reference scans stream shared calls, C# attributes/types/patterns/locals, +JSX elements, JVM documentation links, and Solidity references. A match set +that is intentionally consumed in multiple passes may remain materialized; +single-pass emitters must stay demand-driven and stop owned bounded lists. +All line-based symbol and reference extractors share `SourceLineSplitter`. +It counts newline boundaries once, allocates the exact result array, and then +materializes only the line strings that downstream scanners require; do not +restore separator-index arrays through `string.Split`. + +When structural masking turns a source line into whitespace, do not materialize +a trimmed copy merely to discover that the line has no references. Preserve +documentation handling and the build-automation and markup paths that inspect +the original line, but let ordinary C#, Java, and JavaScript / TypeScript code +paths skip masked multiline payloads before creating a reference context. +Classify prepared-line whitespace once per core-loop iteration and reuse that +result for both special-line dispatch and the ordinary empty-line path; masked +payload lines can be thousands of characters long. + +C / C++ header disambiguation operates on bounded lexical samples. Walk those +samples with spans and newline indices; splitting a sample into a line array +temporarily duplicates every sampled line and scales poorly across large +repositories with many ambiguous `.h` files. The C# value-receiver path is the reference example: local receiver scopes are derived from precomputed block spans for the containing function, and duplicate @@ -2805,7 +2909,7 @@ Contract guarantees: - **Indexing and configured extractor patterns.** Indexing diagnostics use `regex_timeout`; configured pattern diagnostics use `pattern_regex_timeout`. Indexing skips the affected file or pattern so the run can finish and reports bounded diagnostics instead of leaking the pathological pattern input. - **Query/find and MCP find.** CLI human/JSON errors and MCP error envelopes use `regex_timeout` with the same timeout duration text. The recovery hint is surface-specific only where CLI flags and MCP tool arguments differ. - **Redaction surfaces fail closed.** `DiagnosticRedactor`, `GlobalToolLog`, and MCP audit argument values replace the affected value with the configured redaction placeholder. Sensitive-name decisions use `SensitiveNameClassifier`, which normalizes separators and case before checking shared credential fragments so diagnostic and audit redaction cannot drift. `DiagnosticSanitizer` omits the whole message with `[message omitted after sanitization timeout]`. `SuggestionStore` records `redaction_timeout` and persists `[REDACTED:redaction_timeout]`. GitHub API response bodies are replaced with `[response body omitted after redaction timeout]`. -- **Bounded extraction helpers.** `BoundedRegex` keeps extraction best-effort by returning empty matches/`false` or the original input depending on the operation, and records captured timeout diagnostics when a capture scope is active. +- **Bounded extraction helpers.** `BoundedRegex` keeps extraction best-effort by returning empty matches/`false` or the original input depending on the operation, and records captured timeout diagnostics when a capture scope is active. `EnumerateMatches` advances with `Match.NextMatch` only when the consumer requests another result, so bounded extractor loops can stop without materializing the rest of a dense match collection. ## Metrics emission @@ -3480,6 +3584,9 @@ container ownership にも同じ契約を適用する。reference が extracted source range で繰り返し解決する場合は、candidate を name ごとに一度だけ索引化し、その name の ordered range list だけを走査する。duplicate name の first-candidate behavior を維持し、index は 1 回の extraction call 内だけに保持する。 +symbol の container assignment は、sort 済みの per-symbol walk で1つの path buffer を再利用する。 +active stack を buffer へ列挙して outer-to-inner 順に反転し、深く nest した生成ファイルの member +ごとに stack array と新しい path list の両方を実体化してはならない。 hot な抽出ループでの重複検出には、出力 record の完全な identity を key にした `HashSet` などの 定数時間構造を使う。大きな生成ファイルで local variable、parameter、call site、type reference、 @@ -3493,6 +3600,93 @@ hot extractor の delimiter-only parsing では、consumer が item を一度に validation を source string 上で完結できる場合、index / span walk を優先する。`string.Split` は import、dependency、path segment、declaration item ごとに array と substring を作る。置換時は 元の empty-item、trim、quote、first-separator semantics を維持する。 +single delimiter には allocation-free な共通 walker `DelimitedSpanEnumerable` を使う。 +repository metadata、application manifest、VHDL declaration / package path、CUDA parameter +header は split array を作らずこの walker で処理する。 +同様に、dense match loop 内の exclusion-range 判定で capturing LINQ predicate を使っては +ならない。Erlang、OCaml、Raku は remote / qualified / quoted-atom / type-reference の +抑制に `ContainsFunctionalSpan` / `OverlapsFunctionalSpan` を共有する。 +hardware language も同じ規則に従う。Verilog / SystemVerilog / VHDL の shadow scope と、 +CUDA / GLSL / HLSL / Metal / WGSL の binding / resource scope は direct indexed loop を +使い、identifier ごとの predicate closure を作らない。 +repository metadata の character validation は predicate-based enumeration ではなく +`SpanCharacterSearch` を使い、application manifest の dependency ownership は +`assemblyIdentity` ごとの ancestor stack 再走査ではなく XML depth で追跡する。 +state-machine の sentinel 判定も span 上で行う。Erlang specification / callable terminator +と Raku heredoc terminator は、padding を含む copy を実体化せず original line の view を trim する。 +`SpanCharacterSearch.EndsWithAfterTrim` はこれらの sentinel に加え、CSS selector continuation +と C# / Java の body-less declaration termination が共有する suffix primitive である。 +functional-language の exclusion span list は lazy にする。Erlang quoted / remote call、 +OCAML type / qualified call、Raku qualified / method call は、対応する match がない source line +ごとに empty list を割り当ててはならない。 +functional-language の regex loop は match を demand-driven に列挙し、bounded reference list +が満杯になった時点で停止する。Clojure、Elixir、Erlang、OCAML、Raku でこの契約を維持し、 +dense な1行に対して未使用の match object を強制的に作らない。 +JVM-family reference scanner も同じ規則に従う。Java、Kotlin、Scala、Gradle / Groovy の +multi-match loop は `BoundedRegex.EnumerateMatches` を使い、bounded reference list を +所有する loop は上限に達した時点で停止する。 +Python reference scanner は decorator、annotation、runtime type check、typing factory、 +dataclass / attrs integration、dynamic import を逐次走査する。`BoundedRegex.EnumerateMatches` +の start-offset overload により、decorator prefix を再走査せず argument scan も +demand-driven のままにする。offset なしの overload は regex instance の既定開始位置を +維持し、`RegexOptions.RightToLeft` では source の逆順に match する。 +PHP、Ruby、R、Perl の dynamic-language scanner は multi-match の attribute / type、 +DSL target、namespace / member / resource reference、arrow call を逐次走査する。 +bounded reference list へ直接書く loop は dense line の残りを走査せず上限で停止する。 +Prolog goal scan の per-line call list は lazy にする。call がある場合は同じ list 上で +directive metadata を更新してから保存し、call-free rule ごとの empty list や populated +list を射影した2つ目の list を割り当ててはならない。 +secondary reference scanner も Fortran、Visual Basic、F#、Pascal、Objective-C、 +Haskell、Elixir、Smalltalk、Lua、Dart、Razor、JSON、JavaScript、GitHub Actions、 +C++ compound requirement の match を demand-driven に保つ。static pattern の列挙は +extraction timeout を明示的に受け取り、逐次化によって timeout 契約を変えない。 +reference list を所有する regex loop は `ReferenceExtractor.EnumerateReferenceMatches` +を使い、bounded list が満杯なら次の match を要求しない。共有の行単位 pipeline も + type、infrastructure、SQL、call、member、metadata、Razor、Python、R の各 phase 間で + 同じ上限を確認する。 +symbol / dependency extractor も scientific / native、Pascal / Ada、SQL、Python、 +Swift、GraphQL、markup / XAML、shell、Ruby、Perl、Elixir、CSS、HDL、C++、 +manifest parsing をまたいで同じ逐次走査規則に従う。総数だけが必要な場合は +`BoundedRegex.CountMatches` を使い、従来の timeout 時 all-or-nothing 結果を保ったまま +`MatchCollection` を保持しない。先頭 match を分類に、残りを構文解析に使う scanner は、 +input を実体化または再走査せず1つの enumerator を維持する。 +XAML supplemental symbol phase は structured-data symbol budget を共有する。diagnostic +置換用の overflow marker を最大1件だけ保持したら、直ちに +`TrimStructuredDataSymbols` を通って戻る。全 XAML phase の完了後まで無制限の一時 +symbol list を蓄積してから trim してはならない。 +JSON symbol / reference の byte-offset mapping は `Utf8LineStarts` を共有する。先に改行数を +数えて最終 offset array を exact capacity で確保し、dense JSON file で `List` を成長させて +copy 後の array と同時に保持してはならない。 +systems-language scanner は C / C++ construction と template group、Rust call と +value / signature type、Swift property wrapper、Go concurrency と composite / signature +type、共有 scientific / native call group を逐次走査する。source-order emission を維持し、 +所有する bounded list は上限で停止する。 +SQL reference scanner は statement、source、target、generated-column、window-clause、 +procedure-call、一時 object の match を逐次走査する。複数の SQL match を受け取る helper +は sequence を demand-driven のまま保ち、reference を出力する loop は bounded list の +上限到達時に消費を停止する。 +infrastructure / markup scanner は CSS、XAML、HTML / GraphQL / Markdown、HDL、 +MSBuild、Dockerfile、shell、PowerShell の match group を逐次走査する。state-only scan +も demand-driven のまま保ち、bounded-list の停止判定は scanner が reference list を +所有する箇所だけに適用する。 +core reference scan は共有 call、C# attribute / type / pattern / local、JSX element、 +JVM documentation link、Solidity reference を逐次走査する。複数 pass で意図的に再利用する +match set は materialize してよいが、single-pass emitter は demand-driven を維持し、 +所有する bounded list の上限で停止する。 +line-based symbol / reference extractor はすべて `SourceLineSplitter` を共有する。 +newline boundary を一度数えて exact result array を確保し、downstream scanner が必要とする +line string だけを実体化する。`string.Split` による separator-index array を戻してはならない。 + +構造マスクによって source line が空白だけになった場合、reference がないことを確認するため +だけに trim 済み copy を実体化してはならない。documentation handling と、original line を +検査する build-automation / markup 経路は維持しつつ、通常の C#、Java、 +JavaScript / TypeScript 経路では reference context を作る前に multiline payload を skip する。 +prepared-line の whitespace 判定は core-loop iteration ごとに一度だけ行い、special-line +dispatch と通常の empty-line 経路で共有する。masked payload line は数千文字になり得る。 + +C / C++ header の曖昧性解決は bounded lexical sample 上で行う。sample は span と newline index +で走査し、line array に split してはならない。split は sample 内の全行を一時的に複製し、 +曖昧な `.h` file が多い巨大 repository でスケールしにくい。 C# の value receiver 経路を参照例とする。local receiver の scope は containing function 用に 事前計算した block span から導出し、重複 receiver record は hash set で追跡する。この領域の @@ -5264,7 +5458,7 @@ Regex timeout の挙動は `RegexTimeoutPolicy` (`src/CodeIndex/Diagnostics/Rege - **indexing と configured extractor pattern。** indexing 診断は `regex_timeout`、configured pattern 診断は `pattern_regex_timeout` を使う。実行を完了できるよう、影響を受けたファイルまたは pattern を skip し、病的な pattern 入力を漏らさず bounded diagnostics を報告する。 - **query/find と MCP find。** CLI の human/JSON エラーと MCP error envelope は、同じ timeout duration 表記で `regex_timeout` を使う。CLI flag と MCP tool argument が異なる箇所だけ、復旧 hint を surface 別にする。 - **redaction surface は fail closed。** `DiagnosticRedactor`、`GlobalToolLog`、MCP audit の argument value は、対象値を設定済み redaction placeholder へ置換する。sensitive name 判定は `SensitiveNameClassifier` を使い、区切り文字と大小文字を正規化して共有 credential fragment を確認するため、diagnostic と audit の redaction がずれない。`DiagnosticSanitizer` は `[message omitted after sanitization timeout]` でメッセージ全体を省略する。`SuggestionStore` は `redaction_timeout` を記録し `[REDACTED:redaction_timeout]` を永続化する。GitHub API response body は `[response body omitted after redaction timeout]` に置換する。 -- **bounded extraction helper。** `BoundedRegex` は extraction を best-effort に保つため、operation に応じて empty matches / `false` / 元入力を返し、capture scope が有効な場合は timeout diagnostics を記録する。 +- **bounded extraction helper。** `BoundedRegex` は extraction を best-effort に保つため、operation に応じて empty matches / `false` / 元入力を返し、capture scope が有効な場合は timeout diagnostics を記録する。`EnumerateMatches` は consumer が次の結果を要求したときだけ `Match.NextMatch` で進むため、bounded extractor loop は dense match collection の残りを実体化せず停止できる。 ## メトリクス出力 diff --git a/TESTING_GUIDE.md b/TESTING_GUIDE.md index 50f8039a10..cabd5c32b4 100644 --- a/TESTING_GUIDE.md +++ b/TESTING_GUIDE.md @@ -318,6 +318,8 @@ Use `docs/test-doc-maintenance-plan.md` before moving oversized suites or adding CI and release workflow contract tests. Keep repeated related workflow/script string contract assertions, including test-result artifact, retry-output, install, Homebrew, changelog, release-payload job splits, container image, SBOM, NuGet publish, secret scope, SDK pin, tool/action pin, and runner/cache policy contracts, in small grouped helpers so the tests emphasize the contract being checked; use the comparison-aware helpers when the contract intentionally requires ordinal matching. Release workflow package-normalization ZIP fixture helpers live in `ReleaseWorkflowTests.PackageHelpers.cs` so workflow assertions stay near the workflow contracts. - `PackageNormalizeDiagnosticsTests.cs` Package normalizer diagnostic redaction coverage. Keep timeout-budget assertions aligned with the shared diagnostic redaction policy so high-load full-suite runs do not treat expected path/secret placeholders as flaky. +- `BoundedRegexTests.cs` + Shared extractor-regex safety coverage. Demand-driven enumeration tests must take only the requested prefix and keep a catastrophic suffix unvisited, while timeout tests continue to verify best-effort empty results and captured diagnostics. - `DocumentationStatusContractTests.cs`, `DocumentationDriftTests.cs` Checked-in documentation contract tests. They use `RepositoryTestPaths` to keep status fields, workflow references, documented `cdidx` command examples, release/changelog workflow snippets, and representative English/Japanese guide sections synchronized. `DocumentationStatusContractTests.cs` includes readiness, maintenance, and MCP status fields so status JSON support contracts stay visible in the user and agent guides. @@ -576,6 +578,31 @@ Use `docs/test-doc-maintenance-plan.md` before moving oversized suites or adding Concurrent read and read-during-write scenarios (WAL mode validation), including the issue #180 bug-catching snapshot-isolation regressions for all three multi-statement reader entry points: (1) `GetStatus` seeds `refs == files * refsPerFile` and asserts every concurrent observation preserves that invariant; (2) `AnalyzeSymbol` seeds one symbol `S` plus matching reference/caller pairs, toggles a second file symmetrically, and asserts `references.Count == callers.Count` across every `inspect`/`analyze_symbol` bundle; (3) `GetRepoMap` seeds a baseline modified timestamp and toggles a newer file, asserting `latest_modified == workspace_latest_modified` across every map call. Each test fails without the DEFERRED-transaction wrap on the matching reader and passes with it. - `PerformanceTests.cs` Bounded CI smoke coverage plus large-scale data benchmarks. `CiPerformanceSmoke_IndexAndSearchSmallFixture_StaysWithinBudget` and the allocation budget guards run in the default `net8.0` suite, so they are blocking PR/CI checks on the production target, but their broad budgets are intended to catch only severe indexing/search or allocation regressions rather than act as benchmarks. `ReferenceExtraction_RepeatedSymbolMembership_StaysWithinAllocationBudget` uses dense C# private-property receivers and Python imported-type calls to prevent per-candidate full-symbol rescans from returning. `ReferenceExtraction_RepeatedContainerLookup_StaysWithinAllocationBudget` covers dense C# declaration containers and GitHub Actions jobs so name/range ownership resolution stays indexed. `Extraction_DenseDelimitedLists_StayWithinAllocationBudget` covers Python imports, YAML needs, JSON paths, and Fortran procedure lists without temporary split-array growth. `ReferenceDedupe_DenseLongIdentities_StayWithinAllocationBudget` keeps all-language dedupe identities value-based when qualified names are long. The 10K+ large-scale tests remain skip-by-default; run them manually with `--filter`. + `ReferenceExtraction_MaskedMultilinePayloads_StayWithinAllocationBudget` keeps C# raw strings, Java text blocks, and TypeScript template literals from materializing trimmed reference contexts after structural masking has made a line empty. + `CppHeaderDetection_LargeSample_DoesNotMaterializeLineArrays` keeps bounded C / C++ header-disambiguation samples on span-based line walks instead of allocating a string and array for every sampled line. + `DelimitedSpanWalking_DenseExtractorLists_DoesNotAllocate` locks the shared single-delimiter walker to allocation-free trim/remove-empty semantics used by repository metadata, application manifests, VHDL, and CUDA extraction. + `FunctionalSpanMembership_RepeatedCallFiltering_DoesNotAllocate` prevents per-match capturing-predicate allocations from returning to Erlang, OCAML, and Raku exclusion-range checks. + `HardwareScopeMembership_RepeatedIdentifierFiltering_DoesNotAllocate` prevents per-identifier predicate closures from returning to shader binding/resource scope checks shared by CUDA, GLSL, HLSL, Metal, and WGSL; the corresponding HDL extraction regression covers shadowing semantics for Verilog, SystemVerilog, and VHDL. + `SpanCharacterSearch_RepeatedLongMetadataCandidates_DoesNotAllocate` keeps control/whitespace validation allocation-free for long repository-metadata candidates; application-manifest regressions separately preserve dependency ancestry and local probing-path behavior. + `SourceLineSplitting_LargeFiles_AvoidsSeparatorIndexArrays` covers the shared all-language symbol/reference line splitter with 8,192 lines and fixes its exact output plus bounded allocation contract. + `FunctionalTerminatorChecks_LongPaddedLines_DoNotAllocate` keeps Erlang specification/callable and Raku heredoc state-machine sentinels on trimmed spans rather than padded string copies. + `TrimmedSuffixChecks_LongDeclarationLines_DoNotAllocate` keeps CSS selector continuations and C# / Java body-less declaration suffix checks allocation-free on long padded lines. + `FunctionalReferenceExtraction_CallFreeLines_AvoidsEmptySpanLists` covers 4,096 call-free lines each of Erlang, OCAML, and Raku so their exclusion span lists remain lazy. + Functional-language graph fixtures also cover the demand-driven Clojure, Elixir, Erlang, OCAML, and Raku regex loops; keep their bounded-list exit checks adjacent to enumeration. + JVM graph fixtures cover demand-driven Java type/module/method-reference, Kotlin type/infix/constructor, Scala contextual, and Gradle/Groovy DSL matches. Preserve dense-line ordering when adding cap-aware exits. + Python graph fixtures cover streamed decorator arguments, annotations, runtime type checks, typing factories, dataclass/framework integrations, and dynamic imports. `BoundedRegexTests.EnumerateMatches_InstanceRegex_StartsAtRequestedOffset` fixes the no-prefix-rescan contract used by decorator arguments, while `EnumerateMatches_InstanceRightToLeftRegex_PreservesDefaultStartAndOrder` preserves instance-regex default direction. + PHP, Ruby, R, and Perl graph fixtures cover streamed attributes/docblocks/types, DSL command targets, namespace/member/resource references, and arrow calls. Keep nested token/type enumeration cap-aware. + Secondary-language graph fixtures cover streamed Fortran, Visual Basic, F#, Pascal, Objective-C, Haskell, Elixir, Smalltalk, Lua, Dart, Razor, JSON, JavaScript, GitHub Actions, and C++ compound-requirement matches. `BoundedRegexTests.EnumerateMatches_StaticPatternCustomTimeout_ReturnsEmpty` and `EnumerateMatches_StaticPatternCustomTimeout_StopsAfterConsumerBreak` preserve explicit-timeout failure and early-disposal behavior. + `PerformanceTests.ReferenceExtraction_BoundedDenseFSharpPipeline_StopsAtCapacity` fixes the bounded-list contract across line-phase handoffs and action-based call emitters: a 4,000-stage F# pipeline capped at one reference must not enumerate the unused stages. `PerformanceTests.ReferenceMatchEnumeration_BoundedListDoesNotRequestMatchAfterCapacity` additionally proves that the shared wrapper does not call the underlying enumerator's next `MoveNext()` after the cap is filled, while `ReferenceMatchEnumeration_BelowCapacity_DoesNotAllocateWrapperEnumerators` keeps 10,000 ordinary below-cap scans free of wrapper-enumerator heap allocations. + `PerformanceTests.ReferenceExtraction_PrologCallFreeRules_AvoidsPerLineLists` keeps 8,000 call-free Prolog rules from allocating empty goal lists or copying populated directive lists; its `net8.0` allocation budget is blocking. + `PerformanceTests.Utf8LineStarts_DenseInput_AllocatesOnlyFinalOffsetArray` fixes 100,000 UTF-8 line offsets to the single exact-capacity result array used by both JSON symbol and reference extraction; its `net8.0` allocation budget is blocking. + Systems-language graph fixtures cover streamed C/C++ friend/construction/template groups, Rust macro/value/signature types, Swift wrappers, Go concurrency/composite/signature types, and shared scientific/native call groups. + SQL graph fixtures cover streamed statement/source/target, generated-column, window-clause, procedure-call, and temporary-object matches. Preserve source-order output and bounded-list early exits across SQL dialect branches. + Infrastructure and markup graph fixtures cover streamed CSS, XAML, HTML/GraphQL/Markdown, HDL, MSBuild, Dockerfile, shell, and PowerShell matches. Keep state-building scans independent from bounded reference-list ownership. + Core graph fixtures cover streamed shared calls, C# attributes/types/patterns/locals, JSX elements, JVM documentation links, and Solidity references. Preserve intentional multi-pass match reuse for C# `where` constraints. + Symbol-extractor fixtures cover streamed scientific/native, Pascal/Ada, SQL, Python, Swift, GraphQL, markup/XAML, shell, Ruby, Perl, Elixir, CSS, HDL, and C++ matches; dependency-package fixtures cover streamed quoted manifest entries. `BoundedRegexTests.CountMatches_InstanceRegex_CountsWithoutMaterializingCollection` and `CountMatches_InstanceRegexTimeout_ReturnsZero` preserve counting and timeout compatibility without a retained `MatchCollection`. + `SymbolExtractorTests.Extract_XmlBroadXaml_StopsSupplementalScanAtSymbolBudget` uses 50,000 XAML elements to keep supplemental symbol phases bounded to the shared cap plus one diagnostic marker, with blocking `net8.0` allocation and practical-time budgets. + `SymbolExtractorTests.Extract_CSharp_ManyNestedMembers_ReusesContainerPathBuffer` keeps 8,000 members under three nested C# containers on one reusable assignment path buffer; its `net8.0` allocation and practical-time budgets are blocking. `ReferenceExtraction_CSharpNoAliasDenseReferences_StaysWithinAllocationBudget` keeps 12,000 no-alias calls from paying for alias dedupe, `CSharpAliasCompaction_DenseDuplicates_StaysWithinAllocationBudget` guards pre-sized stable compaction after alias rewrites, and `MutualRecursion_DenseRepeatedQualifiedNames_StaysWithinAllocationBudget` covers repeated qualified C#- and Python-style cycle names without per-edge normalization strings. `ReusableStatSnapshot_OnePassMaterialization_StaysWithinAllocationBudget` warms a 1,024-row typed snapshot with its exact capacity, then prevents a second candidate collection or redundant path values from returning to the default `net8.0` path. `ReferenceBatchTransactions_RepositoryScaleAtomicFileScopeEliminatesControlledSqlScopes` models 321,352 references distributed over 856 files in the repository snapshot's five/six-batch shape and fixes the control-SQL contract at 5,009 public batch transaction scopes versus zero explicit atomic-file batch scopes; keep it deterministic and allocation-light instead of inserting all modeled rows or asserting elapsed time. For reference-line window performance audits, measure identical prebuilt reference rows against one-batch and 32-batch limits in alternating order, report both elapsed time and `GC.GetAllocatedBytesForCurrentThread`, and remove the timing harness after recording the result; end-to-end `--memory-trace` rebuilds remain corroborating evidence because extraction, graph finalization, and OS page-cache variance can dominate the persistence delta. @@ -1176,6 +1203,8 @@ dotnet test --filter "FullyQualifiedName~GitHelperTests" CI と release workflow の契約テスト。test-result artifact、retry-output、install、Homebrew、changelog、release-payload job split、container image、SBOM、NuGet publish、secret scope、SDK pin、tool/action pin、runner/cache policy の契約も含め、繰り返しの関連する workflow/script string contract assertion は小さな grouped helper に寄せ、テスト本文が確認している契約を読み取りやすくしてください。contract が ordinal matching を明示的に必要とする場合は comparison-aware helper を使います。Release workflow の package-normalization ZIP fixture helper は `ReleaseWorkflowTests.PackageHelpers.cs` に置き、workflow assertion が workflow 契約の近くに残るようにします。 - `PackageNormalizeDiagnosticsTests.cs` package normalizer の diagnostic redaction カバレッジです。高負荷の full-suite 実行で、期待される path / secret placeholder が flaky に見えないよう、timeout budget の assertion は共有 diagnostic redaction policy と同期させてください。 +- `BoundedRegexTests.cs` + 共有 extractor-regex safety のカバレッジです。demand-driven enumeration test は要求された prefix だけを取得して catastrophic suffix を未走査のまま保ち、timeout test は引き続き best-effort の empty result と captured diagnostic を検証してください。 - `DocumentationStatusContractTests.cs`、`DocumentationDriftTests.cs` checked-in documentation の契約テスト。`RepositoryTestPaths` を使って、status field、workflow 参照、文書化された `cdidx` コマンド例、release/changelog workflow の snippet、代表的な英日 guide セクションの同期を維持します。 `DocumentationStatusContractTests.cs` は readiness、maintenance、MCP status field も含め、status JSON support contract が user guide と agent guide に残るようにします。 @@ -1425,6 +1454,31 @@ dotnet test --filter "FullyQualifiedName~GitHelperTests" 並行読み取りと書き込み中読み取りシナリオ（WALモード検証）。issue #180 の bug-catching な snapshot 隔離回帰テストを 3 つの multi-statement reader 経路について含む。(1) `GetStatus` は `refs == files * refsPerFile` の seed 不変条件を立て、並行観測が常にこの条件を維持することを要求する。(2) `AnalyzeSymbol` はシンボル `S` に対して reference/caller を対称に 1 対 1 で seed し、もう 1 ファイルを対称に toggle することで `inspect` / `analyze_symbol` bundle の `references.Count == callers.Count` を常に保証する。(3) `GetRepoMap` はベースラインの modified と新しい toggle 対象ファイルを用意し、`latest_modified == workspace_latest_modified` が常に一致することを要求する。各テストは対応する reader の DEFERRED transaction を外すと落ち、戻すと通ることを確認済み。 - `PerformanceTests.cs` bounded な CI smoke と大規模データベンチマークを扱います。`CiPerformanceSmoke_IndexAndSearchSmallFixture_StaysWithinBudget` と allocation budget guard は通常の `net8.0` suite で実行されるため production target 上の PR / CI blocking check ですが、benchmark ではなく重大な indexing/search または allocation 退行だけを拾う広めの budget を使います。`ReferenceExtraction_RepeatedSymbolMembership_StaysWithinAllocationBudget` は密な C# private-property receiver と Python imported-type call を使い、candidate ごとの full-symbol 再走査が戻るのを防ぎます。`ReferenceExtraction_RepeatedContainerLookup_StaysWithinAllocationBudget` は密な C# declaration container と GitHub Actions job を扱い、name / range ownership 解決の索引化を維持します。`Extraction_DenseDelimitedLists_StayWithinAllocationBudget` は Python import、YAML needs、JSON path、Fortran procedure list を使い、一時 split-array の増加を防ぎます。`ReferenceDedupe_DenseLongIdentities_StayWithinAllocationBudget` は長い qualified name でも全言語共通 dedupe identity を value-based に維持します。10K+ の大規模テストは引き続きデフォルト Skip で、`--filter` で手動実行します。 + `ReferenceExtraction_MaskedMultilinePayloads_StayWithinAllocationBudget` は、構造マスク後に空行となった C# raw string、Java text block、TypeScript template literal から trim 済み reference context を実体化しないことを固定します。 + `CppHeaderDetection_LargeSample_DoesNotMaterializeLineArrays` は、bounded な C / C++ header 判定 sample を span ベースで行走査し、sampled line ごとの string と array を割り当てないことを固定します。 + `DelimitedSpanWalking_DenseExtractorLists_DoesNotAllocate` は、repository metadata、application manifest、VHDL、CUDA extraction が共有する single-delimiter walker の trim / remove-empty semantics を allocation-free に固定します。 + `FunctionalSpanMembership_RepeatedCallFiltering_DoesNotAllocate` は、Erlang、OCAML、Raku の exclusion-range 判定へ match ごとの capturing-predicate allocation が戻らないことを固定します。 + `HardwareScopeMembership_RepeatedIdentifierFiltering_DoesNotAllocate` は、CUDA、GLSL、HLSL、Metal、WGSL が共有する shader binding / resource scope 判定へ identifier ごとの predicate closure が戻らないことを固定し、対応する HDL extraction regression が Verilog、SystemVerilog、VHDL の shadowing semantics をカバーします。 + `SpanCharacterSearch_RepeatedLongMetadataCandidates_DoesNotAllocate` は、長い repository-metadata candidate の control / whitespace validation を allocation-free に固定し、application-manifest regression が dependency ancestry と local probing-path の意味論を別途維持します。 + `SourceLineSplitting_LargeFiles_AvoidsSeparatorIndexArrays` は、全言語の symbol / reference extraction が共有する line splitter を8,192行で検証し、exact output と bounded allocation contract を固定します。 + `FunctionalTerminatorChecks_LongPaddedLines_DoNotAllocate` は、Erlang specification / callable と Raku heredoc の state-machine sentinel を、padding 付き string copy ではなく trimmed span 上に固定します。 + `TrimmedSuffixChecks_LongDeclarationLines_DoNotAllocate` は、長い padding 付き行に対する CSS selector continuation と C# / Java body-less declaration の suffix 判定を allocation-free に固定します。 + `FunctionalReferenceExtraction_CallFreeLines_AvoidsEmptySpanLists` は、Erlang、OCAML、Raku それぞれ4,096行の call-free input を使い、exclusion span list が lazy なままであることを固定します。 + functional-language graph fixture は Clojure、Elixir、Erlang、OCAML、Raku の demand-driven regex loop も検証します。bounded-list の終了判定は enumeration の直近に維持してください。 + JVM graph fixture は Java type / module / method-reference、Kotlin type / infix / constructor、Scala contextual、Gradle / Groovy DSL の demand-driven match を検証します。cap-aware exit を追加するときも dense-line の順序を維持してください。 + Python graph fixture は decorator argument、annotation、runtime type check、typing factory、dataclass / framework integration、dynamic import の逐次走査を検証します。`BoundedRegexTests.EnumerateMatches_InstanceRegex_StartsAtRequestedOffset` は decorator argument が prefix を再走査しない契約を固定し、`EnumerateMatches_InstanceRightToLeftRegex_PreservesDefaultStartAndOrder` は instance regex の既定方向を維持します。 + PHP、Ruby、R、Perl の graph fixture は attribute / docblock / type、DSL command target、namespace / member / resource reference、arrow call の逐次走査を検証します。nested token / type enumeration も cap-aware のままにしてください。 + secondary-language graph fixture は Fortran、Visual Basic、F#、Pascal、Objective-C、Haskell、Elixir、Smalltalk、Lua、Dart、Razor、JSON、JavaScript、GitHub Actions、C++ compound requirement の逐次走査を検証します。`BoundedRegexTests.EnumerateMatches_StaticPatternCustomTimeout_ReturnsEmpty` と `EnumerateMatches_StaticPatternCustomTimeout_StopsAfterConsumerBreak` は、明示 timeout の失敗時挙動と早期破棄を固定します。 + `PerformanceTests.ReferenceExtraction_BoundedDenseFSharpPipeline_StopsAtCapacity` は、行 phase の引き継ぎと action-based call emitter をまたぐ bounded-list 契約を固定します。上限1の4,000段 F# pipeline は未使用の段を列挙してはなりません。`PerformanceTests.ReferenceMatchEnumeration_BoundedListDoesNotRequestMatchAfterCapacity` はさらに、上限到達後に共有 wrapper が下位 enumerator の次の `MoveNext()` を呼ばないことを固定し、`ReferenceMatchEnumeration_BelowCapacity_DoesNotAllocateWrapperEnumerators` は通常の上限未到達 scan 10,000回で wrapper enumerator の heap allocation がないことを固定します。 + `PerformanceTests.ReferenceExtraction_PrologCallFreeRules_AvoidsPerLineLists` は8,000件の call-free Prolog rule で empty goal list と populated directive list の copy を防ぎます。`net8.0` の allocation budget は blocking です。 + `PerformanceTests.Utf8LineStarts_DenseInput_AllocatesOnlyFinalOffsetArray` は JSON symbol / reference extraction が共有する100,000件の UTF-8 line offset を exact-capacity の結果 array 1つに固定します。`net8.0` の allocation budget は blocking です。 + systems-language graph fixture は C / C++ friend / construction / template group、Rust macro / value / signature type、Swift wrapper、Go concurrency / composite / signature type、共有 scientific / native call group の逐次走査を検証します。 + SQL graph fixture は statement / source / target、generated-column、window-clause、procedure-call、一時 object match の逐次走査を検証します。SQL dialect の分岐をまたいでも source-order output と bounded-list の早期停止を維持してください。 + infrastructure / markup graph fixture は CSS、XAML、HTML / GraphQL / Markdown、HDL、MSBuild、Dockerfile、shell、PowerShell match の逐次走査を検証します。state 構築 scan は bounded reference-list の所有権から独立したままにしてください。 + core graph fixture は共有 call、C# attribute / type / pattern / local、JSX element、JVM documentation link、Solidity reference の逐次走査を検証します。C# `where` constraint の意図的な multi-pass match 再利用は維持してください。 + symbol-extractor fixture は scientific / native、Pascal / Ada、SQL、Python、Swift、GraphQL、markup / XAML、shell、Ruby、Perl、Elixir、CSS、HDL、C++ の match を逐次走査し、dependency-package fixture は引用符付き manifest entry の逐次走査を検証します。`BoundedRegexTests.CountMatches_InstanceRegex_CountsWithoutMaterializingCollection` と `CountMatches_InstanceRegexTimeout_ReturnsZero` は、`MatchCollection` を保持しない count と timeout 互換性を固定します。 + `SymbolExtractorTests.Extract_XmlBroadXaml_StopsSupplementalScanAtSymbolBudget` は50,000件の XAML element を使い、supplemental symbol phase を共有上限と diagnostic marker 1件までに制限します。`net8.0` の allocation と実用時間 budget は blocking です。 + `SymbolExtractorTests.Extract_CSharp_ManyNestedMembers_ReusesContainerPathBuffer` は3階層の C# container 内の8,000 member を1つの再利用 assignment path buffer で処理します。`net8.0` の allocation と実用時間 budget は blocking です。 `ReferenceExtraction_CSharpNoAliasDenseReferences_StaysWithinAllocationBudget` は alias のない 12,000 call が alias dedupe のコストを負わないこと、`CSharpAliasCompaction_DenseDuplicates_StaysWithinAllocationBudget` は alias rewrite 後の事前 capacity 付き stable compaction、`MutualRecursion_DenseRepeatedQualifiedNames_StaysWithinAllocationBudget` は C# / Python 形式の qualified cycle name が edge ごとの正規化文字列を作らないことを保証します。 `ReusableStatSnapshot_OnePassMaterialization_StaysWithinAllocationBudget` は exact capacity を渡した1,024行の typed snapshot を warm-up し、2つ目の候補 collection や重複 path value が通常の `net8.0` 経路へ戻らないようにします。 `ReferenceBatchTransactions_RepositoryScaleAtomicFileScopeEliminatesControlledSqlScopes` は321,352 refsを自己snapshotの5/6 batch形状で856 filesへ分配し、制御SQL契約をpublic batch transaction scope 5,009回対explicit atomic-file scope 0回に固定します。全model rowを挿入したり経過時間をassertしたりせず、deterministicでallocation-lightなまま維持してください。reference-line windowの性能監査では、同一の事前構築済みreference rowを1-batch上限と32-batch上限で交互に測り、経過時間と`GC.GetAllocatedBytesForCurrentThread`の両方を報告して、結果記録後にtiming harnessを削除します。end-to-endの`--memory-trace` rebuildは、extraction・graph finalize・OS page cacheの変動が永続化差を支配し得るため、補助証拠として扱ってください。 diff --git a/changelog.d/unreleased/+cpp-header-line-walk.fixed.md b/changelog.d/unreleased/+cpp-header-line-walk.fixed.md new file mode 100644 index 0000000000..3faaa0d12b --- /dev/null +++ b/changelog.d/unreleased/+cpp-header-line-walk.fixed.md @@ -0,0 +1,20 @@ +--- +category: fixed +affected: + - src/CodeIndex/Indexer/Scanning/FileIndexer.LanguageDetection.cs + - tests/CodeIndex.Tests/PerformanceTests.cs + - DEVELOPER_GUIDE.md + - TESTING_GUIDE.md +--- + +## English + +- **Reduced C / C++ header-detection allocations for large repositories** — + Bounded lexical samples for ambiguous `.h` files are now scanned by newline + index instead of materializing an array and a string for every sampled line. + +## 日本語 + +- **巨大 repository の C / C++ header 判定 allocation を削減しました** — + 曖昧な `.h` file の bounded lexical sample を、sampled line ごとの array と string + に実体化せず newline index で走査するようにしました。 diff --git a/changelog.d/unreleased/+delimited-extractor-spans.fixed.md b/changelog.d/unreleased/+delimited-extractor-spans.fixed.md new file mode 100644 index 0000000000..41613cfc88 --- /dev/null +++ b/changelog.d/unreleased/+delimited-extractor-spans.fixed.md @@ -0,0 +1,27 @@ +--- +category: fixed +affected: + - src/CodeIndex/Indexer/DelimitedSpanEnumerable.cs + - src/CodeIndex/Indexer/Symbols/SymbolExtractor.RepositoryMetadata.cs + - src/CodeIndex/Indexer/References/Languages/RepositoryMetadataReferenceExtractor.cs + - src/CodeIndex/Indexer/References/Languages/HdlReferenceExtractor.cs + - src/CodeIndex/Indexer/References/Languages/HdlReferenceExtractor.Scopes.cs + - src/CodeIndex/Indexer/References/Languages/ShaderReferenceExtractor.cs + - tests/CodeIndex.Tests/PerformanceTests.cs + - DEVELOPER_GUIDE.md + - TESTING_GUIDE.md +--- + +## English + +- **Removed split-array growth from delimiter-only extractor paths** — + Repository metadata, application manifests, VHDL declarations and package + imports, and CUDA kernel parameters now share an allocation-free span walker + while preserving trimming and empty-entry behavior. + +## 日本語 + +- **delimiter-only extractor 経路の split-array 増加を解消しました** — + repository metadata、application manifest、VHDL declaration / package import、 + CUDA kernel parameter は、trim と empty-entry の意味論を維持しつつ allocation-free + な span walker を共有します。 diff --git a/changelog.d/unreleased/+functional-span-membership.fixed.md b/changelog.d/unreleased/+functional-span-membership.fixed.md new file mode 100644 index 0000000000..b7164ac8f4 --- /dev/null +++ b/changelog.d/unreleased/+functional-span-membership.fixed.md @@ -0,0 +1,23 @@ +--- +category: fixed +affected: + - src/CodeIndex/Indexer/References/Languages/FunctionalLanguageReferenceExtractor.cs + - src/CodeIndex/Indexer/References/Languages/FunctionalLanguageReferenceExtractor.Erlang.cs + - src/CodeIndex/Indexer/References/Languages/FunctionalLanguageReferenceExtractor.Ocaml.cs + - src/CodeIndex/Indexer/References/Languages/FunctionalLanguageReferenceExtractor.Raku.cs + - tests/CodeIndex.Tests/PerformanceTests.cs + - DEVELOPER_GUIDE.md + - TESTING_GUIDE.md +--- + +## English + +- **Removed per-match closure allocations from functional-language references** — + Erlang, OCAML, and Raku now use shared indexed span-membership checks when + suppressing remote, qualified, quoted-atom, and type-reference matches. + +## 日本語 + +- **functional-language reference の match ごとの closure allocation を解消しました** — + Erlang、OCAML、Raku は remote、qualified、quoted-atom、type-reference match の抑制時に、 + 共通の indexed span-membership check を使います。 diff --git a/changelog.d/unreleased/+functional-terminator-spans.fixed.md b/changelog.d/unreleased/+functional-terminator-spans.fixed.md new file mode 100644 index 0000000000..69131e02c5 --- /dev/null +++ b/changelog.d/unreleased/+functional-terminator-spans.fixed.md @@ -0,0 +1,21 @@ +--- +category: fixed +affected: + - src/CodeIndex/Indexer/References/Languages/FunctionalLanguageReferenceExtractor.cs + - src/CodeIndex/Indexer/References/Languages/FunctionalLanguageReferenceExtractor.Erlang.cs + - tests/CodeIndex.Tests/PerformanceTests.cs + - DEVELOPER_GUIDE.md + - TESTING_GUIDE.md +--- + +## English + +- **Removed padded-line copies from functional-language state machines** — + Erlang specification/callable terminators and Raku heredoc terminators now + compare trimmed spans, avoiding a new string for every padded sentinel line. + +## 日本語 + +- **functional-language state machine の padded-line copy を解消しました** — + Erlang specification / callable terminator と Raku heredoc terminator は trimmed span + を比較し、padding 付き sentinel line ごとの新しい string を回避します。 diff --git a/changelog.d/unreleased/+hardware-scope-membership.fixed.md b/changelog.d/unreleased/+hardware-scope-membership.fixed.md new file mode 100644 index 0000000000..e6c742085a --- /dev/null +++ b/changelog.d/unreleased/+hardware-scope-membership.fixed.md @@ -0,0 +1,21 @@ +--- +category: fixed +affected: + - src/CodeIndex/Indexer/References/Languages/HdlReferenceExtractor.cs + - src/CodeIndex/Indexer/References/Languages/ShaderReferenceExtractor.cs + - tests/CodeIndex.Tests/PerformanceTests.cs + - DEVELOPER_GUIDE.md + - TESTING_GUIDE.md +--- + +## English + +- **Removed per-identifier closure allocations from hardware-language scopes** — + Verilog, SystemVerilog, and VHDL shadow checks plus CUDA, GLSL, HLSL, Metal, + and WGSL binding/resource checks now use direct indexed membership loops. + +## 日本語 + +- **hardware-language scope の identifier ごとの closure allocation を解消しました** — + Verilog、SystemVerilog、VHDL の shadow 判定と、CUDA、GLSL、HLSL、Metal、WGSL + の binding / resource 判定は direct indexed membership loop を使います。 diff --git a/changelog.d/unreleased/+lazy-functional-spans.fixed.md b/changelog.d/unreleased/+lazy-functional-spans.fixed.md new file mode 100644 index 0000000000..80e6451d72 --- /dev/null +++ b/changelog.d/unreleased/+lazy-functional-spans.fixed.md @@ -0,0 +1,23 @@ +--- +category: fixed +affected: + - src/CodeIndex/Indexer/References/Languages/FunctionalLanguageReferenceExtractor.cs + - src/CodeIndex/Indexer/References/Languages/FunctionalLanguageReferenceExtractor.Erlang.cs + - src/CodeIndex/Indexer/References/Languages/FunctionalLanguageReferenceExtractor.Ocaml.cs + - src/CodeIndex/Indexer/References/Languages/FunctionalLanguageReferenceExtractor.Raku.cs + - tests/CodeIndex.Tests/PerformanceTests.cs + - DEVELOPER_GUIDE.md + - TESTING_GUIDE.md +--- + +## English + +- **Stopped allocating empty exclusion lists on functional-language lines** — + Erlang, OCAML, and Raku now create quoted, remote, type, qualified, and method + span lists only after the first relevant match on a line. + +## 日本語 + +- **functional-language line ごとの empty exclusion list allocation を解消しました** — + Erlang、OCAML、Raku は quoted、remote、type、qualified、method span list を、行内の + 最初の関連 match が見つかった後にだけ作ります。 diff --git a/changelog.d/unreleased/+masked-reference-contexts.fixed.md b/changelog.d/unreleased/+masked-reference-contexts.fixed.md new file mode 100644 index 0000000000..101eaafab8 --- /dev/null +++ b/changelog.d/unreleased/+masked-reference-contexts.fixed.md @@ -0,0 +1,21 @@ +--- +category: fixed +affected: + - src/CodeIndex/Indexer/References/ReferenceExtractor.CoreReferenceLoop.cs + - tests/CodeIndex.Tests/PerformanceTests.cs + - DEVELOPER_GUIDE.md + - TESTING_GUIDE.md +--- + +## English + +- **Reduced reference-extraction allocations for large embedded multiline payloads** — + Structurally masked C# raw strings, Java text blocks, and JavaScript / + TypeScript template literals no longer materialize trimmed reference + contexts for lines that will be skipped. + +## 日本語 + +- **巨大な埋め込み multiline payload に対する reference extraction の allocation を削減しました** — + 構造マスク済みの C# raw string、Java text block、JavaScript / TypeScript template literal + では、skip される行の trim 済み reference context を実体化しません。 diff --git a/changelog.d/unreleased/+metadata-character-scans.fixed.md b/changelog.d/unreleased/+metadata-character-scans.fixed.md new file mode 100644 index 0000000000..066336c5f0 --- /dev/null +++ b/changelog.d/unreleased/+metadata-character-scans.fixed.md @@ -0,0 +1,24 @@ +--- +category: fixed +affected: + - src/CodeIndex/Indexer/SpanCharacterSearch.cs + - src/CodeIndex/Indexer/Symbols/SymbolExtractor.RepositoryMetadata.cs + - src/CodeIndex/Indexer/References/Languages/RepositoryMetadataReferenceExtractor.cs + - tests/CodeIndex.Tests/PerformanceTests.cs + - DEVELOPER_GUIDE.md + - TESTING_GUIDE.md +--- + +## English + +- **Reduced repository-metadata and manifest validation overhead** — + Long metadata candidates now use allocation-free span character scans, while + application manifests track dependency ancestry by XML depth instead of + rescanning an ancestor stack for every identity. + +## 日本語 + +- **repository metadata と manifest の validation overhead を削減しました** — + 長い metadata candidate は allocation-free な span character scan を使い、 + application manifest は identity ごとの ancestor stack 再走査ではなく XML depth + で dependency ancestry を追跡します。 diff --git a/changelog.d/unreleased/+prepared-line-whitespace.fixed.md b/changelog.d/unreleased/+prepared-line-whitespace.fixed.md new file mode 100644 index 0000000000..89c862c1d9 --- /dev/null +++ b/changelog.d/unreleased/+prepared-line-whitespace.fixed.md @@ -0,0 +1,19 @@ +--- +category: fixed +affected: + - src/CodeIndex/Indexer/References/ReferenceExtractor.CoreReferenceLoop.cs + - DEVELOPER_GUIDE.md +--- + +## English + +- **Avoided duplicate full-line whitespace scans during reference extraction** — + Every supported language now classifies each prepared line once and reuses + the result across special-line and ordinary empty-line dispatch, which is + especially important for long structurally masked payloads. + +## 日本語 + +- **reference extraction の重複した全行 whitespace scan を解消しました** — + 全対応言語で prepared line を一度だけ判定し、special-line と通常の empty-line dispatch + で結果を共有するため、長い構造マスク済み payload の再走査を避けます。 diff --git a/changelog.d/unreleased/+source-line-splitting.fixed.md b/changelog.d/unreleased/+source-line-splitting.fixed.md new file mode 100644 index 0000000000..f7d8ac8c98 --- /dev/null +++ b/changelog.d/unreleased/+source-line-splitting.fixed.md @@ -0,0 +1,22 @@ +--- +category: fixed +affected: + - src/CodeIndex/Indexer/SourceLineSplitter.cs + - src/CodeIndex/Indexer/Symbols/SymbolExtractor.cs + - src/CodeIndex/Indexer/References/ReferenceExtractor.Configuration.cs + - tests/CodeIndex.Tests/PerformanceTests.cs + - DEVELOPER_GUIDE.md + - TESTING_GUIDE.md +--- + +## English + +- **Reduced all-language extraction allocations for large source files** — + Symbol and reference extraction now share an exact-capacity line splitter + that avoids the temporary separator-index arrays created by generic splitting. + +## 日本語 + +- **巨大 source file の全言語 extraction allocation を削減しました** — + symbol / reference extraction は exact-capacity line splitter を共有し、generic split + が作る一時 separator-index array を回避します。 diff --git a/changelog.d/unreleased/+stream-bounded-regex-enumeration.fixed.md b/changelog.d/unreleased/+stream-bounded-regex-enumeration.fixed.md new file mode 100644 index 0000000000..818cd7fcad --- /dev/null +++ b/changelog.d/unreleased/+stream-bounded-regex-enumeration.fixed.md @@ -0,0 +1,58 @@ +--- +category: fixed +affected: + - src/CodeIndex/Indexer/BoundedRegex.cs + - src/CodeIndex/Indexer/References/ReferenceExtractor.cs + - tests/CodeIndex.Tests/BoundedRegexTests.cs + - tests/CodeIndex.Tests/PerformanceTests.cs + - DEVELOPER_GUIDE.md + - TESTING_GUIDE.md +--- + +## English + +- **Made bounded regex enumeration demand-driven** — Extractors that stop after + reaching a result limit no longer force every remaining regex match on the + source line to be materialized first, while right-to-left instances preserve + their default reverse match order. Secondary reference scanners now use the + same streaming path across Fortran, Visual Basic, F#, Pascal, Objective-C, + Haskell, Elixir, Smalltalk, Lua, Dart, Razor, JSON, JavaScript, GitHub Actions, + and C++ compound requirements without changing their configured timeouts. + Bounded reference scans now stop before requesting another match and skip + all later extraction phases on the same dense line once capacity is reached. + Symbol and dependency extractors now stream their remaining multi-match + patterns across scientific/native, Pascal/Ada, SQL, Python, Swift, GraphQL, + markup/XAML, shell, Ruby, Perl, Elixir, CSS, HDL, C++, and manifest parsing. + Count-only scans no longer retain a `MatchCollection`, while preserving the + previous all-or-nothing result if matching times out. + Dense XAML supplemental symbol scans now stop as soon as the shared + structured-data symbol budget needs its diagnostic marker, instead of + building an unbounded temporary list and trimming it after all phases. + Symbol container assignment now reuses one path buffer across members instead + of allocating a stack snapshot and a second list for every nested symbol. + Prolog goal scans now allocate per-line call lists only when a known goal is + emitted and update directive metadata in place instead of copying the list. + JSON symbol and reference extraction now build UTF-8 line-offset arrays at + exact capacity instead of retaining a growing list alongside its final copy. + +## 日本語 + +- **bounded regex enumeration を demand-driven にしました** — 結果上限に達して + 停止する extractor は、source line に残るすべての regex match を先に実体化せず、 + right-to-left instance の既定の逆順も維持します。Fortran、Visual Basic、F#、 + Pascal、Objective-C、Haskell、Elixir、Smalltalk、Lua、Dart、Razor、JSON、 + JavaScript、GitHub Actions、C++ compound requirement の secondary reference + scanner も設定済み timeout を変えず同じ逐次経路を使います。bounded reference + scan は上限到達後に次の match を要求せず、同じ dense line の後続 extraction phase + も省略します。symbol / dependency extractor も scientific / native、Pascal / Ada、 + SQL、Python、Swift、GraphQL、markup / XAML、shell、Ruby、Perl、Elixir、CSS、HDL、 + C++、manifest parsing の残存 multi-match pattern を逐次走査します。count のみの scan + は `MatchCollection` を保持せず、matching timeout 時は従来どおり all-or-nothing の + 結果を返します。dense XAML の supplemental symbol scan は、共有 structured-data + symbol budget の diagnostic marker が必要になった時点で停止し、全 phase の後まで + 無制限の一時 list を構築してから trim しません。symbol の container assignment も + member ごとに stack snapshot と2つ目の list を割り当てず、1つの path buffer を再利用します。 + Prolog goal scan も既知 goal を出力するときだけ per-line call list を割り当て、directive + metadata は list を copy せず in-place で更新します。JSON symbol / reference extraction + も UTF-8 line-offset array を exact capacity で構築し、成長中の list と最終 copy を同時に + 保持しません。 diff --git a/changelog.d/unreleased/+stream-core-reference-matches.fixed.md b/changelog.d/unreleased/+stream-core-reference-matches.fixed.md new file mode 100644 index 0000000000..b132df52de --- /dev/null +++ b/changelog.d/unreleased/+stream-core-reference-matches.fixed.md @@ -0,0 +1,30 @@ +--- +category: fixed +affected: + - src/CodeIndex/Indexer/References/Languages/CSharpReferenceExtractor.cs + - src/CodeIndex/Indexer/References/Languages/CSharpReferenceExtractor.Support.cs + - src/CodeIndex/Indexer/References/ReferenceExtractor.CoreCallReferences.cs + - src/CodeIndex/Indexer/References/ReferenceExtractor.CoreLanguageLines.cs + - src/CodeIndex/Indexer/References/ReferenceExtractor.CoreSpecializedLines.cs + - src/CodeIndex/Indexer/References/ReferenceExtractor.CoreTypeReferences.cs + - src/CodeIndex/Indexer/References/ReferenceExtractor.CSharpPatterns.cs + - src/CodeIndex/Indexer/References/ReferenceExtractor.CSharpTypeNames.cs + - src/CodeIndex/Indexer/References/ReferenceExtractor.PatternTypeReferences.cs + - src/CodeIndex/Indexer/References/ReferenceExtractor.PrimaryConstructors.cs + - src/CodeIndex/Indexer/References/ReferenceExtractor.ReferenceRecords.cs + - src/CodeIndex/Indexer/References/ReferenceExtractor.Solidity.cs + - DEVELOPER_GUIDE.md + - TESTING_GUIDE.md +--- + +## English + +- **Streamed core reference matches** — Shared calls, C# attributes, types, + patterns and locals, JSX elements, JVM documentation links, and Solidity + scanners now consume single-pass match groups on demand. + +## 日本語 + +- **core reference match を逐次走査にしました** — 共有 call、C# attribute / type / + pattern / local、JSX element、JVM documentation link、Solidity scanner は + single-pass の match group を demand-driven に消費します。 diff --git a/changelog.d/unreleased/+stream-dynamic-reference-matches.fixed.md b/changelog.d/unreleased/+stream-dynamic-reference-matches.fixed.md new file mode 100644 index 0000000000..d66612c5ce --- /dev/null +++ b/changelog.d/unreleased/+stream-dynamic-reference-matches.fixed.md @@ -0,0 +1,26 @@ +--- +category: fixed +affected: + - src/CodeIndex/Indexer/References/Languages/PhpReferenceExtractor.cs + - src/CodeIndex/Indexer/References/Languages/PhpReferenceExtractor.LanguageTypes.cs + - src/CodeIndex/Indexer/References/Languages/PhpReferenceExtractor.Members.cs + - src/CodeIndex/Indexer/References/Languages/RubyReferenceExtractor.cs + - src/CodeIndex/Indexer/References/Languages/RReferenceExtractor.cs + - src/CodeIndex/Indexer/References/Languages/RReferenceExtractor.Members.cs + - src/CodeIndex/Indexer/References/Languages/RReferenceExtractor.CallsAndResources.cs + - src/CodeIndex/Indexer/References/Languages/PerlReferenceExtractor.cs + - DEVELOPER_GUIDE.md + - TESTING_GUIDE.md +--- + +## English + +- **Streamed dynamic-language reference matches** — PHP, Ruby, R, and Perl + scanners now enumerate dense attribute, type, DSL, namespace/member/resource, + and arrow-call matches only while the reference budget has capacity. + +## 日本語 + +- **dynamic-language の reference match を逐次走査にしました** — PHP、Ruby、R、 + Perl scanner は dense な attribute、type、DSL、namespace / member / resource、 + arrow-call match を reference budget に空きがある間だけ列挙します。 diff --git a/changelog.d/unreleased/+stream-functional-reference-matches.fixed.md b/changelog.d/unreleased/+stream-functional-reference-matches.fixed.md new file mode 100644 index 0000000000..578d6bac07 --- /dev/null +++ b/changelog.d/unreleased/+stream-functional-reference-matches.fixed.md @@ -0,0 +1,23 @@ +--- +category: fixed +affected: + - src/CodeIndex/Indexer/References/Languages/FunctionalLanguageReferenceExtractor.Clojure.cs + - src/CodeIndex/Indexer/References/Languages/FunctionalLanguageReferenceExtractor.Erlang.cs + - src/CodeIndex/Indexer/References/Languages/FunctionalLanguageReferenceExtractor.Ocaml.cs + - src/CodeIndex/Indexer/References/Languages/FunctionalLanguageReferenceExtractor.Raku.cs + - src/CodeIndex/Indexer/References/Languages/ElixirReferenceExtractor.cs + - DEVELOPER_GUIDE.md + - TESTING_GUIDE.md +--- + +## English + +- **Streamed functional-language reference matches** — Clojure, Elixir, + Erlang, OCAML, and Raku now enumerate regex results on demand and stop dense + lines as soon as the per-file reference budget is full. + +## 日本語 + +- **functional-language の reference match を逐次走査にしました** — Clojure、 + Elixir、Erlang、OCAML、Raku は regex result を demand-driven に列挙し、per-file + reference budget が満杯になると dense line の走査を停止します。 diff --git a/changelog.d/unreleased/+stream-infrastructure-reference-matches.fixed.md b/changelog.d/unreleased/+stream-infrastructure-reference-matches.fixed.md new file mode 100644 index 0000000000..3c7732614e --- /dev/null +++ b/changelog.d/unreleased/+stream-infrastructure-reference-matches.fixed.md @@ -0,0 +1,27 @@ +--- +category: fixed +affected: + - src/CodeIndex/Indexer/References/Languages/BuildAutomationReferenceExtractor.cs + - src/CodeIndex/Indexer/References/Languages/CssReferenceExtractor.cs + - src/CodeIndex/Indexer/References/Languages/DockerfileReferenceExtractor.cs + - src/CodeIndex/Indexer/References/Languages/HdlReferenceExtractor.cs + - src/CodeIndex/Indexer/References/Languages/HdlReferenceExtractor.Scopes.cs + - src/CodeIndex/Indexer/References/Languages/MarkupSchemaReferenceExtractor.cs + - src/CodeIndex/Indexer/References/Languages/PowerShellReferenceExtractor.cs + - src/CodeIndex/Indexer/References/Languages/ShellReferenceExtractor.cs + - src/CodeIndex/Indexer/References/Languages/XamlReferenceExtractor.cs + - DEVELOPER_GUIDE.md + - TESTING_GUIDE.md +--- + +## English + +- **Streamed infrastructure and markup reference matches** — CSS, XAML, + HTML/GraphQL/Markdown, HDL, MSBuild, Dockerfile, shell, and PowerShell + scanners now consume dense match groups on demand. + +## 日本語 + +- **infrastructure / markup の reference match を逐次走査にしました** — CSS、 + XAML、HTML / GraphQL / Markdown、HDL、MSBuild、Dockerfile、shell、PowerShell + scanner は dense な match group を demand-driven に消費します。 diff --git a/changelog.d/unreleased/+stream-jvm-reference-matches.fixed.md b/changelog.d/unreleased/+stream-jvm-reference-matches.fixed.md new file mode 100644 index 0000000000..a850856d8a --- /dev/null +++ b/changelog.d/unreleased/+stream-jvm-reference-matches.fixed.md @@ -0,0 +1,24 @@ +--- +category: fixed +affected: + - src/CodeIndex/Indexer/References/Languages/JavaReferenceExtractor.Modules.cs + - src/CodeIndex/Indexer/References/Languages/JavaReferenceExtractor.Types.cs + - src/CodeIndex/Indexer/References/Languages/KotlinReferenceExtractor.cs + - src/CodeIndex/Indexer/References/Languages/ScalaReferenceExtractor.cs + - src/CodeIndex/Indexer/References/Languages/GradleReferenceExtractor.cs + - src/CodeIndex/Indexer/References/Support/JvmMethodReferenceExtractor.cs + - DEVELOPER_GUIDE.md + - TESTING_GUIDE.md +--- + +## English + +- **Streamed JVM-family reference matches** — Java, Kotlin, Scala, and + Gradle/Groovy scanners now consume dense regex results on demand and stop + bounded reference loops once the per-file cap is reached. + +## 日本語 + +- **JVM-family の reference match を逐次走査にしました** — Java、Kotlin、Scala、 + Gradle / Groovy scanner は dense な regex result を demand-driven に消費し、bounded + reference loop は per-file 上限に達すると停止します。 diff --git a/changelog.d/unreleased/+stream-python-reference-matches.fixed.md b/changelog.d/unreleased/+stream-python-reference-matches.fixed.md new file mode 100644 index 0000000000..9cbee512b3 --- /dev/null +++ b/changelog.d/unreleased/+stream-python-reference-matches.fixed.md @@ -0,0 +1,29 @@ +--- +category: fixed +affected: + - src/CodeIndex/Indexer/BoundedRegex.cs + - src/CodeIndex/Indexer/References/Languages/PythonReferenceExtractor.cs + - src/CodeIndex/Indexer/References/Languages/PythonReferenceExtractor.ClassBases.cs + - src/CodeIndex/Indexer/References/Languages/PythonReferenceExtractor.DataclassFields.cs + - src/CodeIndex/Indexer/References/Languages/PythonReferenceExtractor.FrameworkIntegrations.cs + - src/CodeIndex/Indexer/References/Languages/PythonReferenceExtractor.FunctionSignatures.cs + - src/CodeIndex/Indexer/References/Languages/PythonReferenceExtractor.RuntimeTypes.cs + - src/CodeIndex/Indexer/References/Languages/PythonReferenceExtractor.TypingFactories.cs + - tests/CodeIndex.Tests/BoundedRegexTests.cs + - DEVELOPER_GUIDE.md + - TESTING_GUIDE.md +--- + +## English + +- **Streamed Python reference matches** — Python decorators, annotations, + runtime type checks, typing factories, dataclass/framework integrations, and + dynamic imports now stop producing regex matches at the reference cap; + decorator arguments also stream directly from their start offset. + +## 日本語 + +- **Python の reference match を逐次走査にしました** — decorator、annotation、 + runtime type check、typing factory、dataclass / framework integration、dynamic + import は reference 上限で regex match の生成を停止し、decorator argument も指定 + offset から直接逐次走査します。 diff --git a/changelog.d/unreleased/+stream-sql-reference-matches.fixed.md b/changelog.d/unreleased/+stream-sql-reference-matches.fixed.md new file mode 100644 index 0000000000..f6cbeddab0 --- /dev/null +++ b/changelog.d/unreleased/+stream-sql-reference-matches.fixed.md @@ -0,0 +1,27 @@ +--- +category: fixed +affected: + - src/CodeIndex/Indexer/References/Languages/SqlReferenceExtractor.cs + - src/CodeIndex/Indexer/References/Languages/SqlReferenceExtractor.AlterTargets.cs + - src/CodeIndex/Indexer/References/Languages/SqlReferenceExtractor.DropTargets.cs + - src/CodeIndex/Indexer/References/Languages/SqlReferenceExtractor.LineMasking.cs + - src/CodeIndex/Indexer/References/Languages/SqlReferenceExtractor.MaintenanceTargets.cs + - src/CodeIndex/Indexer/References/Languages/SqlReferenceExtractor.QualifiedColumns.cs + - src/CodeIndex/Indexer/References/Languages/SqlReferenceExtractor.Sources.cs + - src/CodeIndex/Indexer/References/Languages/SqlReferenceExtractor.StatementState.cs + - src/CodeIndex/Indexer/References/Languages/SqlReferenceExtractor.Statements.cs + - DEVELOPER_GUIDE.md + - TESTING_GUIDE.md +--- + +## English + +- **Streamed SQL reference matches** — SQL statement, source, target, + generated-column, window-clause, procedure-call, and temporary-object + scanners now consume matches on demand and stop at bounded reference limits. + +## 日本語 + +- **SQL の reference match を逐次走査にしました** — SQL の statement、source、 + target、generated-column、window-clause、procedure-call、一時 object scanner は + match を demand-driven に消費し、bounded reference の上限で停止します。 diff --git a/changelog.d/unreleased/+stream-systems-reference-matches.fixed.md b/changelog.d/unreleased/+stream-systems-reference-matches.fixed.md new file mode 100644 index 0000000000..40be258a47 --- /dev/null +++ b/changelog.d/unreleased/+stream-systems-reference-matches.fixed.md @@ -0,0 +1,29 @@ +--- +category: fixed +affected: + - src/CodeIndex/Indexer/References/Languages/CppReferenceExtractor.cs + - src/CodeIndex/Indexer/References/Languages/GoReferenceExtractor.cs + - src/CodeIndex/Indexer/References/Languages/RustReferenceExtractor.cs + - src/CodeIndex/Indexer/References/Languages/RustReferenceExtractor.SignatureTypes.cs + - src/CodeIndex/Indexer/References/Languages/RustReferenceExtractor.ValueTypes.cs + - src/CodeIndex/Indexer/References/Languages/SwiftReferenceExtractor.cs + - src/CodeIndex/Indexer/References/Languages/ScientificNativeReferenceEmitter.cs + - src/CodeIndex/Indexer/References/Support/LanguageReferenceExtractionSupport.CppTypeGroups.cs + - src/CodeIndex/Indexer/References/Support/LanguageReferenceExtractionSupport.Go.cs + - src/CodeIndex/Indexer/References/Support/LanguageReferenceExtractionSupport.GoCompositeTypes.cs + - src/CodeIndex/Indexer/References/Support/LanguageReferenceExtractionSupport.GoSignatures.cs + - DEVELOPER_GUIDE.md + - TESTING_GUIDE.md +--- + +## English + +- **Streamed systems-language reference matches** — C/C++, Rust, Swift, Go, + and shared scientific/native scanners now consume dense construction, type, + concurrency, wrapper, and call matches on demand. + +## 日本語 + +- **systems-language の reference match を逐次走査にしました** — C / C++、Rust、 + Swift、Go と共有 scientific / native scanner は dense な construction、type、 + concurrency、wrapper、call match を demand-driven に消費します。 diff --git a/changelog.d/unreleased/+trimmed-suffix-spans.fixed.md b/changelog.d/unreleased/+trimmed-suffix-spans.fixed.md new file mode 100644 index 0000000000..f945aaedcd --- /dev/null +++ b/changelog.d/unreleased/+trimmed-suffix-spans.fixed.md @@ -0,0 +1,24 @@ +--- +category: fixed +affected: + - src/CodeIndex/Indexer/SpanCharacterSearch.cs + - src/CodeIndex/Indexer/References/Languages/FunctionalLanguageReferenceExtractor.cs + - src/CodeIndex/Indexer/References/Languages/CssReferenceExtractor.AnimationsAndSelectors.cs + - src/CodeIndex/Indexer/Symbols/SymbolExtractor.CSharpScanner.cs + - src/CodeIndex/Indexer/Symbols/SymbolExtractor.Java.cs + - tests/CodeIndex.Tests/PerformanceTests.cs + - DEVELOPER_GUIDE.md + - TESTING_GUIDE.md +--- + +## English + +- **Removed padded suffix copies across declaration scanners** — CSS selector + continuations, C# and Java body-less declarations, and functional-language + sentinels now share allocation-free trimmed span comparisons. + +## 日本語 + +- **declaration scanner 横断で padded suffix copy を解消しました** — + CSS selector continuation、C# / Java body-less declaration、functional-language + sentinel は allocation-free な trimmed span 比較を共有します。 diff --git a/src/CodeIndex/Indexer/BoundedRegex.cs b/src/CodeIndex/Indexer/BoundedRegex.cs index 49a17252a6..eecdde52a1 100644 --- a/src/CodeIndex/Indexer/BoundedRegex.cs +++ b/src/CodeIndex/Indexer/BoundedRegex.cs @@ -145,41 +145,68 @@ public static BclMatch Match(BclRegex regex, string input) public static IEnumerable EnumerateMatches(BclRegex regex, string input) { - MatchCollection matches; - try - { - matches = regex.Matches(input); - _ = matches.Count; - } - catch (RegexMatchTimeoutException ex) + var pattern = regex.ToString(); + var match = FirstMatchOrEmpty(regex, input, pattern); + while (match.Success) { - RecordTimeout("matches", regex.ToString(), ex); - yield break; + yield return match; + match = NextMatchOrEmpty(match, pattern); } + } - foreach (BclMatch match in matches) + public static IEnumerable EnumerateMatches( + BclRegex regex, + string input, + int startAt) + { + var pattern = regex.ToString(); + var match = FirstMatchOrEmpty(regex, input, startAt, pattern); + while (match.Success) + { yield return match; + match = NextMatchOrEmpty(match, pattern); + } } public static IEnumerable EnumerateMatches(string input, string pattern) => EnumerateMatches(input, pattern, RegexOptions.None); - public static IEnumerable EnumerateMatches(string input, string pattern, RegexOptions options) + public static IEnumerable EnumerateMatches( + string input, + string pattern, + RegexOptions options) => + EnumerateMatches(input, pattern, options, DefaultMatchTimeout); + + public static IEnumerable EnumerateMatches( + string input, + string pattern, + RegexOptions options, + TimeSpan matchTimeout) + { + var match = FirstMatchOrEmpty(input, pattern, options, matchTimeout); + while (match.Success) + { + yield return match; + match = NextMatchOrEmpty(match, pattern); + } + } + + public static int CountMatches(BclRegex regex, string input) { - MatchCollection matches; try { - matches = BclRegex.Matches(input, pattern, options, DefaultMatchTimeout); - _ = matches.Count; + var count = 0; + for (var match = regex.Match(input); match.Success; match = match.NextMatch()) + count++; + return count; } catch (RegexMatchTimeoutException ex) { - RecordTimeout("matches", pattern, ex); - yield break; + // MatchCollection.Count historically returned no matches after a timeout. Preserve + // that all-or-nothing behavior without retaining every Match object. + RecordTimeout("matches", regex.ToString(), ex); + return 0; } - - foreach (BclMatch match in matches) - yield return match; } public static new bool IsMatch(string input, string pattern) => @@ -354,6 +381,73 @@ public static IEnumerable EnumerateMatches(string input, string patter private static void RecordTimeout(string operation, string pattern, RegexMatchTimeoutException ex) => TimeoutCaptureScope.Value?.Record(operation, pattern, ex.MatchTimeout); + private static BclMatch FirstMatchOrEmpty( + BclRegex regex, + string input, + string pattern) + { + try + { + return regex.Match(input); + } + catch (RegexMatchTimeoutException ex) + { + RecordTimeout("matches", pattern, ex); + return BclMatch.Empty; + } + } + + private static BclMatch FirstMatchOrEmpty( + BclRegex regex, + string input, + int startAt, + string pattern) + { + try + { + return regex.Match(input, startAt); + } + catch (RegexMatchTimeoutException ex) + { + RecordTimeout("matches", pattern, ex); + return BclMatch.Empty; + } + } + + private static BclMatch FirstMatchOrEmpty( + string input, + string pattern, + RegexOptions options, + TimeSpan matchTimeout) + { + try + { + return BclRegex.Match( + input, + pattern, + options, + matchTimeout); + } + catch (RegexMatchTimeoutException ex) + { + RecordTimeout("matches", pattern, ex); + return BclMatch.Empty; + } + } + + private static BclMatch NextMatchOrEmpty(BclMatch match, string pattern) + { + try + { + return match.NextMatch(); + } + catch (RegexMatchTimeoutException ex) + { + RecordTimeout("matches", pattern, ex); + return BclMatch.Empty; + } + } + private static string HashPattern(string pattern) { var hash = SHA256.HashData(Encoding.UTF8.GetBytes(pattern)); diff --git a/src/CodeIndex/Indexer/DelimitedSpanEnumerable.cs b/src/CodeIndex/Indexer/DelimitedSpanEnumerable.cs new file mode 100644 index 0000000000..28e7f9fa01 --- /dev/null +++ b/src/CodeIndex/Indexer/DelimitedSpanEnumerable.cs @@ -0,0 +1,84 @@ +namespace CodeIndex.Indexer; + +internal readonly ref struct DelimitedSpanEnumerable +{ + private readonly ReadOnlySpan _value; + private readonly char _delimiter; + private readonly bool _trimEntries; + private readonly bool _removeEmptyEntries; + + internal DelimitedSpanEnumerable( + ReadOnlySpan value, + char delimiter, + bool trimEntries = false, + bool removeEmptyEntries = false) + { + _value = value; + _delimiter = delimiter; + _trimEntries = trimEntries; + _removeEmptyEntries = removeEmptyEntries; + } + + public Enumerator GetEnumerator() => + new(_value, _delimiter, _trimEntries, _removeEmptyEntries); + + internal ref struct Enumerator + { + private readonly ReadOnlySpan _value; + private readonly char _delimiter; + private readonly bool _trimEntries; + private readonly bool _removeEmptyEntries; + private int _nextStart; + private bool _finished; + + internal Enumerator( + ReadOnlySpan value, + char delimiter, + bool trimEntries, + bool removeEmptyEntries) + { + _value = value; + _delimiter = delimiter; + _trimEntries = trimEntries; + _removeEmptyEntries = removeEmptyEntries; + _nextStart = 0; + _finished = false; + Current = default; + CurrentStart = 0; + } + + public ReadOnlySpan Current { get; private set; } + public int CurrentStart { get; private set; } + + public bool MoveNext() + { + while (!_finished) + { + var start = _nextStart; + var relativeEnd = _value[start..].IndexOf(_delimiter); + var end = relativeEnd < 0 + ? _value.Length + : start + relativeEnd; + _finished = relativeEnd < 0; + _nextStart = end + 1; + + var entry = _value[start..end]; + if (_trimEntries) + { + var trimmed = entry.Trim(); + start += entry.Length - entry.TrimStart().Length; + entry = trimmed; + } + + if (_removeEmptyEntries && entry.IsEmpty) + continue; + + Current = entry; + CurrentStart = start; + return true; + } + + return false; + } + } +} diff --git a/src/CodeIndex/Indexer/DependencyPackageExtractor.cs b/src/CodeIndex/Indexer/DependencyPackageExtractor.cs index 71769d6abc..ddd6ca003c 100644 --- a/src/CodeIndex/Indexer/DependencyPackageExtractor.cs +++ b/src/CodeIndex/Indexer/DependencyPackageExtractor.cs @@ -648,7 +648,7 @@ private static void AddQuotedDependencySpecs( List packages, HashSet seen) { - foreach (Match match in QuotedDependencyRegex.Matches(rawLine)) + foreach (Match match in BoundedRegex.EnumerateMatches(QuotedDependencyRegex, rawLine)) { var spec = match.Groups["spec"].Value; if (!TryParseDependencySpec(spec, out var name, out var version)) diff --git a/src/CodeIndex/Indexer/References/Languages/BuildAutomationReferenceExtractor.cs b/src/CodeIndex/Indexer/References/Languages/BuildAutomationReferenceExtractor.cs index 2ce0e59d2d..b1c376b6fa 100644 --- a/src/CodeIndex/Indexer/References/Languages/BuildAutomationReferenceExtractor.cs +++ b/src/CodeIndex/Indexer/References/Languages/BuildAutomationReferenceExtractor.cs @@ -213,15 +213,19 @@ private static void EmitMsBuildReferences( if (line.TrimStart().StartsWith("