From bd5f2e9a3346ceeb592c834d3604fd9233ecd269 Mon Sep 17 00:00:00 2001 From: Widthdom Date: Fri, 24 Jul 2026 12:52:07 +0900 Subject: [PATCH 01/26] Add dynamic language reference graphs (#4746) --- DEVELOPER_GUIDE.md | 4 +- TESTING_GUIDE.md | 2 + USER_GUIDE.md | 10 +- changelog.d/unreleased/4746.added.md | 27 ++ .../DynamicDeclarativeReferenceExtractor.cs | 372 ++++++++++++++++++ .../ReferenceExtractor.CoreExtraction.cs | 24 +- .../References/ReferenceExtractor.State.cs | 3 +- .../Indexer/References/ReferenceExtractor.cs | 37 +- .../Symbols/SymbolExtractor.ExtractCore.cs | 48 +++ .../Indexer/Symbols/SymbolExtractor.cs | 18 + .../QueryCommandRunnerTests.cs | 31 +- .../ReferenceExtractorTests.cs | 131 ++++++ tests/CodeIndex.Tests/SymbolExtractorTests.cs | 20 + tests/CodeIndex.Tests/golden/status.json | 5 + 14 files changed, 714 insertions(+), 18 deletions(-) create mode 100644 changelog.d/unreleased/4746.added.md create mode 100644 src/CodeIndex/Indexer/References/Languages/DynamicDeclarativeReferenceExtractor.cs diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index 9447354b9..4537cc7e9 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -1607,7 +1607,7 @@ Process exit codes are coarse (`0` success including valid zero-row queries, `1` ## Design decisions - **Language capability patterns remain typed at the integration boundary** — CLI/MCP `languages` rows expose suffix-only `extensions`, literal `exact_filenames`, and ``-rendered `filename_prefix_patterns`. `legacy_patterns` preserves the former combined list during deprecation, and `pattern_provenance` identifies built-in, plugin/pattern, and language-map override ownership. Round-trip tests feed every advertised typed pattern back through `FileIndexer.DetectLanguage` (#4617). -- **Ambiguous source extensions stay explicit** — `.m` and `.pl` are not assigned to Objective-C and Perl by default. `FileIndexer` checks an authoritative recognized shebang, then a 64 KiB bounded prefix for strong mutually exclusive Objective-C/MATLAB or Perl/Prolog markers, then at most 256 entries per ancestor directory for conservative project markers. Conflicting or weak evidence is indexed as `ambiguous_m` / `ambiguous_pl`; MATLAB and Prolog have conservative symbol patterns but deliberately advertise no reference/graph support (#4612). +- **Ambiguous source extensions stay explicit** — `.m` and `.pl` are not assigned to Objective-C and Perl by default. `FileIndexer` checks an authoritative recognized shebang, then a 64 KiB bounded prefix for strong mutually exclusive Objective-C/MATLAB or Perl/Prolog markers, then at most 256 entries per ancestor directory for conservative project markers. Conflicting or weak evidence is indexed as `ambiguous_m` / `ambiguous_pl`. MATLAB deliberately remains symbol-only; Prolog and `ambiguous_pl` advertise conservative reference/graph support, with the ambiguous bucket using union symbol/reference rules without changing the content-based classification (#4612, #4746). - **Hotspot marker fingerprints share one bounded tree traversal** — full/update CLI and MCP indexing compute C#, VB, F#, and MSBuild marker fingerprints together instead of walking the directory tree once per language. Each distinct marker glob retains the platform filesystem's matching behavior and is enumerated once per visited directory, while child directories are enumerated once; marker sets, budgets, truncation sentinels, and warning order remain isolated per language. The single-language API delegates to the same engine, preserving ignore rules, nested-repository/submodule boundaries, and MCP authorized-read failures. - **Lock-file dependency graphs model package relationships** — `packages.lock.json`, `package-lock.json`, and `npm-shrinkwrap.json` keep package declarations as symbols, but emit `dependency` references only for explicit parent-package to child-package entries. The parent package is stored as the reference container, so `callers` can identify which package requires a child and `deps` does not infer lock-file-to-lock-file similarity merely because two files contain the same resolved package set (#4409). - **Dependency-cycle audits separate analysis from display** — CLI `deps --cycles` and MCP `deps` with `cycles=true` analyze a deterministic, path-ordered edge set up to the independent `--graph-budget` / `graphBudget` before computing and stably ranking strongly connected components. `--limit` / `limit` only paginates that ranked SCC set, and opaque cursors are bound to the filters, graph budget, and indexed graph that produced them. Machine-readable responses expose `analysis_complete`, graph edge count/budget, stable ranking mode, authoritative total-cycle status, and continuation metadata; exhausting the graph budget is reported as an explicitly incomplete analysis rather than a complete cycle audit (#4731). @@ -4605,7 +4605,7 @@ USER_GUIDEの[終了コード](USER_GUIDE.md#終了コード)セクションを ## 設計判断 - **integration boundary では language capability pattern の型を維持** — CLI/MCP の `languages` 行は suffix のみの `extensions`、literal な `exact_filenames`、`` 表記の `filename_prefix_patterns` を公開します。`legacy_patterns` は deprecation 中に従来の combined list を保持し、`pattern_provenance` は built-in、plugin/pattern、language-map override の所有元を示します。round-trip test は広告した全 typed pattern を `FileIndexer.DetectLanguage` に戻して検証します（#4617）。 -- **曖昧な source extension は曖昧なまま明示** — `.m` と `.pl` を既定で Objective-C / Perl に割り当てません。`FileIndexer` は authoritative な認識済み shebang、64 KiB 上限 prefix 内の相互排他的で強い Objective-C/MATLAB または Perl/Prolog marker、各 ancestor directory 最大 256 entry の保守的な project marker の順に確認します。競合または弱い証拠は `ambiguous_m` / `ambiguous_pl` として index し、MATLAB / Prolog は保守的な symbol pattern を持つ一方、reference / graph 対応を意図的に広告しません（#4612）。 +- **曖昧な source extension は曖昧なまま明示** — `.m` と `.pl` を既定で Objective-C / Perl に割り当てません。`FileIndexer` は authoritative な認識済み shebang、64 KiB 上限 prefix 内の相互排他的で強い Objective-C/MATLAB または Perl/Prolog marker、各 ancestor directory 最大 256 entry の保守的な project marker の順に確認します。競合または弱い証拠は `ambiguous_m` / `ambiguous_pl` として index します。MATLAB は意図的に symbol-only のままですが、Prolog と `ambiguous_pl` は保守的な reference / graph 対応を広告し、曖昧 bucket では content-based classification を変えずに symbol / reference rule の和集合を使います（#4612、#4746）。 - **hotspot marker fingerprint は上限付きtree traversalを1回共有** — full/update CLIとMCP indexingは、directory treeを言語ごとに歩かず、C#、VB、F#、MSBuildのmarker fingerprintをまとめて計算します。各directoryでは固有marker globごとにplatform filesystemのmatching挙動を保って1回ずつ列挙し、child directoryも1回だけ列挙する一方、marker集合、budget、truncation sentinel、warning順は言語別に分離します。single-language APIも同じengineへ委譲し、ignore rule、nested repository/submodule境界、MCP authorized read failureを維持します。 - **lock file の依存グラフは package 間の関係をモデル化** — `packages.lock.json`、`package-lock.json`、`npm-shrinkwrap.json` は package 宣言を symbol として保持しますが、`dependency` reference は明示された親 package → 子 package の項目だけに出力します。親 package を reference container に保存するため、`callers` はどの package が子 package を必要としているかを特定でき、`deps` は同じ resolved package 集合を持つだけの lock file 同士を類似関係として推論しません（#4409）。 - **依存サイクル監査では解析と表示を分離** — CLI の `deps --cycles` と MCP `deps` の `cycles=true` は、独立した `--graph-budget` / `graphBudget` まで path 順で決定的な edge 集合を解析してから、強連結成分を安定順位付けします。`--limit` / `limit` はその SCC 順位集合をページ分割するだけで、不透明 cursor は生成時の filter、graph budget、indexed graph に結び付けます。machine-readable 応答は `analysis_complete`、graph edge 件数/予算、安定 ranking mode、authoritative な総 cycle 件数かどうか、continuation metadata を公開し、graph budget 枯渇時は完全な cycle 監査を装わず明示的な未完了解析として報告します（#4731）。 diff --git a/TESTING_GUIDE.md b/TESTING_GUIDE.md index f0ac1a5d4..be57b64da 100644 --- a/TESTING_GUIDE.md +++ b/TESTING_GUIDE.md @@ -88,6 +88,7 @@ Use `docs/test-doc-maintenance-plan.md` before moving oversized suites or adding Use `AssertReferencesContainInContext(...)` when several reference names share the same kind and exact source context; keep direct predicates when context is only one part of a richer edge contract. Use `AssertReferencesDoNotContain(...)` for negative checks over one reference kind; retain direct predicates when the exclusion depends on container, context, line, or other metadata. `ReferenceExtractorTests.ExtractSymbolsAndReferences(...)` owns the common symbol-then-reference extraction setup for tests that need both lists; use it instead of repeating the two extractor calls when the fixture does not need a specialized path or workspace symbol setup, and discard the symbol tuple element with `_` instead of keeping an unused `symbols` local when the test only asserts references. + Crystal, Groovy, Tcl, Prolog, and `ambiguous_pl` graph fixtures keep imports, parenthesized calls, conservative same-file command/predicate calls, caller containers, and keyword false-positive controls independently diagnosable; capability tests must keep their advertised symbol/reference/graph flags aligned with those extractor fixtures (#4746). JavaScript/TypeScript discriminant-guard coverage shares one JavaScript-compatible fixture across both languages, while `DatabaseTests` pins `type_tag` as an accepted persisted reference kind so extractor-only assertions cannot hide database kind-catalog drift. Dockerfile named-stage reference variants share one multi-stage fixture when ordinary, lowercase, platform-flagged, commented, hyphenated, and dotted forms can be distinguished by exact per-stage call counts; keep external base-image exclusions in that fixture as the negative control. Dockerfile `COPY --from` ONBUILD and quoted-stage forms share a fixture with tagged and digest-qualified external-image controls; assert exact call counts for the genuine stages so false positives cannot hide beside positive edges. @@ -918,6 +919,7 @@ dotnet test --filter "FullyQualifiedName~GitHelperTests" 複数の reference name が同じ kind と完全一致 source context を共有する場合は `AssertReferencesContainInContext(...)` を使い、context がより詳細な edge contract の一部にすぎない場合は直接 predicate を維持します。 1つの reference kind に対する否定チェックには `AssertReferencesDoNotContain(...)` を使い、container、context、line など他の metadata に依存する除外は直接 predicate を維持します。 `ReferenceExtractorTests.ExtractSymbolsAndReferences(...)` は symbol 抽出から reference 抽出までの共通 setup を所有します。fixture が特殊な path や workspace symbol setup を必要としない場合は 2 つの extractor 呼び出しを繰り返さずこの helper を使い、reference だけを検証するテストでは未使用の `symbols` local を残さず symbol 側を `_` で捨ててください。 + Crystal、Groovy、Tcl、Prolog、`ambiguous_pl` の graph fixture では、import、括弧付き call、同一ファイルに限定した保守的な command / predicate call、caller container、keyword の false-positive control を個別に診断可能な状態で維持し、capability test の symbol / reference / graph 広告を extractor fixture と一致させてください（#4746）。 JavaScript / TypeScript の discriminant-guard coverage は両言語で JavaScript-compatible な fixture を共有し、`DatabaseTests` で `type_tag` が永続化可能な reference kind であることも固定して、extractor-only assertion が database kind catalog のずれを隠せないようにします。 Dockerfile の named-stage reference variant は、通常、小文字、platform flag、comment、hyphen、dot 形式を stage ごとの厳密な call 数で区別できる場合、1つの multi-stage fixture を共有します。外部 base image の除外も negative control として同じ fixture に残します。 Dockerfile の `COPY --from` における ONBUILD と quote 付き stage 形式は、tag および digest 付き外部 image の control と fixture を共有します。正しい stage の call 数を厳密に検証し、正例 edge に紛れた false positive を見逃さないようにします。 diff --git a/USER_GUIDE.md b/USER_GUIDE.md index 92717f76f..c53636fbf 100644 --- a/USER_GUIDE.md +++ b/USER_GUIDE.md @@ -2375,7 +2375,8 @@ All indexed languages are searchable through FTS5. Rows with **Symbols = yes** a - SQL: query-time `--lang tsql` is accepted as a SQL alias, and T-SQL aggregate, assembly, and XML schema collection declarations are searchable. - R: function assignments, S4/R6 class declarations, validity/generic/method declarations, inherit vectors, public/private/active methods, and `library` / `require` imports are indexed. - Functional symbol-only languages: Clojure, Erlang, OCaml, and Raku expose conservative declarations as symbols. References and graph queries are not advertised for these languages yet. -- Dynamic symbol-only languages: Crystal, Groovy, Julia, and Tcl expose conservative declarations as symbols. References and graph queries are not advertised for these languages yet. +- Dynamic symbol-only languages: Julia exposes conservative declarations as symbols. References and graph queries are not advertised for Julia yet. +- Dynamic/declarative graph languages: Crystal, Groovy, Tcl, and Prolog expose conservative declarations, imports, and call relationships. Parenthesized calls use the shared extractor; command-style calls are limited to callables declared in the same file, and Tcl proc / Prolog predicate bodies preserve caller containers. - Systems symbol-only languages: Ada, D, and Nim expose conservative declarations as symbols. References and graph queries are not advertised for these languages yet. - Markdown, JSON/YAML, and CSS: Markdown heading and local-anchor symbols are indexed; JSON/YAML configuration keys are indexed as structural key paths; CSS variables, placeholders, and `@extend` references are indexed. - Dockerfile, Assembly, Common Lisp, and Racket: `ARG` build args, labels/PROC/MACRO blocks, package/module forms, definitions, classes/structs, requires, and provides are surfaced as symbols where applicable. @@ -2389,7 +2390,7 @@ All indexed languages are searchable through FTS5. Rows with **Symbols = yes** a - Dependency manifests and lockfiles: use `--lang dependency_manifest` or `--lang dependency_lock` for dependency/security audits. `Directory.Packages.props`, `packages.config`, `requirements.txt`, `pyproject.toml`, `packages.lock.json`, and npm `package-lock.json` / `npm-shrinkwrap.json` expose package symbols and `dependency` references with version, scope, and direct/transitive metadata where the format provides it. - Solution and application manifests: `.sln` files expose project entries as symbols and project path references; `.manifest` files expose assembly identity, requested execution level, supported OS, and long-path settings as symbols. - Shebang scripts: recognized first-line shebangs index extensionless and unknown-extension files for shell (`sh`, `bash`, `zsh`, `fish`, `dash`, `ksh`, `ash`), Python, Ruby, Perl, Tcl (`tclsh`, `wish`), Node.js, PHP, Lua, and PowerShell. Explicit language-map overrides remain authoritative; for ambiguous `.t` files, a recognized shebang overrides the Perl default, while strong known extensions continue to win conflicts. -- Ambiguous `.m` / `.pl`: recognized shebangs win first, then bounded content checks use only strong Objective-C/MATLAB or Perl/Prolog markers, followed by conservative project markers. Scoped updates that add, change, or remove one of those markers automatically rescan the workspace so unchanged ambiguous files do not retain stale classifications. Weak or conflicting evidence remains searchable under `ambiguous_m` or `ambiguous_pl` instead of being assigned unconditionally. MATLAB and Prolog expose declaration symbols but no reference or graph support yet. +- Ambiguous `.m` / `.pl`: recognized shebangs win first, then bounded content checks use only strong Objective-C/MATLAB or Perl/Prolog markers, followed by conservative project markers. Scoped updates that add, change, or remove one of those markers automatically rescan the workspace so unchanged ambiguous files do not retain stale classifications. Weak or conflicting evidence remains under `ambiguous_m` or `ambiguous_pl` instead of being assigned unconditionally. MATLAB exposes declaration symbols without reference or graph support. Prolog and `ambiguous_pl` expose conservative symbols, references, and graph queries after classification; `ambiguous_pl` uses a safe union of Perl and Prolog constructs without overriding the content-based language decision. ### Language extraction matrix @@ -5521,7 +5522,8 @@ indexing はファイル単位の SQLite transaction を commit します。長 - SQL: クエリ時の `--lang tsql` は SQL の別名です。T-SQL の aggregate、assembly、XML schema collection 宣言も検索対象です。 - R: 関数代入、S4/R6 class 宣言、validity/generic/method 宣言、inherit vector、public/private/active method、`library` / `require` import を索引します。 - 関数型言語のシンボル専用対応: Clojure、Erlang、OCaml、Raku は保守的な宣言をシンボルとして公開します。これらの言語では references と graph queries はまだ対応として広告しません。 -- 動的言語のシンボル専用対応: Crystal、Groovy、Julia、Tcl は保守的な宣言をシンボルとして公開します。これらの言語では references と graph queries はまだ対応として広告しません。 +- 動的言語のシンボル専用対応: Julia は保守的な宣言をシンボルとして公開します。Julia の references と graph queries はまだ対応として広告しません。 +- 動的・宣言型言語の graph 対応: Crystal、Groovy、Tcl、Prolog は保守的な宣言、import、call relationship を公開します。括弧付き call は共通 extractor を使い、command-style call は同一ファイルで宣言済みの callable に限定し、Tcl proc / Prolog predicate の本体では caller container を保持します。 - システム系言語のシンボル専用対応: Ada、D、Nim は保守的な宣言をシンボルとして公開します。これらの言語では references と graph queries はまだ対応として広告しません。 - Markdown、JSON/YAML、CSS: Markdown の heading / local anchor、JSON/YAML の configuration key path、CSS の variable、placeholder、`@extend` をシンボルとして扱います。 - Dockerfile、Assembly、Common Lisp、Racket: `ARG` build arg、label、PROC/MACRO、package/module form、definition、class/struct、require/provide を必要に応じて表面化します。 @@ -5535,7 +5537,7 @@ indexing はファイル単位の SQLite transaction を commit します。長 - Dependency manifest / lockfile: dependency / security audit では `--lang dependency_manifest` または `--lang dependency_lock` を使います。`Directory.Packages.props`、`packages.config`、`requirements.txt`、`pyproject.toml`、`packages.lock.json`、npm の `package-lock.json` / `npm-shrinkwrap.json` は、format が提供する範囲で version、scope、direct/transitive metadata を持つ package symbol と `dependency` reference を公開します。 - ソリューションとアプリケーションマニフェスト: `.sln` は project entry をシンボルとして公開し、project path を参照として記録します。`.manifest` は assembly identity、requested execution level、supported OS、long-path 設定をシンボルとして公開します。 - shebang script: 先頭行の shebang を認識できる拡張子なし/未知拡張子ファイルは、shell (`sh`, `bash`, `zsh`, `fish`, `dash`, `ksh`, `ash`)、Python、Ruby、Perl、Tcl (`tclsh`, `wish`)、Node.js、PHP、Lua、PowerShell として index 対象です。明示的な language-map override は常に優先し、曖昧な `.t` では認識済み shebang が Perl の既定値を上書きします。一方、曖昧でない既知拡張子は競合する shebang より優先されます。 -- 曖昧な `.m` / `.pl`: 認識済み shebang を最優先し、その後は bounded content check で Objective-C/MATLAB または Perl/Prolog の強い marker だけを使い、最後に保守的な project marker を確認します。これらの marker を追加・変更・削除する scoped update は workspace を自動的に再 scan し、未変更の曖昧ファイルに古い分類を残しません。弱い証拠や競合する証拠は無条件に言語を割り当てず、`ambiguous_m` / `ambiguous_pl` として全文検索可能なまま残します。MATLAB と Prolog は宣言 symbol を公開しますが、reference / graph はまだ未対応です。 +- 曖昧な `.m` / `.pl`: 認識済み shebang を最優先し、その後は bounded content check で Objective-C/MATLAB または Perl/Prolog の強い marker だけを使い、最後に保守的な project marker を確認します。これらの marker を追加・変更・削除する scoped update は workspace を自動的に再 scan し、未変更の曖昧ファイルに古い分類を残しません。弱い証拠や競合する証拠は無条件に言語を割り当てず、`ambiguous_m` / `ambiguous_pl` のまま保持します。MATLAB は宣言 symbol を公開しますが reference / graph は未対応です。Prolog と `ambiguous_pl` は分類後に保守的な symbol、reference、graph query を公開し、`ambiguous_pl` は content-based の言語判定を上書きせず Perl / Prolog 構文の安全な和集合を使います。 ### 言語別 extraction matrix diff --git a/changelog.d/unreleased/4746.added.md b/changelog.d/unreleased/4746.added.md new file mode 100644 index 000000000..74ba4cb55 --- /dev/null +++ b/changelog.d/unreleased/4746.added.md @@ -0,0 +1,27 @@ +--- +category: added +issues: + - 4746 +affected: + - src/CodeIndex/Indexer/Symbols/SymbolExtractor.cs + - src/CodeIndex/Indexer/Symbols/SymbolExtractor.ExtractCore.cs + - src/CodeIndex/Indexer/References/ReferenceExtractor.cs + - src/CodeIndex/Indexer/References/ReferenceExtractor.State.cs + - src/CodeIndex/Indexer/References/ReferenceExtractor.CoreExtraction.cs + - src/CodeIndex/Indexer/References/Languages/DynamicDeclarativeReferenceExtractor.cs + - tests/CodeIndex.Tests/SymbolExtractorTests.cs + - tests/CodeIndex.Tests/ReferenceExtractorTests.cs + - tests/CodeIndex.Tests/QueryCommandRunnerTests.cs + - tests/CodeIndex.Tests/golden/status.json + - USER_GUIDE.md + - DEVELOPER_GUIDE.md + - TESTING_GUIDE.md +--- + +## English + +- **Crystal, Groovy, Tcl, Prolog, and ambiguous `.pl` files now support conservative reference graphs (#4746)** — indexing records imports and parenthesized calls, recognizes command-style calls only for callables declared in the same file, preserves Tcl proc and Prolog predicate caller containers, and reports structured symbol/reference/graph capabilities after `.pl` content classification. + +## 日本語 + +- **Crystal、Groovy、Tcl、Prolog、曖昧な `.pl` ファイルが保守的な reference graph に対応しました（#4746）** — import と括弧付き call を記録し、command-style call は同一ファイルで宣言済みの callable だけを認識し、Tcl proc / Prolog predicate の caller container を保持して、`.pl` の content classification 後に構造化された symbol / reference / graph capability を報告します。 diff --git a/src/CodeIndex/Indexer/References/Languages/DynamicDeclarativeReferenceExtractor.cs b/src/CodeIndex/Indexer/References/Languages/DynamicDeclarativeReferenceExtractor.cs new file mode 100644 index 000000000..2f4368864 --- /dev/null +++ b/src/CodeIndex/Indexer/References/Languages/DynamicDeclarativeReferenceExtractor.cs @@ -0,0 +1,372 @@ +using System.Text.RegularExpressions; +using CodeIndex.Models; +using Regex = CodeIndex.Indexer.BoundedRegex; + +namespace CodeIndex.Indexer; + +internal static class DynamicDeclarativeReferenceExtractor +{ + internal sealed class ExtractionState + { + public ExtractionState( + HashSet callableNames, + IReadOnlyDictionary containersByLine) + { + CallableNames = callableNames; + ContainersByLine = containersByLine; + } + + public HashSet CallableNames { get; } + public IReadOnlyDictionary ContainersByLine { get; } + + public SymbolRecord? ResolveContainer(int lineNumber, SymbolRecord? fallback) + => ContainersByLine.TryGetValue(lineNumber, out var container) ? container : fallback; + } + + private static readonly Regex TclProcRegex = new( + @"^\s*proc\s+[A-Za-z_:][\w:.-]*\s+", + RegexOptions.Compiled | RegexOptions.CultureInvariant); + private static readonly Regex PrologHeadRegex = new( + @"^\s*(?[a-z][A-Za-z0-9_]*)\s*(?:\([^.\r\n]*\))?\s*(?::-|-->|\.)", + RegexOptions.Compiled | RegexOptions.CultureInvariant); + private static readonly Regex CrystalRequireRegex = new( + @"^\s*require\s+['""](?[^'""]+)['""]", + RegexOptions.Compiled | RegexOptions.CultureInvariant); + private static readonly Regex GroovyImportRegex = new( + @"^\s*import\s+(?:static\s+)?(?[A-Za-z_]\w*(?:\.[A-Za-z_]\w*)*)(?:\.\*)?", + RegexOptions.Compiled | RegexOptions.CultureInvariant); + private static readonly Regex TclPackageRegex = new( + @"^\s*package\s+require\s+(?:-exact\s+)?(?[A-Za-z_:][\w:.-]*)", + RegexOptions.Compiled | RegexOptions.CultureInvariant); + private static readonly Regex PrologImportRegex = new( + @"^\s*:-\s*use_module\s*\(\s*(?:library\s*\(\s*)?['""]?(?[a-z][A-Za-z0-9_./-]*)", + RegexOptions.Compiled | RegexOptions.CultureInvariant); + private static readonly Regex CrystalGroovyBareCallRegex = new( + @"(?:^|[;=])\s*(?:return\s+)?(?[A-Za-z_]\w*)\b(?!\s*\()", + RegexOptions.Compiled | RegexOptions.CultureInvariant); + private static readonly Regex TclCommandRegex = new( + @"(?:^|[;\[])\s*(?[A-Za-z_:][\w:.-]*)", + RegexOptions.Compiled | RegexOptions.CultureInvariant); + private static readonly Regex PrologBareCallRegex = new( + @"(?:^|:-|-->|[,;])\s*(?:\\\+\s*)?(?[a-z][A-Za-z0-9_]*)\b(?!\s*\()", + RegexOptions.Compiled | RegexOptions.CultureInvariant); + + public static ExtractionState? CreateState( + string language, + IReadOnlyList preparedLines, + IReadOnlyList symbols) + { + if (language is not ("crystal" or "groovy" or "tcl" or "prolog" or "ambiguous_pl")) + return null; + + var callableNames = new HashSet(StringComparer.Ordinal); + foreach (var symbol in symbols) + { + if (symbol.Kind is "function" or "lambda" or "operator") + callableNames.Add(symbol.Name); + } + + var containersByLine = new Dictionary(); + if (language == "tcl") + AddTclContainers(preparedLines, symbols, containersByLine); + else if (language is "prolog" or "ambiguous_pl") + AddPrologContainers(preparedLines, symbols, containersByLine); + + return new ExtractionState(callableNames, containersByLine); + } + + public static void EmitAdditionalReferences( + string language, + string preparedLine, + string originalLine, + ExtractionState state, + List references, + ReferenceDedupeSet seen, + long fileId, + string context, + int lineNumber, + Func resolveContainerForCall, + Action addCallLikeReference) + { + EmitImportReference( + language, + originalLine, + references, + seen, + fileId, + context, + lineNumber, + resolveContainerForCall); + + var callRegex = language switch + { + "crystal" or "groovy" => CrystalGroovyBareCallRegex, + "tcl" => TclCommandRegex, + "prolog" or "ambiguous_pl" => PrologBareCallRegex, + _ => null, + }; + if (callRegex == null) + return; + + foreach (Match match in BoundedRegex.EnumerateMatches(callRegex, preparedLine)) + { + var nameGroup = match.Groups["name"]; + if (!state.CallableNames.Contains(nameGroup.Value)) + continue; + + if (language is "prolog" or "ambiguous_pl" + && state.ContainersByLine.TryGetValue(lineNumber, out var prologContainer) + && !string.Equals(prologContainer.Name, nameGroup.Value, StringComparison.Ordinal)) + { + ReferenceExtractor.AddReference( + references, + seen, + fileId, + nameGroup.Value, + nameGroup.Index, + "call", + context, + lineNumber, + prologContainer, + language); + continue; + } + + addCallLikeReference(nameGroup.Value, nameGroup.Index); + } + } + + private static void EmitImportReference( + string language, + string originalLine, + List references, + ReferenceDedupeSet seen, + long fileId, + string context, + int lineNumber, + Func resolveContainerForCall) + { + var match = language switch + { + "crystal" => CrystalRequireRegex.Match(originalLine), + "groovy" => GroovyImportRegex.Match(originalLine), + "tcl" => TclPackageRegex.Match(originalLine), + "prolog" or "ambiguous_pl" => PrologImportRegex.Match(originalLine), + _ => Match.Empty, + }; + if (!match.Success) + return; + + var nameGroup = match.Groups["name"]; + var name = NormalizeImportTarget(language, nameGroup.Value); + if (name.Length == 0) + return; + + ReferenceExtractor.AddReference( + references, + seen, + fileId, + name, + nameGroup.Index, + "type_reference", + context, + lineNumber, + resolveContainerForCall(nameGroup.Index), + language); + } + + private static string NormalizeImportTarget(string language, string name) + { + var normalized = name.Replace('\\', '/').TrimEnd('/'); + if (language == "groovy") + return normalized[(normalized.LastIndexOf('.') + 1)..]; + if (language is "crystal" or "prolog" or "ambiguous_pl") + { + normalized = normalized[(normalized.LastIndexOf('/') + 1)..]; + var extensionIndex = normalized.LastIndexOf('.'); + if (extensionIndex > 0) + normalized = normalized[..extensionIndex]; + } + return normalized; + } + + private static void AddTclContainers( + IReadOnlyList lines, + IReadOnlyList symbols, + Dictionary containersByLine) + { + foreach (var symbol in symbols) + { + if (symbol.Kind != "function" || symbol.StartLine < 1 || symbol.StartLine > lines.Count) + continue; + + var startLineIndex = symbol.StartLine - 1; + var declarationMatch = TclProcRegex.Match(lines[startLineIndex]); + if (!declarationMatch.Success + || !TryFindTclBodyEnd(lines, startLineIndex, declarationMatch.Index + declarationMatch.Length, out var endLineIndex)) + { + continue; + } + + for (var lineIndex = startLineIndex; lineIndex <= endLineIndex; lineIndex++) + containersByLine.TryAdd(lineIndex + 1, symbol); + } + } + + private static bool TryFindTclBodyEnd( + IReadOnlyList lines, + int startLineIndex, + int searchColumn, + out int endLineIndex) + { + endLineIndex = startLineIndex; + if (!TryFindNextNonWhitespace(lines[startLineIndex], searchColumn, out var argsColumn) + || !TryFindTclWordEnd(lines, startLineIndex, argsColumn, out var argsEndLine, out var argsEndColumn) + || !TryFindNextNonWhitespace(lines[argsEndLine], argsEndColumn + 1, out var bodyColumn) + || lines[argsEndLine][bodyColumn] != '{' + || !TryFindMatchingBrace(lines, argsEndLine, bodyColumn, out endLineIndex, out _)) + { + return false; + } + + return true; + } + + private static bool TryFindTclWordEnd( + IReadOnlyList lines, + int startLine, + int startColumn, + out int endLine, + out int endColumn) + { + var line = lines[startLine]; + if (line[startColumn] == '{') + return TryFindMatchingBrace(lines, startLine, startColumn, out endLine, out endColumn); + + if (line[startColumn] == '"') + { + for (var column = startColumn + 1; column < line.Length; column++) + { + if (line[column] == '\\') + { + column++; + continue; + } + if (line[column] == '"') + { + endLine = startLine; + endColumn = column; + return true; + } + } + + endLine = -1; + endColumn = -1; + return false; + } + + var wordEnd = startColumn; + while (wordEnd + 1 < line.Length && !char.IsWhiteSpace(line[wordEnd + 1])) + wordEnd++; + endLine = startLine; + endColumn = wordEnd; + return true; + } + + private static bool TryFindNextNonWhitespace( + string line, + int startColumn, + out int foundColumn) + { + for (var column = startColumn; column < line.Length; column++) + { + if (!char.IsWhiteSpace(line[column])) + { + foundColumn = column; + return true; + } + } + + foundColumn = -1; + return false; + } + + private static bool TryFindMatchingBrace( + IReadOnlyList lines, + int startLine, + int startColumn, + out int endLine, + out int endColumn) + { + var depth = 0; + for (var lineIndex = startLine; lineIndex < lines.Count; lineIndex++) + { + var line = lines[lineIndex]; + for (var column = lineIndex == startLine ? startColumn : 0; column < line.Length; column++) + { + if (line[column] == '\\') + { + column++; + continue; + } + if (line[column] == '{') + depth++; + else if (line[column] == '}' && --depth == 0) + { + endLine = lineIndex; + endColumn = column; + return true; + } + } + } + + endLine = -1; + endColumn = -1; + return false; + } + + private static void AddPrologContainers( + IReadOnlyList lines, + IReadOnlyList symbols, + Dictionary containersByLine) + { + foreach (var symbol in symbols) + { + if (symbol.Kind != "function" || symbol.StartLine < 1 || symbol.StartLine > lines.Count) + continue; + + var startLineIndex = symbol.StartLine - 1; + var headMatch = PrologHeadRegex.Match(lines[startLineIndex]); + if (!headMatch.Success + || !string.Equals(headMatch.Groups["name"].Value, symbol.Name, StringComparison.Ordinal)) + { + continue; + } + + var endLineIndex = FindPrologClauseEnd(lines, startLineIndex); + for (var lineIndex = startLineIndex; lineIndex <= endLineIndex; lineIndex++) + containersByLine.TryAdd(lineIndex + 1, symbol); + } + } + + private static int FindPrologClauseEnd(IReadOnlyList lines, int startLineIndex) + { + for (var lineIndex = startLineIndex; lineIndex < lines.Count; lineIndex++) + { + var line = lines[lineIndex]; + for (var column = 0; column < line.Length; column++) + { + if (line[column] != '.') + continue; + + var previousIsDigit = column > 0 && char.IsDigit(line[column - 1]); + var nextIsIdentifier = column + 1 < line.Length + && (char.IsLetterOrDigit(line[column + 1]) || line[column + 1] == '_'); + if (!previousIsDigit && !nextIsIdentifier) + return lineIndex; + } + } + + return startLineIndex; + } +} diff --git a/src/CodeIndex/Indexer/References/ReferenceExtractor.CoreExtraction.cs b/src/CodeIndex/Indexer/References/ReferenceExtractor.CoreExtraction.cs index 36d333e68..6e5326823 100644 --- a/src/CodeIndex/Indexer/References/ReferenceExtractor.CoreExtraction.cs +++ b/src/CodeIndex/Indexer/References/ReferenceExtractor.CoreExtraction.cs @@ -189,6 +189,10 @@ internal static List ExtractCore(ReferenceExtractionContext req : default; var shellCallableNames = shellNameSets.CallableNames; var shellGlobalAliasNames = shellNameSets.GlobalAliasNames; + var dynamicDeclarativeState = DynamicDeclarativeReferenceExtractor.CreateState( + language, + preparedLines, + symbols); IReadOnlyList<(int StartLine, int EndLine)> csharpNamespaceScopes = language == "csharp" ? BuildCSharpNamespaceScopes(symbols) : Array.Empty<(int StartLine, int EndLine)>(); @@ -1445,7 +1449,7 @@ static string NormalizeCSharpBclRegexQualifiedName(string value) } } - return container; + return dynamicDeclarativeState?.ResolveContainer(lineNumber, container) ?? container; } SymbolRecord? ResolvePythonDefinitionContainer(int line, string kind) @@ -2588,7 +2592,7 @@ bool TryGetKnownPythonTypeCall(string candidate, out string canonicalName) GetMatchedCallIndices(), AddCallLikeReference); } - else if (language == "perl") + else if (language is "perl" or "ambiguous_pl") { PerlReferenceExtractor.EmitAdditionalReferences( preparedLine, @@ -2602,6 +2606,22 @@ bool TryGetKnownPythonTypeCall(string candidate, out string canonicalName) AddCallLikeReference); } + if (dynamicDeclarativeState != null) + { + DynamicDeclarativeReferenceExtractor.EmitAdditionalReferences( + language, + preparedLine, + originalLine, + dynamicDeclarativeState, + references, + seen, + fileId, + context, + lineNumber, + ResolveContainerForCall, + AddCallLikeReference); + } + if (language == "go") LanguageReferenceExtractionSupport.EmitGoBranchLabelReferences(preparedLine, AddCallLikeReference); diff --git a/src/CodeIndex/Indexer/References/ReferenceExtractor.State.cs b/src/CodeIndex/Indexer/References/ReferenceExtractor.State.cs index e74c5bce4..cb258b1f0 100644 --- a/src/CodeIndex/Indexer/References/ReferenceExtractor.State.cs +++ b/src/CodeIndex/Indexer/References/ReferenceExtractor.State.cs @@ -26,7 +26,8 @@ internal sealed class CSharpWhereConstraintState private static readonly string[] BuiltInLanguages = [ "python", "javascript", "typescript", "csharp", "go", "rust", - "java", "kotlin", "ruby", "perl", "c", "cpp", "php", "swift", + "java", "kotlin", "ruby", "crystal", "groovy", "tcl", "perl", "prolog", "ambiguous_pl", + "c", "cpp", "php", "swift", "dart", "scala", "elixir", "lua", "commonlisp", "racket", "vb", "fsharp", "sql", "cobol", "batch", "assembly", "r", "powershell", "shell", "haskell", "solidity", diff --git a/src/CodeIndex/Indexer/References/ReferenceExtractor.cs b/src/CodeIndex/Indexer/References/ReferenceExtractor.cs index a4edf6619..73edd6e26 100644 --- a/src/CodeIndex/Indexer/References/ReferenceExtractor.cs +++ b/src/CodeIndex/Indexer/References/ReferenceExtractor.cs @@ -270,6 +270,41 @@ private static bool IsFunctionLikeSymbolKind(string kind) "print", "say", "die", "warn", "open", "close", "defined", "exists", "delete", "bless", "ref", "scalar", "wantarray", "eval", "do", }, + ["ambiguous_pl"] = new HashSet(StringComparer.Ordinal) + { + "use", "require", "package", "sub", "my", "our", "local", "state", + "if", "elsif", "unless", "while", "until", "foreach", "for", "given", "when", + "print", "say", "die", "warn", "open", "close", "defined", "exists", "delete", + "bless", "ref", "scalar", "wantarray", "eval", "do", + "module", "use_module", "library", "initialization", "dynamic", "multifile", + "discontiguous", "op", + }, + ["crystal"] = new HashSet(StringComparer.Ordinal) + { + "abstract", "alias", "annotation", "begin", "case", "class", "def", "do", "else", + "elsif", "end", "ensure", "enum", "extend", "for", "fun", "if", "include", "lib", + "macro", "module", "next", "of", "private", "protected", "require", "rescue", + "return", "select", "struct", "then", "unless", "until", "when", "while", "with", "yield", + }, + ["groovy"] = new HashSet(StringComparer.Ordinal) + { + "apply", "as", "assert", "break", "case", "catch", "class", "continue", "def", "do", + "else", "enum", "extends", "finally", "for", "if", "implements", "import", "in", + "instanceof", "interface", "new", "package", "return", "switch", "this", "throw", + "throws", "trait", "try", "while", + }, + ["tcl"] = new HashSet(StringComparer.Ordinal) + { + "append", "array", "break", "catch", "concat", "continue", "dict", "error", "eval", + "expr", "for", "foreach", "global", "if", "incr", "info", "lappend", "lindex", + "list", "namespace", "oo::class", "package", "proc", "rename", "return", "set", + "string", "switch", "unset", "upvar", "uplevel", "variable", "while", + }, + ["prolog"] = new HashSet(StringComparer.Ordinal) + { + "module", "use_module", "library", "initialization", "dynamic", "multifile", + "discontiguous", "op", "true", "fail", "false", "is", "not", + }, // F# contextual keywords / F# 文脈キーワード ["fsharp"] = new HashSet(StringComparer.Ordinal) { @@ -849,7 +884,7 @@ private static bool IsFunctionLikeSymbolKind(string kind) // `impact` に metadata edge が混入する。 private static readonly HashSet AnnotationLanguages = new(StringComparer.Ordinal) { - "java", "kotlin", "scala", "typescript", "javascript", "swift", "gradle", "dart", + "java", "kotlin", "scala", "typescript", "javascript", "swift", "gradle", "groovy", "dart", }; // Kotlin use-site target prefixes for annotations (e.g. `@field:Deprecated("msg")`, diff --git a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.ExtractCore.cs b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.ExtractCore.cs index a9776dd88..58eca7dc6 100644 --- a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.ExtractCore.cs +++ b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.ExtractCore.cs @@ -156,6 +156,9 @@ string[] GetJavaScriptTypeScriptSanitizedLines() => var shellScannerLines = lang == "shell" ? MaskShellHeredocLines(lines) : null; + var prologClauseContinuationLines = lang is "prolog" or "ambiguous_pl" + ? BuildPrologClauseContinuationLines(structuralLines) + : null; var powershellEnumBodyLines = lang == "powershell" ? FindPowerShellEnumBodyLines(structuralLines) : null; @@ -399,6 +402,8 @@ bool[] GetCssQualifiedRuleAncestors() => CSharpPropertyMatchCandidate? csharpPropertyCandidateForLine = null; foreach (var pattern in patterns) { + if (prologClauseContinuationLines?[i] == true && pattern.Kind == "function") + continue; if (lang == "csharp" && ReferenceEquals(pattern.Regex, CSharpEnumMemberRegex)) continue; if (lang == "powershell" @@ -2152,6 +2157,49 @@ bool[] GetCssQualifiedRuleAncestors() => return symbols; } + private static readonly Regex PrologOpenClauseRegex = new( + @"^\s*(?:(?:[a-z][A-Za-z0-9_]*\s*(?:\([^.\r\n]*\))?\s*(?::-|-->))|:-)", + RegexOptions.Compiled | RegexOptions.CultureInvariant); + + private static bool[] BuildPrologClauseContinuationLines(IReadOnlyList structuralLines) + { + var continuationLines = new bool[structuralLines.Count]; + var clauseOpen = false; + for (var lineIndex = 0; lineIndex < structuralLines.Count; lineIndex++) + { + continuationLines[lineIndex] = clauseOpen; + var line = structuralLines[lineIndex]; + if (clauseOpen) + { + if (HasPrologClauseTerminator(line)) + clauseOpen = false; + continue; + } + + if (PrologOpenClauseRegex.IsMatch(line) && !HasPrologClauseTerminator(line)) + clauseOpen = true; + } + + return continuationLines; + } + + private static bool HasPrologClauseTerminator(string line) + { + for (var column = 0; column < line.Length; column++) + { + if (line[column] != '.') + continue; + + var previousIsDigit = column > 0 && char.IsDigit(line[column - 1]); + var nextIsIdentifier = column + 1 < line.Length + && (char.IsLetterOrDigit(line[column + 1]) || line[column + 1] == '_'); + if (!previousIsDigit && !nextIsIdentifier) + return true; + } + + return false; + } + private const int CSharpFieldInitializerSignatureLimit = 1024; private static string BoundCSharpFieldInitializerSignature(string signature) diff --git a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.cs b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.cs index fde1e9264..2563900ba 100644 --- a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.cs +++ b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.cs @@ -1713,6 +1713,24 @@ private enum JavaScriptTypeScriptFunctionHeaderConsumeResult new("import", new Regex(@"^\s*:-\s*use_module\s*\(\s*(?[a-z][A-Za-z0-9_]*(?:\([^)]*\))?)", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None), new("function", new Regex(@"^\s*(?[a-z][A-Za-z0-9_]*)\s*(?:\([^\r\n.]*\))?\s*(?::-|-->|\.)", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None), ], + ["ambiguous_pl"] = + [ + // Keep ambiguous .pl files structured without choosing Perl or Prolog prematurely. + // .pl の判定が曖昧でも Perl / Prolog のどちらかへ早計に固定せず、構造を保持する。 + new("namespace", new Regex(@"^\s*package\s+(?" + PerlQualifiedIdentifierPattern + @")\b(?:\s+v?[\d._]+)?\s*\{", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.Brace), + new("namespace", new Regex(@"^\s*package\s+(?" + PerlQualifiedIdentifierPattern + @")\b(?:\s+v?[\d._]+)?\s*;", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None), + new("class", new Regex(@"^\s*class\s+(?" + PerlQualifiedIdentifierPattern + @")\b", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.Brace), + new("interface", new Regex(@"^\s*role\s+(?" + PerlQualifiedIdentifierPattern + @")\b", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.Brace), + new("function", new Regex(@"^\s*use\s+constant\s+(?" + PerlIdentifierPattern + @")\b", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None), + new("import", new Regex(@"^\s*use\s+(?" + PerlQualifiedIdentifierPattern + @")\b", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None), + new("import", new Regex(@"^\s*require\s+(?" + PerlQualifiedIdentifierPattern + @")\b", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None), + new("property", new Regex(@"^\s*our\s+[$@%](?" + PerlIdentifierPattern + @")\b", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None), + new("function", new Regex(@"^\s*(?:(?:my|state)\s+)?sub\s+(?" + PerlQualifiedIdentifierPattern + @")\b(?:\s*:[^{;]+)?", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.Brace), + new("function", new Regex(@"^\s*(?:method|fun)\s+(?" + PerlIdentifierPattern + @")\b(?:\s*:[^{;]+)?", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.Brace), + new("namespace", new Regex(@"^\s*:-\s*module\s*\(\s*(?[a-z][A-Za-z0-9_]*)", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None), + new("import", new Regex(@"^\s*:-\s*use_module\s*\(\s*(?[a-z][A-Za-z0-9_]*(?:\([^)]*\))?)", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None), + new("function", new Regex(@"^\s*(?[a-z][A-Za-z0-9_]*)\s*(?:\([^\r\n.]*\))?\s*(?::-|-->|\.)", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None), + ], ["c"] = [ new("function", new Regex(CFunctionStartBlacklistPattern + CFunctionReturnTypePattern + CFunctionNameBlacklistPattern + @"(?\w+)\s*\(", RegexOptions.Compiled), BodyStyle.Brace, ReturnTypeGroup: "returnType"), diff --git a/tests/CodeIndex.Tests/QueryCommandRunnerTests.cs b/tests/CodeIndex.Tests/QueryCommandRunnerTests.cs index 31ba1f5d3..3bf67d2c8 100644 --- a/tests/CodeIndex.Tests/QueryCommandRunnerTests.cs +++ b/tests/CodeIndex.Tests/QueryCommandRunnerTests.cs @@ -3145,7 +3145,7 @@ public void RunLanguages_JsonReportsLanguageMapOverrideProvenance_Issue4617() } [Fact] - public void RunLanguages_JsonReportsMatlabAndPrologSymbolOnlyCapabilities_Issue4612() + public void RunLanguages_JsonReportsMatlabSymbolOnlyAndPrologGraphCapabilities_Issues4612And4746() { var (exitCode, stdout, stderr) = CaptureConsole(() => QueryCommandRunner.RunLanguages(["--json"], _jsonOptions)); @@ -3155,12 +3155,15 @@ public void RunLanguages_JsonReportsMatlabAndPrologSymbolOnlyCapabilities_Issue4 using var document = ParseJsonOutput(stdout); var languages = document.RootElement.GetProperty("languages").EnumerateArray() .ToDictionary(entry => entry.GetProperty("lang").GetString()!, entry => entry); - foreach (var language in new[] { "matlab", "prolog" }) - { - Assert.True(languages[language].GetProperty("symbol_extraction").GetBoolean()); - Assert.False(languages[language].GetProperty("reference_extraction").GetBoolean()); - Assert.False(languages[language].GetProperty("graph_queries").GetBoolean()); - } + Assert.True(languages["matlab"].GetProperty("symbol_extraction").GetBoolean()); + Assert.False(languages["matlab"].GetProperty("reference_extraction").GetBoolean()); + Assert.False(languages["matlab"].GetProperty("graph_queries").GetBoolean()); + Assert.True(languages["prolog"].GetProperty("symbol_extraction").GetBoolean()); + Assert.True(languages["prolog"].GetProperty("reference_extraction").GetBoolean()); + Assert.True(languages["prolog"].GetProperty("graph_queries").GetBoolean()); + Assert.True(languages["ambiguous_pl"].GetProperty("symbol_extraction").GetBoolean()); + Assert.True(languages["ambiguous_pl"].GetProperty("reference_extraction").GetBoolean()); + Assert.True(languages["ambiguous_pl"].GetProperty("graph_queries").GetBoolean()); Assert.Contains(".m", languages["ambiguous_m"].GetProperty("extensions").EnumerateArray().Select(value => value.GetString())); Assert.Contains(".pl", languages["ambiguous_pl"].GetProperty("extensions").EnumerateArray().Select(value => value.GetString())); @@ -3296,7 +3299,7 @@ public void RunLanguages_Json_SearchOnlyBucketsAdvertiseZeroSymbolAndGraphSuppor var languages = document.RootElement.GetProperty("languages").EnumerateArray() .ToDictionary(entry => entry.GetProperty("lang").GetString()!, entry => entry); - foreach (var symbolOnly in new[] { "ada", "clojure", "crystal", "d", "erlang", "groovy", "julia", "nim", "ocaml", "raku", "tcl" }) + foreach (var symbolOnly in new[] { "ada", "clojure", "d", "erlang", "julia", "nim", "ocaml", "raku" }) { Assert.True(languages.ContainsKey(symbolOnly), $"expected '{symbolOnly}' to be listed"); var entry = languages[symbolOnly]; @@ -3333,6 +3336,18 @@ public void RunLanguages_Json_SearchOnlyBucketsAdvertiseZeroSymbolAndGraphSuppor Assert.Empty(languages[stylesheetPreprocessor].GetProperty("capability_gaps").EnumerateArray()); } + foreach (var dynamicGraph in new[] { "crystal", "groovy", "tcl", "prolog", "ambiguous_pl" }) + { + Assert.True(languages.ContainsKey(dynamicGraph), $"expected '{dynamicGraph}' to be listed"); + Assert.True(languages[dynamicGraph].GetProperty("symbol_extraction").GetBoolean(), + $"{dynamicGraph} must advertise symbol_extraction=true"); + Assert.True(languages[dynamicGraph].GetProperty("reference_extraction").GetBoolean(), + $"{dynamicGraph} must advertise reference_extraction=true"); + Assert.True(languages[dynamicGraph].GetProperty("graph_queries").GetBoolean(), + $"{dynamicGraph} must advertise graph_queries=true"); + Assert.Empty(languages[dynamicGraph].GetProperty("capability_gaps").EnumerateArray()); + } + Assert.True(languages["xml"].GetProperty("symbol_extraction").GetBoolean(), "xml must advertise symbol_extraction=true for XAML/AXAML files"); Assert.True(languages["xml"].GetProperty("reference_extraction").GetBoolean(), diff --git a/tests/CodeIndex.Tests/ReferenceExtractorTests.cs b/tests/CodeIndex.Tests/ReferenceExtractorTests.cs index 6a165df9b..e4ea31959 100644 --- a/tests/CodeIndex.Tests/ReferenceExtractorTests.cs +++ b/tests/CodeIndex.Tests/ReferenceExtractorTests.cs @@ -1050,6 +1050,137 @@ var zero T && reference.ContainerName == "Get"); } + [Fact] + public void Extract_Crystal_IndexesImportsParenthesizedAndCommandCalls_Issue4746() + { + const string content = """ + require "./support" + + def helper(value) + value + end + + def run(value) + helper value + helper(value) + end + """; + + var (symbols, references) = ExtractSymbolsAndReferences("crystal", content); + + Assert.Contains(symbols, symbol => symbol.Kind == "function" && symbol.Name == "run"); + AssertReferencesContain(references, "type_reference", null, "support"); + AssertReferencesContain(references, "call", "run", "helper"); + Assert.DoesNotContain(references, reference => + reference.ReferenceKind == "call" + && reference.SymbolName is "require" or "def"); + } + + [Fact] + public void Extract_Groovy_IndexesImportsParenthesizedAndCommandCalls_Issue4746() + { + const string content = """ + import demo.Support + + class Runner { + static def helper(value) { + value + } + + def run(value) { + helper value + helper(value) + } + } + """; + + var (symbols, references) = ExtractSymbolsAndReferences("groovy", content); + + Assert.Contains(symbols, symbol => symbol.Kind == "function" && symbol.Name == "run"); + AssertReferencesContain(references, "type_reference", null, "Support"); + AssertReferencesContain(references, "call", "run", "helper"); + Assert.DoesNotContain(references, reference => + reference.ReferenceKind == "call" + && reference.SymbolName is "import" or "def"); + } + + [Fact] + public void Extract_Tcl_IndexesPackagesAndProcCommandCallsWithCaller_Issue4746() + { + const string content = """ + package require json + + proc helper {value} { + return $value + } + + proc run {value} { + helper $value + set result [helper $value] + } + + proc variadic args { + helper + } + """; + + var (symbols, references) = ExtractSymbolsAndReferences("tcl", content); + + Assert.Contains(symbols, symbol => symbol.Kind == "function" && symbol.Name == "run"); + AssertReferencesContain(references, "type_reference", null, "json"); + AssertReferencesContain(references, "call", "run", "helper"); + AssertReferencesContain(references, "call", "variadic", "helper"); + Assert.DoesNotContain(references, reference => + reference.ReferenceKind == "call" + && reference.SymbolName is "package" or "proc" or "set"); + } + + [Fact] + public void Extract_Prolog_IndexesModulesImportsAndPredicateCallsWithCaller_Issue4746() + { + const string content = """ + :- module(family, [ancestor/2]). + :- use_module(library(lists)). + parent(alice, bob). + ready. + ancestor(X, Y) :- + parent(X, Y), + ready. + """; + + var (symbols, references) = ExtractSymbolsAndReferences("prolog", content); + + Assert.Contains(symbols, symbol => symbol.Kind == "function" && symbol.Name == "ancestor"); + Assert.Single(symbols, symbol => symbol.Kind == "function" && symbol.Name == "parent"); + Assert.Single(symbols, symbol => symbol.Kind == "function" && symbol.Name == "ready"); + AssertReferencesContain(references, "type_reference", null, "lists"); + AssertReferencesContain(references, "call", "ancestor", "parent", "ready"); + Assert.DoesNotContain(references, reference => + reference.ReferenceKind == "call" + && reference.SymbolName is "module" or "use_module" or "ancestor"); + } + + [Fact] + public void Extract_AmbiguousPl_IndexesPerlAndPrologReferencesWithoutReclassification_Issue4746() + { + const string content = """ + package Hybrid; + use Shared::Tools; + sub perl_helper { return 1; } + :- module(hybrid, [prolog_helper/0]). + prolog_helper :- perl_helper(). + """; + + var (symbols, references) = ExtractSymbolsAndReferences("ambiguous_pl", content); + + Assert.Contains(symbols, symbol => symbol.Kind == "namespace" && symbol.Name == "Hybrid"); + Assert.Contains(symbols, symbol => symbol.Kind == "namespace" && symbol.Name == "hybrid"); + Assert.Contains(references, reference => + reference.SymbolName == "Shared::Tools" + && reference.ReferenceKind == "reference"); + AssertReferencesContain(references, "call", "prolog_helper", "perl_helper"); + } + [Fact] public void TryGetExtractor_RegisteredLanguage_ReturnsAddressableExtractor() { diff --git a/tests/CodeIndex.Tests/SymbolExtractorTests.cs b/tests/CodeIndex.Tests/SymbolExtractorTests.cs index 4054e6a9e..03ce42905 100644 --- a/tests/CodeIndex.Tests/SymbolExtractorTests.cs +++ b/tests/CodeIndex.Tests/SymbolExtractorTests.cs @@ -67,6 +67,25 @@ public void Extract_Prolog_IndexesModuleImportsAndPredicates_Issue4612() AssertSymbolsContain(symbols, "function", "parent", "ancestor"); } + [Fact] + public void Extract_AmbiguousPl_PreservesPerlAndPrologStructure_Issue4746() + { + const string content = """ + package Hybrid; + use Shared::Tools; + sub perl_helper { return 1; } + :- module(hybrid, [prolog_helper/0]). + :- use_module(library(lists)). + prolog_helper :- perl_helper(). + """; + + var symbols = SymbolExtractor.Extract(1, "ambiguous_pl", content); + + AssertSymbolsContain(symbols, "namespace", "Hybrid", "hybrid"); + AssertSymbolsContain(symbols, "import", "Shared::Tools", "library(lists)"); + AssertSymbolsContain(symbols, "function", "perl_helper", "prolog_helper"); + } + [Fact] public void Extract_CancelledToken_ThrowsBeforeWork() { @@ -1301,6 +1320,7 @@ public void Extract_AllPatternLanguages_IsDeterministicUnderParallelCalls() var samples = new Dictionary(StringComparer.Ordinal) { ["ada"] = "with Ada.Text_IO;\npackage Demo is\nprocedure Run;\nend Demo;\n", + ["ambiguous_pl"] = "package Demo;\nsub run { return 1; }\nrun(Result) :- Result is 1.\n", ["assembly"] = "Start:\n call Target\nTarget:\n ret\n", ["batch"] = ":run\necho ok\n", ["c"] = "int answer(void) { return 42; }\n", diff --git a/tests/CodeIndex.Tests/golden/status.json b/tests/CodeIndex.Tests/golden/status.json index 3129c1975..7170a621b 100644 --- a/tests/CodeIndex.Tests/golden/status.json +++ b/tests/CodeIndex.Tests/golden/status.json @@ -44,6 +44,7 @@ } }, "graph_supported_languages": [ + "ambiguous_pl", "assembly", "batch", "blazor", @@ -52,6 +53,7 @@ "cobol", "commonlisp", "cpp", + "crystal", "csharp", "cshtml", "css", @@ -65,6 +67,7 @@ "go", "gradle", "graphql", + "groovy", "haskell", "html", "java", @@ -81,6 +84,7 @@ "perl", "php", "powershell", + "prolog", "protobuf", "python", "r", @@ -98,6 +102,7 @@ "stylus", "svelte", "swift", + "tcl", "terraform", "typescript", "vb", From e1bc74ee056ac8f02180829fefd8bf8123ce3de9 Mon Sep 17 00:00:00 2001 From: Widthdom Date: Fri, 24 Jul 2026 13:26:34 +0900 Subject: [PATCH 02/26] Harden dynamic reference graph extraction (#4746) --- changelog.d/unreleased/4746.added.md | 2 + .../DynamicDeclarativeReferenceExtractor.cs | 328 ++++++++++++++++-- .../ReferenceExtractor.CoreExtraction.cs | 2 +- .../ReferenceExtractor.Preparation.cs | 3 + .../Indexer/Symbols/SymbolExtractor.cs | 10 +- tests/CodeIndex.Tests/DatabaseTests.cs | 28 ++ .../ReferenceExtractorTests.cs | 133 +++++++ tests/CodeIndex.Tests/SymbolExtractorTests.cs | 8 + 8 files changed, 480 insertions(+), 34 deletions(-) diff --git a/changelog.d/unreleased/4746.added.md b/changelog.d/unreleased/4746.added.md index 74ba4cb55..605fc25cc 100644 --- a/changelog.d/unreleased/4746.added.md +++ b/changelog.d/unreleased/4746.added.md @@ -8,7 +8,9 @@ affected: - src/CodeIndex/Indexer/References/ReferenceExtractor.cs - src/CodeIndex/Indexer/References/ReferenceExtractor.State.cs - src/CodeIndex/Indexer/References/ReferenceExtractor.CoreExtraction.cs + - src/CodeIndex/Indexer/References/ReferenceExtractor.Preparation.cs - src/CodeIndex/Indexer/References/Languages/DynamicDeclarativeReferenceExtractor.cs + - tests/CodeIndex.Tests/DatabaseTests.cs - tests/CodeIndex.Tests/SymbolExtractorTests.cs - tests/CodeIndex.Tests/ReferenceExtractorTests.cs - tests/CodeIndex.Tests/QueryCommandRunnerTests.cs diff --git a/src/CodeIndex/Indexer/References/Languages/DynamicDeclarativeReferenceExtractor.cs b/src/CodeIndex/Indexer/References/Languages/DynamicDeclarativeReferenceExtractor.cs index 2f4368864..4b254e63b 100644 --- a/src/CodeIndex/Indexer/References/Languages/DynamicDeclarativeReferenceExtractor.cs +++ b/src/CodeIndex/Indexer/References/Languages/DynamicDeclarativeReferenceExtractor.cs @@ -27,7 +27,7 @@ public ExtractionState( @"^\s*proc\s+[A-Za-z_:][\w:.-]*\s+", RegexOptions.Compiled | RegexOptions.CultureInvariant); private static readonly Regex PrologHeadRegex = new( - @"^\s*(?[a-z][A-Za-z0-9_]*)\s*(?:\([^.\r\n]*\))?\s*(?::-|-->|\.)", + @"^\s*(?[a-z][A-Za-z0-9_]*)\s*(?:\([^\r\n]*\))?\s*(?::-|-->|\.)", RegexOptions.Compiled | RegexOptions.CultureInvariant); private static readonly Regex CrystalRequireRegex = new( @"^\s*require\s+['""](?[^'""]+)['""]", @@ -39,17 +39,191 @@ public ExtractionState( @"^\s*package\s+require\s+(?:-exact\s+)?(?[A-Za-z_:][\w:.-]*)", RegexOptions.Compiled | RegexOptions.CultureInvariant); private static readonly Regex PrologImportRegex = new( - @"^\s*:-\s*use_module\s*\(\s*(?:library\s*\(\s*)?['""]?(?[a-z][A-Za-z0-9_./-]*)", + @"^\s*:-\s*use_module\s*\(\s*(?:library\s*\(\s*)?['""]?(?(?:\.\.?/)*[a-z][A-Za-z0-9_./-]*)", RegexOptions.Compiled | RegexOptions.CultureInvariant); - private static readonly Regex CrystalGroovyBareCallRegex = new( + private static readonly Regex CrystalBareCallRegex = new( + @"(?:^|[;=])\s*(?:return\s+)?(?[A-Za-z_]\w*[?!]?)(?![\w?!])\s*(?!\()", + RegexOptions.Compiled | RegexOptions.CultureInvariant); + private static readonly Regex CrystalSuffixedParenthesizedCallRegex = new( + @"(?[A-Za-z_]\w*[?!])\s*\(", + RegexOptions.Compiled | RegexOptions.CultureInvariant); + private static readonly Regex GroovyBareCallRegex = new( @"(?:^|[;=])\s*(?:return\s+)?(?[A-Za-z_]\w*)\b(?!\s*\()", RegexOptions.Compiled | RegexOptions.CultureInvariant); private static readonly Regex TclCommandRegex = new( - @"(?:^|[;\[])\s*(?[A-Za-z_:][\w:.-]*)", + @"(?:^|[;\[{}])\s*(?[A-Za-z_:][\w:.-]*)", RegexOptions.Compiled | RegexOptions.CultureInvariant); private static readonly Regex PrologBareCallRegex = new( @"(?:^|:-|-->|[,;])\s*(?:\\\+\s*)?(?[a-z][A-Za-z0-9_]*)\b(?!\s*\()", RegexOptions.Compiled | RegexOptions.CultureInvariant); + private static readonly Regex CrystalHeredocOpenerRegex = new( + @"<<-\s*['""]?(?[A-Za-z_]\w*)['""]?", + RegexOptions.Compiled | RegexOptions.CultureInvariant); + + private readonly record struct TclContainerScope( + SymbolRecord Symbol, + int StartLine, + int EndLine); + + private readonly record struct TclBraceEnd( + int Line, + int Column); + + public static string[] MaskNonCodeLines(string language, IReadOnlyList lines) + { + if (language is not ("crystal" or "groovy" or "tcl" or "prolog" or "ambiguous_pl")) + return lines as string[] ?? lines.ToArray(); + + var result = new string[lines.Count]; + var insideBlockComment = false; + char groovyTripleQuote = '\0'; + string? crystalHeredocDelimiter = null; + var insideAmbiguousPerlPod = false; + + for (var lineIndex = 0; lineIndex < lines.Count; lineIndex++) + { + var line = lines[lineIndex]; + if (crystalHeredocDelimiter != null) + { + result[lineIndex] = new string(' ', line.Length); + if (string.Equals(line.Trim(), crystalHeredocDelimiter, StringComparison.Ordinal)) + crystalHeredocDelimiter = null; + continue; + } + + if (language == "ambiguous_pl") + { + var trimmed = line.AsSpan().TrimStart(); + if (insideAmbiguousPerlPod) + { + result[lineIndex] = new string(' ', line.Length); + if (trimmed.StartsWith("=cut", StringComparison.Ordinal)) + insideAmbiguousPerlPod = false; + continue; + } + + if (trimmed.Length > 1 + && trimmed[0] == '=' + && char.IsLetter(trimmed[1])) + { + result[lineIndex] = new string(' ', line.Length); + insideAmbiguousPerlPod = !trimmed.StartsWith("=cut", StringComparison.Ordinal); + continue; + } + } + + var buffer = line.ToCharArray(); + for (var column = 0; column < line.Length;) + { + if (insideBlockComment) + { + buffer[column] = ' '; + if (column + 1 < line.Length && line[column] == '*' && line[column + 1] == '/') + { + buffer[column + 1] = ' '; + column += 2; + insideBlockComment = false; + } + else + { + column++; + } + continue; + } + + if (groovyTripleQuote != '\0') + { + buffer[column] = ' '; + if (column + 2 < line.Length + && line[column] == groovyTripleQuote + && line[column + 1] == groovyTripleQuote + && line[column + 2] == groovyTripleQuote) + { + buffer[column + 1] = ' '; + buffer[column + 2] = ' '; + column += 3; + groovyTripleQuote = '\0'; + } + else + { + column++; + } + continue; + } + + var ch = line[column]; + if (language == "groovy" + && ch is '\'' or '"' + && column + 2 < line.Length + && line[column + 1] == ch + && line[column + 2] == ch) + { + buffer[column] = ' '; + buffer[column + 1] = ' '; + buffer[column + 2] = ' '; + groovyTripleQuote = ch; + column += 3; + continue; + } + + if (ch is '\'' or '"' or '`') + { + column = SkipQuotedToken(line, column, ch); + continue; + } + + if (language == "crystal" + && column + 2 < line.Length + && line[column] == '<' + && line[column + 1] == '<' + && line[column + 2] == '-') + { + var heredocMatch = CrystalHeredocOpenerRegex.Match(line, column); + if (heredocMatch.Success && heredocMatch.Index == column) + crystalHeredocDelimiter = heredocMatch.Groups["delimiter"].Value; + } + + if ((language is "groovy" or "prolog" or "ambiguous_pl") + && column + 1 < line.Length + && line[column] == '/' + && line[column + 1] == '*') + { + buffer[column] = ' '; + buffer[column + 1] = ' '; + column += 2; + insideBlockComment = true; + continue; + } + + if (language == "groovy" + && column + 1 < line.Length + && line[column] == '/' + && line[column + 1] == '/') + { + FillWithSpaces(buffer, column); + break; + } + + if ((language is "crystal" or "tcl" or "ambiguous_pl") && ch == '#') + { + FillWithSpaces(buffer, column); + break; + } + + if ((language is "prolog" or "ambiguous_pl") && ch == '%') + { + FillWithSpaces(buffer, column); + break; + } + + column++; + } + + result[lineIndex] = new string(buffer); + } + + return result; + } public static ExtractionState? CreateState( string language, @@ -78,7 +252,7 @@ public ExtractionState( public static void EmitAdditionalReferences( string language, string preparedLine, - string originalLine, + string structuralLine, ExtractionState state, List references, ReferenceDedupeSet seen, @@ -90,7 +264,7 @@ public static void EmitAdditionalReferences( { EmitImportReference( language, - originalLine, + structuralLine, references, seen, fileId, @@ -100,7 +274,8 @@ public static void EmitAdditionalReferences( var callRegex = language switch { - "crystal" or "groovy" => CrystalGroovyBareCallRegex, + "crystal" => CrystalBareCallRegex, + "groovy" => GroovyBareCallRegex, "tcl" => TclCommandRegex, "prolog" or "ambiguous_pl" => PrologBareCallRegex, _ => null, @@ -108,6 +283,16 @@ public static void EmitAdditionalReferences( if (callRegex == null) return; + if (language == "crystal") + { + foreach (Match match in BoundedRegex.EnumerateMatches(CrystalSuffixedParenthesizedCallRegex, preparedLine)) + { + var nameGroup = match.Groups["name"]; + if (state.CallableNames.Contains(nameGroup.Value)) + addCallLikeReference(nameGroup.Value, nameGroup.Index); + } + } + foreach (Match match in BoundedRegex.EnumerateMatches(callRegex, preparedLine)) { var nameGroup = match.Groups["name"]; @@ -136,6 +321,37 @@ public static void EmitAdditionalReferences( } } + private static int SkipQuotedToken(string line, int startColumn, char delimiter) + { + for (var column = startColumn + 1; column < line.Length; column++) + { + if (line[column] == '\\') + { + column++; + continue; + } + + if (line[column] != delimiter) + continue; + + if (column + 1 < line.Length && line[column + 1] == delimiter) + { + column++; + continue; + } + + return column + 1; + } + + return line.Length; + } + + private static void FillWithSpaces(char[] buffer, int startColumn) + { + for (var column = startColumn; column < buffer.Length; column++) + buffer[column] = ' '; + } + private static void EmitImportReference( string language, string originalLine, @@ -195,6 +411,8 @@ private static void AddTclContainers( IReadOnlyList symbols, Dictionary containersByLine) { + var braceEnds = BuildTclBraceEndPositions(lines); + var scopes = new List(); foreach (var symbol in symbols) { if (symbol.Kind != "function" || symbol.StartLine < 1 || symbol.StartLine > lines.Count) @@ -203,37 +421,77 @@ private static void AddTclContainers( var startLineIndex = symbol.StartLine - 1; var declarationMatch = TclProcRegex.Match(lines[startLineIndex]); if (!declarationMatch.Success - || !TryFindTclBodyEnd(lines, startLineIndex, declarationMatch.Index + declarationMatch.Length, out var endLineIndex)) + || !TryFindTclBodyEnd( + lines, + braceEnds, + startLineIndex, + declarationMatch.Index + declarationMatch.Length, + out var endLineIndex)) { continue; } - for (var lineIndex = startLineIndex; lineIndex <= endLineIndex; lineIndex++) - containersByLine.TryAdd(lineIndex + 1, symbol); + scopes.Add(new TclContainerScope(symbol, symbol.StartLine, endLineIndex + 1)); + } + + scopes.Sort(static (left, right) => + { + var startComparison = left.StartLine.CompareTo(right.StartLine); + return startComparison != 0 + ? startComparison + : right.EndLine.CompareTo(left.EndLine); + }); + + var activeScopes = new Stack(); + var scopeIndex = 0; + for (var lineNumber = 1; lineNumber <= lines.Count; lineNumber++) + { + while (activeScopes.Count > 0 && activeScopes.Peek().EndLine < lineNumber) + activeScopes.Pop(); + + while (scopeIndex < scopes.Count && scopes[scopeIndex].StartLine == lineNumber) + { + var scope = scopes[scopeIndex++]; + while (activeScopes.Count > 0 && activeScopes.Peek().EndLine < lineNumber) + activeScopes.Pop(); + activeScopes.Push(scope); + } + + if (activeScopes.Count > 0) + containersByLine[lineNumber] = activeScopes.Peek().Symbol; } } private static bool TryFindTclBodyEnd( IReadOnlyList lines, + IReadOnlyDictionary braceEnds, int startLineIndex, int searchColumn, out int endLineIndex) { endLineIndex = startLineIndex; if (!TryFindNextNonWhitespace(lines[startLineIndex], searchColumn, out var argsColumn) - || !TryFindTclWordEnd(lines, startLineIndex, argsColumn, out var argsEndLine, out var argsEndColumn) + || !TryFindTclWordEnd( + lines, + braceEnds, + startLineIndex, + argsColumn, + out var argsEndLine, + out var argsEndColumn) || !TryFindNextNonWhitespace(lines[argsEndLine], argsEndColumn + 1, out var bodyColumn) || lines[argsEndLine][bodyColumn] != '{' - || !TryFindMatchingBrace(lines, argsEndLine, bodyColumn, out endLineIndex, out _)) + || !braceEnds.TryGetValue(GetTclPositionKey(argsEndLine, bodyColumn), out var bodyEnd)) { return false; } + endLineIndex = bodyEnd.Line; return true; } private static bool TryFindTclWordEnd( IReadOnlyList lines, + IReadOnlyDictionary braceEnds, int startLine, int startColumn, out int endLine, @@ -241,7 +499,18 @@ private static bool TryFindTclWordEnd( { var line = lines[startLine]; if (line[startColumn] == '{') - return TryFindMatchingBrace(lines, startLine, startColumn, out endLine, out endColumn); + { + if (braceEnds.TryGetValue(GetTclPositionKey(startLine, startColumn), out var braceEnd)) + { + endLine = braceEnd.Line; + endColumn = braceEnd.Column; + return true; + } + + endLine = -1; + endColumn = -1; + return false; + } if (line[startColumn] == '"') { @@ -291,40 +560,39 @@ private static bool TryFindNextNonWhitespace( return false; } - private static bool TryFindMatchingBrace( - IReadOnlyList lines, - int startLine, - int startColumn, - out int endLine, - out int endColumn) + private static Dictionary BuildTclBraceEndPositions(IReadOnlyList lines) { - var depth = 0; - for (var lineIndex = startLine; lineIndex < lines.Count; lineIndex++) + var result = new Dictionary(); + var openings = new Stack<(int Line, int Column)>(); + for (var lineIndex = 0; lineIndex < lines.Count; lineIndex++) { var line = lines[lineIndex]; - for (var column = lineIndex == startLine ? startColumn : 0; column < line.Length; column++) + for (var column = 0; column < line.Length; column++) { if (line[column] == '\\') { column++; continue; } + if (line[column] == '{') - depth++; - else if (line[column] == '}' && --depth == 0) { - endLine = lineIndex; - endColumn = column; - return true; + openings.Push((lineIndex, column)); + } + else if (line[column] == '}' && openings.Count > 0) + { + var opening = openings.Pop(); + result[GetTclPositionKey(opening.Line, opening.Column)] = new TclBraceEnd(lineIndex, column); } } } - endLine = -1; - endColumn = -1; - return false; + return result; } + private static long GetTclPositionKey(int line, int column) => + ((long)line << 32) | (uint)column; + private static void AddPrologContainers( IReadOnlyList lines, IReadOnlyList symbols, diff --git a/src/CodeIndex/Indexer/References/ReferenceExtractor.CoreExtraction.cs b/src/CodeIndex/Indexer/References/ReferenceExtractor.CoreExtraction.cs index 6e5326823..fd8a7f7bd 100644 --- a/src/CodeIndex/Indexer/References/ReferenceExtractor.CoreExtraction.cs +++ b/src/CodeIndex/Indexer/References/ReferenceExtractor.CoreExtraction.cs @@ -2611,7 +2611,7 @@ bool TryGetKnownPythonTypeCall(string candidate, out string canonicalName) DynamicDeclarativeReferenceExtractor.EmitAdditionalReferences( language, preparedLine, - originalLine, + referenceStructuralLines[i], dynamicDeclarativeState, references, seen, diff --git a/src/CodeIndex/Indexer/References/ReferenceExtractor.Preparation.cs b/src/CodeIndex/Indexer/References/ReferenceExtractor.Preparation.cs index 352835ea1..72c7a2dc8 100644 --- a/src/CodeIndex/Indexer/References/ReferenceExtractor.Preparation.cs +++ b/src/CodeIndex/Indexer/References/ReferenceExtractor.Preparation.cs @@ -99,6 +99,9 @@ [new CppLexicalRange(0, maskedContent.Length)], : UsesCStyleBlockComments(language) ? MaskCStyleBlockCommentLines(language, structuralLines) : structuralLines; + referenceStructuralLines = DynamicDeclarativeReferenceExtractor.MaskNonCodeLines( + language, + referenceStructuralLines); if (language == "python") referenceStructuralLines = MaskPythonFStrings(referenceStructuralLines); diff --git a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.cs b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.cs index 2563900ba..56baaba14 100644 --- a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.cs +++ b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.cs @@ -23,6 +23,8 @@ public static partial class SymbolExtractor public const int XmlContractVersion = 3; public const int FunctionalLanguageContractVersion = 2; public const int DynamicLanguageContractVersion = 2; + public const int DynamicReferenceGraphContractVersion = 3; + public const int PrologReferenceGraphContractVersion = 2; public const int SystemsLanguageContractVersion = 2; private static readonly string[] AdditionalSymbolLanguages = [ @@ -79,7 +81,9 @@ public static int GetContractVersion(string? lang) "sass" or "stylus" => StyleAndXamlContractVersion, "xml" => XmlContractVersion, "clojure" or "erlang" or "ocaml" or "raku" => FunctionalLanguageContractVersion, - "crystal" or "groovy" or "julia" or "tcl" => DynamicLanguageContractVersion, + "crystal" or "groovy" or "tcl" => DynamicReferenceGraphContractVersion, + "julia" => DynamicLanguageContractVersion, + "prolog" or "ambiguous_pl" => PrologReferenceGraphContractVersion, "ada" or "d" or "nim" => SystemsLanguageContractVersion, "app_manifest" or "cmake" or "dependency_lock" or "dependency_manifest" or "graphql" or "html" or "json" or "justfile" or "markdown" or "msbuild" or "solution" or "yaml" => ExpandedLanguageContractVersion, _ => DefaultContractVersion, @@ -1711,7 +1715,7 @@ private enum JavaScriptTypeScriptFunctionHeaderConsumeResult [ new("namespace", new Regex(@"^\s*:-\s*module\s*\(\s*(?[a-z][A-Za-z0-9_]*)", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None), new("import", new Regex(@"^\s*:-\s*use_module\s*\(\s*(?[a-z][A-Za-z0-9_]*(?:\([^)]*\))?)", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None), - new("function", new Regex(@"^\s*(?[a-z][A-Za-z0-9_]*)\s*(?:\([^\r\n.]*\))?\s*(?::-|-->|\.)", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None), + new("function", new Regex(@"^\s*(?[a-z][A-Za-z0-9_]*)\s*(?:\([^\r\n]*\))?\s*(?::-|-->|\.)", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None), ], ["ambiguous_pl"] = [ @@ -1729,7 +1733,7 @@ private enum JavaScriptTypeScriptFunctionHeaderConsumeResult new("function", new Regex(@"^\s*(?:method|fun)\s+(?" + PerlIdentifierPattern + @")\b(?:\s*:[^{;]+)?", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.Brace), new("namespace", new Regex(@"^\s*:-\s*module\s*\(\s*(?[a-z][A-Za-z0-9_]*)", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None), new("import", new Regex(@"^\s*:-\s*use_module\s*\(\s*(?[a-z][A-Za-z0-9_]*(?:\([^)]*\))?)", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None), - new("function", new Regex(@"^\s*(?[a-z][A-Za-z0-9_]*)\s*(?:\([^\r\n.]*\))?\s*(?::-|-->|\.)", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None), + new("function", new Regex(@"^\s*(?[a-z][A-Za-z0-9_]*)\s*(?:\([^\r\n]*\))?\s*(?::-|-->|\.)", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None), ], ["c"] = [ diff --git a/tests/CodeIndex.Tests/DatabaseTests.cs b/tests/CodeIndex.Tests/DatabaseTests.cs index d39240509..a416b79cf 100644 --- a/tests/CodeIndex.Tests/DatabaseTests.cs +++ b/tests/CodeIndex.Tests/DatabaseTests.cs @@ -4692,6 +4692,34 @@ public void GetUnchangedFileId_ReturnsNullWhenLanguageExtractorVersionIsStale() Assert.Null(id); } + [Theory] + [InlineData("crystal", 2)] + [InlineData("groovy", 2)] + [InlineData("tcl", 2)] + [InlineData("prolog", 1)] + [InlineData("ambiguous_pl", 1)] + public void GetUnchangedFileId_InvalidatesPreGraphLanguageContracts_Issue4746( + string language, + int previousContractVersion) + { + var modified = new DateTime(2025, 1, 1, 0, 0, 0, DateTimeKind.Utc); + var file = new FileRecord + { + Path = $"src/legacy-{language}.txt", + Lang = language, + Size = 50, + Lines = 5, + Modified = modified, + }; + _writer.UpsertFile(file); + _writer.SetMeta( + DbContext.GetSymbolExtractorVersionMetaKey(language), + previousContractVersion.ToString(System.Globalization.CultureInfo.InvariantCulture)); + + Assert.True(SymbolExtractor.GetContractVersion(language) > previousContractVersion); + Assert.Null(_writer.GetUnchangedFileId(file.Path, modified, language: language)); + } + [Fact] public void GetUnchangedFileId_MatchesByChecksumWhenTimestampDiffers() { diff --git a/tests/CodeIndex.Tests/ReferenceExtractorTests.cs b/tests/CodeIndex.Tests/ReferenceExtractorTests.cs index e4ea31959..5092eb8b4 100644 --- a/tests/CodeIndex.Tests/ReferenceExtractorTests.cs +++ b/tests/CodeIndex.Tests/ReferenceExtractorTests.cs @@ -1181,6 +1181,139 @@ public void Extract_AmbiguousPl_IndexesPerlAndPrologReferencesWithoutReclassific AssertReferencesContain(references, "call", "prolog_helper", "perl_helper"); } + [Fact] + public void Extract_Crystal_PreservesPredicateAndBangCallsWhileMaskingCommentsAndHeredocs_Issue4746() + { + const string content = """ + def ready?(value) + value + end + + def save!(value) + value + end + + def run(value) + # ready?(value) + text = <<-TEXT + ready?(value) + save!(value) + TEXT + ready? value + ready?(value) + save! value + save!(value) + end + """; + + var (_, references) = ExtractSymbolsAndReferences("crystal", content); + + Assert.Equal(2, references.Count(reference => + reference.ReferenceKind == "call" + && reference.ContainerName == "run" + && reference.SymbolName == "ready?")); + Assert.Equal(2, references.Count(reference => + reference.ReferenceKind == "call" + && reference.ContainerName == "run" + && reference.SymbolName == "save!")); + } + + [Fact] + public void Extract_Groovy_MasksTripleQuotedStringsAndComments_Issue4746() + { + const string content = """" + def helper() {} + + def run() { + // helper() + def singleQuoted = ''' + helper() + ''' + def doubleQuoted = """ + helper() + """ + helper() + } + """"; + + var (_, references) = ExtractSymbolsAndReferences("groovy", content); + + Assert.Single(references, reference => + reference.ReferenceKind == "call" + && reference.ContainerName == "run" + && reference.SymbolName == "helper"); + } + + [Fact] + public void Extract_Tcl_IndexesSingleLineProcBodiesAndMasksComments_Issue4746() + { + const string content = """ + proc helper {} { return 1 } + # [helper] + proc run {} { helper } + proc multiline {} { + set value 1 ;# [helper] + helper + } + """; + + var (_, references) = ExtractSymbolsAndReferences("tcl", content); + + AssertReferencesContain(references, "call", "run", "helper"); + AssertReferencesContain(references, "call", "multiline", "helper"); + Assert.Equal(2, references.Count(reference => + reference.ReferenceKind == "call" + && reference.SymbolName == "helper")); + } + + [Fact] + public void Extract_Prolog_IndexesRelativeImportsAndDecimalHeadsWhileMaskingComments_Issue4746() + { + const string content = """ + :- use_module('./support.pl'). + helper. + threshold(1.5) :- + helper. + % helper(). + /* + helper(). + */ + """; + + var (symbols, references) = ExtractSymbolsAndReferences("prolog", content); + + Assert.Contains(symbols, symbol => symbol.Kind == "function" && symbol.Name == "threshold"); + AssertReferencesContain(references, "type_reference", null, "support"); + Assert.Single(references, reference => + reference.ReferenceKind == "call" + && reference.ContainerName == "threshold" + && reference.SymbolName == "helper"); + Assert.DoesNotContain(references, reference => + reference.ReferenceKind == "call" + && reference.SymbolName == "threshold"); + } + + [Fact] + public void Extract_AmbiguousPl_MasksPerlAndPrologCommentsAndPod_Issue4746() + { + const string content = """ + sub helper { return 1; } + prolog_entry :- helper(). + # helper(); + % helper(). + =pod + helper(); + =cut + """; + + var (_, references) = ExtractSymbolsAndReferences("ambiguous_pl", content); + + Assert.Single(references, reference => + reference.ReferenceKind == "call" + && reference.ContainerName == "prolog_entry" + && reference.SymbolName == "helper"); + } + [Fact] public void TryGetExtractor_RegisteredLanguage_ReturnsAddressableExtractor() { diff --git a/tests/CodeIndex.Tests/SymbolExtractorTests.cs b/tests/CodeIndex.Tests/SymbolExtractorTests.cs index 03ce42905..755f6e712 100644 --- a/tests/CodeIndex.Tests/SymbolExtractorTests.cs +++ b/tests/CodeIndex.Tests/SymbolExtractorTests.cs @@ -12542,6 +12542,14 @@ public void GetContractVersion_LanguageSpecificKinds_UseDedicatedVersions() Assert.True(SymbolExtractor.StyleAndXamlContractVersion > SymbolExtractor.DefaultContractVersion); Assert.Equal(SymbolExtractor.XmlContractVersion, SymbolExtractor.GetContractVersion("xml")); Assert.True(SymbolExtractor.XmlContractVersion > SymbolExtractor.StyleAndXamlContractVersion); + Assert.Equal(SymbolExtractor.DynamicReferenceGraphContractVersion, SymbolExtractor.GetContractVersion("crystal")); + Assert.Equal(SymbolExtractor.DynamicReferenceGraphContractVersion, SymbolExtractor.GetContractVersion("groovy")); + Assert.Equal(SymbolExtractor.DynamicReferenceGraphContractVersion, SymbolExtractor.GetContractVersion("tcl")); + Assert.True(SymbolExtractor.DynamicReferenceGraphContractVersion > SymbolExtractor.DynamicLanguageContractVersion); + Assert.Equal(SymbolExtractor.DynamicLanguageContractVersion, SymbolExtractor.GetContractVersion("julia")); + Assert.Equal(SymbolExtractor.PrologReferenceGraphContractVersion, SymbolExtractor.GetContractVersion("prolog")); + Assert.Equal(SymbolExtractor.PrologReferenceGraphContractVersion, SymbolExtractor.GetContractVersion("ambiguous_pl")); + Assert.True(SymbolExtractor.PrologReferenceGraphContractVersion > SymbolExtractor.DefaultContractVersion); } [Fact] From a079695b2c93af31e7a17ed2edaebae576fce4c7 Mon Sep 17 00:00:00 2001 From: Widthdom Date: Fri, 24 Jul 2026 13:46:18 +0900 Subject: [PATCH 03/26] Tighten dynamic graph lexical handling (#4746) --- .../DynamicDeclarativeReferenceExtractor.cs | 312 +++++++++++++++++- .../Symbols/SymbolExtractor.ExtractCore.cs | 36 +- .../ReferenceExtractorTests.cs | 47 ++- tests/CodeIndex.Tests/SymbolExtractorTests.cs | 13 + 4 files changed, 385 insertions(+), 23 deletions(-) diff --git a/src/CodeIndex/Indexer/References/Languages/DynamicDeclarativeReferenceExtractor.cs b/src/CodeIndex/Indexer/References/Languages/DynamicDeclarativeReferenceExtractor.cs index 4b254e63b..5129f6850 100644 --- a/src/CodeIndex/Indexer/References/Languages/DynamicDeclarativeReferenceExtractor.cs +++ b/src/CodeIndex/Indexer/References/Languages/DynamicDeclarativeReferenceExtractor.cs @@ -10,14 +10,17 @@ internal sealed class ExtractionState { public ExtractionState( HashSet callableNames, - IReadOnlyDictionary containersByLine) + IReadOnlyDictionary containersByLine, + IReadOnlyDictionary> tclScriptBodyColumnsByLine) { CallableNames = callableNames; ContainersByLine = containersByLine; + TclScriptBodyColumnsByLine = tclScriptBodyColumnsByLine; } public HashSet CallableNames { get; } public IReadOnlyDictionary ContainersByLine { get; } + public IReadOnlyDictionary> TclScriptBodyColumnsByLine { get; } public SymbolRecord? ResolveContainer(int lineNumber, SymbolRecord? fallback) => ContainersByLine.TryGetValue(lineNumber, out var container) ? container : fallback; @@ -51,7 +54,7 @@ public ExtractionState( @"(?:^|[;=])\s*(?:return\s+)?(?[A-Za-z_]\w*)\b(?!\s*\()", RegexOptions.Compiled | RegexOptions.CultureInvariant); private static readonly Regex TclCommandRegex = new( - @"(?:^|[;\[{}])\s*(?[A-Za-z_:][\w:.-]*)", + @"(?:^|[;\[])\s*(?[A-Za-z_:][\w:.-]*)", RegexOptions.Compiled | RegexOptions.CultureInvariant); private static readonly Regex PrologBareCallRegex = new( @"(?:^|:-|-->|[,;])\s*(?:\\\+\s*)?(?[a-z][A-Za-z0-9_]*)\b(?!\s*\()", @@ -63,7 +66,9 @@ public ExtractionState( private readonly record struct TclContainerScope( SymbolRecord Symbol, int StartLine, - int EndLine); + int EndLine, + int BodyStartLine, + int BodyStartColumn); private readonly record struct TclBraceEnd( int Line, @@ -79,6 +84,11 @@ public static string[] MaskNonCodeLines(string language, IReadOnlyList l char groovyTripleQuote = '\0'; string? crystalHeredocDelimiter = null; var insideAmbiguousPerlPod = false; + var insideSlashyLiteral = false; + var insideGroovyDollarSlashyLiteral = false; + char crystalPercentOpeningDelimiter = '\0'; + char crystalPercentClosingDelimiter = '\0'; + var crystalPercentDelimiterDepth = 0; for (var lineIndex = 0; lineIndex < lines.Count; lineIndex++) { @@ -151,6 +161,70 @@ public static string[] MaskNonCodeLines(string language, IReadOnlyList l continue; } + if (insideGroovyDollarSlashyLiteral) + { + buffer[column] = ' '; + if (column + 1 < line.Length + && line[column] == '/' + && line[column + 1] == '$') + { + buffer[column + 1] = ' '; + column += 2; + insideGroovyDollarSlashyLiteral = false; + } + else + { + column++; + } + continue; + } + + if (insideSlashyLiteral) + { + buffer[column] = ' '; + if (line[column] == '\\' && column + 1 < line.Length) + { + buffer[column + 1] = ' '; + column += 2; + } + else if (line[column] == '/') + { + column++; + insideSlashyLiteral = false; + } + else + { + column++; + } + continue; + } + + if (crystalPercentClosingDelimiter != '\0') + { + buffer[column] = ' '; + if (line[column] == '\\' && column + 1 < line.Length) + { + buffer[column + 1] = ' '; + column += 2; + continue; + } + + if (crystalPercentOpeningDelimiter != crystalPercentClosingDelimiter + && line[column] == crystalPercentOpeningDelimiter) + { + crystalPercentDelimiterDepth++; + } + else if (line[column] == crystalPercentClosingDelimiter + && --crystalPercentDelimiterDepth == 0) + { + crystalPercentOpeningDelimiter = '\0'; + crystalPercentClosingDelimiter = '\0'; + } + + column++; + continue; + } + var ch = line[column]; if (language == "groovy" && ch is '\'' or '"' @@ -204,18 +278,80 @@ public static string[] MaskNonCodeLines(string language, IReadOnlyList l break; } - if ((language is "crystal" or "tcl" or "ambiguous_pl") && ch == '#') + if ((language is "crystal" or "tcl") && ch == '#') { FillWithSpaces(buffer, column); break; } - if ((language is "prolog" or "ambiguous_pl") && ch == '%') + if (language == "prolog" && ch == '%') { FillWithSpaces(buffer, column); break; } + if (language == "ambiguous_pl" && ch == '#') + { + if (IsPrologHashOperator(line, column)) + { + column++; + continue; + } + + FillWithSpaces(buffer, column); + break; + } + + if (language == "ambiguous_pl" && ch == '%') + { + if (IsPerlHashSigil(line, column)) + { + column++; + continue; + } + + FillWithSpaces(buffer, column); + break; + } + + if (language == "groovy" + && ch == '$' + && column + 1 < line.Length + && line[column + 1] == '/') + { + buffer[column] = ' '; + buffer[column + 1] = ' '; + column += 2; + insideGroovyDollarSlashyLiteral = true; + continue; + } + + if (language == "crystal" + && TryBeginCrystalPercentLiteral( + line, + column, + out var percentOpeningDelimiter, + out var percentClosingDelimiter, + out var percentContentColumn)) + { + FillWithSpaces(buffer, column, percentContentColumn); + crystalPercentOpeningDelimiter = percentOpeningDelimiter; + crystalPercentClosingDelimiter = percentClosingDelimiter; + crystalPercentDelimiterDepth = 1; + column = percentContentColumn; + continue; + } + + if (language is "groovy" or "crystal" + && ch == '/' + && IsLikelySlashyLiteralStart(line, column)) + { + buffer[column] = ' '; + column++; + insideSlashyLiteral = true; + continue; + } + column++; } @@ -241,12 +377,20 @@ public static string[] MaskNonCodeLines(string language, IReadOnlyList l } var containersByLine = new Dictionary(); + var tclScriptBodyColumnsByLine = new Dictionary>(); if (language == "tcl") - AddTclContainers(preparedLines, symbols, containersByLine); + AddTclContainers( + preparedLines, + symbols, + containersByLine, + tclScriptBodyColumnsByLine); else if (language is "prolog" or "ambiguous_pl") AddPrologContainers(preparedLines, symbols, containersByLine); - return new ExtractionState(callableNames, containersByLine); + return new ExtractionState( + callableNames, + containersByLine, + tclScriptBodyColumnsByLine); } public static void EmitAdditionalReferences( @@ -319,6 +463,24 @@ public static void EmitAdditionalReferences( addCallLikeReference(nameGroup.Value, nameGroup.Index); } + + if (language == "tcl" + && state.TclScriptBodyColumnsByLine.TryGetValue(lineNumber, out var bodyColumns)) + { + foreach (var bodyColumn in bodyColumns) + { + if (bodyColumn >= preparedLine.Length) + continue; + + var bodyLine = preparedLine[bodyColumn..]; + foreach (Match match in BoundedRegex.EnumerateMatches(TclCommandRegex, bodyLine)) + { + var nameGroup = match.Groups["name"]; + if (state.CallableNames.Contains(nameGroup.Value)) + addCallLikeReference(nameGroup.Value, bodyColumn + nameGroup.Index); + } + } + } } private static int SkipQuotedToken(string line, int startColumn, char delimiter) @@ -352,6 +514,82 @@ private static void FillWithSpaces(char[] buffer, int startColumn) buffer[column] = ' '; } + private static void FillWithSpaces(char[] buffer, int startColumn, int endColumn) + { + for (var column = startColumn; column < endColumn && column < buffer.Length; column++) + buffer[column] = ' '; + } + + private static bool IsPrologHashOperator(string line, int column) + { + if (column + 1 >= line.Length) + return false; + + return line[column + 1] is '=' or '<' or '>' or '\\' or '/' or '#'; + } + + private static bool IsPerlHashSigil(string line, int column) + { + if (column + 1 >= line.Length) + return false; + + return line[column + 1] == '{' + || line[column + 1] == '_' + || char.IsLetter(line[column + 1]); + } + + private static bool IsLikelySlashyLiteralStart(string line, int column) + { + if (column + 1 >= line.Length || line[column + 1] is '/' or '*') + return false; + + var previousColumn = column - 1; + while (previousColumn >= 0 && char.IsWhiteSpace(line[previousColumn])) + previousColumn--; + + if (previousColumn < 0) + return true; + + return line[previousColumn] is '=' or '(' or '[' or '{' or ',' or ':' or ';' + or '!' or '&' or '|' or '?' or '+' or '-' or '*' or '%'; + } + + private static bool TryBeginCrystalPercentLiteral( + string line, + int column, + out char openingDelimiter, + out char closingDelimiter, + out int contentColumn) + { + openingDelimiter = '\0'; + closingDelimiter = '\0'; + contentColumn = column; + if (line[column] != '%' || column + 1 >= line.Length) + return false; + + var delimiterColumn = column + 1; + if (line[delimiterColumn] is 'q' or 'Q' or 'w' or 'W' or 'i' or 'I' or 'x' or 'r') + delimiterColumn++; + if (delimiterColumn >= line.Length + || char.IsLetterOrDigit(line[delimiterColumn]) + || char.IsWhiteSpace(line[delimiterColumn])) + { + return false; + } + + openingDelimiter = line[delimiterColumn]; + closingDelimiter = openingDelimiter switch + { + '(' => ')', + '[' => ']', + '{' => '}', + '<' => '>', + _ => openingDelimiter, + }; + contentColumn = delimiterColumn + 1; + return true; + } + private static void EmitImportReference( string language, string originalLine, @@ -409,7 +647,8 @@ private static string NormalizeImportTarget(string language, string name) private static void AddTclContainers( IReadOnlyList lines, IReadOnlyList symbols, - Dictionary containersByLine) + Dictionary containersByLine, + Dictionary> scriptBodyColumnsByLine) { var braceEnds = BuildTclBraceEndPositions(lines); var scopes = new List(); @@ -426,12 +665,19 @@ private static void AddTclContainers( braceEnds, startLineIndex, declarationMatch.Index + declarationMatch.Length, + out var bodyStartLineIndex, + out var bodyStartColumn, out var endLineIndex)) { continue; } - scopes.Add(new TclContainerScope(symbol, symbol.StartLine, endLineIndex + 1)); + scopes.Add(new TclContainerScope( + symbol, + symbol.StartLine, + endLineIndex + 1, + bodyStartLineIndex + 1, + bodyStartColumn + 1)); } scopes.Sort(static (left, right) => @@ -460,6 +706,15 @@ private static void AddTclContainers( if (activeScopes.Count > 0) containersByLine[lineNumber] = activeScopes.Peek().Symbol; } + + foreach (var lineGroup in scopes.GroupBy(static scope => scope.BodyStartLine)) + { + scriptBodyColumnsByLine[lineGroup.Key] = lineGroup + .Select(static scope => scope.BodyStartColumn) + .Distinct() + .Order() + .ToArray(); + } } private static bool TryFindTclBodyEnd( @@ -467,8 +722,12 @@ private static bool TryFindTclBodyEnd( IReadOnlyDictionary braceEnds, int startLineIndex, int searchColumn, + out int bodyStartLineIndex, + out int bodyStartColumn, out int endLineIndex) { + bodyStartLineIndex = startLineIndex; + bodyStartColumn = -1; endLineIndex = startLineIndex; if (!TryFindNextNonWhitespace(lines[startLineIndex], searchColumn, out var argsColumn) || !TryFindTclWordEnd( @@ -478,9 +737,16 @@ private static bool TryFindTclBodyEnd( argsColumn, out var argsEndLine, out var argsEndColumn) - || !TryFindNextNonWhitespace(lines[argsEndLine], argsEndColumn + 1, out var bodyColumn) - || lines[argsEndLine][bodyColumn] != '{' - || !braceEnds.TryGetValue(GetTclPositionKey(argsEndLine, bodyColumn), out var bodyEnd)) + || !TryFindNextNonWhitespace( + lines, + argsEndLine, + argsEndColumn + 1, + out bodyStartLineIndex, + out bodyStartColumn) + || lines[bodyStartLineIndex][bodyStartColumn] != '{' + || !braceEnds.TryGetValue( + GetTclPositionKey(bodyStartLineIndex, bodyStartColumn), + out var bodyEnd)) { return false; } @@ -560,6 +826,28 @@ private static bool TryFindNextNonWhitespace( return false; } + private static bool TryFindNextNonWhitespace( + IReadOnlyList lines, + int startLine, + int startColumn, + out int foundLine, + out int foundColumn) + { + for (var lineIndex = startLine; lineIndex < lines.Count; lineIndex++) + { + var column = lineIndex == startLine ? startColumn : 0; + if (TryFindNextNonWhitespace(lines[lineIndex], column, out foundColumn)) + { + foundLine = lineIndex; + return true; + } + } + + foundLine = -1; + foundColumn = -1; + return false; + } + private static Dictionary BuildTclBraceEndPositions(IReadOnlyList lines) { var result = new Dictionary(); diff --git a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.ExtractCore.cs b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.ExtractCore.cs index 58eca7dc6..b7dbc50a7 100644 --- a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.ExtractCore.cs +++ b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.ExtractCore.cs @@ -144,6 +144,9 @@ private static List ExtractCore( : null; var structuralLines = StructuralLineMasker.MaskLines(lang, lines); + structuralLines = DynamicDeclarativeReferenceExtractor.MaskNonCodeLines( + lang, + structuralLines); string[]? javaScriptTypeScriptSanitizedLines = null; string[] GetJavaScriptTypeScriptSanitizedLines() => javaScriptTypeScriptSanitizedLines ??= BuildJavaScriptTypeScriptSanitizedLines(lines); @@ -2158,7 +2161,7 @@ bool[] GetCssQualifiedRuleAncestors() => } private static readonly Regex PrologOpenClauseRegex = new( - @"^\s*(?:(?:[a-z][A-Za-z0-9_]*\s*(?:\([^.\r\n]*\))?\s*(?::-|-->))|:-)", + @"^\s*(?:(?:[a-z][A-Za-z0-9_]*\s*(?:\([^\r\n]*\))?\s*(?::-|-->))|:-)", RegexOptions.Compiled | RegexOptions.CultureInvariant); private static bool[] BuildPrologClauseContinuationLines(IReadOnlyList structuralLines) @@ -2187,6 +2190,12 @@ private static bool HasPrologClauseTerminator(string line) { for (var column = 0; column < line.Length; column++) { + if (line[column] is '\'' or '"') + { + column = SkipPrologQuotedTerm(line, column, line[column]) - 1; + continue; + } + if (line[column] != '.') continue; @@ -2200,6 +2209,31 @@ private static bool HasPrologClauseTerminator(string line) return false; } + private static int SkipPrologQuotedTerm(string line, int startColumn, char delimiter) + { + for (var column = startColumn + 1; column < line.Length; column++) + { + if (line[column] == '\\') + { + column++; + continue; + } + + if (line[column] != delimiter) + continue; + + if (column + 1 < line.Length && line[column + 1] == delimiter) + { + column++; + continue; + } + + return column + 1; + } + + return line.Length; + } + private const int CSharpFieldInitializerSignatureLimit = 1024; private static string BoundCSharpFieldInitializerSignature(string signature) diff --git a/tests/CodeIndex.Tests/ReferenceExtractorTests.cs b/tests/CodeIndex.Tests/ReferenceExtractorTests.cs index 5092eb8b4..2db773297 100644 --- a/tests/CodeIndex.Tests/ReferenceExtractorTests.cs +++ b/tests/CodeIndex.Tests/ReferenceExtractorTests.cs @@ -1198,16 +1198,21 @@ def run(value) text = <<-TEXT ready?(value) save!(value) + def fake TEXT + regex = /ready?(value)/ + literal = %q(save!(value)) ready? value ready?(value) save! value save!(value) + fake value end """; - var (_, references) = ExtractSymbolsAndReferences("crystal", content); + var (symbols, references) = ExtractSymbolsAndReferences("crystal", content); + Assert.DoesNotContain(symbols, symbol => symbol.Kind == "function" && symbol.Name == "fake"); Assert.Equal(2, references.Count(reference => reference.ReferenceKind == "call" && reference.ContainerName == "run" @@ -1216,6 +1221,7 @@ def run(value) reference.ReferenceKind == "call" && reference.ContainerName == "run" && reference.SymbolName == "save!")); + Assert.DoesNotContain(references, reference => reference.SymbolName == "fake"); } [Fact] @@ -1228,20 +1234,26 @@ def run() { // helper() def singleQuoted = ''' helper() + def fake() {} ''' def doubleQuoted = """ helper() """ + def slashy = /helper()/ + def dollarSlashy = $/helper()/$ + fake 1 helper() } """"; - var (_, references) = ExtractSymbolsAndReferences("groovy", content); + var (symbols, references) = ExtractSymbolsAndReferences("groovy", content); + Assert.DoesNotContain(symbols, symbol => symbol.Kind == "function" && symbol.Name == "fake"); Assert.Single(references, reference => reference.ReferenceKind == "call" && reference.ContainerName == "run" && reference.SymbolName == "helper"); + Assert.DoesNotContain(references, reference => reference.SymbolName == "fake"); } [Fact] @@ -1250,18 +1262,26 @@ public void Extract_Tcl_IndexesSingleLineProcBodiesAndMasksComments_Issue4746() const string content = """ proc helper {} { return 1 } # [helper] - proc run {} { helper } + proc run {helper} { + set literal {helper} + set substituted [helper] + } proc multiline {} { set value 1 ;# [helper] helper } + proc split {value} + { + helper + } """; var (_, references) = ExtractSymbolsAndReferences("tcl", content); AssertReferencesContain(references, "call", "run", "helper"); AssertReferencesContain(references, "call", "multiline", "helper"); - Assert.Equal(2, references.Count(reference => + AssertReferencesContain(references, "call", "split", "helper"); + Assert.Equal(3, references.Count(reference => reference.ReferenceKind == "call" && reference.SymbolName == "helper")); } @@ -1274,6 +1294,8 @@ public void Extract_Prolog_IndexesRelativeImportsAndDecimalHeadsWhileMaskingComm helper. threshold(1.5) :- helper. + quoted('value.') :- + helper. % helper(). /* helper(). @@ -1283,11 +1305,13 @@ public void Extract_Prolog_IndexesRelativeImportsAndDecimalHeadsWhileMaskingComm var (symbols, references) = ExtractSymbolsAndReferences("prolog", content); Assert.Contains(symbols, symbol => symbol.Kind == "function" && symbol.Name == "threshold"); + Assert.Contains(symbols, symbol => symbol.Kind == "function" && symbol.Name == "quoted"); + Assert.Single(symbols, symbol => symbol.Kind == "function" && symbol.Name == "helper"); AssertReferencesContain(references, "type_reference", null, "support"); - Assert.Single(references, reference => + Assert.Equal(2, references.Count(reference => reference.ReferenceKind == "call" - && reference.ContainerName == "threshold" - && reference.SymbolName == "helper"); + && reference.ContainerName is "threshold" or "quoted" + && reference.SymbolName == "helper")); Assert.DoesNotContain(references, reference => reference.ReferenceKind == "call" && reference.SymbolName == "threshold"); @@ -1299,6 +1323,8 @@ public void Extract_AmbiguousPl_MasksPerlAndPrologCommentsAndPod_Issue4746() const string content = """ sub helper { return 1; } prolog_entry :- helper(). + my %cache = (answer => helper()); + clp_entry(X) :- X #= 1, helper. # helper(); % helper(). =pod @@ -1308,10 +1334,11 @@ public void Extract_AmbiguousPl_MasksPerlAndPrologCommentsAndPod_Issue4746() var (_, references) = ExtractSymbolsAndReferences("ambiguous_pl", content); - Assert.Single(references, reference => + Assert.Equal(3, references.Count(reference => reference.ReferenceKind == "call" - && reference.ContainerName == "prolog_entry" - && reference.SymbolName == "helper"); + && reference.SymbolName == "helper")); + AssertReferencesContain(references, "call", "prolog_entry", "helper"); + AssertReferencesContain(references, "call", "clp_entry", "helper"); } [Fact] diff --git a/tests/CodeIndex.Tests/SymbolExtractorTests.cs b/tests/CodeIndex.Tests/SymbolExtractorTests.cs index 755f6e712..76b1c815f 100644 --- a/tests/CodeIndex.Tests/SymbolExtractorTests.cs +++ b/tests/CodeIndex.Tests/SymbolExtractorTests.cs @@ -77,6 +77,16 @@ public void Extract_AmbiguousPl_PreservesPerlAndPrologStructure_Issue4746() :- module(hybrid, [prolog_helper/0]). :- use_module(library(lists)). prolog_helper :- perl_helper(). + =pod + package PodFake; + sub pod_fake { return 1; } + pod_predicate. + =cut + /* + package BlockFake; + sub block_fake { return 1; } + block_predicate. + */ """; var symbols = SymbolExtractor.Extract(1, "ambiguous_pl", content); @@ -84,6 +94,9 @@ public void Extract_AmbiguousPl_PreservesPerlAndPrologStructure_Issue4746() AssertSymbolsContain(symbols, "namespace", "Hybrid", "hybrid"); AssertSymbolsContain(symbols, "import", "Shared::Tools", "library(lists)"); AssertSymbolsContain(symbols, "function", "perl_helper", "prolog_helper"); + Assert.DoesNotContain(symbols, symbol => + symbol.Name is "PodFake" or "pod_fake" or "pod_predicate" + or "BlockFake" or "block_fake" or "block_predicate"); } [Fact] From 7d0c7e5f6ce6109fff08be6fcf1e533a88ebbdba Mon Sep 17 00:00:00 2001 From: Widthdom Date: Fri, 24 Jul 2026 14:23:10 +0900 Subject: [PATCH 04/26] Close dynamic graph review gaps (#4746) --- DEVELOPER_GUIDE.md | 2 + TESTING_GUIDE.md | 2 + USER_GUIDE.md | 2 + changelog.d/unreleased/4746.added.md | 8 +- .../Cli/QueryCommandRunner.Status.cs | 34 +- .../Cli/QueryCommandRunner.StatusFields.cs | 6 +- src/CodeIndex/Database/DbReader.Status.cs | 49 +- .../Database/DegradationReasonCodes.cs | 7 + .../DynamicDeclarativeReferenceExtractor.cs | 503 +++++++++++++++--- .../ReferenceExtractor.CoreExtraction.cs | 13 +- tests/CodeIndex.Tests/DatabaseTests.cs | 42 ++ .../ReferenceExtractorTests.cs | 18 +- 12 files changed, 579 insertions(+), 107 deletions(-) diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index 4537cc7e9..894ba2846 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -1608,6 +1608,7 @@ Process exit codes are coarse (`0` success including valid zero-row queries, `1` - **Language capability patterns remain typed at the integration boundary** — CLI/MCP `languages` rows expose suffix-only `extensions`, literal `exact_filenames`, and ``-rendered `filename_prefix_patterns`. `legacy_patterns` preserves the former combined list during deprecation, and `pattern_provenance` identifies built-in, plugin/pattern, and language-map override ownership. Round-trip tests feed every advertised typed pattern back through `FileIndexer.DetectLanguage` (#4617). - **Ambiguous source extensions stay explicit** — `.m` and `.pl` are not assigned to Objective-C and Perl by default. `FileIndexer` checks an authoritative recognized shebang, then a 64 KiB bounded prefix for strong mutually exclusive Objective-C/MATLAB or Perl/Prolog markers, then at most 256 entries per ancestor directory for conservative project markers. Conflicting or weak evidence is indexed as `ambiguous_m` / `ambiguous_pl`. MATLAB deliberately remains symbol-only; Prolog and `ambiguous_pl` advertise conservative reference/graph support, with the ambiguous bucket using union symbol/reference rules without changing the content-based classification (#4612, #4746). +- **Dynamic reference-graph readiness follows extractor contracts** — when indexed Crystal, Groovy, Tcl, Prolog, or `ambiguous_pl` rows have a missing or stale symbol-extractor version stamp, status reports `dynamic_reference_graph_contract_stale` and keeps `reference_graph_complete` / `graph_data_current` false until a normal index refresh rewrites those rows (#4746). - **Hotspot marker fingerprints share one bounded tree traversal** — full/update CLI and MCP indexing compute C#, VB, F#, and MSBuild marker fingerprints together instead of walking the directory tree once per language. Each distinct marker glob retains the platform filesystem's matching behavior and is enumerated once per visited directory, while child directories are enumerated once; marker sets, budgets, truncation sentinels, and warning order remain isolated per language. The single-language API delegates to the same engine, preserving ignore rules, nested-repository/submodule boundaries, and MCP authorized-read failures. - **Lock-file dependency graphs model package relationships** — `packages.lock.json`, `package-lock.json`, and `npm-shrinkwrap.json` keep package declarations as symbols, but emit `dependency` references only for explicit parent-package to child-package entries. The parent package is stored as the reference container, so `callers` can identify which package requires a child and `deps` does not infer lock-file-to-lock-file similarity merely because two files contain the same resolved package set (#4409). - **Dependency-cycle audits separate analysis from display** — CLI `deps --cycles` and MCP `deps` with `cycles=true` analyze a deterministic, path-ordered edge set up to the independent `--graph-budget` / `graphBudget` before computing and stably ranking strongly connected components. `--limit` / `limit` only paginates that ranked SCC set, and opaque cursors are bound to the filters, graph budget, and indexed graph that produced them. Machine-readable responses expose `analysis_complete`, graph edge count/budget, stable ranking mode, authoritative total-cycle status, and continuation metadata; exhausting the graph budget is reported as an explicitly incomplete analysis rather than a complete cycle audit (#4731). @@ -4606,6 +4607,7 @@ USER_GUIDEの[終了コード](USER_GUIDE.md#終了コード)セクションを - **integration boundary では language capability pattern の型を維持** — CLI/MCP の `languages` 行は suffix のみの `extensions`、literal な `exact_filenames`、`` 表記の `filename_prefix_patterns` を公開します。`legacy_patterns` は deprecation 中に従来の combined list を保持し、`pattern_provenance` は built-in、plugin/pattern、language-map override の所有元を示します。round-trip test は広告した全 typed pattern を `FileIndexer.DetectLanguage` に戻して検証します（#4617）。 - **曖昧な source extension は曖昧なまま明示** — `.m` と `.pl` を既定で Objective-C / Perl に割り当てません。`FileIndexer` は authoritative な認識済み shebang、64 KiB 上限 prefix 内の相互排他的で強い Objective-C/MATLAB または Perl/Prolog marker、各 ancestor directory 最大 256 entry の保守的な project marker の順に確認します。競合または弱い証拠は `ambiguous_m` / `ambiguous_pl` として index します。MATLAB は意図的に symbol-only のままですが、Prolog と `ambiguous_pl` は保守的な reference / graph 対応を広告し、曖昧 bucket では content-based classification を変えずに symbol / reference rule の和集合を使います（#4612、#4746）。 +- **動的言語の reference-graph readiness は extractor contract に従う** — index 済みの Crystal、Groovy、Tcl、Prolog、`ambiguous_pl` row で symbol-extractor version stamp が欠落または古い場合、status は `dynamic_reference_graph_contract_stale` を報告し、通常の index refresh が対象 row を更新するまで `reference_graph_complete` / `graph_data_current` を false に保ちます（#4746）。 - **hotspot marker fingerprint は上限付きtree traversalを1回共有** — full/update CLIとMCP indexingは、directory treeを言語ごとに歩かず、C#、VB、F#、MSBuildのmarker fingerprintをまとめて計算します。各directoryでは固有marker globごとにplatform filesystemのmatching挙動を保って1回ずつ列挙し、child directoryも1回だけ列挙する一方、marker集合、budget、truncation sentinel、warning順は言語別に分離します。single-language APIも同じengineへ委譲し、ignore rule、nested repository/submodule境界、MCP authorized read failureを維持します。 - **lock file の依存グラフは package 間の関係をモデル化** — `packages.lock.json`、`package-lock.json`、`npm-shrinkwrap.json` は package 宣言を symbol として保持しますが、`dependency` reference は明示された親 package → 子 package の項目だけに出力します。親 package を reference container に保存するため、`callers` はどの package が子 package を必要としているかを特定でき、`deps` は同じ resolved package 集合を持つだけの lock file 同士を類似関係として推論しません（#4409）。 - **依存サイクル監査では解析と表示を分離** — CLI の `deps --cycles` と MCP `deps` の `cycles=true` は、独立した `--graph-budget` / `graphBudget` まで path 順で決定的な edge 集合を解析してから、強連結成分を安定順位付けします。`--limit` / `limit` はその SCC 順位集合をページ分割するだけで、不透明 cursor は生成時の filter、graph budget、indexed graph に結び付けます。machine-readable 応答は `analysis_complete`、graph edge 件数/予算、安定 ranking mode、authoritative な総 cycle 件数かどうか、continuation metadata を公開し、graph budget 枯渇時は完全な cycle 監査を装わず明示的な未完了解析として報告します（#4731）。 diff --git a/TESTING_GUIDE.md b/TESTING_GUIDE.md index be57b64da..fb8234d96 100644 --- a/TESTING_GUIDE.md +++ b/TESTING_GUIDE.md @@ -89,6 +89,7 @@ Use `docs/test-doc-maintenance-plan.md` before moving oversized suites or adding Use `AssertReferencesDoNotContain(...)` for negative checks over one reference kind; retain direct predicates when the exclusion depends on container, context, line, or other metadata. `ReferenceExtractorTests.ExtractSymbolsAndReferences(...)` owns the common symbol-then-reference extraction setup for tests that need both lists; use it instead of repeating the two extractor calls when the fixture does not need a specialized path or workspace symbol setup, and discard the symbol tuple element with `_` instead of keeping an unused `symbols` local when the test only asserts references. Crystal, Groovy, Tcl, Prolog, and `ambiguous_pl` graph fixtures keep imports, parenthesized calls, conservative same-file command/predicate calls, caller containers, and keyword false-positive controls independently diagnosable; capability tests must keep their advertised symbol/reference/graph flags aligned with those extractor fixtures (#4746). + Database status tests must also pin stale and current extractor-version stamps for those languages so pre-graph rows cannot report authoritative graph readiness. JavaScript/TypeScript discriminant-guard coverage shares one JavaScript-compatible fixture across both languages, while `DatabaseTests` pins `type_tag` as an accepted persisted reference kind so extractor-only assertions cannot hide database kind-catalog drift. Dockerfile named-stage reference variants share one multi-stage fixture when ordinary, lowercase, platform-flagged, commented, hyphenated, and dotted forms can be distinguished by exact per-stage call counts; keep external base-image exclusions in that fixture as the negative control. Dockerfile `COPY --from` ONBUILD and quoted-stage forms share a fixture with tagged and digest-qualified external-image controls; assert exact call counts for the genuine stages so false positives cannot hide beside positive edges. @@ -920,6 +921,7 @@ dotnet test --filter "FullyQualifiedName~GitHelperTests" 1つの reference kind に対する否定チェックには `AssertReferencesDoNotContain(...)` を使い、container、context、line など他の metadata に依存する除外は直接 predicate を維持します。 `ReferenceExtractorTests.ExtractSymbolsAndReferences(...)` は symbol 抽出から reference 抽出までの共通 setup を所有します。fixture が特殊な path や workspace symbol setup を必要としない場合は 2 つの extractor 呼び出しを繰り返さずこの helper を使い、reference だけを検証するテストでは未使用の `symbols` local を残さず symbol 側を `_` で捨ててください。 Crystal、Groovy、Tcl、Prolog、`ambiguous_pl` の graph fixture では、import、括弧付き call、同一ファイルに限定した保守的な command / predicate call、caller container、keyword の false-positive control を個別に診断可能な状態で維持し、capability test の symbol / reference / graph 広告を extractor fixture と一致させてください（#4746）。 + さらに database status test でこれらの言語の古い extractor-version stamp と現行 stamp を固定し、graph 対応前の row が authoritative な graph readiness を報告できないことを検証してください。 JavaScript / TypeScript の discriminant-guard coverage は両言語で JavaScript-compatible な fixture を共有し、`DatabaseTests` で `type_tag` が永続化可能な reference kind であることも固定して、extractor-only assertion が database kind catalog のずれを隠せないようにします。 Dockerfile の named-stage reference variant は、通常、小文字、platform flag、comment、hyphen、dot 形式を stage ごとの厳密な call 数で区別できる場合、1つの multi-stage fixture を共有します。外部 base image の除外も negative control として同じ fixture に残します。 Dockerfile の `COPY --from` における ONBUILD と quote 付き stage 形式は、tag および digest 付き外部 image の control と fixture を共有します。正しい stage の call 数を厳密に検証し、正例 edge に紛れた false positive を見逃さないようにします。 diff --git a/USER_GUIDE.md b/USER_GUIDE.md index c53636fbf..e08ad7f5d 100644 --- a/USER_GUIDE.md +++ b/USER_GUIDE.md @@ -2377,6 +2377,7 @@ All indexed languages are searchable through FTS5. Rows with **Symbols = yes** a - Functional symbol-only languages: Clojure, Erlang, OCaml, and Raku expose conservative declarations as symbols. References and graph queries are not advertised for these languages yet. - Dynamic symbol-only languages: Julia exposes conservative declarations as symbols. References and graph queries are not advertised for Julia yet. - Dynamic/declarative graph languages: Crystal, Groovy, Tcl, and Prolog expose conservative declarations, imports, and call relationships. Parenthesized calls use the shared extractor; command-style calls are limited to callables declared in the same file, and Tcl proc / Prolog predicate bodies preserve caller containers. + An index created before this graph contract reports `reference_graph_complete=false` and `graph_data_current=false` with `dynamic_reference_graph_contract_stale`; rerun `cdidx index ` to refresh affected rows before treating absent edges as authoritative. - Systems symbol-only languages: Ada, D, and Nim expose conservative declarations as symbols. References and graph queries are not advertised for these languages yet. - Markdown, JSON/YAML, and CSS: Markdown heading and local-anchor symbols are indexed; JSON/YAML configuration keys are indexed as structural key paths; CSS variables, placeholders, and `@extend` references are indexed. - Dockerfile, Assembly, Common Lisp, and Racket: `ARG` build args, labels/PROC/MACRO blocks, package/module forms, definitions, classes/structs, requires, and provides are surfaced as symbols where applicable. @@ -5524,6 +5525,7 @@ indexing はファイル単位の SQLite transaction を commit します。長 - 関数型言語のシンボル専用対応: Clojure、Erlang、OCaml、Raku は保守的な宣言をシンボルとして公開します。これらの言語では references と graph queries はまだ対応として広告しません。 - 動的言語のシンボル専用対応: Julia は保守的な宣言をシンボルとして公開します。Julia の references と graph queries はまだ対応として広告しません。 - 動的・宣言型言語の graph 対応: Crystal、Groovy、Tcl、Prolog は保守的な宣言、import、call relationship を公開します。括弧付き call は共通 extractor を使い、command-style call は同一ファイルで宣言済みの callable に限定し、Tcl proc / Prolog predicate の本体では caller container を保持します。 + この graph contract より前に作成された index は `dynamic_reference_graph_contract_stale` とともに `reference_graph_complete=false`、`graph_data_current=false` を報告します。欠落 edge を authoritative とみなす前に `cdidx index ` を再実行して対象 row を更新してください。 - システム系言語のシンボル専用対応: Ada、D、Nim は保守的な宣言をシンボルとして公開します。これらの言語では references と graph queries はまだ対応として広告しません。 - Markdown、JSON/YAML、CSS: Markdown の heading / local anchor、JSON/YAML の configuration key path、CSS の variable、placeholder、`@extend` をシンボルとして扱います。 - Dockerfile、Assembly、Common Lisp、Racket: `ARG` build arg、label、PROC/MACRO、package/module form、definition、class/struct、require/provide を必要に応じて表面化します。 diff --git a/changelog.d/unreleased/4746.added.md b/changelog.d/unreleased/4746.added.md index 605fc25cc..0cce6af8f 100644 --- a/changelog.d/unreleased/4746.added.md +++ b/changelog.d/unreleased/4746.added.md @@ -10,6 +10,10 @@ affected: - src/CodeIndex/Indexer/References/ReferenceExtractor.CoreExtraction.cs - src/CodeIndex/Indexer/References/ReferenceExtractor.Preparation.cs - src/CodeIndex/Indexer/References/Languages/DynamicDeclarativeReferenceExtractor.cs + - src/CodeIndex/Database/DbReader.Status.cs + - src/CodeIndex/Database/DegradationReasonCodes.cs + - src/CodeIndex/Cli/QueryCommandRunner.Status.cs + - src/CodeIndex/Cli/QueryCommandRunner.StatusFields.cs - tests/CodeIndex.Tests/DatabaseTests.cs - tests/CodeIndex.Tests/SymbolExtractorTests.cs - tests/CodeIndex.Tests/ReferenceExtractorTests.cs @@ -22,8 +26,8 @@ affected: ## English -- **Crystal, Groovy, Tcl, Prolog, and ambiguous `.pl` files now support conservative reference graphs (#4746)** — indexing records imports and parenthesized calls, recognizes command-style calls only for callables declared in the same file, preserves Tcl proc and Prolog predicate caller containers, and reports structured symbol/reference/graph capabilities after `.pl` content classification. +- **Crystal, Groovy, Tcl, Prolog, and ambiguous `.pl` files now support conservative reference graphs (#4746)** — indexing records imports and parenthesized calls, recognizes command-style calls only for callables declared in the same file, preserves Tcl proc and Prolog predicate caller containers, and reports structured symbol/reference/graph capabilities after `.pl` content classification. Pre-graph extractor stamps keep graph readiness degraded until a normal index refresh rewrites affected rows. ## 日本語 -- **Crystal、Groovy、Tcl、Prolog、曖昧な `.pl` ファイルが保守的な reference graph に対応しました（#4746）** — import と括弧付き call を記録し、command-style call は同一ファイルで宣言済みの callable だけを認識し、Tcl proc / Prolog predicate の caller container を保持して、`.pl` の content classification 後に構造化された symbol / reference / graph capability を報告します。 +- **Crystal、Groovy、Tcl、Prolog、曖昧な `.pl` ファイルが保守的な reference graph に対応しました（#4746）** — import と括弧付き call を記録し、command-style call は同一ファイルで宣言済みの callable だけを認識し、Tcl proc / Prolog predicate の caller container を保持して、`.pl` の content classification 後に構造化された symbol / reference / graph capability を報告します。graph 対応前の extractor stamp がある場合、通常の index refresh で対象 row を更新するまで graph readiness を degraded に保ちます。 diff --git a/src/CodeIndex/Cli/QueryCommandRunner.Status.cs b/src/CodeIndex/Cli/QueryCommandRunner.Status.cs index b88ece5ae..24960f5a1 100644 --- a/src/CodeIndex/Cli/QueryCommandRunner.Status.cs +++ b/src/CodeIndex/Cli/QueryCommandRunner.Status.cs @@ -546,11 +546,20 @@ private static List BuildStatusReadinessDegradations } private static string GetReferenceGraphDegradationRootCause(StatusResult status) - => status.ReferenceGraphIncompleteReasons?.Contains( - DbReader.ReferenceExtractionCapStateUnavailableReason, - StringComparer.Ordinal) == true - ? DegradationReasonCodes.ReferenceExtractionCapStateUnavailable - : DegradationReasonCodes.ReferenceGraphIncomplete; + { + if (status.ReferenceGraphIncompleteReasons?.Contains( + DbReader.DynamicReferenceGraphContractStaleReason, + StringComparer.Ordinal) == true) + { + return DegradationReasonCodes.DynamicReferenceGraphContractStale; + } + + return status.ReferenceGraphIncompleteReasons?.Contains( + DbReader.ReferenceExtractionCapStateUnavailableReason, + StringComparer.Ordinal) == true + ? DegradationReasonCodes.ReferenceExtractionCapStateUnavailable + : DegradationReasonCodes.ReferenceGraphIncomplete; + } private static StatusReadinessDegradation BuildStatusReadinessDegradation(string field, string rootCause, QueryCommandOptions options, StatusResult status) { @@ -672,9 +681,7 @@ private static IReadOnlyList BuildStatusCheckFailures(Status options, failure.Name, rebuild: false, - GetReferenceGraphDegradationRootCause(status) == DegradationReasonCodes.ReferenceExtractionCapStateUnavailable - ? "Refresh indexing to populate current per-file issue state before trusting reference-graph completeness." - : "Reduce or exclude the cap-hitting generated/pathological source before rerunning indexing."), + GetReferenceGraphRepairSafetyNote(status)), "graph_table_available" or "issues_table_available" or "file_issues_data_current" or "sql_graph_contract_ready" or "csharp_symbol_name_ready" or "csharp_metadata_target_ready" => BuildIndexRepairCommand( @@ -700,6 +707,17 @@ private static IReadOnlyList BuildStatusCheckFailures(Status return commands.Count == 0 ? null : commands; } + private static string GetReferenceGraphRepairSafetyNote(StatusResult status) + => GetReferenceGraphDegradationRootCause(status) switch + { + DegradationReasonCodes.DynamicReferenceGraphContractStale => + "Refresh indexing to rewrite stale dynamic-language graph rows and extractor-version stamps.", + DegradationReasonCodes.ReferenceExtractionCapStateUnavailable => + "Refresh indexing to populate current per-file issue state before trusting reference-graph completeness.", + _ => + "Reduce or exclude the cap-hitting generated/pathological source before rerunning indexing.", + }; + private static StatusRepairCommand BuildIndexRepairCommand( StatusResult status, QueryCommandOptions options, diff --git a/src/CodeIndex/Cli/QueryCommandRunner.StatusFields.cs b/src/CodeIndex/Cli/QueryCommandRunner.StatusFields.cs index a118af65f..e85bbe4d4 100644 --- a/src/CodeIndex/Cli/QueryCommandRunner.StatusFields.cs +++ b/src/CodeIndex/Cli/QueryCommandRunner.StatusFields.cs @@ -28,9 +28,9 @@ private sealed record StatusFieldExplanation( new( "reference_graph_complete", "Reference extraction coverage", - "reference-extraction cap state is available and extraction completed without hitting a hard lookup or container-candidate safety cap.", - "reference-extraction cap state is unavailable, or one or more files hit a hard cap; absent graph edges are not authoritative in either case.", - "Inspect `reference_extraction_cap_hits.state_available`: when false, rerun `cdidx index ` to populate current issue state; otherwise inspect `files`, reduce or exclude the cap-hitting generated/pathological source, and rerun indexing."), + "reference-extraction cap state is available, current dynamic-language extractor contracts are stamped, and extraction completed without hitting a hard lookup or container-candidate safety cap.", + "reference-extraction cap state is unavailable, a dynamic-language extractor stamp is stale, or one or more files hit a hard cap; absent graph edges are not authoritative.", + "Inspect `reference_graph_incomplete_reasons` and `reference_extraction_cap_hits.state_available`: refresh indexing for unavailable state or stale contracts; for a hard cap, inspect `files`, reduce or exclude the generated/pathological source, and rerun indexing."), new( "issues_table_available", "Validation issues table", diff --git a/src/CodeIndex/Database/DbReader.Status.cs b/src/CodeIndex/Database/DbReader.Status.cs index a5db73ae6..78118ae18 100644 --- a/src/CodeIndex/Database/DbReader.Status.cs +++ b/src/CodeIndex/Database/DbReader.Status.cs @@ -4,6 +4,18 @@ namespace CodeIndex.Database; public partial class DbReader { + public const string DynamicReferenceGraphContractStaleReason = + "dynamic_reference_graph_contract_stale"; + + private static readonly string[] DynamicReferenceGraphContractLanguages = + [ + "crystal", + "groovy", + "tcl", + "prolog", + "ambiguous_pl", + ]; + /// /// Get database statistics. /// データベースの統計情報を取得する。 @@ -142,8 +154,20 @@ GROUP BY COALESCE(f.lang, 'unknown'), s.kind var indexComplete = !batchInProgress && !string.Equals(indexCompleteness, "incomplete", StringComparison.OrdinalIgnoreCase); var referenceExtractionCapHits = GetReferenceExtractionCapHits(); + var dynamicReferenceGraphContractsCurrent = + AreDynamicReferenceGraphContractsCurrent(langs); + var referenceGraphIncompleteReasons = + referenceExtractionCapHits.Reasons?.ToList() ?? []; + if (!dynamicReferenceGraphContractsCurrent + && !referenceGraphIncompleteReasons.Contains( + DynamicReferenceGraphContractStaleReason, + StringComparer.Ordinal)) + { + referenceGraphIncompleteReasons.Add(DynamicReferenceGraphContractStaleReason); + } var referenceGraphComplete = referenceExtractionCapHits.StateAvailable - && referenceExtractionCapHits.HitCount == 0; + && referenceExtractionCapHits.HitCount == 0 + && dynamicReferenceGraphContractsCurrent; if (batchInProgress) { indexIncompleteReasons ??= []; @@ -176,7 +200,9 @@ GROUP BY COALESCE(f.lang, 'unknown'), s.kind GraphDataCurrent = _hasReferencesTable && indexComplete && referenceGraphComplete, ReferenceExtractionLimits = ReferenceExtractor.GetSafetyLimits(), ReferenceGraphComplete = referenceGraphComplete, - ReferenceGraphIncompleteReasons = referenceGraphComplete ? null : referenceExtractionCapHits.Reasons, + ReferenceGraphIncompleteReasons = referenceGraphComplete + ? null + : referenceGraphIncompleteReasons, ReferenceExtractionCapHits = referenceExtractionCapHits, IssuesTableAvailable = _hasIssuesPhysicalTable, FileIssuesDataCurrent = _hasIssuesTable, @@ -245,6 +271,25 @@ GROUP BY COALESCE(f.lang, 'unknown'), s.kind return result; } + private bool AreDynamicReferenceGraphContractsCurrent( + IReadOnlyDictionary indexedLanguages) + { + foreach (var language in DynamicReferenceGraphContractLanguages) + { + if (!indexedLanguages.ContainsKey(language)) + continue; + + var storedVersion = TryGetMetaStringInternal( + DbContext.GetSymbolExtractorVersionMetaKey(language)); + var currentVersion = SymbolExtractor.GetContractVersion(language).ToString( + System.Globalization.CultureInfo.InvariantCulture); + if (!string.Equals(storedVersion, currentVersion, StringComparison.Ordinal)) + return false; + } + + return true; + } + private StatusPreparedCommandCache? GetPreparedCommandCacheStatus() { if (_commandCache == null) diff --git a/src/CodeIndex/Database/DegradationReasonCodes.cs b/src/CodeIndex/Database/DegradationReasonCodes.cs index 0bf2a15e7..79c9857a5 100644 --- a/src/CodeIndex/Database/DegradationReasonCodes.cs +++ b/src/CodeIndex/Database/DegradationReasonCodes.cs @@ -26,6 +26,7 @@ public static class DegradationReasonCodes public const string IndexIncomplete = "index_complete=false"; public const string ReferenceGraphIncomplete = "reference_graph_complete=false"; public const string ReferenceExtractionCapStateUnavailable = "reference_extraction_cap_state_unavailable"; + public const string DynamicReferenceGraphContractStale = "dynamic_reference_graph_contract_stale"; public const string IssuesTableMissing = "issues_table_available=false"; public const string FileIssuesDataStale = "file_issues_data_current=false"; public const string CSharpSymbolNameNotReady = "csharp_symbol_name_ready=false"; @@ -55,6 +56,7 @@ public static class DegradationReasonCodes IndexIncomplete, ReferenceGraphIncomplete, ReferenceExtractionCapStateUnavailable, + DynamicReferenceGraphContractStale, IssuesTableMissing, FileIssuesDataStale, CSharpSymbolNameNotReady, @@ -193,6 +195,11 @@ private static DegradationReasonMetadata CreateMetadata(string code) "Reference-extraction cap state is unavailable for this legacy or stale issue generation, so graph completeness cannot be established.", "Run `cdidx index ` to populate current per-file issue state before trusting missing callers, callees, dependencies, or impact edges.", "Run `cdidx index --rebuild` only if a normal index refresh cannot restore current issue state."), + DynamicReferenceGraphContractStale => new( + code, + "Crystal, Groovy, Tcl, Prolog, or ambiguous .pl graph rows may predate the current extractor contract.", + "Run `cdidx index ` to refresh graph rows and their extractor-version stamps.", + "Run `cdidx index --rebuild` only if a normal index refresh cannot restore current graph readiness."), IssuesTableMissing => new( code, "Validate output is degraded to empty because the file_issues table is missing.", diff --git a/src/CodeIndex/Indexer/References/Languages/DynamicDeclarativeReferenceExtractor.cs b/src/CodeIndex/Indexer/References/Languages/DynamicDeclarativeReferenceExtractor.cs index 5129f6850..5e3dff7ad 100644 --- a/src/CodeIndex/Indexer/References/Languages/DynamicDeclarativeReferenceExtractor.cs +++ b/src/CodeIndex/Indexer/References/Languages/DynamicDeclarativeReferenceExtractor.cs @@ -11,19 +11,46 @@ internal sealed class ExtractionState public ExtractionState( HashSet callableNames, IReadOnlyDictionary containersByLine, - IReadOnlyDictionary> tclScriptBodyColumnsByLine) + IReadOnlyList tclContainerScopes, + IReadOnlyList? tclCallLines) { CallableNames = callableNames; ContainersByLine = containersByLine; - TclScriptBodyColumnsByLine = tclScriptBodyColumnsByLine; + TclContainerScopes = tclContainerScopes; + TclCallLines = tclCallLines; } public HashSet CallableNames { get; } public IReadOnlyDictionary ContainersByLine { get; } - public IReadOnlyDictionary> TclScriptBodyColumnsByLine { get; } + private IReadOnlyList TclContainerScopes { get; } + private IReadOnlyList? TclCallLines { get; } - public SymbolRecord? ResolveContainer(int lineNumber, SymbolRecord? fallback) - => ContainersByLine.TryGetValue(lineNumber, out var container) ? container : fallback; + public SymbolRecord? ResolveContainer(int lineNumber, int column, SymbolRecord? fallback) + { + for (var scopeIndex = TclContainerScopes.Count - 1; scopeIndex >= 0; scopeIndex--) + { + var scope = TclContainerScopes[scopeIndex]; + if (scope.Contains(lineNumber, column)) + return scope.Symbol; + } + + return ContainersByLine.TryGetValue(lineNumber, out var container) ? container : fallback; + } + + public string GetCallScanLine(string language, int lineNumber, string preparedLine) + { + if (language == "tcl" + && TclCallLines != null + && lineNumber > 0 + && lineNumber <= TclCallLines.Count) + { + return TclCallLines[lineNumber - 1]; + } + + var isClauseContinuation = ContainersByLine.TryGetValue(lineNumber, out var container) + && container.StartLine < lineNumber; + return PreparePrologCallScanLine(language, preparedLine, isClauseContinuation); + } } private static readonly Regex TclProcRegex = new( @@ -63,17 +90,48 @@ public ExtractionState( @"<<-\s*['""]?(?[A-Za-z_]\w*)['""]?", RegexOptions.Compiled | RegexOptions.CultureInvariant); - private readonly record struct TclContainerScope( + internal readonly record struct TclContainerScope( SymbolRecord Symbol, - int StartLine, - int EndLine, int BodyStartLine, - int BodyStartColumn); + int BodyStartColumn, + int BodyEndLine, + int BodyEndColumn) + { + public bool Contains(int lineNumber, int column) + { + if (lineNumber < BodyStartLine || lineNumber > BodyEndLine) + return false; + if (BodyStartLine == BodyEndLine) + return column > BodyStartColumn && column < BodyEndColumn; + if (lineNumber == BodyStartLine) + return column > BodyStartColumn; + if (lineNumber == BodyEndLine) + return column < BodyEndColumn; + return true; + } + } private readonly record struct TclBraceEnd( int Line, int Column); + private enum TclLexicalFrameKind + { + Script, + Quote, + BracedWord, + } + + private sealed class TclLexicalFrame( + TclLexicalFrameKind kind, + char terminator = '\0') + { + public TclLexicalFrameKind Kind { get; } = kind; + public char Terminator { get; } = terminator; + public bool CommandStart { get; set; } = true; + public bool WordStart { get; set; } = true; + } + public static string[] MaskNonCodeLines(string language, IReadOnlyList lines) { if (language is not ("crystal" or "groovy" or "tcl" or "prolog" or "ambiguous_pl")) @@ -278,7 +336,7 @@ public static string[] MaskNonCodeLines(string language, IReadOnlyList l break; } - if ((language is "crystal" or "tcl") && ch == '#') + if (language == "crystal" && ch == '#') { FillWithSpaces(buffer, column); break; @@ -364,6 +422,7 @@ public static string[] MaskNonCodeLines(string language, IReadOnlyList l public static ExtractionState? CreateState( string language, IReadOnlyList preparedLines, + IReadOnlyList structuralLines, IReadOnlyList symbols) { if (language is not ("crystal" or "groovy" or "tcl" or "prolog" or "ambiguous_pl")) @@ -377,20 +436,26 @@ public static string[] MaskNonCodeLines(string language, IReadOnlyList l } var containersByLine = new Dictionary(); - var tclScriptBodyColumnsByLine = new Dictionary>(); + var tclContainerScopes = new List(); + string[]? tclCallLines = null; if (language == "tcl") + { + var tclScriptBodyOpenings = new HashSet(); AddTclContainers( - preparedLines, + structuralLines, symbols, - containersByLine, - tclScriptBodyColumnsByLine); + tclContainerScopes, + tclScriptBodyOpenings); + tclCallLines = BuildTclCallLines(structuralLines, tclScriptBodyOpenings); + } else if (language is "prolog" or "ambiguous_pl") AddPrologContainers(preparedLines, symbols, containersByLine); return new ExtractionState( callableNames, containersByLine, - tclScriptBodyColumnsByLine); + tclContainerScopes, + tclCallLines); } public static void EmitAdditionalReferences( @@ -464,23 +529,6 @@ public static void EmitAdditionalReferences( addCallLikeReference(nameGroup.Value, nameGroup.Index); } - if (language == "tcl" - && state.TclScriptBodyColumnsByLine.TryGetValue(lineNumber, out var bodyColumns)) - { - foreach (var bodyColumn in bodyColumns) - { - if (bodyColumn >= preparedLine.Length) - continue; - - var bodyLine = preparedLine[bodyColumn..]; - foreach (Match match in BoundedRegex.EnumerateMatches(TclCommandRegex, bodyLine)) - { - var nameGroup = match.Groups["name"]; - if (state.CallableNames.Contains(nameGroup.Value)) - addCallLikeReference(nameGroup.Value, bodyColumn + nameGroup.Index); - } - } - } } private static int SkipQuotedToken(string line, int startColumn, char delimiter) @@ -530,12 +578,32 @@ private static bool IsPrologHashOperator(string line, int column) private static bool IsPerlHashSigil(string line, int column) { - if (column + 1 >= line.Length) + if (column + 1 >= line.Length + || (line[column + 1] != '{' + && line[column + 1] != '_' + && !char.IsLetter(line[column + 1]))) + { return false; + } + + var previousColumn = column - 1; + while (previousColumn >= 0 && char.IsWhiteSpace(line[previousColumn])) + previousColumn--; + if (previousColumn >= 0) + return true; + + var tokenEnd = column + 1; + if (line[tokenEnd] == '{') + return true; + while (tokenEnd < line.Length + && (char.IsLetterOrDigit(line[tokenEnd]) || line[tokenEnd] == '_')) + { + tokenEnd++; + } + while (tokenEnd < line.Length && char.IsWhiteSpace(line[tokenEnd])) + tokenEnd++; - return line[column + 1] == '{' - || line[column + 1] == '_' - || char.IsLetter(line[column + 1]); + return tokenEnd >= line.Length || line[tokenEnd] is '=' or ';' or '{' or '[' or ','; } private static bool IsLikelySlashyLiteralStart(string line, int column) @@ -550,8 +618,26 @@ private static bool IsLikelySlashyLiteralStart(string line, int column) if (previousColumn < 0) return true; - return line[previousColumn] is '=' or '(' or '[' or '{' or ',' or ':' or ';' - or '!' or '&' or '|' or '?' or '+' or '-' or '*' or '%'; + if (line[previousColumn] is '=' or '(' or '[' or '{' or ',' or ':' or ';' + or '!' or '&' or '|' or '?' or '+' or '-' or '*' or '%' or '~') + { + return true; + } + + var tokenEnd = previousColumn + 1; + while (previousColumn >= 0 + && (char.IsLetterOrDigit(line[previousColumn]) || line[previousColumn] == '_')) + { + previousColumn--; + } + + var token = line.AsSpan(previousColumn + 1, tokenEnd - previousColumn - 1); + return token.SequenceEqual("return") + || token.SequenceEqual("case") + || token.SequenceEqual("throw") + || token.SequenceEqual("assert") + || token.SequenceEqual("in") + || token.SequenceEqual("when"); } private static bool TryBeginCrystalPercentLiteral( @@ -568,11 +654,13 @@ private static bool TryBeginCrystalPercentLiteral( return false; var delimiterColumn = column + 1; - if (line[delimiterColumn] is 'q' or 'Q' or 'w' or 'W' or 'i' or 'I' or 'x' or 'r') + var hasTypePrefix = line[delimiterColumn] is 'q' or 'Q' or 'w' or 'W' or 'i' or 'I' or 'x' or 'r'; + if (hasTypePrefix) delimiterColumn++; if (delimiterColumn >= line.Length || char.IsLetterOrDigit(line[delimiterColumn]) - || char.IsWhiteSpace(line[delimiterColumn])) + || char.IsWhiteSpace(line[delimiterColumn]) + || (!hasTypePrefix && line[delimiterColumn] is not ('(' or '[' or '{' or '<'))) { return false; } @@ -590,6 +678,75 @@ private static bool TryBeginCrystalPercentLiteral( return true; } + private static string PreparePrologCallScanLine( + string language, + string line, + bool isClauseContinuation) + { + if (language is not ("prolog" or "ambiguous_pl") || isClauseContinuation) + return line; + + var firstColumn = 0; + while (firstColumn < line.Length && char.IsWhiteSpace(line[firstColumn])) + firstColumn++; + if (firstColumn >= line.Length + || !char.IsLower(line[firstColumn]) + || line.AsSpan(firstColumn).StartsWith(":-", StringComparison.Ordinal)) + { + return line; + } + + var parenthesisDepth = 0; + for (var column = firstColumn; column < line.Length; column++) + { + var ch = line[column]; + if (ch is '\'' or '"') + { + column = SkipQuotedToken(line, column, ch) - 1; + continue; + } + if (ch == '(') + { + parenthesisDepth++; + continue; + } + if (ch == ')' && parenthesisDepth > 0) + { + parenthesisDepth--; + continue; + } + if (parenthesisDepth != 0) + continue; + + var separatorLength = line.AsSpan(column).StartsWith("-->", StringComparison.Ordinal) + ? 3 + : line.AsSpan(column).StartsWith(":-", StringComparison.Ordinal) + ? 2 + : 0; + if (separatorLength > 0) + { + var masked = line.ToCharArray(); + FillWithSpaces(masked, 0, column); + return new string(masked); + } + + if (ch == '.' && IsOnlyWhitespaceAfter(line, column + 1)) + return new string(' ', line.Length); + } + + return line; + } + + private static bool IsOnlyWhitespaceAfter(string line, int startColumn) + { + for (var column = startColumn; column < line.Length; column++) + { + if (!char.IsWhiteSpace(line[column])) + return false; + } + return true; + } + private static void EmitImportReference( string language, string originalLine, @@ -647,11 +804,10 @@ private static string NormalizeImportTarget(string language, string name) private static void AddTclContainers( IReadOnlyList lines, IReadOnlyList symbols, - Dictionary containersByLine, - Dictionary> scriptBodyColumnsByLine) + List scopes, + HashSet scriptBodyOpenings) { var braceEnds = BuildTclBraceEndPositions(lines); - var scopes = new List(); foreach (var symbol in symbols) { if (symbol.Kind != "function" || symbol.StartLine < 1 || symbol.StartLine > lines.Count) @@ -667,54 +823,27 @@ private static void AddTclContainers( declarationMatch.Index + declarationMatch.Length, out var bodyStartLineIndex, out var bodyStartColumn, - out var endLineIndex)) + out var bodyEnd)) { continue; } scopes.Add(new TclContainerScope( symbol, - symbol.StartLine, - endLineIndex + 1, bodyStartLineIndex + 1, - bodyStartColumn + 1)); + bodyStartColumn, + bodyEnd.Line + 1, + bodyEnd.Column)); + scriptBodyOpenings.Add(GetTclPositionKey(bodyStartLineIndex, bodyStartColumn)); } scopes.Sort(static (left, right) => { - var startComparison = left.StartLine.CompareTo(right.StartLine); + var startComparison = left.BodyStartLine.CompareTo(right.BodyStartLine); return startComparison != 0 ? startComparison - : right.EndLine.CompareTo(left.EndLine); + : left.BodyStartColumn.CompareTo(right.BodyStartColumn); }); - - var activeScopes = new Stack(); - var scopeIndex = 0; - for (var lineNumber = 1; lineNumber <= lines.Count; lineNumber++) - { - while (activeScopes.Count > 0 && activeScopes.Peek().EndLine < lineNumber) - activeScopes.Pop(); - - while (scopeIndex < scopes.Count && scopes[scopeIndex].StartLine == lineNumber) - { - var scope = scopes[scopeIndex++]; - while (activeScopes.Count > 0 && activeScopes.Peek().EndLine < lineNumber) - activeScopes.Pop(); - activeScopes.Push(scope); - } - - if (activeScopes.Count > 0) - containersByLine[lineNumber] = activeScopes.Peek().Symbol; - } - - foreach (var lineGroup in scopes.GroupBy(static scope => scope.BodyStartLine)) - { - scriptBodyColumnsByLine[lineGroup.Key] = lineGroup - .Select(static scope => scope.BodyStartColumn) - .Distinct() - .Order() - .ToArray(); - } } private static bool TryFindTclBodyEnd( @@ -724,11 +853,11 @@ private static bool TryFindTclBodyEnd( int searchColumn, out int bodyStartLineIndex, out int bodyStartColumn, - out int endLineIndex) + out TclBraceEnd bodyEnd) { bodyStartLineIndex = startLineIndex; bodyStartColumn = -1; - endLineIndex = startLineIndex; + bodyEnd = default; if (!TryFindNextNonWhitespace(lines[startLineIndex], searchColumn, out var argsColumn) || !TryFindTclWordEnd( lines, @@ -746,12 +875,11 @@ private static bool TryFindTclBodyEnd( || lines[bodyStartLineIndex][bodyStartColumn] != '{' || !braceEnds.TryGetValue( GetTclPositionKey(bodyStartLineIndex, bodyStartColumn), - out var bodyEnd)) + out bodyEnd)) { return false; } - endLineIndex = bodyEnd.Line; return true; } @@ -852,26 +980,229 @@ private static Dictionary BuildTclBraceEndPositions(IReadOnly { var result = new Dictionary(); var openings = new Stack<(int Line, int Column)>(); + var commandStart = true; + var wordStart = true; for (var lineIndex = 0; lineIndex < lines.Count; lineIndex++) { var line = lines[lineIndex]; for (var column = 0; column < line.Length; column++) { - if (line[column] == '\\') + var ch = line[column]; + if (openings.Count > 0) + { + if (ch == '\\') + { + column++; + continue; + } + if (ch == '{') + { + openings.Push((lineIndex, column)); + } + else if (ch == '}') + { + var opening = openings.Pop(); + result[GetTclPositionKey(opening.Line, opening.Column)] = + new TclBraceEnd(lineIndex, column); + } + continue; + } + + if (ch == '\\') { column++; + commandStart = false; + wordStart = false; continue; } - if (line[column] == '{') + if (ch == '"') + { + column = SkipQuotedToken(line, column, ch) - 1; + commandStart = false; + wordStart = false; + continue; + } + + if (ch == '#' && commandStart) + break; + if (ch == ';' || ch == '[') + { + commandStart = true; + wordStart = true; + continue; + } + if (char.IsWhiteSpace(ch)) + { + wordStart = true; + continue; + } + if (ch == '{' && wordStart) { openings.Push((lineIndex, column)); + commandStart = false; + wordStart = false; + continue; + } + + commandStart = false; + wordStart = false; + } + + if (openings.Count == 0) + { + commandStart = true; + wordStart = true; + } + } + + return result; + } + + private static string[] BuildTclCallLines( + IReadOnlyList lines, + IReadOnlySet scriptBodyOpenings) + { + var result = new string[lines.Count]; + var frames = new Stack(); + frames.Push(new TclLexicalFrame(TclLexicalFrameKind.Script)); + + for (var lineIndex = 0; lineIndex < lines.Count; lineIndex++) + { + var line = lines[lineIndex]; + var buffer = line.ToCharArray(); + for (var column = 0; column < line.Length;) + { + var frame = frames.Peek(); + var ch = line[column]; + if (frame.Kind == TclLexicalFrameKind.BracedWord) + { + buffer[column] = ' '; + if (ch == '\\' && column + 1 < line.Length) + { + buffer[column + 1] = ' '; + column += 2; + } + else if (ch == '{') + { + frames.Push(new TclLexicalFrame(TclLexicalFrameKind.BracedWord, '}')); + column++; + } + else if (ch == frame.Terminator) + { + frames.Pop(); + column++; + } + else + { + column++; + } + continue; + } + + if (frame.Kind == TclLexicalFrameKind.Quote) + { + buffer[column] = ' '; + if (ch == '\\' && column + 1 < line.Length) + { + buffer[column + 1] = ' '; + column += 2; + } + else if (ch == frame.Terminator) + { + frames.Pop(); + column++; + } + else if (ch == '[') + { + buffer[column] = '['; + frames.Push(new TclLexicalFrame(TclLexicalFrameKind.Script, ']')); + column++; + } + else + { + column++; + } + continue; + } + + if (frame.Terminator != '\0' && ch == frame.Terminator) + { + frames.Pop(); + buffer[column] = frame.Terminator == '}' ? ' ' : ch; + column++; + continue; + } + if (ch == '\\') + { + column += Math.Min(2, line.Length - column); + frame.CommandStart = false; + frame.WordStart = false; + continue; } - else if (line[column] == '}' && openings.Count > 0) + if (ch == '#' && frame.CommandStart) { - var opening = openings.Pop(); - result[GetTclPositionKey(opening.Line, opening.Column)] = new TclBraceEnd(lineIndex, column); + FillWithSpaces(buffer, column); + break; + } + if (ch == '"') + { + buffer[column] = ' '; + frames.Push(new TclLexicalFrame(TclLexicalFrameKind.Quote, '"')); + frame.CommandStart = false; + frame.WordStart = false; + column++; + continue; } + if (ch == '[') + { + frames.Push(new TclLexicalFrame(TclLexicalFrameKind.Script, ']')); + frame.CommandStart = false; + frame.WordStart = false; + column++; + continue; + } + if (ch == '{' && frame.WordStart) + { + if (scriptBodyOpenings.Contains(GetTclPositionKey(lineIndex, column))) + { + buffer[column] = ';'; + frames.Push(new TclLexicalFrame(TclLexicalFrameKind.Script, '}')); + } + else + { + buffer[column] = ' '; + frames.Push(new TclLexicalFrame(TclLexicalFrameKind.BracedWord, '}')); + } + frame.CommandStart = false; + frame.WordStart = false; + column++; + continue; + } + if (ch == ';') + { + frame.CommandStart = true; + frame.WordStart = true; + column++; + continue; + } + if (char.IsWhiteSpace(ch)) + { + frame.WordStart = true; + column++; + continue; + } + + frame.CommandStart = false; + frame.WordStart = false; + column++; + } + + result[lineIndex] = new string(buffer); + if (frames.Peek().Kind == TclLexicalFrameKind.Script) + { + frames.Peek().CommandStart = true; + frames.Peek().WordStart = true; } } diff --git a/src/CodeIndex/Indexer/References/ReferenceExtractor.CoreExtraction.cs b/src/CodeIndex/Indexer/References/ReferenceExtractor.CoreExtraction.cs index fd8a7f7bd..6ee2e01a7 100644 --- a/src/CodeIndex/Indexer/References/ReferenceExtractor.CoreExtraction.cs +++ b/src/CodeIndex/Indexer/References/ReferenceExtractor.CoreExtraction.cs @@ -192,6 +192,7 @@ internal static List ExtractCore(ReferenceExtractionContext req var dynamicDeclarativeState = DynamicDeclarativeReferenceExtractor.CreateState( language, preparedLines, + referenceStructuralLines, symbols); IReadOnlyList<(int StartLine, int EndLine)> csharpNamespaceScopes = language == "csharp" ? BuildCSharpNamespaceScopes(symbols) @@ -1449,7 +1450,7 @@ static string NormalizeCSharpBclRegexQualifiedName(string value) } } - return dynamicDeclarativeState?.ResolveContainer(lineNumber, container) ?? container; + return dynamicDeclarativeState?.ResolveContainer(lineNumber, column, container) ?? container; } SymbolRecord? ResolvePythonDefinitionContainer(int line, string kind) @@ -2493,6 +2494,10 @@ bool TryGetKnownPythonTypeCall(string candidate, out string canonicalName) HashSet? matchedCallIndices = null; HashSet GetMatchedCallIndices() => matchedCallIndices ??= []; + var callScanLine = dynamicDeclarativeState?.GetCallScanLine( + language, + lineNumber, + preparedLine) ?? preparedLine; if (language is "commonlisp" or "racket") { @@ -2538,7 +2543,7 @@ bool TryGetKnownPythonTypeCall(string candidate, out string canonicalName) } else { - foreach (Match match in CallRegex.Matches(preparedLine)) + foreach (Match match in CallRegex.Matches(callScanLine)) { var name = match.Groups["name"].Value; var callIndex = match.Groups["name"].Index; @@ -2595,7 +2600,7 @@ bool TryGetKnownPythonTypeCall(string candidate, out string canonicalName) else if (language is "perl" or "ambiguous_pl") { PerlReferenceExtractor.EmitAdditionalReferences( - preparedLine, + language == "ambiguous_pl" ? callScanLine : preparedLine, originalLine, references, seen, @@ -2610,7 +2615,7 @@ bool TryGetKnownPythonTypeCall(string candidate, out string canonicalName) { DynamicDeclarativeReferenceExtractor.EmitAdditionalReferences( language, - preparedLine, + callScanLine, referenceStructuralLines[i], dynamicDeclarativeState, references, diff --git a/tests/CodeIndex.Tests/DatabaseTests.cs b/tests/CodeIndex.Tests/DatabaseTests.cs index a416b79cf..c24fb6740 100644 --- a/tests/CodeIndex.Tests/DatabaseTests.cs +++ b/tests/CodeIndex.Tests/DatabaseTests.cs @@ -4720,6 +4720,48 @@ public void GetUnchangedFileId_InvalidatesPreGraphLanguageContracts_Issue4746( Assert.Null(_writer.GetUnchangedFileId(file.Path, modified, language: language)); } + [Theory] + [InlineData("crystal", 2)] + [InlineData("groovy", 2)] + [InlineData("tcl", 2)] + [InlineData("prolog", 1)] + [InlineData("ambiguous_pl", 1)] + public void GetStatus_DegradesPreGraphLanguageContractsUntilRefresh_Issue4746( + string language, + int previousContractVersion) + { + _writer.UpsertFile(new FileRecord + { + Path = $"src/legacy-status-{language}.txt", + Lang = language, + Size = 50, + Lines = 5, + Modified = new DateTime(2025, 1, 1, 0, 0, 0, DateTimeKind.Utc), + }); + var versionKey = DbContext.GetSymbolExtractorVersionMetaKey(language); + _writer.SetMeta( + versionKey, + previousContractVersion.ToString(CultureInfo.InvariantCulture)); + + var staleStatus = new DbReader(_db.Connection).GetStatus(); + + Assert.False(staleStatus.ReferenceGraphComplete); + Assert.False(staleStatus.GraphDataCurrent); + Assert.Contains( + DbReader.DynamicReferenceGraphContractStaleReason, + staleStatus.ReferenceGraphIncompleteReasons ?? []); + + _writer.SetMeta( + versionKey, + SymbolExtractor.GetContractVersion(language).ToString(CultureInfo.InvariantCulture)); + + var refreshedStatus = new DbReader(_db.Connection).GetStatus(); + + Assert.DoesNotContain( + DbReader.DynamicReferenceGraphContractStaleReason, + refreshedStatus.ReferenceGraphIncompleteReasons ?? []); + } + [Fact] public void GetUnchangedFileId_MatchesByChecksumWhenTimestampDiffers() { diff --git a/tests/CodeIndex.Tests/ReferenceExtractorTests.cs b/tests/CodeIndex.Tests/ReferenceExtractorTests.cs index 2db773297..747776053 100644 --- a/tests/CodeIndex.Tests/ReferenceExtractorTests.cs +++ b/tests/CodeIndex.Tests/ReferenceExtractorTests.cs @@ -1202,6 +1202,7 @@ def fake TEXT regex = /ready?(value)/ literal = %q(save!(value)) + value %= 2 ready? value ready?(value) save! value @@ -1241,6 +1242,7 @@ def fake() {} """ def slashy = /helper()/ def dollarSlashy = $/helper()/$ + def returned = { return /helper()/ } fake 1 helper() } @@ -1268,12 +1270,16 @@ set substituted [helper] } proc multiline {} { set value 1 ;# [helper] + set color #fff; helper + set quoted "[helper]" + set braced {[helper]} helper } proc split {value} { helper } + proc sameLine {} { helper }; helper """; var (_, references) = ExtractSymbolsAndReferences("tcl", content); @@ -1281,9 +1287,14 @@ proc split {value} AssertReferencesContain(references, "call", "run", "helper"); AssertReferencesContain(references, "call", "multiline", "helper"); AssertReferencesContain(references, "call", "split", "helper"); - Assert.Equal(3, references.Count(reference => + AssertReferencesContain(references, "call", "sameLine", "helper"); + Assert.Equal(7, references.Count(reference => reference.ReferenceKind == "call" && reference.SymbolName == "helper")); + Assert.Single(references, reference => + reference.ReferenceKind == "call" + && reference.SymbolName == "helper" + && reference.ContainerName == null); } [Fact] @@ -1296,6 +1307,8 @@ public void Extract_Prolog_IndexesRelativeImportsAndDecimalHeadsWhileMaskingComm helper. quoted('value.') :- helper. + bar(_). + nested(bar(X)). % helper(). /* helper(). @@ -1314,7 +1327,7 @@ public void Extract_Prolog_IndexesRelativeImportsAndDecimalHeadsWhileMaskingComm && reference.SymbolName == "helper")); Assert.DoesNotContain(references, reference => reference.ReferenceKind == "call" - && reference.SymbolName == "threshold"); + && reference.SymbolName is "threshold" or "bar"); } [Fact] @@ -1327,6 +1340,7 @@ public void Extract_AmbiguousPl_MasksPerlAndPrologCommentsAndPod_Issue4746() clp_entry(X) :- X #= 1, helper. # helper(); % helper(). + %helper(). =pod helper(); =cut From c7a8f7cf0ca994ceece8ac75511b4ded89815b30 Mon Sep 17 00:00:00 2001 From: Widthdom Date: Fri, 24 Jul 2026 14:48:14 +0900 Subject: [PATCH 05/26] Address dynamic graph adversarial findings (#4746) --- AGENT_GUIDE.md | 2 +- README.md | 9 + USER_GUIDE.md | 4 +- changelog.d/unreleased/4746.added.md | 6 +- .../DynamicDeclarativeReferenceExtractor.cs | 218 ++++++++++++++++-- .../ReferenceExtractor.CoreExtraction.cs | 76 +++--- .../Symbols/SymbolExtractor.ExtractCore.cs | 8 +- .../ReferenceExtractorTests.cs | 32 ++- 8 files changed, 285 insertions(+), 70 deletions(-) diff --git a/AGENT_GUIDE.md b/AGENT_GUIDE.md index 48d3a04f2..be3c4ac9d 100644 --- a/AGENT_GUIDE.md +++ b/AGENT_GUIDE.md @@ -141,7 +141,7 @@ CI watching must be bounded. Do not loop indefinitely. - When any readiness field is degraded, the CLI adds `degraded_root_cause`, `degraded_reason`, `recommended_action`, `alternative_action`, and `readiness_degradations[]`. `degraded_root_cause` is the primary stable machine code; `readiness_degradations[]` lists every degraded field with `root_cause`, human reason, and remediation strings. - `hotspot_family_degraded_reason` currently uses `hotspot_family_support_not_indexed`, `hotspot_family_metadata_stale`, `hotspot_family_disabled_at_index_time`, `partial_family_key_population`, and `hotspot_family_marker_fingerprint_incomplete`; the incomplete marker fingerprint code means marker traversal hit safety caps and should stay synchronized with README / developer-guide recovery notes. - `issues_table_available` reports physical `file_issues` table presence only. `file_issues_data_current` reports whether the table is also stamped current for the active index generation. -- `graph_table_available` reports a queryable persisted reference generation, while `graph_data_current`, `reference_graph_complete`, and `index_complete` report current-generation coverage. Reference extraction is bounded at 50,000 lookup symbols, 20,000 lookup lines, 512 names per line, and 20,000 container candidates; `reference_extraction_limits`, `reference_graph_incomplete_reasons`, and `reference_extraction_cap_hits` publish cap state, and `last_index_run.reference_extraction_cap_hits` snapshots it per run. Cap hits persist per file and propagate degraded, non-authoritative absence semantics to callers, callees, deps, and impact. A per-file extraction failure keeps successful graph rows queryable, stamps completeness false with bounded `last_failed_or_partial_index_run.file_errors`, and returns exit `11` unless `index --allow-partial` explicitly opts into exit `0`. While such file failures remain unresolved, a later scoped update automatically uses the normal incremental full-scan path so unrelated targets cannot clear the failure and successful recovery can restore every workspace-wide readiness contract without `--rebuild`. +- `graph_table_available` reports a queryable persisted reference generation, while `graph_data_current`, `reference_graph_complete`, and `index_complete` report current-generation coverage. Reference extraction is bounded at 50,000 lookup symbols, 20,000 lookup lines, 512 names per line, and 20,000 container candidates; `reference_extraction_limits`, `reference_graph_incomplete_reasons`, and `reference_extraction_cap_hits` publish cap state, and `last_index_run.reference_extraction_cap_hits` snapshots it per run. Cap hits persist per file and propagate degraded, non-authoritative absence semantics to callers, callees, deps, and impact. Indexed Crystal, Groovy, Tcl, Prolog, or `ambiguous_pl` rows with a missing or stale extractor stamp add `dynamic_reference_graph_contract_stale` to `reference_graph_incomplete_reasons` and keep graph readiness false until a normal index refresh rewrites them. A per-file extraction failure keeps successful graph rows queryable, stamps completeness false with bounded `last_failed_or_partial_index_run.file_errors`, and returns exit `11` unless `index --allow-partial` explicitly opts into exit `0`. While such file failures remain unresolved, a later scoped update automatically uses the normal incremental full-scan path so unrelated targets cannot clear the failure and successful recovery can restore every workspace-wide readiness contract without `--rebuild`. - `index_writer_version` records the `cdidx` version that last wrote to the DB (stamped into `codeindex_meta` as `cdidx_writer_version` on every full scan, update, and MCP index). `index_newer_than_reader` flips to `true` whenever any persisted numeric contract stamp in `codeindex_meta` (or unknown `PRAGMA user_version` readiness bits) exceeds the current binary's compiled maximum, so an older CLI re-opening a DB written by a newer CLI degrades loudly with an audit trail instead of silently dropping back to text-search fallbacks. `index_newer_than_reader_reason` enumerates the specific newer-than-reader stamps. - `status` also surfaces indexed-HEAD freshness via `indexed_head_sha`, `indexed_head_branch`, `indexed_head_timestamp`, `commits_ahead_of_indexed_head`, and the compact `head_freshness` summary. They are stamped by `cdidx index` on every successful run (full scan AND partial update, distinct from `indexed_head_commit` which is full-scan only) on a best-effort basis (never blocks an otherwise-successful index) and omitted on non-git workspaces, detached HEAD (branch only), or legacy DBs created before this contract. `worktree_head_changed` compares runtime HEAD with this latest stamp when available and falls back to `indexed_head_commit` only for legacy DBs. `head_freshness.state=fresh` requires `status --check` to match the workspace, `fresh_but_incomplete` keeps matching-workspace freshness distinct from incomplete extraction coverage, and `state=head_current` only means the runtime HEAD matches the `indexed_head` selected by `indexed_head_source`. - `status` also surfaces unknown-extension scan coverage via `unknown_extension_file_count`, stamped by successful full-repository index runs (`cdidx index ` and MCP `index_project`) as the number of non-indexed files with non-empty extensions that do not map to a known language. Current scans also stamp `unknown_extension_files` as a path sample bounded by `unknown_extension_file_path_limit` items and the string-list decoded-character budget, `unknown_extension_files_truncated` when more paths existed than were emitted for either bound, and `unknown_extension_file_path_limit` as the item cap rather than a guarantee that that many paths are returned. Newer scans also expose `unknown_extension_extension_counts`, `unknown_extension_category_counts`, and `unknown_extension_groups`; groups classify common non-code buckets such as repository metadata, licenses, binary assets, configuration, structural metadata, and language-support candidates, and include `recommended_action` values of `ignore_configuration`, `first_class_structural_extraction`, or `language_support`. These fields are omitted on legacy DBs or before a current full scan has stamped them. diff --git a/README.md b/README.md index 5f3df7898..6e43b46dd 100644 --- a/README.md +++ b/README.md @@ -232,6 +232,11 @@ last successful run in `reference_extraction_cap_hits`. In that state `reference_graph_complete=false`; CLI/MCP callers, callees, deps, and impact responses carry the same stable reasons, and a zero result is not an authoritative absence. +Indexed Crystal, Groovy, Tcl, Prolog, or `ambiguous_pl` files with a missing or +older extractor stamp similarly report +`dynamic_reference_graph_contract_stale` and keep +`reference_graph_complete=false` / `graph_data_current=false` until +`cdidx index ` refreshes their graph rows. Database Unix-mode hardening defaults to `database_permission_policy=best_effort`. If a filesystem permits SQLite I/O but rejects mode reads or changes, cdidx continues, @@ -624,6 +629,10 @@ CLI の `languages --json` / `status --json` と対応する MCP response は `reference_extraction_cap_hits` に集約します。この状態では `reference_graph_complete=false` となり、CLI/MCP の callers / callees / deps / impact response にも 同じ stable reason が伝播するため、0件という結果は authoritative な不在を意味しません。 +index 済みの Crystal、Groovy、Tcl、Prolog、`ambiguous_pl` file で extractor +stamp が欠落または古い場合も `dynamic_reference_graph_contract_stale` を報告し、 +`cdidx index ` が graph row を更新するまで +`reference_graph_complete=false` / `graph_data_current=false` を維持します。 database の Unix mode hardening は既定で `database_permission_policy=best_effort` です。SQLite I/O は可能でも mode の diff --git a/USER_GUIDE.md b/USER_GUIDE.md index e08ad7f5d..85c5a0788 100644 --- a/USER_GUIDE.md +++ b/USER_GUIDE.md @@ -2376,7 +2376,7 @@ All indexed languages are searchable through FTS5. Rows with **Symbols = yes** a - R: function assignments, S4/R6 class declarations, validity/generic/method declarations, inherit vectors, public/private/active methods, and `library` / `require` imports are indexed. - Functional symbol-only languages: Clojure, Erlang, OCaml, and Raku expose conservative declarations as symbols. References and graph queries are not advertised for these languages yet. - Dynamic symbol-only languages: Julia exposes conservative declarations as symbols. References and graph queries are not advertised for Julia yet. -- Dynamic/declarative graph languages: Crystal, Groovy, Tcl, and Prolog expose conservative declarations, imports, and call relationships. Parenthesized calls use the shared extractor; command-style calls are limited to callables declared in the same file, and Tcl proc / Prolog predicate bodies preserve caller containers. +- Dynamic/declarative graph languages: Crystal, Groovy, Tcl, and Prolog expose conservative declarations, imports, and call relationships. Crystal, Groovy, and Prolog parenthesized calls use the shared extractor; command-style calls are limited to callables declared in the same file. Tcl recognizes command substitutions and common control-command script arguments without treating ordinary `name()` words as calls, while Tcl proc / Prolog predicate bodies preserve caller containers. An index created before this graph contract reports `reference_graph_complete=false` and `graph_data_current=false` with `dynamic_reference_graph_contract_stale`; rerun `cdidx index ` to refresh affected rows before treating absent edges as authoritative. - Systems symbol-only languages: Ada, D, and Nim expose conservative declarations as symbols. References and graph queries are not advertised for these languages yet. - Markdown, JSON/YAML, and CSS: Markdown heading and local-anchor symbols are indexed; JSON/YAML configuration keys are indexed as structural key paths; CSS variables, placeholders, and `@extend` references are indexed. @@ -5524,7 +5524,7 @@ indexing はファイル単位の SQLite transaction を commit します。長 - R: 関数代入、S4/R6 class 宣言、validity/generic/method 宣言、inherit vector、public/private/active method、`library` / `require` import を索引します。 - 関数型言語のシンボル専用対応: Clojure、Erlang、OCaml、Raku は保守的な宣言をシンボルとして公開します。これらの言語では references と graph queries はまだ対応として広告しません。 - 動的言語のシンボル専用対応: Julia は保守的な宣言をシンボルとして公開します。Julia の references と graph queries はまだ対応として広告しません。 -- 動的・宣言型言語の graph 対応: Crystal、Groovy、Tcl、Prolog は保守的な宣言、import、call relationship を公開します。括弧付き call は共通 extractor を使い、command-style call は同一ファイルで宣言済みの callable に限定し、Tcl proc / Prolog predicate の本体では caller container を保持します。 +- 動的・宣言型言語の graph 対応: Crystal、Groovy、Tcl、Prolog は保守的な宣言、import、call relationship を公開します。Crystal、Groovy、Prolog の括弧付き call は共通 extractor を使い、command-style call は同一ファイルで宣言済みの callable に限定します。Tcl は通常の `name()` word を call とみなさず、command substitution と主要な制御 command の script 引数を認識し、Tcl proc / Prolog predicate の本体では caller container を保持します。 この graph contract より前に作成された index は `dynamic_reference_graph_contract_stale` とともに `reference_graph_complete=false`、`graph_data_current=false` を報告します。欠落 edge を authoritative とみなす前に `cdidx index ` を再実行して対象 row を更新してください。 - システム系言語のシンボル専用対応: Ada、D、Nim は保守的な宣言をシンボルとして公開します。これらの言語では references と graph queries はまだ対応として広告しません。 - Markdown、JSON/YAML、CSS: Markdown の heading / local anchor、JSON/YAML の configuration key path、CSS の variable、placeholder、`@extend` をシンボルとして扱います。 diff --git a/changelog.d/unreleased/4746.added.md b/changelog.d/unreleased/4746.added.md index 0cce6af8f..e7cc26573 100644 --- a/changelog.d/unreleased/4746.added.md +++ b/changelog.d/unreleased/4746.added.md @@ -19,6 +19,8 @@ affected: - tests/CodeIndex.Tests/ReferenceExtractorTests.cs - tests/CodeIndex.Tests/QueryCommandRunnerTests.cs - tests/CodeIndex.Tests/golden/status.json + - README.md + - AGENT_GUIDE.md - USER_GUIDE.md - DEVELOPER_GUIDE.md - TESTING_GUIDE.md @@ -26,8 +28,8 @@ affected: ## English -- **Crystal, Groovy, Tcl, Prolog, and ambiguous `.pl` files now support conservative reference graphs (#4746)** — indexing records imports and parenthesized calls, recognizes command-style calls only for callables declared in the same file, preserves Tcl proc and Prolog predicate caller containers, and reports structured symbol/reference/graph capabilities after `.pl` content classification. Pre-graph extractor stamps keep graph readiness degraded until a normal index refresh rewrites affected rows. +- **Crystal, Groovy, Tcl, Prolog, and ambiguous `.pl` files now support conservative reference graphs (#4746)** — indexing records imports and language-appropriate calls, recognizes command-style calls only for callables declared in the same file, parses Tcl command substitutions and common control-command script arguments, preserves Tcl proc and Prolog predicate caller containers, and reports structured symbol/reference/graph capabilities after `.pl` content classification. Pre-graph extractor stamps keep graph readiness degraded until a normal index refresh rewrites affected rows. ## 日本語 -- **Crystal、Groovy、Tcl、Prolog、曖昧な `.pl` ファイルが保守的な reference graph に対応しました（#4746）** — import と括弧付き call を記録し、command-style call は同一ファイルで宣言済みの callable だけを認識し、Tcl proc / Prolog predicate の caller container を保持して、`.pl` の content classification 後に構造化された symbol / reference / graph capability を報告します。graph 対応前の extractor stamp がある場合、通常の index refresh で対象 row を更新するまで graph readiness を degraded に保ちます。 +- **Crystal、Groovy、Tcl、Prolog、曖昧な `.pl` ファイルが保守的な reference graph に対応しました（#4746）** — import と言語ごとの call 構文を記録し、command-style call は同一ファイルで宣言済みの callable だけを認識し、Tcl の command substitution と主要な制御 command の script 引数を解析し、Tcl proc / Prolog predicate の caller container を保持して、`.pl` の content classification 後に構造化された symbol / reference / graph capability を報告します。graph 対応前の extractor stamp がある場合、通常の index refresh で対象 row を更新するまで graph readiness を degraded に保ちます。 diff --git a/src/CodeIndex/Indexer/References/Languages/DynamicDeclarativeReferenceExtractor.cs b/src/CodeIndex/Indexer/References/Languages/DynamicDeclarativeReferenceExtractor.cs index 5e3dff7ad..e6856c56e 100644 --- a/src/CodeIndex/Indexer/References/Languages/DynamicDeclarativeReferenceExtractor.cs +++ b/src/CodeIndex/Indexer/References/Languages/DynamicDeclarativeReferenceExtractor.cs @@ -130,6 +130,20 @@ private sealed class TclLexicalFrame( public char Terminator { get; } = terminator; public bool CommandStart { get; set; } = true; public bool WordStart { get; set; } = true; + public int WordIndex { get; set; } + public string? CommandName { get; set; } + public string? LastBareWord { get; set; } + public int LastBareWordIndex { get; set; } = -1; + + public void ResetCommand() + { + CommandStart = true; + WordStart = true; + WordIndex = 0; + CommandName = null; + LastBareWord = null; + LastBareWordIndex = -1; + } } public static string[] MaskNonCodeLines(string language, IReadOnlyList lines) @@ -144,6 +158,7 @@ public static string[] MaskNonCodeLines(string language, IReadOnlyList l var insideAmbiguousPerlPod = false; var insideSlashyLiteral = false; var insideGroovyDollarSlashyLiteral = false; + char crystalMultilineQuote = '\0'; char crystalPercentOpeningDelimiter = '\0'; char crystalPercentClosingDelimiter = '\0'; var crystalPercentDelimiterDepth = 0; @@ -183,6 +198,26 @@ public static string[] MaskNonCodeLines(string language, IReadOnlyList l var buffer = line.ToCharArray(); for (var column = 0; column < line.Length;) { + if (crystalMultilineQuote != '\0') + { + buffer[column] = ' '; + if (line[column] == '\\' && column + 1 < line.Length) + { + buffer[column + 1] = ' '; + column += 2; + } + else if (line[column] == crystalMultilineQuote) + { + crystalMultilineQuote = '\0'; + column++; + } + else + { + column++; + } + continue; + } + if (insideBlockComment) { buffer[column] = ' '; @@ -300,6 +335,15 @@ public static string[] MaskNonCodeLines(string language, IReadOnlyList l if (ch is '\'' or '"' or '`') { + if (language == "crystal" + && ch is '"' or '`' + && !HasClosingQuotedDelimiter(line, column, ch)) + { + crystalMultilineQuote = ch; + column = line.Length; + continue; + } + column = SkipQuotedToken(line, column, ch); continue; } @@ -441,12 +485,17 @@ public static string[] MaskNonCodeLines(string language, IReadOnlyList l if (language == "tcl") { var tclScriptBodyOpenings = new HashSet(); + var tclBraceEnds = BuildTclBraceEndPositions(structuralLines); AddTclContainers( structuralLines, symbols, + tclBraceEnds, tclContainerScopes, tclScriptBodyOpenings); - tclCallLines = BuildTclCallLines(structuralLines, tclScriptBodyOpenings); + tclCallLines = BuildTclCallLines( + structuralLines, + tclBraceEnds, + tclScriptBodyOpenings); } else if (language is "prolog" or "ambiguous_pl") AddPrologContainers(preparedLines, symbols, containersByLine); @@ -556,6 +605,26 @@ private static int SkipQuotedToken(string line, int startColumn, char delimiter) return line.Length; } + private static bool HasClosingQuotedDelimiter( + string line, + int startColumn, + char delimiter) + { + for (var column = startColumn + 1; column < line.Length; column++) + { + if (line[column] == '\\') + { + column++; + continue; + } + + if (line[column] == delimiter) + return true; + } + + return false; + } + private static void FillWithSpaces(char[] buffer, int startColumn) { for (var column = startColumn; column < buffer.Length; column++) @@ -730,7 +799,8 @@ private static string PreparePrologCallScanLine( return new string(masked); } - if (ch == '.' && IsOnlyWhitespaceAfter(line, column + 1)) + if (IsPrologClauseTerminator(line, column) + && IsOnlyWhitespaceAfter(line, column + 1)) return new string(' ', line.Length); } @@ -747,6 +817,20 @@ private static bool IsOnlyWhitespaceAfter(string line, int startColumn) return true; } + internal static bool IsPrologClauseTerminator(string line, int column) + { + if (column < 0 || column >= line.Length || line[column] != '.') + return false; + + var previous = column > 0 ? line[column - 1] : '\0'; + var next = column + 1 < line.Length ? line[column + 1] : '\0'; + if (previous == '.' || next == '.') + return false; + if (char.IsDigit(previous) && char.IsDigit(next)) + return false; + return IsOnlyWhitespaceAfter(line, column + 1); + } + private static void EmitImportReference( string language, string originalLine, @@ -804,10 +888,10 @@ private static string NormalizeImportTarget(string language, string name) private static void AddTclContainers( IReadOnlyList lines, IReadOnlyList symbols, + IReadOnlyDictionary braceEnds, List scopes, HashSet scriptBodyOpenings) { - var braceEnds = BuildTclBraceEndPositions(lines); foreach (var symbol in symbols) { if (symbol.Kind != "function" || symbol.StartLine < 1 || symbol.StartLine > lines.Count) @@ -1061,6 +1145,7 @@ private static Dictionary BuildTclBraceEndPositions(IReadOnly private static string[] BuildTclCallLines( IReadOnlyList lines, + IReadOnlyDictionary braceEnds, IReadOnlySet scriptBodyOpenings) { var result = new string[lines.Count]; @@ -1071,6 +1156,9 @@ private static string[] BuildTclCallLines( { var line = lines[lineIndex]; var buffer = line.ToCharArray(); + var lineContinued = false; + var suppressLeadingContinuedWord = frames.Peek().Kind != TclLexicalFrameKind.Script + || !frames.Peek().CommandStart; for (var column = 0; column < line.Length;) { var frame = frames.Peek(); @@ -1117,6 +1205,7 @@ private static string[] BuildTclCallLines( { buffer[column] = '['; frames.Push(new TclLexicalFrame(TclLexicalFrameKind.Script, ']')); + suppressLeadingContinuedWord = false; column++; } else @@ -1135,6 +1224,9 @@ private static string[] BuildTclCallLines( } if (ch == '\\') { + if (frame.WordStart) + frame.WordIndex++; + lineContinued = column + 1 >= line.Length; column += Math.Min(2, line.Length - column); frame.CommandStart = false; frame.WordStart = false; @@ -1147,6 +1239,8 @@ private static string[] BuildTclCallLines( } if (ch == '"') { + if (frame.WordStart) + frame.WordIndex++; buffer[column] = ' '; frames.Push(new TclLexicalFrame(TclLexicalFrameKind.Quote, '"')); frame.CommandStart = false; @@ -1156,18 +1250,33 @@ private static string[] BuildTclCallLines( } if (ch == '[') { + if (frame.WordStart) + frame.WordIndex++; frames.Push(new TclLexicalFrame(TclLexicalFrameKind.Script, ']')); frame.CommandStart = false; frame.WordStart = false; + suppressLeadingContinuedWord = false; column++; continue; } if (ch == '{' && frame.WordStart) { - if (scriptBodyOpenings.Contains(GetTclPositionKey(lineIndex, column))) + var wordIndex = frame.WordIndex++; + var positionKey = GetTclPositionKey(lineIndex, column); + var isScriptArgument = + scriptBodyOpenings.Contains(positionKey) + || IsTclScriptArgument( + frame, + wordIndex, + lines, + braceEnds.TryGetValue(positionKey, out var braceEnd) + ? braceEnd + : null); + if (isScriptArgument) { buffer[column] = ';'; frames.Push(new TclLexicalFrame(TclLexicalFrameKind.Script, '}')); + suppressLeadingContinuedWord = false; } else { @@ -1181,8 +1290,8 @@ private static string[] BuildTclCallLines( } if (ch == ';') { - frame.CommandStart = true; - frame.WordStart = true; + frame.ResetCommand(); + suppressLeadingContinuedWord = false; column++; continue; } @@ -1193,22 +1302,101 @@ private static string[] BuildTclCallLines( continue; } + if (frame.WordStart) + { + var wordIndex = frame.WordIndex++; + var token = ReadTclBareWord(line, column); + if (wordIndex == 0) + frame.CommandName = token; + if (token.Length > 0) + { + frame.LastBareWord = token; + frame.LastBareWordIndex = wordIndex; + } + + if (suppressLeadingContinuedWord) + { + FillWithSpaces(buffer, column, column + token.Length); + suppressLeadingContinuedWord = false; + } + } + frame.CommandStart = false; frame.WordStart = false; column++; } result[lineIndex] = new string(buffer); - if (frames.Peek().Kind == TclLexicalFrameKind.Script) - { - frames.Peek().CommandStart = true; - frames.Peek().WordStart = true; - } + if (!lineContinued && frames.Peek().Kind == TclLexicalFrameKind.Script) + frames.Peek().ResetCommand(); } return result; } + private static string ReadTclBareWord(string line, int startColumn) + { + var endColumn = startColumn; + while (endColumn < line.Length + && (char.IsLetterOrDigit(line[endColumn]) + || line[endColumn] is '_' or ':' or '.' or '-')) + { + endColumn++; + } + + return endColumn == startColumn + ? string.Empty + : line[startColumn..endColumn]; + } + + private static bool IsTclScriptArgument( + TclLexicalFrame frame, + int wordIndex, + IReadOnlyList lines, + TclBraceEnd? braceEnd) + { + var isLastCommandWord = braceEnd is { } end + && IsTclLastCommandWord(lines, end); + return frame.CommandName switch + { + "if" => wordIndex == 2 + || (frame.LastBareWord == "then" + && wordIndex == frame.LastBareWordIndex + 1) + || (frame.LastBareWord == "elseif" + && wordIndex == frame.LastBareWordIndex + 2) + || (frame.LastBareWord == "else" + && wordIndex == frame.LastBareWordIndex + 1), + "foreach" or "lmap" => wordIndex >= 3 + && wordIndex % 2 == 1 + && isLastCommandWord, + "while" => wordIndex == 2, + "catch" => wordIndex == 1, + "for" => wordIndex is 1 or 3 or 4, + "eval" => wordIndex >= 1, + "after" => wordIndex == 2, + "try" => wordIndex == 1, + "namespace" => wordIndex == 3, + "dict" => frame.LastBareWord == "for" + && wordIndex == frame.LastBareWordIndex + 3, + _ => false, + }; + } + + private static bool IsTclLastCommandWord( + IReadOnlyList lines, + TclBraceEnd braceEnd) + { + var line = lines[braceEnd.Line]; + for (var column = braceEnd.Column + 1; column < line.Length; column++) + { + if (char.IsWhiteSpace(line[column])) + continue; + return line[column] is ';' or ']' or '}'; + } + + return true; + } + private static long GetTclPositionKey(int line, int column) => ((long)line << 32) | (uint)column; @@ -1243,13 +1431,7 @@ private static int FindPrologClauseEnd(IReadOnlyList lines, int startLin var line = lines[lineIndex]; for (var column = 0; column < line.Length; column++) { - if (line[column] != '.') - continue; - - var previousIsDigit = column > 0 && char.IsDigit(line[column - 1]); - var nextIsIdentifier = column + 1 < line.Length - && (char.IsLetterOrDigit(line[column + 1]) || line[column + 1] == '_'); - if (!previousIsDigit && !nextIsIdentifier) + if (IsPrologClauseTerminator(line, column)) return lineIndex; } } diff --git a/src/CodeIndex/Indexer/References/ReferenceExtractor.CoreExtraction.cs b/src/CodeIndex/Indexer/References/ReferenceExtractor.CoreExtraction.cs index 6ee2e01a7..985e0c602 100644 --- a/src/CodeIndex/Indexer/References/ReferenceExtractor.CoreExtraction.cs +++ b/src/CodeIndex/Indexer/References/ReferenceExtractor.CoreExtraction.cs @@ -2538,49 +2538,51 @@ bool TryGetKnownPythonTypeCall(string candidate, out string canonicalName) } else if (language is "assembly") { - // Assembly references are operand-driven, not `name(...)` call syntax. Running the - // shared CallRegex would misread addressing forms such as `foo(%rip)` as calls. + // Assembly references are operand-driven, not `name(...)` call syntax. } else { - foreach (Match match in CallRegex.Matches(callScanLine)) + if (language != "tcl") { - var name = match.Groups["name"].Value; - var callIndex = match.Groups["name"].Index; - if (language == "rust" && RustReferenceExtractor.IsRawIdentifierPrefix(preparedLine, callIndex)) - continue; - if (language == "objc" && IsObjCSelectorLiteralCall(preparedLine, name, callIndex)) - continue; - if (sqlSuppressedCallIndices != null && sqlSuppressedCallIndices.Contains(callIndex)) - continue; - if (sqlWindowFunctionCallSiteSuppressions != null - && sqlWindowFunctionCallSiteSuppressions.Contains((lineNumber, callIndex))) - continue; - GetMatchedCallIndices().Add(callIndex); - if (TryAddCallLikeReference(name, callIndex)) + foreach (Match match in CallRegex.Matches(callScanLine)) { - EmitGenericInvocationTypeArgumentReferences( - language, - preparedLine, - callIndex, - references, - seen, - fileId, - context, - lineNumber, - ResolveContainerForCall(callIndex)); + var name = match.Groups["name"].Value; + var callIndex = match.Groups["name"].Index; + if (language == "rust" && RustReferenceExtractor.IsRawIdentifierPrefix(preparedLine, callIndex)) + continue; + if (language == "objc" && IsObjCSelectorLiteralCall(preparedLine, name, callIndex)) + continue; + if (sqlSuppressedCallIndices != null && sqlSuppressedCallIndices.Contains(callIndex)) + continue; + if (sqlWindowFunctionCallSiteSuppressions != null + && sqlWindowFunctionCallSiteSuppressions.Contains((lineNumber, callIndex))) + continue; + GetMatchedCallIndices().Add(callIndex); + if (TryAddCallLikeReference(name, callIndex)) + { + EmitGenericInvocationTypeArgumentReferences( + language, + preparedLine, + callIndex, + references, + seen, + fileId, + context, + lineNumber, + ResolveContainerForCall(callIndex)); + } + if (language == "ruby") + RubyReferenceExtractor.EmitCommandTargetReferences( + name, + callIndex, + originalLine, + references, + seen, + fileId, + context, + lineNumber, + ResolveContainerForCall); } - if (language == "ruby") - RubyReferenceExtractor.EmitCommandTargetReferences( - name, - callIndex, - originalLine, - references, - seen, - fileId, - context, - lineNumber, - ResolveContainerForCall); } if (language == "ruby") diff --git a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.ExtractCore.cs b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.ExtractCore.cs index b7dbc50a7..5332f7556 100644 --- a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.ExtractCore.cs +++ b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.ExtractCore.cs @@ -2196,13 +2196,7 @@ private static bool HasPrologClauseTerminator(string line) continue; } - if (line[column] != '.') - continue; - - var previousIsDigit = column > 0 && char.IsDigit(line[column - 1]); - var nextIsIdentifier = column + 1 < line.Length - && (char.IsLetterOrDigit(line[column + 1]) || line[column + 1] == '_'); - if (!previousIsDigit && !nextIsIdentifier) + if (DynamicDeclarativeReferenceExtractor.IsPrologClauseTerminator(line, column)) return true; } diff --git a/tests/CodeIndex.Tests/ReferenceExtractorTests.cs b/tests/CodeIndex.Tests/ReferenceExtractorTests.cs index 747776053..78b1f9779 100644 --- a/tests/CodeIndex.Tests/ReferenceExtractorTests.cs +++ b/tests/CodeIndex.Tests/ReferenceExtractorTests.cs @@ -1202,6 +1202,10 @@ def fake TEXT regex = /ready?(value)/ literal = %q(save!(value)) + ordinary = "first + ready?(value) + def phantom + last" value %= 2 ready? value ready?(value) @@ -1214,6 +1218,7 @@ fake value var (symbols, references) = ExtractSymbolsAndReferences("crystal", content); Assert.DoesNotContain(symbols, symbol => symbol.Kind == "function" && symbol.Name == "fake"); + Assert.DoesNotContain(symbols, symbol => symbol.Kind == "function" && symbol.Name == "phantom"); Assert.Equal(2, references.Count(reference => reference.ReferenceKind == "call" && reference.ContainerName == "run" @@ -1273,6 +1278,22 @@ proc multiline {} { set color #fff; helper set quoted "[helper]" set braced {[helper]} + if {1} { + helper + } + foreach item {1 2} { + helper + } + while {0} { + helper + } + catch { + helper + } + puts \ + helper + set callback helper() + puts helper() helper } proc split {value} @@ -1288,7 +1309,7 @@ proc split {value} AssertReferencesContain(references, "call", "multiline", "helper"); AssertReferencesContain(references, "call", "split", "helper"); AssertReferencesContain(references, "call", "sameLine", "helper"); - Assert.Equal(7, references.Count(reference => + Assert.Equal(11, references.Count(reference => reference.ReferenceKind == "call" && reference.SymbolName == "helper")); Assert.Single(references, reference => @@ -1309,6 +1330,10 @@ public void Extract_Prolog_IndexesRelativeImportsAndDecimalHeadsWhileMaskingComm helper. bar(_). nested(bar(X)). + build(Term) :- + Term =.. [node, + value], + helper. % helper(). /* helper(). @@ -1319,11 +1344,12 @@ public void Extract_Prolog_IndexesRelativeImportsAndDecimalHeadsWhileMaskingComm Assert.Contains(symbols, symbol => symbol.Kind == "function" && symbol.Name == "threshold"); Assert.Contains(symbols, symbol => symbol.Kind == "function" && symbol.Name == "quoted"); + Assert.Contains(symbols, symbol => symbol.Kind == "function" && symbol.Name == "build"); Assert.Single(symbols, symbol => symbol.Kind == "function" && symbol.Name == "helper"); AssertReferencesContain(references, "type_reference", null, "support"); - Assert.Equal(2, references.Count(reference => + Assert.Equal(3, references.Count(reference => reference.ReferenceKind == "call" - && reference.ContainerName is "threshold" or "quoted" + && reference.ContainerName is "threshold" or "quoted" or "build" && reference.SymbolName == "helper")); Assert.DoesNotContain(references, reference => reference.ReferenceKind == "call" From 4eacd137f5b4d246efde7e3964b7cb7d2ec6fbfb Mon Sep 17 00:00:00 2001 From: Widthdom Date: Fri, 24 Jul 2026 15:33:23 +0900 Subject: [PATCH 06/26] Refine dynamic reference graph semantics (#4746) --- USER_GUIDE.md | 6 +- changelog.d/unreleased/4746.added.md | 5 +- .../DynamicDeclarativeReferenceExtractor.cs | 628 +++++++++++++++++- .../ReferenceExtractor.CoreExtraction.cs | 12 +- .../ReferenceExtractor.TypeReferences.cs | 6 +- .../Indexer/References/ReferenceExtractor.cs | 4 +- .../Indexer/Symbols/SymbolExtractor.cs | 4 +- .../ReferenceExtractorTests.cs | 71 +- tests/CodeIndex.Tests/SymbolExtractorTests.cs | 4 +- 9 files changed, 690 insertions(+), 50 deletions(-) diff --git a/USER_GUIDE.md b/USER_GUIDE.md index 85c5a0788..399050c78 100644 --- a/USER_GUIDE.md +++ b/USER_GUIDE.md @@ -2418,7 +2418,8 @@ entries with the unsupported capability, an explanatory message, and | Java / Kotlin / Scala | packages/imports, classes/interfaces, methods, properties | calls, constructors, annotations, type references | Kotlin inline lambda body modeling is limited; verify with `references` before relying on deep call chains. | | JavaScript / TypeScript / Vue / Svelte | functions, classes, exports, imports, variables | calls, constructors, static/dynamic imports, workers, service workers | Dynamic property calls and computed module specifiers are best-effort. `cdidx references render --lang typescript` | | Python / Ruby / PHP / Perl / R | functions, classes/modules, imports where supported | calls, constructors, decorators/annotations where supported | Dynamic dispatch and metaprogramming may require `search`. PHPDoc/static import patterns are indexed when statically visible. | -| MATLAB / Prolog | classes/modules, functions/predicates, imports | none yet | `.m` / `.pl` are classified conservatively; declaration symbols are searchable, but use `search` for reference and graph questions. | +| MATLAB | classes, functions, imports | none yet | `.m` files are classified conservatively; declaration symbols are searchable, but use `search` for reference and graph questions. | +| Prolog / ambiguous `.pl` | modules, predicates, imports | conservative same-file predicate calls and imports | Calls are recorded only in goal positions for predicates declared in the same file. Ambiguous `.pl` retains its content classification while exposing the safe union of Perl and Prolog structure. | | Cython | `cdef` / `cpdef` declarations, cimports, extern declarations | none yet | Cython native-extension declarations are searchable as symbols; use `search` for call/reference questions. | | C / C++ / Objective-C / Swift / Rust / Go / Zig | functions, types, methods, imports/modules | calls, constructors, macro invocations where supported, type references | C++ templates/macros and Rust macro expansion are not evaluated; Rust macro invocations are still reference edges. | | CUDA | C++-style functions/types plus CUDA kernel/device/host sub-kinds | none yet | CUDA kernel/device/host declarations are indexed as C++-style symbols with CUDA sub-kinds; use `search` for call/reference questions. | @@ -5560,7 +5561,8 @@ indexing はファイル単位の SQLite transaction を commit します。長 | Java / Kotlin / Scala | package/import、class/interface、method、property | call、constructor、annotation、type reference | Kotlin inline lambda body の modeling は限定的です。深い call chain を信頼する前に `references` で確認してください。 | | JavaScript / TypeScript / Vue / Svelte | function、class、export、import、variable | call、constructor、static/dynamic import、worker、service worker | dynamic property call と computed module specifier は best-effort です。`cdidx references render --lang typescript` | | Python / Ruby / PHP / Perl / R | function、class/module、対応言語の import | call、constructor、対応言語の decorator/annotation | dynamic dispatch と metaprogramming は `search` が必要な場合があります。PHPDoc/static import pattern は静的に見える範囲で索引されます。 | -| MATLAB / Prolog | class/module、function/predicate、import | まだなし | `.m` / `.pl` は保守的に分類され、宣言 symbol は検索できます。reference / graph の調査には `search` を使ってください。 | +| MATLAB | class、function、import | まだなし | `.m` は保守的に分類され、宣言 symbol は検索できます。reference / graph の調査には `search` を使ってください。 | +| Prolog / 曖昧な `.pl` | module、predicate、import | 同一ファイル内の保守的な predicate call と import | 同一ファイルで宣言された predicate の goal 位置だけを call として記録します。曖昧な `.pl` は content classification を保持しながら、Perl / Prolog 構造の安全な和集合を公開します。 | | Cython | `cdef` / `cpdef` 宣言、cimport、extern 宣言 | まだなし | Cython の native extension 宣言はシンボルとして検索できます。call/reference の調査には `search` を使ってください。 | | C / C++ / Objective-C / Swift / Rust / Go / Zig | function、type、method、import/module | call、constructor、対応言語の macro invocation、type reference | C++ template/macro と Rust macro expansion は評価しません。Rust macro invocation 自体は reference edge です。 | | CUDA | C++ 風の function/type と CUDA kernel/device/host sub-kind | まだなし | CUDA の kernel / device / host 宣言は CUDA sub-kind 付きの C++ 風シンボルとして索引します。call/reference の調査には `search` を使ってください。 | diff --git a/changelog.d/unreleased/4746.added.md b/changelog.d/unreleased/4746.added.md index e7cc26573..1fca0c15e 100644 --- a/changelog.d/unreleased/4746.added.md +++ b/changelog.d/unreleased/4746.added.md @@ -9,6 +9,7 @@ affected: - src/CodeIndex/Indexer/References/ReferenceExtractor.State.cs - src/CodeIndex/Indexer/References/ReferenceExtractor.CoreExtraction.cs - src/CodeIndex/Indexer/References/ReferenceExtractor.Preparation.cs + - src/CodeIndex/Indexer/References/ReferenceExtractor.TypeReferences.cs - src/CodeIndex/Indexer/References/Languages/DynamicDeclarativeReferenceExtractor.cs - src/CodeIndex/Database/DbReader.Status.cs - src/CodeIndex/Database/DegradationReasonCodes.cs @@ -28,8 +29,8 @@ affected: ## English -- **Crystal, Groovy, Tcl, Prolog, and ambiguous `.pl` files now support conservative reference graphs (#4746)** — indexing records imports and language-appropriate calls, recognizes command-style calls only for callables declared in the same file, parses Tcl command substitutions and common control-command script arguments, preserves Tcl proc and Prolog predicate caller containers, and reports structured symbol/reference/graph capabilities after `.pl` content classification. Pre-graph extractor stamps keep graph readiness degraded until a normal index refresh rewrites affected rows. +- **Crystal, Groovy, Tcl, Prolog, and ambiguous `.pl` files now support conservative reference graphs (#4746)** — indexing records imports and language-appropriate calls at their source locations, recognizes command-style calls only for callables declared in the same file, parses Tcl command substitutions, control-command script arguments, and `switch` branch bodies, limits Prolog calls to goal positions, preserves Tcl proc and Prolog predicate caller containers, and reports structured symbol/reference/graph capabilities after `.pl` content classification. Pre-graph extractor stamps keep graph readiness degraded until a normal index refresh rewrites affected rows. ## 日本語 -- **Crystal、Groovy、Tcl、Prolog、曖昧な `.pl` ファイルが保守的な reference graph に対応しました（#4746）** — import と言語ごとの call 構文を記録し、command-style call は同一ファイルで宣言済みの callable だけを認識し、Tcl の command substitution と主要な制御 command の script 引数を解析し、Tcl proc / Prolog predicate の caller container を保持して、`.pl` の content classification 後に構造化された symbol / reference / graph capability を報告します。graph 対応前の extractor stamp がある場合、通常の index refresh で対象 row を更新するまで graph readiness を degraded に保ちます。 +- **Crystal、Groovy、Tcl、Prolog、曖昧な `.pl` ファイルが保守的な reference graph に対応しました（#4746）** — import と言語ごとの call 構文をソース上の位置どおりに記録し、command-style call は同一ファイルで宣言済みの callable だけを認識し、Tcl の command substitution、制御 command の script 引数、`switch` の branch body を解析し、Prolog の call を goal 位置に限定し、Tcl proc / Prolog predicate の caller container を保持して、`.pl` の content classification 後に構造化された symbol / reference / graph capability を報告します。graph 対応前の extractor stamp がある場合、通常の index refresh で対象 row を更新するまで graph readiness を degraded に保ちます。 diff --git a/src/CodeIndex/Indexer/References/Languages/DynamicDeclarativeReferenceExtractor.cs b/src/CodeIndex/Indexer/References/Languages/DynamicDeclarativeReferenceExtractor.cs index e6856c56e..80b217ee2 100644 --- a/src/CodeIndex/Indexer/References/Languages/DynamicDeclarativeReferenceExtractor.cs +++ b/src/CodeIndex/Indexer/References/Languages/DynamicDeclarativeReferenceExtractor.cs @@ -12,18 +12,21 @@ public ExtractionState( HashSet callableNames, IReadOnlyDictionary containersByLine, IReadOnlyList tclContainerScopes, - IReadOnlyList? tclCallLines) + IReadOnlyList? tclCallLines, + IReadOnlyDictionary>? prologGoalCallsByLine) { CallableNames = callableNames; ContainersByLine = containersByLine; TclContainerScopes = tclContainerScopes; TclCallLines = tclCallLines; + PrologGoalCallsByLine = prologGoalCallsByLine; } public HashSet CallableNames { get; } public IReadOnlyDictionary ContainersByLine { get; } private IReadOnlyList TclContainerScopes { get; } private IReadOnlyList? TclCallLines { get; } + private IReadOnlyDictionary>? PrologGoalCallsByLine { get; } public SymbolRecord? ResolveContainer(int lineNumber, int column, SymbolRecord? fallback) { @@ -51,13 +54,27 @@ public string GetCallScanLine(string language, int lineNumber, string preparedLi && container.StartLine < lineNumber; return PreparePrologCallScanLine(language, preparedLine, isClauseContinuation); } + + public IReadOnlyList GetPrologGoalCalls(int lineNumber) + { + if (PrologGoalCallsByLine != null + && PrologGoalCallsByLine.TryGetValue(lineNumber, out var calls)) + { + return calls; + } + + return Array.Empty(); + } + + public bool HasPrologContainer(int lineNumber) => + ContainersByLine.ContainsKey(lineNumber); } private static readonly Regex TclProcRegex = new( @"^\s*proc\s+[A-Za-z_:][\w:.-]*\s+", RegexOptions.Compiled | RegexOptions.CultureInvariant); private static readonly Regex PrologHeadRegex = new( - @"^\s*(?[a-z][A-Za-z0-9_]*)\s*(?:\([^\r\n]*\))?\s*(?::-|-->|\.)", + @"^\s*(?[a-z][A-Za-z0-9_]*)\s*(?:\([^\r\n]*\))?\s*(?::-|-->|\.(?=\s*$))", RegexOptions.Compiled | RegexOptions.CultureInvariant); private static readonly Regex CrystalRequireRegex = new( @"^\s*require\s+['""](?[^'""]+)['""]", @@ -72,20 +89,20 @@ public string GetCallScanLine(string language, int lineNumber, string preparedLi @"^\s*:-\s*use_module\s*\(\s*(?:library\s*\(\s*)?['""]?(?(?:\.\.?/)*[a-z][A-Za-z0-9_./-]*)", RegexOptions.Compiled | RegexOptions.CultureInvariant); private static readonly Regex CrystalBareCallRegex = new( - @"(?:^|[;=])\s*(?:return\s+)?(?[A-Za-z_]\w*[?!]?)(?![\w?!])\s*(?!\()", + @"(?:^|[;=])\s*(?:return\s+)?(?[A-Za-z_]\w*[?!]?)(?![\w?!])(?!\s*(?:\(|(?:<<|>>|[+\-*/%&|^])?=))", RegexOptions.Compiled | RegexOptions.CultureInvariant); private static readonly Regex CrystalSuffixedParenthesizedCallRegex = new( @"(?[A-Za-z_]\w*[?!])\s*\(", RegexOptions.Compiled | RegexOptions.CultureInvariant); private static readonly Regex GroovyBareCallRegex = new( - @"(?:^|[;=])\s*(?:return\s+)?(?[A-Za-z_]\w*)\b(?!\s*\()", + @"(?:^|[;=])\s*(?:return\s+)?(?[A-Za-z_]\w*)\b(?!\s*(?:\(|(?:<<|>>|[+\-*/%&|^])?=))", + RegexOptions.Compiled | RegexOptions.CultureInvariant); + private static readonly Regex GroovyConstructorDeclarationRegex = new( + @"^\s*(?:(?:public|protected|private)\s+)*(?[A-Z]\w*)\s*\([^)]*\)\s*(?:throws\s+[\w.,\s]+)?\{", RegexOptions.Compiled | RegexOptions.CultureInvariant); private static readonly Regex TclCommandRegex = new( @"(?:^|[;\[])\s*(?[A-Za-z_:][\w:.-]*)", RegexOptions.Compiled | RegexOptions.CultureInvariant); - private static readonly Regex PrologBareCallRegex = new( - @"(?:^|:-|-->|[,;])\s*(?:\\\+\s*)?(?[a-z][A-Za-z0-9_]*)\b(?!\s*\()", - RegexOptions.Compiled | RegexOptions.CultureInvariant); private static readonly Regex CrystalHeredocOpenerRegex = new( @"<<-\s*['""]?(?[A-Za-z_]\w*)['""]?", RegexOptions.Compiled | RegexOptions.CultureInvariant); @@ -120,8 +137,54 @@ private enum TclLexicalFrameKind Script, Quote, BracedWord, + SwitchTable, } + internal readonly record struct PrologGoalCall(string Name, int Column); + + private enum PrologLexicalFrameKind + { + PredicateArguments, + GoalGroup, + MetaArguments, + TermGroup, + } + + private sealed class PrologLexicalFrame( + PrologLexicalFrameKind kind, + IReadOnlySet? goalArgumentIndices = null, + char terminator = ')') + { + public PrologLexicalFrameKind Kind { get; } = kind; + public IReadOnlySet? GoalArgumentIndices { get; } = goalArgumentIndices; + public char Terminator { get; } = terminator; + public int ArgumentIndex { get; set; } + public bool CurrentArgumentIsGoal => + GoalArgumentIndices?.Contains(ArgumentIndex) == true; + } + + private static readonly IReadOnlyDictionary> PrologMetaGoalArguments = + new Dictionary>(StringComparer.Ordinal) + { + ["call"] = new HashSet { 0 }, + ["once"] = new HashSet { 0 }, + ["ignore"] = new HashSet { 0 }, + ["not"] = new HashSet { 0 }, + ["catch"] = new HashSet { 0, 2 }, + ["call_cleanup"] = new HashSet { 0, 1 }, + ["setup_call_cleanup"] = new HashSet { 0, 1, 2 }, + ["findall"] = new HashSet { 1 }, + ["bagof"] = new HashSet { 1 }, + ["setof"] = new HashSet { 1 }, + ["forall"] = new HashSet { 0, 1 }, + ["phrase"] = new HashSet { 0 }, + ["maplist"] = new HashSet { 0 }, + ["include"] = new HashSet { 0 }, + ["exclude"] = new HashSet { 0 }, + ["foldl"] = new HashSet { 0 }, + ["convlist"] = new HashSet { 0 }, + }; + private sealed class TclLexicalFrame( TclLexicalFrameKind kind, char terminator = '\0') @@ -134,6 +197,8 @@ private sealed class TclLexicalFrame( public string? CommandName { get; set; } public string? LastBareWord { get; set; } public int LastBareWordIndex { get; set; } = -1; + public int SwitchStringWordIndex { get; set; } = -1; + public bool SwitchOptionsEnded { get; set; } public void ResetCommand() { @@ -143,6 +208,8 @@ public void ResetCommand() CommandName = null; LastBareWord = null; LastBareWordIndex = -1; + SwitchStringWordIndex = -1; + SwitchOptionsEnded = false; } } @@ -482,6 +549,7 @@ public static string[] MaskNonCodeLines(string language, IReadOnlyList l var containersByLine = new Dictionary(); var tclContainerScopes = new List(); string[]? tclCallLines = null; + IReadOnlyDictionary>? prologGoalCallsByLine = null; if (language == "tcl") { var tclScriptBodyOpenings = new HashSet(); @@ -498,13 +566,20 @@ public static string[] MaskNonCodeLines(string language, IReadOnlyList l tclScriptBodyOpenings); } else if (language is "prolog" or "ambiguous_pl") + { AddPrologContainers(preparedLines, symbols, containersByLine); + prologGoalCallsByLine = BuildPrologGoalCalls( + preparedLines, + containersByLine, + callableNames); + } return new ExtractionState( callableNames, containersByLine, tclContainerScopes, - tclCallLines); + tclCallLines, + prologGoalCallsByLine); } public static void EmitAdditionalReferences( @@ -530,12 +605,41 @@ public static void EmitAdditionalReferences( lineNumber, resolveContainerForCall); + if (language is "prolog" or "ambiguous_pl") + { + foreach (var call in state.GetPrologGoalCalls(lineNumber)) + { + if (!state.CallableNames.Contains(call.Name)) + continue; + + if (state.ContainersByLine.TryGetValue(lineNumber, out var prologContainer) + && !string.Equals(prologContainer.Name, call.Name, StringComparison.Ordinal)) + { + ReferenceExtractor.AddReference( + references, + seen, + fileId, + call.Name, + call.Column, + "call", + context, + lineNumber, + prologContainer, + language); + continue; + } + + addCallLikeReference(call.Name, call.Column); + } + + return; + } + var callRegex = language switch { "crystal" => CrystalBareCallRegex, "groovy" => GroovyBareCallRegex, "tcl" => TclCommandRegex, - "prolog" or "ambiguous_pl" => PrologBareCallRegex, _ => null, }; if (callRegex == null) @@ -557,29 +661,32 @@ public static void EmitAdditionalReferences( if (!state.CallableNames.Contains(nameGroup.Value)) continue; - if (language is "prolog" or "ambiguous_pl" - && state.ContainersByLine.TryGetValue(lineNumber, out var prologContainer) - && !string.Equals(prologContainer.Name, nameGroup.Value, StringComparison.Ordinal)) - { - ReferenceExtractor.AddReference( - references, - seen, - fileId, - nameGroup.Value, - nameGroup.Index, - "call", - context, - lineNumber, - prologContainer, - language); - continue; - } - addCallLikeReference(nameGroup.Value, nameGroup.Index); } } + public static bool ShouldSuppressGenericCall( + string language, + string preparedLine, + string name, + int callIndex, + int lineNumber, + ExtractionState? state) + { + if (language == "ambiguous_pl") + return state?.HasPrologContainer(lineNumber) == true; + if (language != "groovy") + return false; + if (name is "super" or "synchronized" or "this") + return true; + + var declaration = GroovyConstructorDeclarationRegex.Match(preparedLine); + return declaration.Success + && declaration.Groups["name"].Index == callIndex + && string.Equals(declaration.Groups["name"].Value, name, StringComparison.Ordinal); + } + private static int SkipQuotedToken(string line, int startColumn, char delimiter) { for (var column = startColumn + 1; column < line.Length; column++) @@ -1163,6 +1270,61 @@ private static string[] BuildTclCallLines( { var frame = frames.Peek(); var ch = line[column]; + if (frame.Kind == TclLexicalFrameKind.SwitchTable) + { + buffer[column] = ' '; + if (ch == frame.Terminator) + { + frames.Pop(); + column++; + } + else if (ch == '\\' && column + 1 < line.Length) + { + buffer[column + 1] = ' '; + frame.WordStart = false; + column += 2; + } + else if (char.IsWhiteSpace(ch)) + { + frame.WordStart = true; + column++; + } + else if (ch == '{' && frame.WordStart) + { + var wordIndex = frame.WordIndex++; + if (wordIndex % 2 == 1) + { + buffer[column] = ';'; + frames.Push(new TclLexicalFrame(TclLexicalFrameKind.Script, '}')); + suppressLeadingContinuedWord = false; + } + else + { + frames.Push(new TclLexicalFrame(TclLexicalFrameKind.BracedWord, '}')); + } + + frame.WordStart = false; + column++; + } + else if (ch == '"' && frame.WordStart) + { + frame.WordIndex++; + var endColumn = SkipQuotedToken(line, column, '"'); + FillWithSpaces(buffer, column, endColumn); + frame.WordStart = false; + column = endColumn; + } + else + { + if (frame.WordStart) + frame.WordIndex++; + frame.WordStart = false; + column++; + } + + continue; + } + if (frame.Kind == TclLexicalFrameKind.BracedWord) { buffer[column] = ' '; @@ -1240,7 +1402,10 @@ private static string[] BuildTclCallLines( if (ch == '"') { if (frame.WordStart) - frame.WordIndex++; + { + var wordIndex = frame.WordIndex++; + UpdateTclSwitchArgumentState(frame, wordIndex, string.Empty); + } buffer[column] = ' '; frames.Push(new TclLexicalFrame(TclLexicalFrameKind.Quote, '"')); frame.CommandStart = false; @@ -1251,7 +1416,10 @@ private static string[] BuildTclCallLines( if (ch == '[') { if (frame.WordStart) - frame.WordIndex++; + { + var wordIndex = frame.WordIndex++; + UpdateTclSwitchArgumentState(frame, wordIndex, string.Empty); + } frames.Push(new TclLexicalFrame(TclLexicalFrameKind.Script, ']')); frame.CommandStart = false; frame.WordStart = false; @@ -1263,16 +1431,28 @@ private static string[] BuildTclCallLines( { var wordIndex = frame.WordIndex++; var positionKey = GetTclPositionKey(lineIndex, column); - var isScriptArgument = - scriptBodyOpenings.Contains(positionKey) + TclBraceEnd? braceEnd = braceEnds.TryGetValue(positionKey, out var foundBraceEnd) + ? foundBraceEnd + : null; + var isSwitchTable = IsTclSwitchTableArgument( + frame, + wordIndex, + lines, + braceEnd); + var isScriptArgument = !isSwitchTable + && (scriptBodyOpenings.Contains(positionKey) || IsTclScriptArgument( frame, wordIndex, lines, - braceEnds.TryGetValue(positionKey, out var braceEnd) - ? braceEnd - : null); - if (isScriptArgument) + braceEnd)); + UpdateTclSwitchArgumentState(frame, wordIndex, string.Empty); + if (isSwitchTable) + { + buffer[column] = ' '; + frames.Push(new TclLexicalFrame(TclLexicalFrameKind.SwitchTable, '}')); + } + else if (isScriptArgument) { buffer[column] = ';'; frames.Push(new TclLexicalFrame(TclLexicalFrameKind.Script, '}')); @@ -1308,6 +1488,8 @@ private static string[] BuildTclCallLines( var token = ReadTclBareWord(line, column); if (wordIndex == 0) frame.CommandName = token; + else + UpdateTclSwitchArgumentState(frame, wordIndex, token); if (token.Length > 0) { frame.LastBareWord = token; @@ -1378,10 +1560,48 @@ private static bool IsTclScriptArgument( "namespace" => wordIndex == 3, "dict" => frame.LastBareWord == "for" && wordIndex == frame.LastBareWordIndex + 3, + "switch" => frame.SwitchStringWordIndex >= 0 + && wordIndex - frame.SwitchStringWordIndex >= 2 + && (wordIndex - frame.SwitchStringWordIndex) % 2 == 0, _ => false, }; } + private static bool IsTclSwitchTableArgument( + TclLexicalFrame frame, + int wordIndex, + IReadOnlyList lines, + TclBraceEnd? braceEnd) + { + return frame.CommandName == "switch" + && frame.SwitchStringWordIndex >= 0 + && wordIndex == frame.SwitchStringWordIndex + 1 + && braceEnd is { } end + && IsTclLastCommandWord(lines, end); + } + + private static void UpdateTclSwitchArgumentState( + TclLexicalFrame frame, + int wordIndex, + string token) + { + if (frame.CommandName != "switch" + || wordIndex == 0 + || frame.SwitchStringWordIndex >= 0) + { + return; + } + + if (!frame.SwitchOptionsEnded && token.StartsWith("-", StringComparison.Ordinal)) + { + if (token == "--") + frame.SwitchOptionsEnded = true; + return; + } + + frame.SwitchStringWordIndex = wordIndex; + } + private static bool IsTclLastCommandWord( IReadOnlyList lines, TclBraceEnd braceEnd) @@ -1400,6 +1620,344 @@ private static bool IsTclLastCommandWord( private static long GetTclPositionKey(int line, int column) => ((long)line << 32) | (uint)column; + private static IReadOnlyDictionary> BuildPrologGoalCalls( + IReadOnlyList lines, + IReadOnlyDictionary containersByLine, + IReadOnlySet callableNames) + { + var result = new Dictionary>(); + var frames = new Stack(); + var expectGoal = true; + SymbolRecord? activeContainer = null; + + for (var lineIndex = 0; lineIndex < lines.Count; lineIndex++) + { + var lineNumber = lineIndex + 1; + if (!containersByLine.TryGetValue(lineNumber, out var container)) + { + frames.Clear(); + activeContainer = null; + expectGoal = true; + continue; + } + + if (activeContainer == null + || activeContainer.StartLine != container.StartLine + || !string.Equals(activeContainer.Name, container.Name, StringComparison.Ordinal)) + { + frames.Clear(); + activeContainer = container; + expectGoal = true; + } + + var callScanLine = PreparePrologCallScanLine( + "prolog", + lines[lineIndex], + container.StartLine < lineNumber); + var lineCalls = new List(); + ScanPrologGoalLine( + callScanLine, + callableNames, + frames, + ref expectGoal, + lineCalls); + if (lineCalls.Count > 0) + result[lineNumber] = lineCalls; + + if (ContainsPrologClauseTerminator(callScanLine)) + { + frames.Clear(); + activeContainer = null; + expectGoal = true; + } + } + + return result; + } + + private static void ScanPrologGoalLine( + string line, + IReadOnlySet callableNames, + Stack frames, + ref bool expectGoal, + List calls) + { + for (var column = 0; column < line.Length;) + { + var ch = line[column]; + if (char.IsWhiteSpace(ch)) + { + column++; + continue; + } + + if (ch is '\'' or '"') + { + column = SkipQuotedToken(line, column, ch); + if (expectGoal) + expectGoal = false; + continue; + } + + if (expectGoal) + { + if (line.AsSpan(column).StartsWith("-->", StringComparison.Ordinal)) + { + column += 3; + continue; + } + if (line.AsSpan(column).StartsWith(":-", StringComparison.Ordinal) + || line.AsSpan(column).StartsWith(@"\+", StringComparison.Ordinal)) + { + column += 2; + continue; + } + if (ch is ',' or ';') + { + column++; + continue; + } + if (line.AsSpan(column).StartsWith("->", StringComparison.Ordinal)) + { + column += 2; + continue; + } + if (ch == '(') + { + frames.Push(new PrologLexicalFrame(PrologLexicalFrameKind.GoalGroup)); + column++; + continue; + } + if (ch == '{') + { + frames.Push(new PrologLexicalFrame( + PrologLexicalFrameKind.GoalGroup, + terminator: '}')); + column++; + continue; + } + if (ch == '[') + { + frames.Push(new PrologLexicalFrame( + PrologLexicalFrameKind.TermGroup, + terminator: ']')); + expectGoal = false; + column++; + continue; + } + if (ch == '!') + { + expectGoal = false; + column++; + continue; + } + if (char.IsLower(ch)) + { + var nameStart = column; + column++; + while (column < line.Length + && (char.IsLetterOrDigit(line[column]) || line[column] == '_')) + { + column++; + } + + var name = line[nameStart..column]; + var nextColumn = column; + while (nextColumn < line.Length && char.IsWhiteSpace(line[nextColumn])) + nextColumn++; + if (callableNames.Contains(name) + && !IsPrologTermBeforeInfixOperator(line, column, nextColumn)) + { + calls.Add(new PrologGoalCall(name, nameStart)); + } + + if (nextColumn < line.Length && line[nextColumn] == '(') + { + if (PrologMetaGoalArguments.TryGetValue(name, out var goalArgumentIndices)) + { + var metaFrame = new PrologLexicalFrame( + PrologLexicalFrameKind.MetaArguments, + goalArgumentIndices); + frames.Push(metaFrame); + expectGoal = metaFrame.CurrentArgumentIsGoal; + } + else + { + frames.Push(new PrologLexicalFrame( + PrologLexicalFrameKind.PredicateArguments)); + expectGoal = false; + } + + column = nextColumn + 1; + } + else + { + expectGoal = false; + } + + continue; + } + + expectGoal = false; + column++; + continue; + } + + if (ch == '(') + { + frames.Push(new PrologLexicalFrame(PrologLexicalFrameKind.PredicateArguments)); + column++; + continue; + } + if (ch is '[' or '{') + { + frames.Push(new PrologLexicalFrame( + PrologLexicalFrameKind.TermGroup, + terminator: ch == '[' ? ']' : '}')); + column++; + continue; + } + if (ch is ')' or ']' or '}') + { + if (frames.TryPeek(out var closingFrame) + && closingFrame.Terminator == ch) + { + frames.Pop(); + } + expectGoal = false; + column++; + continue; + } + if (ch == ',') + { + if (frames.TryPeek(out var frame) + && frame.Kind == PrologLexicalFrameKind.MetaArguments) + { + frame.ArgumentIndex++; + expectGoal = frame.CurrentArgumentIsGoal; + } + else if (CanStartNextPrologGoal(frames)) + { + expectGoal = true; + } + + column++; + continue; + } + if (ch == ';' + || line.AsSpan(column).StartsWith("->", StringComparison.Ordinal)) + { + if (CanStartNextPrologGoal(frames)) + expectGoal = true; + column += ch == ';' ? 1 : 2; + continue; + } + + column++; + } + } + + private static bool IsPrologTermBeforeInfixOperator( + string line, + int nameEndColumn, + int nextColumn) + { + var afterTermColumn = nextColumn; + if (nextColumn < line.Length && line[nextColumn] == '(') + { + var depth = 0; + afterTermColumn = -1; + for (var column = nextColumn; column < line.Length; column++) + { + var ch = line[column]; + if (ch is '\'' or '"') + { + column = SkipQuotedToken(line, column, ch) - 1; + continue; + } + + if (ch == '(') + { + depth++; + } + else if (ch == ')' && --depth == 0) + { + afterTermColumn = column + 1; + break; + } + } + + if (afterTermColumn < 0) + return false; + } + else + { + afterTermColumn = nameEndColumn; + } + + while (afterTermColumn < line.Length && char.IsWhiteSpace(line[afterTermColumn])) + afterTermColumn++; + if (afterTermColumn >= line.Length) + return false; + + var remaining = line.AsSpan(afterTermColumn); + if (remaining.StartsWith("->", StringComparison.Ordinal) + || remaining.StartsWith("*->", StringComparison.Ordinal)) + { + return false; + } + + if (line[afterTermColumn] is '=' or '\\' or '<' or '>' or '@' or '#' + or ':' or '+' or '-' or '*' or '/' or '^') + { + return true; + } + + foreach (var operatorName in PrologInfixOperatorNames) + { + if (!remaining.StartsWith(operatorName, StringComparison.Ordinal)) + continue; + var operatorEnd = afterTermColumn + operatorName.Length; + if (operatorEnd >= line.Length + || !char.IsLetterOrDigit(line[operatorEnd]) && line[operatorEnd] != '_') + { + return true; + } + } + + return false; + } + + private static readonly string[] PrologInfixOperatorNames = + ["is", "mod", "rem", "xor", "div", "rdiv"]; + + private static bool CanStartNextPrologGoal( + IEnumerable frames) + { + foreach (var frame in frames) + { + if (frame.Kind == PrologLexicalFrameKind.PredicateArguments) + return false; + if (frame.Kind == PrologLexicalFrameKind.TermGroup) + return false; + if (frame.Kind == PrologLexicalFrameKind.MetaArguments) + return frame.CurrentArgumentIsGoal; + } + + return true; + } + + private static bool ContainsPrologClauseTerminator(string line) + { + for (var column = 0; column < line.Length; column++) + { + if (IsPrologClauseTerminator(line, column)) + return true; + } + + return false; + } + private static void AddPrologContainers( IReadOnlyList lines, IReadOnlyList symbols, diff --git a/src/CodeIndex/Indexer/References/ReferenceExtractor.CoreExtraction.cs b/src/CodeIndex/Indexer/References/ReferenceExtractor.CoreExtraction.cs index 985e0c602..209518d89 100644 --- a/src/CodeIndex/Indexer/References/ReferenceExtractor.CoreExtraction.cs +++ b/src/CodeIndex/Indexer/References/ReferenceExtractor.CoreExtraction.cs @@ -2542,7 +2542,7 @@ bool TryGetKnownPythonTypeCall(string candidate, out string canonicalName) } else { - if (language != "tcl") + if (language is not ("tcl" or "prolog")) { foreach (Match match in CallRegex.Matches(callScanLine)) { @@ -2557,6 +2557,16 @@ bool TryGetKnownPythonTypeCall(string candidate, out string canonicalName) if (sqlWindowFunctionCallSiteSuppressions != null && sqlWindowFunctionCallSiteSuppressions.Contains((lineNumber, callIndex))) continue; + if (DynamicDeclarativeReferenceExtractor.ShouldSuppressGenericCall( + language, + callScanLine, + name, + callIndex, + lineNumber, + dynamicDeclarativeState)) + { + continue; + } GetMatchedCallIndices().Add(callIndex); if (TryAddCallLikeReference(name, callIndex)) { diff --git a/src/CodeIndex/Indexer/References/ReferenceExtractor.TypeReferences.cs b/src/CodeIndex/Indexer/References/ReferenceExtractor.TypeReferences.cs index 7fe3719bd..ace76cf2b 100644 --- a/src/CodeIndex/Indexer/References/ReferenceExtractor.TypeReferences.cs +++ b/src/CodeIndex/Indexer/References/ReferenceExtractor.TypeReferences.cs @@ -2713,6 +2713,7 @@ private readonly record struct ReferenceLinePrepareOptions( bool UseCSharpTriggerFastPath, bool MaskRustLifetimes, bool MaskStringLiterals, + bool PreserveStringLiteralWidth, bool IncludeBacktickStringDelimiter, bool UsesHashComments, bool UsesRHashComments, @@ -2727,6 +2728,7 @@ private static ReferenceLinePrepareOptions CreateReferenceLinePrepareOptions(str UseCSharpTriggerFastPath: lang == "csharp", MaskRustLifetimes: lang == "rust", MaskStringLiterals: lang != "cobol", + PreserveStringLiteralWidth: lang is "crystal" or "groovy" or "prolog" or "ambiguous_pl", IncludeBacktickStringDelimiter: lang is not ("kotlin" or "r"), UsesHashComments: UsesHashComments(lang), UsesRHashComments: lang == "r", @@ -2755,7 +2757,9 @@ private static string PrepareLine(string line, ReferenceLinePrepareOptions optio var stringLiteralRegex = !options.IncludeBacktickStringDelimiter ? NonBacktickStringLiteralRegex : StringLiteralRegex; - result = stringLiteralRegex.Replace(result, "\"\""); + result = options.PreserveStringLiteralWidth + ? stringLiteralRegex.Replace(result, static match => new string(' ', match.Length)) + : stringLiteralRegex.Replace(result, "\"\""); } if (result.Contains("/*", StringComparison.Ordinal)) result = InlineBlockCommentRegex.Replace(result, " "); diff --git a/src/CodeIndex/Indexer/References/ReferenceExtractor.cs b/src/CodeIndex/Indexer/References/ReferenceExtractor.cs index 73edd6e26..44794d988 100644 --- a/src/CodeIndex/Indexer/References/ReferenceExtractor.cs +++ b/src/CodeIndex/Indexer/References/ReferenceExtractor.cs @@ -290,8 +290,8 @@ private static bool IsFunctionLikeSymbolKind(string kind) { "apply", "as", "assert", "break", "case", "catch", "class", "continue", "def", "do", "else", "enum", "extends", "finally", "for", "if", "implements", "import", "in", - "instanceof", "interface", "new", "package", "return", "switch", "this", "throw", - "throws", "trait", "try", "while", + "instanceof", "interface", "new", "package", "return", "super", "switch", "synchronized", + "this", "throw", "throws", "trait", "try", "while", }, ["tcl"] = new HashSet(StringComparer.Ordinal) { diff --git a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.cs b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.cs index 56baaba14..92e9bd439 100644 --- a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.cs +++ b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.cs @@ -1715,7 +1715,7 @@ private enum JavaScriptTypeScriptFunctionHeaderConsumeResult [ new("namespace", new Regex(@"^\s*:-\s*module\s*\(\s*(?[a-z][A-Za-z0-9_]*)", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None), new("import", new Regex(@"^\s*:-\s*use_module\s*\(\s*(?[a-z][A-Za-z0-9_]*(?:\([^)]*\))?)", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None), - new("function", new Regex(@"^\s*(?[a-z][A-Za-z0-9_]*)\s*(?:\([^\r\n]*\))?\s*(?::-|-->|\.)", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None), + new("function", new Regex(@"^\s*(?[a-z][A-Za-z0-9_]*)\s*(?:\([^\r\n]*\))?\s*(?::-|-->|\.(?=\s*$))", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None), ], ["ambiguous_pl"] = [ @@ -1733,7 +1733,7 @@ private enum JavaScriptTypeScriptFunctionHeaderConsumeResult new("function", new Regex(@"^\s*(?:method|fun)\s+(?" + PerlIdentifierPattern + @")\b(?:\s*:[^{;]+)?", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.Brace), new("namespace", new Regex(@"^\s*:-\s*module\s*\(\s*(?[a-z][A-Za-z0-9_]*)", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None), new("import", new Regex(@"^\s*:-\s*use_module\s*\(\s*(?[a-z][A-Za-z0-9_]*(?:\([^)]*\))?)", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None), - new("function", new Regex(@"^\s*(?[a-z][A-Za-z0-9_]*)\s*(?:\([^\r\n]*\))?\s*(?::-|-->|\.)", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None), + new("function", new Regex(@"^\s*(?[a-z][A-Za-z0-9_]*)\s*(?:\([^\r\n]*\))?\s*(?::-|-->|\.(?=\s*$))", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None), ], ["c"] = [ diff --git a/tests/CodeIndex.Tests/ReferenceExtractorTests.cs b/tests/CodeIndex.Tests/ReferenceExtractorTests.cs index 78b1f9779..8c6b8eeae 100644 --- a/tests/CodeIndex.Tests/ReferenceExtractorTests.cs +++ b/tests/CodeIndex.Tests/ReferenceExtractorTests.cs @@ -1063,6 +1063,8 @@ def helper(value) def run(value) helper value helper(value) + text = "123456789"; helper(value) + helper = 1 end """; @@ -1071,6 +1073,20 @@ helper value Assert.Contains(symbols, symbol => symbol.Kind == "function" && symbol.Name == "run"); AssertReferencesContain(references, "type_reference", null, "support"); AssertReferencesContain(references, "call", "run", "helper"); + Assert.Equal(3, references.Count(reference => + reference.ReferenceKind == "call" + && reference.ContainerName == "run" + && reference.SymbolName == "helper")); + var stringCallLine = Array.FindIndex( + content.Split('\n'), + line => line.Contains("text =", StringComparison.Ordinal)) + 1; + var stringCall = Assert.Single(references, reference => + reference.ReferenceKind == "call" + && reference.SymbolName == "helper" + && reference.Line == stringCallLine); + Assert.Equal( + content.Split('\n')[stringCallLine - 1].IndexOf("helper", StringComparison.Ordinal) + 1, + stringCall.Column); Assert.DoesNotContain(references, reference => reference.ReferenceKind == "call" && reference.SymbolName is "require" or "def"); @@ -1083,6 +1099,10 @@ public void Extract_Groovy_IndexesImportsParenthesizedAndCommandCalls_Issue4746( import demo.Support class Runner { + Runner() { + super(1) + } + static def helper(value) { value } @@ -1090,6 +1110,9 @@ static def helper(value) { def run(value) { helper value helper(value) + def text = "123456789"; helper(value) + helper = 1 + synchronized(value) {} } } """; @@ -1099,9 +1122,23 @@ helper value Assert.Contains(symbols, symbol => symbol.Kind == "function" && symbol.Name == "run"); AssertReferencesContain(references, "type_reference", null, "Support"); AssertReferencesContain(references, "call", "run", "helper"); + Assert.Equal(3, references.Count(reference => + reference.ReferenceKind == "call" + && reference.ContainerName == "run" + && reference.SymbolName == "helper")); + var stringCallLine = Array.FindIndex( + content.Split('\n'), + line => line.Contains("text =", StringComparison.Ordinal)) + 1; + var stringCall = Assert.Single(references, reference => + reference.ReferenceKind == "call" + && reference.SymbolName == "helper" + && reference.Line == stringCallLine); + Assert.Equal( + content.Split('\n')[stringCallLine - 1].IndexOf("helper", StringComparison.Ordinal) + 1, + stringCall.Column); Assert.DoesNotContain(references, reference => reference.ReferenceKind == "call" - && reference.SymbolName is "import" or "def"); + && reference.SymbolName is "import" or "def" or "Runner" or "super" or "synchronized"); } [Fact] @@ -1290,6 +1327,15 @@ set braced {[helper]} catch { helper } + switch $value { + one { helper } + default { + helper + } + } + switch -exact -- $value one { + helper + } default { helper } puts \ helper set callback helper() @@ -1309,7 +1355,7 @@ proc split {value} AssertReferencesContain(references, "call", "multiline", "helper"); AssertReferencesContain(references, "call", "split", "helper"); AssertReferencesContain(references, "call", "sameLine", "helper"); - Assert.Equal(11, references.Count(reference => + Assert.Equal(15, references.Count(reference => reference.ReferenceKind == "call" && reference.SymbolName == "helper")); Assert.Single(references, reference => @@ -1330,6 +1376,19 @@ public void Extract_Prolog_IndexesRelativeImportsAndDecimalHeadsWhileMaskingComm helper. bar(_). nested(bar(X)). + process(_). + data_only(X) :- + X = bar(value), + process(bar(X)). + list_data(X) :- + X = [bar(value), bar(other)], + process(X). + operator_data :- + bar(value) = bar(other), + bar = value, + process(value). + meta :- + call(helper). build(Term) :- Term =.. [node, value], @@ -1347,10 +1406,14 @@ public void Extract_Prolog_IndexesRelativeImportsAndDecimalHeadsWhileMaskingComm Assert.Contains(symbols, symbol => symbol.Kind == "function" && symbol.Name == "build"); Assert.Single(symbols, symbol => symbol.Kind == "function" && symbol.Name == "helper"); AssertReferencesContain(references, "type_reference", null, "support"); - Assert.Equal(3, references.Count(reference => + Assert.Equal(4, references.Count(reference => reference.ReferenceKind == "call" - && reference.ContainerName is "threshold" or "quoted" or "build" + && reference.ContainerName is "threshold" or "quoted" or "meta" or "build" && reference.SymbolName == "helper")); + Assert.Equal(3, references.Count(reference => + reference.ReferenceKind == "call" + && (reference.ContainerName is "data_only" or "list_data" or "operator_data") + && reference.SymbolName == "process")); Assert.DoesNotContain(references, reference => reference.ReferenceKind == "call" && reference.SymbolName is "threshold" or "bar"); diff --git a/tests/CodeIndex.Tests/SymbolExtractorTests.cs b/tests/CodeIndex.Tests/SymbolExtractorTests.cs index 76b1c815f..db43f7d27 100644 --- a/tests/CodeIndex.Tests/SymbolExtractorTests.cs +++ b/tests/CodeIndex.Tests/SymbolExtractorTests.cs @@ -77,6 +77,7 @@ public void Extract_AmbiguousPl_PreservesPerlAndPrologStructure_Issue4746() :- module(hybrid, [prolog_helper/0]). :- use_module(library(lists)). prolog_helper :- perl_helper(). + concat_candidate(1) . other(2); =pod package PodFake; sub pod_fake { return 1; } @@ -96,7 +97,8 @@ public void Extract_AmbiguousPl_PreservesPerlAndPrologStructure_Issue4746() AssertSymbolsContain(symbols, "function", "perl_helper", "prolog_helper"); Assert.DoesNotContain(symbols, symbol => symbol.Name is "PodFake" or "pod_fake" or "pod_predicate" - or "BlockFake" or "block_fake" or "block_predicate"); + or "BlockFake" or "block_fake" or "block_predicate" + or "concat_candidate" or "other"); } [Fact] From 7aed1ce5674dece3ccb7ed8d729dbc81060c7571 Mon Sep 17 00:00:00 2001 From: Widthdom Date: Fri, 24 Jul 2026 15:54:16 +0900 Subject: [PATCH 07/26] Close dynamic graph review findings (#4746) --- .../DynamicDeclarativeReferenceExtractor.cs | 58 +++++++++++++++++-- .../ReferenceExtractor.CoreExtraction.cs | 5 +- .../ReferenceExtractor.TypeReferences.cs | 3 +- .../ReferenceExtractorTests.cs | 30 +++++++--- 4 files changed, 80 insertions(+), 16 deletions(-) diff --git a/src/CodeIndex/Indexer/References/Languages/DynamicDeclarativeReferenceExtractor.cs b/src/CodeIndex/Indexer/References/Languages/DynamicDeclarativeReferenceExtractor.cs index 80b217ee2..0925a6a53 100644 --- a/src/CodeIndex/Indexer/References/Languages/DynamicDeclarativeReferenceExtractor.cs +++ b/src/CodeIndex/Indexer/References/Languages/DynamicDeclarativeReferenceExtractor.cs @@ -98,7 +98,13 @@ public bool HasPrologContainer(int lineNumber) => @"(?:^|[;=])\s*(?:return\s+)?(?[A-Za-z_]\w*)\b(?!\s*(?:\(|(?:<<|>>|[+\-*/%&|^])?=))", RegexOptions.Compiled | RegexOptions.CultureInvariant); private static readonly Regex GroovyConstructorDeclarationRegex = new( - @"^\s*(?:(?:public|protected|private)\s+)*(?[A-Z]\w*)\s*\([^)]*\)\s*(?:throws\s+[\w.,\s]+)?\{", + @"(?:^|[;{])\s*(?:(?:public|protected|private)\s+)*(?[A-Z]\w*)\s*\(", + RegexOptions.Compiled | RegexOptions.CultureInvariant); + private static readonly Regex GroovyMethodDeclarationRegex = new( + @"(?:^|[;{])\s*(?:(?:public|private|protected|static|final|abstract|synchronized|native)\s+)*(?:def|void|boolean|byte|char|short|int|long|float|double|BigDecimal|BigInteger|String|[A-Za-z_$][\w.$<>\[\]?]*)\s+(?[A-Za-z_]\w*)\s*\(", + RegexOptions.Compiled | RegexOptions.CultureInvariant); + private static readonly Regex CrystalMethodDeclarationRegex = new( + @"(?:^|;)\s*(?:(?:private|protected)\s+)?def\s+(?:self\.)?(?[A-Za-z_]\w*[?!]?)\s*\(", RegexOptions.Compiled | RegexOptions.CultureInvariant); private static readonly Regex TclCommandRegex = new( @"(?:^|[;\[])\s*(?[A-Za-z_:][\w:.-]*)", @@ -672,19 +678,42 @@ public static bool ShouldSuppressGenericCall( string name, int callIndex, int lineNumber, - ExtractionState? state) + ExtractionState? state, + SymbolRecord? container) { if (language == "ambiguous_pl") return state?.HasPrologContainer(lineNumber) == true; + if (language == "crystal") + return MatchesDeclarationAt(CrystalMethodDeclarationRegex, preparedLine, name, callIndex); if (language != "groovy") return false; if (name is "super" or "synchronized" or "this") return true; - var declaration = GroovyConstructorDeclarationRegex.Match(preparedLine); - return declaration.Success - && declaration.Groups["name"].Index == callIndex - && string.Equals(declaration.Groups["name"].Value, name, StringComparison.Ordinal); + if (MatchesDeclarationAt(GroovyMethodDeclarationRegex, preparedLine, name, callIndex)) + return true; + return container?.Kind == "class" + && string.Equals(container.Name, name, StringComparison.Ordinal) + && MatchesDeclarationAt(GroovyConstructorDeclarationRegex, preparedLine, name, callIndex); + } + + private static bool MatchesDeclarationAt( + Regex regex, + string line, + string name, + int callIndex) + { + foreach (Match declaration in BoundedRegex.EnumerateMatches(regex, line)) + { + var nameGroup = declaration.Groups["name"]; + if (nameGroup.Index == callIndex + && string.Equals(nameGroup.Value, name, StringComparison.Ordinal)) + { + return true; + } + } + + return false; } private static int SkipQuotedToken(string line, int startColumn, char delimiter) @@ -1258,10 +1287,18 @@ private static string[] BuildTclCallLines( var result = new string[lines.Count]; var frames = new Stack(); frames.Push(new TclLexicalFrame(TclLexicalFrameKind.Script)); + var commentContinued = false; for (var lineIndex = 0; lineIndex < lines.Count; lineIndex++) { var line = lines[lineIndex]; + if (commentContinued) + { + result[lineIndex] = new string(' ', line.Length); + commentContinued = HasTclEscapedNewline(line); + continue; + } + var buffer = line.ToCharArray(); var lineContinued = false; var suppressLeadingContinuedWord = frames.Peek().Kind != TclLexicalFrameKind.Script @@ -1397,6 +1434,7 @@ private static string[] BuildTclCallLines( if (ch == '#' && frame.CommandStart) { FillWithSpaces(buffer, column); + commentContinued = HasTclEscapedNewline(line); break; } if (ch == '"') @@ -1516,6 +1554,14 @@ private static string[] BuildTclCallLines( return result; } + private static bool HasTclEscapedNewline(string line) + { + var backslashCount = 0; + for (var column = line.Length - 1; column >= 0 && line[column] == '\\'; column--) + backslashCount++; + return backslashCount % 2 == 1; + } + private static string ReadTclBareWord(string line, int startColumn) { var endColumn = startColumn; diff --git a/src/CodeIndex/Indexer/References/ReferenceExtractor.CoreExtraction.cs b/src/CodeIndex/Indexer/References/ReferenceExtractor.CoreExtraction.cs index 209518d89..6a4b465eb 100644 --- a/src/CodeIndex/Indexer/References/ReferenceExtractor.CoreExtraction.cs +++ b/src/CodeIndex/Indexer/References/ReferenceExtractor.CoreExtraction.cs @@ -2563,7 +2563,10 @@ bool TryGetKnownPythonTypeCall(string candidate, out string canonicalName) name, callIndex, lineNumber, - dynamicDeclarativeState)) + dynamicDeclarativeState, + language == "groovy" + ? ResolveContainerForCall(callIndex) + : null)) { continue; } diff --git a/src/CodeIndex/Indexer/References/ReferenceExtractor.TypeReferences.cs b/src/CodeIndex/Indexer/References/ReferenceExtractor.TypeReferences.cs index ace76cf2b..b147cc6e6 100644 --- a/src/CodeIndex/Indexer/References/ReferenceExtractor.TypeReferences.cs +++ b/src/CodeIndex/Indexer/References/ReferenceExtractor.TypeReferences.cs @@ -4771,7 +4771,8 @@ lang is "python" or "ruby" or "perl" or "php" or "elixir" or "r" or "powershell" private static bool UsesSlashComments(string lang) => lang is not "python" and not "ruby" and not "r" and not "haskell" and not "makefile" and not "terraform" and not "dockerfile" - and not "css" and not "fortran"; + and not "css" and not "fortran" and not "crystal" and not "tcl" + and not "prolog" and not "ambiguous_pl"; private static bool UsesDashDashComments(string lang) => lang is "lua" or "sql" or "haskell"; diff --git a/tests/CodeIndex.Tests/ReferenceExtractorTests.cs b/tests/CodeIndex.Tests/ReferenceExtractorTests.cs index 8c6b8eeae..12094ed55 100644 --- a/tests/CodeIndex.Tests/ReferenceExtractorTests.cs +++ b/tests/CodeIndex.Tests/ReferenceExtractorTests.cs @@ -1064,8 +1064,11 @@ def run(value) helper value helper(value) text = "123456789"; helper(value) + divided = value // 2; helper(value) helper = 1 end + + class Inline; def inline_helper(); end; end """; var (symbols, references) = ExtractSymbolsAndReferences("crystal", content); @@ -1073,7 +1076,7 @@ helper value Assert.Contains(symbols, symbol => symbol.Kind == "function" && symbol.Name == "run"); AssertReferencesContain(references, "type_reference", null, "support"); AssertReferencesContain(references, "call", "run", "helper"); - Assert.Equal(3, references.Count(reference => + Assert.Equal(4, references.Count(reference => reference.ReferenceKind == "call" && reference.ContainerName == "run" && reference.SymbolName == "helper")); @@ -1089,7 +1092,7 @@ helper value stringCall.Column); Assert.DoesNotContain(references, reference => reference.ReferenceKind == "call" - && reference.SymbolName is "require" or "def"); + && reference.SymbolName is "require" or "def" or "inline_helper"); } [Fact] @@ -1099,8 +1102,10 @@ public void Extract_Groovy_IndexesImportsParenthesizedAndCommandCalls_Issue4746( import demo.Support class Runner { - Runner() { - super(1) + Runner( + int seed + ) { + super(seed) } static def helper(value) { @@ -1115,6 +1120,8 @@ helper value synchronized(value) {} } } + + class InlineRunner { InlineRunner() {}; def inlineHelper() {} } """; var (symbols, references) = ExtractSymbolsAndReferences("groovy", content); @@ -1138,7 +1145,8 @@ helper value stringCall.Column); Assert.DoesNotContain(references, reference => reference.ReferenceKind == "call" - && reference.SymbolName is "import" or "def" or "Runner" or "super" or "synchronized"); + && reference.SymbolName is "import" or "def" or "Runner" or "InlineRunner" + or "inlineHelper" or "super" or "synchronized"); } [Fact] @@ -1312,6 +1320,8 @@ set substituted [helper] } proc multiline {} { set value 1 ;# [helper] + # continued comment \ + helper set color #fff; helper set quoted "[helper]" set braced {[helper]} @@ -1387,6 +1397,9 @@ public void Extract_Prolog_IndexesRelativeImportsAndDecimalHeadsWhileMaskingComm bar(value) = bar(other), bar = value, process(value). + division_data(X) :- + Y is X // 2, + process(Y). meta :- call(helper). build(Term) :- @@ -1410,9 +1423,9 @@ public void Extract_Prolog_IndexesRelativeImportsAndDecimalHeadsWhileMaskingComm reference.ReferenceKind == "call" && reference.ContainerName is "threshold" or "quoted" or "meta" or "build" && reference.SymbolName == "helper")); - Assert.Equal(3, references.Count(reference => + Assert.Equal(4, references.Count(reference => reference.ReferenceKind == "call" - && (reference.ContainerName is "data_only" or "list_data" or "operator_data") + && (reference.ContainerName is "data_only" or "list_data" or "operator_data" or "division_data") && reference.SymbolName == "process")); Assert.DoesNotContain(references, reference => reference.ReferenceKind == "call" @@ -1426,6 +1439,7 @@ public void Extract_AmbiguousPl_MasksPerlAndPrologCommentsAndPod_Issue4746() sub helper { return 1; } prolog_entry :- helper(). my %cache = (answer => helper()); + my $fallback = undef // helper(); clp_entry(X) :- X #= 1, helper. # helper(); % helper(). @@ -1437,7 +1451,7 @@ public void Extract_AmbiguousPl_MasksPerlAndPrologCommentsAndPod_Issue4746() var (_, references) = ExtractSymbolsAndReferences("ambiguous_pl", content); - Assert.Equal(3, references.Count(reference => + Assert.Equal(4, references.Count(reference => reference.ReferenceKind == "call" && reference.SymbolName == "helper")); AssertReferencesContain(references, "call", "prolog_entry", "helper"); From d4b2ff54e70509eb43d5557fb4ce4909b9650f9c Mon Sep 17 00:00:00 2001 From: Widthdom Date: Fri, 24 Jul 2026 16:21:57 +0900 Subject: [PATCH 08/26] Resolve dynamic graph adversarial cases (#4746) --- .../Cli/IndexCommandRunner.FullScan.cs | 5 + .../DynamicDeclarativeReferenceExtractor.cs | 175 +++++++++++++++++- .../ReferenceExtractor.Preparation.cs | 4 +- .../Symbols/SymbolExtractor.ExtractCore.cs | 6 + .../Indexer/Symbols/SymbolExtractor.cs | 3 +- src/CodeIndex/Mcp/McpToolHandlers.Indexing.cs | 5 + .../IndexCommandRunnerFullScanTests.cs | 67 +++++++ .../ReferenceExtractorTests.cs | 49 ++++- 8 files changed, 299 insertions(+), 15 deletions(-) diff --git a/src/CodeIndex/Cli/IndexCommandRunner.FullScan.cs b/src/CodeIndex/Cli/IndexCommandRunner.FullScan.cs index 9f84a5245..6fc6e7cfd 100644 --- a/src/CodeIndex/Cli/IndexCommandRunner.FullScan.cs +++ b/src/CodeIndex/Cli/IndexCommandRunner.FullScan.cs @@ -3517,6 +3517,11 @@ int ReturnBeforeWriteSnapshotFailure(string changedPath) priorHotspotFamilyVersions, priorHotspotFamilyMarkerFingerprints, currentHotspotFamilyMarkerFingerprints); + // Extractor versions describe rows regenerated during this successful run and + // must not depend on whether the independent fold-key contract can be restamped. + // extractor version は今回再生成した row の契約であり、独立した fold-key + // 契約を restamp できるかどうかに依存させない。 + writer.StampSymbolExtractorVersions(indexedSymbolExtractorLanguages); // FoldReady must reflect reality (#86). Full-scan is INCREMENTAL by default — it // skips unchanged files via GetUnchangedFileId, so a legacy DB's pre-#86 rows // keep NULL name_folded / *_folded values. Stamping FoldReady anyway would flip diff --git a/src/CodeIndex/Indexer/References/Languages/DynamicDeclarativeReferenceExtractor.cs b/src/CodeIndex/Indexer/References/Languages/DynamicDeclarativeReferenceExtractor.cs index 0925a6a53..6ce2f8a77 100644 --- a/src/CodeIndex/Indexer/References/Languages/DynamicDeclarativeReferenceExtractor.cs +++ b/src/CodeIndex/Indexer/References/Languages/DynamicDeclarativeReferenceExtractor.cs @@ -10,12 +10,14 @@ internal sealed class ExtractionState { public ExtractionState( HashSet callableNames, + HashSet declarationPositions, IReadOnlyDictionary containersByLine, IReadOnlyList tclContainerScopes, IReadOnlyList? tclCallLines, IReadOnlyDictionary>? prologGoalCallsByLine) { CallableNames = callableNames; + DeclarationPositions = declarationPositions; ContainersByLine = containersByLine; TclContainerScopes = tclContainerScopes; TclCallLines = tclCallLines; @@ -23,6 +25,7 @@ public ExtractionState( } public HashSet CallableNames { get; } + private HashSet DeclarationPositions { get; } public IReadOnlyDictionary ContainersByLine { get; } private IReadOnlyList TclContainerScopes { get; } private IReadOnlyList? TclCallLines { get; } @@ -68,6 +71,9 @@ public IReadOnlyList GetPrologGoalCalls(int lineNumber) public bool HasPrologContainer(int lineNumber) => ContainersByLine.ContainsKey(lineNumber); + + public bool IsDeclarationAt(int lineNumber, int column, string name) => + DeclarationPositions.Contains(new DeclarationPosition(lineNumber, column, name)); } private static readonly Regex TclProcRegex = new( @@ -95,16 +101,16 @@ public bool HasPrologContainer(int lineNumber) => @"(?[A-Za-z_]\w*[?!])\s*\(", RegexOptions.Compiled | RegexOptions.CultureInvariant); private static readonly Regex GroovyBareCallRegex = new( - @"(?:^|[;=])\s*(?:return\s+)?(?[A-Za-z_]\w*)\b(?!\s*(?:\(|(?:<<|>>|[+\-*/%&|^])?=))", + @"(?:^|[;={])\s*(?:return\s+)?(?[A-Za-z_]\w*)\b(?!\s*(?:\(|(?:<<|>>|[+\-*/%&|^])?=))", RegexOptions.Compiled | RegexOptions.CultureInvariant); private static readonly Regex GroovyConstructorDeclarationRegex = new( - @"(?:^|[;{])\s*(?:(?:public|protected|private)\s+)*(?[A-Z]\w*)\s*\(", + @"(?:^|[;{])\s*(?:@[A-Za-z_$][\w.$]*(?:\s*\([^)\r\n]*\))?\s+)*(?:(?:public|protected|private)\s+)*(?[A-Z]\w*)\s*\(", RegexOptions.Compiled | RegexOptions.CultureInvariant); private static readonly Regex GroovyMethodDeclarationRegex = new( - @"(?:^|[;{])\s*(?:(?:public|private|protected|static|final|abstract|synchronized|native)\s+)*(?:def|void|boolean|byte|char|short|int|long|float|double|BigDecimal|BigInteger|String|[A-Za-z_$][\w.$<>\[\]?]*)\s+(?[A-Za-z_]\w*)\s*\(", + @"(?:^|[;{])\s*(?:@[A-Za-z_$][\w.$]*(?:\s*\([^)\r\n]*\))?\s+)*(?:(?:public|private|protected|static|final|abstract|synchronized|native|strictfp)\s+)*(?:def|void|boolean|byte|char|short|int|long|float|double|BigDecimal|BigInteger|String|[A-Za-z_$][\w.$]*(?:\s*<[^(){}\r\n]+>)?(?:\s*\[\])*)\s+(?[A-Za-z_]\w*)\s*\(", RegexOptions.Compiled | RegexOptions.CultureInvariant); private static readonly Regex CrystalMethodDeclarationRegex = new( - @"(?:^|;)\s*(?:(?:private|protected)\s+)?def\s+(?:self\.)?(?[A-Za-z_]\w*[?!]?)\s*\(", + @"(?:^|;)\s*(?:(?:private|protected|abstract)\s+)*def\s+(?:self\.)?(?[A-Za-z_]\w*[?!]?)\s*\(", RegexOptions.Compiled | RegexOptions.CultureInvariant); private static readonly Regex TclCommandRegex = new( @"(?:^|[;\[])\s*(?[A-Za-z_:][\w:.-]*)", @@ -148,6 +154,8 @@ private enum TclLexicalFrameKind internal readonly record struct PrologGoalCall(string Name, int Column); + internal readonly record struct DeclarationPosition(int Line, int Column, string Name); + private enum PrologLexicalFrameKind { PredicateArguments, @@ -235,10 +243,17 @@ public static string[] MaskNonCodeLines(string language, IReadOnlyList l char crystalPercentOpeningDelimiter = '\0'; char crystalPercentClosingDelimiter = '\0'; var crystalPercentDelimiterDepth = 0; + var tclCommentContinued = false; for (var lineIndex = 0; lineIndex < lines.Count; lineIndex++) { var line = lines[lineIndex]; + if (language == "tcl" && tclCommentContinued) + { + result[lineIndex] = new string(' ', line.Length); + tclCommentContinued = HasTclEscapedNewline(line); + continue; + } if (crystalHeredocDelimiter != null) { result[lineIndex] = new string(' ', line.Length); @@ -269,6 +284,9 @@ public static string[] MaskNonCodeLines(string language, IReadOnlyList l } var buffer = line.ToCharArray(); + var tclCommentStart = language == "tcl" + ? FindTclCommentStart(line) + : -1; for (var column = 0; column < line.Length;) { if (crystalMultilineQuote != '\0') @@ -392,6 +410,14 @@ public static string[] MaskNonCodeLines(string language, IReadOnlyList l } var ch = line[column]; + if (language == "tcl" + && ch == '#' + && column == tclCommentStart) + { + FillWithSpaces(buffer, column); + tclCommentContinued = HasTclEscapedNewline(line); + break; + } if (language == "groovy" && ch is '\'' or '"' && column + 2 < line.Length @@ -536,6 +562,49 @@ public static string[] MaskNonCodeLines(string language, IReadOnlyList l return result; } + public static string[] MaskTclContinuedCommentLines( + IReadOnlyList rawLines, + IReadOnlyList maskedLines) + { + var result = maskedLines as string[] ?? maskedLines.ToArray(); + var cloned = false; + var commentContinued = false; + for (var lineIndex = 0; lineIndex < rawLines.Count; lineIndex++) + { + var rawLine = rawLines[lineIndex]; + if (commentContinued) + { + if (!cloned) + { + result = result.ToArray(); + cloned = true; + } + result[lineIndex] = new string(' ', rawLine.Length); + commentContinued = HasTclEscapedNewline(rawLine); + continue; + } + + var commentColumn = FindTclCommentStart(rawLine); + if (commentColumn < 0) + continue; + + commentContinued = HasTclEscapedNewline(rawLine); + if (!commentContinued) + continue; + + if (!cloned) + { + result = result.ToArray(); + cloned = true; + } + var buffer = result[lineIndex].ToCharArray(); + FillWithSpaces(buffer, commentColumn); + result[lineIndex] = new string(buffer); + } + + return result; + } + public static ExtractionState? CreateState( string language, IReadOnlyList preparedLines, @@ -546,10 +615,20 @@ public static string[] MaskNonCodeLines(string language, IReadOnlyList l return null; var callableNames = new HashSet(StringComparer.Ordinal); + var declarationPositions = new HashSet(); foreach (var symbol in symbols) { if (symbol.Kind is "function" or "lambda" or "operator") + { callableNames.Add(symbol.Name); + if (symbol.StartColumn is { } startColumn) + { + declarationPositions.Add(new DeclarationPosition( + symbol.StartLine, + startColumn, + symbol.Name)); + } + } } var containersByLine = new Dictionary(); @@ -582,6 +661,7 @@ public static string[] MaskNonCodeLines(string language, IReadOnlyList l return new ExtractionState( callableNames, + declarationPositions, containersByLine, tclContainerScopes, tclCallLines, @@ -683,6 +763,8 @@ public static bool ShouldSuppressGenericCall( { if (language == "ambiguous_pl") return state?.HasPrologContainer(lineNumber) == true; + if (state?.IsDeclarationAt(lineNumber, callIndex, name) == true) + return true; if (language == "crystal") return MatchesDeclarationAt(CrystalMethodDeclarationRegex, preparedLine, name, callIndex); if (language != "groovy") @@ -1439,15 +1521,25 @@ private static string[] BuildTclCallLines( } if (ch == '"') { + var isScriptArgument = false; if (frame.WordStart) { var wordIndex = frame.WordIndex++; + isScriptArgument = IsTclScriptArgument( + frame, + wordIndex, + lines, + braceEnd: null); UpdateTclSwitchArgumentState(frame, wordIndex, string.Empty); } - buffer[column] = ' '; - frames.Push(new TclLexicalFrame(TclLexicalFrameKind.Quote, '"')); + buffer[column] = isScriptArgument ? ';' : ' '; + frames.Push(new TclLexicalFrame( + isScriptArgument ? TclLexicalFrameKind.Script : TclLexicalFrameKind.Quote, + '"')); frame.CommandStart = false; frame.WordStart = false; + if (isScriptArgument) + suppressLeadingContinuedWord = false; column++; continue; } @@ -1524,6 +1616,8 @@ private static string[] BuildTclCallLines( { var wordIndex = frame.WordIndex++; var token = ReadTclBareWord(line, column); + var isScriptCommand = token.Length > 0 + && IsTclBareScriptCommandArgument(frame, wordIndex); if (wordIndex == 0) frame.CommandName = token; else @@ -1539,6 +1633,10 @@ private static string[] BuildTclCallLines( FillWithSpaces(buffer, column, column + token.Length); suppressLeadingContinuedWord = false; } + else if (isScriptCommand) + { + MarkTclBareScriptCommandBoundary(buffer, column); + } } frame.CommandStart = false; @@ -1562,6 +1660,39 @@ private static bool HasTclEscapedNewline(string line) return backslashCount % 2 == 1; } + private static int FindTclCommentStart(string line) + { + var commandStart = true; + for (var column = 0; column < line.Length; column++) + { + var ch = line[column]; + if (ch is '\'' or '"') + { + column = SkipQuotedToken(line, column, ch) - 1; + commandStart = false; + continue; + } + if (ch == '\\') + { + column++; + commandStart = false; + continue; + } + if (ch == '#' && commandStart) + return column; + if (ch is ';' or '[') + { + commandStart = true; + continue; + } + if (char.IsWhiteSpace(ch)) + continue; + commandStart = false; + } + + return -1; + } + private static string ReadTclBareWord(string line, int startColumn) { var endColumn = startColumn; @@ -1602,6 +1733,7 @@ private static bool IsTclScriptArgument( "for" => wordIndex is 1 or 3 or 4, "eval" => wordIndex >= 1, "after" => wordIndex == 2, + "uplevel" => wordIndex is 1 or 2, "try" => wordIndex == 1, "namespace" => wordIndex == 3, "dict" => frame.LastBareWord == "for" @@ -1613,6 +1745,29 @@ private static bool IsTclScriptArgument( }; } + private static bool IsTclBareScriptCommandArgument( + TclLexicalFrame frame, + int wordIndex) + { + if (frame.CommandName == "eval") + return wordIndex == 1; + if (frame.CommandName == "uplevel") + return wordIndex is 1 or 2; + + return IsTclScriptArgument( + frame, + wordIndex, + Array.Empty(), + braceEnd: null); + } + + private static void MarkTclBareScriptCommandBoundary(char[] buffer, int commandColumn) + { + var boundaryColumn = commandColumn - 1; + if (boundaryColumn >= 0 && char.IsWhiteSpace(buffer[boundaryColumn])) + buffer[boundaryColumn] = ';'; + } + private static bool IsTclSwitchTableArgument( TclLexicalFrame frame, int wordIndex, @@ -1811,6 +1966,14 @@ private static void ScanPrologGoalLine( var nextColumn = column; while (nextColumn < line.Length && char.IsWhiteSpace(line[nextColumn])) nextColumn++; + if (nextColumn < line.Length + && line[nextColumn] == ':' + && (nextColumn + 1 >= line.Length || line[nextColumn + 1] != '-')) + { + column = nextColumn + 1; + expectGoal = true; + continue; + } if (callableNames.Contains(name) && !IsPrologTermBeforeInfixOperator(line, column, nextColumn)) { diff --git a/src/CodeIndex/Indexer/References/ReferenceExtractor.Preparation.cs b/src/CodeIndex/Indexer/References/ReferenceExtractor.Preparation.cs index 72c7a2dc8..ba80f1a5e 100644 --- a/src/CodeIndex/Indexer/References/ReferenceExtractor.Preparation.cs +++ b/src/CodeIndex/Indexer/References/ReferenceExtractor.Preparation.cs @@ -86,7 +86,9 @@ private static bool TryPrepareReferenceLines( : (MultilineStringContent: null, BlockComment: null); var csharpLinesInsideMultilineStringContent = csharpLineState.MultilineStringContent; var csharpLinesInsideBlockComment = csharpLineState.BlockComment; - var referenceStructuralLines = language == "cpp" + var referenceStructuralLines = language == "tcl" + ? lines + : language == "cpp" ? SplitContentLines(MaskCppLexicalRanges( maskedContent, [new CppLexicalRange(0, maskedContent.Length)], diff --git a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.ExtractCore.cs b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.ExtractCore.cs index 5332f7556..9dd6a0f0b 100644 --- a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.ExtractCore.cs +++ b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.ExtractCore.cs @@ -147,6 +147,12 @@ private static List ExtractCore( structuralLines = DynamicDeclarativeReferenceExtractor.MaskNonCodeLines( lang, structuralLines); + if (lang == "tcl") + { + structuralLines = DynamicDeclarativeReferenceExtractor.MaskTclContinuedCommentLines( + lines, + structuralLines); + } string[]? javaScriptTypeScriptSanitizedLines = null; string[] GetJavaScriptTypeScriptSanitizedLines() => javaScriptTypeScriptSanitizedLines ??= BuildJavaScriptTypeScriptSanitizedLines(lines); diff --git a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.cs b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.cs index 92e9bd439..11c7cb5ed 100644 --- a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.cs +++ b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.cs @@ -1618,6 +1618,7 @@ private enum JavaScriptTypeScriptFunctionHeaderConsumeResult new("class", new Regex(@"^\s*(?:abstract\s+)?class\s+(?[A-Z]\w*(?:::[A-Z]\w*)*)", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.RubyEnd), new("struct", new Regex(@"^\s*struct\s+(?[A-Z]\w*(?:::[A-Z]\w*)*)", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.RubyEnd), new("enum", new Regex(@"^\s*enum\s+(?[A-Z]\w*(?:::[A-Z]\w*)*)", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.RubyEnd), + new("function", new Regex(@"^\s*(?:(?:private|protected)\s+)*abstract\s+def\s+(?:self\.)?(?[A-Za-z_]\w*[?!=]?)", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None), new("function", new Regex(@"^\s*(?:private\s+|protected\s+)?def\s+(?:self\.)?(?[A-Za-z_]\w*[?!=]?)", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.RubyEnd), new("function", new Regex(@"^\s*macro\s+(?[A-Za-z_]\w*[?!=]?)", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.RubyEnd), new("typealias", new Regex(@"^\s*alias\s+(?[A-Z]\w*)\s*=", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None), @@ -1629,7 +1630,7 @@ private enum JavaScriptTypeScriptFunctionHeaderConsumeResult new("interface", new Regex(@"^\s*(?:(?:public|private|protected|static|abstract)\s+)*(?:interface|trait)\s+(?[A-Za-z_]\w*)", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.Brace), new("enum", new Regex(@"^\s*(?:(?:public|private|protected|static)\s+)*enum\s+(?[A-Za-z_]\w*)", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.Brace), new("class", new Regex(@"^\s*(?:(?:public|private|protected|static|abstract|final)\s+)*class\s+(?[A-Za-z_]\w*)", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.Brace), - new("function", new Regex(@"^\s*(?!(?:if|for|while|switch|catch|return|throw|new)\b)(?:(?:public|private|protected|static|final|abstract|synchronized|native)\s+)*(?def|void|boolean|byte|char|short|int|long|float|double|BigDecimal|BigInteger|String|[\w.$<>?\[\]]+)\s+(?[A-Za-z_]\w*)\s*\(", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.Brace, ReturnTypeGroup: "returnType"), + new("function", new Regex(@"^\s*(?:@[A-Za-z_$][\w.$]*(?:\s*\([^)\r\n]*\))?\s+)*(?!(?:if|for|while|switch|catch|return|throw|new)\b)(?:(?:public|private|protected|static|final|abstract|synchronized|native|strictfp)\s+)*(?def|void|boolean|byte|char|short|int|long|float|double|BigDecimal|BigInteger|String|[A-Za-z_$][\w.$]*(?:\s*<[^(){}\r\n]+>)?(?:\s*\[\])*)\s+(?[A-Za-z_]\w*)\s*\(", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.Brace, ReturnTypeGroup: "returnType"), new("lambda", new Regex(@"^\s*(?:def\s+)?(?[A-Za-z_]\w*)\s*=\s*\{", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.Brace), new("import", new Regex(@"^\s*import\s+(?:static\s+)?(?[A-Za-z_]\w*(?:\.[A-Za-z_]\w*)*(?:\.\*)?)", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None), ], diff --git a/src/CodeIndex/Mcp/McpToolHandlers.Indexing.cs b/src/CodeIndex/Mcp/McpToolHandlers.Indexing.cs index 584caeaee..d41b27848 100644 --- a/src/CodeIndex/Mcp/McpToolHandlers.Indexing.cs +++ b/src/CodeIndex/Mcp/McpToolHandlers.Indexing.cs @@ -2091,6 +2091,11 @@ await EmitProgressNotificationAsync( priorHotspotFamilyVersions, priorHotspotFamilyMarkerFingerprints, currentHotspotFamilyMarkerFingerprints); + // A successful refresh can stamp the languages it regenerated even when the + // independent fold-key contract remains stale. + // 成功した refresh で再生成した言語は、独立した fold-key 契約が stale の + // ままでも extractor version を stamp できる。 + writer.StampSymbolExtractorVersions(indexedSymbolExtractorLanguages); // FoldReady must reflect reality (#86). Like CLI full-scan, MCP index_project skips // unchanged files via GetUnchangedFileId, so a legacy DB's pre-#86 rows keep NULL // name_folded / *_folded. Stamp only when every row is backfilled; otherwise readers diff --git a/tests/CodeIndex.Tests/IndexCommandRunnerFullScanTests.cs b/tests/CodeIndex.Tests/IndexCommandRunnerFullScanTests.cs index ed29f128c..e34017cad 100644 --- a/tests/CodeIndex.Tests/IndexCommandRunnerFullScanTests.cs +++ b/tests/CodeIndex.Tests/IndexCommandRunnerFullScanTests.cs @@ -4004,6 +4004,73 @@ WHERE kind IN ('route', 'implements', 'attribute', 'layout') } } + [Fact] + public void Run_FullScan_StampsRefreshedDynamicGraphContractWhenFoldContractRemainsStale_Issue4746() + { + var projectRoot = CreateTempProject(); + try + { + File.WriteAllText( + Path.Combine(projectRoot, "commands.tcl"), + """ + proc helper {} { return 1 } + proc run {} { helper } + """); + File.WriteAllText( + Path.Combine(projectRoot, "unchanged.cs"), + "class Unchanged { void Run() { } }"); + + var initialExitCode = IndexCommandRunner.Run([projectRoot, "--json"], _jsonOptions); + Assert.Equal(CommandExitCodes.Success, initialExitCode); + + var dbPath = Path.Combine(projectRoot, ".cdidx", "codeindex.db"); + using (var conn = OpenNonPoolingConnection(dbPath)) + { + conn.Open(); + using var cmd = conn.CreateCommand(); + cmd.CommandText = $""" + UPDATE codeindex_meta + SET value = '2' + WHERE key = '{DbContext.GetSymbolExtractorVersionMetaKey("tcl")}'; + UPDATE codeindex_meta + SET value = '0' + WHERE key = 'fold_key_version'; + """; + cmd.ExecuteNonQuery(); + } + + var (exitCode, json) = RunAndCaptureJson([projectRoot, "--json"]); + + Assert.Equal(CommandExitCodes.Success, exitCode); + Assert.Equal("success", json.GetProperty("status").GetString()); + Assert.Equal(1, json.GetProperty("summary").GetProperty("files_skipped").GetInt32()); + + using var verify = OpenNonPoolingConnection(dbPath); + verify.Open(); + using var versionCmd = verify.CreateCommand(); + versionCmd.CommandText = + $"SELECT value FROM codeindex_meta WHERE key = '{DbContext.GetSymbolExtractorVersionMetaKey("tcl")}'"; + Assert.Equal( + SymbolExtractor.DynamicReferenceGraphContractVersion.ToString( + System.Globalization.CultureInfo.InvariantCulture), + versionCmd.ExecuteScalar() as string); + + using var referenceCmd = verify.CreateCommand(); + referenceCmd.CommandText = """ + SELECT COUNT(*) + FROM symbol_references + WHERE symbol_name = 'helper' + AND container_name = 'run' + AND reference_kind = 'call' + """; + Assert.Equal(1L, referenceCmd.ExecuteScalar()); + } + finally + { + DeleteDirectory(projectRoot); + } + } + [Fact] public void Run_FullScan_ReindexesUnchangedCSharpTupleReadonlyFieldWhenExtractorVersionChanged_Issue4616() { diff --git a/tests/CodeIndex.Tests/ReferenceExtractorTests.cs b/tests/CodeIndex.Tests/ReferenceExtractorTests.cs index 12094ed55..e91dc973b 100644 --- a/tests/CodeIndex.Tests/ReferenceExtractorTests.cs +++ b/tests/CodeIndex.Tests/ReferenceExtractorTests.cs @@ -1069,11 +1069,16 @@ helper value end class Inline; def inline_helper(); end; end + + abstract class Base + abstract def declared(value) + end """; var (symbols, references) = ExtractSymbolsAndReferences("crystal", content); Assert.Contains(symbols, symbol => symbol.Kind == "function" && symbol.Name == "run"); + Assert.Contains(symbols, symbol => symbol.Kind == "function" && symbol.Name == "declared"); AssertReferencesContain(references, "type_reference", null, "support"); AssertReferencesContain(references, "call", "run", "helper"); Assert.Equal(4, references.Count(reference => @@ -1092,7 +1097,7 @@ class Inline; def inline_helper(); end; end stringCall.Column); Assert.DoesNotContain(references, reference => reference.ReferenceKind == "call" - && reference.SymbolName is "require" or "def" or "inline_helper"); + && reference.SymbolName is "require" or "def" or "inline_helper" or "declared"); } [Fact] @@ -1102,7 +1107,7 @@ public void Extract_Groovy_IndexesImportsParenthesizedAndCommandCalls_Issue4746( import demo.Support class Runner { - Runner( + @Inject Runner( int seed ) { super(seed) @@ -1112,10 +1117,21 @@ static def helper(value) { value } + @Override def annotated(value) { + value + } + + Map convert(String value) { + [:] + } + def run(value) { helper value helper(value) def text = "123456789"; helper(value) + [1].each { helper it } + annotated(value) + convert("value") helper = 1 synchronized(value) {} } @@ -1127,12 +1143,22 @@ class InlineRunner { InlineRunner() {}; def inlineHelper() {} } var (symbols, references) = ExtractSymbolsAndReferences("groovy", content); Assert.Contains(symbols, symbol => symbol.Kind == "function" && symbol.Name == "run"); + Assert.Contains(symbols, symbol => symbol.Kind == "function" && symbol.Name == "annotated"); + Assert.Contains(symbols, symbol => symbol.Kind == "function" && symbol.Name == "convert"); AssertReferencesContain(references, "type_reference", null, "Support"); AssertReferencesContain(references, "call", "run", "helper"); - Assert.Equal(3, references.Count(reference => + Assert.Equal(4, references.Count(reference => reference.ReferenceKind == "call" && reference.ContainerName == "run" && reference.SymbolName == "helper")); + Assert.Single(references, reference => + reference.ReferenceKind == "call" + && reference.ContainerName == "run" + && reference.SymbolName == "annotated"); + Assert.Single(references, reference => + reference.ReferenceKind == "call" + && reference.ContainerName == "run" + && reference.SymbolName == "convert"); var stringCallLine = Array.FindIndex( content.Split('\n'), line => line.Contains("text =", StringComparison.Ordinal)) + 1; @@ -1312,6 +1338,8 @@ fake 1 public void Extract_Tcl_IndexesSingleLineProcBodiesAndMasksComments_Issue4746() { const string content = """ + # continued declaration comment \ + proc fake {} { helper } proc helper {} { return 1 } # [helper] proc run {helper} { @@ -1337,6 +1365,10 @@ set braced {[helper]} catch { helper } + if {1} "helper" + eval helper + after 1 helper + uplevel helper switch $value { one { helper } default { @@ -1359,13 +1391,14 @@ proc split {value} proc sameLine {} { helper }; helper """; - var (_, references) = ExtractSymbolsAndReferences("tcl", content); + var (symbols, references) = ExtractSymbolsAndReferences("tcl", content); + Assert.DoesNotContain(symbols, symbol => symbol.Name == "fake"); AssertReferencesContain(references, "call", "run", "helper"); AssertReferencesContain(references, "call", "multiline", "helper"); AssertReferencesContain(references, "call", "split", "helper"); AssertReferencesContain(references, "call", "sameLine", "helper"); - Assert.Equal(15, references.Count(reference => + Assert.Equal(19, references.Count(reference => reference.ReferenceKind == "call" && reference.SymbolName == "helper")); Assert.Single(references, reference => @@ -1406,6 +1439,8 @@ public void Extract_Prolog_IndexesRelativeImportsAndDecimalHeadsWhileMaskingComm Term =.. [node, value], helper. + qualified :- + demo:helper. % helper(). /* helper(). @@ -1419,9 +1454,9 @@ public void Extract_Prolog_IndexesRelativeImportsAndDecimalHeadsWhileMaskingComm Assert.Contains(symbols, symbol => symbol.Kind == "function" && symbol.Name == "build"); Assert.Single(symbols, symbol => symbol.Kind == "function" && symbol.Name == "helper"); AssertReferencesContain(references, "type_reference", null, "support"); - Assert.Equal(4, references.Count(reference => + Assert.Equal(5, references.Count(reference => reference.ReferenceKind == "call" - && reference.ContainerName is "threshold" or "quoted" or "meta" or "build" + && reference.ContainerName is "threshold" or "quoted" or "meta" or "build" or "qualified" && reference.SymbolName == "helper")); Assert.Equal(4, references.Count(reference => reference.ReferenceKind == "call" From fef710f4173948b7d1637a6b862c9b81e493ba7c Mon Sep 17 00:00:00 2001 From: Widthdom Date: Fri, 24 Jul 2026 16:51:30 +0900 Subject: [PATCH 09/26] Complete dynamic graph edge cases (#4746) --- .../DynamicDeclarativeReferenceExtractor.cs | 152 +++++++++++++++++- .../Symbols/SymbolExtractor.ExtractCore.cs | 124 +++++++++++++- .../ReferenceExtractorTests.cs | 64 ++++++++ 3 files changed, 331 insertions(+), 9 deletions(-) diff --git a/src/CodeIndex/Indexer/References/Languages/DynamicDeclarativeReferenceExtractor.cs b/src/CodeIndex/Indexer/References/Languages/DynamicDeclarativeReferenceExtractor.cs index 6ce2f8a77..205bd9111 100644 --- a/src/CodeIndex/Indexer/References/Languages/DynamicDeclarativeReferenceExtractor.cs +++ b/src/CodeIndex/Indexer/References/Languages/DynamicDeclarativeReferenceExtractor.cs @@ -82,6 +82,9 @@ public bool IsDeclarationAt(int lineNumber, int column, string name) => private static readonly Regex PrologHeadRegex = new( @"^\s*(?[a-z][A-Za-z0-9_]*)\s*(?:\([^\r\n]*\))?\s*(?::-|-->|\.(?=\s*$))", RegexOptions.Compiled | RegexOptions.CultureInvariant); + private static readonly Regex PrologMultilineHeadRegex = new( + @"^\s*(?[a-z][A-Za-z0-9_]*)\s*\(", + RegexOptions.Compiled | RegexOptions.CultureInvariant); private static readonly Regex CrystalRequireRegex = new( @"^\s*require\s+['""](?[^'""]+)['""]", RegexOptions.Compiled | RegexOptions.CultureInvariant); @@ -213,6 +216,9 @@ private sealed class TclLexicalFrame( public int LastBareWordIndex { get; set; } = -1; public int SwitchStringWordIndex { get; set; } = -1; public bool SwitchOptionsEnded { get; set; } + public bool SwitchOptionValuePending { get; set; } + public int TryClauseWordIndex { get; set; } = 2; + public int TryScriptWordIndex { get; set; } = 1; public void ResetCommand() { @@ -224,6 +230,9 @@ public void ResetCommand() LastBareWordIndex = -1; SwitchStringWordIndex = -1; SwitchOptionsEnded = false; + SwitchOptionValuePending = false; + TryClauseWordIndex = 2; + TryScriptWordIndex = 1; } } @@ -1531,6 +1540,7 @@ private static string[] BuildTclCallLines( lines, braceEnd: null); UpdateTclSwitchArgumentState(frame, wordIndex, string.Empty); + UpdateTclTryArgumentState(frame, wordIndex, string.Empty, isScriptArgument); } buffer[column] = isScriptArgument ? ';' : ' '; frames.Push(new TclLexicalFrame( @@ -1549,6 +1559,11 @@ private static string[] BuildTclCallLines( { var wordIndex = frame.WordIndex++; UpdateTclSwitchArgumentState(frame, wordIndex, string.Empty); + UpdateTclTryArgumentState( + frame, + wordIndex, + string.Empty, + isScriptArgument: false); } frames.Push(new TclLexicalFrame(TclLexicalFrameKind.Script, ']')); frame.CommandStart = false; @@ -1577,6 +1592,11 @@ private static string[] BuildTclCallLines( lines, braceEnd)); UpdateTclSwitchArgumentState(frame, wordIndex, string.Empty); + UpdateTclTryArgumentState( + frame, + wordIndex, + string.Empty, + isScriptArgument); if (isSwitchTable) { buffer[column] = ' '; @@ -1621,7 +1641,14 @@ private static string[] BuildTclCallLines( if (wordIndex == 0) frame.CommandName = token; else + { UpdateTclSwitchArgumentState(frame, wordIndex, token); + UpdateTclTryArgumentState( + frame, + wordIndex, + token, + isScriptCommand); + } if (token.Length > 0) { frame.LastBareWord = token; @@ -1734,7 +1761,7 @@ private static bool IsTclScriptArgument( "eval" => wordIndex >= 1, "after" => wordIndex == 2, "uplevel" => wordIndex is 1 or 2, - "try" => wordIndex == 1, + "try" => wordIndex == frame.TryScriptWordIndex, "namespace" => wordIndex == 3, "dict" => frame.LastBareWord == "for" && wordIndex == frame.LastBareWordIndex + 3, @@ -1793,16 +1820,51 @@ private static void UpdateTclSwitchArgumentState( return; } + if (frame.SwitchOptionValuePending) + { + frame.SwitchOptionValuePending = false; + return; + } + if (!frame.SwitchOptionsEnded && token.StartsWith("-", StringComparison.Ordinal)) { if (token == "--") frame.SwitchOptionsEnded = true; + else if (token is "-matchvar" or "-indexvar") + frame.SwitchOptionValuePending = true; return; } frame.SwitchStringWordIndex = wordIndex; } + private static void UpdateTclTryArgumentState( + TclLexicalFrame frame, + int wordIndex, + string token, + bool isScriptArgument) + { + if (frame.CommandName != "try") + return; + + if (isScriptArgument) + { + frame.TryClauseWordIndex = wordIndex + 1; + frame.TryScriptWordIndex = -1; + return; + } + + if (wordIndex != frame.TryClauseWordIndex) + return; + + frame.TryScriptWordIndex = token switch + { + "on" or "trap" => wordIndex + 3, + "finally" => wordIndex + 1, + _ => -1, + }; + } + private static bool IsTclLastCommandWord( IReadOnlyList lines, TclBraceEnd braceEnd) @@ -1830,6 +1892,9 @@ private static IReadOnlyDictionary> BuildProl var frames = new Stack(); var expectGoal = true; SymbolRecord? activeContainer = null; + var scanningMultilineHead = false; + var multilineHeadParenthesisDepth = 0; + var multilineHeadParenthesesClosed = false; for (var lineIndex = 0; lineIndex < lines.Count; lineIndex++) { @@ -1839,6 +1904,9 @@ private static IReadOnlyDictionary> BuildProl frames.Clear(); activeContainer = null; expectGoal = true; + scanningMultilineHead = false; + multilineHeadParenthesisDepth = 0; + multilineHeadParenthesesClosed = false; continue; } @@ -1849,12 +1917,30 @@ private static IReadOnlyDictionary> BuildProl frames.Clear(); activeContainer = container; expectGoal = true; + scanningMultilineHead = container.StartLine == lineNumber + && !PrologHeadRegex.IsMatch(lines[lineIndex]) + && PrologMultilineHeadRegex.IsMatch(lines[lineIndex]); + multilineHeadParenthesisDepth = 0; + multilineHeadParenthesesClosed = false; } - var callScanLine = PreparePrologCallScanLine( - "prolog", - lines[lineIndex], - container.StartLine < lineNumber); + string callScanLine; + if (scanningMultilineHead) + { + callScanLine = PreparePrologMultilineHeadScanLine( + lines[lineIndex], + ref multilineHeadParenthesisDepth, + ref multilineHeadParenthesesClosed, + out var headEnded); + scanningMultilineHead = !headEnded; + } + else + { + callScanLine = PreparePrologCallScanLine( + "prolog", + lines[lineIndex], + container.StartLine < lineNumber); + } var lineCalls = new List(); ScanPrologGoalLine( callScanLine, @@ -1876,6 +1962,60 @@ private static IReadOnlyDictionary> BuildProl return result; } + private static string PreparePrologMultilineHeadScanLine( + string line, + ref int parenthesisDepth, + ref bool parenthesesClosed, + out bool headEnded) + { + headEnded = false; + for (var column = 0; column < line.Length; column++) + { + var ch = line[column]; + if (ch is '\'' or '"') + { + column = SkipQuotedToken(line, column, ch) - 1; + continue; + } + + if (!parenthesesClosed) + { + if (ch == '(') + { + parenthesisDepth++; + } + else if (ch == ')' && parenthesisDepth > 0) + { + parenthesisDepth--; + parenthesesClosed = parenthesisDepth == 0; + } + continue; + } + + if (line.AsSpan(column).StartsWith("-->", StringComparison.Ordinal)) + { + var masked = line.ToCharArray(); + FillWithSpaces(masked, 0, column + 3); + headEnded = true; + return new string(masked); + } + if (line.AsSpan(column).StartsWith(":-", StringComparison.Ordinal)) + { + var masked = line.ToCharArray(); + FillWithSpaces(masked, 0, column + 2); + headEnded = true; + return new string(masked); + } + if (IsPrologClauseTerminator(line, column)) + { + headEnded = true; + return new string(' ', line.Length); + } + } + + return new string(' ', line.Length); + } + private static void ScanPrologGoalLine( string line, IReadOnlySet callableNames, @@ -2179,6 +2319,8 @@ private static void AddPrologContainers( var startLineIndex = symbol.StartLine - 1; var headMatch = PrologHeadRegex.Match(lines[startLineIndex]); + if (!headMatch.Success) + headMatch = PrologMultilineHeadRegex.Match(lines[startLineIndex]); if (!headMatch.Success || !string.Equals(headMatch.Groups["name"].Value, symbol.Name, StringComparison.Ordinal)) { diff --git a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.ExtractCore.cs b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.ExtractCore.cs index 9dd6a0f0b..79ec13343 100644 --- a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.ExtractCore.cs +++ b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.ExtractCore.cs @@ -165,9 +165,15 @@ string[] GetJavaScriptTypeScriptSanitizedLines() => var shellScannerLines = lang == "shell" ? MaskShellHeredocLines(lines) : null; - var prologClauseContinuationLines = lang is "prolog" or "ambiguous_pl" - ? BuildPrologClauseContinuationLines(structuralLines) - : null; + bool[]? prologClauseContinuationLines = null; + Dictionary? prologMultilineHeads = null; + if (lang is "prolog" or "ambiguous_pl") + { + prologMultilineHeads = []; + prologClauseContinuationLines = BuildPrologClauseContinuationLines( + structuralLines, + prologMultilineHeads); + } var powershellEnumBodyLines = lang == "powershell" ? FindPowerShellEnumBodyLines(structuralLines) : null; @@ -2145,6 +2151,15 @@ bool[] GetCssQualifiedRuleAncestors() => ExtractSectionHeadingSymbols(fileId, lang, lines, symbols); if (IsRazorLanguage(originalLang) || IsRazorFilePath(filePath)) ExtractRazorDirectiveSymbols(fileId, lines, symbols); + if (prologMultilineHeads is { Count: > 0 }) + { + AddPrologMultilineHeadSymbols( + fileId, + lines, + symbols, + extractionState, + prologMultilineHeads); + } AssignContainers(symbols, lines, getCSharpLineStartStates); if (lang is "shell" or "powershell") AddScriptScopeSymbol(fileId, lines, symbols); @@ -2169,8 +2184,13 @@ bool[] GetCssQualifiedRuleAncestors() => private static readonly Regex PrologOpenClauseRegex = new( @"^\s*(?:(?:[a-z][A-Za-z0-9_]*\s*(?:\([^\r\n]*\))?\s*(?::-|-->))|:-)", RegexOptions.Compiled | RegexOptions.CultureInvariant); + private static readonly Regex PrologMultilineHeadStartRegex = new( + @"^\s*(?[a-z][A-Za-z0-9_]*)\s*(?\()", + RegexOptions.Compiled | RegexOptions.CultureInvariant); - private static bool[] BuildPrologClauseContinuationLines(IReadOnlyList structuralLines) + private static bool[] BuildPrologClauseContinuationLines( + IReadOnlyList structuralLines, + Dictionary multilineHeads) { var continuationLines = new bool[structuralLines.Count]; var clauseOpen = false; @@ -2186,12 +2206,108 @@ private static bool[] BuildPrologClauseContinuationLines(IReadOnlyList s } if (PrologOpenClauseRegex.IsMatch(line) && !HasPrologClauseTerminator(line)) + { + clauseOpen = true; + continue; + } + + var multilineHead = PrologMultilineHeadStartRegex.Match(line); + if (multilineHead.Success + && !HasPrologClauseTerminator(line) + && IsValidatedMultilinePrologHead( + structuralLines, + lineIndex, + multilineHead.Groups["open"].Index)) + { + multilineHeads[lineIndex] = multilineHead.Groups["name"].Value; clauseOpen = true; + } } return continuationLines; } + private static bool IsValidatedMultilinePrologHead( + IReadOnlyList structuralLines, + int startLineIndex, + int openingParenthesisColumn) + { + var parenthesisDepth = 0; + var headClosed = false; + for (var lineIndex = startLineIndex; lineIndex < structuralLines.Count; lineIndex++) + { + var line = structuralLines[lineIndex]; + var startColumn = lineIndex == startLineIndex ? openingParenthesisColumn : 0; + for (var column = startColumn; column < line.Length; column++) + { + var ch = line[column]; + if (ch is '\'' or '"') + { + column = SkipPrologQuotedTerm(line, column, ch) - 1; + continue; + } + + if (!headClosed) + { + if (ch == '(') + { + parenthesisDepth++; + } + else if (ch == ')' && parenthesisDepth > 0) + { + parenthesisDepth--; + headClosed = parenthesisDepth == 0; + } + continue; + } + + if (char.IsWhiteSpace(ch)) + continue; + if (line.AsSpan(column).StartsWith(":-", StringComparison.Ordinal) + || line.AsSpan(column).StartsWith("-->", StringComparison.Ordinal) + || DynamicDeclarativeReferenceExtractor.IsPrologClauseTerminator(line, column)) + { + return true; + } + + return false; + } + } + + return false; + } + + private static void AddPrologMultilineHeadSymbols( + long fileId, + IReadOnlyList lines, + List symbols, + SymbolExtractionState extractionState, + IReadOnlyDictionary multilineHeads) + { + foreach (var (lineIndex, name) in multilineHeads) + { + var lineNumber = lineIndex + 1; + var line = lines[lineIndex]; + AddSymbolRecord( + symbols, + extractionState, + cssSeenSymbols: null, + lineNumber, + new SymbolRecord + { + FileId = fileId, + Kind = "function", + Name = name, + Line = lineNumber, + StartLine = lineNumber, + StartColumn = 0, + EndLine = lineNumber, + Signature = line.Trim(), + }, + line); + } + } + private static bool HasPrologClauseTerminator(string line) { for (var column = 0; column < line.Length; column++) diff --git a/tests/CodeIndex.Tests/ReferenceExtractorTests.cs b/tests/CodeIndex.Tests/ReferenceExtractorTests.cs index e91dc973b..14f347de6 100644 --- a/tests/CodeIndex.Tests/ReferenceExtractorTests.cs +++ b/tests/CodeIndex.Tests/ReferenceExtractorTests.cs @@ -1206,6 +1206,46 @@ proc variadic args { && reference.SymbolName is "package" or "proc" or "set"); } + [Fact] + public void Extract_Tcl_IndexesSwitchOptionAndTryHandlerScripts_Issue4746() + { + const string content = """ + proc helper {} { + return 1 + } + + proc switch_run {value} { + switch -regexp -matchvar matched -indexvar indices -- $value { + one { helper } + default { helper } + } + } + + proc try_run {} { + try { + helper + } on error {message options} { + helper + } trap {POSIX SIG} {message options} { + helper + } finally { + helper + } + } + """; + + var (_, references) = ExtractSymbolsAndReferences("tcl", content); + + Assert.Equal(2, references.Count(reference => + reference.ReferenceKind == "call" + && reference.ContainerName == "switch_run" + && reference.SymbolName == "helper")); + Assert.Equal(4, references.Count(reference => + reference.ReferenceKind == "call" + && reference.ContainerName == "try_run" + && reference.SymbolName == "helper")); + } + [Fact] public void Extract_Prolog_IndexesModulesImportsAndPredicateCallsWithCaller_Issue4746() { @@ -1231,6 +1271,30 @@ public void Extract_Prolog_IndexesModulesImportsAndPredicateCallsWithCaller_Issu && reference.SymbolName is "module" or "use_module" or "ancestor"); } + [Theory] + [InlineData("prolog")] + [InlineData("ambiguous_pl")] + public void Extract_Prolog_IndexesMultilinePredicateHeadsWithoutPhantomDeclarations_Issue4746( + string language) + { + const string content = """ + helper. + multi_line_head( + X + ) :- + helper. + """; + + var (symbols, references) = ExtractSymbolsAndReferences(language, content); + + Assert.Single(symbols, symbol => symbol.Kind == "function" && symbol.Name == "helper"); + Assert.Single(symbols, symbol => symbol.Kind == "function" && symbol.Name == "multi_line_head"); + AssertReferencesContain(references, "call", "multi_line_head", "helper"); + Assert.DoesNotContain(references, reference => + reference.ReferenceKind == "call" + && reference.SymbolName == "multi_line_head"); + } + [Fact] public void Extract_AmbiguousPl_IndexesPerlAndPrologReferencesWithoutReclassification_Issue4746() { From c99a90a57b5de0c1c8f68d0468d4f38781768bdd Mon Sep 17 00:00:00 2001 From: Widthdom Date: Fri, 24 Jul 2026 17:10:30 +0900 Subject: [PATCH 10/26] Harden dynamic graph upgrade paths (#4746) --- changelog.d/unreleased/4746.added.md | 4 ++ .../Database/DbWriter.FoldBackfill.cs | 2 +- .../DynamicDeclarativeReferenceExtractor.cs | 43 +++++++++++++++++-- .../Symbols/SymbolExtractor.ExtractCore.cs | 6 ++- .../Indexer/Symbols/SymbolExtractor.cs | 3 ++ tests/CodeIndex.Tests/DatabaseTests.cs | 23 ++++++++++ .../IndexCommandRunnerFullScanTests.cs | 3 +- .../ReferenceExtractorTests.cs | 10 ++++- tests/CodeIndex.Tests/SymbolExtractorTests.cs | 15 +++++++ 9 files changed, 101 insertions(+), 8 deletions(-) diff --git a/changelog.d/unreleased/4746.added.md b/changelog.d/unreleased/4746.added.md index 1fca0c15e..2fd331414 100644 --- a/changelog.d/unreleased/4746.added.md +++ b/changelog.d/unreleased/4746.added.md @@ -12,10 +12,14 @@ affected: - src/CodeIndex/Indexer/References/ReferenceExtractor.TypeReferences.cs - src/CodeIndex/Indexer/References/Languages/DynamicDeclarativeReferenceExtractor.cs - src/CodeIndex/Database/DbReader.Status.cs + - src/CodeIndex/Database/DbWriter.FoldBackfill.cs - src/CodeIndex/Database/DegradationReasonCodes.cs + - src/CodeIndex/Cli/IndexCommandRunner.FullScan.cs - src/CodeIndex/Cli/QueryCommandRunner.Status.cs - src/CodeIndex/Cli/QueryCommandRunner.StatusFields.cs + - src/CodeIndex/Mcp/McpToolHandlers.Indexing.cs - tests/CodeIndex.Tests/DatabaseTests.cs + - tests/CodeIndex.Tests/IndexCommandRunnerFullScanTests.cs - tests/CodeIndex.Tests/SymbolExtractorTests.cs - tests/CodeIndex.Tests/ReferenceExtractorTests.cs - tests/CodeIndex.Tests/QueryCommandRunnerTests.cs diff --git a/src/CodeIndex/Database/DbWriter.FoldBackfill.cs b/src/CodeIndex/Database/DbWriter.FoldBackfill.cs index f25c617e4..0f73bd093 100644 --- a/src/CodeIndex/Database/DbWriter.FoldBackfill.cs +++ b/src/CodeIndex/Database/DbWriter.FoldBackfill.cs @@ -216,7 +216,7 @@ private bool SymbolExtractorVersionMatchesCurrent(string? lang) var stored = GetMetaString(DbContext.GetSymbolExtractorVersionMetaKey(lang)); if (stored == null) - return true; + return !SymbolExtractor.RequiresExplicitReferenceGraphContractStamp(lang); var current = SymbolExtractor.GetContractVersion(lang).ToString(System.Globalization.CultureInfo.InvariantCulture); return stored == current; diff --git a/src/CodeIndex/Indexer/References/Languages/DynamicDeclarativeReferenceExtractor.cs b/src/CodeIndex/Indexer/References/Languages/DynamicDeclarativeReferenceExtractor.cs index 205bd9111..bb264a27c 100644 --- a/src/CodeIndex/Indexer/References/Languages/DynamicDeclarativeReferenceExtractor.cs +++ b/src/CodeIndex/Indexer/References/Languages/DynamicDeclarativeReferenceExtractor.cs @@ -885,8 +885,6 @@ private static bool IsPerlHashSigil(string line, int column) var previousColumn = column - 1; while (previousColumn >= 0 && char.IsWhiteSpace(line[previousColumn])) previousColumn--; - if (previousColumn >= 0) - return true; var tokenEnd = column + 1; if (line[tokenEnd] == '{') @@ -899,7 +897,46 @@ private static bool IsPerlHashSigil(string line, int column) while (tokenEnd < line.Length && char.IsWhiteSpace(line[tokenEnd])) tokenEnd++; - return tokenEnd >= line.Length || line[tokenEnd] is '=' or ';' or '{' or '[' or ','; + if (previousColumn < 0) + return tokenEnd < line.Length && line[tokenEnd] is '=' or '{' or '['; + + var prefix = line.AsSpan(0, previousColumn + 1); + if (prefix.Contains(":-", StringComparison.Ordinal) + || prefix.Contains("-->", StringComparison.Ordinal) + || line[previousColumn] == '.') + { + return false; + } + + if (line[previousColumn] is '=' or '(' or '[' or '{' or ',' or '\\') + return true; + if (line[previousColumn] == '>' + && previousColumn > 0 + && line[previousColumn - 1] == '=') + { + return true; + } + + var previousTokenStart = previousColumn; + while (previousTokenStart >= 0 + && (char.IsLetterOrDigit(line[previousTokenStart]) + || line[previousTokenStart] == '_')) + { + previousTokenStart--; + } + var previousToken = line.AsSpan(previousTokenStart + 1, previousColumn - previousTokenStart); + return previousToken.Equals("my", StringComparison.Ordinal) + || previousToken.Equals("our", StringComparison.Ordinal) + || previousToken.Equals("state", StringComparison.Ordinal) + || previousToken.Equals("local", StringComparison.Ordinal) + || previousToken.Equals("return", StringComparison.Ordinal) + || previousToken.Equals("keys", StringComparison.Ordinal) + || previousToken.Equals("values", StringComparison.Ordinal) + || previousToken.Equals("each", StringComparison.Ordinal) + || previousToken.Equals("delete", StringComparison.Ordinal) + || previousToken.Equals("exists", StringComparison.Ordinal) + || previousToken.Equals("defined", StringComparison.Ordinal) + || previousToken.Equals("scalar", StringComparison.Ordinal); } private static bool IsLikelySlashyLiteralStart(string line, int column) diff --git a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.ExtractCore.cs b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.ExtractCore.cs index 79ec13343..137068fc6 100644 --- a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.ExtractCore.cs +++ b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.ExtractCore.cs @@ -2187,6 +2187,7 @@ bool[] GetCssQualifiedRuleAncestors() => private static readonly Regex PrologMultilineHeadStartRegex = new( @"^\s*(?[a-z][A-Za-z0-9_]*)\s*(?\()", RegexOptions.Compiled | RegexOptions.CultureInvariant); + private const int PrologMultilineHeadLookaheadLineLimit = 256; private static bool[] BuildPrologClauseContinuationLines( IReadOnlyList structuralLines, @@ -2234,7 +2235,10 @@ private static bool IsValidatedMultilinePrologHead( { var parenthesisDepth = 0; var headClosed = false; - for (var lineIndex = startLineIndex; lineIndex < structuralLines.Count; lineIndex++) + var endLineExclusive = Math.Min( + structuralLines.Count, + startLineIndex + PrologMultilineHeadLookaheadLineLimit); + for (var lineIndex = startLineIndex; lineIndex < endLineExclusive; lineIndex++) { var line = structuralLines[lineIndex]; var startColumn = lineIndex == startLineIndex ? openingParenthesisColumn : 0; diff --git a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.cs b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.cs index 11c7cb5ed..39f08525e 100644 --- a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.cs +++ b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.cs @@ -90,6 +90,9 @@ public static int GetContractVersion(string? lang) }; } + internal static bool RequiresExplicitReferenceGraphContractStamp(string? lang) => + lang is "crystal" or "groovy" or "tcl" or "prolog" or "ambiguous_pl"; + private static IReadOnlyList BuildEnumDeclarationSnapshot(IReadOnlyList symbols, long? fileId = null) { List<(SymbolRecord Symbol, int OriginalIndex)>? candidates = null; diff --git a/tests/CodeIndex.Tests/DatabaseTests.cs b/tests/CodeIndex.Tests/DatabaseTests.cs index c24fb6740..d6cc73589 100644 --- a/tests/CodeIndex.Tests/DatabaseTests.cs +++ b/tests/CodeIndex.Tests/DatabaseTests.cs @@ -4720,6 +4720,29 @@ public void GetUnchangedFileId_InvalidatesPreGraphLanguageContracts_Issue4746( Assert.Null(_writer.GetUnchangedFileId(file.Path, modified, language: language)); } + [Theory] + [InlineData("crystal")] + [InlineData("groovy")] + [InlineData("tcl")] + [InlineData("prolog")] + [InlineData("ambiguous_pl")] + public void GetUnchangedFileId_InvalidatesMissingGraphLanguageContracts_Issue4746( + string language) + { + var modified = new DateTime(2025, 1, 1, 0, 0, 0, DateTimeKind.Utc); + var file = new FileRecord + { + Path = $"src/missing-contract-{language}.txt", + Lang = language, + Size = 50, + Lines = 5, + Modified = modified, + }; + _writer.UpsertFile(file); + + Assert.Null(_writer.GetUnchangedFileId(file.Path, modified, language: language)); + } + [Theory] [InlineData("crystal", 2)] [InlineData("groovy", 2)] diff --git a/tests/CodeIndex.Tests/IndexCommandRunnerFullScanTests.cs b/tests/CodeIndex.Tests/IndexCommandRunnerFullScanTests.cs index e34017cad..36aea390d 100644 --- a/tests/CodeIndex.Tests/IndexCommandRunnerFullScanTests.cs +++ b/tests/CodeIndex.Tests/IndexCommandRunnerFullScanTests.cs @@ -4029,8 +4029,7 @@ proc run {} { helper } conn.Open(); using var cmd = conn.CreateCommand(); cmd.CommandText = $""" - UPDATE codeindex_meta - SET value = '2' + DELETE FROM codeindex_meta WHERE key = '{DbContext.GetSymbolExtractorVersionMetaKey("tcl")}'; UPDATE codeindex_meta SET value = '0' diff --git a/tests/CodeIndex.Tests/ReferenceExtractorTests.cs b/tests/CodeIndex.Tests/ReferenceExtractorTests.cs index 14f347de6..a5f90a054 100644 --- a/tests/CodeIndex.Tests/ReferenceExtractorTests.cs +++ b/tests/CodeIndex.Tests/ReferenceExtractorTests.cs @@ -1540,6 +1540,11 @@ public void Extract_AmbiguousPl_MasksPerlAndPrologCommentsAndPod_Issue4746() my %cache = (answer => helper()); my $fallback = undef // helper(); clp_entry(X) :- X #= 1, helper. + prolog_comment :- + %TODO + helper. + inline_comment :- helper. %comment + after_inline :- helper. # helper(); % helper(). %helper(). @@ -1550,11 +1555,14 @@ public void Extract_AmbiguousPl_MasksPerlAndPrologCommentsAndPod_Issue4746() var (_, references) = ExtractSymbolsAndReferences("ambiguous_pl", content); - Assert.Equal(4, references.Count(reference => + Assert.Equal(7, references.Count(reference => reference.ReferenceKind == "call" && reference.SymbolName == "helper")); AssertReferencesContain(references, "call", "prolog_entry", "helper"); AssertReferencesContain(references, "call", "clp_entry", "helper"); + AssertReferencesContain(references, "call", "prolog_comment", "helper"); + AssertReferencesContain(references, "call", "inline_comment", "helper"); + AssertReferencesContain(references, "call", "after_inline", "helper"); } [Fact] diff --git a/tests/CodeIndex.Tests/SymbolExtractorTests.cs b/tests/CodeIndex.Tests/SymbolExtractorTests.cs index db43f7d27..bb7d8371e 100644 --- a/tests/CodeIndex.Tests/SymbolExtractorTests.cs +++ b/tests/CodeIndex.Tests/SymbolExtractorTests.cs @@ -101,6 +101,21 @@ symbol.Name is "PodFake" or "pod_fake" or "pod_predicate" or "concat_candidate" or "other"); } + [Fact] + public void Extract_Prolog_ManyMalformedMultilineHeadsUsesBoundedLookahead_Issue4746() + { + var content = string.Concat(Enumerable.Repeat("malformed(\n", 20_000)); + var stopwatch = Stopwatch.StartNew(); + + var symbols = SymbolExtractor.Extract(1, "prolog", content); + + stopwatch.Stop(); + Assert.Empty(symbols); + Assert.True( + stopwatch.Elapsed < TimeSpan.FromSeconds(3), + $"Expected bounded multiline-head validation, elapsed={stopwatch.Elapsed}."); + } + [Fact] public void Extract_CancelledToken_ThrowsBeforeWork() { From 8ddf0e7cb797cd7e65a167f6ffebb3546721f625 Mon Sep 17 00:00:00 2001 From: Widthdom Date: Fri, 24 Jul 2026 17:46:10 +0900 Subject: [PATCH 11/26] Fix dynamic graph lexical edge cases (#4746) --- .../DynamicDeclarativeReferenceExtractor.cs | 70 ++++++++++--------- .../ReferenceExtractorTests.cs | 15 +++- 2 files changed, 51 insertions(+), 34 deletions(-) diff --git a/src/CodeIndex/Indexer/References/Languages/DynamicDeclarativeReferenceExtractor.cs b/src/CodeIndex/Indexer/References/Languages/DynamicDeclarativeReferenceExtractor.cs index bb264a27c..526884934 100644 --- a/src/CodeIndex/Indexer/References/Languages/DynamicDeclarativeReferenceExtractor.cs +++ b/src/CodeIndex/Indexer/References/Languages/DynamicDeclarativeReferenceExtractor.cs @@ -447,6 +447,7 @@ public static string[] MaskNonCodeLines(string language, IReadOnlyList l && ch is '"' or '`' && !HasClosingQuotedDelimiter(line, column, ch)) { + FillWithSpaces(buffer, column); crystalMultilineQuote = ch; column = line.Length; continue; @@ -575,37 +576,21 @@ public static string[] MaskTclContinuedCommentLines( IReadOnlyList rawLines, IReadOnlyList maskedLines) { - var result = maskedLines as string[] ?? maskedLines.ToArray(); - var cloned = false; - var commentContinued = false; - for (var lineIndex = 0; lineIndex < rawLines.Count; lineIndex++) + if (rawLines.Count == 0) + return maskedLines as string[] ?? maskedLines.ToArray(); + + var commentColumns = new Dictionary(); + _ = BuildTclCallLines( + maskedLines, + BuildTclBraceEndPositions(maskedLines), + new HashSet(), + commentColumns); + if (commentColumns.Count == 0) + return maskedLines as string[] ?? maskedLines.ToArray(); + + var result = maskedLines.ToArray(); + foreach (var (lineIndex, commentColumn) in commentColumns) { - var rawLine = rawLines[lineIndex]; - if (commentContinued) - { - if (!cloned) - { - result = result.ToArray(); - cloned = true; - } - result[lineIndex] = new string(' ', rawLine.Length); - commentContinued = HasTclEscapedNewline(rawLine); - continue; - } - - var commentColumn = FindTclCommentStart(rawLine); - if (commentColumn < 0) - continue; - - commentContinued = HasTclEscapedNewline(rawLine); - if (!commentContinued) - continue; - - if (!cloned) - { - result = result.ToArray(); - cloned = true; - } var buffer = result[lineIndex].ToCharArray(); FillWithSpaces(buffer, commentColumn); result[lineIndex] = new string(buffer); @@ -1314,6 +1299,12 @@ private static bool TryFindNextNonWhitespace( var column = lineIndex == startLine ? startColumn : 0; if (TryFindNextNonWhitespace(lines[lineIndex], column, out foundColumn)) { + if (foundColumn == lines[lineIndex].Length - 1 + && lines[lineIndex][foundColumn] == '\\') + { + continue; + } + foundLine = lineIndex; return true; } @@ -1410,7 +1401,8 @@ private static Dictionary BuildTclBraceEndPositions(IReadOnly private static string[] BuildTclCallLines( IReadOnlyList lines, IReadOnlyDictionary braceEnds, - IReadOnlySet scriptBodyOpenings) + IReadOnlySet scriptBodyOpenings, + IDictionary? commentColumns = null) { var result = new string[lines.Count]; var frames = new Stack(); @@ -1423,6 +1415,7 @@ private static string[] BuildTclCallLines( if (commentContinued) { result[lineIndex] = new string(' ', line.Length); + commentColumns?.TryAdd(lineIndex, 0); commentContinued = HasTclEscapedNewline(line); continue; } @@ -1551,10 +1544,19 @@ private static string[] BuildTclCallLines( } if (ch == '\\') { + buffer[column] = ' '; + if (column + 1 >= line.Length) + { + lineContinued = true; + frame.WordStart = true; + column++; + continue; + } + if (frame.WordStart) frame.WordIndex++; - lineContinued = column + 1 >= line.Length; - column += Math.Min(2, line.Length - column); + buffer[column + 1] = ' '; + column += 2; frame.CommandStart = false; frame.WordStart = false; continue; @@ -1562,6 +1564,7 @@ private static string[] BuildTclCallLines( if (ch == '#' && frame.CommandStart) { FillWithSpaces(buffer, column); + commentColumns?.TryAdd(lineIndex, column); commentContinued = HasTclEscapedNewline(line); break; } @@ -1795,6 +1798,7 @@ private static bool IsTclScriptArgument( "while" => wordIndex == 2, "catch" => wordIndex == 1, "for" => wordIndex is 1 or 3 or 4, + "proc" => wordIndex == 3, "eval" => wordIndex >= 1, "after" => wordIndex == 2, "uplevel" => wordIndex is 1 or 2, diff --git a/tests/CodeIndex.Tests/ReferenceExtractorTests.cs b/tests/CodeIndex.Tests/ReferenceExtractorTests.cs index a5f90a054..e0a698d09 100644 --- a/tests/CodeIndex.Tests/ReferenceExtractorTests.cs +++ b/tests/CodeIndex.Tests/ReferenceExtractorTests.cs @@ -1064,6 +1064,8 @@ def run(value) helper value helper(value) text = "123456789"; helper(value) + multiline = "helper() + continued" divided = value // 2; helper(value) helper = 1 end @@ -1406,6 +1408,9 @@ public void Extract_Tcl_IndexesSingleLineProcBodiesAndMasksComments_Issue4746() proc fake {} { helper } proc helper {} { return 1 } # [helper] + if {1} { # continued script comment \ + proc script_comment_fake {} { helper } + } proc run {helper} { set literal {helper} set substituted [helper] @@ -1446,23 +1451,31 @@ uplevel helper helper set callback helper() puts helper() + set escaped_command \[helper] + set escaped_separator \; helper helper } proc split {value} { helper } + proc continued {} \ + { + helper + } proc sameLine {} { helper }; helper """; var (symbols, references) = ExtractSymbolsAndReferences("tcl", content); Assert.DoesNotContain(symbols, symbol => symbol.Name == "fake"); + Assert.DoesNotContain(symbols, symbol => symbol.Name == "script_comment_fake"); AssertReferencesContain(references, "call", "run", "helper"); AssertReferencesContain(references, "call", "multiline", "helper"); AssertReferencesContain(references, "call", "split", "helper"); + AssertReferencesContain(references, "call", "continued", "helper"); AssertReferencesContain(references, "call", "sameLine", "helper"); - Assert.Equal(19, references.Count(reference => + Assert.Equal(20, references.Count(reference => reference.ReferenceKind == "call" && reference.SymbolName == "helper")); Assert.Single(references, reference => From 6371c1cb29334d1bb072008851a98af90c11645d Mon Sep 17 00:00:00 2001 From: Widthdom Date: Fri, 24 Jul 2026 18:25:19 +0900 Subject: [PATCH 12/26] Resolve final dynamic graph review findings (#4746) --- .../Cli/IndexCommandRunner.FullScan.cs | 8 +- .../Cli/IndexCommandRunner.Update.cs | 8 +- .../Cli/QueryCommandRunner.GraphOutput.cs | 12 +- src/CodeIndex/Database/DbReader.Status.cs | 40 ++- .../Database/DbReader.WorkspaceHealth.cs | 3 +- .../DynamicDeclarativeReferenceExtractor.cs | 306 ++++++++++++++---- .../Symbols/SymbolExtractor.ExtractCore.cs | 38 +++ .../Indexer/Symbols/SymbolExtractor.cs | 4 +- src/CodeIndex/Mcp/McpToolHandlers.Indexing.cs | 6 +- .../Mcp/McpToolHandlers.QueryTools.cs | 11 +- tests/CodeIndex.Tests/DatabaseTests.cs | 16 +- .../McpServerToolsCallTests.cs | 32 ++ .../QueryCommandRunnerGraphTests.cs | 46 +++ .../ReferenceExtractorTests.cs | 41 ++- 14 files changed, 479 insertions(+), 92 deletions(-) diff --git a/src/CodeIndex/Cli/IndexCommandRunner.FullScan.cs b/src/CodeIndex/Cli/IndexCommandRunner.FullScan.cs index 6fc6e7cfd..37c849a2d 100644 --- a/src/CodeIndex/Cli/IndexCommandRunner.FullScan.cs +++ b/src/CodeIndex/Cli/IndexCommandRunner.FullScan.cs @@ -2201,8 +2201,8 @@ int ReturnBeforeWriteSnapshotFailure(string changedPath) GraphDataCurrent = false, IndexComplete = false, ReferenceExtractionLimits = ReferenceExtractor.GetSafetyLimits(), - ReferenceGraphComplete = referenceExtractionCapHits.StateAvailable - && referenceExtractionCapHits.HitCount == 0, + ReferenceGraphComplete = signalReader.IsReferenceGraphComplete( + referenceExtractionCapHits), ReferenceExtractionCapHits = referenceExtractionCapHits, ErrorCode = CommandErrorCodes.IndexPartial, IssuesTableAvailable = issuesTableAvailable, @@ -3659,8 +3659,8 @@ int ReturnBeforeWriteSnapshotFailure(string changedPath) : writer.GetCounts(); var signalReader = new DbReader(writer.Connection); var referenceExtractionCapHitsAfter = signalReader.GetReferenceExtractionCapHits(); - var referenceGraphCompleteAfter = referenceExtractionCapHitsAfter.StateAvailable - && referenceExtractionCapHitsAfter.HitCount == 0; + var referenceGraphCompleteAfter = signalReader.IsReferenceGraphComplete( + referenceExtractionCapHitsAfter); var sqlGraphContractSignalAfter = signalReader.GetSqlGraphContractSignal(lang: null); if (!hasSqlFilesAfter) { diff --git a/src/CodeIndex/Cli/IndexCommandRunner.Update.cs b/src/CodeIndex/Cli/IndexCommandRunner.Update.cs index 6db2bff17..d63971bf4 100644 --- a/src/CodeIndex/Cli/IndexCommandRunner.Update.cs +++ b/src/CodeIndex/Cli/IndexCommandRunner.Update.cs @@ -1219,8 +1219,8 @@ int ReturnBeforeWriteSnapshotFailure(string changedPath) GraphDataCurrent = false, IndexComplete = false, ReferenceExtractionLimits = ReferenceExtractor.GetSafetyLimits(), - ReferenceGraphComplete = referenceExtractionCapHits.StateAvailable - && referenceExtractionCapHits.HitCount == 0, + ReferenceGraphComplete = signalReader.IsReferenceGraphComplete( + referenceExtractionCapHits), ReferenceExtractionCapHits = referenceExtractionCapHits, ErrorCode = CommandErrorCodes.IndexPartial, IssuesTableAvailable = issuesTableAvailable, @@ -2677,8 +2677,8 @@ or IndexInterruptedException var (totalFiles, totalChunks, totalSymbols, totalReferences) = writer.GetCounts(); var signalReader = new DbReader(writer.Connection); var referenceExtractionCapHitsAfter = signalReader.GetReferenceExtractionCapHits(); - var referenceGraphCompleteAfter = referenceExtractionCapHitsAfter.StateAvailable - && referenceExtractionCapHitsAfter.HitCount == 0; + var referenceGraphCompleteAfter = signalReader.IsReferenceGraphComplete( + referenceExtractionCapHitsAfter); var sqlGraphContractSignalAfter = signalReader.GetSqlGraphContractSignal(lang: null); var hotspotFamilySignalAfter = signalReader.GetHotspotFamilySignal(lang: null); var sqlGraphContractReadyAfter = sqlGraphContractSignalAfter.Ready; diff --git a/src/CodeIndex/Cli/QueryCommandRunner.GraphOutput.cs b/src/CodeIndex/Cli/QueryCommandRunner.GraphOutput.cs index fc86bb49e..0f25e5ecb 100644 --- a/src/CodeIndex/Cli/QueryCommandRunner.GraphOutput.cs +++ b/src/CodeIndex/Cli/QueryCommandRunner.GraphOutput.cs @@ -136,7 +136,8 @@ private static void AddReferenceGraphCompletenessJsonFields( JsonSerializerOptions jsonOptions) { var capHits = reader.GetReferenceExtractionCapHits(); - var complete = capHits.StateAvailable && capHits.HitCount == 0; + var complete = reader.IsReferenceGraphComplete(capHits); + var incompleteReasons = reader.GetReferenceGraphIncompleteReasons(capHits); payload["reference_extraction_limits"] = JsonSerializer.SerializeToNode( ReferenceExtractor.GetSafetyLimits(), CliJsonSerializerContextFactory.Create(jsonOptions).ReferenceExtractionSafetyLimits); @@ -147,7 +148,7 @@ private static void AddReferenceGraphCompletenessJsonFields( if (!complete) { payload["reference_graph_incomplete_reasons"] = JsonSerializer.SerializeToNode( - capHits.Reasons, + incompleteReasons, CliJsonSerializerContextFactory.Create(jsonOptions).ListString); payload["degraded"] = true; } @@ -158,11 +159,12 @@ private static void WriteReferenceGraphCompletenessWarningIfNeeded(bool json, Db if (json) return; var capHits = reader.GetReferenceExtractionCapHits(); - if (capHits.StateAvailable && capHits.HitCount == 0) + if (reader.IsReferenceGraphComplete(capHits)) return; - var reasons = capHits.Reasons.Count == 0 + var incompleteReasons = reader.GetReferenceGraphIncompleteReasons(capHits); + var reasons = incompleteReasons.Count == 0 ? DbReader.ReferenceExtractionCapStateUnavailableReason - : string.Join(", ", capHits.Reasons.Take(4)); + : string.Join(", ", incompleteReasons.Take(4)); CommandErrorWriter.WriteStderr($"WARN: reference graph is incomplete ({reasons}); callers, callees, deps, and impact results are not authoritative absences."); } diff --git a/src/CodeIndex/Database/DbReader.Status.cs b/src/CodeIndex/Database/DbReader.Status.cs index 78118ae18..4719a9ef4 100644 --- a/src/CodeIndex/Database/DbReader.Status.cs +++ b/src/CodeIndex/Database/DbReader.Status.cs @@ -1,4 +1,5 @@ using CodeIndex.Indexer; +using CodeIndex.Models; namespace CodeIndex.Database; @@ -67,14 +68,7 @@ public StatusResult GetStatus() // reader を内側ブロックに閉じ込め、txn.Commit() の前に statement handle を // 解放する。SqliteTransaction.Commit() は同じ connection 上で開いている reader // があると失敗する。 - var langs = new Dictionary(); - { - using var cmd = _conn.CreateCommand(); - cmd.CommandText = "SELECT lang, COUNT(*) FROM files WHERE lang IS NOT NULL GROUP BY lang ORDER BY COUNT(*) DESC"; - using var reader = cmd.ExecuteTrackedReader(); - while (reader.TrackedRead()) - langs[reader.GetString(0)] = reader.GetInt64(1); - } + var langs = GetIndexedLanguageCounts(); var symbolsByLanguage = new Dictionary>(StringComparer.Ordinal); { @@ -290,6 +284,36 @@ private bool AreDynamicReferenceGraphContractsCurrent( return true; } + internal bool IsReferenceGraphComplete(ReferenceExtractionCapHitSummary capHits) => + capHits.StateAvailable + && capHits.HitCount == 0 + && AreDynamicReferenceGraphContractsCurrent(GetIndexedLanguageCounts()); + + internal IReadOnlyList GetReferenceGraphIncompleteReasons( + ReferenceExtractionCapHitSummary capHits) + { + var reasons = capHits.Reasons.ToList(); + if (!AreDynamicReferenceGraphContractsCurrent(GetIndexedLanguageCounts()) + && !reasons.Contains(DynamicReferenceGraphContractStaleReason, StringComparer.Ordinal)) + { + reasons.Add(DynamicReferenceGraphContractStaleReason); + } + + return reasons; + } + + private Dictionary GetIndexedLanguageCounts() + { + var languages = new Dictionary(); + using var cmd = _conn.CreateCommand(); + cmd.CommandText = + "SELECT lang, COUNT(*) FROM files WHERE lang IS NOT NULL GROUP BY lang ORDER BY COUNT(*) DESC"; + using var reader = cmd.ExecuteTrackedReader(); + while (reader.TrackedRead()) + languages[reader.GetString(0)] = reader.GetInt64(1); + return languages; + } + private StatusPreparedCommandCache? GetPreparedCommandCacheStatus() { if (_commandCache == null) diff --git a/src/CodeIndex/Database/DbReader.WorkspaceHealth.cs b/src/CodeIndex/Database/DbReader.WorkspaceHealth.cs index 28f958e35..f4758be20 100644 --- a/src/CodeIndex/Database/DbReader.WorkspaceHealth.cs +++ b/src/CodeIndex/Database/DbReader.WorkspaceHealth.cs @@ -23,8 +23,7 @@ internal WorkspaceIndexHealthSnapshot GetWorkspaceIndexHealth() var indexComplete = !batchInProgress && !string.Equals(indexCompleteness, "incomplete", StringComparison.OrdinalIgnoreCase); var referenceExtractionCapHits = GetReferenceExtractionCapHits(); - var referenceGraphComplete = referenceExtractionCapHits.StateAvailable - && referenceExtractionCapHits.HitCount == 0; + var referenceGraphComplete = IsReferenceGraphComplete(referenceExtractionCapHits); var graphDataCurrent = _hasReferencesTable && indexComplete && referenceGraphComplete; diff --git a/src/CodeIndex/Indexer/References/Languages/DynamicDeclarativeReferenceExtractor.cs b/src/CodeIndex/Indexer/References/Languages/DynamicDeclarativeReferenceExtractor.cs index 526884934..8f2371081 100644 --- a/src/CodeIndex/Indexer/References/Languages/DynamicDeclarativeReferenceExtractor.cs +++ b/src/CodeIndex/Indexer/References/Languages/DynamicDeclarativeReferenceExtractor.cs @@ -13,6 +13,7 @@ public ExtractionState( HashSet declarationPositions, IReadOnlyDictionary containersByLine, IReadOnlyList tclContainerScopes, + IReadOnlyDictionary> prologContainersByLine, IReadOnlyList? tclCallLines, IReadOnlyDictionary>? prologGoalCallsByLine) { @@ -20,6 +21,7 @@ public ExtractionState( DeclarationPositions = declarationPositions; ContainersByLine = containersByLine; TclContainerScopes = tclContainerScopes; + PrologContainersByLine = prologContainersByLine; TclCallLines = tclCallLines; PrologGoalCallsByLine = prologGoalCallsByLine; } @@ -28,6 +30,7 @@ public ExtractionState( private HashSet DeclarationPositions { get; } public IReadOnlyDictionary ContainersByLine { get; } private IReadOnlyList TclContainerScopes { get; } + private IReadOnlyDictionary> PrologContainersByLine { get; } private IReadOnlyList? TclCallLines { get; } private IReadOnlyDictionary>? PrologGoalCallsByLine { get; } @@ -40,6 +43,18 @@ public ExtractionState( return scope.Symbol; } + if (PrologContainersByLine.TryGetValue(lineNumber, out var prologContainers)) + { + for (var containerIndex = prologContainers.Count - 1; + containerIndex >= 0; + containerIndex--) + { + var prologContainer = prologContainers[containerIndex]; + if (prologContainer.StartColumn is { } startColumn && startColumn < column) + return prologContainer; + } + } + return ContainersByLine.TryGetValue(lineNumber, out var container) ? container : fallback; } @@ -70,7 +85,8 @@ public IReadOnlyList GetPrologGoalCalls(int lineNumber) } public bool HasPrologContainer(int lineNumber) => - ContainersByLine.ContainsKey(lineNumber); + ContainersByLine.ContainsKey(lineNumber) + || PrologContainersByLine.ContainsKey(lineNumber); public bool IsDeclarationAt(int lineNumber, int column, string name) => DeclarationPositions.Contains(new DeclarationPosition(lineNumber, column, name)); @@ -80,7 +96,7 @@ public bool IsDeclarationAt(int lineNumber, int column, string name) => @"^\s*proc\s+[A-Za-z_:][\w:.-]*\s+", RegexOptions.Compiled | RegexOptions.CultureInvariant); private static readonly Regex PrologHeadRegex = new( - @"^\s*(?[a-z][A-Za-z0-9_]*)\s*(?:\([^\r\n]*\))?\s*(?::-|-->|\.(?=\s*$))", + @"^\s*(?[a-z][A-Za-z0-9_]*)\s*(?:\([^\r\n]*\))?\s*(?::-|-->|\.(?=\s*(?:$|[a-z][A-Za-z0-9_]*(?:\s*\([^)]*\))?\s*(?::-|-->|\.))))", RegexOptions.Compiled | RegexOptions.CultureInvariant); private static readonly Regex PrologMultilineHeadRegex = new( @"^\s*(?[a-z][A-Za-z0-9_]*)\s*\(", @@ -98,7 +114,7 @@ public bool IsDeclarationAt(int lineNumber, int column, string name) => @"^\s*:-\s*use_module\s*\(\s*(?:library\s*\(\s*)?['""]?(?(?:\.\.?/)*[a-z][A-Za-z0-9_./-]*)", RegexOptions.Compiled | RegexOptions.CultureInvariant); private static readonly Regex CrystalBareCallRegex = new( - @"(?:^|[;=])\s*(?:return\s+)?(?[A-Za-z_]\w*[?!]?)(?![\w?!])(?!\s*(?:\(|(?:<<|>>|[+\-*/%&|^])?=))", + @"(?:^|[;={]|\b(?:then|do)\b|&&|\|\|)\s*(?:return\s+)?(?[A-Za-z_]\w*[?!]?)(?![\w?!])(?!\s*(?:\(|(?:<<|>>|[+\-*/%&|^])?=))", RegexOptions.Compiled | RegexOptions.CultureInvariant); private static readonly Regex CrystalSuffixedParenthesizedCallRegex = new( @"(?[A-Za-z_]\w*[?!])\s*\(", @@ -627,6 +643,7 @@ public static string[] MaskTclContinuedCommentLines( var containersByLine = new Dictionary(); var tclContainerScopes = new List(); + var prologContainersByLine = new Dictionary>(); string[]? tclCallLines = null; IReadOnlyDictionary>? prologGoalCallsByLine = null; if (language == "tcl") @@ -646,7 +663,11 @@ public static string[] MaskTclContinuedCommentLines( } else if (language is "prolog" or "ambiguous_pl") { - AddPrologContainers(preparedLines, symbols, containersByLine); + AddPrologContainers( + preparedLines, + symbols, + containersByLine, + prologContainersByLine); prologGoalCallsByLine = BuildPrologGoalCalls( preparedLines, containersByLine, @@ -658,6 +679,7 @@ public static string[] MaskTclContinuedCommentLines( declarationPositions, containersByLine, tclContainerScopes, + prologContainersByLine, tclCallLines, prologGoalCallsByLine); } @@ -692,7 +714,8 @@ public static void EmitAdditionalReferences( if (!state.CallableNames.Contains(call.Name)) continue; - if (state.ContainersByLine.TryGetValue(lineNumber, out var prologContainer) + var prologContainer = state.ResolveContainer(lineNumber, call.Column, fallback: null); + if (prologContainer != null && !string.Equals(prologContainer.Name, call.Name, StringComparison.Ordinal)) { ReferenceExtractor.AddReference( @@ -1004,18 +1027,65 @@ private static string PreparePrologCallScanLine( if (language is not ("prolog" or "ambiguous_pl") || isClauseContinuation) return line; - var firstColumn = 0; - while (firstColumn < line.Length && char.IsWhiteSpace(line[firstColumn])) - firstColumn++; - if (firstColumn >= line.Length - || !char.IsLower(line[firstColumn]) - || line.AsSpan(firstColumn).StartsWith(":-", StringComparison.Ordinal)) + var masked = line.ToCharArray(); + var clauseStartColumn = 0; + var changed = false; + while (TryFindNextPrologClauseStart(line, clauseStartColumn, out var headStartColumn) + && TryFindPrologHeadBoundary( + line, + headStartColumn, + out var bodyStartColumn, + out var clauseEndColumn)) { - return line; + if (bodyStartColumn >= 0) + { + FillWithSpaces(masked, headStartColumn, bodyStartColumn); + changed = true; + if (clauseEndColumn < 0) + break; + } + else + { + FillWithSpaces(masked, headStartColumn, clauseEndColumn + 1); + changed = true; + } + + clauseStartColumn = clauseEndColumn + 1; + } + + return changed ? new string(masked) : line; + } + + private static bool TryFindNextPrologClauseStart( + string line, + int searchColumn, + out int clauseStartColumn) + { + for (var column = Math.Max(0, searchColumn); column < line.Length; column++) + { + if (char.IsWhiteSpace(line[column])) + continue; + + clauseStartColumn = column; + return char.IsLower(line[column]); } + clauseStartColumn = -1; + return false; + } + + private static bool TryFindPrologHeadBoundary( + string line, + int headStartColumn, + out int bodyStartColumn, + out int clauseEndColumn) + { + bodyStartColumn = -1; + clauseEndColumn = -1; var parenthesisDepth = 0; - for (var column = firstColumn; column < line.Length; column++) + var bracketDepth = 0; + var braceDepth = 0; + for (var column = headStartColumn; column < line.Length; column++) { var ch = line[column]; if (ch is '\'' or '"') @@ -1023,17 +1093,28 @@ private static string PreparePrologCallScanLine( column = SkipQuotedToken(line, column, ch) - 1; continue; } - if (ch == '(') + switch (ch) { - parenthesisDepth++; - continue; - } - if (ch == ')' && parenthesisDepth > 0) - { - parenthesisDepth--; - continue; + case '(': + parenthesisDepth++; + continue; + case ')' when parenthesisDepth > 0: + parenthesisDepth--; + continue; + case '[': + bracketDepth++; + continue; + case ']' when bracketDepth > 0: + bracketDepth--; + continue; + case '{': + braceDepth++; + continue; + case '}' when braceDepth > 0: + braceDepth--; + continue; } - if (parenthesisDepth != 0) + if (parenthesisDepth != 0 || bracketDepth != 0 || braceDepth != 0) continue; var separatorLength = line.AsSpan(column).StartsWith("-->", StringComparison.Ordinal) @@ -1043,27 +1124,18 @@ private static string PreparePrologCallScanLine( : 0; if (separatorLength > 0) { - var masked = line.ToCharArray(); - FillWithSpaces(masked, 0, column); - return new string(masked); + bodyStartColumn = column + separatorLength; + clauseEndColumn = FindPrologClauseTerminator(line, bodyStartColumn); + return true; + } + if (IsPrologClauseTerminator(line, column)) + { + clauseEndColumn = column; + return true; } - - if (IsPrologClauseTerminator(line, column) - && IsOnlyWhitespaceAfter(line, column + 1)) - return new string(' ', line.Length); } - return line; - } - - private static bool IsOnlyWhitespaceAfter(string line, int startColumn) - { - for (var column = startColumn; column < line.Length; column++) - { - if (!char.IsWhiteSpace(line[column])) - return false; - } - return true; + return false; } internal static bool IsPrologClauseTerminator(string line, int column) @@ -1077,7 +1149,59 @@ internal static bool IsPrologClauseTerminator(string line, int column) return false; if (char.IsDigit(previous) && char.IsDigit(next)) return false; - return IsOnlyWhitespaceAfter(line, column + 1); + if (next != '\0' && !char.IsWhiteSpace(next)) + return false; + return IsTopLevelPrologColumn(line, column); + } + + private static bool IsTopLevelPrologColumn(string line, int targetColumn) + { + var parenthesisDepth = 0; + var bracketDepth = 0; + var braceDepth = 0; + for (var column = 0; column < targetColumn; column++) + { + var ch = line[column]; + if (ch is '\'' or '"') + { + column = SkipQuotedToken(line, column, ch) - 1; + continue; + } + switch (ch) + { + case '(': + parenthesisDepth++; + break; + case ')' when parenthesisDepth > 0: + parenthesisDepth--; + break; + case '[': + bracketDepth++; + break; + case ']' when bracketDepth > 0: + bracketDepth--; + break; + case '{': + braceDepth++; + break; + case '}' when braceDepth > 0: + braceDepth--; + break; + } + } + + return parenthesisDepth == 0 && bracketDepth == 0 && braceDepth == 0; + } + + private static int FindPrologClauseTerminator(string line, int startColumn) + { + for (var column = Math.Max(0, startColumn); column < line.Length; column++) + { + if (IsPrologClauseTerminator(line, column)) + return column; + } + + return -1; } private static void EmitImportReference( @@ -1164,10 +1288,15 @@ private static void AddTclContainers( scopes.Add(new TclContainerScope( symbol, bodyStartLineIndex + 1, - bodyStartColumn, + lines[bodyStartLineIndex][bodyStartColumn] is '{' or '"' + ? bodyStartColumn + : bodyStartColumn - 1, bodyEnd.Line + 1, - bodyEnd.Column)); - scriptBodyOpenings.Add(GetTclPositionKey(bodyStartLineIndex, bodyStartColumn)); + lines[bodyStartLineIndex][bodyStartColumn] is '{' or '"' + ? bodyEnd.Column + : bodyEnd.Column + 1)); + if (lines[bodyStartLineIndex][bodyStartColumn] == '{') + scriptBodyOpenings.Add(GetTclPositionKey(bodyStartLineIndex, bodyStartColumn)); } scopes.Sort(static (left, right) => @@ -1205,14 +1334,18 @@ private static bool TryFindTclBodyEnd( argsEndColumn + 1, out bodyStartLineIndex, out bodyStartColumn) - || lines[bodyStartLineIndex][bodyStartColumn] != '{' - || !braceEnds.TryGetValue( - GetTclPositionKey(bodyStartLineIndex, bodyStartColumn), - out bodyEnd)) + || !TryFindTclWordEnd( + lines, + braceEnds, + bodyStartLineIndex, + bodyStartColumn, + out var bodyEndLine, + out var bodyEndColumn)) { return false; } + bodyEnd = new TclBraceEnd(bodyEndLine, bodyEndColumn); return true; } @@ -1466,16 +1599,41 @@ private static string[] BuildTclCallLines( } else if (ch == '"' && frame.WordStart) { - frame.WordIndex++; - var endColumn = SkipQuotedToken(line, column, '"'); - FillWithSpaces(buffer, column, endColumn); + var wordIndex = frame.WordIndex++; + if (wordIndex % 2 == 1) + { + buffer[column] = ';'; + frames.Push(new TclLexicalFrame(TclLexicalFrameKind.Script, '"')); + suppressLeadingContinuedWord = false; + } + else + { + var endColumn = SkipQuotedToken(line, column, '"'); + FillWithSpaces(buffer, column, endColumn); + column = endColumn - 1; + } frame.WordStart = false; - column = endColumn; + column++; } else { if (frame.WordStart) - frame.WordIndex++; + { + var wordIndex = frame.WordIndex++; + if (wordIndex % 2 == 1) + { + var endColumn = column; + while (endColumn < line.Length + && !char.IsWhiteSpace(line[endColumn]) + && line[endColumn] != frame.Terminator) + { + buffer[endColumn] = line[endColumn]; + endColumn++; + } + MarkTclBareScriptCommandBoundary(buffer, column); + column = endColumn - 1; + } + } frame.WordStart = false; column++; } @@ -2080,6 +2238,13 @@ private static void ScanPrologGoalLine( expectGoal = false; continue; } + if (IsPrologClauseTerminator(line, column)) + { + frames.Clear(); + expectGoal = true; + column++; + continue; + } if (expectGoal) { @@ -2351,7 +2516,8 @@ private static bool ContainsPrologClauseTerminator(string line) private static void AddPrologContainers( IReadOnlyList lines, IReadOnlyList symbols, - Dictionary containersByLine) + Dictionary containersByLine, + Dictionary> declarationsByLine) { foreach (var symbol in symbols) { @@ -2359,27 +2525,49 @@ private static void AddPrologContainers( continue; var startLineIndex = symbol.StartLine - 1; - var headMatch = PrologHeadRegex.Match(lines[startLineIndex]); + var startColumn = Math.Clamp( + symbol.StartColumn ?? 0, + 0, + lines[startLineIndex].Length); + var headLine = lines[startLineIndex][startColumn..]; + var headMatch = PrologHeadRegex.Match(headLine); if (!headMatch.Success) - headMatch = PrologMultilineHeadRegex.Match(lines[startLineIndex]); + headMatch = PrologMultilineHeadRegex.Match(headLine); if (!headMatch.Success || !string.Equals(headMatch.Groups["name"].Value, symbol.Name, StringComparison.Ordinal)) { continue; } - var endLineIndex = FindPrologClauseEnd(lines, startLineIndex); + if (!declarationsByLine.TryGetValue(symbol.StartLine, out var declarations)) + { + declarations = []; + declarationsByLine[symbol.StartLine] = declarations; + } + declarations.Add(symbol); + + var endLineIndex = FindPrologClauseEnd(lines, startLineIndex, startColumn); for (var lineIndex = startLineIndex; lineIndex <= endLineIndex; lineIndex++) containersByLine.TryAdd(lineIndex + 1, symbol); } + + foreach (var declarations in declarationsByLine.Values) + { + declarations.Sort(static (left, right) => + (left.StartColumn ?? 0).CompareTo(right.StartColumn ?? 0)); + } } - private static int FindPrologClauseEnd(IReadOnlyList lines, int startLineIndex) + private static int FindPrologClauseEnd( + IReadOnlyList lines, + int startLineIndex, + int startColumn) { for (var lineIndex = startLineIndex; lineIndex < lines.Count; lineIndex++) { var line = lines[lineIndex]; - for (var column = 0; column < line.Length; column++) + var firstColumn = lineIndex == startLineIndex ? startColumn : 0; + for (var column = firstColumn; column < line.Length; column++) { if (IsPrologClauseTerminator(line, column)) return lineIndex; diff --git a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.ExtractCore.cs b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.ExtractCore.cs index 137068fc6..aa4c9ae2b 100644 --- a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.ExtractCore.cs +++ b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.ExtractCore.cs @@ -1713,6 +1713,16 @@ bool[] GetCssQualifiedRuleAncestors() => restartPatternScanOffset = nextJavaSiblingOffset; break; } + if (lang is "prolog" or "ambiguous_pl" + && pattern.Kind == "function" + && TryGetNextPrologClauseOffset( + patternMatchLine, + absoluteStartColumn, + out var nextPrologClauseOffset)) + { + restartPatternScanOffset = nextPrologClauseOffset; + break; + } CollectRecordPrimaryComponentSymbols( fileId, @@ -2189,6 +2199,34 @@ bool[] GetCssQualifiedRuleAncestors() => RegexOptions.Compiled | RegexOptions.CultureInvariant); private const int PrologMultilineHeadLookaheadLineLimit = 256; + private static bool TryGetNextPrologClauseOffset( + string line, + int currentClauseOffset, + out int nextClauseOffset) + { + for (var column = Math.Max(0, currentClauseOffset); column < line.Length; column++) + { + if (!DynamicDeclarativeReferenceExtractor.IsPrologClauseTerminator(line, column)) + continue; + + for (var candidate = column + 1; candidate < line.Length; candidate++) + { + if (char.IsWhiteSpace(line[candidate])) + continue; + + if (char.IsLower(line[candidate])) + { + nextClauseOffset = candidate; + return true; + } + break; + } + } + + nextClauseOffset = -1; + return false; + } + private static bool[] BuildPrologClauseContinuationLines( IReadOnlyList structuralLines, Dictionary multilineHeads) diff --git a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.cs b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.cs index 39f08525e..57d2394bf 100644 --- a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.cs +++ b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.cs @@ -1719,7 +1719,7 @@ private enum JavaScriptTypeScriptFunctionHeaderConsumeResult [ new("namespace", new Regex(@"^\s*:-\s*module\s*\(\s*(?[a-z][A-Za-z0-9_]*)", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None), new("import", new Regex(@"^\s*:-\s*use_module\s*\(\s*(?[a-z][A-Za-z0-9_]*(?:\([^)]*\))?)", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None), - new("function", new Regex(@"^\s*(?[a-z][A-Za-z0-9_]*)\s*(?:\([^\r\n]*\))?\s*(?::-|-->|\.(?=\s*$))", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None), + new("function", new Regex(@"^\s*(?[a-z][A-Za-z0-9_]*)\s*(?:\([^\r\n]*\))?\s*(?::-|-->|\.(?=\s*(?:$|[a-z][A-Za-z0-9_]*(?:\s*\([^)]*\))?\s*(?::-|-->|\.))))", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None), ], ["ambiguous_pl"] = [ @@ -1737,7 +1737,7 @@ private enum JavaScriptTypeScriptFunctionHeaderConsumeResult new("function", new Regex(@"^\s*(?:method|fun)\s+(?" + PerlIdentifierPattern + @")\b(?:\s*:[^{;]+)?", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.Brace), new("namespace", new Regex(@"^\s*:-\s*module\s*\(\s*(?[a-z][A-Za-z0-9_]*)", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None), new("import", new Regex(@"^\s*:-\s*use_module\s*\(\s*(?[a-z][A-Za-z0-9_]*(?:\([^)]*\))?)", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None), - new("function", new Regex(@"^\s*(?[a-z][A-Za-z0-9_]*)\s*(?:\([^\r\n]*\))?\s*(?::-|-->|\.(?=\s*$))", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None), + new("function", new Regex(@"^\s*(?[a-z][A-Za-z0-9_]*)\s*(?:\([^\r\n]*\))?\s*(?::-|-->|\.(?=\s*(?:$|[a-z][A-Za-z0-9_]*(?:\s*\([^)]*\))?\s*(?::-|-->|\.))))", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None), ], ["c"] = [ diff --git a/src/CodeIndex/Mcp/McpToolHandlers.Indexing.cs b/src/CodeIndex/Mcp/McpToolHandlers.Indexing.cs index d41b27848..8b822e8f1 100644 --- a/src/CodeIndex/Mcp/McpToolHandlers.Indexing.cs +++ b/src/CodeIndex/Mcp/McpToolHandlers.Indexing.cs @@ -1324,7 +1324,11 @@ await EmitProgressNotificationAsync( AddMcpIndexDiagnostics(structured, failures, mcpIndexDiagnostics); var referenceExtractionCapHits = writer.GetReferenceExtractionCapHits( issuesStateAvailable: (priorReadiness & DbContext.IssuesReadyFlag) != 0); - AddReferenceGraphCompletenessSignal(structured, referenceExtractionCapHits); + using var referenceSignalReader = new DbReader(writer.Connection, isReadOnly: true); + AddReferenceGraphCompletenessSignal( + structured, + referenceSignalReader, + referenceExtractionCapHits); if (!sqlGraphContractReady) { AddSqlGraphContractSignal( diff --git a/src/CodeIndex/Mcp/McpToolHandlers.QueryTools.cs b/src/CodeIndex/Mcp/McpToolHandlers.QueryTools.cs index a3ac83d6f..b30bcab12 100644 --- a/src/CodeIndex/Mcp/McpToolHandlers.QueryTools.cs +++ b/src/CodeIndex/Mcp/McpToolHandlers.QueryTools.cs @@ -1313,13 +1313,18 @@ private static void AddSqlGraphContractSignal(JsonObject payload, SqlGraphContra } private void AddReferenceGraphCompletenessSignal(JsonObject payload, DbReader reader) - => AddReferenceGraphCompletenessSignal(payload, reader.GetReferenceExtractionCapHits()); + => AddReferenceGraphCompletenessSignal( + payload, + reader, + reader.GetReferenceExtractionCapHits()); private void AddReferenceGraphCompletenessSignal( JsonObject payload, + DbReader reader, ReferenceExtractionCapHitSummary capHits) { - var complete = capHits.StateAvailable && capHits.HitCount == 0; + var complete = reader.IsReferenceGraphComplete(capHits); + var incompleteReasons = reader.GetReferenceGraphIncompleteReasons(capHits); payload["reference_extraction_limits"] = JsonSerializer.SerializeToNode( ReferenceExtractor.GetSafetyLimits(), _jsonOptions); @@ -1330,7 +1335,7 @@ private void AddReferenceGraphCompletenessSignal( if (!complete) { payload["reference_graph_incomplete_reasons"] = JsonSerializer.SerializeToNode( - capHits.Reasons, + incompleteReasons, _jsonOptions); payload["degraded"] = true; } diff --git a/tests/CodeIndex.Tests/DatabaseTests.cs b/tests/CodeIndex.Tests/DatabaseTests.cs index d6cc73589..24c4ffccc 100644 --- a/tests/CodeIndex.Tests/DatabaseTests.cs +++ b/tests/CodeIndex.Tests/DatabaseTests.cs @@ -4766,10 +4766,14 @@ public void GetStatus_DegradesPreGraphLanguageContractsUntilRefresh_Issue4746( versionKey, previousContractVersion.ToString(CultureInfo.InvariantCulture)); - var staleStatus = new DbReader(_db.Connection).GetStatus(); + var staleReader = new DbReader(_db.Connection); + var staleStatus = staleReader.GetStatus(); + var staleWorkspaceHealth = staleReader.GetWorkspaceIndexHealth(); Assert.False(staleStatus.ReferenceGraphComplete); Assert.False(staleStatus.GraphDataCurrent); + Assert.False(staleWorkspaceHealth.ReferenceGraphComplete); + Assert.False(staleWorkspaceHealth.GraphDataCurrent); Assert.Contains( DbReader.DynamicReferenceGraphContractStaleReason, staleStatus.ReferenceGraphIncompleteReasons ?? []); @@ -4778,11 +4782,19 @@ public void GetStatus_DegradesPreGraphLanguageContractsUntilRefresh_Issue4746( versionKey, SymbolExtractor.GetContractVersion(language).ToString(CultureInfo.InvariantCulture)); - var refreshedStatus = new DbReader(_db.Connection).GetStatus(); + var refreshedReader = new DbReader(_db.Connection); + var refreshedStatus = refreshedReader.GetStatus(); + var refreshedWorkspaceHealth = refreshedReader.GetWorkspaceIndexHealth(); Assert.DoesNotContain( DbReader.DynamicReferenceGraphContractStaleReason, refreshedStatus.ReferenceGraphIncompleteReasons ?? []); + Assert.Equal( + refreshedStatus.ReferenceGraphComplete, + refreshedWorkspaceHealth.ReferenceGraphComplete); + Assert.Equal( + refreshedStatus.GraphDataCurrent, + refreshedWorkspaceHealth.GraphDataCurrent); } [Fact] diff --git a/tests/CodeIndex.Tests/McpServerToolsCallTests.cs b/tests/CodeIndex.Tests/McpServerToolsCallTests.cs index 8c1d17cfe..86d627b70 100644 --- a/tests/CodeIndex.Tests/McpServerToolsCallTests.cs +++ b/tests/CodeIndex.Tests/McpServerToolsCallTests.cs @@ -69,6 +69,38 @@ FROM files Assert.Contains(capKind, structured["reference_graph_incomplete_reasons"]!.AsArray().Select(reason => reason!.GetValue())); } + [Fact] + public void ToolsCall_ReferenceGraphCommandsExposeStaleDynamicContract_Issue4746() + { + var writer = new DbWriter(_db.Connection); + writer.UpsertFile(new FileRecord + { + Path = "src/stale.cr", + Lang = "crystal", + Size = 20, + Lines = 3, + Modified = new DateTime(2025, 1, 1, 0, 0, 0, DateTimeKind.Utc), + }); + writer.SetMeta(DbContext.GetSymbolExtractorVersionMetaKey("crystal"), "2"); + + var request = JsonNode.Parse( + """{"jsonrpc":"2.0","id":4746,"method":"tools/call","params":{"name":"callers","arguments":{"query":"MissingSymbol","countOnly":true}}}""")!; + + var response = _server.HandleMessage(request)!; + var structured = response["result"]!["structuredContent"]!; + + Assert.False(structured["reference_graph_complete"]!.GetValue()); + Assert.True(structured["degraded"]!.GetValue()); + Assert.Equal( + 0L, + structured["reference_extraction_cap_hits"]!["hit_count"]!.GetValue()); + Assert.Contains( + DbReader.DynamicReferenceGraphContractStaleReason, + structured["reference_graph_incomplete_reasons"]! + .AsArray() + .Select(reason => reason!.GetValue())); + } + [Fact] public void ToolsCall_LanguagesPublishesReferenceExtractionLimits_Issue4620() { diff --git a/tests/CodeIndex.Tests/QueryCommandRunnerGraphTests.cs b/tests/CodeIndex.Tests/QueryCommandRunnerGraphTests.cs index a4c48da36..34c511d3e 100644 --- a/tests/CodeIndex.Tests/QueryCommandRunnerGraphTests.cs +++ b/tests/CodeIndex.Tests/QueryCommandRunnerGraphTests.cs @@ -69,6 +69,52 @@ FROM files } } + [Fact] + public void GraphCommands_StaleDynamicContractMarksAbsenceQueriesIncomplete_Issue4746() + { + var projectRoot = TestProjectHelper.CreateTempProject("cdidx_graph_dynamic_contract_4746"); + try + { + var dbPath = TestProjectHelper.CreateProjectDb(projectRoot); + TestProjectHelper.InsertIndexedFile( + dbPath, + "src/app.cr", + "crystal", + "def helper(value)\n value\nend\n"); + using (var db = new DbContext(DbOpenIntent.WriteIndex, dbPath)) + { + var writer = new DbWriter(db.Connection); + writer.MarkGraphReady(); + writer.MarkIssuesReady(); + writer.SetMeta(DbContext.GetSymbolExtractorVersionMetaKey("crystal"), "2"); + } + + var (exitCode, stdout, stderr) = CaptureConsole(() => RunGraphCommand( + "callers", + ["MissingSymbol", "--db", dbPath, "--json", "--count", "--exact"], + _jsonOptions)); + + Assert.Equal(CommandExitCodes.Success, exitCode); + Assert.Equal(string.Empty, stderr); + using var document = ParseJsonOutput(stdout); + var json = document.RootElement; + Assert.True(json.GetProperty("degraded").GetBoolean()); + Assert.False(json.GetProperty("reference_graph_complete").GetBoolean()); + Assert.Equal( + 0L, + json.GetProperty("reference_extraction_cap_hits").GetProperty("hit_count").GetInt64()); + Assert.Contains( + DbReader.DynamicReferenceGraphContractStaleReason, + json.GetProperty("reference_graph_incomplete_reasons") + .EnumerateArray() + .Select(value => value.GetString())); + } + finally + { + TestProjectHelper.DeleteDirectory(projectRoot); + } + } + private static void AssertReferenceGraphIncomplete(JsonElement json) { Assert.True(json.GetProperty("degraded").GetBoolean()); diff --git a/tests/CodeIndex.Tests/ReferenceExtractorTests.cs b/tests/CodeIndex.Tests/ReferenceExtractorTests.cs index e0a698d09..98efa7c19 100644 --- a/tests/CodeIndex.Tests/ReferenceExtractorTests.cs +++ b/tests/CodeIndex.Tests/ReferenceExtractorTests.cs @@ -1067,6 +1067,7 @@ helper value multiline = "helper() continued" divided = value // 2; helper(value) + [value].each { helper } helper = 1 end @@ -1083,7 +1084,7 @@ abstract def declared(value) Assert.Contains(symbols, symbol => symbol.Kind == "function" && symbol.Name == "declared"); AssertReferencesContain(references, "type_reference", null, "support"); AssertReferencesContain(references, "call", "run", "helper"); - Assert.Equal(4, references.Count(reference => + Assert.Equal(5, references.Count(reference => reference.ReferenceKind == "call" && reference.ContainerName == "run" && reference.SymbolName == "helper")); @@ -1195,6 +1196,9 @@ set result [helper $value] proc variadic args { helper } + + proc quoted {} "helper" + proc bare {} helper """; var (symbols, references) = ExtractSymbolsAndReferences("tcl", content); @@ -1203,6 +1207,8 @@ proc variadic args { AssertReferencesContain(references, "type_reference", null, "json"); AssertReferencesContain(references, "call", "run", "helper"); AssertReferencesContain(references, "call", "variadic", "helper"); + AssertReferencesContain(references, "call", "quoted", "helper"); + AssertReferencesContain(references, "call", "bare", "helper"); Assert.DoesNotContain(references, reference => reference.ReferenceKind == "call" && reference.SymbolName is "package" or "proc" or "set"); @@ -1219,6 +1225,8 @@ proc helper {} { proc switch_run {value} { switch -regexp -matchvar matched -indexvar indices -- $value { one { helper } + quoted "helper" + bare helper default { helper } } } @@ -1238,7 +1246,7 @@ proc try_run {} { var (_, references) = ExtractSymbolsAndReferences("tcl", content); - Assert.Equal(2, references.Count(reference => + Assert.Equal(4, references.Count(reference => reference.ReferenceKind == "call" && reference.ContainerName == "switch_run" && reference.SymbolName == "helper")); @@ -1297,6 +1305,35 @@ public void Extract_Prolog_IndexesMultilinePredicateHeadsWithoutPhantomDeclarati && reference.SymbolName == "multi_line_head"); } + [Theory] + [InlineData("prolog")] + [InlineData("ambiguous_pl")] + public void Extract_Prolog_IndexesSameLineClausesWithCorrectCallers_Issue4746( + string language) + { + const string content = """ + a. + b. + first. second. + run :- a. other :- b. + """; + + var (symbols, references) = ExtractSymbolsAndReferences(language, content); + + foreach (var name in new[] { "first", "second", "run", "other" }) + Assert.Single(symbols, symbol => symbol.Kind == "function" && symbol.Name == name); + AssertReferencesContain(references, "call", "run", "a"); + AssertReferencesContain(references, "call", "other", "b"); + Assert.DoesNotContain(references, reference => + reference.ReferenceKind == "call" + && reference.ContainerName == "run" + && reference.SymbolName == "b"); + Assert.DoesNotContain(references, reference => + reference.ReferenceKind == "call" + && reference.ContainerName == "other" + && reference.SymbolName == "a"); + } + [Fact] public void Extract_AmbiguousPl_IndexesPerlAndPrologReferencesWithoutReclassification_Issue4746() { From 4390ea9ab844afbffc024312c74ce0f6b6914f84 Mon Sep 17 00:00:00 2001 From: Widthdom Date: Fri, 24 Jul 2026 19:14:52 +0900 Subject: [PATCH 13/26] Close dynamic graph adversarial gaps (#4746) --- changelog.d/unreleased/4746.added.md | 1 + .../Cli/IndexCommandRunner.FullScan.cs | 2 + .../DynamicDeclarativeReferenceExtractor.cs | 171 +++++++++++++++--- .../Languages/PerlReferenceExtractor.cs | 16 +- .../ReferenceExtractor.CoreExtraction.cs | 4 +- .../Symbols/SymbolExtractor.ExtractCore.cs | 23 ++- src/CodeIndex/Mcp/McpToolHandlers.Indexing.cs | 2 + .../IndexCommandRunnerFullScanTests.cs | 42 +++++ .../McpServerToolsCallTests.cs | 62 +++++++ .../ReferenceExtractorTests.cs | 107 +++++++++++ 10 files changed, 389 insertions(+), 41 deletions(-) diff --git a/changelog.d/unreleased/4746.added.md b/changelog.d/unreleased/4746.added.md index 2fd331414..92890975e 100644 --- a/changelog.d/unreleased/4746.added.md +++ b/changelog.d/unreleased/4746.added.md @@ -23,6 +23,7 @@ affected: - tests/CodeIndex.Tests/SymbolExtractorTests.cs - tests/CodeIndex.Tests/ReferenceExtractorTests.cs - tests/CodeIndex.Tests/QueryCommandRunnerTests.cs + - tests/CodeIndex.Tests/McpServerToolsCallTests.cs - tests/CodeIndex.Tests/golden/status.json - README.md - AGENT_GUIDE.md diff --git a/src/CodeIndex/Cli/IndexCommandRunner.FullScan.cs b/src/CodeIndex/Cli/IndexCommandRunner.FullScan.cs index 37c849a2d..f88cbe005 100644 --- a/src/CodeIndex/Cli/IndexCommandRunner.FullScan.cs +++ b/src/CodeIndex/Cli/IndexCommandRunner.FullScan.cs @@ -3039,6 +3039,8 @@ int ReturnBeforeWriteSnapshotFailure(string changedPath) WriteProjectRootOnce(); txn.Commit(); ftsMutated |= fileFtsMutated; + if (!string.IsNullOrWhiteSpace(record.Lang)) + indexedSymbolExtractorLanguages.Add(record.Lang); CountFreshInsertedRows(chunkCount: chunks.Count); processed++; diff --git a/src/CodeIndex/Indexer/References/Languages/DynamicDeclarativeReferenceExtractor.cs b/src/CodeIndex/Indexer/References/Languages/DynamicDeclarativeReferenceExtractor.cs index 8f2371081..57bb3ad0c 100644 --- a/src/CodeIndex/Indexer/References/Languages/DynamicDeclarativeReferenceExtractor.cs +++ b/src/CodeIndex/Indexer/References/Languages/DynamicDeclarativeReferenceExtractor.cs @@ -114,13 +114,13 @@ public bool IsDeclarationAt(int lineNumber, int column, string name) => @"^\s*:-\s*use_module\s*\(\s*(?:library\s*\(\s*)?['""]?(?(?:\.\.?/)*[a-z][A-Za-z0-9_./-]*)", RegexOptions.Compiled | RegexOptions.CultureInvariant); private static readonly Regex CrystalBareCallRegex = new( - @"(?:^|[;={]|\b(?:then|do)\b|&&|\|\|)\s*(?:return\s+)?(?[A-Za-z_]\w*[?!]?)(?![\w?!])(?!\s*(?:\(|(?:<<|>>|[+\-*/%&|^])?=))", + @"(?:^|[;={]|\b(?:then|do)\b|&&|\|\|)\s*(?:return\s+)?(?[A-Za-z_]\w*[?!]?)(?![\w?!])(?!\s*(?::|\(|(?:<<|>>|[+\-*/%&|^])?=))", RegexOptions.Compiled | RegexOptions.CultureInvariant); private static readonly Regex CrystalSuffixedParenthesizedCallRegex = new( @"(?[A-Za-z_]\w*[?!])\s*\(", RegexOptions.Compiled | RegexOptions.CultureInvariant); private static readonly Regex GroovyBareCallRegex = new( - @"(?:^|[;={])\s*(?:return\s+)?(?[A-Za-z_]\w*)\b(?!\s*(?:\(|(?:<<|>>|[+\-*/%&|^])?=))", + @"(?:^|[;={])\s*(?:return\s+)?(?[A-Za-z_]\w*)\b(?!\s*(?::|\(|(?:<<|>>|[+\-*/%&|^])?=))", RegexOptions.Compiled | RegexOptions.CultureInvariant); private static readonly Regex GroovyConstructorDeclarationRegex = new( @"(?:^|[;{])\s*(?:@[A-Za-z_$][\w.$]*(?:\s*\([^)\r\n]*\))?\s+)*(?:(?:public|protected|private)\s+)*(?[A-Z]\w*)\s*\(", @@ -531,7 +531,8 @@ public static string[] MaskNonCodeLines(string language, IReadOnlyList l if (language == "ambiguous_pl" && ch == '%') { - if (IsPerlHashSigil(line, column)) + if (IsPerlHashSigil(line, column) + || IsLikelyPerlModuloOperator(line, column)) { column++; continue; @@ -615,6 +616,12 @@ public static string[] MaskTclContinuedCommentLines( return result; } + public static string[] MaskTclNonScriptLines(IReadOnlyList maskedLines) => + BuildTclCallLines( + maskedLines, + BuildTclBraceEndPositions(maskedLines), + new HashSet()); + public static ExtractionState? CreateState( string language, IReadOnlyList preparedLines, @@ -947,6 +954,50 @@ private static bool IsPerlHashSigil(string line, int column) || previousToken.Equals("scalar", StringComparison.Ordinal); } + private static bool IsLikelyPerlModuloOperator(string line, int column) + { + var prefix = line.AsSpan(0, column); + var hasPerlContext = prefix.Contains('$') + || prefix.Contains('@') + || StartsWithPerlStatementKeyword(prefix); + if (!hasPerlContext) + return false; + + var previousColumn = column - 1; + while (previousColumn >= 0 && char.IsWhiteSpace(line[previousColumn])) + previousColumn--; + if (previousColumn < 0) + return false; + + if (column + 1 < line.Length && line[column + 1] == '=') + return true; + + var nextColumn = column + 1; + while (nextColumn < line.Length && char.IsWhiteSpace(line[nextColumn])) + nextColumn++; + if (nextColumn >= line.Length) + return false; + + var previous = line[previousColumn]; + var next = line[nextColumn]; + return (char.IsLetterOrDigit(previous) || previous is '_' or ')' or ']' or '}') + && (char.IsLetterOrDigit(next) || next is '_' or '$' or '@' or '(' or '+' or '-'); + } + + private static bool StartsWithPerlStatementKeyword(ReadOnlySpan prefix) + { + prefix = prefix.TrimStart(); + foreach (var keyword in new[] { "my", "our", "state", "local", "return" }) + { + if (!prefix.StartsWith(keyword, StringComparison.Ordinal)) + continue; + if (prefix.Length == keyword.Length || char.IsWhiteSpace(prefix[keyword.Length])) + return true; + } + + return false; + } + private static bool IsLikelySlashyLiteralStart(string line, int column) { if (column + 1 >= line.Length || line[column + 1] is '/' or '*') @@ -2142,6 +2193,8 @@ private static IReadOnlyDictionary> BuildProl } var lineCalls = new List(); ScanPrologGoalLine( + lines, + lineIndex, callScanLine, callableNames, frames, @@ -2216,6 +2269,8 @@ private static string PreparePrologMultilineHeadScanLine( } private static void ScanPrologGoalLine( + IReadOnlyList lines, + int lineIndex, string line, IReadOnlySet callableNames, Stack frames, @@ -2321,7 +2376,11 @@ private static void ScanPrologGoalLine( continue; } if (callableNames.Contains(name) - && !IsPrologTermBeforeInfixOperator(line, column, nextColumn)) + && !IsPrologTermBeforeInfixOperator( + lines, + lineIndex, + column, + nextColumn)) { calls.Add(new PrologGoalCall(name, nameStart)); } @@ -2413,56 +2472,79 @@ private static void ScanPrologGoalLine( } private static bool IsPrologTermBeforeInfixOperator( - string line, + IReadOnlyList lines, + int lineIndex, int nameEndColumn, int nextColumn) { + const int lookaheadLineLimit = 256; + var line = lines[lineIndex]; + var afterTermLine = lineIndex; var afterTermColumn = nextColumn; if (nextColumn < line.Length && line[nextColumn] == '(') { var depth = 0; - afterTermColumn = -1; - for (var column = nextColumn; column < line.Length; column++) + var termClosed = false; + var endLineExclusive = Math.Min(lines.Count, lineIndex + lookaheadLineLimit); + for (var scanLineIndex = lineIndex; + scanLineIndex < endLineExclusive && !termClosed; + scanLineIndex++) { - var ch = line[column]; - if (ch is '\'' or '"') + var scanLine = lines[scanLineIndex]; + var startColumn = scanLineIndex == lineIndex ? nextColumn : 0; + for (var column = startColumn; column < scanLine.Length; column++) { - column = SkipQuotedToken(line, column, ch) - 1; - continue; - } + var ch = scanLine[column]; + if (ch is '\'' or '"') + { + column = SkipQuotedToken(scanLine, column, ch) - 1; + continue; + } - if (ch == '(') - { - depth++; - } - else if (ch == ')' && --depth == 0) - { - afterTermColumn = column + 1; - break; + if (ch == '(') + { + depth++; + } + else if (ch == ')' && --depth == 0) + { + afterTermLine = scanLineIndex; + afterTermColumn = column + 1; + termClosed = true; + break; + } } } - if (afterTermColumn < 0) - return false; + // An unterminated compound term is not authoritative evidence of a call. + // 未終端の compound term は call と判断できる根拠にならない。 + if (!termClosed) + return true; } else { afterTermColumn = nameEndColumn; } - while (afterTermColumn < line.Length && char.IsWhiteSpace(line[afterTermColumn])) - afterTermColumn++; - if (afterTermColumn >= line.Length) + if (!TryFindNextPrologToken( + lines, + afterTermLine, + afterTermColumn, + lookaheadLineLimit, + out var operatorLine, + out var operatorColumn)) + { return false; + } - var remaining = line.AsSpan(afterTermColumn); + var operatorSourceLine = lines[operatorLine]; + var remaining = operatorSourceLine.AsSpan(operatorColumn); if (remaining.StartsWith("->", StringComparison.Ordinal) || remaining.StartsWith("*->", StringComparison.Ordinal)) { return false; } - if (line[afterTermColumn] is '=' or '\\' or '<' or '>' or '@' or '#' + if (operatorSourceLine[operatorColumn] is '=' or '\\' or '<' or '>' or '@' or '#' or ':' or '+' or '-' or '*' or '/' or '^') { return true; @@ -2472,14 +2554,43 @@ private static bool IsPrologTermBeforeInfixOperator( { if (!remaining.StartsWith(operatorName, StringComparison.Ordinal)) continue; - var operatorEnd = afterTermColumn + operatorName.Length; - if (operatorEnd >= line.Length - || !char.IsLetterOrDigit(line[operatorEnd]) && line[operatorEnd] != '_') + var operatorEnd = operatorColumn + operatorName.Length; + if (operatorEnd >= operatorSourceLine.Length + || !char.IsLetterOrDigit(operatorSourceLine[operatorEnd]) + && operatorSourceLine[operatorEnd] != '_') + { + return true; + } + } + + return false; + } + + private static bool TryFindNextPrologToken( + IReadOnlyList lines, + int startLine, + int startColumn, + int lookaheadLineLimit, + out int tokenLine, + out int tokenColumn) + { + var endLineExclusive = Math.Min(lines.Count, startLine + lookaheadLineLimit); + for (var lineIndex = startLine; lineIndex < endLineExclusive; lineIndex++) + { + var line = lines[lineIndex]; + var column = lineIndex == startLine ? startColumn : 0; + while (column < line.Length && char.IsWhiteSpace(line[column])) + column++; + if (column < line.Length) { + tokenLine = lineIndex; + tokenColumn = column; return true; } } + tokenLine = -1; + tokenColumn = -1; return false; } diff --git a/src/CodeIndex/Indexer/References/Languages/PerlReferenceExtractor.cs b/src/CodeIndex/Indexer/References/Languages/PerlReferenceExtractor.cs index 3a31a6b13..fa178071c 100644 --- a/src/CodeIndex/Indexer/References/Languages/PerlReferenceExtractor.cs +++ b/src/CodeIndex/Indexer/References/Languages/PerlReferenceExtractor.cs @@ -41,14 +41,26 @@ public static void EmitAdditionalReferences( string context, int lineNumber, Func resolveContainerForCall, - Action addCallLikeReference) + Action addCallLikeReference, + bool emitArrowCallReferences = true) { EmitModuleReference(preparedLine, references, seen, fileId, context, lineNumber, resolveContainerForCall); EmitRequiredModulePathReference(originalLine, references, seen, fileId, context, lineNumber, resolveContainerForCall); EmitBaseModuleReferences(originalLine, references, seen, fileId, context, lineNumber, resolveContainerForCall); EmitMooseInheritanceReferences(originalLine, references, seen, fileId, context, lineNumber, resolveContainerForCall); EmitQualifiedFunctionCallReferences(preparedLine, references, seen, fileId, context, lineNumber, resolveContainerForCall); - EmitArrowCallReferences(preparedLine, references, seen, fileId, context, lineNumber, resolveContainerForCall, addCallLikeReference); + if (emitArrowCallReferences) + { + EmitArrowCallReferences( + preparedLine, + references, + seen, + fileId, + context, + lineNumber, + resolveContainerForCall, + addCallLikeReference); + } } private static void EmitModuleReference( diff --git a/src/CodeIndex/Indexer/References/ReferenceExtractor.CoreExtraction.cs b/src/CodeIndex/Indexer/References/ReferenceExtractor.CoreExtraction.cs index 6a4b465eb..03e722245 100644 --- a/src/CodeIndex/Indexer/References/ReferenceExtractor.CoreExtraction.cs +++ b/src/CodeIndex/Indexer/References/ReferenceExtractor.CoreExtraction.cs @@ -2623,7 +2623,9 @@ bool TryGetKnownPythonTypeCall(string candidate, out string canonicalName) context, lineNumber, ResolveContainerForCall, - AddCallLikeReference); + AddCallLikeReference, + emitArrowCallReferences: language != "ambiguous_pl" + || dynamicDeclarativeState?.HasPrologContainer(lineNumber) != true); } if (dynamicDeclarativeState != null) diff --git a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.ExtractCore.cs b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.ExtractCore.cs index aa4c9ae2b..1581968fb 100644 --- a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.ExtractCore.cs +++ b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.ExtractCore.cs @@ -152,6 +152,8 @@ private static List ExtractCore( structuralLines = DynamicDeclarativeReferenceExtractor.MaskTclContinuedCommentLines( lines, structuralLines); + structuralLines = DynamicDeclarativeReferenceExtractor.MaskTclNonScriptLines( + structuralLines); } string[]? javaScriptTypeScriptSanitizedLines = null; string[] GetJavaScriptTypeScriptSanitizedLines() => @@ -2237,26 +2239,31 @@ private static bool[] BuildPrologClauseContinuationLines( { continuationLines[lineIndex] = clauseOpen; var line = structuralLines[lineIndex]; - if (clauseOpen) + var lastTerminatorColumn = -1; + for (var column = 0; column < line.Length; column++) { - if (HasPrologClauseTerminator(line)) - clauseOpen = false; - continue; + if (DynamicDeclarativeReferenceExtractor.IsPrologClauseTerminator(line, column)) + lastTerminatorColumn = column; } - if (PrologOpenClauseRegex.IsMatch(line) && !HasPrologClauseTerminator(line)) + if (clauseOpen && lastTerminatorColumn < 0) + continue; + + clauseOpen = false; + var clauseCandidateOffset = lastTerminatorColumn + 1; + var clauseCandidate = line[clauseCandidateOffset..]; + if (PrologOpenClauseRegex.IsMatch(clauseCandidate)) { clauseOpen = true; continue; } - var multilineHead = PrologMultilineHeadStartRegex.Match(line); + var multilineHead = PrologMultilineHeadStartRegex.Match(clauseCandidate); if (multilineHead.Success - && !HasPrologClauseTerminator(line) && IsValidatedMultilinePrologHead( structuralLines, lineIndex, - multilineHead.Groups["open"].Index)) + clauseCandidateOffset + multilineHead.Groups["open"].Index)) { multilineHeads[lineIndex] = multilineHead.Groups["name"].Value; clauseOpen = true; diff --git a/src/CodeIndex/Mcp/McpToolHandlers.Indexing.cs b/src/CodeIndex/Mcp/McpToolHandlers.Indexing.cs index 8b822e8f1..2648e66f6 100644 --- a/src/CodeIndex/Mcp/McpToolHandlers.Indexing.cs +++ b/src/CodeIndex/Mcp/McpToolHandlers.Indexing.cs @@ -1635,6 +1635,8 @@ await EmitProgressNotificationAsync( if (!useFullRunBatchMarker) writer.ClearBatchInProgress(); txn.Commit(); + if (!string.IsNullOrWhiteSpace(record.Lang)) + indexedSymbolExtractorLanguages.Add(record.Lang); CountFreshInsertedRows(chunkCount: chunks.Count); ftsMutated = true; processed++; diff --git a/tests/CodeIndex.Tests/IndexCommandRunnerFullScanTests.cs b/tests/CodeIndex.Tests/IndexCommandRunnerFullScanTests.cs index 36aea390d..9d34dcef0 100644 --- a/tests/CodeIndex.Tests/IndexCommandRunnerFullScanTests.cs +++ b/tests/CodeIndex.Tests/IndexCommandRunnerFullScanTests.cs @@ -4070,6 +4070,48 @@ FROM symbol_references } } + [Fact] + public void Run_FullScan_StampsSuppressedDynamicGraphLanguage_Issue4746() + { + var projectRoot = CreateTempProject(); + using var env = EnvironmentVariableScope.Capture( + IndexCommandRunner.GeneratedCodePatternsEnvironmentVariable); + env.Set(IndexCommandRunner.GeneratedCodePatternsEnvironmentVariable, "*.cr"); + try + { + File.WriteAllText( + Path.Combine(projectRoot, "generated.cr"), + """ + def helper + 1 + end + """); + + var (exitCode, json) = RunAndCaptureJson( + [projectRoot, "--json", "--rebuild", "--yes"]); + + Assert.Equal(CommandExitCodes.Success, exitCode); + Assert.Equal("success", json.GetProperty("status").GetString()); + Assert.True(json.GetProperty("reference_graph_complete").GetBoolean()); + Assert.True(json.GetProperty("graph_data_current").GetBoolean()); + + var dbPath = Path.Combine(projectRoot, ".cdidx", "codeindex.db"); + using var verify = OpenNonPoolingConnection(dbPath); + verify.Open(); + using var versionCmd = verify.CreateCommand(); + versionCmd.CommandText = + $"SELECT value FROM codeindex_meta WHERE key = '{DbContext.GetSymbolExtractorVersionMetaKey("crystal")}'"; + Assert.Equal( + SymbolExtractor.DynamicReferenceGraphContractVersion.ToString( + System.Globalization.CultureInfo.InvariantCulture), + versionCmd.ExecuteScalar() as string); + } + finally + { + DeleteDirectory(projectRoot); + } + } + [Fact] public void Run_FullScan_ReindexesUnchangedCSharpTupleReadonlyFieldWhenExtractorVersionChanged_Issue4616() { diff --git a/tests/CodeIndex.Tests/McpServerToolsCallTests.cs b/tests/CodeIndex.Tests/McpServerToolsCallTests.cs index 86d627b70..568dbb8b7 100644 --- a/tests/CodeIndex.Tests/McpServerToolsCallTests.cs +++ b/tests/CodeIndex.Tests/McpServerToolsCallTests.cs @@ -11674,6 +11674,68 @@ public void ToolsCall_Index_DoesNotRestampFoldReadyWhenFoldKeyVersionMismatches( } } + [Fact] + public void ToolsCall_Index_StampsSuppressedDynamicGraphLanguage_Issue4746() + { + var fixtureDir = Path.Combine( + Path.GetFullPath("."), + $"mcp_index_suppressed_dynamic_4746_{Guid.NewGuid():N}"); + Directory.CreateDirectory(fixtureDir); + var dbPath = TestProjectHelper.CreateTempDbPath("cdidx_mcp_suppressed_dynamic_4746"); + using var env = EnvironmentVariableScope.Capture( + IndexCommandRunner.GeneratedCodePatternsEnvironmentVariable); + env.Set(IndexCommandRunner.GeneratedCodePatternsEnvironmentVariable, "*.cr"); + try + { + File.WriteAllText( + Path.Combine(fixtureDir, "generated.cr"), + """ + def helper + 1 + end + """); + using var server = new McpServer(dbPath, ConsoleUi.LoadVersion()); + var request = new JsonObject + { + ["jsonrpc"] = "2.0", + ["id"] = 1, + ["method"] = "tools/call", + ["params"] = new JsonObject + { + ["name"] = "index", + ["arguments"] = new JsonObject + { + ["path"] = fixtureDir, + ["rebuild"] = true + } + } + }; + + var response = server.HandleMessage(request)!; + + Assert.False(response["result"]!["isError"]?.GetValue() ?? false); + var structured = response["result"]!["structuredContent"]!; + Assert.True(structured["reference_graph_complete"]!.GetValue()); + + SqliteConnection.ClearAllPools(); + using var verify = new SqliteConnection($"Data Source={dbPath}"); + verify.Open(); + using var versionCmd = verify.CreateCommand(); + versionCmd.CommandText = + $"SELECT value FROM codeindex_meta WHERE key = '{DbContext.GetSymbolExtractorVersionMetaKey("crystal")}'"; + Assert.Equal( + SymbolExtractor.DynamicReferenceGraphContractVersion.ToString( + System.Globalization.CultureInfo.InvariantCulture), + versionCmd.ExecuteScalar() as string); + } + finally + { + SqliteConnection.ClearAllPools(); + TestProjectHelper.DeleteSqliteDatabaseFiles(dbPath); + TestProjectHelper.DeleteDirectory(fixtureDir); + } + } + [Fact] public void ToolsCall_Index_ClearsHotspotFamilyTrustOnPartialFailure() { diff --git a/tests/CodeIndex.Tests/ReferenceExtractorTests.cs b/tests/CodeIndex.Tests/ReferenceExtractorTests.cs index 98efa7c19..32739d303 100644 --- a/tests/CodeIndex.Tests/ReferenceExtractorTests.cs +++ b/tests/CodeIndex.Tests/ReferenceExtractorTests.cs @@ -1615,6 +1615,113 @@ public void Extract_AmbiguousPl_MasksPerlAndPrologCommentsAndPod_Issue4746() AssertReferencesContain(references, "call", "after_inline", "helper"); } + [Fact] + public void Extract_CrystalAndGroovy_DoNotTreatColonLabelsAsBareCalls_Issue4746() + { + const string crystalContent = """ + def helper + 1 + end + def run + consume( + helper: 1 + ) + end + """; + const string groovyContent = """ + def helper() {} + def run() { + def options = [ + helper: 1 + ] + } + """; + + var (_, crystalReferences) = ExtractSymbolsAndReferences("crystal", crystalContent); + var (_, groovyReferences) = ExtractSymbolsAndReferences("groovy", groovyContent); + + Assert.DoesNotContain(crystalReferences, reference => + reference.ReferenceKind == "call" + && reference.ContainerName == "run" + && reference.SymbolName == "helper"); + Assert.DoesNotContain(groovyReferences, reference => + reference.ReferenceKind == "call" + && reference.ContainerName == "run" + && reference.SymbolName == "helper"); + } + + [Fact] + public void Extract_Prolog_SuppressesMultilineDataTermsAndTracksLaterContinuedClause_Issue4746() + { + const string content = """ + bar(_). + process(_). + operator_data :- + bar( + value + ) + = bar(other), + process(value). + phantom. + run :- seed. other :- + phantom. + """; + + var (symbols, references) = ExtractSymbolsAndReferences("prolog", content); + + Assert.Contains(symbols, symbol => symbol.Kind == "function" && symbol.Name == "other"); + Assert.Single(symbols, symbol => symbol.Kind == "function" && symbol.Name == "phantom"); + AssertReferencesContain(references, "call", "operator_data", "process"); + AssertReferencesContain(references, "call", "other", "phantom"); + Assert.DoesNotContain(references, reference => + reference.ReferenceKind == "call" + && reference.ContainerName == "operator_data" + && reference.SymbolName == "bar"); + } + + [Fact] + public void Extract_Tcl_RejectsProcTextInsideLiteralBracedWords_Issue4746() + { + const string content = """ + set data { + proc fake {} { return 1 } + } + proc helper {} { return 1 } + proc run {} { + fake + helper + } + """; + + var (symbols, references) = ExtractSymbolsAndReferences("tcl", content); + + Assert.DoesNotContain(symbols, symbol => symbol.Kind == "function" && symbol.Name == "fake"); + Assert.DoesNotContain(references, reference => reference.SymbolName == "fake"); + AssertReferencesContain(references, "call", "run", "helper"); + } + + [Fact] + public void Extract_AmbiguousPl_PreservesPerlModuloAndRejectsPrologIfThenArrow_Issue4746() + { + const string content = """ + package Hybrid; + sub helper { return 1; } + :- module(hybrid, [entry/0]). + entry :- condition -> external_goal. + sub run { + my $value = 5 % 2; helper(); + } + """; + + var (_, references) = ExtractSymbolsAndReferences("ambiguous_pl", content); + + AssertReferencesContain(references, "call", "run", "helper"); + Assert.DoesNotContain(references, reference => + reference.ReferenceKind == "call" + && reference.ContainerName == "entry" + && reference.SymbolName == "external_goal"); + } + [Fact] public void TryGetExtractor_RegisteredLanguage_ReturnsAddressableExtractor() { From d33ee3743dd03ad796bfa41943c4e824294a9149 Mon Sep 17 00:00:00 2001 From: Widthdom Date: Fri, 24 Jul 2026 19:36:45 +0900 Subject: [PATCH 14/26] Resolve final Prolog and Tcl review cases (#4746) --- .../DynamicDeclarativeReferenceExtractor.cs | 88 +++++++++++++++---- .../Symbols/SymbolExtractor.ExtractCore.cs | 19 ++-- .../Indexer/Symbols/SymbolExtractor.cs | 4 +- .../ReferenceExtractorTests.cs | 23 ++++- 4 files changed, 108 insertions(+), 26 deletions(-) diff --git a/src/CodeIndex/Indexer/References/Languages/DynamicDeclarativeReferenceExtractor.cs b/src/CodeIndex/Indexer/References/Languages/DynamicDeclarativeReferenceExtractor.cs index 57bb3ad0c..1d0cf8941 100644 --- a/src/CodeIndex/Indexer/References/Languages/DynamicDeclarativeReferenceExtractor.cs +++ b/src/CodeIndex/Indexer/References/Languages/DynamicDeclarativeReferenceExtractor.cs @@ -96,7 +96,7 @@ public bool IsDeclarationAt(int lineNumber, int column, string name) => @"^\s*proc\s+[A-Za-z_:][\w:.-]*\s+", RegexOptions.Compiled | RegexOptions.CultureInvariant); private static readonly Regex PrologHeadRegex = new( - @"^\s*(?[a-z][A-Za-z0-9_]*)\s*(?:\([^\r\n]*\))?\s*(?::-|-->|\.(?=\s*(?:$|[a-z][A-Za-z0-9_]*(?:\s*\([^)]*\))?\s*(?::-|-->|\.))))", + @"^\s*(?[a-z][A-Za-z0-9_]*)\s*(?:\([^\r\n]*\))?\s*(?::-|-->|\.(?=\s*(?:$|:-|[a-z][A-Za-z0-9_]*\s*\(\s*$|[a-z][A-Za-z0-9_]*(?:\s*\([^)]*\))?\s*(?::-|-->|\.))))", RegexOptions.Compiled | RegexOptions.CultureInvariant); private static readonly Regex PrologMultilineHeadRegex = new( @"^\s*(?[a-z][A-Za-z0-9_]*)\s*\(", @@ -1425,18 +1425,23 @@ private static bool TryFindTclWordEnd( if (line[startColumn] == '"') { - for (var column = startColumn + 1; column < line.Length; column++) + for (var lineIndex = startLine; lineIndex < lines.Count; lineIndex++) { - if (line[column] == '\\') + line = lines[lineIndex]; + var firstColumn = lineIndex == startLine ? startColumn + 1 : 0; + for (var column = firstColumn; column < line.Length; column++) { - column++; - continue; - } - if (line[column] == '"') - { - endLine = startLine; - endColumn = column; - return true; + if (line[column] == '\\') + { + column++; + continue; + } + if (line[column] == '"') + { + endLine = lineIndex; + endColumn = column; + return true; + } } } @@ -2167,18 +2172,24 @@ private static IReadOnlyDictionary> BuildProl frames.Clear(); activeContainer = container; expectGoal = true; - scanningMultilineHead = container.StartLine == lineNumber - && !PrologHeadRegex.IsMatch(lines[lineIndex]) - && PrologMultilineHeadRegex.IsMatch(lines[lineIndex]); multilineHeadParenthesisDepth = 0; multilineHeadParenthesesClosed = false; + scanningMultilineHead = TryInitializePrologMultilineHeadScan( + lines, + container, + lineIndex, + ref multilineHeadParenthesisDepth, + ref multilineHeadParenthesesClosed); } string callScanLine; if (scanningMultilineHead) { + var multilineHeadLine = lineNumber == container.StartLine + ? MaskLineBeforeColumn(lines[lineIndex], container.StartColumn ?? 0) + : lines[lineIndex]; callScanLine = PreparePrologMultilineHeadScanLine( - lines[lineIndex], + multilineHeadLine, ref multilineHeadParenthesisDepth, ref multilineHeadParenthesesClosed, out var headEnded); @@ -2214,6 +2225,53 @@ private static IReadOnlyDictionary> BuildProl return result; } + private static bool TryInitializePrologMultilineHeadScan( + IReadOnlyList lines, + SymbolRecord container, + int currentLineIndex, + ref int parenthesisDepth, + ref bool parenthesesClosed) + { + var startLineIndex = container.StartLine - 1; + if (startLineIndex < 0 || startLineIndex >= lines.Count || startLineIndex > currentLineIndex) + return false; + + var startColumn = Math.Clamp( + container.StartColumn ?? 0, + 0, + lines[startLineIndex].Length); + var headLine = lines[startLineIndex][startColumn..]; + if (PrologHeadRegex.IsMatch(headLine) || !PrologMultilineHeadRegex.IsMatch(headLine)) + return false; + + for (var lineIndex = startLineIndex; lineIndex < currentLineIndex; lineIndex++) + { + var line = lineIndex == startLineIndex + ? MaskLineBeforeColumn(lines[lineIndex], startColumn) + : lines[lineIndex]; + _ = PreparePrologMultilineHeadScanLine( + line, + ref parenthesisDepth, + ref parenthesesClosed, + out var headEnded); + if (headEnded) + return false; + } + + return true; + } + + private static string MaskLineBeforeColumn(string line, int startColumn) + { + startColumn = Math.Clamp(startColumn, 0, line.Length); + if (startColumn == 0) + return line; + + var masked = line.ToCharArray(); + FillWithSpaces(masked, 0, startColumn); + return new string(masked); + } + private static string PreparePrologMultilineHeadScanLine( string line, ref int parenthesisDepth, diff --git a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.ExtractCore.cs b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.ExtractCore.cs index 1581968fb..d49c31a46 100644 --- a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.ExtractCore.cs +++ b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.ExtractCore.cs @@ -168,7 +168,7 @@ string[] GetJavaScriptTypeScriptSanitizedLines() => ? MaskShellHeredocLines(lines) : null; bool[]? prologClauseContinuationLines = null; - Dictionary? prologMultilineHeads = null; + Dictionary? prologMultilineHeads = null; if (lang is "prolog" or "ambiguous_pl") { prologMultilineHeads = []; @@ -2200,6 +2200,7 @@ bool[] GetCssQualifiedRuleAncestors() => @"^\s*(?[a-z][A-Za-z0-9_]*)\s*(?\()", RegexOptions.Compiled | RegexOptions.CultureInvariant); private const int PrologMultilineHeadLookaheadLineLimit = 256; + private readonly record struct PrologMultilineHead(string Name, int StartColumn); private static bool TryGetNextPrologClauseOffset( string line, @@ -2231,7 +2232,7 @@ private static bool TryGetNextPrologClauseOffset( private static bool[] BuildPrologClauseContinuationLines( IReadOnlyList structuralLines, - Dictionary multilineHeads) + Dictionary multilineHeads) { var continuationLines = new bool[structuralLines.Count]; var clauseOpen = false; @@ -2265,7 +2266,9 @@ private static bool[] BuildPrologClauseContinuationLines( lineIndex, clauseCandidateOffset + multilineHead.Groups["open"].Index)) { - multilineHeads[lineIndex] = multilineHead.Groups["name"].Value; + multilineHeads[lineIndex] = new PrologMultilineHead( + multilineHead.Groups["name"].Value, + clauseCandidateOffset + multilineHead.Groups["name"].Index); clauseOpen = true; } } @@ -2331,9 +2334,9 @@ private static void AddPrologMultilineHeadSymbols( IReadOnlyList lines, List symbols, SymbolExtractionState extractionState, - IReadOnlyDictionary multilineHeads) + IReadOnlyDictionary multilineHeads) { - foreach (var (lineIndex, name) in multilineHeads) + foreach (var (lineIndex, multilineHead) in multilineHeads) { var lineNumber = lineIndex + 1; var line = lines[lineIndex]; @@ -2346,12 +2349,12 @@ private static void AddPrologMultilineHeadSymbols( { FileId = fileId, Kind = "function", - Name = name, + Name = multilineHead.Name, Line = lineNumber, StartLine = lineNumber, - StartColumn = 0, + StartColumn = multilineHead.StartColumn, EndLine = lineNumber, - Signature = line.Trim(), + Signature = line[multilineHead.StartColumn..].Trim(), }, line); } diff --git a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.cs b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.cs index 57d2394bf..e197a82c8 100644 --- a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.cs +++ b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.cs @@ -1719,7 +1719,7 @@ private enum JavaScriptTypeScriptFunctionHeaderConsumeResult [ new("namespace", new Regex(@"^\s*:-\s*module\s*\(\s*(?[a-z][A-Za-z0-9_]*)", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None), new("import", new Regex(@"^\s*:-\s*use_module\s*\(\s*(?[a-z][A-Za-z0-9_]*(?:\([^)]*\))?)", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None), - new("function", new Regex(@"^\s*(?[a-z][A-Za-z0-9_]*)\s*(?:\([^\r\n]*\))?\s*(?::-|-->|\.(?=\s*(?:$|[a-z][A-Za-z0-9_]*(?:\s*\([^)]*\))?\s*(?::-|-->|\.))))", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None), + new("function", new Regex(@"^\s*(?[a-z][A-Za-z0-9_]*)\s*(?:\([^\r\n]*\))?\s*(?::-|-->|\.(?=\s*(?:$|:-|[a-z][A-Za-z0-9_]*\s*\(\s*$|[a-z][A-Za-z0-9_]*(?:\s*\([^)]*\))?\s*(?::-|-->|\.))))", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None), ], ["ambiguous_pl"] = [ @@ -1737,7 +1737,7 @@ private enum JavaScriptTypeScriptFunctionHeaderConsumeResult new("function", new Regex(@"^\s*(?:method|fun)\s+(?" + PerlIdentifierPattern + @")\b(?:\s*:[^{;]+)?", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.Brace), new("namespace", new Regex(@"^\s*:-\s*module\s*\(\s*(?[a-z][A-Za-z0-9_]*)", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None), new("import", new Regex(@"^\s*:-\s*use_module\s*\(\s*(?[a-z][A-Za-z0-9_]*(?:\([^)]*\))?)", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None), - new("function", new Regex(@"^\s*(?[a-z][A-Za-z0-9_]*)\s*(?:\([^\r\n]*\))?\s*(?::-|-->|\.(?=\s*(?:$|[a-z][A-Za-z0-9_]*(?:\s*\([^)]*\))?\s*(?::-|-->|\.))))", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None), + new("function", new Regex(@"^\s*(?[a-z][A-Za-z0-9_]*)\s*(?:\([^\r\n]*\))?\s*(?::-|-->|\.(?=\s*(?:$|:-|[a-z][A-Za-z0-9_]*\s*\(\s*$|[a-z][A-Za-z0-9_]*(?:\s*\([^)]*\))?\s*(?::-|-->|\.))))", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None), ], ["c"] = [ diff --git a/tests/CodeIndex.Tests/ReferenceExtractorTests.cs b/tests/CodeIndex.Tests/ReferenceExtractorTests.cs index 32739d303..faf5ddb56 100644 --- a/tests/CodeIndex.Tests/ReferenceExtractorTests.cs +++ b/tests/CodeIndex.Tests/ReferenceExtractorTests.cs @@ -1198,6 +1198,9 @@ proc variadic args { } proc quoted {} "helper" + proc multiline_quoted {} " + helper + " proc bare {} helper """; @@ -1208,6 +1211,7 @@ proc bare {} helper AssertReferencesContain(references, "call", "run", "helper"); AssertReferencesContain(references, "call", "variadic", "helper"); AssertReferencesContain(references, "call", "quoted", "helper"); + AssertReferencesContain(references, "call", "multiline_quoted", "helper"); AssertReferencesContain(references, "call", "bare", "helper"); Assert.DoesNotContain(references, reference => reference.ReferenceKind == "call" @@ -1316,14 +1320,31 @@ public void Extract_Prolog_IndexesSameLineClausesWithCorrectCallers_Issue4746( b. first. second. run :- a. other :- b. + before_directive. :- dynamic before_directive/0. + before_multiline. multiline_after( + X + ) :- + a. """; var (symbols, references) = ExtractSymbolsAndReferences(language, content); - foreach (var name in new[] { "first", "second", "run", "other" }) + foreach (var name in new[] + { + "first", + "second", + "run", + "other", + "before_directive", + "before_multiline", + "multiline_after", + }) + { Assert.Single(symbols, symbol => symbol.Kind == "function" && symbol.Name == name); + } AssertReferencesContain(references, "call", "run", "a"); AssertReferencesContain(references, "call", "other", "b"); + AssertReferencesContain(references, "call", "multiline_after", "a"); Assert.DoesNotContain(references, reference => reference.ReferenceKind == "call" && reference.ContainerName == "run" From 7055444129172f53be33b212c07b2d8159dd43aa Mon Sep 17 00:00:00 2001 From: Widthdom Date: Fri, 24 Jul 2026 19:55:41 +0900 Subject: [PATCH 15/26] Close final dynamic graph review gaps (#4746) --- .../DynamicDeclarativeReferenceExtractor.cs | 11 +- .../Symbols/SymbolExtractor.ExtractCore.cs | 103 +++++++++++++++++- .../ReferenceExtractorTests.cs | 21 ++++ tests/CodeIndex.Tests/SymbolExtractorTests.cs | 15 +++ 4 files changed, 141 insertions(+), 9 deletions(-) diff --git a/src/CodeIndex/Indexer/References/Languages/DynamicDeclarativeReferenceExtractor.cs b/src/CodeIndex/Indexer/References/Languages/DynamicDeclarativeReferenceExtractor.cs index 1d0cf8941..eb29bfd91 100644 --- a/src/CodeIndex/Indexer/References/Languages/DynamicDeclarativeReferenceExtractor.cs +++ b/src/CodeIndex/Indexer/References/Languages/DynamicDeclarativeReferenceExtractor.cs @@ -99,7 +99,7 @@ public bool IsDeclarationAt(int lineNumber, int column, string name) => @"^\s*(?[a-z][A-Za-z0-9_]*)\s*(?:\([^\r\n]*\))?\s*(?::-|-->|\.(?=\s*(?:$|:-|[a-z][A-Za-z0-9_]*\s*\(\s*$|[a-z][A-Za-z0-9_]*(?:\s*\([^)]*\))?\s*(?::-|-->|\.))))", RegexOptions.Compiled | RegexOptions.CultureInvariant); private static readonly Regex PrologMultilineHeadRegex = new( - @"^\s*(?[a-z][A-Za-z0-9_]*)\s*\(", + @"^\s*(?[a-z][A-Za-z0-9_]*)(?:(?\s*\()|\s*$)", RegexOptions.Compiled | RegexOptions.CultureInvariant); private static readonly Regex CrystalRequireRegex = new( @"^\s*require\s+['""](?[^'""]+)['""]", @@ -704,9 +704,12 @@ public static void EmitAdditionalReferences( Func resolveContainerForCall, Action addCallLikeReference) { + var importScanLine = language == "tcl" + ? state.GetCallScanLine(language, lineNumber, structuralLine) + : structuralLine; EmitImportReference( language, - structuralLine, + importScanLine, references, seen, fileId, @@ -2241,8 +2244,10 @@ private static bool TryInitializePrologMultilineHeadScan( 0, lines[startLineIndex].Length); var headLine = lines[startLineIndex][startColumn..]; - if (PrologHeadRegex.IsMatch(headLine) || !PrologMultilineHeadRegex.IsMatch(headLine)) + var multilineHeadMatch = PrologMultilineHeadRegex.Match(headLine); + if (PrologHeadRegex.IsMatch(headLine) || !multilineHeadMatch.Success) return false; + parenthesesClosed = !multilineHeadMatch.Groups["open"].Success; for (var lineIndex = startLineIndex; lineIndex < currentLineIndex; lineIndex++) { diff --git a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.ExtractCore.cs b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.ExtractCore.cs index d49c31a46..dfb43ae52 100644 --- a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.ExtractCore.cs +++ b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.ExtractCore.cs @@ -2199,6 +2199,9 @@ bool[] GetCssQualifiedRuleAncestors() => private static readonly Regex PrologMultilineHeadStartRegex = new( @"^\s*(?[a-z][A-Za-z0-9_]*)\s*(?\()", RegexOptions.Compiled | RegexOptions.CultureInvariant); + private static readonly Regex PrologBareMultilineHeadStartRegex = new( + @"^\s*(?[a-z][A-Za-z0-9_]*)\s*$", + RegexOptions.Compiled | RegexOptions.CultureInvariant); private const int PrologMultilineHeadLookaheadLineLimit = 256; private readonly record struct PrologMultilineHead(string Name, int StartColumn); @@ -2240,12 +2243,7 @@ private static bool[] BuildPrologClauseContinuationLines( { continuationLines[lineIndex] = clauseOpen; var line = structuralLines[lineIndex]; - var lastTerminatorColumn = -1; - for (var column = 0; column < line.Length; column++) - { - if (DynamicDeclarativeReferenceExtractor.IsPrologClauseTerminator(line, column)) - lastTerminatorColumn = column; - } + var lastTerminatorColumn = FindLastTopLevelPrologClauseTerminator(line); if (clauseOpen && lastTerminatorColumn < 0) continue; @@ -2270,12 +2268,105 @@ private static bool[] BuildPrologClauseContinuationLines( multilineHead.Groups["name"].Value, clauseCandidateOffset + multilineHead.Groups["name"].Index); clauseOpen = true; + continue; + } + + var bareMultilineHead = PrologBareMultilineHeadStartRegex.Match(clauseCandidate); + if (bareMultilineHead.Success + && IsValidatedBareMultilinePrologHead(structuralLines, lineIndex)) + { + multilineHeads[lineIndex] = new PrologMultilineHead( + bareMultilineHead.Groups["name"].Value, + clauseCandidateOffset + bareMultilineHead.Groups["name"].Index); + clauseOpen = true; } } return continuationLines; } + private static int FindLastTopLevelPrologClauseTerminator(string line) + { + var lastTerminatorColumn = -1; + var parenthesisDepth = 0; + var bracketDepth = 0; + var braceDepth = 0; + for (var column = 0; column < line.Length; column++) + { + var ch = line[column]; + if (ch is '\'' or '"') + { + column = SkipPrologQuotedTerm(line, column, ch) - 1; + continue; + } + + switch (ch) + { + case '(': + parenthesisDepth++; + continue; + case ')' when parenthesisDepth > 0: + parenthesisDepth--; + continue; + case '[': + bracketDepth++; + continue; + case ']' when bracketDepth > 0: + bracketDepth--; + continue; + case '{': + braceDepth++; + continue; + case '}' when braceDepth > 0: + braceDepth--; + continue; + } + + if (ch != '.' + || parenthesisDepth != 0 + || bracketDepth != 0 + || braceDepth != 0) + { + continue; + } + + var previous = column > 0 ? line[column - 1] : '\0'; + var next = column + 1 < line.Length ? line[column + 1] : '\0'; + if (previous != '.' + && next != '.' + && !(char.IsDigit(previous) && char.IsDigit(next)) + && (next == '\0' || char.IsWhiteSpace(next))) + { + lastTerminatorColumn = column; + } + } + + return lastTerminatorColumn; + } + + private static bool IsValidatedBareMultilinePrologHead( + IReadOnlyList structuralLines, + int startLineIndex) + { + var endLineExclusive = Math.Min( + structuralLines.Count, + startLineIndex + PrologMultilineHeadLookaheadLineLimit); + for (var lineIndex = startLineIndex + 1; lineIndex < endLineExclusive; lineIndex++) + { + var line = structuralLines[lineIndex]; + for (var column = 0; column < line.Length; column++) + { + if (char.IsWhiteSpace(line[column])) + continue; + return line.AsSpan(column).StartsWith(":-", StringComparison.Ordinal) + || line.AsSpan(column).StartsWith("-->", StringComparison.Ordinal) + || DynamicDeclarativeReferenceExtractor.IsPrologClauseTerminator(line, column); + } + } + + return false; + } + private static bool IsValidatedMultilinePrologHead( IReadOnlyList structuralLines, int startLineIndex, diff --git a/tests/CodeIndex.Tests/ReferenceExtractorTests.cs b/tests/CodeIndex.Tests/ReferenceExtractorTests.cs index faf5ddb56..8e1c1a597 100644 --- a/tests/CodeIndex.Tests/ReferenceExtractorTests.cs +++ b/tests/CodeIndex.Tests/ReferenceExtractorTests.cs @@ -1297,13 +1297,26 @@ public void Extract_Prolog_IndexesMultilinePredicateHeadsWithoutPhantomDeclarati X ) :- helper. + no_argument_rule + :- + helper. + no_argument_dcg + --> + helper. + no_argument_fact + . """; var (symbols, references) = ExtractSymbolsAndReferences(language, content); Assert.Single(symbols, symbol => symbol.Kind == "function" && symbol.Name == "helper"); Assert.Single(symbols, symbol => symbol.Kind == "function" && symbol.Name == "multi_line_head"); + Assert.Single(symbols, symbol => symbol.Kind == "function" && symbol.Name == "no_argument_rule"); + Assert.Single(symbols, symbol => symbol.Kind == "function" && symbol.Name == "no_argument_dcg"); + Assert.Single(symbols, symbol => symbol.Kind == "function" && symbol.Name == "no_argument_fact"); AssertReferencesContain(references, "call", "multi_line_head", "helper"); + AssertReferencesContain(references, "call", "no_argument_rule", "helper"); + AssertReferencesContain(references, "call", "no_argument_dcg", "helper"); Assert.DoesNotContain(references, reference => reference.ReferenceKind == "call" && reference.SymbolName == "multi_line_head"); @@ -1706,7 +1719,12 @@ public void Extract_Tcl_RejectsProcTextInsideLiteralBracedWords_Issue4746() const string content = """ set data { proc fake {} { return 1 } + package require fakepkg } + set quoted " + package require quotedfake + " + package require realpkg proc helper {} { return 1 } proc run {} { fake @@ -1718,6 +1736,9 @@ proc run {} { Assert.DoesNotContain(symbols, symbol => symbol.Kind == "function" && symbol.Name == "fake"); Assert.DoesNotContain(references, reference => reference.SymbolName == "fake"); + Assert.DoesNotContain(references, reference => + reference.SymbolName is "fakepkg" or "quotedfake"); + AssertReferencesContain(references, "type_reference", null, "realpkg"); AssertReferencesContain(references, "call", "run", "helper"); } diff --git a/tests/CodeIndex.Tests/SymbolExtractorTests.cs b/tests/CodeIndex.Tests/SymbolExtractorTests.cs index bb7d8371e..4c7893a17 100644 --- a/tests/CodeIndex.Tests/SymbolExtractorTests.cs +++ b/tests/CodeIndex.Tests/SymbolExtractorTests.cs @@ -116,6 +116,21 @@ public void Extract_Prolog_ManyMalformedMultilineHeadsUsesBoundedLookahead_Issue $"Expected bounded multiline-head validation, elapsed={stopwatch.Elapsed}."); } + [Fact] + public void Extract_Prolog_ManySameLineTerminatorsUsesLinearScan_Issue4746() + { + var content = string.Concat(Enumerable.Repeat(". ", 30_000)); + var stopwatch = Stopwatch.StartNew(); + + var symbols = SymbolExtractor.Extract(1, "prolog", content); + + stopwatch.Stop(); + Assert.Empty(symbols); + Assert.True( + stopwatch.Elapsed < TimeSpan.FromSeconds(3), + $"Expected linear terminator scanning, elapsed={stopwatch.Elapsed}."); + } + [Fact] public void Extract_CancelledToken_ThrowsBeforeWork() { From 33d65801c4013fa2fb0d60ff08faf8084fab4344 Mon Sep 17 00:00:00 2001 From: Widthdom Date: Fri, 24 Jul 2026 20:24:12 +0900 Subject: [PATCH 16/26] Resolve final dynamic graph review findings (#4746) --- .../Cli/IndexCommandRunner.FullScan.cs | 1 + src/CodeIndex/Database/DbContext.cs | 2 + src/CodeIndex/Database/DbReader.Status.cs | 4 +- .../Database/DbWriter.ContractMetadata.cs | 24 ++ .../Database/DbWriter.FoldBackfill.cs | 11 +- .../DynamicDeclarativeReferenceExtractor.cs | 233 ++++++++++++++++-- .../Indexer/Symbols/SymbolExtractor.cs | 5 +- src/CodeIndex/Mcp/McpToolHandlers.Indexing.cs | 1 + tests/CodeIndex.Tests/DatabaseTests.cs | 61 ++++- .../IndexCommandRunnerFullScanTests.cs | 6 +- .../McpServerToolsCallTests.cs | 2 +- .../ReferenceExtractorTests.cs | 49 +++- 12 files changed, 364 insertions(+), 35 deletions(-) diff --git a/src/CodeIndex/Cli/IndexCommandRunner.FullScan.cs b/src/CodeIndex/Cli/IndexCommandRunner.FullScan.cs index f88cbe005..f5e60803a 100644 --- a/src/CodeIndex/Cli/IndexCommandRunner.FullScan.cs +++ b/src/CodeIndex/Cli/IndexCommandRunner.FullScan.cs @@ -3524,6 +3524,7 @@ int ReturnBeforeWriteSnapshotFailure(string changedPath) // extractor version は今回再生成した row の契約であり、独立した fold-key // 契約を restamp できるかどうかに依存させない。 writer.StampSymbolExtractorVersions(indexedSymbolExtractorLanguages); + writer.StampDynamicReferenceGraphContracts(indexedSymbolExtractorLanguages); // FoldReady must reflect reality (#86). Full-scan is INCREMENTAL by default — it // skips unchanged files via GetUnchangedFileId, so a legacy DB's pre-#86 rows // keep NULL name_folded / *_folded values. Stamping FoldReady anyway would flip diff --git a/src/CodeIndex/Database/DbContext.cs b/src/CodeIndex/Database/DbContext.cs index 07ca6634f..2ce52ed0c 100644 --- a/src/CodeIndex/Database/DbContext.cs +++ b/src/CodeIndex/Database/DbContext.cs @@ -2056,6 +2056,8 @@ public static string BuildIncompleteHotspotFamilyMarkerFingerprint(string? finge public const string SqlGraphContractVersionMetaKey = "sql_graph_contract_version"; public const int ReferenceIdentityContractVersion = 1; public const string ReferenceIdentityContractVersionMetaKey = "reference_identity_contract_version"; + public static string GetDynamicReferenceGraphContractVersionMetaKey(string lang) => + $"dynamic_reference_graph_contract_version_{lang}"; public const string SymbolsOnlyGraphOmittedMetaKey = "symbols_only_graph_omitted"; public const string IndexedProjectRootMetaKey = "indexed_project_root"; public const string IndexedFollowSymlinksPolicyMetaKey = "indexed_follow_symlinks_policy"; diff --git a/src/CodeIndex/Database/DbReader.Status.cs b/src/CodeIndex/Database/DbReader.Status.cs index 4719a9ef4..cc9361cdf 100644 --- a/src/CodeIndex/Database/DbReader.Status.cs +++ b/src/CodeIndex/Database/DbReader.Status.cs @@ -274,8 +274,8 @@ private bool AreDynamicReferenceGraphContractsCurrent( continue; var storedVersion = TryGetMetaStringInternal( - DbContext.GetSymbolExtractorVersionMetaKey(language)); - var currentVersion = SymbolExtractor.GetContractVersion(language).ToString( + DbContext.GetDynamicReferenceGraphContractVersionMetaKey(language)); + var currentVersion = SymbolExtractor.GetReferenceGraphContractVersion(language).ToString( System.Globalization.CultureInfo.InvariantCulture); if (!string.Equals(storedVersion, currentVersion, StringComparison.Ordinal)) return false; diff --git a/src/CodeIndex/Database/DbWriter.ContractMetadata.cs b/src/CodeIndex/Database/DbWriter.ContractMetadata.cs index 5bded3bf9..e2a63f1c8 100644 --- a/src/CodeIndex/Database/DbWriter.ContractMetadata.cs +++ b/src/CodeIndex/Database/DbWriter.ContractMetadata.cs @@ -23,6 +23,30 @@ public void StampSymbolExtractorVersions(IReadOnlyCollection? languagesT SetMetaValues(values.ToArray()); } + /// + /// Stamp reference-graph contracts only for languages whose graph rows were regenerated. + /// This is intentionally separate from symbol-extractor versions because fold-only + /// maintenance may restamp symbol metadata without reparsing references. + /// reference graph を再生成した言語だけに専用 contract を stamp する。fold-only + /// maintenance が symbol metadata を更新しても、旧 graph を current と誤認しない。 + /// + public void StampDynamicReferenceGraphContracts(IReadOnlyCollection languagesToStamp) + { + var values = new List<(string Key, string? Value)>(languagesToStamp.Count); + foreach (var lang in languagesToStamp) + { + if (!SymbolExtractor.RequiresExplicitReferenceGraphContractStamp(lang)) + continue; + + values.Add(( + DbContext.GetDynamicReferenceGraphContractVersionMetaKey(lang), + SymbolExtractor.GetReferenceGraphContractVersion(lang).ToString( + System.Globalization.CultureInfo.InvariantCulture))); + } + + SetMetaValues(values.ToArray()); + } + /// /// Stamp the current C# symbol-name contract version. Readers and indexers use this to /// detect canonical-name upgrades such as operator/conversion/indexer renames. diff --git a/src/CodeIndex/Database/DbWriter.FoldBackfill.cs b/src/CodeIndex/Database/DbWriter.FoldBackfill.cs index 0f73bd093..2796c2e7f 100644 --- a/src/CodeIndex/Database/DbWriter.FoldBackfill.cs +++ b/src/CodeIndex/Database/DbWriter.FoldBackfill.cs @@ -219,7 +219,16 @@ private bool SymbolExtractorVersionMatchesCurrent(string? lang) return !SymbolExtractor.RequiresExplicitReferenceGraphContractStamp(lang); var current = SymbolExtractor.GetContractVersion(lang).ToString(System.Globalization.CultureInfo.InvariantCulture); - return stored == current; + if (stored != current) + return false; + if (!SymbolExtractor.RequiresExplicitReferenceGraphContractStamp(lang)) + return true; + + var storedGraphContract = GetMetaString( + DbContext.GetDynamicReferenceGraphContractVersionMetaKey(lang)); + var currentGraphContract = SymbolExtractor.GetReferenceGraphContractVersion(lang).ToString( + System.Globalization.CultureInfo.InvariantCulture); + return storedGraphContract == currentGraphContract; } /// diff --git a/src/CodeIndex/Indexer/References/Languages/DynamicDeclarativeReferenceExtractor.cs b/src/CodeIndex/Indexer/References/Languages/DynamicDeclarativeReferenceExtractor.cs index eb29bfd91..b6014c895 100644 --- a/src/CodeIndex/Indexer/References/Languages/DynamicDeclarativeReferenceExtractor.cs +++ b/src/CodeIndex/Indexer/References/Languages/DynamicDeclarativeReferenceExtractor.cs @@ -126,11 +126,14 @@ public bool IsDeclarationAt(int lineNumber, int column, string name) => @"(?:^|[;{])\s*(?:@[A-Za-z_$][\w.$]*(?:\s*\([^)\r\n]*\))?\s+)*(?:(?:public|protected|private)\s+)*(?[A-Z]\w*)\s*\(", RegexOptions.Compiled | RegexOptions.CultureInvariant); private static readonly Regex GroovyMethodDeclarationRegex = new( - @"(?:^|[;{])\s*(?:@[A-Za-z_$][\w.$]*(?:\s*\([^)\r\n]*\))?\s+)*(?:(?:public|private|protected|static|final|abstract|synchronized|native|strictfp)\s+)*(?:def|void|boolean|byte|char|short|int|long|float|double|BigDecimal|BigInteger|String|[A-Za-z_$][\w.$]*(?:\s*<[^(){}\r\n]+>)?(?:\s*\[\])*)\s+(?[A-Za-z_]\w*)\s*\(", + @"(?:^|[;{])\s*(?:@[A-Za-z_$][\w.$]*(?:\s*\([^)\r\n]*\))?\s+)*(?:(?:public|private|protected|static|final|abstract|synchronized|native|strictfp)\s+)*(?:<[^(){}\r\n]+>\s+)?(?:def|void|boolean|byte|char|short|int|long|float|double|BigDecimal|BigInteger|String|[A-Za-z_$][\w.$]*(?:\s*<[^(){}\r\n]+>)?(?:\s*\[\])*)\s+(?[A-Za-z_]\w*)\s*\(", RegexOptions.Compiled | RegexOptions.CultureInvariant); private static readonly Regex CrystalMethodDeclarationRegex = new( @"(?:^|;)\s*(?:(?:private|protected|abstract)\s+)*def\s+(?:self\.)?(?[A-Za-z_]\w*[?!]?)\s*\(", RegexOptions.Compiled | RegexOptions.CultureInvariant); + private static readonly Regex CrystalFunDeclarationRegex = new( + @"(?:^|;)\s*fun\s+(?:[A-Za-z_]\w*\.)?(?[A-Za-z_]\w*[?!]?)\s*\(", + RegexOptions.Compiled | RegexOptions.CultureInvariant); private static readonly Regex TclCommandRegex = new( @"(?:^|[;\[])\s*(?[A-Za-z_:][\w:.-]*)", RegexOptions.Compiled | RegexOptions.CultureInvariant); @@ -220,10 +223,12 @@ private sealed class PrologLexicalFrame( private sealed class TclLexicalFrame( TclLexicalFrameKind kind, - char terminator = '\0') + char terminator = '\0', + TclLexicalFrame? concatenationOwner = null) { public TclLexicalFrameKind Kind { get; } = kind; public char Terminator { get; } = terminator; + public TclLexicalFrame? ConcatenationOwner { get; } = concatenationOwner; public bool CommandStart { get; set; } = true; public bool WordStart { get; set; } = true; public int WordIndex { get; set; } @@ -235,6 +240,25 @@ private sealed class TclLexicalFrame( public bool SwitchOptionValuePending { get; set; } public int TryClauseWordIndex { get; set; } = 2; public int TryScriptWordIndex { get; set; } = 1; + public int UplevelScriptWordIndex { get; set; } = -1; + public TclLexicalFrame? ConcatenatedScriptState { get; set; } + + public void CopyCommandStateFrom(TclLexicalFrame source) + { + CommandStart = source.CommandStart; + WordStart = source.WordStart; + WordIndex = source.WordIndex; + CommandName = source.CommandName; + LastBareWord = source.LastBareWord; + LastBareWordIndex = source.LastBareWordIndex; + SwitchStringWordIndex = source.SwitchStringWordIndex; + SwitchOptionsEnded = source.SwitchOptionsEnded; + SwitchOptionValuePending = source.SwitchOptionValuePending; + TryClauseWordIndex = source.TryClauseWordIndex; + TryScriptWordIndex = source.TryScriptWordIndex; + UplevelScriptWordIndex = source.UplevelScriptWordIndex; + ConcatenatedScriptState = source.ConcatenatedScriptState; + } public void ResetCommand() { @@ -249,6 +273,8 @@ public void ResetCommand() SwitchOptionValuePending = false; TryClauseWordIndex = 2; TryScriptWordIndex = 1; + UplevelScriptWordIndex = -1; + ConcatenatedScriptState = null; } } @@ -793,7 +819,10 @@ public static bool ShouldSuppressGenericCall( if (state?.IsDeclarationAt(lineNumber, callIndex, name) == true) return true; if (language == "crystal") - return MatchesDeclarationAt(CrystalMethodDeclarationRegex, preparedLine, name, callIndex); + { + return MatchesDeclarationAt(CrystalMethodDeclarationRegex, preparedLine, name, callIndex) + || MatchesDeclarationAt(CrystalFunDeclarationRegex, preparedLine, name, callIndex); + } if (language != "groovy") return false; if (name is "super" or "synchronized" or "this") @@ -1754,6 +1783,7 @@ private static string[] BuildTclCallLines( if (frame.Terminator != '\0' && ch == frame.Terminator) { + PersistTclConcatenatedScriptState(frame); frames.Pop(); buffer[column] = frame.Terminator == '}' ? ' ' : ch; column++; @@ -1788,21 +1818,31 @@ private static string[] BuildTclCallLines( if (ch == '"') { var isScriptArgument = false; + var isConcatenatedScriptArgument = false; if (frame.WordStart) { var wordIndex = frame.WordIndex++; - isScriptArgument = IsTclScriptArgument( + isConcatenatedScriptArgument = IsTclConcatenatedScriptArgument( frame, wordIndex, - lines, - braceEnd: null); + GetTclQuotedWordToken(line, column)); + isScriptArgument = isConcatenatedScriptArgument + || IsTclScriptArgument( + frame, + wordIndex, + lines, + braceEnd: null); UpdateTclSwitchArgumentState(frame, wordIndex, string.Empty); UpdateTclTryArgumentState(frame, wordIndex, string.Empty, isScriptArgument); } - buffer[column] = isScriptArgument ? ';' : ' '; - frames.Push(new TclLexicalFrame( - isScriptArgument ? TclLexicalFrameKind.Script : TclLexicalFrameKind.Quote, - '"')); + var quotedFrame = isScriptArgument + ? CreateTclScriptFrame(frame, '"', isConcatenatedScriptArgument) + : new TclLexicalFrame(TclLexicalFrameKind.Quote, '"'); + buffer[column] = isScriptArgument + && (!isConcatenatedScriptArgument || quotedFrame.CommandStart) + ? ';' + : ' '; + frames.Push(quotedFrame); frame.CommandStart = false; frame.WordStart = false; if (isScriptArgument) @@ -1841,8 +1881,14 @@ private static string[] BuildTclCallLines( wordIndex, lines, braceEnd); + var isConcatenatedScriptArgument = !isSwitchTable + && IsTclConcatenatedScriptArgument( + frame, + wordIndex, + GetTclBracedWordToken(lines, lineIndex, column, braceEnd)); var isScriptArgument = !isSwitchTable - && (scriptBodyOpenings.Contains(positionKey) + && (isConcatenatedScriptArgument + || scriptBodyOpenings.Contains(positionKey) || IsTclScriptArgument( frame, wordIndex, @@ -1861,8 +1907,14 @@ private static string[] BuildTclCallLines( } else if (isScriptArgument) { - buffer[column] = ';'; - frames.Push(new TclLexicalFrame(TclLexicalFrameKind.Script, '}')); + var scriptFrame = CreateTclScriptFrame( + frame, + '}', + isConcatenatedScriptArgument); + buffer[column] = !isConcatenatedScriptArgument || scriptFrame.CommandStart + ? ';' + : ' '; + frames.Push(scriptFrame); suppressLeadingContinuedWord = false; } else @@ -1893,8 +1945,16 @@ private static string[] BuildTclCallLines( { var wordIndex = frame.WordIndex++; var token = ReadTclBareWord(line, column); + var isConcatenatedScriptArgument = token.Length > 0 + && IsTclConcatenatedScriptArgument(frame, wordIndex, token); var isScriptCommand = token.Length > 0 - && IsTclBareScriptCommandArgument(frame, wordIndex); + && (isConcatenatedScriptArgument + ? ProcessTclConcatenatedBareWord(frame, token) + : IsTclBareScriptCommandArgument( + frame, + wordIndex, + lines, + new TclBraceEnd(lineIndex, column + token.Length - 1))); if (wordIndex == 0) frame.CommandName = token; else @@ -1982,7 +2042,7 @@ private static string ReadTclBareWord(string line, int startColumn) var endColumn = startColumn; while (endColumn < line.Length && (char.IsLetterOrDigit(line[endColumn]) - || line[endColumn] is '_' or ':' or '.' or '-')) + || line[endColumn] is '_' or ':' or '.' or '-' or '#')) { endColumn++; } @@ -2016,9 +2076,7 @@ private static bool IsTclScriptArgument( "catch" => wordIndex == 1, "for" => wordIndex is 1 or 3 or 4, "proc" => wordIndex == 3, - "eval" => wordIndex >= 1, "after" => wordIndex == 2, - "uplevel" => wordIndex is 1 or 2, "try" => wordIndex == frame.TryScriptWordIndex, "namespace" => wordIndex == 3, "dict" => frame.LastBareWord == "for" @@ -2032,18 +2090,143 @@ private static bool IsTclScriptArgument( private static bool IsTclBareScriptCommandArgument( TclLexicalFrame frame, - int wordIndex) + int wordIndex, + IReadOnlyList lines, + TclBraceEnd wordEnd) { - if (frame.CommandName == "eval") - return wordIndex == 1; - if (frame.CommandName == "uplevel") - return wordIndex is 1 or 2; - return IsTclScriptArgument( frame, wordIndex, - Array.Empty(), - braceEnd: null); + lines, + wordEnd); + } + + private static bool IsTclConcatenatedScriptArgument( + TclLexicalFrame frame, + int wordIndex, + string? token) + { + if (frame.CommandName == "eval") + return wordIndex >= 1; + if (frame.CommandName != "uplevel") + return false; + if (frame.UplevelScriptWordIndex >= 0) + return wordIndex >= frame.UplevelScriptWordIndex; + if (wordIndex < 1) + return false; + + if (wordIndex == 1 && IsTclUplevelLevelToken(token)) + { + frame.UplevelScriptWordIndex = 2; + return false; + } + + frame.UplevelScriptWordIndex = wordIndex; + return true; + } + + private static bool IsTclUplevelLevelToken(string? token) + { + if (string.IsNullOrWhiteSpace(token)) + return false; + + var span = token.AsSpan().Trim(); + if (span.Length > 1 && span[0] == '#') + span = span[1..]; + if (span.IsEmpty) + return false; + + var start = span[0] is '+' or '-' ? 1 : 0; + if (start == span.Length) + return false; + for (var index = start; index < span.Length; index++) + { + if (!char.IsDigit(span[index])) + return false; + } + + return true; + } + + private static TclLexicalFrame CreateTclScriptFrame( + TclLexicalFrame owner, + char terminator, + bool concatenateArguments) + { + var frame = new TclLexicalFrame( + TclLexicalFrameKind.Script, + terminator, + concatenateArguments ? owner : null); + if (!concatenateArguments) + return frame; + + if (owner.ConcatenatedScriptState != null) + frame.CopyCommandStateFrom(owner.ConcatenatedScriptState); + // Tcl inserts a separating space while concatenating eval/uplevel arguments. + // eval/uplevel の引数連結では引数間に空白が入るため、次は word boundary。 + frame.WordStart = true; + return frame; + } + + private static void PersistTclConcatenatedScriptState(TclLexicalFrame frame) + { + if (frame.ConcatenationOwner is not { } owner) + return; + + owner.ConcatenatedScriptState ??= new TclLexicalFrame(TclLexicalFrameKind.Script); + owner.ConcatenatedScriptState.CopyCommandStateFrom(frame); + owner.ConcatenatedScriptState.WordStart = true; + } + + private static bool ProcessTclConcatenatedBareWord( + TclLexicalFrame owner, + string token) + { + owner.ConcatenatedScriptState ??= new TclLexicalFrame(TclLexicalFrameKind.Script); + var state = owner.ConcatenatedScriptState; + var isCommand = state.CommandStart; + var wordIndex = state.WordIndex++; + if (wordIndex == 0) + { + state.CommandName = token; + } + else + { + UpdateTclSwitchArgumentState(state, wordIndex, token); + UpdateTclTryArgumentState( + state, + wordIndex, + token, + isScriptArgument: false); + } + + state.LastBareWord = token; + state.LastBareWordIndex = wordIndex; + state.CommandStart = false; + state.WordStart = false; + return isCommand; + } + + private static string? GetTclBracedWordToken( + IReadOnlyList lines, + int startLine, + int startColumn, + TclBraceEnd? braceEnd) + { + if (braceEnd is not { } end || end.Line != startLine) + return null; + var length = end.Column - startColumn - 1; + return length < 0 ? null : lines[startLine].Substring(startColumn + 1, length); + } + + private static string? GetTclQuotedWordToken(string line, int startColumn) + { + var endColumn = SkipQuotedToken(line, startColumn, '"'); + return endColumn <= startColumn + 1 + || endColumn > line.Length + || line[endColumn - 1] != '"' + ? null + : line.Substring(startColumn + 1, endColumn - startColumn - 2); } private static void MarkTclBareScriptCommandBoundary(char[] buffer, int commandColumn) diff --git a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.cs b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.cs index e197a82c8..57d520405 100644 --- a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.cs +++ b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.cs @@ -93,6 +93,9 @@ public static int GetContractVersion(string? lang) internal static bool RequiresExplicitReferenceGraphContractStamp(string? lang) => lang is "crystal" or "groovy" or "tcl" or "prolog" or "ambiguous_pl"; + internal static int GetReferenceGraphContractVersion(string lang) => + GetContractVersion(lang); + private static IReadOnlyList BuildEnumDeclarationSnapshot(IReadOnlyList symbols, long? fileId = null) { List<(SymbolRecord Symbol, int OriginalIndex)>? candidates = null; @@ -1633,7 +1636,7 @@ private enum JavaScriptTypeScriptFunctionHeaderConsumeResult new("interface", new Regex(@"^\s*(?:(?:public|private|protected|static|abstract)\s+)*(?:interface|trait)\s+(?[A-Za-z_]\w*)", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.Brace), new("enum", new Regex(@"^\s*(?:(?:public|private|protected|static)\s+)*enum\s+(?[A-Za-z_]\w*)", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.Brace), new("class", new Regex(@"^\s*(?:(?:public|private|protected|static|abstract|final)\s+)*class\s+(?[A-Za-z_]\w*)", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.Brace), - new("function", new Regex(@"^\s*(?:@[A-Za-z_$][\w.$]*(?:\s*\([^)\r\n]*\))?\s+)*(?!(?:if|for|while|switch|catch|return|throw|new)\b)(?:(?:public|private|protected|static|final|abstract|synchronized|native|strictfp)\s+)*(?def|void|boolean|byte|char|short|int|long|float|double|BigDecimal|BigInteger|String|[A-Za-z_$][\w.$]*(?:\s*<[^(){}\r\n]+>)?(?:\s*\[\])*)\s+(?[A-Za-z_]\w*)\s*\(", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.Brace, ReturnTypeGroup: "returnType"), + new("function", new Regex(@"^\s*(?:@[A-Za-z_$][\w.$]*(?:\s*\([^)\r\n]*\))?\s+)*(?!(?:if|for|while|switch|catch|return|throw|new)\b)(?:(?:public|private|protected|static|final|abstract|synchronized|native|strictfp)\s+)*(?:<[^(){}\r\n]+>\s+)?(?def|void|boolean|byte|char|short|int|long|float|double|BigDecimal|BigInteger|String|[A-Za-z_$][\w.$]*(?:\s*<[^(){}\r\n]+>)?(?:\s*\[\])*)\s+(?[A-Za-z_]\w*)\s*\(", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.Brace, ReturnTypeGroup: "returnType"), new("lambda", new Regex(@"^\s*(?:def\s+)?(?[A-Za-z_]\w*)\s*=\s*\{", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.Brace), new("import", new Regex(@"^\s*import\s+(?:static\s+)?(?[A-Za-z_]\w*(?:\.[A-Za-z_]\w*)*(?:\.\*)?)", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None), ], diff --git a/src/CodeIndex/Mcp/McpToolHandlers.Indexing.cs b/src/CodeIndex/Mcp/McpToolHandlers.Indexing.cs index 2648e66f6..c6a100ede 100644 --- a/src/CodeIndex/Mcp/McpToolHandlers.Indexing.cs +++ b/src/CodeIndex/Mcp/McpToolHandlers.Indexing.cs @@ -2102,6 +2102,7 @@ await EmitProgressNotificationAsync( // 成功した refresh で再生成した言語は、独立した fold-key 契約が stale の // ままでも extractor version を stamp できる。 writer.StampSymbolExtractorVersions(indexedSymbolExtractorLanguages); + writer.StampDynamicReferenceGraphContracts(indexedSymbolExtractorLanguages); // FoldReady must reflect reality (#86). Like CLI full-scan, MCP index_project skips // unchanged files via GetUnchangedFileId, so a legacy DB's pre-#86 rows keep NULL // name_folded / *_folded. Stamp only when every row is backfilled; otherwise readers diff --git a/tests/CodeIndex.Tests/DatabaseTests.cs b/tests/CodeIndex.Tests/DatabaseTests.cs index 24c4ffccc..1c95dde70 100644 --- a/tests/CodeIndex.Tests/DatabaseTests.cs +++ b/tests/CodeIndex.Tests/DatabaseTests.cs @@ -4715,6 +4715,9 @@ public void GetUnchangedFileId_InvalidatesPreGraphLanguageContracts_Issue4746( _writer.SetMeta( DbContext.GetSymbolExtractorVersionMetaKey(language), previousContractVersion.ToString(System.Globalization.CultureInfo.InvariantCulture)); + _writer.SetMeta( + DbContext.GetDynamicReferenceGraphContractVersionMetaKey(language), + previousContractVersion.ToString(System.Globalization.CultureInfo.InvariantCulture)); Assert.True(SymbolExtractor.GetContractVersion(language) > previousContractVersion); Assert.Null(_writer.GetUnchangedFileId(file.Path, modified, language: language)); @@ -4739,6 +4742,9 @@ public void GetUnchangedFileId_InvalidatesMissingGraphLanguageContracts_Issue474 Modified = modified, }; _writer.UpsertFile(file); + _writer.SetMeta( + DbContext.GetSymbolExtractorVersionMetaKey(language), + SymbolExtractor.GetContractVersion(language).ToString(CultureInfo.InvariantCulture)); Assert.Null(_writer.GetUnchangedFileId(file.Path, modified, language: language)); } @@ -4761,7 +4767,7 @@ public void GetStatus_DegradesPreGraphLanguageContractsUntilRefresh_Issue4746( Lines = 5, Modified = new DateTime(2025, 1, 1, 0, 0, 0, DateTimeKind.Utc), }); - var versionKey = DbContext.GetSymbolExtractorVersionMetaKey(language); + var versionKey = DbContext.GetDynamicReferenceGraphContractVersionMetaKey(language); _writer.SetMeta( versionKey, previousContractVersion.ToString(CultureInfo.InvariantCulture)); @@ -4797,6 +4803,59 @@ public void GetStatus_DegradesPreGraphLanguageContractsUntilRefresh_Issue4746( refreshedWorkspaceHealth.GraphDataCurrent); } + [Fact] + public void MarkFoldReady_DoesNotCertifyStaleDynamicReferenceGraph_Issue4746() + { + const string language = "crystal"; + var fileId = _writer.UpsertFile(new FileRecord + { + Path = "src/fold-only.cr", + Lang = language, + Size = 20, + Lines = 2, + Modified = new DateTime(2025, 1, 1, 0, 0, 0, DateTimeKind.Utc), + }); + _writer.InsertSymbols( + [ + new SymbolRecord + { + FileId = fileId, + Kind = "function", + Name = "helper", + StartLine = 1, + EndLine = 2, + }, + ]); + var graphVersionKey = DbContext.GetDynamicReferenceGraphContractVersionMetaKey(language); + _writer.SetMeta(graphVersionKey, "2"); + + Assert.True(_writer.MarkFoldReady()); + + using var metadataCommand = _db.Connection.CreateCommand(); + metadataCommand.CommandText = """ + SELECT key, value + FROM codeindex_meta + WHERE key IN ( + 'symbol_extractor_version_crystal', + 'dynamic_reference_graph_contract_version_crystal') + """; + var metadata = new Dictionary(StringComparer.Ordinal); + using (var metadataReader = metadataCommand.ExecuteReader()) + { + while (metadataReader.Read()) + metadata[metadataReader.GetString(0)] = metadataReader.GetString(1); + } + Assert.Equal( + SymbolExtractor.GetContractVersion(language).ToString(CultureInfo.InvariantCulture), + metadata[DbContext.GetSymbolExtractorVersionMetaKey(language)]); + Assert.Equal("2", metadata[graphVersionKey]); + var status = new DbReader(_db.Connection).GetStatus(); + Assert.False(status.ReferenceGraphComplete); + Assert.Contains( + DbReader.DynamicReferenceGraphContractStaleReason, + status.ReferenceGraphIncompleteReasons ?? []); + } + [Fact] public void GetUnchangedFileId_MatchesByChecksumWhenTimestampDiffers() { diff --git a/tests/CodeIndex.Tests/IndexCommandRunnerFullScanTests.cs b/tests/CodeIndex.Tests/IndexCommandRunnerFullScanTests.cs index 9d34dcef0..968921441 100644 --- a/tests/CodeIndex.Tests/IndexCommandRunnerFullScanTests.cs +++ b/tests/CodeIndex.Tests/IndexCommandRunnerFullScanTests.cs @@ -4030,7 +4030,7 @@ proc run {} { helper } using var cmd = conn.CreateCommand(); cmd.CommandText = $""" DELETE FROM codeindex_meta - WHERE key = '{DbContext.GetSymbolExtractorVersionMetaKey("tcl")}'; + WHERE key = '{DbContext.GetDynamicReferenceGraphContractVersionMetaKey("tcl")}'; UPDATE codeindex_meta SET value = '0' WHERE key = 'fold_key_version'; @@ -4048,7 +4048,7 @@ UPDATE codeindex_meta verify.Open(); using var versionCmd = verify.CreateCommand(); versionCmd.CommandText = - $"SELECT value FROM codeindex_meta WHERE key = '{DbContext.GetSymbolExtractorVersionMetaKey("tcl")}'"; + $"SELECT value FROM codeindex_meta WHERE key = '{DbContext.GetDynamicReferenceGraphContractVersionMetaKey("tcl")}'"; Assert.Equal( SymbolExtractor.DynamicReferenceGraphContractVersion.ToString( System.Globalization.CultureInfo.InvariantCulture), @@ -4100,7 +4100,7 @@ def helper verify.Open(); using var versionCmd = verify.CreateCommand(); versionCmd.CommandText = - $"SELECT value FROM codeindex_meta WHERE key = '{DbContext.GetSymbolExtractorVersionMetaKey("crystal")}'"; + $"SELECT value FROM codeindex_meta WHERE key = '{DbContext.GetDynamicReferenceGraphContractVersionMetaKey("crystal")}'"; Assert.Equal( SymbolExtractor.DynamicReferenceGraphContractVersion.ToString( System.Globalization.CultureInfo.InvariantCulture), diff --git a/tests/CodeIndex.Tests/McpServerToolsCallTests.cs b/tests/CodeIndex.Tests/McpServerToolsCallTests.cs index 568dbb8b7..751cc614d 100644 --- a/tests/CodeIndex.Tests/McpServerToolsCallTests.cs +++ b/tests/CodeIndex.Tests/McpServerToolsCallTests.cs @@ -11722,7 +11722,7 @@ def helper verify.Open(); using var versionCmd = verify.CreateCommand(); versionCmd.CommandText = - $"SELECT value FROM codeindex_meta WHERE key = '{DbContext.GetSymbolExtractorVersionMetaKey("crystal")}'"; + $"SELECT value FROM codeindex_meta WHERE key = '{DbContext.GetDynamicReferenceGraphContractVersionMetaKey("crystal")}'"; Assert.Equal( SymbolExtractor.DynamicReferenceGraphContractVersion.ToString( System.Globalization.CultureInfo.InvariantCulture), diff --git a/tests/CodeIndex.Tests/ReferenceExtractorTests.cs b/tests/CodeIndex.Tests/ReferenceExtractorTests.cs index 8e1c1a597..43d1b166c 100644 --- a/tests/CodeIndex.Tests/ReferenceExtractorTests.cs +++ b/tests/CodeIndex.Tests/ReferenceExtractorTests.cs @@ -1076,6 +1076,10 @@ class Inline; def inline_helper(); end; end abstract class Base abstract def declared(value) end + + lib LibC + fun puts(value : UInt8*) : Int32 + end """; var (symbols, references) = ExtractSymbolsAndReferences("crystal", content); @@ -1100,7 +1104,7 @@ abstract def declared(value) stringCall.Column); Assert.DoesNotContain(references, reference => reference.ReferenceKind == "call" - && reference.SymbolName is "require" or "def" or "inline_helper" or "declared"); + && reference.SymbolName is "require" or "def" or "inline_helper" or "declared" or "puts"); } [Fact] @@ -1128,6 +1132,10 @@ Map convert(String value) { [:] } + public T genericConvert(T value) { + value + } + def run(value) { helper value helper(value) @@ -1135,6 +1143,7 @@ helper value [1].each { helper it } annotated(value) convert("value") + genericConvert(value) helper = 1 synchronized(value) {} } @@ -1148,6 +1157,7 @@ class InlineRunner { InlineRunner() {}; def inlineHelper() {} } Assert.Contains(symbols, symbol => symbol.Kind == "function" && symbol.Name == "run"); Assert.Contains(symbols, symbol => symbol.Kind == "function" && symbol.Name == "annotated"); Assert.Contains(symbols, symbol => symbol.Kind == "function" && symbol.Name == "convert"); + Assert.Contains(symbols, symbol => symbol.Kind == "function" && symbol.Name == "genericConvert"); AssertReferencesContain(references, "type_reference", null, "Support"); AssertReferencesContain(references, "call", "run", "helper"); Assert.Equal(4, references.Count(reference => @@ -1162,6 +1172,10 @@ class InlineRunner { InlineRunner() {}; def inlineHelper() {} } reference.ReferenceKind == "call" && reference.ContainerName == "run" && reference.SymbolName == "convert"); + Assert.Single(references, reference => + reference.ReferenceKind == "call" + && reference.ContainerName == "run" + && reference.SymbolName == "genericConvert"); var stringCallLine = Array.FindIndex( content.Split('\n'), line => line.Contains("text =", StringComparison.Ordinal)) + 1; @@ -1178,6 +1192,39 @@ class InlineRunner { InlineRunner() {}; def inlineHelper() {} } or "inlineHelper" or "super" or "synchronized"); } + [Fact] + public void Extract_Tcl_PreservesConcatenatedScriptStateAndBareLoopBodies_Issue4746() + { + const string content = """ + proc helper {value} { return $value } + proc arg {} { return 2 } + proc extra {} { return 3 } + proc run {} { + eval {helper} {arg} + eval {helper} {; extra} + uplevel helper arg + uplevel #0 helper arg + foreach value {1 2} helper + lmap value {1 2} helper + } + """; + + var (_, references) = ExtractSymbolsAndReferences("tcl", content); + + Assert.Equal(6, references.Count(reference => + reference.ReferenceKind == "call" + && reference.ContainerName == "run" + && reference.SymbolName == "helper")); + Assert.Single(references, reference => + reference.ReferenceKind == "call" + && reference.ContainerName == "run" + && reference.SymbolName == "extra"); + Assert.DoesNotContain(references, reference => + reference.ReferenceKind == "call" + && reference.ContainerName == "run" + && reference.SymbolName == "arg"); + } + [Fact] public void Extract_Tcl_IndexesPackagesAndProcCommandCallsWithCaller_Issue4746() { From e8e76f6c7a869e4fe8d4a2df7db42412478236d3 Mon Sep 17 00:00:00 2001 From: Widthdom Date: Fri, 24 Jul 2026 20:51:57 +0900 Subject: [PATCH 17/26] Resolve dynamic graph review regressions (#4746) --- src/CodeIndex/Database/DbWriter.FileReuse.cs | 10 ++-- .../Database/DbWriter.FoldBackfill.cs | 11 +++- .../DynamicDeclarativeReferenceExtractor.cs | 18 +++++- .../Symbols/SymbolExtractor.ExtractCore.cs | 28 ++++++++-- tests/CodeIndex.Tests/DatabaseTests.cs | 2 + .../IndexCommandRunnerFullScanTests.cs | 55 +++++++++++++++++++ .../ReferenceExtractorTests.cs | 22 +++++++- 7 files changed, 132 insertions(+), 14 deletions(-) diff --git a/src/CodeIndex/Database/DbWriter.FileReuse.cs b/src/CodeIndex/Database/DbWriter.FileReuse.cs index 6b69290b2..ab4e33412 100644 --- a/src/CodeIndex/Database/DbWriter.FileReuse.cs +++ b/src/CodeIndex/Database/DbWriter.FileReuse.cs @@ -59,7 +59,7 @@ internal static Action? ReusableStatSnapshotFilterBatchForTesting if (!allowReuse) return null; ReusableUnchangedFileLookupForTesting?.Invoke(relativePath); - if (!SymbolExtractorVersionMatchesCurrent(language)) + if (!ExtractorContractsMatchCurrentForReuse(language)) return null; if (HasStaleIssueMetadata(relativePath)) return null; @@ -130,7 +130,7 @@ ELSE generated { if (!allowReuse) return null; - if (!SymbolExtractorVersionMatchesCurrent(language)) + if (!ExtractorContractsMatchCurrentForReuse(language)) return null; var hasIssueMetadataColumns = HasIssueMetadataColumns(); @@ -260,7 +260,7 @@ FROM file_issues { if (!allowReuse) return null; - if (!SymbolExtractorVersionMatchesCurrent(language)) + if (!ExtractorContractsMatchCurrentForReuse(language)) return null; if (HasStaleIssueMetadata(relativePath)) return null; @@ -304,7 +304,7 @@ FROM files { if (!allowReuse) return null; - if (!SymbolExtractorVersionMatchesCurrent(language)) + if (!ExtractorContractsMatchCurrentForReuse(language)) return null; var hasIssueMetadataColumns = HasIssueMetadataColumns(); @@ -533,7 +533,7 @@ THEN 1 ELSE 0 END AS reusable_eligible var reusableLanguage = language!; if (!currentVersionByLanguage.TryGetValue(reusableLanguage, out var versionCurrent)) { - versionCurrent = SymbolExtractorVersionMatchesCurrent(reusableLanguage); + versionCurrent = ExtractorContractsMatchCurrentForReuse(reusableLanguage); currentVersionByLanguage.Add(reusableLanguage, versionCurrent); } diff --git a/src/CodeIndex/Database/DbWriter.FoldBackfill.cs b/src/CodeIndex/Database/DbWriter.FoldBackfill.cs index 2796c2e7f..faf268948 100644 --- a/src/CodeIndex/Database/DbWriter.FoldBackfill.cs +++ b/src/CodeIndex/Database/DbWriter.FoldBackfill.cs @@ -216,10 +216,17 @@ private bool SymbolExtractorVersionMatchesCurrent(string? lang) var stored = GetMetaString(DbContext.GetSymbolExtractorVersionMetaKey(lang)); if (stored == null) - return !SymbolExtractor.RequiresExplicitReferenceGraphContractStamp(lang); + return true; var current = SymbolExtractor.GetContractVersion(lang).ToString(System.Globalization.CultureInfo.InvariantCulture); - if (stored != current) + return stored == current; + } + + private bool ExtractorContractsMatchCurrentForReuse(string? lang) + { + if (string.IsNullOrWhiteSpace(lang)) + return true; + if (!SymbolExtractorVersionMatchesCurrent(lang)) return false; if (!SymbolExtractor.RequiresExplicitReferenceGraphContractStamp(lang)) return true; diff --git a/src/CodeIndex/Indexer/References/Languages/DynamicDeclarativeReferenceExtractor.cs b/src/CodeIndex/Indexer/References/Languages/DynamicDeclarativeReferenceExtractor.cs index b6014c895..bfaafc33e 100644 --- a/src/CodeIndex/Indexer/References/Languages/DynamicDeclarativeReferenceExtractor.cs +++ b/src/CodeIndex/Indexer/References/Languages/DynamicDeclarativeReferenceExtractor.cs @@ -119,6 +119,9 @@ public bool IsDeclarationAt(int lineNumber, int column, string name) => private static readonly Regex CrystalSuffixedParenthesizedCallRegex = new( @"(?[A-Za-z_]\w*[?!])\s*\(", RegexOptions.Compiled | RegexOptions.CultureInvariant); + private static readonly Regex CrystalControlPredicateCallRegex = new( + @"\b(?:if|unless|while|until)\s+(?[A-Za-z_]\w*[?!]?)(?![\w?!])(?!\s*(?::|(?:<<|>>|[+\-*/%&|^])?=))", + RegexOptions.Compiled | RegexOptions.CultureInvariant); private static readonly Regex GroovyBareCallRegex = new( @"(?:^|[;={])\s*(?:return\s+)?(?[A-Za-z_]\w*)\b(?!\s*(?::|\(|(?:<<|>>|[+\-*/%&|^])?=))", RegexOptions.Compiled | RegexOptions.CultureInvariant); @@ -792,6 +795,13 @@ public static void EmitAdditionalReferences( if (state.CallableNames.Contains(nameGroup.Value)) addCallLikeReference(nameGroup.Value, nameGroup.Index); } + + foreach (Match match in BoundedRegex.EnumerateMatches(CrystalControlPredicateCallRegex, preparedLine)) + { + var nameGroup = match.Groups["name"]; + if (state.CallableNames.Contains(nameGroup.Value)) + addCallLikeReference(nameGroup.Value, nameGroup.Index); + } } foreach (Match match in BoundedRegex.EnumerateMatches(callRegex, preparedLine)) @@ -1107,11 +1117,15 @@ private static string PreparePrologCallScanLine( string line, bool isClauseContinuation) { - if (language is not ("prolog" or "ambiguous_pl") || isClauseContinuation) + if (language is not ("prolog" or "ambiguous_pl")) return line; var masked = line.ToCharArray(); - var clauseStartColumn = 0; + var clauseStartColumn = isClauseContinuation + ? FindPrologClauseTerminator(line, 0) + 1 + : 0; + if (isClauseContinuation && clauseStartColumn == 0) + return line; var changed = false; while (TryFindNextPrologClauseStart(line, clauseStartColumn, out var headStartColumn) && TryFindPrologHeadBoundary( diff --git a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.ExtractCore.cs b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.ExtractCore.cs index dfb43ae52..c35837c90 100644 --- a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.ExtractCore.cs +++ b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.ExtractCore.cs @@ -412,6 +412,16 @@ bool[] GetCssQualifiedRuleAncestors() => matchLine.Length, line.Length)); } + var prologContinuationResumeOffset = -1; + if (prologClauseContinuationLines?[i] == true) + { + var clauseTerminatorColumn = FindFirstTopLevelPrologClauseTerminator(matchLine); + if (clauseTerminatorColumn >= 0) + { + prologContinuationResumeOffset = clauseTerminatorColumn + 1; + patternStartOffset = Math.Max(patternStartOffset, prologContinuationResumeOffset); + } + } while (patternStartOffset >= 0 && patternStartOffset < matchLine.Length) { var stopAfterFirstPatternMatch = false; @@ -419,7 +429,9 @@ bool[] GetCssQualifiedRuleAncestors() => CSharpPropertyMatchCandidate? csharpPropertyCandidateForLine = null; foreach (var pattern in patterns) { - if (prologClauseContinuationLines?[i] == true && pattern.Kind == "function") + if (prologClauseContinuationLines?[i] == true + && prologContinuationResumeOffset < 0 + && pattern.Kind == "function") continue; if (lang == "csharp" && ReferenceEquals(pattern.Regex, CSharpEnumMemberRegex)) continue; @@ -2286,8 +2298,14 @@ private static bool[] BuildPrologClauseContinuationLines( } private static int FindLastTopLevelPrologClauseTerminator(string line) + => FindTopLevelPrologClauseTerminator(line, findLast: true); + + private static int FindFirstTopLevelPrologClauseTerminator(string line) + => FindTopLevelPrologClauseTerminator(line, findLast: false); + + private static int FindTopLevelPrologClauseTerminator(string line, bool findLast) { - var lastTerminatorColumn = -1; + var terminatorColumn = -1; var parenthesisDepth = 0; var bracketDepth = 0; var braceDepth = 0; @@ -2337,11 +2355,13 @@ private static int FindLastTopLevelPrologClauseTerminator(string line) && !(char.IsDigit(previous) && char.IsDigit(next)) && (next == '\0' || char.IsWhiteSpace(next))) { - lastTerminatorColumn = column; + terminatorColumn = column; + if (!findLast) + break; } } - return lastTerminatorColumn; + return terminatorColumn; } private static bool IsValidatedBareMultilinePrologHead( diff --git a/tests/CodeIndex.Tests/DatabaseTests.cs b/tests/CodeIndex.Tests/DatabaseTests.cs index 1c95dde70..44b20efe2 100644 --- a/tests/CodeIndex.Tests/DatabaseTests.cs +++ b/tests/CodeIndex.Tests/DatabaseTests.cs @@ -4849,7 +4849,9 @@ WHERE key IN ( SymbolExtractor.GetContractVersion(language).ToString(CultureInfo.InvariantCulture), metadata[DbContext.GetSymbolExtractorVersionMetaKey(language)]); Assert.Equal("2", metadata[graphVersionKey]); + Assert.True(_writer.SymbolExtractorVersionsMatchCurrent()); var status = new DbReader(_db.Connection).GetStatus(); + Assert.True(status.FoldReady); Assert.False(status.ReferenceGraphComplete); Assert.Contains( DbReader.DynamicReferenceGraphContractStaleReason, diff --git a/tests/CodeIndex.Tests/IndexCommandRunnerFullScanTests.cs b/tests/CodeIndex.Tests/IndexCommandRunnerFullScanTests.cs index 968921441..33102e80c 100644 --- a/tests/CodeIndex.Tests/IndexCommandRunnerFullScanTests.cs +++ b/tests/CodeIndex.Tests/IndexCommandRunnerFullScanTests.cs @@ -4070,6 +4070,61 @@ FROM symbol_references } } + [Fact] + public void Run_ScopedUpdate_KeepsFoldReadyIndependentOfStaleDynamicGraphContract_Issue4746() + { + var projectRoot = CreateTempProject(); + try + { + File.WriteAllText( + Path.Combine(projectRoot, "commands.tcl"), + """ + proc helper {} { return 1 } + proc run {} { helper } + """); + var csharpPath = Path.Combine(projectRoot, "other.cs"); + File.WriteAllText(csharpPath, "class Other { void Run() { } }"); + + Assert.Equal( + CommandExitCodes.Success, + IndexCommandRunner.Run([projectRoot, "--json"], _jsonOptions)); + + var dbPath = Path.Combine(projectRoot, ".cdidx", "codeindex.db"); + using (var conn = OpenNonPoolingConnection(dbPath)) + { + conn.Open(); + using var cmd = conn.CreateCommand(); + cmd.CommandText = $""" + DELETE FROM codeindex_meta + WHERE key = '{DbContext.GetDynamicReferenceGraphContractVersionMetaKey("tcl")}'; + """; + cmd.ExecuteNonQuery(); + } + + File.WriteAllText(csharpPath, "class Other { void Run() { int value = 1; } }"); + File.SetLastWriteTimeUtc(csharpPath, DateTime.UtcNow.AddSeconds(2)); + var (exitCode, json) = RunAndCaptureJson( + [projectRoot, "--files", csharpPath, "--json", "--quiet"]); + + Assert.Equal(CommandExitCodes.Success, exitCode); + Assert.True(json.GetProperty("fold_ready").GetBoolean()); + Assert.False(json.GetProperty("reference_graph_complete").GetBoolean()); + + using var verify = OpenNonPoolingConnection(dbPath); + verify.Open(); + var status = new DbReader(verify).GetStatus(); + Assert.True(status.FoldReady); + Assert.False(status.ReferenceGraphComplete); + Assert.Contains( + DbReader.DynamicReferenceGraphContractStaleReason, + status.ReferenceGraphIncompleteReasons ?? []); + } + finally + { + DeleteDirectory(projectRoot); + } + } + [Fact] public void Run_FullScan_StampsSuppressedDynamicGraphLanguage_Issue4746() { diff --git a/tests/CodeIndex.Tests/ReferenceExtractorTests.cs b/tests/CodeIndex.Tests/ReferenceExtractorTests.cs index 43d1b166c..572cd7aeb 100644 --- a/tests/CodeIndex.Tests/ReferenceExtractorTests.cs +++ b/tests/CodeIndex.Tests/ReferenceExtractorTests.cs @@ -1063,6 +1063,16 @@ def helper(value) def run(value) helper value helper(value) + if helper + helper value + end + helper value unless helper + while helper + break + end + until helper + break + end text = "123456789"; helper(value) multiline = "helper() continued" @@ -1088,7 +1098,7 @@ fun puts(value : UInt8*) : Int32 Assert.Contains(symbols, symbol => symbol.Kind == "function" && symbol.Name == "declared"); AssertReferencesContain(references, "type_reference", null, "support"); AssertReferencesContain(references, "call", "run", "helper"); - Assert.Equal(5, references.Count(reference => + Assert.Equal(11, references.Count(reference => reference.ReferenceKind == "call" && reference.ContainerName == "run" && reference.SymbolName == "helper")); @@ -1380,6 +1390,8 @@ public void Extract_Prolog_IndexesSameLineClausesWithCorrectCallers_Issue4746( b. first. second. run :- a. other :- b. + continued :- + a. continuation_sibling :- b. before_directive. :- dynamic before_directive/0. before_multiline. multiline_after( X @@ -1395,6 +1407,8 @@ before_multiline. multiline_after( "second", "run", "other", + "continued", + "continuation_sibling", "before_directive", "before_multiline", "multiline_after", @@ -1404,6 +1418,8 @@ before_multiline. multiline_after( } AssertReferencesContain(references, "call", "run", "a"); AssertReferencesContain(references, "call", "other", "b"); + AssertReferencesContain(references, "call", "continued", "a"); + AssertReferencesContain(references, "call", "continuation_sibling", "b"); AssertReferencesContain(references, "call", "multiline_after", "a"); Assert.DoesNotContain(references, reference => reference.ReferenceKind == "call" @@ -1413,6 +1429,10 @@ before_multiline. multiline_after( reference.ReferenceKind == "call" && reference.ContainerName == "other" && reference.SymbolName == "a"); + Assert.DoesNotContain(references, reference => + reference.ReferenceKind == "call" + && reference.ContainerName == "continued" + && reference.SymbolName == "b"); } [Fact] From c1bd36ed48c7c55040a0f1b65bc04c6e66eaa030 Mon Sep 17 00:00:00 2001 From: Widthdom Date: Fri, 24 Jul 2026 21:10:00 +0900 Subject: [PATCH 18/26] Handle Tcl expressions and Crystal regexes (#4746) --- .../DynamicDeclarativeReferenceExtractor.cs | 59 ++++++++++++++++++- .../ReferenceExtractorTests.cs | 24 +++++++- 2 files changed, 81 insertions(+), 2 deletions(-) diff --git a/src/CodeIndex/Indexer/References/Languages/DynamicDeclarativeReferenceExtractor.cs b/src/CodeIndex/Indexer/References/Languages/DynamicDeclarativeReferenceExtractor.cs index bfaafc33e..b924ce997 100644 --- a/src/CodeIndex/Indexer/References/Languages/DynamicDeclarativeReferenceExtractor.cs +++ b/src/CodeIndex/Indexer/References/Languages/DynamicDeclarativeReferenceExtractor.cs @@ -174,6 +174,7 @@ private enum TclLexicalFrameKind Script, Quote, BracedWord, + ExpressionWord, SwitchTable, } @@ -1071,7 +1072,12 @@ private static bool IsLikelySlashyLiteralStart(string line, int column) || token.SequenceEqual("throw") || token.SequenceEqual("assert") || token.SequenceEqual("in") - || token.SequenceEqual("when"); + || token.SequenceEqual("when") + || token.SequenceEqual("if") + || token.SequenceEqual("elsif") + || token.SequenceEqual("unless") + || token.SequenceEqual("while") + || token.SequenceEqual("until"); } private static bool TryBeginCrystalPercentLiteral( @@ -1768,6 +1774,38 @@ private static string[] BuildTclCallLines( continue; } + if (frame.Kind == TclLexicalFrameKind.ExpressionWord) + { + buffer[column] = ' '; + if (ch == '\\' && column + 1 < line.Length) + { + buffer[column + 1] = ' '; + column += 2; + } + else if (ch == '{') + { + frames.Push(new TclLexicalFrame(TclLexicalFrameKind.ExpressionWord, '}')); + column++; + } + else if (ch == frame.Terminator) + { + frames.Pop(); + column++; + } + else if (ch == '[') + { + buffer[column] = '['; + frames.Push(new TclLexicalFrame(TclLexicalFrameKind.Script, ']')); + suppressLeadingContinuedWord = false; + column++; + } + else + { + column++; + } + continue; + } + if (frame.Kind == TclLexicalFrameKind.Quote) { buffer[column] = ' '; @@ -1900,6 +1938,8 @@ private static string[] BuildTclCallLines( frame, wordIndex, GetTclBracedWordToken(lines, lineIndex, column, braceEnd)); + var isExpressionArgument = !isSwitchTable + && IsTclExpressionArgument(frame, wordIndex); var isScriptArgument = !isSwitchTable && (isConcatenatedScriptArgument || scriptBodyOpenings.Contains(positionKey) @@ -1931,6 +1971,11 @@ private static string[] BuildTclCallLines( frames.Push(scriptFrame); suppressLeadingContinuedWord = false; } + else if (isExpressionArgument) + { + buffer[column] = ' '; + frames.Push(new TclLexicalFrame(TclLexicalFrameKind.ExpressionWord, '}')); + } else { buffer[column] = ' '; @@ -2102,6 +2147,18 @@ private static bool IsTclScriptArgument( }; } + private static bool IsTclExpressionArgument(TclLexicalFrame frame, int wordIndex) => + frame.CommandName switch + { + "if" => wordIndex == 1 + || (frame.LastBareWord == "elseif" + && wordIndex == frame.LastBareWordIndex + 1), + "while" => wordIndex == 1, + "for" => wordIndex == 2, + "expr" => wordIndex >= 1, + _ => false, + }; + private static bool IsTclBareScriptCommandArgument( TclLexicalFrame frame, int wordIndex, diff --git a/tests/CodeIndex.Tests/ReferenceExtractorTests.cs b/tests/CodeIndex.Tests/ReferenceExtractorTests.cs index 572cd7aeb..7ceac89ab 100644 --- a/tests/CodeIndex.Tests/ReferenceExtractorTests.cs +++ b/tests/CodeIndex.Tests/ReferenceExtractorTests.cs @@ -1476,6 +1476,18 @@ def run(value) def fake TEXT regex = /ready?(value)/ + if /ready?()/.matches?(value) + value + end + unless /save!()/.matches?(value) + value + end + while /ready?()/.matches?(value) + break + end + until /save!()/.matches?(value) + break + end literal = %q(save!(value)) ordinary = "first ready?(value) @@ -1560,6 +1572,16 @@ proc multiline {} { set color #fff; helper set quoted "[helper]" set braced {[helper]} + if {[helper]} { + set expression_if 1 + } + while {[helper]} { + break + } + for {} {[helper]} {} { + break + } + expr {[helper]} if {1} { helper } @@ -1613,7 +1635,7 @@ proc continued {} \ AssertReferencesContain(references, "call", "split", "helper"); AssertReferencesContain(references, "call", "continued", "helper"); AssertReferencesContain(references, "call", "sameLine", "helper"); - Assert.Equal(20, references.Count(reference => + Assert.Equal(24, references.Count(reference => reference.ReferenceKind == "call" && reference.SymbolName == "helper")); Assert.Single(references, reference => From cc697c12f6755bfeb76587c3b644d522716d0158 Mon Sep 17 00:00:00 2001 From: Widthdom Date: Fri, 24 Jul 2026 21:32:12 +0900 Subject: [PATCH 19/26] Resolve dynamic graph contract review findings (#4746) --- changelog.d/unreleased/4746.added.md | 3 + .../Cli/IndexCommandRunner.Update.cs | 46 +++++++++++++ src/CodeIndex/Database/DbReader.Status.cs | 2 +- src/CodeIndex/Database/DbReader.cs | 9 +++ .../Indexer/References/ReferenceExtractor.cs | 2 + .../Indexer/Symbols/SymbolExtractor.cs | 8 ++- tests/CodeIndex.Tests/DbReaderTests.cs | 24 +++++++ .../IndexCommandRunnerFullScanTests.cs | 65 +++++++++++++++++++ .../ReferenceExtractorTests.cs | 19 ++++++ 9 files changed, 176 insertions(+), 2 deletions(-) diff --git a/changelog.d/unreleased/4746.added.md b/changelog.d/unreleased/4746.added.md index 92890975e..a0c7f8d5e 100644 --- a/changelog.d/unreleased/4746.added.md +++ b/changelog.d/unreleased/4746.added.md @@ -11,14 +11,17 @@ affected: - src/CodeIndex/Indexer/References/ReferenceExtractor.Preparation.cs - src/CodeIndex/Indexer/References/ReferenceExtractor.TypeReferences.cs - src/CodeIndex/Indexer/References/Languages/DynamicDeclarativeReferenceExtractor.cs + - src/CodeIndex/Database/DbReader.cs - src/CodeIndex/Database/DbReader.Status.cs - src/CodeIndex/Database/DbWriter.FoldBackfill.cs - src/CodeIndex/Database/DegradationReasonCodes.cs - src/CodeIndex/Cli/IndexCommandRunner.FullScan.cs + - src/CodeIndex/Cli/IndexCommandRunner.Update.cs - src/CodeIndex/Cli/QueryCommandRunner.Status.cs - src/CodeIndex/Cli/QueryCommandRunner.StatusFields.cs - src/CodeIndex/Mcp/McpToolHandlers.Indexing.cs - tests/CodeIndex.Tests/DatabaseTests.cs + - tests/CodeIndex.Tests/DbReaderTests.cs - tests/CodeIndex.Tests/IndexCommandRunnerFullScanTests.cs - tests/CodeIndex.Tests/SymbolExtractorTests.cs - tests/CodeIndex.Tests/ReferenceExtractorTests.cs diff --git a/src/CodeIndex/Cli/IndexCommandRunner.Update.cs b/src/CodeIndex/Cli/IndexCommandRunner.Update.cs index d63971bf4..c0560d4c4 100644 --- a/src/CodeIndex/Cli/IndexCommandRunner.Update.cs +++ b/src/CodeIndex/Cli/IndexCommandRunner.Update.cs @@ -208,6 +208,34 @@ private static int RunUpdateMode( var priorMetadataTargetCsharpMatchesCurrent = priorMetadataTargetCsharp == currentMetadataTargetVersion; var csharpMetadataTargetsNeedRefresh = !priorMetadataTargetCsharpMatchesCurrent; var symbolsDroppedByKindFilter = 0; + var refreshedDynamicGraphFileCounts = new Dictionary(StringComparer.Ordinal); + + void RecordDynamicGraphFileRefresh(string? language) + { + if (options.SymbolsOnly + || !SymbolExtractor.RequiresExplicitReferenceGraphContractStamp(language)) + { + return; + } + + refreshedDynamicGraphFileCounts.TryGetValue(language!, out var refreshedCount); + refreshedDynamicGraphFileCounts[language!] = refreshedCount + 1; + } + + string[] GetFullyRefreshedDynamicGraphLanguages() + { + if (options.SymbolsOnly || refreshedDynamicGraphFileCounts.Count == 0) + return []; + + using var reader = new DbReader(writer.Connection); + var currentLanguageCounts = reader.GetIndexedLanguageCounts(); + return refreshedDynamicGraphFileCounts + .Where(entry => + currentLanguageCounts.TryGetValue(entry.Key, out var currentCount) + && currentCount == entry.Value) + .Select(entry => entry.Key) + .ToArray(); + } void DemoteReadinessOnce() { @@ -1932,6 +1960,7 @@ FileByteReadSummary MeasureRemainingUpdateReadableFileBytes() writer.ClearBatchInProgress(); txn.Commit(); fileBatchMarked = false; + RecordDynamicGraphFileRefresh(record.Lang); updated++; ftsMutated = true; WriteUpdateVerboseStatus($" [OK ] {relPath} ({chunks.Count} chunks, generated-code extraction skipped)"); @@ -1979,6 +2008,7 @@ FileByteReadSummary MeasureRemainingUpdateReadableFileBytes() writer.ClearBatchInProgress(); txn.Commit(); fileBatchMarked = false; + RecordDynamicGraphFileRefresh(record.Lang); updated++; ftsMutated = true; WriteUpdateVerboseStatus($" [SKIP] {relPath} ({issue.Message})"); @@ -1999,6 +2029,7 @@ FileByteReadSummary MeasureRemainingUpdateReadableFileBytes() writer.ClearBatchInProgress(); txn.Commit(); fileBatchMarked = false; + RecordDynamicGraphFileRefresh(record.Lang); updated++; ftsMutated = true; WriteUpdateVerboseStatus($" [SKIP] {relPath} ({issue.Message})"); @@ -2055,6 +2086,7 @@ FileByteReadSummary MeasureRemainingUpdateReadableFileBytes() writer.ClearBatchInProgress(); txn.Commit(); + RecordDynamicGraphFileRefresh(record.Lang); updated++; ftsMutated = true; if (!options.SymbolsOnly && (symbols.Count > 0 || references.Count > 0)) @@ -2154,6 +2186,7 @@ FileByteReadSummary MeasureRemainingUpdateReadableFileBytes() writer.InsertIssues(fileId, [BuildNullByteIssue(binaryFile)]); writer.ClearBatchInProgress(); txn.Commit(); + RecordDynamicGraphFileRefresh(skippedRecord.Lang); skippedBinaryBatchMarkerOwned = false; } catch (CSharpWorkspaceChangedException workspaceChanged) @@ -2266,6 +2299,7 @@ or IndexInterruptedException ]); writer.ClearBatchInProgress(); txn.Commit(); + RecordDynamicGraphFileRefresh(skippedRecord.Lang); skippedOversizedBatchMarkerOwned = false; } catch (CSharpWorkspaceChangedException workspaceChanged) @@ -2506,6 +2540,9 @@ or IndexInterruptedException (DbContext.LastFailedIndexRunRecoveryHintMetaKey, "Fix the reported file/extractor error, then rerun the same index command. Successful files and graph edges remain persisted; a rebuild is not required."), (DbContext.LastFailedIndexRunFileErrorsMetaKey, JsonSerializer.Serialize(fileErrorList, StatusMetadataJsonContext.Default.ListStatusIndexFileError))); } + var fullyRefreshedDynamicGraphLanguages = errors == 0 && readinessDemoted + ? GetFullyRefreshedDynamicGraphLanguages() + : []; if (readinessDemoted && errors == 0) { writer.MarkBatchInProgress(); @@ -2536,6 +2573,15 @@ or IndexInterruptedException { writer.MarkReferenceIdentityContractReady(); } + // A scoped update can certify a per-language graph contract only when every + // remaining file in that language was regenerated by this run. This covers a + // newly introduced language and an explicitly complete language refresh without + // hiding stale graph rows from untouched files of the same language. + // scoped update では、対象言語の現存ファイルを今回すべて再生成した場合だけ + // graph contract を stamp する。新規言語と全件更新を復元しつつ、未更新の + // stale row を current と誤認しない。 + writer.StampSymbolExtractorVersions(fullyRefreshedDynamicGraphLanguages); + writer.StampDynamicReferenceGraphContracts(fullyRefreshedDynamicGraphLanguages); if ((priorReadiness & DbContext.IssuesReadyFlag) != 0) { writer.MarkIssuesReady(); diff --git a/src/CodeIndex/Database/DbReader.Status.cs b/src/CodeIndex/Database/DbReader.Status.cs index cc9361cdf..fc43e6ad9 100644 --- a/src/CodeIndex/Database/DbReader.Status.cs +++ b/src/CodeIndex/Database/DbReader.Status.cs @@ -302,7 +302,7 @@ internal IReadOnlyList GetReferenceGraphIncompleteReasons( return reasons; } - private Dictionary GetIndexedLanguageCounts() + internal Dictionary GetIndexedLanguageCounts() { var languages = new Dictionary(); using var cmd = _conn.CreateCommand(); diff --git a/src/CodeIndex/Database/DbReader.cs b/src/CodeIndex/Database/DbReader.cs index 3b9d36512..09af2b2c5 100644 --- a/src/CodeIndex/Database/DbReader.cs +++ b/src/CodeIndex/Database/DbReader.cs @@ -726,6 +726,15 @@ private static (bool Newer, string? Reason) DetectNewerThanReaderContracts(Sqlit AppendIfStoredGreater(conn, DbContext.SqlGraphContractVersionMetaKey, DbContext.SqlGraphContractVersion, "sql_graph_contract_version", newerContracts); AppendIfStoredGreater(conn, DbContext.ReferenceIdentityContractVersionMetaKey, DbContext.ReferenceIdentityContractVersion, "reference_identity_contract_version", newerContracts); AppendIfStoredGreater(conn, "fold_key_version", NameFold.Version, "fold_key_version", newerContracts); + foreach (var lang in SymbolExtractor.GetExplicitReferenceGraphContractLanguages()) + { + AppendIfStoredGreater( + conn, + DbContext.GetDynamicReferenceGraphContractVersionMetaKey(lang), + SymbolExtractor.GetReferenceGraphContractVersion(lang), + $"dynamic_reference_graph_contract_version_{lang}", + newerContracts); + } foreach (var lang in FileIndexer.GetHotspotFamilyMarkerLanguages()) AppendIfStoredGreater(conn, DbContext.GetHotspotFamilyVersionMetaKey(lang), DbContext.HotspotFamilyVersion, $"hotspot_family_version_{lang}", newerContracts); diff --git a/src/CodeIndex/Indexer/References/ReferenceExtractor.cs b/src/CodeIndex/Indexer/References/ReferenceExtractor.cs index 44794d988..d35006070 100644 --- a/src/CodeIndex/Indexer/References/ReferenceExtractor.cs +++ b/src/CodeIndex/Indexer/References/ReferenceExtractor.cs @@ -285,6 +285,8 @@ private static bool IsFunctionLikeSymbolKind(string kind) "elsif", "end", "ensure", "enum", "extend", "for", "fun", "if", "include", "lib", "macro", "module", "next", "of", "private", "protected", "require", "rescue", "return", "select", "struct", "then", "unless", "until", "when", "while", "with", "yield", + "as", "alignof", "instance_alignof", "instance_sizeof", "is_a?", "offsetof", "pointerof", + "responds_to?", }, ["groovy"] = new HashSet(StringComparer.Ordinal) { diff --git a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.cs b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.cs index 57d520405..281fdae03 100644 --- a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.cs +++ b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.cs @@ -26,6 +26,8 @@ public static partial class SymbolExtractor public const int DynamicReferenceGraphContractVersion = 3; public const int PrologReferenceGraphContractVersion = 2; public const int SystemsLanguageContractVersion = 2; + private static readonly string[] ExplicitReferenceGraphContractLanguages = + ["crystal", "groovy", "tcl", "prolog", "ambiguous_pl"]; private static readonly string[] AdditionalSymbolLanguages = [ "app_manifest", @@ -90,8 +92,12 @@ public static int GetContractVersion(string? lang) }; } + internal static IReadOnlyList GetExplicitReferenceGraphContractLanguages() => + ExplicitReferenceGraphContractLanguages; + internal static bool RequiresExplicitReferenceGraphContractStamp(string? lang) => - lang is "crystal" or "groovy" or "tcl" or "prolog" or "ambiguous_pl"; + lang != null + && ExplicitReferenceGraphContractLanguages.Contains(lang, StringComparer.Ordinal); internal static int GetReferenceGraphContractVersion(string lang) => GetContractVersion(lang); diff --git a/tests/CodeIndex.Tests/DbReaderTests.cs b/tests/CodeIndex.Tests/DbReaderTests.cs index cde590534..d457e3dd8 100644 --- a/tests/CodeIndex.Tests/DbReaderTests.cs +++ b/tests/CodeIndex.Tests/DbReaderTests.cs @@ -5049,6 +5049,30 @@ public void GetStatus_FlagsIndexNewerThanReaderWhenCSharpMetadataVersionExceedsC Assert.Equal("9.99.0", status.IndexWriterVersion); } + [Theory] + [InlineData("crystal")] + [InlineData("groovy")] + [InlineData("tcl")] + [InlineData("prolog")] + [InlineData("ambiguous_pl")] + public void GetStatus_FlagsIndexNewerThanReaderWhenDynamicGraphVersionExceedsCurrent_Issue4746( + string language) + { + _writer.SetMeta( + DbContext.GetDynamicReferenceGraphContractVersionMetaKey(language), + (SymbolExtractor.GetReferenceGraphContractVersion(language) + 1).ToString( + CultureInfo.InvariantCulture)); + var freshReader = new DbReader(_db.Connection); + + var status = freshReader.GetStatus(); + + Assert.True(status.IndexNewerThanReader); + Assert.NotNull(status.IndexNewerThanReaderReason); + Assert.Contains( + $"dynamic_reference_graph_contract_version_{language}", + status.IndexNewerThanReaderReason); + } + [Fact] public void GetStatus_FlagsIndexNewerThanReaderWhenUserVersionCarriesUnknownReadyBit() { diff --git a/tests/CodeIndex.Tests/IndexCommandRunnerFullScanTests.cs b/tests/CodeIndex.Tests/IndexCommandRunnerFullScanTests.cs index 33102e80c..781aaafe4 100644 --- a/tests/CodeIndex.Tests/IndexCommandRunnerFullScanTests.cs +++ b/tests/CodeIndex.Tests/IndexCommandRunnerFullScanTests.cs @@ -4070,6 +4070,71 @@ FROM symbol_references } } + [Fact] + public void Run_ScopedUpdate_StampsFullyRefreshedDynamicGraphLanguages_Issue4746() + { + var projectRoot = CreateTempProject(); + try + { + File.WriteAllText( + Path.Combine(projectRoot, "initial.cs"), + "class Initial { void Run() { } }"); + Assert.Equal( + CommandExitCodes.Success, + IndexCommandRunner.Run([projectRoot, "--json"], _jsonOptions)); + + var tclPath = Path.Combine(projectRoot, "commands.tcl"); + File.WriteAllText( + tclPath, + """ + proc helper {} { return 1 } + proc run {} { helper } + """); + + var (addedExitCode, addedJson) = RunAndCaptureJson( + [projectRoot, "--files", tclPath, "--json", "--quiet"]); + + Assert.Equal(CommandExitCodes.Success, addedExitCode); + Assert.True(addedJson.GetProperty("reference_graph_complete").GetBoolean()); + Assert.True(addedJson.GetProperty("graph_data_current").GetBoolean()); + + var dbPath = Path.Combine(projectRoot, ".cdidx", "codeindex.db"); + using (var conn = OpenNonPoolingConnection(dbPath)) + { + conn.Open(); + using var cmd = conn.CreateCommand(); + cmd.CommandText = $""" + SELECT value + FROM codeindex_meta + WHERE key = '{DbContext.GetDynamicReferenceGraphContractVersionMetaKey("tcl")}'; + """; + Assert.Equal( + SymbolExtractor.DynamicReferenceGraphContractVersion.ToString( + System.Globalization.CultureInfo.InvariantCulture), + cmd.ExecuteScalar() as string); + + cmd.CommandText = $""" + DELETE FROM codeindex_meta + WHERE key = '{DbContext.GetDynamicReferenceGraphContractVersionMetaKey("tcl")}'; + """; + cmd.ExecuteNonQuery(); + } + + File.AppendAllText(tclPath, "\nproc second {} { helper }\n"); + File.SetLastWriteTimeUtc(tclPath, DateTime.UtcNow.AddSeconds(2)); + var (refreshedExitCode, refreshedJson) = RunAndCaptureJson( + [projectRoot, "--files", tclPath, "--json", "--quiet"]); + + Assert.Equal(CommandExitCodes.Success, refreshedExitCode); + Assert.True(refreshedJson.GetProperty("reference_graph_complete").GetBoolean()); + Assert.True(refreshedJson.GetProperty("graph_data_current").GetBoolean()); + } + finally + { + DeleteDirectory(projectRoot); + } + } + [Fact] public void Run_ScopedUpdate_KeepsFoldReadyIndependentOfStaleDynamicGraphContract_Issue4746() { diff --git a/tests/CodeIndex.Tests/ReferenceExtractorTests.cs b/tests/CodeIndex.Tests/ReferenceExtractorTests.cs index 7ceac89ab..0e04d1d15 100644 --- a/tests/CodeIndex.Tests/ReferenceExtractorTests.cs +++ b/tests/CodeIndex.Tests/ReferenceExtractorTests.cs @@ -1488,6 +1488,15 @@ def fake until /save!()/.matches?(value) break end + cast = value.as(String) + checked_cast = value.as?(String) + pointer = pointerof(value) + instance_size = instance_sizeof(String) + alignment = alignof(String) + instance_alignment = instance_alignof(String) + offset = offsetof(Tuple(Int32), 0) + type = typeof(value) + size = sizeof(String) literal = %q(save!(value)) ordinary = "first ready?(value) @@ -1514,6 +1523,16 @@ fake value reference.ReferenceKind == "call" && reference.ContainerName == "run" && reference.SymbolName == "save!")); + Assert.DoesNotContain(references, reference => + reference.ReferenceKind == "call" + && reference.SymbolName is "as" + or "pointerof" + or "instance_sizeof" + or "alignof" + or "instance_alignof" + or "offsetof" + or "typeof" + or "sizeof"); Assert.DoesNotContain(references, reference => reference.SymbolName == "fake"); } From 3047a3d61fba1f295818a59950bff55539eed14a Mon Sep 17 00:00:00 2001 From: Widthdom Date: Fri, 24 Jul 2026 21:59:34 +0900 Subject: [PATCH 20/26] Resolve final dynamic graph review gaps (#4746) --- changelog.d/unreleased/4746.added.md | 4 +- .../DynamicDeclarativeReferenceExtractor.cs | 440 +++++++++++++++++- .../Symbols/SymbolExtractor.ExtractCore.cs | 8 + .../Indexer/Symbols/SymbolExtractor.cs | 4 +- .../ReferenceExtractorTests.cs | 77 ++- 5 files changed, 523 insertions(+), 10 deletions(-) diff --git a/changelog.d/unreleased/4746.added.md b/changelog.d/unreleased/4746.added.md index a0c7f8d5e..ece8aaffb 100644 --- a/changelog.d/unreleased/4746.added.md +++ b/changelog.d/unreleased/4746.added.md @@ -37,8 +37,8 @@ affected: ## English -- **Crystal, Groovy, Tcl, Prolog, and ambiguous `.pl` files now support conservative reference graphs (#4746)** — indexing records imports and language-appropriate calls at their source locations, recognizes command-style calls only for callables declared in the same file, parses Tcl command substitutions, control-command script arguments, and `switch` branch bodies, limits Prolog calls to goal positions, preserves Tcl proc and Prolog predicate caller containers, and reports structured symbol/reference/graph capabilities after `.pl` content classification. Pre-graph extractor stamps keep graph readiness degraded until a normal index refresh rewrites affected rows. +- **Crystal, Groovy, Tcl, Prolog, and ambiguous `.pl` files now support conservative reference graphs (#4746)** — indexing records imports and language-appropriate calls at their source locations, recognizes command-style calls only for callables declared in the same file, parses Groovy control-statement bodies, Tcl command substitutions, inline proc declarations, control-command script arguments, and `switch` branch bodies, limits Prolog calls to goal positions including top-level directives, masks Perl quote-like regular-expression bodies in ambiguous `.pl` files, preserves Tcl proc and Prolog predicate caller containers, and reports structured symbol/reference/graph capabilities after `.pl` content classification. Pre-graph extractor stamps keep graph readiness degraded until a normal index refresh rewrites affected rows. ## 日本語 -- **Crystal、Groovy、Tcl、Prolog、曖昧な `.pl` ファイルが保守的な reference graph に対応しました（#4746）** — import と言語ごとの call 構文をソース上の位置どおりに記録し、command-style call は同一ファイルで宣言済みの callable だけを認識し、Tcl の command substitution、制御 command の script 引数、`switch` の branch body を解析し、Prolog の call を goal 位置に限定し、Tcl proc / Prolog predicate の caller container を保持して、`.pl` の content classification 後に構造化された symbol / reference / graph capability を報告します。graph 対応前の extractor stamp がある場合、通常の index refresh で対象 row を更新するまで graph readiness を degraded に保ちます。 +- **Crystal、Groovy、Tcl、Prolog、曖昧な `.pl` ファイルが保守的な reference graph に対応しました（#4746）** — import と言語ごとの call 構文をソース上の位置どおりに記録し、command-style call は同一ファイルで宣言済みの callable だけを認識し、Groovy の制御文本体、Tcl の command substitution、inline proc 宣言、制御 command の script 引数、`switch` の branch body を解析し、top-level directive を含む Prolog の call を goal 位置に限定し、曖昧な `.pl` では Perl の quote-like 正規表現本体をマスクし、Tcl proc / Prolog predicate の caller container を保持して、`.pl` の content classification 後に構造化された symbol / reference / graph capability を報告します。graph 対応前の extractor stamp がある場合、通常の index refresh で対象 row を更新するまで graph readiness を degraded に保ちます。 diff --git a/src/CodeIndex/Indexer/References/Languages/DynamicDeclarativeReferenceExtractor.cs b/src/CodeIndex/Indexer/References/Languages/DynamicDeclarativeReferenceExtractor.cs index b924ce997..f7ac0a582 100644 --- a/src/CodeIndex/Indexer/References/Languages/DynamicDeclarativeReferenceExtractor.cs +++ b/src/CodeIndex/Indexer/References/Languages/DynamicDeclarativeReferenceExtractor.cs @@ -93,7 +93,10 @@ public bool IsDeclarationAt(int lineNumber, int column, string name) => } private static readonly Regex TclProcRegex = new( - @"^\s*proc\s+[A-Za-z_:][\w:.-]*\s+", + @"(?[A-Za-z_:][\w:.-]*)\s+", + RegexOptions.Compiled | RegexOptions.CultureInvariant); + private static readonly Regex TclInlineProcRegex = new( + @"(?:^|[;\[])\s*proc\s+(?[A-Za-z_:][\w:.-]*)\s+", RegexOptions.Compiled | RegexOptions.CultureInvariant); private static readonly Regex PrologHeadRegex = new( @"^\s*(?[a-z][A-Za-z0-9_]*)\s*(?:\([^\r\n]*\))?\s*(?::-|-->|\.(?=\s*(?:$|:-|[a-z][A-Za-z0-9_]*\s*\(\s*$|[a-z][A-Za-z0-9_]*(?:\s*\([^)]*\))?\s*(?::-|-->|\.))))", @@ -203,10 +206,23 @@ private sealed class PrologLexicalFrame( GoalArgumentIndices?.Contains(ArgumentIndex) == true; } + private sealed class AmbiguousPerlQuoteLikeState( + char openingDelimiter, + char closingDelimiter, + int remainingSegments) + { + public char OpeningDelimiter { get; set; } = openingDelimiter; + public char ClosingDelimiter { get; set; } = closingDelimiter; + public int RemainingSegments { get; set; } = remainingSegments; + public int DelimiterDepth { get; set; } = openingDelimiter == closingDelimiter ? 0 : 1; + public bool AwaitingNextOpeningDelimiter { get; set; } + } + private static readonly IReadOnlyDictionary> PrologMetaGoalArguments = new Dictionary>(StringComparer.Ordinal) { ["call"] = new HashSet { 0 }, + ["initialization"] = new HashSet { 0 }, ["once"] = new HashSet { 0 }, ["ignore"] = new HashSet { 0 }, ["not"] = new HashSet { 0 }, @@ -292,6 +308,7 @@ public static string[] MaskNonCodeLines(string language, IReadOnlyList l char groovyTripleQuote = '\0'; string? crystalHeredocDelimiter = null; var insideAmbiguousPerlPod = false; + AmbiguousPerlQuoteLikeState? ambiguousPerlQuoteLikeState = null; var insideSlashyLiteral = false; var insideGroovyDollarSlashyLiteral = false; char crystalMultilineQuote = '\0'; @@ -344,6 +361,16 @@ public static string[] MaskNonCodeLines(string language, IReadOnlyList l : -1; for (var column = 0; column < line.Length;) { + if (ambiguousPerlQuoteLikeState != null) + { + MaskAmbiguousPerlQuoteLikeCharacter( + line, + buffer, + ref column, + ref ambiguousPerlQuoteLikeState); + continue; + } + if (crystalMultilineQuote != '\0') { buffer[column] = ' '; @@ -503,6 +530,19 @@ public static string[] MaskNonCodeLines(string language, IReadOnlyList l continue; } + if (language == "ambiguous_pl" + && TryBeginAmbiguousPerlQuoteLikeLiteral( + line, + buffer, + column, + out var perlQuoteLikeState, + out var perlContentColumn)) + { + ambiguousPerlQuoteLikeState = perlQuoteLikeState; + column = perlContentColumn; + continue; + } + if (language == "crystal" && column + 2 < line.Length && line[column] == '<' @@ -652,6 +692,100 @@ public static string[] MaskTclNonScriptLines(IReadOnlyList maskedLines) BuildTclBraceEndPositions(maskedLines), new HashSet()); + public static void AddTclInlineProcSymbols( + long fileId, + IReadOnlyList rawLines, + IReadOnlyList scriptLines, + List symbols) + { + var declarationContainerScopes = BuildTclDeclarationContainerScopes( + rawLines, + symbols); + var existingDeclarations = new HashSet<(int Line, string Name)>( + symbols + .Where(static symbol => symbol.Kind == "function") + .Select(static symbol => (symbol.StartLine, symbol.Name))); + for (var lineIndex = 0; lineIndex < scriptLines.Count; lineIndex++) + { + foreach (Match match in BoundedRegex.EnumerateMatches( + TclInlineProcRegex, + scriptLines[lineIndex])) + { + var nameGroup = match.Groups["name"]; + var lineNumber = lineIndex + 1; + if (!existingDeclarations.Add((lineNumber, nameGroup.Value))) + continue; + + var symbol = new SymbolRecord + { + FileId = fileId, + Kind = "function", + Name = nameGroup.Value, + Line = lineNumber, + StartLine = lineNumber, + StartColumn = nameGroup.Index, + EndLine = lineNumber, + Signature = rawLines[lineIndex].Trim(), + }; + for (var scopeIndex = declarationContainerScopes.Count - 1; + scopeIndex >= 0; + scopeIndex--) + { + var scope = declarationContainerScopes[scopeIndex]; + if (!scope.Contains(lineNumber, nameGroup.Index)) + continue; + symbol.ContainerKind = scope.Symbol.Kind; + symbol.ContainerName = scope.Symbol.Name; + break; + } + symbols.Add(symbol); + } + } + } + + private static IReadOnlyList BuildTclDeclarationContainerScopes( + IReadOnlyList lines, + IReadOnlyList symbols) + { + var braceEnds = BuildTclBraceEndPositions(lines); + var scopes = new List(); + foreach (var symbol in symbols) + { + if (symbol.Kind is not ("namespace" or "class") + || symbol.StartLine < 1 + || symbol.StartLine > lines.Count) + { + continue; + } + + var lineIndex = symbol.StartLine - 1; + var nameColumn = lines[lineIndex].IndexOf( + symbol.Name, + StringComparison.Ordinal); + if (nameColumn < 0) + continue; + var openingColumn = lines[lineIndex].IndexOf( + '{', + nameColumn + symbol.Name.Length); + if (openingColumn < 0 + || !braceEnds.TryGetValue( + GetTclPositionKey(lineIndex, openingColumn), + out var bodyEnd)) + { + continue; + } + + scopes.Add(new TclContainerScope( + symbol, + lineIndex + 1, + openingColumn, + bodyEnd.Line + 1, + bodyEnd.Column)); + } + + return scopes; + } + public static ExtractionState? CreateState( string language, IReadOnlyList preparedLines, @@ -804,6 +938,13 @@ public static void EmitAdditionalReferences( addCallLikeReference(nameGroup.Value, nameGroup.Index); } } + else if (language == "groovy") + { + EmitGroovyControlBodyBareCalls( + preparedLine, + state.CallableNames, + addCallLikeReference); + } foreach (Match match in BoundedRegex.EnumerateMatches(callRegex, preparedLine)) { @@ -816,6 +957,104 @@ public static void EmitAdditionalReferences( } + private static void EmitGroovyControlBodyBareCalls( + string line, + IReadOnlySet callableNames, + Action addCallLikeReference) + { + for (var keywordColumn = 0; keywordColumn < line.Length; keywordColumn++) + { + if (!char.IsLetter(line[keywordColumn])) + continue; + + var keywordEnd = keywordColumn + 1; + while (keywordEnd < line.Length + && (char.IsLetterOrDigit(line[keywordEnd]) || line[keywordEnd] == '_')) + { + keywordEnd++; + } + + var keyword = line.AsSpan(keywordColumn, keywordEnd - keywordColumn); + if (!keyword.SequenceEqual("if") + && !keyword.SequenceEqual("while") + && !keyword.SequenceEqual("for")) + { + keywordColumn = keywordEnd - 1; + continue; + } + + var openingColumn = SkipWhitespace(line, keywordEnd); + if (openingColumn >= line.Length || line[openingColumn] != '(') + { + keywordColumn = keywordEnd - 1; + continue; + } + + var closingColumn = FindMatchingParenthesis(line, openingColumn); + if (closingColumn < 0) + { + keywordColumn = keywordEnd - 1; + continue; + } + + var nameColumn = SkipWhitespace(line, closingColumn + 1); + if (nameColumn >= line.Length + || !IsIdentifierStart(line[nameColumn])) + { + keywordColumn = closingColumn; + continue; + } + + var nameEnd = nameColumn + 1; + while (nameEnd < line.Length + && (char.IsLetterOrDigit(line[nameEnd]) || line[nameEnd] == '_')) + { + nameEnd++; + } + + var name = line[nameColumn..nameEnd]; + var nextColumn = SkipWhitespace(line, nameEnd); + if (callableNames.Contains(name) + && (nextColumn >= line.Length + || (line[nextColumn] != '(' + && line[nextColumn] != ':' + && line[nextColumn] != '='))) + { + addCallLikeReference(name, nameColumn); + } + + keywordColumn = closingColumn; + } + } + + private static int FindMatchingParenthesis(string line, int openingColumn) + { + var depth = 0; + for (var column = openingColumn; column < line.Length; column++) + { + if (line[column] == '(') + { + depth++; + } + else if (line[column] == ')' && --depth == 0) + { + return column; + } + } + + return -1; + } + + private static int SkipWhitespace(string line, int column) + { + while (column < line.Length && char.IsWhiteSpace(line[column])) + column++; + return column; + } + + private static bool IsIdentifierStart(char value) => + value == '_' || char.IsLetter(value); + public static bool ShouldSuppressGenericCall( string language, string preparedLine, @@ -1080,6 +1319,141 @@ private static bool IsLikelySlashyLiteralStart(string line, int column) || token.SequenceEqual("until"); } + private static bool TryBeginAmbiguousPerlQuoteLikeLiteral( + string line, + char[] buffer, + int column, + out AmbiguousPerlQuoteLikeState state, + out int contentColumn) + { + state = null!; + contentColumn = column; + if (column > 0 + && (char.IsLetterOrDigit(line[column - 1]) || line[column - 1] == '_')) + { + return false; + } + + var operatorLength = line.AsSpan(column) switch + { + var span when span.StartsWith("qq", StringComparison.Ordinal) + || span.StartsWith("qr", StringComparison.Ordinal) + || span.StartsWith("qw", StringComparison.Ordinal) + || span.StartsWith("qx", StringComparison.Ordinal) + || span.StartsWith("tr", StringComparison.Ordinal) => 2, + var span when span.StartsWith("q", StringComparison.Ordinal) + || span.StartsWith("m", StringComparison.Ordinal) + || span.StartsWith("s", StringComparison.Ordinal) + || span.StartsWith("y", StringComparison.Ordinal) => 1, + _ => 0, + }; + if (operatorLength == 0) + return false; + + var delimiterColumn = column + operatorLength; + if (delimiterColumn >= line.Length + || char.IsLetterOrDigit(line[delimiterColumn]) + || line[delimiterColumn] == '_' + || char.IsWhiteSpace(line[delimiterColumn])) + { + return false; + } + + var openingDelimiter = line[delimiterColumn]; + var closingDelimiter = GetPairedClosingDelimiter(openingDelimiter); + var remainingSegments = line.AsSpan(column, operatorLength).SequenceEqual("s") + || line.AsSpan(column, operatorLength).SequenceEqual("tr") + || line.AsSpan(column, operatorLength).SequenceEqual("y") + ? 2 + : 1; + FillWithSpaces(buffer, column, delimiterColumn + 1); + state = new AmbiguousPerlQuoteLikeState( + openingDelimiter, + closingDelimiter, + remainingSegments); + contentColumn = delimiterColumn + 1; + return true; + } + + private static void MaskAmbiguousPerlQuoteLikeCharacter( + string line, + char[] buffer, + ref int column, + ref AmbiguousPerlQuoteLikeState? state) + { + var current = state!; + buffer[column] = ' '; + if (current.AwaitingNextOpeningDelimiter) + { + if (char.IsWhiteSpace(line[column])) + { + column++; + return; + } + + current.OpeningDelimiter = line[column]; + current.ClosingDelimiter = GetPairedClosingDelimiter(line[column]); + current.DelimiterDepth = current.OpeningDelimiter == current.ClosingDelimiter ? 0 : 1; + current.AwaitingNextOpeningDelimiter = false; + column++; + return; + } + + if (line[column] == '\\' && column + 1 < line.Length) + { + buffer[column + 1] = ' '; + column += 2; + return; + } + + if (current.OpeningDelimiter != current.ClosingDelimiter + && line[column] == current.OpeningDelimiter) + { + current.DelimiterDepth++; + column++; + return; + } + + if (line[column] != current.ClosingDelimiter) + { + column++; + return; + } + + if (current.OpeningDelimiter != current.ClosingDelimiter + && --current.DelimiterDepth > 0) + { + column++; + return; + } + + current.RemainingSegments--; + if (current.RemainingSegments == 0) + { + state = null; + } + else if (current.OpeningDelimiter != current.ClosingDelimiter) + { + current.AwaitingNextOpeningDelimiter = true; + } + else + { + current.DelimiterDepth = 0; + } + + column++; + } + + private static char GetPairedClosingDelimiter(char openingDelimiter) => + openingDelimiter switch + { + '(' => ')', + '[' => ']', + '{' => '}', + '<' => '>', + _ => openingDelimiter, + }; + private static bool TryBeginCrystalPercentLiteral( string line, int column, @@ -1374,8 +1748,8 @@ private static void AddTclContainers( continue; var startLineIndex = symbol.StartLine - 1; - var declarationMatch = TclProcRegex.Match(lines[startLineIndex]); - if (!declarationMatch.Success + var declarationMatch = FindTclProcDeclaration(lines[startLineIndex], symbol); + if (declarationMatch == null || !TryFindTclBodyEnd( lines, braceEnds, @@ -1411,6 +1785,22 @@ lines[bodyStartLineIndex][bodyStartColumn] is '{' or '"' }); } + private static Match? FindTclProcDeclaration(string line, SymbolRecord symbol) + { + Match? fallback = null; + foreach (Match match in BoundedRegex.EnumerateMatches(TclProcRegex, line)) + { + var nameGroup = match.Groups["name"]; + if (!string.Equals(nameGroup.Value, symbol.Name, StringComparison.Ordinal)) + continue; + if (symbol.StartColumn == nameGroup.Index) + return match; + fallback ??= match; + } + + return fallback; + } + private static bool TryFindTclBodyEnd( IReadOnlyList lines, IReadOnlyDictionary braceEnds, @@ -2407,21 +2797,53 @@ private static IReadOnlyDictionary> BuildProl var scanningMultilineHead = false; var multilineHeadParenthesisDepth = 0; var multilineHeadParenthesesClosed = false; + var scanningDirective = false; for (var lineIndex = 0; lineIndex < lines.Count; lineIndex++) { var lineNumber = lineIndex + 1; if (!containersByLine.TryGetValue(lineNumber, out var container)) { - frames.Clear(); activeContainer = null; - expectGoal = true; scanningMultilineHead = false; multilineHeadParenthesisDepth = 0; multilineHeadParenthesesClosed = false; + if (!scanningDirective + && !StartsWithPrologGoalDirective(lines[lineIndex])) + { + frames.Clear(); + expectGoal = true; + continue; + } + + if (!scanningDirective) + { + frames.Clear(); + expectGoal = true; + scanningDirective = true; + } + + var directiveCalls = new List(); + ScanPrologGoalLine( + lines, + lineIndex, + lines[lineIndex], + callableNames, + frames, + ref expectGoal, + directiveCalls); + if (directiveCalls.Count > 0) + result[lineNumber] = directiveCalls; + if (ContainsPrologClauseTerminator(lines[lineIndex])) + { + frames.Clear(); + expectGoal = true; + scanningDirective = false; + } continue; } + scanningDirective = false; if (activeContainer == null || activeContainer.StartLine != container.StartLine || !string.Equals(activeContainer.Name, container.Name, StringComparison.Ordinal)) @@ -2482,6 +2904,14 @@ private static IReadOnlyDictionary> BuildProl return result; } + private static bool StartsWithPrologGoalDirective(string line) + { + var column = 0; + while (column < line.Length && char.IsWhiteSpace(line[column])) + column++; + return line.AsSpan(column).StartsWith(":-", StringComparison.Ordinal); + } + private static bool TryInitializePrologMultilineHeadScan( IReadOnlyList lines, SymbolRecord container, diff --git a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.ExtractCore.cs b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.ExtractCore.cs index c35837c90..25eef0f28 100644 --- a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.ExtractCore.cs +++ b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.ExtractCore.cs @@ -2184,6 +2184,14 @@ bool[] GetCssQualifiedRuleAncestors() => extractionState, prologMultilineHeads); } + if (lang == "tcl") + { + DynamicDeclarativeReferenceExtractor.AddTclInlineProcSymbols( + fileId, + lines, + structuralLines, + symbols); + } AssignContainers(symbols, lines, getCSharpLineStartStates); if (lang is "shell" or "powershell") AddScriptScopeSymbol(fileId, lines, symbols); diff --git a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.cs b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.cs index 281fdae03..f74a8567b 100644 --- a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.cs +++ b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.cs @@ -23,8 +23,8 @@ public static partial class SymbolExtractor public const int XmlContractVersion = 3; public const int FunctionalLanguageContractVersion = 2; public const int DynamicLanguageContractVersion = 2; - public const int DynamicReferenceGraphContractVersion = 3; - public const int PrologReferenceGraphContractVersion = 2; + public const int DynamicReferenceGraphContractVersion = 4; + public const int PrologReferenceGraphContractVersion = 3; public const int SystemsLanguageContractVersion = 2; private static readonly string[] ExplicitReferenceGraphContractLanguages = ["crystal", "groovy", "tcl", "prolog", "ambiguous_pl"]; diff --git a/tests/CodeIndex.Tests/ReferenceExtractorTests.cs b/tests/CodeIndex.Tests/ReferenceExtractorTests.cs index 0e04d1d15..8eb9199fa 100644 --- a/tests/CodeIndex.Tests/ReferenceExtractorTests.cs +++ b/tests/CodeIndex.Tests/ReferenceExtractorTests.cs @@ -1149,6 +1149,10 @@ Map convert(String value) { def run(value) { helper value helper(value) + if (value) helper value + while (value) helper value + for (item in [value]) helper item + if (nested(value)) helper value def text = "123456789"; helper(value) [1].each { helper it } annotated(value) @@ -1170,7 +1174,7 @@ class InlineRunner { InlineRunner() {}; def inlineHelper() {} } Assert.Contains(symbols, symbol => symbol.Kind == "function" && symbol.Name == "genericConvert"); AssertReferencesContain(references, "type_reference", null, "Support"); AssertReferencesContain(references, "call", "run", "helper"); - Assert.Equal(4, references.Count(reference => + Assert.Equal(8, references.Count(reference => reference.ReferenceKind == "call" && reference.ContainerName == "run" && reference.SymbolName == "helper")); @@ -1235,6 +1239,28 @@ lmap value {1 2} helper && reference.SymbolName == "arg"); } + [Fact] + public void Extract_Tcl_IndexesInlineScriptProcWithCaller_Issue4746() + { + const string content = """ + proc helper {} { return 1 } + namespace eval ::demo { proc run {} { helper } } + set literal { proc fake {} { helper } } + """; + + var (symbols, references) = ExtractSymbolsAndReferences("tcl", content); + + var run = Assert.Single(symbols, symbol => + symbol.Kind == "function" + && symbol.Name == "run"); + Assert.Equal("::demo", run.ContainerName); + Assert.DoesNotContain(symbols, symbol => symbol.Name == "fake"); + Assert.Single(references, reference => + reference.ReferenceKind == "call" + && reference.ContainerName == "run" + && reference.SymbolName == "helper"); + } + [Fact] public void Extract_Tcl_IndexesPackagesAndProcCommandCallsWithCaller_Issue4746() { @@ -1342,6 +1368,28 @@ public void Extract_Prolog_IndexesModulesImportsAndPredicateCallsWithCaller_Issu && reference.SymbolName is "module" or "use_module" or "ancestor"); } + [Theory] + [InlineData("prolog")] + [InlineData("ambiguous_pl")] + public void Extract_Prolog_IndexesTopLevelDirectiveGoals_Issue4746(string language) + { + const string content = """ + helper. + :- helper. + :- initialization(helper). + :- initialization( + helper + ). + """; + + var (_, references) = ExtractSymbolsAndReferences(language, content); + + Assert.Equal(3, references.Count(reference => + reference.ReferenceKind == "call" + && reference.ContainerName == null + && reference.SymbolName == "helper")); + } + [Theory] [InlineData("prolog")] [InlineData("ambiguous_pl")] @@ -1456,6 +1504,33 @@ public void Extract_AmbiguousPl_IndexesPerlAndPrologReferencesWithoutReclassific AssertReferencesContain(references, "call", "prolog_helper", "perl_helper"); } + [Fact] + public void Extract_AmbiguousPl_MasksPerlQuoteLikeRegexBodies_Issue4746() + { + const string content = """ + package Hybrid; + :- module(hybrid, [helper/0]). + sub helper { return 1; } + my $regex = qr/helper()/; + my $match = m{helper()}; + my $subst = s/helper()/replacement/; + my $paired = s{helper()}{replacement}; + my $translated = tr/helper()/replacement/; + my $quoted = q(helper()); + my $multiline = qr{ + helper() + }; + prolog_run :- helper. + """; + + var (_, references) = ExtractSymbolsAndReferences("ambiguous_pl", content); + + Assert.Single(references, reference => + reference.ReferenceKind == "call" + && reference.ContainerName == "prolog_run" + && reference.SymbolName == "helper"); + } + [Fact] public void Extract_Crystal_PreservesPredicateAndBangCallsWhileMaskingCommentsAndHeredocs_Issue4746() { From 61a2012b864c82ac837180ea4e79364aa450d160 Mon Sep 17 00:00:00 2001 From: Widthdom Date: Fri, 24 Jul 2026 22:26:19 +0900 Subject: [PATCH 21/26] Resolve dynamic graph adversarial findings (#4746) --- changelog.d/unreleased/4746.added.md | 5 +- .../Cli/QueryCommandRunner.Status.cs | 2 +- .../DynamicDeclarativeReferenceExtractor.cs | 85 +++++++++++++++++-- .../Indexer/Symbols/SymbolExtractor.cs | 4 +- tests/CodeIndex.Tests/DatabaseTests.cs | 29 +++++++ .../QueryCommandRunnerFilesTests.cs | 43 ++++++++++ .../ReferenceExtractorTests.cs | 40 +++++++++ 7 files changed, 196 insertions(+), 12 deletions(-) diff --git a/changelog.d/unreleased/4746.added.md b/changelog.d/unreleased/4746.added.md index ece8aaffb..0af076fb1 100644 --- a/changelog.d/unreleased/4746.added.md +++ b/changelog.d/unreleased/4746.added.md @@ -25,6 +25,7 @@ affected: - tests/CodeIndex.Tests/IndexCommandRunnerFullScanTests.cs - tests/CodeIndex.Tests/SymbolExtractorTests.cs - tests/CodeIndex.Tests/ReferenceExtractorTests.cs + - tests/CodeIndex.Tests/QueryCommandRunnerFilesTests.cs - tests/CodeIndex.Tests/QueryCommandRunnerTests.cs - tests/CodeIndex.Tests/McpServerToolsCallTests.cs - tests/CodeIndex.Tests/golden/status.json @@ -37,8 +38,8 @@ affected: ## English -- **Crystal, Groovy, Tcl, Prolog, and ambiguous `.pl` files now support conservative reference graphs (#4746)** — indexing records imports and language-appropriate calls at their source locations, recognizes command-style calls only for callables declared in the same file, parses Groovy control-statement bodies, Tcl command substitutions, inline proc declarations, control-command script arguments, and `switch` branch bodies, limits Prolog calls to goal positions including top-level directives, masks Perl quote-like regular-expression bodies in ambiguous `.pl` files, preserves Tcl proc and Prolog predicate caller containers, and reports structured symbol/reference/graph capabilities after `.pl` content classification. Pre-graph extractor stamps keep graph readiness degraded until a normal index refresh rewrites affected rows. +- **Crystal, Groovy, Tcl, Prolog, and ambiguous `.pl` files now support conservative reference graphs (#4746)** — indexing records imports and language-appropriate calls at their source locations, recognizes command-style calls only for callables declared in the same file, parses Groovy control-statement bodies, Tcl command substitutions, inline proc declarations, control-command script arguments, and `switch` branch bodies, excludes Groovy closure parameter headers, limits Prolog calls to goal positions including top-level and same-line directives, masks Perl quote-like regular-expression bodies (including whitespace before delimiters) in ambiguous `.pl` files, preserves Tcl proc and Prolog predicate caller containers, and reports structured symbol/reference/graph capabilities after `.pl` content classification. Pre-graph extractor stamps keep graph readiness degraded until a normal index refresh rewrites affected rows, and human-readable status output identifies that refresh instead of cap-hit remediation. ## 日本語 -- **Crystal、Groovy、Tcl、Prolog、曖昧な `.pl` ファイルが保守的な reference graph に対応しました（#4746）** — import と言語ごとの call 構文をソース上の位置どおりに記録し、command-style call は同一ファイルで宣言済みの callable だけを認識し、Groovy の制御文本体、Tcl の command substitution、inline proc 宣言、制御 command の script 引数、`switch` の branch body を解析し、top-level directive を含む Prolog の call を goal 位置に限定し、曖昧な `.pl` では Perl の quote-like 正規表現本体をマスクし、Tcl proc / Prolog predicate の caller container を保持して、`.pl` の content classification 後に構造化された symbol / reference / graph capability を報告します。graph 対応前の extractor stamp がある場合、通常の index refresh で対象 row を更新するまで graph readiness を degraded に保ちます。 +- **Crystal、Groovy、Tcl、Prolog、曖昧な `.pl` ファイルが保守的な reference graph に対応しました（#4746）** — import と言語ごとの call 構文をソース上の位置どおりに記録し、command-style call は同一ファイルで宣言済みの callable だけを認識し、Groovy の制御文本体、Tcl の command substitution、inline proc 宣言、制御 command の script 引数、`switch` の branch body を解析し、Groovy の closure parameter header を除外し、top-level および同一行の directive を含む Prolog の call を goal 位置に限定し、曖昧な `.pl` では delimiter 前に空白がある場合を含めて Perl の quote-like 正規表現本体をマスクし、Tcl proc / Prolog predicate の caller container を保持して、`.pl` の content classification 後に構造化された symbol / reference / graph capability を報告します。graph 対応前の extractor stamp がある場合、通常の index refresh で対象 row を更新するまで graph readiness を degraded に保ち、人間向け status 出力でも cap-hit 対応ではなくこの refresh を案内します。 diff --git a/src/CodeIndex/Cli/QueryCommandRunner.Status.cs b/src/CodeIndex/Cli/QueryCommandRunner.Status.cs index 24960f5a1..491d9fcbb 100644 --- a/src/CodeIndex/Cli/QueryCommandRunner.Status.cs +++ b/src/CodeIndex/Cli/QueryCommandRunner.Status.cs @@ -323,7 +323,7 @@ public static int RunStatus( ? string.Join(", ", status.ReferenceGraphIncompleteReasons.Take(4)) : DbReader.ReferenceExtractionCapStateUnavailableReason; Console.WriteLine(ConsoleUi.FormatSummaryLine("WARN", $"reference graph is incomplete ({reasons}); absent callers/callees/deps/impact edges are not authoritative.")); - Console.WriteLine(ConsoleUi.FormatSummaryLine("Hint", "inspect reference_extraction_cap_hits.files, reduce or exclude the generated/pathological source, then rerun indexing.")); + Console.WriteLine(ConsoleUi.FormatSummaryLine("Hint", GetReferenceGraphRepairSafetyNote(status))); } if (!status.GraphTableAvailable) Console.WriteLine(ConsoleUi.FormatSummaryLine("WARN", "symbol_references table missing — reference / caller / callee / unused counts are degraded to 0.")); diff --git a/src/CodeIndex/Indexer/References/Languages/DynamicDeclarativeReferenceExtractor.cs b/src/CodeIndex/Indexer/References/Languages/DynamicDeclarativeReferenceExtractor.cs index f7ac0a582..b055f3b45 100644 --- a/src/CodeIndex/Indexer/References/Languages/DynamicDeclarativeReferenceExtractor.cs +++ b/src/CodeIndex/Indexer/References/Languages/DynamicDeclarativeReferenceExtractor.cs @@ -181,7 +181,10 @@ private enum TclLexicalFrameKind SwitchTable, } - internal readonly record struct PrologGoalCall(string Name, int Column); + internal readonly record struct PrologGoalCall( + string Name, + int Column, + bool IsTopLevelDirective = false); internal readonly record struct DeclarationPosition(int Line, int Column, string Name); @@ -888,7 +891,25 @@ public static void EmitAdditionalReferences( if (!state.CallableNames.Contains(call.Name)) continue; - var prologContainer = state.ResolveContainer(lineNumber, call.Column, fallback: null); + var prologContainer = call.IsTopLevelDirective + ? null + : state.ResolveContainer(lineNumber, call.Column, fallback: null); + if (call.IsTopLevelDirective) + { + ReferenceExtractor.AddReference( + references, + seen, + fileId, + call.Name, + call.Column, + "call", + context, + lineNumber, + container: null, + language); + continue; + } + if (prologContainer != null && !string.Equals(prologContainer.Name, call.Name, StringComparison.Ordinal)) { @@ -951,6 +972,11 @@ public static void EmitAdditionalReferences( var nameGroup = match.Groups["name"]; if (!state.CallableNames.Contains(nameGroup.Value)) continue; + if (language == "groovy" + && IsGroovyClosureParameterHeader(preparedLine, nameGroup.Index)) + { + continue; + } addCallLikeReference(nameGroup.Value, nameGroup.Index); } @@ -1045,6 +1071,28 @@ private static int FindMatchingParenthesis(string line, int openingColumn) return -1; } + private static bool IsGroovyClosureParameterHeader(string line, int nameColumn) + { + var openingBrace = line.LastIndexOf('{', Math.Max(0, nameColumn - 1)); + if (openingBrace < 0) + return false; + + var closingBraceBeforeName = line.LastIndexOf('}', Math.Max(0, nameColumn - 1)); + if (closingBraceBeforeName > openingBrace) + return false; + + var arrowColumn = line.IndexOf("->", nameColumn, StringComparison.Ordinal); + if (arrowColumn < 0) + return false; + + var closingBraceAfterName = line.IndexOf('}', nameColumn); + if (closingBraceAfterName >= 0 && closingBraceAfterName < arrowColumn) + return false; + + return line.AsSpan(nameColumn, arrowColumn - nameColumn) + .IndexOfAny(';', '{', '}') < 0; + } + private static int SkipWhitespace(string line, int column) { while (column < line.Length && char.IsWhiteSpace(line[column])) @@ -1350,11 +1398,10 @@ var span when span.StartsWith("q", StringComparison.Ordinal) if (operatorLength == 0) return false; - var delimiterColumn = column + operatorLength; + var delimiterColumn = SkipWhitespace(line, column + operatorLength); if (delimiterColumn >= line.Length || char.IsLetterOrDigit(line[delimiterColumn]) - || line[delimiterColumn] == '_' - || char.IsWhiteSpace(line[delimiterColumn])) + || line[delimiterColumn] == '_') { return false; } @@ -2833,7 +2880,11 @@ private static IReadOnlyDictionary> BuildProl ref expectGoal, directiveCalls); if (directiveCalls.Count > 0) - result[lineNumber] = directiveCalls; + { + result[lineNumber] = directiveCalls + .Select(static call => call with { IsTopLevelDirective = true }) + .ToList(); + } if (ContainsPrologClauseTerminator(lines[lineIndex])) { frames.Clear(); @@ -2891,7 +2942,13 @@ private static IReadOnlyDictionary> BuildProl ref expectGoal, lineCalls); if (lineCalls.Count > 0) - result[lineNumber] = lineCalls; + { + result[lineNumber] = lineCalls + .Select(call => IsTopLevelPrologDirectiveGoal(lines[lineIndex], call.Column) + ? call with { IsTopLevelDirective = true } + : call) + .ToList(); + } if (ContainsPrologClauseTerminator(callScanLine)) { @@ -2912,6 +2969,20 @@ private static bool StartsWithPrologGoalDirective(string line) return line.AsSpan(column).StartsWith(":-", StringComparison.Ordinal); } + private static bool IsTopLevelPrologDirectiveGoal(string line, int goalColumn) + { + var segmentStartColumn = 0; + for (var column = 0; column < goalColumn; column++) + { + if (IsPrologClauseTerminator(line, column)) + segmentStartColumn = column + 1; + } + + segmentStartColumn = SkipWhitespace(line, segmentStartColumn); + return segmentStartColumn + 2 <= goalColumn + && line.AsSpan(segmentStartColumn).StartsWith(":-", StringComparison.Ordinal); + } + private static bool TryInitializePrologMultilineHeadScan( IReadOnlyList lines, SymbolRecord container, diff --git a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.cs b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.cs index f74a8567b..9db5b8bdf 100644 --- a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.cs +++ b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.cs @@ -23,8 +23,8 @@ public static partial class SymbolExtractor public const int XmlContractVersion = 3; public const int FunctionalLanguageContractVersion = 2; public const int DynamicLanguageContractVersion = 2; - public const int DynamicReferenceGraphContractVersion = 4; - public const int PrologReferenceGraphContractVersion = 3; + public const int DynamicReferenceGraphContractVersion = 5; + public const int PrologReferenceGraphContractVersion = 4; public const int SystemsLanguageContractVersion = 2; private static readonly string[] ExplicitReferenceGraphContractLanguages = ["crystal", "groovy", "tcl", "prolog", "ambiguous_pl"]; diff --git a/tests/CodeIndex.Tests/DatabaseTests.cs b/tests/CodeIndex.Tests/DatabaseTests.cs index 44b20efe2..b32ce0af0 100644 --- a/tests/CodeIndex.Tests/DatabaseTests.cs +++ b/tests/CodeIndex.Tests/DatabaseTests.cs @@ -4723,6 +4723,35 @@ public void GetUnchangedFileId_InvalidatesPreGraphLanguageContracts_Issue4746( Assert.Null(_writer.GetUnchangedFileId(file.Path, modified, language: language)); } + [Theory] + [InlineData("groovy", 4)] + [InlineData("prolog", 3)] + [InlineData("ambiguous_pl", 3)] + public void GetUnchangedFileId_InvalidatesPriorDynamicGraphContracts_Issue4746( + string language, + int previousContractVersion) + { + var modified = new DateTime(2025, 1, 1, 0, 0, 0, DateTimeKind.Utc); + var file = new FileRecord + { + Path = $"src/prior-graph-{language}.txt", + Lang = language, + Size = 50, + Lines = 5, + Modified = modified, + }; + _writer.UpsertFile(file); + _writer.SetMeta( + DbContext.GetSymbolExtractorVersionMetaKey(language), + previousContractVersion.ToString(CultureInfo.InvariantCulture)); + _writer.SetMeta( + DbContext.GetDynamicReferenceGraphContractVersionMetaKey(language), + previousContractVersion.ToString(CultureInfo.InvariantCulture)); + + Assert.True(SymbolExtractor.GetContractVersion(language) > previousContractVersion); + Assert.Null(_writer.GetUnchangedFileId(file.Path, modified, language: language)); + } + [Theory] [InlineData("crystal")] [InlineData("groovy")] diff --git a/tests/CodeIndex.Tests/QueryCommandRunnerFilesTests.cs b/tests/CodeIndex.Tests/QueryCommandRunnerFilesTests.cs index fe0c82733..d614fe18e 100644 --- a/tests/CodeIndex.Tests/QueryCommandRunnerFilesTests.cs +++ b/tests/CodeIndex.Tests/QueryCommandRunnerFilesTests.cs @@ -1240,6 +1240,49 @@ public void RunStatus_HumanOutput_WarnsWhenOnlyFoldReadinessIsDegraded() } } + [Fact] + public void RunStatus_HumanOutput_UsesDynamicGraphContractRepairHint_Issue4746() + { + var projectRoot = TestProjectHelper.CreateTempProject("cdidx_status_dynamic_graph_contract_4746"); + try + { + File.WriteAllText( + Path.Combine(projectRoot, "app.groovy"), + "def helper() { }\ndef run() { helper() }\n"); + Assert.Equal( + CommandExitCodes.Success, + IndexCommandRunner.Run([projectRoot, "--json", "--quiet"], _jsonOptions)); + + var dbPath = Path.Combine(projectRoot, ".cdidx", "codeindex.db"); + using (var db = new DbContext(DbOpenIntent.WriteIndex, dbPath)) + { + var writer = new DbWriter(db.Connection); + writer.SetMeta( + DbContext.GetDynamicReferenceGraphContractVersionMetaKey("groovy"), + "0"); + } + + var (exitCode, stdout, stderr) = CaptureConsole(() => QueryCommandRunner.RunStatus( + ["--db", dbPath], + _jsonOptions)); + + Assert.Equal(CommandExitCodes.Success, exitCode); + Assert.Equal(string.Empty, stderr); + Assert.Contains( + "Refresh indexing to rewrite stale dynamic-language graph rows and extractor-version stamps.", + stdout, + StringComparison.Ordinal); + Assert.DoesNotContain( + "reduce or exclude the cap-hitting generated/pathological source", + stdout, + StringComparison.OrdinalIgnoreCase); + } + finally + { + TestProjectHelper.DeleteDirectory(projectRoot); + } + } + [Fact] public void RunStatus_Json_ReadOnlyUriFoldRemediationUsesWritableDbPath() { diff --git a/tests/CodeIndex.Tests/ReferenceExtractorTests.cs b/tests/CodeIndex.Tests/ReferenceExtractorTests.cs index 8eb9199fa..be82605a0 100644 --- a/tests/CodeIndex.Tests/ReferenceExtractorTests.cs +++ b/tests/CodeIndex.Tests/ReferenceExtractorTests.cs @@ -1440,6 +1440,7 @@ first. second. run :- a. other :- b. continued :- a. continuation_sibling :- b. + fact. :- a. before_directive. :- dynamic before_directive/0. before_multiline. multiline_after( X @@ -1457,6 +1458,7 @@ before_multiline. multiline_after( "other", "continued", "continuation_sibling", + "fact", "before_directive", "before_multiline", "multiline_after", @@ -1469,6 +1471,14 @@ before_multiline. multiline_after( AssertReferencesContain(references, "call", "continued", "a"); AssertReferencesContain(references, "call", "continuation_sibling", "b"); AssertReferencesContain(references, "call", "multiline_after", "a"); + Assert.Contains(references, reference => + reference.ReferenceKind == "call" + && reference.ContainerName == null + && reference.SymbolName == "a"); + Assert.DoesNotContain(references, reference => + reference.ReferenceKind == "call" + && reference.ContainerName == "fact" + && reference.SymbolName == "a"); Assert.DoesNotContain(references, reference => reference.ReferenceKind == "call" && reference.ContainerName == "run" @@ -1517,6 +1527,9 @@ public void Extract_AmbiguousPl_MasksPerlQuoteLikeRegexBodies_Issue4746() my $paired = s{helper()}{replacement}; my $translated = tr/helper()/replacement/; my $quoted = q(helper()); + my $spacedQuoted = q {helper()}; + my $spacedRegex = qr {helper()}; + my $spacedSubstitution = s {helper()} {replacement}; my $multiline = qr{ helper() }; @@ -1644,6 +1657,33 @@ fake 1 Assert.DoesNotContain(references, reference => reference.SymbolName == "fake"); } + [Fact] + public void Extract_Groovy_DoesNotTreatClosureParametersAsBareCalls_Issue4746() + { + const string content = """ + def helper() {} + + def run() { + [1].each { helper -> println helper } + } + + def actual() { + helper; [1].each { value -> println value } + } + """; + + var (_, references) = ExtractSymbolsAndReferences("groovy", content); + + Assert.Single(references, reference => + reference.ReferenceKind == "call" + && reference.ContainerName == "actual" + && reference.SymbolName == "helper"); + Assert.DoesNotContain(references, reference => + reference.ReferenceKind == "call" + && reference.ContainerName == "run" + && reference.SymbolName == "helper"); + } + [Fact] public void Extract_Tcl_IndexesSingleLineProcBodiesAndMasksComments_Issue4746() { From c1e4b0f98c4f4a48d3e447f02efb60098ac28c33 Mon Sep 17 00:00:00 2001 From: Widthdom Date: Fri, 24 Jul 2026 22:54:02 +0900 Subject: [PATCH 22/26] Fix ambiguous Prolog graph edge cases (#4746) --- changelog.d/unreleased/4746.added.md | 4 +- .../DynamicDeclarativeReferenceExtractor.cs | 112 +++++++++++------- .../Indexer/Symbols/SymbolExtractor.cs | 2 +- tests/CodeIndex.Tests/DatabaseTests.cs | 4 +- .../ReferenceExtractorTests.cs | 40 +++++++ 5 files changed, 117 insertions(+), 45 deletions(-) diff --git a/changelog.d/unreleased/4746.added.md b/changelog.d/unreleased/4746.added.md index 0af076fb1..90a40dc16 100644 --- a/changelog.d/unreleased/4746.added.md +++ b/changelog.d/unreleased/4746.added.md @@ -38,8 +38,8 @@ affected: ## English -- **Crystal, Groovy, Tcl, Prolog, and ambiguous `.pl` files now support conservative reference graphs (#4746)** — indexing records imports and language-appropriate calls at their source locations, recognizes command-style calls only for callables declared in the same file, parses Groovy control-statement bodies, Tcl command substitutions, inline proc declarations, control-command script arguments, and `switch` branch bodies, excludes Groovy closure parameter headers, limits Prolog calls to goal positions including top-level and same-line directives, masks Perl quote-like regular-expression bodies (including whitespace before delimiters) in ambiguous `.pl` files, preserves Tcl proc and Prolog predicate caller containers, and reports structured symbol/reference/graph capabilities after `.pl` content classification. Pre-graph extractor stamps keep graph readiness degraded until a normal index refresh rewrites affected rows, and human-readable status output identifies that refresh instead of cap-hit remediation. +- **Crystal, Groovy, Tcl, Prolog, and ambiguous `.pl` files now support conservative reference graphs (#4746)** — indexing records imports and language-appropriate calls at their source locations, recognizes command-style calls only for callables declared in the same file, parses Groovy control-statement bodies, Tcl command substitutions, inline proc declarations, control-command script arguments, and `switch` branch bodies, excludes Groovy closure parameter headers, limits Prolog calls to goal positions including top-level and same-line directives, scans Prolog clause terminators in one pass, masks Perl quote-like regular-expression bodies (including whitespace before delimiters) in ambiguous `.pl` files without consuming fat-arrow pairs or short Prolog predicate heads, preserves Tcl proc and Prolog predicate caller containers, and reports structured symbol/reference/graph capabilities after `.pl` content classification. Pre-graph extractor stamps keep graph readiness degraded until a normal index refresh rewrites affected rows, and human-readable status output identifies that refresh instead of cap-hit remediation. ## 日本語 -- **Crystal、Groovy、Tcl、Prolog、曖昧な `.pl` ファイルが保守的な reference graph に対応しました（#4746）** — import と言語ごとの call 構文をソース上の位置どおりに記録し、command-style call は同一ファイルで宣言済みの callable だけを認識し、Groovy の制御文本体、Tcl の command substitution、inline proc 宣言、制御 command の script 引数、`switch` の branch body を解析し、Groovy の closure parameter header を除外し、top-level および同一行の directive を含む Prolog の call を goal 位置に限定し、曖昧な `.pl` では delimiter 前に空白がある場合を含めて Perl の quote-like 正規表現本体をマスクし、Tcl proc / Prolog predicate の caller container を保持して、`.pl` の content classification 後に構造化された symbol / reference / graph capability を報告します。graph 対応前の extractor stamp がある場合、通常の index refresh で対象 row を更新するまで graph readiness を degraded に保ち、人間向け status 出力でも cap-hit 対応ではなくこの refresh を案内します。 +- **Crystal、Groovy、Tcl、Prolog、曖昧な `.pl` ファイルが保守的な reference graph に対応しました（#4746）** — import と言語ごとの call 構文をソース上の位置どおりに記録し、command-style call は同一ファイルで宣言済みの callable だけを認識し、Groovy の制御文本体、Tcl の command substitution、inline proc 宣言、制御 command の script 引数、`switch` の branch body を解析し、Groovy の closure parameter header を除外し、top-level および同一行の directive を含む Prolog の call を goal 位置に限定し、Prolog の clause terminator を1回の走査で検出し、曖昧な `.pl` では fat-arrow pair や短い Prolog predicate head を巻き込まず、delimiter 前に空白がある場合を含めて Perl の quote-like 正規表現本体をマスクし、Tcl proc / Prolog predicate の caller container を保持して、`.pl` の content classification 後に構造化された symbol / reference / graph capability を報告します。graph 対応前の extractor stamp がある場合、通常の index refresh で対象 row を更新するまで graph readiness を degraded に保ち、人間向け status 出力でも cap-hit 対応ではなくこの refresh を案内します。 diff --git a/src/CodeIndex/Indexer/References/Languages/DynamicDeclarativeReferenceExtractor.cs b/src/CodeIndex/Indexer/References/Languages/DynamicDeclarativeReferenceExtractor.cs index b055f3b45..a137dfdfb 100644 --- a/src/CodeIndex/Indexer/References/Languages/DynamicDeclarativeReferenceExtractor.cs +++ b/src/CodeIndex/Indexer/References/Languages/DynamicDeclarativeReferenceExtractor.cs @@ -1,3 +1,4 @@ +using System.Runtime.CompilerServices; using System.Text.RegularExpressions; using CodeIndex.Models; using Regex = CodeIndex.Indexer.BoundedRegex; @@ -6,6 +7,9 @@ namespace CodeIndex.Indexer; internal static class DynamicDeclarativeReferenceExtractor { + private static readonly ConditionalWeakTable + PrologClauseTerminatorMaps = new(); + internal sealed class ExtractionState { public ExtractionState( @@ -362,6 +366,8 @@ public static string[] MaskNonCodeLines(string language, IReadOnlyList l var tclCommentStart = language == "tcl" ? FindTclCommentStart(line) : -1; + var hasPrologLineStructure = language == "ambiguous_pl" + && (StartsWithPrologGoalDirective(line) || PrologHeadRegex.IsMatch(line)); for (var column = 0; column < line.Length;) { if (ambiguousPerlQuoteLikeState != null) @@ -534,6 +540,7 @@ public static string[] MaskNonCodeLines(string language, IReadOnlyList l } if (language == "ambiguous_pl" + && !hasPrologLineStructure && TryBeginAmbiguousPerlQuoteLikeLiteral( line, buffer, @@ -1405,6 +1412,13 @@ var span when span.StartsWith("q", StringComparison.Ordinal) { return false; } + var delimiterSpan = line.AsSpan(delimiterColumn); + if (delimiterSpan.StartsWith("=>", StringComparison.Ordinal) + || delimiterSpan.StartsWith(":-", StringComparison.Ordinal) + || delimiterSpan.StartsWith("-->", StringComparison.Ordinal)) + { + return false; + } var openingDelimiter = line[delimiterColumn]; var closingDelimiter = GetPairedClosingDelimiter(openingDelimiter); @@ -1667,54 +1681,72 @@ internal static bool IsPrologClauseTerminator(string line, int column) if (column < 0 || column >= line.Length || line[column] != '.') return false; - var previous = column > 0 ? line[column - 1] : '\0'; - var next = column + 1 < line.Length ? line[column + 1] : '\0'; - if (previous == '.' || next == '.') - return false; - if (char.IsDigit(previous) && char.IsDigit(next)) - return false; - if (next != '\0' && !char.IsWhiteSpace(next)) - return false; - return IsTopLevelPrologColumn(line, column); + return PrologClauseTerminatorMaps + .GetValue(line, static currentLine => new PrologClauseTerminatorMap(currentLine)) + .IsTerminator(column); } - private static bool IsTopLevelPrologColumn(string line, int targetColumn) + private sealed class PrologClauseTerminatorMap { - var parenthesisDepth = 0; - var bracketDepth = 0; - var braceDepth = 0; - for (var column = 0; column < targetColumn; column++) + private readonly bool[] _terminatorColumns; + + public PrologClauseTerminatorMap(string line) { - var ch = line[column]; - if (ch is '\'' or '"') - { - column = SkipQuotedToken(line, column, ch) - 1; - continue; - } - switch (ch) + _terminatorColumns = new bool[line.Length]; + var parenthesisDepth = 0; + var bracketDepth = 0; + var braceDepth = 0; + for (var column = 0; column < line.Length; column++) { - case '(': - parenthesisDepth++; - break; - case ')' when parenthesisDepth > 0: - parenthesisDepth--; - break; - case '[': - bracketDepth++; - break; - case ']' when bracketDepth > 0: - bracketDepth--; - break; - case '{': - braceDepth++; - break; - case '}' when braceDepth > 0: - braceDepth--; - break; + var ch = line[column]; + if (ch is '\'' or '"') + { + column = SkipQuotedToken(line, column, ch) - 1; + continue; + } + switch (ch) + { + case '(': + parenthesisDepth++; + continue; + case ')' when parenthesisDepth > 0: + parenthesisDepth--; + continue; + case '[': + bracketDepth++; + continue; + case ']' when bracketDepth > 0: + bracketDepth--; + continue; + case '{': + braceDepth++; + continue; + case '}' when braceDepth > 0: + braceDepth--; + continue; + } + + if (ch != '.' + || parenthesisDepth != 0 + || bracketDepth != 0 + || braceDepth != 0) + { + continue; + } + + var previous = column > 0 ? line[column - 1] : '\0'; + var next = column + 1 < line.Length ? line[column + 1] : '\0'; + if (previous != '.' + && next != '.' + && !(char.IsDigit(previous) && char.IsDigit(next)) + && (next == '\0' || char.IsWhiteSpace(next))) + { + _terminatorColumns[column] = true; + } } } - return parenthesisDepth == 0 && bracketDepth == 0 && braceDepth == 0; + public bool IsTerminator(int column) => _terminatorColumns[column]; } private static int FindPrologClauseTerminator(string line, int startColumn) diff --git a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.cs b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.cs index 9db5b8bdf..cd2419693 100644 --- a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.cs +++ b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.cs @@ -24,7 +24,7 @@ public static partial class SymbolExtractor public const int FunctionalLanguageContractVersion = 2; public const int DynamicLanguageContractVersion = 2; public const int DynamicReferenceGraphContractVersion = 5; - public const int PrologReferenceGraphContractVersion = 4; + public const int PrologReferenceGraphContractVersion = 5; public const int SystemsLanguageContractVersion = 2; private static readonly string[] ExplicitReferenceGraphContractLanguages = ["crystal", "groovy", "tcl", "prolog", "ambiguous_pl"]; diff --git a/tests/CodeIndex.Tests/DatabaseTests.cs b/tests/CodeIndex.Tests/DatabaseTests.cs index b32ce0af0..a13c14c99 100644 --- a/tests/CodeIndex.Tests/DatabaseTests.cs +++ b/tests/CodeIndex.Tests/DatabaseTests.cs @@ -4725,8 +4725,8 @@ public void GetUnchangedFileId_InvalidatesPreGraphLanguageContracts_Issue4746( [Theory] [InlineData("groovy", 4)] - [InlineData("prolog", 3)] - [InlineData("ambiguous_pl", 3)] + [InlineData("prolog", 4)] + [InlineData("ambiguous_pl", 4)] public void GetUnchangedFileId_InvalidatesPriorDynamicGraphContracts_Issue4746( string language, int previousContractVersion) diff --git a/tests/CodeIndex.Tests/ReferenceExtractorTests.cs b/tests/CodeIndex.Tests/ReferenceExtractorTests.cs index be82605a0..00620f7d8 100644 --- a/tests/CodeIndex.Tests/ReferenceExtractorTests.cs +++ b/tests/CodeIndex.Tests/ReferenceExtractorTests.cs @@ -1544,6 +1544,46 @@ public void Extract_AmbiguousPl_MasksPerlQuoteLikeRegexBodies_Issue4746() && reference.SymbolName == "helper"); } + [Fact] + public void Extract_AmbiguousPl_PreservesFatArrowPairsAndShortPrologHeads_Issue4746() + { + const string content = """ + package Hybrid; + sub helper { return 1; } + my %dispatch = (s => helper(), qr => helper()); + :- module(hybrid, [q/0]). + q :- helper. + """; + + var (symbols, references) = ExtractSymbolsAndReferences("ambiguous_pl", content); + + Assert.Contains(symbols, symbol => + symbol.Kind == "function" + && symbol.Name == "q"); + Assert.Equal(3, references.Count(reference => + reference.ReferenceKind == "call" + && reference.SymbolName == "helper")); + AssertReferencesContain(references, "call", "q", "helper"); + } + + [Fact] + public void Extract_Prolog_ManyClauseTerminatorsCompletesWithinRunawayGuard_Issue4746() + { + const int terminatorCount = 50_000; + var content = "run :-\n" + string.Concat(Enumerable.Repeat(". ", terminatorCount)); + var symbols = SymbolExtractor.Extract(1, "prolog", content); + + var stopwatch = Stopwatch.StartNew(); + var references = ReferenceExtractor.Extract(1, "prolog", content, symbols); + stopwatch.Stop(); + + Assert.Empty(references); + var runawayBudget = TimeSpan.FromSeconds(5); + Assert.True( + stopwatch.Elapsed < runawayBudget, + $"Prolog clause terminator extraction took {stopwatch.Elapsed.TotalSeconds:F2}s, expected < {runawayBudget.TotalSeconds:F0}s runaway guard budget."); + } + [Fact] public void Extract_Crystal_PreservesPredicateAndBangCallsWhileMaskingCommentsAndHeredocs_Issue4746() { From 994c8952a2a31becb7816321e417002d6f498dc5 Mon Sep 17 00:00:00 2001 From: Widthdom Date: Fri, 24 Jul 2026 23:29:39 +0900 Subject: [PATCH 23/26] Resolve dynamic graph final review findings (#4746) --- changelog.d/unreleased/4746.added.md | 4 +- .../DynamicDeclarativeReferenceExtractor.cs | 75 +++++++++++++++++-- .../ReferenceExtractor.CoreExtraction.cs | 3 +- .../Indexer/Symbols/SymbolExtractor.cs | 4 +- tests/CodeIndex.Tests/DatabaseTests.cs | 8 +- .../ReferenceExtractorTests.cs | 58 +++++++++++++- 6 files changed, 138 insertions(+), 14 deletions(-) diff --git a/changelog.d/unreleased/4746.added.md b/changelog.d/unreleased/4746.added.md index 90a40dc16..b28a299b7 100644 --- a/changelog.d/unreleased/4746.added.md +++ b/changelog.d/unreleased/4746.added.md @@ -38,8 +38,8 @@ affected: ## English -- **Crystal, Groovy, Tcl, Prolog, and ambiguous `.pl` files now support conservative reference graphs (#4746)** — indexing records imports and language-appropriate calls at their source locations, recognizes command-style calls only for callables declared in the same file, parses Groovy control-statement bodies, Tcl command substitutions, inline proc declarations, control-command script arguments, and `switch` branch bodies, excludes Groovy closure parameter headers, limits Prolog calls to goal positions including top-level and same-line directives, scans Prolog clause terminators in one pass, masks Perl quote-like regular-expression bodies (including whitespace before delimiters) in ambiguous `.pl` files without consuming fat-arrow pairs or short Prolog predicate heads, preserves Tcl proc and Prolog predicate caller containers, and reports structured symbol/reference/graph capabilities after `.pl` content classification. Pre-graph extractor stamps keep graph readiness degraded until a normal index refresh rewrites affected rows, and human-readable status output identifies that refresh instead of cap-hit remediation. +- **Crystal, Groovy, Tcl, Prolog, and ambiguous `.pl` files now support conservative reference graphs (#4746)** — indexing records imports and language-appropriate calls at their source locations, recognizes command-style calls only for callables declared in the same file, parses Groovy control-statement bodies, Tcl command substitutions, inline proc declarations, control-command script arguments, and `switch` branch bodies, requires complete Tcl command words and excludes syntax markers, excludes Groovy closure parameter headers, limits Prolog calls to declared predicates in goal positions including top-level and same-line directives, scans Prolog clause terminators in one pass, masks Perl quote-like regular-expression bodies (including whitespace before delimiters) in ambiguous `.pl` files without consuming fat-arrow pairs, short Prolog predicate heads, or Perl `$#array` tokens, stops extraction after Perl `__DATA__` / `__END__` markers, preserves Tcl proc and Prolog predicate caller containers, and reports structured symbol/reference/graph capabilities after `.pl` content classification. Pre-graph extractor stamps keep graph readiness degraded until a normal index refresh rewrites affected rows, and human-readable status output identifies that refresh instead of cap-hit remediation. ## 日本語 -- **Crystal、Groovy、Tcl、Prolog、曖昧な `.pl` ファイルが保守的な reference graph に対応しました（#4746）** — import と言語ごとの call 構文をソース上の位置どおりに記録し、command-style call は同一ファイルで宣言済みの callable だけを認識し、Groovy の制御文本体、Tcl の command substitution、inline proc 宣言、制御 command の script 引数、`switch` の branch body を解析し、Groovy の closure parameter header を除外し、top-level および同一行の directive を含む Prolog の call を goal 位置に限定し、Prolog の clause terminator を1回の走査で検出し、曖昧な `.pl` では fat-arrow pair や短い Prolog predicate head を巻き込まず、delimiter 前に空白がある場合を含めて Perl の quote-like 正規表現本体をマスクし、Tcl proc / Prolog predicate の caller container を保持して、`.pl` の content classification 後に構造化された symbol / reference / graph capability を報告します。graph 対応前の extractor stamp がある場合、通常の index refresh で対象 row を更新するまで graph readiness を degraded に保ち、人間向け status 出力でも cap-hit 対応ではなくこの refresh を案内します。 +- **Crystal、Groovy、Tcl、Prolog、曖昧な `.pl` ファイルが保守的な reference graph に対応しました（#4746）** — import と言語ごとの call 構文をソース上の位置どおりに記録し、command-style call は同一ファイルで宣言済みの callable だけを認識し、Groovy の制御文本体、Tcl の command substitution、inline proc 宣言、制御 command の script 引数、`switch` の branch body を解析し、Tcl の command word 全体を照合して構文 marker を除外し、Groovy の closure parameter header を除外し、top-level および同一行の directive を含む Prolog の call を宣言済み predicate の goal 位置に限定し、Prolog の clause terminator を1回の走査で検出し、曖昧な `.pl` では fat-arrow pair、短い Prolog predicate head、Perl の `$#array` token を巻き込まず、delimiter 前に空白がある場合を含めて Perl の quote-like 正規表現本体をマスクし、Perl の `__DATA__` / `__END__` marker 以降の抽出を停止し、Tcl proc / Prolog predicate の caller container を保持して、`.pl` の content classification 後に構造化された symbol / reference / graph capability を報告します。graph 対応前の extractor stamp がある場合、通常の index refresh で対象 row を更新するまで graph readiness を degraded に保ち、人間向け status 出力でも cap-hit 対応ではなくこの refresh を案内します。 diff --git a/src/CodeIndex/Indexer/References/Languages/DynamicDeclarativeReferenceExtractor.cs b/src/CodeIndex/Indexer/References/Languages/DynamicDeclarativeReferenceExtractor.cs index a137dfdfb..45172a676 100644 --- a/src/CodeIndex/Indexer/References/Languages/DynamicDeclarativeReferenceExtractor.cs +++ b/src/CodeIndex/Indexer/References/Languages/DynamicDeclarativeReferenceExtractor.cs @@ -19,7 +19,8 @@ public ExtractionState( IReadOnlyList tclContainerScopes, IReadOnlyDictionary> prologContainersByLine, IReadOnlyList? tclCallLines, - IReadOnlyDictionary>? prologGoalCallsByLine) + IReadOnlyDictionary>? prologGoalCallsByLine, + IReadOnlySet prologDirectiveLines) { CallableNames = callableNames; DeclarationPositions = declarationPositions; @@ -28,6 +29,7 @@ public ExtractionState( PrologContainersByLine = prologContainersByLine; TclCallLines = tclCallLines; PrologGoalCallsByLine = prologGoalCallsByLine; + PrologDirectiveLines = prologDirectiveLines; } public HashSet CallableNames { get; } @@ -37,6 +39,7 @@ public ExtractionState( private IReadOnlyDictionary> PrologContainersByLine { get; } private IReadOnlyList? TclCallLines { get; } private IReadOnlyDictionary>? PrologGoalCallsByLine { get; } + private IReadOnlySet PrologDirectiveLines { get; } public SymbolRecord? ResolveContainer(int lineNumber, int column, SymbolRecord? fallback) { @@ -92,6 +95,9 @@ public bool HasPrologContainer(int lineNumber) => ContainersByLine.ContainsKey(lineNumber) || PrologContainersByLine.ContainsKey(lineNumber); + public bool IsPrologDirectiveLine(int lineNumber) => + PrologDirectiveLines.Contains(lineNumber); + public bool IsDeclarationAt(int lineNumber, int column, string name) => DeclarationPositions.Contains(new DeclarationPosition(lineNumber, column, name)); } @@ -145,7 +151,7 @@ public bool IsDeclarationAt(int lineNumber, int column, string name) => @"(?:^|;)\s*fun\s+(?:[A-Za-z_]\w*\.)?(?[A-Za-z_]\w*[?!]?)\s*\(", RegexOptions.Compiled | RegexOptions.CultureInvariant); private static readonly Regex TclCommandRegex = new( - @"(?:^|[;\[])\s*(?[A-Za-z_:][\w:.-]*)", + @"(?:^|[;\[])\s*(?[A-Za-z_:][\w:.-]*)(?=$|[\s;\]}""])", RegexOptions.Compiled | RegexOptions.CultureInvariant); private static readonly Regex CrystalHeredocOpenerRegex = new( @"<<-\s*['""]?(?[A-Za-z_]\w*)['""]?", @@ -315,6 +321,7 @@ public static string[] MaskNonCodeLines(string language, IReadOnlyList l char groovyTripleQuote = '\0'; string? crystalHeredocDelimiter = null; var insideAmbiguousPerlPod = false; + var insideAmbiguousPerlData = false; AmbiguousPerlQuoteLikeState? ambiguousPerlQuoteLikeState = null; var insideSlashyLiteral = false; var insideGroovyDollarSlashyLiteral = false; @@ -344,6 +351,21 @@ public static string[] MaskNonCodeLines(string language, IReadOnlyList l if (language == "ambiguous_pl") { var trimmed = line.AsSpan().TrimStart(); + if (insideAmbiguousPerlData) + { + result[lineIndex] = new string(' ', line.Length); + continue; + } + + var trimmedMarker = trimmed.TrimEnd(); + if (trimmedMarker.SequenceEqual("__DATA__") + || trimmedMarker.SequenceEqual("__END__")) + { + result[lineIndex] = new string(' ', line.Length); + insideAmbiguousPerlData = true; + continue; + } + if (insideAmbiguousPerlPod) { result[lineIndex] = new string(' ', line.Length); @@ -599,7 +621,8 @@ public static string[] MaskNonCodeLines(string language, IReadOnlyList l if (language == "ambiguous_pl" && ch == '#') { - if (IsPrologHashOperator(line, column)) + if (IsPrologHashOperator(line, column) + || IsPerlLastIndexVariable(line, column)) { column++; continue; @@ -827,6 +850,7 @@ private static IReadOnlyList BuildTclDeclarationContainerScop var prologContainersByLine = new Dictionary>(); string[]? tclCallLines = null; IReadOnlyDictionary>? prologGoalCallsByLine = null; + IReadOnlySet prologDirectiveLines = new HashSet(); if (language == "tcl") { var tclScriptBodyOpenings = new HashSet(); @@ -853,6 +877,7 @@ private static IReadOnlyList BuildTclDeclarationContainerScop preparedLines, containersByLine, callableNames); + prologDirectiveLines = BuildPrologDirectiveLines(preparedLines); } return new ExtractionState( @@ -862,7 +887,8 @@ private static IReadOnlyList BuildTclDeclarationContainerScop tclContainerScopes, prologContainersByLine, tclCallLines, - prologGoalCallsByLine); + prologGoalCallsByLine, + prologDirectiveLines); } public static void EmitAdditionalReferences( @@ -1120,7 +1146,8 @@ public static bool ShouldSuppressGenericCall( SymbolRecord? container) { if (language == "ambiguous_pl") - return state?.HasPrologContainer(lineNumber) == true; + return state?.HasPrologContainer(lineNumber) == true + || state?.IsPrologDirectiveLine(lineNumber) == true; if (state?.IsDeclarationAt(lineNumber, callIndex, name) == true) return true; if (language == "crystal") @@ -1224,6 +1251,20 @@ private static bool IsPrologHashOperator(string line, int column) return line[column + 1] is '=' or '<' or '>' or '\\' or '/' or '#'; } + private static bool IsPerlLastIndexVariable(string line, int column) + { + if (column <= 0 + || line[column - 1] != '$' + || column + 1 >= line.Length) + { + return false; + } + + return line[column + 1] == '{' + || line[column + 1] == '_' + || char.IsLetter(line[column + 1]); + } + private static bool IsPerlHashSigil(string line, int column) { if (column + 1 >= line.Length @@ -2481,6 +2522,7 @@ private static string[] BuildTclCallLines( : IsTclBareScriptCommandArgument( frame, wordIndex, + token, lines, new TclBraceEnd(lineIndex, column + token.Length - 1))); if (wordIndex == 0) @@ -2631,9 +2673,13 @@ private static bool IsTclExpressionArgument(TclLexicalFrame frame, int wordIndex private static bool IsTclBareScriptCommandArgument( TclLexicalFrame frame, int wordIndex, + string token, IReadOnlyList lines, TclBraceEnd wordEnd) { + if (frame.CommandName == "if" && token == "then") + return false; + return IsTclScriptArgument( frame, wordIndex, @@ -3001,6 +3047,25 @@ private static bool StartsWithPrologGoalDirective(string line) return line.AsSpan(column).StartsWith(":-", StringComparison.Ordinal); } + private static IReadOnlySet BuildPrologDirectiveLines( + IReadOnlyList lines) + { + var directiveLines = new HashSet(); + var scanningDirective = false; + for (var lineIndex = 0; lineIndex < lines.Count; lineIndex++) + { + if (!scanningDirective && !StartsWithPrologGoalDirective(lines[lineIndex])) + continue; + + scanningDirective = true; + directiveLines.Add(lineIndex + 1); + if (ContainsPrologClauseTerminator(lines[lineIndex])) + scanningDirective = false; + } + + return directiveLines; + } + private static bool IsTopLevelPrologDirectiveGoal(string line, int goalColumn) { var segmentStartColumn = 0; diff --git a/src/CodeIndex/Indexer/References/ReferenceExtractor.CoreExtraction.cs b/src/CodeIndex/Indexer/References/ReferenceExtractor.CoreExtraction.cs index 03e722245..788fafd08 100644 --- a/src/CodeIndex/Indexer/References/ReferenceExtractor.CoreExtraction.cs +++ b/src/CodeIndex/Indexer/References/ReferenceExtractor.CoreExtraction.cs @@ -2729,7 +2729,8 @@ void AddGradleDslReference(string name, int callIndex) // 平坦な CallRegex は `<[^>\n]+>` が最初の `>` で止まるため `>>(` 形を取りこぼす。 // depth-aware な fallback を足し、`Foo>()` や `new Dict>()` でも // `call` / `instantiate` を発行する。issue #263 参照。 - if (MayContainNestedGenericSyntax(preparedLine)) + if (language is not ("tcl" or "prolog" or "ambiguous_pl") + && MayContainNestedGenericSyntax(preparedLine)) { foreach (var candidate in EnumerateNestedGenericCallCandidates(preparedLine, matchedCallIndices ?? EmptyMatchedIndices)) { diff --git a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.cs b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.cs index cd2419693..50cae415b 100644 --- a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.cs +++ b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.cs @@ -23,8 +23,8 @@ public static partial class SymbolExtractor public const int XmlContractVersion = 3; public const int FunctionalLanguageContractVersion = 2; public const int DynamicLanguageContractVersion = 2; - public const int DynamicReferenceGraphContractVersion = 5; - public const int PrologReferenceGraphContractVersion = 5; + public const int DynamicReferenceGraphContractVersion = 6; + public const int PrologReferenceGraphContractVersion = 6; public const int SystemsLanguageContractVersion = 2; private static readonly string[] ExplicitReferenceGraphContractLanguages = ["crystal", "groovy", "tcl", "prolog", "ambiguous_pl"]; diff --git a/tests/CodeIndex.Tests/DatabaseTests.cs b/tests/CodeIndex.Tests/DatabaseTests.cs index a13c14c99..9a4823ddd 100644 --- a/tests/CodeIndex.Tests/DatabaseTests.cs +++ b/tests/CodeIndex.Tests/DatabaseTests.cs @@ -4724,9 +4724,11 @@ public void GetUnchangedFileId_InvalidatesPreGraphLanguageContracts_Issue4746( } [Theory] - [InlineData("groovy", 4)] - [InlineData("prolog", 4)] - [InlineData("ambiguous_pl", 4)] + [InlineData("crystal", 5)] + [InlineData("groovy", 5)] + [InlineData("tcl", 5)] + [InlineData("prolog", 5)] + [InlineData("ambiguous_pl", 5)] public void GetUnchangedFileId_InvalidatesPriorDynamicGraphContracts_Issue4746( string language, int previousContractVersion) diff --git a/tests/CodeIndex.Tests/ReferenceExtractorTests.cs b/tests/CodeIndex.Tests/ReferenceExtractorTests.cs index 00620f7d8..bf055cf2e 100644 --- a/tests/CodeIndex.Tests/ReferenceExtractorTests.cs +++ b/tests/CodeIndex.Tests/ReferenceExtractorTests.cs @@ -1343,6 +1343,30 @@ proc try_run {} { && reference.SymbolName == "helper")); } + [Fact] + public void Extract_Tcl_RejectsSyntaxMarkersAndPartialCommandWords_Issue4746() + { + const string content = """ + proc then {} { return 1 } + proc helper {} { return 1 } + proc run {value} { + if {$value} then { helper } + helper() + set generic External>() + } + """; + + var (_, references) = ExtractSymbolsAndReferences("tcl", content); + + Assert.Single(references, reference => + reference.ReferenceKind == "call" + && reference.ContainerName == "run" + && reference.SymbolName == "helper"); + Assert.DoesNotContain(references, reference => + reference.ReferenceKind == "call" + && reference.SymbolName is "then" or "External"); + } + [Fact] public void Extract_Prolog_IndexesModulesImportsAndPredicateCallsWithCaller_Issue4746() { @@ -1380,6 +1404,7 @@ public void Extract_Prolog_IndexesTopLevelDirectiveGoals_Issue4746(string langua :- initialization( helper ). + :- external_goal(). """; var (_, references) = ExtractSymbolsAndReferences(language, content); @@ -1388,6 +1413,9 @@ public void Extract_Prolog_IndexesTopLevelDirectiveGoals_Issue4746(string langua reference.ReferenceKind == "call" && reference.ContainerName == null && reference.SymbolName == "helper")); + Assert.DoesNotContain(references, reference => + reference.ReferenceKind == "call" + && reference.SymbolName == "external_goal"); } [Theory] @@ -1886,6 +1914,9 @@ public void Extract_AmbiguousPl_MasksPerlAndPrologCommentsAndPod_Issue4746() prolog_entry :- helper(). my %cache = (answer => helper()); my $fallback = undef // helper(); + my @items = (1, 2); + my $last = $#items; helper(); + my $braced_last = $#{items}; helper(); clp_entry(X) :- X #= 1, helper. prolog_comment :- %TODO @@ -1902,7 +1933,7 @@ public void Extract_AmbiguousPl_MasksPerlAndPrologCommentsAndPod_Issue4746() var (_, references) = ExtractSymbolsAndReferences("ambiguous_pl", content); - Assert.Equal(7, references.Count(reference => + Assert.Equal(9, references.Count(reference => reference.ReferenceKind == "call" && reference.SymbolName == "helper")); AssertReferencesContain(references, "call", "prolog_entry", "helper"); @@ -1912,6 +1943,31 @@ public void Extract_AmbiguousPl_MasksPerlAndPrologCommentsAndPod_Issue4746() AssertReferencesContain(references, "call", "after_inline", "helper"); } + [Theory] + [InlineData("__DATA__")] + [InlineData("__END__")] + public void Extract_AmbiguousPl_MasksPerlTerminalDataSections_Issue4746( + string marker) + { + var content = $$""" + package Hybrid; + sub helper { return 1; } + :- module(hybrid, [entry/0]). + entry :- helper. + {{marker}} + fake :- helper. + """; + + var (symbols, references) = ExtractSymbolsAndReferences("ambiguous_pl", content); + + AssertReferencesContain(references, "call", "entry", "helper"); + Assert.DoesNotContain(symbols, symbol => + symbol.Kind == "function" + && symbol.Name == "fake"); + Assert.DoesNotContain(references, reference => + reference.ContainerName == "fake"); + } + [Fact] public void Extract_CrystalAndGroovy_DoNotTreatColonLabelsAsBareCalls_Issue4746() { From bdf39dd9931d3b95783e284b671018341b95ce08 Mon Sep 17 00:00:00 2001 From: Widthdom Date: Fri, 24 Jul 2026 23:59:25 +0900 Subject: [PATCH 24/26] Fix Tcl namespace and subcommand graph edges (#4746) --- changelog.d/unreleased/4746.added.md | 4 +- .../DynamicDeclarativeReferenceExtractor.cs | 135 +++++++++++++++++- .../Indexer/References/ReferenceExtractor.cs | 10 +- .../Indexer/Symbols/SymbolExtractor.cs | 2 +- tests/CodeIndex.Tests/DatabaseTests.cs | 6 +- .../DbReaderSymbolIdentityTests.cs | 34 +++++ .../ReferenceExtractorTests.cs | 57 ++++++++ 7 files changed, 235 insertions(+), 13 deletions(-) diff --git a/changelog.d/unreleased/4746.added.md b/changelog.d/unreleased/4746.added.md index b28a299b7..f0f294617 100644 --- a/changelog.d/unreleased/4746.added.md +++ b/changelog.d/unreleased/4746.added.md @@ -38,8 +38,8 @@ affected: ## English -- **Crystal, Groovy, Tcl, Prolog, and ambiguous `.pl` files now support conservative reference graphs (#4746)** — indexing records imports and language-appropriate calls at their source locations, recognizes command-style calls only for callables declared in the same file, parses Groovy control-statement bodies, Tcl command substitutions, inline proc declarations, control-command script arguments, and `switch` branch bodies, requires complete Tcl command words and excludes syntax markers, excludes Groovy closure parameter headers, limits Prolog calls to declared predicates in goal positions including top-level and same-line directives, scans Prolog clause terminators in one pass, masks Perl quote-like regular-expression bodies (including whitespace before delimiters) in ambiguous `.pl` files without consuming fat-arrow pairs, short Prolog predicate heads, or Perl `$#array` tokens, stops extraction after Perl `__DATA__` / `__END__` markers, preserves Tcl proc and Prolog predicate caller containers, and reports structured symbol/reference/graph capabilities after `.pl` content classification. Pre-graph extractor stamps keep graph readiness degraded until a normal index refresh rewrites affected rows, and human-readable status output identifies that refresh instead of cap-hit remediation. +- **Crystal, Groovy, Tcl, Prolog, and ambiguous `.pl` files now support conservative reference graphs (#4746)** — indexing records imports and language-appropriate calls at their source locations, recognizes command-style calls only for callables declared in the same file, parses Groovy control-statement bodies, Tcl command substitutions, inline proc declarations, selected `after` / `namespace` script subcommands, and `switch` branch bodies, resolves Tcl namespace-qualified proc calls to the matching namespace, requires complete Tcl command words and excludes syntax markers, excludes Groovy closure parameter headers, limits Prolog calls to declared predicates in goal positions including top-level and same-line directives, scans Prolog clause terminators in one pass, masks Perl quote-like regular-expression bodies (including whitespace before delimiters) in ambiguous `.pl` files without consuming fat-arrow pairs, short Prolog predicate heads, or Perl `$#array` tokens, stops extraction after Perl `__DATA__` / `__END__` markers, preserves Tcl proc and Prolog predicate caller containers, and reports structured symbol/reference/graph capabilities after `.pl` content classification. Pre-graph extractor stamps keep graph readiness degraded until a normal index refresh rewrites affected rows, and human-readable status output identifies that refresh instead of cap-hit remediation. ## 日本語 -- **Crystal、Groovy、Tcl、Prolog、曖昧な `.pl` ファイルが保守的な reference graph に対応しました（#4746）** — import と言語ごとの call 構文をソース上の位置どおりに記録し、command-style call は同一ファイルで宣言済みの callable だけを認識し、Groovy の制御文本体、Tcl の command substitution、inline proc 宣言、制御 command の script 引数、`switch` の branch body を解析し、Tcl の command word 全体を照合して構文 marker を除外し、Groovy の closure parameter header を除外し、top-level および同一行の directive を含む Prolog の call を宣言済み predicate の goal 位置に限定し、Prolog の clause terminator を1回の走査で検出し、曖昧な `.pl` では fat-arrow pair、短い Prolog predicate head、Perl の `$#array` token を巻き込まず、delimiter 前に空白がある場合を含めて Perl の quote-like 正規表現本体をマスクし、Perl の `__DATA__` / `__END__` marker 以降の抽出を停止し、Tcl proc / Prolog predicate の caller container を保持して、`.pl` の content classification 後に構造化された symbol / reference / graph capability を報告します。graph 対応前の extractor stamp がある場合、通常の index refresh で対象 row を更新するまで graph readiness を degraded に保ち、人間向け status 出力でも cap-hit 対応ではなくこの refresh を案内します。 +- **Crystal、Groovy、Tcl、Prolog、曖昧な `.pl` ファイルが保守的な reference graph に対応しました（#4746）** — import と言語ごとの call 構文をソース上の位置どおりに記録し、command-style call は同一ファイルで宣言済みの callable だけを認識し、Groovy の制御文本体、Tcl の command substitution、inline proc 宣言、選択された `after` / `namespace` subcommand の script 引数、`switch` の branch body を解析し、Tcl の名前空間修飾 proc call を対応する名前空間へ解決し、Tcl の command word 全体を照合して構文 marker を除外し、Groovy の closure parameter header を除外し、top-level および同一行の directive を含む Prolog の call を宣言済み predicate の goal 位置に限定し、Prolog の clause terminator を1回の走査で検出し、曖昧な `.pl` では fat-arrow pair、短い Prolog predicate head、Perl の `$#array` token を巻き込まず、delimiter 前に空白がある場合を含めて Perl の quote-like 正規表現本体をマスクし、Perl の `__DATA__` / `__END__` marker 以降の抽出を停止し、Tcl proc / Prolog predicate の caller container を保持して、`.pl` の content classification 後に構造化された symbol / reference / graph capability を報告します。graph 対応前の extractor stamp がある場合、通常の index refresh で対象 row を更新するまで graph readiness を degraded に保ち、人間向け status 出力でも cap-hit 対応ではなくこの refresh を案内します。 diff --git a/src/CodeIndex/Indexer/References/Languages/DynamicDeclarativeReferenceExtractor.cs b/src/CodeIndex/Indexer/References/Languages/DynamicDeclarativeReferenceExtractor.cs index 45172a676..85fe87aa9 100644 --- a/src/CodeIndex/Indexer/References/Languages/DynamicDeclarativeReferenceExtractor.cs +++ b/src/CodeIndex/Indexer/References/Languages/DynamicDeclarativeReferenceExtractor.cs @@ -17,6 +17,7 @@ public ExtractionState( HashSet declarationPositions, IReadOnlyDictionary containersByLine, IReadOnlyList tclContainerScopes, + IReadOnlyDictionary tclQualifiedCallableTargets, IReadOnlyDictionary> prologContainersByLine, IReadOnlyList? tclCallLines, IReadOnlyDictionary>? prologGoalCallsByLine, @@ -26,6 +27,7 @@ public ExtractionState( DeclarationPositions = declarationPositions; ContainersByLine = containersByLine; TclContainerScopes = tclContainerScopes; + TclQualifiedCallableTargets = tclQualifiedCallableTargets; PrologContainersByLine = prologContainersByLine; TclCallLines = tclCallLines; PrologGoalCallsByLine = prologGoalCallsByLine; @@ -36,6 +38,7 @@ public ExtractionState( private HashSet DeclarationPositions { get; } public IReadOnlyDictionary ContainersByLine { get; } private IReadOnlyList TclContainerScopes { get; } + private IReadOnlyDictionary TclQualifiedCallableTargets { get; } private IReadOnlyDictionary> PrologContainersByLine { get; } private IReadOnlyList? TclCallLines { get; } private IReadOnlyDictionary>? PrologGoalCallsByLine { get; } @@ -80,6 +83,38 @@ public string GetCallScanLine(string language, int lineNumber, string preparedLi return PreparePrologCallScanLine(language, preparedLine, isClauseContinuation); } + public bool TryResolveTclCallable( + string callName, + out string referenceName, + out string? targetQualifier, + out int referenceNameOffset) + { + referenceName = callName; + targetQualifier = null; + referenceNameOffset = 0; + if (CallableNames.Contains(callName)) + return true; + + var normalizedCallName = NormalizeTclQualifiedName(callName); + if (TclQualifiedCallableTargets.TryGetValue(normalizedCallName, out var target)) + { + referenceName = target.Name; + targetQualifier = target.Qualifier; + referenceNameOffset = callName.LastIndexOf("::", StringComparison.Ordinal) + 2; + return true; + } + + if (!normalizedCallName.Contains("::", StringComparison.Ordinal) + && CallableNames.Contains(normalizedCallName)) + { + referenceName = normalizedCallName; + referenceNameOffset = callName.Length - normalizedCallName.Length; + return true; + } + + return false; + } + public IReadOnlyList GetPrologGoalCalls(int lineNumber) { if (PrologGoalCallsByLine != null @@ -182,6 +217,10 @@ private readonly record struct TclBraceEnd( int Line, int Column); + internal readonly record struct TclCallableTarget( + string Name, + string Qualifier); + private enum TclLexicalFrameKind { Script, @@ -266,6 +305,7 @@ private sealed class TclLexicalFrame( public bool WordStart { get; set; } = true; public int WordIndex { get; set; } public string? CommandName { get; set; } + public string? FirstArgument { get; set; } public string? LastBareWord { get; set; } public int LastBareWordIndex { get; set; } = -1; public int SwitchStringWordIndex { get; set; } = -1; @@ -282,6 +322,7 @@ public void CopyCommandStateFrom(TclLexicalFrame source) WordStart = source.WordStart; WordIndex = source.WordIndex; CommandName = source.CommandName; + FirstArgument = source.FirstArgument; LastBareWord = source.LastBareWord; LastBareWordIndex = source.LastBareWordIndex; SwitchStringWordIndex = source.SwitchStringWordIndex; @@ -299,6 +340,7 @@ public void ResetCommand() WordStart = true; WordIndex = 0; CommandName = null; + FirstArgument = null; LastBareWord = null; LastBareWordIndex = -1; SwitchStringWordIndex = -1; @@ -847,12 +889,32 @@ private static IReadOnlyList BuildTclDeclarationContainerScop var containersByLine = new Dictionary(); var tclContainerScopes = new List(); + var tclQualifiedCallableTargets = + new Dictionary(StringComparer.Ordinal); var prologContainersByLine = new Dictionary>(); string[]? tclCallLines = null; IReadOnlyDictionary>? prologGoalCallsByLine = null; IReadOnlySet prologDirectiveLines = new HashSet(); if (language == "tcl") { + foreach (var symbol in symbols) + { + if (symbol.Kind is not ("function" or "lambda" or "operator") + || string.IsNullOrWhiteSpace(symbol.ContainerName)) + { + continue; + } + + var normalizedContainer = NormalizeTclQualifiedName(symbol.ContainerName); + var normalizedName = NormalizeTclQualifiedName(symbol.Name); + if (normalizedContainer.Length == 0 || normalizedName.Length == 0) + continue; + + tclQualifiedCallableTargets[ + $"{normalizedContainer}::{normalizedName}"] = + new TclCallableTarget(symbol.Name, symbol.ContainerName); + } + var tclScriptBodyOpenings = new HashSet(); var tclBraceEnds = BuildTclBraceEndPositions(structuralLines); AddTclContainers( @@ -885,6 +947,7 @@ private static IReadOnlyList BuildTclDeclarationContainerScop declarationPositions, containersByLine, tclContainerScopes, + tclQualifiedCallableTargets, prologContainersByLine, tclCallLines, prologGoalCallsByLine, @@ -1003,6 +1066,41 @@ public static void EmitAdditionalReferences( foreach (Match match in BoundedRegex.EnumerateMatches(callRegex, preparedLine)) { var nameGroup = match.Groups["name"]; + if (language == "tcl") + { + if (!state.TryResolveTclCallable( + nameGroup.Value, + out var referenceName, + out var targetQualifier, + out var referenceNameOffset)) + { + continue; + } + + if (targetQualifier != null + || !string.Equals(referenceName, nameGroup.Value, StringComparison.Ordinal)) + { + ReferenceExtractor.AddReference( + references, + seen, + fileId, + referenceName, + nameGroup.Index + referenceNameOffset, + "call", + context, + lineNumber, + resolveContainerForCall(nameGroup.Index), + language, + targetQualifier); + } + else + { + addCallLikeReference(nameGroup.Value, nameGroup.Index); + } + + continue; + } + if (!state.CallableNames.Contains(nameGroup.Value)) continue; if (language == "groovy" @@ -2384,16 +2482,18 @@ private static string[] BuildTclCallLines( if (frame.WordStart) { var wordIndex = frame.WordIndex++; + var token = GetTclQuotedWordToken(line, column); isConcatenatedScriptArgument = IsTclConcatenatedScriptArgument( frame, wordIndex, - GetTclQuotedWordToken(line, column)); + token); isScriptArgument = isConcatenatedScriptArgument || IsTclScriptArgument( frame, wordIndex, lines, braceEnd: null); + UpdateTclFirstArgument(frame, wordIndex, token); UpdateTclSwitchArgumentState(frame, wordIndex, string.Empty); UpdateTclTryArgumentState(frame, wordIndex, string.Empty, isScriptArgument); } @@ -2443,11 +2543,16 @@ private static string[] BuildTclCallLines( wordIndex, lines, braceEnd); + var token = GetTclBracedWordToken( + lines, + lineIndex, + column, + braceEnd); var isConcatenatedScriptArgument = !isSwitchTable && IsTclConcatenatedScriptArgument( frame, wordIndex, - GetTclBracedWordToken(lines, lineIndex, column, braceEnd)); + token); var isExpressionArgument = !isSwitchTable && IsTclExpressionArgument(frame, wordIndex); var isScriptArgument = !isSwitchTable @@ -2458,6 +2563,7 @@ private static string[] BuildTclCallLines( wordIndex, lines, braceEnd)); + UpdateTclFirstArgument(frame, wordIndex, token); UpdateTclSwitchArgumentState(frame, wordIndex, string.Empty); UpdateTclTryArgumentState( frame, @@ -2529,6 +2635,7 @@ private static string[] BuildTclCallLines( frame.CommandName = token; else { + UpdateTclFirstArgument(frame, wordIndex, token); UpdateTclSwitchArgumentState(frame, wordIndex, token); UpdateTclTryArgumentState( frame, @@ -2646,9 +2753,12 @@ private static bool IsTclScriptArgument( "catch" => wordIndex == 1, "for" => wordIndex is 1 or 3 or 4, "proc" => wordIndex == 3, - "after" => wordIndex == 2, + "after" => wordIndex == 2 + && frame.FirstArgument is not null + && frame.FirstArgument is not ("cancel" or "info"), "try" => wordIndex == frame.TryScriptWordIndex, - "namespace" => wordIndex == 3, + "namespace" => wordIndex == 3 + && frame.FirstArgument == "eval", "dict" => frame.LastBareWord == "for" && wordIndex == frame.LastBareWordIndex + 3, "switch" => frame.SwitchStringWordIndex >= 0 @@ -2778,6 +2888,7 @@ private static bool ProcessTclConcatenatedBareWord( } else { + UpdateTclFirstArgument(state, wordIndex, token); UpdateTclSwitchArgumentState(state, wordIndex, token); UpdateTclTryArgumentState( state, @@ -2815,6 +2926,22 @@ private static bool ProcessTclConcatenatedBareWord( : line.Substring(startColumn + 1, endColumn - startColumn - 2); } + private static string NormalizeTclQualifiedName(string name) + { + while (name.StartsWith("::", StringComparison.Ordinal)) + name = name[2..]; + return name; + } + + private static void UpdateTclFirstArgument( + TclLexicalFrame frame, + int wordIndex, + string? token) + { + if (wordIndex == 1 && token != null) + frame.FirstArgument = token; + } + private static void MarkTclBareScriptCommandBoundary(char[] buffer, int commandColumn) { var boundaryColumn = commandColumn - 1; diff --git a/src/CodeIndex/Indexer/References/ReferenceExtractor.cs b/src/CodeIndex/Indexer/References/ReferenceExtractor.cs index d35006070..a1d85cdb0 100644 --- a/src/CodeIndex/Indexer/References/ReferenceExtractor.cs +++ b/src/CodeIndex/Indexer/References/ReferenceExtractor.cs @@ -1935,7 +1935,8 @@ internal static void AddReference( string context, int lineNumber, SymbolRecord? container, - string? language = null) + string? language = null, + string? targetQualifier = null) { AddReference( references, @@ -1947,7 +1948,8 @@ internal static void AddReference( context, lineNumber, container, - language); + language, + targetQualifier); } internal static void AddReference( @@ -1960,7 +1962,8 @@ internal static void AddReference( string context, int lineNumber, SymbolRecord? container, - string? language = null) + string? language = null, + string? targetQualifier = null) { var column = nameIndex + 1; var dedupeKey = CreateReferenceDedupeKey(fileId, language, lineNumber, column, referenceKind, name, container); @@ -1977,6 +1980,7 @@ internal static void AddReference( Context = context, ContainerKind = container?.Kind, ContainerName = container?.Name, + TargetQualifier = targetQualifier, IsSelfReference = IsSameReferenceName(container?.Name, name), }); } diff --git a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.cs b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.cs index 50cae415b..b1b17d543 100644 --- a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.cs +++ b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.cs @@ -23,7 +23,7 @@ public static partial class SymbolExtractor public const int XmlContractVersion = 3; public const int FunctionalLanguageContractVersion = 2; public const int DynamicLanguageContractVersion = 2; - public const int DynamicReferenceGraphContractVersion = 6; + public const int DynamicReferenceGraphContractVersion = 7; public const int PrologReferenceGraphContractVersion = 6; public const int SystemsLanguageContractVersion = 2; private static readonly string[] ExplicitReferenceGraphContractLanguages = diff --git a/tests/CodeIndex.Tests/DatabaseTests.cs b/tests/CodeIndex.Tests/DatabaseTests.cs index 9a4823ddd..ab448ffaa 100644 --- a/tests/CodeIndex.Tests/DatabaseTests.cs +++ b/tests/CodeIndex.Tests/DatabaseTests.cs @@ -4724,9 +4724,9 @@ public void GetUnchangedFileId_InvalidatesPreGraphLanguageContracts_Issue4746( } [Theory] - [InlineData("crystal", 5)] - [InlineData("groovy", 5)] - [InlineData("tcl", 5)] + [InlineData("crystal", 6)] + [InlineData("groovy", 6)] + [InlineData("tcl", 6)] [InlineData("prolog", 5)] [InlineData("ambiguous_pl", 5)] public void GetUnchangedFileId_InvalidatesPriorDynamicGraphContracts_Issue4746( diff --git a/tests/CodeIndex.Tests/DbReaderSymbolIdentityTests.cs b/tests/CodeIndex.Tests/DbReaderSymbolIdentityTests.cs index de0464932..bfdc01db1 100644 --- a/tests/CodeIndex.Tests/DbReaderSymbolIdentityTests.cs +++ b/tests/CodeIndex.Tests/DbReaderSymbolIdentityTests.cs @@ -122,6 +122,40 @@ public static class Caller Assert.Equal("Invoke", caller.CallerName); } + [Fact] + public void SymbolIdentity_TclNamespaceQualifiedCallResolvesMatchingProc_Issue4746() + { + const string path = "src/identity/namespaced.tcl"; + InsertIndexedFile(path, "tcl", """ + namespace eval ::alpha { proc worker {} { return 1 } } + namespace eval ::beta { proc worker {} { return 2 } } + proc driver {} { + ::beta::worker + } + """); + + var reference = Assert.Single(_reader.SearchReferences( + "worker", + limit: 10, + lang: "tcl", + pathPatterns: [path], + exact: true)); + Assert.Equal("resolved", reference.ResolutionState); + Assert.NotNull(reference.TargetSymbolId); + Assert.Equal(1, reference.ResolutionCandidateCount); + + using var targetContainer = _db.Connection.CreateCommand(); + targetContainer.CommandText = """ + SELECT container_name + FROM symbols + WHERE id = @target_symbol_id + """; + targetContainer.Parameters.AddWithValue( + "@target_symbol_id", + reference.TargetSymbolId!.Value); + Assert.Equal("::beta", (string)targetContainer.ExecuteScalar()!); + } + [Fact] public void SymbolIdentity_CSharpGlobalQualifierResolvesQualifiedEnumMember() { diff --git a/tests/CodeIndex.Tests/ReferenceExtractorTests.cs b/tests/CodeIndex.Tests/ReferenceExtractorTests.cs index bf055cf2e..c00c656fd 100644 --- a/tests/CodeIndex.Tests/ReferenceExtractorTests.cs +++ b/tests/CodeIndex.Tests/ReferenceExtractorTests.cs @@ -1261,6 +1261,63 @@ set literal { proc fake {} { helper } } && reference.SymbolName == "helper"); } + [Fact] + public void Extract_Tcl_OnlyIndexesScriptsForSelectedSubcommands_Issue4746() + { + const string content = """ + proc helper {} { return 1 } + proc run {} { + after idle helper + namespace eval ::tmp { helper } + after cancel helper + after info helper + namespace export alpha helper + } + """; + + var (_, references) = ExtractSymbolsAndReferences("tcl", content); + + Assert.Equal(2, references.Count(reference => + reference.ReferenceKind == "call" + && reference.ContainerName == "run" + && reference.SymbolName == "helper")); + } + + [Fact] + public void Extract_Tcl_ResolvesNamespaceQualifiedProcCalls_Issue4746() + { + const string content = """ + namespace eval ::alpha { proc worker {} { return 1 } } + namespace eval ::beta { proc worker {} { return 2 } } + proc driver {} { + alpha::worker + ::beta::worker + } + """; + + var (_, references) = ExtractSymbolsAndReferences("tcl", content); + var calls = references + .Where(reference => + reference.ReferenceKind == "call" + && reference.ContainerName == "driver" + && reference.SymbolName == "worker") + .OrderBy(reference => reference.Line) + .ToList(); + + Assert.Collection( + calls, + reference => + { + Assert.Equal("::alpha", reference.TargetQualifier); + Assert.Equal(12, reference.Column); + }, + reference => + { + Assert.Equal("::beta", reference.TargetQualifier); + Assert.Equal(13, reference.Column); + }); + } + [Fact] public void Extract_Tcl_IndexesPackagesAndProcCommandCallsWithCaller_Issue4746() { From 1418bec15a7ceb8520972b2b712edf6105d43a3d Mon Sep 17 00:00:00 2001 From: Widthdom Date: Sat, 25 Jul 2026 00:31:18 +0900 Subject: [PATCH 25/26] Resolve final dynamic graph review findings (#4746) --- changelog.d/unreleased/4746.added.md | 4 +- .../DynamicDeclarativeReferenceExtractor.cs | 162 +++++++++++++++++- .../Indexer/Symbols/SymbolExtractor.cs | 4 +- tests/CodeIndex.Tests/DatabaseTests.cs | 10 +- .../ReferenceExtractorTests.cs | 43 ++++- 5 files changed, 205 insertions(+), 18 deletions(-) diff --git a/changelog.d/unreleased/4746.added.md b/changelog.d/unreleased/4746.added.md index f0f294617..202cbfcfc 100644 --- a/changelog.d/unreleased/4746.added.md +++ b/changelog.d/unreleased/4746.added.md @@ -38,8 +38,8 @@ affected: ## English -- **Crystal, Groovy, Tcl, Prolog, and ambiguous `.pl` files now support conservative reference graphs (#4746)** — indexing records imports and language-appropriate calls at their source locations, recognizes command-style calls only for callables declared in the same file, parses Groovy control-statement bodies, Tcl command substitutions, inline proc declarations, selected `after` / `namespace` script subcommands, and `switch` branch bodies, resolves Tcl namespace-qualified proc calls to the matching namespace, requires complete Tcl command words and excludes syntax markers, excludes Groovy closure parameter headers, limits Prolog calls to declared predicates in goal positions including top-level and same-line directives, scans Prolog clause terminators in one pass, masks Perl quote-like regular-expression bodies (including whitespace before delimiters) in ambiguous `.pl` files without consuming fat-arrow pairs, short Prolog predicate heads, or Perl `$#array` tokens, stops extraction after Perl `__DATA__` / `__END__` markers, preserves Tcl proc and Prolog predicate caller containers, and reports structured symbol/reference/graph capabilities after `.pl` content classification. Pre-graph extractor stamps keep graph readiness degraded until a normal index refresh rewrites affected rows, and human-readable status output identifies that refresh instead of cap-hit remediation. +- **Crystal, Groovy, Tcl, Prolog, and ambiguous `.pl` files now support conservative reference graphs (#4746)** — indexing records imports and language-appropriate calls at their source locations, recognizes command-style calls only for callables declared in the same file, parses Groovy control-statement bodies, Tcl command substitutions, inline proc declarations, variadic `after` / `namespace eval` script arguments, `dict for` bodies after bare dictionary words, and `switch` branch bodies, resolves Tcl namespace-qualified proc calls to the matching namespace, requires complete Tcl command words and excludes syntax markers, excludes Groovy closure parameter headers, limits Prolog calls to declared predicates in goal positions including top-level and same-line directives, scans Prolog clause terminators in one pass, masks Perl quote-like regular-expression and heredoc bodies (including whitespace before quote-like delimiters and indented heredocs) in ambiguous `.pl` files without consuming fat-arrow pairs, short Prolog predicate heads, or Perl `$#array` tokens, stops extraction after Perl `__DATA__` / `__END__` markers, preserves Tcl proc and Prolog predicate caller containers, and reports structured symbol/reference/graph capabilities after `.pl` content classification. Pre-graph extractor stamps keep graph readiness degraded until a normal index refresh rewrites affected rows, and human-readable status output identifies that refresh instead of cap-hit remediation. ## 日本語 -- **Crystal、Groovy、Tcl、Prolog、曖昧な `.pl` ファイルが保守的な reference graph に対応しました（#4746）** — import と言語ごとの call 構文をソース上の位置どおりに記録し、command-style call は同一ファイルで宣言済みの callable だけを認識し、Groovy の制御文本体、Tcl の command substitution、inline proc 宣言、選択された `after` / `namespace` subcommand の script 引数、`switch` の branch body を解析し、Tcl の名前空間修飾 proc call を対応する名前空間へ解決し、Tcl の command word 全体を照合して構文 marker を除外し、Groovy の closure parameter header を除外し、top-level および同一行の directive を含む Prolog の call を宣言済み predicate の goal 位置に限定し、Prolog の clause terminator を1回の走査で検出し、曖昧な `.pl` では fat-arrow pair、短い Prolog predicate head、Perl の `$#array` token を巻き込まず、delimiter 前に空白がある場合を含めて Perl の quote-like 正規表現本体をマスクし、Perl の `__DATA__` / `__END__` marker 以降の抽出を停止し、Tcl proc / Prolog predicate の caller container を保持して、`.pl` の content classification 後に構造化された symbol / reference / graph capability を報告します。graph 対応前の extractor stamp がある場合、通常の index refresh で対象 row を更新するまで graph readiness を degraded に保ち、人間向け status 出力でも cap-hit 対応ではなくこの refresh を案内します。 +- **Crystal、Groovy、Tcl、Prolog、曖昧な `.pl` ファイルが保守的な reference graph に対応しました（#4746）** — import と言語ごとの call 構文をソース上の位置どおりに記録し、command-style call は同一ファイルで宣言済みの callable だけを認識し、Groovy の制御文本体、Tcl の command substitution、inline proc 宣言、可変長の `after` / `namespace eval` script 引数、bare な dictionary word に続く `dict for` body、`switch` の branch body を解析し、Tcl の名前空間修飾 proc call を対応する名前空間へ解決し、Tcl の command word 全体を照合して構文 marker を除外し、Groovy の closure parameter header を除外し、top-level および同一行の directive を含む Prolog の call を宣言済み predicate の goal 位置に限定し、Prolog の clause terminator を1回の走査で検出し、曖昧な `.pl` では fat-arrow pair、短い Prolog predicate head、Perl の `$#array` token を巻き込まず、delimiter 前に空白がある場合を含む Perl の quote-like 正規表現本体と indented heredoc を含む heredoc 本体をマスクし、Perl の `__DATA__` / `__END__` marker 以降の抽出を停止し、Tcl proc / Prolog predicate の caller container を保持して、`.pl` の content classification 後に構造化された symbol / reference / graph capability を報告します。graph 対応前の extractor stamp がある場合、通常の index refresh で対象 row を更新するまで graph readiness を degraded に保ち、人間向け status 出力でも cap-hit 対応ではなくこの refresh を案内します。 diff --git a/src/CodeIndex/Indexer/References/Languages/DynamicDeclarativeReferenceExtractor.cs b/src/CodeIndex/Indexer/References/Languages/DynamicDeclarativeReferenceExtractor.cs index 85fe87aa9..6a0d3cfad 100644 --- a/src/CodeIndex/Indexer/References/Languages/DynamicDeclarativeReferenceExtractor.cs +++ b/src/CodeIndex/Indexer/References/Languages/DynamicDeclarativeReferenceExtractor.cs @@ -270,6 +270,19 @@ private sealed class AmbiguousPerlQuoteLikeState( public bool AwaitingNextOpeningDelimiter { get; set; } } + private readonly record struct AmbiguousPerlHeredocDelimiter( + string Value, + bool AllowIndent) + { + public bool MatchesTerminator(string line) + { + var candidate = line.AsSpan().TrimEnd(); + if (AllowIndent) + candidate = candidate.TrimStart(); + return candidate.SequenceEqual(Value); + } + } + private static readonly IReadOnlyDictionary> PrologMetaGoalArguments = new Dictionary>(StringComparer.Ordinal) { @@ -313,6 +326,7 @@ private sealed class TclLexicalFrame( public bool SwitchOptionValuePending { get; set; } public int TryClauseWordIndex { get; set; } = 2; public int TryScriptWordIndex { get; set; } = 1; + public int DictScriptWordIndex { get; set; } = -1; public int UplevelScriptWordIndex { get; set; } = -1; public TclLexicalFrame? ConcatenatedScriptState { get; set; } @@ -330,6 +344,7 @@ public void CopyCommandStateFrom(TclLexicalFrame source) SwitchOptionValuePending = source.SwitchOptionValuePending; TryClauseWordIndex = source.TryClauseWordIndex; TryScriptWordIndex = source.TryScriptWordIndex; + DictScriptWordIndex = source.DictScriptWordIndex; UplevelScriptWordIndex = source.UplevelScriptWordIndex; ConcatenatedScriptState = source.ConcatenatedScriptState; } @@ -348,6 +363,7 @@ public void ResetCommand() SwitchOptionValuePending = false; TryClauseWordIndex = 2; TryScriptWordIndex = 1; + DictScriptWordIndex = -1; UplevelScriptWordIndex = -1; ConcatenatedScriptState = null; } @@ -364,6 +380,8 @@ public static string[] MaskNonCodeLines(string language, IReadOnlyList l string? crystalHeredocDelimiter = null; var insideAmbiguousPerlPod = false; var insideAmbiguousPerlData = false; + var ambiguousPerlHeredocDelimiters = + new Queue(); AmbiguousPerlQuoteLikeState? ambiguousPerlQuoteLikeState = null; var insideSlashyLiteral = false; var insideGroovyDollarSlashyLiteral = false; @@ -393,6 +411,14 @@ public static string[] MaskNonCodeLines(string language, IReadOnlyList l if (language == "ambiguous_pl") { var trimmed = line.AsSpan().TrimStart(); + if (ambiguousPerlHeredocDelimiters.TryPeek(out var heredocDelimiter)) + { + result[lineIndex] = new string(' ', line.Length); + if (heredocDelimiter.MatchesTerminator(line)) + ambiguousPerlHeredocDelimiters.Dequeue(); + continue; + } + if (insideAmbiguousPerlData) { result[lineIndex] = new string(' ', line.Length); @@ -729,6 +755,13 @@ public static string[] MaskNonCodeLines(string language, IReadOnlyList l } result[lineIndex] = new string(buffer); + if (language == "ambiguous_pl") + { + EnqueueAmbiguousPerlHeredocDelimiters( + line, + result[lineIndex], + ambiguousPerlHeredocDelimiters); + } } return result; @@ -1513,6 +1546,101 @@ private static bool IsLikelySlashyLiteralStart(string line, int column) || token.SequenceEqual("until"); } + private static void EnqueueAmbiguousPerlHeredocDelimiters( + string line, + string maskedLine, + Queue delimiters) + { + for (var column = 0; column + 1 < line.Length;) + { + if (line[column] is '\'' or '"' or '`') + { + column = SkipQuotedToken(line, column, line[column]); + continue; + } + + if (line[column] != '<' + || line[column + 1] != '<' + || maskedLine[column] != '<' + || maskedLine[column + 1] != '<') + { + column++; + continue; + } + + var delimiterColumn = column + 2; + var allowIndent = delimiterColumn < line.Length + && line[delimiterColumn] == '~'; + if (allowIndent) + delimiterColumn++; + + var beforeWhitespace = delimiterColumn; + delimiterColumn = SkipWhitespace(line, delimiterColumn); + var hasWhitespace = delimiterColumn > beforeWhitespace; + if (delimiterColumn >= line.Length) + break; + + string? delimiter = null; + var nextColumn = delimiterColumn + 1; + if (line[delimiterColumn] is '\'' or '"' or '`') + { + var quote = line[delimiterColumn]; + var closingColumn = delimiterColumn + 1; + while (closingColumn < line.Length + && line[closingColumn] != quote) + { + if (line[closingColumn] == '\\' + && closingColumn + 1 < line.Length) + { + closingColumn += 2; + } + else + { + closingColumn++; + } + } + + if (closingColumn < line.Length) + { + delimiter = line[(delimiterColumn + 1)..closingColumn]; + nextColumn = closingColumn + 1; + } + } + else + { + if (line[delimiterColumn] == '\\') + { + delimiterColumn++; + } + else if (hasWhitespace) + { + column += 2; + continue; + } + + var delimiterEnd = delimiterColumn; + while (delimiterEnd < line.Length + && (char.IsLetterOrDigit(line[delimiterEnd]) + || line[delimiterEnd] == '_')) + { + delimiterEnd++; + } + + if (delimiterEnd > delimiterColumn + && (char.IsLetter(line[delimiterColumn]) + || line[delimiterColumn] == '_')) + { + delimiter = line[delimiterColumn..delimiterEnd]; + nextColumn = delimiterEnd; + } + } + + if (!string.IsNullOrEmpty(delimiter)) + delimiters.Enqueue(new AmbiguousPerlHeredocDelimiter(delimiter, allowIndent)); + column = Math.Max(nextColumn, column + 2); + } + } + private static bool TryBeginAmbiguousPerlQuoteLikeLiteral( string line, char[] buffer, @@ -2494,6 +2622,7 @@ private static string[] BuildTclCallLines( lines, braceEnd: null); UpdateTclFirstArgument(frame, wordIndex, token); + UpdateTclDictArgumentState(frame, wordIndex, token); UpdateTclSwitchArgumentState(frame, wordIndex, string.Empty); UpdateTclTryArgumentState(frame, wordIndex, string.Empty, isScriptArgument); } @@ -2517,6 +2646,7 @@ private static string[] BuildTclCallLines( if (frame.WordStart) { var wordIndex = frame.WordIndex++; + UpdateTclDictArgumentState(frame, wordIndex, token: null); UpdateTclSwitchArgumentState(frame, wordIndex, string.Empty); UpdateTclTryArgumentState( frame, @@ -2564,6 +2694,7 @@ private static string[] BuildTclCallLines( lines, braceEnd)); UpdateTclFirstArgument(frame, wordIndex, token); + UpdateTclDictArgumentState(frame, wordIndex, token); UpdateTclSwitchArgumentState(frame, wordIndex, string.Empty); UpdateTclTryArgumentState( frame, @@ -2636,6 +2767,7 @@ private static string[] BuildTclCallLines( else { UpdateTclFirstArgument(frame, wordIndex, token); + UpdateTclDictArgumentState(frame, wordIndex, token); UpdateTclSwitchArgumentState(frame, wordIndex, token); UpdateTclTryArgumentState( frame, @@ -2753,14 +2885,8 @@ private static bool IsTclScriptArgument( "catch" => wordIndex == 1, "for" => wordIndex is 1 or 3 or 4, "proc" => wordIndex == 3, - "after" => wordIndex == 2 - && frame.FirstArgument is not null - && frame.FirstArgument is not ("cancel" or "info"), "try" => wordIndex == frame.TryScriptWordIndex, - "namespace" => wordIndex == 3 - && frame.FirstArgument == "eval", - "dict" => frame.LastBareWord == "for" - && wordIndex == frame.LastBareWordIndex + 3, + "dict" => wordIndex == frame.DictScriptWordIndex, "switch" => frame.SwitchStringWordIndex >= 0 && wordIndex - frame.SwitchStringWordIndex >= 2 && (wordIndex - frame.SwitchStringWordIndex) % 2 == 0, @@ -2804,6 +2930,14 @@ private static bool IsTclConcatenatedScriptArgument( { if (frame.CommandName == "eval") return wordIndex >= 1; + if (frame.CommandName == "after") + { + return wordIndex >= 2 + && frame.FirstArgument is not null + && frame.FirstArgument is not ("cancel" or "info"); + } + if (frame.CommandName == "namespace") + return wordIndex >= 3 && frame.FirstArgument == "eval"; if (frame.CommandName != "uplevel") return false; if (frame.UplevelScriptWordIndex >= 0) @@ -2889,6 +3023,7 @@ private static bool ProcessTclConcatenatedBareWord( else { UpdateTclFirstArgument(state, wordIndex, token); + UpdateTclDictArgumentState(state, wordIndex, token); UpdateTclSwitchArgumentState(state, wordIndex, token); UpdateTclTryArgumentState( state, @@ -2942,6 +3077,19 @@ private static void UpdateTclFirstArgument( frame.FirstArgument = token; } + private static void UpdateTclDictArgumentState( + TclLexicalFrame frame, + int wordIndex, + string? token) + { + if (frame.CommandName == "dict" + && wordIndex == 1 + && token == "for") + { + frame.DictScriptWordIndex = wordIndex + 3; + } + } + private static void MarkTclBareScriptCommandBoundary(char[] buffer, int commandColumn) { var boundaryColumn = commandColumn - 1; diff --git a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.cs b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.cs index b1b17d543..d3b415174 100644 --- a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.cs +++ b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.cs @@ -23,8 +23,8 @@ public static partial class SymbolExtractor public const int XmlContractVersion = 3; public const int FunctionalLanguageContractVersion = 2; public const int DynamicLanguageContractVersion = 2; - public const int DynamicReferenceGraphContractVersion = 7; - public const int PrologReferenceGraphContractVersion = 6; + public const int DynamicReferenceGraphContractVersion = 8; + public const int PrologReferenceGraphContractVersion = 7; public const int SystemsLanguageContractVersion = 2; private static readonly string[] ExplicitReferenceGraphContractLanguages = ["crystal", "groovy", "tcl", "prolog", "ambiguous_pl"]; diff --git a/tests/CodeIndex.Tests/DatabaseTests.cs b/tests/CodeIndex.Tests/DatabaseTests.cs index ab448ffaa..772deb8bf 100644 --- a/tests/CodeIndex.Tests/DatabaseTests.cs +++ b/tests/CodeIndex.Tests/DatabaseTests.cs @@ -4724,11 +4724,11 @@ public void GetUnchangedFileId_InvalidatesPreGraphLanguageContracts_Issue4746( } [Theory] - [InlineData("crystal", 6)] - [InlineData("groovy", 6)] - [InlineData("tcl", 6)] - [InlineData("prolog", 5)] - [InlineData("ambiguous_pl", 5)] + [InlineData("crystal", 7)] + [InlineData("groovy", 7)] + [InlineData("tcl", 7)] + [InlineData("prolog", 6)] + [InlineData("ambiguous_pl", 6)] public void GetUnchangedFileId_InvalidatesPriorDynamicGraphContracts_Issue4746( string language, int previousContractVersion) diff --git a/tests/CodeIndex.Tests/ReferenceExtractorTests.cs b/tests/CodeIndex.Tests/ReferenceExtractorTests.cs index c00c656fd..977490bea 100644 --- a/tests/CodeIndex.Tests/ReferenceExtractorTests.cs +++ b/tests/CodeIndex.Tests/ReferenceExtractorTests.cs @@ -1220,12 +1220,13 @@ uplevel helper arg uplevel #0 helper arg foreach value {1 2} helper lmap value {1 2} helper + dict for {key value} key\ value { helper } } """; var (_, references) = ExtractSymbolsAndReferences("tcl", content); - Assert.Equal(6, references.Count(reference => + Assert.Equal(7, references.Count(reference => reference.ReferenceKind == "call" && reference.ContainerName == "run" && reference.SymbolName == "helper")); @@ -1269,6 +1270,8 @@ public void Extract_Tcl_OnlyIndexesScriptsForSelectedSubcommands_Issue4746() proc run {} { after idle helper namespace eval ::tmp { helper } + after 1 {set pending 1;} {helper} + namespace eval ::tmp {set pending 1;} {helper} after cancel helper after info helper namespace export alpha helper @@ -1277,7 +1280,7 @@ namespace export alpha helper var (_, references) = ExtractSymbolsAndReferences("tcl", content); - Assert.Equal(2, references.Count(reference => + Assert.Equal(4, references.Count(reference => reference.ReferenceKind == "call" && reference.ContainerName == "run" && reference.SymbolName == "helper")); @@ -2025,6 +2028,42 @@ public void Extract_AmbiguousPl_MasksPerlTerminalDataSections_Issue4746( reference.ContainerName == "fake"); } + [Fact] + public void Extract_AmbiguousPl_MasksPerlHeredocBodies_Issue4746() + { + const string content = """ + package Hybrid; + sub helper { return 1; } + :- module(hybrid, [entry/0, after_payloads/0]). + entry :- helper. + my $quoted = <<'QUOTED_PAYLOAD'; + helper(); + sub quoted_fake { helper(); } + quoted_fake_predicate :- helper. + QUOTED_PAYLOAD + my $bare = < + reference.ReferenceKind == "call" + && reference.SymbolName == "helper")); + AssertReferencesContain(references, "call", "entry", "helper"); + AssertReferencesContain(references, "call", "after_payloads", "helper"); + Assert.DoesNotContain(symbols, symbol => + symbol.Name is "quoted_fake" or "quoted_fake_predicate" + or "bare_fake" or "indented_fake"); + } + [Fact] public void Extract_CrystalAndGroovy_DoNotTreatColonLabelsAsBareCalls_Issue4746() { From ab3a83e45672feebf1871edca35482b424bccd93 Mon Sep 17 00:00:00 2001 From: Widthdom Date: Sat, 25 Jul 2026 01:53:41 +0900 Subject: [PATCH 26/26] Fix CI capability fixtures and Prolog scan (#4746) --- changelog.d/unreleased/4746.added.md | 4 +- .../Symbols/SymbolExtractor.ExtractCore.cs | 73 ++++++++++++------ .../QueryCommandRunnerSymbolTests.cs | 39 ++++------ .../QueryCommandRunnerTests.cs | 74 +++++++++++++++---- tests/CodeIndex.Tests/SymbolExtractorTests.cs | 7 +- 5 files changed, 135 insertions(+), 62 deletions(-) diff --git a/changelog.d/unreleased/4746.added.md b/changelog.d/unreleased/4746.added.md index 202cbfcfc..13a1d8769 100644 --- a/changelog.d/unreleased/4746.added.md +++ b/changelog.d/unreleased/4746.added.md @@ -38,8 +38,8 @@ affected: ## English -- **Crystal, Groovy, Tcl, Prolog, and ambiguous `.pl` files now support conservative reference graphs (#4746)** — indexing records imports and language-appropriate calls at their source locations, recognizes command-style calls only for callables declared in the same file, parses Groovy control-statement bodies, Tcl command substitutions, inline proc declarations, variadic `after` / `namespace eval` script arguments, `dict for` bodies after bare dictionary words, and `switch` branch bodies, resolves Tcl namespace-qualified proc calls to the matching namespace, requires complete Tcl command words and excludes syntax markers, excludes Groovy closure parameter headers, limits Prolog calls to declared predicates in goal positions including top-level and same-line directives, scans Prolog clause terminators in one pass, masks Perl quote-like regular-expression and heredoc bodies (including whitespace before quote-like delimiters and indented heredocs) in ambiguous `.pl` files without consuming fat-arrow pairs, short Prolog predicate heads, or Perl `$#array` tokens, stops extraction after Perl `__DATA__` / `__END__` markers, preserves Tcl proc and Prolog predicate caller containers, and reports structured symbol/reference/graph capabilities after `.pl` content classification. Pre-graph extractor stamps keep graph readiness degraded until a normal index refresh rewrites affected rows, and human-readable status output identifies that refresh instead of cap-hit remediation. +- **Crystal, Groovy, Tcl, Prolog, and ambiguous `.pl` files now support conservative reference graphs (#4746)** — indexing records imports and language-appropriate calls at their source locations, recognizes command-style calls only for callables declared in the same file, parses Groovy control-statement bodies, Tcl command substitutions, inline proc declarations, variadic `after` / `namespace eval` script arguments, `dict for` bodies after bare dictionary words, and `switch` branch bodies, resolves Tcl namespace-qualified proc calls to the matching namespace, requires complete Tcl command words and excludes syntax markers, excludes Groovy closure parameter headers, limits Prolog calls to declared predicates in goal positions including top-level and same-line directives, scans Prolog clause terminators and precomputes multiline predicate-head parenthesis matches in one pass each, masks Perl quote-like regular-expression and heredoc bodies (including whitespace before quote-like delimiters and indented heredocs) in ambiguous `.pl` files without consuming fat-arrow pairs, short Prolog predicate heads, or Perl `$#array` tokens, stops extraction after Perl `__DATA__` / `__END__` markers, preserves Tcl proc and Prolog predicate caller containers, and reports structured symbol/reference/graph capabilities after `.pl` content classification. Pre-graph extractor stamps keep graph readiness degraded until a normal index refresh rewrites affected rows, and human-readable status output identifies that refresh instead of cap-hit remediation. ## 日本語 -- **Crystal、Groovy、Tcl、Prolog、曖昧な `.pl` ファイルが保守的な reference graph に対応しました（#4746）** — import と言語ごとの call 構文をソース上の位置どおりに記録し、command-style call は同一ファイルで宣言済みの callable だけを認識し、Groovy の制御文本体、Tcl の command substitution、inline proc 宣言、可変長の `after` / `namespace eval` script 引数、bare な dictionary word に続く `dict for` body、`switch` の branch body を解析し、Tcl の名前空間修飾 proc call を対応する名前空間へ解決し、Tcl の command word 全体を照合して構文 marker を除外し、Groovy の closure parameter header を除外し、top-level および同一行の directive を含む Prolog の call を宣言済み predicate の goal 位置に限定し、Prolog の clause terminator を1回の走査で検出し、曖昧な `.pl` では fat-arrow pair、短い Prolog predicate head、Perl の `$#array` token を巻き込まず、delimiter 前に空白がある場合を含む Perl の quote-like 正規表現本体と indented heredoc を含む heredoc 本体をマスクし、Perl の `__DATA__` / `__END__` marker 以降の抽出を停止し、Tcl proc / Prolog predicate の caller container を保持して、`.pl` の content classification 後に構造化された symbol / reference / graph capability を報告します。graph 対応前の extractor stamp がある場合、通常の index refresh で対象 row を更新するまで graph readiness を degraded に保ち、人間向け status 出力でも cap-hit 対応ではなくこの refresh を案内します。 +- **Crystal、Groovy、Tcl、Prolog、曖昧な `.pl` ファイルが保守的な reference graph に対応しました（#4746）** — import と言語ごとの call 構文をソース上の位置どおりに記録し、command-style call は同一ファイルで宣言済みの callable だけを認識し、Groovy の制御文本体、Tcl の command substitution、inline proc 宣言、可変長の `after` / `namespace eval` script 引数、bare な dictionary word に続く `dict for` body、`switch` の branch body を解析し、Tcl の名前空間修飾 proc call を対応する名前空間へ解決し、Tcl の command word 全体を照合して構文 marker を除外し、Groovy の closure parameter header を除外し、top-level および同一行の directive を含む Prolog の call を宣言済み predicate の goal 位置に限定し、Prolog の clause terminator と複数行 predicate head の括弧対応を事前計算した1回の走査で処理し、曖昧な `.pl` では fat-arrow pair、短い Prolog predicate head、Perl の `$#array` token を巻き込まず、delimiter 前に空白がある場合を含む Perl の quote-like 正規表現本体と indented heredoc を含む heredoc 本体をマスクし、Perl の `__DATA__` / `__END__` marker 以降の抽出を停止し、Tcl proc / Prolog predicate の caller container を保持して、`.pl` の content classification 後に構造化された symbol / reference / graph capability を報告します。graph 対応前の extractor stamp がある場合、通常の index refresh で対象 row を更新するまで graph readiness を degraded に保ち、人間向け status 出力でも cap-hit 対応ではなくこの refresh を案内します。 diff --git a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.ExtractCore.cs b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.ExtractCore.cs index 761660249..e25864928 100644 --- a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.ExtractCore.cs +++ b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.ExtractCore.cs @@ -2289,6 +2289,7 @@ bool[] GetCssQualifiedRuleAncestors() => RegexOptions.Compiled | RegexOptions.CultureInvariant); private const int PrologMultilineHeadLookaheadLineLimit = 256; private readonly record struct PrologMultilineHead(string Name, int StartColumn); + private readonly record struct PrologSourcePosition(int LineIndex, int Column); private static bool TryGetNextPrologClauseOffset( string line, @@ -2323,6 +2324,7 @@ private static bool[] BuildPrologClauseContinuationLines( Dictionary multilineHeads) { var continuationLines = new bool[structuralLines.Count]; + var matchingParentheses = BuildPrologMatchingParentheses(structuralLines); var clauseOpen = false; for (var lineIndex = 0; lineIndex < structuralLines.Count; lineIndex++) { @@ -2345,9 +2347,10 @@ private static bool[] BuildPrologClauseContinuationLines( var multilineHead = PrologMultilineHeadStartRegex.Match(clauseCandidate); if (multilineHead.Success && IsValidatedMultilinePrologHead( - structuralLines, - lineIndex, - clauseCandidateOffset + multilineHead.Groups["open"].Index)) + structuralLines, + lineIndex, + clauseCandidateOffset + multilineHead.Groups["open"].Index, + matchingParentheses)) { multilineHeads[lineIndex] = new PrologMultilineHead( multilineHead.Groups["name"].Value, @@ -2370,6 +2373,37 @@ private static bool[] BuildPrologClauseContinuationLines( return continuationLines; } + private static IReadOnlyDictionary + BuildPrologMatchingParentheses(IReadOnlyList structuralLines) + { + var openParentheses = new Stack(); + var matchingParentheses = new Dictionary(); + for (var lineIndex = 0; lineIndex < structuralLines.Count; lineIndex++) + { + var line = structuralLines[lineIndex]; + for (var column = 0; column < line.Length; column++) + { + var ch = line[column]; + if (ch is '\'' or '"') + { + column = SkipPrologQuotedTerm(line, column, ch) - 1; + continue; + } + + if (ch == '(') + { + openParentheses.Push(new PrologSourcePosition(lineIndex, column)); + } + else if (ch == ')' && openParentheses.TryPop(out var openingParenthesis)) + { + matchingParentheses[openingParenthesis] = new PrologSourcePosition(lineIndex, column); + } + } + } + + return matchingParentheses; + } + private static int FindLastTopLevelPrologClauseTerminator(string line) => FindTopLevelPrologClauseTerminator(line, findLast: true); @@ -2463,17 +2497,26 @@ private static bool IsValidatedBareMultilinePrologHead( private static bool IsValidatedMultilinePrologHead( IReadOnlyList structuralLines, int startLineIndex, - int openingParenthesisColumn) + int openingParenthesisColumn, + IReadOnlyDictionary matchingParentheses) { - var parenthesisDepth = 0; - var headClosed = false; var endLineExclusive = Math.Min( structuralLines.Count, startLineIndex + PrologMultilineHeadLookaheadLineLimit); - for (var lineIndex = startLineIndex; lineIndex < endLineExclusive; lineIndex++) + if (!matchingParentheses.TryGetValue( + new PrologSourcePosition(startLineIndex, openingParenthesisColumn), + out var closingParenthesis) + || closingParenthesis.LineIndex >= endLineExclusive) + { + return false; + } + + for (var lineIndex = closingParenthesis.LineIndex; lineIndex < endLineExclusive; lineIndex++) { var line = structuralLines[lineIndex]; - var startColumn = lineIndex == startLineIndex ? openingParenthesisColumn : 0; + var startColumn = lineIndex == closingParenthesis.LineIndex + ? closingParenthesis.Column + 1 + : 0; for (var column = startColumn; column < line.Length; column++) { var ch = line[column]; @@ -2483,20 +2526,6 @@ private static bool IsValidatedMultilinePrologHead( continue; } - if (!headClosed) - { - if (ch == '(') - { - parenthesisDepth++; - } - else if (ch == ')' && parenthesisDepth > 0) - { - parenthesisDepth--; - headClosed = parenthesisDepth == 0; - } - continue; - } - if (char.IsWhiteSpace(ch)) continue; if (line.AsSpan(column).StartsWith(":-", StringComparison.Ordinal) diff --git a/tests/CodeIndex.Tests/QueryCommandRunnerSymbolTests.cs b/tests/CodeIndex.Tests/QueryCommandRunnerSymbolTests.cs index ccbe4b377..0fd3edc07 100644 --- a/tests/CodeIndex.Tests/QueryCommandRunnerSymbolTests.cs +++ b/tests/CodeIndex.Tests/QueryCommandRunnerSymbolTests.cs @@ -1512,31 +1512,24 @@ public void RunSymbols_JsonZeroResults_ReturnEmptyStdout() [Fact] public void RunSymbols_UnsupportedExtractorLanguageExplainsSearchOnlyFallback() { - var projectRoot = TestProjectHelper.CreateTempProject("cdidx_symbols_unsupported_extractor"); - try - { - File.WriteAllText(Path.Combine(projectRoot, "settings.pl"), "enabled = true\n"); - var dbPath = Path.Combine(projectRoot, ".cdidx", "codeindex.db"); - var (indexExitCode, _, indexStderr) = CaptureConsole(() => IndexCommandRunner.Run( - [projectRoot, "--json", "--quiet"], - _jsonOptions)); + using var project = CreateSearchOnlyLanguageProject( + "cdidx_symbols_unsupported_extractor", + out var dbPath); + TestProjectHelper.InsertIndexedFile( + dbPath, + "settings" + SearchOnlyTestExtension, + SearchOnlyTestLanguage, + "enabled = true\n"); - var (exitCode, stdout, stderr) = CaptureConsole(() => QueryCommandRunner.RunSymbols( - ["enabled", "--db", dbPath, "--lang", "ambiguous_pl"], - _jsonOptions)); + var (exitCode, stdout, stderr) = CaptureConsole(() => QueryCommandRunner.RunSymbols( + ["enabled", "--db", dbPath, "--lang", SearchOnlyTestLanguage], + _jsonOptions)); - Assert.Equal(CommandExitCodes.Success, indexExitCode); - Assert.Equal(string.Empty, indexStderr); - Assert.Equal(CommandExitCodes.Success, exitCode); - Assert.Equal(string.Empty, stdout); - Assert.Contains("symbol extraction is not available", stderr); - Assert.Contains("cdidx search --lang ambiguous_pl", stderr); - Assert.Contains("missing-symbols", stderr); - } - finally - { - TestProjectHelper.DeleteDirectory(projectRoot); - } + Assert.Equal(CommandExitCodes.Success, exitCode); + Assert.Equal(string.Empty, stdout); + Assert.Contains("symbol extraction is not available", stderr); + Assert.Contains($"cdidx search --lang {SearchOnlyTestLanguage}", stderr); + Assert.Contains("missing-symbols", stderr); } [Fact] diff --git a/tests/CodeIndex.Tests/QueryCommandRunnerTests.cs b/tests/CodeIndex.Tests/QueryCommandRunnerTests.cs index f1a7dfa97..75bdc0025 100644 --- a/tests/CodeIndex.Tests/QueryCommandRunnerTests.cs +++ b/tests/CodeIndex.Tests/QueryCommandRunnerTests.cs @@ -34,6 +34,8 @@ public partial class QueryCommandRunnerTests { PropertyNamingPolicy = JsonNamingPolicy.SnakeCaseLower, }; + private const string SearchOnlyTestLanguage = "searchonlytest"; + private const string SearchOnlyTestExtension = ".searchonly"; [Fact] public void ParseArgs_ParsesFiltersFlagsAndAcceptsMaxSnippetLines() @@ -2576,6 +2578,27 @@ public IReadOnlyList Extract(long fileId, string source, Extractio => []; } + private static TestProjectHelper.TempProjectScope CreateSearchOnlyLanguageProject( + string prefix, + out string dbPath) + { + var project = TestProjectHelper.CreateTempProjectScope(prefix); + try + { + dbPath = TestProjectHelper.CreateProjectDb(project.Root); + TestProjectHelper.WriteTextFile( + project.Root, + LanguageMapOverrides.WorkspaceFileName, + $"entries:\n- extension: {SearchOnlyTestExtension}\n language: {SearchOnlyTestLanguage}\n"); + return project; + } + catch + { + project.Dispose(); + throw; + } + } + [Theory] [InlineData("--language", "C#", "csharp", 2)] [InlineData("--extension", ".cs", "csharp", 2)] @@ -2687,8 +2710,13 @@ public void RunLanguages_JsonCapabilitySymbolsFiltersSymbolSupport() [InlineData("missing-graph", "graph_queries")] public void RunLanguages_JsonCapabilityMissingFiltersCapabilityGaps(string capability, string propertyName) { + using var project = CreateSearchOnlyLanguageProject( + "cdidx_languages_missing_capability", + out var dbPath); var (exitCode, stdout, stderr) = CaptureConsole(() => - QueryCommandRunner.RunLanguages(["--json", "--capability", capability], _jsonOptions)); + QueryCommandRunner.RunLanguages( + ["--json", "--capability", capability, "--db", dbPath], + _jsonOptions)); Assert.Equal(CommandExitCodes.Success, exitCode); Assert.Equal(string.Empty, stderr); @@ -2696,7 +2724,7 @@ public void RunLanguages_JsonCapabilityMissingFiltersCapabilityGaps(string capab using var document = ParseJsonOutput(stdout); var languages = document.RootElement.GetProperty("languages").EnumerateArray().ToList(); - Assert.NotEmpty(languages); + Assert.Contains(languages, lang => lang.GetProperty("lang").GetString() == SearchOnlyTestLanguage); Assert.All(languages, lang => { Assert.False(lang.GetProperty(propertyName).GetBoolean()); @@ -2707,10 +2735,15 @@ public void RunLanguages_JsonCapabilityMissingFiltersCapabilityGaps(string capab [Fact] public void RunLanguages_FormatCountReturnsCapabilitySummary_Issue4316() { - using var project = TestProjectHelper.CreateTempProjectScope("cdidx_languages_format_count_issue4316"); - var dbPath = TestProjectHelper.CreateProjectDb(project.Root); + using var project = CreateSearchOnlyLanguageProject( + "cdidx_languages_format_count_issue4316", + out var dbPath); TestProjectHelper.InsertIndexedFile(dbPath, "src/App.cs", "csharp", "class App { }\n"); - TestProjectHelper.InsertIndexedFile(dbPath, "src/main.pl", "prolog", "main :- true.\n"); + TestProjectHelper.InsertIndexedFile( + dbPath, + "config/settings.searchonly", + SearchOnlyTestLanguage, + "enabled = true\n"); var (exitCode, stdout, stderr) = CaptureConsole(() => QueryCommandRunner.RunLanguages(["--db", dbPath, "--indexed-only", "--capability", "missing-any", "--format", "count"], _jsonOptions)); @@ -2722,7 +2755,7 @@ public void RunLanguages_FormatCountReturnsCapabilitySummary_Issue4316() var root = document.RootElement; Assert.Equal("count", root.GetProperty("format").GetString()); - Assert.True(root.GetProperty("count").GetInt32() > 0); + Assert.Equal(1, root.GetProperty("count").GetInt32()); Assert.Equal(root.GetProperty("count").GetInt32(), root.GetProperty("language_count").GetInt32()); Assert.Equal(root.GetProperty("count").GetInt32(), root.GetProperty("indexed_language_count").GetInt32()); Assert.True(root.GetProperty("indexed_file_count").GetInt64() > 0); @@ -2782,8 +2815,11 @@ public void RunLanguages_SummaryOnlyJsonReturnsCapabilitySummary_Issue4316() [Fact] public void RunLanguages_JsonIncludesUnsupportedCapabilityGuidance_Issue4122() { + using var project = CreateSearchOnlyLanguageProject( + "cdidx_languages_unsupported_guidance", + out var dbPath); var (exitCode, stdout, stderr) = CaptureConsole(() => - QueryCommandRunner.RunLanguages(["--json"], _jsonOptions)); + QueryCommandRunner.RunLanguages(["--json", "--db", dbPath], _jsonOptions)); Assert.Equal(CommandExitCodes.Success, exitCode); Assert.Equal(string.Empty, stderr); @@ -2791,15 +2827,20 @@ public void RunLanguages_JsonIncludesUnsupportedCapabilityGuidance_Issue4122() using var document = ParseJsonOutput(stdout); var languages = document.RootElement.GetProperty("languages").EnumerateArray() .ToDictionary(entry => entry.GetProperty("lang").GetString()!, entry => entry); - var prologGuidance = languages["prolog"].GetProperty("unsupported_guidance").EnumerateArray().ToList(); + var searchOnlyGuidance = languages[SearchOnlyTestLanguage] + .GetProperty("unsupported_guidance") + .EnumerateArray() + .ToList(); - var referenceGuidance = prologGuidance.Single(guidance => guidance.GetProperty("capability").GetString() == "references"); - Assert.Contains("Reference extraction is not advertised for 'prolog'", referenceGuidance.GetProperty("message").GetString()); + var referenceGuidance = searchOnlyGuidance.Single(guidance => guidance.GetProperty("capability").GetString() == "references"); + Assert.Contains( + $"Reference extraction is not advertised for '{SearchOnlyTestLanguage}'", + referenceGuidance.GetProperty("message").GetString()); var referenceCommands = referenceGuidance.GetProperty("recommended_commands").EnumerateArray().Select(command => command.GetString()).ToList(); Assert.Contains("search", referenceCommands); - Assert.Contains("definition", referenceCommands); + Assert.Contains("excerpt", referenceCommands); - var graphGuidance = prologGuidance.Single(guidance => guidance.GetProperty("capability").GetString() == "graph"); + var graphGuidance = searchOnlyGuidance.Single(guidance => guidance.GetProperty("capability").GetString() == "graph"); Assert.Contains("empty callers, callees, or impact results are not authoritative", graphGuidance.GetProperty("message").GetString()); var graphCommands = graphGuidance.GetProperty("recommended_commands").EnumerateArray().Select(command => command.GetString()).ToList(); Assert.Contains("search", graphCommands); @@ -2813,8 +2854,13 @@ public void RunLanguages_JsonIncludesUnsupportedCapabilityGuidance_Issue4122() [InlineData("none")] public void RunLanguages_JsonCapabilityNoneFiltersAllExtractionGaps_Issue4316(string capability) { + using var project = CreateSearchOnlyLanguageProject( + "cdidx_languages_no_extraction_capability", + out var dbPath); var (exitCode, stdout, stderr) = CaptureConsole(() => - QueryCommandRunner.RunLanguages(["--json", "--capability", capability], _jsonOptions)); + QueryCommandRunner.RunLanguages( + ["--json", "--capability", capability, "--db", dbPath], + _jsonOptions)); Assert.Equal(CommandExitCodes.Success, exitCode); Assert.Equal(string.Empty, stderr); @@ -2823,7 +2869,7 @@ public void RunLanguages_JsonCapabilityNoneFiltersAllExtractionGaps_Issue4316(st var languages = document.RootElement.GetProperty("languages").EnumerateArray().ToList(); var names = languages.Select(lang => lang.GetProperty("lang").GetString()).ToList(); - Assert.NotEmpty(languages); + Assert.Contains(SearchOnlyTestLanguage, names); Assert.DoesNotContain("groovy", names); Assert.All(languages, lang => { diff --git a/tests/CodeIndex.Tests/SymbolExtractorTests.cs b/tests/CodeIndex.Tests/SymbolExtractorTests.cs index 0c9566ea0..99c9e2653 100644 --- a/tests/CodeIndex.Tests/SymbolExtractorTests.cs +++ b/tests/CodeIndex.Tests/SymbolExtractorTests.cs @@ -58,13 +58,18 @@ public void Extract_Prolog_IndexesModuleImportsAndPredicates_Issue4612() :- use_module(library(lists)). parent(alice, bob). ancestor(X, Y) :- parent(X, Y). + multiline_parent( + X, + Y + ) :- + parent(X, Y). """; var symbols = SymbolExtractor.Extract(1, "prolog", content); AssertSymbolsContain(symbols, "namespace", "family"); AssertSymbolsContain(symbols, "import", "library(lists)"); - AssertSymbolsContain(symbols, "function", "parent", "ancestor"); + AssertSymbolsContain(symbols, "function", "parent", "ancestor", "multiline_parent"); } [Fact]