From 465ee0780893a7871d730e8c974aa2bf433e09cf Mon Sep 17 00:00:00 2001 From: Widthdom Date: Fri, 24 Jul 2026 11:28:17 +0900 Subject: [PATCH 01/21] Add scientific language reference graphs (#4738) --- DEVELOPER_GUIDE.md | 4 +- USER_GUIDE.md | 24 +- changelog.d/unreleased/4738.added.md | 30 +++ .../Indexer/AmbiguousMContentMasker.cs | 165 +++++++++++++ .../ScientificNativeCommentMasker.cs | 168 ++++++++++++++ .../ScientificNativeReferenceExtractor.cs | 153 ++++++++++++ .../ReferenceExtractor.CoreExtraction.cs | 79 +++++++ .../ReferenceExtractor.Preparation.cs | 4 +- .../References/ReferenceExtractor.State.cs | 3 +- .../ReferenceExtractor.TypeReferences.cs | 19 +- .../Indexer/References/ReferenceExtractor.cs | 34 ++- .../Symbols/SymbolExtractor.ExtractCore.cs | 29 +++ .../Indexer/Symbols/SymbolExtractor.cs | 6 +- .../QueryCommandRunnerTests.cs | 29 +-- ...eferenceExtractorTests.ScientificNative.cs | 219 ++++++++++++++++++ tests/CodeIndex.Tests/SymbolExtractorTests.cs | 8 + tests/CodeIndex.Tests/golden/status.json | 7 + 17 files changed, 945 insertions(+), 36 deletions(-) create mode 100644 changelog.d/unreleased/4738.added.md create mode 100644 src/CodeIndex/Indexer/AmbiguousMContentMasker.cs create mode 100644 src/CodeIndex/Indexer/References/Languages/ScientificNativeCommentMasker.cs create mode 100644 src/CodeIndex/Indexer/References/Languages/ScientificNativeReferenceExtractor.cs create mode 100644 tests/CodeIndex.Tests/ReferenceExtractorTests.ScientificNative.cs diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index 00a105c42..565824931 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -1597,7 +1597,7 @@ Process exit codes are coarse (`0` success including valid zero-row queries, `1` ## Design decisions - **Language capability patterns remain typed at the integration boundary** — CLI/MCP `languages` rows expose suffix-only `extensions`, literal `exact_filenames`, and ``-rendered `filename_prefix_patterns`. `legacy_patterns` preserves the former combined list during deprecation, and `pattern_provenance` identifies built-in, plugin/pattern, and language-map override ownership. Round-trip tests feed every advertised typed pattern back through `FileIndexer.DetectLanguage` (#4617). -- **Ambiguous source extensions stay explicit** — `.m` and `.pl` are not assigned to Objective-C and Perl by default. `FileIndexer` checks an authoritative recognized shebang, then a 64 KiB bounded prefix for strong mutually exclusive Objective-C/MATLAB or Perl/Prolog markers, then at most 256 entries per ancestor directory for conservative project markers. Conflicting or weak evidence is indexed as `ambiguous_m` / `ambiguous_pl`; MATLAB and Prolog have conservative symbol patterns but deliberately advertise no reference/graph support (#4612). +- **Ambiguous source extensions stay explicit** — `.m` and `.pl` are not assigned to Objective-C and Perl by default. `FileIndexer` checks an authoritative recognized shebang, then a 64 KiB bounded prefix for strong mutually exclusive Objective-C/MATLAB or Perl/Prolog markers, then at most 256 entries per ancestor directory for conservative project markers. Conflicting or weak evidence is indexed as `ambiguous_m` / `ambiguous_pl`; unresolved `.m` files run the bounded MATLAB and Objective-C symbol/reference paths after a shared position-preserving comment mask, while Prolog deliberately remains without reference/graph support (#4612, #4738). - **Hotspot marker fingerprints share one bounded tree traversal** — full/update CLI and MCP indexing compute C#, VB, F#, and MSBuild marker fingerprints together instead of walking the directory tree once per language. Each distinct marker glob retains the platform filesystem's matching behavior and is enumerated once per visited directory, while child directories are enumerated once; marker sets, budgets, truncation sentinels, and warning order remain isolated per language. The single-language API delegates to the same engine, preserving ignore rules, nested-repository/submodule boundaries, and MCP authorized-read failures. - **Lock-file dependency graphs model package relationships** — `packages.lock.json`, `package-lock.json`, and `npm-shrinkwrap.json` keep package declarations as symbols, but emit `dependency` references only for explicit parent-package to child-package entries. The parent package is stored as the reference container, so `callers` can identify which package requires a child and `deps` does not infer lock-file-to-lock-file similarity merely because two files contain the same resolved package set (#4409). - **No ORM** — Raw `Microsoft.Data.Sqlite` with parameterized queries. Keeps dependencies minimal and control explicit. @@ -4573,7 +4573,7 @@ USER_GUIDEの[終了コード](USER_GUIDE.md#終了コード)セクションを ## 設計判断 - **integration boundary では language capability pattern の型を維持** — CLI/MCP の `languages` 行は suffix のみの `extensions`、literal な `exact_filenames`、`` 表記の `filename_prefix_patterns` を公開します。`legacy_patterns` は deprecation 中に従来の combined list を保持し、`pattern_provenance` は built-in、plugin/pattern、language-map override の所有元を示します。round-trip test は広告した全 typed pattern を `FileIndexer.DetectLanguage` に戻して検証します（#4617）。 -- **曖昧な source extension は曖昧なまま明示** — `.m` と `.pl` を既定で Objective-C / Perl に割り当てません。`FileIndexer` は authoritative な認識済み shebang、64 KiB 上限 prefix 内の相互排他的で強い Objective-C/MATLAB または Perl/Prolog marker、各 ancestor directory 最大 256 entry の保守的な project marker の順に確認します。競合または弱い証拠は `ambiguous_m` / `ambiguous_pl` として index し、MATLAB / Prolog は保守的な symbol pattern を持つ一方、reference / graph 対応を意図的に広告しません（#4612）。 +- **曖昧な source extension は曖昧なまま明示** — `.m` と `.pl` を既定で Objective-C / Perl に割り当てません。`FileIndexer` は authoritative な認識済み shebang、64 KiB 上限 prefix 内の相互排他的で強い Objective-C/MATLAB または Perl/Prolog marker、各 ancestor directory 最大 256 entry の保守的な project marker の順に確認します。競合または弱い証拠は `ambiguous_m` / `ambiguous_pl` として index し、未確定の `.m` は位置を保つ共通コメントマスクの後で上限付きの MATLAB / Objective-C symbol・reference 経路を実行します。Prolog は意図的に reference / graph 未対応のままです（#4612、#4738）。 - **hotspot marker fingerprint は上限付きtree traversalを1回共有** — full/update CLIとMCP indexingは、directory treeを言語ごとに歩かず、C#、VB、F#、MSBuildのmarker fingerprintをまとめて計算します。各directoryでは固有marker globごとにplatform filesystemのmatching挙動を保って1回ずつ列挙し、child directoryも1回だけ列挙する一方、marker集合、budget、truncation sentinel、warning順は言語別に分離します。single-language APIも同じengineへ委譲し、ignore rule、nested repository/submodule境界、MCP authorized read failureを維持します。 - **lock file の依存グラフは package 間の関係をモデル化** — `packages.lock.json`、`package-lock.json`、`npm-shrinkwrap.json` は package 宣言を symbol として保持しますが、`dependency` reference は明示された親 package → 子 package の項目だけに出力します。親 package を reference container に保存するため、`callers` はどの package が子 package を必要としているかを特定でき、`deps` は同じ resolved package 集合を持つだけの lock file 同士を類似関係として推論しません（#4409）。 - **ORMなし** — `Microsoft.Data.Sqlite`でパラメータ化クエリを直接使用。依存関係を最小限に、制御を明確に。 diff --git a/USER_GUIDE.md b/USER_GUIDE.md index bb966b9aa..3ab400256 100644 --- a/USER_GUIDE.md +++ b/USER_GUIDE.md @@ -2368,14 +2368,14 @@ All indexed languages are searchable through FTS5. Rows with **Symbols = yes** a - C/C++ headers: `.h` stays on the C path unless lexical code (after comments, strings, and macro payloads are masked) has clear C++ markers such as `namespace`, `template`, `using`, `class`, or `std::`; those headers are promoted to `cpp` at index time. Detection scores the full header up to 48 KiB, then uses head/middle/tail ranges for larger files while retaining lexical state across skipped bytes, so long license blocks do not impose a fixed line cutoff. `index --dry-run --json` reports ambiguous-header decisions in `language_detections` with stable `source` and `confidence` values. - C++ callables: balanced declarators preserve constructors, destructors, conversion operators, ordinary functions, and trailing-return functions as navigable function symbols. Trailing return types populate `return_type` metadata. -- Cython and CUDA: Cython `cdef` / `cpdef` declarations, `cimport` entries, and extern declarations are indexed as symbols. CUDA files reuse C++ symbols and classify `__global__`, `__device__`, and `__host__` functions with CUDA-specific sub-kinds. +- Cython and CUDA: Cython `cdef` / `cpdef` declarations, `cimport` entries, and extern declarations are indexed as symbols; bounded cimport, base-type, and call references feed graph queries. CUDA files reuse C++ symbols and classify `__global__`, `__device__`, and `__host__` functions with CUDA-specific sub-kinds. - Shaders: GLSL, HLSL, Metal, and WGSL entry points, structs, type aliases, resource bindings, constant buffers, samplers, textures, and uniform/input/output declarations are indexed as symbols. - HDL: Verilog, SystemVerilog, and VHDL module/package/type/function/resource declarations are indexed as symbols. References and graph queries are not advertised for HDL yet. - SQL: query-time `--lang tsql` is accepted as a SQL alias, and T-SQL aggregate, assembly, and XML schema collection declarations are searchable. - R: function assignments, S4/R6 class declarations, validity/generic/method declarations, inherit vectors, public/private/active methods, and `library` / `require` imports are indexed. - Functional symbol-only languages: Clojure, Erlang, OCaml, and Raku expose conservative declarations as symbols. References and graph queries are not advertised for these languages yet. -- Dynamic symbol-only languages: Crystal, Groovy, Julia, and Tcl expose conservative declarations as symbols. References and graph queries are not advertised for these languages yet. -- Systems symbol-only languages: Ada, D, and Nim expose conservative declarations as symbols. References and graph queries are not advertised for these languages yet. +- Dynamic symbol-only languages: Crystal, Groovy, and Tcl expose conservative declarations as symbols. References and graph queries are not advertised for these languages yet. +- Scientific and native-extension graphs: Julia, MATLAB, Nim, D, Cython, and Ada emit bounded language-aware import/module, base/type, and call references. Julia macro invocations and Ada procedure-style calls without parentheses are also represented. - Markdown, JSON/YAML, and CSS: Markdown heading and local-anchor symbols are indexed; JSON/YAML configuration keys are indexed as structural key paths; CSS variables, placeholders, and `@extend` references are indexed. - Dockerfile, Assembly, Common Lisp, and Racket: `ARG` build args, labels/PROC/MACRO blocks, package/module forms, definitions, classes/structs, requires, and provides are surfaced as symbols where applicable. - Shell, PowerShell, and Batch: command-style function calls, functions/filters, classes/enums, imports, labels, `goto` / `call` targets, and inline control-flow forms are indexed where the language supports them. @@ -2388,7 +2388,7 @@ All indexed languages are searchable through FTS5. Rows with **Symbols = yes** a - Dependency manifests and lockfiles: use `--lang dependency_manifest` or `--lang dependency_lock` for dependency/security audits. `Directory.Packages.props`, `packages.config`, `requirements.txt`, `pyproject.toml`, `packages.lock.json`, and npm `package-lock.json` / `npm-shrinkwrap.json` expose package symbols and `dependency` references with version, scope, and direct/transitive metadata where the format provides it. - Solution and application manifests: `.sln` files expose project entries as symbols and project path references; `.manifest` files expose assembly identity, requested execution level, supported OS, and long-path settings as symbols. - Shebang scripts: recognized first-line shebangs index extensionless and unknown-extension files for shell (`sh`, `bash`, `zsh`, `fish`, `dash`, `ksh`, `ash`), Python, Ruby, Perl, Tcl (`tclsh`, `wish`), Node.js, PHP, Lua, and PowerShell. Explicit language-map overrides remain authoritative; for ambiguous `.t` files, a recognized shebang overrides the Perl default, while strong known extensions continue to win conflicts. -- Ambiguous `.m` / `.pl`: recognized shebangs win first, then bounded content checks use only strong Objective-C/MATLAB or Perl/Prolog markers, followed by conservative project markers. Scoped updates that add, change, or remove one of those markers automatically rescan the workspace so unchanged ambiguous files do not retain stale classifications. Weak or conflicting evidence remains searchable under `ambiguous_m` or `ambiguous_pl` instead of being assigned unconditionally. MATLAB and Prolog expose declaration symbols but no reference or graph support yet. +- Ambiguous `.m` / `.pl`: recognized shebangs win first, then bounded content checks use only strong Objective-C/MATLAB or Perl/Prolog markers, followed by conservative project markers. Scoped updates that add, change, or remove one of those markers automatically rescan the workspace so unchanged ambiguous files do not retain stale classifications. Weak or conflicting evidence remains searchable under `ambiguous_m` or `ambiguous_pl` instead of being assigned unconditionally. Unresolved `.m` content exposes the conservative union of MATLAB and Objective-C symbols/references after both comment syntaxes are position-preservingly masked; Prolog remains symbol-only. ### Language extraction matrix @@ -2415,8 +2415,8 @@ entries with the unsupported capability, an explanatory message, and | Java / Kotlin / Scala | packages/imports, classes/interfaces, methods, properties | calls, constructors, annotations, type references | Kotlin inline lambda body modeling is limited; verify with `references` before relying on deep call chains. | | JavaScript / TypeScript / Vue / Svelte | functions, classes, exports, imports, variables | calls, constructors, static/dynamic imports, workers, service workers | Dynamic property calls and computed module specifiers are best-effort. `cdidx references render --lang typescript` | | Python / Ruby / PHP / Perl / R | functions, classes/modules, imports where supported | calls, constructors, decorators/annotations where supported | Dynamic dispatch and metaprogramming may require `search`. PHPDoc/static import patterns are indexed when statically visible. | -| MATLAB / Prolog | classes/modules, functions/predicates, imports | none yet | `.m` / `.pl` are classified conservatively; declaration symbols are searchable, but use `search` for reference and graph questions. | -| Cython | `cdef` / `cpdef` declarations, cimports, extern declarations | none yet | Cython native-extension declarations are searchable as symbols; use `search` for call/reference questions. | +| MATLAB / Julia / Nim / D / Ada / Cython | classes/modules/types, functions/procedures, imports | bounded calls, imports/modules, base/type references; Julia macros and Ada procedure-style calls | Static syntax is indexed conservatively; dynamic dispatch, generated code, and macro expansion may still require `search`. | +| Prolog | modules, predicates, imports | none yet | `.pl` is classified conservatively; declaration symbols are searchable, but use `search` for reference and graph questions. | | C / C++ / Objective-C / Swift / Rust / Go / Zig | functions, types, methods, imports/modules | calls, constructors, macro invocations where supported, type references | C++ templates/macros and Rust macro expansion are not evaluated; Rust macro invocations are still reference edges. | | CUDA | C++-style functions/types plus CUDA kernel/device/host sub-kinds | none yet | CUDA kernel/device/host declarations are indexed as C++-style symbols with CUDA sub-kinds; use `search` for call/reference questions. | | GLSL / HLSL / Metal / WGSL | entry points, structs, type aliases, resource bindings, constant buffers, samplers, textures, uniforms/inputs/outputs | none yet | Shader entry points and resource declarations are searchable as symbols; use `search` for data-flow, binding compatibility, and call/reference questions. | @@ -5513,14 +5513,14 @@ indexing はファイル単位の SQLite transaction を commit します。長 - C/C++ ヘッダー: `.h` は既定では C として扱います。コメント、文字列、マクロのペイロードをマスクした後の字句コードに `namespace`、`template`、`using`、`class`、`std::` などの明確な C++ マーカーがある場合だけ、index 時に `cpp` へ昇格します。48 KiB まではヘッダー全体、それを超える場合は評価対象外の byte をまたいで字句状態を保持しながら先頭・中央・末尾 range を評価するため、長いライセンスブロックが固定行数の打ち切りを引き起こしません。`index --dry-run --json` は曖昧なヘッダー判定を、安定した `source` と `confidence` を持つ `language_detections` として報告します。 - C++ callable: 括弧の対応を考慮した declarator 解析により、constructor、destructor、conversion operator、通常関数、後置戻り値関数を移動可能な function シンボルとして保持します。後置戻り値は `return_type` メタデータへ格納します。 -- Cython と CUDA: Cython の `cdef` / `cpdef` 宣言、`cimport`、extern 宣言をシンボルとして索引します。CUDA ファイルは C++ のシンボル抽出を再利用し、`__global__`、`__device__`、`__host__` 関数に CUDA 固有の sub-kind を付けます。 +- Cython と CUDA: Cython の `cdef` / `cpdef` 宣言、`cimport`、extern 宣言をシンボルとして索引し、上限付きの cimport・基底型・call 参照を graph query に提供します。CUDA ファイルは C++ のシンボル抽出を再利用し、`__global__`、`__device__`、`__host__` 関数に CUDA 固有の sub-kind を付けます。 - Shaders: GLSL、HLSL、Metal、WGSL の entry point、struct、type alias、resource binding、constant buffer、sampler、texture、uniform/input/output 宣言をシンボルとして索引します。 - HDL: Verilog、SystemVerilog、VHDL の module / package / type / function / resource 宣言をシンボルとして索引します。HDL の references と graph queries はまだ対応として広告しません。 - SQL: クエリ時の `--lang tsql` は SQL の別名です。T-SQL の aggregate、assembly、XML schema collection 宣言も検索対象です。 - R: 関数代入、S4/R6 class 宣言、validity/generic/method 宣言、inherit vector、public/private/active method、`library` / `require` import を索引します。 - 関数型言語のシンボル専用対応: Clojure、Erlang、OCaml、Raku は保守的な宣言をシンボルとして公開します。これらの言語では references と graph queries はまだ対応として広告しません。 -- 動的言語のシンボル専用対応: Crystal、Groovy、Julia、Tcl は保守的な宣言をシンボルとして公開します。これらの言語では references と graph queries はまだ対応として広告しません。 -- システム系言語のシンボル専用対応: Ada、D、Nim は保守的な宣言をシンボルとして公開します。これらの言語では references と graph queries はまだ対応として広告しません。 +- 動的言語のシンボル専用対応: Crystal、Groovy、Tcl は保守的な宣言をシンボルとして公開します。これらの言語では references と graph queries はまだ対応として広告しません。 +- 科学技術・ネイティブ拡張言語のグラフ: Julia、MATLAB、Nim、D、Cython、Ada は、言語構文に応じた import/module、基底型/type、call 参照を上限付きで出力します。Julia の macro invocation と、括弧を伴わない Ada の procedure call も記録します。 - Markdown、JSON/YAML、CSS: Markdown の heading / local anchor、JSON/YAML の configuration key path、CSS の variable、placeholder、`@extend` をシンボルとして扱います。 - Dockerfile、Assembly、Common Lisp、Racket: `ARG` build arg、label、PROC/MACRO、package/module form、definition、class/struct、require/provide を必要に応じて表面化します。 - Shell、PowerShell、Batch: command-style function call、function/filter、class/enum、import、label、`goto` / `call` target、inline control-flow を言語仕様に合わせて索引します。 @@ -5533,7 +5533,7 @@ indexing はファイル単位の SQLite transaction を commit します。長 - Dependency manifest / lockfile: dependency / security audit では `--lang dependency_manifest` または `--lang dependency_lock` を使います。`Directory.Packages.props`、`packages.config`、`requirements.txt`、`pyproject.toml`、`packages.lock.json`、npm の `package-lock.json` / `npm-shrinkwrap.json` は、format が提供する範囲で version、scope、direct/transitive metadata を持つ package symbol と `dependency` reference を公開します。 - ソリューションとアプリケーションマニフェスト: `.sln` は project entry をシンボルとして公開し、project path を参照として記録します。`.manifest` は assembly identity、requested execution level、supported OS、long-path 設定をシンボルとして公開します。 - shebang script: 先頭行の shebang を認識できる拡張子なし/未知拡張子ファイルは、shell (`sh`, `bash`, `zsh`, `fish`, `dash`, `ksh`, `ash`)、Python、Ruby、Perl、Tcl (`tclsh`, `wish`)、Node.js、PHP、Lua、PowerShell として index 対象です。明示的な language-map override は常に優先し、曖昧な `.t` では認識済み shebang が Perl の既定値を上書きします。一方、曖昧でない既知拡張子は競合する shebang より優先されます。 -- 曖昧な `.m` / `.pl`: 認識済み shebang を最優先し、その後は bounded content check で Objective-C/MATLAB または Perl/Prolog の強い marker だけを使い、最後に保守的な project marker を確認します。これらの marker を追加・変更・削除する scoped update は workspace を自動的に再 scan し、未変更の曖昧ファイルに古い分類を残しません。弱い証拠や競合する証拠は無条件に言語を割り当てず、`ambiguous_m` / `ambiguous_pl` として全文検索可能なまま残します。MATLAB と Prolog は宣言 symbol を公開しますが、reference / graph はまだ未対応です。 +- 曖昧な `.m` / `.pl`: 認識済み shebang を最優先し、その後は bounded content check で Objective-C/MATLAB または Perl/Prolog の強い marker だけを使い、最後に保守的な project marker を確認します。これらの marker を追加・変更・削除する scoped update は workspace を自動的に再 scan し、未変更の曖昧ファイルに古い分類を残しません。弱い証拠や競合する証拠は無条件に言語を割り当てず、`ambiguous_m` / `ambiguous_pl` として全文検索可能なまま残します。未確定の `.m` は両方のコメント構文を位置を保ってマスクした後、MATLAB と Objective-C の symbol/reference を保守的に統合します。Prolog は引き続き symbol-only です。 ### 言語別 extraction matrix @@ -5554,8 +5554,8 @@ indexing はファイル単位の SQLite transaction を commit します。長 | Java / Kotlin / Scala | package/import、class/interface、method、property | call、constructor、annotation、type reference | Kotlin inline lambda body の modeling は限定的です。深い call chain を信頼する前に `references` で確認してください。 | | JavaScript / TypeScript / Vue / Svelte | function、class、export、import、variable | call、constructor、static/dynamic import、worker、service worker | dynamic property call と computed module specifier は best-effort です。`cdidx references render --lang typescript` | | Python / Ruby / PHP / Perl / R | function、class/module、対応言語の import | call、constructor、対応言語の decorator/annotation | dynamic dispatch と metaprogramming は `search` が必要な場合があります。PHPDoc/static import pattern は静的に見える範囲で索引されます。 | -| MATLAB / Prolog | class/module、function/predicate、import | まだなし | `.m` / `.pl` は保守的に分類され、宣言 symbol は検索できます。reference / graph の調査には `search` を使ってください。 | -| Cython | `cdef` / `cpdef` 宣言、cimport、extern 宣言 | まだなし | Cython の native extension 宣言はシンボルとして検索できます。call/reference の調査には `search` を使ってください。 | +| MATLAB / Julia / Nim / D / Ada / Cython | class/module/type、function/procedure、import | 上限付きの call、import/module、基底型/type reference。Julia macro と Ada の procedure-style call | 静的な構文を保守的に索引します。dynamic dispatch、generated code、macro expansion には `search` が必要な場合があります。 | +| Prolog | module、predicate、import | まだなし | `.pl` は保守的に分類され、宣言 symbol は検索できます。reference / graph の調査には `search` を使ってください。 | | C / C++ / Objective-C / Swift / Rust / Go / Zig | function、type、method、import/module | call、constructor、対応言語の macro invocation、type reference | C++ template/macro と Rust macro expansion は評価しません。Rust macro invocation 自体は reference edge です。 | | CUDA | C++ 風の function/type と CUDA kernel/device/host sub-kind | まだなし | CUDA の kernel / device / host 宣言は CUDA sub-kind 付きの C++ 風シンボルとして索引します。call/reference の調査には `search` を使ってください。 | | GLSL / HLSL / Metal / WGSL | entry point、struct、type alias、resource binding、constant buffer、sampler、texture、uniform/input/output | まだなし | Shader entry point と resource 宣言はシンボルとして検索できます。data-flow、binding compatibility、call/reference の調査には `search` を使ってください。 | diff --git a/changelog.d/unreleased/4738.added.md b/changelog.d/unreleased/4738.added.md new file mode 100644 index 000000000..6b7bd7dd6 --- /dev/null +++ b/changelog.d/unreleased/4738.added.md @@ -0,0 +1,30 @@ +--- +category: added +issues: + - 4738 +affected: + - src/CodeIndex/Indexer/AmbiguousMContentMasker.cs + - src/CodeIndex/Indexer/References/Languages/ScientificNativeReferenceExtractor.cs + - src/CodeIndex/Indexer/References/Languages/ScientificNativeCommentMasker.cs + - src/CodeIndex/Indexer/References/ReferenceExtractor.CoreExtraction.cs + - src/CodeIndex/Indexer/References/ReferenceExtractor.Preparation.cs + - src/CodeIndex/Indexer/References/ReferenceExtractor.State.cs + - src/CodeIndex/Indexer/References/ReferenceExtractor.TypeReferences.cs + - src/CodeIndex/Indexer/References/ReferenceExtractor.cs + - src/CodeIndex/Indexer/Symbols/SymbolExtractor.ExtractCore.cs + - src/CodeIndex/Indexer/Symbols/SymbolExtractor.cs + - tests/CodeIndex.Tests/QueryCommandRunnerTests.cs + - tests/CodeIndex.Tests/ReferenceExtractorTests.ScientificNative.cs + - tests/CodeIndex.Tests/SymbolExtractorTests.cs + - tests/CodeIndex.Tests/golden/status.json + - USER_GUIDE.md + - DEVELOPER_GUIDE.md +--- + +## English + +- **Scientific and native-extension languages now expose reference graphs (#4738)** — Nim, MATLAB, Julia, D, Cython, and Ada emit bounded language-aware call, import/module, and type/base references. Ambiguous `.m` files preserve content-based classification and expose a conservative MATLAB/Objective-C symbol and reference union when evidence remains unresolved. + +## 日本語 + +- **科学技術・ネイティブ拡張言語が reference graph に対応しました (#4738)** — Nim、MATLAB、Julia、D、Cython、Ada は、言語構文に応じた call、import/module、type/base reference を上限付きで出力します。曖昧な `.m` は content-based classification を維持し、証拠が未確定のときは MATLAB / Objective-C の symbol・reference を保守的に統合して公開します。 diff --git a/src/CodeIndex/Indexer/AmbiguousMContentMasker.cs b/src/CodeIndex/Indexer/AmbiguousMContentMasker.cs new file mode 100644 index 000000000..afa11f183 --- /dev/null +++ b/src/CodeIndex/Indexer/AmbiguousMContentMasker.cs @@ -0,0 +1,165 @@ +using System.Text; + +namespace CodeIndex.Indexer; + +/// +/// Masks MATLAB and Objective-C comments in unresolved .m content while preserving positions. +/// 未確定の .m 内容にある MATLAB / Objective-C コメントを、位置を保ったままマスクする。 +/// +internal static class AmbiguousMContentMasker +{ + internal static string MaskComments(string content) + { + if (content.IndexOfAny(['%', '/']) < 0) + return content; + + StringBuilder? masked = null; + var inBlockComment = false; + var inMatlabBlockComment = false; + var inLineComment = false; + var quote = '\0'; + + for (var index = 0; index < content.Length; index++) + { + var current = content[index]; + if (current is '\r' or '\n') + { + inLineComment = false; + quote = '\0'; + continue; + } + + if (inLineComment) + { + MaskAt(index); + continue; + } + + if (inMatlabBlockComment) + { + MaskAt(index); + if (current == '%' + && index + 1 < content.Length + && content[index + 1] == '}' + && IsStandaloneMatlabBlockDelimiter(content, index)) + { + MaskAt(++index); + inMatlabBlockComment = false; + } + continue; + } + + if (inBlockComment) + { + MaskAt(index); + if (current == '*' && index + 1 < content.Length && content[index + 1] == '/') + { + MaskAt(++index); + inBlockComment = false; + } + continue; + } + + if (quote != '\0') + { + if (current == '\\' && index + 1 < content.Length) + { + index++; + continue; + } + + if (current == quote) + { + if (quote == '\'' && index + 1 < content.Length && content[index + 1] == '\'') + { + index++; + continue; + } + + quote = '\0'; + } + continue; + } + + if (current == '"' || (current == '\'' && IsSingleQuoteStart(content, index))) + { + quote = current; + continue; + } + + if (current == '%') + { + MaskAt(index); + if (index + 1 < content.Length + && content[index + 1] == '{' + && IsStandaloneMatlabBlockDelimiter(content, index)) + { + inMatlabBlockComment = true; + } + else + { + inLineComment = true; + } + continue; + } + + if (current != '/' || index + 1 >= content.Length) + continue; + + var next = content[index + 1]; + if (next == '/') + { + MaskAt(index); + MaskAt(++index); + inLineComment = true; + } + else if (next == '*') + { + MaskAt(index); + MaskAt(++index); + inBlockComment = true; + } + } + + return masked?.ToString() ?? content; + + void MaskAt(int index) + { + masked ??= new StringBuilder(content); + masked[index] = ' '; + } + } + + private static bool IsSingleQuoteStart(string content, int quoteIndex) + { + for (var index = quoteIndex - 1; index >= 0 && content[index] is not '\r' and not '\n'; index--) + { + if (char.IsWhiteSpace(content[index])) + continue; + + return !IsTransposeOperandEnd(content[index]); + } + + return true; + } + + private static bool IsStandaloneMatlabBlockDelimiter(string content, int percentIndex) + { + for (var index = percentIndex - 1; index >= 0 && content[index] is not '\r' and not '\n'; index--) + { + if (!char.IsWhiteSpace(content[index])) + return false; + } + + for (var index = percentIndex + 2; index < content.Length && content[index] is not '\r' and not '\n'; index++) + { + if (!char.IsWhiteSpace(content[index])) + return false; + } + + return true; + } + + private static bool IsTransposeOperandEnd(char value) => + char.IsLetterOrDigit(value) || value is '_' or ')' or ']' or '}'; +} diff --git a/src/CodeIndex/Indexer/References/Languages/ScientificNativeCommentMasker.cs b/src/CodeIndex/Indexer/References/Languages/ScientificNativeCommentMasker.cs new file mode 100644 index 000000000..e25fe1548 --- /dev/null +++ b/src/CodeIndex/Indexer/References/Languages/ScientificNativeCommentMasker.cs @@ -0,0 +1,168 @@ +namespace CodeIndex.Indexer; + +internal static class ScientificNativeCommentMasker +{ + internal static string[] MaskBlockComments(string language, string[] lines) => + language switch + { + "d" => MaskNestedBlockComments(lines, "/+", "+/", "//"), + "julia" => MaskNestedBlockComments(lines, "#=", "=#", "#", singleQuoteCanBePostfix: true), + "nim" => MaskNestedBlockComments(lines, "#[", "]#", "#"), + "matlab" => MaskMatlabBlockComments(lines), + _ => lines, + }; + + private static string[] MaskNestedBlockComments( + string[] lines, + string opening, + string closing, + string lineComment, + bool singleQuoteCanBePostfix = false) + { + if (!MayContain(lines, opening)) + return lines; + + var result = new string[lines.Length]; + var depth = 0; + for (var lineIndex = 0; lineIndex < lines.Length; lineIndex++) + { + var line = lines[lineIndex]; + char[]? chars = null; + var quote = '\0'; + + void MaskAt(int index) => + (chars ??= line.ToCharArray())[index] = ' '; + + var cursor = 0; + while (cursor < line.Length) + { + if (depth > 0) + { + if (StartsWith(line, cursor, opening)) + { + MaskToken(opening); + depth++; + continue; + } + + if (StartsWith(line, cursor, closing)) + { + MaskToken(closing); + depth--; + continue; + } + + MaskAt(cursor++); + continue; + } + + if (quote != '\0') + { + if (line[cursor] == '\\' && cursor + 1 < line.Length) + { + cursor += 2; + continue; + } + + if (line[cursor] == quote) + quote = '\0'; + cursor++; + continue; + } + + if (line[cursor] is '"' or '\'' or '`') + { + if (line[cursor] == '\'' + && singleQuoteCanBePostfix + && IsPostfixSingleQuote(line, cursor)) + { + cursor++; + continue; + } + + quote = line[cursor++]; + continue; + } + + if (StartsWith(line, cursor, opening)) + { + MaskToken(opening); + depth++; + continue; + } + + if (StartsWith(line, cursor, lineComment)) + break; + + cursor++; + } + + result[lineIndex] = chars is null ? line : new string(chars); + + void MaskToken(string token) + { + for (var tokenIndex = 0; tokenIndex < token.Length; tokenIndex++) + MaskAt(cursor++); + } + } + + return result; + } + + private static bool IsPostfixSingleQuote(string line, int quoteIndex) + { + for (var index = quoteIndex - 1; index >= 0; index--) + { + if (char.IsWhiteSpace(line[index])) + continue; + + return char.IsLetterOrDigit(line[index]) || line[index] is '_' or ')' or ']' or '}'; + } + + return false; + } + + private static string[] MaskMatlabBlockComments(string[] lines) + { + if (!MayContain(lines, "%{")) + return lines; + + var result = new string[lines.Length]; + var inBlockComment = false; + for (var index = 0; index < lines.Length; index++) + { + var line = lines[index]; + var trimmed = line.Trim(); + var opens = !inBlockComment && trimmed.Equals("%{", StringComparison.Ordinal); + if (opens) + inBlockComment = true; + + if (!inBlockComment) + { + result[index] = line; + continue; + } + + result[index] = new string(' ', line.Length); + if (trimmed.Equals("%}", StringComparison.Ordinal)) + inBlockComment = false; + } + + return result; + } + + private static bool MayContain(IEnumerable lines, string token) + { + foreach (var line in lines) + { + if (line.Contains(token, StringComparison.Ordinal)) + return true; + } + + return false; + } + + private static bool StartsWith(string line, int start, string value) => + start + value.Length <= line.Length + && line.AsSpan(start, value.Length).SequenceEqual(value); +} diff --git a/src/CodeIndex/Indexer/References/Languages/ScientificNativeReferenceExtractor.cs b/src/CodeIndex/Indexer/References/Languages/ScientificNativeReferenceExtractor.cs new file mode 100644 index 000000000..f1e330ff0 --- /dev/null +++ b/src/CodeIndex/Indexer/References/Languages/ScientificNativeReferenceExtractor.cs @@ -0,0 +1,153 @@ +using System.Text.RegularExpressions; +using CodeIndex.Models; +using Regex = CodeIndex.Indexer.BoundedRegex; + +namespace CodeIndex.Indexer; + +internal static class ScientificNativeReferenceExtractor +{ + private static readonly HashSet SupportedLanguages = + new(StringComparer.Ordinal) { "ada", "cython", "d", "julia", "matlab", "nim", "objc" }; + + private static readonly Regex NimImportRegex = new( + @"^\s*(?:from\s+(?[A-Za-z_][\w./]*)\s+import\b|(?:import|include)\s+(?[A-Za-z_][\w./]*))", + RegexOptions.Compiled | RegexOptions.CultureInvariant); + private static readonly Regex NimBaseTypeRegex = new( + @"\bobject\s+of\s+(?[A-Za-z_]\w*(?:\.[A-Za-z_]\w*)*)", + RegexOptions.Compiled | RegexOptions.CultureInvariant); + private static readonly Regex NimAnnotatedTypeRegex = new( + @":\s*(?:(?:var|lent|sink)\s+)?(?[A-Z][A-Za-z0-9_]*(?:\.[A-Za-z_]\w*)*)", + RegexOptions.Compiled | RegexOptions.CultureInvariant); + + private static readonly Regex MatlabImportRegex = new( + @"^\s*import\s+(?[A-Za-z]\w*(?:\.[A-Za-z*]\w*)*)", + RegexOptions.Compiled | RegexOptions.CultureInvariant); + private static readonly Regex MatlabBaseTypeRegex = new( + @"^\s*classdef\b[^<\r\n]*<\s*(?[A-Za-z]\w*(?:\.[A-Za-z]\w*)*)", + RegexOptions.Compiled | RegexOptions.CultureInvariant); + + private static readonly Regex JuliaImportRegex = new( + @"^\s*(?:using|import)\s+(?\.*[A-Za-z_]\w*(?:\.[A-Za-z_]\w*)*)", + RegexOptions.Compiled | RegexOptions.CultureInvariant); + private static readonly Regex JuliaTypeRegex = new( + @"(?:<:|::)\s*(?[A-Z][A-Za-z0-9_]*(?:\.[A-Za-z_]\w*)*)", + RegexOptions.Compiled | RegexOptions.CultureInvariant); + private static readonly Regex JuliaMacroCallRegex = new( + @"(?[A-Za-z_]\w*)", + RegexOptions.Compiled | RegexOptions.CultureInvariant); + + private static readonly Regex DImportRegex = new( + @"^\s*(?:(?:public|static)\s+)*import\s+(?:(?:[A-Za-z_]\w*)\s*=\s*)?(?[A-Za-z_]\w*(?:\.[A-Za-z_]\w*)*)", + RegexOptions.Compiled | RegexOptions.CultureInvariant); + private static readonly Regex DBaseTypeRegex = new( + @"^\s*(?:class|interface)\s+[A-Za-z_]\w*(?:\s*\([^)]*\))?\s*:\s*(?[A-Za-z_]\w*(?:\.[A-Za-z_]\w*)*)", + RegexOptions.Compiled | RegexOptions.CultureInvariant); + + private static readonly Regex CythonImportRegex = new( + @"^\s*(?:from\s+(?[A-Za-z_]\w*(?:\.[A-Za-z_]\w*)*)\s+cimport\b|(?:cimport|import)\s+(?[A-Za-z_]\w*(?:\.[A-Za-z_]\w*)*))", + RegexOptions.Compiled | RegexOptions.CultureInvariant); + private static readonly Regex CythonBaseTypeRegex = new( + @"^\s*cdef\s+class\s+[A-Za-z_]\w*\s*\(\s*(?[A-Za-z_]\w*(?:\.[A-Za-z_]\w*)*)", + RegexOptions.Compiled | RegexOptions.CultureInvariant); + + private static readonly Regex AdaImportRegex = new( + @"^\s*(?:(?:limited|private)\s+)*with\s+(?[A-Za-z]\w*(?:\.[A-Za-z]\w*)*)", + RegexOptions.Compiled | RegexOptions.IgnoreCase | RegexOptions.CultureInvariant); + private static readonly Regex AdaDerivedTypeRegex = new( + @"^\s*type\s+[A-Za-z]\w*\s+is\s+new\s+(?[A-Za-z]\w*(?:\.[A-Za-z]\w*)*)", + RegexOptions.Compiled | RegexOptions.IgnoreCase | RegexOptions.CultureInvariant); + private static readonly Regex AdaBareCallRegex = new( + @"^\s*(?!(?:end|null|return|exit|raise|goto)\b)(?[A-Za-z]\w*(?:\.[A-Za-z]\w*)*)\s*;", + RegexOptions.Compiled | RegexOptions.IgnoreCase | RegexOptions.CultureInvariant); + private static readonly Regex ObjectiveCImportRegex = new( + """^\s*#\s*(?:import|include)\s*[<"](?[^>"]+)[>"]""", + RegexOptions.Compiled | RegexOptions.CultureInvariant); + + internal static bool Supports(string language) => SupportedLanguages.Contains(language); + + internal static void EmitReferences( + string language, + string preparedLine, + List references, + ReferenceDedupeSet seen, + long fileId, + string context, + int lineNumber, + Func resolveContainerForColumn, + Action addCallLikeReference) + { + switch (language) + { + case "nim": + EmitMatch(NimImportRegex, "import"); + EmitMatches(NimBaseTypeRegex, "type_reference"); + EmitMatches(NimAnnotatedTypeRegex, "type_reference"); + break; + case "matlab": + EmitMatch(MatlabImportRegex, "import"); + EmitMatches(MatlabBaseTypeRegex, "type_reference"); + break; + case "julia": + EmitMatch(JuliaImportRegex, "import"); + EmitMatches(JuliaTypeRegex, "type_reference"); + foreach (Match match in JuliaMacroCallRegex.Matches(preparedLine)) + { + var group = match.Groups["name"]; + addCallLikeReference(group.Value, group.Index); + } + break; + case "d": + EmitMatch(DImportRegex, "import"); + EmitMatches(DBaseTypeRegex, "type_reference"); + break; + case "cython": + EmitMatch(CythonImportRegex, "import"); + EmitMatches(CythonBaseTypeRegex, "type_reference"); + break; + case "ada": + EmitMatch(AdaImportRegex, "import"); + EmitMatches(AdaDerivedTypeRegex, "type_reference"); + var bareCall = AdaBareCallRegex.Match(preparedLine); + if (bareCall.Success) + { + var group = bareCall.Groups["name"]; + addCallLikeReference(group.Value, group.Index); + } + break; + case "objc": + EmitMatch(ObjectiveCImportRegex, "import"); + break; + } + + void EmitMatch(Regex regex, string referenceKind) + { + var match = regex.Match(preparedLine); + if (match.Success) + EmitGroup(match.Groups["name"], referenceKind); + } + + void EmitMatches(Regex regex, string referenceKind) + { + foreach (Match match in regex.Matches(preparedLine)) + EmitGroup(match.Groups["name"], referenceKind); + } + + void EmitGroup(Group group, string referenceKind) + { + if (!group.Success || group.Length == 0) + return; + + ReferenceExtractor.AddReference( + references, + seen, + fileId, + group.Value, + group.Index, + referenceKind, + context, + lineNumber, + resolveContainerForColumn(group.Index), + language); + } + } +} diff --git a/src/CodeIndex/Indexer/References/ReferenceExtractor.CoreExtraction.cs b/src/CodeIndex/Indexer/References/ReferenceExtractor.CoreExtraction.cs index 36d333e68..086c8b885 100644 --- a/src/CodeIndex/Indexer/References/ReferenceExtractor.CoreExtraction.cs +++ b/src/CodeIndex/Indexer/References/ReferenceExtractor.CoreExtraction.cs @@ -20,6 +20,9 @@ internal static List ExtractCore(ReferenceExtractionContext req var isJsxFile = IsJsxFilePath(path); var isRazorFile = IsRazorFilePath(path) || requestedLanguage is "razor" or "blazor" or "cshtml"; + if (language == "ambiguous_m") + return ExtractAmbiguousMReferences(request); + if (TryExtractStructuralMetadataReferences( fileId, language, @@ -2534,6 +2537,20 @@ bool TryGetKnownPythonTypeCall(string candidate, out string canonicalName) } else { + if (ScientificNativeReferenceExtractor.Supports(language)) + { + ScientificNativeReferenceExtractor.EmitReferences( + language, + preparedLine, + references, + seen, + fileId, + context, + lineNumber, + ResolveContainerForCall, + AddCallLikeReference); + } + foreach (Match match in CallRegex.Matches(preparedLine)) { var name = match.Groups["name"].Value; @@ -3416,5 +3433,67 @@ void AddGradleDslReference(string name, int callIndex) return references; } + private static List ExtractAmbiguousMReferences(ReferenceExtractionContext request) + { + if (string.IsNullOrEmpty(request.Content) + || (request.HasOversizeLine ?? ChunkSplitter.HasOversizeLine(request.Content)) + || (request.ConflictMarkerLine ?? FileIndexer.GetConflictMarkerLine(request.Content)) > 0) + { + return []; + } + + var normalizedContent = request.ContentIsNormalized + ? request.Content + : FileIndexer.NormalizeContentForPrepass(request.Content); + var originalLines = SplitContentLines(normalizedContent); + var maskedContent = AmbiguousMContentMasker.MaskComments(normalizedContent); + var matlabReferences = ExtractCore(request with + { + Language = "matlab", + Content = maskedContent, + RequestedLanguage = "ambiguous_m", + ContentIsNormalized = true, + HasOversizeLine = false, + ConflictMarkerLine = 0, + }); + var objectiveCReferences = ExtractCore(request with + { + Language = "objc", + Content = maskedContent, + RequestedLanguage = "ambiguous_m", + ContentIsNormalized = true, + HasOversizeLine = false, + ConflictMarkerLine = 0, + }); + var merged = CreateReferenceList( + request.MaxReferenceCount, + Math.Min(matlabReferences.Count + objectiveCReferences.Count, ReferenceListInitialCapacityMax)); + var seen = new ReferenceDedupeSet(merged.Capacity); + + AddUnique(matlabReferences); + AddUnique(objectiveCReferences); + return merged; + + void AddUnique(IReadOnlyList candidates) + { + for (var index = 0; index < candidates.Count && !ReferenceLimitReached(merged); index++) + { + var candidate = candidates[index]; + if (candidate.Line > 0 && candidate.Line <= originalLines.Length) + candidate.Context = originalLines[candidate.Line - 1].Trim(); + var key = CreateReferenceDedupeKey( + candidate.FileId, + "ambiguous_m", + candidate.Line, + candidate.Column, + candidate.ReferenceKind, + candidate.SymbolName, + candidate.ContainerKind, + candidate.ContainerName); + if (seen.Add(key)) + TryAddReference(merged, candidate); + } + } + } } diff --git a/src/CodeIndex/Indexer/References/ReferenceExtractor.Preparation.cs b/src/CodeIndex/Indexer/References/ReferenceExtractor.Preparation.cs index 352835ea1..49a3442d6 100644 --- a/src/CodeIndex/Indexer/References/ReferenceExtractor.Preparation.cs +++ b/src/CodeIndex/Indexer/References/ReferenceExtractor.Preparation.cs @@ -80,7 +80,8 @@ private static bool TryPrepareReferenceLines( ? MaskJavaTextBlocks(content) : content; var lines = SplitContentLines(maskedContent); - var structuralLines = StructuralLineMasker.MaskLines(language, lines, out var jsTaggedTemplateHits); + var structuralMaskLanguage = language == "cython" ? "python" : language; + var structuralLines = StructuralLineMasker.MaskLines(structuralMaskLanguage, lines, out var jsTaggedTemplateHits); var csharpLineState = language == "csharp" && MightContainCSharpXmlDocComment(content) ? BuildCSharpLineStateMasks(lines) : (MultilineStringContent: null, BlockComment: null); @@ -99,6 +100,7 @@ [new CppLexicalRange(0, maskedContent.Length)], : UsesCStyleBlockComments(language) ? MaskCStyleBlockCommentLines(language, structuralLines) : structuralLines; + referenceStructuralLines = ScientificNativeCommentMasker.MaskBlockComments(language, referenceStructuralLines); if (language == "python") referenceStructuralLines = MaskPythonFStrings(referenceStructuralLines); diff --git a/src/CodeIndex/Indexer/References/ReferenceExtractor.State.cs b/src/CodeIndex/Indexer/References/ReferenceExtractor.State.cs index e74c5bce4..f898b7b62 100644 --- a/src/CodeIndex/Indexer/References/ReferenceExtractor.State.cs +++ b/src/CodeIndex/Indexer/References/ReferenceExtractor.State.cs @@ -32,7 +32,8 @@ internal sealed class CSharpWhereConstraintState "r", "powershell", "shell", "haskell", "solidity", "gradle", "terraform", "protobuf", "dockerfile", "makefile", "cmake", "justfile", "msbuild", "solution", "dependency_manifest", "dependency_lock", "json", - "zig", "css", "sass", "stylus", "xml", "yaml", "graphql", "html", "markdown", "fortran", "pascal", "objc", "smalltalk" + "zig", "css", "sass", "stylus", "xml", "yaml", "graphql", "html", "markdown", "fortran", "pascal", "objc", "smalltalk", + "ada", "cython", "d", "julia", "matlab", "nim", "ambiguous_m" ]; private static readonly IReadOnlyDictionary Extractors = diff --git a/src/CodeIndex/Indexer/References/ReferenceExtractor.TypeReferences.cs b/src/CodeIndex/Indexer/References/ReferenceExtractor.TypeReferences.cs index 7fe3719bd..739ffa903 100644 --- a/src/CodeIndex/Indexer/References/ReferenceExtractor.TypeReferences.cs +++ b/src/CodeIndex/Indexer/References/ReferenceExtractor.TypeReferences.cs @@ -2718,6 +2718,7 @@ private readonly record struct ReferenceLinePrepareOptions( bool UsesRHashComments, bool UsesSlashComments, bool UsesDashDashComments, + bool UsesPercentComments, bool UsesFortranBangComments, bool UsesPascalBlockComments, bool UsesVisualBasicComments); @@ -2732,6 +2733,7 @@ private static ReferenceLinePrepareOptions CreateReferenceLinePrepareOptions(str UsesRHashComments: lang == "r", UsesSlashComments: UsesSlashComments(lang), UsesDashDashComments: UsesDashDashComments(lang), + UsesPercentComments: lang == "matlab", UsesFortranBangComments: lang == "fortran", UsesPascalBlockComments: lang == "pascal", UsesVisualBasicComments: lang == "vb"); @@ -2784,6 +2786,13 @@ private static string PrepareLine(string line, ReferenceLinePrepareOptions optio result = result[..dashCommentIndex]; } + if (options.UsesPercentComments) + { + var percentCommentIndex = result.IndexOf('%'); + if (percentCommentIndex >= 0) + result = result[..percentCommentIndex]; + } + if (options.UsesFortranBangComments) { var bangCommentIndex = result.IndexOf('!'); @@ -2976,7 +2985,7 @@ private static bool MayContainPascalBlockComment(IReadOnlyList lines) } private static bool UsesCStyleBlockComments(string language) => - language is "c" or "cpp" or "go" or "objc" or "dart"; + language is "c" or "cpp" or "go" or "objc" or "dart" or "d"; private static string[] MaskCStyleBlockCommentLines(string language, IReadOnlyList lines) { @@ -4762,15 +4771,17 @@ private static bool IsAnnotationContext(string line, int probe) private static bool UsesHashComments(string lang) => lang is "python" or "ruby" or "perl" or "php" or "elixir" or "r" or "powershell" - or "shell" or "makefile" or "terraform" or "dockerfile" or "protobuf"; + or "shell" or "makefile" or "terraform" or "dockerfile" or "protobuf" + or "nim" or "julia" or "cython"; private static bool UsesSlashComments(string lang) => lang is not "python" and not "ruby" and not "r" and not "haskell" and not "makefile" and not "terraform" and not "dockerfile" - and not "css" and not "fortran"; + and not "css" and not "fortran" and not "nim" and not "matlab" + and not "julia" and not "cython" and not "ada"; private static bool UsesDashDashComments(string lang) => - lang is "lua" or "sql" or "haskell"; + lang is "lua" or "sql" or "haskell" or "ada"; private static bool IsPythonStringPrefixChar(char c) => c is 'r' or 'R' or 'u' or 'U' or 'b' or 'B' or 'f' or 'F'; diff --git a/src/CodeIndex/Indexer/References/ReferenceExtractor.cs b/src/CodeIndex/Indexer/References/ReferenceExtractor.cs index a4edf6619..c1fefbd54 100644 --- a/src/CodeIndex/Indexer/References/ReferenceExtractor.cs +++ b/src/CodeIndex/Indexer/References/ReferenceExtractor.cs @@ -380,6 +380,38 @@ private static bool IsFunctionLikeSymbolKind(string kind) { "false", "nil", "self", "super", "thisContext", "true", }, + ["ada"] = new HashSet(StringComparer.OrdinalIgnoreCase) + { + "accept", "begin", "case", "declare", "delay", "else", "elsif", "end", "entry", + "exception", "exit", "function", "generic", "if", "loop", "package", "pragma", + "procedure", "raise", "record", "renames", "return", "select", "task", "terminate", + "type", "use", "when", "while", "with", + }, + ["cython"] = new HashSet(StringComparer.Ordinal) + { + "cdef", "cpdef", "ctypedef", "cimport", "def", "extern", "gil", "include", + "nogil", "property", + }, + ["d"] = new HashSet(StringComparer.Ordinal) + { + "assert", "debug", "mixin", "scope", "static", "unittest", "version", + }, + ["julia"] = new HashSet(StringComparer.Ordinal) + { + "abstract", "baremodule", "begin", "do", "export", "finally", "function", "import", + "let", "macro", "module", "mutable", "primitive", "quote", "struct", "using", "where", + }, + ["matlab"] = new HashSet(StringComparer.Ordinal) + { + "case", "catch", "classdef", "elseif", "end", "function", "import", "methods", + "otherwise", "parfor", "properties", "spmd", + }, + ["nim"] = new HashSet(StringComparer.Ordinal) + { + "block", "case", "concept", "converter", "defer", "discard", "distinct", "from", + "func", "import", "include", "iterator", "macro", "method", "mixin", "object", + "proc", "template", "type", "when", + }, // Gradle/Groovy keywords / Gradle/Groovy キーワード ["gradle"] = new HashSet(StringComparer.Ordinal) { @@ -1479,7 +1511,7 @@ private static int GetRustEnumCandidateSpan(SymbolRecord symbol) => (symbol.BodyEndLine ?? symbol.EndLine) - (symbol.BodyStartLine ?? symbol.StartLine); private static StringComparer GetDefinitionNamesComparer(string language) - => language == "sql" + => language is "sql" or "ada" ? StringComparer.OrdinalIgnoreCase : StringComparer.Ordinal; diff --git a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.ExtractCore.cs b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.ExtractCore.cs index 01b137ce3..7f4375a61 100644 --- a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.ExtractCore.cs +++ b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.ExtractCore.cs @@ -81,6 +81,35 @@ private static List ExtractCore( return DependencyPackageExtractor.ExtractSymbols(fileId, content, SplitContentLines(content), filePath, lang); } + if (lang == "ambiguous_m") + { + var maskedContent = AmbiguousMContentMasker.MaskComments(content); + var matlabSymbols = ExtractCore( + fileId, + "matlab", + maskedContent, + contentIsNormalized: true, + hasOversizeLine: false, + conflictMarkerLine: 0, + filePath, + projectRoot, + patternConfigsAlreadyLoaded: true, + cancellationToken); + var objectiveCSymbols = ExtractCore( + fileId, + "objc", + maskedContent, + contentIsNormalized: true, + hasOversizeLine: false, + conflictMarkerLine: 0, + filePath, + projectRoot, + patternConfigsAlreadyLoaded: true, + cancellationToken); + matlabSymbols.AddRange(objectiveCSymbols); + return matlabSymbols; + } + if (lang == "markdown") { var markdownLines = SplitContentLines(content); diff --git a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.cs b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.cs index 8134ab4cd..c6fc2e6e0 100644 --- a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.cs +++ b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.cs @@ -24,6 +24,7 @@ public static partial class SymbolExtractor public const int FunctionalLanguageContractVersion = 2; public const int DynamicLanguageContractVersion = 2; public const int SystemsLanguageContractVersion = 2; + public const int ScientificNativeGraphContractVersion = 3; private static readonly string[] AdditionalSymbolLanguages = [ "app_manifest", @@ -41,6 +42,7 @@ public static partial class SymbolExtractor "solidity", "solution", "cuda", + "ambiguous_m", "dependency_manifest", "dependency_lock", ]; @@ -79,8 +81,8 @@ public static int GetContractVersion(string? lang) "sass" or "stylus" => StyleAndXamlContractVersion, "xml" => XmlContractVersion, "clojure" or "erlang" or "ocaml" or "raku" => FunctionalLanguageContractVersion, - "crystal" or "groovy" or "julia" or "tcl" => DynamicLanguageContractVersion, - "ada" or "d" or "nim" => SystemsLanguageContractVersion, + "crystal" or "groovy" or "tcl" => DynamicLanguageContractVersion, + "ada" or "ambiguous_m" or "cython" or "d" or "julia" or "matlab" or "nim" or "objc" => ScientificNativeGraphContractVersion, "app_manifest" or "cmake" or "dependency_lock" or "dependency_manifest" or "graphql" or "html" or "json" or "justfile" or "markdown" or "msbuild" or "solution" or "yaml" => ExpandedLanguageContractVersion, _ => DefaultContractVersion, }; diff --git a/tests/CodeIndex.Tests/QueryCommandRunnerTests.cs b/tests/CodeIndex.Tests/QueryCommandRunnerTests.cs index 34263738b..6510b6b3e 100644 --- a/tests/CodeIndex.Tests/QueryCommandRunnerTests.cs +++ b/tests/CodeIndex.Tests/QueryCommandRunnerTests.cs @@ -2709,7 +2709,7 @@ public void RunLanguages_FormatCountReturnsCapabilitySummary_Issue4316() using var project = TestProjectHelper.CreateTempProjectScope("cdidx_languages_format_count_issue4316"); var dbPath = TestProjectHelper.CreateProjectDb(project.Root); TestProjectHelper.InsertIndexedFile(dbPath, "src/App.cs", "csharp", "class App { }\n"); - TestProjectHelper.InsertIndexedFile(dbPath, "src/main.adb", "ada", "procedure Main is begin null; end Main;\n"); + TestProjectHelper.InsertIndexedFile(dbPath, "src/main.pl", "prolog", "main :- true.\n"); var (exitCode, stdout, stderr) = CaptureConsole(() => QueryCommandRunner.RunLanguages(["--db", dbPath, "--indexed-only", "--capability", "missing-any", "--format", "count"], _jsonOptions)); @@ -2761,15 +2761,15 @@ public void RunLanguages_JsonIncludesUnsupportedCapabilityGuidance_Issue4122() using var document = ParseJsonOutput(stdout); var languages = document.RootElement.GetProperty("languages").EnumerateArray() .ToDictionary(entry => entry.GetProperty("lang").GetString()!, entry => entry); - var adaGuidance = languages["ada"].GetProperty("unsupported_guidance").EnumerateArray().ToList(); + var prologGuidance = languages["prolog"].GetProperty("unsupported_guidance").EnumerateArray().ToList(); - var referenceGuidance = adaGuidance.Single(guidance => guidance.GetProperty("capability").GetString() == "references"); - Assert.Contains("Reference extraction is not advertised for 'ada'", referenceGuidance.GetProperty("message").GetString()); + var referenceGuidance = prologGuidance.Single(guidance => guidance.GetProperty("capability").GetString() == "references"); + Assert.Contains("Reference extraction is not advertised for 'prolog'", referenceGuidance.GetProperty("message").GetString()); var referenceCommands = referenceGuidance.GetProperty("recommended_commands").EnumerateArray().Select(command => command.GetString()).ToList(); Assert.Contains("search", referenceCommands); Assert.Contains("definition", referenceCommands); - var graphGuidance = adaGuidance.Single(guidance => guidance.GetProperty("capability").GetString() == "graph"); + var graphGuidance = prologGuidance.Single(guidance => guidance.GetProperty("capability").GetString() == "graph"); Assert.Contains("empty callers, callees, or impact results are not authoritative", graphGuidance.GetProperty("message").GetString()); var graphCommands = graphGuidance.GetProperty("recommended_commands").EnumerateArray().Select(command => command.GetString()).ToList(); Assert.Contains("search", graphCommands); @@ -2855,7 +2855,7 @@ public void RunLanguages_JsonListsModernNodeModuleExtensions() } [Fact] - public void RunLanguages_JsonReportsCythonAndCudaSymbolExtraction_Issue3530() + public void RunLanguages_JsonReportsCythonReferenceAndCudaSymbolExtraction_Issue4738() { var (exitCode, stdout, stderr) = CaptureConsole(() => QueryCommandRunner.RunLanguages(["--json"], _jsonOptions)); @@ -2868,8 +2868,8 @@ public void RunLanguages_JsonReportsCythonAndCudaSymbolExtraction_Issue3530() var cuda = languages.EnumerateArray().Single(lang => lang.GetProperty("lang").GetString() == "cuda"); Assert.True(cython.GetProperty("symbol_extraction").GetBoolean()); - Assert.False(cython.GetProperty("reference_extraction").GetBoolean()); - Assert.False(cython.GetProperty("graph_queries").GetBoolean()); + Assert.True(cython.GetProperty("reference_extraction").GetBoolean()); + Assert.True(cython.GetProperty("graph_queries").GetBoolean()); Assert.True(cuda.GetProperty("symbol_extraction").GetBoolean()); Assert.False(cuda.GetProperty("reference_extraction").GetBoolean()); Assert.False(cuda.GetProperty("graph_queries").GetBoolean()); @@ -3145,7 +3145,7 @@ public void RunLanguages_JsonReportsLanguageMapOverrideProvenance_Issue4617() } [Fact] - public void RunLanguages_JsonReportsMatlabAndPrologSymbolOnlyCapabilities_Issue4612() + public void RunLanguages_JsonReportsScientificNativeReferenceCapabilities_Issue4738() { var (exitCode, stdout, stderr) = CaptureConsole(() => QueryCommandRunner.RunLanguages(["--json"], _jsonOptions)); @@ -3155,13 +3155,16 @@ public void RunLanguages_JsonReportsMatlabAndPrologSymbolOnlyCapabilities_Issue4 using var document = ParseJsonOutput(stdout); var languages = document.RootElement.GetProperty("languages").EnumerateArray() .ToDictionary(entry => entry.GetProperty("lang").GetString()!, entry => entry); - foreach (var language in new[] { "matlab", "prolog" }) + foreach (var language in new[] { "ada", "ambiguous_m", "cython", "d", "julia", "matlab", "nim" }) { Assert.True(languages[language].GetProperty("symbol_extraction").GetBoolean()); - Assert.False(languages[language].GetProperty("reference_extraction").GetBoolean()); - Assert.False(languages[language].GetProperty("graph_queries").GetBoolean()); + Assert.True(languages[language].GetProperty("reference_extraction").GetBoolean()); + Assert.True(languages[language].GetProperty("graph_queries").GetBoolean()); } + Assert.True(languages["prolog"].GetProperty("symbol_extraction").GetBoolean()); + Assert.False(languages["prolog"].GetProperty("reference_extraction").GetBoolean()); + Assert.False(languages["prolog"].GetProperty("graph_queries").GetBoolean()); Assert.Contains(".m", languages["ambiguous_m"].GetProperty("extensions").EnumerateArray().Select(value => value.GetString())); Assert.Contains(".pl", languages["ambiguous_pl"].GetProperty("extensions").EnumerateArray().Select(value => value.GetString())); } @@ -3296,7 +3299,7 @@ public void RunLanguages_Json_SearchOnlyBucketsAdvertiseZeroSymbolAndGraphSuppor var languages = document.RootElement.GetProperty("languages").EnumerateArray() .ToDictionary(entry => entry.GetProperty("lang").GetString()!, entry => entry); - foreach (var symbolOnly in new[] { "ada", "clojure", "crystal", "d", "erlang", "groovy", "julia", "nim", "ocaml", "raku", "tcl" }) + foreach (var symbolOnly in new[] { "clojure", "crystal", "erlang", "groovy", "ocaml", "raku", "tcl" }) { Assert.True(languages.ContainsKey(symbolOnly), $"expected '{symbolOnly}' to be listed"); var entry = languages[symbolOnly]; diff --git a/tests/CodeIndex.Tests/ReferenceExtractorTests.ScientificNative.cs b/tests/CodeIndex.Tests/ReferenceExtractorTests.ScientificNative.cs new file mode 100644 index 000000000..99aee0caf --- /dev/null +++ b/tests/CodeIndex.Tests/ReferenceExtractorTests.ScientificNative.cs @@ -0,0 +1,219 @@ +using CodeIndex.Indexer; + +namespace CodeIndex.Tests; + +public partial class ReferenceExtractorTests +{ + public static TheoryData ScientificNativeReferenceCases => new() + { + { + "nim", + """ + import std/math + type Child = object of Base + proc run() = + # ignoredCall() + #[ + ignoredBlockCall() + ]# + helper() + """, + "std/math", + "Base" + }, + { + "matlab", + """ + classdef Child < Base + methods + function run(obj) + import pkg.Tools + % ignoredCall() + %{ + ignoredBlockCall() + %} + helper(); + end + end + end + """, + "pkg.Tools", + "Base" + }, + { + "julia", + """ + module Sample + using LinearAlgebra + struct Child <: Base + end + function run() + # ignoredCall() + value' #= + ignoredBlockCall() + =# + helper() + end + end + """, + "LinearAlgebra", + "Base" + }, + { + "d", + """ + module sample; + import std.stdio; + class Child : Base { + void run() { + /* ignoredCall(); */ + /+ + ignoredBlockCall(); + +/ + helper(); + } + } + """, + "std.stdio", + "Base" + }, + { + "cython", + """" + from libc.stdlib cimport malloc + cdef class Child(Base): + def run(self): + # ignoredCall() + """ + ignoredBlockCall() + """ + helper() + """", + "libc.stdlib", + "Base" + }, + { + "ada", + """ + with Ada.Text_IO; + package body Demo is + type Child is new Base; + procedure Run is + begin + -- IgnoredCall; + Helper; + end Run; + end Demo; + """, + "Ada.Text_IO", + "Base" + }, + }; + + [Theory] + [MemberData(nameof(ScientificNativeReferenceCases))] + public void Extract_ScientificNativeLanguagesEmitBoundedGraphReferences_Issue4738( + string language, + string content, + string importedName, + string baseTypeName) + { + var symbols = SymbolExtractor.Extract(1, language, content); + + var references = ReferenceExtractor.Extract(1, language, content, symbols); + + Assert.Contains(references, reference => + reference.SymbolName == importedName && reference.ReferenceKind == "import"); + Assert.Contains(references, reference => + reference.SymbolName == baseTypeName && reference.ReferenceKind == "type_reference"); + Assert.Contains(references, reference => + reference.SymbolName.Equals("helper", StringComparison.OrdinalIgnoreCase) + && reference.ReferenceKind == "call"); + Assert.DoesNotContain(references, reference => + reference.SymbolName.Equals("ignoredCall", StringComparison.OrdinalIgnoreCase)); + Assert.DoesNotContain(references, reference => + reference.SymbolName.Equals("ignoredBlockCall", StringComparison.OrdinalIgnoreCase)); + } + + [Fact] + public void Extract_AmbiguousMCombinesMatlabAndObjectiveCSymbolsAndReferences_Issue4738() + { + const string content = """ + #import + @interface Widget : NSObject + @end + + function result = run() + import pkg.Tools + % ignoredMatlabCall() + %{ + ignoredMatlabBlockCall() + %} + helper(); + end + """; + + var symbols = SymbolExtractor.Extract(1, "ambiguous_m", content, "mixed.m"); + var references = ReferenceExtractor.Extract(1, "ambiguous_m", content, symbols, "mixed.m"); + + Assert.Contains(symbols, symbol => symbol.Name == "Widget" && symbol.Kind == "class"); + Assert.Contains(symbols, symbol => symbol.Name == "run" && symbol.Kind == "function"); + Assert.Contains(references, reference => + reference.SymbolName == "Foundation/Foundation.h" && reference.ReferenceKind == "import"); + Assert.Contains(references, reference => + reference.SymbolName == "pkg.Tools" && reference.ReferenceKind == "import"); + Assert.Contains(references, reference => + reference.SymbolName == "helper" && reference.ReferenceKind == "call"); + Assert.DoesNotContain(references, reference => reference.SymbolName == "ignoredMatlabCall"); + Assert.DoesNotContain(references, reference => reference.SymbolName == "ignoredMatlabBlockCall"); + } + + [Fact] + public void Extract_AmbiguousMRespectsObjectiveCCommentsDuringMatlabFallback_Issue4738() + { + const string content = """ + // ignoredObjectiveCCall() + void run(void) { + helper(); + } + """; + + var symbols = SymbolExtractor.Extract(1, "ambiguous_m", content, "unknown.m"); + var references = ReferenceExtractor.Extract(1, "ambiguous_m", content, symbols, "unknown.m"); + + Assert.Contains(references, reference => + reference.SymbolName == "helper" && reference.ReferenceKind == "call"); + Assert.DoesNotContain(references, reference => reference.SymbolName == "ignoredObjectiveCCall"); + } + + [Fact] + public void Extract_AmbiguousMRetainsSharedSafetyGuards_Issue4738() + { + var oversizeContent = new string('x', ChunkSplitter.MaxLineLength + 1) + " helper();"; + const string conflictContent = """ + function result = run() + <<<<<<< ours + helper(); + ======= + alternate(); + >>>>>>> theirs + end + """; + + var oversizeReferences = ReferenceExtractor.Extract( + 1, + "ambiguous_m", + oversizeContent, + [], + "oversize.m"); + var conflictReferences = ReferenceExtractor.Extract( + 1, + "ambiguous_m", + conflictContent, + [], + "conflict.m"); + + Assert.Empty(oversizeReferences); + Assert.Empty(conflictReferences); + } +} diff --git a/tests/CodeIndex.Tests/SymbolExtractorTests.cs b/tests/CodeIndex.Tests/SymbolExtractorTests.cs index 4054e6a9e..3777c609b 100644 --- a/tests/CodeIndex.Tests/SymbolExtractorTests.cs +++ b/tests/CodeIndex.Tests/SymbolExtractorTests.cs @@ -12522,6 +12522,14 @@ public void GetContractVersion_LanguageSpecificKinds_UseDedicatedVersions() Assert.True(SymbolExtractor.StyleAndXamlContractVersion > SymbolExtractor.DefaultContractVersion); Assert.Equal(SymbolExtractor.XmlContractVersion, SymbolExtractor.GetContractVersion("xml")); Assert.True(SymbolExtractor.XmlContractVersion > SymbolExtractor.StyleAndXamlContractVersion); + foreach (var language in new[] { "ada", "ambiguous_m", "cython", "d", "julia", "matlab", "nim", "objc" }) + { + Assert.Equal( + SymbolExtractor.ScientificNativeGraphContractVersion, + SymbolExtractor.GetContractVersion(language)); + } + Assert.True(SymbolExtractor.ScientificNativeGraphContractVersion > SymbolExtractor.DynamicLanguageContractVersion); + Assert.True(SymbolExtractor.ScientificNativeGraphContractVersion > SymbolExtractor.SystemsLanguageContractVersion); } [Fact] diff --git a/tests/CodeIndex.Tests/golden/status.json b/tests/CodeIndex.Tests/golden/status.json index 3129c1975..9bd3f72a0 100644 --- a/tests/CodeIndex.Tests/golden/status.json +++ b/tests/CodeIndex.Tests/golden/status.json @@ -44,6 +44,8 @@ } }, "graph_supported_languages": [ + "ada", + "ambiguous_m", "assembly", "batch", "blazor", @@ -55,6 +57,8 @@ "csharp", "cshtml", "css", + "cython", + "d", "dart", "dependency_lock", "dependency_manifest", @@ -70,12 +74,15 @@ "java", "javascript", "json", + "julia", "justfile", "kotlin", "lua", "makefile", "markdown", + "matlab", "msbuild", + "nim", "objc", "pascal", "perl", From 52c6ba16e42c440eb3054b64f097fdec3744312e Mon Sep 17 00:00:00 2001 From: Widthdom Date: Fri, 24 Jul 2026 12:02:46 +0900 Subject: [PATCH 02/21] Address scientific graph review findings (#4738) --- changelog.d/unreleased/4738.added.md | 1 + .../ScientificNativeCommentMasker.cs | 275 +++++++++++++++++- .../ScientificNativeReferenceExtractor.cs | 199 +++++++++++-- .../ReferenceExtractor.TypeReferences.cs | 17 +- .../Indexer/References/ReferenceExtractor.cs | 2 +- .../SymbolExtractor.ScientificNative.cs | 87 ++++++ .../Indexer/Symbols/SymbolExtractor.cs | 16 +- ...eferenceExtractorTests.ScientificNative.cs | 136 ++++++++- 8 files changed, 678 insertions(+), 55 deletions(-) create mode 100644 src/CodeIndex/Indexer/Symbols/SymbolExtractor.ScientificNative.cs diff --git a/changelog.d/unreleased/4738.added.md b/changelog.d/unreleased/4738.added.md index 6b7bd7dd6..66ffd81cb 100644 --- a/changelog.d/unreleased/4738.added.md +++ b/changelog.d/unreleased/4738.added.md @@ -12,6 +12,7 @@ affected: - src/CodeIndex/Indexer/References/ReferenceExtractor.TypeReferences.cs - src/CodeIndex/Indexer/References/ReferenceExtractor.cs - src/CodeIndex/Indexer/Symbols/SymbolExtractor.ExtractCore.cs + - src/CodeIndex/Indexer/Symbols/SymbolExtractor.ScientificNative.cs - src/CodeIndex/Indexer/Symbols/SymbolExtractor.cs - tests/CodeIndex.Tests/QueryCommandRunnerTests.cs - tests/CodeIndex.Tests/ReferenceExtractorTests.ScientificNative.cs diff --git a/src/CodeIndex/Indexer/References/Languages/ScientificNativeCommentMasker.cs b/src/CodeIndex/Indexer/References/Languages/ScientificNativeCommentMasker.cs index e25fe1548..9cda81d6f 100644 --- a/src/CodeIndex/Indexer/References/Languages/ScientificNativeCommentMasker.cs +++ b/src/CodeIndex/Indexer/References/Languages/ScientificNativeCommentMasker.cs @@ -5,13 +5,279 @@ internal static class ScientificNativeCommentMasker internal static string[] MaskBlockComments(string language, string[] lines) => language switch { - "d" => MaskNestedBlockComments(lines, "/+", "+/", "//"), - "julia" => MaskNestedBlockComments(lines, "#=", "=#", "#", singleQuoteCanBePostfix: true), - "nim" => MaskNestedBlockComments(lines, "#[", "]#", "#"), + "d" => MaskNestedBlockComments(MaskDTokenBraceStrings(lines), "/+", "+/", "//"), + "julia" => MaskNestedBlockComments( + MaskTripleQuotedStrings(lines, "#=", "=#", "#"), + "#=", + "=#", + "#", + singleQuoteCanBePostfix: true), + "nim" => MaskNestedBlockComments( + MaskTripleQuotedStrings(lines, "#[", "]#", "#"), + "#[", + "]#", + "#"), "matlab" => MaskMatlabBlockComments(lines), _ => lines, }; + internal static string MaskLineStringLiteralsPreservingPostfixSingleQuotes(string line) + { + char[]? chars = null; + + void MaskAt(int index) => + (chars ??= line.ToCharArray())[index] = ' '; + + for (var cursor = 0; cursor < line.Length; cursor++) + { + var quote = line[cursor]; + if (quote is not ('"' or '\'' or '`')) + continue; + + if (quote == '\'' && IsPostfixSingleQuote(line, cursor)) + continue; + + MaskAt(cursor); + cursor++; + while (cursor < line.Length) + { + var current = line[cursor]; + MaskAt(cursor); + if (current == '\\' && cursor + 1 < line.Length) + { + MaskAt(++cursor); + cursor++; + continue; + } + + if (current != quote) + { + cursor++; + continue; + } + + if (cursor + 1 < line.Length && line[cursor + 1] == quote) + { + MaskAt(++cursor); + cursor++; + continue; + } + + break; + } + } + + return chars is null ? line : new string(chars); + } + + private static string[] MaskTripleQuotedStrings( + string[] lines, + string blockOpening, + string blockClosing, + string lineComment) + { + if (!MayContain(lines, "\"\"\"")) + return lines; + + var result = new string[lines.Length]; + var inTripleQuotedString = false; + var blockCommentDepth = 0; + for (var lineIndex = 0; lineIndex < lines.Length; lineIndex++) + { + var line = lines[lineIndex]; + char[]? chars = null; + var quote = '\0'; + + void MaskAt(int index) => + (chars ??= line.ToCharArray())[index] = ' '; + + var cursor = 0; + while (cursor < line.Length) + { + if (inTripleQuotedString) + { + if (StartsWith(line, cursor, "\"\"\"")) + { + MaskToken("\"\"\""); + inTripleQuotedString = false; + continue; + } + + MaskAt(cursor++); + continue; + } + + if (blockCommentDepth > 0) + { + if (StartsWith(line, cursor, blockOpening)) + { + blockCommentDepth++; + cursor += blockOpening.Length; + continue; + } + + if (StartsWith(line, cursor, blockClosing)) + { + blockCommentDepth--; + cursor += blockClosing.Length; + continue; + } + + cursor++; + continue; + } + + if (quote != '\0') + { + if (line[cursor] == '\\' && cursor + 1 < line.Length) + { + cursor += 2; + continue; + } + + if (line[cursor] == quote) + quote = '\0'; + cursor++; + continue; + } + + if (StartsWith(line, cursor, lineComment)) + break; + + if (StartsWith(line, cursor, blockOpening)) + { + blockCommentDepth++; + cursor += blockOpening.Length; + continue; + } + + if (StartsWith(line, cursor, "\"\"\"")) + { + MaskToken("\"\"\""); + inTripleQuotedString = true; + continue; + } + + if (line[cursor] is '"' or '\'' or '`') + { + quote = line[cursor++]; + continue; + } + + cursor++; + } + + result[lineIndex] = chars is null ? line : new string(chars); + + void MaskToken(string token) + { + for (var tokenIndex = 0; tokenIndex < token.Length; tokenIndex++) + MaskAt(cursor++); + } + } + + return result; + } + + private static string[] MaskDTokenBraceStrings(string[] lines) + { + if (!MayContain(lines, "q{")) + return lines; + + var result = new string[lines.Length]; + var tokenStringDepth = 0; + var blockCommentDepth = 0; + for (var lineIndex = 0; lineIndex < lines.Length; lineIndex++) + { + var line = lines[lineIndex]; + char[]? chars = null; + var quote = '\0'; + + void MaskAt(int index) => + (chars ??= line.ToCharArray())[index] = ' '; + + var cursor = 0; + while (cursor < line.Length) + { + if (tokenStringDepth > 0) + { + var current = line[cursor]; + MaskAt(cursor++); + if (current == '{') + tokenStringDepth++; + else if (current == '}') + tokenStringDepth--; + continue; + } + + if (blockCommentDepth > 0) + { + if (StartsWith(line, cursor, "/+")) + { + blockCommentDepth++; + cursor += 2; + continue; + } + + if (StartsWith(line, cursor, "+/")) + { + blockCommentDepth--; + cursor += 2; + continue; + } + + cursor++; + continue; + } + + if (quote != '\0') + { + if (line[cursor] == '\\' && cursor + 1 < line.Length) + { + cursor += 2; + continue; + } + + if (line[cursor] == quote) + quote = '\0'; + cursor++; + continue; + } + + if (StartsWith(line, cursor, "//")) + break; + + if (StartsWith(line, cursor, "/+")) + { + blockCommentDepth++; + cursor += 2; + continue; + } + + if (line[cursor] is '"' or '\'' or '`') + { + quote = line[cursor++]; + continue; + } + + if (StartsWith(line, cursor, "q{") + && (cursor == 0 || !IsIdentifierChar(line[cursor - 1]))) + { + MaskAt(cursor++); + MaskAt(cursor++); + tokenStringDepth = 1; + continue; + } + + cursor++; + } + + result[lineIndex] = chars is null ? line : new string(chars); + } + + return result; + } + private static string[] MaskNestedBlockComments( string[] lines, string opening, @@ -165,4 +431,7 @@ private static bool MayContain(IEnumerable lines, string token) private static bool StartsWith(string line, int start, string value) => start + value.Length <= line.Length && line.AsSpan(start, value.Length).SequenceEqual(value); + + private static bool IsIdentifierChar(char value) => + char.IsLetterOrDigit(value) || value == '_'; } diff --git a/src/CodeIndex/Indexer/References/Languages/ScientificNativeReferenceExtractor.cs b/src/CodeIndex/Indexer/References/Languages/ScientificNativeReferenceExtractor.cs index f1e330ff0..0778a93ae 100644 --- a/src/CodeIndex/Indexer/References/Languages/ScientificNativeReferenceExtractor.cs +++ b/src/CodeIndex/Indexer/References/Languages/ScientificNativeReferenceExtractor.cs @@ -6,11 +6,16 @@ namespace CodeIndex.Indexer; internal static class ScientificNativeReferenceExtractor { + private const int MaxDependenciesPerDeclaration = 64; + private static readonly HashSet SupportedLanguages = new(StringComparer.Ordinal) { "ada", "cython", "d", "julia", "matlab", "nim", "objc" }; - private static readonly Regex NimImportRegex = new( - @"^\s*(?:from\s+(?[A-Za-z_][\w./]*)\s+import\b|(?:import|include)\s+(?[A-Za-z_][\w./]*))", + private static readonly Regex NimFromImportRegex = new( + @"^\s*from\s+(?[A-Za-z_][\w./]*)\s+import\b", + RegexOptions.Compiled | RegexOptions.CultureInvariant); + private static readonly Regex NimImportListRegex = new( + @"^\s*(?:import|include)\s+(?[^\r\n]+)", RegexOptions.Compiled | RegexOptions.CultureInvariant); private static readonly Regex NimBaseTypeRegex = new( @"\bobject\s+of\s+(?[A-Za-z_]\w*(?:\.[A-Za-z_]\w*)*)", @@ -19,15 +24,15 @@ internal static class ScientificNativeReferenceExtractor @":\s*(?:(?:var|lent|sink)\s+)?(?[A-Z][A-Za-z0-9_]*(?:\.[A-Za-z_]\w*)*)", RegexOptions.Compiled | RegexOptions.CultureInvariant); - private static readonly Regex MatlabImportRegex = new( - @"^\s*import\s+(?[A-Za-z]\w*(?:\.[A-Za-z*]\w*)*)", + private static readonly Regex MatlabImportListRegex = new( + @"^\s*import\s+(?[^\r\n]+)", RegexOptions.Compiled | RegexOptions.CultureInvariant); - private static readonly Regex MatlabBaseTypeRegex = new( - @"^\s*classdef\b[^<\r\n]*<\s*(?[A-Za-z]\w*(?:\.[A-Za-z]\w*)*)", + private static readonly Regex MatlabBaseTypeListRegex = new( + @"^\s*classdef\b[^<\r\n]*<\s*(?[^\r\n]+)", RegexOptions.Compiled | RegexOptions.CultureInvariant); - private static readonly Regex JuliaImportRegex = new( - @"^\s*(?:using|import)\s+(?\.*[A-Za-z_]\w*(?:\.[A-Za-z_]\w*)*)", + private static readonly Regex JuliaImportListRegex = new( + @"^\s*(?:using|import)\s+(?[^\r\n]+)", RegexOptions.Compiled | RegexOptions.CultureInvariant); private static readonly Regex JuliaTypeRegex = new( @"(?:<:|::)\s*(?[A-Z][A-Za-z0-9_]*(?:\.[A-Za-z_]\w*)*)", @@ -36,22 +41,25 @@ internal static class ScientificNativeReferenceExtractor @"(?[A-Za-z_]\w*)", RegexOptions.Compiled | RegexOptions.CultureInvariant); - private static readonly Regex DImportRegex = new( - @"^\s*(?:(?:public|static)\s+)*import\s+(?:(?:[A-Za-z_]\w*)\s*=\s*)?(?[A-Za-z_]\w*(?:\.[A-Za-z_]\w*)*)", + private static readonly Regex DImportListRegex = new( + @"^\s*(?:(?:public|static)\s+)*import\s+(?[^;\r\n]+)", RegexOptions.Compiled | RegexOptions.CultureInvariant); - private static readonly Regex DBaseTypeRegex = new( - @"^\s*(?:class|interface)\s+[A-Za-z_]\w*(?:\s*\([^)]*\))?\s*:\s*(?[A-Za-z_]\w*(?:\.[A-Za-z_]\w*)*)", + private static readonly Regex DBaseTypeListRegex = new( + @"^\s*(?:class|interface)\s+[A-Za-z_]\w*(?:\s*\([^)]*\))?\s*:\s*(?[^{\r\n]+)", RegexOptions.Compiled | RegexOptions.CultureInvariant); - private static readonly Regex CythonImportRegex = new( - @"^\s*(?:from\s+(?[A-Za-z_]\w*(?:\.[A-Za-z_]\w*)*)\s+cimport\b|(?:cimport|import)\s+(?[A-Za-z_]\w*(?:\.[A-Za-z_]\w*)*))", + private static readonly Regex CythonFromImportRegex = new( + @"^\s*from\s+(?[A-Za-z_]\w*(?:\.[A-Za-z_]\w*)*)\s+cimport\b", + RegexOptions.Compiled | RegexOptions.CultureInvariant); + private static readonly Regex CythonImportListRegex = new( + @"^\s*(?:cimport|import)\s+(?[^\r\n]+)", RegexOptions.Compiled | RegexOptions.CultureInvariant); - private static readonly Regex CythonBaseTypeRegex = new( - @"^\s*cdef\s+class\s+[A-Za-z_]\w*\s*\(\s*(?[A-Za-z_]\w*(?:\.[A-Za-z_]\w*)*)", + private static readonly Regex CythonBaseTypeListRegex = new( + @"^\s*cdef\s+class\s+[A-Za-z_]\w*\s*\(\s*(?[^)\r\n]+)", RegexOptions.Compiled | RegexOptions.CultureInvariant); - private static readonly Regex AdaImportRegex = new( - @"^\s*(?:(?:limited|private)\s+)*with\s+(?[A-Za-z]\w*(?:\.[A-Za-z]\w*)*)", + private static readonly Regex AdaImportListRegex = new( + @"^\s*(?:(?:limited|private)\s+)*with\s+(?[^;\r\n]+)", RegexOptions.Compiled | RegexOptions.IgnoreCase | RegexOptions.CultureInvariant); private static readonly Regex AdaDerivedTypeRegex = new( @"^\s*type\s+[A-Za-z]\w*\s+is\s+new\s+(?[A-Za-z]\w*(?:\.[A-Za-z]\w*)*)", @@ -79,16 +87,17 @@ internal static void EmitReferences( switch (language) { case "nim": - EmitMatch(NimImportRegex, "import"); + EmitMatch(NimFromImportRegex, "import"); + EmitNameList(NimImportListRegex, "import", ','); EmitMatches(NimBaseTypeRegex, "type_reference"); EmitMatches(NimAnnotatedTypeRegex, "type_reference"); break; case "matlab": - EmitMatch(MatlabImportRegex, "import"); - EmitMatches(MatlabBaseTypeRegex, "type_reference"); + EmitNameList(MatlabImportListRegex, "import", ',', splitOnWhitespace: true); + EmitNameList(MatlabBaseTypeListRegex, "type_reference", '&'); break; case "julia": - EmitMatch(JuliaImportRegex, "import"); + EmitNameList(JuliaImportListRegex, "import", ',', stopAtColon: true); EmitMatches(JuliaTypeRegex, "type_reference"); foreach (Match match in JuliaMacroCallRegex.Matches(preparedLine)) { @@ -97,15 +106,16 @@ internal static void EmitReferences( } break; case "d": - EmitMatch(DImportRegex, "import"); - EmitMatches(DBaseTypeRegex, "type_reference"); + EmitNameList(DImportListRegex, "import", ',', stopAtColon: true, stripLeadingAlias: true); + EmitNameList(DBaseTypeListRegex, "type_reference", ','); break; case "cython": - EmitMatch(CythonImportRegex, "import"); - EmitMatches(CythonBaseTypeRegex, "type_reference"); + EmitMatch(CythonFromImportRegex, "import"); + EmitNameList(CythonImportListRegex, "import", ','); + EmitNameList(CythonBaseTypeListRegex, "type_reference", ','); break; case "ada": - EmitMatch(AdaImportRegex, "import"); + EmitNameList(AdaImportListRegex, "import", ','); EmitMatches(AdaDerivedTypeRegex, "type_reference"); var bareCall = AdaBareCallRegex.Match(preparedLine); if (bareCall.Success) @@ -132,22 +142,151 @@ void EmitMatches(Regex regex, string referenceKind) EmitGroup(match.Groups["name"], referenceKind); } + void EmitNameList( + Regex regex, + string referenceKind, + char separator, + bool splitOnWhitespace = false, + bool stopAtColon = false, + bool stripLeadingAlias = false) + { + var match = regex.Match(preparedLine); + if (!match.Success) + return; + + var group = match.Groups["names"]; + if (!group.Success || group.Length == 0) + return; + + var names = group.Value; + var namesEnd = names.Length; + if (stopAtColon) + { + var colonIndex = names.IndexOf(':'); + if (colonIndex >= 0) + namesEnd = colonIndex; + } + + var dependencyCount = 0; + var segmentStart = 0; + for (var index = 0; index <= namesEnd && dependencyCount < MaxDependenciesPerDeclaration; index++) + { + var atEnd = index == namesEnd; + var isSeparator = !atEnd + && (names[index] == separator || (splitOnWhitespace && char.IsWhiteSpace(names[index]))); + if (!atEnd && !isSeparator) + continue; + + if (TryEmitDependencySegment( + names, + segmentStart, + index, + group.Index, + referenceKind, + stripLeadingAlias)) + { + dependencyCount++; + } + + segmentStart = index + 1; + while (segmentStart < namesEnd + && (names[segmentStart] == separator + || (splitOnWhitespace && char.IsWhiteSpace(names[segmentStart])))) + { + segmentStart++; + index++; + } + } + } + + bool TryEmitDependencySegment( + string names, + int segmentStart, + int segmentEnd, + int absoluteOffset, + string referenceKind, + bool stripLeadingAlias) + { + while (segmentStart < segmentEnd && char.IsWhiteSpace(names[segmentStart])) + segmentStart++; + while (segmentEnd > segmentStart && char.IsWhiteSpace(names[segmentEnd - 1])) + segmentEnd--; + if (segmentStart >= segmentEnd) + return false; + + if (stripLeadingAlias) + { + var equalsIndex = names.LastIndexOf('=', segmentEnd - 1, segmentEnd - segmentStart); + if (equalsIndex >= segmentStart) + { + segmentStart = equalsIndex + 1; + while (segmentStart < segmentEnd && char.IsWhiteSpace(names[segmentStart])) + segmentStart++; + } + } + + for (var index = segmentStart; index + 3 < segmentEnd; index++) + { + if (!char.IsWhiteSpace(names[index]) + || !names.AsSpan(index + 1, 2).Equals("as", StringComparison.OrdinalIgnoreCase) + || !char.IsWhiteSpace(names[index + 3])) + { + continue; + } + + segmentEnd = index; + break; + } + + while (segmentEnd > segmentStart && char.IsWhiteSpace(names[segmentEnd - 1])) + segmentEnd--; + + var nameEnd = segmentStart; + while (nameEnd < segmentEnd && IsDependencyNameChar(names[nameEnd])) + nameEnd++; + while (nameEnd > segmentStart && names[nameEnd - 1] is '.' or '/') + nameEnd--; + + var firstIdentifierIndex = segmentStart; + while (firstIdentifierIndex < nameEnd && names[firstIdentifierIndex] == '.') + firstIdentifierIndex++; + if (firstIdentifierIndex >= nameEnd + || !(char.IsLetter(names[firstIdentifierIndex]) || names[firstIdentifierIndex] == '_')) + { + return false; + } + + EmitName( + names[segmentStart..nameEnd], + absoluteOffset + segmentStart, + referenceKind); + return true; + } + void EmitGroup(Group group, string referenceKind) { if (!group.Success || group.Length == 0) return; + EmitName(group.Value, group.Index, referenceKind); + } + + void EmitName(string name, int index, string referenceKind) + { ReferenceExtractor.AddReference( references, seen, fileId, - group.Value, - group.Index, + name, + index, referenceKind, context, lineNumber, - resolveContainerForColumn(group.Index), + resolveContainerForColumn(index), language); } } + + private static bool IsDependencyNameChar(char value) => + char.IsLetterOrDigit(value) || value is '_' or '.' or '/' or '*'; } diff --git a/src/CodeIndex/Indexer/References/ReferenceExtractor.TypeReferences.cs b/src/CodeIndex/Indexer/References/ReferenceExtractor.TypeReferences.cs index 739ffa903..cf74161fd 100644 --- a/src/CodeIndex/Indexer/References/ReferenceExtractor.TypeReferences.cs +++ b/src/CodeIndex/Indexer/References/ReferenceExtractor.TypeReferences.cs @@ -2714,6 +2714,7 @@ private readonly record struct ReferenceLinePrepareOptions( bool MaskRustLifetimes, bool MaskStringLiterals, bool IncludeBacktickStringDelimiter, + bool PreservePostfixSingleQuotes, bool UsesHashComments, bool UsesRHashComments, bool UsesSlashComments, @@ -2729,6 +2730,7 @@ private static ReferenceLinePrepareOptions CreateReferenceLinePrepareOptions(str MaskRustLifetimes: lang == "rust", MaskStringLiterals: lang != "cobol", IncludeBacktickStringDelimiter: lang is not ("kotlin" or "r"), + PreservePostfixSingleQuotes: lang is "julia" or "matlab", UsesHashComments: UsesHashComments(lang), UsesRHashComments: lang == "r", UsesSlashComments: UsesSlashComments(lang), @@ -2754,10 +2756,17 @@ private static string PrepareLine(string line, ReferenceLinePrepareOptions optio result = MaskRustLifetimeTokens(result); if (options.MaskStringLiterals && MayContainStringLiteralDelimiter(result, options.IncludeBacktickStringDelimiter)) { - var stringLiteralRegex = !options.IncludeBacktickStringDelimiter - ? NonBacktickStringLiteralRegex - : StringLiteralRegex; - result = stringLiteralRegex.Replace(result, "\"\""); + if (options.PreservePostfixSingleQuotes) + { + result = ScientificNativeCommentMasker.MaskLineStringLiteralsPreservingPostfixSingleQuotes(result); + } + else + { + var stringLiteralRegex = !options.IncludeBacktickStringDelimiter + ? NonBacktickStringLiteralRegex + : StringLiteralRegex; + result = stringLiteralRegex.Replace(result, "\"\""); + } } if (result.Contains("/*", StringComparison.Ordinal)) result = InlineBlockCommentRegex.Replace(result, " "); diff --git a/src/CodeIndex/Indexer/References/ReferenceExtractor.cs b/src/CodeIndex/Indexer/References/ReferenceExtractor.cs index c1fefbd54..79e51f4d9 100644 --- a/src/CodeIndex/Indexer/References/ReferenceExtractor.cs +++ b/src/CodeIndex/Indexer/References/ReferenceExtractor.cs @@ -394,7 +394,7 @@ private static bool IsFunctionLikeSymbolKind(string kind) }, ["d"] = new HashSet(StringComparer.Ordinal) { - "assert", "debug", "mixin", "scope", "static", "unittest", "version", + "assert", "cast", "debug", "mixin", "scope", "static", "unittest", "version", }, ["julia"] = new HashSet(StringComparer.Ordinal) { diff --git a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.ScientificNative.cs b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.ScientificNative.cs new file mode 100644 index 000000000..71ac3798f --- /dev/null +++ b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.ScientificNative.cs @@ -0,0 +1,87 @@ +using System.Text.RegularExpressions; +using Regex = CodeIndex.Indexer.BoundedRegex; + +namespace CodeIndex.Indexer; + +public static partial class SymbolExtractor +{ + private static readonly Regex JuliaScientificBlockTokenRegex = new( + @"\b(?baremodule|module|mutable\s+struct|struct|abstract\s+type|primitive\s+type|function|macro|if|for|while|try|begin|let|quote|do|end)\b", + RegexOptions.Compiled | RegexOptions.CultureInvariant); + private static readonly Regex MatlabScientificBlockTokenRegex = new( + @"\b(?function|classdef|methods|properties|events|enumeration|arguments|if|for|parfor|while|switch|try|spmd|end)\b", + RegexOptions.Compiled | RegexOptions.IgnoreCase | RegexOptions.CultureInvariant); + + private static (int EndLine, int? BodyStartLine, int? BodyEndLine) FindScientificEndRange( + string[] lines, + int startIndex, + string language) + { + var scannerLines = ScientificNativeCommentMasker.MaskBlockComments(language, lines); + var tokenRegex = language == "julia" + ? JuliaScientificBlockTokenRegex + : MatlabScientificBlockTokenRegex; + var depth = 1; + int? bodyStartLine = null; + + for (var lineIndex = startIndex + 1; lineIndex < scannerLines.Length; lineIndex++) + { + var code = MaskScientificBodyScanLine(scannerLines[lineIndex], language); + if (string.IsNullOrWhiteSpace(code)) + continue; + + bodyStartLine ??= lineIndex + 1; + foreach (Match match in tokenRegex.Matches(code)) + { + var keyword = match.Groups["keyword"].Value; + if (!IsScientificBlockTokenAtStatementBoundary(code, match.Index, keyword, language)) + continue; + + if (keyword.Equals("end", StringComparison.OrdinalIgnoreCase)) + { + depth--; + if (depth == 0) + return (lineIndex + 1, bodyStartLine, lineIndex + 1); + continue; + } + + depth++; + } + } + + return bodyStartLine == null + ? (startIndex + 1, null, null) + : (lines.Length, bodyStartLine, lines.Length); + } + + private static string MaskScientificBodyScanLine(string line, string language) + { + var masked = ScientificNativeCommentMasker.MaskLineStringLiteralsPreservingPostfixSingleQuotes(line); + var commentMarker = language == "julia" ? '#' : '%'; + var commentIndex = masked.IndexOf(commentMarker); + return commentIndex >= 0 ? masked[..commentIndex] : masked; + } + + private static bool IsScientificBlockTokenAtStatementBoundary( + string line, + int tokenIndex, + string keyword, + string language) + { + if (language == "julia" && keyword == "do") + return true; + + for (var index = tokenIndex - 1; index >= 0; index--) + { + if (char.IsWhiteSpace(line[index])) + continue; + + if (line[index] == ';') + return true; + + return language == "julia" && line[index] == '='; + } + + return true; + } +} diff --git a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.cs b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.cs index c6fc2e6e0..26d30a206 100644 --- a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.cs +++ b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.cs @@ -481,6 +481,7 @@ private enum BodyStyle RubyEnd, FortranEnd, ElixirEnd, + ScientificEnd, VisualBasicEnd, PascalEnd, SmalltalkMethod, @@ -1630,11 +1631,11 @@ private enum JavaScriptTypeScriptFunctionHeaderConsumeResult ], ["julia"] = [ - new("namespace", new Regex(@"^\s*(?:baremodule|module)\s+(?[A-Za-z_]\w*)", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.ElixirEnd), - new("struct", new Regex(@"^\s*(?:mutable\s+)?struct\s+(?[A-Za-z_]\w*)", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.ElixirEnd), - new("type", new Regex(@"^\s*(?:abstract|primitive)\s+type\s+(?[A-Za-z_]\w*)", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.ElixirEnd), - new("function", new Regex(@"^\s*function\s+(?[A-Za-z_]\w*(?:\.[A-Za-z_]\w*)?)\s*(?:\(|\{)", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.ElixirEnd), - new("function", new Regex(@"^\s*macro\s+(?[A-Za-z_]\w*)\s*(?:\(|$)", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.ElixirEnd), + new("namespace", new Regex(@"^\s*(?:baremodule|module)\s+(?[A-Za-z_]\w*)", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.ScientificEnd), + new("struct", new Regex(@"^\s*(?:mutable\s+)?struct\s+(?[A-Za-z_]\w*)", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.ScientificEnd), + new("type", new Regex(@"^\s*(?:abstract|primitive)\s+type\s+(?[A-Za-z_]\w*)", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.ScientificEnd), + new("function", new Regex(@"^\s*function\s+(?[A-Za-z_]\w*(?:\.[A-Za-z_]\w*)?)\s*(?:\(|\{)", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.ScientificEnd), + new("function", new Regex(@"^\s*macro\s+(?[A-Za-z_]\w*)\s*(?:\(|$)", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.ScientificEnd), new("function", new Regex(@"^\s*(?[A-Za-z_]\w*(?:\.[A-Za-z_]\w*)?)\s*\([^)\r\n]*\)\s*=", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None), new("property", new Regex(@"^\s*const\s+(?[A-Z_]\w*)\s*=", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None), new("import", new Regex(@"^\s*(?:using|import)\s+(?[A-Za-z_]\w*(?:\.[A-Za-z_]\w*)*)", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None), @@ -1702,8 +1703,8 @@ private enum JavaScriptTypeScriptFunctionHeaderConsumeResult ], ["matlab"] = [ - new("class", new Regex(@"^\s*classdef\s*(?:\([^)]*\)\s*)?(?[A-Za-z]\w*)\b", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None), - new("function", new Regex(@"^\s*function\s+(?:(?:\[[^\]]+\]|[A-Za-z]\w*)\s*=\s*)?(?[A-Za-z]\w*)\s*(?:\(|$)", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None), + new("class", new Regex(@"^\s*classdef\s*(?:\([^)]*\)\s*)?(?[A-Za-z]\w*)\b", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.ScientificEnd), + new("function", new Regex(@"^\s*function\s+(?:(?:\[[^\]]+\]|[A-Za-z]\w*)\s*=\s*)?(?[A-Za-z]\w*)\s*(?:\(|$)", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.ScientificEnd), new("import", new Regex(@"^\s*import\s+(?[A-Za-z]\w*(?:\.[A-Za-z*]\w*)*)", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None), ], ["prolog"] = @@ -4798,6 +4799,7 @@ private static (int EndLine, int? BodyStartLine, int? BodyEndLine) ResolveRange( BodyStyle.RubyEnd => FindRubyRange(lines, startIndex), BodyStyle.FortranEnd => FindFortranRange(lines, startIndex), BodyStyle.ElixirEnd => FindElixirRange(lines, startIndex), + BodyStyle.ScientificEnd when lang is "julia" or "matlab" => FindScientificEndRange(lines, startIndex, lang), BodyStyle.VisualBasicEnd => FindVisualBasicRange(lines, startIndex), BodyStyle.PascalEnd => FindPascalRange(lines, startIndex), BodyStyle.SmalltalkMethod => FindSmalltalkMethodRange(lines, startIndex), diff --git a/tests/CodeIndex.Tests/ReferenceExtractorTests.ScientificNative.cs b/tests/CodeIndex.Tests/ReferenceExtractorTests.ScientificNative.cs index 99aee0caf..880721e4e 100644 --- a/tests/CodeIndex.Tests/ReferenceExtractorTests.ScientificNative.cs +++ b/tests/CodeIndex.Tests/ReferenceExtractorTests.ScientificNative.cs @@ -4,12 +4,12 @@ namespace CodeIndex.Tests; public partial class ReferenceExtractorTests { - public static TheoryData ScientificNativeReferenceCases => new() + public static TheoryData ScientificNativeReferenceCases => new() { { "nim", """ - import std/math + import std/math, strutils type Child = object of Base proc run() = # ignoredCall() @@ -19,6 +19,7 @@ proc run() = helper() """, "std/math", + "strutils", "Base" }, { @@ -27,7 +28,7 @@ proc run() = classdef Child < Base methods function run(obj) - import pkg.Tools + import pkg.Tools pkg.Other % ignoredCall() %{ ignoredBlockCall() @@ -38,13 +39,14 @@ import pkg.Tools end """, "pkg.Tools", + "pkg.Other", "Base" }, { "julia", """ module Sample - using LinearAlgebra + using LinearAlgebra, Statistics struct Child <: Base end function run() @@ -57,14 +59,15 @@ function run() end """, "LinearAlgebra", + "Statistics", "Base" }, { "d", """ module sample; - import std.stdio; - class Child : Base { + import std.stdio, std.algorithm; + class Child : Base, IFace { void run() { /* ignoredCall(); */ /+ @@ -75,12 +78,13 @@ void run() { } """, "std.stdio", + "std.algorithm", "Base" }, { "cython", """" - from libc.stdlib cimport malloc + cimport numpy, cython cdef class Child(Base): def run(self): # ignoredCall() @@ -89,13 +93,14 @@ def run(self): """ helper() """", - "libc.stdlib", + "numpy", + "cython", "Base" }, { "ada", """ - with Ada.Text_IO; + with Ada.Text_IO, Ada.Command_Line; package body Demo is type Child is new Base; procedure Run is @@ -106,6 +111,7 @@ procedure Run is end Demo; """, "Ada.Text_IO", + "Ada.Command_Line", "Base" }, }; @@ -116,6 +122,7 @@ public void Extract_ScientificNativeLanguagesEmitBoundedGraphReferences_Issue473 string language, string content, string importedName, + string secondImportedName, string baseTypeName) { var symbols = SymbolExtractor.Extract(1, language, content); @@ -125,16 +132,125 @@ public void Extract_ScientificNativeLanguagesEmitBoundedGraphReferences_Issue473 Assert.Contains(references, reference => reference.SymbolName == importedName && reference.ReferenceKind == "import"); Assert.Contains(references, reference => - reference.SymbolName == baseTypeName && reference.ReferenceKind == "type_reference"); + reference.SymbolName == secondImportedName && reference.ReferenceKind == "import"); Assert.Contains(references, reference => + reference.SymbolName == baseTypeName && reference.ReferenceKind == "type_reference"); + var helperReference = Assert.Single(references, reference => reference.SymbolName.Equals("helper", StringComparison.OrdinalIgnoreCase) && reference.ReferenceKind == "call"); + Assert.Equal("run", helperReference.ContainerName, ignoreCase: true); Assert.DoesNotContain(references, reference => reference.SymbolName.Equals("ignoredCall", StringComparison.OrdinalIgnoreCase)); Assert.DoesNotContain(references, reference => reference.SymbolName.Equals("ignoredBlockCall", StringComparison.OrdinalIgnoreCase)); } + [Theory] + [InlineData( + "matlab", + """ + function result = run(a, b) + text = 'stringCall()'; + result = a' * helper() * b'; + end + """)] + [InlineData( + "julia", + """ + function run(a, b) + text = "stringCall()" + a' * helper() * b' + end + """)] + public void Extract_MatlabAndJuliaPreservePostfixTransposeCalls_Issue4738( + string language, + string content) + { + var symbols = SymbolExtractor.Extract(1, language, content); + + var references = ReferenceExtractor.Extract(1, language, content, symbols); + + var helperReference = Assert.Single(references, reference => + reference.SymbolName == "helper" && reference.ReferenceKind == "call"); + Assert.Equal("run", helperReference.ContainerName); + Assert.DoesNotContain(references, reference => reference.SymbolName == "stringCall"); + } + + public static TheoryData ScientificNativeMultilineLiteralCases => new() + { + { + "julia", + """" + function run() + text = """ + #= tokenOnly() + """ + helper() + end + """" + }, + { + "nim", + """" + proc run() = + let text = """ + #[ tokenOnly() + """ + helper() + """" + }, + { + "d", + """ + void run() { + auto text = q{ + /+ tokenOnly(); + }; + helper(); + } + """ + }, + }; + + [Theory] + [MemberData(nameof(ScientificNativeMultilineLiteralCases))] + public void Extract_ScientificNativeLiteralTokensDoNotSuppressFollowingCalls_Issue4738( + string language, + string content) + { + var symbols = SymbolExtractor.Extract(1, language, content); + + var references = ReferenceExtractor.Extract(1, language, content, symbols); + + Assert.Contains(references, reference => + reference.SymbolName == "helper" && reference.ReferenceKind == "call"); + Assert.DoesNotContain(references, reference => reference.SymbolName == "tokenOnly"); + } + + [Fact] + public void Extract_DMultipleBaseTypesAndCastSyntaxStayGraphAccurate_Issue4738() + { + const string content = """ + class Child : Base, IFace { + void run() { + auto value = cast(int)(helper()); + } + } + """; + var symbols = SymbolExtractor.Extract(1, "d", content); + + var references = ReferenceExtractor.Extract(1, "d", content, symbols); + + Assert.Contains(references, reference => + reference.SymbolName == "Base" && reference.ReferenceKind == "type_reference"); + Assert.Contains(references, reference => + reference.SymbolName == "IFace" && reference.ReferenceKind == "type_reference"); + Assert.Contains(references, reference => + reference.SymbolName == "helper" && reference.ReferenceKind == "call"); + Assert.DoesNotContain(references, reference => + reference.SymbolName == "cast" && reference.ReferenceKind == "call"); + } + [Fact] public void Extract_AmbiguousMCombinesMatlabAndObjectiveCSymbolsAndReferences_Issue4738() { From c8c219b858ec79c0b30c162f386fa946de9b595a Mon Sep 17 00:00:00 2001 From: Widthdom Date: Fri, 24 Jul 2026 12:55:00 +0900 Subject: [PATCH 03/21] Fix scientific graph lexical edge cases (#4738) --- changelog.d/unreleased/4738.added.md | 8 +- src/CodeIndex/Database/DbContext.cs | 2 +- .../DbWriter.ReferenceGraphRefreshScope.cs | 18 +- src/CodeIndex/Database/DbWriter.References.cs | 30 +++- .../Indexer/AmbiguousMContentMasker.cs | 76 ++++++-- .../ScientificNativeCommentMasker.cs | 21 ++- .../ScientificNativeReferenceExtractor.cs | 170 +++++++++++++++++- .../ReferenceExtractor.CoreExtraction.cs | 19 +- .../ReferenceExtractor.TypeReferences.cs | 6 +- .../Symbols/SymbolExtractor.ExtractCore.cs | 25 ++- .../SymbolExtractor.ScientificNative.cs | 26 ++- .../Indexer/Symbols/SymbolExtractor.cs | 15 +- tests/CodeIndex.Tests/DatabaseTests.cs | 77 ++++++++ ...eferenceExtractorTests.ScientificNative.cs | 147 +++++++++++++++ 14 files changed, 585 insertions(+), 55 deletions(-) diff --git a/changelog.d/unreleased/4738.added.md b/changelog.d/unreleased/4738.added.md index 66ffd81cb..5eaa232e3 100644 --- a/changelog.d/unreleased/4738.added.md +++ b/changelog.d/unreleased/4738.added.md @@ -3,6 +3,9 @@ category: added issues: - 4738 affected: + - src/CodeIndex/Database/DbContext.cs + - src/CodeIndex/Database/DbWriter.ReferenceGraphRefreshScope.cs + - src/CodeIndex/Database/DbWriter.References.cs - src/CodeIndex/Indexer/AmbiguousMContentMasker.cs - src/CodeIndex/Indexer/References/Languages/ScientificNativeReferenceExtractor.cs - src/CodeIndex/Indexer/References/Languages/ScientificNativeCommentMasker.cs @@ -15,6 +18,7 @@ affected: - src/CodeIndex/Indexer/Symbols/SymbolExtractor.ScientificNative.cs - src/CodeIndex/Indexer/Symbols/SymbolExtractor.cs - tests/CodeIndex.Tests/QueryCommandRunnerTests.cs + - tests/CodeIndex.Tests/DatabaseTests.cs - tests/CodeIndex.Tests/ReferenceExtractorTests.ScientificNative.cs - tests/CodeIndex.Tests/SymbolExtractorTests.cs - tests/CodeIndex.Tests/golden/status.json @@ -24,8 +28,8 @@ affected: ## English -- **Scientific and native-extension languages now expose reference graphs (#4738)** — Nim, MATLAB, Julia, D, Cython, and Ada emit bounded language-aware call, import/module, and type/base references. Ambiguous `.m` files preserve content-based classification and expose a conservative MATLAB/Objective-C symbol and reference union when evidence remains unresolved. +- **Scientific and native-extension languages now expose reference graphs (#4738)** — Nim, MATLAB, Julia, D, Cython, and Ada emit bounded language-aware call, import/module, and type/base references. Ambiguous `.m` files preserve content-based classification, expose a conservative MATLAB/Objective-C symbol and reference union when evidence remains unresolved, and resolve those references against definitively classified MATLAB or Objective-C files. ## 日本語 -- **科学技術・ネイティブ拡張言語が reference graph に対応しました (#4738)** — Nim、MATLAB、Julia、D、Cython、Ada は、言語構文に応じた call、import/module、type/base reference を上限付きで出力します。曖昧な `.m` は content-based classification を維持し、証拠が未確定のときは MATLAB / Objective-C の symbol・reference を保守的に統合して公開します。 +- **科学技術・ネイティブ拡張言語が reference graph に対応しました (#4738)** — Nim、MATLAB、Julia、D、Cython、Ada は、言語構文に応じた call、import/module、type/base reference を上限付きで出力します。曖昧な `.m` は content-based classification を維持し、証拠が未確定のときは MATLAB / Objective-C の symbol・reference を保守的に統合して公開し、確定済みの MATLAB / Objective-C ファイルにある参照先へ解決します。 diff --git a/src/CodeIndex/Database/DbContext.cs b/src/CodeIndex/Database/DbContext.cs index 07ca6634f..1da5a4cd8 100644 --- a/src/CodeIndex/Database/DbContext.cs +++ b/src/CodeIndex/Database/DbContext.cs @@ -2054,7 +2054,7 @@ public static string BuildIncompleteHotspotFamilyMarkerFingerprint(string? finge public const string CSharpStaticInterfaceSourceEvidenceMetaKey = "csharp_static_interface_source_evidence"; public const int SqlGraphContractVersion = 1; public const string SqlGraphContractVersionMetaKey = "sql_graph_contract_version"; - public const int ReferenceIdentityContractVersion = 1; + public const int ReferenceIdentityContractVersion = 2; public const string ReferenceIdentityContractVersionMetaKey = "reference_identity_contract_version"; public const string SymbolsOnlyGraphOmittedMetaKey = "symbols_only_graph_omitted"; public const string IndexedProjectRootMetaKey = "indexed_project_root"; diff --git a/src/CodeIndex/Database/DbWriter.ReferenceGraphRefreshScope.cs b/src/CodeIndex/Database/DbWriter.ReferenceGraphRefreshScope.cs index 949333775..5f8195471 100644 --- a/src/CodeIndex/Database/DbWriter.ReferenceGraphRefreshScope.cs +++ b/src/CodeIndex/Database/DbWriter.ReferenceGraphRefreshScope.cs @@ -197,7 +197,10 @@ FROM temp.{ReferenceGraphDirtyNamesTable} AS dirty_name CROSS JOIN symbol_references AS r INDEXED BY idx_symbol_refs_symbol_name_folded JOIN files AS source_file ON source_file.id = r.file_id WHERE r.symbol_name_folded = dirty_name.name_folded - AND source_file.lang = dirty_name.lang; + AND ( + source_file.lang = dirty_name.lang + OR (source_file.lang = 'ambiguous_m' AND dirty_name.lang IN ('matlab', 'objc')) + ); """; private const string MaterializeReferenceGraphLookupNamesSql = $""" @@ -209,6 +212,19 @@ FROM temp.{ReferenceGraphDirtyReferencesTable} AS dirty WHERE source_file.lang IS NOT NULL AND r.symbol_name_folded IS NOT NULL; + INSERT OR IGNORE INTO temp.{ReferenceGraphLookupNamesTable}(lang, name_folded) + SELECT target_lang.lang, r.symbol_name_folded + FROM temp.{ReferenceGraphDirtyReferencesTable} AS dirty + JOIN symbol_references AS r ON r.id = dirty.reference_id + JOIN files AS source_file ON source_file.id = r.file_id + CROSS JOIN ( + SELECT 'matlab' AS lang + UNION ALL + SELECT 'objc' + ) AS target_lang + WHERE source_file.lang = 'ambiguous_m' + AND r.symbol_name_folded IS NOT NULL; + INSERT OR IGNORE INTO temp.{ReferenceGraphLookupNamesTable}(lang, name_folded) SELECT 'csharp', r.symbol_name_folded || 'attribute' FROM temp.{ReferenceGraphDirtyReferencesTable} AS dirty diff --git a/src/CodeIndex/Database/DbWriter.References.cs b/src/CodeIndex/Database/DbWriter.References.cs index 40e3e3047..cc19a36f4 100644 --- a/src/CodeIndex/Database/DbWriter.References.cs +++ b/src/CodeIndex/Database/DbWriter.References.cs @@ -142,7 +142,10 @@ ON s.name_folded IN ( THEN r.symbol_name_folded || 'attribute' END ) JOIN files AS target_file ON target_file.id = s.file_id - WHERE source_file.lang = target_file.lang + WHERE ( + source_file.lang = target_file.lang + OR (source_file.lang = 'ambiguous_m' AND target_file.lang IN ('matlab', 'objc')) + ) AND r.target_qualifier IS NOT NULL AND r.target_qualifier NOT LIKE char(31) || 'receiver:%' AND ( @@ -195,7 +198,10 @@ THEN r.symbol_name_folded || 'attribute' END ) JOIN files AS target_file ON target_file.id = s.file_id JOIN symbols AS source ON source.id = r.source_symbol_id - WHERE source_file.lang = target_file.lang + WHERE ( + source_file.lang = target_file.lang + OR (source_file.lang = 'ambiguous_m' AND target_file.lang IN ('matlab', 'objc')) + ) AND r.target_qualifier IS NULL AND s.file_id = r.file_id AND source.container_name IS NOT NULL @@ -221,7 +227,10 @@ THEN r.symbol_name_folded || 'attribute' END ) JOIN files AS target_file ON target_file.id = s.file_id JOIN symbols AS source ON source.id = r.source_symbol_id - WHERE source_file.lang = target_file.lang + WHERE ( + source_file.lang = target_file.lang + OR (source_file.lang = 'ambiguous_m' AND target_file.lang IN ('matlab', 'objc')) + ) AND r.target_qualifier IS NULL AND source.container_qualified_name IS NOT NULL AND source.container_qualified_name <> '' @@ -242,7 +251,10 @@ ON s.name_folded IN ( THEN r.symbol_name_folded || 'attribute' END ) JOIN files AS target_file ON target_file.id = s.file_id - WHERE source_file.lang = target_file.lang + WHERE ( + source_file.lang = target_file.lang + OR (source_file.lang = 'ambiguous_m' AND target_file.lang IN ('matlab', 'objc')) + ) AND r.target_qualifier IS NULL AND s.file_id = r.file_id AND NOT EXISTS ( @@ -262,7 +274,10 @@ THEN r.symbol_name_folded || 'attribute' END ) JOIN files AS target_file ON target_file.id = s.file_id JOIN symbols AS source ON source.id = r.source_symbol_id - WHERE source_file.lang = target_file.lang + WHERE ( + source_file.lang = target_file.lang + OR (source_file.lang = 'ambiguous_m' AND target_file.lang IN ('matlab', 'objc')) + ) AND r.target_qualifier IS NULL AND source.container_name IS NOT NULL AND source.container_name <> '' @@ -277,7 +292,10 @@ INSERT INTO symbol_reference_candidates(reference_id, symbol_id, scope_rank) FROM symbol_references AS r JOIN files AS source_file ON source_file.id = r.file_id JOIN temp.reference_unique_symbol_families AS unique_family - ON unique_family.lang = source_file.lang + ON ( + unique_family.lang = source_file.lang + OR (source_file.lang = 'ambiguous_m' AND unique_family.lang IN ('matlab', 'objc')) + ) AND unique_family.name_folded = r.symbol_name_folded JOIN symbols AS target ON target.name_folded = unique_family.name_folded JOIN files AS target_file diff --git a/src/CodeIndex/Indexer/AmbiguousMContentMasker.cs b/src/CodeIndex/Indexer/AmbiguousMContentMasker.cs index afa11f183..6e64bcbb6 100644 --- a/src/CodeIndex/Indexer/AmbiguousMContentMasker.cs +++ b/src/CodeIndex/Indexer/AmbiguousMContentMasker.cs @@ -8,10 +8,17 @@ namespace CodeIndex.Indexer; /// internal static class AmbiguousMContentMasker { - internal static string MaskComments(string content) + internal static string MaskComments( + string content, + bool maskMatlabComments, + bool maskObjectiveCComments, + bool preserveObjectiveCModuloExpressions = false) { - if (content.IndexOfAny(['%', '/']) < 0) + if ((!maskMatlabComments || content.IndexOf('%') < 0) + && (!maskObjectiveCComments || content.IndexOf('/') < 0)) + { return content; + } StringBuilder? masked = null; var inBlockComment = false; @@ -62,7 +69,9 @@ internal static string MaskComments(string content) if (quote != '\0') { - if (current == '\\' && index + 1 < content.Length) + if (preserveObjectiveCModuloExpressions + && current == '\\' + && index + 1 < content.Length) { index++; continue; @@ -81,14 +90,23 @@ internal static string MaskComments(string content) continue; } - if (current == '"' || (current == '\'' && IsSingleQuoteStart(content, index))) + if (current == '"' + || (current == '\'' + && (preserveObjectiveCModuloExpressions + || IsMatlabSingleQuoteStart(content, index)))) { quote = current; continue; } - if (current == '%') + if (maskMatlabComments && current == '%') { + if (preserveObjectiveCModuloExpressions + && LooksLikeObjectiveCModuloOperator(content, index)) + { + continue; + } + MaskAt(index); if (index + 1 < content.Length && content[index + 1] == '{' @@ -103,7 +121,7 @@ internal static string MaskComments(string content) continue; } - if (current != '/' || index + 1 >= content.Length) + if (!maskObjectiveCComments || current != '/' || index + 1 >= content.Length) continue; var next = content[index + 1]; @@ -130,17 +148,12 @@ void MaskAt(int index) } } - private static bool IsSingleQuoteStart(string content, int quoteIndex) + private static bool IsMatlabSingleQuoteStart(string content, int quoteIndex) { - for (var index = quoteIndex - 1; index >= 0 && content[index] is not '\r' and not '\n'; index--) - { - if (char.IsWhiteSpace(content[index])) - continue; - - return !IsTransposeOperandEnd(content[index]); - } - - return true; + var previousIndex = quoteIndex - 1; + return previousIndex < 0 + || content[previousIndex] is '\r' or '\n' + || !IsTransposeOperandEnd(content[previousIndex]); } private static bool IsStandaloneMatlabBlockDelimiter(string content, int percentIndex) @@ -160,6 +173,37 @@ private static bool IsStandaloneMatlabBlockDelimiter(string content, int percent return true; } + private static bool LooksLikeObjectiveCModuloOperator(string content, int percentIndex) + { + var previousIndex = percentIndex - 1; + while (previousIndex >= 0 + && content[previousIndex] is not '\r' and not '\n' + && char.IsWhiteSpace(content[previousIndex])) + { + previousIndex--; + } + + if (previousIndex < 0 + || content[previousIndex] is '\r' or '\n' + || !IsTransposeOperandEnd(content[previousIndex])) + { + return false; + } + + var nextIndex = percentIndex + 1; + while (nextIndex < content.Length + && content[nextIndex] is not '\r' and not '\n' + && char.IsWhiteSpace(content[nextIndex])) + { + nextIndex++; + } + + return nextIndex < content.Length + && content[nextIndex] is not '\r' and not '\n' + && (char.IsLetterOrDigit(content[nextIndex]) + || content[nextIndex] is '_' or '(' or '\'' or '"'); + } + private static bool IsTransposeOperandEnd(char value) => char.IsLetterOrDigit(value) || value is '_' or ')' or ']' or '}'; } diff --git a/src/CodeIndex/Indexer/References/Languages/ScientificNativeCommentMasker.cs b/src/CodeIndex/Indexer/References/Languages/ScientificNativeCommentMasker.cs index 9cda81d6f..5d9673bed 100644 --- a/src/CodeIndex/Indexer/References/Languages/ScientificNativeCommentMasker.cs +++ b/src/CodeIndex/Indexer/References/Languages/ScientificNativeCommentMasker.cs @@ -21,7 +21,9 @@ internal static string[] MaskBlockComments(string language, string[] lines) => _ => lines, }; - internal static string MaskLineStringLiteralsPreservingPostfixSingleQuotes(string line) + internal static string MaskLineStringLiteralsPreservingPostfixSingleQuotes( + string line, + bool useMatlabStringRules) { char[]? chars = null; @@ -34,7 +36,11 @@ void MaskAt(int index) => if (quote is not ('"' or '\'' or '`')) continue; - if (quote == '\'' && IsPostfixSingleQuote(line, cursor)) + if (quote == '\'' + && IsPostfixSingleQuote( + line, + cursor, + skipWhitespace: !useMatlabStringRules)) continue; MaskAt(cursor); @@ -43,7 +49,9 @@ void MaskAt(int index) => { var current = line[cursor]; MaskAt(cursor); - if (current == '\\' && cursor + 1 < line.Length) + if (!useMatlabStringRules + && current == '\\' + && cursor + 1 < line.Length) { MaskAt(++cursor); cursor++; @@ -375,11 +383,14 @@ void MaskToken(string token) return result; } - private static bool IsPostfixSingleQuote(string line, int quoteIndex) + private static bool IsPostfixSingleQuote( + string line, + int quoteIndex, + bool skipWhitespace = true) { for (var index = quoteIndex - 1; index >= 0; index--) { - if (char.IsWhiteSpace(line[index])) + if (skipWhitespace && char.IsWhiteSpace(line[index])) continue; return char.IsLetterOrDigit(line[index]) || line[index] is '_' or ')' or ']' or '}'; diff --git a/src/CodeIndex/Indexer/References/Languages/ScientificNativeReferenceExtractor.cs b/src/CodeIndex/Indexer/References/Languages/ScientificNativeReferenceExtractor.cs index 0778a93ae..349b36bd0 100644 --- a/src/CodeIndex/Indexer/References/Languages/ScientificNativeReferenceExtractor.cs +++ b/src/CodeIndex/Indexer/References/Languages/ScientificNativeReferenceExtractor.cs @@ -40,6 +40,9 @@ internal static class ScientificNativeReferenceExtractor private static readonly Regex JuliaMacroCallRegex = new( @"(?[A-Za-z_]\w*)", RegexOptions.Compiled | RegexOptions.CultureInvariant); + private static readonly Regex JuliaBroadcastCallRegex = new( + @"(?[A-Za-z_]\w*)\s*\.\s*\(", + RegexOptions.Compiled | RegexOptions.CultureInvariant); private static readonly Regex DImportListRegex = new( @"^\s*(?:(?:public|static)\s+)*import\s+(?[^;\r\n]+)", @@ -47,6 +50,9 @@ internal static class ScientificNativeReferenceExtractor private static readonly Regex DBaseTypeListRegex = new( @"^\s*(?:class|interface)\s+[A-Za-z_]\w*(?:\s*\([^)]*\))?\s*:\s*(?[^{\r\n]+)", RegexOptions.Compiled | RegexOptions.CultureInvariant); + private static readonly Regex DTemplateInvocationRegex = new( + @"(?[A-Za-z_]\w*)\s*!\s*(?:[A-Za-z_]\w*(?:\.[A-Za-z_]\w*)*|\([^()\r\n]*\))\s*\(", + RegexOptions.Compiled | RegexOptions.CultureInvariant); private static readonly Regex CythonFromImportRegex = new( @"^\s*from\s+(?[A-Za-z_]\w*(?:\.[A-Za-z_]\w*)*)\s+cimport\b", @@ -73,6 +79,21 @@ internal static class ScientificNativeReferenceExtractor internal static bool Supports(string language) => SupportedLanguages.Contains(language); + internal static bool IsDTemplateArgumentCall(string line, int callIndex) + { + foreach (Match match in DTemplateInvocationRegex.Matches(line)) + { + var name = match.Groups["name"]; + if (callIndex >= name.Index + name.Length + && callIndex < match.Index + match.Length) + { + return true; + } + } + + return false; + } + internal static void EmitReferences( string language, string preparedLine, @@ -88,7 +109,7 @@ internal static void EmitReferences( { case "nim": EmitMatch(NimFromImportRegex, "import"); - EmitNameList(NimImportListRegex, "import", ','); + EmitNimImportList(); EmitMatches(NimBaseTypeRegex, "type_reference"); EmitMatches(NimAnnotatedTypeRegex, "type_reference"); break; @@ -97,17 +118,32 @@ internal static void EmitReferences( EmitNameList(MatlabBaseTypeListRegex, "type_reference", '&'); break; case "julia": - EmitNameList(JuliaImportListRegex, "import", ',', stopAtColon: true); + EmitNameList( + JuliaImportListRegex, + "import", + ',', + stopAtColon: true, + stripLeadingRelativePrefix: true); EmitMatches(JuliaTypeRegex, "type_reference"); foreach (Match match in JuliaMacroCallRegex.Matches(preparedLine)) { var group = match.Groups["name"]; addCallLikeReference(group.Value, group.Index); } + foreach (Match match in JuliaBroadcastCallRegex.Matches(preparedLine)) + { + var group = match.Groups["name"]; + addCallLikeReference(group.Value, group.Index); + } break; case "d": EmitNameList(DImportListRegex, "import", ',', stopAtColon: true, stripLeadingAlias: true); EmitNameList(DBaseTypeListRegex, "type_reference", ','); + foreach (Match match in DTemplateInvocationRegex.Matches(preparedLine)) + { + var group = match.Groups["name"]; + addCallLikeReference(group.Value, group.Index); + } break; case "cython": EmitMatch(CythonFromImportRegex, "import"); @@ -121,7 +157,9 @@ internal static void EmitReferences( if (bareCall.Success) { var group = bareCall.Groups["name"]; - addCallLikeReference(group.Value, group.Index); + var separatorIndex = group.Value.LastIndexOf('.'); + var leafOffset = separatorIndex + 1; + addCallLikeReference(group.Value[leafOffset..], group.Index + leafOffset); } break; case "objc": @@ -148,7 +186,8 @@ void EmitNameList( char separator, bool splitOnWhitespace = false, bool stopAtColon = false, - bool stripLeadingAlias = false) + bool stripLeadingAlias = false, + bool stripLeadingRelativePrefix = false) { var match = regex.Match(preparedLine); if (!match.Success) @@ -183,7 +222,8 @@ void EmitNameList( index, group.Index, referenceKind, - stripLeadingAlias)) + stripLeadingAlias, + stripLeadingRelativePrefix)) { dependencyCount++; } @@ -205,7 +245,8 @@ bool TryEmitDependencySegment( int segmentEnd, int absoluteOffset, string referenceKind, - bool stripLeadingAlias) + bool stripLeadingAlias, + bool stripLeadingRelativePrefix) { while (segmentStart < segmentEnd && char.IsWhiteSpace(names[segmentStart])) segmentStart++; @@ -256,13 +297,128 @@ bool TryEmitDependencySegment( return false; } + var emittedNameStart = stripLeadingRelativePrefix + ? firstIdentifierIndex + : segmentStart; EmitName( - names[segmentStart..nameEnd], + names[emittedNameStart..nameEnd], absoluteOffset + segmentStart, referenceKind); return true; } + void EmitNimImportList() + { + var match = NimImportListRegex.Match(preparedLine); + if (!match.Success) + return; + + var group = match.Groups["names"]; + if (!group.Success || group.Length == 0) + return; + + var names = group.Value; + var dependencyCount = 0; + var segmentStart = 0; + var bracketDepth = 0; + for (var index = 0; index <= names.Length && dependencyCount < MaxDependenciesPerDeclaration; index++) + { + if (index < names.Length) + { + if (names[index] == '[') + bracketDepth++; + else if (names[index] == ']' && bracketDepth > 0) + bracketDepth--; + } + + if (index < names.Length && (names[index] != ',' || bracketDepth != 0)) + continue; + + dependencyCount += EmitNimImportSegment( + names, + segmentStart, + index, + group.Index, + MaxDependenciesPerDeclaration - dependencyCount); + segmentStart = index + 1; + } + } + + int EmitNimImportSegment( + string names, + int segmentStart, + int segmentEnd, + int absoluteOffset, + int remainingCapacity) + { + while (segmentStart < segmentEnd && char.IsWhiteSpace(names[segmentStart])) + segmentStart++; + while (segmentEnd > segmentStart && char.IsWhiteSpace(names[segmentEnd - 1])) + segmentEnd--; + if (segmentStart >= segmentEnd) + return 0; + + var openingBracket = names.IndexOf('[', segmentStart, segmentEnd - segmentStart); + var closingBracket = openingBracket >= 0 + ? names.IndexOf(']', openingBracket + 1, segmentEnd - openingBracket - 1) + : -1; + if (openingBracket < 0 || closingBracket < 0) + { + return TryEmitDependencySegment( + names, + segmentStart, + segmentEnd, + absoluteOffset, + "import", + stripLeadingAlias: false, + stripLeadingRelativePrefix: false) + ? 1 + : 0; + } + + var prefixStart = segmentStart; + var prefixEnd = openingBracket; + while (prefixEnd > prefixStart && char.IsWhiteSpace(names[prefixEnd - 1])) + prefixEnd--; + if (prefixEnd <= prefixStart || names[prefixEnd - 1] != '/') + return 0; + + var prefix = names[prefixStart..prefixEnd]; + var emittedCount = 0; + var itemStart = openingBracket + 1; + for (var index = itemStart; index <= closingBracket && emittedCount < remainingCapacity; index++) + { + if (index < closingBracket && names[index] != ',') + continue; + + var itemEnd = index; + while (itemStart < itemEnd && char.IsWhiteSpace(names[itemStart])) + itemStart++; + while (itemEnd > itemStart && char.IsWhiteSpace(names[itemEnd - 1])) + itemEnd--; + if (itemStart < itemEnd) + { + var nameEnd = itemStart; + while (nameEnd < itemEnd && IsDependencyNameChar(names[nameEnd])) + nameEnd++; + while (nameEnd > itemStart && names[nameEnd - 1] is '.' or '/') + nameEnd--; + if (nameEnd > itemStart) + { + EmitName( + prefix + names[itemStart..nameEnd], + absoluteOffset + itemStart, + "import"); + emittedCount++; + } + } + + itemStart = index + 1; + } + + return emittedCount; + } + void EmitGroup(Group group, string referenceKind) { if (!group.Success || group.Length == 0) diff --git a/src/CodeIndex/Indexer/References/ReferenceExtractor.CoreExtraction.cs b/src/CodeIndex/Indexer/References/ReferenceExtractor.CoreExtraction.cs index 086c8b885..979b22543 100644 --- a/src/CodeIndex/Indexer/References/ReferenceExtractor.CoreExtraction.cs +++ b/src/CodeIndex/Indexer/References/ReferenceExtractor.CoreExtraction.cs @@ -2557,6 +2557,11 @@ bool TryGetKnownPythonTypeCall(string candidate, out string canonicalName) var callIndex = match.Groups["name"].Index; if (language == "rust" && RustReferenceExtractor.IsRawIdentifierPrefix(preparedLine, callIndex)) continue; + if (language == "d" + && ScientificNativeReferenceExtractor.IsDTemplateArgumentCall(preparedLine, callIndex)) + { + continue; + } if (language == "objc" && IsObjCSelectorLiteralCall(preparedLine, name, callIndex)) continue; if (sqlSuppressedCallIndices != null && sqlSuppressedCallIndices.Contains(callIndex)) @@ -3446,11 +3451,19 @@ private static List ExtractAmbiguousMReferences(ReferenceExtrac ? request.Content : FileIndexer.NormalizeContentForPrepass(request.Content); var originalLines = SplitContentLines(normalizedContent); - var maskedContent = AmbiguousMContentMasker.MaskComments(normalizedContent); + var matlabContent = AmbiguousMContentMasker.MaskComments( + normalizedContent, + maskMatlabComments: true, + maskObjectiveCComments: true); + var objectiveCContent = AmbiguousMContentMasker.MaskComments( + normalizedContent, + maskMatlabComments: true, + maskObjectiveCComments: true, + preserveObjectiveCModuloExpressions: true); var matlabReferences = ExtractCore(request with { Language = "matlab", - Content = maskedContent, + Content = matlabContent, RequestedLanguage = "ambiguous_m", ContentIsNormalized = true, HasOversizeLine = false, @@ -3459,7 +3472,7 @@ private static List ExtractAmbiguousMReferences(ReferenceExtrac var objectiveCReferences = ExtractCore(request with { Language = "objc", - Content = maskedContent, + Content = objectiveCContent, RequestedLanguage = "ambiguous_m", ContentIsNormalized = true, HasOversizeLine = false, diff --git a/src/CodeIndex/Indexer/References/ReferenceExtractor.TypeReferences.cs b/src/CodeIndex/Indexer/References/ReferenceExtractor.TypeReferences.cs index cf74161fd..ebe80ae1a 100644 --- a/src/CodeIndex/Indexer/References/ReferenceExtractor.TypeReferences.cs +++ b/src/CodeIndex/Indexer/References/ReferenceExtractor.TypeReferences.cs @@ -2715,6 +2715,7 @@ private readonly record struct ReferenceLinePrepareOptions( bool MaskStringLiterals, bool IncludeBacktickStringDelimiter, bool PreservePostfixSingleQuotes, + bool UseMatlabStringRules, bool UsesHashComments, bool UsesRHashComments, bool UsesSlashComments, @@ -2731,6 +2732,7 @@ private static ReferenceLinePrepareOptions CreateReferenceLinePrepareOptions(str MaskStringLiterals: lang != "cobol", IncludeBacktickStringDelimiter: lang is not ("kotlin" or "r"), PreservePostfixSingleQuotes: lang is "julia" or "matlab", + UseMatlabStringRules: lang == "matlab", UsesHashComments: UsesHashComments(lang), UsesRHashComments: lang == "r", UsesSlashComments: UsesSlashComments(lang), @@ -2758,7 +2760,9 @@ private static string PrepareLine(string line, ReferenceLinePrepareOptions optio { if (options.PreservePostfixSingleQuotes) { - result = ScientificNativeCommentMasker.MaskLineStringLiteralsPreservingPostfixSingleQuotes(result); + result = ScientificNativeCommentMasker.MaskLineStringLiteralsPreservingPostfixSingleQuotes( + result, + options.UseMatlabStringRules); } else { diff --git a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.ExtractCore.cs b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.ExtractCore.cs index e94c0bd63..b8930d342 100644 --- a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.ExtractCore.cs +++ b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.ExtractCore.cs @@ -83,11 +83,19 @@ private static List ExtractCore( if (lang == "ambiguous_m") { - var maskedContent = AmbiguousMContentMasker.MaskComments(content); + var matlabContent = AmbiguousMContentMasker.MaskComments( + content, + maskMatlabComments: true, + maskObjectiveCComments: true); + var objectiveCContent = AmbiguousMContentMasker.MaskComments( + content, + maskMatlabComments: true, + maskObjectiveCComments: true, + preserveObjectiveCModuloExpressions: true); var matlabSymbols = ExtractCore( fileId, "matlab", - maskedContent, + matlabContent, contentIsNormalized: true, hasOversizeLine: false, conflictMarkerLine: 0, @@ -98,7 +106,7 @@ private static List ExtractCore( var objectiveCSymbols = ExtractCore( fileId, "objc", - maskedContent, + objectiveCContent, contentIsNormalized: true, hasOversizeLine: false, conflictMarkerLine: 0, @@ -173,6 +181,9 @@ private static List ExtractCore( : null; var structuralLines = StructuralLineMasker.MaskLines(lang, lines); + var scientificBodyScannerLines = lang is "julia" or "matlab" + ? PrepareScientificBodyScannerLines(structuralLines, lang) + : null; string[]? javaScriptTypeScriptSanitizedLines = null; string[] GetJavaScriptTypeScriptSanitizedLines() => javaScriptTypeScriptSanitizedLines ??= BuildJavaScriptTypeScriptSanitizedLines(lines); @@ -877,7 +888,13 @@ bool[] GetCssQualifiedRuleAncestors() => i, absoluteStartColumn, csharpGateRawStartColumn) - : ResolveRange(rangeLines, i, pattern.BodyStyle, lang, absoluteStartColumn); + : ResolveRange( + rangeLines, + i, + pattern.BodyStyle, + lang, + absoluteStartColumn, + scientificBodyScannerLines); if (fortranContinuationCandidate != null) endLine = Math.Max(endLine, fortranContinuationCandidate.Value.LastConsumedLineIndex + 1); var startLine = i + 1; diff --git a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.ScientificNative.cs b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.ScientificNative.cs index 71ac3798f..51cb75daa 100644 --- a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.ScientificNative.cs +++ b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.ScientificNative.cs @@ -13,11 +13,10 @@ public static partial class SymbolExtractor RegexOptions.Compiled | RegexOptions.IgnoreCase | RegexOptions.CultureInvariant); private static (int EndLine, int? BodyStartLine, int? BodyEndLine) FindScientificEndRange( - string[] lines, + string[] scannerLines, int startIndex, string language) { - var scannerLines = ScientificNativeCommentMasker.MaskBlockComments(language, lines); var tokenRegex = language == "julia" ? JuliaScientificBlockTokenRegex : MatlabScientificBlockTokenRegex; @@ -26,7 +25,7 @@ private static (int EndLine, int? BodyStartLine, int? BodyEndLine) FindScientifi for (var lineIndex = startIndex + 1; lineIndex < scannerLines.Length; lineIndex++) { - var code = MaskScientificBodyScanLine(scannerLines[lineIndex], language); + var code = scannerLines[lineIndex]; if (string.IsNullOrWhiteSpace(code)) continue; @@ -51,12 +50,26 @@ private static (int EndLine, int? BodyStartLine, int? BodyEndLine) FindScientifi return bodyStartLine == null ? (startIndex + 1, null, null) - : (lines.Length, bodyStartLine, lines.Length); + : (scannerLines.Length, bodyStartLine, scannerLines.Length); + } + + private static string[] PrepareScientificBodyScannerLines( + string[] lines, + string language) + { + var blockMaskedLines = ScientificNativeCommentMasker.MaskBlockComments(language, lines); + var scannerLines = new string[blockMaskedLines.Length]; + for (var lineIndex = 0; lineIndex < blockMaskedLines.Length; lineIndex++) + scannerLines[lineIndex] = MaskScientificBodyScanLine(blockMaskedLines[lineIndex], language); + + return scannerLines; } private static string MaskScientificBodyScanLine(string line, string language) { - var masked = ScientificNativeCommentMasker.MaskLineStringLiteralsPreservingPostfixSingleQuotes(line); + var masked = ScientificNativeCommentMasker.MaskLineStringLiteralsPreservingPostfixSingleQuotes( + line, + useMatlabStringRules: language == "matlab"); var commentMarker = language == "julia" ? '#' : '%'; var commentIndex = masked.IndexOf(commentMarker); return commentIndex >= 0 ? masked[..commentIndex] : masked; @@ -76,7 +89,8 @@ private static bool IsScientificBlockTokenAtStatementBoundary( if (char.IsWhiteSpace(line[index])) continue; - if (line[index] == ';') + if (line[index] == ';' + || (language == "matlab" && line[index] == ',')) return true; return language == "julia" && line[index] == '='; diff --git a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.cs b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.cs index 5396811c8..5d2a9b307 100644 --- a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.cs +++ b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.cs @@ -4787,9 +4787,15 @@ private static bool HasRemainingSameLineSignatureOccurrence(SymbolRecord symbol, private static (int EndLine, int? BodyStartLine, int? BodyEndLine) ResolveRange(string[] lines, int startIndex, BodyStyle bodyStyle) => - ResolveRange(lines, startIndex, bodyStyle, null, 0); + ResolveRange(lines, startIndex, bodyStyle, null, 0, null); - private static (int EndLine, int? BodyStartLine, int? BodyEndLine) ResolveRange(string[] lines, int startIndex, BodyStyle bodyStyle, string? lang = null, int startColumn = 0) + private static (int EndLine, int? BodyStartLine, int? BodyEndLine) ResolveRange( + string[] lines, + int startIndex, + BodyStyle bodyStyle, + string? lang = null, + int startColumn = 0, + string[]? scientificBodyScannerLines = null) { return bodyStyle switch { @@ -4802,7 +4808,10 @@ private static (int EndLine, int? BodyStartLine, int? BodyEndLine) ResolveRange( BodyStyle.RubyEnd => FindRubyRange(lines, startIndex), BodyStyle.FortranEnd => FindFortranRange(lines, startIndex), BodyStyle.ElixirEnd => FindElixirRange(lines, startIndex), - BodyStyle.ScientificEnd when lang is "julia" or "matlab" => FindScientificEndRange(lines, startIndex, lang), + BodyStyle.ScientificEnd when lang is "julia" or "matlab" => FindScientificEndRange( + scientificBodyScannerLines ?? PrepareScientificBodyScannerLines(lines, lang), + startIndex, + lang), BodyStyle.VisualBasicEnd => FindVisualBasicRange(lines, startIndex), BodyStyle.PascalEnd => FindPascalRange(lines, startIndex), BodyStyle.SmalltalkMethod => FindSmalltalkMethodRange(lines, startIndex), diff --git a/tests/CodeIndex.Tests/DatabaseTests.cs b/tests/CodeIndex.Tests/DatabaseTests.cs index d39240509..a1b67c89c 100644 --- a/tests/CodeIndex.Tests/DatabaseTests.cs +++ b/tests/CodeIndex.Tests/DatabaseTests.cs @@ -637,6 +637,83 @@ WHERE reference.id IS NULL OR symbol.id IS NULL } } + [Fact] + public void ReferenceGraph_AmbiguousMResolvesAgainstDefinitiveDialects_Issue4738() + { + var callerId = UpsertTestFileWithLanguage("src/caller.m", "ambiguous_m", "ambiguous-caller"); + _writer.InsertReferences([ + new ReferenceRecord + { + FileId = callerId, + SymbolName = "MatlabTarget", + ReferenceKind = "call", + Line = 1, + Column = 1, + Context = "MatlabTarget();", + }, + new ReferenceRecord + { + FileId = callerId, + SymbolName = "ObjectiveCTarget", + ReferenceKind = "call", + Line = 2, + Column = 1, + Context = "ObjectiveCTarget();", + }, + ], refreshMutualRecursionFlags: false); + _writer.RefreshMutualRecursionFlags(); + Assert.Equal(2, ExecuteScalarLong($""" + SELECT COUNT(*) + FROM symbol_references + WHERE file_id = {callerId.ToString(CultureInfo.InvariantCulture)} + AND resolution_state = 'unresolved' + """)); + + using (var scope = _writer.BeginReferenceGraphRefreshScope()) + { + using var transaction = _writer.BeginTransaction(); + var matlabTargetId = _writer.InsertNewFile(new FileRecord + { + Path = "src/matlab-target.m", + Lang = "matlab", + Size = 32, + Lines = 2, + Modified = new DateTime(2026, 7, 24, 0, 0, 0, DateTimeKind.Utc), + Checksum = "matlab-target", + }); + var objectiveCTargetId = _writer.InsertNewFile(new FileRecord + { + Path = "src/objective-c-target.m", + Lang = "objc", + Size = 32, + Lines = 2, + Modified = new DateTime(2026, 7, 24, 0, 0, 0, DateTimeKind.Utc), + Checksum = "objective-c-target", + }); + _writer.InsertSymbols([ + new SymbolRecord { FileId = matlabTargetId, Kind = "function", Name = "MatlabTarget", Line = 1 }, + new SymbolRecord { FileId = objectiveCTargetId, Kind = "function", Name = "ObjectiveCTarget", Line = 1 }, + ]); + transaction.Commit(); + _writer.RefreshMutualRecursionFlags(); + } + + Assert.Equal(2, ExecuteScalarLong($""" + SELECT COUNT(*) + FROM symbol_references + WHERE file_id = {callerId.ToString(CultureInfo.InvariantCulture)} + AND resolution_state = 'resolved' + """)); + + _writer.RefreshMutualRecursionFlags(); + Assert.Equal(2, ExecuteScalarLong($""" + SELECT COUNT(*) + FROM symbol_references + WHERE file_id = {callerId.ToString(CultureInfo.InvariantCulture)} + AND resolution_state = 'resolved' + """)); + } + [Fact] public void ReferenceGraphDirtyScope_RollbackAndCancellationPreserveRetryState() { diff --git a/tests/CodeIndex.Tests/ReferenceExtractorTests.ScientificNative.cs b/tests/CodeIndex.Tests/ReferenceExtractorTests.ScientificNative.cs index 880721e4e..ec0bf100c 100644 --- a/tests/CodeIndex.Tests/ReferenceExtractorTests.ScientificNative.cs +++ b/tests/CodeIndex.Tests/ReferenceExtractorTests.ScientificNative.cs @@ -176,6 +176,137 @@ public void Extract_MatlabAndJuliaPreservePostfixTransposeCalls_Issue4738( Assert.DoesNotContain(references, reference => reference.SymbolName == "stringCall"); } + [Fact] + public void Extract_MatlabAppliesCharacterVectorQuoteRules_Issue4738() + { + const string content = """ + function run() + path = 'C:\'; helper(); + items = [prefix 'fake()']; + end + """; + var symbols = SymbolExtractor.Extract(1, "matlab", content); + + var references = ReferenceExtractor.Extract(1, "matlab", content, symbols); + + var helperReference = Assert.Single(references, reference => + reference.SymbolName == "helper" && reference.ReferenceKind == "call"); + Assert.Equal("run", helperReference.ContainerName); + Assert.DoesNotContain(references, reference => reference.SymbolName == "fake"); + } + + [Fact] + public void Extract_NimExpandsGroupedImports_Issue4738() + { + const string content = """ + import std/[strutils, sequtils], os + proc run() = + helper() + """; + var symbols = SymbolExtractor.Extract(1, "nim", content); + + var references = ReferenceExtractor.Extract(1, "nim", content, symbols); + + Assert.Contains(references, reference => + reference.SymbolName == "std/strutils" && reference.ReferenceKind == "import"); + Assert.Contains(references, reference => + reference.SymbolName == "std/sequtils" && reference.ReferenceKind == "import"); + Assert.Contains(references, reference => + reference.SymbolName == "os" && reference.ReferenceKind == "import"); + Assert.DoesNotContain(references, reference => reference.SymbolName is "std" or "sequtils"); + } + + [Fact] + public void Extract_JuliaNormalizesRelativeImportsAndBroadcastCalls_Issue4738() + { + const string content = """ + module Main + using .Utils, ..Parent + function run(xs) + helper.(xs) + end + end + """; + var symbols = SymbolExtractor.Extract(1, "julia", content); + + var references = ReferenceExtractor.Extract(1, "julia", content, symbols); + + Assert.Contains(references, reference => + reference.SymbolName == "Utils" && reference.ReferenceKind == "import"); + Assert.Contains(references, reference => + reference.SymbolName == "Parent" && reference.ReferenceKind == "import"); + var helperReference = Assert.Single(references, reference => + reference.SymbolName == "helper" && reference.ReferenceKind == "call"); + Assert.Equal("run", helperReference.ContainerName); + Assert.DoesNotContain(references, reference => + reference.SymbolName is ".Utils" or "..Parent"); + } + + [Fact] + public void Extract_DTemplateInvocationsEmitCalleeCalls_Issue4738() + { + const string content = """ + void run() { + helper!int(); + other!(string, int)(42); + } + """; + var symbols = SymbolExtractor.Extract(1, "d", content); + + var references = ReferenceExtractor.Extract(1, "d", content, symbols); + + Assert.Contains(references, reference => + reference.SymbolName == "helper" && reference.ReferenceKind == "call"); + Assert.Contains(references, reference => + reference.SymbolName == "other" && reference.ReferenceKind == "call"); + Assert.DoesNotContain(references, reference => + reference.SymbolName is "int" or "string" && reference.ReferenceKind == "call"); + } + + [Fact] + public void Extract_AdaQualifiedBareCallUsesResolvableLeafName_Issue4738() + { + const string content = """ + package body Demo is + procedure Run is + begin + Helpers.Flush; + end Run; + end Demo; + """; + var symbols = SymbolExtractor.Extract(1, "ada", content); + + var references = ReferenceExtractor.Extract(1, "ada", content, symbols); + + var flushReference = Assert.Single(references, reference => + reference.SymbolName == "Flush" && reference.ReferenceKind == "call"); + Assert.Equal("Run", flushReference.ContainerName); + Assert.DoesNotContain(references, reference => reference.SymbolName == "Helpers.Flush"); + } + + [Fact] + public void Extract_MatlabCommaSeparatedEndKeepsFollowingCallTopLevel_Issue4738() + { + const string content = """ + function first() + if true, helper(), end + end + toplevel(); + function second() + other(); + end + """; + var symbols = SymbolExtractor.Extract(1, "matlab", content); + + var references = ReferenceExtractor.Extract(1, "matlab", content, symbols); + + var first = Assert.Single(symbols, symbol => symbol.Name == "first"); + Assert.Equal(3, first.EndLine); + var topLevelReference = Assert.Single(references, reference => + reference.SymbolName == "toplevel" && reference.ReferenceKind == "call"); + Assert.Null(topLevelReference.ContainerName); + } + public static TheoryData ScientificNativeMultilineLiteralCases => new() { { @@ -302,6 +433,22 @@ void run(void) { Assert.DoesNotContain(references, reference => reference.SymbolName == "ignoredObjectiveCCall"); } + [Fact] + public void Extract_AmbiguousMPreservesObjectiveCModuloExpressions_Issue4738() + { + const string content = """ + void run(void) { + int value = left % helper(); + } + """; + + var symbols = SymbolExtractor.Extract(1, "ambiguous_m", content, "unknown.m"); + var references = ReferenceExtractor.Extract(1, "ambiguous_m", content, symbols, "unknown.m"); + + Assert.Single(references, reference => + reference.SymbolName == "helper" && reference.ReferenceKind == "call"); + } + [Fact] public void Extract_AmbiguousMRetainsSharedSafetyGuards_Issue4738() { From 9cff38e5307f21a8811f01e1a74f17c64a6669ca Mon Sep 17 00:00:00 2001 From: Widthdom Date: Fri, 24 Jul 2026 13:29:49 +0900 Subject: [PATCH 04/21] Fix scientific graph review regressions (#4738) --- changelog.d/unreleased/4738.added.md | 2 + .../Database/DbReader.GraphQueries.cs | 6 +- src/CodeIndex/Database/DbReader.References.cs | 9 +- .../Indexer/AmbiguousMContentMasker.cs | 5 +- .../ScientificNativeCommentMasker.cs | 116 ++++++++++++++++-- .../ScientificNativeReferenceExtractor.cs | 2 +- .../SymbolExtractor.ScientificNative.cs | 87 ++++++++++++- tests/CodeIndex.Tests/DatabaseTests.cs | 38 ++++++ ...eferenceExtractorTests.ScientificNative.cs | 106 ++++++++++++++++ 9 files changed, 352 insertions(+), 19 deletions(-) diff --git a/changelog.d/unreleased/4738.added.md b/changelog.d/unreleased/4738.added.md index 5eaa232e3..87822929d 100644 --- a/changelog.d/unreleased/4738.added.md +++ b/changelog.d/unreleased/4738.added.md @@ -4,6 +4,8 @@ issues: - 4738 affected: - src/CodeIndex/Database/DbContext.cs + - src/CodeIndex/Database/DbReader.GraphQueries.cs + - src/CodeIndex/Database/DbReader.References.cs - src/CodeIndex/Database/DbWriter.ReferenceGraphRefreshScope.cs - src/CodeIndex/Database/DbWriter.References.cs - src/CodeIndex/Indexer/AmbiguousMContentMasker.cs diff --git a/src/CodeIndex/Database/DbReader.GraphQueries.cs b/src/CodeIndex/Database/DbReader.GraphQueries.cs index 2a80eb176..25ae552f6 100644 --- a/src/CodeIndex/Database/DbReader.GraphQueries.cs +++ b/src/CodeIndex/Database/DbReader.GraphQueries.cs @@ -166,7 +166,11 @@ FROM symbol_reference_candidates AS identity_candidate ? $" AND (r.symbol_name LIKE @query ESCAPE '\\' OR (r.symbol_name = @queryCssScssVariableAlias COLLATE NOCASE{cssScssVariableAliasScope}) OR (f.lang = 'sql' AND r.symbol_name = sql_leaf_name(@aliasQuery) COLLATE NOCASE))" : " AND (r.symbol_name LIKE @query ESCAPE '\\' OR (f.lang = 'sql' AND r.symbol_name = sql_leaf_name(@aliasQuery) COLLATE NOCASE))"; if (lang != null) - sql += " AND f.lang = @lang"; + { + sql += IncludeAmbiguousMSourceForIdentityTarget(lang, targetSymbolId) + ? " AND (f.lang = @lang OR f.lang = 'ambiguous_m')" + : " AND f.lang = @lang"; + } sql += BuildCSharpBareMemberGraphReferenceFilter(query, lang, exact, contextSql, "f", "r"); AppendPathFilters(ref sql, pathPatterns, excludePathPatterns, excludeTests); sql += @" diff --git a/src/CodeIndex/Database/DbReader.References.cs b/src/CodeIndex/Database/DbReader.References.cs index b03786505..f13bf0f41 100644 --- a/src/CodeIndex/Database/DbReader.References.cs +++ b/src/CodeIndex/Database/DbReader.References.cs @@ -8,6 +8,9 @@ public partial class DbReader private const int CSharpUsingStaticReferenceFilterChunkSize = 64; private const int CSharpUsingStaticReferenceFilterMaxRawLimit = 65536; private sealed record SearchReferenceRawRow(string Path, string? Lang, string SymbolName, string ReferenceKind, int Line, int Column, string Context, string? ContainerKind, string? ContainerName, bool IsSelfReference, bool IsMutualRecursion, long? TargetSymbolId, string? TargetSymbolKey, string? ResolutionState, int ResolutionCandidateCount); + private static bool IncludeAmbiguousMSourceForIdentityTarget(string? language, long? targetSymbolId) => + targetSymbolId != null && language is "matlab" or "objc"; + internal sealed record ReferencePositionCandidate(SymbolResult Definition, bool Authoritative); internal sealed record ReferencePositionResolution( bool IdentityAvailable, @@ -379,7 +382,11 @@ FROM symbol_reference_candidates AS identity_candidate )"; } if (lang != null) - sql += " AND f.lang = @lang"; + { + sql += IncludeAmbiguousMSourceForIdentityTarget(lang, targetSymbolId) + ? " AND (f.lang = @lang OR f.lang = 'ambiguous_m')" + : " AND f.lang = @lang"; + } AppendPathFilters(ref sql, pathPatterns, excludePathPatterns, excludeTests); if (referenceKind == null) { diff --git a/src/CodeIndex/Indexer/AmbiguousMContentMasker.cs b/src/CodeIndex/Indexer/AmbiguousMContentMasker.cs index 6e64bcbb6..f1b9b6710 100644 --- a/src/CodeIndex/Indexer/AmbiguousMContentMasker.cs +++ b/src/CodeIndex/Indexer/AmbiguousMContentMasker.cs @@ -153,7 +153,10 @@ private static bool IsMatlabSingleQuoteStart(string content, int quoteIndex) var previousIndex = quoteIndex - 1; return previousIndex < 0 || content[previousIndex] is '\r' or '\n' - || !IsTransposeOperandEnd(content[previousIndex]); + || (!IsTransposeOperandEnd(content[previousIndex]) + && !(content[previousIndex] == '.' + && previousIndex > 0 + && IsTransposeOperandEnd(content[previousIndex - 1]))); } private static bool IsStandaloneMatlabBlockDelimiter(string content, int percentIndex) diff --git a/src/CodeIndex/Indexer/References/Languages/ScientificNativeCommentMasker.cs b/src/CodeIndex/Indexer/References/Languages/ScientificNativeCommentMasker.cs index 5d9673bed..4b6d287e3 100644 --- a/src/CodeIndex/Indexer/References/Languages/ScientificNativeCommentMasker.cs +++ b/src/CodeIndex/Indexer/References/Languages/ScientificNativeCommentMasker.cs @@ -5,7 +5,7 @@ internal static class ScientificNativeCommentMasker internal static string[] MaskBlockComments(string language, string[] lines) => language switch { - "d" => MaskNestedBlockComments(MaskDTokenBraceStrings(lines), "/+", "+/", "//"), + "d" => MaskNestedBlockComments(MaskDMultilineStrings(lines), "/+", "+/", "//"), "julia" => MaskNestedBlockComments( MaskTripleQuotedStrings(lines, "#=", "=#", "#"), "#=", @@ -187,14 +187,20 @@ void MaskToken(string token) return result; } - private static string[] MaskDTokenBraceStrings(string[] lines) + private static string[] MaskDMultilineStrings(string[] lines) { - if (!MayContain(lines, "q{")) + if (!MayContain(lines, "q{") + && !MayContain(lines, "q\"") + && !MayContain(lines, "`")) + { return lines; + } var result = new string[lines.Length]; var tokenStringDepth = 0; var blockCommentDepth = 0; + var inBacktickString = false; + string? tokenStringClosing = null; for (var lineIndex = 0; lineIndex < lines.Length; lineIndex++) { var line = lines[lineIndex]; @@ -207,6 +213,28 @@ void MaskAt(int index) => var cursor = 0; while (cursor < line.Length) { + if (inBacktickString) + { + var current = line[cursor]; + MaskAt(cursor++); + if (current == '`') + inBacktickString = false; + continue; + } + + if (tokenStringClosing != null) + { + if (StartsWith(line, cursor, tokenStringClosing)) + { + MaskToken(tokenStringClosing); + tokenStringClosing = null; + continue; + } + + MaskAt(cursor++); + continue; + } + if (tokenStringDepth > 0) { var current = line[cursor]; @@ -262,12 +290,6 @@ void MaskAt(int index) => continue; } - if (line[cursor] is '"' or '\'' or '`') - { - quote = line[cursor++]; - continue; - } - if (StartsWith(line, cursor, "q{") && (cursor == 0 || !IsIdentifierChar(line[cursor - 1]))) { @@ -277,15 +299,83 @@ void MaskAt(int index) => continue; } + if (StartsWith(line, cursor, "q\"") + && (cursor == 0 || !IsIdentifierChar(line[cursor - 1])) + && TryGetDTokenStringClosing(line, cursor + 2, out var openingLength, out var closing)) + { + for (var openingIndex = 0; openingIndex < 2 + openingLength; openingIndex++) + MaskAt(cursor++); + tokenStringClosing = closing; + continue; + } + + if (line[cursor] == '`') + { + MaskAt(cursor++); + inBacktickString = true; + continue; + } + + if (line[cursor] is '"' or '\'') + { + quote = line[cursor++]; + continue; + } + cursor++; } result[lineIndex] = chars is null ? line : new string(chars); + + void MaskToken(string token) + { + for (var tokenIndex = 0; tokenIndex < token.Length; tokenIndex++) + MaskAt(cursor++); + } } return result; } + private static bool TryGetDTokenStringClosing( + string line, + int delimiterIndex, + out int openingLength, + out string closing) + { + openingLength = 0; + closing = string.Empty; + if (delimiterIndex >= line.Length) + return false; + + closing = line[delimiterIndex] switch + { + '[' => "]\"", + '(' => ")\"", + '{' => "}\"", + '<' => ">\"", + _ => string.Empty, + }; + if (closing.Length != 0) + { + openingLength = 1; + return true; + } + + var end = delimiterIndex; + while (end < line.Length && IsIdentifierChar(line[end])) + end++; + if (end == delimiterIndex + || (end < line.Length && !char.IsWhiteSpace(line[end]))) + { + return false; + } + + openingLength = end - delimiterIndex; + closing = line[delimiterIndex..end] + '"'; + return true; + } + private static string[] MaskNestedBlockComments( string[] lines, string opening, @@ -393,7 +483,13 @@ private static bool IsPostfixSingleQuote( if (skipWhitespace && char.IsWhiteSpace(line[index])) continue; - return char.IsLetterOrDigit(line[index]) || line[index] is '_' or ')' or ']' or '}'; + if (char.IsLetterOrDigit(line[index]) || line[index] is '_' or ')' or ']' or '}') + return true; + + return line[index] == '.' + && index > 0 + && (char.IsLetterOrDigit(line[index - 1]) + || line[index - 1] is '_' or ')' or ']' or '}'); } return false; diff --git a/src/CodeIndex/Indexer/References/Languages/ScientificNativeReferenceExtractor.cs b/src/CodeIndex/Indexer/References/Languages/ScientificNativeReferenceExtractor.cs index 349b36bd0..1d895beb5 100644 --- a/src/CodeIndex/Indexer/References/Languages/ScientificNativeReferenceExtractor.cs +++ b/src/CodeIndex/Indexer/References/Languages/ScientificNativeReferenceExtractor.cs @@ -302,7 +302,7 @@ bool TryEmitDependencySegment( : segmentStart; EmitName( names[emittedNameStart..nameEnd], - absoluteOffset + segmentStart, + absoluteOffset + emittedNameStart, referenceKind); return true; } diff --git a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.ScientificNative.cs b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.ScientificNative.cs index 51cb75daa..9cc7a70a9 100644 --- a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.ScientificNative.cs +++ b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.ScientificNative.cs @@ -23,15 +23,24 @@ private static (int EndLine, int? BodyStartLine, int? BodyEndLine) FindScientifi var depth = 1; int? bodyStartLine = null; - for (var lineIndex = startIndex + 1; lineIndex < scannerLines.Length; lineIndex++) + for (var lineIndex = startIndex; lineIndex < scannerLines.Length; lineIndex++) { var code = scannerLines[lineIndex]; if (string.IsNullOrWhiteSpace(code)) continue; - bodyStartLine ??= lineIndex + 1; + var skipDeclarationToken = lineIndex == startIndex; + if (!skipDeclarationToken) + bodyStartLine ??= lineIndex + 1; + foreach (Match match in tokenRegex.Matches(code)) { + if (skipDeclarationToken) + { + skipDeclarationToken = false; + continue; + } + var keyword = match.Groups["keyword"].Value; if (!IsScientificBlockTokenAtStatementBoundary(code, match.Index, keyword, language)) continue; @@ -40,7 +49,7 @@ private static (int EndLine, int? BodyStartLine, int? BodyEndLine) FindScientifi { depth--; if (depth == 0) - return (lineIndex + 1, bodyStartLine, lineIndex + 1); + return (lineIndex + 1, bodyStartLine ?? lineIndex + 1, lineIndex + 1); continue; } @@ -91,9 +100,77 @@ private static bool IsScientificBlockTokenAtStatementBoundary( if (line[index] == ';' || (language == "matlab" && line[index] == ',')) - return true; + { + return IsScientificStatementSeparatorAtTopLevel(line, index); + } + + return language == "julia" + && (line[index] == '=' || IsJuliaMacroBlockPrefix(line, tokenIndex)); + } + + return true; + } + + private static bool IsScientificStatementSeparatorAtTopLevel(string line, int separatorIndex) + { + var parentheses = 0; + var brackets = 0; + var braces = 0; + for (var index = 0; index < separatorIndex; index++) + { + switch (line[index]) + { + case '(': + parentheses++; + break; + case ')': + parentheses = Math.Max(0, parentheses - 1); + break; + case '[': + brackets++; + break; + case ']': + brackets = Math.Max(0, brackets - 1); + break; + case '{': + braces++; + break; + case '}': + braces = Math.Max(0, braces - 1); + break; + } + } + + return parentheses == 0 && brackets == 0 && braces == 0; + } + + private static bool IsJuliaMacroBlockPrefix(string line, int tokenIndex) + { + var index = tokenIndex - 1; + while (index >= 0 && char.IsWhiteSpace(line[index])) + index--; + if (index < 0) + return false; + + var tokenEnd = index + 1; + while (index >= 0 + && (char.IsLetterOrDigit(line[index]) || line[index] is '_' or '.' or '@')) + { + index--; + } + + var token = line.AsSpan(index + 1, tokenEnd - index - 1); + var atIndex = token.LastIndexOf('@'); + if (atIndex < 0 || atIndex == token.Length - 1) + return false; - return language == "julia" && line[index] == '='; + for (var tokenIndexOffset = 0; tokenIndexOffset < token.Length; tokenIndexOffset++) + { + if (!(char.IsLetterOrDigit(token[tokenIndexOffset]) + || token[tokenIndexOffset] is '_' or '.' or '@')) + { + return false; + } } return true; diff --git a/tests/CodeIndex.Tests/DatabaseTests.cs b/tests/CodeIndex.Tests/DatabaseTests.cs index a1b67c89c..b9ba6508f 100644 --- a/tests/CodeIndex.Tests/DatabaseTests.cs +++ b/tests/CodeIndex.Tests/DatabaseTests.cs @@ -712,6 +712,44 @@ FROM symbol_references WHERE file_id = {callerId.ToString(CultureInfo.InvariantCulture)} AND resolution_state = 'resolved' """)); + + _writer.MarkGraphReady(); + _writer.MarkReferenceIdentityContractReady(); + var reader = new DbReader(_db.Connection); + foreach (var (name, language, path) in new[] + { + ("MatlabTarget", "matlab", "src/matlab-target.m"), + ("ObjectiveCTarget", "objc", "src/objective-c-target.m"), + }) + { + var symbolId = ExecuteScalarLong($""" + SELECT id + FROM symbols + WHERE name = '{name}' + """); + var definition = new DefinitionResult + { + SymbolId = symbolId, + Path = path, + Lang = language, + Kind = "function", + Name = name, + Line = 1, + StartLine = 1, + EndLine = 1, + }; + Assert.Equal(1, ExecuteScalarLong($""" + SELECT COUNT(*) + FROM symbol_reference_candidates + WHERE symbol_id = {symbolId.ToString(CultureInfo.InvariantCulture)} + """)); + + var identityScopedReference = Assert.Single( + reader.GetReferencesForDefinition(definition, limit: 20)); + Assert.Equal("src/caller.m", identityScopedReference.Path); + Assert.Equal("ambiguous_m", identityScopedReference.Lang); + Assert.Equal(name, identityScopedReference.SymbolName); + } } [Fact] diff --git a/tests/CodeIndex.Tests/ReferenceExtractorTests.ScientificNative.cs b/tests/CodeIndex.Tests/ReferenceExtractorTests.ScientificNative.cs index ec0bf100c..381ae63f1 100644 --- a/tests/CodeIndex.Tests/ReferenceExtractorTests.ScientificNative.cs +++ b/tests/CodeIndex.Tests/ReferenceExtractorTests.ScientificNative.cs @@ -195,6 +195,23 @@ function run() Assert.DoesNotContain(references, reference => reference.SymbolName == "fake"); } + [Fact] + public void Extract_MatlabPreservesCallsAfterDotTranspose_Issue4738() + { + const string content = """ + function run(A) + value = A.'; helper(); + end + """; + var symbols = SymbolExtractor.Extract(1, "matlab", content); + + var references = ReferenceExtractor.Extract(1, "matlab", content, symbols); + + var helperReference = Assert.Single(references, reference => + reference.SymbolName == "helper" && reference.ReferenceKind == "call"); + Assert.Equal("run", helperReference.ContainerName); + } + [Fact] public void Extract_NimExpandsGroupedImports_Issue4738() { @@ -235,6 +252,10 @@ function run(xs) reference.SymbolName == "Utils" && reference.ReferenceKind == "import"); Assert.Contains(references, reference => reference.SymbolName == "Parent" && reference.ReferenceKind == "import"); + Assert.Equal(8, Assert.Single(references, reference => + reference.SymbolName == "Utils" && reference.ReferenceKind == "import").Column); + Assert.Equal(17, Assert.Single(references, reference => + reference.SymbolName == "Parent" && reference.ReferenceKind == "import").Column); var helperReference = Assert.Single(references, reference => reference.SymbolName == "helper" && reference.ReferenceKind == "call"); Assert.Equal("run", helperReference.ContainerName); @@ -307,6 +328,69 @@ function second() Assert.Null(topLevelReference.ContainerName); } + [Theory] + [InlineData("matlab", "function run(), helper(), end\noutside();\n")] + [InlineData("julia", "function run(); helper(); end\noutside()\n")] + public void Extract_CompactScientificFunctionEndsOnDeclarationLine_Issue4738( + string language, + string content) + { + var symbols = SymbolExtractor.Extract(1, language, content); + + var references = ReferenceExtractor.Extract(1, language, content, symbols); + + var function = Assert.Single(symbols, symbol => symbol.Name == "run"); + Assert.Equal(1, function.EndLine); + Assert.Equal("run", Assert.Single(references, reference => + reference.SymbolName == "helper" && reference.ReferenceKind == "call").ContainerName); + Assert.Null(Assert.Single(references, reference => + reference.SymbolName == "outside" && reference.ReferenceKind == "call").ContainerName); + } + + [Fact] + public void Extract_MatlabIndexEndDoesNotCloseFunction_Issue4738() + { + const string content = """ + function run(A) + value = A(:, end); + helper(); + end + outside(); + """; + var symbols = SymbolExtractor.Extract(1, "matlab", content); + + var references = ReferenceExtractor.Extract(1, "matlab", content, symbols); + + Assert.Equal(4, Assert.Single(symbols, symbol => symbol.Name == "run").EndLine); + Assert.Equal("run", Assert.Single(references, reference => + reference.SymbolName == "helper" && reference.ReferenceKind == "call").ContainerName); + Assert.Null(Assert.Single(references, reference => + reference.SymbolName == "outside" && reference.ReferenceKind == "call").ContainerName); + } + + [Fact] + public void Extract_JuliaMacroBlockDoesNotCloseFunctionEarly_Issue4738() + { + const string content = """ + function run() + @async begin + helper() + end + after() + end + outside() + """; + var symbols = SymbolExtractor.Extract(1, "julia", content); + + var references = ReferenceExtractor.Extract(1, "julia", content, symbols); + + Assert.Equal(6, Assert.Single(symbols, symbol => symbol.Name == "run").EndLine); + Assert.Equal("run", Assert.Single(references, reference => + reference.SymbolName == "after" && reference.ReferenceKind == "call").ContainerName); + Assert.Null(Assert.Single(references, reference => + reference.SymbolName == "outside" && reference.ReferenceKind == "call").ContainerName); + } + public static TheoryData ScientificNativeMultilineLiteralCases => new() { { @@ -341,6 +425,28 @@ void run() { } """ }, + { + "d", + """ + void run() { + auto text = `literal + tokenOnly() + literal`; + helper(); + } + """ + }, + { + "d", + """ + void run() { + auto text = q"EOS + tokenOnly() + EOS"; + helper(); + } + """ + }, }; [Theory] From 6c93c2db681eab7a5636c3463d9cb3d1d8a9156a Mon Sep 17 00:00:00 2001 From: Widthdom Date: Fri, 24 Jul 2026 13:52:38 +0900 Subject: [PATCH 05/21] Resolve final scientific graph review findings (#4738) --- .../ScientificNativeCommentMasker.cs | 27 +++- .../ReferenceExtractor.TypeReferences.cs | 4 +- .../Symbols/SymbolExtractor.ExtractCore.cs | 5 +- .../SymbolExtractor.ScientificNative.cs | 109 +++++++++++++- .../Indexer/Symbols/SymbolExtractor.cs | 6 +- ...eferenceExtractorTests.ScientificNative.cs | 134 +++++++++++++++++- 6 files changed, 271 insertions(+), 14 deletions(-) diff --git a/src/CodeIndex/Indexer/References/Languages/ScientificNativeCommentMasker.cs b/src/CodeIndex/Indexer/References/Languages/ScientificNativeCommentMasker.cs index 4b6d287e3..eb39c8ccc 100644 --- a/src/CodeIndex/Indexer/References/Languages/ScientificNativeCommentMasker.cs +++ b/src/CodeIndex/Indexer/References/Languages/ScientificNativeCommentMasker.cs @@ -5,9 +5,15 @@ internal static class ScientificNativeCommentMasker internal static string[] MaskBlockComments(string language, string[] lines) => language switch { - "d" => MaskNestedBlockComments(MaskDMultilineStrings(lines), "/+", "+/", "//"), + "d" => ReferenceExtractor.MaskCStyleBlockCommentLines( + "d", + MaskNestedBlockComments( + MaskDMultilineStrings(lines), + "/+", + "+/", + "//")), "julia" => MaskNestedBlockComments( - MaskTripleQuotedStrings(lines, "#=", "=#", "#"), + MaskTripleQuotedStrings(lines, "#=", "=#", "#", singleQuoteCanBePostfix: true), "#=", "=#", "#", @@ -82,7 +88,8 @@ private static string[] MaskTripleQuotedStrings( string[] lines, string blockOpening, string blockClosing, - string lineComment) + string lineComment, + bool singleQuoteCanBePostfix = false) { if (!MayContain(lines, "\"\"\"")) return lines; @@ -149,9 +156,6 @@ void MaskAt(int index) => continue; } - if (StartsWith(line, cursor, lineComment)) - break; - if (StartsWith(line, cursor, blockOpening)) { blockCommentDepth++; @@ -159,6 +163,9 @@ void MaskAt(int index) => continue; } + if (StartsWith(line, cursor, lineComment)) + break; + if (StartsWith(line, cursor, "\"\"\"")) { MaskToken("\"\"\""); @@ -168,6 +175,14 @@ void MaskAt(int index) => if (line[cursor] is '"' or '\'' or '`') { + if (line[cursor] == '\'' + && singleQuoteCanBePostfix + && IsPostfixSingleQuote(line, cursor)) + { + cursor++; + continue; + } + quote = line[cursor++]; continue; } diff --git a/src/CodeIndex/Indexer/References/ReferenceExtractor.TypeReferences.cs b/src/CodeIndex/Indexer/References/ReferenceExtractor.TypeReferences.cs index ebe80ae1a..ff6be6d48 100644 --- a/src/CodeIndex/Indexer/References/ReferenceExtractor.TypeReferences.cs +++ b/src/CodeIndex/Indexer/References/ReferenceExtractor.TypeReferences.cs @@ -2998,9 +2998,9 @@ private static bool MayContainPascalBlockComment(IReadOnlyList lines) } private static bool UsesCStyleBlockComments(string language) => - language is "c" or "cpp" or "go" or "objc" or "dart" or "d"; + language is "c" or "cpp" or "go" or "objc" or "dart"; - private static string[] MaskCStyleBlockCommentLines(string language, IReadOnlyList lines) + internal static string[] MaskCStyleBlockCommentLines(string language, IReadOnlyList lines) { if (lines is string[] lineArray && !MayContainCStyleMaskingTrigger(language, lines)) return lineArray; diff --git a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.ExtractCore.cs b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.ExtractCore.cs index b8930d342..30fe5af24 100644 --- a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.ExtractCore.cs +++ b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.ExtractCore.cs @@ -180,7 +180,10 @@ private static List ExtractCore( ? GetPythonModulePrefix(filePath) : null; - var structuralLines = StructuralLineMasker.MaskLines(lang, lines); + var structuralMaskLanguage = lang == "cython" ? "python" : lang; + var structuralLines = StructuralLineMasker.MaskLines(structuralMaskLanguage, lines); + if (lang is "d" or "julia" or "matlab" or "nim") + structuralLines = ScientificNativeCommentMasker.MaskBlockComments(lang, structuralLines); var scientificBodyScannerLines = lang is "julia" or "matlab" ? PrepareScientificBodyScannerLines(structuralLines, lang) : null; diff --git a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.ScientificNative.cs b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.ScientificNative.cs index 9cc7a70a9..7bf41a517 100644 --- a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.ScientificNative.cs +++ b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.ScientificNative.cs @@ -15,13 +15,15 @@ public static partial class SymbolExtractor private static (int EndLine, int? BodyStartLine, int? BodyEndLine) FindScientificEndRange( string[] scannerLines, int startIndex, - string language) + string language, + int? openingTokenLineIndex = null) { var tokenRegex = language == "julia" ? JuliaScientificBlockTokenRegex : MatlabScientificBlockTokenRegex; var depth = 1; int? bodyStartLine = null; + var declarationColumn = GetFirstNonWhitespaceColumn(scannerLines[startIndex]); for (var lineIndex = startIndex; lineIndex < scannerLines.Length; lineIndex++) { @@ -29,11 +31,22 @@ private static (int EndLine, int? BodyStartLine, int? BodyEndLine) FindScientifi if (string.IsNullOrWhiteSpace(code)) continue; - var skipDeclarationToken = lineIndex == startIndex; + var skipDeclarationToken = lineIndex == (openingTokenLineIndex ?? startIndex); + var matches = tokenRegex.Matches(code); + if (language == "matlab" + && lineIndex > startIndex + && depth == 1 + && IsMatlabPeerDeclaration(code, matches, declarationColumn)) + { + return bodyStartLine == null + ? (lineIndex, null, null) + : (lineIndex, bodyStartLine, lineIndex); + } + if (!skipDeclarationToken) bodyStartLine ??= lineIndex + 1; - foreach (Match match in tokenRegex.Matches(code)) + foreach (Match match in matches) { if (skipDeclarationToken) { @@ -62,6 +75,96 @@ private static (int EndLine, int? BodyStartLine, int? BodyEndLine) FindScientifi : (scannerLines.Length, bodyStartLine, scannerLines.Length); } + private static (int EndLine, int? BodyStartLine, int? BodyEndLine) FindJuliaShortFunctionRange( + string[] scannerLines, + int startIndex) + { + if (!TryGetJuliaShortBlockExpressionStartLine(scannerLines, startIndex, out var blockStartLine)) + return (startIndex + 1, startIndex + 1, startIndex + 1); + + var blockRange = FindScientificEndRange( + scannerLines, + startIndex, + "julia", + openingTokenLineIndex: blockStartLine); + return blockRange.BodyStartLine == null + ? (startIndex + 1, startIndex + 1, startIndex + 1) + : (blockRange.EndLine, startIndex + 1, blockRange.BodyEndLine); + } + + private static bool TryGetJuliaShortBlockExpressionStartLine( + string[] scannerLines, + int startIndex, + out int blockStartLine) + { + blockStartLine = startIndex; + var startLine = scannerLines[startIndex]; + var parameterEnd = startLine.LastIndexOf(')'); + var assignmentIndex = parameterEnd >= 0 + ? startLine.IndexOf('=', parameterEnd + 1) + : -1; + if (assignmentIndex < 0) + return false; + + var expression = startLine[(assignmentIndex + 1)..].TrimStart(); + if (expression.Length == 0) + { + for (var lineIndex = startIndex + 1; lineIndex < scannerLines.Length; lineIndex++) + { + expression = scannerLines[lineIndex].TrimStart(); + if (expression.Length != 0) + { + blockStartLine = lineIndex; + break; + } + } + } + + foreach (var keyword in JuliaShortBlockExpressionKeywords) + { + if (expression.Equals(keyword, StringComparison.Ordinal) + || expression.StartsWith(keyword + " ", StringComparison.Ordinal) + || expression.StartsWith(keyword + ";", StringComparison.Ordinal)) + { + return true; + } + } + + return false; + } + + private static readonly string[] JuliaShortBlockExpressionKeywords = + ["begin", "for", "if", "let", "quote", "try", "while"]; + + private static bool IsMatlabPeerDeclaration( + string code, + MatchCollection matches, + int declarationColumn) + { + foreach (Match match in matches) + { + if (match.Index > declarationColumn) + return false; + + var keyword = match.Groups["keyword"].Value; + return keyword.Equals("function", StringComparison.OrdinalIgnoreCase) + || keyword.Equals("classdef", StringComparison.OrdinalIgnoreCase); + } + + return false; + } + + private static int GetFirstNonWhitespaceColumn(string line) + { + for (var index = 0; index < line.Length; index++) + { + if (!char.IsWhiteSpace(line[index])) + return index; + } + + return 0; + } + private static string[] PrepareScientificBodyScannerLines( string[] lines, string language) diff --git a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.cs b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.cs index 5d2a9b307..4eba3bf5c 100644 --- a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.cs +++ b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.cs @@ -482,6 +482,7 @@ private enum BodyStyle FortranEnd, ElixirEnd, ScientificEnd, + JuliaShortFunction, VisualBasicEnd, PascalEnd, SmalltalkMethod, @@ -1639,7 +1640,7 @@ private enum JavaScriptTypeScriptFunctionHeaderConsumeResult new("type", new Regex(@"^\s*(?:abstract|primitive)\s+type\s+(?[A-Za-z_]\w*)", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.ScientificEnd), new("function", new Regex(@"^\s*function\s+(?[A-Za-z_]\w*(?:\.[A-Za-z_]\w*)?)\s*(?:\(|\{)", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.ScientificEnd), new("function", new Regex(@"^\s*macro\s+(?[A-Za-z_]\w*)\s*(?:\(|$)", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.ScientificEnd), - new("function", new Regex(@"^\s*(?[A-Za-z_]\w*(?:\.[A-Za-z_]\w*)?)\s*\([^)\r\n]*\)\s*=", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None), + new("function", new Regex(@"^\s*(?[A-Za-z_]\w*(?:\.[A-Za-z_]\w*)?)\s*\([^)\r\n]*\)\s*=", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.JuliaShortFunction), new("property", new Regex(@"^\s*const\s+(?[A-Z_]\w*)\s*=", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None), new("import", new Regex(@"^\s*(?:using|import)\s+(?[A-Za-z_]\w*(?:\.[A-Za-z_]\w*)*)", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None), ], @@ -4812,6 +4813,9 @@ private static (int EndLine, int? BodyStartLine, int? BodyEndLine) ResolveRange( scientificBodyScannerLines ?? PrepareScientificBodyScannerLines(lines, lang), startIndex, lang), + BodyStyle.JuliaShortFunction when lang == "julia" => FindJuliaShortFunctionRange( + scientificBodyScannerLines ?? PrepareScientificBodyScannerLines(lines, lang), + startIndex), BodyStyle.VisualBasicEnd => FindVisualBasicRange(lines, startIndex), BodyStyle.PascalEnd => FindPascalRange(lines, startIndex), BodyStyle.SmalltalkMethod => FindSmalltalkMethodRange(lines, startIndex), diff --git a/tests/CodeIndex.Tests/ReferenceExtractorTests.ScientificNative.cs b/tests/CodeIndex.Tests/ReferenceExtractorTests.ScientificNative.cs index 381ae63f1..89ec2ad89 100644 --- a/tests/CodeIndex.Tests/ReferenceExtractorTests.ScientificNative.cs +++ b/tests/CodeIndex.Tests/ReferenceExtractorTests.ScientificNative.cs @@ -145,6 +145,78 @@ public void Extract_ScientificNativeLanguagesEmitBoundedGraphReferences_Issue473 reference.SymbolName.Equals("ignoredBlockCall", StringComparison.OrdinalIgnoreCase)); } + public static TheoryData ScientificNativePhantomSymbolCases => new() + { + { + "d", + """ + /+ + void Phantom() {} + +/ + void real() { Phantom(); } + """ + }, + { + "nim", + """ + #[ + proc Phantom() = discard + ]# + proc real() = Phantom() + """ + }, + { + "julia", + """ + #= + function Phantom() + end + =# + function real() + Phantom() + end + """ + }, + { + "matlab", + """ + %{ + function Phantom() + end + %} + function real() + Phantom(); + end + """ + }, + { + "cython", + """" + """ + def Phantom(): + pass + """ + def real(): + Phantom() + """" + }, + }; + + [Theory] + [MemberData(nameof(ScientificNativePhantomSymbolCases))] + public void Extract_ScientificNativeNonCodeDeclarationsDoNotBecomeResolutionTargets_Issue4738( + string language, + string content) + { + var symbols = SymbolExtractor.Extract(1, language, content); + + var references = ReferenceExtractor.Extract(1, language, content, symbols); + + Assert.DoesNotContain(symbols, symbol => symbol.Name == "Phantom"); + Assert.Single(references, reference => + reference.SymbolName == "Phantom" && reference.ReferenceKind == "call"); + } + [Theory] [InlineData( "matlab", @@ -391,6 +463,66 @@ @async begin reference.SymbolName == "outside" && reference.ReferenceKind == "call").ContainerName); } + [Theory] + [InlineData("run(x) = helper(x)\nhelper(x) = x\n", 1)] + [InlineData("run(x) = begin\n helper(x)\nend\nhelper(x) = x\n", 3)] + [InlineData("run(x) =\nbegin\n helper(x)\nend\nhelper(x) = x\n", 4)] + public void Extract_JuliaShortFunctionsOwnTheirCallReferences_Issue4738( + string content, + int expectedEndLine) + { + var symbols = SymbolExtractor.Extract(1, "julia", content); + + var references = ReferenceExtractor.Extract(1, "julia", content, symbols); + + var run = Assert.Single(symbols, symbol => symbol.Name == "run"); + Assert.Equal(expectedEndLine, run.EndLine); + Assert.Equal("run", Assert.Single(references, reference => + reference.SymbolName == "helper" && reference.ReferenceKind == "call").ContainerName); + } + + [Fact] + public void Extract_MatlabPeerFunctionsWithoutClosingEndHaveSeparateRanges_Issue4738() + { + const string content = """ + function first() + helper1(); + function second() + helper2(); + """; + var symbols = SymbolExtractor.Extract(1, "matlab", content); + + var references = ReferenceExtractor.Extract(1, "matlab", content, symbols); + + var first = Assert.Single(symbols, symbol => symbol.Name == "first"); + var second = Assert.Single(symbols, symbol => symbol.Name == "second"); + Assert.Equal(2, first.EndLine); + Assert.Equal(4, second.EndLine); + Assert.Equal("first", Assert.Single(references, reference => + reference.SymbolName == "helper1" && reference.ReferenceKind == "call").ContainerName); + Assert.Equal("second", Assert.Single(references, reference => + reference.SymbolName == "helper2" && reference.ReferenceKind == "call").ContainerName); + } + + [Fact] + public void Extract_JuliaTransposeBeforeBlockCommentDoesNotMaskFollowingCalls_Issue4738() + { + const string content = """" + function run(value) + value' #= + """ + =# + helper() + end + """"; + var symbols = SymbolExtractor.Extract(1, "julia", content); + + var references = ReferenceExtractor.Extract(1, "julia", content, symbols); + + Assert.Equal("run", Assert.Single(references, reference => + reference.SymbolName == "helper" && reference.ReferenceKind == "call").ContainerName); + } + public static TheoryData ScientificNativeMultilineLiteralCases => new() { { @@ -419,7 +551,7 @@ proc run() = """ void run() { auto text = q{ - /+ tokenOnly(); + /* tokenOnly(); }; helper(); } From 3d2d4fc15d0264cd00743e3f561d1556cb98a671 Mon Sep 17 00:00:00 2001 From: Widthdom Date: Fri, 24 Jul 2026 14:11:35 +0900 Subject: [PATCH 06/21] Harden scientific graph language parsing (#4738) --- .../ScientificNativeCommentMasker.cs | 120 +++++++++++++---- .../ReferenceExtractor.CoreExtraction.cs | 6 + .../ReferenceExtractor.TypeReferences.cs | 8 +- .../SymbolExtractor.ScientificNative.cs | 114 +++++++++++++--- ...eferenceExtractorTests.ScientificNative.cs | 126 ++++++++++++++++++ 5 files changed, 327 insertions(+), 47 deletions(-) diff --git a/src/CodeIndex/Indexer/References/Languages/ScientificNativeCommentMasker.cs b/src/CodeIndex/Indexer/References/Languages/ScientificNativeCommentMasker.cs index eb39c8ccc..e840ca81d 100644 --- a/src/CodeIndex/Indexer/References/Languages/ScientificNativeCommentMasker.cs +++ b/src/CodeIndex/Indexer/References/Languages/ScientificNativeCommentMasker.cs @@ -5,13 +5,7 @@ internal static class ScientificNativeCommentMasker internal static string[] MaskBlockComments(string language, string[] lines) => language switch { - "d" => ReferenceExtractor.MaskCStyleBlockCommentLines( - "d", - MaskNestedBlockComments( - MaskDMultilineStrings(lines), - "/+", - "+/", - "//")), + "d" => MaskDNonCodeRegions(lines), "julia" => MaskNestedBlockComments( MaskTripleQuotedStrings(lines, "#=", "=#", "#", singleQuoteCanBePostfix: true), "#=", @@ -202,25 +196,60 @@ void MaskToken(string token) return result; } - private static string[] MaskDMultilineStrings(string[] lines) + internal static string MaskNimRawStringLiterals(string line) { - if (!MayContain(lines, "q{") - && !MayContain(lines, "q\"") - && !MayContain(lines, "`")) + if (!line.Contains("r\"", StringComparison.Ordinal)) + return line; + + char[]? chars = null; + for (var cursor = 0; cursor + 1 < line.Length; cursor++) { - return lines; + if (line[cursor] != 'r' + || line[cursor + 1] != '"' + || (cursor > 0 && IsIdentifierChar(line[cursor - 1]))) + { + continue; + } + + chars ??= line.ToCharArray(); + chars[cursor++] = ' '; + chars[cursor++] = ' '; + while (cursor < line.Length) + { + var current = line[cursor]; + chars[cursor++] = ' '; + if (current != '"') + continue; + + if (cursor < line.Length && line[cursor] == '"') + { + chars[cursor++] = ' '; + continue; + } + + break; + } + + cursor--; } + return chars is null ? line : new string(chars); + } + + private static string[] MaskDNonCodeRegions(string[] lines) + { var result = new string[lines.Length]; var tokenStringDepth = 0; - var blockCommentDepth = 0; + var nestedCommentDepth = 0; + var inCStyleBlockComment = false; var inBacktickString = false; string? tokenStringClosing = null; + var quote = '\0'; + var quoteUsesEscapes = false; for (var lineIndex = 0; lineIndex < lines.Length; lineIndex++) { var line = lines[lineIndex]; char[]? chars = null; - var quote = '\0'; void MaskAt(int index) => (chars ??= line.ToCharArray())[index] = ' '; @@ -261,47 +290,72 @@ void MaskAt(int index) => continue; } - if (blockCommentDepth > 0) + if (nestedCommentDepth > 0) { if (StartsWith(line, cursor, "/+")) { - blockCommentDepth++; - cursor += 2; + MaskToken("/+"); + nestedCommentDepth++; continue; } if (StartsWith(line, cursor, "+/")) { - blockCommentDepth--; - cursor += 2; + MaskToken("+/"); + nestedCommentDepth--; continue; } - cursor++; + MaskAt(cursor++); + continue; + } + + if (inCStyleBlockComment) + { + if (StartsWith(line, cursor, "*/")) + { + MaskToken("*/"); + inCStyleBlockComment = false; + continue; + } + + MaskAt(cursor++); continue; } if (quote != '\0') { - if (line[cursor] == '\\' && cursor + 1 < line.Length) + var current = line[cursor]; + MaskAt(cursor++); + if (quoteUsesEscapes && current == '\\' && cursor < line.Length) { - cursor += 2; + MaskAt(cursor++); continue; } - if (line[cursor] == quote) + if (current == quote) quote = '\0'; - cursor++; continue; } if (StartsWith(line, cursor, "//")) + { + while (cursor < line.Length) + MaskAt(cursor++); break; + } + + if (StartsWith(line, cursor, "/*")) + { + MaskToken("/*"); + inCStyleBlockComment = true; + continue; + } if (StartsWith(line, cursor, "/+")) { - blockCommentDepth++; - cursor += 2; + MaskToken("/+"); + nestedCommentDepth++; continue; } @@ -331,9 +385,21 @@ void MaskAt(int index) => continue; } + if (StartsWith(line, cursor, "r\"") + && (cursor == 0 || !IsIdentifierChar(line[cursor - 1]))) + { + MaskAt(cursor++); + MaskAt(cursor++); + quote = '"'; + quoteUsesEscapes = false; + continue; + } + if (line[cursor] is '"' or '\'') { - quote = line[cursor++]; + quote = line[cursor]; + quoteUsesEscapes = true; + MaskAt(cursor++); continue; } diff --git a/src/CodeIndex/Indexer/References/ReferenceExtractor.CoreExtraction.cs b/src/CodeIndex/Indexer/References/ReferenceExtractor.CoreExtraction.cs index 979b22543..8cfc22961 100644 --- a/src/CodeIndex/Indexer/References/ReferenceExtractor.CoreExtraction.cs +++ b/src/CodeIndex/Indexer/References/ReferenceExtractor.CoreExtraction.cs @@ -2562,6 +2562,12 @@ bool TryGetKnownPythonTypeCall(string candidate, out string canonicalName) { continue; } + if (language == "ada" + && callIndex > 0 + && preparedLine[callIndex - 1] == '\'') + { + continue; + } if (language == "objc" && IsObjCSelectorLiteralCall(preparedLine, name, callIndex)) continue; if (sqlSuppressedCallIndices != null && sqlSuppressedCallIndices.Contains(callIndex)) diff --git a/src/CodeIndex/Indexer/References/ReferenceExtractor.TypeReferences.cs b/src/CodeIndex/Indexer/References/ReferenceExtractor.TypeReferences.cs index ff6be6d48..7e865cfd9 100644 --- a/src/CodeIndex/Indexer/References/ReferenceExtractor.TypeReferences.cs +++ b/src/CodeIndex/Indexer/References/ReferenceExtractor.TypeReferences.cs @@ -2713,6 +2713,7 @@ private readonly record struct ReferenceLinePrepareOptions( bool UseCSharpTriggerFastPath, bool MaskRustLifetimes, bool MaskStringLiterals, + bool MaskNimRawStrings, bool IncludeBacktickStringDelimiter, bool PreservePostfixSingleQuotes, bool UseMatlabStringRules, @@ -2730,8 +2731,9 @@ private static ReferenceLinePrepareOptions CreateReferenceLinePrepareOptions(str UseCSharpTriggerFastPath: lang == "csharp", MaskRustLifetimes: lang == "rust", MaskStringLiterals: lang != "cobol", + MaskNimRawStrings: lang == "nim", IncludeBacktickStringDelimiter: lang is not ("kotlin" or "r"), - PreservePostfixSingleQuotes: lang is "julia" or "matlab", + PreservePostfixSingleQuotes: lang is "ada" or "julia" or "matlab", UseMatlabStringRules: lang == "matlab", UsesHashComments: UsesHashComments(lang), UsesRHashComments: lang == "r", @@ -2756,6 +2758,8 @@ private static string PrepareLine(string line, ReferenceLinePrepareOptions optio var result = line; if (options.MaskRustLifetimes) result = MaskRustLifetimeTokens(result); + if (options.MaskNimRawStrings) + result = ScientificNativeCommentMasker.MaskNimRawStringLiterals(result); if (options.MaskStringLiterals && MayContainStringLiteralDelimiter(result, options.IncludeBacktickStringDelimiter)) { if (options.PreservePostfixSingleQuotes) @@ -3000,7 +3004,7 @@ private static bool MayContainPascalBlockComment(IReadOnlyList lines) private static bool UsesCStyleBlockComments(string language) => language is "c" or "cpp" or "go" or "objc" or "dart"; - internal static string[] MaskCStyleBlockCommentLines(string language, IReadOnlyList lines) + private static string[] MaskCStyleBlockCommentLines(string language, IReadOnlyList lines) { if (lines is string[] lineArray && !MayContainCStyleMaskingTrigger(language, lines)) return lineArray; diff --git a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.ScientificNative.cs b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.ScientificNative.cs index 7bf41a517..dfbeaf7d7 100644 --- a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.ScientificNative.cs +++ b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.ScientificNative.cs @@ -36,7 +36,12 @@ private static (int EndLine, int? BodyStartLine, int? BodyEndLine) FindScientifi if (language == "matlab" && lineIndex > startIndex && depth == 1 - && IsMatlabPeerDeclaration(code, matches, declarationColumn)) + && IsMatlabPeerDeclaration( + scannerLines, + lineIndex, + code, + matches, + declarationColumn)) { return bodyStartLine == null ? (lineIndex, null, null) @@ -99,20 +104,20 @@ private static bool TryGetJuliaShortBlockExpressionStartLine( { blockStartLine = startIndex; var startLine = scannerLines[startIndex]; - var parameterEnd = startLine.LastIndexOf(')'); + var parameterEnd = startLine.IndexOf(')'); var assignmentIndex = parameterEnd >= 0 ? startLine.IndexOf('=', parameterEnd + 1) : -1; if (assignmentIndex < 0) return false; - var expression = startLine[(assignmentIndex + 1)..].TrimStart(); - if (expression.Length == 0) + var expression = startLine[(assignmentIndex + 1)..]; + if (string.IsNullOrWhiteSpace(expression)) { for (var lineIndex = startIndex + 1; lineIndex < scannerLines.Length; lineIndex++) { - expression = scannerLines[lineIndex].TrimStart(); - if (expression.Length != 0) + expression = scannerLines[lineIndex]; + if (!string.IsNullOrWhiteSpace(expression)) { blockStartLine = lineIndex; break; @@ -120,23 +125,20 @@ private static bool TryGetJuliaShortBlockExpressionStartLine( } } - foreach (var keyword in JuliaShortBlockExpressionKeywords) + foreach (Match match in JuliaScientificBlockTokenRegex.Matches(expression)) { - if (expression.Equals(keyword, StringComparison.Ordinal) - || expression.StartsWith(keyword + " ", StringComparison.Ordinal) - || expression.StartsWith(keyword + ";", StringComparison.Ordinal)) - { + var keyword = match.Groups["keyword"].Value; + if (IsJuliaExpressionPositionBlockOpener(expression, match.Index, keyword) + || (keyword == "for" && IsJuliaShortForBlockStart(expression, match.Index))) return true; - } } return false; } - private static readonly string[] JuliaShortBlockExpressionKeywords = - ["begin", "for", "if", "let", "quote", "try", "while"]; - private static bool IsMatlabPeerDeclaration( + string[] scannerLines, + int lineIndex, string code, MatchCollection matches, int declarationColumn) @@ -147,8 +149,49 @@ private static bool IsMatlabPeerDeclaration( return false; var keyword = match.Groups["keyword"].Value; - return keyword.Equals("function", StringComparison.OrdinalIgnoreCase) - || keyword.Equals("classdef", StringComparison.OrdinalIgnoreCase); + if (!keyword.Equals("function", StringComparison.OrdinalIgnoreCase) + && !keyword.Equals("classdef", StringComparison.OrdinalIgnoreCase)) + { + return false; + } + + return !HasMatlabExplicitOuterClosure(scannerLines, lineIndex); + } + + return false; + } + + private static bool HasMatlabExplicitOuterClosure(string[] scannerLines, int nestedDeclarationLineIndex) + { + var depth = 1; + for (var lineIndex = nestedDeclarationLineIndex; lineIndex < scannerLines.Length; lineIndex++) + { + var code = scannerLines[lineIndex]; + var skipNestedDeclaration = lineIndex == nestedDeclarationLineIndex; + foreach (Match match in MatlabScientificBlockTokenRegex.Matches(code)) + { + if (skipNestedDeclaration) + { + skipNestedDeclaration = false; + depth++; + continue; + } + + var keyword = match.Groups["keyword"].Value; + if (!IsScientificBlockTokenAtStatementBoundary(code, match.Index, keyword, "matlab")) + continue; + + if (keyword.Equals("end", StringComparison.OrdinalIgnoreCase)) + { + depth--; + if (depth == 0) + return true; + } + else + { + depth++; + } + } } return false; @@ -193,7 +236,7 @@ private static bool IsScientificBlockTokenAtStatementBoundary( string keyword, string language) { - if (language == "julia" && keyword == "do") + if (language == "julia" && IsJuliaExpressionPositionBlockOpener(line, tokenIndex, keyword)) return true; for (var index = tokenIndex - 1; index >= 0; index--) @@ -214,6 +257,41 @@ private static bool IsScientificBlockTokenAtStatementBoundary( return true; } + private static bool IsJuliaExpressionPositionBlockOpener( + string line, + int tokenIndex, + string keyword) + { + if (keyword is "begin" or "do" or "function" or "let" or "quote" or "try" or "while") + return true; + if (keyword != "if") + return false; + + var index = tokenIndex - 1; + while (index >= 0 && char.IsWhiteSpace(line[index])) + index--; + if (index < 0 || line[index] is '(' or ',' or '=' or ';') + return true; + + var tokenEnd = index + 1; + while (index >= 0 && (char.IsLetterOrDigit(line[index]) || line[index] == '_')) + index--; + var precedingToken = line.AsSpan(index + 1, tokenEnd - index - 1); + return precedingToken.Equals("else", StringComparison.Ordinal) + || precedingToken.Equals("return", StringComparison.Ordinal); + } + + private static bool IsJuliaShortForBlockStart(string expression, int tokenIndex) + { + for (var index = 0; index < tokenIndex; index++) + { + if (!char.IsWhiteSpace(expression[index]) && expression[index] != '(') + return false; + } + + return true; + } + private static bool IsScientificStatementSeparatorAtTopLevel(string line, int separatorIndex) { var parentheses = 0; diff --git a/tests/CodeIndex.Tests/ReferenceExtractorTests.ScientificNative.cs b/tests/CodeIndex.Tests/ReferenceExtractorTests.ScientificNative.cs index 89ec2ad89..07b30cd7a 100644 --- a/tests/CodeIndex.Tests/ReferenceExtractorTests.ScientificNative.cs +++ b/tests/CodeIndex.Tests/ReferenceExtractorTests.ScientificNative.cs @@ -467,6 +467,8 @@ @async begin [InlineData("run(x) = helper(x)\nhelper(x) = x\n", 1)] [InlineData("run(x) = begin\n helper(x)\nend\nhelper(x) = x\n", 3)] [InlineData("run(x) =\nbegin\n helper(x)\nend\nhelper(x) = x\n", 4)] + [InlineData("run(x) = map(x) do item\n helper(item)\nend\nhelper(x) = x\n", 3)] + [InlineData("run(x) = (if x\n helper(x)\nend)\nhelper(x) = x\n", 3)] public void Extract_JuliaShortFunctionsOwnTheirCallReferences_Issue4738( string content, int expectedEndLine) @@ -481,6 +483,36 @@ public void Extract_JuliaShortFunctionsOwnTheirCallReferences_Issue4738( reference.SymbolName == "helper" && reference.ReferenceKind == "call").ContainerName); } + [Fact] + public void Extract_JuliaExpressionPositionBlocksKeepOuterFunctionRange_Issue4738() + { + const string content = """ + function outer(xs) + map(function (item) + inner(item) + end, xs) + push!(xs, begin + nested() + end) + values = [item for item in xs if item > 0] + after() + end + outside() + """; + var symbols = SymbolExtractor.Extract(1, "julia", content); + + var references = ReferenceExtractor.Extract(1, "julia", content, symbols); + + Assert.Equal(10, Assert.Single(symbols, symbol => symbol.Name == "outer").EndLine); + foreach (var name in new[] { "inner", "nested", "after" }) + { + Assert.Equal("outer", Assert.Single(references, reference => + reference.SymbolName == name && reference.ReferenceKind == "call").ContainerName); + } + Assert.Null(Assert.Single(references, reference => + reference.SymbolName == "outside" && reference.ReferenceKind == "call").ContainerName); + } + [Fact] public void Extract_MatlabPeerFunctionsWithoutClosingEndHaveSeparateRanges_Issue4738() { @@ -504,6 +536,31 @@ function second() reference.SymbolName == "helper2" && reference.ReferenceKind == "call").ContainerName); } + [Fact] + public void Extract_MatlabSameIndentNestedFunctionKeepsExplicitOuterRange_Issue4738() + { + const string content = """ + function outer() + helper1(); + function nested() + inner(); + end + helper2(); + end + outside(); + """; + var symbols = SymbolExtractor.Extract(1, "matlab", content); + + var references = ReferenceExtractor.Extract(1, "matlab", content, symbols); + + Assert.Equal(7, Assert.Single(symbols, symbol => symbol.Name == "outer").EndLine); + Assert.Equal(5, Assert.Single(symbols, symbol => symbol.Name == "nested").EndLine); + Assert.Equal("outer", Assert.Single(references, reference => + reference.SymbolName == "helper2" && reference.ReferenceKind == "call").ContainerName); + Assert.Null(Assert.Single(references, reference => + reference.SymbolName == "outside" && reference.ReferenceKind == "call").ContainerName); + } + [Fact] public void Extract_JuliaTransposeBeforeBlockCommentDoesNotMaskFollowingCalls_Issue4738() { @@ -596,6 +653,75 @@ public void Extract_ScientificNativeLiteralTokensDoNotSuppressFollowingCalls_Iss Assert.DoesNotContain(references, reference => reference.SymbolName == "tokenOnly"); } + [Fact] + public void Extract_DCommentTextCannotOpenTokenStrings_Issue4738() + { + const string content = """ + /* documentation mentions q{ without a closing brace */ + /* documentation mentions an unmatched ` delimiter */ + void run() { + helper(); + } + """; + var symbols = SymbolExtractor.Extract(1, "d", content); + + var references = ReferenceExtractor.Extract(1, "d", content, symbols); + + Assert.Equal("run", Assert.Single(references, reference => + reference.SymbolName == "helper" && reference.ReferenceKind == "call").ContainerName); + } + + [Theory] + [InlineData( + "d", + """ + void run() { + auto text = r"fake()\"; + helper(); + } + """)] + [InlineData( + "nim", + """ + proc run() = + let text = r"fake()\" + helper() + """)] + public void Extract_DAndNimRawStringsUseLiteralBackslashRules_Issue4738( + string language, + string content) + { + var symbols = SymbolExtractor.Extract(1, language, content); + + var references = ReferenceExtractor.Extract(1, language, content, symbols); + + Assert.Equal("run", Assert.Single(references, reference => + reference.SymbolName == "helper" && reference.ReferenceKind == "call").ContainerName); + Assert.DoesNotContain(references, reference => reference.SymbolName == "fake"); + } + + [Fact] + public void Extract_AdaAttributesPreserveNestedCallsWithoutPhantomAttributeCalls_Issue4738() + { + const string content = """ + procedure Run is + begin + First(Integer'Image(helper()) & Float'Image(other())); + end Run; + """; + var symbols = SymbolExtractor.Extract(1, "ada", content); + + var references = ReferenceExtractor.Extract(1, "ada", content, symbols); + + foreach (var name in new[] { "First", "helper", "other" }) + { + Assert.Equal("Run", Assert.Single(references, reference => + reference.SymbolName == name && reference.ReferenceKind == "call").ContainerName); + } + Assert.DoesNotContain(references, reference => + reference.SymbolName == "Image" && reference.ReferenceKind == "call"); + } + [Fact] public void Extract_DMultipleBaseTypesAndCastSyntaxStayGraphAccurate_Issue4738() { From a8002ced083dd307f1b5631f0af1400c04505cbb Mon Sep 17 00:00:00 2001 From: Widthdom Date: Fri, 24 Jul 2026 14:45:51 +0900 Subject: [PATCH 07/21] Resolve scientific graph review edge cases (#4738) --- DEVELOPER_GUIDE.md | 4 +- .../Indexer/AmbiguousMContentMasker.cs | 13 ++ .../ScientificNativeCommentMasker.cs | 24 ++- .../ScientificNativeReferenceExtractor.cs | 109 +++++++++---- .../ReferenceExtractor.CoreExtraction.cs | 7 +- .../Indexer/References/ReferenceExtractor.cs | 1 + .../SymbolExtractor.ScientificNative.cs | 140 ++++++++++++++++- ...eferenceExtractorTests.ScientificNative.cs | 144 +++++++++++++++++- 8 files changed, 404 insertions(+), 38 deletions(-) diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index 19d9cefcd..13612272a 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -1626,8 +1626,8 @@ Process exit codes are coarse (`0` success including valid zero-row queries, `1` - **Freshness metadata for trust decisions** — `status` exposes whole-workspace freshness and git state, plus trust metadata such as `sql_graph_contract_ready` / `sql_graph_contract_degraded_reason`, `hotspot_family_ready` / `hotspot_family_degraded_reason`, forward-compatibility audit fields (`index_writer_version`, `index_newer_than_reader`, `index_newer_than_reader_reason` — see "Forward-compatibility readiness audit"), and fold remediation fields (`fold_ready_reason`, `degraded_reason`, `recommended_action`, `alternative_action`) so AI clients can tell up front whether SQL graph/dependency/impact answers, duplicate-name hotspot families, and Unicode `--exact` are authoritative. CLI `status --json` and MCP `status` both populate those fold remediation fields when `fold_ready=false`. It also carries `unknown_extension_file_count`, a capped `unknown_extension_files` path sample, `unknown_extension_files_truncated`, and `unknown_extension_file_path_limit` after a current full-repository scan so extension-table coverage gaps are visible and actionable even when those files were excluded from indexing. When those fold remediation fields are derived from an explicit read-only `file:` DB URI, they are normalized back to a writable filesystem path for both absolute (`file:///...?...`) and relative (`file:codeindex.db?...`) forms instead of echoing the read-only URI into commands that would fail. `cdidx index` JSON/human readiness output also surfaces the same trust bits, keeping the post-index readiness summary aligned with `status`. `impact` / MCP `impact_analysis` also mirror the SQL graph-contract signal in JSON so stale SQL rows do not masquerade as authoritative zero-impact answers. `inspect` / MCP `analyze_symbol` and `references` / MCP `references` now mirror that same SQL graph-contract signal whenever SQL-backed graph reads contribute to their payloads, so stale SQL rows do not look like authoritative hits or zero-result answers there either. `map` keeps `indexed_at` / `latest_modified` scoped to the filtered result set and also exposes `workspace_indexed_at` / `workspace_latest_modified` for whole-workspace freshness. `inspect` mirrors those whole-workspace timestamps and git fields so symbol-oriented AI flows can make trust decisions without a separate `status` call. `files` exposes per-file checksum plus modified/indexed timestamps. File-column migrations are applied opportunistically for older DBs, and read paths are designed to avoid crashing if in-place migration is unavailable. CLI and MCP zero-result JSON responses for `search`, `files`, `symbols`, `definition`, `references`, `callers`, `callees`, `deps`, `unused`, `hotspots`, and `impact` include `indexed_file_count`, `indexed_at`, and `freshness_available`. `indexed_at:null` with `freshness_available=true` means the index is empty, while `freshness_available=false` means a legacy/read-only DB could not expose freshness timestamps and `freshness_degraded_reason` explains why. **HEAD-aware staleness signal**: every successful `cdidx index` full scan now stamps the captured `git HEAD` into `codeindex_meta` so subsequent runs can compare it against the workspace HEAD. When they differ and the user did not pass `--rebuild`, the CLI emits a `head_changed` warning recommending `cdidx index --rebuild` and exposes `head_changed` / `prior_indexed_head_commit` / `current_head_commit` / `head_change_notice` in `index --json`. `status --check` mirrors the same comparison through `workspace_check.head_changed` (alongside `indexed_head_commit` / `workspace_head_commit` when they differ), so AI clients that already gate on freshness can refuse to trust a default incremental scan after `git switch ` without a separate query. `--commits` / `--files` partial updates deliberately preserve the captured HEAD so the staleness signal survives until a real full scan reindexes the worktree. Non-git workspaces and legacy DBs that never captured a HEAD skip the comparison instead of false-positive flagging. - **Folded-key upgrade without reparse** — `backfill-fold` and MCP `backfill_fold` recompute `name_folded` / `*_folded` directly from existing DB rows, then stamp `FoldReadyFlag` once verification confirms no required folded values remain NULL. This gives AI clients and users a low-cost upgrade path from pre-#86 DBs without re-reading every source file, and it also rewrites all folded rows when `fold_key_version` is missing or mismatched so future `NameFold.Version` bumps cannot silently restamp stale keys. - **Bundled symbol analysis** — `inspect` and MCP `analyze_symbol` return definition, nearby symbols, references, callers, callees, file metadata, workspace trust metadata, and graph-support metadata in one request so AI clients can answer common symbol questions with fewer round-trips. -- **Language-aware reference extraction** — `references`, `callers`, `callees`, and `impact` are backed by an indexed reference table built only for languages where regex-based call/reference extraction is meaningful (30 of 46 languages). Unsupported languages intentionally fall back to text search instead of returning low-confidence pseudo-graph data. When a language is removed from graph support, `PurgeUnsupportedReferences` deletes its stale `symbol_references` rows on the next indexing run, and graph read paths additionally filter by supported languages to prevent stale edges from surviving between index runs. Shell is intentionally excluded because its command-style invocations (`foo arg1 arg2`) cannot be detected by the parenthesized-call regex. **Nested generic call sites**: C#/Java constructor calls like `new Dictionary>()` and C# generic method calls like `Helper.DoWork>()` are recovered by a depth-aware fallback scanner so the outer target still reaches the reference table even though the flat regex fast-path cannot balance `>>`. **JS/TS no-paren constructors**: JavaScript / TypeScript zero-argument constructor calls that legally omit `()` — for example `new Foo;`, `new Date;`, qualified targets like `new Demo.Provider;`, and one-level generic TypeScript forms like `new Box;` — are emitted as `instantiate` edges via a dedicated language-gated path, while next-line `.bar()` / `[0]` continuations are suppressed so a line-ended `new Foo` does not become a phantom standalone instantiation. **Constructor chain calls**: C# `: this(...)` / `: base(...)` initializers and Java `this(...)` / `super(...)` first-statement calls are detected separately from the generic call regex and rewritten so the reference target is the real constructor (enclosing class/record for `this`, the parsed base type from the class signature for `base` / `super`). Cross-line C# initializers are attributed to the owning constructor rather than the enclosing class. Base-type parsing strips generics, record primary-ctor args, `where` constraints, and `global::` / dotted namespace qualifiers; Java `super.method()` stays a normal method call. **Type-position dependency edges**: C#/Java base lists, declaration types, generic constraints, `throws`, `is`/`as`/`instanceof`, and real C# XML-doc `cref` sites are indexed as `type_reference` rows so `references` / `impact` can see compile-time rename dependencies without polluting the default dynamic call graph exposed by `callers` / `callees`. C# XML-doc `cref` extraction accepts declaration-attached XML-doc comments from both `///` lines and delimited `/** ... */` blocks, including declarations that begin later on the same physical line after the closing `*/` only when no unrelated same-line code or declaration intervenes, while ordinary `//` / `////` comments, non-documenting block comments, method-body XML-doc comments that merely precede a later declaration, brace-free field/property initializer continuations, brace-free expression lambdas, intervening top-level executable statements, same-line non-target code after `*/`, other nested executable continuations, and multiline raw/verbatim string content whose line happens to start with `/**` stay excluded. Non-doc code or string content after the closing `*/` on the same physical line is still outside the doc-comment slice. Even though the regex now runs against that narrower slice, the extractor preserves `symbol_references.column` relative to the original physical source line. On the C# read path, `using static` constant-pattern suppression is token-aware around `is` / `case`, reconstructs a small indexed multi-line window when the anchor lives on a previous line, and keeps trivia-bearing forms such as `value is/*comment*/Red`, `value is\n Red or Blue`, and `case\tRed:` filtered. Same-name type rescue also honors `file` visibility so file-local types only rescue references from the same physical file; inherited protected/public/internal nested types from real base classes rescue derived-class pattern heads only after the base reference is normalized through active type and namespace aliases, and alias-expanded constructed generic bases are canonicalized again before containing-type lookup so `AliasBase = Probe.Base` resolves the same way as `Probe.Base`; implemented interfaces do not contribute inherited nested-type rescue; and same-file `using Namespace;`, project-wide `global using Namespace;`, and active type aliases all participate in the rescue set. The extractor deliberately leaves ambiguous unqualified `using static` heads such as `value is Red` in the DB, because file-local parsing alone cannot know whether another file in the same namespace declares the real `Red` type; the workspace-aware read path is responsible for suppressing the pure constant-only cases. **SQL qualified-name alignment**: SQL definitions still persist their schema-qualified symbol name (`dbo.fn_X`), but graph/`deps`/unused/hotspot readers now resolve each SQL reference row through its stored source-line context, recorded call column, and enclosing container before they compare it to definitions, so qualified `references` / `callers` / `impact` queries stay schema-scoped even when one line contains multiple qualified calls or the lookup is non-exact. Those readers fall back to the bare leaf only when the source site itself is genuinely unqualified, which keeps `deps`, `unused`, and `hotspots` aligned with qualified SQL calls without regressing bare-call support or double-counting `EXEC dbo.fn_Target; EXEC sales.fn_Target;`. Once a row already has a recorded call column, those downstream readers no longer whole-line-upgrade that row to a later qualified token, so trailing comments, string literals, or a second qualified call cannot steal the earlier unqualified edge. Exact SQL graph/dependency readers also preserve the resolved segment count, so a quoted single identifier containing a dot such as `"sales.fn_Target"` stays distinct from the real qualified name `sales.fn_Target` across exact `references` / `callers` / `impact` and aggregate `deps` / `unused` / `hotspots`. SQL CTE body source rows use the raw `cte_body_reference` kind, so `references --kind cte_body_reference` can distinguish anchor/recursive-member internals from outer-query table references. Qualified SQL `callees` queries also keep leaf fallback disabled unless the caller query itself is unqualified, so `callees sales.Caller` no longer widens to `dbo.Caller`. SQL extractors also accept optional whitespace around qualified-name dots, so definitions/calls such as `[sales] . [fn_Target]` and `[dbo] . [fn_Target]` keep their full qualified identity instead of truncating at the first segment. The same SQL no-parens extractor now preserves ANSI / PostgreSQL double-quoted call targets such as `CALL "sales"."proc_name"` and `EXEC "dbo"."fn_Target"` instead of stripping them as string literals, while true single-quoted SQL string literals remain masked. Definition-oriented readers also canonicalize quoted qualified SQL names (`[dbo].[fn_X]` → `dbo.fn_X`) before matching, and they only fall back to the leaf identifier for unqualified queries so exact qualified lookups do not widen to sibling schemas that merely share the same leaf name. Exact SQL definition matching also preserves segment count, so a quoted single identifier that contains a dot (`"sales.fn_Target"`) does not collide with a real qualified name (`sales.fn_Target`). SQL exact graph leaf fallback also stays on the Unicode folded exact path, and both quoted qualified and unqualified Unicode exact definition lookups now use the folded normalized path, so queries such as `dbo.Äpfel` / `dbo.äpfel` and bare `Äpfel` / `äpfel` keep matching leaf call/reference rows such as `äpfel` plus stored definitions such as `[dbo].[Äpfel]` or `dbo.Äpfel` instead of silently degrading to ASCII-only `NOCASE`. Exact multi-name SQL `symbols --count` lookups also bind the folded leaf parameters on that same `_foldReady` path, so Unicode leaf query sets no longer fail with missing-parameter database errors. -- **Language-aware reference extraction** — `references`, `callers`, `callees`, and `impact` are backed by an indexed reference table built only for languages where regex-based call/reference extraction is meaningful (30 of 46 languages). Unsupported languages intentionally fall back to text search instead of returning low-confidence pseudo-graph data. When a language is removed from graph support, `PurgeUnsupportedReferences` deletes its stale `symbol_references` rows on the next indexing run, and graph read paths additionally filter by supported languages to prevent stale edges from surviving between index runs. Shell is intentionally excluded because its command-style invocations (`foo arg1 arg2`) cannot be detected by the parenthesized-call regex. **Nested generic call sites**: C#/Java constructor calls like `new Dictionary>()` and C# generic method calls like `Helper.DoWork>()` are recovered by a depth-aware fallback scanner so the outer target still reaches the reference table even though the flat regex fast-path cannot balance `>>`. **JS/TS no-paren constructors**: JavaScript / TypeScript zero-argument constructor calls that legally omit `()` — for example `new Foo;`, `new Date;`, qualified targets like `new Demo.Provider;`, and one-level generic TypeScript forms like `new Box;` — are emitted as `instantiate` edges via a dedicated language-gated path, while next-line `.bar()` / `[0]` continuations are suppressed so a line-ended `new Foo` does not become a phantom standalone instantiation. **Constructor chain calls**: C# `: this(...)` / `: base(...)` initializers and Java `this(...)` / `super(...)` first-statement calls are detected separately from the generic call regex and rewritten so the reference target is the real constructor (enclosing class/record for `this`, the parsed base type from the class signature for `base` / `super`). Cross-line C# initializers are attributed to the owning constructor rather than the enclosing class. Base-type parsing strips generics, record primary-ctor args, `where` constraints, and `global::` / dotted namespace qualifiers; Java `super.method()` stays a normal method call. **Type-position dependency edges**: C#/Java base lists, declaration types, generic constraints, `throws`, `is`/`as`/`instanceof`, and real C# XML-doc `cref` sites are indexed as `type_reference` rows so `references` / `impact` can see compile-time rename dependencies without polluting the default dynamic call graph exposed by `callers` / `callees`. C# XML-doc `cref` extraction accepts declaration-attached XML-doc comments from both `///` lines and delimited `/** ... */` blocks, including declarations that begin later on the same physical line after the closing `*/` only when no unrelated same-line code or declaration intervenes, while ordinary `//` / `////` comments, non-documenting block comments, method-body XML-doc comments that merely precede a later declaration, brace-free field/property initializer continuations, brace-free expression lambdas, intervening top-level executable statements, same-line non-target code after `*/`, other nested executable continuations, and multiline raw/verbatim string content whose line happens to start with `/**` stay excluded. Non-doc code or string content after the closing `*/` on the same physical line is still outside the doc-comment slice. Even though the regex now runs against that narrower slice, the extractor preserves `symbol_references.column` relative to the original physical source line. On the C# read path, `using static` constant-pattern suppression is token-aware around `is` / `case`, reconstructs an anchor-aware indexed multi-line window when the anchor lives on a previous line, and keeps trivia-bearing forms such as `value is/*comment*/Red`, `value is\n Red or Blue`, `value is\n // comment\n Red`, `case\n // comment\n Point:`, long `case` / `or` chains, and `case\tRed:` filtered or rescued correctly. Qualified constant/member patterns stay qualifier-driven on that exact-name read path, so an unrelated same-name type such as `class Red {}` no longer cancels suppression for `case Color.Red or Color.Blue:` just because the leaf name matches. The extractor-side pending type-pattern carry now also survives trivia-only separator lines, standalone continuation-line `not`, and multiline `case` heads/logical continuations, so comment-only or `not`-only continuation lines no longer drop the later type head before the real token arrives. Non-type `case` labels such as `case > 0:` and `case not > 0:` do not arm that pending carry, so the next-line call/identifier token stays out of `type_reference`. Same-name type rescue also honors `file` visibility so file-local types only rescue references from the same physical file; inherited protected/public/internal nested types from real base classes rescue derived-class pattern heads only after the base reference is normalized through active type and namespace aliases, and alias-expanded constructed generic bases are canonicalized again before containing-type lookup so `AliasBase = Probe.Base` resolves the same way as `Probe.Base`; implemented interfaces do not contribute inherited nested-type rescue; and same-file `using Namespace;`, project-wide `global using Namespace;`, and active type aliases all participate in the rescue set. The extractor deliberately leaves ambiguous unqualified `using static` heads such as `value is Red` in the DB, because file-local parsing alone cannot know whether another file in the same namespace declares the real `Red` type; the workspace-aware read path is responsible for suppressing the pure constant-only cases. **SQL qualified-name alignment**: SQL definitions still persist their schema-qualified symbol name (`dbo.fn_X`), but graph/`deps`/unused/hotspot readers now resolve each SQL reference row through its stored source-line context, recorded call column, and enclosing container before they compare it to definitions, so qualified `references` / `callers` / `impact` queries stay schema-scoped even when one line contains multiple qualified calls or the lookup is non-exact. Those readers fall back to the bare leaf only when the source site itself is genuinely unqualified, which keeps `deps`, `unused`, and `hotspots` aligned with qualified SQL calls without regressing bare-call support or double-counting `EXEC dbo.fn_Target; EXEC sales.fn_Target;`. Once a row already has a recorded call column, those downstream readers no longer whole-line-upgrade that row to a later qualified token, so trailing comments, string literals, or a second qualified call cannot steal the earlier unqualified edge. Exact SQL graph/dependency readers also preserve the resolved segment count, so a quoted single identifier containing a dot such as `"sales.fn_Target"` stays distinct from the real qualified name `sales.fn_Target` across exact `references` / `callers` / `impact` and aggregate `deps` / `unused` / `hotspots`. SQL CTE body source rows use the raw `cte_body_reference` kind, so `references --kind cte_body_reference` can distinguish anchor/recursive-member internals from outer-query table references. Qualified SQL `callees` queries also keep leaf fallback disabled unless the caller query itself is unqualified, so `callees sales.Caller` no longer widens to `dbo.Caller`. SQL extractors also accept optional whitespace around qualified-name dots, so definitions/calls such as `[sales] . [fn_Target]` and `[dbo] . [fn_Target]` keep their full qualified identity instead of truncating at the first segment. The same SQL no-parens extractor now preserves ANSI / PostgreSQL double-quoted call targets such as `CALL "sales"."proc_name"` and `EXEC "dbo"."fn_Target"` instead of stripping them as string literals, while true single-quoted SQL string literals remain masked. Definition-oriented readers also canonicalize quoted qualified SQL names (`[dbo].[fn_X]` → `dbo.fn_X`) before matching, and they only fall back to the leaf identifier for unqualified queries so exact qualified lookups do not widen to sibling schemas that merely share the same leaf name. Exact SQL definition matching also preserves segment count, so a quoted single identifier that contains a dot (`"sales.fn_Target"`) does not collide with a real qualified name (`sales.fn_Target`). SQL exact graph leaf fallback also stays on the Unicode folded exact path, and both quoted qualified and unqualified Unicode exact definition lookups now use the folded normalized path, so queries such as `dbo.Äpfel` / `dbo.äpfel` and bare `Äpfel` / `äpfel` keep matching leaf call/reference rows such as `äpfel` plus stored definitions such as `[dbo].[Äpfel]` or `dbo.Äpfel` instead of silently degrading to ASCII-only `NOCASE`. Exact multi-name SQL `symbols --count` lookups also bind the folded leaf parameters on that same `_foldReady` path, so Unicode leaf query sets no longer fail with missing-parameter database errors. +- **Language-aware reference extraction** — `references`, `callers`, `callees`, and `impact` are backed by an indexed reference table built only for languages where regex-based call/reference extraction is meaningful (65 of 90 built-in languages). Unsupported languages intentionally fall back to text search instead of returning low-confidence pseudo-graph data. When a language is removed from graph support, `PurgeUnsupportedReferences` deletes its stale `symbol_references` rows on the next indexing run, and graph read paths additionally filter by supported languages to prevent stale edges from surviving between index runs. Shell is intentionally excluded because its command-style invocations (`foo arg1 arg2`) cannot be detected by the parenthesized-call regex. **Nested generic call sites**: C#/Java constructor calls like `new Dictionary>()` and C# generic method calls like `Helper.DoWork>()` are recovered by a depth-aware fallback scanner so the outer target still reaches the reference table even though the flat regex fast-path cannot balance `>>`. **JS/TS no-paren constructors**: JavaScript / TypeScript zero-argument constructor calls that legally omit `()` — for example `new Foo;`, `new Date;`, qualified targets like `new Demo.Provider;`, and one-level generic TypeScript forms like `new Box;` — are emitted as `instantiate` edges via a dedicated language-gated path, while next-line `.bar()` / `[0]` continuations are suppressed so a line-ended `new Foo` does not become a phantom standalone instantiation. **Constructor chain calls**: C# `: this(...)` / `: base(...)` initializers and Java `this(...)` / `super(...)` first-statement calls are detected separately from the generic call regex and rewritten so the reference target is the real constructor (enclosing class/record for `this`, the parsed base type from the class signature for `base` / `super`). Cross-line C# initializers are attributed to the owning constructor rather than the enclosing class. Base-type parsing strips generics, record primary-ctor args, `where` constraints, and `global::` / dotted namespace qualifiers; Java `super.method()` stays a normal method call. **Type-position dependency edges**: C#/Java base lists, declaration types, generic constraints, `throws`, `is`/`as`/`instanceof`, and real C# XML-doc `cref` sites are indexed as `type_reference` rows so `references` / `impact` can see compile-time rename dependencies without polluting the default dynamic call graph exposed by `callers` / `callees`. C# XML-doc `cref` extraction accepts declaration-attached XML-doc comments from both `///` lines and delimited `/** ... */` blocks, including declarations that begin later on the same physical line after the closing `*/` only when no unrelated same-line code or declaration intervenes, while ordinary `//` / `////` comments, non-documenting block comments, method-body XML-doc comments that merely precede a later declaration, brace-free field/property initializer continuations, brace-free expression lambdas, intervening top-level executable statements, same-line non-target code after `*/`, other nested executable continuations, and multiline raw/verbatim string content whose line happens to start with `/**` stay excluded. Non-doc code or string content after the closing `*/` on the same physical line is still outside the doc-comment slice. Even though the regex now runs against that narrower slice, the extractor preserves `symbol_references.column` relative to the original physical source line. On the C# read path, `using static` constant-pattern suppression is token-aware around `is` / `case`, reconstructs a small indexed multi-line window when the anchor lives on a previous line, and keeps trivia-bearing forms such as `value is/*comment*/Red`, `value is\n Red or Blue`, and `case\tRed:` filtered. Same-name type rescue also honors `file` visibility so file-local types only rescue references from the same physical file; inherited protected/public/internal nested types from real base classes rescue derived-class pattern heads only after the base reference is normalized through active type and namespace aliases, and alias-expanded constructed generic bases are canonicalized again before containing-type lookup so `AliasBase = Probe.Base` resolves the same way as `Probe.Base`; implemented interfaces do not contribute inherited nested-type rescue; and same-file `using Namespace;`, project-wide `global using Namespace;`, and active type aliases all participate in the rescue set. The extractor deliberately leaves ambiguous unqualified `using static` heads such as `value is Red` in the DB, because file-local parsing alone cannot know whether another file in the same namespace declares the real `Red` type; the workspace-aware read path is responsible for suppressing the pure constant-only cases. **SQL qualified-name alignment**: SQL definitions still persist their schema-qualified symbol name (`dbo.fn_X`), but graph/`deps`/unused/hotspot readers now resolve each SQL reference row through its stored source-line context, recorded call column, and enclosing container before they compare it to definitions, so qualified `references` / `callers` / `impact` queries stay schema-scoped even when one line contains multiple qualified calls or the lookup is non-exact. Those readers fall back to the bare leaf only when the source site itself is genuinely unqualified, which keeps `deps`, `unused`, and `hotspots` aligned with qualified SQL calls without regressing bare-call support or double-counting `EXEC dbo.fn_Target; EXEC sales.fn_Target;`. Once a row already has a recorded call column, those downstream readers no longer whole-line-upgrade that row to a later qualified token, so trailing comments, string literals, or a second qualified call cannot steal the earlier unqualified edge. Exact SQL graph/dependency readers also preserve the resolved segment count, so a quoted single identifier containing a dot such as `"sales.fn_Target"` stays distinct from the real qualified name `sales.fn_Target` across exact `references` / `callers` / `impact` and aggregate `deps` / `unused` / `hotspots`. SQL CTE body source rows use the raw `cte_body_reference` kind, so `references --kind cte_body_reference` can distinguish anchor/recursive-member internals from outer-query table references. Qualified SQL `callees` queries also keep leaf fallback disabled unless the caller query itself is unqualified, so `callees sales.Caller` no longer widens to `dbo.Caller`. SQL extractors also accept optional whitespace around qualified-name dots, so definitions/calls such as `[sales] . [fn_Target]` and `[dbo] . [fn_Target]` keep their full qualified identity instead of truncating at the first segment. The same SQL no-parens extractor now preserves ANSI / PostgreSQL double-quoted call targets such as `CALL "sales"."proc_name"` and `EXEC "dbo"."fn_Target"` instead of stripping them as string literals, while true single-quoted SQL string literals remain masked. Definition-oriented readers also canonicalize quoted qualified SQL names (`[dbo].[fn_X]` → `dbo.fn_X`) before matching, and they only fall back to the leaf identifier for unqualified queries so exact qualified lookups do not widen to sibling schemas that merely share the same leaf name. Exact SQL definition matching also preserves segment count, so a quoted single identifier that contains a dot (`"sales.fn_Target"`) does not collide with a real qualified name (`sales.fn_Target`). SQL exact graph leaf fallback also stays on the Unicode folded exact path, and both quoted qualified and unqualified Unicode exact definition lookups now use the folded normalized path, so queries such as `dbo.Äpfel` / `dbo.äpfel` and bare `Äpfel` / `äpfel` keep matching leaf call/reference rows such as `äpfel` plus stored definitions such as `[dbo].[Äpfel]` or `dbo.Äpfel` instead of silently degrading to ASCII-only `NOCASE`. Exact multi-name SQL `symbols --count` lookups also bind the folded leaf parameters on that same `_foldReady` path, so Unicode leaf query sets no longer fail with missing-parameter database errors. +- **Language-aware reference extraction** — `references`, `callers`, `callees`, and `impact` are backed by an indexed reference table built only for languages where regex-based call/reference extraction is meaningful (65 of 90 built-in languages). Unsupported languages intentionally fall back to text search instead of returning low-confidence pseudo-graph data. When a language is removed from graph support, `PurgeUnsupportedReferences` deletes its stale `symbol_references` rows on the next indexing run, and graph read paths additionally filter by supported languages to prevent stale edges from surviving between index runs. Shell is intentionally excluded because its command-style invocations (`foo arg1 arg2`) cannot be detected by the parenthesized-call regex. **Nested generic call sites**: C#/Java constructor calls like `new Dictionary>()` and C# generic method calls like `Helper.DoWork>()` are recovered by a depth-aware fallback scanner so the outer target still reaches the reference table even though the flat regex fast-path cannot balance `>>`. **JS/TS no-paren constructors**: JavaScript / TypeScript zero-argument constructor calls that legally omit `()` — for example `new Foo;`, `new Date;`, qualified targets like `new Demo.Provider;`, and one-level generic TypeScript forms like `new Box;` — are emitted as `instantiate` edges via a dedicated language-gated path, while next-line `.bar()` / `[0]` continuations are suppressed so a line-ended `new Foo` does not become a phantom standalone instantiation. **Constructor chain calls**: C# `: this(...)` / `: base(...)` initializers and Java `this(...)` / `super(...)` first-statement calls are detected separately from the generic call regex and rewritten so the reference target is the real constructor (enclosing class/record for `this`, the parsed base type from the class signature for `base` / `super`). Cross-line C# initializers are attributed to the owning constructor rather than the enclosing class. Base-type parsing strips generics, record primary-ctor args, `where` constraints, and `global::` / dotted namespace qualifiers; Java `super.method()` stays a normal method call. **Type-position dependency edges**: C#/Java base lists, declaration types, generic constraints, `throws`, `is`/`as`/`instanceof`, and real C# XML-doc `cref` sites are indexed as `type_reference` rows so `references` / `impact` can see compile-time rename dependencies without polluting the default dynamic call graph exposed by `callers` / `callees`. C# XML-doc `cref` extraction accepts declaration-attached XML-doc comments from both `///` lines and delimited `/** ... */` blocks, including declarations that begin later on the same physical line after the closing `*/` only when no unrelated same-line code or declaration intervenes, while ordinary `//` / `////` comments, non-documenting block comments, method-body XML-doc comments that merely precede a later declaration, brace-free field/property initializer continuations, brace-free expression lambdas, intervening top-level executable statements, same-line non-target code after `*/`, other nested executable continuations, and multiline raw/verbatim string content whose line happens to start with `/**` stay excluded. Non-doc code or string content after the closing `*/` on the same physical line is still outside the doc-comment slice. Even though the regex now runs against that narrower slice, the extractor preserves `symbol_references.column` relative to the original physical source line. On the C# read path, `using static` constant-pattern suppression is token-aware around `is` / `case`, reconstructs an anchor-aware indexed multi-line window when the anchor lives on a previous line, and keeps trivia-bearing forms such as `value is/*comment*/Red`, `value is\n Red or Blue`, `value is\n // comment\n Red`, `case\n // comment\n Point:`, long `case` / `or` chains, and `case\tRed:` filtered or rescued correctly. Qualified constant/member patterns stay qualifier-driven on that exact-name read path, so an unrelated same-name type such as `class Red {}` no longer cancels suppression for `case Color.Red or Color.Blue:` just because the leaf name matches. The extractor-side pending type-pattern carry now also survives trivia-only separator lines, standalone continuation-line `not`, and multiline `case` heads/logical continuations, so comment-only or `not`-only continuation lines no longer drop the later type head before the real token arrives. Non-type `case` labels such as `case > 0:` and `case not > 0:` do not arm that pending carry, so the next-line call/identifier token stays out of `type_reference`. Same-name type rescue also honors `file` visibility so file-local types only rescue references from the same physical file; inherited protected/public/internal nested types from real base classes rescue derived-class pattern heads only after the base reference is normalized through active type and namespace aliases, and alias-expanded constructed generic bases are canonicalized again before containing-type lookup so `AliasBase = Probe.Base` resolves the same way as `Probe.Base`; implemented interfaces do not contribute inherited nested-type rescue; and same-file `using Namespace;`, project-wide `global using Namespace;`, and active type aliases all participate in the rescue set. The extractor deliberately leaves ambiguous unqualified `using static` heads such as `value is Red` in the DB, because file-local parsing alone cannot know whether another file in the same namespace declares the real `Red` type; the workspace-aware read path is responsible for suppressing the pure constant-only cases. **SQL qualified-name alignment**: SQL definitions still persist their schema-qualified symbol name (`dbo.fn_X`), but graph/`deps`/unused/hotspot readers now resolve each SQL reference row through its stored source-line context, recorded call column, and enclosing container before they compare it to definitions, so qualified `references` / `callers` / `impact` queries stay schema-scoped even when one line contains multiple qualified calls or the lookup is non-exact. Those readers fall back to the bare leaf only when the source site itself is genuinely unqualified, which keeps `deps`, `unused`, and `hotspots` aligned with qualified SQL calls without regressing bare-call support or double-counting `EXEC dbo.fn_Target; EXEC sales.fn_Target;`. Once a row already has a recorded call column, those downstream readers no longer whole-line-upgrade that row to a later qualified token, so trailing comments, string literals, or a second qualified call cannot steal the earlier unqualified edge. Exact SQL graph/dependency readers also preserve the resolved segment count, so a quoted single identifier containing a dot such as `"sales.fn_Target"` stays distinct from the real qualified name `sales.fn_Target` across exact `references` / `callers` / `impact` and aggregate `deps` / `unused` / `hotspots`. SQL CTE body source rows use the raw `cte_body_reference` kind, so `references --kind cte_body_reference` can distinguish anchor/recursive-member internals from outer-query table references. Qualified SQL `callees` queries also keep leaf fallback disabled unless the caller query itself is unqualified, so `callees sales.Caller` no longer widens to `dbo.Caller`. SQL extractors also accept optional whitespace around qualified-name dots, so definitions/calls such as `[sales] . [fn_Target]` and `[dbo] . [fn_Target]` keep their full qualified identity instead of truncating at the first segment. The same SQL no-parens extractor now preserves ANSI / PostgreSQL double-quoted call targets such as `CALL "sales"."proc_name"` and `EXEC "dbo"."fn_Target"` instead of stripping them as string literals, while true single-quoted SQL string literals remain masked. Definition-oriented readers also canonicalize quoted qualified SQL names (`[dbo].[fn_X]` → `dbo.fn_X`) before matching, and they only fall back to the leaf identifier for unqualified queries so exact qualified lookups do not widen to sibling schemas that merely share the same leaf name. Exact SQL definition matching also preserves segment count, so a quoted single identifier that contains a dot (`"sales.fn_Target"`) does not collide with a real qualified name (`sales.fn_Target`). SQL exact graph leaf fallback also stays on the Unicode folded exact path, and both quoted qualified and unqualified Unicode exact definition lookups now use the folded normalized path, so queries such as `dbo.Äpfel` / `dbo.äpfel` and bare `Äpfel` / `äpfel` keep matching leaf call/reference rows such as `äpfel` plus stored definitions such as `[dbo].[Äpfel]` or `dbo.Äpfel` instead of silently degrading to ASCII-only `NOCASE`. Exact multi-name SQL `symbols --count` lookups also bind the folded leaf parameters on that same `_foldReady` path, so Unicode leaf query sets no longer fail with missing-parameter database errors. - **Transitive impact analysis** — `impact` and MCP `impact_analysis` compute the transitive caller chain of a symbol using BFS. Design constraints refined through adversarial review: caller matching uses case-insensitive exact match (`lower() = lower()`) to avoid both substring expansion and case-sensitivity brittleness; symbol names are pre-resolved through definitions with exact-case preference; the read path filters to graph-supported languages to prevent stale edges from removed languages; the definition set used for heuristic fallback must also respect active `--lang` / `--path` / `--exclude-path` / `--exclude-tests` filters and graph-supported languages so out-of-scope or unsupported duplicates do not suppress in-scope hints; fallback eligibility is keyed off class-like definitions only, so same-name namespace/import siblings do not block a single resolved class / struct / interface target, while pure non-callable `namespace` / `import` queries surface `non_callable_symbol_kind` guidance; heuristic file-level hints still return a successful result and encode their non-authoritative status via `impact_mode`, `heuristic`, `hint_count`, and `truncated`; caller rows include `result_kind: "graph"` and heuristic `file_impacts` rows include `result_kind: "file_heuristic"` so clients can distinguish authoritative hop-depth graph results from boundary fallback hints without inferring from list position or depth values; when `truncated` is `true`, the JSON / MCP payload also exposes `truncated_reason` so callers can distinguish actionable cases from runaway-graph cases — `user_limit` means the caller-supplied `--limit` was reached and raising `--limit` will return more results, while `safety_cap` means an internal per-symbol BFS fetch-iteration cap fired (the graph is likely pathological / cyclic and raising `--limit` alone will not help). `impact` / MCP `impact_analysis` also expose `termination_reason` (`completed`, `max_depth_reached`, `cycle_detected`, `row_limit_truncated`, `safety_cap`, or `cancelled`), `cycle_detected`, and `cycles` so caller cycles are distinguishable from natural traversal completion or limit/depth termination (#1883). `safety_cap` outranks `user_limit` whenever both are encountered, and the heuristic file-level hints path is `user_limit`-only because hint truncation is always driven by the caller's `--limit`. The field is omitted whenever `truncated` is `false`. (#1533) `count` / `file_count` now describe the visible returned set while `confirmed_count` / `confirmed_file_count` preserve symbol-level caller totals for heuristic-success payloads, and `impact --json --count` uses the same `*_count` field names as the full payload; to reduce general-name collisions, a file only qualifies for type fallback if it both references one of the candidate member names and also exposes same-file evidence anchoring the source/target pair — either a `call` / `instantiate` reference to the resolved target name (the call-graph itself authoritatively pins the relationship, so this path runs before the metadata-attribute bypass and does not depend on the looser ambiguity guard) or structured type evidence through indexed symbol metadata such as signatures or return types — rather than raw comment/string text matches. The call/instantiate anchor matches the resolved name exactly with no suffix-strip alias, because callable references already carry the authoritative identifier and applying the C# `[Foo]` → `FooAttribute` alias there would let unrelated `Foo()` method calls falsely anchor `impact FooAttribute` (#1881); the metadata bypass keeps the C# `Attribute` suffix alias because attribute use sites legitimately abbreviate the target name. The signature evidence path is Unicode-aware so fullwidth/accented identifiers are tokenized consistently with exact-name resolution; hint `reference_count` reflects the real number of matching reference rows while the symbol list stays deduplicated; only multiple class-like definitions are treated as fallback ambiguity, even when they share one file; and `PurgeUnsupportedReferences` runs in all three indexing paths (CLI full scan, CLI update mode, MCP index). - **Extractor regex backtracking policy** — Built-in symbol and reference extractors must not use unbounded regular expression matching on repository-controlled file content. Backtracking regexes use `BoundedRegex.DefaultMatchTimeout`, while `RegexOptions.NonBacktracking` is allowed for patterns that are compatible with the non-backtracking engine. Patterns that deliberately remain backtracking-only, such as lookaround-heavy or balancing-group extractors, are acceptable only because the shared timeout audit covers them. If a future extractor must use `System.Text.RegularExpressions.Regex` directly, it must pass an explicit timeout and document why `BoundedRegex` or `NonBacktracking` is not suitable. - **Hybrid symbol extraction** — No AST parsers and no heavyweight language-specific dependencies. Most languages still use compiled regex patterns, while JavaScript/TypeScript add a lightweight lexer/state machine for class-body method extraction, private-scope filtering, synthetic class-expression binding detection, and JS/TS-specific range resolution that regex alone could not handle reliably. The trade-off still favors speed and portability over full parser accuracy, but the index stores richer symbol metadata such as definition ranges, optional body ranges, signatures, enclosing symbols, qualified container paths, authoritative family keys, visibility, and return types when the language patterns or JS/TS state machine can infer them. Visual Basic patterns also treat `Namespace ... End Namespace` as a real container and allow implicit-visibility declarations plus leading modifiers (`Shared`, `Overrides`, `Partial`, etc.), so VB projects expose the same top-level orientation and member coverage that other class-based languages already get. Visual Basic container patterns use case-insensitive `VisualBasicEnd` range tracking so cross-file partial families still get stable body ranges and can participate in hotspot-family grouping. **Pattern externalization**: Language patterns are currently defined inline in `SymbolExtractor.cs` using compiled `Regex` objects. This keeps the extraction pipeline self-contained and allows compile-time validation, but means adding a new language requires a code change and rebuild. A future iteration could externalize patterns to JSON/TOML files (loaded at startup), which would lower the barrier for community contributions and enable hot-reload during development. The trade-off is losing compile-time safety and slightly increasing startup cost. If externalized, patterns should include: language name, kind (function/class/import/namespace), regex string, body style (brace/indent/ruby-end/none), and optional capture group names for visibility and return type. diff --git a/src/CodeIndex/Indexer/AmbiguousMContentMasker.cs b/src/CodeIndex/Indexer/AmbiguousMContentMasker.cs index f1b9b6710..f3989c7e8 100644 --- a/src/CodeIndex/Indexer/AmbiguousMContentMasker.cs +++ b/src/CodeIndex/Indexer/AmbiguousMContentMasker.cs @@ -201,6 +201,19 @@ private static bool LooksLikeObjectiveCModuloOperator(string content, int percen nextIndex++; } + while (nextIndex < content.Length + && content[nextIndex] is not '\r' and not '\n' + && content[nextIndex] is '+' or '-' or '!' or '~' or '*' or '&') + { + nextIndex++; + while (nextIndex < content.Length + && content[nextIndex] is not '\r' and not '\n' + && char.IsWhiteSpace(content[nextIndex])) + { + nextIndex++; + } + } + return nextIndex < content.Length && content[nextIndex] is not '\r' and not '\n' && (char.IsLetterOrDigit(content[nextIndex]) diff --git a/src/CodeIndex/Indexer/References/Languages/ScientificNativeCommentMasker.cs b/src/CodeIndex/Indexer/References/Languages/ScientificNativeCommentMasker.cs index e840ca81d..085b6632d 100644 --- a/src/CodeIndex/Indexer/References/Languages/ScientificNativeCommentMasker.cs +++ b/src/CodeIndex/Indexer/References/Languages/ScientificNativeCommentMasker.cs @@ -7,7 +7,13 @@ internal static string[] MaskBlockComments(string language, string[] lines) => { "d" => MaskDNonCodeRegions(lines), "julia" => MaskNestedBlockComments( - MaskTripleQuotedStrings(lines, "#=", "=#", "#", singleQuoteCanBePostfix: true), + MaskTripleQuotedStrings( + lines, + "#=", + "=#", + "#", + singleQuoteCanBePostfix: true, + tripleQuoteUsesBackslashEscapes: true), "#=", "=#", "#", @@ -83,7 +89,8 @@ private static string[] MaskTripleQuotedStrings( string blockOpening, string blockClosing, string lineComment, - bool singleQuoteCanBePostfix = false) + bool singleQuoteCanBePostfix = false, + bool tripleQuoteUsesBackslashEscapes = false) { if (!MayContain(lines, "\"\"\"")) return lines; @@ -105,7 +112,9 @@ void MaskAt(int index) => { if (inTripleQuotedString) { - if (StartsWith(line, cursor, "\"\"\"")) + if (StartsWith(line, cursor, "\"\"\"") + && (!tripleQuoteUsesBackslashEscapes + || !HasOddBackslashPrefix(line, cursor))) { MaskToken("\"\"\""); inTripleQuotedString = false; @@ -196,6 +205,15 @@ void MaskToken(string token) return result; } + private static bool HasOddBackslashPrefix(string line, int index) + { + var backslashCount = 0; + for (index--; index >= 0 && line[index] == '\\'; index--) + backslashCount++; + + return (backslashCount & 1) != 0; + } + internal static string MaskNimRawStringLiterals(string line) { if (!line.Contains("r\"", StringComparison.Ordinal)) diff --git a/src/CodeIndex/Indexer/References/Languages/ScientificNativeReferenceExtractor.cs b/src/CodeIndex/Indexer/References/Languages/ScientificNativeReferenceExtractor.cs index 1d895beb5..9968f7cd6 100644 --- a/src/CodeIndex/Indexer/References/Languages/ScientificNativeReferenceExtractor.cs +++ b/src/CodeIndex/Indexer/References/Languages/ScientificNativeReferenceExtractor.cs @@ -6,8 +6,6 @@ namespace CodeIndex.Indexer; internal static class ScientificNativeReferenceExtractor { - private const int MaxDependenciesPerDeclaration = 64; - private static readonly HashSet SupportedLanguages = new(StringComparer.Ordinal) { "ada", "cython", "d", "julia", "matlab", "nim", "objc" }; @@ -55,7 +53,7 @@ internal static class ScientificNativeReferenceExtractor RegexOptions.Compiled | RegexOptions.CultureInvariant); private static readonly Regex CythonFromImportRegex = new( - @"^\s*from\s+(?[A-Za-z_]\w*(?:\.[A-Za-z_]\w*)*)\s+cimport\b", + @"^\s*from\s+(?\.*[A-Za-z_]\w*(?:\.[A-Za-z_]\w*)*)\s+cimport\b", RegexOptions.Compiled | RegexOptions.CultureInvariant); private static readonly Regex CythonImportListRegex = new( @"^\s*(?:cimport|import)\s+(?[^\r\n]+)", @@ -103,8 +101,12 @@ internal static void EmitReferences( string context, int lineNumber, Func resolveContainerForColumn, - Action addCallLikeReference) + Action addCallLikeReference, + int maxDependenciesPerDeclaration, + Action? reportDiagnostic) { + var dependencyLimitReported = false; + switch (language) { case "nim": @@ -146,7 +148,7 @@ internal static void EmitReferences( } break; case "cython": - EmitMatch(CythonFromImportRegex, "import"); + EmitMatch(CythonFromImportRegex, "import", stripLeadingRelativePrefix: true); EmitNameList(CythonImportListRegex, "import", ','); EmitNameList(CythonBaseTypeListRegex, "type_reference", ','); break; @@ -167,11 +169,27 @@ internal static void EmitReferences( break; } - void EmitMatch(Regex regex, string referenceKind) + void EmitMatch( + Regex regex, + string referenceKind, + bool stripLeadingRelativePrefix = false) { var match = regex.Match(preparedLine); - if (match.Success) - EmitGroup(match.Groups["name"], referenceKind); + if (!match.Success) + return; + + var group = match.Groups["name"]; + if (!stripLeadingRelativePrefix) + { + EmitGroup(group, referenceKind); + return; + } + + var nameStart = 0; + while (nameStart < group.Length && group.Value[nameStart] == '.') + nameStart++; + if (nameStart < group.Length) + EmitName(group.Value[nameStart..], group.Index + nameStart, referenceKind); } void EmitMatches(Regex regex, string referenceKind) @@ -208,7 +226,7 @@ void EmitNameList( var dependencyCount = 0; var segmentStart = 0; - for (var index = 0; index <= namesEnd && dependencyCount < MaxDependenciesPerDeclaration; index++) + for (var index = 0; index <= namesEnd; index++) { var atEnd = index == namesEnd; var isSeparator = !atEnd @@ -216,6 +234,7 @@ void EmitNameList( if (!atEnd && !isSeparator) continue; + var canEmit = dependencyCount < maxDependenciesPerDeclaration; if (TryEmitDependencySegment( names, segmentStart, @@ -223,8 +242,15 @@ void EmitNameList( group.Index, referenceKind, stripLeadingAlias, - stripLeadingRelativePrefix)) + stripLeadingRelativePrefix, + emit: canEmit)) { + if (!canEmit) + { + ReportDependencyLimit(); + return; + } + dependencyCount++; } @@ -246,7 +272,8 @@ bool TryEmitDependencySegment( int absoluteOffset, string referenceKind, bool stripLeadingAlias, - bool stripLeadingRelativePrefix) + bool stripLeadingRelativePrefix, + bool emit = true) { while (segmentStart < segmentEnd && char.IsWhiteSpace(names[segmentStart])) segmentStart++; @@ -300,10 +327,14 @@ bool TryEmitDependencySegment( var emittedNameStart = stripLeadingRelativePrefix ? firstIdentifierIndex : segmentStart; - EmitName( - names[emittedNameStart..nameEnd], - absoluteOffset + emittedNameStart, - referenceKind); + if (emit) + { + EmitName( + names[emittedNameStart..nameEnd], + absoluteOffset + emittedNameStart, + referenceKind); + } + return true; } @@ -321,7 +352,7 @@ void EmitNimImportList() var dependencyCount = 0; var segmentStart = 0; var bracketDepth = 0; - for (var index = 0; index <= names.Length && dependencyCount < MaxDependenciesPerDeclaration; index++) + for (var index = 0; index <= names.Length; index++) { if (index < names.Length) { @@ -334,17 +365,24 @@ void EmitNimImportList() if (index < names.Length && (names[index] != ',' || bracketDepth != 0)) continue; - dependencyCount += EmitNimImportSegment( + var (emittedCount, truncated) = EmitNimImportSegment( names, segmentStart, index, group.Index, - MaxDependenciesPerDeclaration - dependencyCount); + Math.Max(0, maxDependenciesPerDeclaration - dependencyCount)); + dependencyCount += emittedCount; + if (truncated) + { + ReportDependencyLimit(); + return; + } + segmentStart = index + 1; } } - int EmitNimImportSegment( + (int EmittedCount, bool Truncated) EmitNimImportSegment( string names, int segmentStart, int segmentEnd, @@ -356,7 +394,7 @@ int EmitNimImportSegment( while (segmentEnd > segmentStart && char.IsWhiteSpace(names[segmentEnd - 1])) segmentEnd--; if (segmentStart >= segmentEnd) - return 0; + return (0, false); var openingBracket = names.IndexOf('[', segmentStart, segmentEnd - segmentStart); var closingBracket = openingBracket >= 0 @@ -364,16 +402,19 @@ int EmitNimImportSegment( : -1; if (openingBracket < 0 || closingBracket < 0) { - return TryEmitDependencySegment( + var canEmit = remainingCapacity > 0; + var hasDependency = TryEmitDependencySegment( names, segmentStart, segmentEnd, absoluteOffset, "import", stripLeadingAlias: false, - stripLeadingRelativePrefix: false) - ? 1 - : 0; + stripLeadingRelativePrefix: false, + emit: canEmit); + return hasDependency + ? (canEmit ? 1 : 0, !canEmit) + : (0, false); } var prefixStart = segmentStart; @@ -381,12 +422,12 @@ int EmitNimImportSegment( while (prefixEnd > prefixStart && char.IsWhiteSpace(names[prefixEnd - 1])) prefixEnd--; if (prefixEnd <= prefixStart || names[prefixEnd - 1] != '/') - return 0; + return (0, false); var prefix = names[prefixStart..prefixEnd]; var emittedCount = 0; var itemStart = openingBracket + 1; - for (var index = itemStart; index <= closingBracket && emittedCount < remainingCapacity; index++) + for (var index = itemStart; index <= closingBracket; index++) { if (index < closingBracket && names[index] != ',') continue; @@ -405,6 +446,9 @@ int EmitNimImportSegment( nameEnd--; if (nameEnd > itemStart) { + if (emittedCount >= remainingCapacity) + return (emittedCount, true); + EmitName( prefix + names[itemStart..nameEnd], absoluteOffset + itemStart, @@ -416,7 +460,18 @@ int EmitNimImportSegment( itemStart = index + 1; } - return emittedCount; + return (emittedCount, false); + } + + void ReportDependencyLimit() + { + if (dependencyLimitReported) + return; + + dependencyLimitReported = true; + reportDiagnostic?.Invoke(new ReferenceExtractionDiagnostic( + "reference_scientific_native_dependency_name_budget_exceeded", + $"Scientific/native dependency extraction used the first {maxDependenciesPerDeclaration:N0} names on line {lineNumber:N0} and skipped additional names.")); } void EmitGroup(Group group, string referenceKind) diff --git a/src/CodeIndex/Indexer/References/ReferenceExtractor.CoreExtraction.cs b/src/CodeIndex/Indexer/References/ReferenceExtractor.CoreExtraction.cs index 8cfc22961..9c1c87201 100644 --- a/src/CodeIndex/Indexer/References/ReferenceExtractor.CoreExtraction.cs +++ b/src/CodeIndex/Indexer/References/ReferenceExtractor.CoreExtraction.cs @@ -59,6 +59,9 @@ internal static List ExtractCore(ReferenceExtractionContext req var csharpLinesInsideBlockComment = preparedInput.CSharpLinesInsideBlockComment; var referenceStructuralLines = preparedInput.ReferenceStructuralLines; var preparedLines = preparedInput.PreparedLines; + var scientificNativeDependencyLimit = ScientificNativeReferenceExtractor.Supports(language) + ? GetSafetyLimits().MaxNamesPerLine + : 0; var goImportBlockLines = preparedInput.GoImportBlockLines; var luaReferenceLines = preparedInput.LuaReferenceLines; var luaPreparedLines = preparedInput.LuaPreparedLines; @@ -2548,7 +2551,9 @@ bool TryGetKnownPythonTypeCall(string candidate, out string canonicalName) context, lineNumber, ResolveContainerForCall, - AddCallLikeReference); + AddCallLikeReference, + scientificNativeDependencyLimit, + request.ReportDiagnostic); } foreach (Match match in CallRegex.Matches(preparedLine)) diff --git a/src/CodeIndex/Indexer/References/ReferenceExtractor.cs b/src/CodeIndex/Indexer/References/ReferenceExtractor.cs index 79e51f4d9..ddfea8d56 100644 --- a/src/CodeIndex/Indexer/References/ReferenceExtractor.cs +++ b/src/CodeIndex/Indexer/References/ReferenceExtractor.cs @@ -64,6 +64,7 @@ public static partial class ReferenceExtractor "reference_definition_lookup_line_name_budget_exceeded", "reference_definition_lookup_symbol_budget_exceeded", "reference_enclosing_type_candidate_budget_exceeded", + "reference_scientific_native_dependency_name_budget_exceeded", "reference_swift_property_line_budget_exceeded", "reference_swift_property_line_name_budget_exceeded", "reference_swift_property_symbol_budget_exceeded", diff --git a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.ScientificNative.cs b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.ScientificNative.cs index dfbeaf7d7..62d10a939 100644 --- a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.ScientificNative.cs +++ b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.ScientificNative.cs @@ -21,9 +21,10 @@ private static (int EndLine, int? BodyStartLine, int? BodyEndLine) FindScientifi var tokenRegex = language == "julia" ? JuliaScientificBlockTokenRegex : MatlabScientificBlockTokenRegex; - var depth = 1; + var depth = 0; int? bodyStartLine = null; var declarationColumn = GetFirstNonWhitespaceColumn(scannerLines[startIndex]); + var delimiterFrames = new Stack<(char ClosingDelimiter, int BlockDepth)>(); for (var lineIndex = startIndex; lineIndex < scannerLines.Length; lineIndex++) { @@ -36,6 +37,7 @@ private static (int EndLine, int? BodyStartLine, int? BodyEndLine) FindScientifi if (language == "matlab" && lineIndex > startIndex && depth == 1 + && delimiterFrames.Count == 0 && IsMatlabPeerDeclaration( scannerLines, lineIndex, @@ -51,11 +53,21 @@ private static (int EndLine, int? BodyStartLine, int? BodyEndLine) FindScientifi if (!skipDeclarationToken) bodyStartLine ??= lineIndex + 1; + var delimiterScanIndex = 0; foreach (Match match in matches) { + ScanScientificDelimiterFrames( + code, + delimiterScanIndex, + match.Index, + depth, + delimiterFrames); + delimiterScanIndex = match.Index + match.Length; + if (skipDeclarationToken) { skipDeclarationToken = false; + depth++; continue; } @@ -65,6 +77,12 @@ private static (int EndLine, int? BodyStartLine, int? BodyEndLine) FindScientifi if (keyword.Equals("end", StringComparison.OrdinalIgnoreCase)) { + if (delimiterFrames.TryPeek(out var delimiterFrame) + && depth <= delimiterFrame.BlockDepth) + { + continue; + } + depth--; if (depth == 0) return (lineIndex + 1, bodyStartLine ?? lineIndex + 1, lineIndex + 1); @@ -73,6 +91,13 @@ private static (int EndLine, int? BodyStartLine, int? BodyEndLine) FindScientifi depth++; } + + ScanScientificDelimiterFrames( + code, + delimiterScanIndex, + code.Length, + depth, + delimiterFrames); } return bodyStartLine == null @@ -85,7 +110,14 @@ private static (int EndLine, int? BodyStartLine, int? BodyEndLine) FindJuliaShor int startIndex) { if (!TryGetJuliaShortBlockExpressionStartLine(scannerLines, startIndex, out var blockStartLine)) + { + if (TryFindJuliaShortDelimitedExpressionEnd(scannerLines, startIndex, out var expressionEndLine)) + { + return (expressionEndLine, startIndex + 1, expressionEndLine); + } + return (startIndex + 1, startIndex + 1, startIndex + 1); + } var blockRange = FindScientificEndRange( scannerLines, @@ -136,6 +168,59 @@ private static bool TryGetJuliaShortBlockExpressionStartLine( return false; } + private static bool TryFindJuliaShortDelimitedExpressionEnd( + string[] scannerLines, + int startIndex, + out int expressionEndLine) + { + expressionEndLine = startIndex + 1; + var startLine = scannerLines[startIndex]; + var parameterEnd = startLine.IndexOf(')'); + var assignmentIndex = parameterEnd >= 0 + ? startLine.IndexOf('=', parameterEnd + 1) + : -1; + if (assignmentIndex < 0) + return false; + + var delimiters = new Stack(); + var sawOpeningDelimiter = false; + for (var lineIndex = startIndex; lineIndex < scannerLines.Length; lineIndex++) + { + var line = scannerLines[lineIndex]; + var cursor = lineIndex == startIndex ? assignmentIndex + 1 : 0; + for (; cursor < line.Length; cursor++) + { + var value = line[cursor]; + var closingDelimiter = value switch + { + '(' => ')', + '[' => ']', + '{' => '}', + _ => '\0', + }; + if (closingDelimiter != '\0') + { + delimiters.Push(closingDelimiter); + sawOpeningDelimiter = true; + continue; + } + + if (delimiters.TryPeek(out var expectedClosingDelimiter) + && value == expectedClosingDelimiter) + { + delimiters.Pop(); + if (sawOpeningDelimiter && delimiters.Count == 0) + { + expressionEndLine = lineIndex + 1; + return true; + } + } + } + } + + return false; + } + private static bool IsMatlabPeerDeclaration( string[] scannerLines, int lineIndex, @@ -164,12 +249,22 @@ private static bool IsMatlabPeerDeclaration( private static bool HasMatlabExplicitOuterClosure(string[] scannerLines, int nestedDeclarationLineIndex) { var depth = 1; + var delimiterFrames = new Stack<(char ClosingDelimiter, int BlockDepth)>(); for (var lineIndex = nestedDeclarationLineIndex; lineIndex < scannerLines.Length; lineIndex++) { var code = scannerLines[lineIndex]; var skipNestedDeclaration = lineIndex == nestedDeclarationLineIndex; + var delimiterScanIndex = 0; foreach (Match match in MatlabScientificBlockTokenRegex.Matches(code)) { + ScanScientificDelimiterFrames( + code, + delimiterScanIndex, + match.Index, + depth, + delimiterFrames); + delimiterScanIndex = match.Index + match.Length; + if (skipNestedDeclaration) { skipNestedDeclaration = false; @@ -183,6 +278,12 @@ private static bool HasMatlabExplicitOuterClosure(string[] scannerLines, int nes if (keyword.Equals("end", StringComparison.OrdinalIgnoreCase)) { + if (delimiterFrames.TryPeek(out var delimiterFrame) + && depth <= delimiterFrame.BlockDepth) + { + continue; + } + depth--; if (depth == 0) return true; @@ -192,11 +293,48 @@ private static bool HasMatlabExplicitOuterClosure(string[] scannerLines, int nes depth++; } } + + ScanScientificDelimiterFrames( + code, + delimiterScanIndex, + code.Length, + depth, + delimiterFrames); } return false; } + private static void ScanScientificDelimiterFrames( + string line, + int startIndex, + int endIndex, + int blockDepth, + Stack<(char ClosingDelimiter, int BlockDepth)> delimiterFrames) + { + for (var index = startIndex; index < endIndex; index++) + { + var closingDelimiter = line[index] switch + { + '(' => ')', + '[' => ']', + '{' => '}', + _ => '\0', + }; + if (closingDelimiter != '\0') + { + delimiterFrames.Push((closingDelimiter, blockDepth)); + continue; + } + + if (delimiterFrames.TryPeek(out var delimiterFrame) + && line[index] == delimiterFrame.ClosingDelimiter) + { + delimiterFrames.Pop(); + } + } + } + private static int GetFirstNonWhitespaceColumn(string line) { for (var index = 0; index < line.Length; index++) diff --git a/tests/CodeIndex.Tests/ReferenceExtractorTests.ScientificNative.cs b/tests/CodeIndex.Tests/ReferenceExtractorTests.ScientificNative.cs index 07b30cd7a..8092e7482 100644 --- a/tests/CodeIndex.Tests/ReferenceExtractorTests.ScientificNative.cs +++ b/tests/CodeIndex.Tests/ReferenceExtractorTests.ScientificNative.cs @@ -1,4 +1,5 @@ using CodeIndex.Indexer; +using CodeIndex.Models; namespace CodeIndex.Tests; @@ -580,6 +581,134 @@ function run(value) reference.SymbolName == "helper" && reference.ReferenceKind == "call").ContainerName); } + [Theory] + [InlineData( + "matlab", + """ + function run(A) + value = A(1, ... + end); + helper(); + end + outside(); + """)] + [InlineData( + "julia", + """ + function run(A) + value = A[ + end] + helper() + end + outside() + """)] + public void Extract_ScientificIndexEndOnContinuationLinePreservesFunctionScope_Issue4738( + string language, + string content) + { + var symbols = SymbolExtractor.Extract(1, language, content); + + var references = ReferenceExtractor.Extract(1, language, content, symbols); + + Assert.Equal(5, Assert.Single(symbols, symbol => symbol.Name == "run").EndLine); + Assert.Equal("run", Assert.Single(references, reference => + reference.SymbolName == "helper" && reference.ReferenceKind == "call").ContainerName); + Assert.Null(Assert.Single(references, reference => + reference.SymbolName == "outside" && reference.ReferenceKind == "call").ContainerName); + } + + [Fact] + public void Extract_JuliaMultilineDelimitedShortFunctionKeepsItsCallScope_Issue4738() + { + const string content = """ + run(value) = ( + helper(value) + ) + outside() + """; + var symbols = SymbolExtractor.Extract(1, "julia", content); + + var references = ReferenceExtractor.Extract(1, "julia", content, symbols); + + Assert.Equal(3, Assert.Single(symbols, symbol => symbol.Name == "run").EndLine); + Assert.Equal("run", Assert.Single(references, reference => + reference.SymbolName == "helper" && reference.ReferenceKind == "call").ContainerName); + Assert.Null(Assert.Single(references, reference => + reference.SymbolName == "outside" && reference.ReferenceKind == "call").ContainerName); + } + + [Fact] + public void Extract_JuliaEscapedTripleQuoteStaysInsideMultilineString_Issue4738() + { + const string content = """" + function run() + text = """ + escaped \""" fake() + still literal + """ + helper() + end + """"; + var symbols = SymbolExtractor.Extract(1, "julia", content); + + var references = ReferenceExtractor.Extract(1, "julia", content, symbols); + + Assert.DoesNotContain(references, reference => reference.SymbolName == "fake"); + Assert.Equal("run", Assert.Single(references, reference => + reference.SymbolName == "helper" && reference.ReferenceKind == "call").ContainerName); + } + + [Theory] + [InlineData("from .helpers cimport thing\n", "helpers")] + [InlineData("from ..pkg.helpers cimport thing\n", "pkg.helpers")] + public void Extract_CythonRelativeCimportsNormalizeTheirModuleName_Issue4738( + string content, + string expectedModule) + { + var references = ReferenceExtractor.Extract(1, "cython", content, []); + + Assert.Single(references, reference => + reference.SymbolName == expectedModule && reference.ReferenceKind == "import"); + } + + [Fact] + public void Extract_ScientificDependencyNameLimitReportsOnlyAfterTheSharedBoundary_Issue4738() + { + var previousLimits = ReferenceExtractor.SafetyLimitsForTesting; + ReferenceExtractor.SafetyLimitsForTesting = new ReferenceExtractionSafetyLimits + { + MaxLookupSymbols = 100, + MaxLookupLines = 100, + MaxNamesPerLine = 2, + MaxContainerCandidates = 100, + }; + + try + { + var exact = ReferenceExtractor.ExtractDetailed( + 1, + "ada", + "with Alpha, Beta;\n", + []); + var exceeded = ReferenceExtractor.ExtractDetailed( + 1, + "ada", + "with Alpha, Beta, Gamma;\n", + []); + + Assert.Equal(2, exact.References.Count(reference => reference.ReferenceKind == "import")); + Assert.Equal(2, exceeded.References.Count(reference => reference.ReferenceKind == "import")); + Assert.DoesNotContain(exact.Diagnostics, diagnostic => + diagnostic.Kind == "reference_scientific_native_dependency_name_budget_exceeded"); + Assert.Contains(exceeded.Diagnostics, diagnostic => + diagnostic.Kind == "reference_scientific_native_dependency_name_budget_exceeded"); + } + finally + { + ReferenceExtractor.SafetyLimitsForTesting = previousLimits; + } + } + public static TheoryData ScientificNativeMultilineLiteralCases => new() { { @@ -801,16 +930,23 @@ void run(void) { public void Extract_AmbiguousMPreservesObjectiveCModuloExpressions_Issue4738() { const string content = """ - void run(void) { - int value = left % helper(); + @implementation Widget + - (void)run { + int first = left % helper(); + int second = left % -other(); + int third = left % *pointer(); } + @end """; var symbols = SymbolExtractor.Extract(1, "ambiguous_m", content, "unknown.m"); var references = ReferenceExtractor.Extract(1, "ambiguous_m", content, symbols, "unknown.m"); - Assert.Single(references, reference => - reference.SymbolName == "helper" && reference.ReferenceKind == "call"); + foreach (var name in new[] { "helper", "other", "pointer" }) + { + Assert.Single(references, reference => + reference.SymbolName == name && reference.ReferenceKind == "call"); + } } [Fact] From c0c8d240494e253086e32ae77b88164f65eef9a4 Mon Sep 17 00:00:00 2001 From: Widthdom Date: Fri, 24 Jul 2026 15:26:10 +0900 Subject: [PATCH 08/21] Resolve scientific graph review findings (#4738) --- .../Indexer/AmbiguousMContentMasker.cs | 3 + .../ScientificNativeCommentMasker.cs | 172 +++++++++++++++++- .../ScientificNativeReferenceExtractor.cs | 6 +- .../SymbolExtractor.ScientificNative.cs | 76 +++++++- ...eferenceExtractorTests.ScientificNative.cs | 89 ++++++++- 5 files changed, 333 insertions(+), 13 deletions(-) diff --git a/src/CodeIndex/Indexer/AmbiguousMContentMasker.cs b/src/CodeIndex/Indexer/AmbiguousMContentMasker.cs index f3989c7e8..340f4f863 100644 --- a/src/CodeIndex/Indexer/AmbiguousMContentMasker.cs +++ b/src/CodeIndex/Indexer/AmbiguousMContentMasker.cs @@ -194,6 +194,9 @@ private static bool LooksLikeObjectiveCModuloOperator(string content, int percen } var nextIndex = percentIndex + 1; + if (nextIndex < content.Length && content[nextIndex] == '=') + nextIndex++; + while (nextIndex < content.Length && content[nextIndex] is not '\r' and not '\n' && char.IsWhiteSpace(content[nextIndex])) diff --git a/src/CodeIndex/Indexer/References/Languages/ScientificNativeCommentMasker.cs b/src/CodeIndex/Indexer/References/Languages/ScientificNativeCommentMasker.cs index 085b6632d..7e299ee72 100644 --- a/src/CodeIndex/Indexer/References/Languages/ScientificNativeCommentMasker.cs +++ b/src/CodeIndex/Indexer/References/Languages/ScientificNativeCommentMasker.cs @@ -13,7 +13,8 @@ internal static string[] MaskBlockComments(string language, string[] lines) => "=#", "#", singleQuoteCanBePostfix: true, - tripleQuoteUsesBackslashEscapes: true), + tripleQuoteUsesBackslashEscapes: true, + maskMultilineBacktickStrings: true), "#=", "=#", "#", @@ -90,13 +91,18 @@ private static string[] MaskTripleQuotedStrings( string blockClosing, string lineComment, bool singleQuoteCanBePostfix = false, - bool tripleQuoteUsesBackslashEscapes = false) + bool tripleQuoteUsesBackslashEscapes = false, + bool maskMultilineBacktickStrings = false) { - if (!MayContain(lines, "\"\"\"")) + if (!MayContain(lines, "\"\"\"") + && (!maskMultilineBacktickStrings || !MayContain(lines, "`"))) + { return lines; + } var result = new string[lines.Length]; var inTripleQuotedString = false; + var inBacktickString = false; var blockCommentDepth = 0; for (var lineIndex = 0; lineIndex < lines.Length; lineIndex++) { @@ -110,6 +116,15 @@ void MaskAt(int index) => var cursor = 0; while (cursor < line.Length) { + if (inBacktickString) + { + var current = line[cursor]; + MaskAt(cursor++); + if (current == '`' && !HasOddBackslashPrefix(line, cursor - 1)) + inBacktickString = false; + continue; + } + if (inTripleQuotedString) { if (StartsWith(line, cursor, "\"\"\"") @@ -176,6 +191,13 @@ void MaskAt(int index) => continue; } + if (maskMultilineBacktickStrings && line[cursor] == '`') + { + MaskAt(cursor++); + inBacktickString = true; + continue; + } + if (line[cursor] is '"' or '\'' or '`') { if (line[cursor] == '\'' @@ -262,6 +284,12 @@ private static string[] MaskDNonCodeRegions(string[] lines) var inCStyleBlockComment = false; var inBacktickString = false; string? tokenStringClosing = null; + var tokenStringQuote = '\0'; + var tokenStringQuoteUsesEscapes = false; + var tokenStringInBacktickString = false; + var tokenStringNestedCommentDepth = 0; + var tokenStringInCStyleBlockComment = false; + string? nestedTokenStringClosing = null; var quote = '\0'; var quoteUsesEscapes = false; for (var lineIndex = 0; lineIndex < lines.Length; lineIndex++) @@ -299,11 +327,143 @@ void MaskAt(int index) => if (tokenStringDepth > 0) { - var current = line[cursor]; + if (nestedTokenStringClosing != null) + { + if (StartsWith(line, cursor, nestedTokenStringClosing)) + { + MaskToken(nestedTokenStringClosing); + nestedTokenStringClosing = null; + continue; + } + + MaskAt(cursor++); + continue; + } + + if (tokenStringInBacktickString) + { + var backtickCurrent = line[cursor]; + MaskAt(cursor++); + if (backtickCurrent == '`') + tokenStringInBacktickString = false; + continue; + } + + if (tokenStringNestedCommentDepth > 0) + { + if (StartsWith(line, cursor, "/+")) + { + MaskToken("/+"); + tokenStringNestedCommentDepth++; + continue; + } + + if (StartsWith(line, cursor, "+/")) + { + MaskToken("+/"); + tokenStringNestedCommentDepth--; + continue; + } + + MaskAt(cursor++); + continue; + } + + if (tokenStringInCStyleBlockComment) + { + if (StartsWith(line, cursor, "*/")) + { + MaskToken("*/"); + tokenStringInCStyleBlockComment = false; + continue; + } + + MaskAt(cursor++); + continue; + } + + if (tokenStringQuote != '\0') + { + var quotedCurrent = line[cursor]; + MaskAt(cursor++); + if (tokenStringQuoteUsesEscapes + && quotedCurrent == '\\' + && cursor < line.Length) + { + MaskAt(cursor++); + continue; + } + + if (quotedCurrent == tokenStringQuote) + tokenStringQuote = '\0'; + continue; + } + + if (StartsWith(line, cursor, "//")) + { + while (cursor < line.Length) + MaskAt(cursor++); + break; + } + + if (StartsWith(line, cursor, "/*")) + { + MaskToken("/*"); + tokenStringInCStyleBlockComment = true; + continue; + } + + if (StartsWith(line, cursor, "/+")) + { + MaskToken("/+"); + tokenStringNestedCommentDepth++; + continue; + } + + if (StartsWith(line, cursor, "q\"") + && (cursor == 0 || !IsIdentifierChar(line[cursor - 1])) + && TryGetDTokenStringClosing( + line, + cursor + 2, + out var nestedOpeningLength, + out var nestedClosing)) + { + for (var openingIndex = 0; openingIndex < 2 + nestedOpeningLength; openingIndex++) + MaskAt(cursor++); + nestedTokenStringClosing = nestedClosing; + continue; + } + + if (line[cursor] == '`') + { + MaskAt(cursor++); + tokenStringInBacktickString = true; + continue; + } + + if (StartsWith(line, cursor, "r\"") + && (cursor == 0 || !IsIdentifierChar(line[cursor - 1]))) + { + MaskAt(cursor++); + MaskAt(cursor++); + tokenStringQuote = '"'; + tokenStringQuoteUsesEscapes = false; + continue; + } + + if (line[cursor] is '"' or '\'') + { + tokenStringQuote = line[cursor]; + tokenStringQuoteUsesEscapes = true; + MaskAt(cursor++); + continue; + } + + var structuralCurrent = line[cursor]; MaskAt(cursor++); - if (current == '{') + if (structuralCurrent == '{') tokenStringDepth++; - else if (current == '}') + else if (structuralCurrent == '}') tokenStringDepth--; continue; } diff --git a/src/CodeIndex/Indexer/References/Languages/ScientificNativeReferenceExtractor.cs b/src/CodeIndex/Indexer/References/Languages/ScientificNativeReferenceExtractor.cs index 9968f7cd6..2e39fe83d 100644 --- a/src/CodeIndex/Indexer/References/Languages/ScientificNativeReferenceExtractor.cs +++ b/src/CodeIndex/Indexer/References/Languages/ScientificNativeReferenceExtractor.cs @@ -43,10 +43,10 @@ internal static class ScientificNativeReferenceExtractor RegexOptions.Compiled | RegexOptions.CultureInvariant); private static readonly Regex DImportListRegex = new( - @"^\s*(?:(?:public|static)\s+)*import\s+(?[^;\r\n]+)", + @"^\s*(?:(?:public|private|protected|package|static|export)\s+)*import\s+(?[^;\r\n]+)", RegexOptions.Compiled | RegexOptions.CultureInvariant); private static readonly Regex DBaseTypeListRegex = new( - @"^\s*(?:class|interface)\s+[A-Za-z_]\w*(?:\s*\([^)]*\))?\s*:\s*(?[^{\r\n]+)", + @"^\s*(?:(?:public|private|protected|package|static|abstract|final|extern)\s+)*(?:class|interface)\s+[A-Za-z_]\w*(?:\s*\([^)]*\))?\s*:\s*(?[^{\r\n]+)", RegexOptions.Compiled | RegexOptions.CultureInvariant); private static readonly Regex DTemplateInvocationRegex = new( @"(?[A-Za-z_]\w*)\s*!\s*(?:[A-Za-z_]\w*(?:\.[A-Za-z_]\w*)*|\([^()\r\n]*\))\s*\(", @@ -59,7 +59,7 @@ internal static class ScientificNativeReferenceExtractor @"^\s*(?:cimport|import)\s+(?[^\r\n]+)", RegexOptions.Compiled | RegexOptions.CultureInvariant); private static readonly Regex CythonBaseTypeListRegex = new( - @"^\s*cdef\s+class\s+[A-Za-z_]\w*\s*\(\s*(?[^)\r\n]+)", + @"^\s*(?:cdef\s+)?class\s+[A-Za-z_]\w*\s*\(\s*(?[^)\r\n]+)", RegexOptions.Compiled | RegexOptions.CultureInvariant); private static readonly Regex AdaImportListRegex = new( diff --git a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.ScientificNative.cs b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.ScientificNative.cs index 62d10a939..9312232b7 100644 --- a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.ScientificNative.cs +++ b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.ScientificNative.cs @@ -116,6 +116,11 @@ private static (int EndLine, int? BodyStartLine, int? BodyEndLine) FindJuliaShor return (expressionEndLine, startIndex + 1, expressionEndLine); } + if (TryFindJuliaShortContinuationEnd(scannerLines, startIndex, out expressionEndLine)) + { + return (expressionEndLine, startIndex + 1, expressionEndLine); + } + return (startIndex + 1, startIndex + 1, startIndex + 1); } @@ -182,12 +187,21 @@ private static bool TryFindJuliaShortDelimitedExpressionEnd( if (assignmentIndex < 0) return false; + var expressionStart = assignmentIndex + 1; + while (expressionStart < startLine.Length && char.IsWhiteSpace(startLine[expressionStart])) + expressionStart++; + if (expressionStart >= startLine.Length + || startLine[expressionStart] is not ('(' or '[' or '{')) + { + return false; + } + var delimiters = new Stack(); var sawOpeningDelimiter = false; for (var lineIndex = startIndex; lineIndex < scannerLines.Length; lineIndex++) { var line = scannerLines[lineIndex]; - var cursor = lineIndex == startIndex ? assignmentIndex + 1 : 0; + var cursor = lineIndex == startIndex ? expressionStart : 0; for (; cursor < line.Length; cursor++) { var value = line[cursor]; @@ -221,6 +235,66 @@ private static bool TryFindJuliaShortDelimitedExpressionEnd( return false; } + private static bool TryFindJuliaShortContinuationEnd( + string[] scannerLines, + int startIndex, + out int expressionEndLine) + { + expressionEndLine = startIndex + 1; + var startLine = scannerLines[startIndex]; + var parameterEnd = startLine.IndexOf(')'); + var assignmentIndex = parameterEnd >= 0 + ? startLine.IndexOf('=', parameterEnd + 1) + : -1; + if (assignmentIndex < 0 + || !EndsWithJuliaContinuationOperator(startLine[(assignmentIndex + 1)..])) + { + return false; + } + + var declarationColumn = GetFirstNonWhitespaceColumn(startLine); + for (var lineIndex = startIndex + 1; lineIndex < scannerLines.Length; lineIndex++) + { + var line = scannerLines[lineIndex]; + if (string.IsNullOrWhiteSpace(line)) + continue; + + if (GetFirstNonWhitespaceColumn(line) <= declarationColumn) + return expressionEndLine > startIndex + 1; + + expressionEndLine = lineIndex + 1; + if (!EndsWithJuliaContinuationOperator(line)) + return true; + } + + return expressionEndLine > startIndex + 1; + } + + private static bool EndsWithJuliaContinuationOperator(string expression) + { + var trimmed = expression.AsSpan().TrimEnd(); + if (trimmed.IsEmpty) + return false; + + return trimmed[^1] is '+' + or '-' + or '*' + or '/' + or '\\' + or '^' + or '%' + or '&' + or '|' + or '<' + or '>' + or '=' + or '?' + or ':' + or ',' + or '÷' + or '⊻'; + } + private static bool IsMatlabPeerDeclaration( string[] scannerLines, int lineIndex, diff --git a/tests/CodeIndex.Tests/ReferenceExtractorTests.ScientificNative.cs b/tests/CodeIndex.Tests/ReferenceExtractorTests.ScientificNative.cs index 8092e7482..460fe6f2d 100644 --- a/tests/CodeIndex.Tests/ReferenceExtractorTests.ScientificNative.cs +++ b/tests/CodeIndex.Tests/ReferenceExtractorTests.ScientificNative.cs @@ -737,7 +737,7 @@ proc run() = """ void run() { auto text = q{ - /* tokenOnly(); + /* tokenOnly(); */ }; helper(); } @@ -782,6 +782,68 @@ public void Extract_ScientificNativeLiteralTokensDoNotSuppressFollowingCalls_Iss Assert.DoesNotContain(references, reference => reference.SymbolName == "tokenOnly"); } + [Fact] + public void Extract_JuliaMultilineCommandLiteralDoesNotEmitOrRescopePhantomCode_Issue4738() + { + const string content = """ + function real() + command = `echo + phantomCall() + function Phantom() + ` + helper() + end + """; + var symbols = SymbolExtractor.Extract(1, "julia", content); + + var references = ReferenceExtractor.Extract(1, "julia", content, symbols); + + Assert.DoesNotContain(symbols, symbol => symbol.Name == "Phantom"); + Assert.DoesNotContain(references, reference => reference.SymbolName == "phantomCall"); + Assert.Equal("real", Assert.Single(references, reference => + reference.SymbolName == "helper" && reference.ReferenceKind == "call").ContainerName); + } + + [Fact] + public void Extract_JuliaShortFunctionOperatorContinuationKeepsReferenceContainer_Issue4738() + { + const string content = """ + f(x) = first(x) + + second(x) + """; + var symbol = Assert.Single(SymbolExtractor.Extract(1, "julia", content)); + + var references = ReferenceExtractor.Extract(1, "julia", content, [symbol]); + + Assert.Equal(2, symbol.EndLine); + Assert.Equal("f", Assert.Single(references, reference => + reference.SymbolName == "second" && reference.ReferenceKind == "call").ContainerName); + } + + [Fact] + public void Extract_DTokenStringNestedLiteralsAndCommentsDoNotChangeBraceDepth_Issue4738() + { + const string content = """ + void run() { + enum code = q{ + auto first = "}"; + auto second = q"[}]"; + /* } */ + /+ { } +/ + phantomCall(); + }; + helper(); + } + """; + var symbols = SymbolExtractor.Extract(1, "d", content); + + var references = ReferenceExtractor.Extract(1, "d", content, symbols); + + Assert.DoesNotContain(references, reference => reference.SymbolName == "phantomCall"); + Assert.Equal("run", Assert.Single(references, reference => + reference.SymbolName == "helper" && reference.ReferenceKind == "call").ContainerName); + } + [Fact] public void Extract_DCommentTextCannotOpenTokenStrings_Issue4738() { @@ -855,7 +917,8 @@ procedure Run is public void Extract_DMultipleBaseTypesAndCastSyntaxStayGraphAccurate_Issue4738() { const string content = """ - class Child : Base, IFace { + private import pkg.mod; + public abstract class Child : Base, IFace { void run() { auto value = cast(int)(helper()); } @@ -869,12 +932,30 @@ void run() { reference.SymbolName == "Base" && reference.ReferenceKind == "type_reference"); Assert.Contains(references, reference => reference.SymbolName == "IFace" && reference.ReferenceKind == "type_reference"); + Assert.Contains(references, reference => + reference.SymbolName == "pkg.mod" && reference.ReferenceKind == "import"); Assert.Contains(references, reference => reference.SymbolName == "helper" && reference.ReferenceKind == "call"); Assert.DoesNotContain(references, reference => reference.SymbolName == "cast" && reference.ReferenceKind == "call"); } + [Fact] + public void Extract_CythonRegularClassEmitsBaseTypeReference_Issue4738() + { + const string content = """ + class Child(Base): + def run(self): + helper() + """; + var symbols = SymbolExtractor.Extract(1, "cython", content); + + var references = ReferenceExtractor.Extract(1, "cython", content, symbols); + + Assert.Contains(references, reference => + reference.SymbolName == "Base" && reference.ReferenceKind == "type_reference"); + } + [Fact] public void Extract_AmbiguousMCombinesMatlabAndObjectiveCSymbolsAndReferences_Issue4738() { @@ -935,6 +1016,8 @@ @implementation Widget int first = left % helper(); int second = left % -other(); int third = left % *pointer(); + value %= divisor; + afterAssignment(); } @end """; @@ -942,7 +1025,7 @@ @implementation Widget var symbols = SymbolExtractor.Extract(1, "ambiguous_m", content, "unknown.m"); var references = ReferenceExtractor.Extract(1, "ambiguous_m", content, symbols, "unknown.m"); - foreach (var name in new[] { "helper", "other", "pointer" }) + foreach (var name in new[] { "helper", "other", "pointer", "afterAssignment" }) { Assert.Single(references, reference => reference.SymbolName == name && reference.ReferenceKind == "call"); From fa43a67d47349060786c3ca66c3eb3736bac8882 Mon Sep 17 00:00:00 2001 From: Widthdom Date: Fri, 24 Jul 2026 15:56:01 +0900 Subject: [PATCH 09/21] Close scientific graph adversarial gaps (#4738) --- .../Indexer/AmbiguousMContentMasker.cs | 13 ++++ .../ScientificNativeCommentMasker.cs | 27 ++++++-- .../ScientificNativeReferenceExtractor.cs | 32 +++++---- .../ReferenceExtractor.CoreExtraction.cs | 10 ++- .../SymbolExtractor.ScientificNative.cs | 20 +++--- ...eferenceExtractorTests.ScientificNative.cs | 68 ++++++++++++++++++- 6 files changed, 139 insertions(+), 31 deletions(-) diff --git a/src/CodeIndex/Indexer/AmbiguousMContentMasker.cs b/src/CodeIndex/Indexer/AmbiguousMContentMasker.cs index 340f4f863..ba534ffd9 100644 --- a/src/CodeIndex/Indexer/AmbiguousMContentMasker.cs +++ b/src/CodeIndex/Indexer/AmbiguousMContentMasker.cs @@ -186,6 +186,19 @@ private static bool LooksLikeObjectiveCModuloOperator(string content, int percen previousIndex--; } + if (previousIndex > 0 + && content[previousIndex] is '+' or '-' + && content[previousIndex - 1] == content[previousIndex]) + { + previousIndex -= 2; + while (previousIndex >= 0 + && content[previousIndex] is not '\r' and not '\n' + && char.IsWhiteSpace(content[previousIndex])) + { + previousIndex--; + } + } + if (previousIndex < 0 || content[previousIndex] is '\r' or '\n' || !IsTransposeOperandEnd(content[previousIndex])) diff --git a/src/CodeIndex/Indexer/References/Languages/ScientificNativeCommentMasker.cs b/src/CodeIndex/Indexer/References/Languages/ScientificNativeCommentMasker.cs index 7e299ee72..b0eafd761 100644 --- a/src/CodeIndex/Indexer/References/Languages/ScientificNativeCommentMasker.cs +++ b/src/CodeIndex/Indexer/References/Languages/ScientificNativeCommentMasker.cs @@ -19,11 +19,7 @@ internal static string[] MaskBlockComments(string language, string[] lines) => "=#", "#", singleQuoteCanBePostfix: true), - "nim" => MaskNestedBlockComments( - MaskTripleQuotedStrings(lines, "#[", "]#", "#"), - "#[", - "]#", - "#"), + "nim" => MaskNimNonCodeRegions(lines), "matlab" => MaskMatlabBlockComments(lines), _ => lines, }; @@ -276,6 +272,27 @@ internal static string MaskNimRawStringLiterals(string line) return chars is null ? line : new string(chars); } + private static string[] MaskNimNonCodeRegions(string[] lines) + { + string[]? rawStringMaskedLines = null; + for (var lineIndex = 0; lineIndex < lines.Length; lineIndex++) + { + var maskedLine = MaskNimRawStringLiterals(lines[lineIndex]); + if (ReferenceEquals(maskedLine, lines[lineIndex])) + continue; + + rawStringMaskedLines ??= (string[])lines.Clone(); + rawStringMaskedLines[lineIndex] = maskedLine; + } + + var preparedLines = rawStringMaskedLines ?? lines; + return MaskNestedBlockComments( + MaskTripleQuotedStrings(preparedLines, "#[", "]#", "#"), + "#[", + "]#", + "#"); + } + private static string[] MaskDNonCodeRegions(string[] lines) { var result = new string[lines.Length]; diff --git a/src/CodeIndex/Indexer/References/Languages/ScientificNativeReferenceExtractor.cs b/src/CodeIndex/Indexer/References/Languages/ScientificNativeReferenceExtractor.cs index 2e39fe83d..bc26e86d7 100644 --- a/src/CodeIndex/Indexer/References/Languages/ScientificNativeReferenceExtractor.cs +++ b/src/CodeIndex/Indexer/References/Languages/ScientificNativeReferenceExtractor.cs @@ -6,6 +6,8 @@ namespace CodeIndex.Indexer; internal static class ScientificNativeReferenceExtractor { + internal readonly record struct DTemplateArgumentCallSpan(int Start, int EndExclusive); + private static readonly HashSet SupportedLanguages = new(StringComparer.Ordinal) { "ada", "cython", "d", "julia", "matlab", "nim", "objc" }; @@ -77,22 +79,21 @@ internal static class ScientificNativeReferenceExtractor internal static bool Supports(string language) => SupportedLanguages.Contains(language); - internal static bool IsDTemplateArgumentCall(string line, int callIndex) + internal static bool IsDTemplateArgumentCall( + IReadOnlyList? spans, + ref int spanIndex, + int callIndex) { - foreach (Match match in DTemplateInvocationRegex.Matches(line)) - { - var name = match.Groups["name"]; - if (callIndex >= name.Index + name.Length - && callIndex < match.Index + match.Length) - { - return true; - } - } + if (spans == null) + return false; + + while (spanIndex < spans.Count && callIndex >= spans[spanIndex].EndExclusive) + spanIndex++; - return false; + return spanIndex < spans.Count && callIndex >= spans[spanIndex].Start; } - internal static void EmitReferences( + internal static IReadOnlyList? EmitReferences( string language, string preparedLine, List references, @@ -106,6 +107,7 @@ internal static void EmitReferences( Action? reportDiagnostic) { var dependencyLimitReported = false; + List? dTemplateArgumentCallSpans = null; switch (language) { @@ -145,6 +147,10 @@ internal static void EmitReferences( { var group = match.Groups["name"]; addCallLikeReference(group.Value, group.Index); + (dTemplateArgumentCallSpans ??= []).Add( + new DTemplateArgumentCallSpan( + group.Index + group.Length, + match.Index + match.Length)); } break; case "cython": @@ -169,6 +175,8 @@ internal static void EmitReferences( break; } + return dTemplateArgumentCallSpans; + void EmitMatch( Regex regex, string referenceKind, diff --git a/src/CodeIndex/Indexer/References/ReferenceExtractor.CoreExtraction.cs b/src/CodeIndex/Indexer/References/ReferenceExtractor.CoreExtraction.cs index 47bca094e..c59ee3a6a 100644 --- a/src/CodeIndex/Indexer/References/ReferenceExtractor.CoreExtraction.cs +++ b/src/CodeIndex/Indexer/References/ReferenceExtractor.CoreExtraction.cs @@ -2565,9 +2565,11 @@ bool TryGetKnownPythonTypeCall(string candidate, out string canonicalName) } else { + IReadOnlyList? + dTemplateArgumentCallSpans = null; if (ScientificNativeReferenceExtractor.Supports(language)) { - ScientificNativeReferenceExtractor.EmitReferences( + dTemplateArgumentCallSpans = ScientificNativeReferenceExtractor.EmitReferences( language, preparedLine, references, @@ -2581,6 +2583,7 @@ bool TryGetKnownPythonTypeCall(string candidate, out string canonicalName) request.ReportDiagnostic); } + var dTemplateArgumentCallSpanIndex = 0; foreach (Match match in CallRegex.Matches(preparedLine)) { var name = match.Groups["name"].Value; @@ -2588,7 +2591,10 @@ bool TryGetKnownPythonTypeCall(string candidate, out string canonicalName) if (language == "rust" && RustReferenceExtractor.IsRawIdentifierPrefix(preparedLine, callIndex)) continue; if (language == "d" - && ScientificNativeReferenceExtractor.IsDTemplateArgumentCall(preparedLine, callIndex)) + && ScientificNativeReferenceExtractor.IsDTemplateArgumentCall( + dTemplateArgumentCallSpans, + ref dTemplateArgumentCallSpanIndex, + callIndex)) { continue; } diff --git a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.ScientificNative.cs b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.ScientificNative.cs index 9312232b7..f16573ed5 100644 --- a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.ScientificNative.cs +++ b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.ScientificNative.cs @@ -23,7 +23,6 @@ private static (int EndLine, int? BodyStartLine, int? BodyEndLine) FindScientifi : MatlabScientificBlockTokenRegex; var depth = 0; int? bodyStartLine = null; - var declarationColumn = GetFirstNonWhitespaceColumn(scannerLines[startIndex]); var delimiterFrames = new Stack<(char ClosingDelimiter, int BlockDepth)>(); for (var lineIndex = startIndex; lineIndex < scannerLines.Length; lineIndex++) @@ -42,8 +41,7 @@ private static (int EndLine, int? BodyStartLine, int? BodyEndLine) FindScientifi scannerLines, lineIndex, code, - matches, - declarationColumn)) + matches)) { return bodyStartLine == null ? (lineIndex, null, null) @@ -246,8 +244,14 @@ private static bool TryFindJuliaShortContinuationEnd( var assignmentIndex = parameterEnd >= 0 ? startLine.IndexOf('=', parameterEnd + 1) : -1; - if (assignmentIndex < 0 - || !EndsWithJuliaContinuationOperator(startLine[(assignmentIndex + 1)..])) + if (assignmentIndex < 0) + { + return false; + } + + var initialExpression = startLine[(assignmentIndex + 1)..]; + if (!string.IsNullOrWhiteSpace(initialExpression) + && !EndsWithJuliaContinuationOperator(initialExpression)) { return false; } @@ -299,14 +303,10 @@ private static bool IsMatlabPeerDeclaration( string[] scannerLines, int lineIndex, string code, - MatchCollection matches, - int declarationColumn) + MatchCollection matches) { foreach (Match match in matches) { - if (match.Index > declarationColumn) - return false; - var keyword = match.Groups["keyword"].Value; if (!keyword.Equals("function", StringComparison.OrdinalIgnoreCase) && !keyword.Equals("classdef", StringComparison.OrdinalIgnoreCase)) diff --git a/tests/CodeIndex.Tests/ReferenceExtractorTests.ScientificNative.cs b/tests/CodeIndex.Tests/ReferenceExtractorTests.ScientificNative.cs index 460fe6f2d..5e3a23241 100644 --- a/tests/CodeIndex.Tests/ReferenceExtractorTests.ScientificNative.cs +++ b/tests/CodeIndex.Tests/ReferenceExtractorTests.ScientificNative.cs @@ -357,6 +357,20 @@ void run() { reference.SymbolName is "int" or "string" && reference.ReferenceKind == "call"); } + [Fact] + public void Extract_DManyTemplateInvocationsSuppressArgumentCallsInOnePass_Issue4738() + { + var content = "void run() { " + string.Join(' ', Enumerable.Repeat("helper!Type();", 512)) + " }"; + var symbols = SymbolExtractor.Extract(1, "d", content); + + var references = ReferenceExtractor.Extract(1, "d", content, symbols); + + Assert.Equal(512, references.Count(reference => + reference.SymbolName == "helper" && reference.ReferenceKind == "call")); + Assert.DoesNotContain(references, reference => + reference.SymbolName == "Type" && reference.ReferenceKind == "call"); + } + [Fact] public void Extract_AdaQualifiedBareCallUsesResolvableLeafName_Issue4738() { @@ -520,7 +534,7 @@ public void Extract_MatlabPeerFunctionsWithoutClosingEndHaveSeparateRanges_Issue const string content = """ function first() helper1(); - function second() + function second() helper2(); """; var symbols = SymbolExtractor.Extract(1, "matlab", content); @@ -820,6 +834,25 @@ public void Extract_JuliaShortFunctionOperatorContinuationKeepsReferenceContaine reference.SymbolName == "second" && reference.ReferenceKind == "call").ContainerName); } + [Fact] + public void Extract_JuliaShortFunctionTrailingAssignmentKeepsIndentedRhsContainer_Issue4738() + { + const string content = """ + f(x) = + helper(x) + outside() + """; + var symbol = Assert.Single(SymbolExtractor.Extract(1, "julia", content)); + + var references = ReferenceExtractor.Extract(1, "julia", content, [symbol]); + + Assert.Equal(2, symbol.EndLine); + Assert.Equal("f", Assert.Single(references, reference => + reference.SymbolName == "helper" && reference.ReferenceKind == "call").ContainerName); + Assert.Null(Assert.Single(references, reference => + reference.SymbolName == "outside" && reference.ReferenceKind == "call").ContainerName); + } + [Fact] public void Extract_DTokenStringNestedLiteralsAndCommentsDoNotChangeBraceDepth_Issue4738() { @@ -891,6 +924,27 @@ public void Extract_DAndNimRawStringsUseLiteralBackslashRules_Issue4738( Assert.DoesNotContain(references, reference => reference.SymbolName == "fake"); } + [Fact] + public void Extract_NimRawStringBeforeBlockCommentDoesNotExposeCommentCode_Issue4738() + { + const string content = """ + let text = r"literal\" #[ + proc Phantom() = discard + phantomCall() + ]# + proc real() = + helper() + """; + var symbols = SymbolExtractor.Extract(1, "nim", content); + + var references = ReferenceExtractor.Extract(1, "nim", content, symbols); + + Assert.DoesNotContain(symbols, symbol => symbol.Name == "Phantom"); + Assert.DoesNotContain(references, reference => reference.SymbolName == "phantomCall"); + Assert.Equal("real", Assert.Single(references, reference => + reference.SymbolName == "helper" && reference.ReferenceKind == "call").ContainerName); + } + [Fact] public void Extract_AdaAttributesPreserveNestedCallsWithoutPhantomAttributeCalls_Issue4738() { @@ -1018,6 +1072,8 @@ @implementation Widget int third = left % *pointer(); value %= divisor; afterAssignment(); + int fourth = index++ % postIncrement(); + int fifth = index-- % postDecrement(); } @end """; @@ -1025,7 +1081,15 @@ @implementation Widget var symbols = SymbolExtractor.Extract(1, "ambiguous_m", content, "unknown.m"); var references = ReferenceExtractor.Extract(1, "ambiguous_m", content, symbols, "unknown.m"); - foreach (var name in new[] { "helper", "other", "pointer", "afterAssignment" }) + foreach (var name in new[] + { + "helper", + "other", + "pointer", + "afterAssignment", + "postIncrement", + "postDecrement", + }) { Assert.Single(references, reference => reference.SymbolName == name && reference.ReferenceKind == "call"); From f7ce7d0272b90dc89d3e61d61c3698c9fd2d6114 Mon Sep 17 00:00:00 2001 From: Widthdom Date: Fri, 24 Jul 2026 16:25:59 +0900 Subject: [PATCH 10/21] Resolve scientific graph adversarial findings (#4738) --- .../ScientificNativeCommentMasker.cs | 5 +- .../ScientificNativeReferenceExtractor.cs | 107 +++++++++++++--- .../ReferenceExtractor.CoreExtraction.cs | 1 + .../ReferenceExtractor.TypeReferences.cs | 5 +- .../Indexer/References/ReferenceExtractor.cs | 10 +- .../Symbols/SymbolExtractor.ExtractCore.cs | 6 +- .../SymbolExtractor.ScientificNative.cs | 104 ++++++++------- .../Indexer/Symbols/SymbolExtractor.cs | 12 +- ...eferenceExtractorTests.ScientificNative.cs | 118 ++++++++++++++++++ tests/CodeIndex.Tests/golden/status.json | 2 +- 10 files changed, 295 insertions(+), 75 deletions(-) diff --git a/src/CodeIndex/Indexer/References/Languages/ScientificNativeCommentMasker.cs b/src/CodeIndex/Indexer/References/Languages/ScientificNativeCommentMasker.cs index b0eafd761..6e16942c7 100644 --- a/src/CodeIndex/Indexer/References/Languages/ScientificNativeCommentMasker.cs +++ b/src/CodeIndex/Indexer/References/Languages/ScientificNativeCommentMasker.cs @@ -26,7 +26,8 @@ internal static string[] MaskBlockComments(string language, string[] lines) => internal static string MaskLineStringLiteralsPreservingPostfixSingleQuotes( string line, - bool useMatlabStringRules) + bool useMatlabStringRules, + bool useBackslashEscapes) { char[]? chars = null; @@ -52,7 +53,7 @@ void MaskAt(int index) => { var current = line[cursor]; MaskAt(cursor); - if (!useMatlabStringRules + if (useBackslashEscapes && current == '\\' && cursor + 1 < line.Length) { diff --git a/src/CodeIndex/Indexer/References/Languages/ScientificNativeReferenceExtractor.cs b/src/CodeIndex/Indexer/References/Languages/ScientificNativeReferenceExtractor.cs index bc26e86d7..032c44c78 100644 --- a/src/CodeIndex/Indexer/References/Languages/ScientificNativeReferenceExtractor.cs +++ b/src/CodeIndex/Indexer/References/Languages/ScientificNativeReferenceExtractor.cs @@ -71,11 +71,14 @@ internal static class ScientificNativeReferenceExtractor @"^\s*type\s+[A-Za-z]\w*\s+is\s+new\s+(?[A-Za-z]\w*(?:\.[A-Za-z]\w*)*)", RegexOptions.Compiled | RegexOptions.IgnoreCase | RegexOptions.CultureInvariant); private static readonly Regex AdaBareCallRegex = new( - @"^\s*(?!(?:end|null|return|exit|raise|goto)\b)(?[A-Za-z]\w*(?:\.[A-Za-z]\w*)*)\s*;", + @"(?:^|;)\s*(?!(?:end|null|return|exit|raise|goto)\b)(?[A-Za-z]\w*(?:\.[A-Za-z]\w*)*)\s*;", RegexOptions.Compiled | RegexOptions.IgnoreCase | RegexOptions.CultureInvariant); private static readonly Regex ObjectiveCImportRegex = new( """^\s*#\s*(?:import|include)\s*[<"](?[^>"]+)[>"]""", RegexOptions.Compiled | RegexOptions.CultureInvariant); + private static readonly Regex ObjectiveCImportDirectiveRegex = new( + @"^\s*#\s*(?:import|include)\b", + RegexOptions.Compiled | RegexOptions.CultureInvariant); internal static bool Supports(string language) => SupportedLanguages.Contains(language); @@ -96,6 +99,7 @@ internal static bool IsDTemplateArgumentCall( internal static IReadOnlyList? EmitReferences( string language, string preparedLine, + string originalLine, List references, ReferenceDedupeSet seen, long fileId, @@ -114,12 +118,16 @@ internal static bool IsDTemplateArgumentCall( case "nim": EmitMatch(NimFromImportRegex, "import"); EmitNimImportList(); - EmitMatches(NimBaseTypeRegex, "type_reference"); - EmitMatches(NimAnnotatedTypeRegex, "type_reference"); + EmitMatches(NimBaseTypeRegex, "type_reference", normalizeQualifiedTypeName: true); + EmitMatches(NimAnnotatedTypeRegex, "type_reference", normalizeQualifiedTypeName: true); break; case "matlab": EmitNameList(MatlabImportListRegex, "import", ',', splitOnWhitespace: true); - EmitNameList(MatlabBaseTypeListRegex, "type_reference", '&'); + EmitNameList( + MatlabBaseTypeListRegex, + "type_reference", + '&', + normalizeQualifiedTypeName: true); break; case "julia": EmitNameList( @@ -128,7 +136,7 @@ internal static bool IsDTemplateArgumentCall( ',', stopAtColon: true, stripLeadingRelativePrefix: true); - EmitMatches(JuliaTypeRegex, "type_reference"); + EmitMatches(JuliaTypeRegex, "type_reference", normalizeQualifiedTypeName: true); foreach (Match match in JuliaMacroCallRegex.Matches(preparedLine)) { var group = match.Groups["name"]; @@ -142,7 +150,11 @@ internal static bool IsDTemplateArgumentCall( break; case "d": EmitNameList(DImportListRegex, "import", ',', stopAtColon: true, stripLeadingAlias: true); - EmitNameList(DBaseTypeListRegex, "type_reference", ','); + EmitNameList( + DBaseTypeListRegex, + "type_reference", + ',', + normalizeQualifiedTypeName: true); foreach (Match match in DTemplateInvocationRegex.Matches(preparedLine)) { var group = match.Groups["name"]; @@ -156,13 +168,16 @@ internal static bool IsDTemplateArgumentCall( case "cython": EmitMatch(CythonFromImportRegex, "import", stripLeadingRelativePrefix: true); EmitNameList(CythonImportListRegex, "import", ','); - EmitNameList(CythonBaseTypeListRegex, "type_reference", ','); + EmitNameList( + CythonBaseTypeListRegex, + "type_reference", + ',', + normalizeQualifiedTypeName: true); break; case "ada": EmitNameList(AdaImportListRegex, "import", ','); - EmitMatches(AdaDerivedTypeRegex, "type_reference"); - var bareCall = AdaBareCallRegex.Match(preparedLine); - if (bareCall.Success) + EmitMatches(AdaDerivedTypeRegex, "type_reference", normalizeQualifiedTypeName: true); + foreach (Match bareCall in AdaBareCallRegex.Matches(preparedLine)) { var group = bareCall.Groups["name"]; var separatorIndex = group.Value.LastIndexOf('.'); @@ -171,7 +186,7 @@ internal static bool IsDTemplateArgumentCall( } break; case "objc": - EmitMatch(ObjectiveCImportRegex, "import"); + EmitObjectiveCImport(); break; } @@ -200,10 +215,13 @@ void EmitMatch( EmitName(group.Value[nameStart..], group.Index + nameStart, referenceKind); } - void EmitMatches(Regex regex, string referenceKind) + void EmitMatches( + Regex regex, + string referenceKind, + bool normalizeQualifiedTypeName = false) { foreach (Match match in regex.Matches(preparedLine)) - EmitGroup(match.Groups["name"], referenceKind); + EmitGroup(match.Groups["name"], referenceKind, normalizeQualifiedTypeName); } void EmitNameList( @@ -213,7 +231,8 @@ void EmitNameList( bool splitOnWhitespace = false, bool stopAtColon = false, bool stripLeadingAlias = false, - bool stripLeadingRelativePrefix = false) + bool stripLeadingRelativePrefix = false, + bool normalizeQualifiedTypeName = false) { var match = regex.Match(preparedLine); if (!match.Success) @@ -251,6 +270,7 @@ void EmitNameList( referenceKind, stripLeadingAlias, stripLeadingRelativePrefix, + normalizeQualifiedTypeName, emit: canEmit)) { if (!canEmit) @@ -281,6 +301,7 @@ bool TryEmitDependencySegment( string referenceKind, bool stripLeadingAlias, bool stripLeadingRelativePrefix, + bool normalizeQualifiedTypeName = false, bool emit = true) { while (segmentStart < segmentEnd && char.IsWhiteSpace(names[segmentStart])) @@ -335,12 +356,26 @@ bool TryEmitDependencySegment( var emittedNameStart = stripLeadingRelativePrefix ? firstIdentifierIndex : segmentStart; + string? targetQualifier = null; + if (normalizeQualifiedTypeName) + { + var lastDotIndex = names.LastIndexOf( + '.', + nameEnd - 1, + nameEnd - emittedNameStart); + if (lastDotIndex >= emittedNameStart) + { + targetQualifier = names[emittedNameStart..lastDotIndex]; + emittedNameStart = lastDotIndex + 1; + } + } if (emit) { EmitName( names[emittedNameStart..nameEnd], absoluteOffset + emittedNameStart, - referenceKind); + referenceKind, + targetQualifier); } return true; @@ -482,15 +517,33 @@ void ReportDependencyLimit() $"Scientific/native dependency extraction used the first {maxDependenciesPerDeclaration:N0} names on line {lineNumber:N0} and skipped additional names.")); } - void EmitGroup(Group group, string referenceKind) + void EmitGroup( + Group group, + string referenceKind, + bool normalizeQualifiedTypeName = false) { if (!group.Success || group.Length == 0) return; - EmitName(group.Value, group.Index, referenceKind); + if (!normalizeQualifiedTypeName) + { + EmitName(group.Value, group.Index, referenceKind); + return; + } + + var lastDotIndex = group.Value.LastIndexOf('.'); + EmitName( + lastDotIndex >= 0 ? group.Value[(lastDotIndex + 1)..] : group.Value, + group.Index + lastDotIndex + 1, + referenceKind, + lastDotIndex >= 0 ? group.Value[..lastDotIndex] : null); } - void EmitName(string name, int index, string referenceKind) + void EmitName( + string name, + int index, + string referenceKind, + string? targetQualifier = null) { ReferenceExtractor.AddReference( references, @@ -502,7 +555,23 @@ void EmitName(string name, int index, string referenceKind) context, lineNumber, resolveContainerForColumn(index), - language); + language, + targetQualifier); + } + + void EmitObjectiveCImport() + { + var directiveLine = ObjectiveCImportRegex.IsMatch(preparedLine) + ? preparedLine + : ObjectiveCImportDirectiveRegex.IsMatch(preparedLine) + ? originalLine + : null; + if (directiveLine == null) + return; + + var match = ObjectiveCImportRegex.Match(directiveLine); + if (match.Success) + EmitGroup(match.Groups["name"], "import"); } } diff --git a/src/CodeIndex/Indexer/References/ReferenceExtractor.CoreExtraction.cs b/src/CodeIndex/Indexer/References/ReferenceExtractor.CoreExtraction.cs index c59ee3a6a..38171bc46 100644 --- a/src/CodeIndex/Indexer/References/ReferenceExtractor.CoreExtraction.cs +++ b/src/CodeIndex/Indexer/References/ReferenceExtractor.CoreExtraction.cs @@ -2572,6 +2572,7 @@ bool TryGetKnownPythonTypeCall(string candidate, out string canonicalName) dTemplateArgumentCallSpans = ScientificNativeReferenceExtractor.EmitReferences( language, preparedLine, + originalLine, references, seen, fileId, diff --git a/src/CodeIndex/Indexer/References/ReferenceExtractor.TypeReferences.cs b/src/CodeIndex/Indexer/References/ReferenceExtractor.TypeReferences.cs index 7e4401ef3..2bd14e7c1 100644 --- a/src/CodeIndex/Indexer/References/ReferenceExtractor.TypeReferences.cs +++ b/src/CodeIndex/Indexer/References/ReferenceExtractor.TypeReferences.cs @@ -2717,6 +2717,7 @@ private readonly record struct ReferenceLinePrepareOptions( bool IncludeBacktickStringDelimiter, bool PreservePostfixSingleQuotes, bool UseMatlabStringRules, + bool ScientificStringUsesBackslashEscapes, bool UsesHashComments, bool UsesRHashComments, bool UsesSlashComments, @@ -2735,6 +2736,7 @@ private static ReferenceLinePrepareOptions CreateReferenceLinePrepareOptions(str IncludeBacktickStringDelimiter: lang is not ("kotlin" or "r"), PreservePostfixSingleQuotes: lang is "ada" or "julia" or "matlab", UseMatlabStringRules: lang == "matlab", + ScientificStringUsesBackslashEscapes: lang == "julia", UsesHashComments: UsesHashComments(lang), UsesRHashComments: lang == "r", UsesSlashComments: UsesSlashComments(lang), @@ -2766,7 +2768,8 @@ private static string PrepareLine(string line, ReferenceLinePrepareOptions optio { result = ScientificNativeCommentMasker.MaskLineStringLiteralsPreservingPostfixSingleQuotes( result, - options.UseMatlabStringRules); + options.UseMatlabStringRules, + options.ScientificStringUsesBackslashEscapes); } else { diff --git a/src/CodeIndex/Indexer/References/ReferenceExtractor.cs b/src/CodeIndex/Indexer/References/ReferenceExtractor.cs index c792806e1..0c3c8c0c9 100644 --- a/src/CodeIndex/Indexer/References/ReferenceExtractor.cs +++ b/src/CodeIndex/Indexer/References/ReferenceExtractor.cs @@ -1960,7 +1960,8 @@ internal static void AddReference( string context, int lineNumber, SymbolRecord? container, - string? language = null) + string? language = null, + string? targetQualifier = null) { AddReference( references, @@ -1972,7 +1973,8 @@ internal static void AddReference( context, lineNumber, container, - language); + language, + targetQualifier); } internal static void AddReference( @@ -1985,7 +1987,8 @@ internal static void AddReference( string context, int lineNumber, SymbolRecord? container, - string? language = null) + string? language = null, + string? targetQualifier = null) { var column = nameIndex + 1; var dedupeKey = CreateReferenceDedupeKey(fileId, language, lineNumber, column, referenceKind, name, container); @@ -2002,6 +2005,7 @@ internal static void AddReference( Context = context, ContainerKind = container?.Kind, ContainerName = container?.Name, + TargetQualifier = targetQualifier, IsSelfReference = IsSameReferenceName(container?.Name, name), }); } diff --git a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.ExtractCore.cs b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.ExtractCore.cs index 30fe5af24..702485709 100644 --- a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.ExtractCore.cs +++ b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.ExtractCore.cs @@ -187,6 +187,9 @@ private static List ExtractCore( var scientificBodyScannerLines = lang is "julia" or "matlab" ? PrepareScientificBodyScannerLines(structuralLines, lang) : null; + var matlabExplicitOuterClosureByLine = lang == "matlab" && scientificBodyScannerLines != null + ? BuildMatlabExplicitOuterClosureMap(scientificBodyScannerLines) + : null; string[]? javaScriptTypeScriptSanitizedLines = null; string[] GetJavaScriptTypeScriptSanitizedLines() => javaScriptTypeScriptSanitizedLines ??= BuildJavaScriptTypeScriptSanitizedLines(lines); @@ -897,7 +900,8 @@ bool[] GetCssQualifiedRuleAncestors() => pattern.BodyStyle, lang, absoluteStartColumn, - scientificBodyScannerLines); + scientificBodyScannerLines, + matlabExplicitOuterClosureByLine); if (fortranContinuationCandidate != null) endLine = Math.Max(endLine, fortranContinuationCandidate.Value.LastConsumedLineIndex + 1); var startLine = i + 1; diff --git a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.ScientificNative.cs b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.ScientificNative.cs index f16573ed5..c569babcb 100644 --- a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.ScientificNative.cs +++ b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.ScientificNative.cs @@ -16,7 +16,8 @@ private static (int EndLine, int? BodyStartLine, int? BodyEndLine) FindScientifi string[] scannerLines, int startIndex, string language, - int? openingTokenLineIndex = null) + int? openingTokenLineIndex = null, + bool[]? matlabExplicitOuterClosureByLine = null) { var tokenRegex = language == "julia" ? JuliaScientificBlockTokenRegex @@ -38,10 +39,10 @@ private static (int EndLine, int? BodyStartLine, int? BodyEndLine) FindScientifi && depth == 1 && delimiterFrames.Count == 0 && IsMatlabPeerDeclaration( - scannerLines, lineIndex, code, - matches)) + matches, + matlabExplicitOuterClosureByLine)) { return bodyStartLine == null ? (lineIndex, null, null) @@ -185,24 +186,18 @@ private static bool TryFindJuliaShortDelimitedExpressionEnd( if (assignmentIndex < 0) return false; - var expressionStart = assignmentIndex + 1; - while (expressionStart < startLine.Length && char.IsWhiteSpace(startLine[expressionStart])) - expressionStart++; - if (expressionStart >= startLine.Length - || startLine[expressionStart] is not ('(' or '[' or '{')) - { - return false; - } - var delimiters = new Stack(); - var sawOpeningDelimiter = false; + var sawExpressionContent = false; + var sawUnclosedDelimiter = false; for (var lineIndex = startIndex; lineIndex < scannerLines.Length; lineIndex++) { var line = scannerLines[lineIndex]; - var cursor = lineIndex == startIndex ? expressionStart : 0; + var cursor = lineIndex == startIndex ? assignmentIndex + 1 : 0; for (; cursor < line.Length; cursor++) { var value = line[cursor]; + if (!char.IsWhiteSpace(value)) + sawExpressionContent = true; var closingDelimiter = value switch { '(' => ')', @@ -213,7 +208,6 @@ private static bool TryFindJuliaShortDelimitedExpressionEnd( if (closingDelimiter != '\0') { delimiters.Push(closingDelimiter); - sawOpeningDelimiter = true; continue; } @@ -221,13 +215,23 @@ private static bool TryFindJuliaShortDelimitedExpressionEnd( && value == expectedClosingDelimiter) { delimiters.Pop(); - if (sawOpeningDelimiter && delimiters.Count == 0) - { - expressionEndLine = lineIndex + 1; - return true; - } } } + + if (delimiters.Count > 0) + { + sawUnclosedDelimiter = true; + continue; + } + + if (sawUnclosedDelimiter) + { + expressionEndLine = lineIndex + 1; + return true; + } + + if (sawExpressionContent) + return false; } return false; @@ -300,10 +304,10 @@ private static bool EndsWithJuliaContinuationOperator(string expression) } private static bool IsMatlabPeerDeclaration( - string[] scannerLines, int lineIndex, string code, - MatchCollection matches) + MatchCollection matches, + bool[]? matlabExplicitOuterClosureByLine) { foreach (Match match in matches) { @@ -314,20 +318,23 @@ private static bool IsMatlabPeerDeclaration( return false; } - return !HasMatlabExplicitOuterClosure(scannerLines, lineIndex); + return matlabExplicitOuterClosureByLine == null + || lineIndex >= matlabExplicitOuterClosureByLine.Length + || !matlabExplicitOuterClosureByLine[lineIndex]; } return false; } - private static bool HasMatlabExplicitOuterClosure(string[] scannerLines, int nestedDeclarationLineIndex) + private static bool[] BuildMatlabExplicitOuterClosureMap(string[] scannerLines) { - var depth = 1; + var result = new bool[scannerLines.Length]; + var tokens = new List<(int LineIndex, int BalanceAfter, bool IsDeclaration)>(); + var balance = 0; var delimiterFrames = new Stack<(char ClosingDelimiter, int BlockDepth)>(); - for (var lineIndex = nestedDeclarationLineIndex; lineIndex < scannerLines.Length; lineIndex++) + for (var lineIndex = 0; lineIndex < scannerLines.Length; lineIndex++) { var code = scannerLines[lineIndex]; - var skipNestedDeclaration = lineIndex == nestedDeclarationLineIndex; var delimiterScanIndex = 0; foreach (Match match in MatlabScientificBlockTokenRegex.Matches(code)) { @@ -335,17 +342,10 @@ private static bool HasMatlabExplicitOuterClosure(string[] scannerLines, int nes code, delimiterScanIndex, match.Index, - depth, + balance, delimiterFrames); delimiterScanIndex = match.Index + match.Length; - if (skipNestedDeclaration) - { - skipNestedDeclaration = false; - depth++; - continue; - } - var keyword = match.Groups["keyword"].Value; if (!IsScientificBlockTokenAtStatementBoundary(code, match.Index, keyword, "matlab")) continue; @@ -353,30 +353,47 @@ private static bool HasMatlabExplicitOuterClosure(string[] scannerLines, int nes if (keyword.Equals("end", StringComparison.OrdinalIgnoreCase)) { if (delimiterFrames.TryPeek(out var delimiterFrame) - && depth <= delimiterFrame.BlockDepth) + && balance <= delimiterFrame.BlockDepth) { continue; } - depth--; - if (depth == 0) - return true; + balance--; } else { - depth++; + balance++; } + + tokens.Add(( + lineIndex, + balance, + keyword.Equals("function", StringComparison.OrdinalIgnoreCase) + || keyword.Equals("classdef", StringComparison.OrdinalIgnoreCase))); } ScanScientificDelimiterFrames( code, delimiterScanIndex, code.Length, - depth, + balance, delimiterFrames); } - return false; + var suffixMinimumBalance = int.MaxValue; + for (var tokenIndex = tokens.Count - 1; tokenIndex >= 0; tokenIndex--) + { + var token = tokens[tokenIndex]; + if (token.IsDeclaration) + { + result[token.LineIndex] = + suffixMinimumBalance <= token.BalanceAfter - 2; + } + + suffixMinimumBalance = Math.Min(suffixMinimumBalance, token.BalanceAfter); + } + + return result; } private static void ScanScientificDelimiterFrames( @@ -436,7 +453,8 @@ private static string MaskScientificBodyScanLine(string line, string language) { var masked = ScientificNativeCommentMasker.MaskLineStringLiteralsPreservingPostfixSingleQuotes( line, - useMatlabStringRules: language == "matlab"); + useMatlabStringRules: language == "matlab", + useBackslashEscapes: language == "julia"); var commentMarker = language == "julia" ? '#' : '%'; var commentIndex = masked.IndexOf(commentMarker); return commentIndex >= 0 ? masked[..commentIndex] : masked; diff --git a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.cs b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.cs index 1b374455f..c3d9631ed 100644 --- a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.cs +++ b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.cs @@ -4788,7 +4788,7 @@ private static bool HasRemainingSameLineSignatureOccurrence(SymbolRecord symbol, private static (int EndLine, int? BodyStartLine, int? BodyEndLine) ResolveRange(string[] lines, int startIndex, BodyStyle bodyStyle) => - ResolveRange(lines, startIndex, bodyStyle, null, 0, null); + ResolveRange(lines, startIndex, bodyStyle, null, 0, null, null); private static (int EndLine, int? BodyStartLine, int? BodyEndLine) ResolveRange( string[] lines, @@ -4796,7 +4796,8 @@ private static (int EndLine, int? BodyStartLine, int? BodyEndLine) ResolveRange( BodyStyle bodyStyle, string? lang = null, int startColumn = 0, - string[]? scientificBodyScannerLines = null) + string[]? scientificBodyScannerLines = null, + bool[]? matlabExplicitOuterClosureByLine = null) { return bodyStyle switch { @@ -4810,9 +4811,10 @@ private static (int EndLine, int? BodyStartLine, int? BodyEndLine) ResolveRange( BodyStyle.FortranEnd => FindFortranRange(lines, startIndex), BodyStyle.ElixirEnd => FindElixirRange(lines, startIndex), BodyStyle.ScientificEnd when lang is "julia" or "matlab" => FindScientificEndRange( - scientificBodyScannerLines ?? PrepareScientificBodyScannerLines(lines, lang), - startIndex, - lang), + scientificBodyScannerLines ?? PrepareScientificBodyScannerLines(lines, lang), + startIndex, + lang, + matlabExplicitOuterClosureByLine: matlabExplicitOuterClosureByLine), BodyStyle.JuliaShortFunction when lang == "julia" => FindJuliaShortFunctionRange( scientificBodyScannerLines ?? PrepareScientificBodyScannerLines(lines, lang), startIndex), diff --git a/tests/CodeIndex.Tests/ReferenceExtractorTests.ScientificNative.cs b/tests/CodeIndex.Tests/ReferenceExtractorTests.ScientificNative.cs index 5e3a23241..c37aef225 100644 --- a/tests/CodeIndex.Tests/ReferenceExtractorTests.ScientificNative.cs +++ b/tests/CodeIndex.Tests/ReferenceExtractorTests.ScientificNative.cs @@ -1126,4 +1126,122 @@ public void Extract_AmbiguousMRetainsSharedSafetyGuards_Issue4738() Assert.Empty(oversizeReferences); Assert.Empty(conflictReferences); } + + [Theory] + [InlineData("nim", "type Child = object of pkg.Base\n", "pkg")] + [InlineData("matlab", "classdef Child < pkg.Base\nend\n", "pkg")] + [InlineData("julia", "struct Child <: Pkg.Base\nend\n", "Pkg")] + [InlineData("d", "class Child : pkg.Base {}\n", "pkg")] + [InlineData("cython", "cdef class Child(pkg.Base):\n pass\n", "pkg")] + [InlineData("ada", "type Child is new Pkg.Base;\n", "Pkg")] + public void Extract_QualifiedScientificBaseTypesUseResolvableLeafNames_Issue4738( + string language, + string content, + string expectedQualifier) + { + var symbols = SymbolExtractor.Extract(1, language, content); + + var reference = Assert.Single( + ReferenceExtractor.Extract(1, language, content, symbols), + candidate => candidate.ReferenceKind == "type_reference"); + + Assert.Equal("Base", reference.SymbolName); + Assert.Equal(expectedQualifier, reference.TargetQualifier); + } + + [Theory] + [InlineData("objc")] + [InlineData("ambiguous_m")] + public void Extract_ObjectiveCQuotedImportsSurviveStringMasking_Issue4738(string language) + { + const string content = """ + /* + #import "Ignored.h" + */ + #import "Local/Header.h" + #include + """; + var symbols = SymbolExtractor.Extract(1, language, content, "sample.m"); + + var references = ReferenceExtractor.Extract(1, language, content, symbols, "sample.m"); + + Assert.Single(references, reference => + reference.SymbolName == "Local/Header.h" && reference.ReferenceKind == "import"); + Assert.Single(references, reference => + reference.SymbolName == "Framework/Header.h" && reference.ReferenceKind == "import"); + Assert.DoesNotContain(references, reference => reference.SymbolName == "Ignored.h"); + } + + [Fact] + public void Extract_AdaBackslashesDoNotEscapeClosingStringQuotes_Issue4738() + { + const string content = """ + procedure Run is + begin + Put("C:\"); Helper; + end Run; + """; + var symbols = SymbolExtractor.Extract(1, "ada", content); + + var references = ReferenceExtractor.Extract(1, "ada", content, symbols); + + Assert.Single(references, reference => + reference.SymbolName == "Helper" + && reference.ReferenceKind == "call" + && reference.ContainerName == "Run"); + } + + [Theory] + [InlineData( + """ + f(x) = helper( + nested(x) + ) + outside() + """)] + [InlineData( + """ + f(x) = + helper( + nested(x) + ) + outside() + """)] + public void Extract_JuliaShortFunctionMultilineCallsKeepNestedReferencesScoped_Issue4738( + string content) + { + var symbols = SymbolExtractor.Extract(1, "julia", content); + + var references = ReferenceExtractor.Extract(1, "julia", content, symbols); + + Assert.Equal("f", Assert.Single(references, reference => + reference.SymbolName == "helper" && reference.ReferenceKind == "call").ContainerName); + Assert.Equal("f", Assert.Single(references, reference => + reference.SymbolName == "nested" && reference.ReferenceKind == "call").ContainerName); + Assert.Null(Assert.Single(references, reference => + reference.SymbolName == "outside" && reference.ReferenceKind == "call").ContainerName); + } + + [Fact] + public void Extract_MatlabManyEndlessPeerFunctionsKeepIndependentLinearRanges_Issue4738() + { + const int functionCount = 2_048; + var content = new System.Text.StringBuilder(functionCount * 40); + for (var index = 0; index < functionCount; index++) + { + content.Append("function f") + .Append(index) + .Append("()\n helper") + .Append(index) + .Append("();\n"); + } + + var symbols = SymbolExtractor.Extract(1, "matlab", content.ToString()); + + Assert.Equal(functionCount, symbols.Count(symbol => symbol.Kind == "function")); + Assert.Equal(2, Assert.Single(symbols, symbol => symbol.Name == "f0").EndLine); + Assert.Equal(functionCount * 2 + 1, Assert.Single( + symbols, + symbol => symbol.Name == $"f{functionCount - 1}").EndLine); + } } diff --git a/tests/CodeIndex.Tests/golden/status.json b/tests/CodeIndex.Tests/golden/status.json index 8a08ca2f7..541e0dcad 100644 --- a/tests/CodeIndex.Tests/golden/status.json +++ b/tests/CodeIndex.Tests/golden/status.json @@ -58,9 +58,9 @@ "csharp", "cshtml", "css", + "cuda", "cython", "d", - "cuda", "dart", "dependency_lock", "dependency_manifest", From 1cd43349d92a0becfdf91ae2390a39477be1b32d Mon Sep 17 00:00:00 2001 From: Widthdom Date: Fri, 24 Jul 2026 16:55:37 +0900 Subject: [PATCH 11/21] Resolve scientific graph final review findings (#4738) --- src/CodeIndex/Database/DbWriter.References.cs | 18 +++++--- .../ScientificNativeCommentMasker.cs | 9 ++++ .../ScientificNativeReferenceExtractor.cs | 22 ++++++++++ tests/CodeIndex.Tests/DatabaseTests.cs | 16 +++++-- ...eferenceExtractorTests.ScientificNative.cs | 43 +++++++++++++++++++ 5 files changed, 99 insertions(+), 9 deletions(-) diff --git a/src/CodeIndex/Database/DbWriter.References.cs b/src/CodeIndex/Database/DbWriter.References.cs index cc19a36f4..d11fe4d1b 100644 --- a/src/CodeIndex/Database/DbWriter.References.cs +++ b/src/CodeIndex/Database/DbWriter.References.cs @@ -143,7 +143,8 @@ THEN r.symbol_name_folded || 'attribute' END ) JOIN files AS target_file ON target_file.id = s.file_id WHERE ( - source_file.lang = target_file.lang + (source_file.lang = target_file.lang + AND (source_file.lang <> 'ambiguous_m' OR source_file.id = target_file.id)) OR (source_file.lang = 'ambiguous_m' AND target_file.lang IN ('matlab', 'objc')) ) AND r.target_qualifier IS NOT NULL @@ -199,7 +200,8 @@ THEN r.symbol_name_folded || 'attribute' END JOIN files AS target_file ON target_file.id = s.file_id JOIN symbols AS source ON source.id = r.source_symbol_id WHERE ( - source_file.lang = target_file.lang + (source_file.lang = target_file.lang + AND (source_file.lang <> 'ambiguous_m' OR source_file.id = target_file.id)) OR (source_file.lang = 'ambiguous_m' AND target_file.lang IN ('matlab', 'objc')) ) AND r.target_qualifier IS NULL @@ -228,7 +230,8 @@ THEN r.symbol_name_folded || 'attribute' END JOIN files AS target_file ON target_file.id = s.file_id JOIN symbols AS source ON source.id = r.source_symbol_id WHERE ( - source_file.lang = target_file.lang + (source_file.lang = target_file.lang + AND (source_file.lang <> 'ambiguous_m' OR source_file.id = target_file.id)) OR (source_file.lang = 'ambiguous_m' AND target_file.lang IN ('matlab', 'objc')) ) AND r.target_qualifier IS NULL @@ -252,7 +255,8 @@ THEN r.symbol_name_folded || 'attribute' END ) JOIN files AS target_file ON target_file.id = s.file_id WHERE ( - source_file.lang = target_file.lang + (source_file.lang = target_file.lang + AND (source_file.lang <> 'ambiguous_m' OR source_file.id = target_file.id)) OR (source_file.lang = 'ambiguous_m' AND target_file.lang IN ('matlab', 'objc')) ) AND r.target_qualifier IS NULL @@ -275,7 +279,8 @@ THEN r.symbol_name_folded || 'attribute' END JOIN files AS target_file ON target_file.id = s.file_id JOIN symbols AS source ON source.id = r.source_symbol_id WHERE ( - source_file.lang = target_file.lang + (source_file.lang = target_file.lang + AND (source_file.lang <> 'ambiguous_m' OR source_file.id = target_file.id)) OR (source_file.lang = 'ambiguous_m' AND target_file.lang IN ('matlab', 'objc')) ) AND r.target_qualifier IS NULL @@ -293,7 +298,8 @@ FROM symbol_references AS r JOIN files AS source_file ON source_file.id = r.file_id JOIN temp.reference_unique_symbol_families AS unique_family ON ( - unique_family.lang = source_file.lang + (unique_family.lang = source_file.lang + AND source_file.lang <> 'ambiguous_m') OR (source_file.lang = 'ambiguous_m' AND unique_family.lang IN ('matlab', 'objc')) ) AND unique_family.name_folded = r.symbol_name_folded diff --git a/src/CodeIndex/Indexer/References/Languages/ScientificNativeCommentMasker.cs b/src/CodeIndex/Indexer/References/Languages/ScientificNativeCommentMasker.cs index 6e16942c7..d240d08c2 100644 --- a/src/CodeIndex/Indexer/References/Languages/ScientificNativeCommentMasker.cs +++ b/src/CodeIndex/Indexer/References/Languages/ScientificNativeCommentMasker.cs @@ -249,6 +249,15 @@ internal static string MaskNimRawStringLiterals(string line) } chars ??= line.ToCharArray(); + if (cursor + 3 < line.Length + && line[cursor + 2] == '"' + && line[cursor + 3] == '"') + { + // Leave the triple-quote delimiter for the stateful multiline masker. + chars[cursor] = ' '; + continue; + } + chars[cursor++] = ' '; chars[cursor++] = ' '; while (cursor < line.Length) diff --git a/src/CodeIndex/Indexer/References/Languages/ScientificNativeReferenceExtractor.cs b/src/CodeIndex/Indexer/References/Languages/ScientificNativeReferenceExtractor.cs index 032c44c78..181ea5236 100644 --- a/src/CodeIndex/Indexer/References/Languages/ScientificNativeReferenceExtractor.cs +++ b/src/CodeIndex/Indexer/References/Languages/ScientificNativeReferenceExtractor.cs @@ -60,6 +60,12 @@ internal static class ScientificNativeReferenceExtractor private static readonly Regex CythonImportListRegex = new( @"^\s*(?:cimport|import)\s+(?[^\r\n]+)", RegexOptions.Compiled | RegexOptions.CultureInvariant); + private static readonly Regex CythonStringDependencyRegex = new( + """^\s*(?:include\s+|cdef\s+extern\s+from\s+)(?:'(?[^']+)'|"(?[^"]+)")""", + RegexOptions.Compiled | RegexOptions.CultureInvariant); + private static readonly Regex CythonStringDependencyDirectiveRegex = new( + @"^\s*(?:include\b|cdef\s+extern\s+from\b)", + RegexOptions.Compiled | RegexOptions.CultureInvariant); private static readonly Regex CythonBaseTypeListRegex = new( @"^\s*(?:cdef\s+)?class\s+[A-Za-z_]\w*\s*\(\s*(?[^)\r\n]+)", RegexOptions.Compiled | RegexOptions.CultureInvariant); @@ -168,6 +174,7 @@ internal static bool IsDTemplateArgumentCall( case "cython": EmitMatch(CythonFromImportRegex, "import", stripLeadingRelativePrefix: true); EmitNameList(CythonImportListRegex, "import", ','); + EmitCythonStringDependency(); EmitNameList( CythonBaseTypeListRegex, "type_reference", @@ -573,6 +580,21 @@ void EmitObjectiveCImport() if (match.Success) EmitGroup(match.Groups["name"], "import"); } + + void EmitCythonStringDependency() + { + var directiveLine = CythonStringDependencyRegex.IsMatch(preparedLine) + ? preparedLine + : CythonStringDependencyDirectiveRegex.IsMatch(preparedLine) + ? originalLine + : null; + if (directiveLine == null) + return; + + var match = CythonStringDependencyRegex.Match(directiveLine); + if (match.Success) + EmitGroup(match.Groups["name"], "import"); + } } private static bool IsDependencyNameChar(char value) => diff --git a/tests/CodeIndex.Tests/DatabaseTests.cs b/tests/CodeIndex.Tests/DatabaseTests.cs index b9ba6508f..e44d2cc85 100644 --- a/tests/CodeIndex.Tests/DatabaseTests.cs +++ b/tests/CodeIndex.Tests/DatabaseTests.cs @@ -641,6 +641,14 @@ WHERE reference.id IS NULL OR symbol.id IS NULL public void ReferenceGraph_AmbiguousMResolvesAgainstDefinitiveDialects_Issue4738() { var callerId = UpsertTestFileWithLanguage("src/caller.m", "ambiguous_m", "ambiguous-caller"); + var unresolvedTargetId = UpsertTestFileWithLanguage( + "src/unresolved-target.m", + "ambiguous_m", + "ambiguous-target"); + _writer.InsertSymbols([ + new SymbolRecord { FileId = unresolvedTargetId, Kind = "function", Name = "MatlabTarget", Line = 1 }, + new SymbolRecord { FileId = unresolvedTargetId, Kind = "function", Name = "ObjectiveCTarget", Line = 2 }, + ]); _writer.InsertReferences([ new ReferenceRecord { @@ -723,9 +731,11 @@ FROM symbol_references }) { var symbolId = ExecuteScalarLong($""" - SELECT id - FROM symbols - WHERE name = '{name}' + SELECT symbol.id + FROM symbols AS symbol + JOIN files AS file ON file.id = symbol.file_id + WHERE symbol.name = '{name}' + AND file.path = '{path}' """); var definition = new DefinitionResult { diff --git a/tests/CodeIndex.Tests/ReferenceExtractorTests.ScientificNative.cs b/tests/CodeIndex.Tests/ReferenceExtractorTests.ScientificNative.cs index c37aef225..32ac01734 100644 --- a/tests/CodeIndex.Tests/ReferenceExtractorTests.ScientificNative.cs +++ b/tests/CodeIndex.Tests/ReferenceExtractorTests.ScientificNative.cs @@ -945,6 +945,49 @@ proc real() = reference.SymbolName == "helper" && reference.ReferenceKind == "call").ContainerName); } + [Fact] + public void Extract_NimRawTripleStringUsesMultilineMask_Issue4738() + { + const string content = """" + let text = r""" + proc Phantom() = discard + phantomCall() + """ + proc real() = + helper() + """"; + var symbols = SymbolExtractor.Extract(1, "nim", content); + + var references = ReferenceExtractor.Extract(1, "nim", content, symbols); + + Assert.DoesNotContain(symbols, symbol => symbol.Name == "Phantom"); + Assert.DoesNotContain(references, reference => reference.SymbolName == "phantomCall"); + Assert.Equal("real", Assert.Single(references, reference => + reference.SymbolName == "helper" && reference.ReferenceKind == "call").ContainerName); + } + + [Fact] + public void Extract_CythonStringDependenciesPreserveQuotedNames_Issue4738() + { + const string content = """" + include "helpers.pxi" + cdef extern from 'native.h': + void native_call() + """ + include "phantom.pxi" + """ + """"; + var symbols = SymbolExtractor.Extract(1, "cython", content); + + var references = ReferenceExtractor.Extract(1, "cython", content, symbols); + + Assert.Contains(references, reference => + reference.SymbolName == "helpers.pxi" && reference.ReferenceKind == "import"); + Assert.Contains(references, reference => + reference.SymbolName == "native.h" && reference.ReferenceKind == "import"); + Assert.DoesNotContain(references, reference => reference.SymbolName == "phantom.pxi"); + } + [Fact] public void Extract_AdaAttributesPreserveNestedCallsWithoutPhantomAttributeCalls_Issue4738() { From ae73bef916110b0cf51d47e056f7409c47e6ba69 Mon Sep 17 00:00:00 2001 From: Widthdom Date: Fri, 24 Jul 2026 17:14:16 +0900 Subject: [PATCH 12/21] Fix scientific graph language smoke cases (#4738) --- .../Indexer/AmbiguousMContentMasker.cs | 7 ++- .../ScientificNativeCommentMasker.cs | 36 ++++++++++++-- .../ScientificNativeReferenceExtractor.cs | 2 +- ...eferenceExtractorTests.ScientificNative.cs | 47 +++++++++++++++++++ 4 files changed, 84 insertions(+), 8 deletions(-) diff --git a/src/CodeIndex/Indexer/AmbiguousMContentMasker.cs b/src/CodeIndex/Indexer/AmbiguousMContentMasker.cs index ba534ffd9..ae9c4c9e7 100644 --- a/src/CodeIndex/Indexer/AmbiguousMContentMasker.cs +++ b/src/CodeIndex/Indexer/AmbiguousMContentMasker.cs @@ -201,7 +201,7 @@ private static bool LooksLikeObjectiveCModuloOperator(string content, int percen if (previousIndex < 0 || content[previousIndex] is '\r' or '\n' - || !IsTransposeOperandEnd(content[previousIndex])) + || !IsObjectiveCModuloLeftOperandEnd(content[previousIndex])) { return false; } @@ -233,9 +233,12 @@ private static bool LooksLikeObjectiveCModuloOperator(string content, int percen return nextIndex < content.Length && content[nextIndex] is not '\r' and not '\n' && (char.IsLetterOrDigit(content[nextIndex]) - || content[nextIndex] is '_' or '(' or '\'' or '"'); + || content[nextIndex] is '_' or '(' or '[' or '@' or '\'' or '"'); } + private static bool IsObjectiveCModuloLeftOperandEnd(char value) => + IsTransposeOperandEnd(value) || value is '\'' or '"'; + private static bool IsTransposeOperandEnd(char value) => char.IsLetterOrDigit(value) || value is '_' or ')' or ']' or '}'; } diff --git a/src/CodeIndex/Indexer/References/Languages/ScientificNativeCommentMasker.cs b/src/CodeIndex/Indexer/References/Languages/ScientificNativeCommentMasker.cs index d240d08c2..d6fb35e6e 100644 --- a/src/CodeIndex/Indexer/References/Languages/ScientificNativeCommentMasker.cs +++ b/src/CodeIndex/Indexer/References/Languages/ScientificNativeCommentMasker.cs @@ -311,18 +311,26 @@ private static string[] MaskDNonCodeRegions(string[] lines) var inCStyleBlockComment = false; var inBacktickString = false; string? tokenStringClosing = null; + var tokenStringClosingRequiresLineBoundary = false; var tokenStringQuote = '\0'; var tokenStringQuoteUsesEscapes = false; var tokenStringInBacktickString = false; var tokenStringNestedCommentDepth = 0; var tokenStringInCStyleBlockComment = false; string? nestedTokenStringClosing = null; + var nestedTokenStringClosingRequiresLineBoundary = false; var quote = '\0'; var quoteUsesEscapes = false; for (var lineIndex = 0; lineIndex < lines.Length; lineIndex++) { var line = lines[lineIndex]; char[]? chars = null; + var firstNonWhitespaceIndex = 0; + while (firstNonWhitespaceIndex < line.Length + && char.IsWhiteSpace(line[firstNonWhitespaceIndex])) + { + firstNonWhitespaceIndex++; + } void MaskAt(int index) => (chars ??= line.ToCharArray())[index] = ' '; @@ -341,10 +349,13 @@ void MaskAt(int index) => if (tokenStringClosing != null) { - if (StartsWith(line, cursor, tokenStringClosing)) + if ((!tokenStringClosingRequiresLineBoundary + || cursor == firstNonWhitespaceIndex) + && StartsWith(line, cursor, tokenStringClosing)) { MaskToken(tokenStringClosing); tokenStringClosing = null; + tokenStringClosingRequiresLineBoundary = false; continue; } @@ -356,10 +367,13 @@ void MaskAt(int index) => { if (nestedTokenStringClosing != null) { - if (StartsWith(line, cursor, nestedTokenStringClosing)) + if ((!nestedTokenStringClosingRequiresLineBoundary + || cursor == firstNonWhitespaceIndex) + && StartsWith(line, cursor, nestedTokenStringClosing)) { MaskToken(nestedTokenStringClosing); nestedTokenStringClosing = null; + nestedTokenStringClosingRequiresLineBoundary = false; continue; } @@ -453,11 +467,14 @@ void MaskAt(int index) => line, cursor + 2, out var nestedOpeningLength, - out var nestedClosing)) + out var nestedClosing, + out var nestedClosingRequiresLineBoundary)) { for (var openingIndex = 0; openingIndex < 2 + nestedOpeningLength; openingIndex++) MaskAt(cursor++); nestedTokenStringClosing = nestedClosing; + nestedTokenStringClosingRequiresLineBoundary = + nestedClosingRequiresLineBoundary; continue; } @@ -575,11 +592,17 @@ void MaskAt(int index) => if (StartsWith(line, cursor, "q\"") && (cursor == 0 || !IsIdentifierChar(line[cursor - 1])) - && TryGetDTokenStringClosing(line, cursor + 2, out var openingLength, out var closing)) + && TryGetDTokenStringClosing( + line, + cursor + 2, + out var openingLength, + out var closing, + out var closingRequiresLineBoundary)) { for (var openingIndex = 0; openingIndex < 2 + openingLength; openingIndex++) MaskAt(cursor++); tokenStringClosing = closing; + tokenStringClosingRequiresLineBoundary = closingRequiresLineBoundary; continue; } @@ -627,10 +650,12 @@ private static bool TryGetDTokenStringClosing( string line, int delimiterIndex, out int openingLength, - out string closing) + out string closing, + out bool closingRequiresLineBoundary) { openingLength = 0; closing = string.Empty; + closingRequiresLineBoundary = false; if (delimiterIndex >= line.Length) return false; @@ -659,6 +684,7 @@ private static bool TryGetDTokenStringClosing( openingLength = end - delimiterIndex; closing = line[delimiterIndex..end] + '"'; + closingRequiresLineBoundary = true; return true; } diff --git a/src/CodeIndex/Indexer/References/Languages/ScientificNativeReferenceExtractor.cs b/src/CodeIndex/Indexer/References/Languages/ScientificNativeReferenceExtractor.cs index 181ea5236..705726e27 100644 --- a/src/CodeIndex/Indexer/References/Languages/ScientificNativeReferenceExtractor.cs +++ b/src/CodeIndex/Indexer/References/Languages/ScientificNativeReferenceExtractor.cs @@ -77,7 +77,7 @@ internal static class ScientificNativeReferenceExtractor @"^\s*type\s+[A-Za-z]\w*\s+is\s+new\s+(?[A-Za-z]\w*(?:\.[A-Za-z]\w*)*)", RegexOptions.Compiled | RegexOptions.IgnoreCase | RegexOptions.CultureInvariant); private static readonly Regex AdaBareCallRegex = new( - @"(?:^|;)\s*(?!(?:end|null|return|exit|raise|goto)\b)(?[A-Za-z]\w*(?:\.[A-Za-z]\w*)*)\s*;", + @"(?:^|(?<=;))\s*(?!(?:end|null|return|exit|raise|goto)\b)(?[A-Za-z]\w*(?:\.[A-Za-z]\w*)*)\s*;", RegexOptions.Compiled | RegexOptions.IgnoreCase | RegexOptions.CultureInvariant); private static readonly Regex ObjectiveCImportRegex = new( """^\s*#\s*(?:import|include)\s*[<"](?[^>"]+)[>"]""", diff --git a/tests/CodeIndex.Tests/ReferenceExtractorTests.ScientificNative.cs b/tests/CodeIndex.Tests/ReferenceExtractorTests.ScientificNative.cs index 32ac01734..9e08b9cac 100644 --- a/tests/CodeIndex.Tests/ReferenceExtractorTests.ScientificNative.cs +++ b/tests/CodeIndex.Tests/ReferenceExtractorTests.ScientificNative.cs @@ -392,6 +392,26 @@ procedure Run is Assert.DoesNotContain(references, reference => reference.SymbolName == "Helpers.Flush"); } + [Fact] + public void Extract_AdaConsecutiveBareCallsDoNotConsumeNextSeparator_Issue4738() + { + const string content = """ + procedure Run is + begin + First; Second; Third; + end Run; + """; + var symbols = SymbolExtractor.Extract(1, "ada", content); + + var references = ReferenceExtractor.Extract(1, "ada", content, symbols); + + foreach (var name in new[] { "First", "Second", "Third" }) + { + Assert.Equal("Run", Assert.Single(references, reference => + reference.SymbolName == name && reference.ReferenceKind == "call").ContainerName); + } + } + [Fact] public void Extract_MatlabCommaSeparatedEndKeepsFollowingCallTopLevel_Issue4738() { @@ -877,6 +897,27 @@ enum code = q{ reference.SymbolName == "helper" && reference.ReferenceKind == "call").ContainerName); } + [Fact] + public void Extract_DIdentifierTokenStringRequiresDelimiterLineBoundary_Issue4738() + { + const string content = """ + void run() { + enum text = q"EOS + inside text contains EOS"; phantomCall() + still inside + EOS"; + helper(); + } + """; + var symbols = SymbolExtractor.Extract(1, "d", content); + + var references = ReferenceExtractor.Extract(1, "d", content, symbols); + + Assert.DoesNotContain(references, reference => reference.SymbolName == "phantomCall"); + Assert.Equal("run", Assert.Single(references, reference => + reference.SymbolName == "helper" && reference.ReferenceKind == "call").ContainerName); + } + [Fact] public void Extract_DCommentTextCannotOpenTokenStrings_Issue4738() { @@ -1117,6 +1158,9 @@ @implementation Widget afterAssignment(); int fourth = index++ % postIncrement(); int fifth = index-- % postDecrement(); + int sixth = left % [self computeValue]; + int seventh = 'x' % charRhs(); + int eighth = left % @(boxed()); } @end """; @@ -1132,6 +1176,9 @@ @implementation Widget "afterAssignment", "postIncrement", "postDecrement", + "computeValue", + "charRhs", + "boxed", }) { Assert.Single(references, reference => From 41de90d9a12f5e50b71501bc25edcedbe9e257ee Mon Sep 17 00:00:00 2001 From: Widthdom Date: Fri, 24 Jul 2026 17:35:20 +0900 Subject: [PATCH 13/21] Handle scientific graph syntax variants (#4738) --- .../ScientificNativeCommentMasker.cs | 28 +-- .../ScientificNativeReferenceExtractor.cs | 161 ++++++++++++++++-- ...eferenceExtractorTests.ScientificNative.cs | 73 ++++++++ 3 files changed, 240 insertions(+), 22 deletions(-) diff --git a/src/CodeIndex/Indexer/References/Languages/ScientificNativeCommentMasker.cs b/src/CodeIndex/Indexer/References/Languages/ScientificNativeCommentMasker.cs index d6fb35e6e..c3df39d7c 100644 --- a/src/CodeIndex/Indexer/References/Languages/ScientificNativeCommentMasker.cs +++ b/src/CodeIndex/Indexer/References/Languages/ScientificNativeCommentMasker.cs @@ -235,30 +235,38 @@ private static bool HasOddBackslashPrefix(string line, int index) internal static string MaskNimRawStringLiterals(string line) { - if (!line.Contains("r\"", StringComparison.Ordinal)) + if (!line.Contains('"')) return line; char[]? chars = null; - for (var cursor = 0; cursor + 1 < line.Length; cursor++) + for (var cursor = 0; cursor < line.Length;) { - if (line[cursor] != 'r' - || line[cursor + 1] != '"' + if (!(char.IsLetter(line[cursor]) || line[cursor] == '_') || (cursor > 0 && IsIdentifierChar(line[cursor - 1]))) { + cursor++; continue; } + var prefixStart = cursor; + while (cursor < line.Length && IsIdentifierChar(line[cursor])) + cursor++; + if (cursor >= line.Length || line[cursor] != '"') + continue; + chars ??= line.ToCharArray(); - if (cursor + 3 < line.Length - && line[cursor + 2] == '"' - && line[cursor + 3] == '"') + for (var prefixIndex = prefixStart; prefixIndex < cursor; prefixIndex++) + chars[prefixIndex] = ' '; + + if (cursor + 2 < line.Length + && line[cursor + 1] == '"' + && line[cursor + 2] == '"') { // Leave the triple-quote delimiter for the stateful multiline masker. - chars[cursor] = ' '; + cursor += 3; continue; } - chars[cursor++] = ' '; chars[cursor++] = ' '; while (cursor < line.Length) { @@ -275,8 +283,6 @@ internal static string MaskNimRawStringLiterals(string line) break; } - - cursor--; } return chars is null ? line : new string(chars); diff --git a/src/CodeIndex/Indexer/References/Languages/ScientificNativeReferenceExtractor.cs b/src/CodeIndex/Indexer/References/Languages/ScientificNativeReferenceExtractor.cs index 705726e27..bb353cdac 100644 --- a/src/CodeIndex/Indexer/References/Languages/ScientificNativeReferenceExtractor.cs +++ b/src/CodeIndex/Indexer/References/Languages/ScientificNativeReferenceExtractor.cs @@ -7,6 +7,11 @@ namespace CodeIndex.Indexer; internal static class ScientificNativeReferenceExtractor { internal readonly record struct DTemplateArgumentCallSpan(int Start, int EndExclusive); + private readonly record struct DTemplateInvocation( + string Name, + int NameIndex, + int ArgumentStart, + int EndExclusive); private static readonly HashSet SupportedLanguages = new(StringComparer.Ordinal) { "ada", "cython", "d", "julia", "matlab", "nim", "objc" }; @@ -50,12 +55,8 @@ internal static class ScientificNativeReferenceExtractor private static readonly Regex DBaseTypeListRegex = new( @"^\s*(?:(?:public|private|protected|package|static|abstract|final|extern)\s+)*(?:class|interface)\s+[A-Za-z_]\w*(?:\s*\([^)]*\))?\s*:\s*(?[^{\r\n]+)", RegexOptions.Compiled | RegexOptions.CultureInvariant); - private static readonly Regex DTemplateInvocationRegex = new( - @"(?[A-Za-z_]\w*)\s*!\s*(?:[A-Za-z_]\w*(?:\.[A-Za-z_]\w*)*|\([^()\r\n]*\))\s*\(", - RegexOptions.Compiled | RegexOptions.CultureInvariant); - private static readonly Regex CythonFromImportRegex = new( - @"^\s*from\s+(?\.*[A-Za-z_]\w*(?:\.[A-Za-z_]\w*)*)\s+cimport\b", + @"^\s*from\s+(?\.*[A-Za-z_]\w*(?:\.[A-Za-z_]\w*)*)\s+(?:cimport|import)\b", RegexOptions.Compiled | RegexOptions.CultureInvariant); private static readonly Regex CythonImportListRegex = new( @"^\s*(?:cimport|import)\s+(?[^\r\n]+)", @@ -77,7 +78,7 @@ internal static class ScientificNativeReferenceExtractor @"^\s*type\s+[A-Za-z]\w*\s+is\s+new\s+(?[A-Za-z]\w*(?:\.[A-Za-z]\w*)*)", RegexOptions.Compiled | RegexOptions.IgnoreCase | RegexOptions.CultureInvariant); private static readonly Regex AdaBareCallRegex = new( - @"(?:^|(?<=;))\s*(?!(?:end|null|return|exit|raise|goto)\b)(?[A-Za-z]\w*(?:\.[A-Za-z]\w*)*)\s*;", + @"(?:^|;|\b(?:begin|then|else|loop)\b|=>)\s*(?!(?:end|null|return|exit|raise|goto)\b)(?[A-Za-z]\w*(?:\.[A-Za-z]\w*)*)\s*(?=;)", RegexOptions.Compiled | RegexOptions.IgnoreCase | RegexOptions.CultureInvariant); private static readonly Regex ObjectiveCImportRegex = new( """^\s*#\s*(?:import|include)\s*[<"](?[^>"]+)[>"]""", @@ -161,14 +162,13 @@ internal static bool IsDTemplateArgumentCall( "type_reference", ',', normalizeQualifiedTypeName: true); - foreach (Match match in DTemplateInvocationRegex.Matches(preparedLine)) + foreach (var invocation in FindDTemplateInvocations(preparedLine)) { - var group = match.Groups["name"]; - addCallLikeReference(group.Value, group.Index); + addCallLikeReference(invocation.Name, invocation.NameIndex); (dTemplateArgumentCallSpans ??= []).Add( new DTemplateArgumentCallSpan( - group.Index + group.Length, - match.Index + match.Length)); + invocation.ArgumentStart, + invocation.EndExclusive)); } break; case "cython": @@ -599,4 +599,143 @@ void EmitCythonStringDependency() private static bool IsDependencyNameChar(char value) => char.IsLetterOrDigit(value) || value is '_' or '.' or '/' or '*'; + + private static IReadOnlyList FindDTemplateInvocations(string line) + { + List? invocations = null; + var cursor = 0; + while (cursor < line.Length) + { + if (!IsDIdentifierStart(line[cursor]) + || (cursor > 0 && IsDIdentifierPart(line[cursor - 1]))) + { + cursor++; + continue; + } + + var nameIndex = cursor; + cursor = ScanDIdentifier(line, cursor); + var nameEnd = cursor; + var scan = cursor; + while (true) + { + SkipWhitespace(line, ref scan); + if (scan >= line.Length || line[scan] != '.') + break; + + var nextNameIndex = scan + 1; + SkipWhitespace(line, ref nextNameIndex); + if (nextNameIndex >= line.Length || !IsDIdentifierStart(line[nextNameIndex])) + break; + + nameIndex = nextNameIndex; + nameEnd = ScanDIdentifier(line, nextNameIndex); + scan = nameEnd; + } + + SkipWhitespace(line, ref scan); + if (scan >= line.Length || line[scan] != '!') + { + cursor = scan; + continue; + } + + if (scan + 1 < line.Length && line[scan + 1] == '=') + { + cursor = scan + 2; + continue; + } + + scan++; + SkipWhitespace(line, ref scan); + var argumentStart = nameEnd; + if (scan < line.Length && line[scan] == '(') + { + if (!TryScanBalancedDTemplateArguments(line, scan, out scan)) + { + cursor = line.Length; + continue; + } + } + else + { + var tokenStart = scan; + while (scan < line.Length + && !char.IsWhiteSpace(line[scan]) + && line[scan] is not ('(' or ';' or ',')) + { + scan++; + } + + if (scan == tokenStart) + continue; + } + + SkipWhitespace(line, ref scan); + if (scan >= line.Length || line[scan] != '(') + { + cursor = scan; + continue; + } + + (invocations ??= []).Add(new DTemplateInvocation( + line[nameIndex..nameEnd], + nameIndex, + argumentStart, + scan + 1)); + cursor = scan + 1; + } + + return invocations ?? []; + } + + private static bool TryScanBalancedDTemplateArguments( + string line, + int openingParenthesis, + out int endExclusive) + { + var depth = 0; + for (var cursor = openingParenthesis; cursor < line.Length; cursor++) + { + if (line[cursor] == '(') + { + depth++; + continue; + } + + if (line[cursor] != ')') + continue; + + depth--; + if (depth == 0) + { + endExclusive = cursor + 1; + return true; + } + } + + endExclusive = line.Length; + return false; + } + + private static int ScanDIdentifier(string line, int start) + { + var cursor = start + 1; + while (cursor < line.Length && IsDIdentifierPart(line[cursor])) + cursor++; + + return cursor; + } + + private static void SkipWhitespace(string line, ref int cursor) + { + while (cursor < line.Length && char.IsWhiteSpace(line[cursor])) + cursor++; + } + + private static bool IsDIdentifierStart(char value) => + char.IsLetter(value) || value == '_'; + + private static bool IsDIdentifierPart(char value) => + char.IsLetterOrDigit(value) || value == '_'; } diff --git a/tests/CodeIndex.Tests/ReferenceExtractorTests.ScientificNative.cs b/tests/CodeIndex.Tests/ReferenceExtractorTests.ScientificNative.cs index 9e08b9cac..7e6e29aa6 100644 --- a/tests/CodeIndex.Tests/ReferenceExtractorTests.ScientificNative.cs +++ b/tests/CodeIndex.Tests/ReferenceExtractorTests.ScientificNative.cs @@ -357,6 +357,27 @@ void run() { reference.SymbolName is "int" or "string" && reference.ReferenceKind == "call"); } + [Fact] + public void Extract_DTemplateInvocationsAcceptLiteralAndNestedArguments_Issue4738() + { + const string content = """ + void run() { + literal!42(); + nested!(Tuple!(int))(); + } + """; + var symbols = SymbolExtractor.Extract(1, "d", content); + + var references = ReferenceExtractor.Extract(1, "d", content, symbols); + + Assert.Contains(references, reference => + reference.SymbolName == "literal" && reference.ReferenceKind == "call"); + Assert.Contains(references, reference => + reference.SymbolName == "nested" && reference.ReferenceKind == "call"); + Assert.DoesNotContain(references, reference => + reference.SymbolName is "Tuple" or "int" && reference.ReferenceKind == "call"); + } + [Fact] public void Extract_DManyTemplateInvocationsSuppressArgumentCallsInOnePass_Issue4738() { @@ -412,6 +433,24 @@ procedure Run is } } + [Fact] + public void Extract_AdaInlineStatementBoundariesPreserveBareCalls_Issue4738() + { + const string content = """ + procedure Run is begin First; end Run; + procedure Check is begin if Ready then Second; else Third; end if; end Check; + """; + var symbols = SymbolExtractor.Extract(1, "ada", content); + + var references = ReferenceExtractor.Extract(1, "ada", content, symbols); + + foreach (var name in new[] { "First", "Second", "Third" }) + { + Assert.Single(references, reference => + reference.SymbolName == name && reference.ReferenceKind == "call"); + } + } + [Fact] public void Extract_MatlabCommaSeparatedEndKeepsFollowingCallTopLevel_Issue4738() { @@ -965,6 +1004,23 @@ public void Extract_DAndNimRawStringsUseLiteralBackslashRules_Issue4738( Assert.DoesNotContain(references, reference => reference.SymbolName == "fake"); } + [Fact] + public void Extract_NimGeneralizedRawStringsUseLiteralBackslashRules_Issue4738() + { + const string content = """ + proc run() = + let text = foo"notACall()\" + helper() + """; + var symbols = SymbolExtractor.Extract(1, "nim", content); + + var references = ReferenceExtractor.Extract(1, "nim", content, symbols); + + Assert.Equal("run", Assert.Single(references, reference => + reference.SymbolName == "helper" && reference.ReferenceKind == "call").ContainerName); + Assert.DoesNotContain(references, reference => reference.SymbolName == "notACall"); + } + [Fact] public void Extract_NimRawStringBeforeBlockCommentDoesNotExposeCommentCode_Issue4738() { @@ -1029,6 +1085,23 @@ void native_call() Assert.DoesNotContain(references, reference => reference.SymbolName == "phantom.pxi"); } + [Fact] + public void Extract_CythonOrdinaryFromImportsEmitModuleDependencies_Issue4738() + { + const string content = """ + from helpers import thing + from .local import other + """; + var symbols = SymbolExtractor.Extract(1, "cython", content); + + var references = ReferenceExtractor.Extract(1, "cython", content, symbols); + + Assert.Contains(references, reference => + reference.SymbolName == "helpers" && reference.ReferenceKind == "import"); + Assert.Contains(references, reference => + reference.SymbolName == "local" && reference.ReferenceKind == "import"); + } + [Fact] public void Extract_AdaAttributesPreserveNestedCallsWithoutPhantomAttributeCalls_Issue4738() { From ac51e1874cca77bfd4d9413b8fb3e83bb90507cf Mon Sep 17 00:00:00 2001 From: Widthdom Date: Fri, 24 Jul 2026 17:55:42 +0900 Subject: [PATCH 14/21] Resolve qualified scientific graph references (#4738) --- src/CodeIndex/Database/DbWriter.References.cs | 29 ++++++- .../Indexer/AmbiguousMContentMasker.cs | 71 ++++++++++++++++ .../ScientificNativeReferenceExtractor.cs | 6 +- tests/CodeIndex.Tests/DatabaseTests.cs | 81 +++++++++++++++++++ ...eferenceExtractorTests.ScientificNative.cs | 12 +++ 5 files changed, 197 insertions(+), 2 deletions(-) diff --git a/src/CodeIndex/Database/DbWriter.References.cs b/src/CodeIndex/Database/DbWriter.References.cs index d11fe4d1b..f9514e8e1 100644 --- a/src/CodeIndex/Database/DbWriter.References.cs +++ b/src/CodeIndex/Database/DbWriter.References.cs @@ -153,6 +153,21 @@ AND r.target_qualifier NOT LIKE char(31) || 'receiver:%' s.container_name = r.target_qualifier COLLATE NOCASE OR s.container_qualified_name = r.target_qualifier COLLATE NOCASE OR s.container_qualified_name LIKE '%.' || r.target_qualifier COLLATE NOCASE + OR EXISTS ( + SELECT 1 + FROM symbols AS target_scope + WHERE target_scope.file_id = s.file_id + AND target_scope.kind IN ('namespace', 'module', 'package') + AND ( + target_scope.name = r.target_qualifier COLLATE NOCASE + OR target_scope.container_qualified_name = r.target_qualifier COLLATE NOCASE + OR substr( + target_scope.name, + 1, + length(r.target_qualifier) + 1 + ) = (r.target_qualifier || '.') COLLATE NOCASE + ) + ) ); INSERT INTO symbol_reference_candidates(reference_id, symbol_id, scope_rank) @@ -311,7 +326,19 @@ JOIN files AS target_file COALESCE(target.container_qualified_name, target.container_name, '') || char(31) || COALESCE(target.name, '') = unique_family.family_key WHERE source_file.lang <> 'csharp' - AND r.target_qualifier IS NULL + AND ( + r.target_qualifier IS NULL + OR source_file.lang IN ( + 'ada', + 'ambiguous_m', + 'cython', + 'd', + 'julia', + 'matlab', + 'nim', + 'objc' + ) + ) AND NOT EXISTS ( SELECT 1 FROM symbol_reference_candidates AS existing WHERE existing.reference_id = r.id diff --git a/src/CodeIndex/Indexer/AmbiguousMContentMasker.cs b/src/CodeIndex/Indexer/AmbiguousMContentMasker.cs index ae9c4c9e7..85f7103b5 100644 --- a/src/CodeIndex/Indexer/AmbiguousMContentMasker.cs +++ b/src/CodeIndex/Indexer/AmbiguousMContentMasker.cs @@ -20,6 +20,8 @@ internal static string MaskComments( return content; } + preserveObjectiveCModuloExpressions &= + HasStrongObjectiveCModuloEvidence(content); StringBuilder? masked = null; var inBlockComment = false; var inMatlabBlockComment = false; @@ -239,6 +241,75 @@ private static bool LooksLikeObjectiveCModuloOperator(string content, int percen private static bool IsObjectiveCModuloLeftOperandEnd(char value) => IsTransposeOperandEnd(value) || value is '\'' or '"'; + private static bool HasStrongObjectiveCModuloEvidence(string content) + { + var lineStart = 0; + while (lineStart < content.Length) + { + var lineEnd = lineStart; + while (lineEnd < content.Length && content[lineEnd] is not ('\r' or '\n')) + lineEnd++; + + var line = content.AsSpan(lineStart, lineEnd - lineStart).TrimStart(); + if (StartsWithObjectiveCPreprocessorDirective(line) + || StartsWithObjectiveCAtKeyword(line) + || StartsWithObjectiveCMethodDeclaration(line)) + { + return true; + } + + lineStart = lineEnd + 1; + if (lineEnd < content.Length + && content[lineEnd] == '\r' + && lineStart < content.Length + && content[lineStart] == '\n') + { + lineStart++; + } + } + + return false; + } + + private static bool StartsWithObjectiveCPreprocessorDirective(ReadOnlySpan line) + { + if (line.IsEmpty || line[0] != '#') + return false; + + var keywordStart = 1; + while (keywordStart < line.Length && char.IsWhiteSpace(line[keywordStart])) + keywordStart++; + + return StartsWithToken(line[keywordStart..], "import") + || StartsWithToken(line[keywordStart..], "include"); + } + + private static bool StartsWithObjectiveCAtKeyword(ReadOnlySpan line) => + StartsWithToken(line, "@interface") + || StartsWithToken(line, "@implementation") + || StartsWithToken(line, "@protocol") + || StartsWithToken(line, "@class") + || StartsWithToken(line, "@property") + || StartsWithToken(line, "@synthesize") + || StartsWithToken(line, "@dynamic") + || StartsWithToken(line, "@autoreleasepool"); + + private static bool StartsWithObjectiveCMethodDeclaration(ReadOnlySpan line) + { + if (line.IsEmpty || line[0] is not ('-' or '+')) + return false; + + var openingParenthesis = 1; + while (openingParenthesis < line.Length && char.IsWhiteSpace(line[openingParenthesis])) + openingParenthesis++; + + return openingParenthesis < line.Length && line[openingParenthesis] == '('; + } + + private static bool StartsWithToken(ReadOnlySpan line, string token) => + line.StartsWith(token, StringComparison.Ordinal) + && (line.Length == token.Length || !IsTransposeOperandEnd(line[token.Length])); + private static bool IsTransposeOperandEnd(char value) => char.IsLetterOrDigit(value) || value is '_' or ')' or ']' or '}'; } diff --git a/src/CodeIndex/Indexer/References/Languages/ScientificNativeReferenceExtractor.cs b/src/CodeIndex/Indexer/References/Languages/ScientificNativeReferenceExtractor.cs index bb353cdac..d2eee6c0a 100644 --- a/src/CodeIndex/Indexer/References/Languages/ScientificNativeReferenceExtractor.cs +++ b/src/CodeIndex/Indexer/References/Languages/ScientificNativeReferenceExtractor.cs @@ -189,7 +189,11 @@ internal static bool IsDTemplateArgumentCall( var group = bareCall.Groups["name"]; var separatorIndex = group.Value.LastIndexOf('.'); var leafOffset = separatorIndex + 1; - addCallLikeReference(group.Value[leafOffset..], group.Index + leafOffset); + EmitName( + group.Value[leafOffset..], + group.Index + leafOffset, + "call", + separatorIndex >= 0 ? group.Value[..separatorIndex] : null); } break; case "objc": diff --git a/tests/CodeIndex.Tests/DatabaseTests.cs b/tests/CodeIndex.Tests/DatabaseTests.cs index e44d2cc85..9e9ac9bcc 100644 --- a/tests/CodeIndex.Tests/DatabaseTests.cs +++ b/tests/CodeIndex.Tests/DatabaseTests.cs @@ -762,6 +762,87 @@ FROM symbol_reference_candidates } } + [Fact] + public void ReferenceGraph_ScientificQualifiersUseModuleEvidenceOrSafeUniqueFallback_Issue4738() + { + var dCallerId = UpsertTestFileWithLanguage("src/child.d", "d", "qualified-d-caller"); + var dTargetId = UpsertTestFileWithLanguage("src/pkg/base.d", "d", "qualified-d-target"); + var cythonCallerId = UpsertTestFileWithLanguage( + "src/child.pyx", + "cython", + "qualified-cython-caller"); + var cythonTargetId = UpsertTestFileWithLanguage( + "src/pkg/base.pyx", + "cython", + "qualified-cython-target"); + var adaCallerId = UpsertTestFileWithLanguage("src/main.adb", "ada", "qualified-ada-caller"); + var adaP1Id = UpsertTestFileWithLanguage("src/p1.adb", "ada", "qualified-ada-p1"); + var adaP2Id = UpsertTestFileWithLanguage("src/p2.adb", "ada", "qualified-ada-p2"); + _writer.InsertSymbols([ + new SymbolRecord { FileId = dTargetId, Kind = "namespace", Name = "pkg.base", Line = 1 }, + new SymbolRecord { FileId = dTargetId, Kind = "class", Name = "Base", Line = 2 }, + new SymbolRecord { FileId = cythonTargetId, Kind = "class", Name = "NativeBase", Line = 1 }, + new SymbolRecord { FileId = adaP1Id, Kind = "namespace", Name = "P1", Line = 1 }, + new SymbolRecord { FileId = adaP1Id, Kind = "function", Name = "Flush", Line = 2 }, + new SymbolRecord { FileId = adaP2Id, Kind = "namespace", Name = "P2", Line = 1 }, + new SymbolRecord { FileId = adaP2Id, Kind = "function", Name = "Flush", Line = 2 }, + ]); + _writer.InsertReferences([ + new ReferenceRecord + { + FileId = dCallerId, + SymbolName = "Base", + TargetQualifier = "pkg", + ReferenceKind = "type_reference", + Line = 1, + Column = 1, + Context = "class Child : pkg.Base {}", + }, + new ReferenceRecord + { + FileId = cythonCallerId, + SymbolName = "NativeBase", + TargetQualifier = "pkg", + ReferenceKind = "type_reference", + Line = 1, + Column = 1, + Context = "cdef class Child(pkg.NativeBase):", + }, + new ReferenceRecord + { + FileId = adaCallerId, + SymbolName = "Flush", + TargetQualifier = "P1", + ReferenceKind = "call", + Line = 1, + Column = 1, + Context = "P1.Flush;", + }, + ], refreshMutualRecursionFlags: false); + + _writer.RefreshMutualRecursionFlags(); + + AssertResolvedTo(dCallerId, dTargetId, "Base"); + AssertResolvedTo(cythonCallerId, cythonTargetId, "NativeBase"); + AssertResolvedTo(adaCallerId, adaP1Id, "Flush"); + + void AssertResolvedTo(long callerId, long targetFileId, string symbolName) + { + var expectedTargetId = ExecuteScalarLong($""" + SELECT id + FROM symbols + WHERE file_id = {targetFileId.ToString(CultureInfo.InvariantCulture)} + AND name = '{symbolName}' + """); + Assert.Equal(expectedTargetId, ExecuteScalarLong($""" + SELECT target_symbol_id + FROM symbol_references + WHERE file_id = {callerId.ToString(CultureInfo.InvariantCulture)} + """)); + Assert.Equal("resolved", ReadReferenceResolutionState(callerId)); + } + } + [Fact] public void ReferenceGraphDirtyScope_RollbackAndCancellationPreserveRetryState() { diff --git a/tests/CodeIndex.Tests/ReferenceExtractorTests.ScientificNative.cs b/tests/CodeIndex.Tests/ReferenceExtractorTests.ScientificNative.cs index 7e6e29aa6..eaf4ad03d 100644 --- a/tests/CodeIndex.Tests/ReferenceExtractorTests.ScientificNative.cs +++ b/tests/CodeIndex.Tests/ReferenceExtractorTests.ScientificNative.cs @@ -410,6 +410,7 @@ procedure Run is var flushReference = Assert.Single(references, reference => reference.SymbolName == "Flush" && reference.ReferenceKind == "call"); Assert.Equal("Run", flushReference.ContainerName); + Assert.Equal("Helpers", flushReference.TargetQualifier); Assert.DoesNotContain(references, reference => reference.SymbolName == "Helpers.Flush"); } @@ -1259,6 +1260,17 @@ @implementation Widget } } + [Fact] + public void Extract_AmbiguousMMatlabCommentsDoNotMasqueradeAsModuloExpressions_Issue4738() + { + const string content = "x = left % helper();"; + + var symbols = SymbolExtractor.Extract(1, "ambiguous_m", content, "unknown.m"); + var references = ReferenceExtractor.Extract(1, "ambiguous_m", content, symbols, "unknown.m"); + + Assert.DoesNotContain(references, reference => reference.SymbolName == "helper"); + } + [Fact] public void Extract_AmbiguousMRetainsSharedSafetyGuards_Issue4738() { From 22c4095ff86dced87052e6da98c0a72bffe54c7c Mon Sep 17 00:00:00 2001 From: Widthdom Date: Fri, 24 Jul 2026 18:26:04 +0900 Subject: [PATCH 15/21] Harden scientific qualified references (#4738) --- src/CodeIndex/Database/DbWriter.References.cs | 40 ++++-- .../Indexer/AmbiguousMContentMasker.cs | 88 +++++++++++-- .../ScientificNativeReferenceExtractor.cs | 43 ++++++ .../ReferenceExtractor.CoreExtraction.cs | 68 +++++++++- tests/CodeIndex.Tests/DatabaseTests.cs | 124 ++++++++++++++++++ ...eferenceExtractorTests.ScientificNative.cs | 80 +++++++++++ 6 files changed, 412 insertions(+), 31 deletions(-) diff --git a/src/CodeIndex/Database/DbWriter.References.cs b/src/CodeIndex/Database/DbWriter.References.cs index f9514e8e1..84779e69c 100644 --- a/src/CodeIndex/Database/DbWriter.References.cs +++ b/src/CodeIndex/Database/DbWriter.References.cs @@ -153,20 +153,32 @@ AND r.target_qualifier NOT LIKE char(31) || 'receiver:%' s.container_name = r.target_qualifier COLLATE NOCASE OR s.container_qualified_name = r.target_qualifier COLLATE NOCASE OR s.container_qualified_name LIKE '%.' || r.target_qualifier COLLATE NOCASE - OR EXISTS ( - SELECT 1 - FROM symbols AS target_scope - WHERE target_scope.file_id = s.file_id - AND target_scope.kind IN ('namespace', 'module', 'package') - AND ( - target_scope.name = r.target_qualifier COLLATE NOCASE - OR target_scope.container_qualified_name = r.target_qualifier COLLATE NOCASE - OR substr( - target_scope.name, - 1, - length(r.target_qualifier) + 1 - ) = (r.target_qualifier || '.') COLLATE NOCASE - ) + OR ( + source_file.lang IN ( + 'ada', + 'ambiguous_m', + 'cython', + 'd', + 'julia', + 'matlab', + 'nim', + 'objc' + ) + AND EXISTS ( + SELECT 1 + FROM symbols AS target_scope + WHERE target_scope.file_id = s.file_id + AND target_scope.kind IN ('namespace', 'module', 'package') + AND ( + target_scope.name = r.target_qualifier COLLATE NOCASE + OR target_scope.container_qualified_name = r.target_qualifier COLLATE NOCASE + OR substr( + target_scope.name, + 1, + length(r.target_qualifier) + 1 + ) = (r.target_qualifier || '.') COLLATE NOCASE + ) + ) ) ); diff --git a/src/CodeIndex/Indexer/AmbiguousMContentMasker.cs b/src/CodeIndex/Indexer/AmbiguousMContentMasker.cs index 85f7103b5..2bb217fe9 100644 --- a/src/CodeIndex/Indexer/AmbiguousMContentMasker.cs +++ b/src/CodeIndex/Indexer/AmbiguousMContentMasker.cs @@ -13,15 +13,36 @@ internal static string MaskComments( bool maskMatlabComments, bool maskObjectiveCComments, bool preserveObjectiveCModuloExpressions = false) + { + if (preserveObjectiveCModuloExpressions) + { + preserveObjectiveCModuloExpressions = + HasStrongObjectiveCModuloEvidence(content); + } + + return MaskContent( + content, + maskMatlabComments, + maskObjectiveCComments, + preserveObjectiveCModuloExpressions, + maskStrings: false); + } + + private static string MaskContent( + string content, + bool maskMatlabComments, + bool maskObjectiveCComments, + bool preserveObjectiveCModuloExpressions, + bool maskStrings) { if ((!maskMatlabComments || content.IndexOf('%') < 0) - && (!maskObjectiveCComments || content.IndexOf('/') < 0)) + && (!maskObjectiveCComments || content.IndexOf('/') < 0) + && (!maskStrings + || (content.IndexOf('"') < 0 && content.IndexOf('\'') < 0))) { return content; } - preserveObjectiveCModuloExpressions &= - HasStrongObjectiveCModuloEvidence(content); StringBuilder? masked = null; var inBlockComment = false; var inMatlabBlockComment = false; @@ -34,7 +55,12 @@ internal static string MaskComments( if (current is '\r' or '\n') { inLineComment = false; - quote = '\0'; + if (!maskStrings + || quote == '\0' + || !IsEscapedLineBreak(content, index)) + { + quote = '\0'; + } continue; } @@ -71,10 +97,15 @@ internal static string MaskComments( if (quote != '\0') { - if (preserveObjectiveCModuloExpressions + if (maskStrings) + MaskAt(index); + + if ((preserveObjectiveCModuloExpressions || maskStrings) && current == '\\' && index + 1 < content.Length) { + if (maskStrings && content[index + 1] is not ('\r' or '\n')) + MaskAt(index + 1); index++; continue; } @@ -83,6 +114,8 @@ internal static string MaskComments( { if (quote == '\'' && index + 1 < content.Length && content[index + 1] == '\'') { + if (maskStrings) + MaskAt(index + 1); index++; continue; } @@ -98,6 +131,8 @@ internal static string MaskComments( || IsMatlabSingleQuoteStart(content, index)))) { quote = current; + if (maskStrings) + MaskAt(index); continue; } @@ -243,14 +278,23 @@ private static bool IsObjectiveCModuloLeftOperandEnd(char value) => private static bool HasStrongObjectiveCModuloEvidence(string content) { + var evidenceContent = MaskContent( + content, + maskMatlabComments: true, + maskObjectiveCComments: true, + preserveObjectiveCModuloExpressions: false, + maskStrings: true); var lineStart = 0; - while (lineStart < content.Length) + while (lineStart < evidenceContent.Length) { var lineEnd = lineStart; - while (lineEnd < content.Length && content[lineEnd] is not ('\r' or '\n')) + while (lineEnd < evidenceContent.Length + && evidenceContent[lineEnd] is not ('\r' or '\n')) + { lineEnd++; + } - var line = content.AsSpan(lineStart, lineEnd - lineStart).TrimStart(); + var line = evidenceContent.AsSpan(lineStart, lineEnd - lineStart).TrimStart(); if (StartsWithObjectiveCPreprocessorDirective(line) || StartsWithObjectiveCAtKeyword(line) || StartsWithObjectiveCMethodDeclaration(line)) @@ -259,10 +303,10 @@ private static bool HasStrongObjectiveCModuloEvidence(string content) } lineStart = lineEnd + 1; - if (lineEnd < content.Length - && content[lineEnd] == '\r' - && lineStart < content.Length - && content[lineStart] == '\n') + if (lineEnd < evidenceContent.Length + && evidenceContent[lineEnd] == '\r' + && lineStart < evidenceContent.Length + && evidenceContent[lineStart] == '\n') { lineStart++; } @@ -271,6 +315,26 @@ private static bool HasStrongObjectiveCModuloEvidence(string content) return false; } + private static bool IsEscapedLineBreak(string content, int lineBreakIndex) + { + var previousIndex = lineBreakIndex - 1; + if (content[lineBreakIndex] == '\n' + && previousIndex >= 0 + && content[previousIndex] == '\r') + { + previousIndex--; + } + + var backslashCount = 0; + while (previousIndex >= 0 && content[previousIndex] == '\\') + { + backslashCount++; + previousIndex--; + } + + return backslashCount % 2 != 0; + } + private static bool StartsWithObjectiveCPreprocessorDirective(ReadOnlySpan line) { if (line.IsEmpty || line[0] != '#') diff --git a/src/CodeIndex/Indexer/References/Languages/ScientificNativeReferenceExtractor.cs b/src/CodeIndex/Indexer/References/Languages/ScientificNativeReferenceExtractor.cs index d2eee6c0a..438e82bf7 100644 --- a/src/CodeIndex/Indexer/References/Languages/ScientificNativeReferenceExtractor.cs +++ b/src/CodeIndex/Indexer/References/Languages/ScientificNativeReferenceExtractor.cs @@ -89,6 +89,43 @@ private readonly record struct DTemplateInvocation( internal static bool Supports(string language) => SupportedLanguages.Contains(language); + internal static string? GetParenthesizedCallTargetQualifier( + string preparedLine, + int callIndex) + { + var separatorIndex = callIndex - 1; + while (separatorIndex >= 0 && char.IsWhiteSpace(preparedLine[separatorIndex])) + separatorIndex--; + if (separatorIndex < 0 || preparedLine[separatorIndex] != '.') + return null; + + var segments = new List(); + while (separatorIndex >= 0 && preparedLine[separatorIndex] == '.') + { + var segmentEnd = separatorIndex; + var segmentStart = segmentEnd - 1; + while (segmentStart >= 0 && char.IsWhiteSpace(preparedLine[segmentStart])) + segmentStart--; + segmentEnd = segmentStart + 1; + while (segmentStart >= 0 && IsQualifierIdentifierPart(preparedLine[segmentStart])) + segmentStart--; + segmentStart++; + if (segmentStart >= segmentEnd + || !IsQualifierIdentifierStart(preparedLine[segmentStart])) + { + return null; + } + + segments.Add(preparedLine[segmentStart..segmentEnd]); + separatorIndex = segmentStart - 1; + while (separatorIndex >= 0 && char.IsWhiteSpace(preparedLine[separatorIndex])) + separatorIndex--; + } + + segments.Reverse(); + return string.Join('.', segments); + } + internal static bool IsDTemplateArgumentCall( IReadOnlyList? spans, ref int spanIndex, @@ -742,4 +779,10 @@ private static bool IsDIdentifierStart(char value) => private static bool IsDIdentifierPart(char value) => char.IsLetterOrDigit(value) || value == '_'; + + private static bool IsQualifierIdentifierStart(char value) => + char.IsLetter(value) || value == '_'; + + private static bool IsQualifierIdentifierPart(char value) => + char.IsLetterOrDigit(value) || value == '_'; } diff --git a/src/CodeIndex/Indexer/References/ReferenceExtractor.CoreExtraction.cs b/src/CodeIndex/Indexer/References/ReferenceExtractor.CoreExtraction.cs index 38171bc46..163686912 100644 --- a/src/CodeIndex/Indexer/References/ReferenceExtractor.CoreExtraction.cs +++ b/src/CodeIndex/Indexer/References/ReferenceExtractor.CoreExtraction.cs @@ -1610,6 +1610,26 @@ bool ShouldSuppressDefinitionCall(string resolvedName, string rawName, int callI } } + if (language == "julia") + { + var targetQualifier = + ScientificNativeReferenceExtractor.GetParenthesizedCallTargetQualifier( + preparedLine, + callIndex); + if (targetQualifier != null) + { + var qualifiedName = $"{targetQualifier}.{resolvedName}"; + var qualifiedDefinitionIndex = + preparedLine.IndexOf(qualifiedName, StringComparison.Ordinal); + if (qualifiedDefinitionIndex >= 0 + && callIndex == qualifiedDefinitionIndex + targetQualifier.Length + 1 + && definitionNames.Contains(qualifiedName)) + { + return true; + } + } + } + if (language != "sql") return TryGetDefinitionNameIndex(resolvedName, out var definitionIndex) && callIndex == definitionIndex; @@ -2298,7 +2318,14 @@ bool TryGetDefinitionNameIndex(string resolvedName, out int definitionIndex) } void AddCallLikeReference(string name, int callIndex) => - _ = TryAddCallLikeReference(name, callIndex); + _ = TryAddCallLikeReference( + name, + callIndex, + ScientificNativeReferenceExtractor.Supports(language) + ? ScientificNativeReferenceExtractor.GetParenthesizedCallTargetQualifier( + preparedLine, + callIndex) + : null); void AddPowerShellParameterReference(string name, int callIndex) { @@ -2306,7 +2333,10 @@ void AddPowerShellParameterReference(string name, int callIndex) AddReference(references, seen, fileId, name, callIndex, "parameter", context, lineNumber, callContainer, language); } - bool TryAddCallLikeReference(string name, int callIndex) + bool TryAddCallLikeReference( + string name, + int callIndex, + string? targetQualifier = null) { var normalizedName = language == "fsharp" && FSharpReferenceExtractor.IsOperatorCallName(name) ? $"operator {name}" @@ -2405,7 +2435,18 @@ bool TryAddCallLikeReference(string name, int callIndex) } if (IsConstructorCallName(language, preparedLine, callIndex)) { - AddReference(references, seen, fileId, normalizedName, callIndex, "instantiate", context, lineNumber, callContainer, language); + AddReference( + references, + seen, + fileId, + normalizedName, + callIndex, + "instantiate", + context, + lineNumber, + callContainer, + language, + targetQualifier); return true; } if (language == "rust" @@ -2472,7 +2513,17 @@ bool TryAddCallLikeReference(string name, int callIndex) return true; } - AddReference(references, seen, fileId, normalizedName, callIndex, "call", context, lineNumber, callContainer); + AddReference( + references, + seen, + fileId, + normalizedName, + callIndex, + "call", + context, + lineNumber, + callContainer, + targetQualifier: targetQualifier); return true; bool TryGetKnownPythonTypeCall(string candidate, out string canonicalName) @@ -2613,7 +2664,14 @@ bool TryGetKnownPythonTypeCall(string candidate, out string canonicalName) && sqlWindowFunctionCallSiteSuppressions.Contains((lineNumber, callIndex))) continue; GetMatchedCallIndices().Add(callIndex); - if (TryAddCallLikeReference(name, callIndex)) + if (TryAddCallLikeReference( + name, + callIndex, + ScientificNativeReferenceExtractor.Supports(language) + ? ScientificNativeReferenceExtractor.GetParenthesizedCallTargetQualifier( + preparedLine, + callIndex) + : null)) { EmitGenericInvocationTypeArgumentReferences( language, diff --git a/tests/CodeIndex.Tests/DatabaseTests.cs b/tests/CodeIndex.Tests/DatabaseTests.cs index 9e9ac9bcc..0fa653fa2 100644 --- a/tests/CodeIndex.Tests/DatabaseTests.cs +++ b/tests/CodeIndex.Tests/DatabaseTests.cs @@ -843,6 +843,130 @@ FROM symbol_references } } + [Fact] + public void ReferenceGraph_ModuleEvidenceDoesNotBroadenCSharpQualifiedCandidates_Issue4738() + { + var callerId = UpsertTestFileWithLanguage( + "src/caller.cs", + "csharp", + "qualified-csharp-caller"); + var targetId = UpsertTestFileWithLanguage( + "src/targets.cs", + "csharp", + "qualified-csharp-target"); + _writer.InsertSymbols([ + new SymbolRecord + { + FileId = targetId, + Kind = "namespace", + Name = "A", + Line = 1, + }, + new SymbolRecord + { + FileId = targetId, + Kind = "namespace", + Name = "B", + Line = 2, + }, + new SymbolRecord + { + FileId = targetId, + Kind = "class", + Name = "Thing", + ContainerName = "A", + Line = 3, + }, + new SymbolRecord + { + FileId = targetId, + Kind = "class", + Name = "Thing", + ContainerName = "B", + Line = 4, + }, + ]); + _writer.InsertReferences([ + new ReferenceRecord + { + FileId = callerId, + SymbolName = "Thing", + TargetQualifier = "A", + ReferenceKind = "type_reference", + Line = 1, + Column = 1, + Context = "A.Thing value;", + }, + ], refreshMutualRecursionFlags: false); + + _writer.RefreshMutualRecursionFlags(); + + var expectedTargetId = ExecuteScalarLong($""" + SELECT id + FROM symbols + WHERE file_id = {targetId.ToString(CultureInfo.InvariantCulture)} + AND name = 'Thing' + AND container_name = 'A' + """); + Assert.Equal(expectedTargetId, ExecuteScalarLong($""" + SELECT target_symbol_id + FROM symbol_references + WHERE file_id = {callerId.ToString(CultureInfo.InvariantCulture)} + """)); + Assert.Equal("resolved", ReadReferenceResolutionState(callerId)); + } + + [Fact] + public void ReferenceGraph_ExtractedScientificQualifierResolvesDuplicateLeaf_Issue4738() + { + const string callerContent = "void Run() { p1.Flush(); }\n"; + var callerId = UpsertTestFileWithLanguage( + "src/main.d", + "d", + "qualified-d-call-caller"); + var p1Id = UpsertTestFileWithLanguage( + "src/p1.d", + "d", + "qualified-d-call-p1"); + var p2Id = UpsertTestFileWithLanguage( + "src/p2.d", + "d", + "qualified-d-call-p2"); + var callerSymbols = SymbolExtractor.Extract(callerId, "d", callerContent); + _writer.InsertSymbols([ + .. callerSymbols, + new SymbolRecord { FileId = p1Id, Kind = "namespace", Name = "p1", Line = 1 }, + new SymbolRecord { FileId = p1Id, Kind = "function", Name = "Flush", Line = 2 }, + new SymbolRecord { FileId = p2Id, Kind = "namespace", Name = "p2", Line = 1 }, + new SymbolRecord { FileId = p2Id, Kind = "function", Name = "Flush", Line = 2 }, + ]); + var callerReferences = ReferenceExtractor.Extract( + callerId, + "d", + callerContent, + callerSymbols); + _writer.InsertReferences(callerReferences, refreshMutualRecursionFlags: false); + + _writer.RefreshMutualRecursionFlags(); + + var call = Assert.Single(callerReferences, reference => + reference.SymbolName == "Flush" && reference.ReferenceKind == "call"); + Assert.Equal("p1", call.TargetQualifier); + var expectedTargetId = ExecuteScalarLong($""" + SELECT id + FROM symbols + WHERE file_id = {p1Id.ToString(CultureInfo.InvariantCulture)} + AND name = 'Flush' + """); + Assert.Equal(expectedTargetId, ExecuteScalarLong($""" + SELECT target_symbol_id + FROM symbol_references + WHERE file_id = {callerId.ToString(CultureInfo.InvariantCulture)} + AND symbol_name = 'Flush' + """)); + Assert.Equal("resolved", ReadReferenceResolutionState(callerId)); + } + [Fact] public void ReferenceGraphDirtyScope_RollbackAndCancellationPreserveRetryState() { diff --git a/tests/CodeIndex.Tests/ReferenceExtractorTests.ScientificNative.cs b/tests/CodeIndex.Tests/ReferenceExtractorTests.ScientificNative.cs index eaf4ad03d..14c1be455 100644 --- a/tests/CodeIndex.Tests/ReferenceExtractorTests.ScientificNative.cs +++ b/tests/CodeIndex.Tests/ReferenceExtractorTests.ScientificNative.cs @@ -1271,6 +1271,39 @@ public void Extract_AmbiguousMMatlabCommentsDoNotMasqueradeAsModuloExpressions_I Assert.DoesNotContain(references, reference => reference.SymbolName == "helper"); } + [Theory] + [InlineData( + """ + %{ + @interface Fake + %} + function result = run(left) + result = left % helper(); + end + """)] + [InlineData( + """ + const char *text = "\ + @interface Fake"; + function result = run(left) + result = left % helper(); + end + """)] + public void Extract_AmbiguousMCommentedOrQuotedObjectiveCMarkersDoNotEnableModulo_Issue4738( + string content) + { + var symbols = SymbolExtractor.Extract(1, "ambiguous_m", content, "unknown.m"); + + var references = ReferenceExtractor.Extract( + 1, + "ambiguous_m", + content, + symbols, + "unknown.m"); + + Assert.DoesNotContain(references, reference => reference.SymbolName == "helper"); + } + [Fact] public void Extract_AmbiguousMRetainsSharedSafetyGuards_Issue4738() { @@ -1324,6 +1357,53 @@ public void Extract_QualifiedScientificBaseTypesUseResolvableLeafNames_Issue4738 Assert.Equal(expectedQualifier, reference.TargetQualifier); } + [Theory] + [InlineData("d", "void run() { pkg.tools.flush(); }\n", "flush", "pkg.tools")] + [InlineData("ada", "procedure Run is begin Pkg.Tools.Flush(); end Run;\n", "Flush", "Pkg.Tools")] + [InlineData("nim", "proc run() = pkg.tools.flush()\n", "flush", "pkg.tools")] + [InlineData("julia", "function run()\n Pkg.Tools.flush()\nend\n", "flush", "Pkg.Tools")] + [InlineData("cython", "def run():\n pkg.tools.flush()\n", "flush", "pkg.tools")] + [InlineData("matlab", "function run()\n pkg.tools.flush();\nend\n", "flush", "pkg.tools")] + public void Extract_QualifiedScientificCallsPreserveTargetQualifiers_Issue4738( + string language, + string content, + string expectedName, + string expectedQualifier) + { + var symbols = SymbolExtractor.Extract(1, language, content); + + var reference = Assert.Single( + ReferenceExtractor.Extract(1, language, content, symbols), + candidate => candidate.SymbolName == expectedName + && candidate.ReferenceKind == "call"); + + Assert.Equal(expectedQualifier, reference.TargetQualifier); + } + + [Fact] + public void Extract_QualifiedJuliaDefinitionsDoNotEmitPhantomCalls_Issue4738() + { + const string content = """ + module Base + function Base.foo(x) + helper() + end + Base.bar(x) = helper2() + end + """; + var symbols = SymbolExtractor.Extract(1, "julia", content); + + var references = ReferenceExtractor.Extract(1, "julia", content, symbols); + + Assert.DoesNotContain(references, reference => + reference.SymbolName is "foo" or "bar" + && reference.ReferenceKind == "call"); + Assert.Single(references, reference => + reference.SymbolName == "helper" && reference.ReferenceKind == "call"); + Assert.Single(references, reference => + reference.SymbolName == "helper2" && reference.ReferenceKind == "call"); + } + [Theory] [InlineData("objc")] [InlineData("ambiguous_m")] From 2755f7e7323ecb781440a082d96936373b2b4b04 Mon Sep 17 00:00:00 2001 From: Widthdom Date: Fri, 24 Jul 2026 19:16:58 +0900 Subject: [PATCH 16/21] Address scientific graph review findings (#4738) --- .../ReferenceExtractor.CoreExtraction.cs | 17 ++ .../ReferenceExtractor.TypeReferences.cs | 6 + .../Indexer/References/ReferenceExtractor.cs | 113 +++++++++++++- .../Symbols/SymbolExtractor.ExtractCore.cs | 6 +- .../SymbolExtractor.ScientificNative.cs | 9 ++ ...eferenceExtractorTests.ScientificNative.cs | 145 ++++++++++++++++++ 6 files changed, 291 insertions(+), 5 deletions(-) diff --git a/src/CodeIndex/Indexer/References/ReferenceExtractor.CoreExtraction.cs b/src/CodeIndex/Indexer/References/ReferenceExtractor.CoreExtraction.cs index 163686912..0aef032df 100644 --- a/src/CodeIndex/Indexer/References/ReferenceExtractor.CoreExtraction.cs +++ b/src/CodeIndex/Indexer/References/ReferenceExtractor.CoreExtraction.cs @@ -97,6 +97,11 @@ internal static List ExtractCore(ReferenceExtractionContext req var csharpAttrTopLevelRanges = csharpAttrTables.Item2; var definitionNamesComparer = GetDefinitionNamesComparer(language); var definitionNamesByLine = BuildDefinitionNamesByLine(language, symbols, request.ReportDiagnostic); + var scientificDefinitionNameIndicesByLine = BuildScientificDefinitionNameIndicesByLine( + language, + lines, + symbols, + definitionNamesByLine); var allDefinitionNames = language == "stylus" ? BuildAllDefinitionNames(language, symbols, request.ReportDiagnostic) : null; @@ -1360,6 +1365,10 @@ static string NormalizeCSharpBclRegexQualifiedName(string value) var definitionNames = definitionNamesByLine.TryGetValue(lineNumber, out var namesOnLine) ? namesOnLine : null; + Dictionary>? scientificDefinitionNameIndices = null; + scientificDefinitionNameIndicesByLine?.TryGetValue( + lineNumber, + out scientificDefinitionNameIndices); Dictionary? definitionNameIndices = null; List? sqlDefinitionLeafSpans = null; if (language == "sql") @@ -1610,6 +1619,14 @@ bool ShouldSuppressDefinitionCall(string resolvedName, string rawName, int callI } } + if (scientificDefinitionNameIndices != null + && scientificDefinitionNameIndices.TryGetValue( + resolvedName, + out var scientificDefinitionIndices)) + { + return scientificDefinitionIndices.Contains(callIndex); + } + if (language == "julia") { var targetQualifier = diff --git a/src/CodeIndex/Indexer/References/ReferenceExtractor.TypeReferences.cs b/src/CodeIndex/Indexer/References/ReferenceExtractor.TypeReferences.cs index 2bd14e7c1..1477651cd 100644 --- a/src/CodeIndex/Indexer/References/ReferenceExtractor.TypeReferences.cs +++ b/src/CodeIndex/Indexer/References/ReferenceExtractor.TypeReferences.cs @@ -2808,6 +2808,12 @@ private static string PrepareLine(string line, ReferenceLinePrepareOptions optio if (options.UsesPercentComments) { + // Outside strings, MATLAB treats `...` and the rest of the physical line as a + // continuation comment. MATLAB では文字列外の `...` 以降は継続コメントになる。 + var continuationIndex = result.IndexOf("...", StringComparison.Ordinal); + if (continuationIndex >= 0) + result = result[..continuationIndex]; + var percentCommentIndex = result.IndexOf('%'); if (percentCommentIndex >= 0) result = result[..percentCommentIndex]; diff --git a/src/CodeIndex/Indexer/References/ReferenceExtractor.cs b/src/CodeIndex/Indexer/References/ReferenceExtractor.cs index 72f2ac243..6dafbcdc9 100644 --- a/src/CodeIndex/Indexer/References/ReferenceExtractor.cs +++ b/src/CodeIndex/Indexer/References/ReferenceExtractor.cs @@ -424,7 +424,8 @@ private static bool IsFunctionLikeSymbolKind(string kind) }, ["d"] = new HashSet(StringComparer.Ordinal) { - "assert", "cast", "debug", "mixin", "scope", "static", "unittest", "version", + "__traits", "assert", "cast", "debug", "extern", "is", "mixin", "pragma", "scope", + "static", "unittest", "version", }, ["julia"] = new HashSet(StringComparer.Ordinal) { @@ -433,8 +434,8 @@ private static bool IsFunctionLikeSymbolKind(string kind) }, ["matlab"] = new HashSet(StringComparer.Ordinal) { - "case", "catch", "classdef", "elseif", "end", "function", "import", "methods", - "otherwise", "parfor", "properties", "spmd", + "arguments", "case", "catch", "classdef", "elseif", "end", "function", "import", + "methods", "otherwise", "parfor", "properties", "spmd", }, ["nim"] = new HashSet(StringComparer.Ordinal) { @@ -1426,6 +1427,112 @@ private static IReadOnlyDictionary> BuildDefinitionNamesByL return namesByLine; } + private static IReadOnlyDictionary>>? + BuildScientificDefinitionNameIndicesByLine( + string language, + IReadOnlyList lines, + IReadOnlyList symbols, + IReadOnlyDictionary> definitionNamesByLine) + { + if (!ScientificNativeReferenceExtractor.Supports(language) || symbols.Count == 0) + return null; + + var limits = GetSafetyLimits(); + var comparer = GetDefinitionNamesComparer(language); + var comparison = comparer == StringComparer.OrdinalIgnoreCase + ? StringComparison.OrdinalIgnoreCase + : StringComparison.Ordinal; + var indicesByLine = new Dictionary>>(); + for (var symbolIndex = 0; + symbolIndex < symbols.Count && symbolIndex < limits.MaxLookupSymbols; + symbolIndex++) + { + var symbol = symbols[symbolIndex]; + if (symbol.Line <= 0 + || symbol.Line > lines.Count + || !definitionNamesByLine.TryGetValue(symbol.Line, out var retainedNames) + || !retainedNames.Contains(symbol.Name)) + { + continue; + } + + var line = lines[symbol.Line - 1]; + var searchStart = Math.Clamp(symbol.StartColumn ?? 0, 0, line.Length); + var definitionIndex = FindScientificDefinitionNameIndex( + line, + symbol.Name, + searchStart, + comparison); + if (definitionIndex < 0) + continue; + + if (!indicesByLine.TryGetValue(symbol.Line, out var indicesByName)) + { + indicesByName = new Dictionary>(comparer); + indicesByLine[symbol.Line] = indicesByName; + } + + AddScientificDefinitionNameIndex( + indicesByName, + symbol.Name, + definitionIndex); + + var leafSeparatorIndex = symbol.Name.LastIndexOf('.'); + if (leafSeparatorIndex >= 0 && leafSeparatorIndex + 1 < symbol.Name.Length) + { + AddScientificDefinitionNameIndex( + indicesByName, + symbol.Name[(leafSeparatorIndex + 1)..], + definitionIndex + leafSeparatorIndex + 1); + } + } + + return indicesByLine; + } + + private static int FindScientificDefinitionNameIndex( + string line, + string name, + int searchStart, + StringComparison comparison) + { + while (searchStart <= line.Length - name.Length) + { + var index = line.IndexOf(name, searchStart, comparison); + if (index < 0) + return -1; + + var beforeIsBoundary = index == 0 + || !IsScientificDefinitionIdentifierChar(line[index - 1]); + var end = index + name.Length; + var afterIsBoundary = end == line.Length + || !IsScientificDefinitionIdentifierChar(line[end]); + if (beforeIsBoundary && afterIsBoundary) + return index; + + searchStart = index + 1; + } + + return -1; + } + + private static bool IsScientificDefinitionIdentifierChar(char value) + => char.IsLetterOrDigit(value) || value is '_' or '!' or '?' or '$'; + + private static void AddScientificDefinitionNameIndex( + Dictionary> indicesByName, + string name, + int index) + { + if (!indicesByName.TryGetValue(name, out var indices)) + { + indices = []; + indicesByName[name] = indices; + } + + indices.Add(index); + } + private static IReadOnlySet BuildAllDefinitionNames( string language, IReadOnlyList symbols, diff --git a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.ExtractCore.cs b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.ExtractCore.cs index 96d509ade..bb8b1a2b0 100644 --- a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.ExtractCore.cs +++ b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.ExtractCore.cs @@ -1609,8 +1609,10 @@ bool[] GetCssQualifiedRuleAncestors() => Name = name, Line = startLine, StartLine = startLine, - StartColumn = lang == "rust" && pattern.Kind == "function" - ? match.Groups["name"].Index + StartColumn = lang is "ada" or "cython" or "d" or "julia" or "matlab" or "nim" + ? lineOffset + match.Groups["name"].Index + : lang == "rust" && pattern.Kind == "function" + ? match.Groups["name"].Index : (csharpSingleLineCollapsedMatch ? csharpSignatureRawStartColumn : absoluteStartColumn), diff --git a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.ScientificNative.cs b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.ScientificNative.cs index c569babcb..5f5c813c4 100644 --- a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.ScientificNative.cs +++ b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.ScientificNative.cs @@ -88,6 +88,15 @@ private static (int EndLine, int? BodyStartLine, int? BodyEndLine) FindScientifi continue; } + // Julia comprehension clauses inside delimiters do not open `end`-terminated + // blocks. Julia の区切り記号内にある内包表記句は `end` 終端ブロックを開かない。 + if (language == "julia" + && delimiterFrames.Count > 0 + && keyword is "for" or "if") + { + continue; + } + depth++; } diff --git a/tests/CodeIndex.Tests/ReferenceExtractorTests.ScientificNative.cs b/tests/CodeIndex.Tests/ReferenceExtractorTests.ScientificNative.cs index 14c1be455..c3a239cdd 100644 --- a/tests/CodeIndex.Tests/ReferenceExtractorTests.ScientificNative.cs +++ b/tests/CodeIndex.Tests/ReferenceExtractorTests.ScientificNative.cs @@ -1499,4 +1499,149 @@ public void Extract_MatlabManyEndlessPeerFunctionsKeepIndependentLinearRanges_Is symbols, symbol => symbol.Name == $"f{functionCount - 1}").EndLine); } + + [Theory] + [InlineData( + "julia", + """ + function f() + f() + end + """, + "f")] + [InlineData( + "matlab", + """ + function f() + f(); + end + """, + "f")] + [InlineData( + "cython", + """ + def f(): + f() + """, + "f")] + [InlineData( + "nim", + """ + proc p() = + p() + """, + "p")] + [InlineData( + "d", + """ + id id() { + id(); + } + """, + "id")] + public void Extract_ScientificDeclarationIdentifierSpanDoesNotHideRecursiveCall_Issue4738( + string language, + string content, + string functionName) + { + var symbols = SymbolExtractor.Extract(1, language, content); + + var references = ReferenceExtractor.Extract(1, language, content, symbols); + + Assert.Single(references, reference => + reference.SymbolName == functionName && reference.ReferenceKind == "call"); + } + + [Fact] + public void Extract_JuliaMultilineComprehensionDoesNotExtendFunctionRange_Issue4738() + { + const string content = """ + function run(xs) + values = [ + helper(x) + for x in xs + if x > 0 + ] + after() + end + outside() + """; + var symbols = SymbolExtractor.Extract(1, "julia", content); + + var references = ReferenceExtractor.Extract(1, "julia", content, symbols); + + Assert.Equal(8, Assert.Single(symbols, symbol => symbol.Name == "run").EndLine); + Assert.Equal("run", Assert.Single(references, reference => + reference.SymbolName == "after" && reference.ReferenceKind == "call").ContainerName); + Assert.Null(Assert.Single(references, reference => + reference.SymbolName == "outside" && reference.ReferenceKind == "call").ContainerName); + } + + [Fact] + public void Extract_MatlabContinuationTailDoesNotEmitCalls_Issue4738() + { + const string content = """ + function run(value) + first(value, ... ignoredCall() + value); + realCall(); + end + """; + var symbols = SymbolExtractor.Extract(1, "matlab", content); + + var references = ReferenceExtractor.Extract(1, "matlab", content, symbols); + + Assert.Single(references, reference => + reference.SymbolName == "first" && reference.ReferenceKind == "call"); + Assert.Single(references, reference => + reference.SymbolName == "realCall" && reference.ReferenceKind == "call"); + Assert.DoesNotContain(references, reference => reference.SymbolName == "ignoredCall"); + } + + [Fact] + public void Extract_DCompileTimeConstructsDoNotEmitCalls_Issue4738() + { + const string content = """ + extern(C) void exported() { + static if (is(typeof(value) == int)) { + pragma(msg, __traits(compiles, helper())); + } + realCall(); + } + """; + var symbols = SymbolExtractor.Extract(1, "d", content); + + var references = ReferenceExtractor.Extract(1, "d", content, symbols); + + foreach (var name in new[] { "extern", "is", "pragma", "__traits" }) + { + Assert.DoesNotContain(references, reference => + reference.SymbolName == name && reference.ReferenceKind == "call"); + } + Assert.Single(references, reference => + reference.SymbolName == "helper" && reference.ReferenceKind == "call"); + Assert.Single(references, reference => + reference.SymbolName == "realCall" && reference.ReferenceKind == "call"); + } + + [Fact] + public void Extract_MatlabArgumentsBlockHeaderDoesNotEmitCall_Issue4738() + { + const string content = """ + function run(values) + arguments (Repeating) + values + end + helper(); + end + """; + var symbols = SymbolExtractor.Extract(1, "matlab", content); + + var references = ReferenceExtractor.Extract(1, "matlab", content, symbols); + + Assert.DoesNotContain(references, reference => + reference.SymbolName == "arguments" && reference.ReferenceKind == "call"); + Assert.Equal("run", Assert.Single(references, reference => + reference.SymbolName == "helper" && reference.ReferenceKind == "call").ContainerName); + } } From db271cd6ddc4505a8ce7837ac3f2edb98e4aa962 Mon Sep 17 00:00:00 2001 From: Widthdom Date: Fri, 24 Jul 2026 19:41:57 +0900 Subject: [PATCH 17/21] Fix adversarial scientific graph gaps (#4738) --- .../Database/DbReader.GraphQueries.cs | 38 ++++++---- src/CodeIndex/Database/DbWriter.References.cs | 2 + .../ScientificNativeReferenceExtractor.cs | 21 +++++- .../Indexer/Symbols/SymbolExtractor.cs | 7 +- tests/CodeIndex.Tests/DatabaseTests.cs | 72 +++++++++++++++++++ tests/CodeIndex.Tests/DbReaderImpactTests.cs | 46 ++++++++++++ ...eferenceExtractorTests.ScientificNative.cs | 29 ++++++++ 7 files changed, 199 insertions(+), 16 deletions(-) diff --git a/src/CodeIndex/Database/DbReader.GraphQueries.cs b/src/CodeIndex/Database/DbReader.GraphQueries.cs index 25ae552f6..dfecdf83d 100644 --- a/src/CodeIndex/Database/DbReader.GraphQueries.cs +++ b/src/CodeIndex/Database/DbReader.GraphQueries.cs @@ -936,13 +936,13 @@ ELSE 5 /// SQL 側で要求された LIMIT/OFFSET を適用し、呼び出し側が要求以上の中間ページを /// materialize しないようにする。 /// - private List GetCallersExact(string symbolName, int limit, int offset = 0, string? lang = null, IReadOnlyList? pathPatterns = null, IReadOnlyList? excludePathPatterns = null, bool excludeTests = false) - => GetCallersExactCore(symbolName, limit, offset, lang, pathPatterns, excludePathPatterns, excludeTests, targetSymbolId: null); + private List GetCallersExact(string symbolName, int limit, int offset = 0, string? lang = null, IReadOnlyList? pathPatterns = null, IReadOnlyList? excludePathPatterns = null, bool excludeTests = false, bool includeAmbiguousMSource = false) + => GetCallersExactCore(symbolName, limit, offset, lang, pathPatterns, excludePathPatterns, excludeTests, targetSymbolId: null, includeAmbiguousMSource); - private List GetCallersExactForTarget(string symbolName, long targetSymbolId, int limit, int offset, string? lang, IReadOnlyList? pathPatterns, IReadOnlyList? excludePathPatterns, bool excludeTests) - => GetCallersExactCore(symbolName, limit, offset, lang, pathPatterns, excludePathPatterns, excludeTests, targetSymbolId); + private List GetCallersExactForTarget(string symbolName, long targetSymbolId, int limit, int offset, string? lang, IReadOnlyList? pathPatterns, IReadOnlyList? excludePathPatterns, bool excludeTests, bool includeAmbiguousMSource = false) + => GetCallersExactCore(symbolName, limit, offset, lang, pathPatterns, excludePathPatterns, excludeTests, targetSymbolId, includeAmbiguousMSource); - private List GetCallersExactCore(string symbolName, int limit, int offset, string? lang, IReadOnlyList? pathPatterns, IReadOnlyList? excludePathPatterns, bool excludeTests, long? targetSymbolId) + private List GetCallersExactCore(string symbolName, int limit, int offset, string? lang, IReadOnlyList? pathPatterns, IReadOnlyList? excludePathPatterns, bool excludeTests, long? targetSymbolId, bool includeAmbiguousMSource) { if (!_hasReferencesTable) return new List(); using var cmd = _conn.CreateCommand(); @@ -1014,7 +1014,11 @@ AND r.reference_kind IN {CallGraphReferenceKindsSql} AND {supportedLangFilter} {targetCondition}"; if (lang != null) - sql += " AND f.lang = @lang"; + { + sql += includeAmbiguousMSource + ? " AND (f.lang = @lang OR f.lang = 'ambiguous_m')" + : " AND f.lang = @lang"; + } sql += BuildCSharpBareMemberGraphReferenceFilter(symbolName, lang, exact: true, contextSql, "f", "r"); AppendPathFilters(ref sql, pathPatterns, excludePathPatterns, excludeTests); sql += @" @@ -1142,6 +1146,14 @@ private static string BuildImpactVisitedKey(CallerResult caller, string callerNa && rootDefinitions[0].Lang == "csharp" ? rootDefinitions[0].SymbolId : null; + var ambiguousMRootSymbolId = hasResolvedIdentityGraph + && rootDefinitions.Count == 1 + && lang is "matlab" or "objc" + && string.Equals(rootDefinitions[0].Lang, lang, StringComparison.Ordinal) + ? rootDefinitions[0].SymbolId + : null; + var identityRootSymbolId = qualifiedRootSymbolId ?? ambiguousMRootSymbolId; + var includeAmbiguousMSource = ambiguousMRootSymbolId != null; var results = new List(); resultOffset = Math.Max(0, resultOffset); @@ -1204,9 +1216,9 @@ private static string BuildImpactVisitedKey(CallerResult caller, string callerNa while (discoveredResultCount < resultWindowEnd && fetchIterations < maxFetchIterations && !graphStateBudgetHit && !boundaryProbeBudgetHit) { fetchIterations++; - var page = depth == 0 && qualifiedRootSymbolId is long targetSymbolId - ? GetCallersExactForTarget(currentSymbol, targetSymbolId, pageSize, pageOffset, lang, pathPatterns, excludePathPatterns, excludeTests) - : GetCallersExact(currentSymbol, pageSize, pageOffset, lang, pathPatterns, excludePathPatterns, excludeTests); + var page = depth == 0 && identityRootSymbolId is long targetSymbolId + ? GetCallersExactForTarget(currentSymbol, targetSymbolId, pageSize, pageOffset, lang, pathPatterns, excludePathPatterns, excludeTests, includeAmbiguousMSource) + : GetCallersExact(currentSymbol, pageSize, pageOffset, lang, pathPatterns, excludePathPatterns, excludeTests, includeAmbiguousMSource); if (page.Count == 0) break; // No more callers for this symbol / このシンボルの caller は尽きた @@ -1353,7 +1365,8 @@ private static string BuildImpactVisitedKey(CallerResult caller, string callerNa lang, pathPatterns, excludePathPatterns, - excludeTests); + excludeTests, + includeAmbiguousMSource); maxDepthReached |= boundaryInspection.HasUnvisitedCaller; if (boundaryInspection.ProbeBudgetHit) { @@ -1452,7 +1465,8 @@ private ImpactBoundaryInspection InspectBoundaryCallers( string? lang, IReadOnlyList? pathPatterns, IReadOnlyList? excludePathPatterns, - bool excludeTests) + bool excludeTests, + bool includeAmbiguousMSource) { var offset = 0; var probes = 0; @@ -1462,7 +1476,7 @@ private ImpactBoundaryInspection InspectBoundaryCallers( return new ImpactBoundaryInspection(HasUnvisitedCaller: true, ProbeBudgetHit: true); var pageSize = Math.Min(ImpactBoundaryCallerProbePageSize, ImpactBoundaryCallerProbeBudget - probes); - var page = GetCallersExact(symbolName, pageSize, offset, lang, pathPatterns, excludePathPatterns, excludeTests); + var page = GetCallersExact(symbolName, pageSize, offset, lang, pathPatterns, excludePathPatterns, excludeTests, includeAmbiguousMSource); if (page.Count == 0) return new ImpactBoundaryInspection(HasUnvisitedCaller: false, ProbeBudgetHit: false); probes += page.Count; diff --git a/src/CodeIndex/Database/DbWriter.References.cs b/src/CodeIndex/Database/DbWriter.References.cs index 84779e69c..b17fb2ea1 100644 --- a/src/CodeIndex/Database/DbWriter.References.cs +++ b/src/CodeIndex/Database/DbWriter.References.cs @@ -164,6 +164,8 @@ source_file.lang IN ( 'nim', 'objc' ) + AND COALESCE(s.container_name, '') = '' + AND COALESCE(s.container_qualified_name, '') = '' AND EXISTS ( SELECT 1 FROM symbols AS target_scope diff --git a/src/CodeIndex/Indexer/References/Languages/ScientificNativeReferenceExtractor.cs b/src/CodeIndex/Indexer/References/Languages/ScientificNativeReferenceExtractor.cs index 438e82bf7..214bf2837 100644 --- a/src/CodeIndex/Indexer/References/Languages/ScientificNativeReferenceExtractor.cs +++ b/src/CodeIndex/Indexer/References/Languages/ScientificNativeReferenceExtractor.cs @@ -6,6 +6,9 @@ namespace CodeIndex.Indexer; internal static class ScientificNativeReferenceExtractor { + private const string JuliaIdentifierPattern = @"[\p{L}_]\w*"; + private const string JuliaCallableIdentifierPattern = JuliaIdentifierPattern + @"!?"; + internal readonly record struct DTemplateArgumentCallSpan(int Start, int EndExclusive); private readonly record struct DTemplateInvocation( string Name, @@ -43,10 +46,13 @@ private readonly record struct DTemplateInvocation( @"(?:<:|::)\s*(?[A-Z][A-Za-z0-9_]*(?:\.[A-Za-z_]\w*)*)", RegexOptions.Compiled | RegexOptions.CultureInvariant); private static readonly Regex JuliaMacroCallRegex = new( - @"(?[A-Za-z_]\w*)", + $@"(?{JuliaIdentifierPattern})", + RegexOptions.Compiled | RegexOptions.CultureInvariant); + private static readonly Regex JuliaBangCallRegex = new( + $@"(?{JuliaIdentifierPattern}!)\s*\(", RegexOptions.Compiled | RegexOptions.CultureInvariant); private static readonly Regex JuliaBroadcastCallRegex = new( - @"(?[A-Za-z_]\w*)\s*\.\s*\(", + $@"(?{JuliaCallableIdentifierPattern})\s*\.\s*\(", RegexOptions.Compiled | RegexOptions.CultureInvariant); private static readonly Regex DImportListRegex = new( @@ -96,6 +102,12 @@ private readonly record struct DTemplateInvocation( var separatorIndex = callIndex - 1; while (separatorIndex >= 0 && char.IsWhiteSpace(preparedLine[separatorIndex])) separatorIndex--; + if (separatorIndex >= 0 && preparedLine[separatorIndex] == '@') + { + separatorIndex--; + while (separatorIndex >= 0 && char.IsWhiteSpace(preparedLine[separatorIndex])) + separatorIndex--; + } if (separatorIndex < 0 || preparedLine[separatorIndex] != '.') return null; @@ -186,6 +198,11 @@ internal static bool IsDTemplateArgumentCall( var group = match.Groups["name"]; addCallLikeReference(group.Value, group.Index); } + foreach (Match match in JuliaBangCallRegex.Matches(preparedLine)) + { + var group = match.Groups["name"]; + addCallLikeReference(group.Value, group.Index); + } foreach (Match match in JuliaBroadcastCallRegex.Matches(preparedLine)) { var group = match.Groups["name"]; diff --git a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.cs b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.cs index 40a632392..c2bf5ba84 100644 --- a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.cs +++ b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.cs @@ -58,6 +58,9 @@ public static partial class SymbolExtractor private const int SymbolListInitialCapacityLineThreshold = 128; private const int SymbolListInitialCapacityMax = 1024; + private const string JuliaIdentifierPattern = @"[\p{L}_]\w*"; + private const string JuliaQualifiedCallableIdentifierPattern = + JuliaIdentifierPattern + @"(?:\." + JuliaIdentifierPattern + @")*!?"; private static string[] SplitContentLines(string content) => content.IndexOf('\n', StringComparison.Ordinal) < 0 ? [content] : content.Split('\n'); @@ -1649,9 +1652,9 @@ private enum JavaScriptTypeScriptFunctionHeaderConsumeResult new("namespace", new Regex(@"^\s*(?:baremodule|module)\s+(?[A-Za-z_]\w*)", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.ScientificEnd), new("struct", new Regex(@"^\s*(?:mutable\s+)?struct\s+(?[A-Za-z_]\w*)", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.ScientificEnd), new("type", new Regex(@"^\s*(?:abstract|primitive)\s+type\s+(?[A-Za-z_]\w*)", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.ScientificEnd), - new("function", new Regex(@"^\s*function\s+(?[A-Za-z_]\w*(?:\.[A-Za-z_]\w*)?)\s*(?:\(|\{)", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.ScientificEnd), + new("function", new Regex(@"^\s*function\s+(?" + JuliaQualifiedCallableIdentifierPattern + @")\s*(?:\(|\{)", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.ScientificEnd), new("function", new Regex(@"^\s*macro\s+(?[A-Za-z_]\w*)\s*(?:\(|$)", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.ScientificEnd), - new("function", new Regex(@"^\s*(?[A-Za-z_]\w*(?:\.[A-Za-z_]\w*)?)\s*\([^)\r\n]*\)\s*=", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.JuliaShortFunction), + new("function", new Regex(@"^\s*(?" + JuliaQualifiedCallableIdentifierPattern + @")\s*\([^)\r\n]*\)\s*=", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.JuliaShortFunction), new("property", new Regex(@"^\s*const\s+(?[A-Z_]\w*)\s*=", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None), new("import", new Regex(@"^\s*(?:using|import)\s+(?[A-Za-z_]\w*(?:\.[A-Za-z_]\w*)*)", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None), ], diff --git a/tests/CodeIndex.Tests/DatabaseTests.cs b/tests/CodeIndex.Tests/DatabaseTests.cs index 0fa653fa2..0d8d9aa1d 100644 --- a/tests/CodeIndex.Tests/DatabaseTests.cs +++ b/tests/CodeIndex.Tests/DatabaseTests.cs @@ -967,6 +967,78 @@ FROM symbol_references Assert.Equal("resolved", ReadReferenceResolutionState(callerId)); } + [Fact] + public void ReferenceGraph_JuliaQualifierSelectsContainerWithinSharedModuleFile_Issue4738() + { + const string targetContent = """ + module A + function foo() + end + macro trace(value) + value + end + end + module B + function foo() + end + macro trace(value) + value + end + end + """; + const string callerContent = """ + function run() + A.foo() + A.@trace 1 + end + """; + var targetId = UpsertTestFileWithLanguage( + "src/modules.jl", + "julia", + "qualified-julia-shared-module-target"); + var callerId = UpsertTestFileWithLanguage( + "src/caller.jl", + "julia", + "qualified-julia-shared-module-caller"); + var targetSymbols = SymbolExtractor.Extract(targetId, "julia", targetContent); + var callerSymbols = SymbolExtractor.Extract(callerId, "julia", callerContent); + _writer.InsertSymbols([.. targetSymbols, .. callerSymbols]); + var callerReferences = ReferenceExtractor.Extract( + callerId, + "julia", + callerContent, + callerSymbols); + _writer.InsertReferences(callerReferences, refreshMutualRecursionFlags: false); + + _writer.RefreshMutualRecursionFlags(); + + foreach (var name in new[] { "foo", "trace" }) + { + var reference = Assert.Single(callerReferences, candidate => + candidate.SymbolName == name && candidate.ReferenceKind == "call"); + Assert.Equal("A", reference.TargetQualifier); + var expectedTargetId = ExecuteScalarLong($""" + SELECT id + FROM symbols + WHERE file_id = {targetId.ToString(CultureInfo.InvariantCulture)} + AND name = '{name}' + AND container_name = 'A' + """); + Assert.Equal(expectedTargetId, ExecuteScalarLong($""" + SELECT target_symbol_id + FROM symbol_references + WHERE file_id = {callerId.ToString(CultureInfo.InvariantCulture)} + AND symbol_name = '{name}' + """)); + Assert.Equal("resolved", ExecuteScalarString($""" + SELECT resolution_state + FROM symbol_references + WHERE file_id = {callerId.ToString(CultureInfo.InvariantCulture)} + AND symbol_name = '{name}' + """)); + } + } + [Fact] public void ReferenceGraphDirtyScope_RollbackAndCancellationPreserveRetryState() { diff --git a/tests/CodeIndex.Tests/DbReaderImpactTests.cs b/tests/CodeIndex.Tests/DbReaderImpactTests.cs index 4af9ddb49..aa2ade755 100644 --- a/tests/CodeIndex.Tests/DbReaderImpactTests.cs +++ b/tests/CodeIndex.Tests/DbReaderImpactTests.cs @@ -40,6 +40,52 @@ public void Leaf() { } Assert.Contains("--max-hops 1", analysis.Suggestion, StringComparison.Ordinal); } + [Fact] + public void AnalyzeImpact_DefinitiveMatlabTargetTraversesAmbiguousMCallers_Issue4738() + { + InsertIndexedFile( + "src/target.m", + "matlab", + """ + function Target() + end + """); + InsertIndexedFile( + "src/caller.m", + "ambiguous_m", + """ + function Bridge() + Target(); + end + function Outer() + Bridge(); + end + """); + + var analysis = _reader.AnalyzeImpact( + "Target", + maxDepth: 2, + limit: 10, + lang: "matlab", + pathPatterns: ["src/*.m"]); + + Assert.Equal("callers", analysis.ImpactMode); + Assert.Collection( + analysis.Callers.OrderBy(caller => caller.Depth), + caller => + { + Assert.Equal("ambiguous_m", caller.Lang); + Assert.Equal("Bridge", caller.CallerName); + Assert.Equal(1, caller.Depth); + }, + caller => + { + Assert.Equal("ambiguous_m", caller.Lang); + Assert.Equal("Outer", caller.CallerName); + Assert.Equal(2, caller.Depth); + }); + } + [Fact] public void AnalyzeImpact_CSharpVerbatimQueryKeepsOriginalInputOnMiss() { diff --git a/tests/CodeIndex.Tests/ReferenceExtractorTests.ScientificNative.cs b/tests/CodeIndex.Tests/ReferenceExtractorTests.ScientificNative.cs index c3a239cdd..4441d9d09 100644 --- a/tests/CodeIndex.Tests/ReferenceExtractorTests.ScientificNative.cs +++ b/tests/CodeIndex.Tests/ReferenceExtractorTests.ScientificNative.cs @@ -1577,6 +1577,35 @@ function run(xs) reference.SymbolName == "outside" && reference.ReferenceKind == "call").ContainerName); } + [Fact] + public void Extract_JuliaBangCallablesAndQualifiedMacrosPreserveGraphIdentity_Issue4738() + { + const string content = """ + function mutate!(value) + value + end + push!(values, value) = values + function run(values) + A.mutate!(values) + A.push!.(values) + A.@trace values + end + """; + var symbols = SymbolExtractor.Extract(1, "julia", content); + + var references = ReferenceExtractor.Extract(1, "julia", content, symbols); + + Assert.Contains(symbols, symbol => symbol.Name == "mutate!"); + Assert.Contains(symbols, symbol => symbol.Name == "push!"); + foreach (var name in new[] { "mutate!", "push!", "trace" }) + { + var reference = Assert.Single(references, candidate => + candidate.SymbolName == name && candidate.ReferenceKind == "call"); + Assert.Equal("A", reference.TargetQualifier); + Assert.Equal("run", reference.ContainerName); + } + } + [Fact] public void Extract_MatlabContinuationTailDoesNotEmitCalls_Issue4738() { From 154d0dde2b6bdc4ebba65b77c11d8063d73a3c3b Mon Sep 17 00:00:00 2001 From: Widthdom Date: Fri, 24 Jul 2026 20:10:47 +0900 Subject: [PATCH 18/21] Fix scientific graph review edge cases (#4738) --- .../DbWriter.ReferenceGraphRefreshScope.cs | 22 ++++++ src/CodeIndex/Database/DbWriter.References.cs | 39 ++++++++-- .../ScientificNativeCommentMasker.cs | 33 ++++++++- .../ReferenceExtractor.TypeReferences.cs | 9 ++- tests/CodeIndex.Tests/DatabaseTests.cs | 74 +++++++++++++++++++ ...eferenceExtractorTests.ScientificNative.cs | 36 +++++++++ 6 files changed, 201 insertions(+), 12 deletions(-) diff --git a/src/CodeIndex/Database/DbWriter.ReferenceGraphRefreshScope.cs b/src/CodeIndex/Database/DbWriter.ReferenceGraphRefreshScope.cs index 5f8195471..bbc65bc9d 100644 --- a/src/CodeIndex/Database/DbWriter.ReferenceGraphRefreshScope.cs +++ b/src/CodeIndex/Database/DbWriter.ReferenceGraphRefreshScope.cs @@ -61,7 +61,29 @@ CROSS JOIN symbols AS s INDEXED BY idx_symbols_name_folded JOIN files AS target_file ON target_file.id = s.file_id WHERE s.name_folded = dirty_name.name_folded AND target_file.lang = dirty_name.lang + AND target_file.lang <> 'ambiguous_m' GROUP BY target_file.lang, s.name_folded + HAVING COUNT(DISTINCT target_file.path || char(31) || + COALESCE(s.container_qualified_name, s.container_name, '') || char(31) || + COALESCE(s.name, '')) = 1; + + -- Keep the scoped projection aligned with the full-refresh union-wide + -- uniqueness contract for callers whose .m dialect is unresolved. + -- .m 方言が未確定な呼出し元について、差分更新でも全件更新と同じ + -- 言語横断の一意性契約を維持する。 + INSERT INTO temp.reference_unique_symbol_families(lang, name_folded, family_key) + SELECT 'ambiguous_m', + s.name_folded, + MIN(target_file.path || char(31) || + COALESCE(s.container_qualified_name, s.container_name, '') || char(31) || + COALESCE(s.name, '')) AS family_key + FROM temp.{ReferenceGraphLookupNamesTable} AS dirty_name + CROSS JOIN symbols AS s INDEXED BY idx_symbols_name_folded + JOIN files AS target_file ON target_file.id = s.file_id + WHERE dirty_name.lang = 'ambiguous_m' + AND s.name_folded = dirty_name.name_folded + AND target_file.lang IN ('matlab', 'objc') + GROUP BY s.name_folded HAVING COUNT(DISTINCT target_file.path || char(31) || COALESCE(s.container_qualified_name, s.container_name, '') || char(31) || COALESCE(s.name, '')) = 1; diff --git a/src/CodeIndex/Database/DbWriter.References.cs b/src/CodeIndex/Database/DbWriter.References.cs index b17fb2ea1..0222ce28f 100644 --- a/src/CodeIndex/Database/DbWriter.References.cs +++ b/src/CodeIndex/Database/DbWriter.References.cs @@ -122,7 +122,27 @@ INSERT INTO temp.reference_unique_symbol_families(lang, name_folded, family_key) FROM symbols AS s JOIN files AS target_file ON target_file.id = s.file_id WHERE s.name_folded IS NOT NULL + AND target_file.lang <> 'ambiguous_m' GROUP BY target_file.lang, s.name_folded + HAVING COUNT(DISTINCT target_file.path || char(31) || + COALESCE(s.container_qualified_name, s.container_name, '') || char(31) || + COALESCE(s.name, '')) = 1; + + -- An ambiguous .m caller can bind to either dialect, so uniqueness must hold + -- across the MATLAB/Objective-C union rather than within either language alone. + -- ambiguous .m の呼出し先は両方の方言になり得るため、一意性は各言語内ではなく + -- MATLAB/Objective-C の和集合全体で成立させる。 + INSERT INTO temp.reference_unique_symbol_families(lang, name_folded, family_key) + SELECT 'ambiguous_m', + s.name_folded, + MIN(target_file.path || char(31) || + COALESCE(s.container_qualified_name, s.container_name, '') || char(31) || + COALESCE(s.name, '')) AS family_key + FROM symbols AS s + JOIN files AS target_file ON target_file.id = s.file_id + WHERE s.name_folded IS NOT NULL + AND target_file.lang IN ('matlab', 'objc') + GROUP BY s.name_folded HAVING COUNT(DISTINCT target_file.path || char(31) || COALESCE(s.container_qualified_name, s.container_name, '') || char(31) || COALESCE(s.name, '')) = 1; @@ -326,16 +346,21 @@ INSERT INTO symbol_reference_candidates(reference_id, symbol_id, scope_rank) FROM symbol_references AS r JOIN files AS source_file ON source_file.id = r.file_id JOIN temp.reference_unique_symbol_families AS unique_family - ON ( - (unique_family.lang = source_file.lang - AND source_file.lang <> 'ambiguous_m') - OR (source_file.lang = 'ambiguous_m' AND unique_family.lang IN ('matlab', 'objc')) - ) + ON unique_family.lang = source_file.lang AND unique_family.name_folded = r.symbol_name_folded JOIN symbols AS target ON target.name_folded = unique_family.name_folded JOIN files AS target_file - ON target_file.id = target.file_id - AND target_file.lang = unique_family.lang + ON target_file.id = target.file_id + AND ( + ( + unique_family.lang <> 'ambiguous_m' + AND target_file.lang = unique_family.lang + ) + OR ( + unique_family.lang = 'ambiguous_m' + AND target_file.lang IN ('matlab', 'objc') + ) + ) AND target_file.path || char(31) || COALESCE(target.container_qualified_name, target.container_name, '') || char(31) || COALESCE(target.name, '') = unique_family.family_key diff --git a/src/CodeIndex/Indexer/References/Languages/ScientificNativeCommentMasker.cs b/src/CodeIndex/Indexer/References/Languages/ScientificNativeCommentMasker.cs index c3df39d7c..ca5ee7421 100644 --- a/src/CodeIndex/Indexer/References/Languages/ScientificNativeCommentMasker.cs +++ b/src/CodeIndex/Indexer/References/Languages/ScientificNativeCommentMasker.cs @@ -27,7 +27,8 @@ internal static string[] MaskBlockComments(string language, string[] lines) => internal static string MaskLineStringLiteralsPreservingPostfixSingleQuotes( string line, bool useMatlabStringRules, - bool useBackslashEscapes) + bool useBackslashEscapes, + bool singleQuoteCanBePostfix = true) { char[]? chars = null; @@ -41,6 +42,7 @@ void MaskAt(int index) => continue; if (quote == '\'' + && singleQuoteCanBePostfix && IsPostfixSingleQuote( line, cursor, @@ -241,6 +243,32 @@ internal static string MaskNimRawStringLiterals(string line) char[]? chars = null; for (var cursor = 0; cursor < line.Length;) { + if (line[cursor] == '"') + { + if (cursor + 2 < line.Length + && line[cursor + 1] == '"' + && line[cursor + 2] == '"') + { + var closing = line.IndexOf("\"\"\"", cursor + 3, StringComparison.Ordinal); + cursor = closing < 0 ? line.Length : closing + 3; + continue; + } + + cursor++; + while (cursor < line.Length) + { + if (line[cursor] == '\\' && cursor + 1 < line.Length) + { + cursor += 2; + continue; + } + + if (line[cursor++] == '"') + break; + } + continue; + } + if (!(char.IsLetter(line[cursor]) || line[cursor] == '_') || (cursor > 0 && IsIdentifierChar(line[cursor - 1]))) { @@ -263,7 +291,8 @@ internal static string MaskNimRawStringLiterals(string line) && line[cursor + 2] == '"') { // Leave the triple-quote delimiter for the stateful multiline masker. - cursor += 3; + var closing = line.IndexOf("\"\"\"", cursor + 3, StringComparison.Ordinal); + cursor = closing < 0 ? line.Length : closing + 3; continue; } diff --git a/src/CodeIndex/Indexer/References/ReferenceExtractor.TypeReferences.cs b/src/CodeIndex/Indexer/References/ReferenceExtractor.TypeReferences.cs index 1477651cd..5320f86a5 100644 --- a/src/CodeIndex/Indexer/References/ReferenceExtractor.TypeReferences.cs +++ b/src/CodeIndex/Indexer/References/ReferenceExtractor.TypeReferences.cs @@ -2715,6 +2715,7 @@ private readonly record struct ReferenceLinePrepareOptions( bool MaskStringLiterals, bool MaskNimRawStrings, bool IncludeBacktickStringDelimiter, + bool PreserveStringLiteralLength, bool PreservePostfixSingleQuotes, bool UseMatlabStringRules, bool ScientificStringUsesBackslashEscapes, @@ -2734,9 +2735,10 @@ private static ReferenceLinePrepareOptions CreateReferenceLinePrepareOptions(str MaskStringLiterals: lang != "cobol", MaskNimRawStrings: lang == "nim", IncludeBacktickStringDelimiter: lang is not ("kotlin" or "r"), + PreserveStringLiteralLength: ScientificNativeReferenceExtractor.Supports(lang), PreservePostfixSingleQuotes: lang is "ada" or "julia" or "matlab", UseMatlabStringRules: lang == "matlab", - ScientificStringUsesBackslashEscapes: lang == "julia", + ScientificStringUsesBackslashEscapes: lang is "cython" or "d" or "julia" or "nim" or "objc", UsesHashComments: UsesHashComments(lang), UsesRHashComments: lang == "r", UsesSlashComments: UsesSlashComments(lang), @@ -2764,12 +2766,13 @@ private static string PrepareLine(string line, ReferenceLinePrepareOptions optio result = ScientificNativeCommentMasker.MaskNimRawStringLiterals(result); if (options.MaskStringLiterals && MayContainStringLiteralDelimiter(result, options.IncludeBacktickStringDelimiter)) { - if (options.PreservePostfixSingleQuotes) + if (options.PreserveStringLiteralLength) { result = ScientificNativeCommentMasker.MaskLineStringLiteralsPreservingPostfixSingleQuotes( result, options.UseMatlabStringRules, - options.ScientificStringUsesBackslashEscapes); + options.ScientificStringUsesBackslashEscapes, + options.PreservePostfixSingleQuotes); } else { diff --git a/tests/CodeIndex.Tests/DatabaseTests.cs b/tests/CodeIndex.Tests/DatabaseTests.cs index 0d8d9aa1d..da1bab41b 100644 --- a/tests/CodeIndex.Tests/DatabaseTests.cs +++ b/tests/CodeIndex.Tests/DatabaseTests.cs @@ -762,6 +762,80 @@ FROM symbol_reference_candidates } } + [Fact] + public void ReferenceGraph_AmbiguousMUniqueFallbackRequiresUnionWideUniqueness_Issue4738() + { + var callerId = UpsertTestFileWithLanguage( + "src/caller.m", + "ambiguous_m", + "ambiguous-union-caller"); + var objectiveCTargetId = UpsertTestFileWithLanguage( + "src/objective-c-target.m", + "objc", + "ambiguous-union-objc"); + _writer.InsertSymbols([ + new SymbolRecord { FileId = objectiveCTargetId, Kind = "function", Name = "Foo", Line = 1 }, + ]); + _writer.InsertReferences([ + new ReferenceRecord + { + FileId = callerId, + SymbolName = "Foo", + ReferenceKind = "call", + Line = 1, + Column = 1, + Context = "Foo();", + }, + ], refreshMutualRecursionFlags: false); + + _writer.RefreshMutualRecursionFlags(); + + Assert.Equal("resolved", ReadReferenceResolutionState(callerId)); + + using (var scope = _writer.BeginReferenceGraphRefreshScope()) + { + using var transaction = _writer.BeginTransaction(); + var firstMatlabTargetId = _writer.InsertNewFile(new FileRecord + { + Path = "src/first-matlab-target.m", + Lang = "matlab", + Size = 32, + Lines = 1, + Modified = new DateTime(2026, 7, 24, 0, 0, 0, DateTimeKind.Utc), + Checksum = "ambiguous-union-matlab-first", + }); + var secondMatlabTargetId = _writer.InsertNewFile(new FileRecord + { + Path = "src/second-matlab-target.m", + Lang = "matlab", + Size = 32, + Lines = 1, + Modified = new DateTime(2026, 7, 24, 0, 0, 0, DateTimeKind.Utc), + Checksum = "ambiguous-union-matlab-second", + }); + _writer.InsertSymbols([ + new SymbolRecord { FileId = firstMatlabTargetId, Kind = "function", Name = "Foo", Line = 1 }, + new SymbolRecord { FileId = secondMatlabTargetId, Kind = "function", Name = "Foo", Line = 1 }, + ]); + transaction.Commit(); + _writer.RefreshMutualRecursionFlags(); + } + + Assert.Equal("unresolved", ReadReferenceResolutionState(callerId)); + Assert.Equal(0, ExecuteScalarLong($""" + SELECT COUNT(*) + FROM symbol_references + WHERE file_id = {callerId.ToString(CultureInfo.InvariantCulture)} + AND target_symbol_id IS NOT NULL + """)); + Assert.Equal(0, ExecuteScalarLong($""" + SELECT COUNT(*) + FROM symbol_reference_candidates AS candidate + JOIN symbol_references AS reference ON reference.id = candidate.reference_id + WHERE reference.file_id = {callerId.ToString(CultureInfo.InvariantCulture)} + """)); + } + [Fact] public void ReferenceGraph_ScientificQualifiersUseModuleEvidenceOrSafeUniqueFallback_Issue4738() { diff --git a/tests/CodeIndex.Tests/ReferenceExtractorTests.ScientificNative.cs b/tests/CodeIndex.Tests/ReferenceExtractorTests.ScientificNative.cs index 4441d9d09..7bd534f6b 100644 --- a/tests/CodeIndex.Tests/ReferenceExtractorTests.ScientificNative.cs +++ b/tests/CodeIndex.Tests/ReferenceExtractorTests.ScientificNative.cs @@ -1606,6 +1606,42 @@ A.@trace values } } + public static TheoryData ScientificNativeFollowingStringReferenceCases => new() + { + { + "cython", + """def run(): consume("a very long string", pkg.helper())""" + }, + { + "d", + """void run() { consume("a very long string", pkg.helper()); }""" + }, + { + "nim", + """proc run() = consume("a very long string", pkg.helper())""" + }, + { + "objc", + """void run() { consume(@"a very long string", pkg.helper()); }""" + }, + }; + + [Theory] + [MemberData(nameof(ScientificNativeFollowingStringReferenceCases))] + public void Extract_ScientificNativeStringsPreserveFollowingReferenceColumns_Issue4738( + string language, + string content) + { + var symbols = SymbolExtractor.Extract(1, language, content); + + var references = ReferenceExtractor.Extract(1, language, content, symbols); + + var helper = Assert.Single(references, reference => + reference.SymbolName == "helper" && reference.ReferenceKind == "call"); + Assert.Equal("pkg", helper.TargetQualifier); + Assert.Equal(content.IndexOf("helper", StringComparison.Ordinal) + 1, helper.Column); + } + [Fact] public void Extract_MatlabContinuationTailDoesNotEmitCalls_Issue4738() { From 94a4f915c20812c3b3f57e42ca5731efce9de0a1 Mon Sep 17 00:00:00 2001 From: Widthdom Date: Fri, 24 Jul 2026 21:02:04 +0900 Subject: [PATCH 19/21] Resolve final scientific graph review findings (#4738) --- .../Database/DbReader.GraphQueries.cs | 20 +-- src/CodeIndex/Database/DbReader.References.cs | 6 +- src/CodeIndex/Database/DbReader.cs | 5 + .../Database/DbSymbolReader.Definitions.cs | 2 +- .../Database/DbSymbolReader.Search.cs | 16 ++- src/CodeIndex/Database/DbWriter.BatchSql.cs | 8 ++ .../Database/DbWriter.ChunkSymbolBatches.cs | 5 +- .../Database/DbWriter.FoldBackfill.cs | 89 ++++++++++---- src/CodeIndex/Database/DbWriter.References.cs | 13 +- .../Indexer/NimIdentifierIdentity.cs | 32 +++++ .../ScientificNativeReferenceExtractor.cs | 8 ++ .../ReferenceExtractor.CoreExtraction.cs | 4 + .../ReferenceExtractor.Preparation.cs | 2 +- .../Indexer/References/ReferenceExtractor.cs | 13 +- .../Symbols/SymbolExtractor.ExtractCore.cs | 5 + .../Indexer/Symbols/SymbolExtractor.Pascal.cs | 90 ++++++++++++++ .../Indexer/Symbols/SymbolExtractor.cs | 12 +- src/CodeIndex/Models/ReferenceRecord.cs | 9 ++ src/CodeIndex/Models/SymbolRecord.cs | 3 + tests/CodeIndex.Tests/DatabaseTests.cs | 116 ++++++++++++++++++ ...eferenceExtractorTests.ScientificNative.cs | 93 ++++++++++++++ 21 files changed, 498 insertions(+), 53 deletions(-) create mode 100644 src/CodeIndex/Indexer/NimIdentifierIdentity.cs diff --git a/src/CodeIndex/Database/DbReader.GraphQueries.cs b/src/CodeIndex/Database/DbReader.GraphQueries.cs index dfecdf83d..89398dede 100644 --- a/src/CodeIndex/Database/DbReader.GraphQueries.cs +++ b/src/CodeIndex/Database/DbReader.GraphQueries.cs @@ -195,7 +195,7 @@ FROM logical_references r if (!exact) callersQueryParam = $"%{EscapeLikeQuery(query)}%"; else if (_foldReady) - callersQueryParam = NameFold.Fold(query) ?? query; + callersQueryParam = FoldNameForLanguage(query, lang); else callersQueryParam = query; SqliteCommandPolicy.Add(cmd, "@query", callersQueryParam); @@ -339,7 +339,7 @@ FROM symbol_references r var value = !exact ? $"%{EscapeLikeQuery(query)}%" : _foldReady - ? NameFold.Fold(query) ?? query + ? FoldNameForLanguage(query, lang) : query; SqliteCommandPolicy.Add(cmd, "@query", value); SqliteCommandPolicy.Add(cmd, "@aliasQuery", query); @@ -446,7 +446,7 @@ FROM symbol_references r var value = !exact ? $"%{EscapeLikeQuery(query)}%" : _foldReady - ? NameFold.Fold(query) ?? query + ? FoldNameForLanguage(query, lang) : query; SqliteCommandPolicy.Add(cmd, "@query", value); SqliteCommandPolicy.Add(cmd, "@aliasQuery", query); @@ -572,7 +572,7 @@ FROM logical_references r if (!exact) calleesQueryParam = $"%{EscapeLikeQuery(query)}%"; else if (_foldReady) - calleesQueryParam = NameFold.Fold(query) ?? query; + calleesQueryParam = FoldNameForLanguage(query, lang); else calleesQueryParam = query; SqliteCommandPolicy.Add(cmd, "@query", calleesQueryParam); @@ -704,7 +704,7 @@ FROM symbol_references r var value = !exact ? $"%{EscapeLikeQuery(query)}%" : _foldReady - ? NameFold.Fold(query) ?? query + ? FoldNameForLanguage(query, lang) : query; SqliteCommandPolicy.Add(cmd, "@query", value); SqliteCommandPolicy.Add(cmd, "@aliasQuery", query); @@ -803,7 +803,7 @@ FROM symbol_references r var value = !exact ? $"%{EscapeLikeQuery(query)}%" : _foldReady - ? NameFold.Fold(query) ?? query + ? FoldNameForLanguage(query, lang) : query; SqliteCommandPolicy.Add(cmd, "@query", value); SqliteCommandPolicy.Add(cmd, "@aliasQuery", query); @@ -1042,7 +1042,7 @@ FROM logical_references r SqliteCommandPolicy.Add(cmd, "@aliasQueryLeafFolded", NameFold.Fold(SqlNameResolver.GetLeafName(symbolName)) ?? SqlNameResolver.GetLeafName(symbolName)); SqliteCommandPolicy.Add(cmd, "@symbolNameLeafFolded", NameFold.Fold(SqlNameResolver.GetLeafName(symbolName)) ?? SqlNameResolver.GetLeafName(symbolName)); if (_foldReady) - SqliteCommandPolicy.Add(cmd, "@symbolNameFolded", NameFold.Fold(symbolName) ?? symbolName); + SqliteCommandPolicy.Add(cmd, "@symbolNameFolded", FoldNameForLanguage(symbolName, lang)); for (var i = 0; i < polymorphicCSharpSymbolNames.Count; i++) { if (_foldReady) @@ -2177,9 +2177,9 @@ CROSS JOIN definition_stats stats cmd.CommandText = sql; SqliteCommandPolicy.Add(cmd, "@resolvedName", resolvedName); SqliteCommandPolicy.Add(cmd, "@resolvedNameNormalized", normalizedName); - SqliteCommandPolicy.Add(cmd, "@resolvedNameNormalizedFolded", NameFold.Fold(normalizedName) ?? normalizedName); + SqliteCommandPolicy.Add(cmd, "@resolvedNameNormalizedFolded", FoldNameForLanguage(normalizedName, lang)); SqliteCommandPolicy.Add(cmd, "@resolvedNameLeaf", leafName); - SqliteCommandPolicy.Add(cmd, "@resolvedNameLeafFolded", NameFold.Fold(leafName) ?? leafName); + SqliteCommandPolicy.Add(cmd, "@resolvedNameLeafFolded", FoldNameForLanguage(leafName, lang)); SqliteCommandPolicy.Add(cmd, "@resolvedNameSegmentCount", segmentCount); SqliteCommandPolicy.Add(cmd, "@allowLeafFallback", allowLeafFallback ? 1 : 0); if (SqlNameResolver.HasQualifier(resolvedName)) @@ -2189,7 +2189,7 @@ CROSS JOIN definition_stats stats SqliteCommandPolicy.Add(cmd, "@resolvedNameContainerSuffix", $"%.{EscapeLikeQuery(container)}"); } if (_foldReady) - SqliteCommandPolicy.Add(cmd, "@resolvedNameFolded", NameFold.Fold(resolvedName) ?? resolvedName); + SqliteCommandPolicy.Add(cmd, "@resolvedNameFolded", FoldNameForLanguage(resolvedName, lang)); if (lang != null) SqliteCommandPolicy.Add(cmd, "@lang", lang); SqliteCommandPolicy.Add(cmd, "@definitionLimit", Math.Max(1, representativeLimit)); diff --git a/src/CodeIndex/Database/DbReader.References.cs b/src/CodeIndex/Database/DbReader.References.cs index f13bf0f41..43005ce1c 100644 --- a/src/CodeIndex/Database/DbReader.References.cs +++ b/src/CodeIndex/Database/DbReader.References.cs @@ -408,7 +408,7 @@ FROM symbol_reference_candidates AS identity_candidate if (!exact) queryParam = $"%{EscapeLikeQuery(query)}%"; else if (_foldReady) - queryParam = NameFold.Fold(query) ?? query; + queryParam = FoldNameForLanguage(query, lang); else queryParam = query; SqliteCommandPolicy.Add(cmd, "@query", queryParam); @@ -784,7 +784,7 @@ FROM symbol_references r var value = !exact ? $"%{EscapeLikeQuery(query)}%" : _foldReady - ? NameFold.Fold(query) ?? query + ? FoldNameForLanguage(query, lang) : query; SqliteCommandPolicy.Add(cmd, "@query", value); SqliteCommandPolicy.Add(cmd, "@aliasQuery", query); @@ -923,7 +923,7 @@ FROM symbol_references r var value = !exact ? $"%{EscapeLikeQuery(query)}%" : _foldReady - ? NameFold.Fold(query) ?? query + ? FoldNameForLanguage(query, lang) : query; SqliteCommandPolicy.Add(cmd, "@query", value); SqliteCommandPolicy.Add(cmd, "@aliasQuery", query); diff --git a/src/CodeIndex/Database/DbReader.cs b/src/CodeIndex/Database/DbReader.cs index 7d8286df2..18927b04f 100644 --- a/src/CodeIndex/Database/DbReader.cs +++ b/src/CodeIndex/Database/DbReader.cs @@ -1659,6 +1659,11 @@ internal static bool IsSqlLanguage(string? lang) : normalized; } + internal static string FoldNameForLanguage(string value, string? lang) => + string.Equals(NormalizeQueryLanguage(lang), "nim", StringComparison.Ordinal) + ? NimIdentifierIdentity.Fold(value) ?? value + : NameFold.Fold(value) ?? value; + internal static bool ContainsSqlLanguage(IEnumerable langs) => langs.Any(IsSqlLanguage); diff --git a/src/CodeIndex/Database/DbSymbolReader.Definitions.cs b/src/CodeIndex/Database/DbSymbolReader.Definitions.cs index 024a9a0aa..e5f056c24 100644 --- a/src/CodeIndex/Database/DbSymbolReader.Definitions.cs +++ b/src/CodeIndex/Database/DbSymbolReader.Definitions.cs @@ -367,7 +367,7 @@ FROM chunks c var paramValue = !exact ? $"%{EscapeLikeQuery(normalizedQuery)}%" : _foldReady - ? NameFold.Fold(normalizedQuery) ?? normalizedQuery + ? FoldNameForLanguage(normalizedQuery, lang) : normalizedQuery; SqliteCommandPolicy.Add(cmd, "@query", paramValue); SqliteCommandPolicy.Add(cmd, "@queryNormalized", SqlNameResolver.NormalizeQualifiedName(normalizedQuery)); diff --git a/src/CodeIndex/Database/DbSymbolReader.Search.cs b/src/CodeIndex/Database/DbSymbolReader.Search.cs index a32f7989f..02a2f6529 100644 --- a/src/CodeIndex/Database/DbSymbolReader.Search.cs +++ b/src/CodeIndex/Database/DbSymbolReader.Search.cs @@ -246,7 +246,7 @@ FROM symbols s var paramValue = !exact ? $"%{EscapeLikeQuery(value)}%" : _foldReady - ? NameFold.Fold(value) ?? value + ? FoldNameForLanguage(value, lang) : value; SqliteCommandPolicy.Add(cmd, "@query0", paramValue); SqliteCommandPolicy.Add(cmd, "@query0Normalized", SqlNameResolver.NormalizeQualifiedName(value)); @@ -363,7 +363,7 @@ FROM symbols s var paramValue = !exact ? $"%{EscapeLikeQuery(value)}%" : _foldReady - ? NameFold.Fold(value) ?? value + ? FoldNameForLanguage(value, lang) : value; SqliteCommandPolicy.Add(cmd, $"@query{i}", paramValue); SqliteCommandPolicy.Add(cmd, $"@query{i}Normalized", SqlNameResolver.NormalizeQualifiedName(value)); @@ -652,7 +652,7 @@ FROM symbols s if (!exact) paramValue = $"%{EscapeLikeQuery(effectiveQueries[idx])}%"; else if (_foldReady) - paramValue = NameFold.Fold(effectiveQueries[idx]) ?? effectiveQueries[idx]; + paramValue = FoldNameForLanguage(effectiveQueries[idx], lang); else paramValue = effectiveQueries[idx]; SqliteCommandPolicy.Add(cmd, $"@query{idx}", paramValue); @@ -854,6 +854,14 @@ private string BuildSymbolSortOrderBy( if (!string.IsNullOrWhiteSpace(lang) && string.Equals(lang, "javascript", StringComparison.OrdinalIgnoreCase)) return NormalizeJavaScriptSymbolSearchQuery(query); + if (exact + && !string.IsNullOrWhiteSpace(lang) + && string.Equals(lang, "nim", StringComparison.OrdinalIgnoreCase) + && !string.IsNullOrWhiteSpace(query)) + { + return NimIdentifierIdentity.Fold(query.Trim()); + } + // Terraform dotted prefixes (var.X / local.X / module.X / data.TYPE.X) are stored as bare names in // the references and symbols tables. Strip the prefix so queries pasted from HCL still resolve. // Terraform の dotted prefix（var.X / local.X / module.X / data.TYPE.X）は参照/シンボルの bare 名で格納されるため、 @@ -1052,7 +1060,7 @@ private static bool ShouldPreserveRustQualifiedExactQuery(string? query, string? if (ShouldPreserveRustQualifiedExactQuery(query, lang, exact)) return query?.Trim(); - return NormalizeSymbolSearchQuery(query, lang) ?? query; + return NormalizeSymbolSearchQuery(query, lang, exact) ?? query; } private static (string? QualifiedPath, string? ContainerPath, string? LeafName) NormalizeRustQualifiedExactQueryParts(string query) diff --git a/src/CodeIndex/Database/DbWriter.BatchSql.cs b/src/CodeIndex/Database/DbWriter.BatchSql.cs index fa6849b70..cb1f3eb0d 100644 --- a/src/CodeIndex/Database/DbWriter.BatchSql.cs +++ b/src/CodeIndex/Database/DbWriter.BatchSql.cs @@ -30,6 +30,14 @@ private static object FoldedNameDbValue(string? name, Dictionary cache) => + identityNameFolded != null + ? identityNameFolded + : FoldedNameDbValue(name, cache); + private static Dictionary CreateFoldedNameCache(int rowCount, int namesPerRow) { if (rowCount <= 0 || namesPerRow <= 0) diff --git a/src/CodeIndex/Database/DbWriter.ChunkSymbolBatches.cs b/src/CodeIndex/Database/DbWriter.ChunkSymbolBatches.cs index 469dc6699..b0ff53bec 100644 --- a/src/CodeIndex/Database/DbWriter.ChunkSymbolBatches.cs +++ b/src/CodeIndex/Database/DbWriter.ChunkSymbolBatches.cs @@ -244,7 +244,10 @@ private void InsertSymbolBatch(IReadOnlyList symbols, int start, i ? (symbol.IsMetadataTarget.Value ? 1 : 0) : (object)DBNull.Value; cmd.Parameters[parameterIndex++].Value = (object?)symbol.MetadataTargetSource ?? DBNull.Value; - cmd.Parameters[parameterIndex++].Value = FoldedNameDbValue(symbol.Name, foldedNameCache); + cmd.Parameters[parameterIndex++].Value = FoldedNameDbValue( + symbol.Name, + symbol.IdentityNameFolded, + foldedNameCache); } cmd.ExecuteNonQuery(); diff --git a/src/CodeIndex/Database/DbWriter.FoldBackfill.cs b/src/CodeIndex/Database/DbWriter.FoldBackfill.cs index f25c617e4..0f8ca8236 100644 --- a/src/CodeIndex/Database/DbWriter.FoldBackfill.cs +++ b/src/CodeIndex/Database/DbWriter.FoldBackfill.cs @@ -94,14 +94,21 @@ private bool AllFoldedColumnsBackfilledCore( public bool AllFoldedColumnValuesMatchCurrentFold() { var symbols = RentCommand( - "SELECT name, name_folded FROM symbols WHERE name IS NOT NULL", + """ + SELECT s.name, s.name_folded, f.lang + FROM symbols s + JOIN files f ON f.id = s.file_id + WHERE s.name IS NOT NULL + """, static _ => { }); try { using var reader = symbols.ExecuteTrackedReader(); while (reader.TrackedRead()) { - var expected = NameFold.Fold(reader.GetString(0)); + var expected = DbReader.FoldNameForLanguage( + reader.GetString(0), + reader.IsDBNull(2) ? null : reader.GetString(2)); var actual = reader.IsDBNull(1) ? null : reader.GetString(1); if (!string.Equals(actual, expected, StringComparison.Ordinal)) return false; @@ -114,9 +121,12 @@ public bool AllFoldedColumnValuesMatchCurrentFold() var references = RentCommand( @" - SELECT symbol_name, symbol_name_folded, container_name, container_name_folded - FROM symbol_references - WHERE symbol_name IS NOT NULL OR container_name IS NOT NULL", + SELECT r.symbol_name, r.symbol_name_folded, + r.container_name, r.container_name_folded, + f.lang + FROM symbol_references r + JOIN files f ON f.id = r.file_id + WHERE r.symbol_name IS NOT NULL OR r.container_name IS NOT NULL", static _ => { }); try { @@ -125,7 +135,9 @@ FROM symbol_references { if (!reader.IsDBNull(0)) { - var expected = NameFold.Fold(reader.GetString(0)); + var expected = DbReader.FoldNameForLanguage( + reader.GetString(0), + reader.IsDBNull(4) ? null : reader.GetString(4)); var actual = reader.IsDBNull(1) ? null : reader.GetString(1); if (!string.Equals(actual, expected, StringComparison.Ordinal)) return false; @@ -133,7 +145,9 @@ FROM symbol_references if (!reader.IsDBNull(2)) { - var expected = NameFold.Fold(reader.GetString(2)); + var expected = DbReader.FoldNameForLanguage( + reader.GetString(2), + reader.IsDBNull(4) ? null : reader.GetString(4)); var actual = reader.IsDBNull(3) ? null : reader.GetString(3); if (!string.Equals(actual, expected, StringComparison.Ordinal)) return false; @@ -318,10 +332,21 @@ private int BackfillSymbolFoldedRows(bool rewriteAll, CancellationToken cancella return 0; var lastSymbolId = rewriteAll ? GetFoldBackfillCheckpoint(FoldBackfillLastSymbolIdMetaKey) : 0; - var rows = new List<(long Id, string Name)>(); + var rows = new List<(long Id, string Name, string? Lang)>(); var selectSql = rewriteAll - ? "SELECT id, name FROM symbols WHERE name IS NOT NULL AND id > @lastSymbolId ORDER BY id" - : "SELECT id, name FROM symbols WHERE name IS NOT NULL AND name_folded IS NULL"; + ? """ + SELECT s.id, s.name, f.lang + FROM symbols s + JOIN files f ON f.id = s.file_id + WHERE s.name IS NOT NULL AND s.id > @lastSymbolId + ORDER BY s.id + """ + : """ + SELECT s.id, s.name, f.lang + FROM symbols s + JOIN files f ON f.id = s.file_id + WHERE s.name IS NOT NULL AND s.name_folded IS NULL + """; var select = RentCommand( selectSql, rewriteAll @@ -335,7 +360,10 @@ private int BackfillSymbolFoldedRows(bool rewriteAll, CancellationToken cancella while (reader.TrackedRead()) { cancellationToken.ThrowIfCancellationRequested(); - rows.Add((reader.GetInt64(0), reader.GetString(1))); + rows.Add(( + reader.GetInt64(0), + reader.GetString(1), + reader.IsDBNull(2) ? null : reader.GetString(2))); } } finally @@ -360,7 +388,7 @@ private int BackfillSymbolFoldedRows(bool rewriteAll, CancellationToken cancella foreach (var row in rows) { cancellationToken.ThrowIfCancellationRequested(); - pFolded.Value = (object?)NameFold.Fold(row.Name) ?? DBNull.Value; + pFolded.Value = DbReader.FoldNameForLanguage(row.Name, row.Lang); pId.Value = row.Id; update.ExecuteNonQuery(); if (rewriteAll) @@ -379,17 +407,23 @@ private int BackfillSymbolFoldedRows(bool rewriteAll, CancellationToken cancella private int BackfillReferenceFoldedRows(bool rewriteAll, CancellationToken cancellationToken) { var lastReferenceId = rewriteAll ? GetFoldBackfillCheckpoint(FoldBackfillLastReferenceIdMetaKey) : 0; - var rows = new List<(long Id, string? SymbolName, string? ContainerName)>(); + var rows = new List<(long Id, string? SymbolName, string? ContainerName, string? Lang)>(); var selectSql = rewriteAll - ? @"SELECT id, symbol_name, container_name - FROM symbol_references - WHERE id > @lastReferenceId - AND (symbol_name IS NOT NULL OR container_name IS NOT NULL) - ORDER BY id" - : @"SELECT id, symbol_name, container_name - FROM symbol_references - WHERE (symbol_name IS NOT NULL AND symbol_name_folded IS NULL) - OR (container_name IS NOT NULL AND container_name_folded IS NULL)"; + ? """ + SELECT r.id, r.symbol_name, r.container_name, f.lang + FROM symbol_references r + JOIN files f ON f.id = r.file_id + WHERE r.id > @lastReferenceId + AND (r.symbol_name IS NOT NULL OR r.container_name IS NOT NULL) + ORDER BY r.id + """ + : """ + SELECT r.id, r.symbol_name, r.container_name, f.lang + FROM symbol_references r + JOIN files f ON f.id = r.file_id + WHERE (r.symbol_name IS NOT NULL AND r.symbol_name_folded IS NULL) + OR (r.container_name IS NOT NULL AND r.container_name_folded IS NULL) + """; var select = RentCommand( selectSql, rewriteAll @@ -406,7 +440,8 @@ FROM symbol_references rows.Add(( reader.GetInt64(0), reader.IsDBNull(1) ? null : reader.GetString(1), - reader.IsDBNull(2) ? null : reader.GetString(2))); + reader.IsDBNull(2) ? null : reader.GetString(2), + reader.IsDBNull(3) ? null : reader.GetString(3))); } } finally @@ -436,8 +471,12 @@ FROM symbol_references foreach (var row in rows) { cancellationToken.ThrowIfCancellationRequested(); - pSymbolNameFolded.Value = (object?)NameFold.Fold(row.SymbolName) ?? DBNull.Value; - pContainerNameFolded.Value = (object?)NameFold.Fold(row.ContainerName) ?? DBNull.Value; + pSymbolNameFolded.Value = row.SymbolName == null + ? DBNull.Value + : DbReader.FoldNameForLanguage(row.SymbolName, row.Lang); + pContainerNameFolded.Value = row.ContainerName == null + ? DBNull.Value + : DbReader.FoldNameForLanguage(row.ContainerName, row.Lang); pId.Value = row.Id; update.ExecuteNonQuery(); if (rewriteAll) diff --git a/src/CodeIndex/Database/DbWriter.References.cs b/src/CodeIndex/Database/DbWriter.References.cs index 0222ce28f..2a0cecdfa 100644 --- a/src/CodeIndex/Database/DbWriter.References.cs +++ b/src/CodeIndex/Database/DbWriter.References.cs @@ -836,8 +836,14 @@ private void InsertReferenceBatch( cmd.Parameters[parameterIndex++].Value = previousReferenceLineId; cmd.Parameters[parameterIndex++].Value = (object?)reference.ContainerKind ?? DBNull.Value; cmd.Parameters[parameterIndex++].Value = (object?)reference.ContainerName ?? DBNull.Value; - cmd.Parameters[parameterIndex++].Value = FoldedNameDbValue(reference.SymbolName, foldedNameCache); - cmd.Parameters[parameterIndex++].Value = FoldedNameDbValue(reference.ContainerName, foldedNameCache); + cmd.Parameters[parameterIndex++].Value = FoldedNameDbValue( + reference.SymbolName, + reference.IdentitySymbolNameFolded, + foldedNameCache); + cmd.Parameters[parameterIndex++].Value = FoldedNameDbValue( + reference.ContainerName, + reference.IdentityContainerNameFolded, + foldedNameCache); cmd.Parameters[parameterIndex++].Value = reference.IsSelfReference ? 1 : 0; cmd.Parameters[parameterIndex++].Value = reference.IsMutualRecursion ? 1 : 0; cmd.Parameters[parameterIndex++].Value = (object?)ExtractTargetQualifier(reference) ?? DBNull.Value; @@ -1210,6 +1216,9 @@ internal void RefreshMutualRecursionFlags(CancellationToken cancellationToken = private static string? ExtractTargetQualifier(ReferenceRecord reference) { + if (reference.SuppressInferredTargetQualifier) + return null; + if (!string.IsNullOrWhiteSpace(reference.TargetQualifier)) { var explicitQualifier = reference.TargetQualifier.Trim(); diff --git a/src/CodeIndex/Indexer/NimIdentifierIdentity.cs b/src/CodeIndex/Indexer/NimIdentifierIdentity.cs new file mode 100644 index 000000000..a1eda664a --- /dev/null +++ b/src/CodeIndex/Indexer/NimIdentifierIdentity.cs @@ -0,0 +1,32 @@ +using System.Text; + +namespace CodeIndex.Indexer; + +internal static class NimIdentifierIdentity +{ + internal static string? Fold(string? value) + { + if (value == null) + return null; + + var folded = new StringBuilder(value.Length); + var atIdentifierStart = true; + foreach (var character in value) + { + if (character == '_') + continue; + + if (char.IsLetterOrDigit(character)) + { + folded.Append(atIdentifierStart ? character : char.ToLowerInvariant(character)); + atIdentifierStart = false; + continue; + } + + folded.Append(character); + atIdentifierStart = true; + } + + return folded.ToString(); + } +} diff --git a/src/CodeIndex/Indexer/References/Languages/ScientificNativeReferenceExtractor.cs b/src/CodeIndex/Indexer/References/Languages/ScientificNativeReferenceExtractor.cs index 214bf2837..7eb3e36f5 100644 --- a/src/CodeIndex/Indexer/References/Languages/ScientificNativeReferenceExtractor.cs +++ b/src/CodeIndex/Indexer/References/Languages/ScientificNativeReferenceExtractor.cs @@ -6,6 +6,7 @@ namespace CodeIndex.Indexer; internal static class ScientificNativeReferenceExtractor { + internal const string CurrentContainerReceiverMarker = "\u001fcurrent-container"; private const string JuliaIdentifierPattern = @"[\p{L}_]\w*"; private const string JuliaCallableIdentifierPattern = JuliaIdentifierPattern + @"!?"; @@ -96,6 +97,7 @@ private readonly record struct DTemplateInvocation( internal static bool Supports(string language) => SupportedLanguages.Contains(language); internal static string? GetParenthesizedCallTargetQualifier( + string language, string preparedLine, int callIndex) { @@ -135,6 +137,12 @@ private readonly record struct DTemplateInvocation( } segments.Reverse(); + if ((language == "cython" && segments[0] is "self" or "cls") + || (language == "d" && segments[0] is "this" or "super")) + { + return CurrentContainerReceiverMarker; + } + return string.Join('.', segments); } diff --git a/src/CodeIndex/Indexer/References/ReferenceExtractor.CoreExtraction.cs b/src/CodeIndex/Indexer/References/ReferenceExtractor.CoreExtraction.cs index 0aef032df..4803582d5 100644 --- a/src/CodeIndex/Indexer/References/ReferenceExtractor.CoreExtraction.cs +++ b/src/CodeIndex/Indexer/References/ReferenceExtractor.CoreExtraction.cs @@ -1631,6 +1631,7 @@ bool ShouldSuppressDefinitionCall(string resolvedName, string rawName, int callI { var targetQualifier = ScientificNativeReferenceExtractor.GetParenthesizedCallTargetQualifier( + language, preparedLine, callIndex); if (targetQualifier != null) @@ -2340,6 +2341,7 @@ void AddCallLikeReference(string name, int callIndex) => callIndex, ScientificNativeReferenceExtractor.Supports(language) ? ScientificNativeReferenceExtractor.GetParenthesizedCallTargetQualifier( + language, preparedLine, callIndex) : null); @@ -2540,6 +2542,7 @@ bool TryAddCallLikeReference( context, lineNumber, callContainer, + language, targetQualifier: targetQualifier); return true; @@ -2686,6 +2689,7 @@ bool TryGetKnownPythonTypeCall(string candidate, out string canonicalName) callIndex, ScientificNativeReferenceExtractor.Supports(language) ? ScientificNativeReferenceExtractor.GetParenthesizedCallTargetQualifier( + language, preparedLine, callIndex) : null)) diff --git a/src/CodeIndex/Indexer/References/ReferenceExtractor.Preparation.cs b/src/CodeIndex/Indexer/References/ReferenceExtractor.Preparation.cs index 49a3442d6..e93b64bb9 100644 --- a/src/CodeIndex/Indexer/References/ReferenceExtractor.Preparation.cs +++ b/src/CodeIndex/Indexer/References/ReferenceExtractor.Preparation.cs @@ -101,7 +101,7 @@ [new CppLexicalRange(0, maskedContent.Length)], ? MaskCStyleBlockCommentLines(language, structuralLines) : structuralLines; referenceStructuralLines = ScientificNativeCommentMasker.MaskBlockComments(language, referenceStructuralLines); - if (language == "python") + if (language is "python" or "cython") referenceStructuralLines = MaskPythonFStrings(referenceStructuralLines); var linePrepareOptions = CreateReferenceLinePrepareOptions(language); diff --git a/src/CodeIndex/Indexer/References/ReferenceExtractor.cs b/src/CodeIndex/Indexer/References/ReferenceExtractor.cs index 6dafbcdc9..de01e4ac5 100644 --- a/src/CodeIndex/Indexer/References/ReferenceExtractor.cs +++ b/src/CodeIndex/Indexer/References/ReferenceExtractor.cs @@ -2101,18 +2101,29 @@ internal static void AddReference( var dedupeKey = CreateReferenceDedupeKey(fileId, language, lineNumber, column, referenceKind, name, container); if (!seen.Add(dedupeKey)) return; + var currentContainerReceiver = string.Equals( + targetQualifier, + ScientificNativeReferenceExtractor.CurrentContainerReceiverMarker, + StringComparison.Ordinal); TryAddReference(references, new ReferenceRecord { FileId = fileId, SymbolName = name, + IdentitySymbolNameFolded = language == "nim" + ? NimIdentifierIdentity.Fold(name) + : null, ReferenceKind = referenceKind, Line = lineNumber, Column = column, Context = context, ContainerKind = container?.Kind, ContainerName = container?.Name, - TargetQualifier = targetQualifier, + IdentityContainerNameFolded = language == "nim" + ? NimIdentifierIdentity.Fold(container?.Name) + : null, + TargetQualifier = currentContainerReceiver ? null : targetQualifier, + SuppressInferredTargetQualifier = currentContainerReceiver, IsSelfReference = IsSameReferenceName(container?.Name, name), }); } diff --git a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.ExtractCore.cs b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.ExtractCore.cs index bb8b1a2b0..a845c4e34 100644 --- a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.ExtractCore.cs +++ b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.ExtractCore.cs @@ -2209,6 +2209,11 @@ bool[] GetCssQualifiedRuleAncestors() => if (lang == "shell") ExpandShellAliasSymbols(fileId, lines, symbols, extractionState); PopulateDeclaredContainerQualifiedNames(symbols); + if (lang == "nim") + { + foreach (var symbol in symbols) + symbol.IdentityNameFolded = NimIdentifierIdentity.Fold(symbol.Name); + } return symbols; } diff --git a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.Pascal.cs b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.Pascal.cs index f4a094807..4ff5d6a1b 100644 --- a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.Pascal.cs +++ b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.Pascal.cs @@ -10,6 +10,96 @@ public static partial class SymbolExtractor private static readonly Regex PascalNestedEndBlockStartRegex = new(@"\b(?:case|try)\b", RegexOptions.Compiled | RegexOptions.IgnoreCase | RegexOptions.CultureInvariant); private static readonly Regex PascalRoutineStartRegex = new(@"^\s*(?:(?:class|static)\s+)?(?:procedure|function|constructor|destructor)\b", RegexOptions.Compiled | RegexOptions.IgnoreCase | RegexOptions.CultureInvariant); private static readonly Regex PascalRangeBoundaryRegex = new(@"^\s*(?:interface|implementation|initialization|finalization)\b", RegexOptions.Compiled | RegexOptions.IgnoreCase | RegexOptions.CultureInvariant); + private static readonly Regex AdaRangeDeclarationNameRegex = new( + @"^\s*(?:(?:overriding|not\s+overriding)\s+)?(?:(?:package\s+(?:body\s+)?)|(?:function|procedure)\s+(?:(?:[A-Za-z]\w*)\.)*|(?:task|protected)\s+(?:type\s+)?)(?[A-Za-z]\w*)\b", + RegexOptions.Compiled | RegexOptions.IgnoreCase | RegexOptions.CultureInvariant); + private static readonly Regex AdaNamedEndRegex = new( + @"\bend\s+(?[A-Za-z]\w*(?:\.[A-Za-z]\w*)*)\s*;", + RegexOptions.Compiled | RegexOptions.IgnoreCase | RegexOptions.CultureInvariant); + private static readonly Regex AdaBeginRegex = new( + @"\bbegin\b", + RegexOptions.Compiled | RegexOptions.IgnoreCase | RegexOptions.CultureInvariant); + private static readonly Regex AdaUnnamedOuterEndRegex = new( + @"\bend\s*;", + RegexOptions.Compiled | RegexOptions.IgnoreCase | RegexOptions.CultureInvariant); + + private static (int EndLine, int? BodyStartLine, int? BodyEndLine) FindAdaRange( + string[] lines, + int startIndex) + { + var declaration = AdaRangeDeclarationNameRegex.Match(lines[startIndex]); + if (!declaration.Success) + return (startIndex + 1, null, null); + + var declarationName = declaration.Groups["name"].Value; + int? bodyStartLine = null; + for (var i = startIndex; i < lines.Length; i++) + { + var code = MaskAdaRangeStringsAndComments(lines[i]); + if (bodyStartLine == null && AdaBeginRegex.IsMatch(code)) + bodyStartLine = i + 1; + + foreach (Match endMatch in AdaNamedEndRegex.Matches(code)) + { + var endName = endMatch.Groups["name"].Value; + var endLeaf = endName[(endName.LastIndexOf('.') + 1)..]; + if (string.Equals(endLeaf, declarationName, StringComparison.OrdinalIgnoreCase)) + return (i + 1, bodyStartLine, i + 1); + } + } + + if (bodyStartLine == null) + return (startIndex + 1, null, null); + + for (var i = bodyStartLine.Value - 1; i < lines.Length; i++) + { + if (AdaUnnamedOuterEndRegex.IsMatch(MaskAdaRangeStringsAndComments(lines[i]))) + return (i + 1, bodyStartLine, i + 1); + } + + return (lines.Length, bodyStartLine, lines.Length); + } + + private static string MaskAdaRangeStringsAndComments(string line) + { + char[]? chars = null; + var inString = false; + for (var i = 0; i < line.Length; i++) + { + if (inString) + { + chars![i] = ' '; + if (line[i] != '"') + continue; + if (i + 1 < line.Length && line[i + 1] == '"') + { + chars[++i] = ' '; + continue; + } + + inString = false; + continue; + } + + if (line[i] == '"') + { + chars ??= line.ToCharArray(); + chars[i] = ' '; + inString = true; + continue; + } + + if (line[i] == '-' && i + 1 < line.Length && line[i + 1] == '-') + { + chars ??= line.ToCharArray(); + for (; i < line.Length; i++) + chars[i] = ' '; + break; + } + } + + return chars == null ? line : new string(chars); + } private static (int EndLine, int? BodyStartLine, int? BodyEndLine) FindPascalRange(string[] lines, int startIndex) { diff --git a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.cs b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.cs index c2bf5ba84..e32c17bc9 100644 --- a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.cs +++ b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.cs @@ -24,7 +24,7 @@ public static partial class SymbolExtractor public const int FunctionalLanguageContractVersion = 3; public const int DynamicLanguageContractVersion = 2; public const int SystemsLanguageContractVersion = 2; - public const int ScientificNativeGraphContractVersion = 3; + public const int ScientificNativeGraphContractVersion = 4; public const int RepositoryMetadataContractVersion = 2; public const int ApplicationManifestContractVersion = 3; private static readonly string[] AdditionalSymbolLanguages = @@ -499,6 +499,7 @@ private enum BodyStyle JuliaShortFunction, VisualBasicEnd, PascalEnd, + AdaEnd, SmalltalkMethod, SqlProcBody, } @@ -1654,7 +1655,7 @@ private enum JavaScriptTypeScriptFunctionHeaderConsumeResult new("type", new Regex(@"^\s*(?:abstract|primitive)\s+type\s+(?[A-Za-z_]\w*)", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.ScientificEnd), new("function", new Regex(@"^\s*function\s+(?" + JuliaQualifiedCallableIdentifierPattern + @")\s*(?:\(|\{)", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.ScientificEnd), new("function", new Regex(@"^\s*macro\s+(?[A-Za-z_]\w*)\s*(?:\(|$)", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.ScientificEnd), - new("function", new Regex(@"^\s*(?" + JuliaQualifiedCallableIdentifierPattern + @")\s*\([^)\r\n]*\)\s*=", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.JuliaShortFunction), + new("function", new Regex(@"^\s*(?" + JuliaQualifiedCallableIdentifierPattern + @")\s*\([^)\r\n]*\)\s*(?:where\s*(?:\{[^}\r\n]*\}|[A-Za-z_]\w*)\s*)?=", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.JuliaShortFunction), new("property", new Regex(@"^\s*const\s+(?[A-Z_]\w*)\s*=", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None), new("import", new Regex(@"^\s*(?:using|import)\s+(?[A-Za-z_]\w*(?:\.[A-Za-z_]\w*)*)", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None), ], @@ -1668,10 +1669,10 @@ private enum JavaScriptTypeScriptFunctionHeaderConsumeResult ], ["ada"] = [ - new("namespace", new Regex(@"^\s*package\s+(?:body\s+)?(?[A-Za-z]\w*(?:\.[A-Za-z]\w*)*)\s+is\b", RegexOptions.Compiled | RegexOptions.IgnoreCase | RegexOptions.CultureInvariant), BodyStyle.PascalEnd), + new("namespace", new Regex(@"^\s*package\s+(?:body\s+)?(?[A-Za-z]\w*(?:\.[A-Za-z]\w*)*)\s+is\b", RegexOptions.Compiled | RegexOptions.IgnoreCase | RegexOptions.CultureInvariant), BodyStyle.AdaEnd), new("type", new Regex(@"^\s*(?:subtype|type)\s+(?[A-Za-z]\w*)\s+is\b", RegexOptions.Compiled | RegexOptions.IgnoreCase | RegexOptions.CultureInvariant), BodyStyle.None), - new("type", new Regex(@"^\s*(?:task|protected)\s+(?:type\s+)?(?[A-Za-z]\w*)\b", RegexOptions.Compiled | RegexOptions.IgnoreCase | RegexOptions.CultureInvariant), BodyStyle.PascalEnd), - new("function", new Regex(@"^\s*(?:(?:overriding|not\s+overriding)\s+)?(?:function|procedure)\s+(?:(?:[A-Za-z]\w*)\.)*(?[A-Za-z]\w*)\b", RegexOptions.Compiled | RegexOptions.IgnoreCase | RegexOptions.CultureInvariant), BodyStyle.PascalEnd), + new("type", new Regex(@"^\s*(?:task|protected)\s+(?:type\s+)?(?[A-Za-z]\w*)\b", RegexOptions.Compiled | RegexOptions.IgnoreCase | RegexOptions.CultureInvariant), BodyStyle.AdaEnd), + new("function", new Regex(@"^\s*(?:(?:overriding|not\s+overriding)\s+)?(?:function|procedure)\s+(?:(?:[A-Za-z]\w*)\.)*(?[A-Za-z]\w*)\b", RegexOptions.Compiled | RegexOptions.IgnoreCase | RegexOptions.CultureInvariant), BodyStyle.AdaEnd), new("import", new Regex(@"^\s*with\s+(?[A-Za-z]\w*(?:\.[A-Za-z]\w*)*)\s*;", RegexOptions.Compiled | RegexOptions.IgnoreCase | RegexOptions.CultureInvariant), BodyStyle.None), ], ["d"] = @@ -4834,6 +4835,7 @@ private static (int EndLine, int? BodyStartLine, int? BodyEndLine) ResolveRange( startIndex), BodyStyle.VisualBasicEnd => FindVisualBasicRange(lines, startIndex), BodyStyle.PascalEnd => FindPascalRange(lines, startIndex), + BodyStyle.AdaEnd => FindAdaRange(lines, startIndex), BodyStyle.SmalltalkMethod => FindSmalltalkMethodRange(lines, startIndex), BodyStyle.SqlProcBody => FindSqlProcBodyRange(lines, startIndex), _ => (startIndex + 1, null, null), diff --git a/src/CodeIndex/Models/ReferenceRecord.cs b/src/CodeIndex/Models/ReferenceRecord.cs index 872598609..c4b3d03da 100644 --- a/src/CodeIndex/Models/ReferenceRecord.cs +++ b/src/CodeIndex/Models/ReferenceRecord.cs @@ -12,6 +12,9 @@ public class ReferenceRecord /// Referenced symbol name / 参照先シンボル名 public string SymbolName { get; set; } = string.Empty; + /// Language-specific persisted identity key when generic folding is insufficient / 一般的なfoldでは不十分な場合の言語固有identity key + internal string? IdentitySymbolNameFolded { get; set; } + /// Reference kind such as call or instantiate / 参照種別 public string ReferenceKind { get; set; } = string.Empty; @@ -30,6 +33,9 @@ public class ReferenceRecord /// Enclosing symbol name when known / 親シンボル名 public string? ContainerName { get; set; } + /// Language-specific persisted container identity key / 言語固有の永続化container identity key + internal string? IdentityContainerNameFolded { get; set; } + /// /// Receiver/type qualifier immediately before the referenced name when it is a stable /// type-like identifier (for example FileShare in FileShare.ReadWrite). @@ -37,6 +43,9 @@ public class ReferenceRecord /// public string? TargetQualifier { get; set; } + /// True when a language-specific receiver explicitly denotes the current container / 言語固有receiverが現在のcontainerを明示する場合はtrue + internal bool SuppressInferredTargetQualifier { get; set; } + /// True when the enclosing symbol references itself / 親シンボル自身への参照なら true public bool IsSelfReference { get; set; } diff --git a/src/CodeIndex/Models/SymbolRecord.cs b/src/CodeIndex/Models/SymbolRecord.cs index 3dd1e6d6f..aad15d2be 100644 --- a/src/CodeIndex/Models/SymbolRecord.cs +++ b/src/CodeIndex/Models/SymbolRecord.cs @@ -21,6 +21,9 @@ public class SymbolRecord /// Symbol name / シンボル名 public string Name { get; set; } = string.Empty; + /// Language-specific persisted identity key when generic folding is insufficient / 一般的なfoldでは不十分な場合の言語固有identity key + internal string? IdentityNameFolded { get; set; } + /// Line number (1-based) / 行番号（1始まり） public int Line { get; set; } diff --git a/tests/CodeIndex.Tests/DatabaseTests.cs b/tests/CodeIndex.Tests/DatabaseTests.cs index da1bab41b..4ab14d644 100644 --- a/tests/CodeIndex.Tests/DatabaseTests.cs +++ b/tests/CodeIndex.Tests/DatabaseTests.cs @@ -1113,6 +1113,122 @@ FROM symbol_references } } + [Fact] + public void ReferenceGraph_CythonSelfReceiverResolvesSameClassDuplicateLeaf_Issue4738() + { + const string content = """ + cdef class A: + def helper(self): + pass + def run(self): + self.helper() + cdef class B: + def helper(self): + pass + """; + var fileId = UpsertTestFileWithLanguage( + "src/workers.pyx", + "cython", + "cython-self-receiver"); + var symbols = SymbolExtractor.Extract(fileId, "cython", content); + _writer.InsertSymbols(symbols); + var references = ReferenceExtractor.Extract(fileId, "cython", content, symbols); + _writer.InsertReferences(references, refreshMutualRecursionFlags: false); + + _writer.RefreshMutualRecursionFlags(); + + var helperReference = Assert.Single(references, reference => + reference.SymbolName == "helper" && reference.ReferenceKind == "call"); + Assert.Null(helperReference.TargetQualifier); + var expectedTargetId = ExecuteScalarLong($""" + SELECT id + FROM symbols + WHERE file_id = {fileId.ToString(CultureInfo.InvariantCulture)} + AND name = 'helper' + AND container_name = 'A' + """); + Assert.Equal(expectedTargetId, ExecuteScalarLong($""" + SELECT target_symbol_id + FROM symbol_references + WHERE file_id = {fileId.ToString(CultureInfo.InvariantCulture)} + AND symbol_name = 'helper' + """)); + Assert.Equal("resolved", ReadReferenceResolutionState(fileId)); + } + + [Fact] + public void ReferenceGraph_NimStyleInsensitiveIdentityResolvesAndSearches_Issue4738() + { + const string content = """ + proc myProc() = discard + proc RunGraph() = + my_proc() + """; + var fileId = UpsertTestFileWithLanguage( + "src/style.nim", + "nim", + "nim-style-insensitive"); + var symbols = SymbolExtractor.Extract(fileId, "nim", content); + _writer.InsertSymbols(symbols); + var references = ReferenceExtractor.Extract(fileId, "nim", content, symbols); + _writer.InsertReferences(references, refreshMutualRecursionFlags: false); + + _writer.RefreshMutualRecursionFlags(); + Assert.True(_writer.MarkFoldReady( + stampCurrentSymbolExtractorVersions: true, + symbolExtractorLanguagesToStamp: ["nim"])); + _writer.MarkGraphReady(); + + var expectedTargetId = ExecuteScalarLong($""" + SELECT id + FROM symbols + WHERE file_id = {fileId.ToString(CultureInfo.InvariantCulture)} + AND name = 'myProc' + """); + Assert.Equal(expectedTargetId, ExecuteScalarLong($""" + SELECT target_symbol_id + FROM symbol_references + WHERE file_id = {fileId.ToString(CultureInfo.InvariantCulture)} + AND symbol_name = 'my_proc' + """)); + Assert.Equal("resolved", ReadReferenceResolutionState(fileId)); + var expectedSourceId = ExecuteScalarLong($""" + SELECT id + FROM symbols + WHERE file_id = {fileId.ToString(CultureInfo.InvariantCulture)} + AND name = 'RunGraph' + """); + Assert.Equal(expectedSourceId, ExecuteScalarLong($""" + SELECT source_symbol_id + FROM symbol_references + WHERE file_id = {fileId.ToString(CultureInfo.InvariantCulture)} + AND symbol_name = 'my_proc' + """)); + Assert.Equal("myproc", ExecuteScalarString($""" + SELECT name_folded + FROM symbols + WHERE id = {expectedTargetId.ToString(CultureInfo.InvariantCulture)} + """)); + Assert.Equal("myproc", ExecuteScalarString($""" + SELECT symbol_name_folded + FROM symbol_references + WHERE file_id = {fileId.ToString(CultureInfo.InvariantCulture)} + AND symbol_name = 'my_proc' + """)); + Assert.Equal("Myproc", DbReader.FoldNameForLanguage("My_proc", "nim")); + Assert.Equal(0, ExecuteScalarLong(""" + SELECT COUNT(*) + FROM symbols + WHERE name_folded = 'Myproc' + """)); + + var reader = new DbReader(_db.Connection); + Assert.Single(reader.SearchSymbols("my_proc", lang: "nim", exact: true)); + Assert.Single(reader.SearchReferences("myProc", lang: "nim", exact: true)); + Assert.Empty(reader.SearchSymbols("My_proc", lang: "nim", exact: true)); + Assert.Empty(reader.SearchReferences("MyProc", lang: "nim", exact: true)); + } + [Fact] public void ReferenceGraphDirtyScope_RollbackAndCancellationPreserveRetryState() { diff --git a/tests/CodeIndex.Tests/ReferenceExtractorTests.ScientificNative.cs b/tests/CodeIndex.Tests/ReferenceExtractorTests.ScientificNative.cs index 7bd534f6b..f612816de 100644 --- a/tests/CodeIndex.Tests/ReferenceExtractorTests.ScientificNative.cs +++ b/tests/CodeIndex.Tests/ReferenceExtractorTests.ScientificNative.cs @@ -1709,4 +1709,97 @@ function run(values) Assert.Equal("run", Assert.Single(references, reference => reference.SymbolName == "helper" && reference.ReferenceKind == "call").ContainerName); } + + [Theory] + [InlineData( + "cython", + """ + cdef class Worker: + def run(self): + self.helper() + """)] + [InlineData( + "d", + """ + class Worker { + void run() { + this.helper(); + } + } + """)] + public void Extract_InstanceReceiversUseCurrentContainerResolution_Issue4738( + string language, + string content) + { + var symbols = SymbolExtractor.Extract(1, language, content); + + var references = ReferenceExtractor.Extract(1, language, content, symbols); + + var helper = Assert.Single(references, reference => + reference.SymbolName == "helper" && reference.ReferenceKind == "call"); + Assert.Equal("run", helper.ContainerName); + Assert.Null(helper.TargetQualifier); + } + + [Fact] + public void Extract_AdaNestedEndsPreserveOuterProcedureScope_Issue4738() + { + const string content = """ + procedure Run is + begin + if Ready then + First; + end if; + Later; + end Run; + """; + var symbols = SymbolExtractor.Extract(1, "ada", content); + + var references = ReferenceExtractor.Extract(1, "ada", content, symbols); + + Assert.Equal(7, Assert.Single(symbols, symbol => symbol.Name == "Run").EndLine); + foreach (var name in new[] { "First", "Later" }) + { + Assert.Equal("Run", Assert.Single(references, reference => + reference.SymbolName == name && reference.ReferenceKind == "call").ContainerName); + } + } + + [Fact] + public void Extract_CythonFStringExpressionsEmitCalls_Issue4738() + { + const string content = """ + def run(): + value = f"{pkg.helper()}" + """; + var symbols = SymbolExtractor.Extract(1, "cython", content); + + var references = ReferenceExtractor.Extract(1, "cython", content, symbols); + + var helper = Assert.Single(references, reference => + reference.SymbolName == "helper" && reference.ReferenceKind == "call"); + Assert.Equal("run", helper.ContainerName); + Assert.Equal("pkg", helper.TargetQualifier); + Assert.Equal(20, helper.Column); + } + + [Fact] + public void Extract_JuliaWhereShortFunctionOwnsRhsCalls_Issue4738() + { + const string content = """ + f(x::T) where {T} = helper(x) + outside() + """; + var symbols = SymbolExtractor.Extract(1, "julia", content); + + var references = ReferenceExtractor.Extract(1, "julia", content, symbols); + + Assert.Equal(1, Assert.Single(symbols, symbol => symbol.Name == "f").EndLine); + Assert.Equal("f", Assert.Single(references, reference => + reference.SymbolName == "helper" && reference.ReferenceKind == "call").ContainerName); + Assert.Null(Assert.Single(references, reference => + reference.SymbolName == "outside" && reference.ReferenceKind == "call").ContainerName); + Assert.DoesNotContain(references, reference => + reference.SymbolName == "f" && reference.ReferenceKind == "call"); + } } From b8833b792df80e5c8a7cca8cb5b9842db418d0b4 Mon Sep 17 00:00:00 2001 From: Widthdom Date: Fri, 24 Jul 2026 21:34:47 +0900 Subject: [PATCH 20/21] Resolve persisted scientific graph review findings (#4738) --- .../Database/DbSymbolReader.Search.cs | 8 - .../Database/DbWriter.FoldBackfill.cs | 25 +++ src/CodeIndex/Database/NameFold.cs | 18 +- .../ReferenceExtractor.CoreExtraction.cs | 2 +- src/CodeIndex/Models/ReferenceRecord.cs | 5 + src/CodeIndex/Models/SymbolRecord.cs | 3 + tests/CodeIndex.Tests/DatabaseTests.cs | 154 ++++++++++++++++++ .../IndexCommandRunnerTests.cs | 26 +++ tests/CodeIndex.Tests/NameFoldTests.cs | 7 + .../PostExtractionHookContractTests.cs | 43 +++++ 10 files changed, 273 insertions(+), 18 deletions(-) diff --git a/src/CodeIndex/Database/DbSymbolReader.Search.cs b/src/CodeIndex/Database/DbSymbolReader.Search.cs index 02a2f6529..29b7bb350 100644 --- a/src/CodeIndex/Database/DbSymbolReader.Search.cs +++ b/src/CodeIndex/Database/DbSymbolReader.Search.cs @@ -854,14 +854,6 @@ private string BuildSymbolSortOrderBy( if (!string.IsNullOrWhiteSpace(lang) && string.Equals(lang, "javascript", StringComparison.OrdinalIgnoreCase)) return NormalizeJavaScriptSymbolSearchQuery(query); - if (exact - && !string.IsNullOrWhiteSpace(lang) - && string.Equals(lang, "nim", StringComparison.OrdinalIgnoreCase) - && !string.IsNullOrWhiteSpace(query)) - { - return NimIdentifierIdentity.Fold(query.Trim()); - } - // Terraform dotted prefixes (var.X / local.X / module.X / data.TYPE.X) are stored as bare names in // the references and symbols tables. Strip the prefix so queries pasted from HCL still resolve. // Terraform の dotted prefix（var.X / local.X / module.X / data.TYPE.X）は参照/シンボルの bare 名で格納されるため、 diff --git a/src/CodeIndex/Database/DbWriter.FoldBackfill.cs b/src/CodeIndex/Database/DbWriter.FoldBackfill.cs index 0f8ca8236..cd1659175 100644 --- a/src/CodeIndex/Database/DbWriter.FoldBackfill.cs +++ b/src/CodeIndex/Database/DbWriter.FoldBackfill.cs @@ -8,6 +8,7 @@ public partial class DbWriter private const string FoldBackfillPhaseMetaKey = "fold_backfill_phase"; private const string FoldBackfillLastSymbolIdMetaKey = "fold_backfill_last_symbol_id"; private const string FoldBackfillLastReferenceIdMetaKey = "fold_backfill_last_reference_id"; + private const string FoldBackfillGraphRefreshPendingMetaKey = "fold_backfill_graph_refresh_pending"; private static readonly AsyncLocal ScopedFoldBackfillRowUpdatedForTesting = new(); private static readonly AsyncLocal ScopedFoldBackfillVerificationForTesting = new(); @@ -254,6 +255,21 @@ private bool SymbolExtractorVersionMatchesCurrent(string? lang) CancellationToken cancellationToken = default) { cancellationToken.ThrowIfCancellationRequested(); + var graphRefreshPending = string.Equals( + GetMetaString(FoldBackfillGraphRefreshPendingMetaKey), + "1", + StringComparison.Ordinal); + var pendingRows = CountBackfillFoldedColumns(rewriteAll); + if (!graphRefreshPending && (pendingRows.Symbols > 0 || pendingRows.SymbolReferences > 0)) + { + // Persist this before the first row mutation so cancellation after the rewrite but + // before graph refresh cannot make a retry mistake the operation for a no-op. + // 最初の行を書き換える前に pending を永続化し、書換え後から graph refresh + // までの中断を retry が no-op と誤認しないようにする。 + SetMeta(FoldBackfillGraphRefreshPendingMetaKey, "1"); + graphRefreshPending = true; + } + var foldBackfillPhase = rewriteAll ? GetMetaString(FoldBackfillPhaseMetaKey) : null; var symbols = BackfillSymbolFoldedRows(rewriteAll, cancellationToken); if (rewriteAll && foldBackfillPhase != "references") @@ -264,6 +280,15 @@ private bool SymbolExtractorVersionMatchesCurrent(string? lang) var symbolReferences = BackfillReferenceFoldedRows(rewriteAll, cancellationToken); cancellationToken.ThrowIfCancellationRequested(); + if (graphRefreshPending) + { + // Candidate membership and resolved identities depend on the persisted folded keys. + // Refresh them before advertising the rewritten rows as current. + // candidate と解決済み identity は永続化 folded key に依存するため、 + // 書換え後の key を current と公開する前に graph を再解決する。 + RefreshMutualRecursionFlags(cancellationToken); + SetMeta(FoldBackfillGraphRefreshPendingMetaKey, null); + } if (rewriteAll) ClearFoldBackfillCheckpoint(); diff --git a/src/CodeIndex/Database/NameFold.cs b/src/CodeIndex/Database/NameFold.cs index fa522dae8..f84b8b05b 100644 --- a/src/CodeIndex/Database/NameFold.cs +++ b/src/CodeIndex/Database/NameFold.cs @@ -51,16 +51,16 @@ public static class NameFold ]; /// - /// Algorithm version for the persisted `name_folded` / `*_folded` keys. Bumped whenever - /// changes semantics so an upgraded reader can detect that a previously - /// stamped DB's folded keys were generated by a different fold function and fall back to - /// the `COLLATE NOCASE` path until a rebuild regenerates the keys. Without this, a - /// Unicode CaseFold table refresh (or any future fold tweak) would silently mismatch - /// against legacy keys and return wrong zero results on `--exact` queries. - /// fold アルゴリズムのバージョン。Fold の意味が変わるたびに bump し、旧 key 版の DB を - /// NOCASE fallback に降格させる（fold table 更新などに備えるメタデータ）。 + /// Persisted `name_folded` / `*_folded` key contract version. Bumped whenever + /// or language-specific key generation changes semantics so an upgraded + /// reader can detect previously stamped incompatible keys and fall back to the + /// `COLLATE NOCASE` path until a rebuild/backfill regenerates them. Without this, a + /// Unicode CaseFold table refresh, language-specific identity rule, or future fold tweak + /// would silently mismatch legacy keys and return wrong zero results on `--exact` queries. + /// 永続化 folded key 契約のバージョン。Fold 自体または言語固有 key 生成の意味が + /// 変わるたびに bump し、旧 key 版の DB を NOCASE fallback に降格させる。 /// - public const int Version = 2; + public const int Version = 3; /// /// Runtime-sensitive canary fingerprint for the current observable fold output. diff --git a/src/CodeIndex/Indexer/References/ReferenceExtractor.CoreExtraction.cs b/src/CodeIndex/Indexer/References/ReferenceExtractor.CoreExtraction.cs index 4803582d5..a31ec6c63 100644 --- a/src/CodeIndex/Indexer/References/ReferenceExtractor.CoreExtraction.cs +++ b/src/CodeIndex/Indexer/References/ReferenceExtractor.CoreExtraction.cs @@ -2542,7 +2542,7 @@ bool TryAddCallLikeReference( context, lineNumber, callContainer, - language, + ScientificNativeReferenceExtractor.Supports(language) ? language : null, targetQualifier: targetQualifier); return true; diff --git a/src/CodeIndex/Models/ReferenceRecord.cs b/src/CodeIndex/Models/ReferenceRecord.cs index c4b3d03da..f89e34dae 100644 --- a/src/CodeIndex/Models/ReferenceRecord.cs +++ b/src/CodeIndex/Models/ReferenceRecord.cs @@ -1,3 +1,5 @@ +using System.Text.Json.Serialization; + namespace CodeIndex.Models; /// @@ -13,6 +15,7 @@ public class ReferenceRecord public string SymbolName { get; set; } = string.Empty; /// Language-specific persisted identity key when generic folding is insufficient / 一般的なfoldでは不十分な場合の言語固有identity key + [JsonInclude] internal string? IdentitySymbolNameFolded { get; set; } /// Reference kind such as call or instantiate / 参照種別 @@ -34,6 +37,7 @@ public class ReferenceRecord public string? ContainerName { get; set; } /// Language-specific persisted container identity key / 言語固有の永続化container identity key + [JsonInclude] internal string? IdentityContainerNameFolded { get; set; } /// @@ -44,6 +48,7 @@ public class ReferenceRecord public string? TargetQualifier { get; set; } /// True when a language-specific receiver explicitly denotes the current container / 言語固有receiverが現在のcontainerを明示する場合はtrue + [JsonInclude] internal bool SuppressInferredTargetQualifier { get; set; } /// True when the enclosing symbol references itself / 親シンボル自身への参照なら true diff --git a/src/CodeIndex/Models/SymbolRecord.cs b/src/CodeIndex/Models/SymbolRecord.cs index aad15d2be..0f7847854 100644 --- a/src/CodeIndex/Models/SymbolRecord.cs +++ b/src/CodeIndex/Models/SymbolRecord.cs @@ -1,3 +1,5 @@ +using System.Text.Json.Serialization; + namespace CodeIndex.Models; /// @@ -22,6 +24,7 @@ public class SymbolRecord public string Name { get; set; } = string.Empty; /// Language-specific persisted identity key when generic folding is insufficient / 一般的なfoldでは不十分な場合の言語固有identity key + [JsonInclude] internal string? IdentityNameFolded { get; set; } /// Line number (1-based) / 行番号（1始まり） diff --git a/tests/CodeIndex.Tests/DatabaseTests.cs b/tests/CodeIndex.Tests/DatabaseTests.cs index 4ab14d644..d48595948 100644 --- a/tests/CodeIndex.Tests/DatabaseTests.cs +++ b/tests/CodeIndex.Tests/DatabaseTests.cs @@ -1229,6 +1229,160 @@ FROM symbols Assert.Empty(reader.SearchReferences("MyProc", lang: "nim", exact: true)); } + [Fact] + public void ReferenceGraph_NimBackfillRewritesLegacyKeysAndRefreshesCandidates_Issue4738() + { + var fileId = UpsertTestFileWithLanguage( + "src/legacy-style.nim", + "nim", + "nim-legacy-style"); + _writer.InsertSymbols([ + new SymbolRecord + { + FileId = fileId, + Kind = "function", + Name = "my_proc", + Line = 1, + StartLine = 1, + EndLine = 1, + }, + new SymbolRecord + { + FileId = fileId, + Kind = "function", + Name = "RunGraph", + Line = 2, + StartLine = 2, + EndLine = 3, + BodyStartLine = 3, + BodyEndLine = 3, + }, + ]); + _writer.InsertReferences([ + new ReferenceRecord + { + FileId = fileId, + SymbolName = "myProc", + ReferenceKind = "call", + Line = 3, + Column = 3, + ContainerKind = "function", + ContainerName = "RunGraph", + Context = "myProc()", + }, + ], refreshMutualRecursionFlags: false); + _writer.RefreshMutualRecursionFlags(); + + Assert.Equal("unresolved", ReadReferenceResolutionState(fileId)); + + var rewritten = _writer.BackfillFoldedColumns(rewriteAll: true); + + Assert.Equal((2, 1), rewritten); + var expectedTargetId = ExecuteScalarLong($""" + SELECT id + FROM symbols + WHERE file_id = {fileId.ToString(CultureInfo.InvariantCulture)} + AND name = 'my_proc' + """); + Assert.Equal(expectedTargetId, ExecuteScalarLong($""" + SELECT target_symbol_id + FROM symbol_references + WHERE file_id = {fileId.ToString(CultureInfo.InvariantCulture)} + """)); + Assert.Equal("resolved", ReadReferenceResolutionState(fileId)); + } + + [Fact] + public void ReferenceGraph_NimBackfillRetriesInterruptedGraphRefresh_Issue4738() + { + var fileId = UpsertTestFileWithLanguage( + "src/interrupted-style.nim", + "nim", + "nim-interrupted-style"); + _writer.InsertSymbols([ + new SymbolRecord + { + FileId = fileId, + Kind = "function", + Name = "my_proc", + Line = 1, + StartLine = 1, + EndLine = 1, + }, + ]); + _writer.InsertReferences([ + new ReferenceRecord + { + FileId = fileId, + SymbolName = "myProc", + ReferenceKind = "call", + Line = 2, + Column = 1, + Context = "myProc()", + }, + ], refreshMutualRecursionFlags: false); + + var previousRefreshHook = DbWriter.MutualRecursionRefreshForTesting; + var interruptRefresh = true; + try + { + DbWriter.MutualRecursionRefreshForTesting = () => + { + previousRefreshHook?.Invoke(); + if (interruptRefresh) + throw new OperationCanceledException("interrupt fold graph refresh"); + }; + + Assert.Throws( + () => _writer.BackfillFoldedColumns(rewriteAll: true)); + + interruptRefresh = false; + Assert.Equal((0, 0), _writer.BackfillFoldedColumns(rewriteAll: true)); + } + finally + { + DbWriter.MutualRecursionRefreshForTesting = previousRefreshHook; + } + + Assert.Equal("resolved", ReadReferenceResolutionState(fileId)); + } + + [Fact] + public void SearchSymbols_NimExactDegradedPathPreservesRawUnderscores_Issue4738() + { + var fileId = UpsertTestFileWithLanguage( + "src/degraded-style.nim", + "nim", + "nim-degraded-style"); + _writer.InsertSymbols([ + new SymbolRecord + { + FileId = fileId, + Kind = "function", + Name = "my_proc", + Line = 1, + StartLine = 1, + EndLine = 1, + }, + ]); + _writer.SetMeta( + "fold_key_version", + (NameFold.Version - 1).ToString(CultureInfo.InvariantCulture)); + _writer.SetMeta("fold_key_fingerprint", NameFold.Fingerprint()); + using (var ready = _db.Connection.CreateCommand()) + { + ready.CommandText = + $"PRAGMA user_version = {_db.GetUserVersion() | DbContext.FoldReadyFlag}"; + ready.ExecuteNonQuery(); + } + + var reader = new DbReader(_db.Connection); + + Assert.False(reader._foldReady); + Assert.Single(reader.SearchSymbols("my_proc", lang: "nim", exact: true)); + Assert.Empty(reader.SearchSymbols("myProc", lang: "nim", exact: true)); + } + [Fact] public void ReferenceGraphDirtyScope_RollbackAndCancellationPreserveRetryState() { diff --git a/tests/CodeIndex.Tests/IndexCommandRunnerTests.cs b/tests/CodeIndex.Tests/IndexCommandRunnerTests.cs index 34b2ee511..2df866e91 100644 --- a/tests/CodeIndex.Tests/IndexCommandRunnerTests.cs +++ b/tests/CodeIndex.Tests/IndexCommandRunnerTests.cs @@ -498,6 +498,32 @@ public void SymbolExtractionWorker_Utf8RequestsPreserveUnicodeAcrossLanguages() } } + [Fact] + public void SymbolExtractionWorker_NimIdentityKeySurvivesProtocolRoundTrip_Issue4738() + { + var projectRoot = CreateTempProject(); + try + { + using var worker = new SymbolExtractionWorkerClient(); + var result = worker.Invoke( + 0, + "nim", + "proc my_proc() = discard\n", + Path.Combine(projectRoot, "sample.nim"), + projectRoot, + TimeSpan.FromSeconds(5)); + + Assert.True(result.Success, result.WorkerError); + var symbol = Assert.Single(result.Symbols!); + Assert.Equal("my_proc", symbol.Name); + Assert.Equal("myproc", symbol.IdentityNameFolded); + } + finally + { + DeleteDirectory(projectRoot); + } + } + [Fact] public void SymbolExtractionWorker_StreamResponseWritesBomlessUtf8Frame() { diff --git a/tests/CodeIndex.Tests/NameFoldTests.cs b/tests/CodeIndex.Tests/NameFoldTests.cs index 35310cfc4..2e6efcacf 100644 --- a/tests/CodeIndex.Tests/NameFoldTests.cs +++ b/tests/CodeIndex.Tests/NameFoldTests.cs @@ -42,4 +42,11 @@ public void Fingerprint_ReturnsLowercaseHex() Assert.Equal(fingerprint.ToLowerInvariant(), fingerprint); Assert.DoesNotContain(fingerprint, c => c is >= 'A' and <= 'F'); } + + [Fact] + public void PersistedKeyContract_VersionsNimStyleInsensitiveIdentity_Issue4738() + { + Assert.Equal(3, NameFold.Version); + Assert.Equal("myproc", DbReader.FoldNameForLanguage("my_proc", "nim")); + } } diff --git a/tests/CodeIndex.Tests/PostExtractionHookContractTests.cs b/tests/CodeIndex.Tests/PostExtractionHookContractTests.cs index 5c9dc6990..006b28e18 100644 --- a/tests/CodeIndex.Tests/PostExtractionHookContractTests.cs +++ b/tests/CodeIndex.Tests/PostExtractionHookContractTests.cs @@ -45,6 +45,49 @@ public void CallbackProtocol_RoundTripsRequestAndResponseContracts_Issue4185() Assert.Equal("TrimmedApp", Assert.Single(roundTrippedResponse.Symbols!).Name); } + [Fact] + public void CallbackProtocol_RoundTripsLanguageIdentityState_Issue4738() + { + var request = new PostExtractionHookCallbackProtocol.WorkerRequest( + nameof(IPostExtractionHook.OnReferencesExtracted), + new FileContext("project", "src/style.nim", "/project/src/style.nim", "nim"), + [ + new SymbolRecord + { + FileId = 5, + Kind = "function", + Name = "my_proc", + IdentityNameFolded = "myproc", + Line = 1, + StartLine = 1, + EndLine = 1, + }, + ], + [ + new ReferenceRecord + { + FileId = 5, + SymbolName = "myProc", + IdentitySymbolNameFolded = "myproc", + ReferenceKind = "call", + Line = 2, + Column = 5, + ContainerName = "RunGraph", + IdentityContainerNameFolded = "Rungraph", + SuppressInferredTargetQualifier = true, + }, + ]); + + var roundTripped = PostExtractionHookCallbackProtocol.DeserializeRequest( + PostExtractionHookCallbackProtocol.SerializeRequest(request)); + + Assert.Equal("myproc", Assert.Single(roundTripped.Symbols!).IdentityNameFolded); + var reference = Assert.Single(roundTripped.References!); + Assert.Equal("myproc", reference.IdentitySymbolNameFolded); + Assert.Equal("Rungraph", reference.IdentityContainerNameFolded); + Assert.True(reference.SuppressInferredTargetQualifier); + } + [Fact] public void MutationMaterializer_ClonesAndTrimsRecordsWithinContracts_Issue4185() { From 94a17fdb6d0666a6b0b85f3a0f7e7b877ac12a70 Mon Sep 17 00:00:00 2001 From: Widthdom Date: Fri, 24 Jul 2026 23:02:16 +0900 Subject: [PATCH 21/21] Resolve second adversarial review findings (#4738) --- .../Database/DbReader.GraphQueries.cs | 90 +++++++++++-------- src/CodeIndex/Database/DbReader.References.cs | 49 +++++----- src/CodeIndex/Database/DbReader.cs | 17 ++++ .../Database/DbSymbolReader.Analysis.cs | 6 +- .../Database/DbSymbolReader.Definitions.cs | 9 +- .../Database/DbSymbolReader.Search.cs | 35 ++++++-- .../PostExtractionHookMutationMaterializer.cs | 26 ++++++ .../Indexer/Hooks/PostExtractionHooks.cs | 8 ++ .../ScientificNativeReferenceExtractor.cs | 2 +- .../Indexer/References/ReferenceExtractor.cs | 3 +- .../Indexer/Symbols/SymbolExtractor.Pascal.cs | 76 +++++++++++++++- .../SymbolExtractor.ScientificNative.cs | 43 +++++++++ .../Indexer/Symbols/SymbolExtractor.cs | 25 +++++- tests/CodeIndex.Tests/DatabaseTests.cs | 27 ++++++ .../PostExtractionHookContractTests.cs | 65 +++++++++++++- .../QueryCommandRunnerImpactTests.cs | 1 + .../QueryCommandRunnerSymbolTests.cs | 6 +- ...eferenceExtractorTests.ScientificNative.cs | 90 +++++++++++++++++++ 18 files changed, 498 insertions(+), 80 deletions(-) diff --git a/src/CodeIndex/Database/DbReader.GraphQueries.cs b/src/CodeIndex/Database/DbReader.GraphQueries.cs index 89398dede..3d382ccaf 100644 --- a/src/CodeIndex/Database/DbReader.GraphQueries.cs +++ b/src/CodeIndex/Database/DbReader.GraphQueries.cs @@ -126,7 +126,7 @@ FROM symbol_reference_candidates AS identity_candidate var qualifiedContextSql = BuildQualifiedContextMatchSql(contextSql, "r.column_number", folded: true, like: false); var csharpQualifiedContextSql = BuildCSharpQualifiedContextFallbackSql(qualifiedContextSql); var qualifiedLeafFallbackSql = BuildQualifiedLeafFallbackSql("r.symbol_name", "r.symbol_name_folded", folded: true); - sql += $" AND (((f.lang = 'sql') AND {qualifiedContextSql}) OR ((f.lang != 'sql') AND r.symbol_name_folded = @query) OR {csharpQualifiedContextSql} OR {qualifiedLeafFallbackSql})"; + sql += $" AND (((f.lang = 'sql') AND {qualifiedContextSql}) OR ((f.lang != 'sql') AND {BuildPersistedFoldedNameMatchSql("r.symbol_name_folded", "@query")}) OR {csharpQualifiedContextSql} OR {qualifiedLeafFallbackSql})"; } else if (useSqlQualifiedContextMatch && exact) { @@ -152,9 +152,9 @@ FROM symbol_reference_candidates AS identity_candidate else if (exact && _foldReady) sql += allowSqlLeafFallback ? cssScssVariableAlias != null - ? $" AND (r.symbol_name_folded = @query OR (r.symbol_name_folded = @queryCssScssVariableAlias{cssScssVariableAliasScope}) OR (f.lang = 'sql' AND r.symbol_name_folded = @aliasQueryLeafFolded))" - : " AND (r.symbol_name_folded = @query OR (f.lang = 'sql' AND r.symbol_name_folded = @aliasQueryLeafFolded))" - : " AND r.symbol_name_folded = @query"; + ? $" AND ({BuildPersistedFoldedNameMatchSql("r.symbol_name_folded", "@query")} OR (r.symbol_name_folded = @queryCssScssVariableAlias{cssScssVariableAliasScope}) OR (f.lang = 'sql' AND r.symbol_name_folded = @aliasQueryLeafFolded))" + : $" AND ({BuildPersistedFoldedNameMatchSql("r.symbol_name_folded", "@query")} OR (f.lang = 'sql' AND r.symbol_name_folded = @aliasQueryLeafFolded))" + : $" AND {BuildPersistedFoldedNameMatchSql("r.symbol_name_folded", "@query")}"; else if (exact) sql += allowSqlLeafFallback ? cssScssVariableAlias != null @@ -198,7 +198,10 @@ FROM logical_references r callersQueryParam = FoldNameForLanguage(query, lang); else callersQueryParam = query; - SqliteCommandPolicy.Add(cmd, "@query", callersQueryParam); + if (exact && _foldReady) + AddPersistedFoldedNameQueryParameters(cmd, "@query", query, lang); + else + SqliteCommandPolicy.Add(cmd, "@query", callersQueryParam); SqliteCommandPolicy.Add(cmd, "@aliasQuery", query); AddQualifiedGraphQueryParameters(cmd, query, allowQualifiedLeafFallback, allowCSharpQualifiedContextMatch); SqliteCommandPolicy.Add(cmd, "@aliasQueryLeafFolded", NameFold.Fold(SqlNameResolver.GetLeafName(query)) ?? SqlNameResolver.GetLeafName(query)); @@ -289,7 +292,7 @@ FROM symbol_references r var qualifiedContextSql = BuildQualifiedContextMatchSql(contextSql, "r.column_number", folded: true, like: false); var csharpQualifiedContextSql = BuildCSharpQualifiedContextFallbackSql(qualifiedContextSql); var qualifiedLeafFallbackSql = BuildQualifiedLeafFallbackSql("r.symbol_name", "r.symbol_name_folded", folded: true); - groupedSql += $" AND (((f.lang = 'sql') AND {qualifiedContextSql}) OR ((f.lang != 'sql') AND r.symbol_name_folded = @query) OR {csharpQualifiedContextSql} OR {qualifiedLeafFallbackSql})"; + groupedSql += $" AND (((f.lang = 'sql') AND {qualifiedContextSql}) OR ((f.lang != 'sql') AND {BuildPersistedFoldedNameMatchSql("r.symbol_name_folded", "@query")}) OR {csharpQualifiedContextSql} OR {qualifiedLeafFallbackSql})"; } else if (useSqlQualifiedContextMatch && exact) { @@ -315,9 +318,9 @@ FROM symbol_references r else if (exact && _foldReady) groupedSql += allowSqlLeafFallback ? cssScssVariableAlias != null - ? $" AND (r.symbol_name_folded = @query OR (r.symbol_name_folded = @queryCssScssVariableAlias{cssScssVariableAliasScope}) OR (f.lang = 'sql' AND r.symbol_name_folded = @aliasQueryLeafFolded))" - : " AND (r.symbol_name_folded = @query OR (f.lang = 'sql' AND r.symbol_name_folded = @aliasQueryLeafFolded))" - : " AND r.symbol_name_folded = @query"; + ? $" AND ({BuildPersistedFoldedNameMatchSql("r.symbol_name_folded", "@query")} OR (r.symbol_name_folded = @queryCssScssVariableAlias{cssScssVariableAliasScope}) OR (f.lang = 'sql' AND r.symbol_name_folded = @aliasQueryLeafFolded))" + : $" AND ({BuildPersistedFoldedNameMatchSql("r.symbol_name_folded", "@query")} OR (f.lang = 'sql' AND r.symbol_name_folded = @aliasQueryLeafFolded))" + : $" AND {BuildPersistedFoldedNameMatchSql("r.symbol_name_folded", "@query")}"; else if (exact) groupedSql += allowSqlLeafFallback ? cssScssVariableAlias != null @@ -341,7 +344,10 @@ FROM symbol_references r : _foldReady ? FoldNameForLanguage(query, lang) : query; - SqliteCommandPolicy.Add(cmd, "@query", value); + if (exact && _foldReady) + AddPersistedFoldedNameQueryParameters(cmd, "@query", query, lang); + else + SqliteCommandPolicy.Add(cmd, "@query", value); SqliteCommandPolicy.Add(cmd, "@aliasQuery", query); AddQualifiedGraphQueryParameters(cmd, query, allowQualifiedLeafFallback, allowCSharpQualifiedContextMatch); SqliteCommandPolicy.Add(cmd, "@aliasQueryLeafFolded", NameFold.Fold(SqlNameResolver.GetLeafName(query)) ?? SqlNameResolver.GetLeafName(query)); @@ -396,7 +402,7 @@ FROM symbol_references r var qualifiedContextSql = BuildQualifiedContextMatchSql(contextSql, "r.column_number", folded: true, like: false); var csharpQualifiedContextSql = BuildCSharpQualifiedContextFallbackSql(qualifiedContextSql); var qualifiedLeafFallbackSql = BuildQualifiedLeafFallbackSql("r.symbol_name", "r.symbol_name_folded", folded: true); - groupedSql += $" AND (((f.lang = 'sql') AND {qualifiedContextSql}) OR ((f.lang != 'sql') AND r.symbol_name_folded = @query) OR {csharpQualifiedContextSql} OR {qualifiedLeafFallbackSql})"; + groupedSql += $" AND (((f.lang = 'sql') AND {qualifiedContextSql}) OR ((f.lang != 'sql') AND {BuildPersistedFoldedNameMatchSql("r.symbol_name_folded", "@query")}) OR {csharpQualifiedContextSql} OR {qualifiedLeafFallbackSql})"; } else if (useSqlQualifiedContextMatch && exact) { @@ -422,9 +428,9 @@ FROM symbol_references r else if (exact && _foldReady) groupedSql += allowSqlLeafFallback ? cssScssVariableAlias != null - ? $" AND (r.symbol_name_folded = @query OR (r.symbol_name_folded = @queryCssScssVariableAlias{cssScssVariableAliasScope}) OR (f.lang = 'sql' AND r.symbol_name_folded = @aliasQueryLeafFolded))" - : " AND (r.symbol_name_folded = @query OR (f.lang = 'sql' AND r.symbol_name_folded = @aliasQueryLeafFolded))" - : " AND r.symbol_name_folded = @query"; + ? $" AND ({BuildPersistedFoldedNameMatchSql("r.symbol_name_folded", "@query")} OR (r.symbol_name_folded = @queryCssScssVariableAlias{cssScssVariableAliasScope}) OR (f.lang = 'sql' AND r.symbol_name_folded = @aliasQueryLeafFolded))" + : $" AND ({BuildPersistedFoldedNameMatchSql("r.symbol_name_folded", "@query")} OR (f.lang = 'sql' AND r.symbol_name_folded = @aliasQueryLeafFolded))" + : $" AND {BuildPersistedFoldedNameMatchSql("r.symbol_name_folded", "@query")}"; else if (exact) groupedSql += allowSqlLeafFallback ? cssScssVariableAlias != null @@ -448,7 +454,10 @@ FROM symbol_references r : _foldReady ? FoldNameForLanguage(query, lang) : query; - SqliteCommandPolicy.Add(cmd, "@query", value); + if (exact && _foldReady) + AddPersistedFoldedNameQueryParameters(cmd, "@query", query, lang); + else + SqliteCommandPolicy.Add(cmd, "@query", value); SqliteCommandPolicy.Add(cmd, "@aliasQuery", query); AddQualifiedGraphQueryParameters(cmd, query, allowQualifiedLeafFallback, allowCSharpQualifiedContextMatch); SqliteCommandPolicy.Add(cmd, "@aliasQueryLeafFolded", NameFold.Fold(SqlNameResolver.GetLeafName(query)) ?? SqlNameResolver.GetLeafName(query)); @@ -519,7 +528,7 @@ WHERE r.container_name IS NOT NULL if (exact && useSqlQualifiedContainerMatch && _foldReady) { var qualifiedLeafFallbackSql = BuildQualifiedLeafFallbackSql("r.container_name", "r.container_name_folded", folded: true); - sql += $" AND (((f.lang = 'sql') AND sql_segment_count(r.container_name) = @aliasQuerySegmentCount AND sql_normalize_name_folded(r.container_name) = @aliasQueryNormalizedFolded) OR ((f.lang != 'sql') AND r.container_name_folded = @query) OR {qualifiedLeafFallbackSql})"; + sql += $" AND (((f.lang = 'sql') AND sql_segment_count(r.container_name) = @aliasQuerySegmentCount AND sql_normalize_name_folded(r.container_name) = @aliasQueryNormalizedFolded) OR ((f.lang != 'sql') AND {BuildPersistedFoldedNameMatchSql("r.container_name_folded", "@query")}) OR {qualifiedLeafFallbackSql})"; } else if (exact && useSqlQualifiedContainerMatch) { @@ -539,9 +548,9 @@ WHERE r.container_name IS NOT NULL else if (exact && _foldReady) sql += allowSqlLeafFallback ? cssScssVariableAlias != null - ? $" AND (r.container_name_folded = @query OR (r.container_name_folded = @queryCssScssVariableAlias{cssScssVariableAliasScope}) OR (f.lang = 'sql' AND sql_leaf_name_folded(r.container_name) = @aliasQueryLeafFolded))" - : " AND (r.container_name_folded = @query OR (f.lang = 'sql' AND sql_leaf_name_folded(r.container_name) = @aliasQueryLeafFolded))" - : " AND r.container_name_folded = @query"; + ? $" AND ({BuildPersistedFoldedNameMatchSql("r.container_name_folded", "@query")} OR (r.container_name_folded = @queryCssScssVariableAlias{cssScssVariableAliasScope}) OR (f.lang = 'sql' AND sql_leaf_name_folded(r.container_name) = @aliasQueryLeafFolded))" + : $" AND ({BuildPersistedFoldedNameMatchSql("r.container_name_folded", "@query")} OR (f.lang = 'sql' AND sql_leaf_name_folded(r.container_name) = @aliasQueryLeafFolded))" + : $" AND {BuildPersistedFoldedNameMatchSql("r.container_name_folded", "@query")}"; else if (exact) sql += allowSqlLeafFallback ? cssScssVariableAlias != null @@ -575,7 +584,10 @@ FROM logical_references r calleesQueryParam = FoldNameForLanguage(query, lang); else calleesQueryParam = query; - SqliteCommandPolicy.Add(cmd, "@query", calleesQueryParam); + if (exact && _foldReady) + AddPersistedFoldedNameQueryParameters(cmd, "@query", query, lang); + else + SqliteCommandPolicy.Add(cmd, "@query", calleesQueryParam); SqliteCommandPolicy.Add(cmd, "@aliasQuery", query); SqliteCommandPolicy.Add(cmd, "@aliasQueryLeafFolded", NameFold.Fold(SqlNameResolver.GetLeafName(query)) ?? SqlNameResolver.GetLeafName(query)); SqliteCommandPolicy.Add(cmd, "@aliasQueryNormalized", SqlNameResolver.NormalizeQualifiedName(query)); @@ -661,7 +673,7 @@ FROM symbol_references r if (exact && useSqlQualifiedContainerMatch && _foldReady) { var qualifiedLeafFallbackSql = BuildQualifiedLeafFallbackSql("r.container_name", "r.container_name_folded", folded: true); - groupedSql += $" AND (((f.lang = 'sql') AND sql_segment_count(r.container_name) = @aliasQuerySegmentCount AND sql_normalize_name_folded(r.container_name) = @aliasQueryNormalizedFolded) OR ((f.lang != 'sql') AND r.container_name_folded = @query) OR {qualifiedLeafFallbackSql})"; + groupedSql += $" AND (((f.lang = 'sql') AND sql_segment_count(r.container_name) = @aliasQuerySegmentCount AND sql_normalize_name_folded(r.container_name) = @aliasQueryNormalizedFolded) OR ((f.lang != 'sql') AND {BuildPersistedFoldedNameMatchSql("r.container_name_folded", "@query")}) OR {qualifiedLeafFallbackSql})"; } else if (exact && useSqlQualifiedContainerMatch) { @@ -681,9 +693,9 @@ FROM symbol_references r else if (exact && _foldReady) groupedSql += allowSqlLeafFallback ? cssScssVariableAlias != null - ? $" AND (r.container_name_folded = @query OR (r.container_name_folded = @queryCssScssVariableAlias{cssScssVariableAliasScope}) OR (f.lang = 'sql' AND sql_leaf_name_folded(r.container_name) = @aliasQueryLeafFolded))" - : " AND (r.container_name_folded = @query OR (f.lang = 'sql' AND sql_leaf_name_folded(r.container_name) = @aliasQueryLeafFolded))" - : " AND r.container_name_folded = @query"; + ? $" AND ({BuildPersistedFoldedNameMatchSql("r.container_name_folded", "@query")} OR (r.container_name_folded = @queryCssScssVariableAlias{cssScssVariableAliasScope}) OR (f.lang = 'sql' AND sql_leaf_name_folded(r.container_name) = @aliasQueryLeafFolded))" + : $" AND ({BuildPersistedFoldedNameMatchSql("r.container_name_folded", "@query")} OR (f.lang = 'sql' AND sql_leaf_name_folded(r.container_name) = @aliasQueryLeafFolded))" + : $" AND {BuildPersistedFoldedNameMatchSql("r.container_name_folded", "@query")}"; else if (exact) groupedSql += allowSqlLeafFallback ? cssScssVariableAlias != null @@ -706,7 +718,10 @@ FROM symbol_references r : _foldReady ? FoldNameForLanguage(query, lang) : query; - SqliteCommandPolicy.Add(cmd, "@query", value); + if (exact && _foldReady) + AddPersistedFoldedNameQueryParameters(cmd, "@query", query, lang); + else + SqliteCommandPolicy.Add(cmd, "@query", value); SqliteCommandPolicy.Add(cmd, "@aliasQuery", query); SqliteCommandPolicy.Add(cmd, "@aliasQueryLeafFolded", NameFold.Fold(SqlNameResolver.GetLeafName(query)) ?? SqlNameResolver.GetLeafName(query)); SqliteCommandPolicy.Add(cmd, "@aliasQueryNormalized", SqlNameResolver.NormalizeQualifiedName(query)); @@ -760,7 +775,7 @@ FROM symbol_references r if (exact && useSqlQualifiedContainerMatch && _foldReady) { var qualifiedLeafFallbackSql = BuildQualifiedLeafFallbackSql("r.container_name", "r.container_name_folded", folded: true); - groupedSql += $" AND (((f.lang = 'sql') AND sql_segment_count(r.container_name) = @aliasQuerySegmentCount AND sql_normalize_name_folded(r.container_name) = @aliasQueryNormalizedFolded) OR ((f.lang != 'sql') AND r.container_name_folded = @query) OR {qualifiedLeafFallbackSql})"; + groupedSql += $" AND (((f.lang = 'sql') AND sql_segment_count(r.container_name) = @aliasQuerySegmentCount AND sql_normalize_name_folded(r.container_name) = @aliasQueryNormalizedFolded) OR ((f.lang != 'sql') AND {BuildPersistedFoldedNameMatchSql("r.container_name_folded", "@query")}) OR {qualifiedLeafFallbackSql})"; } else if (exact && useSqlQualifiedContainerMatch) { @@ -780,9 +795,9 @@ FROM symbol_references r else if (exact && _foldReady) groupedSql += allowSqlLeafFallback ? cssScssVariableAlias != null - ? $" AND (r.container_name_folded = @query OR (r.container_name_folded = @queryCssScssVariableAlias{cssScssVariableAliasScope}) OR (f.lang = 'sql' AND sql_leaf_name_folded(r.container_name) = @aliasQueryLeafFolded))" - : " AND (r.container_name_folded = @query OR (f.lang = 'sql' AND sql_leaf_name_folded(r.container_name) = @aliasQueryLeafFolded))" - : " AND r.container_name_folded = @query"; + ? $" AND ({BuildPersistedFoldedNameMatchSql("r.container_name_folded", "@query")} OR (r.container_name_folded = @queryCssScssVariableAlias{cssScssVariableAliasScope}) OR (f.lang = 'sql' AND sql_leaf_name_folded(r.container_name) = @aliasQueryLeafFolded))" + : $" AND ({BuildPersistedFoldedNameMatchSql("r.container_name_folded", "@query")} OR (f.lang = 'sql' AND sql_leaf_name_folded(r.container_name) = @aliasQueryLeafFolded))" + : $" AND {BuildPersistedFoldedNameMatchSql("r.container_name_folded", "@query")}"; else if (exact) groupedSql += allowSqlLeafFallback ? cssScssVariableAlias != null @@ -805,7 +820,10 @@ FROM symbol_references r : _foldReady ? FoldNameForLanguage(query, lang) : query; - SqliteCommandPolicy.Add(cmd, "@query", value); + if (exact && _foldReady) + AddPersistedFoldedNameQueryParameters(cmd, "@query", query, lang); + else + SqliteCommandPolicy.Add(cmd, "@query", value); SqliteCommandPolicy.Add(cmd, "@aliasQuery", query); SqliteCommandPolicy.Add(cmd, "@aliasQueryLeafFolded", NameFold.Fold(SqlNameResolver.GetLeafName(query)) ?? SqlNameResolver.GetLeafName(query)); SqliteCommandPolicy.Add(cmd, "@aliasQueryNormalized", SqlNameResolver.NormalizeQualifiedName(query)); @@ -974,9 +992,9 @@ private List GetCallersExactCore(string symbolName, int limit, int var nameCondition = _foldReady ? allowSqlLeafFallback ? @" - AND (r.symbol_name_folded = @symbolNameFolded OR (f.lang = 'sql' AND r.symbol_name_folded = @symbolNameLeafFolded)" + polymorphicNameCondition + " OR (f.lang = 'solution' AND r.reference_kind = 'project_reference' AND r.container_name = @symbolName COLLATE NOCASE))" + AND (" + BuildPersistedFoldedNameMatchSql("r.symbol_name_folded", "@symbolNameFolded") + " OR (f.lang = 'sql' AND r.symbol_name_folded = @symbolNameLeafFolded)" + polymorphicNameCondition + " OR (f.lang = 'solution' AND r.reference_kind = 'project_reference' AND r.container_name = @symbolName COLLATE NOCASE))" : @" - AND (((f.lang = 'sql') AND sql_context_has_name_folded_at(" + contextSql + @", @symbolName, r.column_number) = 1) OR ((f.lang != 'sql') AND r.symbol_name_folded = @symbolNameFolded) OR " + BuildCSharpQualifiedContextFallbackSql(BuildQualifiedContextMatchSql(contextSql, "r.column_number", folded: true, like: false)) + " OR " + BuildQualifiedLeafFallbackSql("r.symbol_name", "r.symbol_name_folded", folded: true) + polymorphicNameCondition + " OR (f.lang = 'solution' AND r.reference_kind = 'project_reference' AND r.container_name = @symbolName COLLATE NOCASE))" + AND (((f.lang = 'sql') AND sql_context_has_name_folded_at(" + contextSql + @", @symbolName, r.column_number) = 1) OR ((f.lang != 'sql') AND " + BuildPersistedFoldedNameMatchSql("r.symbol_name_folded", "@symbolNameFolded") + ") OR " + BuildCSharpQualifiedContextFallbackSql(BuildQualifiedContextMatchSql(contextSql, "r.column_number", folded: true, like: false)) + " OR " + BuildQualifiedLeafFallbackSql("r.symbol_name", "r.symbol_name_folded", folded: true) + polymorphicNameCondition + " OR (f.lang = 'solution' AND r.reference_kind = 'project_reference' AND r.container_name = @symbolName COLLATE NOCASE))" : allowSqlLeafFallback ? @" AND (r.symbol_name = @symbolName COLLATE NOCASE OR (f.lang = 'sql' AND r.symbol_name = sql_leaf_name(@symbolName) COLLATE NOCASE)" + polymorphicNameCondition + " OR (f.lang = 'solution' AND r.reference_kind = 'project_reference' AND r.container_name = @symbolName COLLATE NOCASE))" @@ -1042,7 +1060,7 @@ FROM logical_references r SqliteCommandPolicy.Add(cmd, "@aliasQueryLeafFolded", NameFold.Fold(SqlNameResolver.GetLeafName(symbolName)) ?? SqlNameResolver.GetLeafName(symbolName)); SqliteCommandPolicy.Add(cmd, "@symbolNameLeafFolded", NameFold.Fold(SqlNameResolver.GetLeafName(symbolName)) ?? SqlNameResolver.GetLeafName(symbolName)); if (_foldReady) - SqliteCommandPolicy.Add(cmd, "@symbolNameFolded", FoldNameForLanguage(symbolName, lang)); + AddPersistedFoldedNameQueryParameters(cmd, "@symbolNameFolded", symbolName, lang); for (var i = 0; i < polymorphicCSharpSymbolNames.Count; i++) { if (_foldReady) @@ -2051,8 +2069,8 @@ private ImpactDefinitionResolution ResolveImpactDefinitions( GetSymbolColumnSql("family_key")); var nameCondition = _foldReady ? allowLeafFallback - ? "(s.name_folded = @resolvedNameFolded OR (f.lang = 'sql' AND ((sql_segment_count(s.name) = @resolvedNameSegmentCount AND sql_normalize_name_folded(s.name) = @resolvedNameNormalizedFolded) OR sql_leaf_name_folded(s.name) = @resolvedNameLeafFolded)))" - : "(s.name_folded = @resolvedNameFolded OR (f.lang = 'sql' AND sql_segment_count(s.name) = @resolvedNameSegmentCount AND sql_normalize_name_folded(s.name) = @resolvedNameNormalizedFolded))" + ? $"({BuildPersistedFoldedNameMatchSql("s.name_folded", "@resolvedNameFolded")} OR (f.lang = 'sql' AND ((sql_segment_count(s.name) = @resolvedNameSegmentCount AND sql_normalize_name_folded(s.name) = @resolvedNameNormalizedFolded) OR sql_leaf_name_folded(s.name) = @resolvedNameLeafFolded)))" + : $"({BuildPersistedFoldedNameMatchSql("s.name_folded", "@resolvedNameFolded")} OR (f.lang = 'sql' AND sql_segment_count(s.name) = @resolvedNameSegmentCount AND sql_normalize_name_folded(s.name) = @resolvedNameNormalizedFolded))" : allowLeafFallback ? "(s.name = @resolvedName COLLATE NOCASE OR (f.lang = 'sql' AND ((sql_segment_count(s.name) = @resolvedNameSegmentCount AND sql_normalize_name(s.name) = @resolvedNameNormalized COLLATE NOCASE) OR sql_leaf_name(s.name) = @resolvedNameLeaf COLLATE NOCASE)))" : "(s.name = @resolvedName COLLATE NOCASE OR (f.lang = 'sql' AND sql_segment_count(s.name) = @resolvedNameSegmentCount AND sql_normalize_name(s.name) = @resolvedNameNormalized COLLATE NOCASE))"; @@ -2189,7 +2207,7 @@ CROSS JOIN definition_stats stats SqliteCommandPolicy.Add(cmd, "@resolvedNameContainerSuffix", $"%.{EscapeLikeQuery(container)}"); } if (_foldReady) - SqliteCommandPolicy.Add(cmd, "@resolvedNameFolded", FoldNameForLanguage(resolvedName, lang)); + AddPersistedFoldedNameQueryParameters(cmd, "@resolvedNameFolded", resolvedName, lang); if (lang != null) SqliteCommandPolicy.Add(cmd, "@lang", lang); SqliteCommandPolicy.Add(cmd, "@definitionLimit", Math.Max(1, representativeLimit)); diff --git a/src/CodeIndex/Database/DbReader.References.cs b/src/CodeIndex/Database/DbReader.References.cs index 43005ce1c..58f06294d 100644 --- a/src/CodeIndex/Database/DbReader.References.cs +++ b/src/CodeIndex/Database/DbReader.References.cs @@ -314,8 +314,8 @@ FROM symbol_references r var csharpQualifiedContextSql = BuildCSharpQualifiedContextFallbackSql(qualifiedContextSql); var qualifiedLeafFallbackSql = BuildQualifiedLeafFallbackSql("r.symbol_name", "r.symbol_name_folded", folded: true); sql += referencesSuffixAlias != null - ? $" AND (((f.lang = 'sql') AND {qualifiedContextSql}) OR ((f.lang != 'sql') AND (r.symbol_name_folded = @query OR (r.symbol_name_folded = @queryAttributeAlias{referencesAliasScope}))) OR {csharpQualifiedContextSql} OR {qualifiedLeafFallbackSql})" - : $" AND (((f.lang = 'sql') AND {qualifiedContextSql}) OR ((f.lang != 'sql') AND r.symbol_name_folded = @query) OR {csharpQualifiedContextSql} OR {qualifiedLeafFallbackSql})"; + ? $" AND (((f.lang = 'sql') AND {qualifiedContextSql}) OR ((f.lang != 'sql') AND ({BuildPersistedFoldedNameMatchSql("r.symbol_name_folded", "@query")} OR (r.symbol_name_folded = @queryAttributeAlias{referencesAliasScope}))) OR {csharpQualifiedContextSql} OR {qualifiedLeafFallbackSql})" + : $" AND (((f.lang = 'sql') AND {qualifiedContextSql}) OR ((f.lang != 'sql') AND {BuildPersistedFoldedNameMatchSql("r.symbol_name_folded", "@query")}) OR {csharpQualifiedContextSql} OR {qualifiedLeafFallbackSql})"; } else if (useSqlQualifiedContextMatch && exact) { @@ -346,12 +346,12 @@ FROM symbol_references r } else if (exact && _foldReady) sql += referencesSuffixAlias != null - ? $" AND (r.symbol_name_folded = @query OR (r.symbol_name_folded = @queryAttributeAlias{referencesAliasScope}){(allowSqlLeafFallback ? $" OR (r.symbol_name_folded = @aliasQueryLeafFolded{sqlLeafReferenceScope})" : string.Empty)})" + ? $" AND ({BuildPersistedFoldedNameMatchSql("r.symbol_name_folded", "@query")} OR (r.symbol_name_folded = @queryAttributeAlias{referencesAliasScope}){(allowSqlLeafFallback ? $" OR (r.symbol_name_folded = @aliasQueryLeafFolded{sqlLeafReferenceScope})" : string.Empty)})" : referencesCssScssVariableAlias != null - ? $" AND (r.symbol_name_folded = @query OR (r.symbol_name_folded = @queryCssScssVariableAlias{referencesCssScssVariableAliasScope}){(allowSqlLeafFallback ? $" OR (r.symbol_name_folded = @aliasQueryLeafFolded{sqlLeafReferenceScope})" : string.Empty)})" + ? $" AND ({BuildPersistedFoldedNameMatchSql("r.symbol_name_folded", "@query")} OR (r.symbol_name_folded = @queryCssScssVariableAlias{referencesCssScssVariableAliasScope}){(allowSqlLeafFallback ? $" OR (r.symbol_name_folded = @aliasQueryLeafFolded{sqlLeafReferenceScope})" : string.Empty)})" : allowSqlLeafFallback - ? $" AND (r.symbol_name_folded = @query OR (r.symbol_name_folded = @aliasQueryLeafFolded{sqlLeafReferenceScope}))" - : " AND r.symbol_name_folded = @query"; + ? $" AND ({BuildPersistedFoldedNameMatchSql("r.symbol_name_folded", "@query")} OR (r.symbol_name_folded = @aliasQueryLeafFolded{sqlLeafReferenceScope}))" + : $" AND {BuildPersistedFoldedNameMatchSql("r.symbol_name_folded", "@query")}"; else if (exact) sql += referencesSuffixAlias != null ? $" AND (r.symbol_name = @query COLLATE NOCASE OR (r.symbol_name = @queryAttributeAlias COLLATE NOCASE{referencesAliasScope}){(allowSqlLeafFallback ? $" OR (r.symbol_name = sql_leaf_name(@aliasQuery) COLLATE NOCASE{sqlLeafReferenceScope})" : string.Empty)})" @@ -411,7 +411,10 @@ FROM symbol_reference_candidates AS identity_candidate queryParam = FoldNameForLanguage(query, lang); else queryParam = query; - SqliteCommandPolicy.Add(cmd, "@query", queryParam); + if (exact && _foldReady) + AddPersistedFoldedNameQueryParameters(cmd, "@query", query, lang); + else + SqliteCommandPolicy.Add(cmd, "@query", queryParam); SqliteCommandPolicy.Add(cmd, "@aliasQuery", query); AddQualifiedGraphQueryParameters(cmd, query, allowQualifiedLeafFallback, allowCSharpQualifiedContextMatch); SqliteCommandPolicy.Add(cmd, "@aliasQueryLeafFolded", NameFold.Fold(SqlNameResolver.GetLeafName(query)) ?? SqlNameResolver.GetLeafName(query)); @@ -722,8 +725,8 @@ FROM symbol_references r var csharpQualifiedContextSql = BuildCSharpQualifiedContextFallbackSql(qualifiedContextSql); var qualifiedLeafFallbackSql = BuildQualifiedLeafFallbackSql("r.symbol_name", "r.symbol_name_folded", folded: true); innerSql += countSuffixAlias != null - ? $" AND (((f.lang = 'sql') AND {qualifiedContextSql}) OR ((f.lang != 'sql') AND (r.symbol_name_folded = @query OR (r.symbol_name_folded = @queryAttributeAlias{countAliasScope}))) OR {csharpQualifiedContextSql} OR {qualifiedLeafFallbackSql})" - : $" AND (((f.lang = 'sql') AND {qualifiedContextSql}) OR ((f.lang != 'sql') AND r.symbol_name_folded = @query) OR {csharpQualifiedContextSql} OR {qualifiedLeafFallbackSql})"; + ? $" AND (((f.lang = 'sql') AND {qualifiedContextSql}) OR ((f.lang != 'sql') AND ({BuildPersistedFoldedNameMatchSql("r.symbol_name_folded", "@query")} OR (r.symbol_name_folded = @queryAttributeAlias{countAliasScope}))) OR {csharpQualifiedContextSql} OR {qualifiedLeafFallbackSql})" + : $" AND (((f.lang = 'sql') AND {qualifiedContextSql}) OR ((f.lang != 'sql') AND {BuildPersistedFoldedNameMatchSql("r.symbol_name_folded", "@query")}) OR {csharpQualifiedContextSql} OR {qualifiedLeafFallbackSql})"; } else if (useSqlQualifiedContextMatch && exact) { @@ -754,10 +757,10 @@ FROM symbol_references r } else if (exact && _foldReady) innerSql += countSuffixAlias != null - ? $" AND (r.symbol_name_folded = @query OR (r.symbol_name_folded = @queryAttributeAlias{countAliasScope}){(allowSqlLeafFallback ? $" OR (r.symbol_name_folded = @aliasQueryLeafFolded{sqlLeafCountScope})" : string.Empty)})" + ? $" AND ({BuildPersistedFoldedNameMatchSql("r.symbol_name_folded", "@query")} OR (r.symbol_name_folded = @queryAttributeAlias{countAliasScope}){(allowSqlLeafFallback ? $" OR (r.symbol_name_folded = @aliasQueryLeafFolded{sqlLeafCountScope})" : string.Empty)})" : allowSqlLeafFallback - ? $" AND (r.symbol_name_folded = @query OR (r.symbol_name_folded = @aliasQueryLeafFolded{sqlLeafCountScope}))" - : " AND r.symbol_name_folded = @query"; + ? $" AND ({BuildPersistedFoldedNameMatchSql("r.symbol_name_folded", "@query")} OR (r.symbol_name_folded = @aliasQueryLeafFolded{sqlLeafCountScope}))" + : $" AND {BuildPersistedFoldedNameMatchSql("r.symbol_name_folded", "@query")}"; else if (exact) innerSql += countSuffixAlias != null ? $" AND (r.symbol_name = @query COLLATE NOCASE OR (r.symbol_name = @queryAttributeAlias COLLATE NOCASE{countAliasScope}){(allowSqlLeafFallback ? $" OR (r.symbol_name = sql_leaf_name(@aliasQuery) COLLATE NOCASE{sqlLeafCountScope})" : string.Empty)})" @@ -786,7 +789,10 @@ FROM symbol_references r : _foldReady ? FoldNameForLanguage(query, lang) : query; - SqliteCommandPolicy.Add(cmd, "@query", value); + if (exact && _foldReady) + AddPersistedFoldedNameQueryParameters(cmd, "@query", query, lang); + else + SqliteCommandPolicy.Add(cmd, "@query", value); SqliteCommandPolicy.Add(cmd, "@aliasQuery", query); AddQualifiedGraphQueryParameters(cmd, query, allowQualifiedLeafFallback, allowCSharpQualifiedContextMatch); SqliteCommandPolicy.Add(cmd, "@aliasQueryLeafFolded", NameFold.Fold(SqlNameResolver.GetLeafName(query)) ?? SqlNameResolver.GetLeafName(query)); @@ -855,8 +861,8 @@ FROM symbol_references r var csharpQualifiedContextSql = BuildCSharpQualifiedContextFallbackSql(qualifiedContextSql); var qualifiedLeafFallbackSql = BuildQualifiedLeafFallbackSql("r.symbol_name", "r.symbol_name_folded", folded: true); innerSql += totalSuffixAlias != null - ? $" AND (((f.lang = 'sql') AND {qualifiedContextSql}) OR ((f.lang != 'sql') AND (r.symbol_name_folded = @query OR (r.symbol_name_folded = @queryAttributeAlias{totalAliasScope}))) OR {csharpQualifiedContextSql} OR {qualifiedLeafFallbackSql})" - : $" AND (((f.lang = 'sql') AND {qualifiedContextSql}) OR ((f.lang != 'sql') AND r.symbol_name_folded = @query) OR {csharpQualifiedContextSql} OR {qualifiedLeafFallbackSql})"; + ? $" AND (((f.lang = 'sql') AND {qualifiedContextSql}) OR ((f.lang != 'sql') AND ({BuildPersistedFoldedNameMatchSql("r.symbol_name_folded", "@query")} OR (r.symbol_name_folded = @queryAttributeAlias{totalAliasScope}))) OR {csharpQualifiedContextSql} OR {qualifiedLeafFallbackSql})" + : $" AND (((f.lang = 'sql') AND {qualifiedContextSql}) OR ((f.lang != 'sql') AND {BuildPersistedFoldedNameMatchSql("r.symbol_name_folded", "@query")}) OR {csharpQualifiedContextSql} OR {qualifiedLeafFallbackSql})"; } else if (useSqlQualifiedContextMatch && exact) { @@ -887,12 +893,12 @@ FROM symbol_references r } else if (exact && _foldReady) innerSql += totalSuffixAlias != null - ? $" AND (r.symbol_name_folded = @query OR (r.symbol_name_folded = @queryAttributeAlias{totalAliasScope}){(allowSqlLeafFallback ? $" OR (r.symbol_name_folded = @aliasQueryLeafFolded{sqlLeafTotalScope})" : string.Empty)})" + ? $" AND ({BuildPersistedFoldedNameMatchSql("r.symbol_name_folded", "@query")} OR (r.symbol_name_folded = @queryAttributeAlias{totalAliasScope}){(allowSqlLeafFallback ? $" OR (r.symbol_name_folded = @aliasQueryLeafFolded{sqlLeafTotalScope})" : string.Empty)})" : totalCssScssVariableAlias != null - ? $" AND (r.symbol_name_folded = @query OR (r.symbol_name_folded = @queryCssScssVariableAlias{totalCssScssVariableAliasScope}){(allowSqlLeafFallback ? $" OR (r.symbol_name_folded = @aliasQueryLeafFolded{sqlLeafTotalScope})" : string.Empty)})" + ? $" AND ({BuildPersistedFoldedNameMatchSql("r.symbol_name_folded", "@query")} OR (r.symbol_name_folded = @queryCssScssVariableAlias{totalCssScssVariableAliasScope}){(allowSqlLeafFallback ? $" OR (r.symbol_name_folded = @aliasQueryLeafFolded{sqlLeafTotalScope})" : string.Empty)})" : allowSqlLeafFallback - ? $" AND (r.symbol_name_folded = @query OR (r.symbol_name_folded = @aliasQueryLeafFolded{sqlLeafTotalScope}))" - : " AND r.symbol_name_folded = @query"; + ? $" AND ({BuildPersistedFoldedNameMatchSql("r.symbol_name_folded", "@query")} OR (r.symbol_name_folded = @aliasQueryLeafFolded{sqlLeafTotalScope}))" + : $" AND {BuildPersistedFoldedNameMatchSql("r.symbol_name_folded", "@query")}"; else if (exact) innerSql += totalSuffixAlias != null ? $" AND (r.symbol_name = @query COLLATE NOCASE OR (r.symbol_name = @queryAttributeAlias COLLATE NOCASE{totalAliasScope}){(allowSqlLeafFallback ? $" OR (r.symbol_name = sql_leaf_name(@aliasQuery) COLLATE NOCASE{sqlLeafTotalScope})" : string.Empty)})" @@ -925,7 +931,10 @@ FROM symbol_references r : _foldReady ? FoldNameForLanguage(query, lang) : query; - SqliteCommandPolicy.Add(cmd, "@query", value); + if (exact && _foldReady) + AddPersistedFoldedNameQueryParameters(cmd, "@query", query, lang); + else + SqliteCommandPolicy.Add(cmd, "@query", value); SqliteCommandPolicy.Add(cmd, "@aliasQuery", query); AddQualifiedGraphQueryParameters(cmd, query, allowQualifiedLeafFallback, allowCSharpQualifiedContextMatch); SqliteCommandPolicy.Add(cmd, "@aliasQueryLeafFolded", NameFold.Fold(SqlNameResolver.GetLeafName(query)) ?? SqlNameResolver.GetLeafName(query)); diff --git a/src/CodeIndex/Database/DbReader.cs b/src/CodeIndex/Database/DbReader.cs index 18927b04f..709786bfd 100644 --- a/src/CodeIndex/Database/DbReader.cs +++ b/src/CodeIndex/Database/DbReader.cs @@ -1664,6 +1664,23 @@ internal static string FoldNameForLanguage(string value, string? lang) => ? NimIdentifierIdentity.Fold(value) ?? value : NameFold.Fold(value) ?? value; + private static string BuildPersistedFoldedNameMatchSql( + string foldedNameSql, + string parameterSql, + string fileLanguageSql = "f.lang") + => $"(({fileLanguageSql} <> 'nim' AND {foldedNameSql} = {parameterSql})" + + $" OR ({fileLanguageSql} = 'nim' AND {foldedNameSql} = {parameterSql}Nim))"; + + private static void AddPersistedFoldedNameQueryParameters( + SqliteCommand command, + string parameterName, + string value, + string? lang) + { + SqliteCommandPolicy.Add(command, parameterName, FoldNameForLanguage(value, lang)); + SqliteCommandPolicy.Add(command, $"{parameterName}Nim", FoldNameForLanguage(value, "nim")); + } + internal static bool ContainsSqlLanguage(IEnumerable langs) => langs.Any(IsSqlLanguage); diff --git a/src/CodeIndex/Database/DbSymbolReader.Analysis.cs b/src/CodeIndex/Database/DbSymbolReader.Analysis.cs index 3f2f5e601..a8050fa45 100644 --- a/src/CodeIndex/Database/DbSymbolReader.Analysis.cs +++ b/src/CodeIndex/Database/DbSymbolReader.Analysis.cs @@ -612,8 +612,8 @@ public bool HasExactGraphSupportedDefinition( var allowLeafFallback = !SqlNameResolver.HasQualifier(normalizedQuery); var nameCondition = _foldReady ? allowLeafFallback - ? "(s.name_folded = @queryFolded OR (f.lang = 'sql' AND ((sql_segment_count(s.name) = @querySegmentCount AND sql_normalize_name_folded(s.name) = @queryNormalizedFolded) OR sql_leaf_name_folded(s.name) = @queryLeafFolded)))" - : "(s.name_folded = @queryFolded OR (f.lang = 'sql' AND sql_segment_count(s.name) = @querySegmentCount AND sql_normalize_name_folded(s.name) = @queryNormalizedFolded))" + ? $"({BuildPersistedFoldedNameMatchSql("s.name_folded", "@queryFolded")} OR (f.lang = 'sql' AND ((sql_segment_count(s.name) = @querySegmentCount AND sql_normalize_name_folded(s.name) = @queryNormalizedFolded) OR sql_leaf_name_folded(s.name) = @queryLeafFolded)))" + : $"({BuildPersistedFoldedNameMatchSql("s.name_folded", "@queryFolded")} OR (f.lang = 'sql' AND sql_segment_count(s.name) = @querySegmentCount AND sql_normalize_name_folded(s.name) = @queryNormalizedFolded))" : allowLeafFallback ? "(s.name = @queryRaw COLLATE NOCASE OR (f.lang = 'sql' AND ((sql_segment_count(s.name) = @querySegmentCount AND sql_normalize_name(s.name) = @queryNormalized COLLATE NOCASE) OR sql_leaf_name(s.name) = @queryLeaf COLLATE NOCASE)))" : "(s.name = @queryRaw COLLATE NOCASE OR (f.lang = 'sql' AND sql_segment_count(s.name) = @querySegmentCount AND sql_normalize_name(s.name) = @queryNormalized COLLATE NOCASE))"; @@ -635,7 +635,7 @@ FROM symbols s cmd.CommandText = sql; SqliteCommandPolicy.Add(cmd, "@queryRaw", query); - SqliteCommandPolicy.Add(cmd, "@queryFolded", NameFold.Fold(query) ?? query); + AddPersistedFoldedNameQueryParameters(cmd, "@queryFolded", query, lang); SqliteCommandPolicy.Add(cmd, "@queryNormalized", SqlNameResolver.NormalizeQualifiedName(query)); SqliteCommandPolicy.Add(cmd, "@queryNormalizedFolded", NameFold.Fold(SqlNameResolver.NormalizeQualifiedName(query)) ?? SqlNameResolver.NormalizeQualifiedName(query)); SqliteCommandPolicy.Add(cmd, "@queryLeaf", SqlNameResolver.GetLeafName(query)); diff --git a/src/CodeIndex/Database/DbSymbolReader.Definitions.cs b/src/CodeIndex/Database/DbSymbolReader.Definitions.cs index e5f056c24..26ba01a4f 100644 --- a/src/CodeIndex/Database/DbSymbolReader.Definitions.cs +++ b/src/CodeIndex/Database/DbSymbolReader.Definitions.cs @@ -335,8 +335,8 @@ FROM symbols s : " AND ((s.container_qualified_name = @queryRustContainer COLLATE NOCASE OR s.container_name = @queryRustContainer COLLATE NOCASE) AND s.name = @queryRustLeaf COLLATE NOCASE)" : _foldReady ? allowLeafFallback - ? " AND (s.name_folded = @query OR (f.lang = 'sql' AND ((sql_segment_count(s.name) = @querySegmentCount AND sql_normalize_name_folded(s.name) = @queryNormalizedFolded) OR sql_leaf_name_folded(s.name) = @queryLeafFolded)))" - : $" AND (s.name_folded = @query OR (f.lang = 'sql' AND sql_segment_count(s.name) = @querySegmentCount AND sql_normalize_name_folded(s.name) = @queryNormalizedFolded){(qualifiedSymbolClause != null ? $" OR {qualifiedSymbolClause}" : string.Empty)})" + ? $" AND ({BuildPersistedFoldedNameMatchSql("s.name_folded", "@query")} OR (f.lang = 'sql' AND ((sql_segment_count(s.name) = @querySegmentCount AND sql_normalize_name_folded(s.name) = @queryNormalizedFolded) OR sql_leaf_name_folded(s.name) = @queryLeafFolded)))" + : $" AND ({BuildPersistedFoldedNameMatchSql("s.name_folded", "@query")} OR (f.lang = 'sql' AND sql_segment_count(s.name) = @querySegmentCount AND sql_normalize_name_folded(s.name) = @queryNormalizedFolded){(qualifiedSymbolClause != null ? $" OR {qualifiedSymbolClause}" : string.Empty)})" : allowLeafFallback ? " AND (s.name = @query COLLATE NOCASE OR (f.lang = 'sql' AND ((sql_segment_count(s.name) = @querySegmentCount AND sql_normalize_name(s.name) = @queryNormalized COLLATE NOCASE) OR sql_leaf_name(s.name) = @queryLeaf COLLATE NOCASE)))" : $" AND (s.name = @query COLLATE NOCASE OR (f.lang = 'sql' AND sql_segment_count(s.name) = @querySegmentCount AND sql_normalize_name(s.name) = @queryNormalized COLLATE NOCASE){(qualifiedSymbolClause != null ? $" OR {qualifiedSymbolClause}" : string.Empty)})" @@ -369,7 +369,10 @@ FROM chunks c : _foldReady ? FoldNameForLanguage(normalizedQuery, lang) : normalizedQuery; - SqliteCommandPolicy.Add(cmd, "@query", paramValue); + if (exact && _foldReady) + AddPersistedFoldedNameQueryParameters(cmd, "@query", normalizedQuery, lang); + else + SqliteCommandPolicy.Add(cmd, "@query", paramValue); SqliteCommandPolicy.Add(cmd, "@queryNormalized", SqlNameResolver.NormalizeQualifiedName(normalizedQuery)); SqliteCommandPolicy.Add(cmd, "@queryNormalizedFolded", NameFold.Fold(SqlNameResolver.NormalizeQualifiedName(normalizedQuery)) ?? SqlNameResolver.NormalizeQualifiedName(normalizedQuery)); SqliteCommandPolicy.Add(cmd, "@queryLeaf", SqlNameResolver.GetLeafName(normalizedQuery)); diff --git a/src/CodeIndex/Database/DbSymbolReader.Search.cs b/src/CodeIndex/Database/DbSymbolReader.Search.cs index 29b7bb350..707420db4 100644 --- a/src/CodeIndex/Database/DbSymbolReader.Search.cs +++ b/src/CodeIndex/Database/DbSymbolReader.Search.cs @@ -220,8 +220,8 @@ FROM symbols s : " AND ((s.container_qualified_name = @query0RustContainer COLLATE NOCASE OR s.container_name = @query0RustContainer COLLATE NOCASE) AND s.name = @query0RustLeaf COLLATE NOCASE)" : _foldReady ? allowLeafFallback - ? " AND (s.name_folded = @query0 OR (f.lang = 'sql' AND ((sql_segment_count(s.name) = @query0SegmentCount AND sql_normalize_name_folded(s.name) = @query0NormalizedFolded) OR sql_leaf_name_folded(s.name) = @query0LeafFolded)))" - : $" AND (s.name_folded = @query0 OR (f.lang = 'sql' AND sql_segment_count(s.name) = @query0SegmentCount AND sql_normalize_name_folded(s.name) = @query0NormalizedFolded){(qualifiedSymbolClause != null ? $" OR {qualifiedSymbolClause}" : string.Empty)})" + ? $" AND ({BuildPersistedFoldedNameMatchSql("s.name_folded", "@query0")} OR (f.lang = 'sql' AND ((sql_segment_count(s.name) = @query0SegmentCount AND sql_normalize_name_folded(s.name) = @query0NormalizedFolded) OR sql_leaf_name_folded(s.name) = @query0LeafFolded)))" + : $" AND ({BuildPersistedFoldedNameMatchSql("s.name_folded", "@query0")} OR (f.lang = 'sql' AND sql_segment_count(s.name) = @query0SegmentCount AND sql_normalize_name_folded(s.name) = @query0NormalizedFolded){(qualifiedSymbolClause != null ? $" OR {qualifiedSymbolClause}" : string.Empty)})" : allowLeafFallback ? " AND (s.name = @query0 COLLATE NOCASE OR (f.lang = 'sql' AND ((sql_segment_count(s.name) = @query0SegmentCount AND sql_normalize_name(s.name) = @query0Normalized COLLATE NOCASE) OR sql_leaf_name(s.name) = @query0Leaf COLLATE NOCASE)))" : $" AND (s.name = @query0 COLLATE NOCASE OR (f.lang = 'sql' AND sql_segment_count(s.name) = @query0SegmentCount AND sql_normalize_name(s.name) = @query0Normalized COLLATE NOCASE){(qualifiedSymbolClause != null ? $" OR {qualifiedSymbolClause}" : string.Empty)})" @@ -248,7 +248,10 @@ FROM symbols s : _foldReady ? FoldNameForLanguage(value, lang) : value; - SqliteCommandPolicy.Add(cmd, "@query0", paramValue); + if (exact && _foldReady) + AddPersistedFoldedNameQueryParameters(cmd, "@query0", value, lang); + else + SqliteCommandPolicy.Add(cmd, "@query0", paramValue); SqliteCommandPolicy.Add(cmd, "@query0Normalized", SqlNameResolver.NormalizeQualifiedName(value)); SqliteCommandPolicy.Add(cmd, "@query0NormalizedFolded", NameFold.Fold(SqlNameResolver.NormalizeQualifiedName(value)) ?? SqlNameResolver.NormalizeQualifiedName(value)); SqliteCommandPolicy.Add(cmd, "@query0Leaf", SqlNameResolver.GetLeafName(value)); @@ -330,8 +333,8 @@ FROM symbols s : $"((s.container_qualified_name = @query{idx}RustContainer COLLATE NOCASE OR s.container_name = @query{idx}RustContainer COLLATE NOCASE) AND s.name = @query{idx}RustLeaf COLLATE NOCASE)"; return _foldReady ? allowLeafFallback - ? $"(s.name_folded = @query{idx}{swiftBacktickClause} OR (f.lang = 'sql' AND ((sql_segment_count(s.name) = @query{idx}SegmentCount AND sql_normalize_name_folded(s.name) = @query{idx}NormalizedFolded) OR sql_leaf_name_folded(s.name) = @query{idx}LeafFolded)))" - : $"(s.name_folded = @query{idx}{swiftBacktickClause} OR (f.lang = 'sql' AND sql_segment_count(s.name) = @query{idx}SegmentCount AND sql_normalize_name_folded(s.name) = @query{idx}NormalizedFolded){(qualifiedSymbolClause != null ? $" OR {qualifiedSymbolClause}" : string.Empty)})" + ? $"({BuildPersistedFoldedNameMatchSql("s.name_folded", $"@query{idx}")}{swiftBacktickClause} OR (f.lang = 'sql' AND ((sql_segment_count(s.name) = @query{idx}SegmentCount AND sql_normalize_name_folded(s.name) = @query{idx}NormalizedFolded) OR sql_leaf_name_folded(s.name) = @query{idx}LeafFolded)))" + : $"({BuildPersistedFoldedNameMatchSql("s.name_folded", $"@query{idx}")}{swiftBacktickClause} OR (f.lang = 'sql' AND sql_segment_count(s.name) = @query{idx}SegmentCount AND sql_normalize_name_folded(s.name) = @query{idx}NormalizedFolded){(qualifiedSymbolClause != null ? $" OR {qualifiedSymbolClause}" : string.Empty)})" : allowLeafFallback ? $"(s.name = @query{idx} COLLATE NOCASE{swiftBacktickClause} OR (f.lang = 'sql' AND ((sql_segment_count(s.name) = @query{idx}SegmentCount AND sql_normalize_name(s.name) = @query{idx}Normalized COLLATE NOCASE) OR sql_leaf_name(s.name) = @query{idx}Leaf COLLATE NOCASE)))" : $"(s.name = @query{idx} COLLATE NOCASE{swiftBacktickClause} OR (f.lang = 'sql' AND sql_segment_count(s.name) = @query{idx}SegmentCount AND sql_normalize_name(s.name) = @query{idx}Normalized COLLATE NOCASE){(qualifiedSymbolClause != null ? $" OR {qualifiedSymbolClause}" : string.Empty)})"; @@ -365,7 +368,10 @@ FROM symbols s : _foldReady ? FoldNameForLanguage(value, lang) : value; - SqliteCommandPolicy.Add(cmd, $"@query{i}", paramValue); + if (exact && _foldReady) + AddPersistedFoldedNameQueryParameters(cmd, $"@query{i}", value, lang); + else + SqliteCommandPolicy.Add(cmd, $"@query{i}", paramValue); SqliteCommandPolicy.Add(cmd, $"@query{i}Normalized", SqlNameResolver.NormalizeQualifiedName(value)); SqliteCommandPolicy.Add(cmd, $"@query{i}NormalizedFolded", NameFold.Fold(SqlNameResolver.NormalizeQualifiedName(value)) ?? SqlNameResolver.NormalizeQualifiedName(value)); SqliteCommandPolicy.Add(cmd, $"@query{i}Leaf", SqlNameResolver.GetLeafName(value)); @@ -608,8 +614,8 @@ FROM symbols s : $"((s.container_qualified_name = @query{idx}RustContainer COLLATE NOCASE OR s.container_name = @query{idx}RustContainer COLLATE NOCASE) AND s.name = @query{idx}RustLeaf COLLATE NOCASE)"; return _foldReady ? allowLeafFallback - ? $"(s.name_folded = @query{idx}{swiftBacktickClause} OR (f.lang = 'sql' AND ((sql_segment_count(s.name) = @query{idx}SegmentCount AND sql_normalize_name_folded(s.name) = @query{idx}NormalizedFolded) OR sql_leaf_name_folded(s.name) = @query{idx}LeafFolded)))" - : $"(s.name_folded = @query{idx}{swiftBacktickClause} OR (f.lang = 'sql' AND sql_segment_count(s.name) = @query{idx}SegmentCount AND sql_normalize_name_folded(s.name) = @query{idx}NormalizedFolded){(qualifiedSymbolClause != null ? $" OR {qualifiedSymbolClause}" : string.Empty)})" + ? $"({BuildPersistedFoldedNameMatchSql("s.name_folded", $"@query{idx}")}{swiftBacktickClause} OR (f.lang = 'sql' AND ((sql_segment_count(s.name) = @query{idx}SegmentCount AND sql_normalize_name_folded(s.name) = @query{idx}NormalizedFolded) OR sql_leaf_name_folded(s.name) = @query{idx}LeafFolded)))" + : $"({BuildPersistedFoldedNameMatchSql("s.name_folded", $"@query{idx}")}{swiftBacktickClause} OR (f.lang = 'sql' AND sql_segment_count(s.name) = @query{idx}SegmentCount AND sql_normalize_name_folded(s.name) = @query{idx}NormalizedFolded){(qualifiedSymbolClause != null ? $" OR {qualifiedSymbolClause}" : string.Empty)})" : allowLeafFallback ? $"(s.name = @query{idx} COLLATE NOCASE{swiftBacktickClause} OR (f.lang = 'sql' AND ((sql_segment_count(s.name) = @query{idx}SegmentCount AND sql_normalize_name(s.name) = @query{idx}Normalized COLLATE NOCASE) OR sql_leaf_name(s.name) = @query{idx}Leaf COLLATE NOCASE)))" : $"(s.name = @query{idx} COLLATE NOCASE{swiftBacktickClause} OR (f.lang = 'sql' AND sql_segment_count(s.name) = @query{idx}SegmentCount AND sql_normalize_name(s.name) = @query{idx}Normalized COLLATE NOCASE){(qualifiedSymbolClause != null ? $" OR {qualifiedSymbolClause}" : string.Empty)})"; @@ -655,7 +661,18 @@ FROM symbols s paramValue = FoldNameForLanguage(effectiveQueries[idx], lang); else paramValue = effectiveQueries[idx]; - SqliteCommandPolicy.Add(cmd, $"@query{idx}", paramValue); + if (exact && _foldReady) + { + AddPersistedFoldedNameQueryParameters( + cmd, + $"@query{idx}", + effectiveQueries[idx], + lang); + } + else + { + SqliteCommandPolicy.Add(cmd, $"@query{idx}", paramValue); + } SqliteCommandPolicy.Add(cmd, $"@query{idx}Normalized", SqlNameResolver.NormalizeQualifiedName(effectiveQueries[idx])); SqliteCommandPolicy.Add(cmd, $"@query{idx}NormalizedFolded", NameFold.Fold(SqlNameResolver.NormalizeQualifiedName(effectiveQueries[idx])) ?? SqlNameResolver.NormalizeQualifiedName(effectiveQueries[idx])); SqliteCommandPolicy.Add(cmd, $"@query{idx}Leaf", SqlNameResolver.GetLeafName(effectiveQueries[idx])); diff --git a/src/CodeIndex/Indexer/Hooks/PostExtractionHookMutationMaterializer.cs b/src/CodeIndex/Indexer/Hooks/PostExtractionHookMutationMaterializer.cs index 42b5ddc35..273edd63c 100644 --- a/src/CodeIndex/Indexer/Hooks/PostExtractionHookMutationMaterializer.cs +++ b/src/CodeIndex/Indexer/Hooks/PostExtractionHookMutationMaterializer.cs @@ -59,6 +59,27 @@ internal static bool TrimToLimit(List? items, int? maxCount) return true; } + internal static void RefreshLanguageIdentity(string? language, IEnumerable symbols) + { + if (!string.Equals(language, "nim", StringComparison.Ordinal)) + return; + + foreach (var symbol in symbols) + symbol.IdentityNameFolded = NimIdentifierIdentity.Fold(symbol.Name); + } + + internal static void RefreshLanguageIdentity(string? language, IEnumerable references) + { + if (!string.Equals(language, "nim", StringComparison.Ordinal)) + return; + + foreach (var reference in references) + { + reference.IdentitySymbolNameFolded = NimIdentifierIdentity.Fold(reference.SymbolName); + reference.IdentityContainerNameFolded = NimIdentifierIdentity.Fold(reference.ContainerName); + } + } + private static SymbolRecord CloneSymbol(SymbolRecord symbol) => new() { @@ -67,6 +88,7 @@ private static SymbolRecord CloneSymbol(SymbolRecord symbol) Kind = symbol.Kind, SubKind = symbol.SubKind, Name = symbol.Name, + IdentityNameFolded = symbol.IdentityNameFolded, Line = symbol.Line, StartLine = symbol.StartLine, StartColumn = symbol.StartColumn, @@ -91,12 +113,16 @@ private static ReferenceRecord CloneReference(ReferenceRecord reference) Id = reference.Id, FileId = reference.FileId, SymbolName = reference.SymbolName, + IdentitySymbolNameFolded = reference.IdentitySymbolNameFolded, ReferenceKind = reference.ReferenceKind, Line = reference.Line, Column = reference.Column, Context = reference.Context, ContainerKind = reference.ContainerKind, ContainerName = reference.ContainerName, + IdentityContainerNameFolded = reference.IdentityContainerNameFolded, + TargetQualifier = reference.TargetQualifier, + SuppressInferredTargetQualifier = reference.SuppressInferredTargetQualifier, IsSelfReference = reference.IsSelfReference, IsMutualRecursion = reference.IsMutualRecursion, }; diff --git a/src/CodeIndex/Indexer/Hooks/PostExtractionHooks.cs b/src/CodeIndex/Indexer/Hooks/PostExtractionHooks.cs index b957ac002..4eda16fe5 100644 --- a/src/CodeIndex/Indexer/Hooks/PostExtractionHooks.cs +++ b/src/CodeIndex/Indexer/Hooks/PostExtractionHooks.cs @@ -400,6 +400,12 @@ private void OnSymbolsExtractedCore( PostExtractionHookMutationMaterializer.ReplaceList(symbols, workingSymbols); } } + + // Hooks can rename or add records but cannot set the internal persisted identity key. + // Re-derive it from the accepted public name after all mutations. + // hook は record の rename/add はできるが内部の永続化 identity key は設定できないため、 + // 全 mutation 受理後の公開名から再導出する。 + PostExtractionHookMutationMaterializer.RefreshLanguageIdentity(context.Language, symbols); } public void OnReferencesExtracted(FileContext context, IList references, CancellationToken cancellationToken = default) @@ -434,6 +440,8 @@ public void OnReferencesExtracted(FileContext context, IList re PostExtractionHookMutationMaterializer.ReplaceList(references, workingReferences); } } + + PostExtractionHookMutationMaterializer.RefreshLanguageIdentity(context.Language, references); } private bool InvokeHookWithBudget( diff --git a/src/CodeIndex/Indexer/References/Languages/ScientificNativeReferenceExtractor.cs b/src/CodeIndex/Indexer/References/Languages/ScientificNativeReferenceExtractor.cs index 7eb3e36f5..683d1445a 100644 --- a/src/CodeIndex/Indexer/References/Languages/ScientificNativeReferenceExtractor.cs +++ b/src/CodeIndex/Indexer/References/Languages/ScientificNativeReferenceExtractor.cs @@ -138,7 +138,7 @@ private readonly record struct DTemplateInvocation( segments.Reverse(); if ((language == "cython" && segments[0] is "self" or "cls") - || (language == "d" && segments[0] is "this" or "super")) + || (language == "d" && segments[0] == "this")) { return CurrentContainerReceiverMarker; } diff --git a/src/CodeIndex/Indexer/References/ReferenceExtractor.cs b/src/CodeIndex/Indexer/References/ReferenceExtractor.cs index de01e4ac5..31f0550c6 100644 --- a/src/CodeIndex/Indexer/References/ReferenceExtractor.cs +++ b/src/CodeIndex/Indexer/References/ReferenceExtractor.cs @@ -2124,7 +2124,8 @@ internal static void AddReference( : null, TargetQualifier = currentContainerReceiver ? null : targetQualifier, SuppressInferredTargetQualifier = currentContainerReceiver, - IsSelfReference = IsSameReferenceName(container?.Name, name), + IsSelfReference = (targetQualifier == null || currentContainerReceiver) + && IsSameReferenceName(container?.Name, name), }); } diff --git a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.Pascal.cs b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.Pascal.cs index 4ff5d6a1b..6d113fe5a 100644 --- a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.Pascal.cs +++ b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.Pascal.cs @@ -22,6 +22,12 @@ public static partial class SymbolExtractor private static readonly Regex AdaUnnamedOuterEndRegex = new( @"\bend\s*;", RegexOptions.Compiled | RegexOptions.IgnoreCase | RegexOptions.CultureInvariant); + private static readonly Regex AdaBodylessAfterIsRegex = new( + @"^(?:abstract|separate|null|new)\b", + RegexOptions.Compiled | RegexOptions.IgnoreCase | RegexOptions.CultureInvariant); + private static readonly Regex AdaRoutineBodySignatureRegex = new( + @"^\s*(?:(?:overriding|not\s+overriding)\s+)?(?:procedure\b[\s\S]*?\bis\b|function\b[\s\S]*?\breturn\b[\s\S]*?\bis\b)", + RegexOptions.Compiled | RegexOptions.IgnoreCase | RegexOptions.CultureInvariant); private static (int EndLine, int? BodyStartLine, int? BodyEndLine) FindAdaRange( string[] lines, @@ -31,6 +37,9 @@ private static (int EndLine, int? BodyStartLine, int? BodyEndLine) FindAdaRange( if (!declaration.Success) return (startIndex + 1, null, null); + if (TryFindAdaBodylessDeclarationEnd(lines, startIndex, out var declarationEndLine)) + return (declarationEndLine, null, null); + var declarationName = declaration.Groups["name"].Value; int? bodyStartLine = null; for (var i = startIndex; i < lines.Length; i++) @@ -51,15 +60,78 @@ private static (int EndLine, int? BodyStartLine, int? BodyEndLine) FindAdaRange( if (bodyStartLine == null) return (startIndex + 1, null, null); + var beginDepth = 0; for (var i = bodyStartLine.Value - 1; i < lines.Length; i++) { - if (AdaUnnamedOuterEndRegex.IsMatch(MaskAdaRangeStringsAndComments(lines[i]))) - return (i + 1, bodyStartLine, i + 1); + var code = MaskAdaRangeStringsAndComments(lines[i]); + beginDepth += AdaBeginRegex.Matches(code).Count; + foreach (Match _ in AdaUnnamedOuterEndRegex.Matches(code)) + { + if (beginDepth > 0) + beginDepth--; + if (beginDepth == 0) + return (i + 1, bodyStartLine, i + 1); + } } return (lines.Length, bodyStartLine, lines.Length); } + private static bool TryFindAdaBodylessDeclarationEnd( + string[] lines, + int startIndex, + out int declarationEndLine) + { + declarationEndLine = startIndex + 1; + var delimiterDepth = 0; + for (var lineIndex = startIndex; lineIndex < lines.Length; lineIndex++) + { + var code = MaskAdaRangeStringsAndComments(lines[lineIndex]); + for (var index = 0; index < code.Length; index++) + { + if (code[index] is '(' or '[') + { + delimiterDepth++; + continue; + } + if (code[index] is ')' or ']') + { + delimiterDepth = Math.Max(0, delimiterDepth - 1); + continue; + } + if (delimiterDepth != 0) + continue; + + if (code[index] == ';') + { + declarationEndLine = lineIndex + 1; + return true; + } + + if (index + 2 > code.Length + || !code.AsSpan(index, 2).Equals("is", StringComparison.OrdinalIgnoreCase) + || (index > 0 && (char.IsLetterOrDigit(code[index - 1]) || code[index - 1] == '_')) + || (index + 2 < code.Length + && (char.IsLetterOrDigit(code[index + 2]) || code[index + 2] == '_'))) + { + continue; + } + + var tail = code[(index + 2)..].TrimStart(); + for (var tailLineIndex = lineIndex + 1; + tail.Length == 0 && tailLineIndex < lines.Length; + tailLineIndex++) + { + tail = MaskAdaRangeStringsAndComments(lines[tailLineIndex]).TrimStart(); + } + + return tail.StartsWith('(') || AdaBodylessAfterIsRegex.IsMatch(tail); + } + } + + return false; + } + private static string MaskAdaRangeStringsAndComments(string line) { char[]? chars = null; diff --git a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.ScientificNative.cs b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.ScientificNative.cs index 5f5c813c4..d75148b88 100644 --- a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.ScientificNative.cs +++ b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.ScientificNative.cs @@ -74,6 +74,18 @@ private static (int EndLine, int? BodyStartLine, int? BodyEndLine) FindScientifi if (!IsScientificBlockTokenAtStatementBoundary(code, match.Index, keyword, language)) continue; + // In Julia, `begin` inside square-bracket indexing is a first-index sentinel, + // not a `begin ... end` block opener (`A[begin]`). + // Julia の角括弧 index 内の `begin` は先頭 index sentinel であり、 + // `begin ... end` block の開始ではない。 + if (language == "julia" + && keyword == "begin" + && delimiterFrames.TryPeek(out var indexingFrame) + && indexingFrame.ClosingDelimiter == ']') + { + continue; + } + if (keyword.Equals("end", StringComparison.OrdinalIgnoreCase)) { if (delimiterFrames.TryPeek(out var delimiterFrame) @@ -173,6 +185,8 @@ private static bool TryGetJuliaShortBlockExpressionStartLine( foreach (Match match in JuliaScientificBlockTokenRegex.Matches(expression)) { var keyword = match.Groups["keyword"].Value; + if (keyword == "begin" && IsJuliaSquareBracketIndexToken(expression, match.Index)) + continue; if (IsJuliaExpressionPositionBlockOpener(expression, match.Index, keyword) || (keyword == "for" && IsJuliaShortForBlockStart(expression, match.Index))) return true; @@ -181,6 +195,35 @@ private static bool TryGetJuliaShortBlockExpressionStartLine( return false; } + private static bool IsJuliaSquareBracketIndexToken(string expression, int tokenIndex) + { + var delimiters = new Stack(); + for (var index = 0; index < tokenIndex; index++) + { + var closingDelimiter = expression[index] switch + { + '(' => ')', + '[' => ']', + '{' => '}', + _ => '\0', + }; + if (closingDelimiter != '\0') + { + delimiters.Push(closingDelimiter); + continue; + } + + if (delimiters.TryPeek(out var expectedClosingDelimiter) + && expression[index] == expectedClosingDelimiter) + { + delimiters.Pop(); + } + } + + return delimiters.TryPeek(out var enclosingDelimiter) + && enclosingDelimiter == ']'; + } + private static bool TryFindJuliaShortDelimitedExpressionEnd( string[] scannerLines, int startIndex, diff --git a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.cs b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.cs index e32c17bc9..1b0c79857 100644 --- a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.cs +++ b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.cs @@ -4771,7 +4771,8 @@ private static void RemoveTrailingSameNameDeclarationOnlyFunctions( var signature = prior.Signature?.TrimStart(); if (signature != null && (signature.StartsWith("declare ", StringComparison.Ordinal) - || CSharpPartialFunctionDeclarationSignatureRegex.IsMatch(signature))) + || CSharpPartialFunctionDeclarationSignatureRegex.IsMatch(signature) + || IsAdaForwardDeclarationPair(signature, symbol.Signature))) { break; } @@ -4781,6 +4782,28 @@ private static void RemoveTrailingSameNameDeclarationOnlyFunctions( } } + private static bool IsAdaForwardDeclarationPair( + string declarationSignature, + string? implementationSignature) + { + if (implementationSignature == null + || !AdaRoutineBodySignatureRegex.IsMatch(implementationSignature)) + { + return false; + } + + var trimmed = declarationSignature.Trim(); + if (!trimmed.EndsWith(';')) + return false; + + return trimmed.StartsWith("procedure ", StringComparison.OrdinalIgnoreCase) + || trimmed.StartsWith("function ", StringComparison.OrdinalIgnoreCase) + || trimmed.StartsWith("overriding procedure ", StringComparison.OrdinalIgnoreCase) + || trimmed.StartsWith("overriding function ", StringComparison.OrdinalIgnoreCase) + || trimmed.StartsWith("not overriding procedure ", StringComparison.OrdinalIgnoreCase) + || trimmed.StartsWith("not overriding function ", StringComparison.OrdinalIgnoreCase); + } + // Some compact same-line C# fixtures can legitimately contain two distinct siblings with // the same short signature on the same physical line // (`Child { } } public partial class Child { }`). Allow as many identical rows as the raw diff --git a/tests/CodeIndex.Tests/DatabaseTests.cs b/tests/CodeIndex.Tests/DatabaseTests.cs index d48595948..b609d9151 100644 --- a/tests/CodeIndex.Tests/DatabaseTests.cs +++ b/tests/CodeIndex.Tests/DatabaseTests.cs @@ -1168,6 +1168,16 @@ proc RunGraph() = "src/style.nim", "nim", "nim-style-insensitive"); + _writer.InsertChunks([ + new() + { + FileId = fileId, + ChunkIndex = 0, + StartLine = 1, + EndLine = 3, + Content = content, + }, + ]); var symbols = SymbolExtractor.Extract(fileId, "nim", content); _writer.InsertSymbols(symbols); var references = ReferenceExtractor.Extract(fileId, "nim", content, symbols); @@ -1227,6 +1237,23 @@ FROM symbols Assert.Single(reader.SearchReferences("myProc", lang: "nim", exact: true)); Assert.Empty(reader.SearchSymbols("My_proc", lang: "nim", exact: true)); Assert.Empty(reader.SearchReferences("MyProc", lang: "nim", exact: true)); + + Assert.Single(reader.SearchSymbols("my_proc", exact: true)); + Assert.Equal(1, reader.CountSearchSymbols("my_proc", exact: true)); + Assert.Equal(1, reader.CountDefinitionsTotal("my_proc", exact: true).Count); + Assert.Single(reader.SearchReferences("my_proc", exact: true)); + Assert.Equal(1, reader.CountSearchReferences("my_proc", exact: true)); + Assert.Equal(1, reader.CountSearchReferencesTotal("my_proc", exact: true).Count); + Assert.Single(reader.GetCallers("my_proc", exact: true)); + Assert.Equal(1, reader.CountCallers("my_proc", exact: true)); + Assert.Equal(1, reader.CountCallersTotal("my_proc", exact: true).Count); + Assert.Single(reader.GetCallees("Run_Graph", exact: true)); + Assert.Equal(1, reader.CountCallees("Run_Graph", exact: true)); + Assert.Equal(1, reader.CountCalleesTotal("Run_Graph", exact: true).Count); + Assert.Empty(reader.SearchSymbols("My_proc", exact: true)); + Assert.Empty(reader.SearchReferences("My_proc", exact: true)); + Assert.Empty(reader.GetCallers("My_proc", exact: true)); + Assert.Empty(reader.GetCallees("run_graph", exact: true)); } [Fact] diff --git a/tests/CodeIndex.Tests/PostExtractionHookContractTests.cs b/tests/CodeIndex.Tests/PostExtractionHookContractTests.cs index 006b28e18..314626ec3 100644 --- a/tests/CodeIndex.Tests/PostExtractionHookContractTests.cs +++ b/tests/CodeIndex.Tests/PostExtractionHookContractTests.cs @@ -93,7 +93,16 @@ public void MutationMaterializer_ClonesAndTrimsRecordsWithinContracts_Issue4185( { var symbols = new List { - new() { FileId = 7, Kind = "class", Name = "Original", Line = 1, StartLine = 1, EndLine = 1 }, + new() + { + FileId = 7, + Kind = "class", + Name = "Original", + IdentityNameFolded = "original-key", + Line = 1, + StartLine = 1, + EndLine = 1, + }, new() { FileId = 7, Kind = "method", Name = "Extra", Line = 2, StartLine = 2, EndLine = 2 }, }; @@ -101,6 +110,7 @@ public void MutationMaterializer_ClonesAndTrimsRecordsWithinContracts_Issue4185( Assert.True(inputTruncated); var clonedSymbol = Assert.Single(cloned); + Assert.Equal("original-key", clonedSymbol.IdentityNameFolded); clonedSymbol.Name = "ChangedByHook"; Assert.Equal("Original", symbols[0].Name); @@ -114,4 +124,57 @@ public void MutationMaterializer_ClonesAndTrimsRecordsWithinContracts_Issue4185( Assert.True(PostExtractionHookMutationMaterializer.TrimToLimit(references, maxCount: 2)); Assert.Equal(["Original", "Extra"], references.Select(reference => reference.SymbolName)); } + + [Fact] + public void MutationMaterializer_RecomputesNimIdentityAfterHookMutation_Issue4738() + { + var symbols = new List + { + new() + { + FileId = 7, + Kind = "function", + Name = "renamed_proc", + IdentityNameFolded = "stale", + Line = 1, + StartLine = 1, + EndLine = 1, + }, + }; + var references = new List + { + new() + { + FileId = 7, + SymbolName = "renamedProc", + IdentitySymbolNameFolded = "stale", + ReferenceKind = "call", + Line = 2, + Column = 1, + ContainerName = "Run_Graph", + IdentityContainerNameFolded = "stale", + TargetQualifier = "pkg", + SuppressInferredTargetQualifier = true, + }, + }; + + var clonedReferences = PostExtractionHookMutationMaterializer.CloneReferences( + references, + maxCount: null, + out var referencesTruncated); + Assert.False(referencesTruncated); + var clonedReference = Assert.Single(clonedReferences); + Assert.Equal("pkg", clonedReference.TargetQualifier); + Assert.True(clonedReference.SuppressInferredTargetQualifier); + + PostExtractionHookMutationMaterializer.RefreshLanguageIdentity("nim", symbols); + PostExtractionHookMutationMaterializer.RefreshLanguageIdentity("nim", clonedReferences); + + Assert.Equal("renamedproc", Assert.Single(symbols).IdentityNameFolded); + var reference = Assert.Single(clonedReferences); + Assert.Equal("renamedproc", reference.IdentitySymbolNameFolded); + Assert.Equal("Rungraph", reference.IdentityContainerNameFolded); + Assert.Equal("pkg", reference.TargetQualifier); + Assert.True(reference.SuppressInferredTargetQualifier); + } } diff --git a/tests/CodeIndex.Tests/QueryCommandRunnerImpactTests.cs b/tests/CodeIndex.Tests/QueryCommandRunnerImpactTests.cs index ed0ab5a9b..0ece2adb3 100644 --- a/tests/CodeIndex.Tests/QueryCommandRunnerImpactTests.cs +++ b/tests/CodeIndex.Tests/QueryCommandRunnerImpactTests.cs @@ -105,6 +105,7 @@ public void GetTransitiveCallers_MaxDepthBoundaryProbeBudgetTerminatesStably_Iss null, null, false, + false, ]); Assert.NotNull(inspection); var type = inspection!.GetType(); diff --git a/tests/CodeIndex.Tests/QueryCommandRunnerSymbolTests.cs b/tests/CodeIndex.Tests/QueryCommandRunnerSymbolTests.cs index b50d7e074..8610a5b15 100644 --- a/tests/CodeIndex.Tests/QueryCommandRunnerSymbolTests.cs +++ b/tests/CodeIndex.Tests/QueryCommandRunnerSymbolTests.cs @@ -1515,14 +1515,14 @@ public void RunSymbols_UnsupportedExtractorLanguageExplainsSearchOnlyFallback() var projectRoot = TestProjectHelper.CreateTempProject("cdidx_symbols_unsupported_extractor"); try { - File.WriteAllText(Path.Combine(projectRoot, "settings.m"), "enabled = true\n"); + File.WriteAllText(Path.Combine(projectRoot, "settings.pl"), "enabled = true\n"); var dbPath = Path.Combine(projectRoot, ".cdidx", "codeindex.db"); var (indexExitCode, _, indexStderr) = CaptureConsole(() => IndexCommandRunner.Run( [projectRoot, "--json", "--quiet"], _jsonOptions)); var (exitCode, stdout, stderr) = CaptureConsole(() => QueryCommandRunner.RunSymbols( - ["enabled", "--db", dbPath, "--lang", "ambiguous_m"], + ["enabled", "--db", dbPath, "--lang", "ambiguous_pl"], _jsonOptions)); Assert.Equal(CommandExitCodes.Success, indexExitCode); @@ -1530,7 +1530,7 @@ public void RunSymbols_UnsupportedExtractorLanguageExplainsSearchOnlyFallback() Assert.Equal(CommandExitCodes.Success, exitCode); Assert.Equal(string.Empty, stdout); Assert.Contains("symbol extraction is not available", stderr); - Assert.Contains("cdidx search --lang ambiguous_m", stderr); + Assert.Contains("cdidx search --lang ambiguous_pl", stderr); Assert.Contains("missing-symbols", stderr); } finally diff --git a/tests/CodeIndex.Tests/ReferenceExtractorTests.ScientificNative.cs b/tests/CodeIndex.Tests/ReferenceExtractorTests.ScientificNative.cs index f612816de..3ac27aae4 100644 --- a/tests/CodeIndex.Tests/ReferenceExtractorTests.ScientificNative.cs +++ b/tests/CodeIndex.Tests/ReferenceExtractorTests.ScientificNative.cs @@ -1802,4 +1802,94 @@ public void Extract_JuliaWhereShortFunctionOwnsRhsCalls_Issue4738() Assert.DoesNotContain(references, reference => reference.SymbolName == "f" && reference.ReferenceKind == "call"); } + + [Theory] + [InlineData( + """ + function f(A) + value = A[begin] + end + outside() + """)] + [InlineData( + """ + f(A) = A[begin] + outside() + """)] + public void Extract_JuliaIndexBeginDoesNotExtendFunctionRange_Issue4738(string content) + { + var symbols = SymbolExtractor.Extract(1, "julia", content); + + var references = ReferenceExtractor.Extract(1, "julia", content, symbols); + + Assert.Null(Assert.Single(references, reference => + reference.SymbolName == "outside" && reference.ReferenceKind == "call").ContainerName); + } + + [Fact] + public void Extract_DSuperReceiverDoesNotBecomeCurrentContainerSelfCall_Issue4738() + { + const string content = """ + class Base { + void run() { } + } + class Child : Base { + override void run() { super.run(); } + } + """; + var symbols = SymbolExtractor.Extract(1, "d", content); + + var references = ReferenceExtractor.Extract(1, "d", content, symbols); + + var call = Assert.Single(references, reference => + reference.SymbolName == "run" && reference.ReferenceKind == "call"); + Assert.Equal("super", call.TargetQualifier); + Assert.False(call.IsSelfReference); + } + + [Fact] + public void Extract_AdaUnnamedOuterEndTracksNestedDeclareBlock_Issue4738() + { + const string content = """ + procedure Outer is + begin + declare + begin + Inner; + end; + Later; + end; + outside; + """; + var symbols = SymbolExtractor.Extract(1, "ada", content); + + var references = ReferenceExtractor.Extract(1, "ada", content, symbols); + + Assert.Equal(8, Assert.Single(symbols, symbol => symbol.Name == "Outer").EndLine); + Assert.Equal("Outer", Assert.Single(references, reference => + reference.SymbolName == "Later" && reference.ReferenceKind == "call").ContainerName); + Assert.Null(Assert.Single(references, reference => + reference.SymbolName == "outside" && reference.ReferenceKind == "call").ContainerName); + } + + [Fact] + public void Extract_AdaForwardDeclarationDoesNotBorrowLaterBodyRange_Issue4738() + { + const string content = """ + procedure Inner; + procedure Inner is + begin + Helper; + end Inner; + """; + + var symbols = SymbolExtractor.Extract(1, "ada", content); + + var declarations = symbols.Where(symbol => symbol.Name == "Inner").ToList(); + Assert.Equal(2, declarations.Count); + Assert.Equal(1, declarations[0].EndLine); + Assert.Null(declarations[0].BodyStartLine); + Assert.Equal(5, declarations[1].EndLine); + Assert.Equal(3, declarations[1].BodyStartLine); + } }