From 1b491341d0f97e97fdb8e51df6ccd91fde1a2c87 Mon Sep 17 00:00:00 2001 From: Widthdom Date: Wed, 29 Jul 2026 06:12:44 +0900 Subject: [PATCH 1/3] Fix Markdown dependency noise evidence (#4868) --- DEVELOPER_GUIDE.md | 29 +- README.md | 11 + USER_GUIDE.md | 4 +- changelog.d/unreleased/4868.fixed.md | 24 ++ src/CodeIndex/Cli/CliFlagSchema.cs | 2 +- src/CodeIndex/Cli/JsonOutputContracts.cs | 2 + .../Cli/QueryCommandRunner.Dependencies.cs | 214 +++++++++++-- .../Database/DbReader.Dependencies.cs | 303 ++++++++++++++++-- src/CodeIndex/Models/QueryResults.cs | 11 + .../DbReaderFileDependencyTests.cs | 72 ++++- .../QueryCommandRunnerTests.cs | 103 ++++++ 11 files changed, 717 insertions(+), 58 deletions(-) create mode 100644 changelog.d/unreleased/4868.fixed.md diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index ddedfe90c..a8cdcbf4c 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -1978,7 +1978,21 @@ Different graph entry points walk different `reference_kind` subsets by design. | `deps` (default = forward) | source file → target file | all kinds; metadata rows require class-like + metadata-eligible targets (`has_metadata_target_kind`) and a unique resolution (`target_ambiguity`); MSBuild imports/project references resolve paths relative to the declaring project instead of matching shared package names | `DbReader.GetFileDependencies` | | `deps --reverse` | target file → source file | same as forward `deps` (same SQL) | `DbReader.GetFileDependencies` | -`deps --symbol`, `--symbol-family`, and `--suppress-noise` are pushed into the logical-reference and target-candidate SQL scopes before candidate ranking and `--limit`; cycle and cross-workspace reads apply the same filters before their candidate limits. Consequently, `reference_count`, ranking, and the `symbol_filter` before/after counters describe the SQL-filtered scope rather than the whole pre-filter workspace. Long SQLite dependency reads also register command cancellation with the query token. +`deps --symbol`, `--symbol-family`, and the generic-symbol part of +`--suppress-noise` are pushed into the logical-reference and target-candidate +SQL scopes before candidate ranking and `--limit`; cycle and cross-workspace +reads apply the same name filters before their candidate limits. Markdown +heading-name matches are classified separately as +`markdown_heading_name_match` evidence. Suppressed queries prioritize retained +evidence before candidate limits, then remove only that evidence in the CLI +layer, so explicit Markdown path links remain visible even when an edge also +contains legacy heading fanout. Machine-readable edges expose +`source_language`, `origin`, `reference_kind`, `target_kind`, and +`reference_count` distributions in `evidence`; `symbol_filter` adds reference +before/after totals and per-reason affected/removed counts. Generic-symbol +counters still describe the SQL-filtered scope rather than the whole pre-filter +workspace. Long SQLite dependency reads also register command cancellation with +the query token. Practical consequence: `impact ` on a class-like symbol returns the heuristic file-dependency-hint fallback (with metadata edges) when no member-level callers exist, whereas default `callers ` returns only executable edges. Both are correct under their own contracts; counts will not match. To reconcile, run `references --kind attribute` (or `annotation`), or pass an explicitly supported non-default kind to `callers` / `callees`, to surface edges that the default call graph intentionally drops. @@ -5256,7 +5270,18 @@ USER_GUIDEの[終了コード](USER_GUIDE.md#終了コード)セクションを | `deps` (デフォルト = forward) | source file → target file | 全 kind。metadata 行は class-like かつ metadata-eligible な target (`has_metadata_target_kind`) と一意解決 (`target_ambiguity`) を要求。MSBuild の import / project reference は共有 package 名との一致ではなく、宣言元 project 相対の path として解決 | `DbReader.GetFileDependencies` | | `deps --reverse` | target file → source file | forward `deps` と同じ SQL を共有 | `DbReader.GetFileDependencies` | -`deps --symbol`、`--symbol-family`、`--suppress-noise` は、候補の ranking と `--limit` より前に logical-reference と target-candidate の SQL scope へ push down される。cycle と cross-workspace の read も、各候補上限より前に同じ filter を適用する。そのため `reference_count`、ranking、`symbol_filter` の before/after counter は、絞り込み前の workspace 全体ではなく SQL で絞り込まれた scope を表す。長時間の SQLite dependency read では query token による command cancellation も登録する。 +`deps --symbol`、`--symbol-family`、`--suppress-noise` の汎用 symbol 部分は、候補の +ranking と `--limit` より前に logical-reference と target-candidate の SQL scope へ +push down される。cycle と cross-workspace の read も、各候補上限より前に同じ名前 +filter を適用する。Markdown の見出し名一致は +`markdown_heading_name_match` evidence として別に分類する。抑制時は候補上限より前に +保持対象 evidence を優先し、CLI 層でこの evidence だけを除くため、旧 index 由来の +見出し fanout と同じ edge に含まれる明示的な Markdown path link も残る。 +machine-readable edge の `evidence` は `source_language`、`origin`、`reference_kind`、 +`target_kind`、`reference_count` の分布を公開し、`symbol_filter` は reference の +before/after 合計と理由別の affected / removed 件数を加える。汎用 symbol の counter は +引き続き、絞り込み前の workspace 全体ではなく SQL で絞り込まれた scope を表す。 +長時間の SQLite dependency read では query token による command cancellation も登録する。 実運用上の帰結: クラスのようなシンボルに対する `impact ` は、member-level の caller が存在しない場合 heuristic file-dependency-hint fallback (metadata エッジを含む) を返し、一方の既定 `callers ` は実行可能 edge だけを返す。両方とも個々の契約上は正しいが、件数は一致しない。差分を埋めるには `references --kind attribute`（または `annotation`）を使うか、`callers` / `callees` に明示的に対応する非既定 kind を渡し、既定 call graph が意図的に落としている edge を確認する。 diff --git a/README.md b/README.md index 6e6e1f895..1f3923b3e 100644 --- a/README.md +++ b/README.md @@ -159,6 +159,12 @@ For generated or dense source that emits excessive reference rows, use `cdidx . --max-references-per-file ` to keep text search and symbols indexed while skipping references for only the over-limit file. +For file-dependency audits, `cdidx deps --suppress-noise` removes generic +framework symbols and legacy Markdown heading-name fanout without hiding +explicit Markdown links. JSON edges expose the origin, reference kind, target +kind, and count in `evidence`; `symbol_filter` reports before/after counts and +the `markdown_heading_name_match` suppression reason. + If one file throws during indexing, cdidx commits successful files and their graph edges, reports structured `file_errors`, and exits with partial-result code `11`. Use `--allow-partial` only when automation deliberately accepts exit `0` for that @@ -583,6 +589,11 @@ structured identity key を保持します。 `cdidx . --max-references-per-file ` を使うと、text search と symbols は保持しつつ 上限を超えたファイルだけ references をスキップできます。 +ファイル依存関係の監査では、`cdidx deps --suppress-noise` が明示的な Markdown link を +隠さずに、汎用 framework symbol と旧 index 由来の Markdown 見出し名 fanout を除外します。 +JSON edge の `evidence` は origin、reference kind、target kind、件数を公開し、 +`symbol_filter` は before/after 件数と `markdown_heading_name_match` の抑制理由を返します。 + index 中に 1 ファイルで例外が発生した場合、cdidx は成功ファイルとその graph edge を commit し、構造化 `file_errors` を返して partial-result 終了コード `11` で終了します。 不完全 generation の終了コード `0` を automation が意図的に許容するときだけ diff --git a/USER_GUIDE.md b/USER_GUIDE.md index ef16a9a5e..52b30160c 100644 --- a/USER_GUIDE.md +++ b/USER_GUIDE.md @@ -2070,7 +2070,7 @@ same source location. | `--since ` | `search`, `definition`, `symbols`, `files` | Filter to files modified since this ISO 8601 timestamp. Offsetless values (e.g. `2024-01-01T00:00:00`) are treated as UTC so the same flag resolves to the same instant in every timezone; append `Z` or an explicit offset (`+09:00`) to be explicit. | | `--no-dedup` | `search` | Disable overlapping-chunk deduplication and return every raw chunk hit; useful for debugging chunk boundaries or measuring raw match density | | `--reverse` | `deps` | Reverse lookup: show files that depend ON the matched path | -| `--symbol ` / `--symbol-family ` / `--suppress-noise` | `deps` | Restrict dependency edges by an exact symbol, a symbol-name prefix, or the built-in noise profile. These filters run in SQLite before candidate ranking and `--limit`, including cycle and cross-workspace queries. `reference_count`, ranking, and JSON `symbol_filter` counters therefore describe the filtered scope. | +| `--symbol ` / `--symbol-family ` / `--suppress-noise` | `deps` | Restrict dependency edges by an exact symbol, a symbol-name prefix, or the built-in noise profile. Exact/prefix filters and generic-symbol suppression run in SQLite before candidate ranking and `--limit`, including cycle and cross-workspace queries. `--suppress-noise` also classifies legacy Markdown heading-name matches as evidence, prioritizes retained evidence before candidate limits, and removes only that evidence so explicit Markdown links remain visible. JSON edges expose the `source_language` / `origin` / `reference_kind` / `target_kind` / `reference_count` distribution in `evidence`; `symbol_filter` reports reference before/after totals and per-reason affected/removed counts. | | `--cycles` / `--graph-budget ` / `--cursor ` | `deps` | Compute deterministic, stably ranked dependency SCCs. `--graph-budget` independently bounds analyzed edges (default `10000`), while `--limit` pages the ranked SCCs and an opaque `next_cursor` continues the same filtered graph. JSON reports `analysis_complete`, `graph_edge_count`, `graph_edge_budget`, ranking metadata, authoritative-total status, and continuation metadata. When the graph budget is exhausted, the SCC set and total are explicitly non-authoritative; increase `--graph-budget` or narrow the graph with `--suppress-noise`, `--symbol`, `--symbol-family`, or `--path`. | | `--strict-not-found` | Query commands | Return exit code `2` when a valid query produces zero rows. Without this flag, zero-result queries normally exit `0` and keep their normal empty/zero-result output; the default-format `definition --json` miss is an intentional exception that always emits `E018_QUERY_NOT_FOUND` and exits `2`. | | `--top ` | Query commands | Alias for `--limit` | @@ -5262,7 +5262,7 @@ raw match density を正確に測る、といった理由で全 raw chunk hit | `--since ` | `search`, `definition`, `symbols`, `files` | 指定タイムスタンプ以降に変更されたファイルのみ（ISO 8601）。オフセットなしの値（例: `2024-01-01T00:00:00`）は UTC として解釈されるため、どのタイムゾーンから呼び出しても同じ UTC 時点になります。明示したい場合は末尾に `Z` または `+09:00` 等のオフセットを付与してください。 | | `--no-dedup` | `search` | overlap chunk の重複排除を無効化し、全 raw chunk hit を返す。chunk 境界の debug や raw match density 計測向け | | `--reverse` | `deps` | 逆引き: 指定パスに依存しているファイルを表示 | -| `--symbol ` / `--symbol-family ` / `--suppress-noise` | `deps` | 完全一致のシンボル、シンボル名の接頭辞、または組み込み noise profile で依存 edge を絞り込む。cycle と cross-workspace query を含め、これらの filter は候補の ranking と `--limit` より前に SQLite 内で適用される。そのため `reference_count`、ranking、JSON の `symbol_filter` counter は絞り込み後の scope を表す。 | +| `--symbol ` / `--symbol-family ` / `--suppress-noise` | `deps` | 完全一致のシンボル、シンボル名の接頭辞、または組み込み noise profile で依存 edge を絞り込む。完全一致 / 接頭辞 filter と汎用 symbol 抑制は、cycle と cross-workspace query を含め、候補の ranking と `--limit` より前に SQLite 内で適用される。`--suppress-noise` は旧 index 由来の Markdown 見出し名一致も evidence として分類し、候補上限より前に保持対象 evidence を優先したうえで、その evidence だけを除くため明示的な Markdown link は残る。JSON edge の `evidence` は `source_language` / `origin` / `reference_kind` / `target_kind` / `reference_count` の分布を公開し、`symbol_filter` は reference の before/after 合計と理由別の affected / removed 件数を返す。 | | `--cycles` / `--graph-budget ` / `--cursor ` | `deps` | 決定的かつ安定順位付きの依存 SCC を計算する。`--graph-budget` は解析する edge 数を独立して制限し（既定値 `10000`）、`--limit` は順位付け済み SCC をページ分割し、不透明な `next_cursor` で同じ filter 済み graph の続きを取得する。JSON は `analysis_complete`、`graph_edge_count`、`graph_edge_budget`、ranking metadata、総件数が authoritative かどうか、continuation metadata を返す。graph budget 枯渇時は SCC 集合と総件数が non-authoritative であることを明示するため、`--graph-budget` を増やすか、`--suppress-noise`、`--symbol`、`--symbol-family`、`--path` で graph を絞り込む。 | | `--workspace-db ` | `deps` | file dependency query に別の CodeIndex DB を追加する。最大 7 個の distinct な追加 DB（`--db` を含め合計 8 個）まで繰り返し指定でき、JSON edge には同じ相対パスを区別できるよう `source_db` / `target_db` が含まれる。 | | `--strict-not-found` | クエリ系 | 有効な query の結果が 0 件なら終了コード `2` を返す。この flag がない場合、0 件の query は通常、既存の empty / zero-result output を維持して終了コード `0` を返す。ただし既定 format の `definition --json` 未検出は意図的な例外で、常に `E018_QUERY_NOT_FOUND` と終了コード `2` を返す。 | diff --git a/changelog.d/unreleased/4868.fixed.md b/changelog.d/unreleased/4868.fixed.md new file mode 100644 index 000000000..404891a1c --- /dev/null +++ b/changelog.d/unreleased/4868.fixed.md @@ -0,0 +1,24 @@ +--- +category: fixed +issues: + - 4868 +affected: + - src/CodeIndex/Models/QueryResults.cs + - src/CodeIndex/Database/DbReader.Dependencies.cs + - src/CodeIndex/Cli/QueryCommandRunner.Dependencies.cs + - src/CodeIndex/Cli/JsonOutputContracts.cs + - src/CodeIndex/Cli/CliFlagSchema.cs + - tests/CodeIndex.Tests/DbReaderFileDependencyTests.cs + - tests/CodeIndex.Tests/QueryCommandRunnerTests.cs + - README.md + - USER_GUIDE.md + - DEVELOPER_GUIDE.md +--- + +## English + +- **Dependency noise suppression now recognizes legacy Markdown heading fanout (#4868)** — `deps --suppress-noise` classifies heading-name matches separately from explicit Markdown path links, prioritizes retained evidence before dependency and cycle candidate limits, and removes only the heading-derived reference counts. JSON and JSON-graph edges expose origin, reference-kind, target-kind, and count distributions through `evidence`, while `symbol_filter` reports reference-count deltas and per-reason affected/removed totals. + +## 日本語 + +- **依存ノイズ抑制が旧 Markdown 見出し fanout を識別するようになりました (#4868)** — `deps --suppress-noise` は見出し名一致を明示的な Markdown path link と分けて分類し、依存辺と cycle の候補上限より前に保持対象 evidence を優先したうえで、見出し由来の reference 件数だけを除外します。JSON / JSON-graph edge は `evidence` で origin、reference kind、target kind、件数の分布を公開し、`symbol_filter` は reference 件数の差分と理由別の affected / removed 合計を返します。 diff --git a/src/CodeIndex/Cli/CliFlagSchema.cs b/src/CodeIndex/Cli/CliFlagSchema.cs index c5a6d38a3..bd66f63e9 100644 --- a/src/CodeIndex/Cli/CliFlagSchema.cs +++ b/src/CodeIndex/Cli/CliFlagSchema.cs @@ -311,7 +311,7 @@ private static IReadOnlyList BuildAll() new() { Name = "--summary-only", Description = "Map/Diff/Recipes/Audit/Files/Symbols/Deps/Hotspots/Languages: return only aggregate summary fields where supported", PrimaryCommands = Set(SummaryOnlyCommands) }, new() { Name = "--cycles", Description = "Deps: return deterministically ranked dependency SCCs with stable pagination", PrimaryCommands = Set(DependencyCycleCommands) }, new() { Name = "--graph-budget", ValuePlaceholder = "", Description = $"Deps cycles: maximum graph edges analyzed for SCC completeness (default: {QueryCommandRunner.DefaultDependencyCycleGraphBudget})", PrimaryCommands = Set(DependencyCycleCommands) }, - new() { Name = "--suppress-noise", Description = "Deps: suppress generic framework/noise symbols in edge symbol samples", PrimaryCommands = Set("deps") }, + new() { Name = "--suppress-noise", Description = "Deps: suppress generic framework symbols and legacy Markdown heading-name fanout while preserving explicit links", PrimaryCommands = Set("deps") }, new() { Name = "--symbol", ValuePlaceholder = "", Description = "Deps: keep only edges with an exact sampled symbol name", PrimaryCommands = Set("deps") }, new() { Name = "--symbol-family", ValuePlaceholder = "", Description = "Deps: keep only edges with a sampled symbol prefix/family", PrimaryCommands = Set("deps") }, new() { Name = "--indexed-only", Description = "Languages: list only languages present in the current index", PrimaryCommands = Set(LanguagesFilterCommands) }, diff --git a/src/CodeIndex/Cli/JsonOutputContracts.cs b/src/CodeIndex/Cli/JsonOutputContracts.cs index fd54f4aec..8b0bdff38 100644 --- a/src/CodeIndex/Cli/JsonOutputContracts.cs +++ b/src/CodeIndex/Cli/JsonOutputContracts.cs @@ -1109,6 +1109,7 @@ internal sealed record ValidateConfigJsonResult( [JsonSerializable(typeof(ExcerptContentLineSpan))] [JsonSerializable(typeof(ExcerptRecoveryHint))] [JsonSerializable(typeof(ExcerptSemanticToken))] +[JsonSerializable(typeof(FileDependencyEvidence))] [JsonSerializable(typeof(FileDependencyResult))] [JsonSerializable(typeof(FileExcerptResult))] [JsonSerializable(typeof(FileFindResult))] @@ -1158,6 +1159,7 @@ internal sealed record ValidateConfigJsonResult( [JsonSerializable(typeof(List))] [JsonSerializable(typeof(List))] [JsonSerializable(typeof(List))] +[JsonSerializable(typeof(List))] [JsonSerializable(typeof(List))] [JsonSerializable(typeof(List))] [JsonSerializable(typeof(List))] diff --git a/src/CodeIndex/Cli/QueryCommandRunner.Dependencies.cs b/src/CodeIndex/Cli/QueryCommandRunner.Dependencies.cs index 3498d4224..0a531de6d 100644 --- a/src/CodeIndex/Cli/QueryCommandRunner.Dependencies.cs +++ b/src/CodeIndex/Cli/QueryCommandRunner.Dependencies.cs @@ -1409,6 +1409,7 @@ private static DependencySymbolFilterResult ApplyDependencySymbolFilters(IReadOn if (!applied) { var unchangedSymbolCount = edges.Sum(edge => SplitDependencySymbols(edge.Symbols).Count); + var unchangedReferenceCount = edges.Sum(static edge => (long)edge.ReferenceCount); return new DependencySymbolFilterResult( edges.ToList(), new DependencySymbolFilterSummary( @@ -1419,25 +1420,76 @@ private static DependencySymbolFilterResult ApplyDependencySymbolFilters(IReadOn EdgesBefore: edges.Count, EdgesAfter: edges.Count, SymbolsBefore: unchangedSymbolCount, - SymbolsAfter: unchangedSymbolCount)); + SymbolsAfter: unchangedSymbolCount, + ReferencesBefore: unchangedReferenceCount, + ReferencesAfter: unchangedReferenceCount, + SuppressionReasons: [])); } var filteredEdges = new List(edges.Count); var symbolsBefore = 0; var symbolsAfter = 0; + long referencesBefore = 0; + long referencesAfter = 0; + var headingEdgesAffected = 0; + var headingEdgesRemoved = 0; + long headingReferencesRemoved = 0; foreach (var edge in edges) { + referencesBefore += edge.ReferenceCount; + var edgeEvidence = edge.Evidence ?? []; + var preservesExplicitMarkdownLink = edgeEvidence.Any( + static evidence => evidence.Origin == "markdown_explicit_link"); + var keptEvidence = edgeEvidence; + var referenceCount = edge.ReferenceCount; + if (options.DependencySuppressNoise && edgeEvidence.Count > 0) + { + keptEvidence = edgeEvidence + .Where(static evidence => evidence.Origin != "markdown_heading_name_match") + .ToList(); + var removedReferenceCount = edgeEvidence + .Where(static evidence => evidence.Origin == "markdown_heading_name_match") + .Sum(static evidence => (long)evidence.ReferenceCount); + if (removedReferenceCount > 0) + { + headingEdgesAffected++; + headingReferencesRemoved += removedReferenceCount; + referenceCount = (int)Math.Max(0L, edge.ReferenceCount - removedReferenceCount); + if (referenceCount == 0) + headingEdgesRemoved++; + } + } + var symbols = SplitDependencySymbols(edge.Symbols); symbolsBefore += symbols.Count; var keptSymbols = symbols - .Where(symbol => KeepDependencySymbol(symbol, options)) + .Where(symbol => KeepDependencySymbol(symbol, options, preservesExplicitMarkdownLink)) .ToList(); - symbolsAfter += keptSymbols.Count; if (keptSymbols.Count == 0) continue; - filteredEdges.Add(CopyDependencyEdge(edge, string.Join(",", keptSymbols))); + + if (referenceCount == 0) + continue; + + symbolsAfter += keptSymbols.Count; + referencesAfter += referenceCount; + filteredEdges.Add(CopyDependencyEdge( + edge, + string.Join(",", keptSymbols), + referenceCount, + keptEvidence)); } + IReadOnlyList suppressionReasons = + headingReferencesRemoved > 0 + ? [ + new DependencySuppressionReasonSummary( + Reason: "markdown_heading_name_match", + EdgesAffected: headingEdgesAffected, + EdgesRemoved: headingEdgesRemoved, + ReferencesRemoved: headingReferencesRemoved), + ] + : []; return new DependencySymbolFilterResult( filteredEdges, new DependencySymbolFilterSummary( @@ -1448,12 +1500,20 @@ private static DependencySymbolFilterResult ApplyDependencySymbolFilters(IReadOn EdgesBefore: edges.Count, EdgesAfter: filteredEdges.Count, SymbolsBefore: symbolsBefore, - SymbolsAfter: symbolsAfter)); + SymbolsAfter: symbolsAfter, + ReferencesBefore: referencesBefore, + ReferencesAfter: referencesAfter, + SuppressionReasons: suppressionReasons)); } - private static bool KeepDependencySymbol(string symbol, QueryCommandOptions options) + private static bool KeepDependencySymbol( + string symbol, + QueryCommandOptions options, + bool preservesExplicitMarkdownLink) { - if (options.DependencySuppressNoise && DependencyNoiseProfile.IsNoiseSymbol(symbol)) + if (options.DependencySuppressNoise + && !preservesExplicitMarkdownLink + && DependencyNoiseProfile.IsNoiseSymbol(symbol)) return false; var hasNameFilters = options.DependencySymbols.Count > 0 || options.DependencySymbolFamilies.Count > 0; @@ -1467,7 +1527,11 @@ private static bool KeepDependencySymbol(string symbol, QueryCommandOptions opti private static List SplitDependencySymbols(string symbols) => symbols.Split(',', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries).ToList(); - private static FileDependencyResult CopyDependencyEdge(FileDependencyResult edge, string symbols) + private static FileDependencyResult CopyDependencyEdge( + FileDependencyResult edge, + string symbols, + int referenceCount, + List evidence) => new() { ResultKind = edge.ResultKind, @@ -1475,9 +1539,10 @@ private static FileDependencyResult CopyDependencyEdge(FileDependencyResult edge TargetPath = edge.TargetPath, SourceDb = edge.SourceDb, TargetDb = edge.TargetDb, - ReferenceCount = edge.ReferenceCount, - RankingScore = edge.RankingScore, + ReferenceCount = referenceCount, + RankingScore = DependencyNoiseProfile.ComputeRankingScore(referenceCount, symbols), Symbols = symbols, + Evidence = evidence, }; private static void AddDependencySchemaJsonFields( @@ -1527,7 +1592,23 @@ private static void AddDependencySymbolFilterJsonFields(JsonObject payload, Depe ["symbols_before"] = symbolFilter.SymbolsBefore, ["symbols_after"] = symbolFilter.SymbolsAfter, ["symbols_removed"] = symbolFilter.SymbolsBefore - symbolFilter.SymbolsAfter, + ["references_before"] = symbolFilter.ReferencesBefore, + ["references_after"] = symbolFilter.ReferencesAfter, + ["references_removed"] = symbolFilter.ReferencesBefore - symbolFilter.ReferencesAfter, }; + if (symbolFilter.SuppressionReasons.Count > 0) + { + filter["suppression_reasons"] = new JsonArray( + symbolFilter.SuppressionReasons + .Select(reason => (JsonNode?)new JsonObject + { + ["reason"] = reason.Reason, + ["edges_affected"] = reason.EdgesAffected, + ["edges_removed"] = reason.EdgesRemoved, + ["references_removed"] = reason.ReferencesRemoved, + }) + .ToArray()); + } if (symbolFilter.Symbols.Count > 0) filter["symbol"] = JsonSerializer.SerializeToNode(symbolFilter.Symbols.ToList(), CliJsonSerializerContextFactory.Create(jsonOptions).ListString); if (symbolFilter.SymbolFamilies.Count > 0) @@ -1538,6 +1619,16 @@ private static void AddDependencySymbolFilterJsonFields(JsonObject payload, Depe private static JsonArray BuildDependencySymbolsJson(string symbols) => new(SplitDependencySymbols(symbols).Select(symbol => JsonValue.Create(symbol)).ToArray()); + private static JsonArray BuildDependencyEvidenceJson(IReadOnlyList evidence) + => new(evidence.Select(item => (JsonNode?)new JsonObject + { + ["source_language"] = item.SourceLanguage, + ["origin"] = item.Origin, + ["reference_kind"] = item.ReferenceKind, + ["target_kind"] = item.TargetKind, + ["reference_count"] = item.ReferenceCount, + }).ToArray()); + private static bool DepsEmitsJson(QueryCommandOptions options, string depsFormat) => depsFormat == OutputFormatJsonGraph || (options.Json && depsFormat == OutputFormatEdgeList); @@ -1560,7 +1651,16 @@ private sealed record DependencySymbolFilterSummary( int EdgesBefore, int EdgesAfter, int SymbolsBefore, - int SymbolsAfter); + int SymbolsAfter, + long ReferencesBefore, + long ReferencesAfter, + IReadOnlyList SuppressionReasons); + + private sealed record DependencySuppressionReasonSummary( + string Reason, + int EdgesAffected, + int EdgesRemoved, + long ReferencesRemoved); private static int WriteDependencyGraph( IReadOnlyList edges, @@ -1628,6 +1728,7 @@ private static int WriteDependencyJsonGraph( ["reference_count"] = edge.ReferenceCount, ["ranking_score"] = edge.RankingScore, ["symbols"] = BuildDependencySymbolsJson(edge.Symbols), + ["evidence"] = BuildDependencyEvidenceJson(edge.Evidence ?? []), }).ToArray()); } addExtraJsonFields?.Invoke(payload); @@ -1862,11 +1963,25 @@ private static List GetCrossDatabaseFileDependencies( using var cmd = connection.CreateCommand(); var sourcePathExpr = reverse ? "dst.path" : "src.path"; var targetPathExpr = reverse ? "src.path" : "dst.path"; + var crossReferenceOrderSql = options.DependencySuppressNoise + ? "edge_totals.retained_reference_count DESC, edge_totals.reference_count DESC, edge_totals.source_path, edge_totals.target_path" + : "edge_totals.reference_count DESC, edge_totals.source_path, edge_totals.target_path"; + var retainedCrossSymbolFilterSql = options.DependencySuppressNoise + ? " WHERE origin <> 'markdown_heading_name_match'" + : string.Empty; cmd.CommandText = $@" WITH edges AS ( SELECT {sourcePathExpr} AS source_path, {targetPathExpr} AS target_path, - r.symbol_name + r.symbol_name, + src.lang AS source_lang, + CASE + WHEN src.lang = 'markdown' AND s.kind = 'heading' + THEN 'markdown_heading_name_match' + ELSE 'cross_database_symbol_name_match' + END AS origin, + r.reference_kind AS raw_reference_kind, + CASE WHEN s.kind = 'heading' THEN 'heading' ELSE 'symbol' END AS target_kind FROM symbol_references r JOIN files src ON src.id = r.file_id JOIN targetdb.symbols s ON s.name = r.symbol_name @@ -1892,21 +2007,65 @@ FROM symbol_references r "r.symbol_name", options.DependencySymbols, options.DependencySymbolFamilies, - options.DependencySuppressNoise, - "crossDependency"); + suppressDependencyNoise: false, + parameterPrefix: "crossDependencyNames"); + DbReader.AppendDependencySymbolFilter( + cmd, + ref crossDatabaseSql, + "r.symbol_name", + dependencySymbols: null, + dependencySymbolFamilies: null, + suppressDependencyNoise: options.DependencySuppressNoise, + parameterPrefix: "crossDependencyNoise", + filterScopeSql: "NOT (src.lang = 'markdown' AND s.kind = 'heading')"); cmd.CommandText = crossDatabaseSql; cmd.CommandText += @" ), edge_totals AS ( SELECT source_path, target_path, - COUNT(*) AS reference_count + COUNT(*) AS reference_count, + SUM(CASE WHEN origin = 'markdown_heading_name_match' THEN 0 ELSE 1 END) AS retained_reference_count FROM edges GROUP BY source_path, target_path ), + edge_evidence_rows AS ( + SELECT source_path, + target_path, + source_lang, + origin, + raw_reference_kind, + target_kind, + COUNT(*) AS evidence_reference_count + FROM edges + GROUP BY source_path, + target_path, + source_lang, + origin, + raw_reference_kind, + target_kind + ), + ordered_edge_evidence AS ( + SELECT source_path, + target_path, + source_lang || char(31) || + origin || char(31) || + raw_reference_kind || char(31) || + target_kind || char(31) || + evidence_reference_count AS evidence_item + FROM edge_evidence_rows + ORDER BY source_path, target_path, source_lang, origin, raw_reference_kind, target_kind + ), + edge_evidence_payloads AS ( + SELECT source_path, + target_path, + GROUP_CONCAT(evidence_item, char(30)) AS evidence_payload + FROM ordered_edge_evidence + GROUP BY source_path, target_path + ), distinct_edge_symbols AS ( SELECT DISTINCT source_path, target_path, symbol_name - FROM edges + FROM edges" + retainedCrossSymbolFilterSql + @" ), ranked_edge_symbols AS ( SELECT source_path, @@ -1918,13 +2077,20 @@ FROM distinct_edge_symbols SELECT edge_totals.source_path, edge_totals.target_path, edge_totals.reference_count, - COALESCE(GROUP_CONCAT(CASE WHEN ranked_edge_symbols.symbol_rank <= @symbolSampleLimit THEN ranked_edge_symbols.symbol_name END), '') AS symbols + COALESCE(GROUP_CONCAT(CASE WHEN ranked_edge_symbols.symbol_rank <= @symbolSampleLimit THEN ranked_edge_symbols.symbol_name END), '') AS symbols, + COALESCE(edge_evidence_payloads.evidence_payload, '') AS evidence_payload FROM edge_totals LEFT JOIN ranked_edge_symbols ON ranked_edge_symbols.source_path = edge_totals.source_path AND ranked_edge_symbols.target_path = edge_totals.target_path - GROUP BY edge_totals.source_path, edge_totals.target_path, edge_totals.reference_count - ORDER BY edge_totals.reference_count DESC, edge_totals.source_path, edge_totals.target_path + LEFT JOIN edge_evidence_payloads + ON edge_evidence_payloads.source_path = edge_totals.source_path + AND edge_evidence_payloads.target_path = edge_totals.target_path + GROUP BY edge_totals.source_path, + edge_totals.target_path, + edge_totals.reference_count, + edge_evidence_payloads.evidence_payload + ORDER BY " + crossReferenceOrderSql + @" LIMIT @limit"; SqliteCommandPolicy.Add(cmd, "@limit", DependencyNoiseProfile.GetRankingCandidateLimit(limit)); SqliteCommandPolicy.Add(cmd, "@symbolSampleLimit", DbReader.DependencySymbolSampleLimit); @@ -1945,6 +2111,7 @@ LEFT JOIN ranked_edge_symbols TargetDb = reverse ? sourceDbPath : targetDbPath, ReferenceCount = reader.GetInt32(2), Symbols = reader.IsDBNull(3) ? string.Empty : reader.GetString(3), + Evidence = DbReader.ParseDependencyEvidence(reader.GetString(4)), }); } } @@ -1953,7 +2120,14 @@ LEFT JOIN ranked_edge_symbols throw new OperationCanceledException(cancellationToken); } foreach (var result in results) - result.RankingScore = DependencyNoiseProfile.ComputeRankingScore(result.ReferenceCount, result.Symbols); + { + var rankingReferenceCount = options.DependencySuppressNoise + ? (result.Evidence ?? []) + .Where(static evidence => evidence.Origin != "markdown_heading_name_match") + .Sum(static evidence => evidence.ReferenceCount) + : result.ReferenceCount; + result.RankingScore = DependencyNoiseProfile.ComputeRankingScore(rankingReferenceCount, result.Symbols); + } return results .OrderByDescending(result => result.RankingScore) diff --git a/src/CodeIndex/Database/DbReader.Dependencies.cs b/src/CodeIndex/Database/DbReader.Dependencies.cs index 173efef6e..b73c4b7fa 100644 --- a/src/CodeIndex/Database/DbReader.Dependencies.cs +++ b/src/CodeIndex/Database/DbReader.Dependencies.cs @@ -263,12 +263,19 @@ public List GetFileDependencies( SELECT resolved.source_path, resolved.target_path, resolved.symbol_name, - COUNT(*) AS ref_count + COUNT(*) AS ref_count, + resolved.source_lang, + 'resolved_identity' AS origin, + resolved.raw_reference_kind, + resolved.target_kind FROM ( SELECT DISTINCT lrp.source_path, target_file.path AS target_path, lrp.symbol_name, - lrp.reference_id + lrp.reference_id, + lrp.source_lang, + lrp.raw_reference_kind, + target.kind AS target_kind FROM logical_references_primary lrp JOIN symbol_reference_candidates candidate ON candidate.reference_id = lrp.reference_id @@ -294,7 +301,8 @@ AND lrp.resolution_state IN ('resolved', 'resolved_group') resolvedIdentityEdgesSql += @" ORDER BY lrp.source_path, lrp.symbol_name, lrp.reference_id" + resolvedIdentityLimitSql + @" ) resolved - GROUP BY resolved.source_path, resolved.target_path, resolved.symbol_name + GROUP BY resolved.source_path, resolved.target_path, resolved.symbol_name, + resolved.source_lang, resolved.raw_reference_kind, resolved.target_kind UNION ALL "; } @@ -313,6 +321,7 @@ UNION ALL var sqlDependencyTargetMatchExpr = @"( (tf.target_lang != 'sql' AND NOT (snc.source_lang IN ('msbuild', 'solution') AND snc.logical_reference_kind IN ('import', 'project_reference')) + AND NOT (snc.source_lang = 'markdown' AND snc.logical_reference_kind = 'import') AND NOT (snc.source_lang = 'markdown' AND snc.logical_reference_kind = 'reference') AND tf.symbol_name = snc.symbol_name) OR (snc.source_lang = 'python' @@ -323,9 +332,6 @@ SELECT 1 FROM symbols py_import_match AND py_import_match.kind = 'import' AND python_import_target_name(snc.source_path, snc.symbol_name, snc.context, snc.column_number, " + pythonImportMatchSignatureExpr + @") = tf.symbol_name )) - OR (snc.source_lang = 'markdown' - AND snc.logical_reference_kind = 'import' - AND tf.target_path = markdown_resolve_path(snc.source_path, snc.symbol_name)) OR (tf.target_lang = 'sql' AND ( (tf.symbol_segment_count = snc.symbol_segment_count AND tf.symbol_name = snc.symbol_name COLLATE NOCASE) OR (sql_segment_count(snc.raw_symbol_name) = 1 @@ -575,6 +581,7 @@ target_files AS ( " + targetLogicalSymbolNameExpr + @" AS symbol_name, " + targetLogicalSymbolSegmentCountExpr + @" AS symbol_segment_count, MAX(CASE WHEN s.kind IN ('class','struct','interface') THEN 1 ELSE 0 END) AS has_class_like_kind, + MAX(CASE WHEN s.kind = 'heading' THEN 1 ELSE 0 END) AS has_heading_kind, MAX(CASE WHEN " + BuildMetadataTargetKindExpr("dst") + @" THEN 1 ELSE 0 END) AS has_metadata_target_kind FROM symbols s @@ -684,7 +691,15 @@ edges AS ( SELECT snc.source_path, tf.target_path, tf.symbol_name, - snc.ref_count + snc.ref_count, + snc.source_lang, + CASE + WHEN snc.source_lang = 'markdown' AND tf.has_heading_kind = 1 + THEN 'markdown_heading_name_match' + ELSE 'symbol_name_match' + END AS origin, + snc.raw_reference_kind, + CASE WHEN tf.has_heading_kind = 1 THEN 'heading' ELSE 'symbol' END AS target_kind FROM bounded_source_name_counts snc JOIN target_files tf ON " + sqlDependencyTargetMatchExpr + @" @@ -745,8 +760,47 @@ AND python_import_resolves(snc.source_path, tf.target_path, snc.symbol_name, snc "tf.symbol_name", dependencySymbols, dependencySymbolFamilies, - suppressDependencyNoise, - "edgeDependency"); + suppressDependencyNoise: false, + parameterPrefix: "edgeDependencyNames"); + AppendDependencySymbolFilter( + cmd, + ref sql, + "tf.symbol_name", + dependencySymbols: null, + dependencySymbolFamilies: null, + suppressDependencyNoise: suppressDependencyNoise, + parameterPrefix: "edgeDependencyNoise", + filterScopeSql: "NOT (snc.source_lang = 'markdown' AND tf.has_heading_kind = 1)"); + sql += @" + UNION ALL + -- Resolve explicit Markdown links once per target file. Joining these + -- path references through target_files would multiply one link by every + -- heading or symbol declared in the destination document. + -- 明示的な Markdown link は target file ごとに一度だけ解決する。 + -- target_files 経由で結合すると、1 link が宛先 document 内の全見出し + -- / symbol の件数だけ増幅されるため、file-level path として扱う。 + SELECT snc.source_path, + ptf.target_path, + snc.raw_symbol_name, + snc.ref_count, + snc.source_lang, + 'markdown_explicit_link', + snc.raw_reference_kind, + 'file' + FROM bounded_source_name_counts snc + JOIN path_target_files ptf + ON ptf.target_path = markdown_resolve_path(snc.source_path, snc.symbol_name) + WHERE snc.source_lang = 'markdown' + AND snc.logical_reference_kind = 'import' + AND snc.source_path != ptf.target_path"; + AppendDependencySymbolFilter( + cmd, + ref sql, + "snc.raw_symbol_name", + dependencySymbols, + dependencySymbolFamilies, + suppressDependencyNoise: false, + parameterPrefix: "markdownPathDependency"); sql += @" UNION ALL -- Dockerfile stages are symbols within one file, so their dependency edge is @@ -758,7 +812,11 @@ UNION ALL SELECT snc.source_path, snc.source_path, snc.symbol_name, - snc.ref_count + snc.ref_count, + snc.source_lang, + 'docker_stage_reference', + snc.raw_reference_kind, + 'stage' FROM source_name_counts snc JOIN files self_dst ON self_dst.id = snc.source_file_id WHERE snc.source_lang = 'dockerfile' @@ -797,7 +855,11 @@ UNION ALL SELECT snc.source_path, ptf.target_path, snc.symbol_name, - snc.ref_count + snc.ref_count, + snc.source_lang, + 'explicit_path_reference', + snc.raw_reference_kind, + 'file' FROM bounded_source_name_counts snc JOIN path_target_files ptf ON ptf.target_path = markdown_resolve_path(snc.source_path, snc.symbol_name) @@ -812,23 +874,71 @@ AND snc.logical_reference_kind IN ('import', 'project_reference') dependencySymbolFamilies, suppressDependencyNoise, "pathDependency"); + var dependencyReferenceOrderSql = suppressDependencyNoise + ? "retained_reference_count DESC, reference_count DESC, source_path, target_path" + : "reference_count DESC, source_path, target_path"; + var finalDependencyReferenceOrderSql = suppressDependencyNoise + ? "limited_edge_totals.retained_reference_count DESC, limited_edge_totals.reference_count DESC, limited_edge_totals.source_path, limited_edge_totals.target_path" + : "limited_edge_totals.reference_count DESC, limited_edge_totals.source_path, limited_edge_totals.target_path"; + var retainedDependencySymbolFilterSql = suppressDependencyNoise + ? " WHERE edges.origin <> 'markdown_heading_name_match'" + : string.Empty; sql += @" ), edge_totals AS ( SELECT source_path, target_path, - SUM(ref_count) AS reference_count + SUM(ref_count) AS reference_count, + SUM(CASE WHEN origin = 'markdown_heading_name_match' THEN 0 ELSE ref_count END) AS retained_reference_count FROM edges GROUP BY source_path, target_path ), limited_edge_totals AS ( SELECT source_path, target_path, - reference_count + reference_count, + retained_reference_count FROM edge_totals - ORDER BY reference_count DESC, source_path, target_path + ORDER BY " + dependencyReferenceOrderSql + @" LIMIT @limit ), + edge_evidence_rows AS ( + SELECT edges.source_path, + edges.target_path, + edges.source_lang, + edges.origin, + edges.raw_reference_kind, + edges.target_kind, + SUM(edges.ref_count) AS evidence_reference_count + FROM edges + JOIN limited_edge_totals + ON limited_edge_totals.source_path = edges.source_path + AND limited_edge_totals.target_path = edges.target_path + GROUP BY edges.source_path, + edges.target_path, + edges.source_lang, + edges.origin, + edges.raw_reference_kind, + edges.target_kind + ), + ordered_edge_evidence AS ( + SELECT source_path, + target_path, + source_lang || char(31) || + origin || char(31) || + raw_reference_kind || char(31) || + target_kind || char(31) || + evidence_reference_count AS evidence_item + FROM edge_evidence_rows + ORDER BY source_path, target_path, source_lang, origin, raw_reference_kind, target_kind + ), + edge_evidence_payloads AS ( + SELECT source_path, + target_path, + GROUP_CONCAT(evidence_item, char(30)) AS evidence_payload + FROM ordered_edge_evidence + GROUP BY source_path, target_path + ), distinct_edge_symbols AS ( SELECT DISTINCT edges.source_path, edges.target_path, @@ -836,7 +946,7 @@ distinct_edge_symbols AS ( FROM edges JOIN limited_edge_totals ON limited_edge_totals.source_path = edges.source_path - AND limited_edge_totals.target_path = edges.target_path + AND limited_edge_totals.target_path = edges.target_path" + retainedDependencySymbolFilterSql + @" ), ranked_edge_symbols AS ( SELECT source_path, @@ -848,13 +958,20 @@ FROM distinct_edge_symbols SELECT limited_edge_totals.source_path, limited_edge_totals.target_path, limited_edge_totals.reference_count, - COALESCE(GROUP_CONCAT(CASE WHEN ranked_edge_symbols.symbol_rank <= @symbolSampleLimit THEN ranked_edge_symbols.symbol_name END), '') AS symbols + COALESCE(GROUP_CONCAT(CASE WHEN ranked_edge_symbols.symbol_rank <= @symbolSampleLimit THEN ranked_edge_symbols.symbol_name END), '') AS symbols, + COALESCE(edge_evidence_payloads.evidence_payload, '') AS evidence_payload FROM limited_edge_totals LEFT JOIN ranked_edge_symbols ON ranked_edge_symbols.source_path = limited_edge_totals.source_path AND ranked_edge_symbols.target_path = limited_edge_totals.target_path - GROUP BY limited_edge_totals.source_path, limited_edge_totals.target_path, limited_edge_totals.reference_count - ORDER BY limited_edge_totals.reference_count DESC, limited_edge_totals.source_path, limited_edge_totals.target_path"; + LEFT JOIN edge_evidence_payloads + ON edge_evidence_payloads.source_path = limited_edge_totals.source_path + AND edge_evidence_payloads.target_path = limited_edge_totals.target_path + GROUP BY limited_edge_totals.source_path, + limited_edge_totals.target_path, + limited_edge_totals.reference_count, + edge_evidence_payloads.evidence_payload + ORDER BY " + finalDependencyReferenceOrderSql; cmd.CommandText = sql; if (lang != null) @@ -883,6 +1000,7 @@ LEFT JOIN ranked_edge_symbols TargetPath = reader.GetString(1), ReferenceCount = reader.GetInt32(2), Symbols = reader.GetString(3), + Evidence = ParseDependencyEvidence(reader.GetString(4)), }); } } @@ -890,13 +1008,48 @@ LEFT JOIN ranked_edge_symbols { throw new OperationCanceledException(cancellationToken); } - return RankDependencyResults(results, limit); + return RankDependencyResults(results, limit, suppressDependencyNoise); } - private static List RankDependencyResults(List results, int limit) + internal static List ParseDependencyEvidence(string payload) + { + if (string.IsNullOrEmpty(payload)) + return []; + + var evidence = new List(); + foreach (var item in payload.Split('\u001e', StringSplitOptions.RemoveEmptyEntries)) + { + var fields = item.Split('\u001f'); + if (fields.Length != 5 || !int.TryParse(fields[4], out var referenceCount)) + continue; + + evidence.Add(new FileDependencyEvidence + { + SourceLanguage = fields[0], + Origin = fields[1], + ReferenceKind = fields[2], + TargetKind = fields[3], + ReferenceCount = referenceCount, + }); + } + + return evidence; + } + + private static List RankDependencyResults( + List results, + int limit, + bool suppressDependencyNoise) { foreach (var result in results) - result.RankingScore = DependencyNoiseProfile.ComputeRankingScore(result.ReferenceCount, result.Symbols); + { + var rankingReferenceCount = suppressDependencyNoise && result.Evidence is { Count: > 0 } + ? result.Evidence + .Where(static evidence => evidence.Origin != "markdown_heading_name_match") + .Sum(static evidence => evidence.ReferenceCount) + : result.ReferenceCount; + result.RankingScore = DependencyNoiseProfile.ComputeRankingScore(rankingReferenceCount, result.Symbols); + } return results .OrderByDescending(result => result.RankingScore) @@ -928,10 +1081,27 @@ public List GetFileDependencyCycleCandidates( cancellationToken.ThrowIfCancellationRequested(); using var cmd = _conn.CreateCommand(); var constrainedAlias = reverse ? "dst" : "src"; + var cycleMarkdownExplicitLinkSql = _referenceColumns.Contains("target_qualifier") + ? "(src.lang = 'markdown' AND r.reference_kind = 'reference' AND r.target_qualifier IS NOT NULL AND dst.path = markdown_resolve_path(src.path, r.target_qualifier))" + : "0 = 1"; + var cycleNoiseEvidenceScopeSql = + "(" + cycleMarkdownExplicitLinkSql + " OR (src.lang = 'markdown' AND s.kind = 'heading'))"; + var cycleCandidateOrderSql = suppressDependencyNoise + ? "retained_evidence DESC, source_path, target_path" + : "source_path, target_path"; + var retainedCycleSymbolFilterSql = suppressDependencyNoise + ? " WHERE origin <> 'markdown_heading_name_match'" + : string.Empty; var sql = @" WITH candidate_edges AS ( - SELECT DISTINCT src.path AS source_path, - dst.path AS target_path + SELECT src.path AS source_path, + dst.path AS target_path, + MAX(CASE + WHEN src.lang = 'markdown' + AND s.kind = 'heading' + AND NOT " + cycleMarkdownExplicitLinkSql + @" THEN 0 + ELSE 1 + END) AS retained_evidence FROM symbol_references r JOIN files src ON r.file_id = src.id JOIN symbols s ON s.name = r.symbol_name @@ -967,17 +1137,37 @@ FROM symbol_references r "r.symbol_name", dependencySymbols, dependencySymbolFamilies, - suppressDependencyNoise, - "cycleDependency"); + suppressDependencyNoise: false, + parameterPrefix: "cycleDependencyNames"); + AppendDependencySymbolFilter( + cmd, + ref sql, + "r.symbol_name", + dependencySymbols: null, + dependencySymbolFamilies: null, + suppressDependencyNoise: suppressDependencyNoise, + parameterPrefix: "cycleDependencyNoise", + filterScopeSql: "NOT " + cycleNoiseEvidenceScopeSql); sql += @" - ORDER BY source_path, target_path + GROUP BY src.path, dst.path + ORDER BY " + cycleCandidateOrderSql + @" LIMIT @limit ), candidate_symbols AS ( SELECT candidate_edges.source_path, candidate_edges.target_path, r.id AS reference_id, - r.symbol_name + r.symbol_name, + src.lang AS source_lang, + CASE + WHEN " + cycleMarkdownExplicitLinkSql + @" + THEN 'markdown_explicit_link' + WHEN src.lang = 'markdown' AND s.kind = 'heading' + THEN 'markdown_heading_name_match' + ELSE 'symbol_name_match' + END AS origin, + r.reference_kind AS raw_reference_kind, + CASE WHEN s.kind = 'heading' THEN 'heading' ELSE 'symbol' END AS target_kind FROM candidate_edges JOIN files src ON src.path = candidate_edges.source_path JOIN symbol_references r ON r.file_id = src.id @@ -992,8 +1182,17 @@ FROM candidate_edges "r.symbol_name", dependencySymbols, dependencySymbolFamilies, - suppressDependencyNoise, - "cycleAggregate"); + suppressDependencyNoise: false, + parameterPrefix: "cycleAggregateNames"); + AppendDependencySymbolFilter( + cmd, + ref sql, + "r.symbol_name", + dependencySymbols: null, + dependencySymbolFamilies: null, + suppressDependencyNoise: suppressDependencyNoise, + parameterPrefix: "cycleAggregateNoise", + filterScopeSql: "NOT " + cycleNoiseEvidenceScopeSql); sql += @" ), edge_reference_totals AS ( @@ -1003,11 +1202,45 @@ edge_reference_totals AS ( FROM candidate_symbols GROUP BY source_path, target_path ), + edge_evidence_rows AS ( + SELECT source_path, + target_path, + source_lang, + origin, + raw_reference_kind, + target_kind, + COUNT(DISTINCT reference_id) AS evidence_reference_count + FROM candidate_symbols + GROUP BY source_path, + target_path, + source_lang, + origin, + raw_reference_kind, + target_kind + ), + ordered_edge_evidence AS ( + SELECT source_path, + target_path, + source_lang || char(31) || + origin || char(31) || + raw_reference_kind || char(31) || + target_kind || char(31) || + evidence_reference_count AS evidence_item + FROM edge_evidence_rows + ORDER BY source_path, target_path, source_lang, origin, raw_reference_kind, target_kind + ), + edge_evidence_payloads AS ( + SELECT source_path, + target_path, + GROUP_CONCAT(evidence_item, char(30)) AS evidence_payload + FROM ordered_edge_evidence + GROUP BY source_path, target_path + ), distinct_edge_symbols AS ( SELECT DISTINCT source_path, target_path, symbol_name - FROM candidate_symbols + FROM candidate_symbols" + retainedCycleSymbolFilterSql + @" ), ranked_edge_symbols AS ( SELECT source_path, @@ -1019,14 +1252,19 @@ FROM distinct_edge_symbols SELECT edge_reference_totals.source_path, edge_reference_totals.target_path, edge_reference_totals.reference_count, - COALESCE(GROUP_CONCAT(CASE WHEN symbol_rank <= @symbolSampleLimit THEN symbol_name END), '') AS symbols + COALESCE(GROUP_CONCAT(CASE WHEN symbol_rank <= @symbolSampleLimit THEN symbol_name END), '') AS symbols, + COALESCE(edge_evidence_payloads.evidence_payload, '') AS evidence_payload FROM edge_reference_totals LEFT JOIN ranked_edge_symbols ON ranked_edge_symbols.source_path = edge_reference_totals.source_path AND ranked_edge_symbols.target_path = edge_reference_totals.target_path + LEFT JOIN edge_evidence_payloads + ON edge_evidence_payloads.source_path = edge_reference_totals.source_path + AND edge_evidence_payloads.target_path = edge_reference_totals.target_path GROUP BY edge_reference_totals.source_path, edge_reference_totals.target_path, - edge_reference_totals.reference_count + edge_reference_totals.reference_count, + edge_evidence_payloads.evidence_payload ORDER BY edge_reference_totals.source_path, edge_reference_totals.target_path"; cmd.CommandText = sql; @@ -1055,6 +1293,7 @@ LEFT JOIN ranked_edge_symbols ReferenceCount = reader.GetInt32(2), RankingScore = reader.GetInt32(2), Symbols = reader.GetString(3), + Evidence = ParseDependencyEvidence(reader.GetString(4)), }); } } diff --git a/src/CodeIndex/Models/QueryResults.cs b/src/CodeIndex/Models/QueryResults.cs index e609537a6..a3b54bf62 100644 --- a/src/CodeIndex/Models/QueryResults.cs +++ b/src/CodeIndex/Models/QueryResults.cs @@ -2248,4 +2248,15 @@ public class FileDependencyResult [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)] public double RankingScore { get; set; } public string Symbols { get; set; } = string.Empty; + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public List? Evidence { get; set; } +} + +public class FileDependencyEvidence +{ + public string SourceLanguage { get; set; } = string.Empty; + public string Origin { get; set; } = string.Empty; + public string ReferenceKind { get; set; } = string.Empty; + public string TargetKind { get; set; } = string.Empty; + public int ReferenceCount { get; set; } } diff --git a/tests/CodeIndex.Tests/DbReaderFileDependencyTests.cs b/tests/CodeIndex.Tests/DbReaderFileDependencyTests.cs index f550b0a42..ba4883c26 100644 --- a/tests/CodeIndex.Tests/DbReaderFileDependencyTests.cs +++ b/tests/CodeIndex.Tests/DbReaderFileDependencyTests.cs @@ -285,14 +285,84 @@ JOIN symbols AS target } var dependencies = _reader.GetFileDependencies(limit: 20, lang: "markdown"); - Assert.Contains(dependencies, dependency => + var explicitDependency = Assert.Single(dependencies, dependency => dependency.SourcePath == "docs/issue4846/source.md" && dependency.TargetPath == "docs/issue4846/target.md"); + var explicitEvidence = Assert.IsType>(explicitDependency.Evidence); + Assert.Contains(explicitEvidence, evidence => + evidence.Origin == "markdown_explicit_link" + && evidence.SourceLanguage == "markdown" + && evidence.ReferenceCount > 0); + Assert.DoesNotContain(explicitEvidence, evidence => + evidence.Origin == "markdown_heading_name_match"); Assert.DoesNotContain(dependencies, dependency => dependency.SourcePath == "docs/issue4846/source.md" && dependency.TargetPath == "docs/issue4846/unrelated.md"); } + [Fact] + public void GetFileDependencies_ReportsLegacyMarkdownHeadingEvidenceSeparatelyFromExplicitLinks_Issue4868() + { + InsertIndexedFile( + "docs/issue4868/target.md", + "markdown", + "# Error\n\n# Shared heading\n"); + InsertIndexedFile( + "docs/issue4868/noise.md", + "markdown", + "# Shared heading\n"); + InsertIndexedFile( + "docs/issue4868/unrelated.md", + "markdown", + "# Shared heading\n"); + InsertIndexedFile( + "docs/issue4868/source.md", + "markdown", + "# Source\n\n[target](target.md#error)\n"); + + long sourceFileId; + using (var sourceFile = _db.Connection.CreateCommand()) + { + sourceFile.CommandText = "SELECT id FROM files WHERE path = 'docs/issue4868/source.md'"; + sourceFileId = Convert.ToInt64(sourceFile.ExecuteScalar()); + } + _writer.InsertReferences(Enumerable.Range(0, 2_000).Select(index => new ReferenceRecord + { + FileId = sourceFileId, + SymbolName = "Shared heading", + ReferenceKind = "use", + Line = index + 10, + Column = 1, + Context = $"legacy heading reference {index}", + }).ToArray()); + + var dependencies = _reader.GetFileDependencies( + limit: 20, + lang: "markdown", + suppressDependencyNoise: true); + + var explicitDependency = Assert.Single(dependencies, dependency => + dependency.SourcePath == "docs/issue4868/source.md" + && dependency.TargetPath == "docs/issue4868/target.md"); + var explicitEvidence = Assert.IsType>(explicitDependency.Evidence); + Assert.Contains(explicitEvidence, evidence => + evidence.Origin == "markdown_explicit_link" + && evidence.ReferenceKind == "import" + && evidence.TargetKind == "file" + && evidence.ReferenceCount == 1); + Assert.Contains(explicitEvidence, evidence => + evidence.Origin == "markdown_heading_name_match" + && evidence.ReferenceCount == 2_000); + + var noisyDependency = Assert.Single(dependencies, dependency => + dependency.SourcePath == "docs/issue4868/source.md" + && dependency.TargetPath == "docs/issue4868/noise.md"); + var noiseEvidence = Assert.Single( + Assert.IsType>(noisyDependency.Evidence)); + Assert.Equal("markdown_heading_name_match", noiseEvidence.Origin); + Assert.Equal(2_000, noiseEvidence.ReferenceCount); + } + [Fact] public void GetFileDependencies_CapsDenseSymbolSample_Issue3155() { diff --git a/tests/CodeIndex.Tests/QueryCommandRunnerTests.cs b/tests/CodeIndex.Tests/QueryCommandRunnerTests.cs index f144a6add..aff21723d 100644 --- a/tests/CodeIndex.Tests/QueryCommandRunnerTests.cs +++ b/tests/CodeIndex.Tests/QueryCommandRunnerTests.cs @@ -5685,6 +5685,8 @@ public void RunDeps_JsonGraph_WritesValidGraphPayload() Assert.True(json.GetProperty("edges")[0].TryGetProperty("reference_count", out _)); Assert.True(json.GetProperty("edges")[0].TryGetProperty("ranking_score", out _)); Assert.True(json.GetProperty("edges")[0].TryGetProperty("symbols", out _)); + Assert.True(json.GetProperty("edges")[0].TryGetProperty("evidence", out var evidence)); + Assert.NotEmpty(evidence.EnumerateArray()); } [Fact] @@ -5737,6 +5739,84 @@ public void RunDeps_JsonPushesSymbolFiltersIntoSqlAndAddsMetadata_Issues3943And4 Assert.Equal(0, symbolFilter.GetProperty("symbols_removed").GetInt32()); } + [Fact] + public void RunDeps_SuppressNoiseRemovesLegacyMarkdownHeadingEvidenceButKeepsExplicitLinks_Issue4868() + { + using var project = TestProjectHelper.CreateTempProjectScope("cdidx_deps_markdown_heading_noise"); + var projectRoot = project.Root; + var dbPath = TestProjectHelper.CreateProjectDb(projectRoot); + TestProjectHelper.InsertIndexedFile( + dbPath, + "docs/issue4868/target.md", + "markdown", + "# Error\n\n# Shared heading\n"); + TestProjectHelper.InsertIndexedFile( + dbPath, + "docs/issue4868/noise.md", + "markdown", + "# Shared heading\n"); + TestProjectHelper.InsertIndexedFile( + dbPath, + "docs/issue4868/unrelated.md", + "markdown", + "# Shared heading\n"); + TestProjectHelper.InsertIndexedFile( + dbPath, + "docs/issue4868/source.md", + "markdown", + "# Source\n\n[target](target.md#error)\n"); + InsertLegacyMarkdownHeadingReferences( + dbPath, + "docs/issue4868/source.md", + "Shared heading", + count: 2_000); + MarkDependencyGraphReady(dbPath); + + var (defaultExitCode, defaultStdout, defaultStderr) = CaptureConsole(() => QueryCommandRunner.RunDeps( + ["--db", dbPath, "--json", "--limit", "10", "--lang", "markdown"], + _jsonOptions)); + + using var defaultDocument = ParseJsonOutput(defaultStdout); + var defaultJson = defaultDocument.RootElement; + var noisyEdge = Assert.Single(defaultJson.GetProperty("edges").EnumerateArray(), edge => + edge.GetProperty("target_path").GetString() == "docs/issue4868/unrelated.md"); + var noisyEvidence = Assert.Single(noisyEdge.GetProperty("evidence").EnumerateArray()); + + Assert.Equal(CommandExitCodes.Success, defaultExitCode); + Assert.Equal(string.Empty, defaultStderr); + Assert.Equal("markdown_heading_name_match", noisyEvidence.GetProperty("origin").GetString()); + Assert.Equal("use", noisyEvidence.GetProperty("reference_kind").GetString()); + Assert.Equal("heading", noisyEvidence.GetProperty("target_kind").GetString()); + Assert.Equal(2_000, noisyEvidence.GetProperty("reference_count").GetInt32()); + + var (filteredExitCode, filteredStdout, filteredStderr) = CaptureConsole(() => QueryCommandRunner.RunDeps( + ["--db", dbPath, "--json", "--limit", "10", "--lang", "markdown", "--suppress-noise"], + _jsonOptions)); + + using var filteredDocument = ParseJsonOutput(filteredStdout); + var filteredJson = filteredDocument.RootElement; + var retainedEdge = Assert.Single(filteredJson.GetProperty("edges").EnumerateArray()); + var retainedEvidence = Assert.Single(retainedEdge.GetProperty("evidence").EnumerateArray()); + var symbolFilter = filteredJson.GetProperty("symbol_filter"); + var suppressionReason = Assert.Single(symbolFilter.GetProperty("suppression_reasons").EnumerateArray()); + + Assert.Equal(CommandExitCodes.Success, filteredExitCode); + Assert.Equal(string.Empty, filteredStderr); + Assert.Equal("docs/issue4868/target.md", retainedEdge.GetProperty("target_path").GetString()); + Assert.Equal(1, retainedEdge.GetProperty("reference_count").GetInt32()); + Assert.Equal("target.md#error", retainedEdge.GetProperty("symbols").GetString()); + Assert.Equal("markdown_explicit_link", retainedEvidence.GetProperty("origin").GetString()); + Assert.Equal("import", retainedEvidence.GetProperty("reference_kind").GetString()); + Assert.Equal("file", retainedEvidence.GetProperty("target_kind").GetString()); + Assert.Equal(6_001, symbolFilter.GetProperty("references_before").GetInt32()); + Assert.Equal(1, symbolFilter.GetProperty("references_after").GetInt32()); + Assert.Equal(6_000, symbolFilter.GetProperty("references_removed").GetInt32()); + Assert.Equal("markdown_heading_name_match", suppressionReason.GetProperty("reason").GetString()); + Assert.Equal(3, suppressionReason.GetProperty("edges_affected").GetInt32()); + Assert.Equal(2, suppressionReason.GetProperty("edges_removed").GetInt32()); + Assert.Equal(6_000, suppressionReason.GetProperty("references_removed").GetInt32()); + } + [Fact] public void RunDeps_CyclesPushesNoiseFilterBeforeCandidateLimit_Issues4114And4619() { @@ -6192,6 +6272,29 @@ private static void InsertFileWithReference(string dbPath, string path, string s private static void InsertFileWithReferences(string dbPath, string path, IReadOnlyList symbolNames) => InsertFileWithSymbolsAndReferences(dbPath, path, [], symbolNames); + private static void InsertLegacyMarkdownHeadingReferences( + string dbPath, + string sourcePath, + string headingName, + int count) + { + using var db = new DbContext(DbOpenIntent.WriteIndex, dbPath); + using var sourceFile = db.Connection.CreateCommand(); + sourceFile.CommandText = "SELECT id FROM files WHERE path = $path"; + sourceFile.Parameters.AddWithValue("$path", sourcePath); + var sourceFileId = Convert.ToInt64(sourceFile.ExecuteScalar()); + var writer = new DbWriter(db.Connection); + writer.InsertReferences(Enumerable.Range(0, count).Select(index => new ReferenceRecord + { + FileId = sourceFileId, + SymbolName = headingName, + ReferenceKind = "use", + Line = index + 10, + Column = 1, + Context = $"legacy heading reference {index}", + }).ToArray()); + } + private static void MarkDependencyGraphReady(string dbPath) { using var db = new DbContext(DbOpenIntent.WriteIndex, dbPath); From c9be5e3f48e1f9e10d2199a1341efdad99c07c30 Mon Sep 17 00:00:00 2001 From: Widthdom Date: Wed, 29 Jul 2026 06:58:40 +0900 Subject: [PATCH 2/3] Address dependency review edge cases (#4868) --- .../Cli/QueryCommandRunner.Dependencies.cs | 138 ++++++++++++++-- .../Database/DbReader.Dependencies.cs | 17 +- src/CodeIndex/Models/QueryResults.cs | 2 + .../QueryCommandRunnerTests.cs | 149 ++++++++++++++++++ 4 files changed, 286 insertions(+), 20 deletions(-) diff --git a/src/CodeIndex/Cli/QueryCommandRunner.Dependencies.cs b/src/CodeIndex/Cli/QueryCommandRunner.Dependencies.cs index 0a531de6d..e74b4f2b2 100644 --- a/src/CodeIndex/Cli/QueryCommandRunner.Dependencies.cs +++ b/src/CodeIndex/Cli/QueryCommandRunner.Dependencies.cs @@ -1460,7 +1460,7 @@ private static DependencySymbolFilterResult ApplyDependencySymbolFilters(IReadOn } } - var symbols = SplitDependencySymbols(edge.Symbols); + var symbols = GetDependencySymbols(edge); symbolsBefore += symbols.Count; var keptSymbols = symbols .Where(symbol => KeepDependencySymbol(symbol, options, preservesExplicitMarkdownLink)) @@ -1475,7 +1475,7 @@ private static DependencySymbolFilterResult ApplyDependencySymbolFilters(IReadOn referencesAfter += referenceCount; filteredEdges.Add(CopyDependencyEdge( edge, - string.Join(",", keptSymbols), + keptSymbols, referenceCount, keptEvidence)); } @@ -1527,9 +1527,12 @@ private static bool KeepDependencySymbol( private static List SplitDependencySymbols(string symbols) => symbols.Split(',', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries).ToList(); + private static List GetDependencySymbols(FileDependencyResult edge) + => edge.SymbolSamples?.ToList() ?? SplitDependencySymbols(edge.Symbols); + private static FileDependencyResult CopyDependencyEdge( FileDependencyResult edge, - string symbols, + List symbolSamples, int referenceCount, List evidence) => new() @@ -1540,8 +1543,9 @@ private static FileDependencyResult CopyDependencyEdge( SourceDb = edge.SourceDb, TargetDb = edge.TargetDb, ReferenceCount = referenceCount, - RankingScore = DependencyNoiseProfile.ComputeRankingScore(referenceCount, symbols), - Symbols = symbols, + RankingScore = DependencyNoiseProfile.ComputeRankingScore(referenceCount, string.Join(",", symbolSamples)), + Symbols = string.Join(",", symbolSamples), + SymbolSamples = symbolSamples, Evidence = evidence, }; @@ -1616,8 +1620,8 @@ private static void AddDependencySymbolFilterJsonFields(JsonObject payload, Depe payload["symbol_filter"] = filter; } - private static JsonArray BuildDependencySymbolsJson(string symbols) - => new(SplitDependencySymbols(symbols).Select(symbol => JsonValue.Create(symbol)).ToArray()); + private static JsonArray BuildDependencySymbolsJson(FileDependencyResult edge) + => new(GetDependencySymbols(edge).Select(symbol => JsonValue.Create(symbol)).ToArray()); private static JsonArray BuildDependencyEvidenceJson(IReadOnlyList evidence) => new(evidence.Select(item => (JsonNode?)new JsonObject @@ -1727,7 +1731,7 @@ private static int WriteDependencyJsonGraph( ["target"] = edge.TargetPath, ["reference_count"] = edge.ReferenceCount, ["ranking_score"] = edge.RankingScore, - ["symbols"] = BuildDependencySymbolsJson(edge.Symbols), + ["symbols"] = BuildDependencySymbolsJson(edge), ["evidence"] = BuildDependencyEvidenceJson(edge.Evidence ?? []), }).ToArray()); } @@ -1823,12 +1827,18 @@ private static List GetWorkspaceFileDependencyCycleCandida options.DependencySuppressNoise); candidateRowCount += primaryCandidateRows; if (options.WorkspaceDbPaths.Count == 0) - return results.Take(limit).ToList(); + { + if (!options.DependencySuppressNoise) + return results.Take(limit).ToList(); + + candidateRowCount = results.Count(HasRetainedDependencyEvidence); + return OrderWorkspaceCycleCandidates(results, limit); + } var memberDbs = BuildWorkspaceDependencyDatabaseList(options); var primaryDb = memberDbs[0]; TagFileDependencyResults(results, primaryDb); - if (results.Count >= limit) + if (!options.DependencySuppressNoise && results.Count >= limit) return results.Take(limit).ToList(); foreach (var normalizedDbPath in memberDbs.Skip(1)) { @@ -1850,7 +1860,7 @@ private static List GetWorkspaceFileDependencyCycleCandida candidateRowCount += memberCandidateRows; TagFileDependencyResults(memberResults, normalizedDbPath); results.AddRange(memberResults); - if (results.Count >= limit) + if (!options.DependencySuppressNoise && results.Count >= limit) return results.Take(limit).ToList(); } @@ -1863,13 +1873,35 @@ private static List GetWorkspaceFileDependencyCycleCandida var crossDbResults = GetCrossDatabaseFileDependencies(sourceDb, targetDb, options, reverse, limit, cancellationToken); candidateRowCount += crossDbResults.Count; results.AddRange(crossDbResults); - if (results.Count >= limit) + if (!options.DependencySuppressNoise && results.Count >= limit) return results.Take(limit).ToList(); } - return results.Take(limit).ToList(); + if (!options.DependencySuppressNoise) + return results.Take(limit).ToList(); + + candidateRowCount = results.Count(HasRetainedDependencyEvidence); + return OrderWorkspaceCycleCandidates(results, limit); } + private static List OrderWorkspaceCycleCandidates( + IEnumerable results, + int limit) + => results + .OrderByDescending(HasRetainedDependencyEvidence) + .ThenByDescending(result => result.RankingScore) + .ThenByDescending(result => result.ReferenceCount) + .ThenBy(result => result.SourceDb, StringComparer.Ordinal) + .ThenBy(result => result.SourcePath, StringComparer.Ordinal) + .ThenBy(result => result.TargetDb, StringComparer.Ordinal) + .ThenBy(result => result.TargetPath, StringComparer.Ordinal) + .Take(limit) + .ToList(); + + private static bool HasRetainedDependencyEvidence(FileDependencyResult result) + => result.Evidence is not { Count: > 0 } + || result.Evidence.Any(static evidence => evidence.Origin != "markdown_heading_name_match"); + internal static List BuildWorkspaceDependencyDatabaseList(QueryCommandOptions options) { var primaryDb = Path.GetFullPath(DbPathResolver.NormalizeDbPath(options.DbPath)); @@ -1952,6 +1984,22 @@ private static List GetCrossDatabaseFileDependencies( using var targetDb = new DbContext(DbOpenIntent.QueryOnly, targetDbPath, cancellationToken); using var sourceDb = new DbContext(DbOpenIntent.QueryOnly, sourceDbPath, cancellationToken); var connection = sourceDb.Connection; + var sourceReader = new DbReader(sourceDb); + var targetReader = new DbReader(targetDb); + var sourceProjectRoot = sourceReader.GetIndexedProjectRoot(); + var targetProjectRoot = targetReader.GetIndexedProjectRoot(); + var hasTargetQualifier = sourceDb.SchemaCache.GetColumns("symbol_references").Contains("target_qualifier"); + connection.CreateFunction( + "markdown_cross_database_path_matches", + (string? sourcePath, string? targetPath, string? targetQualifier) + => MarkdownCrossDatabasePathMatches( + sourceProjectRoot, + sourcePath, + targetProjectRoot, + targetPath, + targetQualifier) + ? 1 + : 0); // Keep the target context alive for the whole attached query. WAL-backed targets may // resolve to a private artifact-preserving snapshot whose cleanup is owned by that // context; attaching the original path would let SQLite create/touch source sidecars. @@ -1969,6 +2017,11 @@ private static List GetCrossDatabaseFileDependencies( var retainedCrossSymbolFilterSql = options.DependencySuppressNoise ? " WHERE origin <> 'markdown_heading_name_match'" : string.Empty; + var crossMarkdownExplicitLinkSql = hasTargetQualifier + ? "(src.lang = 'markdown' AND r.reference_kind = 'reference' AND r.target_qualifier IS NOT NULL AND markdown_cross_database_path_matches(src.path, dst.path, r.target_qualifier) = 1)" + : "(0 = 1)"; + var crossMarkdownNoiseEvidenceSql = + $"({crossMarkdownExplicitLinkSql} OR (src.lang = 'markdown' AND s.kind = 'heading'))"; cmd.CommandText = $@" WITH edges AS ( SELECT {sourcePathExpr} AS source_path, @@ -1976,6 +2029,8 @@ WITH edges AS ( r.symbol_name, src.lang AS source_lang, CASE + WHEN {crossMarkdownExplicitLinkSql} + THEN 'markdown_explicit_link' WHEN src.lang = 'markdown' AND s.kind = 'heading' THEN 'markdown_heading_name_match' ELSE 'cross_database_symbol_name_match' @@ -2017,7 +2072,7 @@ FROM symbol_references r dependencySymbolFamilies: null, suppressDependencyNoise: options.DependencySuppressNoise, parameterPrefix: "crossDependencyNoise", - filterScopeSql: "NOT (src.lang = 'markdown' AND s.kind = 'heading')"); + filterScopeSql: $"NOT {crossMarkdownNoiseEvidenceSql}"); cmd.CommandText = crossDatabaseSql; cmd.CommandText += @" ), @@ -2077,7 +2132,7 @@ FROM distinct_edge_symbols SELECT edge_totals.source_path, edge_totals.target_path, edge_totals.reference_count, - COALESCE(GROUP_CONCAT(CASE WHEN ranked_edge_symbols.symbol_rank <= @symbolSampleLimit THEN ranked_edge_symbols.symbol_name END), '') AS symbols, + COALESCE(GROUP_CONCAT(CASE WHEN ranked_edge_symbols.symbol_rank <= @symbolSampleLimit THEN ranked_edge_symbols.symbol_name END, char(31)), '') AS symbols, COALESCE(edge_evidence_payloads.evidence_payload, '') AS evidence_payload FROM edge_totals LEFT JOIN ranked_edge_symbols @@ -2103,6 +2158,9 @@ ORDER BY " + crossReferenceOrderSql + @" while (reader.TrackedRead()) { cancellationToken.ThrowIfCancellationRequested(); + var symbolSamples = reader.IsDBNull(3) + ? [] + : DbReader.ParseDependencySymbols(reader.GetString(3)); results.Add(new FileDependencyResult { SourcePath = reader.GetString(0), @@ -2110,7 +2168,8 @@ ORDER BY " + crossReferenceOrderSql + @" SourceDb = reverse ? targetDbPath : sourceDbPath, TargetDb = reverse ? sourceDbPath : targetDbPath, ReferenceCount = reader.GetInt32(2), - Symbols = reader.IsDBNull(3) ? string.Empty : reader.GetString(3), + SymbolSamples = symbolSamples, + Symbols = string.Join(",", symbolSamples), Evidence = DbReader.ParseDependencyEvidence(reader.GetString(4)), }); } @@ -2140,6 +2199,53 @@ ORDER BY " + crossReferenceOrderSql + @" .ToList(); } + private static bool MarkdownCrossDatabasePathMatches( + string? sourceProjectRoot, + string? sourcePath, + string? targetProjectRoot, + string? targetPath, + string? targetQualifier) + { + if (string.IsNullOrWhiteSpace(sourceProjectRoot) + || string.IsNullOrWhiteSpace(sourcePath) + || string.IsNullOrWhiteSpace(targetProjectRoot) + || string.IsNullOrWhiteSpace(targetPath) + || string.IsNullOrWhiteSpace(targetQualifier)) + return false; + + var qualifier = targetQualifier.Replace('\\', '/').Trim(); + var fragmentIndex = qualifier.IndexOf('#', StringComparison.Ordinal); + if (fragmentIndex >= 0) + qualifier = qualifier[..fragmentIndex]; + var queryIndex = qualifier.IndexOf('?', StringComparison.Ordinal); + if (queryIndex >= 0) + qualifier = qualifier[..queryIndex]; + if (qualifier.Length == 0 + || qualifier.Contains("://", StringComparison.Ordinal) + || qualifier.StartsWith("mailto:", StringComparison.OrdinalIgnoreCase)) + return false; + + try + { + var sourceDirectory = Path.GetDirectoryName(sourcePath.Replace('/', Path.DirectorySeparatorChar)) + ?? string.Empty; + var resolvedSourceTarget = qualifier.StartsWith("/", StringComparison.Ordinal) + ? Path.GetFullPath(Path.Combine(sourceProjectRoot, qualifier.TrimStart('/'))) + : Path.GetFullPath(Path.Combine( + sourceProjectRoot, + sourceDirectory, + qualifier.Replace('/', Path.DirectorySeparatorChar))); + var resolvedTarget = Path.GetFullPath(Path.Combine( + targetProjectRoot, + targetPath.Replace('/', Path.DirectorySeparatorChar))); + return PathCasing.PathsEqual(resolvedSourceTarget, resolvedTarget); + } + catch (Exception ex) when (ex is ArgumentException or NotSupportedException or PathTooLongException) + { + return false; + } + } + private static void AttachCrossDatabaseTarget(SqliteConnection connection, string targetDbPath) { try diff --git a/src/CodeIndex/Database/DbReader.Dependencies.cs b/src/CodeIndex/Database/DbReader.Dependencies.cs index b73c4b7fa..32888f746 100644 --- a/src/CodeIndex/Database/DbReader.Dependencies.cs +++ b/src/CodeIndex/Database/DbReader.Dependencies.cs @@ -958,7 +958,7 @@ FROM distinct_edge_symbols SELECT limited_edge_totals.source_path, limited_edge_totals.target_path, limited_edge_totals.reference_count, - COALESCE(GROUP_CONCAT(CASE WHEN ranked_edge_symbols.symbol_rank <= @symbolSampleLimit THEN ranked_edge_symbols.symbol_name END), '') AS symbols, + COALESCE(GROUP_CONCAT(CASE WHEN ranked_edge_symbols.symbol_rank <= @symbolSampleLimit THEN ranked_edge_symbols.symbol_name END, char(31)), '') AS symbols, COALESCE(edge_evidence_payloads.evidence_payload, '') AS evidence_payload FROM limited_edge_totals LEFT JOIN ranked_edge_symbols @@ -994,12 +994,14 @@ LEFT JOIN edge_evidence_payloads while (reader.TrackedRead()) { cancellationToken.ThrowIfCancellationRequested(); + var symbolSamples = ParseDependencySymbols(reader.GetString(3)); results.Add(new FileDependencyResult { SourcePath = reader.GetString(0), TargetPath = reader.GetString(1), ReferenceCount = reader.GetInt32(2), - Symbols = reader.GetString(3), + SymbolSamples = symbolSamples, + Symbols = string.Join(",", symbolSamples), Evidence = ParseDependencyEvidence(reader.GetString(4)), }); } @@ -1036,6 +1038,11 @@ internal static List ParseDependencyEvidence(string payl return evidence; } + internal static List ParseDependencySymbols(string payload) + => string.IsNullOrEmpty(payload) + ? [] + : payload.Split('\u001f', StringSplitOptions.RemoveEmptyEntries).ToList(); + private static List RankDependencyResults( List results, int limit, @@ -1252,7 +1259,7 @@ FROM distinct_edge_symbols SELECT edge_reference_totals.source_path, edge_reference_totals.target_path, edge_reference_totals.reference_count, - COALESCE(GROUP_CONCAT(CASE WHEN symbol_rank <= @symbolSampleLimit THEN symbol_name END), '') AS symbols, + COALESCE(GROUP_CONCAT(CASE WHEN symbol_rank <= @symbolSampleLimit THEN symbol_name END, char(31)), '') AS symbols, COALESCE(edge_evidence_payloads.evidence_payload, '') AS evidence_payload FROM edge_reference_totals LEFT JOIN ranked_edge_symbols @@ -1286,13 +1293,15 @@ LEFT JOIN edge_evidence_payloads { cancellationToken.ThrowIfCancellationRequested(); candidateRowCount++; + var symbolSamples = ParseDependencySymbols(reader.GetString(3)); results.Add(new FileDependencyResult { SourcePath = reader.GetString(0), TargetPath = reader.GetString(1), ReferenceCount = reader.GetInt32(2), RankingScore = reader.GetInt32(2), - Symbols = reader.GetString(3), + SymbolSamples = symbolSamples, + Symbols = string.Join(",", symbolSamples), Evidence = ParseDependencyEvidence(reader.GetString(4)), }); } diff --git a/src/CodeIndex/Models/QueryResults.cs b/src/CodeIndex/Models/QueryResults.cs index a3b54bf62..54447aecc 100644 --- a/src/CodeIndex/Models/QueryResults.cs +++ b/src/CodeIndex/Models/QueryResults.cs @@ -2248,6 +2248,8 @@ public class FileDependencyResult [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)] public double RankingScore { get; set; } public string Symbols { get; set; } = string.Empty; + [JsonIgnore] + public List? SymbolSamples { get; set; } [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public List? Evidence { get; set; } } diff --git a/tests/CodeIndex.Tests/QueryCommandRunnerTests.cs b/tests/CodeIndex.Tests/QueryCommandRunnerTests.cs index aff21723d..469645791 100644 --- a/tests/CodeIndex.Tests/QueryCommandRunnerTests.cs +++ b/tests/CodeIndex.Tests/QueryCommandRunnerTests.cs @@ -5817,6 +5817,148 @@ public void RunDeps_SuppressNoiseRemovesLegacyMarkdownHeadingEvidenceButKeepsExp Assert.Equal(6_000, suppressionReason.GetProperty("references_removed").GetInt32()); } + [Fact] + public void RunDeps_SymbolFilterAndJsonGraphKeepCommaInExplicitMarkdownLink_Issue4868() + { + using var project = TestProjectHelper.CreateTempProjectScope("cdidx_deps_markdown_comma_link"); + var dbPath = TestProjectHelper.CreateProjectDb(project.Root); + TestProjectHelper.InsertIndexedFile( + dbPath, + "docs/source.md", + "markdown", + "# Source\n\n[target](target,file.md#error)\n"); + TestProjectHelper.InsertIndexedFile( + dbPath, + "docs/target,file.md", + "markdown", + "# error\n"); + MarkDependencyGraphReady(dbPath); + + var symbol = "target,file.md#error"; + var (jsonExitCode, jsonStdout, jsonStderr) = CaptureConsole(() => QueryCommandRunner.RunDeps( + ["--db", dbPath, "--json", "--lang", "markdown", "--suppress-noise", "--symbol", symbol], + _jsonOptions)); + + using var jsonDocument = ParseJsonOutput(jsonStdout); + var jsonEdge = Assert.Single(jsonDocument.RootElement.GetProperty("edges").EnumerateArray()); + + Assert.Equal(CommandExitCodes.Success, jsonExitCode); + Assert.Equal(string.Empty, jsonStderr); + Assert.Equal(symbol, jsonEdge.GetProperty("symbols").GetString()); + + var (graphExitCode, graphStdout, graphStderr) = CaptureConsole(() => QueryCommandRunner.RunDeps( + ["--db", dbPath, "--format", "json-graph", "--lang", "markdown", "--suppress-noise", "--symbol", symbol], + _jsonOptions)); + + using var graphDocument = ParseJsonOutput(graphStdout); + var graphEdge = Assert.Single(graphDocument.RootElement.GetProperty("edges").EnumerateArray()); + var graphSymbol = Assert.Single(graphEdge.GetProperty("symbols").EnumerateArray()); + + Assert.Equal(CommandExitCodes.Success, graphExitCode); + Assert.Equal(string.Empty, graphStderr); + Assert.Equal(symbol, graphSymbol.GetString()); + } + + [Fact] + public void RunDeps_WorkspaceSuppressNoiseKeepsPathQualifiedMarkdownLink_Issue4868() + { + using var workspace = TestProjectHelper.CreateTempProjectScope("cdidx_deps_markdown_cross_db_link"); + var sourceRoot = Path.Combine(workspace.Root, "a"); + var targetRoot = Path.Combine(workspace.Root, "b"); + Directory.CreateDirectory(sourceRoot); + Directory.CreateDirectory(targetRoot); + var sourceDb = TestProjectHelper.CreateProjectDb(sourceRoot); + var targetDb = TestProjectHelper.CreateProjectDb(targetRoot); + TestProjectHelper.InsertIndexedFile( + sourceDb, + "source.md", + "markdown", + "# Source\n\n[target](../b/target.md#error)\n"); + TestProjectHelper.InsertIndexedFile( + targetDb, + "target.md", + "markdown", + "# error\n"); + SetIndexedProjectRoot(sourceDb, sourceRoot); + SetIndexedProjectRoot(targetDb, targetRoot); + MarkDependencyGraphReady(sourceDb); + MarkDependencyGraphReady(targetDb); + + var (exitCode, stdout, stderr) = CaptureConsole(() => QueryCommandRunner.RunDeps( + [ + "--db", sourceDb, + "--workspace-db", targetDb, + "--json", + "--lang", "markdown", + "--suppress-noise", + ], + _jsonOptions)); + + using var document = ParseJsonOutput(stdout); + var edge = Assert.Single(document.RootElement.GetProperty("edges").EnumerateArray()); + var evidence = Assert.Single(edge.GetProperty("evidence").EnumerateArray()); + + Assert.Equal(CommandExitCodes.Success, exitCode); + Assert.Equal(string.Empty, stderr); + Assert.Equal("source.md", edge.GetProperty("source_path").GetString()); + Assert.Equal("target.md", edge.GetProperty("target_path").GetString()); + Assert.Equal("markdown_explicit_link", evidence.GetProperty("origin").GetString()); + Assert.Equal("reference", evidence.GetProperty("reference_kind").GetString()); + } + + [Fact] + public void RunDeps_WorkspaceCyclesApplyBudgetAfterMarkdownNoiseSuppression_Issue4868() + { + using var primaryProject = TestProjectHelper.CreateTempProjectScope("cdidx_deps_markdown_cycle_noise_primary"); + using var memberProject = TestProjectHelper.CreateTempProjectScope("cdidx_deps_markdown_cycle_member"); + var primaryDb = TestProjectHelper.CreateProjectDb(primaryProject.Root); + var memberDb = TestProjectHelper.CreateProjectDb(memberProject.Root); + TestProjectHelper.InsertIndexedFile(primaryDb, "source.md", "markdown", "# source\n"); + foreach (var heading in new[] { "noise one", "noise two", "noise three" }) + { + var targetPath = heading.Replace(' ', '-') + ".md"; + TestProjectHelper.InsertIndexedFile(primaryDb, targetPath, "markdown", $"# {heading}\n"); + InsertLegacyMarkdownHeadingReferences(primaryDb, "source.md", heading, count: 1); + } + TestProjectHelper.InsertIndexedFile( + memberDb, + "one.md", + "markdown", + "# one\n\n[two](two.md#two)\n"); + TestProjectHelper.InsertIndexedFile( + memberDb, + "two.md", + "markdown", + "# two\n\n[one](one.md#one)\n"); + MarkDependencyGraphReady(primaryDb); + MarkDependencyGraphReady(memberDb); + + var (exitCode, stdout, stderr) = CaptureConsole(() => QueryCommandRunner.RunDeps( + [ + "--db", primaryDb, + "--workspace-db", memberDb, + "--json", + "--cycles", + "--graph-budget", "2", + "--limit", "10", + "--lang", "markdown", + "--suppress-noise", + ], + _jsonOptions)); + + using var document = ParseJsonOutput(stdout); + var json = document.RootElement; + var cycle = Assert.Single(json.GetProperty("cycles").EnumerateArray()); + var nodes = cycle.GetProperty("nodes").EnumerateArray().Select(node => node.GetString()).ToArray(); + + Assert.Equal(CommandExitCodes.Success, exitCode); + Assert.Equal(string.Empty, stderr); + Assert.Equal(["one.md", "two.md"], nodes); + Assert.True(json.GetProperty("analysis_complete").GetBoolean()); + Assert.False(json.GetProperty("truncated").GetBoolean()); + Assert.Equal(2, json.GetProperty("graph_edge_count").GetInt32()); + } + [Fact] public void RunDeps_CyclesPushesNoiseFilterBeforeCandidateLimit_Issues4114And4619() { @@ -6303,6 +6445,13 @@ private static void MarkDependencyGraphReady(string dbPath) writer.MarkCSharpSymbolNameContractReady(); } + private static void SetIndexedProjectRoot(string dbPath, string projectRoot) + { + using var db = new DbContext(DbOpenIntent.WriteIndex, dbPath); + var writer = new DbWriter(db.Connection); + writer.SetMeta(DbContext.IndexedProjectRootMetaKey, Path.GetFullPath(projectRoot)); + } + private static void CreatePlainSqliteDatabase(string dbPath) { var builder = new SqliteConnectionStringBuilder { DataSource = dbPath }; From 3f390cb63fb65218e6780cef06edaae9aace480b Mon Sep 17 00:00:00 2001 From: Widthdom Date: Wed, 29 Jul 2026 07:29:48 +0900 Subject: [PATCH 3/3] Bound reviewed dependency aggregation (#4868) --- changelog.d/unreleased/4868.fixed.md | 1 + .../Cli/QueryCommandRunner.Dependencies.cs | 96 +++++++++++++++++-- .../Database/DbReader.Dependencies.cs | 4 +- .../Models/DependencyNoiseProfile.cs | 12 ++- .../QueryCommandRunnerTests.cs | 60 +++++++++++- 5 files changed, 157 insertions(+), 16 deletions(-) diff --git a/changelog.d/unreleased/4868.fixed.md b/changelog.d/unreleased/4868.fixed.md index 404891a1c..f8b392f8c 100644 --- a/changelog.d/unreleased/4868.fixed.md +++ b/changelog.d/unreleased/4868.fixed.md @@ -3,6 +3,7 @@ category: fixed issues: - 4868 affected: + - src/CodeIndex/Models/DependencyNoiseProfile.cs - src/CodeIndex/Models/QueryResults.cs - src/CodeIndex/Database/DbReader.Dependencies.cs - src/CodeIndex/Cli/QueryCommandRunner.Dependencies.cs diff --git a/src/CodeIndex/Cli/QueryCommandRunner.Dependencies.cs b/src/CodeIndex/Cli/QueryCommandRunner.Dependencies.cs index e74b4f2b2..a799fad8d 100644 --- a/src/CodeIndex/Cli/QueryCommandRunner.Dependencies.cs +++ b/src/CodeIndex/Cli/QueryCommandRunner.Dependencies.cs @@ -1543,7 +1543,7 @@ private static FileDependencyResult CopyDependencyEdge( SourceDb = edge.SourceDb, TargetDb = edge.TargetDb, ReferenceCount = referenceCount, - RankingScore = DependencyNoiseProfile.ComputeRankingScore(referenceCount, string.Join(",", symbolSamples)), + RankingScore = DependencyNoiseProfile.ComputeRankingScore(referenceCount, symbolSamples), Symbols = string.Join(",", symbolSamples), SymbolSamples = symbolSamples, Evidence = evidence, @@ -1838,8 +1838,29 @@ private static List GetWorkspaceFileDependencyCycleCandida var memberDbs = BuildWorkspaceDependencyDatabaseList(options); var primaryDb = memberDbs[0]; TagFileDependencyResults(results, primaryDb); - if (!options.DependencySuppressNoise && results.Count >= limit) + List? retainedResults = null; + List? suppressedResults = null; + if (options.DependencySuppressNoise) + { + candidateRowCount = 0; + retainedResults = []; + suppressedResults = []; + if (AddBoundedWorkspaceCycleCandidates( + results, + retainedResults, + suppressedResults, + limit)) + { + candidateRowCount = retainedResults.Count; + return OrderWorkspaceCycleCandidates(retainedResults, limit); + } + results.Clear(); + } + else if (results.Count >= limit) + { return results.Take(limit).ToList(); + } + foreach (var normalizedDbPath in memberDbs.Skip(1)) { cancellationToken.ThrowIfCancellationRequested(); @@ -1857,9 +1878,24 @@ private static List GetWorkspaceFileDependencyCycleCandida options.DependencySymbols, options.DependencySymbolFamilies, options.DependencySuppressNoise); - candidateRowCount += memberCandidateRows; TagFileDependencyResults(memberResults, normalizedDbPath); - results.AddRange(memberResults); + if (options.DependencySuppressNoise) + { + if (AddBoundedWorkspaceCycleCandidates( + memberResults, + retainedResults!, + suppressedResults!, + limit)) + { + candidateRowCount = retainedResults!.Count; + return OrderWorkspaceCycleCandidates(retainedResults, limit); + } + } + else + { + candidateRowCount += memberCandidateRows; + results.AddRange(memberResults); + } if (!options.DependencySuppressNoise && results.Count >= limit) return results.Take(limit).ToList(); } @@ -1871,8 +1907,23 @@ private static List GetWorkspaceFileDependencyCycleCandida if (string.Equals(sourceDb, targetDb, StringComparison.Ordinal)) continue; var crossDbResults = GetCrossDatabaseFileDependencies(sourceDb, targetDb, options, reverse, limit, cancellationToken); - candidateRowCount += crossDbResults.Count; - results.AddRange(crossDbResults); + if (options.DependencySuppressNoise) + { + if (AddBoundedWorkspaceCycleCandidates( + crossDbResults, + retainedResults!, + suppressedResults!, + limit)) + { + candidateRowCount = retainedResults!.Count; + return OrderWorkspaceCycleCandidates(retainedResults, limit); + } + } + else + { + candidateRowCount += crossDbResults.Count; + results.AddRange(crossDbResults); + } if (!options.DependencySuppressNoise && results.Count >= limit) return results.Take(limit).ToList(); } @@ -1880,8 +1931,33 @@ private static List GetWorkspaceFileDependencyCycleCandida if (!options.DependencySuppressNoise) return results.Take(limit).ToList(); - candidateRowCount = results.Count(HasRetainedDependencyEvidence); - return OrderWorkspaceCycleCandidates(results, limit); + candidateRowCount = retainedResults!.Count; + return OrderWorkspaceCycleCandidates(retainedResults.Concat(suppressedResults!), limit); + } + + internal static bool AddBoundedWorkspaceCycleCandidates( + IEnumerable candidates, + List retainedResults, + List suppressedResults, + int limit) + { + foreach (var candidate in candidates) + { + if (HasRetainedDependencyEvidence(candidate)) + { + if (retainedResults.Count < limit) + retainedResults.Add(candidate); + } + else if (suppressedResults.Count < limit) + { + suppressedResults.Add(candidate); + } + + if (retainedResults.Count >= limit) + return true; + } + + return false; } private static List OrderWorkspaceCycleCandidates( @@ -2185,7 +2261,9 @@ ORDER BY " + crossReferenceOrderSql + @" .Where(static evidence => evidence.Origin != "markdown_heading_name_match") .Sum(static evidence => evidence.ReferenceCount) : result.ReferenceCount; - result.RankingScore = DependencyNoiseProfile.ComputeRankingScore(rankingReferenceCount, result.Symbols); + result.RankingScore = result.SymbolSamples is { } symbolSamples + ? DependencyNoiseProfile.ComputeRankingScore(rankingReferenceCount, symbolSamples) + : DependencyNoiseProfile.ComputeRankingScore(rankingReferenceCount, result.Symbols); } return results diff --git a/src/CodeIndex/Database/DbReader.Dependencies.cs b/src/CodeIndex/Database/DbReader.Dependencies.cs index 32888f746..f0e502e40 100644 --- a/src/CodeIndex/Database/DbReader.Dependencies.cs +++ b/src/CodeIndex/Database/DbReader.Dependencies.cs @@ -1055,7 +1055,9 @@ private static List RankDependencyResults( .Where(static evidence => evidence.Origin != "markdown_heading_name_match") .Sum(static evidence => evidence.ReferenceCount) : result.ReferenceCount; - result.RankingScore = DependencyNoiseProfile.ComputeRankingScore(rankingReferenceCount, result.Symbols); + result.RankingScore = result.SymbolSamples is { } symbolSamples + ? DependencyNoiseProfile.ComputeRankingScore(rankingReferenceCount, symbolSamples) + : DependencyNoiseProfile.ComputeRankingScore(rankingReferenceCount, result.Symbols); } return results diff --git a/src/CodeIndex/Models/DependencyNoiseProfile.cs b/src/CodeIndex/Models/DependencyNoiseProfile.cs index ec7979947..bba97d4c8 100644 --- a/src/CodeIndex/Models/DependencyNoiseProfile.cs +++ b/src/CodeIndex/Models/DependencyNoiseProfile.cs @@ -76,15 +76,21 @@ internal static int GetRankingCandidateLimit(int limit) } internal static double ComputeRankingScore(int referenceCount, string symbols) + => ComputeRankingScore( + referenceCount, + symbols.Split(',', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries)); + + internal static double ComputeRankingScore( + int referenceCount, + IReadOnlyList symbolNames) { - var symbolNames = symbols.Split(',', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries); - if (symbolNames.Length == 0) + if (symbolNames.Count == 0) return referenceCount; var penaltySum = 0.0; foreach (var symbolName in symbolNames) penaltySum += IsNoiseSymbol(symbolName) ? SymbolRankPenalty : 1.0; - return referenceCount * penaltySum / symbolNames.Length; + return referenceCount * penaltySum / symbolNames.Count; } } diff --git a/tests/CodeIndex.Tests/QueryCommandRunnerTests.cs b/tests/CodeIndex.Tests/QueryCommandRunnerTests.cs index 469645791..619c76a57 100644 --- a/tests/CodeIndex.Tests/QueryCommandRunnerTests.cs +++ b/tests/CodeIndex.Tests/QueryCommandRunnerTests.cs @@ -5826,15 +5826,32 @@ public void RunDeps_SymbolFilterAndJsonGraphKeepCommaInExplicitMarkdownLink_Issu dbPath, "docs/source.md", "markdown", - "# Source\n\n[target](target,file.md#error)\n"); + "# Source\n\n[comma](Path,target.md#error)\n[plain](z-target.md#error)\n"); TestProjectHelper.InsertIndexedFile( dbPath, - "docs/target,file.md", + "docs/Path,target.md", + "markdown", + "# error\n"); + TestProjectHelper.InsertIndexedFile( + dbPath, + "docs/z-target.md", "markdown", "# error\n"); MarkDependencyGraphReady(dbPath); - var symbol = "target,file.md#error"; + var (rankingExitCode, rankingStdout, rankingStderr) = CaptureConsole(() => QueryCommandRunner.RunDeps( + ["--db", dbPath, "--json", "--lang", "markdown", "--suppress-noise", "--limit", "1"], + _jsonOptions)); + + using var rankingDocument = ParseJsonOutput(rankingStdout); + var rankingEdge = Assert.Single(rankingDocument.RootElement.GetProperty("edges").EnumerateArray()); + + Assert.Equal(CommandExitCodes.Success, rankingExitCode); + Assert.Equal(string.Empty, rankingStderr); + Assert.Equal("docs/Path,target.md", rankingEdge.GetProperty("target_path").GetString()); + Assert.Equal(1.0, rankingEdge.GetProperty("ranking_score").GetDouble()); + + var symbol = "Path,target.md#error"; var (jsonExitCode, jsonStdout, jsonStderr) = CaptureConsole(() => QueryCommandRunner.RunDeps( ["--db", dbPath, "--json", "--lang", "markdown", "--suppress-noise", "--symbol", symbol], _jsonOptions)); @@ -5959,6 +5976,43 @@ public void RunDeps_WorkspaceCyclesApplyBudgetAfterMarkdownNoiseSuppression_Issu Assert.Equal(2, json.GetProperty("graph_edge_count").GetInt32()); } + [Fact] + public void AddBoundedWorkspaceCycleCandidates_CapsRetainedAndSuppressedBuckets_Issue4868() + { + static FileDependencyResult CreateCandidate(string targetPath, string origin) + => new() + { + TargetPath = targetPath, + Evidence = + [ + new FileDependencyEvidence + { + Origin = origin, + ReferenceCount = 1, + }, + ], + }; + + var candidates = Enumerable.Range(0, 1_000) + .Select(index => CreateCandidate($"noise-{index}", "markdown_heading_name_match")) + .Concat(Enumerable.Range(0, 10) + .Select(index => CreateCandidate($"retained-{index}", "markdown_explicit_link"))); + var retainedResults = new List(); + var suppressedResults = new List(); + + var limitReached = QueryCommandRunner.AddBoundedWorkspaceCycleCandidates( + candidates, + retainedResults, + suppressedResults, + limit: 3); + + Assert.True(limitReached); + Assert.Equal(3, retainedResults.Count); + Assert.Equal(3, suppressedResults.Count); + Assert.All(retainedResults, result => Assert.StartsWith("retained-", result.TargetPath)); + Assert.All(suppressedResults, result => Assert.StartsWith("noise-", result.TargetPath)); + } + [Fact] public void RunDeps_CyclesPushesNoiseFilterBeforeCandidateLimit_Issues4114And4619() {