From 59ebc51a0b594cc7487e6d17901f6e22d6702501 Mon Sep 17 00:00:00 2001 From: Widthdom Date: Fri, 12 Jun 2026 09:08:05 +0900 Subject: [PATCH 1/5] Expose language capability gaps (#3518) --- USER_GUIDE.md | 26 ++++---- changelog.d/unreleased/3518.changed.md | 19 ++++++ src/CodeIndex/Cli/CliFlagSchema.cs | 2 +- src/CodeIndex/Cli/ConsoleUi.cs | 2 +- src/CodeIndex/Cli/JsonOutputContracts.cs | 4 +- src/CodeIndex/Cli/QueryCommandRunner.cs | 65 ++++++++++++++---- src/CodeIndex/Mcp/McpServer.cs | 4 +- src/CodeIndex/Mcp/McpToolHandlers.cs | 30 +++++++-- tests/CodeIndex.Tests/McpServerTests.cs | 6 ++ .../QueryCommandRunnerTests.cs | 66 +++++++++++++++++-- 10 files changed, 187 insertions(+), 37 deletions(-) create mode 100644 changelog.d/unreleased/3518.changed.md diff --git a/USER_GUIDE.md b/USER_GUIDE.md index 30ed7283e6..492d5d1877 100644 --- a/USER_GUIDE.md +++ b/USER_GUIDE.md @@ -282,7 +282,7 @@ sections below show examples and option details for the most common workflows. | Analysis | `hotspots` | Rank high-impact symbols or statements by reference volume | `symbol_hotspots` | | Analysis | `validate` | Report encoding and line-ending issues in indexed files; U+FFFD rows include origin/severity metadata | `validate` | | Status | `status` | Show DB statistics, freshness, and readiness metadata | `status` | -| Status | `languages` | List language extensions and symbol/graph capabilities; add `--indexed-only` and `--capability graph|symbols|references` for workspace audits | `languages` | +| Status | `languages` | List language extensions and symbol/reference/graph capabilities; add `--indexed-only` and `--capability graph|references|symbols|missing-graph|missing-references|missing-symbols|search-only` for workspace audits | `languages` | | Diagnostics | `db --integrity-check` | Run SQLite `PRAGMA integrity_check` against the DB | -- | | Diagnostics | `report --output ` | Build a redacted bug-report bundle | -- | | Feedback | `suggestions` | List, inspect, and export local suggestion history | -- | @@ -1629,7 +1629,7 @@ The database reflects the working tree at the time of the last index. After swit ## Supported languages -All indexed languages are searchable through FTS5. Rows with **Symbols = yes** also support structured queries by function, class, import, or language-specific symbol name. Use `cdidx languages --indexed-only --json` to list only languages present in the current DB, and add `--capability graph|symbols|references` to narrow the table to languages that support a specific structured capability. +All indexed languages are searchable through FTS5. Rows with **Symbols = yes** also support structured queries by function, class, import, or language-specific symbol name. Use `cdidx languages --indexed-only --json` to list only languages present in the current DB; JSON rows expose `symbol_extraction`, `reference_extraction`, `graph_queries`, and `capability_gaps`. Add `--capability graph|references|symbols|missing-graph|missing-references|missing-symbols|search-only` to narrow the table to languages that support a structured capability or still have a capability gap. | Language | Extensions | Symbols | |---|---|:---:| @@ -1725,11 +1725,14 @@ All indexed languages are searchable through FTS5. Rows with **Symbols = yes** a ### Language extraction matrix -Use `cdidx languages --json` as the live capability probe. Add `--indexed-only` -when you only want languages present in the current DB, and add -`--capability graph|symbols|references` when auditing a specific structured -capability. This matrix explains the common extraction behavior so users know -when to trust structured commands and when to fall back to `search`. +Use `cdidx languages --json` as the live capability probe. JSON rows expose +`symbol_extraction`, `reference_extraction`, `graph_queries`, and +`capability_gaps`. Add `--indexed-only` when you only want languages present in +the current DB, and add +`--capability graph|references|symbols|missing-graph|missing-references|missing-symbols|search-only` +when auditing a specific structured capability or capability gap. This matrix +explains the common extraction behavior so users know when to trust structured +commands and when to fall back to `search`. | Language family | Symbols | References / graph | Notes and example query | |---|---|---|---| @@ -4075,9 +4078,10 @@ indexing はファイル単位の SQLite transaction を commit します。長 ### 言語別 extraction matrix -現在の capability は `cdidx languages --json` を live probe として確認してください。 -現在の DB に存在する言語だけを見たい場合は `--indexed-only`、特定の構造化 capability を監査する場合は -`--capability graph|symbols|references` を追加します。この matrix は、構造化 command を信頼できる場面と +現在の capability は `cdidx languages --json` を live probe として確認してください。JSON 行には +`symbol_extraction`、`reference_extraction`、`graph_queries`、`capability_gaps` が含まれます。 +現在の DB に存在する言語だけを見たい場合は `--indexed-only`、特定の構造化 capability や capability gap を監査する場合は +`--capability graph|references|symbols|missing-graph|missing-references|missing-symbols|search-only` を追加します。この matrix は、構造化 command を信頼できる場面と `search` に戻るべき場面を判断するための概要です。 | 言語ファミリ | Symbols | References / graph | メモと例 | @@ -4490,7 +4494,7 @@ OpenAI Codex CLI (`codex.json` または `~/.codex/config.json`): | `symbol_hotspots` | 影響の大きい hotspot を検索。`groupBy` は `symbol` / `file` / `statement` を指定でき、SQL scope は statement grouping、非 SQL scope は symbol grouping が既定。 | | `batch_query` | 複数クエリを1回で実行（MCP専用、最大10件）。レスポンスにはトップレベル `metadata`（`submitted` / `executed` / `errors` / `total_elapsed_ms` / `success_count` / `failure_count`）と各 `results` エントリの `request_index`、任意の client `slot_id`、`ok`、`elapsed_ms`、`summary`、`args_summary` が含まれ、位置だけに依存せず部分失敗や遅い内部クエリを把握できます。 | | `validate` | エンコーディング問題（origin/severity 付き U+FFFD、BOM、null バイト、改行混在 / CR-only 行末、UTF-16 BOM 検出、UTF-8 以外と推定されるエンコーディング）を報告 | -| `languages` | 対応言語一覧を拡張子・機能付きで表示。`--indexed-only` と `--capability graph|symbols|references` で現在の DB や機能別に絞り込み可能 | +| `languages` | 対応言語一覧を拡張子・機能付きで表示。`--indexed-only` と `--capability graph|references|symbols|missing-graph|missing-references|missing-symbols|search-only` で現在の DB、機能別、または capability gap 別に絞り込み可能 | | `ping` | 軽量な接続確認 | | `index` | プロジェクトのインデックス作成・更新 | | `backfill_fold` | 既存 DB の folded-name key をソース再解析なしで更新 | diff --git a/changelog.d/unreleased/3518.changed.md b/changelog.d/unreleased/3518.changed.md new file mode 100644 index 0000000000..04b439ec52 --- /dev/null +++ b/changelog.d/unreleased/3518.changed.md @@ -0,0 +1,19 @@ +--- +category: changed +issues: + - 3518 +affected: + - src/CodeIndex/Cli/QueryCommandRunner.cs + - src/CodeIndex/Cli/JsonOutputContracts.cs + - src/CodeIndex/Mcp/McpToolHandlers.cs + - src/CodeIndex/Mcp/McpServer.cs + - USER_GUIDE.md +--- + +## English + +- **Language capability output now separates references from graph support (#3518)** — `cdidx languages --json` and the MCP `languages` tool now expose `reference_extraction` and `capability_gaps`, and `--capability` can filter missing capability gaps or search-only languages. + +## 日本語 + +- **言語 capability 出力で references と graph 対応を分離しました (#3518)** — `cdidx languages --json` と MCP の `languages` ツールは `reference_extraction` と `capability_gaps` を返すようになり、`--capability` で未対応 gap や search-only 言語を絞り込めます。 diff --git a/src/CodeIndex/Cli/CliFlagSchema.cs b/src/CodeIndex/Cli/CliFlagSchema.cs index db8519235c..d0d2b91865 100644 --- a/src/CodeIndex/Cli/CliFlagSchema.cs +++ b/src/CodeIndex/Cli/CliFlagSchema.cs @@ -273,7 +273,7 @@ private static IReadOnlyList BuildAll() new() { Name = "--sections", ValuePlaceholder = "", Description = "Map: comma-separated response sections to include", Commands = Set(MapSectionCommands) }, new() { Name = "--cycles", Description = "Deps: return dependency cycles instead of edge rows", Commands = Set(DependencyCycleCommands) }, new() { Name = "--indexed-only", Description = "Languages: list only languages present in the current index", Commands = Set(LanguagesFilterCommands) }, - new() { Name = "--capability", ValuePlaceholder = "", Description = "Languages: filter by language capability", Commands = Set(LanguagesFilterCommands) }, + new() { Name = "--capability", ValuePlaceholder = "", Description = "Languages: filter by language capability or capability gap", Commands = Set(LanguagesFilterCommands) }, new() { Name = "--query", ValuePlaceholder = "", Description = "Literal query", Commands = Set(QueryCommands) }, new() { Name = "--recipe", ValuePlaceholder = "", Description = "Search: run a built-in audit recipe query set", Commands = Set("search") }, new() { Name = "--list-recipes", Description = "Search: list built-in audit recipes", Commands = Set("search") }, diff --git a/src/CodeIndex/Cli/ConsoleUi.cs b/src/CodeIndex/Cli/ConsoleUi.cs index 8e4d22deba..d0d70be8f2 100644 --- a/src/CodeIndex/Cli/ConsoleUi.cs +++ b/src/CodeIndex/Cli/ConsoleUi.cs @@ -117,7 +117,7 @@ private static readonly (string Command, string Usage)[] CommandUsageLines = ("export", "cdidx export [--db ] [--json]"), ("export", "cdidx export ctags [--output ] [--db ]"), ("import", "cdidx import [--db ] [--prune-paths] [--dry-run|--check] [--json]"), - ("languages", "cdidx languages [--db ] [--json] [--indexed-only] [--capability ]"), + ("languages", "cdidx languages [--db ] [--json] [--indexed-only] [--capability ]"), ("batch", "cdidx batch [--db ] # reads JSON string arrays from stdin, one query command per line; max 1,048,576 chars/line and 256 arguments"), ("mcp", "cdidx mcp [--db ] [--transport stdio|http] [--http-listen ] [--audit-log ] [--audit-log-include-values] [--audit-log-max-bytes ] [--suggestion-dedup-threshold <0..1>]"), ("lsp", "cdidx lsp [--db ]"), diff --git a/src/CodeIndex/Cli/JsonOutputContracts.cs b/src/CodeIndex/Cli/JsonOutputContracts.cs index f3c550643a..7510fba28d 100644 --- a/src/CodeIndex/Cli/JsonOutputContracts.cs +++ b/src/CodeIndex/Cli/JsonOutputContracts.cs @@ -192,7 +192,9 @@ internal sealed record LanguageEntryJsonResult( [property: JsonPropertyName("extensions")] List Extensions, [property: JsonPropertyName("aliases")] List Aliases, [property: JsonPropertyName("symbol_extraction")] bool SymbolExtraction, - [property: JsonPropertyName("graph_queries")] bool GraphQueries); + [property: JsonPropertyName("reference_extraction")] bool ReferenceExtraction, + [property: JsonPropertyName("graph_queries")] bool GraphQueries, + [property: JsonPropertyName("capability_gaps")] List CapabilityGaps); internal sealed record LanguagesJsonResult( [property: JsonPropertyName("languages")] List Languages); diff --git a/src/CodeIndex/Cli/QueryCommandRunner.cs b/src/CodeIndex/Cli/QueryCommandRunner.cs index 5bf0eeecdd..32b8f3c832 100644 --- a/src/CodeIndex/Cli/QueryCommandRunner.cs +++ b/src/CodeIndex/Cli/QueryCommandRunner.cs @@ -39,6 +39,10 @@ public static class QueryCommandRunner private const string LanguageCapabilityGraph = "graph"; private const string LanguageCapabilityReferences = "references"; private const string LanguageCapabilitySymbols = "symbols"; + private const string LanguageCapabilityMissingGraph = "missing-graph"; + private const string LanguageCapabilityMissingReferences = "missing-references"; + private const string LanguageCapabilityMissingSymbols = "missing-symbols"; + private const string LanguageCapabilitySearchOnly = "search-only"; internal static readonly TimeSpan DefaultStaleAfter = TimeSpan.FromHours(24); internal static readonly TimeSpan MaxStaleAfter = TimeSpan.FromDays(30); internal const string MaxStaleAfterDisplay = "30d"; @@ -6239,15 +6243,23 @@ public static int RunLanguages(string[] cmdArgs, JsonSerializerOptions jsonOptio var symbolLangs = SymbolExtractor.GetSupportedLanguages(); var graphLangs = ReferenceExtractor.GetSupportedLanguages(); - // Build a consolidated view: language -> (extensions, hasSymbols, hasGraph) - // 統合ビュー: 言語 -> (拡張子, シンボル対応, グラフ対応) + // Build a consolidated view: language -> capability flags and gaps. + // 統合ビュー: 言語 -> capability flag と gap。 var allLangs = new Dictionary(StringComparer.Ordinal); foreach (var (ext, lang) in langExtensions) { if (!allLangs.TryGetValue(lang, out var info)) { - info = new LanguageSupportInfo([], GetLanguageAliases(lang).ToList(), symbolLangs.Contains(lang), graphLangs.Contains(lang)); + var hasSymbols = symbolLangs.Contains(lang); + var hasReferences = graphLangs.Contains(lang); + info = new LanguageSupportInfo( + [], + GetLanguageAliases(lang).ToList(), + hasSymbols, + hasReferences, + hasReferences, + BuildLanguageCapabilityGaps(hasSymbols, hasReferences, hasReferences)); allLangs[lang] = info; } info.Extensions.Add(ext); @@ -6281,7 +6293,9 @@ int WriteLanguages(IEnumerable> langua kv.Value.Extensions.OrderBy(e => e).ToList(), kv.Value.Aliases.OrderBy(a => a).ToList(), kv.Value.Symbols, - kv.Value.Graph)).ToList(); + kv.Value.References, + kv.Value.Graph, + kv.Value.CapabilityGaps)).ToList(); Console.WriteLine(JsonSerializer.Serialize(new LanguagesJsonResult(entries), CliJsonSerializerContextFactory.Create(jsonOptions).LanguagesJsonResult)); } else @@ -6292,25 +6306,28 @@ int WriteLanguages(IEnumerable> langua // Symbols / Graph 列が拡張子文字列に埋もれないようにする。 const int ExtensionColumnWidth = 36; const int AliasColumnWidth = 12; - Console.WriteLine($"{"Language",-14} {"Extensions",-36} {"Aliases",-12} {"Symbols",-9} {"Graph",-7}"); - Console.WriteLine(new string('-', 79)); + Console.WriteLine($"{"Language",-14} {"Extensions",-36} {"Aliases",-12} {"Symbols",-9} {"Refs",-5} {"Graph",-7}"); + Console.WriteLine(new string('-', 85)); foreach (var (lang, info) in filtered) { var exts = string.Join(" ", info.Extensions.OrderBy(e => e)); var aliases = string.Join(" ", info.Aliases.OrderBy(a => a)); var aliasCell = string.IsNullOrWhiteSpace(aliases) ? "-" : aliases; var sym = info.Symbols ? "yes" : "-"; + var refs = info.References ? "yes" : "-"; var graph = info.Graph ? "yes" : "-"; if (exts.Length <= ExtensionColumnWidth && aliases.Length <= AliasColumnWidth) { - Console.WriteLine($"{lang,-14} {exts,-36} {aliasCell,-12} {sym,-9} {graph,-7}"); + Console.WriteLine($"{lang,-14} {exts,-36} {aliasCell,-12} {sym,-9} {refs,-5} {graph,-7}"); } else { - Console.WriteLine($"{lang,-14} {"",-36} {"",-12} {sym,-9} {graph,-7}"); + Console.WriteLine($"{lang,-14} {"",-36} {"",-12} {sym,-9} {refs,-5} {graph,-7}"); Console.WriteLine($" Extensions: {exts}"); if (!string.IsNullOrWhiteSpace(aliases)) Console.WriteLine($" Aliases: {aliases}"); + if (info.CapabilityGaps.Count > 0) + Console.WriteLine($" Gaps: {string.Join(", ", info.CapabilityGaps)}"); } } Console.Error.WriteLine($"\n({filtered.Count} languages)"); @@ -6320,20 +6337,44 @@ int WriteLanguages(IEnumerable> langua } } - private sealed record LanguageSupportInfo(List Extensions, List Aliases, bool Symbols, bool Graph); + private sealed record LanguageSupportInfo(List Extensions, List Aliases, bool Symbols, bool References, bool Graph, List CapabilityGaps); private static bool LanguageMatchesCapability(LanguageSupportInfo language, string capability) => capability switch { LanguageCapabilitySymbols => language.Symbols, - LanguageCapabilityGraph or LanguageCapabilityReferences => language.Graph, + LanguageCapabilityReferences => language.References, + LanguageCapabilityGraph => language.Graph, + LanguageCapabilityMissingSymbols => !language.Symbols, + LanguageCapabilityMissingReferences => !language.References, + LanguageCapabilityMissingGraph => !language.Graph, + LanguageCapabilitySearchOnly => !language.Symbols && !language.References && !language.Graph, _ => false, }; private static bool TryNormalizeLanguageCapability(string value, out string capability) { capability = value.Trim().ToLowerInvariant(); - return capability is LanguageCapabilityGraph or LanguageCapabilityReferences or LanguageCapabilitySymbols; + return capability is + LanguageCapabilityGraph or + LanguageCapabilityReferences or + LanguageCapabilitySymbols or + LanguageCapabilityMissingGraph or + LanguageCapabilityMissingReferences or + LanguageCapabilityMissingSymbols or + LanguageCapabilitySearchOnly; + } + + private static List BuildLanguageCapabilityGaps(bool symbols, bool references, bool graph) + { + var gaps = new List(); + if (!symbols) + gaps.Add("missing-symbols"); + if (!references) + gaps.Add("missing-references"); + if (!graph) + gaps.Add("missing-graph"); + return gaps; } public static QueryCommandOptions ParseArgs( @@ -6614,7 +6655,7 @@ void WarnIfDuplicateSingleValueOption(string canonicalName, string newValue) } else { - AddParseError($"Error: unsupported --capability value '{ConsoleUi.FormatBoundedValue(capabilityValue)}'. Use graph, symbols, or references."); + AddParseError($"Error: unsupported --capability value '{ConsoleUi.FormatBoundedValue(capabilityValue)}'. Use graph, references, symbols, missing-graph, missing-references, missing-symbols, or search-only."); } break; case "--format": diff --git a/src/CodeIndex/Mcp/McpServer.cs b/src/CodeIndex/Mcp/McpServer.cs index 8403e97c58..494dc3e234 100644 --- a/src/CodeIndex/Mcp/McpServer.cs +++ b/src/CodeIndex/Mcp/McpServer.cs @@ -4070,14 +4070,14 @@ private static string AppendLanguageSupportClause(string name, string descriptio "excerpt" or "status" or "validate" => $"Language support: Language-agnostic over indexed files and diagnostics for every detected language listed by `languages`: {DetectedLanguageList()}. This tool does not interpret a `lang` filter.", "languages" - => "Language support: This is the authoritative language catalog for MCP tools; it lists every detected language plus symbol_extraction and graph_queries capability flags.", + => "Language support: This is the authoritative language catalog for MCP tools; it lists every detected language plus symbol_extraction, reference_extraction, graph_queries, and capability_gaps fields.", "index" => $"Language support: Indexes every detected language listed by `languages`: {DetectedLanguageList()}, then extracts symbols and graph references only where the catalog advertises those capabilities.", "batch_query" => "Language support: Language behavior is inherited from each nested read-only tool; consult each returned payload and the `languages` tool for capabilities.", "backfill_fold" or "ping" or "suggest_improvement" => "Language support: Language-independent tool; it does not interpret `lang` filters.", - _ => "Language support: See the `languages` tool for detected languages and per-language symbol_extraction / graph_queries capabilities.", + _ => "Language support: See the `languages` tool for detected languages and per-language symbol_extraction / reference_extraction / graph_queries capabilities.", }; return $"{description} {clause}"; diff --git a/src/CodeIndex/Mcp/McpToolHandlers.cs b/src/CodeIndex/Mcp/McpToolHandlers.cs index 33498999d0..8184571bd5 100644 --- a/src/CodeIndex/Mcp/McpToolHandlers.cs +++ b/src/CodeIndex/Mcp/McpToolHandlers.cs @@ -4059,15 +4059,23 @@ private JsonNode ExecuteLanguages(JsonNode? id) { var langExtensions = FileIndexer.GetLanguageExtensions(); var symbolLangs = SymbolExtractor.GetSupportedLanguages(); - var graphLangs = ReferenceExtractor.GetSupportedLanguages(); + var referenceLangs = ReferenceExtractor.GetSupportedLanguages(); // Build consolidated language info / 統合言語情報を構築 - var allLangs = new Dictionary Extensions, List Aliases, bool Symbols, bool Graph)>(StringComparer.Ordinal); + var allLangs = new Dictionary Extensions, List Aliases, bool Symbols, bool References, bool Graph, List CapabilityGaps)>(StringComparer.Ordinal); foreach (var (ext, lang) in langExtensions) { if (!allLangs.TryGetValue(lang, out var info)) { - info = (new List(), QueryCommandRunner.GetLanguageAliases(lang).ToList(), symbolLangs.Contains(lang), graphLangs.Contains(lang)); + var hasSymbols = symbolLangs.Contains(lang); + var hasReferences = referenceLangs.Contains(lang); + info = ( + new List(), + QueryCommandRunner.GetLanguageAliases(lang).ToList(), + hasSymbols, + hasReferences, + hasReferences, + BuildLanguageCapabilityGaps(hasSymbols, hasReferences, hasReferences)); allLangs[lang] = info; } info.Extensions.Add(ext); @@ -4087,15 +4095,29 @@ private JsonNode ExecuteLanguages(JsonNode? id) ["extensions"] = extArray, ["aliases"] = new JsonArray(info.Aliases.OrderBy(alias => alias).Select(alias => JsonValue.Create(alias)).ToArray()), ["symbol_extraction"] = info.Symbols, + ["reference_extraction"] = info.References, ["graph_queries"] = info.Graph, + ["capability_gaps"] = new JsonArray(info.CapabilityGaps.Select(gap => JsonValue.Create(gap)).ToArray()), }); } var payload = new JsonObject { ["languages"] = languagesArray }; - var summary = $"{sorted.Count} languages supported. {symbolLangs.Count} with symbol extraction, {graphLangs.Count} with call-graph queries."; + var summary = $"{sorted.Count} languages supported. {symbolLangs.Count} with symbol extraction, {referenceLangs.Count} with reference extraction, {referenceLangs.Count} with call-graph queries."; return CreateToolResult(id, summary, payload); } + private static List BuildLanguageCapabilityGaps(bool symbols, bool references, bool graph) + { + var gaps = new List(); + if (!symbols) + gaps.Add("missing-symbols"); + if (!references) + gaps.Add("missing-references"); + if (!graph) + gaps.Add("missing-graph"); + return gaps; + } + private JsonNode ExecuteIndex(JsonNode? id, JsonNode? args, JsonNode? progressToken = null) => ExecuteIndexAsync(id, args, progressToken).GetAwaiter().GetResult(); diff --git a/tests/CodeIndex.Tests/McpServerTests.cs b/tests/CodeIndex.Tests/McpServerTests.cs index 41c197455c..7bae875907 100644 --- a/tests/CodeIndex.Tests/McpServerTests.cs +++ b/tests/CodeIndex.Tests/McpServerTests.cs @@ -9153,6 +9153,7 @@ public void ToolsCall_Languages_ReturnsCapabilities() // Verify a known language has the right capabilities / 既知の言語の機能を検証 var csharp = languages.First(l => l!["lang"]!.GetValue() == "csharp")!; Assert.True(csharp["symbol_extraction"]!.GetValue()); + Assert.True(csharp["reference_extraction"]!.GetValue()); Assert.True(csharp["graph_queries"]!.GetValue()); Assert.Contains(".cs", csharp["extensions"]!.AsArray().Select(e => e!.GetValue())); @@ -9166,6 +9167,7 @@ public void ToolsCall_Languages_ReturnsCapabilities() var assembly = languages.First(l => l!["lang"]!.GetValue() == "assembly")!; Assert.True(assembly["symbol_extraction"]!.GetValue()); + Assert.True(assembly["reference_extraction"]!.GetValue()); Assert.True(assembly["graph_queries"]!.GetValue()); Assert.Contains(".asm", assembly["extensions"]!.AsArray().Select(e => e!.GetValue())); Assert.Contains(".S", assembly["extensions"]!.AsArray().Select(e => e!.GetValue())); @@ -9174,10 +9176,13 @@ public void ToolsCall_Languages_ReturnsCapabilities() // Verify a detection-only language / 検出のみの言語を検証 var markdown = languages.First(l => l!["lang"]!.GetValue() == "markdown")!; Assert.True(markdown["symbol_extraction"]!.GetValue()); + Assert.False(markdown["reference_extraction"]!.GetValue()); Assert.False(markdown["graph_queries"]!.GetValue()); + Assert.Contains("missing-references", markdown["capability_gaps"]!.AsArray().Select(e => e!.GetValue())); var yaml = languages.First(l => l!["lang"]!.GetValue() == "yaml")!; Assert.Contains("yml", yaml["aliases"]!.AsArray().Select(e => e!.GetValue())); + Assert.Contains("missing-symbols", yaml["capability_gaps"]!.AsArray().Select(e => e!.GetValue())); // Pin #215: HTML must report symbol_extraction=true and list all four // extensions so AI tools discover HTML support via the MCP languages tool. @@ -9185,6 +9190,7 @@ public void ToolsCall_Languages_ReturnsCapabilities() // の 4 拡張子を MCP languages ツールから返すこと。 var html = languages.First(l => l!["lang"]!.GetValue() == "html")!; Assert.True(html["symbol_extraction"]!.GetValue()); + Assert.False(html["reference_extraction"]!.GetValue()); var htmlExtensions = html["extensions"]!.AsArray().Select(e => e!.GetValue()).ToList(); Assert.Contains(".html", htmlExtensions); Assert.Contains(".htm", htmlExtensions); diff --git a/tests/CodeIndex.Tests/QueryCommandRunnerTests.cs b/tests/CodeIndex.Tests/QueryCommandRunnerTests.cs index cf878f044b..acdfd738f3 100644 --- a/tests/CodeIndex.Tests/QueryCommandRunnerTests.cs +++ b/tests/CodeIndex.Tests/QueryCommandRunnerTests.cs @@ -2033,9 +2033,9 @@ public void RunLanguages_JsonIndexedOnlyCombinesWithCapabilityFilter() } [Theory] - [InlineData("graph")] - [InlineData("references")] - public void RunLanguages_JsonCapabilityGraphFiltersGraphSupport(string capability) + [InlineData("graph", "graph_queries")] + [InlineData("references", "reference_extraction")] + public void RunLanguages_JsonCapabilityFiltersPositiveSupport(string capability, string propertyName) { var (exitCode, stdout, stderr) = CaptureConsole(() => QueryCommandRunner.RunLanguages(["--json", "--capability", capability], _jsonOptions)); @@ -2048,7 +2048,7 @@ public void RunLanguages_JsonCapabilityGraphFiltersGraphSupport(string capabilit Assert.NotEmpty(languages); Assert.Contains(languages, lang => lang.GetProperty("lang").GetString() == "csharp"); - Assert.All(languages, lang => Assert.True(lang.GetProperty("graph_queries").GetBoolean())); + Assert.All(languages, lang => Assert.True(lang.GetProperty(propertyName).GetBoolean())); } [Fact] @@ -2069,6 +2069,51 @@ public void RunLanguages_JsonCapabilitySymbolsFiltersSymbolSupport() Assert.All(languages, lang => Assert.True(lang.GetProperty("symbol_extraction").GetBoolean())); } + [Theory] + [InlineData("missing-symbols", "symbol_extraction")] + [InlineData("missing-references", "reference_extraction")] + [InlineData("missing-graph", "graph_queries")] + public void RunLanguages_JsonCapabilityMissingFiltersCapabilityGaps(string capability, string propertyName) + { + var (exitCode, stdout, stderr) = CaptureConsole(() => + QueryCommandRunner.RunLanguages(["--json", "--capability", capability], _jsonOptions)); + + Assert.Equal(CommandExitCodes.Success, exitCode); + Assert.Equal(string.Empty, stderr); + + using var document = ParseJsonOutput(stdout); + var languages = document.RootElement.GetProperty("languages").EnumerateArray().ToList(); + + Assert.NotEmpty(languages); + Assert.All(languages, lang => + { + Assert.False(lang.GetProperty(propertyName).GetBoolean()); + Assert.Contains(capability, lang.GetProperty("capability_gaps").EnumerateArray().Select(gap => gap.GetString())); + }); + } + + [Fact] + public void RunLanguages_JsonCapabilitySearchOnlyFiltersAllExtractionGaps() + { + var (exitCode, stdout, stderr) = CaptureConsole(() => + QueryCommandRunner.RunLanguages(["--json", "--capability", "search-only"], _jsonOptions)); + + Assert.Equal(CommandExitCodes.Success, exitCode); + Assert.Equal(string.Empty, stderr); + + using var document = ParseJsonOutput(stdout); + var languages = document.RootElement.GetProperty("languages").EnumerateArray().ToList(); + + Assert.NotEmpty(languages); + Assert.Contains(languages, lang => lang.GetProperty("lang").GetString() == "cython"); + Assert.All(languages, lang => + { + Assert.False(lang.GetProperty("symbol_extraction").GetBoolean()); + Assert.False(lang.GetProperty("reference_extraction").GetBoolean()); + Assert.False(lang.GetProperty("graph_queries").GetBoolean()); + }); + } + [Fact] public void RunLanguages_InvalidCapabilityReturnsUsageError() { @@ -2077,7 +2122,8 @@ public void RunLanguages_InvalidCapabilityReturnsUsageError() Assert.Equal(CommandExitCodes.UsageError, exitCode); Assert.Contains("unsupported --capability value 'lint'", stderr); - Assert.Contains("graph, symbols, or references", stderr); + Assert.Contains("missing-references", stderr); + Assert.Contains("search-only", stderr); } [Fact] @@ -2134,6 +2180,7 @@ public void RunLanguages_JsonListsHtmlWithSymbolExtractionAndAllExtensions() var html = languages.EnumerateArray().First(lang => lang.GetProperty("lang").GetString() == "html"); Assert.True(html.GetProperty("symbol_extraction").GetBoolean()); + Assert.False(html.GetProperty("reference_extraction").GetBoolean()); var extensions = html.GetProperty("extensions").EnumerateArray().Select(ext => ext.GetString()).ToList(); Assert.Contains(".html", extensions); Assert.Contains(".htm", extensions); @@ -2154,6 +2201,7 @@ public void RunLanguages_JsonListsAssemblyWithSymbolExtractionGraphAndAliases() var assembly = languages.EnumerateArray().First(lang => lang.GetProperty("lang").GetString() == "assembly"); Assert.True(assembly.GetProperty("symbol_extraction").GetBoolean()); + Assert.True(assembly.GetProperty("reference_extraction").GetBoolean()); Assert.True(assembly.GetProperty("graph_queries").GetBoolean()); var extensions = assembly.GetProperty("extensions").EnumerateArray().Select(ext => ext.GetString()).ToList(); @@ -2214,8 +2262,12 @@ public void RunLanguages_Json_SearchOnlyBucketsAdvertiseZeroSymbolAndGraphSuppor var entry = languages[searchOnly]; Assert.False(entry.GetProperty("symbol_extraction").GetBoolean(), $"{searchOnly} must advertise symbol_extraction=false"); + Assert.False(entry.GetProperty("reference_extraction").GetBoolean(), + $"{searchOnly} must advertise reference_extraction=false"); Assert.False(entry.GetProperty("graph_queries").GetBoolean(), $"{searchOnly} must advertise graph_queries=false"); + Assert.Contains("missing-symbols", entry.GetProperty("capability_gaps").EnumerateArray().Select(gap => gap.GetString())); + Assert.Contains("missing-references", entry.GetProperty("capability_gaps").EnumerateArray().Select(gap => gap.GetString())); } foreach (var searchOnly in new[] { "crystal", "clojure", "d", "erlang", "julia", "nim", "ocaml", "solidity", "tcl" }) @@ -2224,6 +2276,8 @@ public void RunLanguages_Json_SearchOnlyBucketsAdvertiseZeroSymbolAndGraphSuppor var entry = languages[searchOnly]; Assert.False(entry.GetProperty("symbol_extraction").GetBoolean(), $"{searchOnly} must advertise symbol_extraction=false"); + Assert.False(entry.GetProperty("reference_extraction").GetBoolean(), + $"{searchOnly} must advertise reference_extraction=false"); Assert.False(entry.GetProperty("graph_queries").GetBoolean(), $"{searchOnly} must advertise graph_queries=false"); } @@ -2235,6 +2289,8 @@ public void RunLanguages_Json_SearchOnlyBucketsAdvertiseZeroSymbolAndGraphSuppor Assert.True(languages.ContainsKey("perl"), "expected 'perl' to be listed"); Assert.True(languages["perl"].GetProperty("symbol_extraction").GetBoolean(), "perl must advertise symbol_extraction=true"); + Assert.True(languages["perl"].GetProperty("reference_extraction").GetBoolean(), + "perl must advertise reference_extraction=true"); Assert.True(languages["perl"].GetProperty("graph_queries").GetBoolean(), "perl must advertise graph_queries=true"); From 75b4b7c320c36a54ae4343dcb56b7827f1516fa5 Mon Sep 17 00:00:00 2001 From: Widthdom Date: Fri, 12 Jun 2026 09:15:40 +0900 Subject: [PATCH 2/5] Index JSON and YAML configuration symbols (#3482) --- USER_GUIDE.md | 12 +- changelog.d/unreleased/3482.added.md | 18 ++ src/CodeIndex/Cli/QueryCommandRunner.cs | 15 + .../Symbols/SymbolExtractor.StructuredData.cs | 262 ++++++++++++++++++ .../Indexer/Symbols/SymbolExtractor.cs | 12 +- tests/CodeIndex.Tests/McpServerTests.cs | 3 +- .../QueryCommandRunnerSymbolTests.cs | 30 ++ .../QueryCommandRunnerTests.cs | 4 + tests/CodeIndex.Tests/SymbolExtractorTests.cs | 54 ++++ 9 files changed, 402 insertions(+), 8 deletions(-) create mode 100644 changelog.d/unreleased/3482.added.md create mode 100644 src/CodeIndex/Indexer/Symbols/SymbolExtractor.StructuredData.cs diff --git a/USER_GUIDE.md b/USER_GUIDE.md index 492d5d1877..9c3466db72 100644 --- a/USER_GUIDE.md +++ b/USER_GUIDE.md @@ -1697,8 +1697,8 @@ All indexed languages are searchable through FTS5. Rows with **Symbols = yes** a | CMake | `.cmake`, `CMakeLists.txt` | -- | | SQL | `.sql`, `.pgsql`, `.tsql`, `.plsql`, `.pks`, `.pkb`, `.pls`, `.plb`, `.psql` | yes | | Markdown | `.md` | yes | -| YAML | `.yaml`, `.yml` | -- | -| JSON | `.json` | -- | +| YAML | `.yaml`, `.yml` | yes | +| JSON | `.json` | yes | | TOML | `.toml` | -- | | HTML | `.html`, `.htm`, `.xhtml`, `.shtml` | yes | | CSS | `.css`, `.scss`, `.less`, `.pcss` | yes | @@ -1713,7 +1713,7 @@ All indexed languages are searchable through FTS5. Rows with **Symbols = yes** a - C/C++ headers: `.h` stays on the C path unless the file has clear C++ markers such as `namespace`, `template`, `using`, `class`, or `std::`; those headers are promoted to `cpp` at index time. - SQL: query-time `--lang tsql` is accepted as a SQL alias, and T-SQL aggregate, assembly, and XML schema collection declarations are searchable. - R: function assignments, S4/R6 class declarations, validity/generic/method declarations, inherit vectors, public/private/active methods, and `library` / `require` imports are indexed. -- Markdown and CSS: Markdown heading and local-anchor symbols are indexed; CSS variables, placeholders, and `@extend` references are indexed. +- Markdown, JSON/YAML, and CSS: Markdown heading and local-anchor symbols are indexed; JSON/YAML configuration keys are indexed as structural key paths; CSS variables, placeholders, and `@extend` references are indexed. - Dockerfile, Assembly, Common Lisp, and Racket: `ARG` build args, labels/PROC/MACRO blocks, package/module forms, definitions, classes/structs, requires, and provides are surfaced as symbols where applicable. - Shell, PowerShell, and Batch: command-style function calls, functions/filters, classes/enums, imports, labels, `goto` / `call` targets, and inline control-flow forms are indexed where the language supports them. - C# and Java: modern C# partial members remain visible to `symbols`, `definition`, and `outline`; Java sealed `permits` lists are recorded as `type_reference` graph edges. @@ -4050,8 +4050,8 @@ indexing はファイル単位の SQLite transaction を commit します。長 | CMake | `.cmake`, `CMakeLists.txt` | -- | | SQL | `.sql`, `.pgsql`, `.tsql`, `.plsql`, `.pks`, `.pkb`, `.pls`, `.plb`, `.psql` | yes | | Markdown | `.md` | yes | -| YAML | `.yaml`, `.yml` | -- | -| JSON | `.json` | -- | +| YAML | `.yaml`, `.yml` | yes | +| JSON | `.json` | yes | | TOML | `.toml` | -- | | HTML | `.html`, `.htm`, `.xhtml`, `.shtml` | yes | | CSS | `.css`, `.scss`, `.less`, `.pcss` | yes | @@ -4066,7 +4066,7 @@ indexing はファイル単位の SQLite transaction を commit します。長 - C/C++ ヘッダー: `.h` は既定では C として扱います。`namespace`、`template`、`using`、`class`、`std::` などの明確な C++ マーカーがある場合だけ、index 時に `cpp` へ昇格します。 - SQL: クエリ時の `--lang tsql` は SQL の別名です。T-SQL の aggregate、assembly、XML schema collection 宣言も検索対象です。 - R: 関数代入、S4/R6 class 宣言、validity/generic/method 宣言、inherit vector、public/private/active method、`library` / `require` import を索引します。 -- Markdown と CSS: Markdown の heading / local anchor、CSS の variable、placeholder、`@extend` をシンボルとして扱います。 +- Markdown、JSON/YAML、CSS: Markdown の heading / local anchor、JSON/YAML の configuration key path、CSS の variable、placeholder、`@extend` をシンボルとして扱います。 - Dockerfile、Assembly、Common Lisp、Racket: `ARG` build arg、label、PROC/MACRO、package/module form、definition、class/struct、require/provide を必要に応じて表面化します。 - Shell、PowerShell、Batch: command-style function call、function/filter、class/enum、import、label、`goto` / `call` target、inline control-flow を言語仕様に合わせて索引します。 - C# と Java: C# の近年の partial member は `symbols`、`definition`、`outline` から見えます。Java の sealed `permits` list は `type_reference` graph edge として記録します。 diff --git a/changelog.d/unreleased/3482.added.md b/changelog.d/unreleased/3482.added.md new file mode 100644 index 0000000000..c4733ff059 --- /dev/null +++ b/changelog.d/unreleased/3482.added.md @@ -0,0 +1,18 @@ +--- +category: added +issues: + - 3482 +affected: + - src/CodeIndex/Indexer/Symbols/SymbolExtractor.cs + - src/CodeIndex/Indexer/Symbols/SymbolExtractor.StructuredData.cs + - src/CodeIndex/Cli/QueryCommandRunner.cs + - USER_GUIDE.md +--- + +## English + +- **JSON and YAML files now expose structural symbols (#3482)** — `symbols`, `definition`, `outline`, and language capability probes can now see JSON/YAML configuration key paths, while unsupported symbol extractor languages explain the `search` fallback instead of returning an unexplained empty result. + +## 日本語 + +- **JSON と YAML が構造シンボルを公開するようになりました (#3482)** — `symbols`、`definition`、`outline`、言語 capability probe で JSON/YAML の configuration key path が見えるようになり、シンボル抽出未対応言語では空結果だけでなく `search` fallback の説明を返します。 diff --git a/src/CodeIndex/Cli/QueryCommandRunner.cs b/src/CodeIndex/Cli/QueryCommandRunner.cs index 32b8f3c832..ef9ec32191 100644 --- a/src/CodeIndex/Cli/QueryCommandRunner.cs +++ b/src/CodeIndex/Cli/QueryCommandRunner.cs @@ -2573,6 +2573,7 @@ public static int RunSymbols(string[] cmdArgs, JsonSerializerOptions jsonOptions WriteExactZeroHint(exactZeroHint); WriteKindHint(options.Kind, reader); WriteLangHint(options.Lang, reader); + WriteSymbolExtractionCapabilityHint(options.Lang, reader); WriteZeroResultHints(options, reader); } return ZeroResultExitCode(options); @@ -9653,6 +9654,20 @@ private static void WriteLangHint(string? lang, DbReader reader) Console.Error.WriteLine($"Did you mean: --lang {suggestion}?"); } + private static void WriteSymbolExtractionCapabilityHint(string? lang, DbReader reader) + { + if (string.IsNullOrWhiteSpace(lang)) + return; + if (SymbolExtractor.GetSupportedLanguages().Contains(lang, StringComparer.Ordinal)) + return; + + var status = reader.GetStatus(); + if (status.Languages.Count == 0 || !status.Languages.ContainsKey(lang)) + return; + + Console.Error.WriteLine($"Hint: '{lang}' is indexed for full-text search, but symbol extraction is not available for that language. Use `cdidx search --lang {lang}` for text matches or `cdidx languages --capability missing-symbols` to audit capability gaps."); + } + // All valid symbol kinds emitted by SymbolExtractor / SymbolExtractor が出力する全有効シンボル種別 private static readonly string[] AllValidKinds = KnownSymbolKindFilters.OrderBy(kind => kind, StringComparer.Ordinal).ToArray(); diff --git a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.StructuredData.cs b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.StructuredData.cs new file mode 100644 index 0000000000..323a593ece --- /dev/null +++ b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.StructuredData.cs @@ -0,0 +1,262 @@ +using System.Text.Json; +using System.Text.RegularExpressions; +using CodeIndex.Models; +using Regex = CodeIndex.Indexer.BoundedRegex; + +namespace CodeIndex.Indexer; + +public static partial class SymbolExtractor +{ + private static readonly Regex JsonFallbackPropertyRegex = new( + @"^\s*""(?(?:\\.|[^""\\])+)""\s*:", + RegexOptions.Compiled | RegexOptions.CultureInvariant); + + private static readonly Regex YamlMappingKeyRegex = new( + @"^(?[ \t]*)(?:-\s*)?(?:""(?(?:[^""]|"""")+)""|'(?(?:[^']|'')+)'|(?[A-Za-z0-9_.-][A-Za-z0-9_. -]*))\s*:\s*(?.*)$", + RegexOptions.Compiled | RegexOptions.CultureInvariant); + + private readonly record struct YamlPathFrame(int Indent, string Path); + + private static List ExtractJsonSymbols(long fileId, string content, string[] lines) + { + var symbols = new List(); + var lineStarts = BuildLineStarts(lines); + var searchOffset = 0; + + try + { + using var document = JsonDocument.Parse(content, new JsonDocumentOptions + { + AllowTrailingCommas = true, + CommentHandling = JsonCommentHandling.Skip, + }); + + if (document.RootElement.ValueKind == JsonValueKind.Object) + ExtractJsonObjectSymbols(fileId, content, lines, lineStarts, document.RootElement, parentPath: null, ref searchOffset, symbols); + return symbols; + } + catch (JsonException) + { + return ExtractJsonFallbackSymbols(fileId, lines); + } + } + + private static void ExtractJsonObjectSymbols( + long fileId, + string content, + string[] lines, + int[] lineStarts, + JsonElement element, + string? parentPath, + ref int searchOffset, + List symbols) + { + foreach (var property in element.EnumerateObject()) + { + var name = string.IsNullOrEmpty(parentPath) + ? property.Name + : parentPath + "." + property.Name; + var propertyOffset = FindJsonPropertyOffset(content, property.Name, ref searchOffset); + var line = FindLineNumberForOffset(lineStarts, propertyOffset); + var kind = property.Value.ValueKind is JsonValueKind.Object or JsonValueKind.Array + ? "namespace" + : "property"; + + symbols.Add(CreateStructuredDataSymbol(fileId, kind, name, line, lines, parentPath)); + + if (property.Value.ValueKind == JsonValueKind.Object) + { + ExtractJsonObjectSymbols(fileId, content, lines, lineStarts, property.Value, name, ref searchOffset, symbols); + } + else if (property.Value.ValueKind == JsonValueKind.Array) + { + foreach (var item in property.Value.EnumerateArray()) + { + if (item.ValueKind == JsonValueKind.Object) + ExtractJsonObjectSymbols(fileId, content, lines, lineStarts, item, name, ref searchOffset, symbols); + } + } + } + } + + private static List ExtractJsonFallbackSymbols(long fileId, string[] lines) + { + var symbols = new List(); + for (var i = 0; i < lines.Length; i++) + { + var match = JsonFallbackPropertyRegex.Match(lines[i]); + if (!match.Success) + continue; + + var name = UnescapeJsonPropertyName(match.Groups["name"].Value); + if (string.IsNullOrWhiteSpace(name)) + continue; + + symbols.Add(CreateStructuredDataSymbol(fileId, "property", name, i + 1, lines, parentPath: null)); + } + + return symbols; + } + + private static List ExtractYamlSymbols(long fileId, string[] lines) + { + var symbols = new List(); + var stack = new List(); + int? blockScalarIndent = null; + + for (var i = 0; i < lines.Length; i++) + { + var line = lines[i]; + if (string.IsNullOrWhiteSpace(line)) + continue; + + var indent = CountLeadingSpaces(line); + if (blockScalarIndent.HasValue) + { + if (indent > blockScalarIndent.Value) + continue; + blockScalarIndent = null; + } + + var trimmed = line.TrimStart(); + if (trimmed.StartsWith('#') || trimmed is "---" or "...") + continue; + + var match = YamlMappingKeyRegex.Match(line); + if (!match.Success) + continue; + + var key = ExtractYamlKey(match); + if (string.IsNullOrWhiteSpace(key)) + continue; + + while (stack.Count > 0 && indent <= stack[^1].Indent) + stack.RemoveAt(stack.Count - 1); + + var parentPath = stack.Count == 0 ? null : stack[^1].Path; + var path = string.IsNullOrEmpty(parentPath) ? key : parentPath + "." + key; + var value = StripYamlInlineComment(match.Groups["value"].Value).Trim(); + var isContainer = value.Length == 0 || value is "|" or ">" or "|-" or ">-" or "|+" or ">+"; + var kind = isContainer ? "namespace" : "property"; + + symbols.Add(CreateStructuredDataSymbol(fileId, kind, path, i + 1, lines, parentPath)); + + if (isContainer) + { + stack.Add(new YamlPathFrame(indent, path)); + if (value.StartsWith('|') || value.StartsWith('>')) + blockScalarIndent = indent; + } + } + + return symbols; + } + + private static SymbolRecord CreateStructuredDataSymbol( + long fileId, + string kind, + string name, + int line, + string[] lines, + string? parentPath) + { + var signatureIndex = Math.Clamp(line - 1, 0, Math.Max(0, lines.Length - 1)); + return new SymbolRecord + { + FileId = fileId, + Kind = kind, + Name = name, + Line = line, + StartLine = line, + EndLine = line, + Signature = lines.Length == 0 ? null : lines[signatureIndex].Trim(), + ContainerKind = parentPath == null ? null : "namespace", + ContainerName = parentPath, + ContainerQualifiedName = parentPath, + }; + } + + private static int FindJsonPropertyOffset(string content, string propertyName, ref int searchOffset) + { + var encodedName = JsonSerializer.Serialize(propertyName); + var offset = content.IndexOf(encodedName, searchOffset, StringComparison.Ordinal); + if (offset < 0 && searchOffset > 0) + offset = content.IndexOf(encodedName, StringComparison.Ordinal); + + if (offset >= 0) + { + searchOffset = offset + encodedName.Length; + return offset; + } + + return Math.Clamp(searchOffset, 0, Math.Max(0, content.Length - 1)); + } + + private static int[] BuildLineStarts(string[] lines) + { + var starts = new int[lines.Length == 0 ? 1 : lines.Length]; + var offset = 0; + for (var i = 0; i < lines.Length; i++) + { + starts[i] = offset; + offset += lines[i].Length + 1; + } + + return starts; + } + + private static int FindLineNumberForOffset(int[] lineStarts, int offset) + { + var index = Array.BinarySearch(lineStarts, offset); + if (index < 0) + index = Math.Max(0, ~index - 1); + return index + 1; + } + + private static int CountLeadingSpaces(string line) + { + var count = 0; + while (count < line.Length && line[count] == ' ') + count++; + return count; + } + + private static string ExtractYamlKey(Match match) + { + if (match.Groups["double"].Success) + return match.Groups["double"].Value.Replace("\"\"", "\"", StringComparison.Ordinal).Trim(); + if (match.Groups["single"].Success) + return match.Groups["single"].Value.Replace("''", "'", StringComparison.Ordinal).Trim(); + return match.Groups["plain"].Value.Trim(); + } + + private static string StripYamlInlineComment(string value) + { + var inSingle = false; + var inDouble = false; + for (var i = 0; i < value.Length; i++) + { + var ch = value[i]; + if (ch == '\'' && !inDouble) + inSingle = !inSingle; + else if (ch == '"' && !inSingle) + inDouble = !inDouble; + else if (ch == '#' && !inSingle && !inDouble && (i == 0 || char.IsWhiteSpace(value[i - 1]))) + return value[..i]; + } + + return value; + } + + private static string UnescapeJsonPropertyName(string value) + { + try + { + return JsonSerializer.Deserialize("\"" + value + "\"") ?? value; + } + catch (JsonException) + { + return value; + } + } +} diff --git a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.cs b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.cs index 3b2879f2ee..000e8c9c35 100644 --- a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.cs +++ b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.cs @@ -2122,7 +2122,7 @@ private enum JavaScriptTypeScriptFunctionHeaderConsumeResult /// public static IReadOnlyCollection GetSupportedLanguages() => PatternCache.Keys - .Concat(new[] { "commonlisp", "racket", "vue", "svelte", "markdown", "razor", "blazor", "cshtml" }) + .Concat(new[] { "commonlisp", "racket", "vue", "svelte", "markdown", "json", "yaml", "razor", "blazor", "cshtml" }) .Concat(ExtractorPluginRegistry.SymbolLanguages) .Distinct(StringComparer.Ordinal) .ToArray(); @@ -2262,6 +2262,16 @@ public static List Extract(long fileId, string? lang, string conte return ExtractXmlSymbols(fileId, content.Split('\n')); } + if (lang == "json") + { + return ExtractJsonSymbols(fileId, content, content.Split('\n')); + } + + if (lang == "yaml") + { + return ExtractYamlSymbols(fileId, content.Split('\n')); + } + if (lang == "markdown") { var markdownLines = content.Split('\n'); diff --git a/tests/CodeIndex.Tests/McpServerTests.cs b/tests/CodeIndex.Tests/McpServerTests.cs index 7bae875907..4fc503d416 100644 --- a/tests/CodeIndex.Tests/McpServerTests.cs +++ b/tests/CodeIndex.Tests/McpServerTests.cs @@ -9181,8 +9181,9 @@ public void ToolsCall_Languages_ReturnsCapabilities() Assert.Contains("missing-references", markdown["capability_gaps"]!.AsArray().Select(e => e!.GetValue())); var yaml = languages.First(l => l!["lang"]!.GetValue() == "yaml")!; + Assert.True(yaml["symbol_extraction"]!.GetValue()); Assert.Contains("yml", yaml["aliases"]!.AsArray().Select(e => e!.GetValue())); - Assert.Contains("missing-symbols", yaml["capability_gaps"]!.AsArray().Select(e => e!.GetValue())); + Assert.DoesNotContain("missing-symbols", yaml["capability_gaps"]!.AsArray().Select(e => e!.GetValue())); // Pin #215: HTML must report symbol_extraction=true and list all four // extensions so AI tools discover HTML support via the MCP languages tool. diff --git a/tests/CodeIndex.Tests/QueryCommandRunnerSymbolTests.cs b/tests/CodeIndex.Tests/QueryCommandRunnerSymbolTests.cs index 4c0c174c9b..e499945cb4 100644 --- a/tests/CodeIndex.Tests/QueryCommandRunnerSymbolTests.cs +++ b/tests/CodeIndex.Tests/QueryCommandRunnerSymbolTests.cs @@ -655,6 +655,36 @@ public void RunSymbols_JsonZeroResults_ReturnEmptyStdout() } } + [Fact] + public void RunSymbols_UnsupportedExtractorLanguageExplainsSearchOnlyFallback() + { + var projectRoot = TestProjectHelper.CreateTempProject("cdidx_symbols_unsupported_extractor"); + try + { + File.WriteAllText(Path.Combine(projectRoot, "settings.toml"), "enabled = true\n"); + var dbPath = Path.Combine(projectRoot, ".cdidx", "codeindex.db"); + var (indexExitCode, _, indexStderr) = CaptureConsole(() => IndexCommandRunner.Run( + [projectRoot, "--json", "--quiet"], + _jsonOptions)); + + var (exitCode, stdout, stderr) = CaptureConsole(() => QueryCommandRunner.RunSymbols( + ["enabled", "--db", dbPath, "--lang", "toml"], + _jsonOptions)); + + Assert.Equal(CommandExitCodes.Success, indexExitCode); + Assert.Equal(string.Empty, indexStderr); + Assert.Equal(CommandExitCodes.Success, exitCode); + Assert.Equal(string.Empty, stdout); + Assert.Contains("symbol extraction is not available", stderr); + Assert.Contains("cdidx search --lang toml", stderr); + Assert.Contains("missing-symbols", stderr); + } + finally + { + TestProjectHelper.DeleteDirectory(projectRoot); + } + } + [Fact] public void RunUnused_WithJsonIncludesConfidenceBuckets() { diff --git a/tests/CodeIndex.Tests/QueryCommandRunnerTests.cs b/tests/CodeIndex.Tests/QueryCommandRunnerTests.cs index acdfd738f3..1e3bacd56d 100644 --- a/tests/CodeIndex.Tests/QueryCommandRunnerTests.cs +++ b/tests/CodeIndex.Tests/QueryCommandRunnerTests.cs @@ -2285,6 +2285,10 @@ public void RunLanguages_Json_SearchOnlyBucketsAdvertiseZeroSymbolAndGraphSuppor var yamlAliases = languages["yaml"].GetProperty("aliases").EnumerateArray() .Select(alias => alias.GetString()).ToList(); Assert.Contains("yml", yamlAliases); + Assert.True(languages["yaml"].GetProperty("symbol_extraction").GetBoolean(), + "yaml must advertise symbol_extraction=true"); + Assert.True(languages["json"].GetProperty("symbol_extraction").GetBoolean(), + "json must advertise symbol_extraction=true"); Assert.True(languages.ContainsKey("perl"), "expected 'perl' to be listed"); Assert.True(languages["perl"].GetProperty("symbol_extraction").GetBoolean(), diff --git a/tests/CodeIndex.Tests/SymbolExtractorTests.cs b/tests/CodeIndex.Tests/SymbolExtractorTests.cs index 2c1e6ee289..1df0e7b6e8 100644 --- a/tests/CodeIndex.Tests/SymbolExtractorTests.cs +++ b/tests/CodeIndex.Tests/SymbolExtractorTests.cs @@ -147,6 +147,60 @@ public void Extract_ConfiguredPatternYaml_HandlesOutOfTreeLanguage() } } + [Fact] + public void Extract_Json_IndexesConfigurationKeyPaths() + { + const string content = """ + { + "scripts": { + "build": "dotnet build", + "test": "dotnet test" + }, + "dependencies": { + "xunit": "2.9.3" + } + } + """; + + var symbols = SymbolExtractor.Extract(1, "json", content); + + Assert.Contains(symbols, symbol => symbol.Kind == "namespace" && symbol.Name == "scripts" && symbol.Line == 2); + Assert.Contains(symbols, symbol => symbol.Kind == "property" && symbol.Name == "scripts.build" && symbol.ContainerName == "scripts"); + Assert.Contains(symbols, symbol => symbol.Kind == "property" && symbol.Name == "scripts.test" && symbol.ContainerName == "scripts"); + Assert.Contains(symbols, symbol => symbol.Kind == "property" && symbol.Name == "dependencies.xunit" && symbol.ContainerName == "dependencies"); + } + + [Fact] + public void Extract_Yaml_IndexesIndentedConfigurationKeyPaths() + { + const string content = """ + name: ci + on: + push: + branches: + - main + jobs: + build: + runs-on: ubuntu-latest + steps: + - name: Restore + uses: actions/setup-dotnet@v4 + - name: Test + run: dotnet test + notes: | + jobs.fake: + run: ignored + """; + + var symbols = SymbolExtractor.Extract(1, "yaml", content); + + Assert.Contains(symbols, symbol => symbol.Kind == "property" && symbol.Name == "name" && symbol.Line == 1); + Assert.Contains(symbols, symbol => symbol.Kind == "namespace" && symbol.Name == "jobs.build" && symbol.ContainerName == "jobs"); + Assert.Contains(symbols, symbol => symbol.Kind == "property" && symbol.Name == "jobs.build.runs-on"); + Assert.Contains(symbols, symbol => symbol.Kind == "property" && symbol.Name == "jobs.build.steps.uses"); + Assert.DoesNotContain(symbols, symbol => symbol.Name.Contains("jobs.fake", StringComparison.Ordinal)); + } + From 2dff4f59082e46bef27b56f9c1935391eab3cf3c Mon Sep 17 00:00:00 2001 From: Widthdom Date: Fri, 12 Jun 2026 09:38:49 +0900 Subject: [PATCH 3/5] Index build automation structure (#3526) --- USER_GUIDE.md | 14 +- changelog.d/unreleased/3526.added.md | 19 + .../BuildAutomationReferenceExtractor.cs | 335 ++++++++++++++++++ .../References/ReferenceExtractor.Core.cs | 18 +- .../References/ReferenceExtractor.State.cs | 2 +- .../SymbolExtractor.BuildAutomation.cs | 195 ++++++++++ .../Indexer/Symbols/SymbolExtractor.cs | 19 + .../QueryCommandRunnerTests.cs | 16 +- .../ReferenceExtractorTests.cs | 118 ++++++ tests/CodeIndex.Tests/SymbolExtractorTests.cs | 86 +++++ 10 files changed, 812 insertions(+), 10 deletions(-) create mode 100644 changelog.d/unreleased/3526.added.md create mode 100644 src/CodeIndex/Indexer/References/Languages/BuildAutomationReferenceExtractor.cs create mode 100644 src/CodeIndex/Indexer/Symbols/SymbolExtractor.BuildAutomation.cs diff --git a/USER_GUIDE.md b/USER_GUIDE.md index 9c3466db72..9b6788bfb7 100644 --- a/USER_GUIDE.md +++ b/USER_GUIDE.md @@ -1690,11 +1690,12 @@ All indexed languages are searchable through FTS5. Rows with **Symbols = yes** a | Perl test | `.t` | -- | | Zig | `.zig` | yes | | XAML | `.xaml`, `.axaml` | -- | -| MSBuild | `.csproj`, `.fsproj`, `.vbproj`, `.props`, `.targets` | -- | +| MSBuild | `.csproj`, `.fsproj`, `.vbproj`, `.props`, `.targets` | yes | | Shell | `.sh`, `.bash`, `.zsh`, `.fish` | partial | | PowerShell | `.ps1`, `.psm1`, `.psd1` | yes | | Batch | `.bat`, `.cmd` | yes | -| CMake | `.cmake`, `CMakeLists.txt` | -- | +| CMake | `.cmake`, `CMakeLists.txt` | yes | +| Justfile | `Justfile` | yes | | SQL | `.sql`, `.pgsql`, `.tsql`, `.plsql`, `.pks`, `.pkb`, `.pls`, `.plb`, `.psql` | yes | | Markdown | `.md` | yes | | YAML | `.yaml`, `.yml` | yes | @@ -1741,7 +1742,7 @@ commands and when to fall back to `search`. | JavaScript / TypeScript / Vue / Svelte | functions, classes, exports, imports, variables | calls, constructors, static/dynamic imports, workers, service workers | Dynamic property calls and computed module specifiers are best-effort. `cdidx references render --lang typescript` | | Python / Ruby / PHP / Perl / R | functions, classes/modules, imports where supported | calls, constructors, decorators/annotations where supported | Dynamic dispatch and metaprogramming may require `search`. PHPDoc/static import patterns are indexed when statically visible. | | C / C++ / Objective-C / Swift / Rust / Go / Zig | functions, types, methods, imports/modules | calls, constructors, macro invocations where supported, type references | C++ templates/macros and Rust macro expansion are not evaluated; Rust macro invocations are still reference edges. | -| Shell / PowerShell / Batch / Makefile / Gradle | functions, labels, tasks, imports where applicable | command-style calls and control-flow targets | Runtime command construction is not resolved. | +| Shell / PowerShell / Batch / Makefile / CMake / Justfile / MSBuild / Gradle | functions, labels, targets, recipes, tasks, imports where applicable | command-style calls, target dependencies, and control-flow targets | Runtime command construction is not resolved. | | SQL / Terraform / Dockerfile | statements/resources/stages/labels | table/resource/stage references, Dockerfile stage dependencies, Terraform dotted refs | SQL hotspot grouping defaults to statements; Dockerfile `COPY --from=` follows named stages. | | Markdown / HTML / CSS / GraphQL / Protobuf | headings, anchors, selectors, schema types/messages where supported | local anchors, CSS extends/variables, schema references where supported | Use `search` for prose and generated markup. | | Other indexed text formats | file/chunk search only unless `languages` reports symbols | no graph unless `languages` reports support | `cdidx search "literal" --lang yaml` is the reliable fallback. | @@ -4043,11 +4044,12 @@ indexing はファイル単位の SQLite transaction を commit します。長 | Perl test | `.t` | -- | | Zig | `.zig` | yes | | XAML | `.xaml`, `.axaml` | -- | -| MSBuild | `.csproj`, `.fsproj`, `.vbproj`, `.props`, `.targets` | -- | +| MSBuild | `.csproj`, `.fsproj`, `.vbproj`, `.props`, `.targets` | yes | | Shell | `.sh`, `.bash`, `.zsh`, `.fish` | partial | | PowerShell | `.ps1`, `.psm1`, `.psd1` | yes | | Batch | `.bat`, `.cmd` | yes | -| CMake | `.cmake`, `CMakeLists.txt` | -- | +| CMake | `.cmake`, `CMakeLists.txt` | yes | +| Justfile | `Justfile` | yes | | SQL | `.sql`, `.pgsql`, `.tsql`, `.plsql`, `.pks`, `.pkb`, `.pls`, `.plb`, `.psql` | yes | | Markdown | `.md` | yes | | YAML | `.yaml`, `.yml` | yes | @@ -4091,7 +4093,7 @@ indexing はファイル単位の SQLite transaction を commit します。長 | JavaScript / TypeScript / Vue / Svelte | function、class、export、import、variable | call、constructor、static/dynamic import、worker、service worker | dynamic property call と computed module specifier は best-effort です。`cdidx references render --lang typescript` | | Python / Ruby / PHP / Perl / R | function、class/module、対応言語の import | call、constructor、対応言語の decorator/annotation | dynamic dispatch と metaprogramming は `search` が必要な場合があります。PHPDoc/static import pattern は静的に見える範囲で索引されます。 | | C / C++ / Objective-C / Swift / Rust / Go / Zig | function、type、method、import/module | call、constructor、対応言語の macro invocation、type reference | C++ template/macro と Rust macro expansion は評価しません。Rust macro invocation 自体は reference edge です。 | -| Shell / PowerShell / Batch / Makefile / Gradle | function、label、task、対応言語の import | command-style call と control-flow target | runtime で組み立てられる command は解決しません。 | +| Shell / PowerShell / Batch / Makefile / CMake / Justfile / MSBuild / Gradle | function、label、target、recipe、task、対応言語の import | command-style call、target dependency、control-flow target | runtime で組み立てられる command は解決しません。 | | SQL / Terraform / Dockerfile | statement/resource/stage/label | table/resource/stage reference、Dockerfile stage dependency、Terraform dotted refs | SQL hotspot grouping は既定で statement、Dockerfile `COPY --from=` は named stage を追跡します。 | | Markdown / HTML / CSS / GraphQL / Protobuf | heading、anchor、selector、対応 schema type/message | local anchor、CSS extend/variable、対応 schema reference | prose や generated markup には `search` を使ってください。 | | その他の indexed text format | `languages` が symbol 対応を示す場合を除き file/chunk search のみ | `languages` が graph 対応を示す場合を除きなし | `cdidx search "literal" --lang yaml` が信頼できる fallback です。 | diff --git a/changelog.d/unreleased/3526.added.md b/changelog.d/unreleased/3526.added.md new file mode 100644 index 0000000000..091e94265f --- /dev/null +++ b/changelog.d/unreleased/3526.added.md @@ -0,0 +1,19 @@ +--- +category: added +issues: + - 3526 +affected: + - src/CodeIndex/Indexer/Symbols/SymbolExtractor.cs + - src/CodeIndex/Indexer/Symbols/SymbolExtractor.BuildAutomation.cs + - src/CodeIndex/Indexer/References/ReferenceExtractor.Core.cs + - src/CodeIndex/Indexer/References/Languages/BuildAutomationReferenceExtractor.cs + - USER_GUIDE.md +--- + +## English + +- **Build automation files now expose structural symbols and references (#3526)** — CMake, Justfile, and MSBuild files now surface targets, recipes, properties, imports, and dependency references for graph-aware commands and capability probes. + +## 日本語 + +- **ビルド自動化ファイルが構造シンボルと参照を公開するようになりました (#3526)** — CMake、Justfile、MSBuild の target、recipe、property、import、依存参照を graph 対応コマンドと capability probe から扱えるようになりました。 diff --git a/src/CodeIndex/Indexer/References/Languages/BuildAutomationReferenceExtractor.cs b/src/CodeIndex/Indexer/References/Languages/BuildAutomationReferenceExtractor.cs new file mode 100644 index 0000000000..7c5c6b56e1 --- /dev/null +++ b/src/CodeIndex/Indexer/References/Languages/BuildAutomationReferenceExtractor.cs @@ -0,0 +1,335 @@ +using System.Text.RegularExpressions; +using CodeIndex.Models; +using Regex = CodeIndex.Indexer.BoundedRegex; + +namespace CodeIndex.Indexer; + +internal static class BuildAutomationReferenceExtractor +{ + private static readonly Regex CMakeCommandRegex = new( + @"^\s*(?[A-Za-z_]\w*)\s*\((?.*)\)\s*(?:#.*)?$", + RegexOptions.Compiled | RegexOptions.IgnoreCase | RegexOptions.CultureInvariant); + + private static readonly Regex JustImportRegex = new( + @"^\s*(?:import|mod)\s+[""'](?[^""']+)[""']", + RegexOptions.Compiled | RegexOptions.CultureInvariant); + + private static readonly Regex JustRecipeRegex = new( + @"^(?[A-Za-z_][\w.-]*)(?:\s+[^:#\r\n]+)?\s*:(?![:=])(?.*)$", + RegexOptions.Compiled | RegexOptions.CultureInvariant); + + private static readonly Regex MsBuildElementRegex = new( + @"<\s*(?[A-Za-z_][\w.-]*)(?[^<>]*)>", + RegexOptions.Compiled | RegexOptions.CultureInvariant); + + private static readonly Regex MsBuildAttributeRegex = new( + @"(?[A-Za-z_:][\w:.-]*)\s*=\s*(?:""(?[^""]*)""|'(?[^']*)')", + RegexOptions.Compiled | RegexOptions.CultureInvariant); + + private static readonly HashSet CMakeIgnoredDependencyTokens = new(StringComparer.OrdinalIgnoreCase) + { + "debug", + "general", + "INTERFACE", + "LINK_PRIVATE", + "LINK_PUBLIC", + "optimized", + "PRIVATE", + "PUBLIC", + }; + + private static readonly HashSet MsBuildImportItemElements = new(StringComparer.OrdinalIgnoreCase) + { + "Analyzer", + "Compile", + "Content", + "None", + "PackageReference", + "ProjectReference", + "Reference", + }; + + public static void EmitReferences( + string language, + string preparedLine, + string context, + int lineNumber, + List references, + HashSet seen, + long fileId, + SymbolRecord? container) + { + switch (language) + { + case "cmake": + EmitCMakeReferences(preparedLine, context, lineNumber, references, seen, fileId, container); + break; + case "justfile": + EmitJustfileReferences(preparedLine, context, lineNumber, references, seen, fileId, container); + break; + case "msbuild": + EmitMsBuildReferences(preparedLine, context, lineNumber, references, seen, fileId, container); + break; + } + } + + private static void EmitCMakeReferences( + string line, + string context, + int lineNumber, + List references, + HashSet seen, + long fileId, + SymbolRecord? container) + { + var match = CMakeCommandRegex.Match(line); + if (!match.Success) + return; + + var command = match.Groups["command"].Value; + var argsGroup = match.Groups["args"]; + var args = TokenizeBuildArguments(argsGroup.Value, argsGroup.Index); + if (args.Count == 0) + return; + + if (IsCMakeImportCommand(command)) + { + AddReference(references, seen, fileId, args[0], "import", context, lineNumber, container, "cmake"); + return; + } + + if (string.Equals(command, "target_link_libraries", StringComparison.OrdinalIgnoreCase) + || string.Equals(command, "add_dependencies", StringComparison.OrdinalIgnoreCase)) + { + var targetContainer = IsIgnoredCMakeDependency(args[0].Text) + ? container + : new SymbolRecord { Kind = "function", Name = args[0].Text }; + for (var i = 1; i < args.Count; i++) + { + var token = args[i]; + if (IsIgnoredCMakeDependency(token.Text)) + continue; + + AddReference(references, seen, fileId, token, "call", context, lineNumber, targetContainer, "cmake"); + } + } + } + + private static void EmitJustfileReferences( + string line, + string context, + int lineNumber, + List references, + HashSet seen, + long fileId, + SymbolRecord? container) + { + var importMatch = JustImportRegex.Match(line); + if (importMatch.Success) + { + AddReference(references, seen, fileId, importMatch, "import", context, lineNumber, container, "justfile"); + return; + } + + var recipeMatch = JustRecipeRegex.Match(line); + if (!recipeMatch.Success) + return; + + var deps = recipeMatch.Groups["deps"]; + var recipeContainer = new SymbolRecord + { + Kind = "function", + Name = recipeMatch.Groups["name"].Value, + }; + foreach (var token in TokenizeBuildArguments(StripJustComment(deps.Value), deps.Index)) + { + if (!IsBuildIdentifier(token.Text)) + continue; + + AddReference(references, seen, fileId, token, "call", context, lineNumber, recipeContainer, "justfile"); + } + } + + private static void EmitMsBuildReferences( + string line, + string context, + int lineNumber, + List references, + HashSet seen, + long fileId, + SymbolRecord? container) + { + if (line.TrimStart().StartsWith("", StringComparison.Ordinal); + if (commentEnd < 0) + return false; + + state.InHtmlComment = false; + line = new string(' ', commentEnd + 3) + line[(commentEnd + 3)..]; + } + + var chars = line.ToCharArray(); + var searchStart = 0; + while (searchStart < chars.Length) + { + var commentStart = line.IndexOf("", commentStart + 4, StringComparison.Ordinal); + var endExclusive = commentEnd < 0 ? chars.Length : commentEnd + 3; + for (var i = commentStart; i < endExclusive; i++) + chars[i] = ' '; + + if (commentEnd < 0) + { + state.InHtmlComment = true; + break; + } + + searchStart = endExclusive; + } + + scanLine = new string(chars); + return !string.IsNullOrWhiteSpace(scanLine); + } + + private static void UpdateHtmlRawTextState(string line, MarkupState state) + { + foreach (Match tagMatch in HtmlTagRegex.Matches(line)) + { + if (tagMatch.Groups["closing"].Success) + continue; + + var tagName = tagMatch.Groups["name"].Value; + if (!tagName.Equals("script", StringComparison.OrdinalIgnoreCase) + && !tagName.Equals("style", StringComparison.OrdinalIgnoreCase)) + { + continue; + } + + if (line.IndexOf(" references, + HashSet seen, + long fileId, + SymbolRecord? container) + { + var target = rawTarget.Trim(); + if (target.Length == 0) + return; + + if (referenceKind == "reference") + target = NormalizeMarkdownAnchor(target); + + AddHtmlReference(references, seen, fileId, target, targetIndex, referenceKind, context, lineNumber, container); + } + + private static void AddHtmlReference( + List references, + HashSet seen, + long fileId, + string name, + int nameIndex, + string referenceKind, + string context, + int lineNumber, + SymbolRecord? container) + => ReferenceExtractor.AddReference( + references, + seen, + fileId, + name, + nameIndex, + referenceKind, + context, + lineNumber, + container, + "html"); + + private static IEnumerable<(string Url, int Index)> EnumerateHtmlSrcsetUrls(string value, int absoluteStartIndex) + { + var start = 0; + while (start < value.Length) + { + while (start < value.Length && (char.IsWhiteSpace(value[start]) || value[start] == ',')) + start++; + + if (start >= value.Length) + yield break; + + var end = start; + while (end < value.Length && !char.IsWhiteSpace(value[end]) && value[end] != ',') + end++; + + if (end > start) + yield return (value[start..end], absoluteStartIndex + start); + + var comma = value.IndexOf(',', end); + if (comma < 0) + yield break; + start = comma + 1; + } + } + + private static IEnumerable<(string Token, int Index)> EnumerateWhitespaceTokens(string value, int absoluteStartIndex) + { + var index = 0; + while (index < value.Length) + { + while (index < value.Length && char.IsWhiteSpace(value[index])) + index++; + + var start = index; + while (index < value.Length && !char.IsWhiteSpace(value[index])) + index++; + + if (index > start) + yield return (value[start..index], absoluteStartIndex + start); + } + } + + private static bool TryToggleMarkdownFence(string line, MarkupState state) + { + var index = 0; + while (index < line.Length && index < 3 && line[index] == ' ') + index++; + + if (index > 3 || index >= line.Length) + return false; + + var marker = line[index]; + if (marker is not ('`' or '~')) + return false; + + var length = index; + while (length < line.Length && line[length] == marker) + length++; + + var markerLength = length - index; + if (markerLength < 3) + return false; + + if (state.InMarkdownFence) + { + if (marker != state.MarkdownFenceChar || markerLength < state.MarkdownFenceLength) + return false; + + state.InMarkdownFence = false; + state.MarkdownFenceChar = '\0'; + state.MarkdownFenceLength = 0; + return true; + } + + state.InMarkdownFence = true; + state.MarkdownFenceChar = marker; + state.MarkdownFenceLength = markerLength; + return true; + } + + private static string StripMarkdownInlineCode(string line) + { + var chars = line.ToCharArray(); + var inCode = false; + for (var i = 0; i < chars.Length; i++) + { + if (chars[i] != '`') + continue; + + inCode = !inCode; + chars[i] = ' '; + i++; + while (inCode && i < chars.Length) + { + if (chars[i] == '`') + { + inCode = false; + chars[i] = ' '; + break; + } + + chars[i] = ' '; + i++; + } + } + + return new string(chars); + } + + private static string NormalizeMarkdownLinkTarget(string rawTarget) + { + var target = rawTarget.Trim(); + if (target.Length >= 2 && target[0] == '<' && target[^1] == '>') + target = target[1..^1].Trim(); + return target; + } + + private static string NormalizeMarkdownAnchor(string value) + { + var anchor = value.Trim().TrimStart('#').Trim(); + if (anchor.Length == 0) + return string.Empty; + + var chars = new List(anchor.Length); + var previousDash = false; + foreach (var ch in anchor.ToLowerInvariant()) + { + if (char.IsLetterOrDigit(ch) || ch == '_' || ch == '-') + { + chars.Add(ch); + previousDash = ch == '-'; + } + else if (char.IsWhiteSpace(ch) && !previousDash) + { + chars.Add('-'); + previousDash = true; + } + } + + return new string(chars.ToArray()).Trim('-'); + } + + private static void AddReference( + List references, + HashSet seen, + long fileId, + Match match, + string referenceKind, + string context, + int lineNumber, + SymbolRecord? container, + string language) + => ReferenceExtractor.AddReference( + references, + seen, + fileId, + match, + referenceKind, + context, + lineNumber, + container, + language); +} diff --git a/src/CodeIndex/Indexer/References/ReferenceExtractor.Core.cs b/src/CodeIndex/Indexer/References/ReferenceExtractor.Core.cs index 7978fa4071..d02fddc743 100644 --- a/src/CodeIndex/Indexer/References/ReferenceExtractor.Core.cs +++ b/src/CodeIndex/Indexer/References/ReferenceExtractor.Core.cs @@ -196,6 +196,9 @@ bool HasActiveSameFileCSharpTypeCandidate(string typeExpression, int lineNumber) var csharpInDelimitedDocComment = false; var jvmInDelimitedDocComment = false; var phpInDocblock = false; + var markupSchemaState = language is "graphql" or "html" or "markdown" + ? new MarkupSchemaReferenceExtractor.MarkupState() + : null; SymbolRecord? phpDocblockContainer = null; HashSet? phpDocblockPropertyNames = null; @@ -370,6 +373,23 @@ bool HasActiveSameFileCSharpTypeCandidate(string typeExpression, int lineNumber) continue; } + if (language is "graphql" or "html" or "markdown" + && context.Length > 0) + { + var markupContainer = containerResolver.Find(lineNumber); + MarkupSchemaReferenceExtractor.EmitReferences( + language, + originalLine, + context, + lineNumber, + references, + seen, + fileId, + markupContainer, + markupSchemaState); + continue; + } + if (string.IsNullOrWhiteSpace(preparedLine)) { if (language == "csharp" diff --git a/src/CodeIndex/Indexer/References/ReferenceExtractor.State.cs b/src/CodeIndex/Indexer/References/ReferenceExtractor.State.cs index 1c86351b3f..6e59f6cad3 100644 --- a/src/CodeIndex/Indexer/References/ReferenceExtractor.State.cs +++ b/src/CodeIndex/Indexer/References/ReferenceExtractor.State.cs @@ -31,7 +31,7 @@ internal sealed class CSharpWhereConstraintState "assembly", "r", "powershell", "shell", "haskell", "gradle", "terraform", "protobuf", "dockerfile", "makefile", "cmake", "justfile", "msbuild", - "zig", "css", "fortran", "pascal", "objc", "smalltalk" + "zig", "css", "graphql", "html", "markdown", "fortran", "pascal", "objc", "smalltalk" ]; private static readonly IReadOnlyDictionary Extractors = diff --git a/src/CodeIndex/Indexer/References/ReferenceExtractor.cs b/src/CodeIndex/Indexer/References/ReferenceExtractor.cs index 7f1c9923ef..9efce95ae0 100644 --- a/src/CodeIndex/Indexer/References/ReferenceExtractor.cs +++ b/src/CodeIndex/Indexer/References/ReferenceExtractor.cs @@ -955,7 +955,7 @@ private static List BuildReferenceContainerCandidates(IReadOnlyLis .Where(symbol => symbol.BodyStartLine != null && symbol.BodyEndLine != null && (IsFunctionLikeSymbolKind(symbol.Kind) || symbol.Kind == "hook" || symbol.Kind == "accessor" || symbol.Kind == "class" || symbol.Kind == "struct" || symbol.Kind == "namespace" - || symbol.Kind == "object" || symbol.Kind == "property" || symbol.Kind == "class_hook")) + || symbol.Kind == "object" || symbol.Kind == "property" || symbol.Kind == "heading" || symbol.Kind == "class_hook")) .OrderBy(symbol => (symbol.BodyEndLine ?? symbol.EndLine) - (symbol.BodyStartLine ?? symbol.StartLine)) .ToList(); diff --git a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.cs b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.cs index 5adccfa54e..da911e2365 100644 --- a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.cs +++ b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.cs @@ -15,6 +15,7 @@ namespace CodeIndex.Indexer; public static partial class SymbolExtractor { public const int DefaultContractVersion = 1; + public const int ExpandedLanguageContractVersion = 2; public const int CSharpContractVersion = 2; private static readonly Regex GraphQLInputBlockRegex = new( @"^\s*(?:extend\s+)?input\s+(?\w+)[^{]*\{(?.*?)^\s*\}", @@ -47,6 +48,7 @@ public static int GetContractVersion(string? lang) { null or "" => DefaultContractVersion, "csharp" => CSharpContractVersion, + "cmake" or "graphql" or "html" or "json" or "justfile" or "markdown" or "msbuild" or "yaml" => ExpandedLanguageContractVersion, _ => DefaultContractVersion, }; } diff --git a/tests/CodeIndex.Tests/DbReaderTests.cs b/tests/CodeIndex.Tests/DbReaderTests.cs index bafbcc215f..bc365e7792 100644 --- a/tests/CodeIndex.Tests/DbReaderTests.cs +++ b/tests/CodeIndex.Tests/DbReaderTests.cs @@ -13571,9 +13571,9 @@ public void Ｒｕｎ() { } [Fact] public void AnalyzeSymbol_UnsupportedLanguage_ReportsGraphSupportMetadata() { - var analysis = _reader.AnalyzeSymbol("Heading", limit: 5, lang: "markdown"); + var analysis = _reader.AnalyzeSymbol("Heading", limit: 5, lang: "toml"); - Assert.Equal("markdown", analysis.GraphLanguage); + Assert.Equal("toml", analysis.GraphLanguage); Assert.False(analysis.GraphSupported); Assert.Contains("not indexed", analysis.GraphSupportReason); Assert.Empty(analysis.Definitions); diff --git a/tests/CodeIndex.Tests/IndexCommandRunnerTests.cs b/tests/CodeIndex.Tests/IndexCommandRunnerTests.cs index 94477f5da4..0ed586e95a 100644 --- a/tests/CodeIndex.Tests/IndexCommandRunnerTests.cs +++ b/tests/CodeIndex.Tests/IndexCommandRunnerTests.cs @@ -3000,8 +3000,8 @@ long CountReferences() var writer = new DbWriter(db.Connection); var fileId = writer.UpsertFile(new FileRecord { - Path = "docs/readme.md", - Lang = "markdown", + Path = "docs/readme.toml", + Lang = "toml", Size = 12, Lines = 1, Modified = new DateTime(2026, 1, 1, 0, 0, 0, DateTimeKind.Utc), @@ -3246,8 +3246,8 @@ int CountReferences() var writer = new DbWriter(db.Connection); var fileId = writer.UpsertFile(new FileRecord { - Path = "docs/readme.md", - Lang = "markdown", + Path = "docs/readme.toml", + Lang = "toml", Size = 12, Lines = 1, Modified = new DateTime(2026, 1, 1, 0, 0, 0, DateTimeKind.Utc), @@ -7732,6 +7732,65 @@ def target(): } } + [Fact] + public void Run_FilesUpdate_ReindexesUnchangedJsonFileWhenExpandedLanguageExtractorVersionChanged() + { + var projectRoot = CreateTempProject(); + try + { + var sourcePath = Path.Combine(projectRoot, "package.json"); + File.WriteAllText( + sourcePath, + """ + { + "scripts": { + "build": "dotnet build" + } + } + """); + + var initialExitCode = IndexCommandRunner.Run([projectRoot, "--json"], _jsonOptions); + Assert.Equal(CommandExitCodes.Success, initialExitCode); + + var dbPath = Path.Combine(projectRoot, ".cdidx", "codeindex.db"); + using (var conn = OpenNonPoolingConnection(dbPath)) + { + conn.Open(); + using var cmd = conn.CreateCommand(); + cmd.CommandText = """ + DELETE FROM symbols WHERE file_id = (SELECT id FROM files WHERE path = 'package.json'); + DELETE FROM symbol_references WHERE file_id = (SELECT id FROM files WHERE path = 'package.json'); + INSERT OR REPLACE INTO codeindex_meta(key, value) VALUES('symbol_extractor_version_json', '1'); + """; + cmd.ExecuteNonQuery(); + } + + var (exitCode, json) = RunAndCaptureJson([projectRoot, "--files", sourcePath, "--json"]); + + Assert.Equal(CommandExitCodes.Success, exitCode); + Assert.Equal("success", json.GetProperty("status").GetString()); + Assert.Equal(1, json.GetProperty("summary").GetProperty("updated").GetInt32()); + Assert.Equal(0, json.GetProperty("summary").GetProperty("skipped").GetInt32()); + + using var verify = OpenNonPoolingConnection(dbPath); + verify.Open(); + + using var symbolCmd = verify.CreateCommand(); + symbolCmd.CommandText = """ + SELECT COUNT(*) + FROM symbols s + JOIN files f ON f.id = s.file_id + WHERE f.path = 'package.json' + AND s.name = 'scripts.build' + """; + Assert.Equal(1L, (long)symbolCmd.ExecuteScalar()!); + } + finally + { + DeleteDirectory(projectRoot); + } + } + [Fact] public void Run_FullScan_RestampsExtractorVersionWhenOnlyStaleLanguageWasReindexed() { diff --git a/tests/CodeIndex.Tests/McpServerTests.cs b/tests/CodeIndex.Tests/McpServerTests.cs index 4fc503d416..82590d4bb4 100644 --- a/tests/CodeIndex.Tests/McpServerTests.cs +++ b/tests/CodeIndex.Tests/McpServerTests.cs @@ -4826,10 +4826,10 @@ public void ToolsCall_AnalyzeSymbol_ReturnsBundledContext() [Fact] public void ToolsCall_AnalyzeSymbol_UnsupportedLanguage_ReturnsGraphSupportHint() { - var request = JsonNode.Parse("""{"jsonrpc":"2.0","id":1,"method":"tools/call","params":{"name":"analyze_symbol","arguments":{"query":"Heading","lang":"markdown"}}}""")!; + var request = JsonNode.Parse("""{"jsonrpc":"2.0","id":1,"method":"tools/call","params":{"name":"analyze_symbol","arguments":{"query":"Heading","lang":"toml"}}}""")!; var response = _server.HandleMessage(request)!; - Assert.Equal("markdown", response["result"]!["structuredContent"]!["graph_language"]!.GetValue()); + Assert.Equal("toml", response["result"]!["structuredContent"]!["graph_language"]!.GetValue()); Assert.False(response["result"]!["structuredContent"]!["graph_supported"]!.GetValue()); Assert.Contains("Use search, definition, excerpt, or files instead.", response["result"]!["structuredContent"]!["graph_support_reason"]!.GetValue()); } @@ -5172,10 +5172,10 @@ public void ToolsCall_References_MaxLineWidthZeroDisablesTruncation() [Fact] public void ToolsCall_References_UnsupportedLanguage_ReturnsGraphSupportHint() { - var request = JsonNode.Parse("""{"jsonrpc":"2.0","id":1,"method":"tools/call","params":{"name":"references","arguments":{"query":"Run","lang":"markdown"}}}""")!; + var request = JsonNode.Parse("""{"jsonrpc":"2.0","id":1,"method":"tools/call","params":{"name":"references","arguments":{"query":"Run","lang":"toml"}}}""")!; var response = _server.HandleMessage(request)!; - Assert.Equal("markdown", response["result"]!["structuredContent"]!["graph_language"]!.GetValue()); + Assert.Equal("toml", response["result"]!["structuredContent"]!["graph_language"]!.GetValue()); Assert.False(response["result"]!["structuredContent"]!["graph_supported"]!.GetValue()); Assert.Contains("not indexed", response["result"]!["structuredContent"]!["graph_support_reason"]!.GetValue()); } @@ -5373,10 +5373,10 @@ public enum Status [Fact] public void ToolsCall_Callers_UnsupportedLanguage_ReturnsGraphSupportHint() { - var request = JsonNode.Parse("""{"jsonrpc":"2.0","id":1,"method":"tools/call","params":{"name":"callers","arguments":{"query":"Run","lang":"markdown"}}}""")!; + var request = JsonNode.Parse("""{"jsonrpc":"2.0","id":1,"method":"tools/call","params":{"name":"callers","arguments":{"query":"Run","lang":"toml"}}}""")!; var response = _server.HandleMessage(request)!; - Assert.Equal("markdown", response["result"]!["structuredContent"]!["graph_language"]!.GetValue()); + Assert.Equal("toml", response["result"]!["structuredContent"]!["graph_language"]!.GetValue()); Assert.False(response["result"]!["structuredContent"]!["graph_supported"]!.GetValue()); Assert.Contains("not indexed", response["result"]!["structuredContent"]!["graph_support_reason"]!.GetValue()); } @@ -5462,10 +5462,10 @@ public enum Status [Fact] public void ToolsCall_Callees_UnsupportedLanguage_ReturnsGraphSupportHint() { - var request = JsonNode.Parse("""{"jsonrpc":"2.0","id":1,"method":"tools/call","params":{"name":"callees","arguments":{"query":"Run","lang":"markdown"}}}""")!; + var request = JsonNode.Parse("""{"jsonrpc":"2.0","id":1,"method":"tools/call","params":{"name":"callees","arguments":{"query":"Run","lang":"toml"}}}""")!; var response = _server.HandleMessage(request)!; - Assert.Equal("markdown", response["result"]!["structuredContent"]!["graph_language"]!.GetValue()); + Assert.Equal("toml", response["result"]!["structuredContent"]!["graph_language"]!.GetValue()); Assert.False(response["result"]!["structuredContent"]!["graph_supported"]!.GetValue()); Assert.Contains("not indexed", response["result"]!["structuredContent"]!["graph_support_reason"]!.GetValue()); } @@ -6535,12 +6535,12 @@ public void ToolsCall_AnalyzeSymbol_ExactOnReadOnlyLegacyDb_WithMissingSymbolFal [Fact] public void ToolsCall_AnalyzeSymbol_ExactOnReadOnlyLegacyDb_UnsupportedGraphLanguage_SkipsGraphDegradedSignal() { - InsertIndexedFile("docs/guide.md", "markdown", "# Heading\n\nSee also `Run`.\n"); + InsertIndexedFile("docs/guide.toml", "toml", "title = \"Heading\"\nrun = \"Run\"\n"); ForceLegacyExactFallbackMode(); DropGraphExactFallbackIndexes(); using var readOnlyServer = new McpServer(new Uri(_dbPath).AbsoluteUri + "?immutable=1", ConsoleUi.LoadVersion()); - var request = JsonNode.Parse("""{"jsonrpc":"2.0","id":1,"method":"tools/call","params":{"name":"analyze_symbol","arguments":{"query":"Heading","lang":"markdown","exact":true}}}""")!; + var request = JsonNode.Parse("""{"jsonrpc":"2.0","id":1,"method":"tools/call","params":{"name":"analyze_symbol","arguments":{"query":"Heading","lang":"toml","exact":true}}}""")!; var response = readOnlyServer.HandleMessage(request)!; var structured = response["result"]!["structuredContent"]!; @@ -6554,7 +6554,7 @@ public void ToolsCall_AnalyzeSymbol_ExactOnReadOnlyLegacyDb_UnsupportedGraphLang [Fact] public void ToolsCall_AnalyzeSymbol_ExactOnReadOnlyLegacyDb_PathOnlyUnsupportedSlice_SkipsGraphDegradedSignal() { - InsertIndexedFile("docs/guide.md", "markdown", "# Heading\n\nSee also `Run`.\n"); + InsertIndexedFile("docs/guide.toml", "toml", "title = \"Heading\"\nrun = \"Run\"\n"); ForceLegacyExactFallbackMode(); DropGraphExactFallbackIndexes(); using var readOnlyServer = new McpServer(new Uri(_dbPath).AbsoluteUri + "?immutable=1", ConsoleUi.LoadVersion()); @@ -9173,12 +9173,12 @@ public void ToolsCall_Languages_ReturnsCapabilities() Assert.Contains(".S", assembly["extensions"]!.AsArray().Select(e => e!.GetValue())); Assert.Contains("assembler", assembly["aliases"]!.AsArray().Select(e => e!.GetValue())); - // Verify a detection-only language / 検出のみの言語を検証 + // Verify a markup language / markup 言語を検証 var markdown = languages.First(l => l!["lang"]!.GetValue() == "markdown")!; Assert.True(markdown["symbol_extraction"]!.GetValue()); - Assert.False(markdown["reference_extraction"]!.GetValue()); - Assert.False(markdown["graph_queries"]!.GetValue()); - Assert.Contains("missing-references", markdown["capability_gaps"]!.AsArray().Select(e => e!.GetValue())); + Assert.True(markdown["reference_extraction"]!.GetValue()); + Assert.True(markdown["graph_queries"]!.GetValue()); + Assert.DoesNotContain("missing-references", markdown["capability_gaps"]!.AsArray().Select(e => e!.GetValue())); var yaml = languages.First(l => l!["lang"]!.GetValue() == "yaml")!; Assert.True(yaml["symbol_extraction"]!.GetValue()); @@ -9191,7 +9191,8 @@ public void ToolsCall_Languages_ReturnsCapabilities() // の 4 拡張子を MCP languages ツールから返すこと。 var html = languages.First(l => l!["lang"]!.GetValue() == "html")!; Assert.True(html["symbol_extraction"]!.GetValue()); - Assert.False(html["reference_extraction"]!.GetValue()); + Assert.True(html["reference_extraction"]!.GetValue()); + Assert.True(html["graph_queries"]!.GetValue()); var htmlExtensions = html["extensions"]!.AsArray().Select(e => e!.GetValue()).ToList(); Assert.Contains(".html", htmlExtensions); Assert.Contains(".htm", htmlExtensions); diff --git a/tests/CodeIndex.Tests/QueryCommandRunnerGraphTests.cs b/tests/CodeIndex.Tests/QueryCommandRunnerGraphTests.cs index ba4ea5d4ea..408b696dde 100644 --- a/tests/CodeIndex.Tests/QueryCommandRunnerGraphTests.cs +++ b/tests/CodeIndex.Tests/QueryCommandRunnerGraphTests.cs @@ -1716,12 +1716,12 @@ public void RunReferences_UnsupportedLanguageWithoutMatches_PrintsGraphSupportHi var dbPath = TestProjectHelper.CreateProjectDb(projectRoot); var (exitCode, _, stderr) = CaptureConsole(() => QueryCommandRunner.RunReferences( - ["MissingSymbol", "--db", dbPath, "--lang", "markdown"], + ["MissingSymbol", "--db", dbPath, "--lang", "toml"], _jsonOptions)); Assert.Equal(CommandExitCodes.Success, exitCode); Assert.Contains("No references found.", stderr); - Assert.Contains("call-graph queries are not indexed for 'markdown'", stderr); + Assert.Contains("call-graph queries are not indexed for 'toml'", stderr); } finally { diff --git a/tests/CodeIndex.Tests/QueryCommandRunnerInspectTests.cs b/tests/CodeIndex.Tests/QueryCommandRunnerInspectTests.cs index ff3f90653a..0259d31bc8 100644 --- a/tests/CodeIndex.Tests/QueryCommandRunnerInspectTests.cs +++ b/tests/CodeIndex.Tests/QueryCommandRunnerInspectTests.cs @@ -4647,21 +4647,21 @@ public void RunInspect_ExactOnReadOnlyLegacyDb_WithMissingSymbolIndexAndGraphTab [Fact] public void RunInspect_ExactOnReadOnlyLegacyDb_UnsupportedGraphLanguage_DoesNotReportFalseDegradedSignal() { - var projectRoot = TestProjectHelper.CreateTempProject("cdidx_query_runner_inspect_markdown_exact_ok"); + var projectRoot = TestProjectHelper.CreateTempProject("cdidx_query_runner_inspect_toml_exact_ok"); try { var dbPath = TestProjectHelper.CreateProjectDb(projectRoot); TestProjectHelper.InsertIndexedFile( dbPath, - "docs/guide.md", - "markdown", - "# Heading\n\nSee also `Run`.\n"); + "docs/guide.toml", + "toml", + "title = \"Heading\"\nrun = \"Run\"\n"); ForceLegacyExactFallbackMode(dbPath); DropGraphExactFallbackIndexes(dbPath); var readOnlyUri = new Uri(dbPath).AbsoluteUri + "?immutable=1"; var (exitCode, stdout, stderr) = CaptureConsole(() => QueryCommandRunner.RunInspect( - ["Heading", "--db", readOnlyUri, "--exact", "--lang", "markdown", "--json"], + ["Heading", "--db", readOnlyUri, "--exact", "--lang", "toml", "--json"], _jsonOptions)); using var document = ParseJsonOutput(stdout); @@ -4688,9 +4688,9 @@ public void RunInspect_ExactOnReadOnlyLegacyDb_PathOnlyUnsupportedSlice_DoesNotR var dbPath = TestProjectHelper.CreateProjectDb(projectRoot); TestProjectHelper.InsertIndexedFile( dbPath, - "docs/guide.md", - "markdown", - "# Heading\n\nSee also `Run`.\n"); + "docs/guide.toml", + "toml", + "title = \"Heading\"\nrun = \"Run\"\n"); ForceLegacyExactFallbackMode(dbPath); DropGraphExactFallbackIndexes(dbPath); diff --git a/tests/CodeIndex.Tests/QueryCommandRunnerSymbolTests.cs b/tests/CodeIndex.Tests/QueryCommandRunnerSymbolTests.cs index e499945cb4..f2c166611a 100644 --- a/tests/CodeIndex.Tests/QueryCommandRunnerSymbolTests.cs +++ b/tests/CodeIndex.Tests/QueryCommandRunnerSymbolTests.cs @@ -1149,7 +1149,7 @@ public void RunUnused_WithJsonUnsupportedLanguageZeroResults_UsesUnusedSchema() try { var (exitCode, stdout, stderr) = CaptureConsole(() => QueryCommandRunner.RunUnused( - ["--db", dbPath, "--json", "--lang", "markdown"], + ["--db", dbPath, "--json", "--lang", "toml"], _jsonOptions)); using var document = ParseJsonOutput(stdout); diff --git a/tests/CodeIndex.Tests/QueryCommandRunnerTests.cs b/tests/CodeIndex.Tests/QueryCommandRunnerTests.cs index 0e22879667..9ffbca580c 100644 --- a/tests/CodeIndex.Tests/QueryCommandRunnerTests.cs +++ b/tests/CodeIndex.Tests/QueryCommandRunnerTests.cs @@ -2024,7 +2024,7 @@ public void RunLanguages_JsonIndexedOnlyCombinesWithCapabilityFilter() .Select(lang => lang.GetProperty("lang").GetString()) .ToList(); - Assert.Equal(["csharp"], names); + Assert.Equal(["csharp", "markdown"], names); } finally { @@ -2165,11 +2165,11 @@ public void RunLanguages_JsonListsModernNodeModuleExtensions() public void RunLanguages_JsonListsHtmlWithSymbolExtractionAndAllExtensions() { // Pin the #215 surface: `cdidx languages --json` must report html with - // symbol_extraction=true and list all four extensions (.html, .htm, .xhtml, .shtml) - // so AI tools can discover HTML symbol support without indexing first. - // #215 の表面契約を pin: `cdidx languages --json` は html を symbol_extraction=true で - // 返し、`.html` / `.htm` / `.xhtml` / `.shtml` の 4 拡張子を列挙する必要がある。 - // AI ツールがインデックス前でも HTML のシンボル対応を検出できるようにするため。 + // symbol_extraction/reference_extraction=true and list all four extensions + // (.html, .htm, .xhtml, .shtml) so AI tools can discover HTML support without indexing first. + // #215 の表面契約を pin: `cdidx languages --json` は html を symbol_extraction / + // reference_extraction=true で返し、`.html` / `.htm` / `.xhtml` / `.shtml` の 4 拡張子を + // 列挙する必要がある。AI ツールがインデックス前でも HTML 対応を検出できるようにするため。 var (exitCode, stdout, stderr) = CaptureConsole(() => QueryCommandRunner.RunLanguages(["--json"], _jsonOptions)); Assert.Equal(CommandExitCodes.Success, exitCode); @@ -2180,7 +2180,7 @@ public void RunLanguages_JsonListsHtmlWithSymbolExtractionAndAllExtensions() var html = languages.EnumerateArray().First(lang => lang.GetProperty("lang").GetString() == "html"); Assert.True(html.GetProperty("symbol_extraction").GetBoolean()); - Assert.False(html.GetProperty("reference_extraction").GetBoolean()); + Assert.True(html.GetProperty("reference_extraction").GetBoolean()); var extensions = html.GetProperty("extensions").EnumerateArray().Select(ext => ext.GetString()).ToList(); Assert.Contains(".html", extensions); Assert.Contains(".htm", extensions); @@ -2310,6 +2310,18 @@ public void RunLanguages_Json_SearchOnlyBucketsAdvertiseZeroSymbolAndGraphSuppor Assert.Empty(languages[buildAutomation].GetProperty("capability_gaps").EnumerateArray()); } + foreach (var markupSchema in new[] { "graphql", "html", "markdown" }) + { + Assert.True(languages.ContainsKey(markupSchema), $"expected '{markupSchema}' to be listed"); + Assert.True(languages[markupSchema].GetProperty("symbol_extraction").GetBoolean(), + $"{markupSchema} must advertise symbol_extraction=true"); + Assert.True(languages[markupSchema].GetProperty("reference_extraction").GetBoolean(), + $"{markupSchema} must advertise reference_extraction=true"); + Assert.True(languages[markupSchema].GetProperty("graph_queries").GetBoolean(), + $"{markupSchema} must advertise graph_queries=true"); + Assert.Empty(languages[markupSchema].GetProperty("capability_gaps").EnumerateArray()); + } + // Cython owns .pyx / .pxd exclusively; python keeps .py / .pyi / .pyw and Bazel filenames. // Cython は .pyx / .pxd を専有し、python は .py / .pyi / .pyw と Bazel ファイル名を維持。 var cythonExts = languages["cython"].GetProperty("extensions").EnumerateArray() diff --git a/tests/CodeIndex.Tests/ReferenceExtractorTests.cs b/tests/CodeIndex.Tests/ReferenceExtractorTests.cs index 53bd8dc099..6c381331e9 100644 --- a/tests/CodeIndex.Tests/ReferenceExtractorTests.cs +++ b/tests/CodeIndex.Tests/ReferenceExtractorTests.cs @@ -246,6 +246,142 @@ public void Extract_MsBuild_BuildAutomationReferences() && reference.ContainerName == "Generate"); } + [Fact] + public void Extract_GraphQL_MarkupSchemaReferences() + { + const string content = """ + directive @auth on FIELD_DEFINITION + + interface Node { + id: ID! + } + + type User implements Node { + bestFriend: User + avatar: Avatar + } + + union SearchResult = User | Team + + fragment UserCard on User { + ...AvatarFields + bestFriend { id } + } + + query GetUser { + user @auth { + ...UserCard + ... on User { id } + } + } + """; + + var symbols = SymbolExtractor.Extract(1, "graphql", content); + var references = ReferenceExtractor.Extract(1, "graphql", content, symbols); + + Assert.Contains(references, reference => + reference.ReferenceKind == "type_reference" + && reference.SymbolName == "Node" + && reference.ContainerName == "User"); + Assert.Contains(references, reference => + reference.ReferenceKind == "type_reference" + && reference.SymbolName == "Avatar" + && reference.ContainerName == "User"); + Assert.Contains(references, reference => + reference.ReferenceKind == "type_reference" + && reference.SymbolName == "Team" + && reference.ContainerName == "SearchResult"); + Assert.Contains(references, reference => + reference.ReferenceKind == "call" + && reference.SymbolName == "AvatarFields" + && reference.ContainerName == "UserCard"); + Assert.Contains(references, reference => + reference.ReferenceKind == "call" + && reference.SymbolName == "UserCard" + && reference.ContainerName == "GetUser"); + Assert.Contains(references, reference => + reference.ReferenceKind == "call" + && reference.SymbolName == "auth" + && reference.ContainerName == "GetUser"); + Assert.DoesNotContain(references, reference => reference.SymbolName == "ID"); + } + + [Fact] + public void Extract_Html_MarkupReferences() + { + const string content = """ + + + + Details + + + """; + + var symbols = SymbolExtractor.Extract(1, "html", content); + var references = ReferenceExtractor.Extract(1, "html", content, symbols); + + Assert.Contains(references, reference => + reference.ReferenceKind == "import" + && reference.SymbolName == "/assets/app.css"); + Assert.Contains(references, reference => + reference.ReferenceKind == "import" + && reference.SymbolName == "app.js"); + Assert.Contains(references, reference => + reference.ReferenceKind == "import" + && reference.SymbolName == "small.png"); + Assert.Contains(references, reference => + reference.ReferenceKind == "import" + && reference.SymbolName == "large.png"); + Assert.Contains(references, reference => + reference.ReferenceKind == "reference" + && reference.SymbolName == "details"); + Assert.Contains(references, reference => + reference.ReferenceKind == "reference" + && reference.SymbolName == "profile"); + Assert.Contains(references, reference => + reference.ReferenceKind == "call" + && reference.SymbolName == "user-card"); + Assert.DoesNotContain(references, reference => reference.SymbolName == "ignored.png"); + } + + [Fact] + public void Extract_Markdown_MarkupReferences() + { + const string content = """ + # Intro + + See [Guide](docs/guide.md), [Details](#deep-details), and [API][api]. + + ```markdown + [Ignored](ignored.md) + ``` + + [api]: docs/api.md + """; + + var symbols = SymbolExtractor.Extract(1, "markdown", content); + var references = ReferenceExtractor.Extract(1, "markdown", content, symbols); + + Assert.Contains(references, reference => + reference.ReferenceKind == "import" + && reference.SymbolName == "docs/guide.md" + && reference.ContainerName == "Intro"); + Assert.Contains(references, reference => + reference.ReferenceKind == "reference" + && reference.SymbolName == "deep-details" + && reference.ContainerName == "Intro"); + Assert.Contains(references, reference => + reference.ReferenceKind == "reference" + && reference.SymbolName == "api" + && reference.ContainerName == "Intro"); + Assert.Contains(references, reference => + reference.ReferenceKind == "import" + && reference.SymbolName == "docs/api.md" + && reference.ContainerName == "Intro"); + Assert.DoesNotContain(references, reference => reference.SymbolName == "ignored.md"); + } + [Fact] public void Extract_Go_EmitsGoroutineAndChannelReferences() { @@ -4234,7 +4370,7 @@ public void Extract_UnsupportedLanguage_ReturnsEmpty() { const string content = "hello = world"; - var references = ReferenceExtractor.Extract(1, "markdown", content, []); + var references = ReferenceExtractor.Extract(1, "toml", content, []); Assert.Empty(references); } diff --git a/tests/CodeIndex.Tests/golden/status.json b/tests/CodeIndex.Tests/golden/status.json index c53b85d00d..ee9cef41b2 100644 --- a/tests/CodeIndex.Tests/golden/status.json +++ b/tests/CodeIndex.Tests/golden/status.json @@ -32,6 +32,7 @@ "batch", "blazor", "c", + "cmake", "cobol", "commonlisp", "cpp", @@ -45,12 +46,17 @@ "fsharp", "go", "gradle", + "graphql", "haskell", + "html", "java", "javascript", + "justfile", "kotlin", "lua", "makefile", + "markdown", + "msbuild", "objc", "pascal", "perl", From 75b6627f66f617897d4dab98bc2ef533fc4c66a4 Mon Sep 17 00:00:00 2001 From: Widthdom Date: Fri, 12 Jun 2026 17:51:28 +0900 Subject: [PATCH 5/5] Fix JSON contract reindex test fixture (#3482) --- tests/CodeIndex.Tests/IndexCommandRunnerTests.cs | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/CodeIndex.Tests/IndexCommandRunnerTests.cs b/tests/CodeIndex.Tests/IndexCommandRunnerTests.cs index 1da287f475..eb0502313c 100644 --- a/tests/CodeIndex.Tests/IndexCommandRunnerTests.cs +++ b/tests/CodeIndex.Tests/IndexCommandRunnerTests.cs @@ -7922,13 +7922,13 @@ public void Run_FilesUpdate_ReindexesUnchangedJsonFileWhenExpandedLanguageExtrac var projectRoot = CreateTempProject(); try { - var sourcePath = Path.Combine(projectRoot, "package.json"); + var sourcePath = Path.Combine(projectRoot, "settings.json"); File.WriteAllText( sourcePath, """ { - "scripts": { - "build": "dotnet build" + "features": { + "preview": true } } """); @@ -7942,8 +7942,8 @@ public void Run_FilesUpdate_ReindexesUnchangedJsonFileWhenExpandedLanguageExtrac conn.Open(); using var cmd = conn.CreateCommand(); cmd.CommandText = """ - DELETE FROM symbols WHERE file_id = (SELECT id FROM files WHERE path = 'package.json'); - DELETE FROM symbol_references WHERE file_id = (SELECT id FROM files WHERE path = 'package.json'); + DELETE FROM symbols WHERE file_id = (SELECT id FROM files WHERE path = 'settings.json'); + DELETE FROM symbol_references WHERE file_id = (SELECT id FROM files WHERE path = 'settings.json'); INSERT OR REPLACE INTO codeindex_meta(key, value) VALUES('symbol_extractor_version_json', '1'); """; cmd.ExecuteNonQuery(); @@ -7964,8 +7964,8 @@ public void Run_FilesUpdate_ReindexesUnchangedJsonFileWhenExpandedLanguageExtrac SELECT COUNT(*) FROM symbols s JOIN files f ON f.id = s.file_id - WHERE f.path = 'package.json' - AND s.name = 'scripts.build' + WHERE f.path = 'settings.json' + AND s.name = 'features.preview' """; Assert.Equal(1L, (long)symbolCmd.ExecuteScalar()!); }