Expand language capabilities and extraction coverage

changelog.d/unreleased/3482.added.md

@@ -0,0 +1,18 @@
+---
+category: added
+issues:
+  - 3482
+affected:
+  - src/CodeIndex/Indexer/Symbols/SymbolExtractor.cs
+  - src/CodeIndex/Indexer/Symbols/SymbolExtractor.StructuredData.cs
+  - src/CodeIndex/Cli/QueryCommandRunner.cs
+  - USER_GUIDE.md
+---
+
+## English
+
+- **JSON and YAML files now expose structural symbols (#3482)** — `symbols`, `definition`, `outline`, and language capability probes can now see JSON/YAML configuration key paths, while unsupported symbol extractor languages explain the `search` fallback instead of returning an unexplained empty result.
+
+## 日本語
+
+- **JSON と YAML が構造シンボルを公開するようになりました (#3482)** — `symbols`、`definition`、`outline`、言語 capability probe で JSON/YAML の configuration key path が見えるようになり、シンボル抽出未対応言語では空結果だけでなく `search` fallback の説明を返します。

changelog.d/unreleased/3518.changed.md

@@ -0,0 +1,19 @@
+---
+category: changed
+issues:
+  - 3518
+affected:
+  - src/CodeIndex/Cli/QueryCommandRunner.cs
+  - src/CodeIndex/Cli/JsonOutputContracts.cs
+  - src/CodeIndex/Mcp/McpToolHandlers.cs
+  - src/CodeIndex/Mcp/McpServer.cs
+  - USER_GUIDE.md
+---
+
+## English
+
+- **Language capability output now separates references from graph support (#3518)** — `cdidx languages --json` and the MCP `languages` tool now expose `reference_extraction` and `capability_gaps`, and `--capability` can filter missing capability gaps or search-only languages.
+
+## 日本語
+
+- **言語 capability 出力で references と graph 対応を分離しました (#3518)** — `cdidx languages --json` と MCP の `languages` ツールは `reference_extraction` と `capability_gaps` を返すようになり、`--capability` で未対応 gap や search-only 言語を絞り込めます。

changelog.d/unreleased/3525.added.md

@@ -0,0 +1,18 @@
+---
+category: added
+issues:
+  - 3525
+affected:
+  - src/CodeIndex/Indexer/References/ReferenceExtractor.Core.cs
+  - src/CodeIndex/Indexer/References/ReferenceExtractor.State.cs
+  - src/CodeIndex/Indexer/References/Languages/MarkupSchemaReferenceExtractor.cs
+  - USER_GUIDE.md
+---
+
+## English
+
+- **Markup and schema languages now expose references and graph support (#3525)** — GraphQL, HTML, and Markdown now emit conservative references for schema types/fragments/directives, assets/components, links, anchors, and reference definitions.
+
+## 日本語
+
+- **マークアップと schema 言語が references と graph 対応を公開するようになりました (#3525)** — GraphQL、HTML、Markdown で schema type / fragment / directive、asset / component、link、anchor、reference definition の保守的な参照を抽出します。

changelog.d/unreleased/3526.added.md

@@ -0,0 +1,19 @@
+---
+category: added
+issues:
+  - 3526
+affected:
+  - src/CodeIndex/Indexer/Symbols/SymbolExtractor.cs
+  - src/CodeIndex/Indexer/Symbols/SymbolExtractor.BuildAutomation.cs
+  - src/CodeIndex/Indexer/References/ReferenceExtractor.Core.cs
+  - src/CodeIndex/Indexer/References/Languages/BuildAutomationReferenceExtractor.cs
+  - USER_GUIDE.md
+---
+
+## English
+
+- **Build automation files now expose structural symbols and references (#3526)** — CMake, Justfile, and MSBuild files now surface targets, recipes, properties, imports, and dependency references for graph-aware commands and capability probes.
+
+## 日本語
+
+- **ビルド自動化ファイルが構造シンボルと参照を公開するようになりました (#3526)** — CMake、Justfile、MSBuild の target、recipe、property、import、依存参照を graph 対応コマンドと capability probe から扱えるようになりました。

src/CodeIndex/Cli/CliFlagSchema.cs

@@ -273,7 +273,7 @@ private static IReadOnlyList<CliFlag> BuildAll()
             new() { Name = "--sections", ValuePlaceholder = "<tree,languages,hotspots,metrics>", Description = "Map: comma-separated response sections to include", Commands = Set(MapSectionCommands) },
             new() { Name = "--cycles", Description = "Deps: return dependency cycles instead of edge rows", Commands = Set(DependencyCycleCommands) },
             new() { Name = "--indexed-only", Description = "Languages: list only languages present in the current index", Commands = Set(LanguagesFilterCommands) },
-            new() { Name = "--capability", ValuePlaceholder = "<graph|symbols|references>", Description = "Languages: filter by language capability", Commands = Set(LanguagesFilterCommands) },
+            new() { Name = "--capability", ValuePlaceholder = "<graph|references|symbols|missing-graph|missing-references|missing-symbols|search-only>", Description = "Languages: filter by language capability or capability gap", Commands = Set(LanguagesFilterCommands) },
             new() { Name = "--query", ValuePlaceholder = "<query>", Description = "Literal query", Commands = Set(QueryCommands) },
             new() { Name = "--recipe", ValuePlaceholder = "<name>", Description = "Search: run a built-in audit recipe query set", Commands = Set("search") },
             new() { Name = "--list-recipes", Description = "Search: list built-in audit recipes", Commands = Set("search") },

src/CodeIndex/Cli/ConsoleUi.cs

@@ -117,7 +117,7 @@ private static readonly (string Command, string Usage)[] CommandUsageLines =
         ("export", "cdidx export <archive> [--db <path>] [--json]"),
         ("export", "cdidx export ctags [--output <path>] [--db <path>]"),
         ("import", "cdidx import <archive> [--db <path>] [--prune-paths] [--dry-run|--check] [--json]"),
-        ("languages", "cdidx languages [--db <path>] [--json] [--indexed-only] [--capability <graph|symbols|references>]"),
+        ("languages", "cdidx languages [--db <path>] [--json] [--indexed-only] [--capability <graph|references|symbols|missing-graph|missing-references|missing-symbols|search-only>]"),
         ("batch", "cdidx batch [--db <path>]  # reads JSON string arrays from stdin, one query command per line; max 1,048,576 chars/line and 256 arguments"),
         ("mcp", "cdidx mcp [--db <path>] [--transport stdio|http] [--http-listen <host:port>] [--audit-log <path>] [--audit-log-include-values] [--audit-log-max-bytes <n>] [--suggestion-dedup-threshold <0..1>]"),
         ("lsp", "cdidx lsp [--db <path>]"),

src/CodeIndex/Cli/JsonOutputContracts.cs

@@ -225,7 +225,9 @@ internal sealed record LanguageEntryJsonResult(
     [property: JsonPropertyName("extensions")] List<string> Extensions,
     [property: JsonPropertyName("aliases")] List<string> Aliases,
     [property: JsonPropertyName("symbol_extraction")] bool SymbolExtraction,
-    [property: JsonPropertyName("graph_queries")] bool GraphQueries);
+    [property: JsonPropertyName("reference_extraction")] bool ReferenceExtraction,
+    [property: JsonPropertyName("graph_queries")] bool GraphQueries,
+    [property: JsonPropertyName("capability_gaps")] List<string> CapabilityGaps);
 
 internal sealed record LanguagesJsonResult(
     [property: JsonPropertyName("languages")] List<LanguageEntryJsonResult> Languages);

src/CodeIndex/Cli/QueryCommandRunner.cs

@@ -42,6 +42,10 @@ public static class QueryCommandRunner
     private const string LanguageCapabilityGraph = "graph";
     private const string LanguageCapabilityReferences = "references";
     private const string LanguageCapabilitySymbols = "symbols";
+    private const string LanguageCapabilityMissingGraph = "missing-graph";
+    private const string LanguageCapabilityMissingReferences = "missing-references";
+    private const string LanguageCapabilityMissingSymbols = "missing-symbols";
+    private const string LanguageCapabilitySearchOnly = "search-only";
     internal static readonly TimeSpan DefaultStaleAfter = TimeSpan.FromHours(24);
     internal static readonly TimeSpan MaxStaleAfter = TimeSpan.FromDays(30);
     internal const string MaxStaleAfterDisplay = "30d";
@@ -2995,6 +2999,7 @@ public static int RunSymbols(string[] cmdArgs, JsonSerializerOptions jsonOptions
                     WriteExactZeroHint(exactZeroHint);
                     WriteKindHint(options.Kind, reader);
                     WriteLangHint(options.Lang, reader);
+                    WriteSymbolExtractionCapabilityHint(options.Lang, reader);
                     WriteZeroResultHints(options, reader);
                 }
                 return ZeroResultExitCode(options);
@@ -6704,15 +6709,23 @@ public static int RunLanguages(string[] cmdArgs, JsonSerializerOptions jsonOptio
         var symbolLangs = SymbolExtractor.GetSupportedLanguages();
         var graphLangs = ReferenceExtractor.GetSupportedLanguages();
 
-        // Build a consolidated view: language -> (extensions, hasSymbols, hasGraph)
-        // 統合ビュー: 言語 -> (拡張子, シンボル対応, グラフ対応)
+        // Build a consolidated view: language -> capability flags and gaps.
+        // 統合ビュー: 言語 -> capability flag と gap。
         var allLangs = new Dictionary<string, LanguageSupportInfo>(StringComparer.Ordinal);
 
         foreach (var (ext, lang) in langExtensions)
         {
             if (!allLangs.TryGetValue(lang, out var info))
             {
-                info = new LanguageSupportInfo([], GetLanguageAliases(lang).ToList(), symbolLangs.Contains(lang), graphLangs.Contains(lang));
+                var hasSymbols = symbolLangs.Contains(lang);
+                var hasReferences = graphLangs.Contains(lang);
+                info = new LanguageSupportInfo(
+                    [],
+                    GetLanguageAliases(lang).ToList(),
+                    hasSymbols,
+                    hasReferences,
+                    hasReferences,
+                    BuildLanguageCapabilityGaps(hasSymbols, hasReferences, hasReferences));
                 allLangs[lang] = info;
             }
             info.Extensions.Add(ext);
@@ -6746,7 +6759,9 @@ int WriteLanguages(IEnumerable<KeyValuePair<string, LanguageSupportInfo>> langua
                     kv.Value.Extensions.OrderBy(e => e).ToList(),
                     kv.Value.Aliases.OrderBy(a => a).ToList(),
                     kv.Value.Symbols,
-                    kv.Value.Graph)).ToList();
+                    kv.Value.References,
+                    kv.Value.Graph,
+                    kv.Value.CapabilityGaps)).ToList();
                 Console.WriteLine(JsonSerializer.Serialize(new LanguagesJsonResult(entries), CliJsonSerializerContextFactory.Create(jsonOptions).LanguagesJsonResult));
             }
             else
@@ -6757,25 +6772,28 @@ int WriteLanguages(IEnumerable<KeyValuePair<string, LanguageSupportInfo>> langua
                 // Symbols / Graph 列が拡張子文字列に埋もれないようにする。
                 const int ExtensionColumnWidth = 36;
                 const int AliasColumnWidth = 12;
-                Console.WriteLine($"{"Language",-14} {"Extensions",-36} {"Aliases",-12} {"Symbols",-9} {"Graph",-7}");
-                Console.WriteLine(new string('-', 79));
+                Console.WriteLine($"{"Language",-14} {"Extensions",-36} {"Aliases",-12} {"Symbols",-9} {"Refs",-5} {"Graph",-7}");
+                Console.WriteLine(new string('-', 85));
                 foreach (var (lang, info) in filtered)
                 {
                     var exts = string.Join(" ", info.Extensions.OrderBy(e => e));
                     var aliases = string.Join(" ", info.Aliases.OrderBy(a => a));
                     var aliasCell = string.IsNullOrWhiteSpace(aliases) ? "-" : aliases;
                     var sym = info.Symbols ? "yes" : "-";
+                    var refs = info.References ? "yes" : "-";
                     var graph = info.Graph ? "yes" : "-";
                     if (exts.Length <= ExtensionColumnWidth && aliases.Length <= AliasColumnWidth)
                     {
-                        Console.WriteLine($"{lang,-14} {exts,-36} {aliasCell,-12} {sym,-9} {graph,-7}");
+                        Console.WriteLine($"{lang,-14} {exts,-36} {aliasCell,-12} {sym,-9} {refs,-5} {graph,-7}");
                     }
                     else
                     {
-                        Console.WriteLine($"{lang,-14} {"",-36} {"",-12} {sym,-9} {graph,-7}");
+                        Console.WriteLine($"{lang,-14} {"",-36} {"",-12} {sym,-9} {refs,-5} {graph,-7}");
                         Console.WriteLine($"  Extensions: {exts}");
                         if (!string.IsNullOrWhiteSpace(aliases))
                             Console.WriteLine($"  Aliases: {aliases}");
+                        if (info.CapabilityGaps.Count > 0)
+                            Console.WriteLine($"  Gaps: {string.Join(", ", info.CapabilityGaps)}");
                     }
                 }
                 Console.Error.WriteLine($"\n({filtered.Count} languages)");
@@ -6785,20 +6803,44 @@ int WriteLanguages(IEnumerable<KeyValuePair<string, LanguageSupportInfo>> langua
         }
     }
 
-    private sealed record LanguageSupportInfo(List<string> Extensions, List<string> Aliases, bool Symbols, bool Graph);
+    private sealed record LanguageSupportInfo(List<string> Extensions, List<string> Aliases, bool Symbols, bool References, bool Graph, List<string> CapabilityGaps);
 
     private static bool LanguageMatchesCapability(LanguageSupportInfo language, string capability)
         => capability switch
         {
             LanguageCapabilitySymbols => language.Symbols,
-            LanguageCapabilityGraph or LanguageCapabilityReferences => language.Graph,
+            LanguageCapabilityReferences => language.References,
+            LanguageCapabilityGraph => language.Graph,
+            LanguageCapabilityMissingSymbols => !language.Symbols,
+            LanguageCapabilityMissingReferences => !language.References,
+            LanguageCapabilityMissingGraph => !language.Graph,
+            LanguageCapabilitySearchOnly => !language.Symbols && !language.References && !language.Graph,
             _ => false,
         };
 
     private static bool TryNormalizeLanguageCapability(string value, out string capability)
     {
         capability = value.Trim().ToLowerInvariant();
-        return capability is LanguageCapabilityGraph or LanguageCapabilityReferences or LanguageCapabilitySymbols;
+        return capability is
+            LanguageCapabilityGraph or
+            LanguageCapabilityReferences or
+            LanguageCapabilitySymbols or
+            LanguageCapabilityMissingGraph or
+            LanguageCapabilityMissingReferences or
+            LanguageCapabilityMissingSymbols or
+            LanguageCapabilitySearchOnly;
+    }
+
+    private static List<string> BuildLanguageCapabilityGaps(bool symbols, bool references, bool graph)
+    {
+        var gaps = new List<string>();
+        if (!symbols)
+            gaps.Add("missing-symbols");
+        if (!references)
+            gaps.Add("missing-references");
+        if (!graph)
+            gaps.Add("missing-graph");
+        return gaps;
     }
 
     public static QueryCommandOptions ParseArgs(
@@ -7081,7 +7123,7 @@ void WarnIfDuplicateSingleValueOption(string canonicalName, string newValue)
                     }
                     else
                     {
-                        AddParseError($"Error: unsupported --capability value '{ConsoleUi.FormatBoundedValue(capabilityValue)}'. Use graph, symbols, or references.");
+                        AddParseError($"Error: unsupported --capability value '{ConsoleUi.FormatBoundedValue(capabilityValue)}'. Use graph, references, symbols, missing-graph, missing-references, missing-symbols, or search-only.");
                     }
                     break;
                 case "--format":
@@ -10275,6 +10317,20 @@ private static void WriteLangHint(string? lang, DbReader reader)
             Console.Error.WriteLine($"Did you mean: --lang {suggestion}?");
     }
 
+    private static void WriteSymbolExtractionCapabilityHint(string? lang, DbReader reader)
+    {
+        if (string.IsNullOrWhiteSpace(lang))
+            return;
+        if (SymbolExtractor.GetSupportedLanguages().Contains(lang, StringComparer.Ordinal))
+            return;
+
+        var status = reader.GetStatus();
+        if (status.Languages.Count == 0 || !status.Languages.ContainsKey(lang))
+            return;
+
+        Console.Error.WriteLine($"Hint: '{lang}' is indexed for full-text search, but symbol extraction is not available for that language. Use `cdidx search <query> --lang {lang}` for text matches or `cdidx languages --capability missing-symbols` to audit capability gaps.");
+    }
+
     // All valid symbol kinds emitted by SymbolExtractor / SymbolExtractor が出力する全有効シンボル種別
     private static readonly string[] AllValidKinds =
         KnownSymbolKindFilters.OrderBy(kind => kind, StringComparer.Ordinal).ToArray();