From 694841ec95ace8615b27086a6ab5e80994250cc4 Mon Sep 17 00:00:00 2001 From: Widthdom Date: Sun, 31 May 2026 13:30:12 +0900 Subject: [PATCH 1/6] Add YAML symbol pattern sidecars (#1756) --- changelog.d/unreleased/1756.added.md | 16 ++++ .../ConfiguredSymbolExtractor.cs | 50 +++++++++++ .../Extensibility/ExtractorPluginRegistry.cs | 90 +++++++++++++++++++ tests/CodeIndex.Tests/SymbolExtractorTests.cs | 33 +++++++ 4 files changed, 189 insertions(+) create mode 100644 changelog.d/unreleased/1756.added.md create mode 100644 src/CodeIndex/Indexer/Extensibility/ConfiguredSymbolExtractor.cs diff --git a/changelog.d/unreleased/1756.added.md b/changelog.d/unreleased/1756.added.md new file mode 100644 index 0000000000..b4a414e57d --- /dev/null +++ b/changelog.d/unreleased/1756.added.md @@ -0,0 +1,16 @@ +--- +category: added +issues: + - 1756 +affected: + - src/CodeIndex/Indexer/Extensibility/ConfiguredSymbolExtractor.cs + - src/CodeIndex/Indexer/Extensibility/ExtractorPluginRegistry.cs +--- + +## English + +- **Added YAML-backed custom symbol patterns (#1756)** — `cdidx` now loads regex symbol extractors from `.cdidx/patterns/*.yaml` and `~/.config/cdidx/patterns/*.yaml` so lightweight language patterns can live outside the binary. + +## 日本語 + +- **YAML ベースのカスタムシンボルパターンを追加しました (#1756)** — `.cdidx/patterns/*.yaml` と `~/.config/cdidx/patterns/*.yaml` から regex symbol extractor を読み込めるようになり、軽量な言語パターンをバイナリ外に置けます。 diff --git a/src/CodeIndex/Indexer/Extensibility/ConfiguredSymbolExtractor.cs b/src/CodeIndex/Indexer/Extensibility/ConfiguredSymbolExtractor.cs new file mode 100644 index 0000000000..3ea1bba37f --- /dev/null +++ b/src/CodeIndex/Indexer/Extensibility/ConfiguredSymbolExtractor.cs @@ -0,0 +1,50 @@ +using System.Text.RegularExpressions; +using CodeIndex.Models; + +namespace CodeIndex.Indexer.Extensibility; + +internal sealed class ConfiguredSymbolExtractor( + string language, + IReadOnlyCollection fileExtensions, + IReadOnlyList patterns) : ISymbolExtractor +{ + internal sealed record PatternRule(string Kind, Regex Regex); + + public string Language { get; } = language; + + public IReadOnlyCollection FileExtensions { get; } = fileExtensions; + + public IReadOnlyList Extract(long fileId, string source, ExtractionContext context) + { + var symbols = new List(); + var lineNumber = 0; + foreach (var line in source.Replace("\r\n", "\n", StringComparison.Ordinal).Replace('\r', '\n').Split('\n')) + { + lineNumber++; + foreach (var pattern in patterns) + { + var match = pattern.Regex.Match(line); + if (!match.Success) + continue; + + var name = match.Groups["name"].Success ? match.Groups["name"].Value : match.Value.Trim(); + if (string.IsNullOrWhiteSpace(name)) + continue; + + symbols.Add(new SymbolRecord + { + FileId = fileId, + Kind = pattern.Kind, + Name = name, + Line = lineNumber, + StartLine = lineNumber, + EndLine = lineNumber, + Signature = line.Trim(), + }); + break; + } + } + + return symbols; + } +} diff --git a/src/CodeIndex/Indexer/Extensibility/ExtractorPluginRegistry.cs b/src/CodeIndex/Indexer/Extensibility/ExtractorPluginRegistry.cs index e2a531bf81..c5feb01c5a 100644 --- a/src/CodeIndex/Indexer/Extensibility/ExtractorPluginRegistry.cs +++ b/src/CodeIndex/Indexer/Extensibility/ExtractorPluginRegistry.cs @@ -1,5 +1,6 @@ using System.Reflection; using System.Runtime.Loader; +using System.Text.RegularExpressions; namespace CodeIndex.Indexer.Extensibility; @@ -87,6 +88,16 @@ internal static void ResetForTests() } } + internal static void ReloadForTests() + { + lock (Gate) + { + SymbolExtractors.Clear(); + ReferenceExtractors.Clear(); + pluginsLoaded = false; + } + } + private static void EnsurePluginsLoaded() { if (Volatile.Read(ref pluginsLoaded)) @@ -99,6 +110,8 @@ private static void EnsurePluginsLoaded() foreach (var pluginPath in EnumeratePluginAssemblyPaths()) TryLoadPlugin(pluginPath); + foreach (var patternPath in EnumeratePatternConfigPaths()) + TryLoadPatternConfig(patternPath); pluginsLoaded = true; } @@ -125,6 +138,83 @@ private static IEnumerable EnumeratePluginDirectories() yield return Path.Combine(home, ".cdidx", "plugins"); } + private static IEnumerable EnumeratePatternConfigPaths() + { + foreach (var directory in EnumeratePatternDirectories()) + { + if (!Directory.Exists(directory)) + continue; + + foreach (var path in Directory.EnumerateFiles(directory, "*.yaml", SearchOption.TopDirectoryOnly)) + yield return path; + foreach (var path in Directory.EnumerateFiles(directory, "*.yml", SearchOption.TopDirectoryOnly)) + yield return path; + } + } + + private static IEnumerable EnumeratePatternDirectories() + { + yield return Path.Combine(Environment.CurrentDirectory, ".cdidx", "patterns"); + + var home = Environment.GetFolderPath(Environment.SpecialFolder.UserProfile); + if (!string.IsNullOrWhiteSpace(home)) + yield return Path.Combine(home, ".config", "cdidx", "patterns"); + } + + private static void TryLoadPatternConfig(string path) + { + try + { + var language = string.Empty; + var extensions = new List(); + var patterns = new List(); + string? pendingKind = null; + foreach (var rawLine in File.ReadLines(path)) + { + var line = rawLine.Trim(); + if (line.Length == 0 || line.StartsWith('#')) + continue; + + if (TryReadScalar(line, "language", out var value)) + { + language = NormalizePluginLanguage(value); + } + else if (TryReadScalar(line.TrimStart('-').Trim(), "extension", out value)) + { + extensions.Add(NormalizePluginExtension(value) ?? value); + } + else if (TryReadScalar(line.TrimStart('-').Trim(), "kind", out value)) + { + pendingKind = value.Trim(); + } + else if (TryReadScalar(line.TrimStart('-').Trim(), "regex", out value) && pendingKind != null) + { + patterns.Add(new ConfiguredSymbolExtractor.PatternRule( + pendingKind, + new Regex(value, RegexOptions.Compiled | RegexOptions.CultureInvariant))); + pendingKind = null; + } + } + + if (language.Length > 0 && patterns.Count > 0) + Register(new ConfiguredSymbolExtractor(language, extensions, patterns)); + } + catch + { + // Pattern sidecars are best-effort like binary plugins. + } + } + + private static bool TryReadScalar(string line, string key, out string value) + { + value = string.Empty; + var prefix = key + ":"; + if (!line.StartsWith(prefix, StringComparison.OrdinalIgnoreCase)) + return false; + value = line[prefix.Length..].Trim().Trim('"', '\'').Replace("\\\\", "\\", StringComparison.Ordinal); + return value.Length > 0; + } + private static void TryLoadPlugin(string pluginPath) { try diff --git a/tests/CodeIndex.Tests/SymbolExtractorTests.cs b/tests/CodeIndex.Tests/SymbolExtractorTests.cs index b4107e5a3f..68c066785b 100644 --- a/tests/CodeIndex.Tests/SymbolExtractorTests.cs +++ b/tests/CodeIndex.Tests/SymbolExtractorTests.cs @@ -44,6 +44,39 @@ public void Extract_CustomSymbolPlugin_HandlesUnsupportedLanguage() } } + [Fact] + public void Extract_ConfiguredPatternYaml_HandlesOutOfTreeLanguage() + { + lock (TestConsoleLock.Gate) + { + var tempDir = Path.Combine(Path.GetTempPath(), $"cdidx_patterns_{Guid.NewGuid():N}"); + var originalDirectory = Environment.CurrentDirectory; + try + { + Directory.CreateDirectory(Path.Combine(tempDir, ".cdidx", "patterns")); + File.WriteAllText( + Path.Combine(tempDir, ".cdidx", "patterns", "toydsl.yaml"), + "language: \"toydsl\"\nextensions:\n - extension: \".toy\"\npatterns:\n - kind: \"class\"\n regex: \"^entity (?\\\\w+)\"\n"); + Environment.CurrentDirectory = tempDir; + ExtractorPluginRegistry.ReloadForTests(); + + var symbols = SymbolExtractor.Extract(2, "toydsl", "entity Widget", "demo.toy"); + + var symbol = Assert.Single(symbols); + Assert.Equal("class", symbol.Kind); + Assert.Equal("Widget", symbol.Name); + Assert.Equal("toydsl", FileIndexer.DetectLanguage("demo.toy")); + } + finally + { + ExtractorPluginRegistry.ResetForTests(); + Environment.CurrentDirectory = originalDirectory; + if (Directory.Exists(tempDir)) + Directory.Delete(tempDir, recursive: true); + } + } + } + [Fact] public void Extract_CsharpFileScopedNamespace_DoesNotEnterMemberHeaderMerge() { From 6c08d91ae032c7b61dd36ac2d273e443a68db21c Mon Sep 17 00:00:00 2001 From: Widthdom Date: Sun, 31 May 2026 13:30:23 +0900 Subject: [PATCH 2/6] Add language map overrides (#1757) --- changelog.d/unreleased/1757.added.md | 16 ++++ src/CodeIndex/Indexer/Scanning/FileIndexer.cs | 21 +++++ .../Indexer/Scanning/LanguageMapOverrides.cs | 76 +++++++++++++++++++ tests/CodeIndex.Tests/FileIndexerTests.cs | 27 +++++++ 4 files changed, 140 insertions(+) create mode 100644 changelog.d/unreleased/1757.added.md create mode 100644 src/CodeIndex/Indexer/Scanning/LanguageMapOverrides.cs diff --git a/changelog.d/unreleased/1757.added.md b/changelog.d/unreleased/1757.added.md new file mode 100644 index 0000000000..f51f101a53 --- /dev/null +++ b/changelog.d/unreleased/1757.added.md @@ -0,0 +1,16 @@ +--- +category: added +issues: + - 1757 +affected: + - src/CodeIndex/Indexer/Scanning/FileIndexer.cs + - src/CodeIndex/Indexer/Scanning/LanguageMapOverrides.cs +--- + +## English + +- **Added per-user and per-workspace language-map overrides (#1757)** — extension aliases from `~/.config/cdidx/langmap.yaml` and `.cdidx-langmap.yaml` now override built-in language detection, including compound suffixes such as `.kts.in`. + +## 日本語 + +- **ユーザー単位・workspace 単位の language-map override を追加しました (#1757)** — `~/.config/cdidx/langmap.yaml` と `.cdidx-langmap.yaml` の拡張子 alias が組み込み言語判定を上書きし、`.kts.in` のような複合サフィックスにも対応します。 diff --git a/src/CodeIndex/Indexer/Scanning/FileIndexer.cs b/src/CodeIndex/Indexer/Scanning/FileIndexer.cs index a3b737ee30..f72a17f2f8 100644 --- a/src/CodeIndex/Indexer/Scanning/FileIndexer.cs +++ b/src/CodeIndex/Indexer/Scanning/FileIndexer.cs @@ -1084,6 +1084,8 @@ public static IReadOnlyDictionary GetLanguageExtensions() merged.TryAdd($"{prefix}", lang); foreach (var (extension, lang) in ExtractorPluginRegistry.LanguageExtensions) merged.TryAdd(extension, lang); + foreach (var (extension, lang) in LanguageMapOverrides.LoadEffectiveMap()) + merged[extension] = lang; return merged; } @@ -1117,6 +1119,9 @@ internal static LanguageDetectionResult TryDetectLanguage(string filePath, strin } var ext = Path.GetExtension(filePath); + if (TryDetectLanguageOverride(filePath, out var overrideLang)) + return new LanguageDetectionResult(FileProbeStatus.Supported, overrideLang); + if (LangMap.TryGetValue(ext, out var lang)) { if (lang == "c" && string.Equals(ext, ".h", StringComparison.OrdinalIgnoreCase) && !string.IsNullOrEmpty(content)) @@ -1138,6 +1143,22 @@ internal static LanguageDetectionResult TryDetectLanguage(string filePath, strin return TryDetectLanguageFromShebang(filePath); } + private static bool TryDetectLanguageOverride(string filePath, out string language) + { + language = string.Empty; + var fileName = Path.GetFileName(filePath); + foreach (var (extension, mappedLanguage) in LanguageMapOverrides.LoadEffectiveMap()) + { + if (fileName.EndsWith(extension, StringComparison.OrdinalIgnoreCase)) + { + language = mappedLanguage; + return true; + } + } + + return false; + } + private static string? TryDetectCppHeaderLanguage(string content) { const int maxLines = 200; diff --git a/src/CodeIndex/Indexer/Scanning/LanguageMapOverrides.cs b/src/CodeIndex/Indexer/Scanning/LanguageMapOverrides.cs new file mode 100644 index 0000000000..a88604e7fa --- /dev/null +++ b/src/CodeIndex/Indexer/Scanning/LanguageMapOverrides.cs @@ -0,0 +1,76 @@ +namespace CodeIndex.Indexer; + +internal static class LanguageMapOverrides +{ + internal const string WorkspaceFileName = ".cdidx-langmap.yaml"; + + internal static IReadOnlyDictionary LoadEffectiveMap() + { + var map = new Dictionary(StringComparer.OrdinalIgnoreCase); + foreach (var path in EnumerateConfigPaths()) + LoadInto(path, map); + return map; + } + + private static IEnumerable EnumerateConfigPaths() + { + var home = Environment.GetFolderPath(Environment.SpecialFolder.UserProfile); + if (!string.IsNullOrWhiteSpace(home)) + yield return Path.Combine(home, ".config", "cdidx", "langmap.yaml"); + + var directory = Environment.CurrentDirectory; + while (!string.IsNullOrEmpty(directory)) + { + var candidate = Path.Combine(directory, WorkspaceFileName); + if (File.Exists(candidate)) + { + yield return candidate; + yield break; + } + + directory = Directory.GetParent(directory)?.FullName ?? string.Empty; + } + } + + private static void LoadInto(string path, Dictionary target) + { + if (!File.Exists(path)) + return; + + string? pendingExtension = null; + foreach (var rawLine in File.ReadLines(path)) + { + var line = rawLine.Trim(); + if (line.Length == 0 || line.StartsWith('#')) + continue; + + if (TryReadScalar(line.TrimStart('-').Trim(), "extension", out var value)) + { + pendingExtension = NormalizeExtension(value); + continue; + } + + if (TryReadScalar(line.TrimStart('-').Trim(), "language", out value) && pendingExtension != null) + { + target[pendingExtension] = value.Trim().ToLowerInvariant(); + pendingExtension = null; + } + } + } + + private static bool TryReadScalar(string line, string key, out string value) + { + value = string.Empty; + var prefix = key + ":"; + if (!line.StartsWith(prefix, StringComparison.OrdinalIgnoreCase)) + return false; + value = line[prefix.Length..].Trim().Trim('"', '\''); + return value.Length > 0; + } + + private static string NormalizeExtension(string extension) + { + extension = extension.Trim().ToLowerInvariant(); + return extension.StartsWith(".", StringComparison.Ordinal) ? extension : "." + extension; + } +} diff --git a/tests/CodeIndex.Tests/FileIndexerTests.cs b/tests/CodeIndex.Tests/FileIndexerTests.cs index 867fe0b024..3d2ab2f932 100644 --- a/tests/CodeIndex.Tests/FileIndexerTests.cs +++ b/tests/CodeIndex.Tests/FileIndexerTests.cs @@ -326,6 +326,33 @@ public void DetectLanguage_KnownExtensions_ReturnsCorrectLang(string filename, s Assert.Equal(expected, FileIndexer.DetectLanguage(filename)); } + [Fact] + public void DetectLanguage_WorkspaceLangMapYaml_AliasesExtension() + { + lock (TestConsoleLock.Gate) + { + var tempDir = Path.Combine(Path.GetTempPath(), $"cdidx_langmap_{Guid.NewGuid():N}"); + var originalDirectory = Environment.CurrentDirectory; + try + { + Directory.CreateDirectory(tempDir); + File.WriteAllText( + Path.Combine(tempDir, LanguageMapOverrides.WorkspaceFileName), + "entries:\n - extension: \".kts.in\"\n language: \"kotlin\"\n"); + Environment.CurrentDirectory = tempDir; + + Assert.Equal("kotlin", FileIndexer.DetectLanguage(Path.Combine(tempDir, "build.kts.in"))); + Assert.Equal("kotlin", FileIndexer.GetLanguageExtensions()[".kts.in"]); + } + finally + { + Environment.CurrentDirectory = originalDirectory; + if (Directory.Exists(tempDir)) + Directory.Delete(tempDir, recursive: true); + } + } + } + [Theory] [InlineData("App.csproj")] [InlineData("Directory.Build.props")] From 008a3b360b5ed237cc31411d3b60f03689a87d73 Mon Sep 17 00:00:00 2001 From: Widthdom Date: Sun, 31 May 2026 13:30:32 +0900 Subject: [PATCH 3/6] Add extractor test harness (#1759) --- changelog.d/unreleased/1759.added.md | 15 +++++ src/CodeIndex/Cli/ProgramRunner.cs | 69 +++++++++++++++++++++ tests/CodeIndex.Tests/ProgramRunnerTests.cs | 31 +++++++++ 3 files changed, 115 insertions(+) create mode 100644 changelog.d/unreleased/1759.added.md diff --git a/changelog.d/unreleased/1759.added.md b/changelog.d/unreleased/1759.added.md new file mode 100644 index 0000000000..9b78f5eab6 --- /dev/null +++ b/changelog.d/unreleased/1759.added.md @@ -0,0 +1,15 @@ +--- +category: added +issues: + - 1759 +affected: + - src/CodeIndex/Cli/ProgramRunner.cs +--- + +## English + +- **Added `cdidx test-extractor` (#1759)** — contributors can run symbol extraction for one file with `--language` and `--file`, emit JSON, and compare against an expected symbols fixture without building a full index. + +## 日本語 + +- **`cdidx test-extractor` を追加しました (#1759)** — contributor は full index を作らず、`--language` と `--file` で単一ファイルの symbol extraction を実行し、JSON 出力や expected symbols fixture との比較ができます。 diff --git a/src/CodeIndex/Cli/ProgramRunner.cs b/src/CodeIndex/Cli/ProgramRunner.cs index f7dcba0acb..ab56bf355a 100644 --- a/src/CodeIndex/Cli/ProgramRunner.cs +++ b/src/CodeIndex/Cli/ProgramRunner.cs @@ -238,6 +238,7 @@ internal static int Run( "validate-config" => CdidxConfigFile.RunValidate(subArgs, jsonOptions), "db" => DbCommandRunner.RunIntegrityCheck(subArgs, jsonOptions), "report" => ReportCommandRunner.Run(subArgs, jsonOptions, appVersion), + "test-extractor" => RunTestExtractor(subArgs, jsonOptions), _ when IsProjectPathArg(commandName) => IndexCommandRunner.Run(args, jsonOptions), _ => ShowError(args, $"Unknown command: {commandName}") @@ -285,6 +286,74 @@ _ when IsProjectPathArg(commandName) internal static bool IsProjectPathArg(string arg) => !arg.StartsWith('-') && (Directory.Exists(arg) || arg.Contains('/') || arg.Contains('\\') || arg == "."); + private static int RunTestExtractor(string[] args, JsonSerializerOptions jsonOptions) + { + string? language = null; + string? file = null; + string? expect = null; + var json = false; + for (var i = 0; i < args.Length; i++) + { + var arg = args[i]; + if (TryConsumeInlineOrNext(args, ref i, arg, "--language", out var value)) + language = value; + else if (TryConsumeInlineOrNext(args, ref i, arg, "--file", out value)) + file = value; + else if (TryConsumeInlineOrNext(args, ref i, arg, "--expect-symbols", out value) || TryConsumeInlineOrNext(args, ref i, arg, "--expect", out value)) + expect = value; + else if (arg == "--json") + json = true; + else + return CommandErrorWriter.Write($"Unknown test-extractor argument: {arg}", CommandExitCodes.InvalidArgument, "use --language --file [--expect-symbols ] [--json]."); + } + + if (string.IsNullOrWhiteSpace(language) || string.IsNullOrWhiteSpace(file)) + return CommandErrorWriter.Write("test-extractor requires --language and --file.", CommandExitCodes.InvalidArgument, "use --language --file [--expect-symbols ] [--json]."); + if (!File.Exists(file)) + return CommandErrorWriter.Write($"File not found: {file}", CommandExitCodes.NotFound); + + var source = File.ReadAllText(file); + var symbols = Indexer.SymbolExtractor.Extract(1, language, source, file); + if (expect != null) + { + var expected = File.ReadAllText(expect); + var actual = JsonSerializer.Serialize(symbols); + if (!JsonEquivalent(expected, actual)) + { + Console.Error.WriteLine("Expected symbols did not match extracted symbols."); + Console.Error.WriteLine(actual); + return CommandExitCodes.InvalidArgument; + } + } + + if (json || expect == null) + Console.WriteLine(JsonSerializer.Serialize(symbols)); + return CommandExitCodes.Success; + } + + private static bool TryConsumeInlineOrNext(string[] args, ref int index, string arg, string flag, out string value) + { + value = string.Empty; + if (arg.StartsWith(flag + "=", StringComparison.Ordinal)) + { + value = arg[(flag.Length + 1)..]; + return true; + } + + if (arg != flag || index + 1 >= args.Length) + return false; + + value = args[++index]; + return true; + } + + private static bool JsonEquivalent(string expected, string actual) + { + using var expectedDoc = JsonDocument.Parse(expected); + using var actualDoc = JsonDocument.Parse(actual); + return JsonSerializer.Serialize(expectedDoc.RootElement) == JsonSerializer.Serialize(actualDoc.RootElement); + } + internal static void EnsureRedirectedStdoutUsesUtf8() { if (!Console.IsOutputRedirected || Console.Out is StringWriter || Console.Out.GetType().Assembly != typeof(Console).Assembly) diff --git a/tests/CodeIndex.Tests/ProgramRunnerTests.cs b/tests/CodeIndex.Tests/ProgramRunnerTests.cs index ce87b1e7f2..fe256f5429 100644 --- a/tests/CodeIndex.Tests/ProgramRunnerTests.cs +++ b/tests/CodeIndex.Tests/ProgramRunnerTests.cs @@ -26,6 +26,37 @@ public void ContainsJsonOutputFlag_AfterPassthrough_ReturnsFalse() Assert.False(ProgramRunner.ContainsJsonOutputFlag(["search", "--", "--json"])); } + [Fact] + public void Run_TestExtractor_PrintsIsolatedSymbols() + { + lock (TestConsoleLock.Gate) + { + var tempDir = Path.Combine(Path.GetTempPath(), $"cdidx_test_extractor_{Guid.NewGuid():N}"); + try + { + Directory.CreateDirectory(tempDir); + var file = Path.Combine(tempDir, "app.py"); + File.WriteAllText(file, "def hello():\n pass\n"); + + var (exitCode, stdout, stderr) = CaptureConsole(() => ProgramRunner.Run( + ["test-extractor", "--language", "python", "--file", file, "--json"], + appVersion: "1.10.0")); + + Assert.Equal(CommandExitCodes.Success, exitCode); + Assert.Empty(stderr); + using var document = JsonDocument.Parse(stdout); + Assert.Contains(document.RootElement.EnumerateArray(), item => + item.GetProperty("Kind").GetString() == "function" + && item.GetProperty("Name").GetString() == "hello"); + } + finally + { + if (Directory.Exists(tempDir)) + Directory.Delete(tempDir, recursive: true); + } + } + } + [Fact] public void TryConsumeQueryTraceFlag_StripsTraceAndPreservesEscapedQuery() { From b9b399d74fe1ce253d61776953b18091d5c82294 Mon Sep 17 00:00:00 2001 From: Widthdom Date: Sun, 31 May 2026 13:30:44 +0900 Subject: [PATCH 4/6] Document custom extractor workflows (#1756 #1757 #1759) --- DEVELOPER_GUIDE.md | 48 ++++++++++++++++++++++++++++++++++++++++++++++ README.md | 12 ++++++++++++ 2 files changed, 60 insertions(+) diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index abed5af659..f7a216d7e5 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -3000,6 +3000,54 @@ Cloud セッションは開発ループの中で `dotnet build` にフォール - ドキュメント（README, CHANGELOG）は前半英語、後半日本語の構成。 - 不要な本番パッケージは入れない。test-only package は、テストハーネスの改善に明確に寄与し、`tests/CodeIndex.Tests/` に閉じる限り許容されるが、本番依存ルールを緩めるものではない。 +## Custom Language Extraction + +Downstream users can add lightweight language support without rebuilding +`cdidx`: + +- extension aliases are read from `~/.config/cdidx/langmap.yaml` and the first + workspace ancestor `.cdidx-langmap.yaml`; workspace entries override user + entries; +- regex-backed symbol patterns are read from `.cdidx/patterns/*.yaml` and + `~/.config/cdidx/patterns/*.yaml`; +- `cdidx test-extractor --language --file --json` runs symbol + extraction without building an index, and `--expect-symbols ` compares + the extracted JSON to a fixture. + +Minimal examples: + +```yaml +# .cdidx-langmap.yaml +entries: + - extension: ".kts.in" + language: "kotlin" +``` + +```yaml +# .cdidx/patterns/toydsl.yaml +language: "toydsl" +extensions: + - extension: ".toy" +patterns: + - kind: "class" + regex: "^entity (?\\w+)" +``` + +Each configured regex should expose a named `name` capture. If it does not, +`cdidx` uses the full match text as the symbol name. Invalid sidecar files are +ignored so a broken local experiment does not prevent indexing. + +## カスタム言語抽出 + +下流ユーザーは `cdidx` を再ビルドせずに軽量な言語対応を追加できます。 + +- 拡張子 alias は `~/.config/cdidx/langmap.yaml` と、最初に見つかった workspace + 祖先の `.cdidx-langmap.yaml` から読み込まれ、workspace 側が user 側を上書きします。 +- regex ベースのシンボルパターンは `.cdidx/patterns/*.yaml` と + `~/.config/cdidx/patterns/*.yaml` から読み込まれます。 +- `cdidx test-extractor --language --file --json` は index を作らずに + symbol extraction だけを実行し、`--expect-symbols ` で fixture JSON と比較できます。 + ## SQLite reader のデバッグ `Database/DbDebug.cs` は `ExecuteTrackedReader` / `TrackedRead` の最後に流れた SQL、パラメーター、行ごとの状態を記録し、ループ途中で `SqliteException` が発生した場合に再現に十分な文脈を stderr へダンプする。インデックス済みのソースバイトが想定外の経路に漏れないよう、ダンプ経路はゲート制御されている: diff --git a/README.md b/README.md index fdb362a64b..291d7466ff 100644 --- a/README.md +++ b/README.md @@ -57,6 +57,12 @@ cdidx search "Handle" --project MyApp cdidx mcp ``` +Custom language loops can stay out of tree: put extension aliases in +`.cdidx-langmap.yaml`, put regex symbol patterns in `.cdidx/patterns/*.yaml`, +and run `cdidx test-extractor --language --file --json` to test +an extractor fixture without building a full index. See +[Custom Language Extraction](DEVELOPER_GUIDE.md#custom-language-extraction). + After the first command, use these cues and follow-up commands: | Situation | What to expect or run | @@ -274,6 +280,12 @@ cdidx search "Handle" --project MyApp cdidx mcp ``` +カスタム言語の開発ループは out-of-tree で回せます。拡張子 alias は +`.cdidx-langmap.yaml`、regex シンボルパターンは `.cdidx/patterns/*.yaml` に置き、 +`cdidx test-extractor --language --file --json` で full index を作らずに +extractor fixture を確認できます。詳細は +[Custom Language Extraction](DEVELOPER_GUIDE.md#custom-language-extraction) を参照してください。 + 初回実行後は、次の見方と追加コマンドをよく使います。 | 状況 | 見るもの / 使うもの | From 63750deccb0783f8f3ced8adae292dda9709db0a Mon Sep 17 00:00:00 2001 From: Widthdom Date: Sun, 31 May 2026 13:41:33 +0900 Subject: [PATCH 5/6] Resolve workspace custom config roots (#1756 #1757) --- .../Extensibility/ExtractorPluginRegistry.cs | 30 +++++++++++++++---- src/CodeIndex/Indexer/Scanning/FileIndexer.cs | 2 +- .../Indexer/Scanning/LanguageMapOverrides.cs | 19 +++++++++--- .../Indexer/Symbols/SymbolExtractor.cs | 3 ++ tests/CodeIndex.Tests/FileIndexerTests.cs | 4 ++- tests/CodeIndex.Tests/SymbolExtractorTests.cs | 6 ++-- 6 files changed, 51 insertions(+), 13 deletions(-) diff --git a/src/CodeIndex/Indexer/Extensibility/ExtractorPluginRegistry.cs b/src/CodeIndex/Indexer/Extensibility/ExtractorPluginRegistry.cs index c5feb01c5a..0d10d3ee93 100644 --- a/src/CodeIndex/Indexer/Extensibility/ExtractorPluginRegistry.cs +++ b/src/CodeIndex/Indexer/Extensibility/ExtractorPluginRegistry.cs @@ -11,6 +11,7 @@ public static class ExtractorPluginRegistry private static readonly object Gate = new(); private static readonly Dictionary SymbolExtractors = new(StringComparer.Ordinal); private static readonly Dictionary ReferenceExtractors = new(StringComparer.Ordinal); + private static readonly HashSet LoadedPatternConfigPaths = new(StringComparer.OrdinalIgnoreCase); private static bool pluginsLoaded; public static IReadOnlyCollection SymbolLanguages @@ -84,6 +85,7 @@ internal static void ResetForTests() { SymbolExtractors.Clear(); ReferenceExtractors.Clear(); + LoadedPatternConfigPaths.Clear(); pluginsLoaded = true; } } @@ -94,10 +96,21 @@ internal static void ReloadForTests() { SymbolExtractors.Clear(); ReferenceExtractors.Clear(); + LoadedPatternConfigPaths.Clear(); pluginsLoaded = false; } } + internal static void LoadPatternConfigsForProjectRoot(string? projectRoot) + { + EnsurePluginsLoaded(); + if (string.IsNullOrWhiteSpace(projectRoot)) + return; + + foreach (var patternPath in EnumeratePatternConfigPaths(Path.GetFullPath(projectRoot))) + TryLoadPatternConfig(patternPath); + } + private static void EnsurePluginsLoaded() { if (Volatile.Read(ref pluginsLoaded)) @@ -110,7 +123,7 @@ private static void EnsurePluginsLoaded() foreach (var pluginPath in EnumeratePluginAssemblyPaths()) TryLoadPlugin(pluginPath); - foreach (var patternPath in EnumeratePatternConfigPaths()) + foreach (var patternPath in EnumeratePatternConfigPaths(Environment.CurrentDirectory)) TryLoadPatternConfig(patternPath); pluginsLoaded = true; @@ -138,9 +151,9 @@ private static IEnumerable EnumeratePluginDirectories() yield return Path.Combine(home, ".cdidx", "plugins"); } - private static IEnumerable EnumeratePatternConfigPaths() + private static IEnumerable EnumeratePatternConfigPaths(string workspaceRoot) { - foreach (var directory in EnumeratePatternDirectories()) + foreach (var directory in EnumeratePatternDirectories(workspaceRoot)) { if (!Directory.Exists(directory)) continue; @@ -152,9 +165,9 @@ private static IEnumerable EnumeratePatternConfigPaths() } } - private static IEnumerable EnumeratePatternDirectories() + private static IEnumerable EnumeratePatternDirectories(string workspaceRoot) { - yield return Path.Combine(Environment.CurrentDirectory, ".cdidx", "patterns"); + yield return Path.Combine(workspaceRoot, ".cdidx", "patterns"); var home = Environment.GetFolderPath(Environment.SpecialFolder.UserProfile); if (!string.IsNullOrWhiteSpace(home)) @@ -165,6 +178,13 @@ private static void TryLoadPatternConfig(string path) { try { + path = Path.GetFullPath(path); + lock (Gate) + { + if (!LoadedPatternConfigPaths.Add(path)) + return; + } + var language = string.Empty; var extensions = new List(); var patterns = new List(); diff --git a/src/CodeIndex/Indexer/Scanning/FileIndexer.cs b/src/CodeIndex/Indexer/Scanning/FileIndexer.cs index f72a17f2f8..35a3839694 100644 --- a/src/CodeIndex/Indexer/Scanning/FileIndexer.cs +++ b/src/CodeIndex/Indexer/Scanning/FileIndexer.cs @@ -1147,7 +1147,7 @@ private static bool TryDetectLanguageOverride(string filePath, out string langua { language = string.Empty; var fileName = Path.GetFileName(filePath); - foreach (var (extension, mappedLanguage) in LanguageMapOverrides.LoadEffectiveMap()) + foreach (var (extension, mappedLanguage) in LanguageMapOverrides.LoadEffectiveMap(filePath)) { if (fileName.EndsWith(extension, StringComparison.OrdinalIgnoreCase)) { diff --git a/src/CodeIndex/Indexer/Scanning/LanguageMapOverrides.cs b/src/CodeIndex/Indexer/Scanning/LanguageMapOverrides.cs index a88604e7fa..5d9bfda34f 100644 --- a/src/CodeIndex/Indexer/Scanning/LanguageMapOverrides.cs +++ b/src/CodeIndex/Indexer/Scanning/LanguageMapOverrides.cs @@ -4,21 +4,21 @@ internal static class LanguageMapOverrides { internal const string WorkspaceFileName = ".cdidx-langmap.yaml"; - internal static IReadOnlyDictionary LoadEffectiveMap() + internal static IReadOnlyDictionary LoadEffectiveMap(string? startPath = null) { var map = new Dictionary(StringComparer.OrdinalIgnoreCase); - foreach (var path in EnumerateConfigPaths()) + foreach (var path in EnumerateConfigPaths(startPath)) LoadInto(path, map); return map; } - private static IEnumerable EnumerateConfigPaths() + private static IEnumerable EnumerateConfigPaths(string? startPath) { var home = Environment.GetFolderPath(Environment.SpecialFolder.UserProfile); if (!string.IsNullOrWhiteSpace(home)) yield return Path.Combine(home, ".config", "cdidx", "langmap.yaml"); - var directory = Environment.CurrentDirectory; + var directory = ResolveStartDirectory(startPath); while (!string.IsNullOrEmpty(directory)) { var candidate = Path.Combine(directory, WorkspaceFileName); @@ -32,6 +32,17 @@ private static IEnumerable EnumerateConfigPaths() } } + private static string ResolveStartDirectory(string? startPath) + { + if (string.IsNullOrWhiteSpace(startPath)) + return Environment.CurrentDirectory; + + var fullPath = Path.GetFullPath(startPath); + return Directory.Exists(fullPath) + ? fullPath + : Path.GetDirectoryName(fullPath) ?? Environment.CurrentDirectory; + } + private static void LoadInto(string path, Dictionary target) { if (!File.Exists(path)) diff --git a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.cs b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.cs index 332b496d9e..f7947ecaa1 100644 --- a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.cs +++ b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.cs @@ -2139,6 +2139,7 @@ private static bool TryPrepareSymbolExtraction( string? originalLang, string content, string? filePath, + string? projectRoot, CancellationToken cancellationToken, out string? lang, out string preparedContent, @@ -2182,6 +2183,7 @@ private static bool TryPrepareSymbolExtraction( content = content.Replace("\r\n", "\n").Replace("\r", "\n"); preparedContent = FileIndexer.StripLineLeadingInvisibles(content); cancellationToken.ThrowIfCancellationRequested(); + ExtractorPluginRegistry.LoadPatternConfigsForProjectRoot(projectRoot); if (pluginLanguage != null && !PatternCache.ContainsKey(pluginLanguage) @@ -2215,6 +2217,7 @@ public static List Extract(long fileId, string? lang, string conte originalLang, content, filePath, + projectRoot, cancellationToken, out lang, out content, diff --git a/tests/CodeIndex.Tests/FileIndexerTests.cs b/tests/CodeIndex.Tests/FileIndexerTests.cs index 3d2ab2f932..eb5e2580d8 100644 --- a/tests/CodeIndex.Tests/FileIndexerTests.cs +++ b/tests/CodeIndex.Tests/FileIndexerTests.cs @@ -339,7 +339,9 @@ public void DetectLanguage_WorkspaceLangMapYaml_AliasesExtension() File.WriteAllText( Path.Combine(tempDir, LanguageMapOverrides.WorkspaceFileName), "entries:\n - extension: \".kts.in\"\n language: \"kotlin\"\n"); - Environment.CurrentDirectory = tempDir; + var outsideDir = Path.Combine(tempDir, "outside"); + Directory.CreateDirectory(outsideDir); + Environment.CurrentDirectory = outsideDir; Assert.Equal("kotlin", FileIndexer.DetectLanguage(Path.Combine(tempDir, "build.kts.in"))); Assert.Equal("kotlin", FileIndexer.GetLanguageExtensions()[".kts.in"]); diff --git a/tests/CodeIndex.Tests/SymbolExtractorTests.cs b/tests/CodeIndex.Tests/SymbolExtractorTests.cs index 68c066785b..bfe3c009bf 100644 --- a/tests/CodeIndex.Tests/SymbolExtractorTests.cs +++ b/tests/CodeIndex.Tests/SymbolExtractorTests.cs @@ -57,10 +57,12 @@ public void Extract_ConfiguredPatternYaml_HandlesOutOfTreeLanguage() File.WriteAllText( Path.Combine(tempDir, ".cdidx", "patterns", "toydsl.yaml"), "language: \"toydsl\"\nextensions:\n - extension: \".toy\"\npatterns:\n - kind: \"class\"\n regex: \"^entity (?\\\\w+)\"\n"); - Environment.CurrentDirectory = tempDir; + var outsideDir = Path.Combine(tempDir, "outside"); + Directory.CreateDirectory(outsideDir); + Environment.CurrentDirectory = outsideDir; ExtractorPluginRegistry.ReloadForTests(); - var symbols = SymbolExtractor.Extract(2, "toydsl", "entity Widget", "demo.toy"); + var symbols = SymbolExtractor.Extract(2, "toydsl", "entity Widget", "demo.toy", tempDir); var symbol = Assert.Single(symbols); Assert.Equal("class", symbol.Kind); From 62725c4480ce9d1ea1426d47de3933561658b4ec Mon Sep 17 00:00:00 2001 From: Widthdom Date: Sun, 31 May 2026 13:46:19 +0900 Subject: [PATCH 6/6] Load custom pattern extensions during scanning (#1756) --- .../Extensibility/ExtractorPluginRegistry.cs | 27 ++++++++++++++++--- src/CodeIndex/Indexer/Scanning/FileIndexer.cs | 6 +++++ 2 files changed, 30 insertions(+), 3 deletions(-) diff --git a/src/CodeIndex/Indexer/Extensibility/ExtractorPluginRegistry.cs b/src/CodeIndex/Indexer/Extensibility/ExtractorPluginRegistry.cs index 0d10d3ee93..023c53aadd 100644 --- a/src/CodeIndex/Indexer/Extensibility/ExtractorPluginRegistry.cs +++ b/src/CodeIndex/Indexer/Extensibility/ExtractorPluginRegistry.cs @@ -111,6 +111,24 @@ internal static void LoadPatternConfigsForProjectRoot(string? projectRoot) TryLoadPatternConfig(patternPath); } + internal static void LoadPatternConfigsForPath(string? path) + { + EnsurePluginsLoaded(); + if (string.IsNullOrWhiteSpace(path)) + return; + + var directory = Path.GetFullPath(path); + if (!Directory.Exists(directory)) + directory = Path.GetDirectoryName(directory) ?? string.Empty; + + while (!string.IsNullOrEmpty(directory)) + { + foreach (var patternPath in EnumeratePatternConfigPaths(directory, includeUserDirectory: false)) + TryLoadPatternConfig(patternPath); + directory = Directory.GetParent(directory)?.FullName ?? string.Empty; + } + } + private static void EnsurePluginsLoaded() { if (Volatile.Read(ref pluginsLoaded)) @@ -151,9 +169,9 @@ private static IEnumerable EnumeratePluginDirectories() yield return Path.Combine(home, ".cdidx", "plugins"); } - private static IEnumerable EnumeratePatternConfigPaths(string workspaceRoot) + private static IEnumerable EnumeratePatternConfigPaths(string workspaceRoot, bool includeUserDirectory = true) { - foreach (var directory in EnumeratePatternDirectories(workspaceRoot)) + foreach (var directory in EnumeratePatternDirectories(workspaceRoot, includeUserDirectory)) { if (!Directory.Exists(directory)) continue; @@ -165,10 +183,13 @@ private static IEnumerable EnumeratePatternConfigPaths(string workspaceR } } - private static IEnumerable EnumeratePatternDirectories(string workspaceRoot) + private static IEnumerable EnumeratePatternDirectories(string workspaceRoot, bool includeUserDirectory) { yield return Path.Combine(workspaceRoot, ".cdidx", "patterns"); + if (!includeUserDirectory) + yield break; + var home = Environment.GetFolderPath(Environment.SpecialFolder.UserProfile); if (!string.IsNullOrWhiteSpace(home)) yield return Path.Combine(home, ".config", "cdidx", "patterns"); diff --git a/src/CodeIndex/Indexer/Scanning/FileIndexer.cs b/src/CodeIndex/Indexer/Scanning/FileIndexer.cs index 35a3839694..7d9914d4af 100644 --- a/src/CodeIndex/Indexer/Scanning/FileIndexer.cs +++ b/src/CodeIndex/Indexer/Scanning/FileIndexer.cs @@ -1138,7 +1138,13 @@ internal static LanguageDetectionResult TryDetectLanguage(string filePath, strin return new LanguageDetectionResult(FileProbeStatus.Supported, pluginLang); if (!string.IsNullOrEmpty(ext)) + { + ExtractorPluginRegistry.LoadPatternConfigsForPath(filePath); + if (ExtractorPluginRegistry.LanguageExtensions.TryGetValue(ext, out pluginLang)) + return new LanguageDetectionResult(FileProbeStatus.Supported, pluginLang); + return new LanguageDetectionResult(FileProbeStatus.Unsupported, null); + } return TryDetectLanguageFromShebang(filePath); }