From f95ca2e8f3ae863801e187bc8e64be5c4cd248e7 Mon Sep 17 00:00:00 2001 From: Widthdom Date: Fri, 5 Jun 2026 14:48:15 +0900 Subject: [PATCH] Fix pattern config discovery cap for #3051 --- DEVELOPER_GUIDE.md | 12 ++-- README.md | 7 +- changelog.d/unreleased/3051.security.md | 18 +++++ .../Extensibility/ExtractorPluginRegistry.cs | 68 ++++++++++++++++--- .../SymbolExtractorConfiguredPatternTests.cs | 35 ++++++++++ 5 files changed, 122 insertions(+), 18 deletions(-) create mode 100644 changelog.d/unreleased/3051.security.md diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index a895550e14..98e76811bc 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -2048,9 +2048,10 @@ Downstream users can add lightweight language support without rebuilding entries; - regex-backed symbol patterns are read from `.cdidx/patterns/*.yaml` and `~/.config/cdidx/patterns/*.yaml`; sidecars must be regular files under - non-symlink pattern directories, each file is capped at 64 KiB / 128 rules, - the process loads at most 128 configured rules total, and regex matches use a - 100 ms timeout; + non-symlink pattern directories, discovery accepts at most 128 candidates per + pattern directory, each file is capped at 64 KiB / 128 rules, the process + loads at most 128 configured rules total, and regex matches use a 100 ms + timeout; - `cdidx test-extractor --language --file --json` runs symbol extraction without building an index, and `--expect-symbols ` compares the extracted JSON to a fixture. The source and expectation files are capped @@ -3632,8 +3633,9 @@ Cloud セッションは開発ループの中で `dotnet build` にフォール 祖先の `.cdidx-langmap.yaml` から読み込まれ、workspace 側が user 側を上書きします。 - regex ベースのシンボルパターンは `.cdidx/patterns/*.yaml` と `~/.config/cdidx/patterns/*.yaml` から読み込まれます。sidecar は symlink ではない - pattern directory 配下の通常ファイルのみが対象で、各ファイルは 64 KiB / 128 ルール、 - プロセス全体では configured rule 128 件に制限され、regex match には 100 ms の timeout が付きます。 + pattern directory 配下の通常ファイルのみが対象で、探索候補は pattern directory ごとに + 128 件まで、各ファイルは 64 KiB / 128 ルール、プロセス全体では configured rule 128 件に制限され、 + regex match には 100 ms の timeout が付きます。 - `cdidx test-extractor --language --file --json` は index を作らずに symbol extraction だけを実行し、`--expect-symbols ` で fixture JSON と比較できます。 source と expectation file はそれぞれ 4 MiB に制限されます。 diff --git a/README.md b/README.md index ef392c6d34..70f31d45fa 100644 --- a/README.md +++ b/README.md @@ -81,8 +81,9 @@ Custom language loops can stay out of tree: put extension aliases in and run `cdidx test-extractor --language --file --json` to test an extractor fixture without building a full index. `test-extractor` source and `--expect-symbols` files are capped at 4 MiB each. Pattern sidecars are -limited to regular files under non-symlink pattern directories, size/count -bounded per file and per process, and regex matches are time-limited. See +limited to regular files under non-symlink pattern directories, discovery is +capped per directory, size/count is bounded per file and per process, and regex +matches are time-limited. See [Custom Language Extraction](DEVELOPER_GUIDE.md#custom-language-extraction). After the first command, use these cues and follow-up commands: @@ -388,7 +389,7 @@ list metadata ごとに grouped されるため usage error で拒否します `cdidx test-extractor --language --file --json` で full index を作らずに extractor fixture を確認できます。`test-extractor` の source と `--expect-symbols` ファイルはそれぞれ 4 MiB に制限されます。pattern sidecar は symlink ではない pattern directory -配下の通常ファイルだけが対象で、size / count は file 単位と process 単位で制限され、 +配下の通常ファイルだけが対象で、探索候補数は directory 単位、size / count は file 単位と process 単位で制限され、 regex match には timeout が付きます。詳細は [Custom Language Extraction](DEVELOPER_GUIDE.md#custom-language-extraction) を参照してください。 diff --git a/changelog.d/unreleased/3051.security.md b/changelog.d/unreleased/3051.security.md new file mode 100644 index 0000000000..e2f804a573 --- /dev/null +++ b/changelog.d/unreleased/3051.security.md @@ -0,0 +1,18 @@ +--- +category: security +issues: + - 3051 +affected: + - src/CodeIndex/Indexer/Extensibility/ExtractorPluginRegistry.cs + - tests/CodeIndex.Tests/SymbolExtractorConfiguredPatternTests.cs + - README.md + - DEVELOPER_GUIDE.md +--- + +## English + +- **Pattern config discovery is now streamed and capped per directory (#3051)** — `cdidx` now streams `.yaml` and `.yml` pattern sidecar discovery, accepts at most 128 pattern config candidates per directory, and emits a diagnostic when the directory cap is reached. + +## 日本語 + +- **pattern config の探索を streaming 化し directory ごとの上限を追加しました (#3051)** — `cdidx` は `.yaml` / `.yml` の pattern sidecar 探索を streaming で行い、pattern config 候補を directory ごとに最大 128 件まで受け付け、上限に達した場合は diagnostic を出力します。 diff --git a/src/CodeIndex/Indexer/Extensibility/ExtractorPluginRegistry.cs b/src/CodeIndex/Indexer/Extensibility/ExtractorPluginRegistry.cs index 12e3121462..9417024672 100644 --- a/src/CodeIndex/Indexer/Extensibility/ExtractorPluginRegistry.cs +++ b/src/CodeIndex/Indexer/Extensibility/ExtractorPluginRegistry.cs @@ -17,6 +17,7 @@ public static class ExtractorPluginRegistry internal const int MaxPatternRulesPerConfig = 128; internal const int MaxPatternRulesTotal = 128; internal const int MaxPatternRegexLength = 4096; + internal const int MaxPatternConfigCandidatesPerDirectory = 128; internal const int MaxPluginAssemblyCandidatesPerDirectory = 128; internal const int MaxPluginAssemblyCandidatesTotal = 256; internal const long MaxPluginAssemblyBytes = 64 * 1024 * 1024; @@ -27,6 +28,7 @@ public static class ExtractorPluginRegistry private static readonly Dictionary ReferenceExtractors = new(StringComparer.Ordinal); private static readonly HashSet LoadedPluginAssemblyPaths = new(StringComparer.OrdinalIgnoreCase); private static readonly HashSet LoadedPatternConfigPaths = new(StringComparer.OrdinalIgnoreCase); + private static readonly IReadOnlyList PatternConfigSearchPatterns = ["*.yaml", "*.yml"]; private static readonly List Diagnostics = []; private const int DiagnosticLimit = 20; private static int pluginAssemblyCount; @@ -166,6 +168,9 @@ internal static IReadOnlyList EnumeratePluginAssemblyPathsForTests(strin internal static IReadOnlyList EnumeratePluginAssemblyPathsForTests(IReadOnlyList directories) => EnumeratePluginAssemblyPaths(directories).ToArray(); + internal static IReadOnlyList EnumeratePatternConfigPathsFromDirectoryForTests(string directory) + => EnumeratePatternConfigPathsFromDirectory(directory, workspaceRoot: null).ToArray(); + internal static void LoadPluginAssembliesForTests(IReadOnlyList directories) => LoadPluginAssemblies(directories); @@ -360,27 +365,58 @@ private static IEnumerable EnumeratePatternConfigPathsFromDirectory(stri if (!Directory.Exists(directory) || !PatternDirectoryIsSafe(directory, workspaceRoot)) yield break; - foreach (var path in EnumeratePatternFiles(directory, "*.yaml")) - yield return path; - foreach (var path in EnumeratePatternFiles(directory, "*.yml")) - yield return path; + var directoryCandidates = 0; + foreach (var searchPattern in PatternConfigSearchPatterns) + { + using var enumerator = TryEnumeratePatternFiles(directory, searchPattern); + if (enumerator == null) + continue; + + while (TryMoveNextPatternFile(directory, enumerator, out var path)) + { + if (directoryCandidates >= MaxPatternConfigCandidatesPerDirectory) + { + ReportPatternDirectorySkipped( + directory, + $"too many pattern config candidates (maximum {MaxPatternConfigCandidatesPerDirectory} per directory)"); + yield break; + } + + directoryCandidates++; + yield return path; + } + } } - private static IEnumerable EnumeratePatternFiles(string directory, string searchPattern) + private static IEnumerator? TryEnumeratePatternFiles(string directory, string searchPattern) { - string[] paths; try { - paths = Directory.GetFiles(directory, searchPattern, SearchOption.TopDirectoryOnly); + return Directory.EnumerateFiles(directory, searchPattern, SearchOption.TopDirectoryOnly).GetEnumerator(); } catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) { ReportPatternDirectoryRejected(directory, "could not enumerate pattern directory"); - yield break; + return null; } + } - foreach (var path in paths) - yield return path; + private static bool TryMoveNextPatternFile(string directory, IEnumerator enumerator, out string patternPath) + { + patternPath = string.Empty; + try + { + if (!enumerator.MoveNext()) + return false; + + patternPath = enumerator.Current; + return true; + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + ReportPatternDirectoryRejected(directory, "could not enumerate pattern directory"); + return false; + } } private static bool PatternDirectoryIsSafe(string directory, string? workspaceRoot) @@ -547,6 +583,18 @@ private static void ReportPatternDirectoryRejected(string path, string reason) countsAsSkippedFile: false); } + private static void ReportPatternDirectorySkipped(string path, string reason) + { + Console.Error.WriteLine($"[cdidx] Skipped pattern directory '{DiagnosticSanitizer.ForPath(path)}': {DiagnosticSanitizer.ForMessage(reason)}."); + RecordDiagnostic( + "pattern_directory", + path, + typeName: null, + severity: "skipped", + $"Pattern directory skipped: {reason}.", + countsAsSkippedFile: false); + } + private static void ReportPluginDirectorySkipped(string path, string reason) { RecordDiagnostic( diff --git a/tests/CodeIndex.Tests/SymbolExtractorConfiguredPatternTests.cs b/tests/CodeIndex.Tests/SymbolExtractorConfiguredPatternTests.cs index d7ca2cea55..3336269cb3 100644 --- a/tests/CodeIndex.Tests/SymbolExtractorConfiguredPatternTests.cs +++ b/tests/CodeIndex.Tests/SymbolExtractorConfiguredPatternTests.cs @@ -53,6 +53,41 @@ public void EnumeratePluginAssemblyPaths_UsesExplicitProjectRootForWorkspacePlug } } + [Fact] + public void EnumeratePatternConfigPathsFromDirectory_CapsCandidatesAcrossExtensionsWithDiagnostic() + { + lock (TestConsoleLock.Gate) + { + var tempDir = Path.Combine(Path.GetTempPath(), $"cdidx_patterns_candidate_cap_{Guid.NewGuid():N}"); + try + { + var patternDir = Path.Combine(tempDir, ".cdidx", "patterns"); + Directory.CreateDirectory(patternDir); + for (var i = 0; i < ExtractorPluginRegistry.MaxPatternConfigCandidatesPerDirectory; i++) + File.WriteAllText(Path.Combine(patternDir, $"candidate{i:D3}.yaml"), string.Empty); + File.WriteAllText(Path.Combine(patternDir, "overflow.yml"), string.Empty); + ExtractorPluginRegistry.ResetForTests(); + + IReadOnlyList paths = Array.Empty(); + var stderr = ConsoleCapture.CaptureError(() => + { + paths = ExtractorPluginRegistry.EnumeratePatternConfigPathsFromDirectoryForTests(patternDir); + }); + + Assert.Equal(ExtractorPluginRegistry.MaxPatternConfigCandidatesPerDirectory, paths.Count); + Assert.DoesNotContain(paths, path => Path.GetFileName(path) == "overflow.yml"); + Assert.Contains("Skipped pattern directory", stderr, StringComparison.Ordinal); + Assert.Contains("too many pattern config candidates", stderr, StringComparison.Ordinal); + } + finally + { + ExtractorPluginRegistry.ResetForTests(); + if (Directory.Exists(tempDir)) + Directory.Delete(tempDir, recursive: true); + } + } + } + [Fact] public void Extract_ConfiguredPatternYaml_RejectsInvalidRegexWithDiagnostic() {