diff --git a/changelog.d/unreleased/3052.security.md b/changelog.d/unreleased/3052.security.md new file mode 100644 index 0000000000..a1c3d8dfb0 --- /dev/null +++ b/changelog.d/unreleased/3052.security.md @@ -0,0 +1,16 @@ +--- +category: security +issues: + - 3052 +affected: + - src/CodeIndex/Indexer/Extensibility/ExtractorPluginRegistry.cs + - tests/CodeIndex.Tests/SymbolExtractorConfiguredPatternTests.cs +--- + +## English + +- **Pattern config loading now avoids newline split arrays (#3052)** — configured extractor YAML is parsed line-by-line from bounded UTF-8 text, avoiding whole-file `Split('\n')` allocations for configs with many tiny lines. + +## 日本語 + +- **pattern config 読み込みで改行 Split 配列を作らないようにしました (#3052)** — configured extractor YAML を上限付き UTF-8 テキストから行ごとに解析し、小さな行が多い config でファイル全体の `Split('\n')` 割り当てを避けるようにしました。 diff --git a/src/CodeIndex/Indexer/Extensibility/ExtractorPluginRegistry.cs b/src/CodeIndex/Indexer/Extensibility/ExtractorPluginRegistry.cs index 12e3121462..38ce9d2a0b 100644 --- a/src/CodeIndex/Indexer/Extensibility/ExtractorPluginRegistry.cs +++ b/src/CodeIndex/Indexer/Extensibility/ExtractorPluginRegistry.cs @@ -430,33 +430,35 @@ private static void TryLoadPatternConfig(string path) return; } - var configLines = TryReadPatternConfigLines(path); - if (configLines == null) + var configText = TryReadPatternConfigText(path); + if (configText == null) return; var language = string.Empty; var extensions = new List(); var patterns = new List(); string? pendingKind = null; - foreach (var rawLine in configLines) + var remaining = configText.AsSpan(); + while (TryReadNextPatternConfigLine(ref remaining, out var rawLine)) { var line = rawLine.Trim(); - if (line.Length == 0 || line.StartsWith('#')) + if (line.Length == 0 || line[0] == '#') continue; + var itemLine = TrimPatternConfigListMarker(line); if (TryReadScalar(line, "language", out var value)) { language = NormalizePluginLanguage(value); } - else if (TryReadScalar(line.TrimStart('-').Trim(), "extension", out value)) + else if (TryReadScalar(itemLine, "extension", out value)) { extensions.Add(NormalizePluginExtension(value) ?? value); } - else if (TryReadScalar(line.TrimStart('-').Trim(), "kind", out value)) + else if (TryReadScalar(itemLine, "kind", out value)) { pendingKind = value.Trim(); } - else if (TryReadScalar(line.TrimStart('-').Trim(), "regex", out value) && pendingKind != null) + else if (TryReadScalar(itemLine, "regex", out value) && pendingKind != null) { if (patterns.Count >= MaxPatternRulesPerConfig) { @@ -573,7 +575,7 @@ private static bool TryReservePatternRuleBudget(string path) } } - private static IReadOnlyList? TryReadPatternConfigLines(string path) + private static string? TryReadPatternConfigText(string path) { var fileInfo = new FileInfo(path); if (!fileInfo.Exists) @@ -607,11 +609,38 @@ private static bool TryReservePatternRuleBudget(string path) if (bytes == null) return null; - var text = Encoding.UTF8.GetString(bytes); - return text - .Replace("\r\n", "\n", StringComparison.Ordinal) - .Replace('\r', '\n') - .Split('\n'); + return Encoding.UTF8.GetString(bytes); + } + + private static bool TryReadNextPatternConfigLine(ref ReadOnlySpan remaining, out ReadOnlySpan line) + { + if (remaining.IsEmpty) + { + line = default; + return false; + } + + var lineBreakIndex = remaining.IndexOfAny('\r', '\n'); + if (lineBreakIndex < 0) + { + line = remaining; + remaining = default; + return true; + } + + line = remaining[..lineBreakIndex]; + var nextIndex = lineBreakIndex + 1; + if (remaining[lineBreakIndex] == '\r' && nextIndex < remaining.Length && remaining[nextIndex] == '\n') + nextIndex++; + remaining = remaining[nextIndex..]; + return true; + } + + private static ReadOnlySpan TrimPatternConfigListMarker(ReadOnlySpan line) + { + while (!line.IsEmpty && line[0] == '-') + line = line[1..]; + return line.Trim(); } private static byte[]? TryReadWindowsPatternConfigBytes(string path) @@ -831,16 +860,32 @@ private struct WindowsFileInformation public uint FileIndexLow; } - private static bool TryReadScalar(string line, string key, out string value) + private static bool TryReadScalar(ReadOnlySpan line, string key, out string value) { value = string.Empty; - var prefix = key + ":"; - if (!line.StartsWith(prefix, StringComparison.OrdinalIgnoreCase)) + if (line.Length <= key.Length || line[key.Length] != ':') return false; - value = line[prefix.Length..].Trim().Trim('"', '\'').Replace("\\\\", "\\", StringComparison.Ordinal); + + if (!line.StartsWith(key.AsSpan(), StringComparison.OrdinalIgnoreCase)) + return false; + + var scalar = TrimScalarQuotes(line[(key.Length + 1)..].Trim()); + if (scalar.IsEmpty) + return false; + + value = scalar.ToString().Replace("\\\\", "\\", StringComparison.Ordinal); return value.Length > 0; } + private static ReadOnlySpan TrimScalarQuotes(ReadOnlySpan value) + { + while (!value.IsEmpty && (value[0] == '"' || value[0] == '\'')) + value = value[1..]; + while (!value.IsEmpty && (value[^1] == '"' || value[^1] == '\'')) + value = value[..^1]; + return value; + } + private static void TryLoadPlugin(string pluginPath) { var fullPath = pluginPath; diff --git a/tests/CodeIndex.Tests/SymbolExtractorConfiguredPatternTests.cs b/tests/CodeIndex.Tests/SymbolExtractorConfiguredPatternTests.cs index d7ca2cea55..d08bcab651 100644 --- a/tests/CodeIndex.Tests/SymbolExtractorConfiguredPatternTests.cs +++ b/tests/CodeIndex.Tests/SymbolExtractorConfiguredPatternTests.cs @@ -84,6 +84,35 @@ public void Extract_ConfiguredPatternYaml_RejectsInvalidRegexWithDiagnostic() } } + [Fact] + public void Extract_ConfiguredPatternYaml_ReadsManyLineConfig() + { + lock (TestConsoleLock.Gate) + { + var tempDir = Path.Combine(Path.GetTempPath(), $"cdidx_patterns_many_lines_{Guid.NewGuid():N}"); + try + { + var commentLines = string.Join("\n", Enumerable.Repeat("#", 4096)); + WritePatternConfig( + tempDir, + $"{commentLines}\nlanguage: \"toydsl\"\nextensions:\n - extension: \".toy\"\npatterns:\n - kind: \"class\"\n regex: \"^entity (?\\\\w+)\"\n"); + ExtractorPluginRegistry.ReloadForTests(); + + var symbols = SymbolExtractor.Extract(2, "toydsl", "entity Widget", "demo.toy", tempDir); + + var symbol = Assert.Single(symbols); + Assert.Equal("class", symbol.Kind); + Assert.Equal("Widget", symbol.Name); + } + finally + { + ExtractorPluginRegistry.ResetForTests(); + if (Directory.Exists(tempDir)) + Directory.Delete(tempDir, recursive: true); + } + } + } + [Fact] public void Extract_ConfiguredPatternYaml_RejectsOversizeConfigWithDiagnostic() {