From 742aeb2e35a55dae61152d2e5284c7e760c86065 Mon Sep 17 00:00:00 2001 From: Widthdom Date: Sat, 20 Jun 2026 11:58:18 +0900 Subject: [PATCH 01/18] Add per-file reference cap (#3719) --- README.md | 8 ++ changelog.d/unreleased/3719.fixed.md | 19 +++ src/CodeIndex/Cli/CliFlagSchema.cs | 1 + src/CodeIndex/Cli/ConsoleUi.cs | 3 +- .../Cli/IndexCommandRunner.FullScan.cs | 47 ++++++- src/CodeIndex/Cli/IndexCommandRunner.Parse.cs | 26 +++- .../Cli/IndexCommandRunner.Update.cs | 28 ++-- src/CodeIndex/Cli/IndexCommandRunner.cs | 3 + src/CodeIndex/Cli/IndexWatchRunner.cs | 5 + src/CodeIndex/Database/DbWriter.cs | 8 ++ src/CodeIndex/Mcp/McpToolDefinitions.cs | 1 + src/CodeIndex/Mcp/McpToolHandlers.cs | 29 +++- tests/CodeIndex.Tests/ConsoleUiTests.cs | 2 +- .../IndexCommandRunnerTests.cs | 131 ++++++++++++++++++ 14 files changed, 287 insertions(+), 24 deletions(-) create mode 100644 changelog.d/unreleased/3719.fixed.md diff --git a/README.md b/README.md index b01b9dc5c5..dde6050a36 100644 --- a/README.md +++ b/README.md @@ -90,6 +90,10 @@ For a faster first pass when you only need text search, `definition`, `symbols`, or `map`, run `cdidx . --symbols-only`. Reference graph commands remain degraded until you rerun `cdidx .` without that flag. +For generated or dense source that emits excessive reference rows, use +`cdidx . --max-references-per-file ` to keep text search and symbols indexed +while skipping references for only the over-limit file. + ## Highlights | Area | What to use | @@ -265,6 +269,10 @@ cdidx lsp --db .cdidx/codeindex.db `cdidx . --symbols-only` を使えます。reference graph 系コマンドは、このフラグなしで `cdidx .` を再実行するまで degraded のままです。 +生成コードや高密度なソースが過剰な reference 行を生成する場合は +`cdidx . --max-references-per-file ` を使うと、text search と symbols は保持しつつ +上限を超えたファイルだけ references をスキップできます。 + ## 特長 | 分野 | 使うもの | diff --git a/changelog.d/unreleased/3719.fixed.md b/changelog.d/unreleased/3719.fixed.md new file mode 100644 index 0000000000..289ae6fc4e --- /dev/null +++ b/changelog.d/unreleased/3719.fixed.md @@ -0,0 +1,19 @@ +--- +category: fixed +issues: + - 3719 +affected: + - src/CodeIndex/Cli/IndexCommandRunner.cs + - src/CodeIndex/Cli/IndexCommandRunner.FullScan.cs + - src/CodeIndex/Cli/IndexCommandRunner.Update.cs + - src/CodeIndex/Mcp/McpToolHandlers.cs + - src/CodeIndex/Mcp/McpToolDefinitions.cs +--- + +## English + +- **Reference extraction now has a per-file cap (#3719)** — `cdidx index` and MCP indexing accept `--max-references-per-file` / `maxReferencesPerFile`, preserve text search and symbols, and emit `reference_count_exceeded` file issues when a file's references are skipped. + +## 日本語 + +- **reference 抽出にファイル単位の上限を追加しました (#3719)** — `cdidx index` と MCP index が `--max-references-per-file` / `maxReferencesPerFile` を受け付け、text search と symbols は保持しつつ、references をスキップしたファイルには `reference_count_exceeded` file issue を出します。 diff --git a/src/CodeIndex/Cli/CliFlagSchema.cs b/src/CodeIndex/Cli/CliFlagSchema.cs index 812e0c7a1c..de7d4cb2bd 100644 --- a/src/CodeIndex/Cli/CliFlagSchema.cs +++ b/src/CodeIndex/Cli/CliFlagSchema.cs @@ -367,6 +367,7 @@ private static IReadOnlyList BuildAll() new() { Name = "--duration-format", ValuePlaceholder = "", Description = "Index elapsed time display format", Commands = Set("index") }, new() { Name = "--max-file-bytes", ValuePlaceholder = "", Description = "Override the per-file indexing size limit", Commands = Set("index") }, new() { Name = "--max-symbols-per-file", ValuePlaceholder = "", Description = "Skip file content, symbols, and references when one file emits too many symbols (max 50000)", Commands = Set("index") }, + new() { Name = "--max-references-per-file", ValuePlaceholder = "", Description = "Skip references when one file emits too many references (max 1000000)", Commands = Set("index") }, new() { Name = "--parallelism", ValuePlaceholder = "", Description = "Full-scan extraction worker count (default: CPU count capped at 16; also honors CDIDX_INDEX_PARALLELISM)", Commands = Set("index") }, new() { Name = "--memory-trace", Description = "Include phase memory samples in index JSON output", Commands = Set("index") }, new() { Name = "--commits", ValuePlaceholder = "", Description = "Update files changed in given git commits", Commands = Set("index") }, diff --git a/src/CodeIndex/Cli/ConsoleUi.cs b/src/CodeIndex/Cli/ConsoleUi.cs index e220f14d3a..b309d04713 100644 --- a/src/CodeIndex/Cli/ConsoleUi.cs +++ b/src/CodeIndex/Cli/ConsoleUi.cs @@ -80,7 +80,7 @@ public static class ConsoleUi private static readonly (string Command, string Usage)[] CommandUsageLines = [ - ("index", "cdidx index [--db ] [--rebuild] [--optimize] [--symbols-only] [--verbose] [--dry-run] [--force] [--quiet] [--json] [--memory-trace] [--duration-format ] [--notify ] [--max-file-bytes ] [--max-symbols-per-file ] [--follow-symlinks ] [--include-symbol-kind [,]] [--exclude-symbol-kind [,]] [--watch [--debounce ]]"), + ("index", "cdidx index [--db ] [--rebuild] [--optimize] [--symbols-only] [--verbose] [--dry-run] [--force] [--quiet] [--json] [--memory-trace] [--duration-format ] [--notify ] [--max-file-bytes ] [--max-symbols-per-file ] [--max-references-per-file ] [--follow-symlinks ] [--include-symbol-kind [,]] [--exclude-symbol-kind [,]] [--watch [--debounce ]]"), ("hooks", "cdidx hooks [--project ] [--force] [--json]"), ("backfill-fold", "cdidx backfill-fold [--db ] [--dry-run] [--no-checkpoint] [--json]"), ("optimize", "cdidx optimize [--db ] [--json]"), @@ -990,6 +990,7 @@ private static void PrintFlagReference(Action WriteHelpLine) WriteHelpLine(" --notify Long index completion signal: auto, bell, osc9, desktop, or none (also honors CDIDX_NOTIFY; quiet/json suppress it)"); WriteHelpLine(" --max-file-bytes Index only files up to this size (default: 4MiB; also honors CDIDX_MAX_FILE_BYTES; accepts K/M/G suffixes)"); WriteHelpLine(" --max-symbols-per-file Skip file content, symbols, and references when one file emits too many symbols (default: 5000; max: 50000)"); + WriteHelpLine(" --max-references-per-file Skip references when one file emits too many references (default: 100000; max: 1000000)"); WriteHelpLine(" --parallelism Full-scan extraction workers (default: CPU count capped at 16; also honors CDIDX_INDEX_PARALLELISM)"); WriteHelpLine(" --follow-symlinks Symlink policy for directories and files: none (default), internal, or all"); WriteHelpLine(" --include-symbol-kind [,] Keep only matching symbol kinds during indexing"); diff --git a/src/CodeIndex/Cli/IndexCommandRunner.FullScan.cs b/src/CodeIndex/Cli/IndexCommandRunner.FullScan.cs index 72301d7463..a5ba98b440 100644 --- a/src/CodeIndex/Cli/IndexCommandRunner.FullScan.cs +++ b/src/CodeIndex/Cli/IndexCommandRunner.FullScan.cs @@ -47,6 +47,31 @@ private static FileIssue BuildSymbolCountExceededIssue(string path, int symbolCo Message = $"Symbol extraction produced {symbolCount:N0} symbols, exceeding the --max-symbols-per-file limit of {maxSymbolsPerFile:N0}; file content, symbols, and references were not indexed. Exclude the generated/pathological file or raise --max-symbols-per-file if this is expected.", }; + private static FileIssue BuildReferenceCountExceededIssue(string path, int referenceCount, int maxReferencesPerFile) => + new() + { + Path = path, + Kind = "reference_count_exceeded", + Line = 0, + Message = $"Reference extraction produced {referenceCount:N0} references, exceeding the --max-references-per-file limit of {maxReferencesPerFile:N0}; references were not indexed for this file. Exclude the generated/pathological file or raise --max-references-per-file if this is expected.", + }; + + private static bool ExistingFileViolatesExtractionCaps(DbWriter writer, long fileId, int maxSymbolsPerFile, int maxReferencesPerFile) => + writer.CountSymbolsForFile(fileId) > maxSymbolsPerFile + || writer.HasIssueForFile(fileId, "symbol_count_exceeded") + || writer.CountReferencesForFile(fileId) > maxReferencesPerFile + || writer.HasIssueForFile(fileId, "reference_count_exceeded"); + + internal static IReadOnlyList AppendIssue(IReadOnlyList issues, FileIssue issue) + { + if (issues.Count == 0) + return [issue]; + + var combined = issues.ToList(); + combined.Add(issue); + return combined; + } + internal static string FormatIndexPhasePath(string path, string phase) => $"{path} ({phase})"; @@ -1054,6 +1079,12 @@ void StopJsonHeartbeat() } activeJsonExtractionPhases[workerIndex] = FormatIndexPhasePath(record.Path, "validating"); issues = FileIndexer.ValidateContent(record.Path, rawBytes, content, record.Lang); + if (references.Count > options.MaxReferencesPerFile) + { + var issue = BuildReferenceCountExceededIssue(record.Path, references.Count, options.MaxReferencesPerFile); + references = []; + issues = AppendIssue(issues, issue); + } } extractionResults.Add( FullScanFileWorkItem.Success(filePath, record, content, rawBytes, warning, chunks, symbols, references, issues), @@ -1202,13 +1233,10 @@ void StopJsonHeartbeat() && (record.Lang != "sql" || sqlGraphContractMatchesCurrent) && AllowReuseWithCurrentHotspotFamilyTrust(record.Lang, hotspotFamilyTrustMatchesCurrent)); } - if (existingId != null) + if (existingId != null + && ExistingFileViolatesExtractionCaps(writer, existingId.Value, options.MaxSymbolsPerFile, options.MaxReferencesPerFile)) { - if (writer.CountSymbolsForFile(existingId.Value) > options.MaxSymbolsPerFile - || writer.HasIssueForFile(existingId.Value, "symbol_count_exceeded")) - { - existingId = null; - } + existingId = null; } if (existingId != null) { @@ -1326,6 +1354,13 @@ void StopJsonHeartbeat() cancellationToken) : ReassignReferenceFileIds(item.References, fileId); postExtractionHooks.OnReferencesExtracted(fileContext, AsMutableList(references)); + if (references.Count > options.MaxReferencesPerFile) + { + var issue = BuildReferenceCountExceededIssue(record.Path, references.Count, options.MaxReferencesPerFile); + references = []; + var baseIssues = item.Issues ?? FileIndexer.ValidateContent(record.Path, item.RawBytes!, item.Content!, record.Lang); + item = item with { Issues = AppendIssue(baseIssues, issue) }; + } } writer.InsertReferences(references, refreshMutualRecursionFlags: false); if (references.Count > 0) diff --git a/src/CodeIndex/Cli/IndexCommandRunner.Parse.cs b/src/CodeIndex/Cli/IndexCommandRunner.Parse.cs index 009c78ff9b..4329dbb10f 100644 --- a/src/CodeIndex/Cli/IndexCommandRunner.Parse.cs +++ b/src/CodeIndex/Cli/IndexCommandRunner.Parse.cs @@ -14,7 +14,7 @@ public static partial class IndexCommandRunner [ "--db", "--data-dir", "--rebuild", "--verbose", "--json", "--quiet", "--dry-run", "--force", "--yes", "--watch", "--debounce", "--duration-format", "--max-file-bytes", "--max-symbols-per-file", - "--notify", + "--max-references-per-file", "--notify", "--parallelism", "--memory-trace", "--follow-symlinks", "--symbols-only", "--commits", "--changed-between", "--files", "--solution", "--project", "--include-symbol-kind", "--exclude-symbol-kind", "--optimize", "--help", @@ -49,6 +49,7 @@ public static IndexCommandOptions ParseArgs(string[] args) var notifyMode = ReadCompletionNotificationModeFromEnvironment(); long? maxFileSizeBytes = ReadMaxFileSizeBytesFromEnvironment(); var maxSymbolsPerFile = DefaultMaxSymbolsPerFile; + var maxReferencesPerFile = DefaultMaxReferencesPerFile; var parallelism = ReadIndexParallelismFromEnvironment(); var symlinkPolicy = FileIndexer.SymlinkPolicy.None; string? easterEgg = null; @@ -170,6 +171,12 @@ public static IndexCommandOptions ParseArgs(string[] args) case var option when option.StartsWith("--max-symbols-per-file=", StringComparison.Ordinal): maxSymbolsPerFile = ParseMaxSymbolsPerFile(option["--max-symbols-per-file=".Length..], maxSymbolsPerFile, "--max-symbols-per-file", ref parseError); break; + case "--max-references-per-file" when i + 1 < args.Length: + maxReferencesPerFile = ParseMaxReferencesPerFile(args[++i], maxReferencesPerFile, "--max-references-per-file", ref parseError); + break; + case var option when option.StartsWith("--max-references-per-file=", StringComparison.Ordinal): + maxReferencesPerFile = ParseMaxReferencesPerFile(option["--max-references-per-file=".Length..], maxReferencesPerFile, "--max-references-per-file", ref parseError); + break; case "--parallelism" when i + 1 < args.Length: parallelism = ParseIndexParallelism(args[++i], parallelism, "--parallelism"); break; @@ -328,6 +335,7 @@ public static IndexCommandOptions ParseArgs(string[] args) NotifyMode = notifyMode, MaxFileSizeBytes = maxFileSizeBytes, MaxSymbolsPerFile = maxSymbolsPerFile, + MaxReferencesPerFile = maxReferencesPerFile, Parallelism = parallelism, SymlinkPolicy = symlinkPolicy, SymbolKindFilter = SymbolKindFilter.Create(includeSymbolKinds, excludeSymbolKinds, symbolKindFilterError), @@ -510,6 +518,22 @@ private static int ParseMaxSymbolsPerFile(string value, int fallback, string sou return fallback; } + private static int ParseMaxReferencesPerFile(string value, int fallback, string source, ref string? parseError) + { + if (int.TryParse(value, System.Globalization.NumberStyles.Integer, System.Globalization.CultureInfo.InvariantCulture, out var parsed) && parsed > 0) + { + if (parsed <= MaxReferencesPerFileLimit) + return parsed; + + parseError ??= $"{source} must be less than or equal to {MaxReferencesPerFileLimit}"; + return fallback; + } + + var displayValue = ConsoleUi.FormatBoundedValue(value); + Console.Error.WriteLine($"Warning: invalid {source} value '{displayValue}' (ignored; use a positive integer) / 不正な {source} 値 '{displayValue}'（無視。正の整数を指定）"); + return fallback; + } + private static DurationOutputFormat ParseDurationFormat(string value, DurationOutputFormat fallback) { return value.Trim().ToLowerInvariant() switch diff --git a/src/CodeIndex/Cli/IndexCommandRunner.Update.cs b/src/CodeIndex/Cli/IndexCommandRunner.Update.cs index 9362c10904..2a56b13725 100644 --- a/src/CodeIndex/Cli/IndexCommandRunner.Update.cs +++ b/src/CodeIndex/Cli/IndexCommandRunner.Update.cs @@ -579,13 +579,10 @@ void ThrowIfUpdateCancelled() && (statReusableLanguage != "csharp" || csharpSymbolNameContractMatchesCurrent) && (statReusableLanguage != "csharp" || !csharpWorkspace.HasStaticInterfaceContracts) && (statReusableLanguage != "sql" || sqlGraphContractMatchesCurrent)); - if (statMatchedId != null) + if (statMatchedId != null + && ExistingFileViolatesExtractionCaps(writer, statMatchedId.Value, options.MaxSymbolsPerFile, options.MaxReferencesPerFile)) { - if (writer.CountSymbolsForFile(statMatchedId.Value) > options.MaxSymbolsPerFile - || writer.HasIssueForFile(statMatchedId.Value, "symbol_count_exceeded")) - { - statMatchedId = null; - } + statMatchedId = null; } if (statMatchedId != null) { @@ -621,13 +618,10 @@ void ThrowIfUpdateCancelled() && (record.Lang != "csharp" || csharpSymbolNameContractMatchesCurrent) && (record.Lang != "csharp" || !csharpWorkspace.HasStaticInterfaceContracts) && (record.Lang != "sql" || sqlGraphContractMatchesCurrent)); - if (existingId != null) + if (existingId != null + && ExistingFileViolatesExtractionCaps(writer, existingId.Value, options.MaxSymbolsPerFile, options.MaxReferencesPerFile)) { - if (writer.CountSymbolsForFile(existingId.Value) > options.MaxSymbolsPerFile - || writer.HasIssueForFile(existingId.Value, "symbol_count_exceeded")) - { - existingId = null; - } + existingId = null; } if (existingId != null) { @@ -722,10 +716,18 @@ void ThrowIfUpdateCancelled() record.Lang == "csharp" ? csharpWorkspace.Symbols : null, cancellationToken); postExtractionHooks.OnReferencesExtracted(fileContext, references); + FileIssue? referenceCapIssue = null; + if (references.Count > options.MaxReferencesPerFile) + { + referenceCapIssue = BuildReferenceCountExceededIssue(record.Path, references.Count, options.MaxReferencesPerFile); + references = []; + } writer.InsertReferences(references); // Validate content for encoding issues / エンコーディング問題を検証 currentUpdatePath = FormatIndexPhasePath(relPath, "validating"); - var issues = FileIndexer.ValidateContent(record.Path, rawBytes, content, record.Lang); + IReadOnlyList issues = FileIndexer.ValidateContent(record.Path, rawBytes, content, record.Lang); + if (referenceCapIssue != null) + issues = AppendIssue(issues, referenceCapIssue); writer.InsertIssues(fileId, issues); currentUpdatePath = FormatIndexPhasePath(relPath, "committing"); writer.ClearBatchInProgress(); diff --git a/src/CodeIndex/Cli/IndexCommandRunner.cs b/src/CodeIndex/Cli/IndexCommandRunner.cs index 1c79e1e445..eabd6d457f 100644 --- a/src/CodeIndex/Cli/IndexCommandRunner.cs +++ b/src/CodeIndex/Cli/IndexCommandRunner.cs @@ -21,6 +21,8 @@ public static partial class IndexCommandRunner internal const string ExcludeSymbolKindsEnvironmentVariable = "CDIDX_INDEX_EXCLUDE_SYMBOL_KINDS"; internal const int DefaultMaxSymbolsPerFile = 5000; internal const int MaxSymbolsPerFileLimit = 50_000; + internal const int DefaultMaxReferencesPerFile = 100_000; + internal const int MaxReferencesPerFileLimit = 1_000_000; internal const int MaxCommitRefCount = 64; internal const int MaxCommitRefLength = 256; internal const int MaxGitExcludeBytes = 256 * 1024; @@ -1592,6 +1594,7 @@ public sealed class IndexCommandOptions public CompletionNotificationMode NotifyMode { get; init; } = CompletionNotificationMode.Auto; public long? MaxFileSizeBytes { get; init; } public int MaxSymbolsPerFile { get; init; } = IndexCommandRunner.DefaultMaxSymbolsPerFile; + public int MaxReferencesPerFile { get; init; } = IndexCommandRunner.DefaultMaxReferencesPerFile; public int Parallelism { get; init; } = IndexCommandRunner.DefaultIndexParallelism(); public bool MemoryTrace { get; init; } public FileIndexer.SymlinkPolicy SymlinkPolicy { get; init; } = FileIndexer.SymlinkPolicy.None; diff --git a/src/CodeIndex/Cli/IndexWatchRunner.cs b/src/CodeIndex/Cli/IndexWatchRunner.cs index ff423d16e3..b8eb3bb659 100644 --- a/src/CodeIndex/Cli/IndexWatchRunner.cs +++ b/src/CodeIndex/Cli/IndexWatchRunner.cs @@ -224,6 +224,11 @@ private static List BuildSubRunArgs(IndexCommandOptions baseOptions, str args.Add("--max-symbols-per-file"); args.Add(baseOptions.MaxSymbolsPerFile.ToString(CultureInfo.InvariantCulture)); } + if (baseOptions.MaxReferencesPerFile != IndexCommandRunner.DefaultMaxReferencesPerFile) + { + args.Add("--max-references-per-file"); + args.Add(baseOptions.MaxReferencesPerFile.ToString(CultureInfo.InvariantCulture)); + } if (baseOptions.Parallelism != IndexCommandRunner.DefaultIndexParallelism()) { args.Add("--parallelism"); diff --git a/src/CodeIndex/Database/DbWriter.cs b/src/CodeIndex/Database/DbWriter.cs index a14c2d5116..4053d8d9dc 100644 --- a/src/CodeIndex/Database/DbWriter.cs +++ b/src/CodeIndex/Database/DbWriter.cs @@ -720,6 +720,14 @@ public int CountSymbolsForFile(long fileId) return Convert.ToInt32(cmd.ExecuteScalar()); } + public int CountReferencesForFile(long fileId) + { + using var cmd = _conn.CreateCommand(); + cmd.CommandText = "SELECT COUNT(*) FROM symbol_references WHERE file_id = @file_id"; + cmd.Parameters.AddWithValue("@file_id", fileId); + return Convert.ToInt32(cmd.ExecuteScalar()); + } + public bool HasIssueForFile(long fileId, string kind) { using var cmd = _conn.CreateCommand(); diff --git a/src/CodeIndex/Mcp/McpToolDefinitions.cs b/src/CodeIndex/Mcp/McpToolDefinitions.cs index 585ee31462..dc27e72c5e 100644 --- a/src/CodeIndex/Mcp/McpToolDefinitions.cs +++ b/src/CodeIndex/Mcp/McpToolDefinitions.cs @@ -478,6 +478,7 @@ private JsonNode HandleToolsList(JsonNode? id) ["dryRun"] = new JsonObject { ["type"] = "boolean", ["description"] = "Plan the index run without mutating the database. Reports scan counts, effective options, and unsupported MCP modes.", ["default"] = false }, ["maxFileBytes"] = new JsonObject { ["type"] = "integer", ["description"] = "Override the per-file indexing size limit for this run. Defaults to CDIDX_MAX_FILE_BYTES or 4MiB.", ["minimum"] = 1, ["maximum"] = int.MaxValue }, ["maxSymbolsPerFile"] = new JsonObject { ["type"] = "integer", ["description"] = "Skip symbol/reference indexing for files that produce more symbols than this limit, matching CLI --max-symbols-per-file.", ["default"] = IndexCommandRunner.DefaultMaxSymbolsPerFile, ["minimum"] = 1, ["maximum"] = IndexCommandRunner.MaxSymbolsPerFileLimit }, + ["maxReferencesPerFile"] = new JsonObject { ["type"] = "integer", ["description"] = "Skip references for files that produce more references than this limit, matching CLI --max-references-per-file.", ["default"] = IndexCommandRunner.DefaultMaxReferencesPerFile, ["minimum"] = 1, ["maximum"] = IndexCommandRunner.MaxReferencesPerFileLimit }, ["followSymlinks"] = new JsonObject { ["type"] = "string", ["enum"] = new JsonArray { "none", "internal", "all" }, ["description"] = "Directory and file symlink policy matching CLI --follow-symlinks.", ["default"] = "none" }, ["includeSymbolKind"] = new JsonObject { ["oneOf"] = new JsonArray { new JsonObject { ["type"] = "string" }, new JsonObject { ["type"] = "array", ["items"] = new JsonObject { ["type"] = "string" } } }, ["description"] = "Only index symbols with these kinds. Accepts a value, comma-separated string, or array." }, ["excludeSymbolKind"] = new JsonObject { ["oneOf"] = new JsonArray { new JsonObject { ["type"] = "string" }, new JsonObject { ["type"] = "array", ["items"] = new JsonObject { ["type"] = "string" } } }, ["description"] = "Drop symbols with these kinds before indexing. Accepts a value, comma-separated string, or array." }, diff --git a/src/CodeIndex/Mcp/McpToolHandlers.cs b/src/CodeIndex/Mcp/McpToolHandlers.cs index d31ee62a10..3481ae3c77 100644 --- a/src/CodeIndex/Mcp/McpToolHandlers.cs +++ b/src/CodeIndex/Mcp/McpToolHandlers.cs @@ -5331,6 +5331,15 @@ private static FileIssue BuildMcpSymbolCountExceededIssue(string path, int symbo Message = $"Symbol extraction produced {symbolCount:N0} symbols, exceeding the maxSymbolsPerFile limit of {maxSymbolsPerFile:N0}; file content, symbols, and references were not indexed. Exclude the generated/pathological file or raise maxSymbolsPerFile if this is expected.", }; + private static FileIssue BuildMcpReferenceCountExceededIssue(string path, int referenceCount, int maxReferencesPerFile) => + new() + { + Path = path, + Kind = "reference_count_exceeded", + Line = 0, + Message = $"Reference extraction produced {referenceCount:N0} references, exceeding the maxReferencesPerFile limit of {maxReferencesPerFile:N0}; references were not indexed for this file. Exclude the generated/pathological file or raise maxReferencesPerFile if this is expected.", + }; + private static bool TryReadMcpIndexSymlinkPolicy(JsonNode? args, out FileIndexer.SymlinkPolicy symlinkPolicy, out string? error) { symlinkPolicy = FileIndexer.SymlinkPolicy.None; @@ -5421,6 +5430,7 @@ private static JsonObject BuildMcpIndexOptionsPayload( bool rebuild, long? maxFileBytes, int maxSymbolsPerFile, + int maxReferencesPerFile, FileIndexer.SymlinkPolicy symlinkPolicy, IReadOnlyList includeSymbolKinds, IReadOnlyList excludeSymbolKinds, @@ -5434,6 +5444,7 @@ private static JsonObject BuildMcpIndexOptionsPayload( ["rebuild"] = rebuild, ["maxFileBytes"] = maxFileBytes.HasValue ? JsonValue.Create(maxFileBytes.Value) : null, ["maxSymbolsPerFile"] = maxSymbolsPerFile, + ["maxReferencesPerFile"] = maxReferencesPerFile, ["followSymlinks"] = FormatMcpIndexSymlinkPolicy(symlinkPolicy), ["includeSymbolKind"] = ToJsonStringArray(includeSymbolKinds), ["excludeSymbolKind"] = ToJsonStringArray(excludeSymbolKinds), @@ -5514,6 +5525,9 @@ private async Task ExecuteIndexAsync(JsonNode? id, JsonNode? args, Jso var maxSymbolsPerFile = args?["maxSymbolsPerFile"]?.GetValue() ?? IndexCommandRunner.DefaultMaxSymbolsPerFile; if (maxSymbolsPerFile <= 0 || maxSymbolsPerFile > IndexCommandRunner.MaxSymbolsPerFileLimit) return CreateToolErrorResponse(id, $"maxSymbolsPerFile must be between 1 and {IndexCommandRunner.MaxSymbolsPerFileLimit}"); + var maxReferencesPerFile = args?["maxReferencesPerFile"]?.GetValue() ?? IndexCommandRunner.DefaultMaxReferencesPerFile; + if (maxReferencesPerFile <= 0 || maxReferencesPerFile > IndexCommandRunner.MaxReferencesPerFileLimit) + return CreateToolErrorResponse(id, $"maxReferencesPerFile must be between 1 and {IndexCommandRunner.MaxReferencesPerFileLimit}"); if (!TryReadMcpIndexSymlinkPolicy(args, out var symlinkPolicy, out var symlinkPolicyError)) return CreateToolErrorResponse(id, symlinkPolicyError!); var includeSymbolKinds = ReadStringOrCommaSeparatedList(args, "includeSymbolKind"); @@ -5547,6 +5561,7 @@ private async Task ExecuteIndexAsync(JsonNode? id, JsonNode? args, Jso rebuild, maxFileBytes, maxSymbolsPerFile, + maxReferencesPerFile, symlinkPolicy, includeSymbolKinds, excludeSymbolKinds, @@ -5831,7 +5846,9 @@ static string FormatDiagnosticPath(string projectRoot, string path) if (existingId != null) { if (writer.CountSymbolsForFile(existingId.Value) > maxSymbolsPerFile - || writer.HasIssueForFile(existingId.Value, "symbol_count_exceeded")) + || writer.HasIssueForFile(existingId.Value, "symbol_count_exceeded") + || writer.CountReferencesForFile(existingId.Value) > maxReferencesPerFile + || writer.HasIssueForFile(existingId.Value, "reference_count_exceeded")) { existingId = null; } @@ -5876,10 +5893,18 @@ static string FormatDiagnosticPath(string projectRoot, string path) record.Lang == "csharp" ? csharpWorkspace.Symbols : null, requestToken); postExtractionHooks.OnReferencesExtracted(fileContext, references); + FileIssue? referenceCapIssue = null; + if (references.Count > maxReferencesPerFile) + { + referenceCapIssue = BuildMcpReferenceCountExceededIssue(record.Path, references.Count, maxReferencesPerFile); + references = []; + } writer.InsertReferences(references); // Keep MCP index parity with CLI index: persist file-level validation issues too. // MCPインデックスもCLIインデックスと同等に、ファイル検証issueを保存する。 - var issues = FileIndexer.ValidateContent(record.Path, rawBytes, content, record.Lang); + IReadOnlyList issues = FileIndexer.ValidateContent(record.Path, rawBytes, content, record.Lang); + if (referenceCapIssue != null) + issues = IndexCommandRunner.AppendIssue(issues, referenceCapIssue); writer.InsertIssues(fileId, issues); } WriteProjectRootOnce(); diff --git a/tests/CodeIndex.Tests/ConsoleUiTests.cs b/tests/CodeIndex.Tests/ConsoleUiTests.cs index e6fa58acbf..8996886eaa 100644 --- a/tests/CodeIndex.Tests/ConsoleUiTests.cs +++ b/tests/CodeIndex.Tests/ConsoleUiTests.cs @@ -103,7 +103,7 @@ public void PrintUsage_WithoutBanner_HidesAsciiArtAndEasterEggFlags() Assert.DoesNotContain("██████╗", output); Assert.Contains("Usage:", output); - Assert.Contains("cdidx index [--db ] [--rebuild] [--optimize] [--symbols-only] [--verbose] [--dry-run] [--force] [--quiet] [--json] [--memory-trace] [--duration-format ] [--notify ] [--max-file-bytes ] [--max-symbols-per-file ] [--follow-symlinks ]", output); + Assert.Contains("cdidx index [--db ] [--rebuild] [--optimize] [--symbols-only] [--verbose] [--dry-run] [--force] [--quiet] [--json] [--memory-trace] [--duration-format ] [--notify ] [--max-file-bytes ] [--max-symbols-per-file ] [--max-references-per-file ] [--follow-symlinks ]", output); Assert.Contains("cdidx hooks [--project ] [--force] [--json]", output); Assert.Contains("cdidx index --commits [commit-ref ...] [--db ] [--verbose] [--dry-run] [--json] [--memory-trace] [--duration-format ]", output); Assert.Contains("cdidx index --files [path ...] [--db ] [--verbose] [--dry-run] [--json] [--memory-trace] [--duration-format ]", output); diff --git a/tests/CodeIndex.Tests/IndexCommandRunnerTests.cs b/tests/CodeIndex.Tests/IndexCommandRunnerTests.cs index 44fdab716e..b5affae01b 100644 --- a/tests/CodeIndex.Tests/IndexCommandRunnerTests.cs +++ b/tests/CodeIndex.Tests/IndexCommandRunnerTests.cs @@ -1126,6 +1126,111 @@ public void Run_FileAboveMaxSymbolsPerFile_PersistsSymbolCountExceededIssueOnly( } } + [Fact] + public void Run_FileAboveMaxReferencesPerFile_FullScanPersistsReferenceCountExceededIssueOnly_Issue3719() + { + var projectRoot = CreateTempProject(); + try + { + var filePath = Path.Combine(projectRoot, "DenseReferences.cs"); + File.WriteAllText(filePath, BuildDenseReferenceCSharpSource(8)); + + var (exitCode, json) = RunAndCaptureJson([projectRoot, "--max-references-per-file", "2", "--json"]); + + Assert.Equal(CommandExitCodes.Success, exitCode); + Assert.Equal("success", json.GetProperty("status").GetString()); + Assert.Equal(0, json.GetProperty("summary").GetProperty("errors").GetInt32()); + + var dbPath = Path.Combine(projectRoot, ".cdidx", "codeindex.db"); + Assert.Equal(1, CountRows(dbPath, "files")); + Assert.True(CountRows(dbPath, "chunks") > 0); + Assert.True(CountRows(dbPath, "symbols") > 0); + Assert.Equal(0, CountRows(dbPath, "symbol_references")); + + using var db = new DbContext(dbPath); + db.TryMigrateForRead(); + var reader = new DbReader(db.Connection, db.IsReadOnly); + var issue = Assert.Single(reader.GetIssues("reference_count_exceeded")); + Assert.Equal("DenseReferences.cs", issue.Path); + Assert.Equal(0, issue.Line); + Assert.Contains("--max-references-per-file", issue.Message); + + var (raisedExitCode, raisedJson) = RunAndCaptureJson([projectRoot, "--max-references-per-file", "100", "--json"]); + + Assert.Equal(CommandExitCodes.Success, raisedExitCode); + Assert.Equal("success", raisedJson.GetProperty("status").GetString()); + Assert.True(CountRows(dbPath, "symbol_references") > 0); + Assert.Empty(reader.GetIssues("reference_count_exceeded")); + } + finally + { + SqliteConnection.ClearAllPools(); + DeleteDirectory(projectRoot); + } + } + + [Fact] + public void Run_FileAboveMaxReferencesPerFile_UpdatePersistsReferenceCountExceededIssueOnly_Issue3719() + { + var projectRoot = CreateTempProject(); + try + { + var filePath = Path.Combine(projectRoot, "DenseReferences.cs"); + File.WriteAllText(filePath, BuildDenseReferenceCSharpSource(8)); + + var (initialExitCode, _) = RunAndCaptureJson([projectRoot, "--max-references-per-file", "100", "--json"]); + Assert.Equal(CommandExitCodes.Success, initialExitCode); + + var dbPath = Path.Combine(projectRoot, ".cdidx", "codeindex.db"); + Assert.True(CountRows(dbPath, "symbol_references") > 0); + + var (exitCode, json) = RunAndCaptureJson([projectRoot, "--files", filePath, "--max-references-per-file", "2", "--json"]); + + Assert.Equal(CommandExitCodes.Success, exitCode); + Assert.Equal("success", json.GetProperty("status").GetString()); + Assert.Equal(0, json.GetProperty("summary").GetProperty("errors").GetInt32()); + Assert.Equal(1, CountRows(dbPath, "files")); + Assert.True(CountRows(dbPath, "chunks") > 0); + Assert.True(CountRows(dbPath, "symbols") > 0); + Assert.Equal(0, CountRows(dbPath, "symbol_references")); + + using var db = new DbContext(dbPath); + db.TryMigrateForRead(); + var reader = new DbReader(db.Connection, db.IsReadOnly); + var issue = Assert.Single(reader.GetIssues("reference_count_exceeded")); + Assert.Equal("DenseReferences.cs", issue.Path); + Assert.Contains("--max-references-per-file", issue.Message); + } + finally + { + SqliteConnection.ClearAllPools(); + DeleteDirectory(projectRoot); + } + } + + private static string BuildDenseReferenceCSharpSource(int callCount) + { + var calls = string.Join('\n', Enumerable.Range(0, callCount).Select(static _ => " Target.Ping();")); + return $$""" +namespace DenseReferences; + +public static class Target +{ + public static void Ping() + { + } +} + +public sealed class Caller +{ + public void Run() + { +{{calls}} + } +} +"""; + } + [Fact] public void Run_SymbolsOnly_FullScanSkipsReferenceGraphUntilNormalIndex() { @@ -2707,6 +2812,22 @@ public void ParseArgs_MaxSymbolsPerFileInlineFlag_ParsesPositiveValue() Assert.Equal(43, options.MaxSymbolsPerFile); } + [Fact] + public void ParseArgs_MaxReferencesPerFileFlag_ParsesPositiveValue_Issue3719() + { + var options = IndexCommandRunner.ParseArgs([".", "--max-references-per-file", "42"]); + + Assert.Equal(42, options.MaxReferencesPerFile); + } + + [Fact] + public void ParseArgs_MaxReferencesPerFileInlineFlag_ParsesPositiveValue_Issue3719() + { + var options = IndexCommandRunner.ParseArgs([".", "--max-references-per-file=43"]); + + Assert.Equal(43, options.MaxReferencesPerFile); + } + [Fact] public void ParseArgs_SymbolsOnlyFlag_SetsOption() { @@ -2734,6 +2855,16 @@ public void ParseArgs_MaxSymbolsPerFileFlag_RejectsValueAboveMaximum_Issue3172() Assert.Contains("--max-symbols-per-file must be less than or equal to 50000", options.ParseError); } + [Fact] + public void ParseArgs_MaxReferencesPerFileFlag_RejectsValueAboveMaximum_Issue3719() + { + var aboveMaximum = $"{IndexCommandRunner.MaxReferencesPerFileLimit + 1}"; + var options = IndexCommandRunner.ParseArgs([".", $"--max-references-per-file={aboveMaximum}"]); + + Assert.Equal(IndexCommandRunner.DefaultMaxReferencesPerFile, options.MaxReferencesPerFile); + Assert.Contains("--max-references-per-file must be less than or equal to 1000000", options.ParseError); + } + [Fact] public void ParseArgs_MaxFileBytesInvalidValue_IsIgnored() { From 2b34e4bcdcc07635411a7b69b40572c119ce11e8 Mon Sep 17 00:00:00 2001 From: Widthdom Date: Sat, 20 Jun 2026 12:09:24 +0900 Subject: [PATCH 02/18] Surface regex timeout diagnostics (#3800) --- changelog.d/unreleased/3800.fixed.md | 18 +++ .../Cli/IndexCommandRunner.FullScan.cs | 45 ++++++- .../Cli/IndexCommandRunner.Update.cs | 24 ++-- src/CodeIndex/Indexer/BoundedRegex.cs | 114 +++++++++++++++--- src/CodeIndex/Mcp/McpToolHandlers.cs | 24 ++-- tests/CodeIndex.Tests/BoundedRegexTests.cs | 23 ++++ 6 files changed, 211 insertions(+), 37 deletions(-) create mode 100644 changelog.d/unreleased/3800.fixed.md diff --git a/changelog.d/unreleased/3800.fixed.md b/changelog.d/unreleased/3800.fixed.md new file mode 100644 index 0000000000..3a53106b84 --- /dev/null +++ b/changelog.d/unreleased/3800.fixed.md @@ -0,0 +1,18 @@ +--- +category: fixed +issues: + - 3800 +affected: + - src/CodeIndex/Indexer/BoundedRegex.cs + - src/CodeIndex/Cli/IndexCommandRunner.FullScan.cs + - src/CodeIndex/Cli/IndexCommandRunner.Update.cs + - src/CodeIndex/Mcp/McpToolHandlers.cs +--- + +## English + +- **Regex timeout fallbacks are now visible in file issues (#3800)** — reference extraction records bounded `regex_timeout` diagnostics with hashed pattern metadata when a guarded regex falls back to no-match behavior, so skipped matches no longer disappear silently. + +## 日本語 + +- **regex timeout fallback が file issue に表示されるようになりました (#3800)** — reference 抽出中に保護された regex が no-match fallback へ切り替わった場合、ハッシュ化した pattern メタデータ付きの bounded な `regex_timeout` 診断を記録し、match の欠落が無言で起きないようにしました。 diff --git a/src/CodeIndex/Cli/IndexCommandRunner.FullScan.cs b/src/CodeIndex/Cli/IndexCommandRunner.FullScan.cs index a5ba98b440..37c7cb79d6 100644 --- a/src/CodeIndex/Cli/IndexCommandRunner.FullScan.cs +++ b/src/CodeIndex/Cli/IndexCommandRunner.FullScan.cs @@ -56,6 +56,25 @@ private static FileIssue BuildReferenceCountExceededIssue(string path, int refer Message = $"Reference extraction produced {referenceCount:N0} references, exceeding the --max-references-per-file limit of {maxReferencesPerFile:N0}; references were not indexed for this file. Exclude the generated/pathological file or raise --max-references-per-file if this is expected.", }; + internal static FileIssue? BuildRegexTimeoutIssue(string path, BoundedRegex.RegexTimeoutCaptureScope capture) + { + if (!capture.HasTimeouts) + return null; + + var samples = capture.Diagnostics.Count == 0 + ? "none" + : string.Join(", ", capture.Diagnostics.Select(static diagnostic => + $"{diagnostic.Operation}:{diagnostic.PatternHash} len={diagnostic.PatternLength} timeout={diagnostic.TimeoutMs:0.###}ms")); + var truncationSuffix = capture.DiagnosticsTruncated ? "; additional timeout diagnostics omitted" : string.Empty; + return new FileIssue + { + Path = path, + Kind = "regex_timeout", + Line = 0, + Message = $"Regex timeout fallback occurred during {capture.PatternFamily} for language {capture.Language} ({capture.TimeoutCount:N0} timeout(s); samples {samples}{truncationSuffix}); extraction used a safe no-match fallback and may be incomplete for this file.", + }; + } + private static bool ExistingFileViolatesExtractionCaps(DbWriter writer, long fileId, int maxSymbolsPerFile, int maxReferencesPerFile) => writer.CountSymbolsForFile(fileId) > maxSymbolsPerFile || writer.HasIssueForFile(fileId, "symbol_count_exceeded") @@ -1061,6 +1080,7 @@ void StopJsonHeartbeat() continue; } SymbolExtractor.ApplyFamilyScope(symbols, indexer.GetFamilyScopeKey(filePath, record.Lang)); + FileIssue? regexTimeoutIssue = null; if (options.SymbolsOnly) { references = []; @@ -1068,6 +1088,7 @@ void StopJsonHeartbeat() else { activeJsonExtractionPhases[workerIndex] = FormatIndexPhasePath(record.Path, "references"); + using var regexTimeouts = BoundedRegex.CaptureTimeouts(record.Lang, "reference_extraction"); references = ReferenceExtractor.Extract( 0, record.Lang, @@ -1076,9 +1097,12 @@ void StopJsonHeartbeat() record.Path, record.Lang == "csharp" ? csharpWorkspace.Symbols : null, extractionCancellationToken); + regexTimeoutIssue = BuildRegexTimeoutIssue(record.Path, regexTimeouts); } activeJsonExtractionPhases[workerIndex] = FormatIndexPhasePath(record.Path, "validating"); issues = FileIndexer.ValidateContent(record.Path, rawBytes, content, record.Lang); + if (regexTimeoutIssue != null) + issues = AppendIssue(issues, regexTimeoutIssue); if (references.Count > options.MaxReferencesPerFile) { var issue = BuildReferenceCountExceededIssue(record.Path, references.Count, options.MaxReferencesPerFile); @@ -1343,17 +1367,30 @@ void StopJsonHeartbeat() } else { - references = item.References == null - ? ReferenceExtractor.Extract( + FileIssue? regexTimeoutIssue = null; + if (item.References == null) + { + using var regexTimeouts = BoundedRegex.CaptureTimeouts(record.Lang, "reference_extraction"); + references = ReferenceExtractor.Extract( fileId, record.Lang, item.Content!, symbols, record.Path, record.Lang == "csharp" ? csharpWorkspace.Symbols : null, - cancellationToken) - : ReassignReferenceFileIds(item.References, fileId); + cancellationToken); + regexTimeoutIssue = BuildRegexTimeoutIssue(record.Path, regexTimeouts); + } + else + { + references = ReassignReferenceFileIds(item.References, fileId); + } postExtractionHooks.OnReferencesExtracted(fileContext, AsMutableList(references)); + if (regexTimeoutIssue != null) + { + var baseIssues = item.Issues ?? FileIndexer.ValidateContent(record.Path, item.RawBytes!, item.Content!, record.Lang); + item = item with { Issues = AppendIssue(baseIssues, regexTimeoutIssue) }; + } if (references.Count > options.MaxReferencesPerFile) { var issue = BuildReferenceCountExceededIssue(record.Path, references.Count, options.MaxReferencesPerFile); diff --git a/src/CodeIndex/Cli/IndexCommandRunner.Update.cs b/src/CodeIndex/Cli/IndexCommandRunner.Update.cs index 2a56b13725..b4aac83395 100644 --- a/src/CodeIndex/Cli/IndexCommandRunner.Update.cs +++ b/src/CodeIndex/Cli/IndexCommandRunner.Update.cs @@ -707,14 +707,20 @@ void ThrowIfUpdateCancelled() FileIndexer.ValidateSymbolLineRanges(record, symbols); writer.InsertSymbols(symbols); currentUpdatePath = FormatIndexPhasePath(relPath, "references"); - var references = ReferenceExtractor.Extract( - fileId, - record.Lang, - content, - symbols, - record.Path, - record.Lang == "csharp" ? csharpWorkspace.Symbols : null, - cancellationToken); + List references; + FileIssue? regexTimeoutIssue; + using (var regexTimeouts = BoundedRegex.CaptureTimeouts(record.Lang, "reference_extraction")) + { + references = ReferenceExtractor.Extract( + fileId, + record.Lang, + content, + symbols, + record.Path, + record.Lang == "csharp" ? csharpWorkspace.Symbols : null, + cancellationToken); + regexTimeoutIssue = BuildRegexTimeoutIssue(record.Path, regexTimeouts); + } postExtractionHooks.OnReferencesExtracted(fileContext, references); FileIssue? referenceCapIssue = null; if (references.Count > options.MaxReferencesPerFile) @@ -726,6 +732,8 @@ void ThrowIfUpdateCancelled() // Validate content for encoding issues / エンコーディング問題を検証 currentUpdatePath = FormatIndexPhasePath(relPath, "validating"); IReadOnlyList issues = FileIndexer.ValidateContent(record.Path, rawBytes, content, record.Lang); + if (regexTimeoutIssue != null) + issues = AppendIssue(issues, regexTimeoutIssue); if (referenceCapIssue != null) issues = AppendIssue(issues, referenceCapIssue); writer.InsertIssues(fileId, issues); diff --git a/src/CodeIndex/Indexer/BoundedRegex.cs b/src/CodeIndex/Indexer/BoundedRegex.cs index f92dc11fd5..5705cca2c5 100644 --- a/src/CodeIndex/Indexer/BoundedRegex.cs +++ b/src/CodeIndex/Indexer/BoundedRegex.cs @@ -1,4 +1,7 @@ +using System.Security.Cryptography; +using System.Text; using System.Text.RegularExpressions; +using System.Threading; using BclMatch = System.Text.RegularExpressions.Match; using BclRegex = System.Text.RegularExpressions.Regex; @@ -8,6 +11,57 @@ internal sealed class BoundedRegex : BclRegex { // Keep regex matches bounded, but leave enough scheduler headroom for full-suite CI contention. internal static readonly TimeSpan DefaultMatchTimeout = TimeSpan.FromSeconds(2); + private const int MaxCapturedTimeoutDiagnostics = 8; + private static readonly AsyncLocal TimeoutCaptureScope = new(); + + internal readonly record struct RegexTimeoutDiagnostic( + string Operation, + string PatternHash, + int PatternLength, + double TimeoutMs); + + internal sealed class RegexTimeoutCaptureScope : IDisposable + { + private readonly RegexTimeoutCaptureScope? _previous; + private readonly List _diagnostics = []; + + internal RegexTimeoutCaptureScope(string? language, string patternFamily) + { + _previous = TimeoutCaptureScope.Value; + Language = string.IsNullOrWhiteSpace(language) ? "unknown" : language; + PatternFamily = patternFamily; + TimeoutCaptureScope.Value = this; + } + + public string Language { get; } + public string PatternFamily { get; } + public int TimeoutCount { get; private set; } + public IReadOnlyList Diagnostics => _diagnostics; + public bool DiagnosticsTruncated => TimeoutCount > _diagnostics.Count; + public bool HasTimeouts => TimeoutCount > 0; + + internal void Record(string operation, string pattern, TimeSpan timeout) + { + TimeoutCount++; + if (_diagnostics.Count >= MaxCapturedTimeoutDiagnostics) + return; + + _diagnostics.Add(new RegexTimeoutDiagnostic( + operation, + HashPattern(pattern), + pattern.Length, + timeout.TotalMilliseconds)); + } + + public void Dispose() + { + if (ReferenceEquals(TimeoutCaptureScope.Value, this)) + TimeoutCaptureScope.Value = _previous; + } + } + + internal static RegexTimeoutCaptureScope CaptureTimeouts(string? language, string patternFamily) => + new(language, patternFamily); public BoundedRegex(string pattern) : base(pattern, RegexOptions.None, DefaultMatchTimeout) @@ -37,8 +91,9 @@ public BoundedRegex(string pattern, RegexOptions options, TimeSpan matchTimeout) { return BclRegex.Match(input, pattern, options, DefaultMatchTimeout); } - catch (RegexMatchTimeoutException) + catch (RegexMatchTimeoutException ex) { + RecordTimeout("match", pattern, ex); return BclMatch.Empty; } } @@ -49,8 +104,9 @@ public static BclMatch Match(BclRegex regex, string input) { return regex.Match(input); } - catch (RegexMatchTimeoutException) + catch (RegexMatchTimeoutException ex) { + RecordTimeout("match", regex.ToString(), ex); return BclMatch.Empty; } } @@ -66,8 +122,9 @@ public static BclMatch Match(BclRegex regex, string input) _ = matches.Count; return matches; } - catch (RegexMatchTimeoutException) + catch (RegexMatchTimeoutException ex) { + RecordTimeout("matches", pattern, ex); return EmptyMatches(); } } @@ -80,8 +137,9 @@ public static IEnumerable EnumerateMatches(BclRegex regex, string inpu matches = regex.Matches(input); _ = matches.Count; } - catch (RegexMatchTimeoutException) + catch (RegexMatchTimeoutException ex) { + RecordTimeout("matches", regex.ToString(), ex); yield break; } @@ -100,8 +158,9 @@ public static IEnumerable EnumerateMatches(string input, string patter matches = BclRegex.Matches(input, pattern, options, DefaultMatchTimeout); _ = matches.Count; } - catch (RegexMatchTimeoutException) + catch (RegexMatchTimeoutException ex) { + RecordTimeout("matches", pattern, ex); yield break; } @@ -118,8 +177,9 @@ public static IEnumerable EnumerateMatches(string input, string patter { return BclRegex.IsMatch(input, pattern, options, DefaultMatchTimeout); } - catch (RegexMatchTimeoutException) + catch (RegexMatchTimeoutException ex) { + RecordTimeout("is_match", pattern, ex); return false; } } @@ -133,8 +193,9 @@ public static IEnumerable EnumerateMatches(string input, string patter { return BclRegex.Replace(input, pattern, replacement, options, DefaultMatchTimeout); } - catch (RegexMatchTimeoutException) + catch (RegexMatchTimeoutException ex) { + RecordTimeout("replace", pattern, ex); return input; } } @@ -148,8 +209,9 @@ public static IEnumerable EnumerateMatches(string input, string patter { return BclRegex.Replace(input, pattern, evaluator, options, DefaultMatchTimeout); } - catch (RegexMatchTimeoutException) + catch (RegexMatchTimeoutException ex) { + RecordTimeout("replace", pattern, ex); return input; } } @@ -160,8 +222,9 @@ public static IEnumerable EnumerateMatches(string input, string patter { return base.Match(input); } - catch (RegexMatchTimeoutException) + catch (RegexMatchTimeoutException ex) { + RecordTimeout("match", ToString(), ex); return BclMatch.Empty; } } @@ -172,8 +235,9 @@ public static IEnumerable EnumerateMatches(string input, string patter { return base.Match(input, startat); } - catch (RegexMatchTimeoutException) + catch (RegexMatchTimeoutException ex) { + RecordTimeout("match", ToString(), ex); return BclMatch.Empty; } } @@ -184,8 +248,9 @@ public static IEnumerable EnumerateMatches(string input, string patter { return base.Match(input, beginning, length); } - catch (RegexMatchTimeoutException) + catch (RegexMatchTimeoutException ex) { + RecordTimeout("match", ToString(), ex); return BclMatch.Empty; } } @@ -198,8 +263,9 @@ public static IEnumerable EnumerateMatches(string input, string patter _ = matches.Count; return matches; } - catch (RegexMatchTimeoutException) + catch (RegexMatchTimeoutException ex) { + RecordTimeout("matches", ToString(), ex); return EmptyMatches(); } } @@ -212,8 +278,9 @@ public static IEnumerable EnumerateMatches(string input, string patter _ = matches.Count; return matches; } - catch (RegexMatchTimeoutException) + catch (RegexMatchTimeoutException ex) { + RecordTimeout("matches", ToString(), ex); return EmptyMatches(); } } @@ -224,8 +291,9 @@ public static IEnumerable EnumerateMatches(string input, string patter { return base.IsMatch(input); } - catch (RegexMatchTimeoutException) + catch (RegexMatchTimeoutException ex) { + RecordTimeout("is_match", ToString(), ex); return false; } } @@ -236,8 +304,9 @@ public static IEnumerable EnumerateMatches(string input, string patter { return base.IsMatch(input, startat); } - catch (RegexMatchTimeoutException) + catch (RegexMatchTimeoutException ex) { + RecordTimeout("is_match", ToString(), ex); return false; } } @@ -248,8 +317,9 @@ public static IEnumerable EnumerateMatches(string input, string patter { return base.Replace(input, replacement); } - catch (RegexMatchTimeoutException) + catch (RegexMatchTimeoutException ex) { + RecordTimeout("replace", ToString(), ex); return input; } } @@ -260,12 +330,22 @@ public static IEnumerable EnumerateMatches(string input, string patter { return base.Replace(input, evaluator); } - catch (RegexMatchTimeoutException) + catch (RegexMatchTimeoutException ex) { + RecordTimeout("replace", ToString(), ex); return input; } } + private static void RecordTimeout(string operation, string pattern, RegexMatchTimeoutException ex) => + TimeoutCaptureScope.Value?.Record(operation, pattern, ex.MatchTimeout); + + private static string HashPattern(string pattern) + { + var hash = SHA256.HashData(Encoding.UTF8.GetBytes(pattern)); + return Convert.ToHexString(hash.AsSpan(0, 8)).ToLowerInvariant(); + } + private static MatchCollection EmptyMatches() => BclRegex.Matches(string.Empty, @"\b\B", RegexOptions.None, DefaultMatchTimeout); } diff --git a/src/CodeIndex/Mcp/McpToolHandlers.cs b/src/CodeIndex/Mcp/McpToolHandlers.cs index 3481ae3c77..3132d1ccfe 100644 --- a/src/CodeIndex/Mcp/McpToolHandlers.cs +++ b/src/CodeIndex/Mcp/McpToolHandlers.cs @@ -5884,14 +5884,20 @@ static string FormatDiagnosticPath(string projectRoot, string path) { writer.InsertChunks(chunks); writer.InsertSymbols(symbols); - var references = ReferenceExtractor.Extract( - fileId, - record.Lang, - content, - symbols, - record.Path, - record.Lang == "csharp" ? csharpWorkspace.Symbols : null, - requestToken); + List references; + FileIssue? regexTimeoutIssue; + using (var regexTimeouts = BoundedRegex.CaptureTimeouts(record.Lang, "reference_extraction")) + { + references = ReferenceExtractor.Extract( + fileId, + record.Lang, + content, + symbols, + record.Path, + record.Lang == "csharp" ? csharpWorkspace.Symbols : null, + requestToken); + regexTimeoutIssue = IndexCommandRunner.BuildRegexTimeoutIssue(record.Path, regexTimeouts); + } postExtractionHooks.OnReferencesExtracted(fileContext, references); FileIssue? referenceCapIssue = null; if (references.Count > maxReferencesPerFile) @@ -5903,6 +5909,8 @@ static string FormatDiagnosticPath(string projectRoot, string path) // Keep MCP index parity with CLI index: persist file-level validation issues too. // MCPインデックスもCLIインデックスと同等に、ファイル検証issueを保存する。 IReadOnlyList issues = FileIndexer.ValidateContent(record.Path, rawBytes, content, record.Lang); + if (regexTimeoutIssue != null) + issues = IndexCommandRunner.AppendIssue(issues, regexTimeoutIssue); if (referenceCapIssue != null) issues = IndexCommandRunner.AppendIssue(issues, referenceCapIssue); writer.InsertIssues(fileId, issues); diff --git a/tests/CodeIndex.Tests/BoundedRegexTests.cs b/tests/CodeIndex.Tests/BoundedRegexTests.cs index 7ee6666f26..8c771336f3 100644 --- a/tests/CodeIndex.Tests/BoundedRegexTests.cs +++ b/tests/CodeIndex.Tests/BoundedRegexTests.cs @@ -44,6 +44,29 @@ public void InstanceMatch_Timeout_ReturnsEmpty() Assert.False(match.Success); } + [Fact] + public void CaptureTimeouts_RecordsHashedDiagnosticsForFileIssue() + { + const string pattern = "(a+)+$"; + var regex = new BoundedRegex(pattern, default, TimeSpan.FromMilliseconds(1)); + var input = new string('a', 10_000) + "!"; + + using var capture = BoundedRegex.CaptureTimeouts("csharp", "reference_extraction"); + var match = regex.Match(input); + var issue = IndexCommandRunner.BuildRegexTimeoutIssue("src/Pathological.cs", capture); + + Assert.False(match.Success); + Assert.True(capture.HasTimeouts); + Assert.NotNull(issue); + Assert.Equal("regex_timeout", issue.Kind); + Assert.Equal("src/Pathological.cs", issue.Path); + Assert.Contains("reference_extraction", issue.Message); + Assert.Contains(capture.Diagnostics[0].PatternHash, issue.Message); + Assert.Contains("len=6", issue.Message); + Assert.DoesNotContain(pattern, issue.Message); + Assert.DoesNotContain(input, issue.Message); + } + [Fact] public void InstanceMatches_Timeout_ReturnsEmpty() { From 2b30505aa1797d4b0b07a6969535afac253d85d6 Mon Sep 17 00:00:00 2001 From: Widthdom Date: Sat, 20 Jun 2026 12:15:54 +0900 Subject: [PATCH 03/18] Report scan checkpoint schema diagnostics (#3712) --- DEVELOPER_GUIDE.md | 2 + changelog.d/unreleased/3712.fixed.md | 18 ++++ .../Cli/IndexCommandRunner.FullScan.cs | 88 ++++++++++++++----- src/CodeIndex/Cli/IndexCommandRunner.cs | 4 + .../IndexCommandRunnerTests.cs | 40 ++++++++- 5 files changed, 131 insertions(+), 21 deletions(-) create mode 100644 changelog.d/unreleased/3712.fixed.md diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index e56cb76df7..a1ac5bfbd8 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -153,6 +153,8 @@ git status --short -- '**/packages.lock.json' | DTOs | `Models/FileRecord.cs`, `Models/ChunkRecord.cs`, `Models/SymbolRecord.cs`, `Models/ReferenceRecord.cs` | Records shared by indexing, storage, query, and MCP layers. | | Tests | `tests/CodeIndex.Tests/*Tests.cs`, `TestProjectHelper.cs`, `TestConsoleLock.cs` | Focused unit/integration coverage for chunking, extraction, DB reads/writes, CLI behavior, MCP behavior, git helpers, and shared test harness utilities. | +Full-scan resume checkpoints live at `.cdidx/scan-checkpoint.json`. The internal JSON schema is versioned (`Version` is currently `1`) and binds `GitHead` to a bounded `Directories` array; malformed, stale, future-version, oversized, or over-depth payloads are ignored with a bounded warning and a full scan. + Large command and extractor files have a tracked decomposition plan in [docs/large-file-decomposition-plan.md](docs/large-file-decomposition-plan.md). Use that plan when splitting `QueryCommandRunner`, `SymbolExtractor`, diff --git a/changelog.d/unreleased/3712.fixed.md b/changelog.d/unreleased/3712.fixed.md new file mode 100644 index 0000000000..9314188a52 --- /dev/null +++ b/changelog.d/unreleased/3712.fixed.md @@ -0,0 +1,18 @@ +--- +category: fixed +issues: + - 3712 +affected: + - src/CodeIndex/Cli/IndexCommandRunner.cs + - src/CodeIndex/Cli/IndexCommandRunner.FullScan.cs + - tests/CodeIndex.Tests/IndexCommandRunnerTests.cs + - DEVELOPER_GUIDE.md +--- + +## English + +- **Scan checkpoint recovery now reports schema/version diagnostics (#3712)** — malformed, stale, future-version, oversized, over-depth, or invalid-directory checkpoint payloads fall back to a full scan with a bounded `` warning instead of being ignored silently. + +## 日本語 + +- **scan checkpoint の復旧で schema/version 診断を出すようになりました (#3712)** — malformed、stale、future-version、oversized、over-depth、invalid-directory な checkpoint payload は、無言で無視される代わりに bounded な `` warning を出して full scan へフォールバックします。 diff --git a/src/CodeIndex/Cli/IndexCommandRunner.FullScan.cs b/src/CodeIndex/Cli/IndexCommandRunner.FullScan.cs index 37c7cb79d6..4b2007deae 100644 --- a/src/CodeIndex/Cli/IndexCommandRunner.FullScan.cs +++ b/src/CodeIndex/Cli/IndexCommandRunner.FullScan.cs @@ -317,66 +317,109 @@ ex is UnauthorizedAccessException internal const int MaxScanCheckpointDirectories = 4096; internal const int MaxScanCheckpointDirectoryLength = 4096; - internal static IReadOnlySet LoadScanCheckpoint(string path, string? currentHead) + internal static IReadOnlySet LoadScanCheckpoint(string path, string? currentHead) => + LoadScanCheckpointDetailed(path, currentHead).Directories; + + internal static ScanCheckpointLoadResult LoadScanCheckpointDetailed(string path, string? currentHead) { try { - if (string.IsNullOrWhiteSpace(currentHead) || !File.Exists(path)) - return EmptyScanCheckpointDirectories(); + if (!File.Exists(path)) + return EmptyScanCheckpointLoadResult(); + if (string.IsNullOrWhiteSpace(currentHead)) + return IgnoredScanCheckpoint(path, "current Git HEAD is unavailable"); var text = DataDirectorySecurity.ReadTextWithinLimit(path, MaxScanCheckpointBytes, FileShare.ReadWrite); if (text is null) - return EmptyScanCheckpointDirectories(); + return IgnoredScanCheckpoint(path, $"file exceeds the scan checkpoint size limit of {MaxScanCheckpointBytes:N0} bytes"); var checkpoint = JsonSerializer.Deserialize( text, new JsonSerializerOptions { MaxDepth = MaxScanCheckpointJsonDepth }); - if (checkpoint is not { Version: ScanCheckpointVersion } - || !string.Equals(checkpoint.GitHead, currentHead, StringComparison.Ordinal) - || !TryBuildScanCheckpointDirectories(checkpoint.Directories, out var directories)) - { - return EmptyScanCheckpointDirectories(); - } - - return directories; + if (checkpoint is null) + return IgnoredScanCheckpoint(path, "JSON root is null or not a scan checkpoint object"); + if (checkpoint.Version != ScanCheckpointVersion) + return IgnoredScanCheckpoint(path, FormatScanCheckpointVersionMismatch(checkpoint.Version)); + if (!string.Equals(checkpoint.GitHead, currentHead, StringComparison.Ordinal)) + return IgnoredScanCheckpoint(path, "checkpoint GitHead does not match current HEAD; checkpoint is stale"); + if (!TryBuildScanCheckpointDirectories(checkpoint.Directories, out var directories, out var directoryFailureReason)) + return IgnoredScanCheckpoint(path, directoryFailureReason); + + return new ScanCheckpointLoadResult(directories, WarningMessage: null); } - catch (JsonException) + catch (JsonException ex) { - return EmptyScanCheckpointDirectories(); + return IgnoredScanCheckpoint( + path, + $"malformed checkpoint JSON or depth exceeds {MaxScanCheckpointJsonDepth:N0} ({CommandErrorWriter.FormatSanitizedException(ex)})"); } - catch (IOException) + catch (IOException ex) { - return EmptyScanCheckpointDirectories(); + return IgnoredScanCheckpoint(path, $"read failed ({CommandErrorWriter.FormatSanitizedException(ex)})"); } - catch (UnauthorizedAccessException) + catch (UnauthorizedAccessException ex) { - return EmptyScanCheckpointDirectories(); + return IgnoredScanCheckpoint(path, $"read failed ({CommandErrorWriter.FormatSanitizedException(ex)})"); } } - private static bool TryBuildScanCheckpointDirectories(IReadOnlyList? rawDirectories, out IReadOnlySet directories) + private static string FormatScanCheckpointVersionMismatch(int version) => + version > ScanCheckpointVersion + ? $"future checkpoint version {version:N0} exceeds supported version {ScanCheckpointVersion:N0}" + : $"unsupported checkpoint version {version:N0}; supported version is {ScanCheckpointVersion:N0}"; + + private static ScanCheckpointLoadResult EmptyScanCheckpointLoadResult() => + new(EmptyScanCheckpointDirectories(), WarningMessage: null); + + private static ScanCheckpointLoadResult IgnoredScanCheckpoint(string path, string reason) => + new( + EmptyScanCheckpointDirectories(), + $"scan checkpoint ignored for {ConsoleUi.FormatBoundedValue(path)}: {reason}; continuing with a full scan."); + + private static bool TryBuildScanCheckpointDirectories( + IReadOnlyList? rawDirectories, + out IReadOnlySet directories, + out string failureReason) { directories = EmptyScanCheckpointDirectories(); + failureReason = string.Empty; if (rawDirectories is not { Count: > 0 }) + { + failureReason = "Directories must be a non-empty JSON array"; return false; + } if (rawDirectories.Count > MaxScanCheckpointDirectories) + { + failureReason = + $"Directories contains {rawDirectories.Count:N0} entries, exceeding the limit of {MaxScanCheckpointDirectories:N0}"; return false; + } var result = new HashSet(StringComparer.Ordinal); foreach (var directory in rawDirectories) { if (directory is null) + { + failureReason = "Directories contains a null entry"; return false; + } if (directory.Length == 0) continue; if (directory.Length > MaxScanCheckpointDirectoryLength) + { + failureReason = + $"Directories contains an entry longer than {MaxScanCheckpointDirectoryLength:N0} characters"; return false; + } result.Add(directory); } if (result.Count == 0) + { + failureReason = "Directories contains only empty entries"; return false; + } directories = result; return true; @@ -503,7 +546,8 @@ void ThrowIfDiscoveryCancelled() } var scanCheckpointPath = Path.Combine(projectRoot, ".cdidx", ScanCheckpointFileName); - var checkpointedDirectories = LoadScanCheckpoint(scanCheckpointPath, currentHeadForCheckpoint); + var checkpointLoadResult = LoadScanCheckpointDetailed(scanCheckpointPath, currentHeadForCheckpoint); + var checkpointedDirectories = checkpointLoadResult.Directories; WriteFullScanJsonLiveness(options, "scanning files..."); var scanHeartbeat = StartFullScanJsonPhaseHeartbeat(options, "scanning files"); FileIndexer.ScanFilesResult scanResult; @@ -536,11 +580,15 @@ void ThrowIfDiscoveryCancelled() var warningList = warningScanErrors .Select(error => new CliJsonMessage(error.Path, error.Message)) .ToList(); + if (checkpointLoadResult.WarningMessage != null) + warningList.Add(new CliJsonMessage("", checkpointLoadResult.WarningMessage)); if (!options.Json && !options.Quiet) { Console.WriteLine($" Found {ConsoleUi.Counted(files.Count, "file", format: "N0")}"); foreach (var error in scanResult.Errors) ConsoleUi.PrintWarning($"{error.Path}: {error.Message}"); + if (checkpointLoadResult.WarningMessage != null) + ConsoleUi.PrintWarning(checkpointLoadResult.WarningMessage); Console.WriteLine(); } diff --git a/src/CodeIndex/Cli/IndexCommandRunner.cs b/src/CodeIndex/Cli/IndexCommandRunner.cs index eabd6d457f..6721953310 100644 --- a/src/CodeIndex/Cli/IndexCommandRunner.cs +++ b/src/CodeIndex/Cli/IndexCommandRunner.cs @@ -39,6 +39,10 @@ private sealed record ScanCheckpoint( string? GitHead, IReadOnlyList Directories); + internal sealed record ScanCheckpointLoadResult( + IReadOnlySet Directories, + string? WarningMessage); + internal static Action? FullScanWritePhaseStartedForTesting { get; set; } internal static Action? FullScanExtractionSchedulingForTesting { get; set; } internal static Func? IndexExtractionStallTimeoutForTesting { get; set; } diff --git a/tests/CodeIndex.Tests/IndexCommandRunnerTests.cs b/tests/CodeIndex.Tests/IndexCommandRunnerTests.cs index b5affae01b..d5800beaf3 100644 --- a/tests/CodeIndex.Tests/IndexCommandRunnerTests.cs +++ b/tests/CodeIndex.Tests/IndexCommandRunnerTests.cs @@ -987,6 +987,39 @@ public void LoadScanCheckpoint_DirectoryPayloadOutsideBounds_ReturnsEmpty() } } + [Fact] + public void LoadScanCheckpointDetailed_InvalidPayloads_ReturnsReasonedWarning() + { + var projectRoot = CreateTempProject(); + var checkpointPath = Path.Combine(projectRoot, "scan-checkpoint.json"); + try + { + File.WriteAllText(checkpointPath, """{"Version":999,"GitHead":"abc123","Directories":["src"]}"""); + var futureVersion = IndexCommandRunner.LoadScanCheckpointDetailed(checkpointPath, "abc123"); + Assert.Empty(futureVersion.Directories); + Assert.Contains("future checkpoint version", futureVersion.WarningMessage); + + File.WriteAllText(checkpointPath, """{"Version":1,"GitHead":"old","Directories":["src"]}"""); + var stale = IndexCommandRunner.LoadScanCheckpointDetailed(checkpointPath, "abc123"); + Assert.Empty(stale.Directories); + Assert.Contains("checkpoint GitHead does not match current HEAD", stale.WarningMessage); + + File.WriteAllText(checkpointPath, """{"Version":1,"GitHead":"abc123","Directories":["src"]"""); + var malformed = IndexCommandRunner.LoadScanCheckpointDetailed(checkpointPath, "abc123"); + Assert.Empty(malformed.Directories); + Assert.Contains("malformed checkpoint JSON", malformed.WarningMessage); + + File.WriteAllText(checkpointPath, """{"Version":1,"GitHead":"abc123","Directories":[null]}"""); + var invalidDirectories = IndexCommandRunner.LoadScanCheckpointDetailed(checkpointPath, "abc123"); + Assert.Empty(invalidDirectories.Directories); + Assert.Contains("Directories contains a null entry", invalidDirectories.WarningMessage); + } + finally + { + DeleteDirectory(projectRoot); + } + } + [Fact] public void LoadScanCheckpoint_JsonDepthOutsideBounds_ReturnsEmpty() { @@ -7919,9 +7952,14 @@ public void Run_FullScan_IgnoresOversizedCheckpoint() padding.Append(' ', 1024).Append('\n'); File.WriteAllText(checkpointPath, checkpoint + padding); - var (exitCode, _) = RunAndCaptureJson([projectRoot, "--json"]); + var (exitCode, json) = RunAndCaptureJson([projectRoot, "--json"]); Assert.Equal(CommandExitCodes.Success, exitCode); + Assert.Contains( + json.GetProperty("warnings").EnumerateArray(), + warning => + warning.GetProperty("file").GetString() == "" + && warning.GetProperty("message").GetString()!.Contains("file exceeds the scan checkpoint size limit", StringComparison.Ordinal)); var indexedPaths = ReadIndexedPaths(Path.Combine(projectRoot, ".cdidx", "codeindex.db")); Assert.Contains("src/a.cs", indexedPaths); From 0b948de1a236b432042414dade3df54c13dc01b8 Mon Sep 17 00:00:00 2001 From: Widthdom Date: Sat, 20 Jun 2026 12:21:46 +0900 Subject: [PATCH 04/18] Cap dangling filesystem entry scans (#3763) --- changelog.d/unreleased/3763.fixed.md | 16 ++++++++++ src/CodeIndex/Indexer/Scanning/FileIndexer.cs | 21 +++++++++++- tests/CodeIndex.Tests/FileIndexerTests.cs | 32 +++++++++++++++++++ 3 files changed, 68 insertions(+), 1 deletion(-) create mode 100644 changelog.d/unreleased/3763.fixed.md diff --git a/changelog.d/unreleased/3763.fixed.md b/changelog.d/unreleased/3763.fixed.md new file mode 100644 index 0000000000..74868b9649 --- /dev/null +++ b/changelog.d/unreleased/3763.fixed.md @@ -0,0 +1,16 @@ +--- +category: fixed +issues: + - 3763 +affected: + - src/CodeIndex/Indexer/Scanning/FileIndexer.cs + - tests/CodeIndex.Tests/FileIndexerTests.cs +--- + +## English + +- **Dangling filesystem entry probes are now capped (#3763)** — per-directory dangling symlink checks stop after a bounded candidate count and emit a warning when additional diagnostics may be omitted, preserving normal scan correctness while avoiding unbounded probe work. + +## 日本語 + +- **dangling filesystem entry probe に上限を追加しました (#3763)** — ディレクトリ単位の dangling symlink 確認は bounded な candidate 数で停止し、追加診断が省略される可能性を warning として出すため、通常の scan correctness を保ちながら無制限な probe 作業を避けます。 diff --git a/src/CodeIndex/Indexer/Scanning/FileIndexer.cs b/src/CodeIndex/Indexer/Scanning/FileIndexer.cs index c6b66dedc6..964fa9041e 100644 --- a/src/CodeIndex/Indexer/Scanning/FileIndexer.cs +++ b/src/CodeIndex/Indexer/Scanning/FileIndexer.cs @@ -16,6 +16,7 @@ namespace CodeIndex.Indexer; /// public class FileIndexer { + internal const int MaxDanglingFileSystemEntryScanCandidates = 4096; internal static Func? FileSystemIgnoreCaseProbeForTesting { get; set; } internal static Func? ResolveDirectoryLinkTargetForTesting { get; set; } @@ -473,6 +474,7 @@ internal static bool IsDefaultExcludedFileName(string fileName) private readonly long _maxFileSizeBytes; private readonly FileContentLoader _contentLoader; private readonly SymlinkPolicy _symlinkPolicy; + private readonly int _maxDanglingFileSystemEntryScanCandidates; // Submodule working-tree paths declared in /.gitmodules, relative to // _projectRoot and slash-normalized. Used to override SkipDirs so that submodules // hosted under SkipDirs-named directories (e.g. vendor/foo) remain visible to the @@ -1018,7 +1020,8 @@ internal FileIndexer( long? maxFileSizeBytes, Func? directoryIgnoreCaseProbe, Func>? enumerateFiles = null, - SymlinkPolicy symlinkPolicy = SymlinkPolicy.None) + SymlinkPolicy symlinkPolicy = SymlinkPolicy.None, + int? maxDanglingFileSystemEntryScanCandidates = null) { _projectRoot = Path.GetFullPath(projectRoot); _ignoreRuleRoot = NormalizeIgnoreRuleRoot(ignoreRuleRoot); @@ -1030,6 +1033,9 @@ internal FileIndexer( _maxFileSizeBytes = ResolveMaxFileSizeBytes(maxFileSizeBytes); _contentLoader = new FileContentLoader(_maxFileSizeBytes); _symlinkPolicy = symlinkPolicy; + _maxDanglingFileSystemEntryScanCandidates = Math.Max( + 1, + maxDanglingFileSystemEntryScanCandidates ?? MaxDanglingFileSystemEntryScanCandidates); ExtractorPluginRegistry.LoadPatternConfigsForProjectRoot(_projectRoot); var pathComparer = _ignoreCase ? StringComparer.OrdinalIgnoreCase : StringComparer.Ordinal; (_submodulePaths, _submoduleAncestorPaths, _submoduleLoadWarnings) = LoadGitSubmodulePaths(_ignoreRuleRoot, _projectRoot, pathComparer); @@ -2528,9 +2534,22 @@ private void RecordDanglingFileSystemEntries( DirectoryScanState scanState, CancellationToken cancellationToken) { + var candidateLimit = _maxDanglingFileSystemEntryScanCandidates; + var candidateCount = 0; foreach (var enumeratedEntry in Directory.EnumerateFileSystemEntries(LongPath.EnsureWindowsPrefix(dir))) { cancellationToken.ThrowIfCancellationRequested(); + candidateCount++; + if (candidateCount > candidateLimit) + { + var relativeDir = ToRelativePath(dir); + scanState.Errors.Add(new ScanError( + relativeDir, + $"Dangling filesystem entry scan truncated after {candidateLimit:N0} candidate(s); additional dangling symlink diagnostics in this directory may be omitted.", + ScanIssueSeverity.Warning)); + return; + } + var entry = LongPath.RemoveWindowsPrefix(enumeratedEntry); if (!IsReparsePoint(entry) || ReparsePointTargetExists(entry)) continue; diff --git a/tests/CodeIndex.Tests/FileIndexerTests.cs b/tests/CodeIndex.Tests/FileIndexerTests.cs index cd21c01ca7..832f758725 100644 --- a/tests/CodeIndex.Tests/FileIndexerTests.cs +++ b/tests/CodeIndex.Tests/FileIndexerTests.cs @@ -48,6 +48,38 @@ public void ScanFilesDetailed_CancelledToken_ThrowsBeforeEnumeration() } } + [Fact] + public void ScanFilesDetailed_DanglingFileSystemEntryScanCapsCandidatesWithWarning() + { + var tempDir = Path.Combine(Path.GetTempPath(), $"cdidx-dangling-cap-{Guid.NewGuid():N}"); + Directory.CreateDirectory(tempDir); + try + { + for (var i = 0; i < 5; i++) + File.WriteAllText(Path.Combine(tempDir, $"file{i}.cs"), $"public class C{i} {{ }}\n"); + + var result = new FileIndexer( + tempDir, + ignoreCase: false, + ignoreRuleRoot: null, + maxFileSizeBytes: null, + directoryIgnoreCaseProbe: null, + maxDanglingFileSystemEntryScanCandidates: 3).ScanFilesDetailed(); + + Assert.Equal(5, result.Files.Count); + var warning = Assert.Single( + result.Errors, + error => error.Message.Contains("Dangling filesystem entry scan truncated", StringComparison.Ordinal)); + Assert.Equal(FileIndexer.ScanIssueSeverity.Warning, warning.Severity); + Assert.Contains("Dangling filesystem entry scan truncated after 3 candidate", warning.Message); + Assert.False(result.HadErrors); + } + finally + { + TestProjectHelper.DeleteDirectory(tempDir); + } + } + [Fact] public void Constructor_CaseProbeAvoidsRootProbeArtifacts_Issue3174() { From df4c3944a501200ee62ae665aaf5a241cf0fc15b Mon Sep 17 00:00:00 2001 From: Widthdom Date: Sat, 20 Jun 2026 12:25:44 +0900 Subject: [PATCH 05/18] Clarify project discovery budget diagnostics (#3762) --- changelog.d/unreleased/3762.fixed.md | 17 +++++ src/CodeIndex/Indexer/Scanning/FileIndexer.cs | 72 +++++++++++++++++-- tests/CodeIndex.Tests/FileIndexerTests.cs | 10 +++ .../IndexCommandRunnerTests.cs | 22 ++++++ 4 files changed, 114 insertions(+), 7 deletions(-) create mode 100644 changelog.d/unreleased/3762.fixed.md diff --git a/changelog.d/unreleased/3762.fixed.md b/changelog.d/unreleased/3762.fixed.md new file mode 100644 index 0000000000..8692995026 --- /dev/null +++ b/changelog.d/unreleased/3762.fixed.md @@ -0,0 +1,17 @@ +--- +category: fixed +issues: + - 3762 +affected: + - src/CodeIndex/Indexer/Scanning/FileIndexer.cs + - tests/CodeIndex.Tests/FileIndexerTests.cs + - tests/CodeIndex.Tests/IndexCommandRunnerTests.cs +--- + +## English + +- **Project discovery budget diagnostics now name the budget that fired (#3762)** — project marker fingerprint truncation records directory/marker-file budget reasons in warnings and fingerprint salt, and solution fallback discovery has explicit directory-budget coverage. + +## 日本語 + +- **project discovery budget 診断が発火した budget 名を示すようになりました (#3762)** — project marker fingerprint の truncation は directory / marker-file budget の理由を warning と fingerprint salt に記録し、solution fallback discovery には directory budget の明示テストを追加しました。 diff --git a/src/CodeIndex/Indexer/Scanning/FileIndexer.cs b/src/CodeIndex/Indexer/Scanning/FileIndexer.cs index 964fa9041e..3ce46a5ccb 100644 --- a/src/CodeIndex/Indexer/Scanning/FileIndexer.cs +++ b/src/CodeIndex/Indexer/Scanning/FileIndexer.cs @@ -94,6 +94,7 @@ private sealed class ProjectMarkerFingerprintTraversalState public int DirectoriesVisited { get; set; } public int MarkerFilesCollected { get; set; } public bool Truncated { get; set; } + public string TruncationReason { get; set; } = "unknown"; } private readonly record struct ProjectMarkerFingerprintDirectory(string Path, IgnoreRuleSet IgnoreRules, bool IsProjectRoot); @@ -1768,7 +1769,7 @@ private ProjectMarkerFingerprintResult GetProjectMarkerFingerprintResult( if (traversalState.Truncated) { projectMarkers.Add( - $"__cdidx_project_marker_fingerprint_truncated__:directories={traversalState.DirectoriesVisited};markers={traversalState.MarkerFilesCollected}"); + $"__cdidx_project_marker_fingerprint_truncated__:reason={traversalState.TruncationReason};directories={traversalState.DirectoriesVisited};markers={traversalState.MarkerFilesCollected}"); } projectMarkers.Sort(StringComparer.Ordinal); @@ -1891,7 +1892,11 @@ private void CollectProjectMarkerFiles( if (traversalState.DirectoriesVisited >= maxDirectories) { - traversalState.Truncated = true; + TruncateProjectMarkerTraversal( + traversalState, + errors, + current.Path, + $"directory budget {maxDirectories:N0} exhausted after visiting {traversalState.DirectoriesVisited:N0} directories"); return; } @@ -1903,7 +1908,11 @@ private void CollectProjectMarkerFiles( var loadResult = LoadIgnoreRulesForDirectory(currentDirectory, current.IgnoreRules, errors, ref fullyScanned); if (!loadResult.IgnoreRulesAvailable) { - traversalState.Truncated = true; + TruncateProjectMarkerTraversal( + traversalState, + errors, + currentDirectory, + "ignore-rule loading failed"); return; } @@ -1916,7 +1925,11 @@ private void CollectProjectMarkerFiles( if (traversalState.MarkerFilesCollected >= maxMarkerFiles) { - traversalState.Truncated = true; + TruncateProjectMarkerTraversal( + traversalState, + errors, + currentDirectory, + $"marker file budget {maxMarkerFiles:N0} exhausted after collecting {traversalState.MarkerFilesCollected:N0} marker files"); return; } @@ -1940,7 +1953,11 @@ private void CollectProjectMarkerFiles( if (traversalState.DirectoriesVisited + pendingDirectories.Count >= maxDirectories) { - traversalState.Truncated = true; + TruncateProjectMarkerTraversal( + traversalState, + errors, + currentDirectory, + $"directory budget {maxDirectories:N0} would be exceeded while queuing subdirectories after visiting {traversalState.DirectoriesVisited:N0} directories"); return; } @@ -1950,16 +1967,39 @@ private void CollectProjectMarkerFiles( catch (UnauthorizedAccessException) { AddProjectMarkerTraversalWarning(errors, currentDirectory, nameof(UnauthorizedAccessException)); - traversalState.Truncated = true; + MarkProjectMarkerTraversalTruncated( + traversalState, + $"traversal failed with {nameof(UnauthorizedAccessException)}"); } catch (IOException) { AddProjectMarkerTraversalWarning(errors, currentDirectory, nameof(IOException)); - traversalState.Truncated = true; + MarkProjectMarkerTraversalTruncated( + traversalState, + $"traversal failed with {nameof(IOException)}"); } } } + private void TruncateProjectMarkerTraversal( + ProjectMarkerFingerprintTraversalState traversalState, + List errors, + string directory, + string reason) + { + MarkProjectMarkerTraversalTruncated(traversalState, reason); + AddProjectMarkerBudgetWarning(errors, directory, reason); + } + + private static void MarkProjectMarkerTraversalTruncated( + ProjectMarkerFingerprintTraversalState traversalState, + string reason) + { + if (!traversalState.Truncated) + traversalState.TruncationReason = reason; + traversalState.Truncated = true; + } + private static IEnumerable EnumerateProjectMarkerDirectories(string dir) => EnumerateProjectMarkerDirectoriesForTesting is { } enumerate ? enumerate(dir) @@ -1983,6 +2023,24 @@ private void AddProjectMarkerTraversalWarning(List errors, string dir ScanIssueSeverity.Warning)); } + private void AddProjectMarkerBudgetWarning(List errors, string directory, string reason) + { + if (errors.Count(static error => error.Message.StartsWith("Project marker discovery truncated", StringComparison.Ordinal)) + >= MaxProjectMarkerTraversalWarnings) + { + return; + } + + var relativePath = NormalizeIgnorePath(Path.GetRelativePath(_projectRoot, directory)); + if (string.IsNullOrEmpty(relativePath)) + relativePath = "."; + + errors.Add(new ScanError( + relativePath, + $"Project marker discovery truncated because {reason}.", + ScanIssueSeverity.Warning)); + } + private static IReadOnlyList? GetProjectMarkerPatterns(string? lang) => lang switch { "csharp" => ["*.csproj"], diff --git a/tests/CodeIndex.Tests/FileIndexerTests.cs b/tests/CodeIndex.Tests/FileIndexerTests.cs index 832f758725..4edcbaf970 100644 --- a/tests/CodeIndex.Tests/FileIndexerTests.cs +++ b/tests/CodeIndex.Tests/FileIndexerTests.cs @@ -861,6 +861,11 @@ public void GetProjectMarkerFingerprint_DirectoryCapReportsIncompleteTraversal() Assert.False(result.IsComplete); Assert.False(string.IsNullOrWhiteSpace(result.Fingerprint)); + var warning = Assert.Single( + result.Warnings, + error => error.Message.Contains("directory budget 1", StringComparison.Ordinal)); + Assert.Equal(FileIndexer.ScanIssueSeverity.Warning, warning.Severity); + Assert.Contains("Project marker discovery truncated", warning.Message, StringComparison.Ordinal); } finally { @@ -940,10 +945,15 @@ public void GetProjectMarkerFingerprint_FileCapTruncatesMarkerCollection() var fullFingerprint = indexer.GetProjectMarkerFingerprint("msbuild"); var cappedFingerprint = indexer.GetProjectMarkerFingerprintForTesting("msbuild", maxDirectories: 100, maxMarkerFiles: 1); + var cappedResult = indexer.GetProjectMarkerFingerprintResultForTesting("msbuild", maxDirectories: 100, maxMarkerFiles: 1); Assert.False(string.IsNullOrWhiteSpace(fullFingerprint)); Assert.False(string.IsNullOrWhiteSpace(cappedFingerprint)); Assert.NotEqual(fullFingerprint, cappedFingerprint); + Assert.False(cappedResult.IsComplete); + Assert.Contains( + cappedResult.Warnings, + error => error.Message.Contains("marker file budget 1", StringComparison.Ordinal)); } finally { diff --git a/tests/CodeIndex.Tests/IndexCommandRunnerTests.cs b/tests/CodeIndex.Tests/IndexCommandRunnerTests.cs index d5800beaf3..f228c3f8af 100644 --- a/tests/CodeIndex.Tests/IndexCommandRunnerTests.cs +++ b/tests/CodeIndex.Tests/IndexCommandRunnerTests.cs @@ -2467,6 +2467,28 @@ public void ResolveProjects_RejectsFallbackDiscoveryFileTraversalLimit_Issue3213 } } + [Fact] + public void ResolveProjects_RejectsFallbackDiscoveryDirectoryTraversalLimit() + { + var projectRoot = TestProjectHelper.CreateTempProject("cdidx_solution_fallback_directory_limit"); + try + { + Directory.CreateDirectory(Path.Combine(projectRoot, "A")); + Directory.CreateDirectory(Path.Combine(projectRoot, "B")); + var limits = SolutionProjectResolverLimits.Default with { MaxFallbackDiscoveryDirectories = 1 }; + + var ex = Assert.Throws( + () => SolutionProjectResolver.ResolveProjects(projectRoot, solutionPath: null, limits)); + + Assert.Contains("fallback project discovery traversed more than 1 directories", ex.Message); + Assert.Contains("pass --solution ", ex.Message); + } + finally + { + TestProjectHelper.DeleteDirectory(projectRoot); + } + } + [Fact] public void ResolveProjectFiles_HonorsGitRootIgnoreRulesForNestedWorkspace_Issue2862() { From 88363502342d1608bac8d229b23ef0235bdf0958 Mon Sep 17 00:00:00 2001 From: Widthdom Date: Sat, 20 Jun 2026 12:37:45 +0900 Subject: [PATCH 06/18] Improve file scan edge diagnostics (#3835) --- USER_GUIDE.md | 29 ++-- changelog.d/unreleased/3835.fixed.md | 18 +++ .../Indexer/Scanning/FileContentLoader.cs | 27 +++- src/CodeIndex/Indexer/Scanning/FileIndexer.cs | 126 ++++++++++++++++-- tests/CodeIndex.Tests/FileIndexerTests.cs | 41 +++++- 5 files changed, 210 insertions(+), 31 deletions(-) create mode 100644 changelog.d/unreleased/3835.fixed.md diff --git a/USER_GUIDE.md b/USER_GUIDE.md index ffe44182cc..6e47132a37 100644 --- a/USER_GUIDE.md +++ b/USER_GUIDE.md @@ -454,9 +454,10 @@ cdidx validate --json --limit 50 --path legacy/ ``` `validate` reports indexed files that are likely to produce misleading snippets -or symbol names: U+FFFD replacement characters, UTF-16 BOMs, null bytes, mixed or -CR-only line endings, likely non-UTF-8 content, Git LFS pointer placeholders, and -malformed or truncated Dockerfile JSON-form instruction payloads. +or symbol names: U+FFFD replacement characters, UTF-16 BOM and BOM-less +heuristic detections, null bytes, mixed or CR-only line endings, likely non-UTF-8 +content, Git LFS pointer placeholders, and malformed or truncated Dockerfile +JSON-form instruction payloads. For `replacement_char`, JSON and MCP responses include `origin` (`source_literal` or `decode_replacement`) and `severity` so agents can distinguish intentional U+FFFD literals from likely encoding damage. @@ -464,9 +465,9 @@ Use `--severity warning` to hide informational source literals and focus on findings that indicate likely encoding damage. Use `--json=array` when a pipeline expects a bare issue array instead of the default `{ "count": ..., "issues": [...] }` object. -LFS pointers are recorded as `lfs_pointer_skipped` and their placeholder body is -not indexed; run `git lfs pull` and then `cdidx index .` to index the real file -content. +LFS pointers are recorded as `lfs_pointer_skipped`; their placeholder body is +not indexed, and their checksum stays tied to the pointer identity until you run +`git lfs pull` and then `cdidx index .` to index the real file content. ### Find potentially unused symbols @@ -2305,7 +2306,7 @@ The MCP `tools/list` response includes an `examples` array for every registered | `unused_symbols` | Find symbols defined but never referenced, with confidence buckets for dead-code triage | | `symbol_hotspots` | Find high-impact hotspots. `groupBy` supports `symbol`, `file`, and `statement`; SQL scopes default to statement grouping while non-SQL scopes default to symbol grouping. | | `batch_query` | Execute multiple queries in a single call (MCP only, max 10). The response includes a top-level `metadata` object with `submitted`, `executed`, `errors`, `total_elapsed_ms`, `success_count`, and `failure_count`; every entry in `results` carries `request_index`, optional client `slot_id`, `ok`, `elapsed_ms`, `summary`, and compact `args_summary` fields so callers can correlate partial failures and slow inner queries without relying on positional guesses. | -| `validate` | Report encoding and file-content issues (U+FFFD with origin/severity, BOM, null bytes, mixed/CR-only line endings, UTF-16 BOM detection, likely non-UTF8 encodings, Dockerfile JSON-form diagnostics) | +| `validate` | Report encoding and file-content issues (U+FFFD with origin/severity, BOM, null bytes, mixed/CR-only line endings, UTF-16 BOM/heuristic detection, likely non-UTF8 encodings, Git LFS pointer placeholders, Dockerfile JSON-form diagnostics) | | `languages` | List all supported languages, file extensions, and capabilities | | `ping` | Lightweight connection check | | `index` | Index or re-index a project directory | @@ -2958,17 +2959,19 @@ cdidx validate --json --limit 50 --path legacy/ ``` `validate` は、snippet や symbol name を誤らせやすい indexed file を報告します。 -対象は U+FFFD replacement character、UTF-16 BOM、null byte、mixed / CR-only line -ending、likely non-UTF-8 content、Git LFS pointer placeholder、Dockerfile の -JSON-form instruction payload の parse / truncation 診断などです。 +対象は U+FFFD replacement character、UTF-16 BOM と BOM なし heuristic 検出、 +null byte、mixed / CR-only line ending、likely non-UTF-8 content、Git LFS pointer +placeholder、Dockerfile の JSON-form instruction payload の parse / truncation +診断などです。 `replacement_char` の JSON / MCP response には `origin` (`source_literal` / `decode_replacement`) と `severity` が入り、意図的な U+FFFD literal と エンコーディング破損の可能性を agent が区別できます。`--severity warning` を使うと、informational な source literal を隠して、エンコーディング破損の 可能性がある finding に集中できます。pipeline が既定の `{ "count": ..., "issues": [...] }` object ではなく bare issue array を期待する場合は `--json=array` を使えます。LFS pointer -は `lfs_pointer_skipped` として記録され、placeholder 本文は index されません。 -実体を index するには `git lfs pull` の後に `cdidx index .` を再実行してください。 +は `lfs_pointer_skipped` として記録され、placeholder 本文は index されず、checksum は +実体を取得するまで pointer identity に紐づきます。実体を index するには `git lfs pull` +の後に `cdidx index .` を再実行してください。 ### 未使用の可能性がある symbols を探す @@ -4804,7 +4807,7 @@ OpenAI Codex CLI (`codex.json` または `~/.codex/config.json`): | `unused_symbols` | 定義されているが参照されていないシンボルを bucket 付きで検索（デッドコード検出向け） | | `symbol_hotspots` | 影響の大きい hotspot を検索。`groupBy` は `symbol` / `file` / `statement` を指定でき、SQL scope は statement grouping、非 SQL scope は symbol grouping が既定。 | | `batch_query` | 複数クエリを1回で実行（MCP専用、最大10件）。レスポンスにはトップレベル `metadata`（`submitted` / `executed` / `errors` / `total_elapsed_ms` / `success_count` / `failure_count`）と各 `results` エントリの `request_index`、任意の client `slot_id`、`ok`、`elapsed_ms`、`summary`、`args_summary` が含まれ、位置だけに依存せず部分失敗や遅い内部クエリを把握できます。 | -| `validate` | エンコーディングと file-content の問題（origin/severity 付き U+FFFD、BOM、null バイト、改行混在 / CR-only 行末、UTF-16 BOM 検出、UTF-8 以外と推定されるエンコーディング、Dockerfile JSON-form 診断）を報告 | +| `validate` | エンコーディングと file-content の問題（origin/severity 付き U+FFFD、BOM、null バイト、改行混在 / CR-only 行末、UTF-16 BOM / heuristic 検出、UTF-8 以外と推定されるエンコーディング、Git LFS pointer placeholder、Dockerfile JSON-form 診断）を報告 | | `languages` | 対応言語一覧を拡張子・機能付きで表示。`--indexed-only` と `--capability graph|references|symbols|missing-graph|missing-references|missing-symbols|search-only` で現在の DB、機能別、または capability gap 別に絞り込み可能 | | `ping` | 軽量な接続確認 | | `index` | プロジェクトのインデックス作成・更新 | diff --git a/changelog.d/unreleased/3835.fixed.md b/changelog.d/unreleased/3835.fixed.md new file mode 100644 index 0000000000..66e40df29d --- /dev/null +++ b/changelog.d/unreleased/3835.fixed.md @@ -0,0 +1,18 @@ +--- +category: fixed +issues: + - 3835 +affected: + - src/CodeIndex/Indexer/Scanning/FileContentLoader.cs + - src/CodeIndex/Indexer/Scanning/FileIndexer.cs + - tests/CodeIndex.Tests/FileIndexerTests.cs + - USER_GUIDE.md +--- + +## English + +- **File scanning diagnostics now cover more edge cases (#3835)** — Git LFS pointer files keep pointer-identity checksums, BOM-less UTF-16 heuristic decoding reports the chosen encoding, NUL-byte binary skips include the byte offset, out-of-policy symlink warnings no longer expose absolute external targets, and extensionless `env -S` shebangs resolve their interpreter. + +## 日本語 + +- **ファイル走査診断がより多くの端ケースを扱うようになりました (#3835)** — Git LFS pointer ファイルは pointer identity に基づく checksum を保持し、BOM なし UTF-16 heuristic decode は選択した encoding を報告し、NUL バイトによる binary skip は byte offset を含め、ポリシー外 symlink の warning は外部絶対 target を露出せず、拡張子なし `env -S` shebang は interpreter を解決するようになりました。 diff --git a/src/CodeIndex/Indexer/Scanning/FileContentLoader.cs b/src/CodeIndex/Indexer/Scanning/FileContentLoader.cs index 1f435793a6..dd19793032 100644 --- a/src/CodeIndex/Indexer/Scanning/FileContentLoader.cs +++ b/src/CodeIndex/Indexer/Scanning/FileContentLoader.cs @@ -20,8 +20,10 @@ internal LoadedFileContent Load( var (content, warning) = DecodeIndexableContent(bytes, relativePath); content = NormalizeLineEndings(content); content = StripLineLeadingInvisibles(content); - if (IsGitLfsPointer(bytes)) + var isGitLfsPointer = IsGitLfsPointer(bytes); + if (isGitLfsPointer) content = string.Empty; + var checksumBytes = isGitLfsPointer ? bytes : Encoding.UTF8.GetBytes(content); return new LoadedFileContent( content, @@ -29,7 +31,7 @@ internal LoadedFileContent Load( sizeBytes, modifiedUtc, FileIndexer.CountPhysicalLines(content), - ComputeChecksum(Encoding.UTF8.GetBytes(content)), + ComputeChecksum(checksumBytes), warning); } @@ -105,14 +107,17 @@ internal LoadedFileContent Load( { var isUtf16Encoded = TryDetectUtf16Encoding(bytes, allowHeuristic: true, out var utf16BigEndian, out var hasUtf16Bom); - if (!isUtf16Encoded && ContainsIndexBlockingNullByte(bytes)) - throw new FileIndexer.BinaryFileSkippedException($"{relativePath}: binary file skipped because it contains NULL bytes"); + if (!isUtf16Encoded && TryFindIndexBlockingNullByte(bytes, out var nullByteOffset)) + throw new FileIndexer.BinaryFileSkippedException($"{relativePath}: binary file skipped because it contains NULL byte at byte offset {nullByteOffset}"); if (isUtf16Encoded) { var content = new UnicodeEncoding(utf16BigEndian, byteOrderMark: hasUtf16Bom, throwOnInvalidBytes: false) .GetString(bytes); - return (content, null); + var warning = hasUtf16Bom + ? null + : $"{relativePath}: decoded as {(utf16BigEndian ? "UTF-16BE" : "UTF-16LE")} without BOM by NUL-byte heuristic"; + return (content, warning); } try @@ -274,7 +279,17 @@ private static bool IsGitLfsSizeLine(string line) internal static bool ContainsIndexBlockingNullByte(byte[] rawBytes) { - return !TryDetectUtf16Encoding(rawBytes, allowHeuristic: true, out _, out _) && rawBytes.Any(b => b == 0); + return TryFindIndexBlockingNullByte(rawBytes, out _); + } + + internal static bool TryFindIndexBlockingNullByte(byte[] rawBytes, out int offset) + { + offset = -1; + if (TryDetectUtf16Encoding(rawBytes, allowHeuristic: true, out _, out _)) + return false; + + offset = Array.IndexOf(rawBytes, (byte)0); + return offset >= 0; } internal static bool TryDetectUtf16Encoding( diff --git a/src/CodeIndex/Indexer/Scanning/FileIndexer.cs b/src/CodeIndex/Indexer/Scanning/FileIndexer.cs index 3ce46a5ccb..0824bd8523 100644 --- a/src/CodeIndex/Indexer/Scanning/FileIndexer.cs +++ b/src/CodeIndex/Indexer/Scanning/FileIndexer.cs @@ -1533,10 +1533,22 @@ private bool ShouldSkipDirectoryLink(string subDir, List errors, Hash if (_symlinkPolicy == SymlinkPolicy.Internal && IsPathEqualOrParent(_projectRoot, targetPath)) return false; - errors.Add(new ScanError(relative, $"Skipped symlinked directory outside the active symlink policy: {targetPath}", ScanIssueSeverity.Warning)); + errors.Add(new ScanError( + relative, + $"Skipped symlinked directory outside the active symlink policy: target {FormatSymlinkPolicyTargetForDiagnostic(targetPath)}", + ScanIssueSeverity.Warning)); return true; } + private string FormatSymlinkPolicyTargetForDiagnostic(string targetPath) + { + if (!IsPathEqualOrParent(_projectRoot, targetPath)) + return ""; + + var relative = NormalizePathSeparators(Path.GetRelativePath(_projectRoot, targetPath)); + return relative == "." ? "" : relative; + } + internal bool ShouldSkipDirectoryTraversal(string directory) => ShouldSkipDirectoryLink( directory, @@ -1568,7 +1580,7 @@ private static string GetDirectoryTraversalIdentity(string directory) { } - return directory; + return $"unresolved-reparse:{Path.GetFullPath(directory)}"; } internal static FileProbeStatus GetFileIndexability(string filePath) @@ -3689,8 +3701,13 @@ public static List ValidateContent(string relativePath, byte[] rawByt // 不正サロゲートペアに備え content 側 U+FFFD 走査は継続する。Closes #1540. var isUtf16 = TryDetectUtf16Encoding(rawBytes, allowHeuristic: true, out var utf16BigEndian, out var hasUtf16Bom); - if (isUtf16 && hasUtf16Bom) - AddUtf16BomIssue(issues, relativePath, utf16BigEndian); + if (isUtf16) + { + if (hasUtf16Bom) + AddUtf16BomIssue(issues, relativePath, utf16BigEndian); + else + AddUtf16HeuristicIssue(issues, relativePath, utf16BigEndian); + } if (TryGetConflictMarkerLine(content, out var conflictMarkerLine)) { @@ -3960,6 +3977,19 @@ private static void AddUtf16BomIssue(List issues, string relativePath }); } + private static void AddUtf16HeuristicIssue(List issues, string relativePath, bool utf16BigEndian) + { + issues.Add(new FileIssue + { + Path = relativePath, + Kind = "utf16_heuristic", + Line = 1, + Message = utf16BigEndian + ? "BOM-less UTF-16 BE detected by NUL-byte heuristic (decoded as UTF-16)" + : "BOM-less UTF-16 LE detected by NUL-byte heuristic (decoded as UTF-16)", + }); + } + private static void AddReplacementCharacterIssues( List issues, string relativePath, @@ -4424,9 +4454,8 @@ private static LanguageDetectionResult TryDetectLanguageFromShebang( if (string.IsNullOrWhiteSpace(commandLine)) return new LanguageDetectionResult(FileProbeStatus.Unsupported, null); - var tokens = commandLine - .Split([' ', '\t'], StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries); - if (tokens.Length == 0) + var tokens = TokenizeShebangCommandLine(commandLine); + if (tokens.Count == 0) return new LanguageDetectionResult(FileProbeStatus.Unsupported, null); var interpreter = ResolveShebangInterpreter(tokens); @@ -4524,15 +4553,77 @@ private static int FindShebangLineEnd(ReadOnlySpan bytes, ShebangEncoding _ => new UTF8Encoding(false, throwOnInvalidBytes: true).GetString(bytes), }; + private static IReadOnlyList TokenizeShebangCommandLine(string commandLine) + { + var tokens = new List(); + var token = new StringBuilder(commandLine.Length); + char? quote = null; + var escaped = false; + + foreach (var ch in commandLine) + { + if (escaped) + { + token.Append(ch); + escaped = false; + continue; + } + + if (ch == '\\') + { + escaped = true; + continue; + } + + if (quote is { } activeQuote) + { + if (ch == activeQuote) + quote = null; + else + token.Append(ch); + continue; + } + + if (ch is '\'' or '"') + { + quote = ch; + continue; + } + + if (ch is ' ' or '\t') + { + if (token.Length > 0) + { + tokens.Add(token.ToString()); + token.Clear(); + } + continue; + } + + token.Append(ch); + } + + if (escaped) + token.Append('\\'); + if (token.Length > 0) + tokens.Add(token.ToString()); + + return tokens; + } + private static string? ResolveShebangInterpreter(IReadOnlyList tokens) { - var interpreter = Path.GetFileName(tokens[0]).ToLowerInvariant(); + var interpreter = NormalizeShebangInterpreterToken(tokens[0]); + if (interpreter == null) + return null; if (interpreter is not "env") return interpreter; for (var i = 1; i < tokens.Count; i++) { var token = tokens[i]; + if (token == "--") + continue; if (token.StartsWith("-", StringComparison.Ordinal)) continue; @@ -4541,12 +4632,29 @@ private static int FindShebangLineEnd(ReadOnlySpan bytes, ShebangEncoding if (token.Contains('=')) continue; - return Path.GetFileName(token).ToLowerInvariant(); + return NormalizeShebangInterpreterToken(token); } return null; } + private static string? NormalizeShebangInterpreterToken(string token) + { + var candidate = token; + if (token.IndexOfAny([' ', '\t']) >= 0) + { + var nestedTokens = TokenizeShebangCommandLine(token); + if (nestedTokens.Count == 0) + return null; + candidate = nestedTokens[0]; + } + + if (string.IsNullOrWhiteSpace(candidate)) + return null; + + return Path.GetFileName(candidate).ToLowerInvariant(); + } + private static string? MapShebangInterpreterToLanguage(string interpreter) => interpreter switch { "bash" or "sh" or "zsh" or "fish" or "dash" or "ksh" or "ash" => "shell", diff --git a/tests/CodeIndex.Tests/FileIndexerTests.cs b/tests/CodeIndex.Tests/FileIndexerTests.cs index 4edcbaf970..939fabe099 100644 --- a/tests/CodeIndex.Tests/FileIndexerTests.cs +++ b/tests/CodeIndex.Tests/FileIndexerTests.cs @@ -1062,6 +1062,8 @@ public void DetectLanguage_UnknownExtensions_ReturnsNull(string filename) [InlineData("worker", "#!/usr/bin/python3\nprint('hi')\n", "python")] [InlineData("bundle", "#!/usr/bin/env ruby\nputs 'hi'\n", "ruby")] [InlineData("cli", "#!/usr/bin/env node\nconsole.log('hi')\n", "javascript")] + [InlineData("envsplit", "#!/usr/bin/env -S python -O\nprint('hi')\n", "python")] + [InlineData("envquoted", "#!/usr/bin/env -S \"python -O\"\nprint('hi')\n", "python")] [InlineData("script", "#!/usr/bin/env pwsh\nWrite-Host hi\n", "powershell")] public void DetectLanguage_ExtensionlessShebangScripts_ReturnCorrectLang(string fileName, string content, string expected) { @@ -4121,7 +4123,9 @@ public void ScanFiles_FollowSymlinksInternal_SkipsOutOfTreeDirectorySymlink() result.Errors, error => error.Path == "external" && error.Severity == FileIndexer.ScanIssueSeverity.Warning - && error.Message.Contains("symlinked directory", StringComparison.OrdinalIgnoreCase)); + && error.Message.Contains("symlinked directory", StringComparison.OrdinalIgnoreCase) + && error.Message.Contains("", StringComparison.Ordinal) + && !error.Message.Contains(externalDir, StringComparison.Ordinal)); } finally { @@ -4882,7 +4886,9 @@ public void BuildRecord_Utf16LeWithoutBomFile_DecodedAsUtf16() var indexer = new FileIndexer(tempDir); var (_, content, _, warning) = indexer.BuildRecordWithRawBytes(filePath); - Assert.Null(warning); + Assert.NotNull(warning); + Assert.Contains("UTF-16LE without BOM", warning, StringComparison.Ordinal); + Assert.Contains("NUL-byte heuristic", warning, StringComparison.Ordinal); Assert.Contains("namespace Utf16LeNoBom;", content); Assert.False(FileIndexer.ContainsIndexBlockingNullByte(System.Text.Encoding.Unicode.GetBytes(payload))); } @@ -4910,7 +4916,9 @@ public void BuildRecord_Utf16BeWithoutBomFile_DecodedAsUtf16() var indexer = new FileIndexer(tempDir); var (_, content, _, warning) = indexer.BuildRecordWithRawBytes(filePath); - Assert.Null(warning); + Assert.NotNull(warning); + Assert.Contains("UTF-16BE without BOM", warning, StringComparison.Ordinal); + Assert.Contains("NUL-byte heuristic", warning, StringComparison.Ordinal); Assert.Contains("namespace Utf16BeNoBom;", content); Assert.False(FileIndexer.ContainsIndexBlockingNullByte(System.Text.Encoding.BigEndianUnicode.GetBytes(payload))); } @@ -4955,11 +4963,36 @@ public void ValidateContent_Utf16WithoutBom_DoesNotEmitNullByteIssue() var issues = FileIndexer.ValidateContent("utf16le-nobom.cs", rawBytes, payload); + var issue = Assert.Single(issues.Where(i => i.Kind == "utf16_heuristic")); + Assert.Equal(1, issue.Line); + Assert.Contains("UTF-16 LE", issue.Message, StringComparison.Ordinal); + Assert.Contains("NUL-byte heuristic", issue.Message, StringComparison.Ordinal); Assert.DoesNotContain(issues, i => i.Kind == "utf16_bom"); Assert.DoesNotContain(issues, i => i.Kind == "null_byte"); Assert.DoesNotContain(issues, i => i.Kind == "mixed_line_endings"); } + [Fact] + public void BuildRecord_NonUtf16NullByte_ThrowsOffsetDiagnostic() + { + var tempDir = Path.Combine(Path.GetTempPath(), $"codeindex_test_{Guid.NewGuid():N}"); + try + { + Directory.CreateDirectory(tempDir); + var filePath = Path.Combine(tempDir, "binary.cs"); + File.WriteAllBytes(filePath, [(byte)'c', (byte)'l', (byte)'a', (byte)'s', (byte)'s', (byte)' ', 0x00]); + + var indexer = new FileIndexer(tempDir); + var ex = Assert.Throws(() => indexer.BuildRecordWithRawBytes(filePath)); + + Assert.Contains("NULL byte at byte offset 6", ex.Message, StringComparison.Ordinal); + } + finally + { + Directory.Delete(tempDir, true); + } + } + [Fact] public void ValidateContent_HighFffdRatio_EmitsAggregateNonUtf8Likely() { @@ -5180,6 +5213,8 @@ size 12345 Assert.Equal(string.Empty, content); Assert.Equal(0, record.Lines); + Assert.Equal(FileIndexer.ComputeChecksum(rawBytes), record.Checksum); + Assert.NotEqual(FileIndexer.ComputeChecksum(System.Text.Encoding.UTF8.GetBytes(string.Empty)), record.Checksum); var issue = Assert.Single(issues, i => i.Kind == "lfs_pointer_skipped"); Assert.Equal("asset.cs", issue.Path); Assert.Equal(1, issue.Line); From 552118eb8425dd166f48f633c90e968bf522c821 Mon Sep 17 00:00:00 2001 From: Widthdom Date: Sat, 20 Jun 2026 12:43:10 +0900 Subject: [PATCH 07/18] Reduce chunk line materialization (#3785) --- changelog.d/unreleased/3785.fixed.md | 16 ++++++++ .../Indexer/Scanning/ChunkSplitter.cs | 40 ++++++++++++++----- tests/CodeIndex.Tests/ChunkSplitterTests.cs | 18 +++++++++ 3 files changed, 65 insertions(+), 9 deletions(-) create mode 100644 changelog.d/unreleased/3785.fixed.md diff --git a/changelog.d/unreleased/3785.fixed.md b/changelog.d/unreleased/3785.fixed.md new file mode 100644 index 0000000000..9830513c68 --- /dev/null +++ b/changelog.d/unreleased/3785.fixed.md @@ -0,0 +1,16 @@ +--- +category: fixed +issues: + - 3785 +affected: + - src/CodeIndex/Indexer/Scanning/ChunkSplitter.cs + - tests/CodeIndex.Tests/ChunkSplitterTests.cs +--- + +## English + +- **Chunk preparation now avoids full line-string materialization (#3785)** — chunk splitting tracks line start offsets and slices only the persisted chunk bodies, reducing duplicate allocations for large but valid files while preserving line ranges and trailing-newline behavior. + +## 日本語 + +- **chunk preparation が全行 string 配列の materialization を避けるようになりました (#3785)** — chunk splitting は行開始 offset を追跡し、永続化する chunk 本文だけを切り出すため、大きくても有効なファイルでの重複 allocation を減らしつつ、行範囲と末尾改行の挙動を維持します。 diff --git a/src/CodeIndex/Indexer/Scanning/ChunkSplitter.cs b/src/CodeIndex/Indexer/Scanning/ChunkSplitter.cs index 86d94427e4..bf61d09574 100644 --- a/src/CodeIndex/Indexer/Scanning/ChunkSplitter.cs +++ b/src/CodeIndex/Indexer/Scanning/ChunkSplitter.cs @@ -96,19 +96,29 @@ public static List Split(long fileId, string content) // 既存の issues 経路で観測できる。Closes #1542. if (HasOversizeLine(content)) return []; - // Remove trailing newline to avoid phantom empty line / 末尾改行による空行を除去 - var lines = content.EndsWith('\n') - ? content[..^1].Split('\n') - : content.Split('\n'); + // Track line start offsets instead of materializing every line string. Large + // source files can still be valid and under the file-size cap, and chunking + // should only allocate the persisted chunk bodies rather than a duplicate + // full-file string[] plus per-chunk slice arrays. A trailing newline is not + // treated as an extra phantom line, matching the previous Split('\n') path. + // 各行の string 配列を作らず、行開始 offset だけを保持する。大きな source file + // でも file-size 上限内なら有効なため、chunking では永続化する chunk 本文以外に + // ファイル全体分の string[] や chunk ごとの slice 配列を作らない。末尾改行を + // 余分な空行として扱わない点は従来の Split('\n') 経路と同じ。 + var lineStarts = GetLineStartOffsets(content); var chunks = new List(); int step = ChunkSize - Overlap; int chunkIndex = 0; + var effectiveContentLength = content.EndsWith('\n') ? content.Length - 1 : content.Length; - for (int start = 0; start < lines.Length; start += step) + for (int start = 0; start < lineStarts.Count; start += step) { - int end = Math.Min(start + ChunkSize, lines.Length); - var chunkLines = lines[start..end]; - var chunkContent = string.Join('\n', chunkLines); + int end = Math.Min(start + ChunkSize, lineStarts.Count); + var startOffset = lineStarts[start]; + var endOffset = end < lineStarts.Count + ? lineStarts[end] - 1 + : effectiveContentLength; + var chunkContent = content.Substring(startOffset, endOffset - startOffset); chunks.Add(new ChunkRecord { @@ -122,10 +132,22 @@ public static List Split(long fileId, string content) chunkIndex++; // Stop if we've reached the end / 末尾に到達したら終了 - if (end >= lines.Length) + if (end >= lineStarts.Count) break; } return chunks; } + + private static List GetLineStartOffsets(string content) + { + var lineStarts = new List { 0 }; + for (var i = 0; i < content.Length; i++) + { + if (content[i] == '\n' && i + 1 < content.Length) + lineStarts.Add(i + 1); + } + + return lineStarts; + } } diff --git a/tests/CodeIndex.Tests/ChunkSplitterTests.cs b/tests/CodeIndex.Tests/ChunkSplitterTests.cs index c77a2ede5e..16065e8d34 100644 --- a/tests/CodeIndex.Tests/ChunkSplitterTests.cs +++ b/tests/CodeIndex.Tests/ChunkSplitterTests.cs @@ -49,6 +49,24 @@ public void Split_LargeFile_CreatesOverlappingChunks() Assert.Equal(71, chunks[1].StartLine); } + [Fact] + public void Split_ManyValidLines_KeepsAllocationBounded() + { + var content = string.Join('\n', Enumerable.Range(1, 5000).Select(i => $"line {i:D4} value")); + _ = ChunkSplitter.Split(1, "warmup\nline\n"); + + var before = GC.GetAllocatedBytesForCurrentThread(); + var chunks = ChunkSplitter.Split(1, content); + var allocated = GC.GetAllocatedBytesForCurrentThread() - before; + + Assert.Equal(72, chunks.Count); + Assert.Equal(1, chunks[0].StartLine); + Assert.Equal(80, chunks[0].EndLine); + Assert.Equal(4971, chunks[^1].StartLine); + Assert.Equal(5000, chunks[^1].EndLine); + Assert.True(allocated < 450_000, $"Chunk splitting allocated {allocated} bytes."); + } + [Fact] public void Split_EmptyFile_ReturnsNoChunks() { From c20ada41dbad95ebc8002d17b13288ad49507ea3 Mon Sep 17 00:00:00 2001 From: Widthdom Date: Sat, 20 Jun 2026 13:03:34 +0900 Subject: [PATCH 08/18] Make generated-code suppression configurable (#3720) --- USER_GUIDE.md | 10 +- changelog.d/unreleased/3720.fixed.md | 23 +++ src/CodeIndex/Cli/CdidxConfigFile.cs | 10 +- .../Cli/IndexCommandRunner.DryRun.cs | 9 +- .../Cli/IndexCommandRunner.FullScan.cs | 56 ++++++++ src/CodeIndex/Cli/IndexCommandRunner.Parse.cs | 34 ++++- .../Cli/IndexCommandRunner.Update.cs | 30 ++++ src/CodeIndex/Cli/IndexCommandRunner.cs | 15 +- src/CodeIndex/Indexer/Scanning/FileIndexer.cs | 38 ++++- .../Scanning/GeneratedCodePatternMatcher.cs | 135 ++++++++++++++++++ src/CodeIndex/Mcp/McpToolHandlers.cs | 31 +++- tests/CodeIndex.Tests/CdidxConfigFileTests.cs | 4 +- tests/CodeIndex.Tests/FileIndexerTests.cs | 34 +++++ .../IndexCommandRunnerTests.cs | 97 +++++++++++++ 14 files changed, 510 insertions(+), 16 deletions(-) create mode 100644 changelog.d/unreleased/3720.fixed.md create mode 100644 src/CodeIndex/Indexer/Scanning/GeneratedCodePatternMatcher.cs diff --git a/USER_GUIDE.md b/USER_GUIDE.md index 6e47132a37..6a183637a8 100644 --- a/USER_GUIDE.md +++ b/USER_GUIDE.md @@ -1707,7 +1707,8 @@ Supported schema (top-level keys are snake_case; nested indexing kind keys keep "suggestion_max_count": 5000, // → CDIDX_SUGGESTION_MAX_COUNT "indexing": { "includeKinds": ["class"], // → CDIDX_INDEX_INCLUDE_SYMBOL_KINDS - "excludeKinds": ["test_method"] // → CDIDX_INDEX_EXCLUDE_SYMBOL_KINDS + "excludeKinds": ["test_method"], // → CDIDX_INDEX_EXCLUDE_SYMBOL_KINDS + "generatedCodePatterns": ["src/generated/**", "*.client.ts"] // → CDIDX_INDEX_GENERATED_CODE_PATTERNS }, "mcp": { "tools": { @@ -1723,7 +1724,7 @@ Supported schema (top-level keys are snake_case; nested indexing kind keys keep } ``` -JSON5-style line comments (`//`) and trailing commas are accepted so the file stays human-editable. The optional `$schema` key is ignored at runtime; it is honored only so editors that recognize JSON Schema references can offer completion. Setting `disable_persistent_log` to `false` is a no-op (absence already means "logging enabled") — only `true` exports `CDIDX_DISABLE_PERSISTENT_LOG=1`. Config-sourced `metrics_path` and `global_tool_log_dir` values are resolved from the config workspace root and must stay inside that workspace; use the CLI flag or a real environment variable when you intentionally need an outside destination. `stale_after` uses the same compact duration format as `status --check --stale-after`: `30m`, `2h`, or `7d`, up to `30d`. `suggestion_dedup_threshold` sets the MCP suggestion fuzzy-deduplication cutoff as a number from `0` to `1`; the built-in default is `0.85`, and `cdidx mcp --suggestion-dedup-threshold <0..1>` overrides it for one MCP session. `suggestion_max_age_days` and `suggestion_max_count` bound the live `.cdidx/suggestions-*.json` store; pruned records are appended to `.cdidx/suggestions-*.archive.jsonl`, whose active file is capped at 8 MiB and rotates up to three retained generations (`.1` through `.3`). Defaults are 365 days and 5000 records, and config-file values may not exceed 3650 days or 100000 records. Matching environment variables above those caps fall back to the defaults. `mcp.rate_limit.bucket_idle_seconds` sets the same idle bucket TTL as `CDIDX_MCP_RATE_LIMIT_BUCKET_IDLE_SECONDS`; invalid runtime values fall back to the default with a warning. String-array settings such as `indexing.includeKinds`, `indexing.excludeKinds`, `mcp.tools.allow`, and `mcp.tools.deny` are capped at 128 entries and 256 characters per item before they are joined into environment variables. `indexing.includeKinds` and `indexing.excludeKinds` set the default symbol-kind filter for `cdidx index`; CLI flags `--include-symbol-kind [,]` and `--exclude-symbol-kind [,]` override those env-backed defaults for a single run. +JSON5-style line comments (`//`) and trailing commas are accepted so the file stays human-editable. The optional `$schema` key is ignored at runtime; it is honored only so editors that recognize JSON Schema references can offer completion. Setting `disable_persistent_log` to `false` is a no-op (absence already means "logging enabled") — only `true` exports `CDIDX_DISABLE_PERSISTENT_LOG=1`. Config-sourced `metrics_path` and `global_tool_log_dir` values are resolved from the config workspace root and must stay inside that workspace; use the CLI flag or a real environment variable when you intentionally need an outside destination. `stale_after` uses the same compact duration format as `status --check --stale-after`: `30m`, `2h`, or `7d`, up to `30d`. `suggestion_dedup_threshold` sets the MCP suggestion fuzzy-deduplication cutoff as a number from `0` to `1`; the built-in default is `0.85`, and `cdidx mcp --suggestion-dedup-threshold <0..1>` overrides it for one MCP session. `suggestion_max_age_days` and `suggestion_max_count` bound the live `.cdidx/suggestions-*.json` store; pruned records are appended to `.cdidx/suggestions-*.archive.jsonl`, whose active file is capped at 8 MiB and rotates up to three retained generations (`.1` through `.3`). Defaults are 365 days and 5000 records, and config-file values may not exceed 3650 days or 100000 records. Matching environment variables above those caps fall back to the defaults. `mcp.rate_limit.bucket_idle_seconds` sets the same idle bucket TTL as `CDIDX_MCP_RATE_LIMIT_BUCKET_IDLE_SECONDS`; invalid runtime values fall back to the default with a warning. String-array settings such as `indexing.includeKinds`, `indexing.excludeKinds`, `indexing.generatedCodePatterns`, `mcp.tools.allow`, and `mcp.tools.deny` are capped at 128 entries and 256 characters per item before they are joined into environment variables. `indexing.generatedCodePatterns` marks matching relative paths or basenames as generated code. Matching files remain indexed for text search and chunk retrieval, but symbol/reference extraction is skipped and `file_issues` records `generated_code_extraction_skipped`; patterns with a slash match slash-normalized relative paths, patterns without a slash match basenames, and `*`, `?`, and `**` are supported. `indexing.includeKinds` and `indexing.excludeKinds` set the default symbol-kind filter for `cdidx index`; CLI flags `--include-symbol-kind [,]` and `--exclude-symbol-kind [,]` override those env-backed defaults for a single run. ## How it works @@ -4223,7 +4224,8 @@ MCP のレスポンスサイズ上限は、環境変数 override で guard が "suggestion_max_count": 5000, // → CDIDX_SUGGESTION_MAX_COUNT "indexing": { "includeKinds": ["class"], // → CDIDX_INDEX_INCLUDE_SYMBOL_KINDS - "excludeKinds": ["test_method"] // → CDIDX_INDEX_EXCLUDE_SYMBOL_KINDS + "excludeKinds": ["test_method"], // → CDIDX_INDEX_EXCLUDE_SYMBOL_KINDS + "generatedCodePatterns": ["src/generated/**", "*.client.ts"] // → CDIDX_INDEX_GENERATED_CODE_PATTERNS }, "mcp": { "tools": { @@ -4239,7 +4241,7 @@ MCP のレスポンスサイズ上限は、環境変数 override で guard が } ``` -人手で編集しやすいよう JSON5 形式の行コメント（`//`）と末尾カンマを許容します。任意の `$schema` キーはランタイムでは無視され、JSON Schema 参照をサポートするエディタが補完を提供するためだけに認識されます。`disable_persistent_log` を `false` に設定しても何も起きません（不在のままで "ログ有効" が既定）— `true` の場合のみ `CDIDX_DISABLE_PERSISTENT_LOG=1` を export します。config 由来の `metrics_path` と `global_tool_log_dir` は設定ファイルの workspace root から解決され、その workspace 内に収まる必要があります。意図的に外部の出力先を使う場合は CLI フラグまたは実際の環境変数を使ってください。`stale_after` は `status --check --stale-after` と同じ compact duration 形式（`30m` / `2h` / `7d`、最大 `30d`）です。`suggestion_dedup_threshold` は MCP suggestion の fuzzy deduplication しきい値を `0` から `1` の数値で設定します。組み込み既定値は `0.85` で、`cdidx mcp --suggestion-dedup-threshold <0..1>` は 1 回の MCP session だけこの値を上書きします。`suggestion_max_age_days` と `suggestion_max_count` は live の `.cdidx/suggestions-*.json` store の上限を設定し、prune された record は `.cdidx/suggestions-*.archive.jsonl` に追記されます。この active archive は 8 MiB で上限管理され、最大 3 世代（`.1` から `.3`）までローテーションされます。既定値は 365 日と 5000 件で、config-file 値は 3650 日または 100000 件を超えられません。同じ環境変数がこの上限を超えた場合は既定値へ戻ります。`mcp.rate_limit.bucket_idle_seconds` は `CDIDX_MCP_RATE_LIMIT_BUCKET_IDLE_SECONDS` と同じ idle bucket TTL を設定します。不正な runtime 値は警告付きで既定値へ戻ります。`indexing.includeKinds`、`indexing.excludeKinds`、`mcp.tools.allow`、`mcp.tools.deny` のような string array 設定は、環境変数へ join される前に 128 件、1 要素 256 文字までに制限されます。`indexing.includeKinds` と `indexing.excludeKinds` は `cdidx index` の symbol-kind filter 既定値を設定し、CLI フラグ `--include-symbol-kind [,]` / `--exclude-symbol-kind [,]` はその env 経由の既定値を 1 回の実行だけ上書きします。 +人手で編集しやすいよう JSON5 形式の行コメント（`//`）と末尾カンマを許容します。任意の `$schema` キーはランタイムでは無視され、JSON Schema 参照をサポートするエディタが補完を提供するためだけに認識されます。`disable_persistent_log` を `false` に設定しても何も起きません（不在のままで "ログ有効" が既定）— `true` の場合のみ `CDIDX_DISABLE_PERSISTENT_LOG=1` を export します。config 由来の `metrics_path` と `global_tool_log_dir` は設定ファイルの workspace root から解決され、その workspace 内に収まる必要があります。意図的に外部の出力先を使う場合は CLI フラグまたは実際の環境変数を使ってください。`stale_after` は `status --check --stale-after` と同じ compact duration 形式（`30m` / `2h` / `7d`、最大 `30d`）です。`suggestion_dedup_threshold` は MCP suggestion の fuzzy deduplication しきい値を `0` から `1` の数値で設定します。組み込み既定値は `0.85` で、`cdidx mcp --suggestion-dedup-threshold <0..1>` は 1 回の MCP session だけこの値を上書きします。`suggestion_max_age_days` と `suggestion_max_count` は live の `.cdidx/suggestions-*.json` store の上限を設定し、prune された record は `.cdidx/suggestions-*.archive.jsonl` に追記されます。この active archive は 8 MiB で上限管理され、最大 3 世代（`.1` から `.3`）までローテーションされます。既定値は 365 日と 5000 件で、config-file 値は 3650 日または 100000 件を超えられません。同じ環境変数がこの上限を超えた場合は既定値へ戻ります。`mcp.rate_limit.bucket_idle_seconds` は `CDIDX_MCP_RATE_LIMIT_BUCKET_IDLE_SECONDS` と同じ idle bucket TTL を設定します。不正な runtime 値は警告付きで既定値へ戻ります。`indexing.includeKinds`、`indexing.excludeKinds`、`indexing.generatedCodePatterns`、`mcp.tools.allow`、`mcp.tools.deny` のような string array 設定は、環境変数へ join される前に 128 件、1 要素 256 文字までに制限されます。`indexing.generatedCodePatterns` は一致した相対パスまたはベース名を generated code として扱います。一致したファイルは全文検索と chunk 取得用には引き続き index されますが、symbol/reference 抽出はスキップされ、`file_issues` に `generated_code_extraction_skipped` が記録されます。スラッシュを含む pattern は slash-normalized relative path、スラッシュを含まない pattern は basename に一致し、`*`、`?`、`**` を利用できます。`indexing.includeKinds` と `indexing.excludeKinds` は `cdidx index` の symbol-kind filter 既定値を設定し、CLI フラグ `--include-symbol-kind [,]` / `--exclude-symbol-kind [,]` はその env 経由の既定値を 1 回の実行だけ上書きします。 ## 動作の仕組み diff --git a/changelog.d/unreleased/3720.fixed.md b/changelog.d/unreleased/3720.fixed.md new file mode 100644 index 0000000000..ff7a42d219 --- /dev/null +++ b/changelog.d/unreleased/3720.fixed.md @@ -0,0 +1,23 @@ +--- +category: fixed +issues: + - 3720 +affected: + - src/CodeIndex/Cli/CdidxConfigFile.cs + - src/CodeIndex/Cli/IndexCommandRunner.cs + - src/CodeIndex/Indexer/Scanning/FileIndexer.cs + - src/CodeIndex/Indexer/Scanning/GeneratedCodePatternMatcher.cs + - src/CodeIndex/Mcp/McpToolHandlers.cs + - tests/CodeIndex.Tests/CdidxConfigFileTests.cs + - tests/CodeIndex.Tests/FileIndexerTests.cs + - tests/CodeIndex.Tests/IndexCommandRunnerTests.cs + - USER_GUIDE.md +--- + +## English + +- **Generated-code indexing suppression is now configurable (#3720)** — project config can mark filename or path patterns as generated code, preserving text chunks while skipping symbol/reference extraction and recording `generated_code_extraction_skipped` in `file_issues`. + +## 日本語 + +- **generated-code indexing suppression を設定可能にしました (#3720)** — project config で filename/path pattern を generated code として指定でき、text chunk は保持しながら symbol/reference 抽出をスキップし、`file_issues` に `generated_code_extraction_skipped` を記録します。 diff --git a/src/CodeIndex/Cli/CdidxConfigFile.cs b/src/CodeIndex/Cli/CdidxConfigFile.cs index 3b30c4d49a..6f9f2a2881 100644 --- a/src/CodeIndex/Cli/CdidxConfigFile.cs +++ b/src/CodeIndex/Cli/CdidxConfigFile.cs @@ -56,7 +56,7 @@ internal static class CdidxConfigFile "mcp", }; - private static readonly IReadOnlyList KnownIndexingKeys = new[] { "includeKinds", "excludeKinds" }; + private static readonly IReadOnlyList KnownIndexingKeys = new[] { "includeKinds", "excludeKinds", "generatedCodePatterns" }; private static readonly IReadOnlyList KnownSearchKeys = new[] { "limit", "snippet_lines", "max_line_width" }; private static readonly IReadOnlyList KnownOutputKeys = new[] { "format", "locale" }; private static readonly IReadOnlyList KnownGraphKeys = new[] { "max_hops" }; @@ -277,6 +277,14 @@ private static void AddIndexingEnvironmentSettings(JsonElement root, string path else if (value!.Length > 0) pending.Add((IndexCommandRunner.ExcludeSymbolKindsEnvironmentVariable, string.Join(",", value))); } + + if (indexing.TryGetProperty("generatedCodePatterns", out var generatedCodePatterns)) + { + if (!TryReadStringArray(generatedCodePatterns, "indexing.generatedCodePatterns", path, out var value, out var err)) + errors.Add(err!); + else if (value!.Length > 0) + pending.Add((IndexCommandRunner.GeneratedCodePatternsEnvironmentVariable, string.Join(",", value))); + } } private static void AddSearchEnvironmentSettings(JsonElement root, string path, List<(string EnvName, string Value)> pending, List errors) diff --git a/src/CodeIndex/Cli/IndexCommandRunner.DryRun.cs b/src/CodeIndex/Cli/IndexCommandRunner.DryRun.cs index 0b5e62881d..609e394f25 100644 --- a/src/CodeIndex/Cli/IndexCommandRunner.DryRun.cs +++ b/src/CodeIndex/Cli/IndexCommandRunner.DryRun.cs @@ -20,7 +20,14 @@ private static int RunDryRun( CancellationToken cancellationToken) { var projectPath = options.ProjectPath!; - var dryIndexer = new FileIndexer(projectPath, ignoreCase, ignoreRuleRoot, options.MaxFileSizeBytes, directoryIgnoreCaseProbe: null, symlinkPolicy: options.SymlinkPolicy); + var dryIndexer = new FileIndexer( + projectPath, + ignoreCase, + ignoreRuleRoot, + options.MaxFileSizeBytes, + directoryIgnoreCaseProbe: null, + symlinkPolicy: options.SymlinkPolicy, + generatedCodePatterns: options.GeneratedCodePatterns); IReadOnlyList dryCandidates; IReadOnlyList dryDeleteCandidates; bool authoritativeFullScan; diff --git a/src/CodeIndex/Cli/IndexCommandRunner.FullScan.cs b/src/CodeIndex/Cli/IndexCommandRunner.FullScan.cs index 4b2007deae..9adbbd56c3 100644 --- a/src/CodeIndex/Cli/IndexCommandRunner.FullScan.cs +++ b/src/CodeIndex/Cli/IndexCommandRunner.FullScan.cs @@ -81,6 +81,9 @@ private static bool ExistingFileViolatesExtractionCaps(DbWriter writer, long fil || writer.CountReferencesForFile(fileId) > maxReferencesPerFile || writer.HasIssueForFile(fileId, "reference_count_exceeded"); + internal static bool ExistingFileGeneratedSuppressionMismatch(DbWriter writer, long fileId, FileIssue? generatedSuppressionIssue) + => writer.HasIssueForFile(fileId, FileIndexer.GeneratedCodeExtractionSkippedIssueKind) != (generatedSuppressionIssue != null); + internal static IReadOnlyList AppendIssue(IReadOnlyList issues, FileIssue issue) { if (issues.Count == 0) @@ -91,6 +94,13 @@ internal static IReadOnlyList AppendIssue(IReadOnlyList is return combined; } + internal static IReadOnlyList AppendIssueIfMissing(IReadOnlyList issues, FileIssue issue) + { + if (issues.Any(existing => string.Equals(existing.Kind, issue.Kind, StringComparison.Ordinal))) + return issues; + return AppendIssue(issues, issue); + } + internal static string FormatIndexPhasePath(string path, string phase) => $"{path} ({phase})"; @@ -1105,10 +1115,24 @@ void StopJsonHeartbeat() IReadOnlyList? symbols = null; IReadOnlyList? references = null; IReadOnlyList? issues = null; + var generatedSuppressionIssue = indexer.BuildGeneratedCodeExtractionSkippedIssue(record.Path); if (parallelizeExtraction) { activeJsonExtractionPhases[workerIndex] = FormatIndexPhasePath(record.Path, "chunking"); chunks = ChunkSplitter.Split(0, content); + if (generatedSuppressionIssue != null) + { + symbols = []; + references = []; + activeJsonExtractionPhases[workerIndex] = FormatIndexPhasePath(record.Path, "validating"); + issues = AppendIssueIfMissing( + FileIndexer.ValidateContent(record.Path, rawBytes, content, record.Lang), + generatedSuppressionIssue); + extractionResults.Add( + FullScanFileWorkItem.Success(filePath, record, content, rawBytes, warning, chunks, symbols, references, issues), + extractionCancellationToken); + continue; + } activeJsonExtractionPhases[workerIndex] = FormatIndexPhasePath(record.Path, "symbols"); symbols = ExtractSymbolsWithStallTimeout( 0, @@ -1310,6 +1334,11 @@ void StopJsonHeartbeat() { existingId = null; } + if (existingId != null + && ExistingFileGeneratedSuppressionMismatch(writer, existingId.Value, indexer.BuildGeneratedCodeExtractionSkippedIssue(record.Path))) + { + existingId = null; + } if (existingId != null) { writer.PurgeStaleFilesSharingChecksum(projectRoot, record.Path, record.Checksum); @@ -1345,6 +1374,33 @@ void StopJsonHeartbeat() var chunks = item.Chunks == null ? ChunkSplitter.Split(fileId, item.Content!) : ReassignChunkFileIds(item.Chunks, fileId); + var generatedSuppressionIssue = indexer.BuildGeneratedCodeExtractionSkippedIssue(record.Path); + if (generatedSuppressionIssue != null) + { + writer.InsertChunks(chunks); + writer.InsertSymbols([]); + writer.InsertReferences([]); + var generatedIssues = AppendIssueIfMissing( + item.Issues ?? FileIndexer.ValidateContent(record.Path, item.RawBytes!, item.Content!, record.Lang), + generatedSuppressionIssue); + writer.InsertIssues(fileId, generatedIssues); + if (options.Verbose) + WriteIndexVerboseStatus($" [OK ] {record.Path} ({chunks.Count} chunks, generated-code extraction skipped)"); + currentJsonIndexFile = FormatIndexPhasePath(record.Path, "committing"); + WriteProjectRootOnce(); + txn.Commit(); + + processed++; + if (!options.Json && !options.Quiet) + { + PauseIndexSpinnerForConsoleWrite(); + ConsoleUi.PrintProgress(processed, files.Count); + ResumeIndexSpinnerAfterConsoleWrite(); + } + ReportJsonIndexProgressIfNeeded(); + currentJsonIndexFile = null; + continue; + } currentJsonIndexFile = FormatIndexPhasePath(record.Path, "symbols"); var symbols = item.Symbols == null ? ExtractSymbolsWithStallTimeout( diff --git a/src/CodeIndex/Cli/IndexCommandRunner.Parse.cs b/src/CodeIndex/Cli/IndexCommandRunner.Parse.cs index 4329dbb10f..664156ee38 100644 --- a/src/CodeIndex/Cli/IndexCommandRunner.Parse.cs +++ b/src/CodeIndex/Cli/IndexCommandRunner.Parse.cs @@ -68,6 +68,8 @@ public static IndexCommandOptions ParseArgs(string[] args) string? symbolKindFilterError = null; var includeSymbolKindsSpecifiedOnCli = false; var excludeSymbolKindsSpecifiedOnCli = false; + string? generatedCodePatternError = null; + var generatedCodePatterns = ReadGeneratedCodePatternsFromEnvironment(ref generatedCodePatternError); AddSymbolKindFilterValues( IncludeSymbolKindsEnvironmentVariable, @@ -320,7 +322,7 @@ public static IndexCommandOptions ParseArgs(string[] args) ProjectFilters = projectFilters, SolutionPath = solutionPath, ProjectFilterError = projectFilterError, - ParseError = parseError, + ParseError = parseError ?? generatedCodePatternError, EasterEgg = easterEgg, DryRun = dryRun, Force = force, @@ -339,9 +341,39 @@ public static IndexCommandOptions ParseArgs(string[] args) Parallelism = parallelism, SymlinkPolicy = symlinkPolicy, SymbolKindFilter = SymbolKindFilter.Create(includeSymbolKinds, excludeSymbolKinds, symbolKindFilterError), + GeneratedCodePatterns = generatedCodePatterns, }; } + internal static IReadOnlyList ReadGeneratedCodePatternsFromEnvironment() + { + string? parseError = null; + return ReadGeneratedCodePatternsFromEnvironment(ref parseError); + } + + private static IReadOnlyList ReadGeneratedCodePatternsFromEnvironment(ref string? parseError) + { + var value = CdidxEnvironment.GetEnvironmentVariable(GeneratedCodePatternsEnvironmentVariable); + if (string.IsNullOrWhiteSpace(value)) + return []; + if (!ValidateCsvBounds(GeneratedCodePatternsEnvironmentVariable, value, MaxGeneratedCodePatternCsvLength, MaxGeneratedCodePatternCount, ref parseError)) + return []; + + var patterns = new List(); + foreach (var raw in value.Split(',', StringSplitOptions.TrimEntries)) + { + if (raw.Length == 0) + { + parseError ??= $"{GeneratedCodePatternsEnvironmentVariable} contains an empty generated-code pattern"; + continue; + } + + patterns.Add(raw); + } + + return patterns; + } + private static FileIndexer.SymlinkPolicy ParseSymlinkPolicy(string value, FileIndexer.SymlinkPolicy fallback, ref string? parseError) { switch (value.Trim().ToLowerInvariant()) diff --git a/src/CodeIndex/Cli/IndexCommandRunner.Update.cs b/src/CodeIndex/Cli/IndexCommandRunner.Update.cs index b4aac83395..5c3591c498 100644 --- a/src/CodeIndex/Cli/IndexCommandRunner.Update.cs +++ b/src/CodeIndex/Cli/IndexCommandRunner.Update.cs @@ -584,6 +584,11 @@ void ThrowIfUpdateCancelled() { statMatchedId = null; } + if (statMatchedId != null + && ExistingFileGeneratedSuppressionMismatch(writer, statMatchedId.Value, indexer.BuildGeneratedCodeExtractionSkippedIssue(dbPath))) + { + statMatchedId = null; + } if (statMatchedId != null) { skipped++; @@ -623,6 +628,11 @@ void ThrowIfUpdateCancelled() { existingId = null; } + if (existingId != null + && ExistingFileGeneratedSuppressionMismatch(writer, existingId.Value, indexer.BuildGeneratedCodeExtractionSkippedIssue(record.Path))) + { + existingId = null; + } if (existingId != null) { using var purgeTxn = writer.BeginTransaction(); @@ -661,6 +671,26 @@ void ThrowIfUpdateCancelled() var fileId = writer.UpsertFile(record); currentUpdatePath = FormatIndexPhasePath(relPath, "chunking"); var chunks = ChunkSplitter.Split(fileId, content); + var generatedSuppressionIssue = indexer.BuildGeneratedCodeExtractionSkippedIssue(record.Path); + if (generatedSuppressionIssue != null) + { + writer.InsertChunks(chunks); + writer.InsertSymbols([]); + writer.InsertReferences([]); + currentUpdatePath = FormatIndexPhasePath(relPath, "validating"); + var generatedIssues = AppendIssueIfMissing( + FileIndexer.ValidateContent(record.Path, rawBytes, content, record.Lang), + generatedSuppressionIssue); + writer.InsertIssues(fileId, generatedIssues); + currentUpdatePath = FormatIndexPhasePath(relPath, "committing"); + writer.ClearBatchInProgress(); + txn.Commit(); + fileBatchMarked = false; + updated++; + ftsMutated = true; + WriteUpdateVerboseStatus($" [OK ] {relPath} ({chunks.Count} chunks, generated-code extraction skipped)"); + continue; + } currentUpdatePath = FormatIndexPhasePath(relPath, "symbols"); var symbols = ExtractSymbolsWithStallTimeout( fileId, diff --git a/src/CodeIndex/Cli/IndexCommandRunner.cs b/src/CodeIndex/Cli/IndexCommandRunner.cs index 6721953310..fd614a31a6 100644 --- a/src/CodeIndex/Cli/IndexCommandRunner.cs +++ b/src/CodeIndex/Cli/IndexCommandRunner.cs @@ -19,12 +19,15 @@ public static partial class IndexCommandRunner { internal const string IncludeSymbolKindsEnvironmentVariable = "CDIDX_INDEX_INCLUDE_SYMBOL_KINDS"; internal const string ExcludeSymbolKindsEnvironmentVariable = "CDIDX_INDEX_EXCLUDE_SYMBOL_KINDS"; + internal const string GeneratedCodePatternsEnvironmentVariable = "CDIDX_INDEX_GENERATED_CODE_PATTERNS"; internal const int DefaultMaxSymbolsPerFile = 5000; internal const int MaxSymbolsPerFileLimit = 50_000; internal const int DefaultMaxReferencesPerFile = 100_000; internal const int MaxReferencesPerFileLimit = 1_000_000; internal const int MaxCommitRefCount = 64; internal const int MaxCommitRefLength = 256; + internal const int MaxGeneratedCodePatternCsvLength = 32_768; + internal const int MaxGeneratedCodePatternCount = 128; internal const int MaxGitExcludeBytes = 256 * 1024; internal const string SymbolKindFilterMetaKey = "index_symbol_kind_filter"; private const int MaxIndexRunDiagnosticLength = 512; @@ -281,7 +284,14 @@ internal static int Run(string[] indexArgs, JsonSerializerOptions jsonOptions, C AddToGitExclude(options.ProjectPath, dbPath, indexRunDiagnostics); var writer = new DbWriter(db); - var indexer = new FileIndexer(options.ProjectPath, ignoreCase, ignoreRuleRoot, options.MaxFileSizeBytes, directoryIgnoreCaseProbe: null, symlinkPolicy: options.SymlinkPolicy); + var indexer = new FileIndexer( + options.ProjectPath, + ignoreCase, + ignoreRuleRoot, + options.MaxFileSizeBytes, + directoryIgnoreCaseProbe: null, + symlinkPolicy: options.SymlinkPolicy, + generatedCodePatterns: options.GeneratedCodePatterns); var currentHotspotFamilyMarkerFingerprints = GetHotspotFamilyMarkerFingerprints(indexer, indexCancellation.Token); var projectRoot = Path.GetFullPath(options.ProjectPath!); @@ -1094,6 +1104,8 @@ private static CSharpStaticInterfaceWorkspaceSymbols BuildCSharpStaticInterfaceW var (record, content, _, _) = indexer.BuildRecordWithRawBytes(absolutePath, cancellationToken); if (record.Lang != "csharp") continue; + if (indexer.BuildGeneratedCodeExtractionSkippedIssue(record.Path) != null) + continue; if (!MayContainCSharpStaticInterfaceContract(content)) continue; @@ -1603,6 +1615,7 @@ public sealed class IndexCommandOptions public bool MemoryTrace { get; init; } public FileIndexer.SymlinkPolicy SymlinkPolicy { get; init; } = FileIndexer.SymlinkPolicy.None; public SymbolKindFilter SymbolKindFilter { get; init; } = SymbolKindFilter.Empty; + public IReadOnlyList GeneratedCodePatterns { get; init; } = []; } public sealed class SymbolKindFilter diff --git a/src/CodeIndex/Indexer/Scanning/FileIndexer.cs b/src/CodeIndex/Indexer/Scanning/FileIndexer.cs index 0824bd8523..aef37600cc 100644 --- a/src/CodeIndex/Indexer/Scanning/FileIndexer.cs +++ b/src/CodeIndex/Indexer/Scanning/FileIndexer.cs @@ -476,6 +476,7 @@ internal static bool IsDefaultExcludedFileName(string fileName) private readonly FileContentLoader _contentLoader; private readonly SymlinkPolicy _symlinkPolicy; private readonly int _maxDanglingFileSystemEntryScanCandidates; + private readonly GeneratedCodePatternMatcher _generatedCodePatterns; // Submodule working-tree paths declared in /.gitmodules, relative to // _projectRoot and slash-normalized. Used to override SkipDirs so that submodules // hosted under SkipDirs-named directories (e.g. vendor/foo) remain visible to the @@ -1009,8 +1010,13 @@ public FileIndexer(string projectRoot, bool ignoreCase) { } - public FileIndexer(string projectRoot, bool ignoreCase, string? ignoreRuleRoot, long? maxFileSizeBytes = null) - : this(projectRoot, ignoreCase, ignoreRuleRoot, maxFileSizeBytes, directoryIgnoreCaseProbe: null) + public FileIndexer( + string projectRoot, + bool ignoreCase, + string? ignoreRuleRoot, + long? maxFileSizeBytes = null, + IReadOnlyList? generatedCodePatterns = null) + : this(projectRoot, ignoreCase, ignoreRuleRoot, maxFileSizeBytes, directoryIgnoreCaseProbe: null, generatedCodePatterns: generatedCodePatterns) { } @@ -1022,7 +1028,8 @@ internal FileIndexer( Func? directoryIgnoreCaseProbe, Func>? enumerateFiles = null, SymlinkPolicy symlinkPolicy = SymlinkPolicy.None, - int? maxDanglingFileSystemEntryScanCandidates = null) + int? maxDanglingFileSystemEntryScanCandidates = null, + IReadOnlyList? generatedCodePatterns = null) { _projectRoot = Path.GetFullPath(projectRoot); _ignoreRuleRoot = NormalizeIgnoreRuleRoot(ignoreRuleRoot); @@ -1037,6 +1044,7 @@ internal FileIndexer( _maxDanglingFileSystemEntryScanCandidates = Math.Max( 1, maxDanglingFileSystemEntryScanCandidates ?? MaxDanglingFileSystemEntryScanCandidates); + _generatedCodePatterns = GeneratedCodePatternMatcher.FromPatterns(generatedCodePatterns, ignoreCase); ExtractorPluginRegistry.LoadPatternConfigsForProjectRoot(_projectRoot); var pathComparer = _ignoreCase ? StringComparer.OrdinalIgnoreCase : StringComparer.Ordinal; (_submodulePaths, _submoduleAncestorPaths, _submoduleLoadWarnings) = LoadGitSubmodulePaths(_ignoreRuleRoot, _projectRoot, pathComparer); @@ -3476,7 +3484,8 @@ public static string NormalizeIndexPath(string path) Lines = loaded.LineCount, Checksum = loaded.Checksum, Modified = loaded.ModifiedUtc, - Generated = IsGeneratedCodeFile(normalizedRelativePath, loaded.Content), + Generated = IsGeneratedCodeFile(normalizedRelativePath, loaded.Content) + || IsConfiguredGeneratedCodeFile(normalizedRelativePath), }; return (record, loaded.Content, loaded.RawBytes, loaded.Warning); @@ -3499,7 +3508,8 @@ public FileRecord BuildSkippedFileRecord(string absolutePath) Lines = 0, Checksum = null, Modified = info.Exists ? info.LastWriteTimeUtc : DateTime.MinValue, - Generated = HasGeneratedCodeFileName(normalizedRelativePath), + Generated = HasGeneratedCodeFileName(normalizedRelativePath) + || IsConfiguredGeneratedCodeFile(normalizedRelativePath), }; } @@ -3561,6 +3571,24 @@ private static string EscapeControlCharacters(string value) internal static bool IsGeneratedCodeFile(string relativePath, string content) => HasGeneratedCodeFileName(relativePath) || HasGeneratedCodeHeader(content); + internal const string GeneratedCodeExtractionSkippedIssueKind = "generated_code_extraction_skipped"; + + internal bool IsConfiguredGeneratedCodeFile(string relativePath) + => _generatedCodePatterns.TryMatch(relativePath, out _); + + internal FileIssue? BuildGeneratedCodeExtractionSkippedIssue(string relativePath) + => _generatedCodePatterns.TryMatch(relativePath, out _) + ? new FileIssue + { + Path = relativePath, + Kind = GeneratedCodeExtractionSkippedIssueKind, + Line = 0, + Message = "Generated-code extraction suppressed by project configuration; file content and chunks were indexed, but symbols and references were skipped.", + Origin = "generated_code_pattern", + Severity = FileIssue.SeverityInfo, + } + : null; + internal static int CountPhysicalLines(string content) { if (content.Length == 0) diff --git a/src/CodeIndex/Indexer/Scanning/GeneratedCodePatternMatcher.cs b/src/CodeIndex/Indexer/Scanning/GeneratedCodePatternMatcher.cs new file mode 100644 index 0000000000..a39eff9552 --- /dev/null +++ b/src/CodeIndex/Indexer/Scanning/GeneratedCodePatternMatcher.cs @@ -0,0 +1,135 @@ +using System.Text; +using System.Text.RegularExpressions; + +namespace CodeIndex.Indexer; + +internal sealed class GeneratedCodePatternMatcher +{ + internal static readonly GeneratedCodePatternMatcher Empty = new([]); + + private static readonly TimeSpan MatchTimeout = TimeSpan.FromMilliseconds(50); + + private readonly Rule[] _rules; + + private GeneratedCodePatternMatcher(Rule[] rules) + { + _rules = rules; + } + + internal static GeneratedCodePatternMatcher FromPatterns(IEnumerable? patterns, bool ignoreCase) + { + if (patterns == null) + return Empty; + + var rules = new List(); + foreach (var raw in patterns) + { + var pattern = NormalizePattern(raw); + if (pattern.Length == 0) + continue; + + rules.Add(new Rule( + pattern, + MatchBasenameOnly: !pattern.Contains('/', StringComparison.Ordinal), + BuildMatcher(pattern, ignoreCase))); + } + + return rules.Count == 0 ? Empty : new GeneratedCodePatternMatcher(rules.ToArray()); + } + + internal bool TryMatch(string relativePath, out string pattern) + { + pattern = string.Empty; + if (_rules.Length == 0) + return false; + + var normalizedPath = NormalizePath(relativePath); + var fileName = GetFileName(normalizedPath); + foreach (var rule in _rules) + { + var candidate = rule.MatchBasenameOnly ? fileName : normalizedPath; + if (rule.Matcher.IsMatch(candidate)) + { + pattern = rule.Pattern; + return true; + } + } + + return false; + } + + private static string NormalizePattern(string? raw) + { + if (string.IsNullOrWhiteSpace(raw)) + return string.Empty; + + var pattern = raw.Trim().Replace('\\', '/'); + while (pattern.StartsWith("./", StringComparison.Ordinal)) + pattern = pattern[2..]; + while (pattern.StartsWith("/", StringComparison.Ordinal)) + pattern = pattern[1..]; + return pattern; + } + + private static string NormalizePath(string path) + { + var normalized = path.Replace('\\', '/'); + while (normalized.StartsWith("./", StringComparison.Ordinal)) + normalized = normalized[2..]; + return normalized; + } + + private static string GetFileName(string normalizedPath) + { + var slash = normalizedPath.LastIndexOf('/'); + return slash < 0 ? normalizedPath : normalizedPath[(slash + 1)..]; + } + + private static Regex BuildMatcher(string pattern, bool ignoreCase) + { + var builder = new StringBuilder(pattern.Length * 2); + builder.Append('^'); + for (var i = 0; i < pattern.Length; i++) + { + var ch = pattern[i]; + if (ch == '*') + { + var isDoubleStar = i + 1 < pattern.Length && pattern[i + 1] == '*'; + if (isDoubleStar) + { + if (i + 2 < pattern.Length && pattern[i + 2] == '/') + { + builder.Append("(?:[^/]+/)*"); + i += 2; + } + else + { + builder.Append(".*"); + i++; + } + } + else + { + builder.Append("[^/]*"); + } + continue; + } + + if (ch == '?') + { + builder.Append("[^/]"); + continue; + } + + builder.Append(Regex.Escape(ch.ToString())); + } + builder.Append('$'); + + var options = RegexOptions.CultureInvariant | RegexOptions.NonBacktracking; + if (ignoreCase) + options |= RegexOptions.IgnoreCase; + return new Regex(builder.ToString(), options, MatchTimeout); + } + + private sealed record Rule(string Pattern, bool MatchBasenameOnly, Regex Matcher); +} diff --git a/src/CodeIndex/Mcp/McpToolHandlers.cs b/src/CodeIndex/Mcp/McpToolHandlers.cs index 3132d1ccfe..a7c02d5faf 100644 --- a/src/CodeIndex/Mcp/McpToolHandlers.cs +++ b/src/CodeIndex/Mcp/McpToolHandlers.cs @@ -5592,7 +5592,8 @@ private async Task ExecuteIndexAsync(JsonNode? id, JsonNode? args, Jso GitHelper.TryGetRepositoryRoot(projectPath, _currentRequestToken.Value) ?? Path.GetFullPath(projectPath), maxFileBytes, directoryIgnoreCaseProbe: null, - symlinkPolicy: symlinkPolicy); + symlinkPolicy: symlinkPolicy, + generatedCodePatterns: IndexCommandRunner.ReadGeneratedCodePatternsFromEnvironment()); var scan = dryRunIndexer.ScanFilesDetailed(cancellationToken: _currentRequestToken.Value); if (memorySamples != null) memorySamples.Add(CaptureMcpIndexMemorySample("scan", runStopwatch)); @@ -5697,7 +5698,8 @@ private async Task ExecuteIndexAsync(JsonNode? id, JsonNode? args, Jso GitHelper.TryGetRepositoryRoot(projectPath, requestToken) ?? Path.GetFullPath(projectPath), maxFileBytes, directoryIgnoreCaseProbe: null, - symlinkPolicy: symlinkPolicy); + symlinkPolicy: symlinkPolicy, + generatedCodePatterns: IndexCommandRunner.ReadGeneratedCodePatternsFromEnvironment()); using var postExtractionHooks = PostExtractionHookRunner.DiscoverDefault(maxFileBytes); var currentHotspotFamilyMarkerFingerprints = GetHotspotFamilyMarkerFingerprints(indexer, requestToken); var currentCSharpSymbolNameContractVersion = DbContext.CSharpSymbolNameContractVersion.ToString(System.Globalization.CultureInfo.InvariantCulture); @@ -5836,7 +5838,9 @@ static string FormatDiagnosticPath(string projectRoot, string path) record.Modified, record.Checksum, size: record.Size, + lines: record.Lines, language: record.Lang, + generated: record.Generated, allowReuse: symbolKindFilterMatchesPrior && record.Lang is not ("javascript" or "typescript") && (record.Lang != "csharp" || csharpSymbolNameContractMatchesCurrent) @@ -5853,6 +5857,11 @@ static string FormatDiagnosticPath(string projectRoot, string path) existingId = null; } } + if (existingId != null + && IndexCommandRunner.ExistingFileGeneratedSuppressionMismatch(writer, existingId.Value, indexer.BuildGeneratedCodeExtractionSkippedIssue(record.Path))) + { + existingId = null; + } if (existingId != null) { skipped++; @@ -5868,6 +5877,22 @@ static string FormatDiagnosticPath(string projectRoot, string path) using var txn = writer.BeginTransaction(); var fileId = writer.UpsertFile(record); var chunks = ChunkSplitter.Split(fileId, content); + var generatedSuppressionIssue = indexer.BuildGeneratedCodeExtractionSkippedIssue(record.Path); + if (generatedSuppressionIssue != null) + { + writer.InsertChunks(chunks); + writer.InsertSymbols([]); + writer.InsertReferences([]); + var issues = IndexCommandRunner.AppendIssueIfMissing( + FileIndexer.ValidateContent(record.Path, rawBytes, content, record.Lang), + generatedSuppressionIssue); + writer.InsertIssues(fileId, issues); + WriteProjectRootOnce(); + writer.ClearBatchInProgress(); + txn.Commit(); + McpIndexFileCommittedForTesting?.Invoke(record.Path); + continue; + } var symbols = SymbolExtractor.Extract(fileId, record.Lang, content, filePath, projectPath, requestToken).ToList(); SymbolExtractor.ApplyFamilyScope(symbols, indexer.GetFamilyScopeKey(filePath, record.Lang)); var fileContext = new FileContext(projectPath, record.Path, filePath, record.Lang); @@ -7115,6 +7140,8 @@ private static CSharpStaticInterfaceWorkspaceSymbols BuildMcpCSharpStaticInterfa var (record, content, _, _) = indexer.BuildRecordWithRawBytes(absolutePath, cancellationToken); if (record.Lang != "csharp") continue; + if (indexer.BuildGeneratedCodeExtractionSkippedIssue(record.Path) != null) + continue; pendingSymbols.AddRange(SymbolExtractor.Extract(0, record.Lang, content, record.Path, cancellationToken: cancellationToken)); } diff --git a/tests/CodeIndex.Tests/CdidxConfigFileTests.cs b/tests/CodeIndex.Tests/CdidxConfigFileTests.cs index 9bc4a977be..5b9d7cd325 100644 --- a/tests/CodeIndex.Tests/CdidxConfigFileTests.cs +++ b/tests/CodeIndex.Tests/CdidxConfigFileTests.cs @@ -45,7 +45,8 @@ public void LoadAndApply_MaterializesKnownKeysIntoEnvironment() "suggestion_max_count": 250, "indexing": { "includeKinds": ["class"], - "excludeKinds": ["test_method", "generated_parser"] + "excludeKinds": ["test_method", "generated_parser"], + "generatedCodePatterns": ["src/generated/**", "*.client.ts"] }, "mcp": { "tools": { "allow": ["search", "definition"], "deny": ["index"] }, @@ -69,6 +70,7 @@ public void LoadAndApply_MaterializesKnownKeysIntoEnvironment() Assert.Equal("250", result.Settings["CDIDX_SUGGESTION_MAX_COUNT"]); Assert.Equal("class", result.Settings["CDIDX_INDEX_INCLUDE_SYMBOL_KINDS"]); Assert.Equal("test_method,generated_parser", result.Settings["CDIDX_INDEX_EXCLUDE_SYMBOL_KINDS"]); + Assert.Equal("src/generated/**,*.client.ts", result.Settings[IndexCommandRunner.GeneratedCodePatternsEnvironmentVariable]); Assert.Equal("search,definition", result.Settings["CDIDX_MCP_TOOLS_ALLOW"]); Assert.Equal("index", result.Settings["CDIDX_MCP_TOOLS_DENY"]); Assert.Equal("5", result.Settings[RateLimiterOptions.RpsEnvVar]); diff --git a/tests/CodeIndex.Tests/FileIndexerTests.cs b/tests/CodeIndex.Tests/FileIndexerTests.cs index 939fabe099..8c77c22e18 100644 --- a/tests/CodeIndex.Tests/FileIndexerTests.cs +++ b/tests/CodeIndex.Tests/FileIndexerTests.cs @@ -5225,6 +5225,40 @@ size 12345 } } + [Fact] + public void BuildRecord_ConfiguredGeneratedPatternMarksGeneratedAndBuildsExtractionIssue() + { + var tempDir = Path.Combine(Path.GetTempPath(), $"codeindex_test_{Guid.NewGuid():N}"); + try + { + var generatedDir = Path.Combine(tempDir, "src", "generated"); + Directory.CreateDirectory(generatedDir); + var filePath = Path.Combine(generatedDir, "Client.cs"); + File.WriteAllText(filePath, "public class Client { public string Lookup() => \"ok\"; }\n"); + + var indexer = new FileIndexer( + tempDir, + ignoreCase: false, + ignoreRuleRoot: null, + generatedCodePatterns: ["src/generated/**"]); + var (record, content, rawBytes, _) = indexer.BuildRecordWithRawBytes(filePath); + var issue = indexer.BuildGeneratedCodeExtractionSkippedIssue(record.Path); + + Assert.True(record.Generated); + Assert.Equal("src/generated/Client.cs", record.Path); + Assert.Contains("public class Client", content, StringComparison.Ordinal); + Assert.True(rawBytes.Length > 0); + Assert.NotNull(issue); + Assert.Equal(FileIndexer.GeneratedCodeExtractionSkippedIssueKind, issue.Kind); + Assert.Equal("src/generated/Client.cs", issue.Path); + Assert.Contains("symbols and references were skipped", issue.Message, StringComparison.Ordinal); + } + finally + { + Directory.Delete(tempDir, true); + } + } + [Fact] public void BuildRecord_GitLfsVersionLineWithoutPointerShapePreservesContent() { diff --git a/tests/CodeIndex.Tests/IndexCommandRunnerTests.cs b/tests/CodeIndex.Tests/IndexCommandRunnerTests.cs index f228c3f8af..21bbcdda30 100644 --- a/tests/CodeIndex.Tests/IndexCommandRunnerTests.cs +++ b/tests/CodeIndex.Tests/IndexCommandRunnerTests.cs @@ -6014,6 +6014,103 @@ FROM symbol_references r } } + [Fact] + public void Run_FullScan_ConfiguredGeneratedCodePatternsKeepChunksButSkipExtraction() + { + using var env = EnvironmentVariableScope.Capture(IndexCommandRunner.GeneratedCodePatternsEnvironmentVariable); + var projectRoot = CreateTempProject(); + try + { + env.Set(IndexCommandRunner.GeneratedCodePatternsEnvironmentVariable, "src/generated/**"); + Directory.CreateDirectory(Path.Combine(projectRoot, "src", "generated")); + File.WriteAllText( + Path.Combine(projectRoot, "src", "generated", "GeneratedClient.cs"), + """ + public class GeneratedClient + { + public string Lookup() => "generated"; + } + """); + File.WriteAllText( + Path.Combine(projectRoot, "NormalClient.cs"), + """ + public class NormalClient + { + public string Lookup() => "normal"; + } + """); + + var exitCode = IndexCommandRunner.Run([projectRoot, "--json", "--quiet"], _jsonOptions); + Assert.Equal(CommandExitCodes.Success, exitCode); + + var dbPath = Path.Combine(projectRoot, ".cdidx", "codeindex.db"); + using (var conn = OpenNonPoolingConnection(dbPath)) + { + conn.Open(); + using var generatedCmd = conn.CreateCommand(); + generatedCmd.CommandText = """ + SELECT f.generated, + (SELECT COUNT(*) FROM chunks c WHERE c.file_id = f.id AND c.content LIKE '%GeneratedClient%'), + (SELECT COUNT(*) FROM symbols s WHERE s.file_id = f.id), + (SELECT COUNT(*) FROM symbol_references r WHERE r.file_id = f.id), + (SELECT COUNT(*) FROM file_issues i WHERE i.file_id = f.id AND i.kind = @issueKind) + FROM files f + WHERE f.path = @path + """; + generatedCmd.Parameters.AddWithValue("@issueKind", FileIndexer.GeneratedCodeExtractionSkippedIssueKind); + generatedCmd.Parameters.AddWithValue("@path", "src/generated/GeneratedClient.cs"); + using (var reader = generatedCmd.ExecuteReader()) + { + Assert.True(reader.Read()); + Assert.Equal(1, reader.GetInt32(0)); + Assert.True(reader.GetInt32(1) > 0); + Assert.Equal(0, reader.GetInt32(2)); + Assert.Equal(0, reader.GetInt32(3)); + Assert.Equal(1, reader.GetInt32(4)); + } + + using var normalCmd = conn.CreateCommand(); + normalCmd.CommandText = """ + SELECT COUNT(*) + FROM symbols s + JOIN files f ON f.id = s.file_id + WHERE f.path = 'NormalClient.cs' + AND s.name = 'NormalClient' + """; + Assert.Equal(1L, (long)normalCmd.ExecuteScalar()!); + } + + env.Set(IndexCommandRunner.GeneratedCodePatternsEnvironmentVariable, null); + var updateExitCode = IndexCommandRunner.Run([projectRoot, "--files", "src/generated/GeneratedClient.cs", "--json", "--quiet"], _jsonOptions); + Assert.Equal(CommandExitCodes.Success, updateExitCode); + + using (var conn = OpenNonPoolingConnection(dbPath)) + { + conn.Open(); + using var updatedCmd = conn.CreateCommand(); + updatedCmd.CommandText = """ + SELECT f.generated, + (SELECT COUNT(*) FROM symbols s WHERE s.file_id = f.id AND s.name = 'GeneratedClient'), + (SELECT COUNT(*) FROM file_issues i WHERE i.file_id = f.id AND i.kind = @issueKind) + FROM files f + WHERE f.path = @path + """; + updatedCmd.Parameters.AddWithValue("@issueKind", FileIndexer.GeneratedCodeExtractionSkippedIssueKind); + updatedCmd.Parameters.AddWithValue("@path", "src/generated/GeneratedClient.cs"); + using var reader = updatedCmd.ExecuteReader(); + Assert.True(reader.Read()); + Assert.Equal(0, reader.GetInt32(0)); + Assert.Equal(1, reader.GetInt32(1)); + Assert.Equal(0, reader.GetInt32(2)); + } + } + finally + { + DeleteDirectory(projectRoot); + SqliteConnection.ClearAllPools(); + } + } + [Fact] public void Run_UpdateFiles_CsharpStaticInterfaceContractChange_ReindexesImplementers() { From a7758812404566a5fa046d8178dc7dae7f66e768 Mon Sep 17 00:00:00 2001 From: Widthdom Date: Sat, 20 Jun 2026 13:17:30 +0900 Subject: [PATCH 09/18] Align MCP reference cap contract (#3719) --- changelog.d/unreleased/3719.fixed.md | 1 + src/CodeIndex/Mcp/McpToolArgumentContracts.cs | 2 +- src/CodeIndex/Mcp/McpToolHandlers.cs | 4 ++-- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/changelog.d/unreleased/3719.fixed.md b/changelog.d/unreleased/3719.fixed.md index 289ae6fc4e..e7c9fcf103 100644 --- a/changelog.d/unreleased/3719.fixed.md +++ b/changelog.d/unreleased/3719.fixed.md @@ -8,6 +8,7 @@ affected: - src/CodeIndex/Cli/IndexCommandRunner.Update.cs - src/CodeIndex/Mcp/McpToolHandlers.cs - src/CodeIndex/Mcp/McpToolDefinitions.cs + - src/CodeIndex/Mcp/McpToolArgumentContracts.cs --- ## English diff --git a/src/CodeIndex/Mcp/McpToolArgumentContracts.cs b/src/CodeIndex/Mcp/McpToolArgumentContracts.cs index 2f2f988952..7e98f77691 100644 --- a/src/CodeIndex/Mcp/McpToolArgumentContracts.cs +++ b/src/CodeIndex/Mcp/McpToolArgumentContracts.cs @@ -33,7 +33,7 @@ public partial class McpServer "validate" => new HashSet(StringComparer.Ordinal) { "kind", "severity", "limit", "path", "excludePaths", "excludeTests", "countOnly", "format", "project", "solution" }, "unused_symbols" => new HashSet(StringComparer.Ordinal) { "kind", "lang", "limit", "visibility", "excludeVisibility", "path", "excludePaths", "excludeTests", "bucket", "minConfidence", "byBucket", "project", "solution" }, "symbol_hotspots" => new HashSet(StringComparer.Ordinal) { "kind", "lang", "limit", "visibility", "excludeVisibility", "groupBy", "path", "excludePaths", "excludeTests", "project", "solution" }, - "index" => new HashSet(StringComparer.Ordinal) { "path", "rebuild", "dryRun", "dry_run", "maxFileBytes", "maxSymbolsPerFile", "followSymlinks", "includeSymbolKind", "excludeSymbolKind", "memoryTrace", "parallelism", "commits", "changedBetween", "files", "watch", "debounce" }, + "index" => new HashSet(StringComparer.Ordinal) { "path", "rebuild", "dryRun", "dry_run", "maxFileBytes", "maxSymbolsPerFile", "maxReferencesPerFile", "followSymlinks", "includeSymbolKind", "excludeSymbolKind", "memoryTrace", "parallelism", "commits", "changedBetween", "files", "watch", "debounce" }, "backfill_fold" => new HashSet(StringComparer.Ordinal) { "dry_run", "dryRun", "force" }, "suggest_improvement" => new HashSet(StringComparer.Ordinal) { "category", "language", "description", "context", "toolInvocationContext", "evidencePaths", "evidence_paths" }, _ => new HashSet(StringComparer.Ordinal), diff --git a/src/CodeIndex/Mcp/McpToolHandlers.cs b/src/CodeIndex/Mcp/McpToolHandlers.cs index a7c02d5faf..acb53c0d82 100644 --- a/src/CodeIndex/Mcp/McpToolHandlers.cs +++ b/src/CodeIndex/Mcp/McpToolHandlers.cs @@ -856,8 +856,8 @@ private static bool TryGetExpectedJsonType(string toolName, string argumentName, { "limit" or "offset" or "snippetLines" or "maxLineWidth" or "before" or "after" or "focusLine" or "focusColumn" or "focusLength" or "startLine" or "endLine" or - "maxHops" or "maxDepth" or "depth" or "parallelism" or "maxFileBytes" or "maxSymbolsPerFile" or "debounce" or - "maxHops" or "maxDepth" or "depth" or "parallelism" or "maxFileBytes" or "maxSymbolsPerFile" or "debounce" or + "maxHops" or "maxDepth" or "depth" or "parallelism" or "maxFileBytes" or "maxSymbolsPerFile" or "maxReferencesPerFile" or "debounce" or + "maxHops" or "maxDepth" or "depth" or "parallelism" or "maxFileBytes" or "maxSymbolsPerFile" or "maxReferencesPerFile" or "debounce" or "staleAfterSeconds" or "guardWindow" or "maxOutputBytes" or "maxResponseBytes" => "integer", "check" or "excludeTests" or "includeGenerated" or "indexedOnly" or "rawQuery" or "noDedup" or "exactSubstring" or From 8797f7d17ea81116d255816601518a75801b1023 Mon Sep 17 00:00:00 2001 From: Widthdom Date: Sat, 20 Jun 2026 13:49:02 +0900 Subject: [PATCH 10/18] Preserve generated suppression search visibility (#3720) --- USER_GUIDE.md | 4 +- changelog.d/unreleased/3720.fixed.md | 5 +- src/CodeIndex/Indexer/Scanning/FileIndexer.cs | 9 +--- src/CodeIndex/Mcp/McpToolHandlers.cs | 2 + tests/CodeIndex.Tests/FileIndexerTests.cs | 4 +- .../IndexCommandRunnerTests.cs | 2 +- tests/CodeIndex.Tests/McpServerTests.cs | 54 +++++++++++++++++++ 7 files changed, 66 insertions(+), 14 deletions(-) diff --git a/USER_GUIDE.md b/USER_GUIDE.md index ce10bd627d..6c3fb7bab3 100644 --- a/USER_GUIDE.md +++ b/USER_GUIDE.md @@ -1724,7 +1724,7 @@ Supported schema (top-level keys are snake_case; nested indexing kind keys keep } ``` -JSON5-style line comments (`//`) and trailing commas are accepted so the file stays human-editable. The optional `$schema` key is ignored at runtime; it is honored only so editors that recognize JSON Schema references can offer completion. Setting `disable_persistent_log` to `false` is a no-op (absence already means "logging enabled") — only `true` exports `CDIDX_DISABLE_PERSISTENT_LOG=1`. Config-sourced `metrics_path` and `global_tool_log_dir` values are resolved from the config workspace root and must stay inside that workspace; use the CLI flag or a real environment variable when you intentionally need an outside destination. `stale_after` uses the same compact duration format as `status --check --stale-after`: `30m`, `2h`, or `7d`, up to `30d`. `suggestion_dedup_threshold` sets the MCP suggestion fuzzy-deduplication cutoff as a number from `0` to `1`; the built-in default is `0.85`, and `cdidx mcp --suggestion-dedup-threshold <0..1>` overrides it for one MCP session. `suggestion_max_age_days` and `suggestion_max_count` bound the live `.cdidx/suggestions-*.json` store; pruned records are appended to `.cdidx/suggestions-*.archive.jsonl`, whose active file is capped at 8 MiB and rotates up to three retained generations (`.1` through `.3`). Defaults are 365 days and 5000 records, and config-file values may not exceed 3650 days or 100000 records. Matching environment variables above those caps fall back to the defaults. `mcp.rate_limit.bucket_idle_seconds` sets the same idle bucket TTL as `CDIDX_MCP_RATE_LIMIT_BUCKET_IDLE_SECONDS`; invalid runtime values fall back to the default with a warning. String-array settings such as `indexing.includeKinds`, `indexing.excludeKinds`, `indexing.generatedCodePatterns`, `mcp.tools.allow`, and `mcp.tools.deny` are capped at 128 entries and 256 characters per item before they are joined into environment variables. `indexing.generatedCodePatterns` marks matching relative paths or basenames as generated code. Matching files remain indexed for text search and chunk retrieval, but symbol/reference extraction is skipped and `file_issues` records `generated_code_extraction_skipped`; patterns with a slash match slash-normalized relative paths, patterns without a slash match basenames, and `*`, `?`, and `**` are supported. `indexing.includeKinds` and `indexing.excludeKinds` set the default symbol-kind filter for `cdidx index`; CLI flags `--include-symbol-kind [,]` and `--exclude-symbol-kind [,]` override those env-backed defaults for a single run. +JSON5-style line comments (`//`) and trailing commas are accepted so the file stays human-editable. The optional `$schema` key is ignored at runtime; it is honored only so editors that recognize JSON Schema references can offer completion. Setting `disable_persistent_log` to `false` is a no-op (absence already means "logging enabled") — only `true` exports `CDIDX_DISABLE_PERSISTENT_LOG=1`. Config-sourced `metrics_path` and `global_tool_log_dir` values are resolved from the config workspace root and must stay inside that workspace; use the CLI flag or a real environment variable when you intentionally need an outside destination. `stale_after` uses the same compact duration format as `status --check --stale-after`: `30m`, `2h`, or `7d`, up to `30d`. `suggestion_dedup_threshold` sets the MCP suggestion fuzzy-deduplication cutoff as a number from `0` to `1`; the built-in default is `0.85`, and `cdidx mcp --suggestion-dedup-threshold <0..1>` overrides it for one MCP session. `suggestion_max_age_days` and `suggestion_max_count` bound the live `.cdidx/suggestions-*.json` store; pruned records are appended to `.cdidx/suggestions-*.archive.jsonl`, whose active file is capped at 8 MiB and rotates up to three retained generations (`.1` through `.3`). Defaults are 365 days and 5000 records, and config-file values may not exceed 3650 days or 100000 records. Matching environment variables above those caps fall back to the defaults. `mcp.rate_limit.bucket_idle_seconds` sets the same idle bucket TTL as `CDIDX_MCP_RATE_LIMIT_BUCKET_IDLE_SECONDS`; invalid runtime values fall back to the default with a warning. String-array settings such as `indexing.includeKinds`, `indexing.excludeKinds`, `indexing.generatedCodePatterns`, `mcp.tools.allow`, and `mcp.tools.deny` are capped at 128 entries and 256 characters per item before they are joined into environment variables. `indexing.generatedCodePatterns` treats matching relative paths or basenames as extraction-suppressed generated-code sources. Matching files remain indexed for normal text search and chunk retrieval because the query-filtered `generated` flag is not set by this option; symbol/reference extraction is skipped and `file_issues` records `generated_code_extraction_skipped`. Patterns with a slash match slash-normalized relative paths, patterns without a slash match basenames, and `*`, `?`, and `**` are supported. `indexing.includeKinds` and `indexing.excludeKinds` set the default symbol-kind filter for `cdidx index`; CLI flags `--include-symbol-kind [,]` and `--exclude-symbol-kind [,]` override those env-backed defaults for a single run. ## How it works @@ -4245,7 +4245,7 @@ MCP のレスポンスサイズ上限は、環境変数 override で guard が } ``` -人手で編集しやすいよう JSON5 形式の行コメント（`//`）と末尾カンマを許容します。任意の `$schema` キーはランタイムでは無視され、JSON Schema 参照をサポートするエディタが補完を提供するためだけに認識されます。`disable_persistent_log` を `false` に設定しても何も起きません（不在のままで "ログ有効" が既定）— `true` の場合のみ `CDIDX_DISABLE_PERSISTENT_LOG=1` を export します。config 由来の `metrics_path` と `global_tool_log_dir` は設定ファイルの workspace root から解決され、その workspace 内に収まる必要があります。意図的に外部の出力先を使う場合は CLI フラグまたは実際の環境変数を使ってください。`stale_after` は `status --check --stale-after` と同じ compact duration 形式（`30m` / `2h` / `7d`、最大 `30d`）です。`suggestion_dedup_threshold` は MCP suggestion の fuzzy deduplication しきい値を `0` から `1` の数値で設定します。組み込み既定値は `0.85` で、`cdidx mcp --suggestion-dedup-threshold <0..1>` は 1 回の MCP session だけこの値を上書きします。`suggestion_max_age_days` と `suggestion_max_count` は live の `.cdidx/suggestions-*.json` store の上限を設定し、prune された record は `.cdidx/suggestions-*.archive.jsonl` に追記されます。この active archive は 8 MiB で上限管理され、最大 3 世代（`.1` から `.3`）までローテーションされます。既定値は 365 日と 5000 件で、config-file 値は 3650 日または 100000 件を超えられません。同じ環境変数がこの上限を超えた場合は既定値へ戻ります。`mcp.rate_limit.bucket_idle_seconds` は `CDIDX_MCP_RATE_LIMIT_BUCKET_IDLE_SECONDS` と同じ idle bucket TTL を設定します。不正な runtime 値は警告付きで既定値へ戻ります。`indexing.includeKinds`、`indexing.excludeKinds`、`indexing.generatedCodePatterns`、`mcp.tools.allow`、`mcp.tools.deny` のような string array 設定は、環境変数へ join される前に 128 件、1 要素 256 文字までに制限されます。`indexing.generatedCodePatterns` は一致した相対パスまたはベース名を generated code として扱います。一致したファイルは全文検索と chunk 取得用には引き続き index されますが、symbol/reference 抽出はスキップされ、`file_issues` に `generated_code_extraction_skipped` が記録されます。スラッシュを含む pattern は slash-normalized relative path、スラッシュを含まない pattern は basename に一致し、`*`、`?`、`**` を利用できます。`indexing.includeKinds` と `indexing.excludeKinds` は `cdidx index` の symbol-kind filter 既定値を設定し、CLI フラグ `--include-symbol-kind [,]` / `--exclude-symbol-kind [,]` はその env 経由の既定値を 1 回の実行だけ上書きします。 +人手で編集しやすいよう JSON5 形式の行コメント（`//`）と末尾カンマを許容します。任意の `$schema` キーはランタイムでは無視され、JSON Schema 参照をサポートするエディタが補完を提供するためだけに認識されます。`disable_persistent_log` を `false` に設定しても何も起きません（不在のままで "ログ有効" が既定）— `true` の場合のみ `CDIDX_DISABLE_PERSISTENT_LOG=1` を export します。config 由来の `metrics_path` と `global_tool_log_dir` は設定ファイルの workspace root から解決され、その workspace 内に収まる必要があります。意図的に外部の出力先を使う場合は CLI フラグまたは実際の環境変数を使ってください。`stale_after` は `status --check --stale-after` と同じ compact duration 形式（`30m` / `2h` / `7d`、最大 `30d`）です。`suggestion_dedup_threshold` は MCP suggestion の fuzzy deduplication しきい値を `0` から `1` の数値で設定します。組み込み既定値は `0.85` で、`cdidx mcp --suggestion-dedup-threshold <0..1>` は 1 回の MCP session だけこの値を上書きします。`suggestion_max_age_days` と `suggestion_max_count` は live の `.cdidx/suggestions-*.json` store の上限を設定し、prune された record は `.cdidx/suggestions-*.archive.jsonl` に追記されます。この active archive は 8 MiB で上限管理され、最大 3 世代（`.1` から `.3`）までローテーションされます。既定値は 365 日と 5000 件で、config-file 値は 3650 日または 100000 件を超えられません。同じ環境変数がこの上限を超えた場合は既定値へ戻ります。`mcp.rate_limit.bucket_idle_seconds` は `CDIDX_MCP_RATE_LIMIT_BUCKET_IDLE_SECONDS` と同じ idle bucket TTL を設定します。不正な runtime 値は警告付きで既定値へ戻ります。`indexing.includeKinds`、`indexing.excludeKinds`、`indexing.generatedCodePatterns`、`mcp.tools.allow`、`mcp.tools.deny` のような string array 設定は、環境変数へ join される前に 128 件、1 要素 256 文字までに制限されます。`indexing.generatedCodePatterns` は一致した相対パスまたはベース名を generated-code extraction の抑制対象として扱います。この設定では query filter 用の `generated` flag を立てないため、一致したファイルも通常の全文検索と chunk 取得用には引き続き index されます。symbol/reference 抽出はスキップされ、`file_issues` に `generated_code_extraction_skipped` が記録されます。スラッシュを含む pattern は slash-normalized relative path、スラッシュを含まない pattern は basename に一致し、`*`、`?`、`**` を利用できます。`indexing.includeKinds` と `indexing.excludeKinds` は `cdidx index` の symbol-kind filter 既定値を設定し、CLI フラグ `--include-symbol-kind [,]` / `--exclude-symbol-kind [,]` はその env 経由の既定値を 1 回の実行だけ上書きします。 ## 動作の仕組み diff --git a/changelog.d/unreleased/3720.fixed.md b/changelog.d/unreleased/3720.fixed.md index ff7a42d219..ab7e2496e6 100644 --- a/changelog.d/unreleased/3720.fixed.md +++ b/changelog.d/unreleased/3720.fixed.md @@ -11,13 +11,14 @@ affected: - tests/CodeIndex.Tests/CdidxConfigFileTests.cs - tests/CodeIndex.Tests/FileIndexerTests.cs - tests/CodeIndex.Tests/IndexCommandRunnerTests.cs + - tests/CodeIndex.Tests/McpServerTests.cs - USER_GUIDE.md --- ## English -- **Generated-code indexing suppression is now configurable (#3720)** — project config can mark filename or path patterns as generated code, preserving text chunks while skipping symbol/reference extraction and recording `generated_code_extraction_skipped` in `file_issues`. +- **Generated-code extraction suppression is now configurable (#3720)** — project config can suppress symbol/reference extraction for filename or path patterns while preserving normal text search visibility and recording `generated_code_extraction_skipped` in `file_issues`. ## 日本語 -- **generated-code indexing suppression を設定可能にしました (#3720)** — project config で filename/path pattern を generated code として指定でき、text chunk は保持しながら symbol/reference 抽出をスキップし、`file_issues` に `generated_code_extraction_skipped` を記録します。 +- **generated-code extraction suppression を設定可能にしました (#3720)** — project config で filename/path pattern の symbol/reference 抽出を抑制でき、通常の text search での可視性は維持しつつ、`file_issues` に `generated_code_extraction_skipped` を記録します。 diff --git a/src/CodeIndex/Indexer/Scanning/FileIndexer.cs b/src/CodeIndex/Indexer/Scanning/FileIndexer.cs index aef37600cc..56766e423e 100644 --- a/src/CodeIndex/Indexer/Scanning/FileIndexer.cs +++ b/src/CodeIndex/Indexer/Scanning/FileIndexer.cs @@ -3484,8 +3484,7 @@ public static string NormalizeIndexPath(string path) Lines = loaded.LineCount, Checksum = loaded.Checksum, Modified = loaded.ModifiedUtc, - Generated = IsGeneratedCodeFile(normalizedRelativePath, loaded.Content) - || IsConfiguredGeneratedCodeFile(normalizedRelativePath), + Generated = IsGeneratedCodeFile(normalizedRelativePath, loaded.Content), }; return (record, loaded.Content, loaded.RawBytes, loaded.Warning); @@ -3508,8 +3507,7 @@ public FileRecord BuildSkippedFileRecord(string absolutePath) Lines = 0, Checksum = null, Modified = info.Exists ? info.LastWriteTimeUtc : DateTime.MinValue, - Generated = HasGeneratedCodeFileName(normalizedRelativePath) - || IsConfiguredGeneratedCodeFile(normalizedRelativePath), + Generated = HasGeneratedCodeFileName(normalizedRelativePath), }; } @@ -3573,9 +3571,6 @@ internal static bool IsGeneratedCodeFile(string relativePath, string content) internal const string GeneratedCodeExtractionSkippedIssueKind = "generated_code_extraction_skipped"; - internal bool IsConfiguredGeneratedCodeFile(string relativePath) - => _generatedCodePatterns.TryMatch(relativePath, out _); - internal FileIssue? BuildGeneratedCodeExtractionSkippedIssue(string relativePath) => _generatedCodePatterns.TryMatch(relativePath, out _) ? new FileIssue diff --git a/src/CodeIndex/Mcp/McpToolHandlers.cs b/src/CodeIndex/Mcp/McpToolHandlers.cs index 930421310a..9acbf8393f 100644 --- a/src/CodeIndex/Mcp/McpToolHandlers.cs +++ b/src/CodeIndex/Mcp/McpToolHandlers.cs @@ -5975,6 +5975,8 @@ static string FormatDiagnosticPath(string projectRoot, string path) WriteProjectRootOnce(); writer.ClearBatchInProgress(); txn.Commit(); + processed++; + await EmitProgressNotificationAsync(progressToken, processed, files.Count).ConfigureAwait(false); McpIndexFileCommittedForTesting?.Invoke(record.Path); continue; } diff --git a/tests/CodeIndex.Tests/FileIndexerTests.cs b/tests/CodeIndex.Tests/FileIndexerTests.cs index 8c77c22e18..77dabd01b1 100644 --- a/tests/CodeIndex.Tests/FileIndexerTests.cs +++ b/tests/CodeIndex.Tests/FileIndexerTests.cs @@ -5226,7 +5226,7 @@ size 12345 } [Fact] - public void BuildRecord_ConfiguredGeneratedPatternMarksGeneratedAndBuildsExtractionIssue() + public void BuildRecord_ConfiguredGeneratedPatternBuildsExtractionIssueWithoutGeneratedFlag() { var tempDir = Path.Combine(Path.GetTempPath(), $"codeindex_test_{Guid.NewGuid():N}"); try @@ -5244,7 +5244,7 @@ public void BuildRecord_ConfiguredGeneratedPatternMarksGeneratedAndBuildsExtract var (record, content, rawBytes, _) = indexer.BuildRecordWithRawBytes(filePath); var issue = indexer.BuildGeneratedCodeExtractionSkippedIssue(record.Path); - Assert.True(record.Generated); + Assert.False(record.Generated); Assert.Equal("src/generated/Client.cs", record.Path); Assert.Contains("public class Client", content, StringComparison.Ordinal); Assert.True(rawBytes.Length > 0); diff --git a/tests/CodeIndex.Tests/IndexCommandRunnerTests.cs b/tests/CodeIndex.Tests/IndexCommandRunnerTests.cs index 21bbcdda30..72c18275b9 100644 --- a/tests/CodeIndex.Tests/IndexCommandRunnerTests.cs +++ b/tests/CodeIndex.Tests/IndexCommandRunnerTests.cs @@ -6062,7 +6062,7 @@ FROM files f using (var reader = generatedCmd.ExecuteReader()) { Assert.True(reader.Read()); - Assert.Equal(1, reader.GetInt32(0)); + Assert.Equal(0, reader.GetInt32(0)); Assert.True(reader.GetInt32(1) > 0); Assert.Equal(0, reader.GetInt32(2)); Assert.Equal(0, reader.GetInt32(3)); diff --git a/tests/CodeIndex.Tests/McpServerTests.cs b/tests/CodeIndex.Tests/McpServerTests.cs index c7b79eb376..d96f5916af 100644 --- a/tests/CodeIndex.Tests/McpServerTests.cs +++ b/tests/CodeIndex.Tests/McpServerTests.cs @@ -10632,6 +10632,60 @@ public void ToolsCall_Index_ReprocessesAfterPartialSymbolKindFilterChange_Issue3 } } + [Fact] + public void ToolsCall_Index_GeneratedCodePatternCountsProcessedAndSkipsExtraction_Issue3720() + { + using var env = EnvironmentVariableScope.Capture(IndexCommandRunner.GeneratedCodePatternsEnvironmentVariable); + var fixtureDir = Path.Combine(Path.GetFullPath("."), $"mcp_index_generated_pattern_{Guid.NewGuid():N}"); + Directory.CreateDirectory(fixtureDir); + var dbPath = Path.Combine(Path.GetTempPath(), $"cdidx_mcp_index_generated_pattern_{Guid.NewGuid():N}.db"); + try + { + env.Set(IndexCommandRunner.GeneratedCodePatternsEnvironmentVariable, "generated/**"); + Directory.CreateDirectory(Path.Combine(fixtureDir, "generated")); + File.WriteAllText( + Path.Combine(fixtureDir, "generated", "Client.cs"), + "public class GeneratedClient { public string Lookup() => \"generated\"; }\n"); + using var server = new McpServer(dbPath, ConsoleUi.LoadVersion()); + + var response = CallIndex(server, fixtureDir); + + Assert.False(response["result"]?["isError"]?.GetValue() ?? false); + using var verifyDb = new DbContext(dbPath); + Assert.Equal("1", verifyDb.GetMetaString(DbContext.LastIndexRunRowsUpsertedMetaKey)); + var connectionString = new SqliteConnectionStringBuilder + { + DataSource = dbPath, + }.ToString(); + using var connection = new SqliteConnection(connectionString); + connection.Open(); + using var command = connection.CreateCommand(); + command.CommandText = """ + SELECT f.generated, + (SELECT COUNT(*) FROM chunks c WHERE c.file_id = f.id AND c.content LIKE '%GeneratedClient%'), + (SELECT COUNT(*) FROM symbols s WHERE s.file_id = f.id), + (SELECT COUNT(*) FROM symbol_references r WHERE r.file_id = f.id), + (SELECT COUNT(*) FROM file_issues i WHERE i.file_id = f.id AND i.kind = @issueKind) + FROM files f + WHERE f.path = @path + """; + command.Parameters.AddWithValue("@issueKind", FileIndexer.GeneratedCodeExtractionSkippedIssueKind); + command.Parameters.AddWithValue("@path", "generated/Client.cs"); + using var reader = command.ExecuteReader(); + Assert.True(reader.Read()); + Assert.Equal(0, reader.GetInt32(0)); + Assert.True(reader.GetInt32(1) > 0); + Assert.Equal(0, reader.GetInt32(2)); + Assert.Equal(0, reader.GetInt32(3)); + Assert.Equal(1, reader.GetInt32(4)); + } + finally + { + TestProjectHelper.DeleteDirectory(fixtureDir); + DeleteSqliteDatabaseFiles(dbPath); + } + } + [Fact] public void ToolsCall_UnknownArgumentName_TruncatesDisplay_Issue3117() { From b0eead089574d8dc0ed8c66e059eb0ec95745828 Mon Sep 17 00:00:00 2001 From: Widthdom Date: Sat, 20 Jun 2026 14:50:59 +0900 Subject: [PATCH 11/18] Reuse unchanged reference prep lines (#3785) --- changelog.d/unreleased/3785.fixed.md | 5 ++-- .../ReferenceExtractor.Preparation.cs | 26 ++++++++++++++++--- 2 files changed, 26 insertions(+), 5 deletions(-) diff --git a/changelog.d/unreleased/3785.fixed.md b/changelog.d/unreleased/3785.fixed.md index 9830513c68..72894b6c43 100644 --- a/changelog.d/unreleased/3785.fixed.md +++ b/changelog.d/unreleased/3785.fixed.md @@ -3,14 +3,15 @@ category: fixed issues: - 3785 affected: + - src/CodeIndex/Indexer/References/ReferenceExtractor.Preparation.cs - src/CodeIndex/Indexer/Scanning/ChunkSplitter.cs - tests/CodeIndex.Tests/ChunkSplitterTests.cs --- ## English -- **Chunk preparation now avoids full line-string materialization (#3785)** — chunk splitting tracks line start offsets and slices only the persisted chunk bodies, reducing duplicate allocations for large but valid files while preserving line ranges and trailing-newline behavior. +- **Chunk and reference preparation now avoid more full line-string materialization (#3785)** — chunk splitting tracks line start offsets and slices only the persisted chunk bodies, and reference preparation reuses structural line arrays when sanitizer output is unchanged, reducing duplicate allocations for large but valid files while preserving line ranges and trailing-newline behavior. ## 日本語 -- **chunk preparation が全行 string 配列の materialization を避けるようになりました (#3785)** — chunk splitting は行開始 offset を追跡し、永続化する chunk 本文だけを切り出すため、大きくても有効なファイルでの重複 allocation を減らしつつ、行範囲と末尾改行の挙動を維持します。 +- **chunk/reference preparation がより多くの全行 string 配列 materialization を避けるようになりました (#3785)** — chunk splitting は行開始 offset を追跡して永続化する chunk 本文だけを切り出し、reference preparation は sanitizer output が変わらない場合に structural line 配列を再利用するため、大きくても有効なファイルでの重複 allocation を減らしつつ、行範囲と末尾改行の挙動を維持します。 diff --git a/src/CodeIndex/Indexer/References/ReferenceExtractor.Preparation.cs b/src/CodeIndex/Indexer/References/ReferenceExtractor.Preparation.cs index aa63dadf03..292086a336 100644 --- a/src/CodeIndex/Indexer/References/ReferenceExtractor.Preparation.cs +++ b/src/CodeIndex/Indexer/References/ReferenceExtractor.Preparation.cs @@ -91,9 +91,7 @@ private static bool TryPrepareReferenceLines( if (language == "python") referenceStructuralLines = MaskPythonFStrings(referenceStructuralLines); - var preparedLines = new string[lines.Length]; - for (var pi = 0; pi < lines.Length; pi++) - preparedLines[pi] = PrepareLine(language, referenceStructuralLines[pi]); + var preparedLines = PrepareReferenceLines(language, referenceStructuralLines); var goImportBlockLines = language == "go" ? GoReferenceExtractor.BuildImportBlockLineMap(lines) : null; @@ -139,6 +137,28 @@ private static bool TryPrepareReferenceLines( return true; } + private static string[] PrepareReferenceLines(string language, string[] referenceStructuralLines) + { + string[]? preparedLines = null; + for (var index = 0; index < referenceStructuralLines.Length; index++) + { + var structuralLine = referenceStructuralLines[index]; + var preparedLine = PrepareLine(language, structuralLine); + if (preparedLines == null) + { + if (string.Equals(preparedLine, structuralLine, StringComparison.Ordinal)) + continue; + + preparedLines = new string[referenceStructuralLines.Length]; + Array.Copy(referenceStructuralLines, preparedLines, index); + } + + preparedLines[index] = preparedLine; + } + + return preparedLines ?? referenceStructuralLines; + } + private static IReadOnlyDictionary>? GroupJsTaggedTemplatesByLine( IReadOnlyList? jsTaggedTemplateHits) { From a9783e3f96a6167cdd733d2f96aad1538aa87176 Mon Sep 17 00:00:00 2001 From: Widthdom Date: Sat, 20 Jun 2026 14:53:35 +0900 Subject: [PATCH 12/18] Stop reference extraction at cap (#3719) --- changelog.d/unreleased/3719.fixed.md | 7 ++- .../Cli/IndexCommandRunner.FullScan.cs | 6 +- .../Cli/IndexCommandRunner.Update.cs | 3 +- .../Languages/ElixirReferenceExtractor.cs | 2 +- .../References/ReferenceExtractionContext.cs | 3 +- .../References/ReferenceExtractor.Core.cs | 17 ++++-- .../ReferenceExtractor.TypeReferences.cs | 4 +- .../Indexer/References/ReferenceExtractor.cs | 58 ++++++++++++++----- .../Support/JvmMethodReferenceExtractor.cs | 2 +- src/CodeIndex/Mcp/McpToolHandlers.cs | 5 +- 10 files changed, 76 insertions(+), 31 deletions(-) diff --git a/changelog.d/unreleased/3719.fixed.md b/changelog.d/unreleased/3719.fixed.md index e7c9fcf103..978732d81a 100644 --- a/changelog.d/unreleased/3719.fixed.md +++ b/changelog.d/unreleased/3719.fixed.md @@ -6,6 +6,9 @@ affected: - src/CodeIndex/Cli/IndexCommandRunner.cs - src/CodeIndex/Cli/IndexCommandRunner.FullScan.cs - src/CodeIndex/Cli/IndexCommandRunner.Update.cs + - src/CodeIndex/Indexer/References/ReferenceExtractionContext.cs + - src/CodeIndex/Indexer/References/ReferenceExtractor.Core.cs + - src/CodeIndex/Indexer/References/ReferenceExtractor.cs - src/CodeIndex/Mcp/McpToolHandlers.cs - src/CodeIndex/Mcp/McpToolDefinitions.cs - src/CodeIndex/Mcp/McpToolArgumentContracts.cs @@ -13,8 +16,8 @@ affected: ## English -- **Reference extraction now has a per-file cap (#3719)** — `cdidx index` and MCP indexing accept `--max-references-per-file` / `maxReferencesPerFile`, preserve text search and symbols, and emit `reference_count_exceeded` file issues when a file's references are skipped. +- **Reference extraction now has a per-file cap (#3719)** — `cdidx index` and MCP indexing accept `--max-references-per-file` / `maxReferencesPerFile`, stop extraction after the diagnostic sentinel count, preserve text search and symbols, and emit `reference_count_exceeded` file issues when a file's references are skipped. ## 日本語 -- **reference 抽出にファイル単位の上限を追加しました (#3719)** — `cdidx index` と MCP index が `--max-references-per-file` / `maxReferencesPerFile` を受け付け、text search と symbols は保持しつつ、references をスキップしたファイルには `reference_count_exceeded` file issue を出します。 +- **reference 抽出にファイル単位の上限を追加しました (#3719)** — `cdidx index` と MCP index が `--max-references-per-file` / `maxReferencesPerFile` を受け付け、診断用 sentinel 件数で抽出を止め、text search と symbols は保持しつつ、references をスキップしたファイルには `reference_count_exceeded` file issue を出します。 diff --git a/src/CodeIndex/Cli/IndexCommandRunner.FullScan.cs b/src/CodeIndex/Cli/IndexCommandRunner.FullScan.cs index 9adbbd56c3..e03ef7fa5b 100644 --- a/src/CodeIndex/Cli/IndexCommandRunner.FullScan.cs +++ b/src/CodeIndex/Cli/IndexCommandRunner.FullScan.cs @@ -1168,7 +1168,8 @@ void StopJsonHeartbeat() symbols, record.Path, record.Lang == "csharp" ? csharpWorkspace.Symbols : null, - extractionCancellationToken); + extractionCancellationToken, + maxReferenceCount: options.MaxReferencesPerFile + 1); regexTimeoutIssue = BuildRegexTimeoutIssue(record.Path, regexTimeouts); } activeJsonExtractionPhases[workerIndex] = FormatIndexPhasePath(record.Path, "validating"); @@ -1482,7 +1483,8 @@ void StopJsonHeartbeat() symbols, record.Path, record.Lang == "csharp" ? csharpWorkspace.Symbols : null, - cancellationToken); + cancellationToken, + maxReferenceCount: options.MaxReferencesPerFile + 1); regexTimeoutIssue = BuildRegexTimeoutIssue(record.Path, regexTimeouts); } else diff --git a/src/CodeIndex/Cli/IndexCommandRunner.Update.cs b/src/CodeIndex/Cli/IndexCommandRunner.Update.cs index 5c3591c498..dd687f3782 100644 --- a/src/CodeIndex/Cli/IndexCommandRunner.Update.cs +++ b/src/CodeIndex/Cli/IndexCommandRunner.Update.cs @@ -748,7 +748,8 @@ void ThrowIfUpdateCancelled() symbols, record.Path, record.Lang == "csharp" ? csharpWorkspace.Symbols : null, - cancellationToken); + cancellationToken, + maxReferenceCount: options.MaxReferencesPerFile + 1); regexTimeoutIssue = BuildRegexTimeoutIssue(record.Path, regexTimeouts); } postExtractionHooks.OnReferencesExtracted(fileContext, references); diff --git a/src/CodeIndex/Indexer/References/Languages/ElixirReferenceExtractor.cs b/src/CodeIndex/Indexer/References/Languages/ElixirReferenceExtractor.cs index 6c3cae88dc..e1c4129b97 100644 --- a/src/CodeIndex/Indexer/References/Languages/ElixirReferenceExtractor.cs +++ b/src/CodeIndex/Indexer/References/Languages/ElixirReferenceExtractor.cs @@ -89,7 +89,7 @@ void AddDefimplTypeReference(Group group, int column) if (!seen.Add(key)) return; - references.Add(new ReferenceRecord + ReferenceExtractor.TryAddReference(references, new ReferenceRecord { FileId = fileId, SymbolName = name, diff --git a/src/CodeIndex/Indexer/References/ReferenceExtractionContext.cs b/src/CodeIndex/Indexer/References/ReferenceExtractionContext.cs index 7f5a870665..c40a0da051 100644 --- a/src/CodeIndex/Indexer/References/ReferenceExtractionContext.cs +++ b/src/CodeIndex/Indexer/References/ReferenceExtractionContext.cs @@ -14,4 +14,5 @@ public sealed record ReferenceExtractionContext( string? Path = null, IReadOnlyList? WorkspaceSymbols = null, string? RequestedLanguage = null, - CancellationToken CancellationToken = default); + CancellationToken CancellationToken = default, + int? MaxReferenceCount = null); diff --git a/src/CodeIndex/Indexer/References/ReferenceExtractor.Core.cs b/src/CodeIndex/Indexer/References/ReferenceExtractor.Core.cs index 3bcce34f29..59c02fd00d 100644 --- a/src/CodeIndex/Indexer/References/ReferenceExtractor.Core.cs +++ b/src/CodeIndex/Indexer/References/ReferenceExtractor.Core.cs @@ -337,7 +337,7 @@ void EmitCSharpBclRegexWithoutTimeoutReferences(List references if (!seen.Add(dedupeKey)) continue; - references.Add(new ReferenceRecord + if (!TryAddReference(references, new ReferenceRecord { FileId = reference.FileId, SymbolName = reference.SymbolName, @@ -348,7 +348,10 @@ void EmitCSharpBclRegexWithoutTimeoutReferences(List references ContainerKind = reference.ContainerKind, ContainerName = reference.ContainerName, IsSelfReference = reference.IsSelfReference, - }); + })) + { + return; + } } } @@ -784,7 +787,7 @@ static string NormalizeCSharpBclRegexQualifiedName(string value) return normalized; } - var references = new List(); + var references = CreateReferenceList(request.MaxReferenceCount); var seen = new HashSet(StringComparer.Ordinal); if (language == "csharp") { @@ -839,6 +842,9 @@ static string NormalizeCSharpBclRegexQualifiedName(string value) for (int i = 0; i < lines.Length; i++) { + if (ReferenceLimitReached(references)) + break; + if ((i & 0x3f) == 0) request.CancellationToken.ThrowIfCancellationRequested(); @@ -2954,7 +2960,7 @@ void AddGradleDslReference(string name, int callIndex) } } - if (language == "csharp") + if (!ReferenceLimitReached(references) && language == "csharp") { CSharpReferenceExtractor.EmitSwitchExpressionTypePatternReferences( lines, @@ -2981,7 +2987,8 @@ void AddGradleDslReference(string name, int callIndex) } ApplyCSharpUsingAliasReferenceNames(references); - EmitCSharpBclRegexWithoutTimeoutReferences(references, seen); + if (!ReferenceLimitReached(references)) + EmitCSharpBclRegexWithoutTimeoutReferences(references, seen); MarkMutualRecursionReferences(references); return references; } diff --git a/src/CodeIndex/Indexer/References/ReferenceExtractor.TypeReferences.cs b/src/CodeIndex/Indexer/References/ReferenceExtractor.TypeReferences.cs index ec892e96df..5c637d1bba 100644 --- a/src/CodeIndex/Indexer/References/ReferenceExtractor.TypeReferences.cs +++ b/src/CodeIndex/Indexer/References/ReferenceExtractor.TypeReferences.cs @@ -579,7 +579,7 @@ internal static void AddTypeReferenceSegment( if (!seen.Add(dedupeKey)) return; - references.Add(new ReferenceRecord + TryAddReference(references, new ReferenceRecord { FileId = fileId, SymbolName = segment, @@ -1664,7 +1664,7 @@ private static void AddChainReference( if (!seen.Add(dedupeKey)) return; - references.Add(new ReferenceRecord + TryAddReference(references, new ReferenceRecord { FileId = fileId, SymbolName = name, diff --git a/src/CodeIndex/Indexer/References/ReferenceExtractor.cs b/src/CodeIndex/Indexer/References/ReferenceExtractor.cs index 9a2ad52639..a9f138cb5b 100644 --- a/src/CodeIndex/Indexer/References/ReferenceExtractor.cs +++ b/src/CodeIndex/Indexer/References/ReferenceExtractor.cs @@ -893,7 +893,8 @@ public static List Extract( IReadOnlyList symbols, string? path = null, IReadOnlyList? workspaceSymbols = null, - CancellationToken cancellationToken = default) + CancellationToken cancellationToken = default, + int? maxReferenceCount = null) { cancellationToken.ThrowIfCancellationRequested(); var requestedLanguage = lang; @@ -916,6 +917,7 @@ public static List Extract( fileId, content, new ExtractionContext(pluginLanguage, path, symbols, workspaceSymbols)) + .Take(maxReferenceCount ?? int.MaxValue) .ToList(); } @@ -929,8 +931,33 @@ public static List Extract( path, workspaceSymbols, requestedLanguage, - cancellationToken)); + cancellationToken, + maxReferenceCount)); } + + private sealed class BoundedReferenceList(int maxReferenceCount) : List + { + internal int MaxReferenceCount { get; } = maxReferenceCount; + } + + internal static List CreateReferenceList(int? maxReferenceCount) + => maxReferenceCount is > 0 + ? new BoundedReferenceList(maxReferenceCount.Value) + : []; + + internal static bool ReferenceLimitReached(List references) + => references is BoundedReferenceList bounded + && bounded.Count >= bounded.MaxReferenceCount; + + internal static bool TryAddReference(List references, ReferenceRecord reference) + { + if (ReferenceLimitReached(references)) + return false; + + references.Add(reference); + return true; + } + private static Dictionary> BuildDefinitionNamesByLine( string language, IReadOnlyList symbols) @@ -1256,7 +1283,7 @@ internal static void AddReference( if (!seen.Add(dedupeKey)) return; - references.Add(new ReferenceRecord + TryAddReference(references, new ReferenceRecord { FileId = fileId, SymbolName = name, @@ -1819,17 +1846,22 @@ internal static void AddTypeReferenceSegments( var dedupeKey = BuildReferenceDedupeKey(fileId, language, lineNumber, column, "type_reference", normalizedSegment, container); if (seen.Add(dedupeKey)) { - references.Add(new ReferenceRecord + if (!TryAddReference( + references, + new ReferenceRecord + { + FileId = fileId, + SymbolName = normalizedSegment, + ReferenceKind = "type_reference", + Line = lineNumber, + Column = column, + Context = context, + ContainerKind = container?.Kind, + ContainerName = container?.Name, + })) { - FileId = fileId, - SymbolName = normalizedSegment, - ReferenceKind = "type_reference", - Line = lineNumber, - Column = column, - Context = context, - ContainerKind = container?.Kind, - ContainerName = container?.Name, - }); + return; + } } } diff --git a/src/CodeIndex/Indexer/References/Support/JvmMethodReferenceExtractor.cs b/src/CodeIndex/Indexer/References/Support/JvmMethodReferenceExtractor.cs index 9063f238ab..6002cf50d5 100644 --- a/src/CodeIndex/Indexer/References/Support/JvmMethodReferenceExtractor.cs +++ b/src/CodeIndex/Indexer/References/Support/JvmMethodReferenceExtractor.cs @@ -227,7 +227,7 @@ private static void AddChainReference( if (!seen.Add(dedupeKey)) return; - references.Add(new ReferenceRecord + ReferenceExtractor.TryAddReference(references, new ReferenceRecord { FileId = fileId, SymbolName = name, diff --git a/src/CodeIndex/Mcp/McpToolHandlers.cs b/src/CodeIndex/Mcp/McpToolHandlers.cs index 9acbf8393f..41f98c89a3 100644 --- a/src/CodeIndex/Mcp/McpToolHandlers.cs +++ b/src/CodeIndex/Mcp/McpToolHandlers.cs @@ -858,7 +858,6 @@ private static bool TryGetExpectedJsonType(string toolName, string argumentName, "limit" or "offset" or "snippetLines" or "maxLineWidth" or "before" or "after" or "focusLine" or "focusColumn" or "focusLength" or "startLine" or "endLine" or "maxHops" or "maxDepth" or "depth" or "parallelism" or "maxFileBytes" or "maxSymbolsPerFile" or "maxReferencesPerFile" or "debounce" or - "maxHops" or "maxDepth" or "depth" or "parallelism" or "maxFileBytes" or "maxSymbolsPerFile" or "maxReferencesPerFile" or "debounce" or "staleAfterSeconds" or "guardWindow" or "maxOutputBytes" or "maxResponseBytes" => "integer", "check" or "excludeTests" or "includeGenerated" or "indexedOnly" or "rawQuery" or "noDedup" or "exactSubstring" or @@ -866,7 +865,6 @@ private static bool TryGetExpectedJsonType(string toolName, string argumentName, "lspCompatible" or "regex" or "withPaths" or "rebuild" or "dryRun" or "dry_run" or "force" or "optimize" or "reverse" or "cycles" or "config" or "logPath" or "updateCheck" or - "optimize" or "reverse" or "cycles" or "config" or "logPath" or "updateCheck" or "rawKinds" or "orderBySize" or "rawBytes" or "byBucket" or "memoryTrace" or "watch" or "estimateOnly" or "listRecipes" => "boolean", "project" or "capability" or "scopes" or "visibility" or "excludeVisibility" or "includeSymbolKind" or "excludeSymbolKind" or @@ -6007,7 +6005,8 @@ static string FormatDiagnosticPath(string projectRoot, string path) symbols, record.Path, record.Lang == "csharp" ? csharpWorkspace.Symbols : null, - requestToken); + requestToken, + maxReferenceCount: maxReferencesPerFile + 1); regexTimeoutIssue = IndexCommandRunner.BuildRegexTimeoutIssue(record.Path, regexTimeouts); } postExtractionHooks.OnReferencesExtracted(fileContext, references); From a632ad3fecef9b7332a18d23bae1253bae7e7db8 Mon Sep 17 00:00:00 2001 From: Widthdom Date: Sat, 20 Jun 2026 14:55:37 +0900 Subject: [PATCH 13/18] Persist null-byte scan issues (#3835) --- changelog.d/unreleased/3835.fixed.md | 7 +++- .../Cli/IndexCommandRunner.FullScan.cs | 15 ++++++- .../Cli/IndexCommandRunner.Update.cs | 39 +++++++++++-------- .../Indexer/Scanning/FileContentLoader.cs | 5 ++- src/CodeIndex/Indexer/Scanning/FileIndexer.cs | 9 ++++- src/CodeIndex/Mcp/McpToolHandlers.cs | 23 +++++------ tests/CodeIndex.Tests/FileIndexerTests.cs | 2 + .../IndexCommandRunnerTests.cs | 32 ++++++++++++++- 8 files changed, 97 insertions(+), 35 deletions(-) diff --git a/changelog.d/unreleased/3835.fixed.md b/changelog.d/unreleased/3835.fixed.md index 66e40df29d..cd90566c09 100644 --- a/changelog.d/unreleased/3835.fixed.md +++ b/changelog.d/unreleased/3835.fixed.md @@ -3,16 +3,19 @@ category: fixed issues: - 3835 affected: + - src/CodeIndex/Cli/IndexCommandRunner.FullScan.cs + - src/CodeIndex/Cli/IndexCommandRunner.Update.cs - src/CodeIndex/Indexer/Scanning/FileContentLoader.cs - src/CodeIndex/Indexer/Scanning/FileIndexer.cs + - src/CodeIndex/Mcp/McpToolHandlers.cs - tests/CodeIndex.Tests/FileIndexerTests.cs - USER_GUIDE.md --- ## English -- **File scanning diagnostics now cover more edge cases (#3835)** — Git LFS pointer files keep pointer-identity checksums, BOM-less UTF-16 heuristic decoding reports the chosen encoding, NUL-byte binary skips include the byte offset, out-of-policy symlink warnings no longer expose absolute external targets, and extensionless `env -S` shebangs resolve their interpreter. +- **File scanning diagnostics now cover more edge cases (#3835)** — Git LFS pointer files keep pointer-identity checksums, BOM-less UTF-16 heuristic decoding reports the chosen encoding, NUL-byte binary skips include the byte offset and persist a `null_byte` file issue during indexing, out-of-policy symlink warnings no longer expose absolute external targets, and extensionless `env -S` shebangs resolve their interpreter. ## 日本語 -- **ファイル走査診断がより多くの端ケースを扱うようになりました (#3835)** — Git LFS pointer ファイルは pointer identity に基づく checksum を保持し、BOM なし UTF-16 heuristic decode は選択した encoding を報告し、NUL バイトによる binary skip は byte offset を含め、ポリシー外 symlink の warning は外部絶対 target を露出せず、拡張子なし `env -S` shebang は interpreter を解決するようになりました。 +- **ファイル走査診断がより多くの端ケースを扱うようになりました (#3835)** — Git LFS pointer ファイルは pointer identity に基づく checksum を保持し、BOM なし UTF-16 heuristic decode は選択した encoding を報告し、NUL バイトによる binary skip は byte offset を含めて index 時にも `null_byte` file issue として永続化し、ポリシー外 symlink の warning は外部絶対 target を露出せず、拡張子なし `env -S` shebang は interpreter を解決するようになりました。 diff --git a/src/CodeIndex/Cli/IndexCommandRunner.FullScan.cs b/src/CodeIndex/Cli/IndexCommandRunner.FullScan.cs index e03ef7fa5b..1a23282610 100644 --- a/src/CodeIndex/Cli/IndexCommandRunner.FullScan.cs +++ b/src/CodeIndex/Cli/IndexCommandRunner.FullScan.cs @@ -56,6 +56,15 @@ private static FileIssue BuildReferenceCountExceededIssue(string path, int refer Message = $"Reference extraction produced {referenceCount:N0} references, exceeding the --max-references-per-file limit of {maxReferencesPerFile:N0}; references were not indexed for this file. Exclude the generated/pathological file or raise --max-references-per-file if this is expected.", }; + internal static FileIssue BuildNullByteIssue(FileIndexer.BinaryFileSkippedException ex) => + new() + { + Path = ex.RelativePath, + Kind = "null_byte", + Line = 0, + Message = ex.Message, + }; + internal static FileIssue? BuildRegexTimeoutIssue(string path, BoundedRegex.RegexTimeoutCaptureScope capture) { if (!capture.HasTimeouts) @@ -1193,7 +1202,11 @@ void StopJsonHeartbeat() } catch (FileIndexer.BinaryFileSkippedException ex) { - extractionResults.Add(FullScanFileWorkItem.Skipped(filePath, ex.Message), extractionCancellationToken); + var record = indexer.BuildSkippedFileRecord(filePath); + var issue = BuildNullByteIssue(ex); + extractionResults.Add( + FullScanFileWorkItem.Success(filePath, record, string.Empty, [], ex.Message, [], [], [], [issue]), + extractionCancellationToken); } catch (FileIndexer.FileTooLargeSkippedException ex) { diff --git a/src/CodeIndex/Cli/IndexCommandRunner.Update.cs b/src/CodeIndex/Cli/IndexCommandRunner.Update.cs index dd687f3782..ada4a58f09 100644 --- a/src/CodeIndex/Cli/IndexCommandRunner.Update.cs +++ b/src/CodeIndex/Cli/IndexCommandRunner.Update.cs @@ -783,8 +783,11 @@ void ThrowIfUpdateCancelled() } catch (Exception ex) { - if (ex is FileIndexer.BinaryFileSkippedException) + if (ex is FileIndexer.BinaryFileSkippedException binaryFile) { + if (fileBatchMarked) + writer.ClearBatchInProgress(); + warnings++; warningList.Add(new CliJsonMessage(relPath, ex.Message)); if (!options.Json && !options.Quiet) @@ -794,22 +797,24 @@ void ThrowIfUpdateCancelled() ResumeUpdateSpinnerAfterConsoleWrite(); } - if (writer.HasFileAtPath(dbPath)) - { - DemoteReadinessOnce(); - using var deleteTxn = writer.BeginTransaction(); - if (writer.DeleteFileByPath(dbPath)) - { - WriteProjectRootOnce(); - deleteTxn.Commit(); - removed++; - ftsMutated = true; - } - } - else - { - skipped++; - } + DemoteReadinessOnce(); + writer.MarkBatchInProgress(); + using var txn = writer.BeginTransaction(); + var skippedRecord = indexer.BuildSkippedFileRecord(absPath); + writer.PurgeStaleFilesSharingChecksum(projectRoot, skippedRecord.Path, skippedRecord.Checksum); + if (projectRootWritten) + writer.PurgeStaleFilesSharingDirectoryAndStem(projectRoot, skippedRecord.Path); + WriteProjectRootOnce(); + var fileId = writer.UpsertFile(skippedRecord); + writer.InsertChunks([]); + writer.InsertSymbols([]); + writer.InsertReferences([]); + writer.InsertIssues(fileId, [BuildNullByteIssue(binaryFile)]); + writer.ClearBatchInProgress(); + txn.Commit(); + + updated++; + ftsMutated = true; continue; } diff --git a/src/CodeIndex/Indexer/Scanning/FileContentLoader.cs b/src/CodeIndex/Indexer/Scanning/FileContentLoader.cs index dd19793032..40898b2c6c 100644 --- a/src/CodeIndex/Indexer/Scanning/FileContentLoader.cs +++ b/src/CodeIndex/Indexer/Scanning/FileContentLoader.cs @@ -108,7 +108,10 @@ internal LoadedFileContent Load( var isUtf16Encoded = TryDetectUtf16Encoding(bytes, allowHeuristic: true, out var utf16BigEndian, out var hasUtf16Bom); if (!isUtf16Encoded && TryFindIndexBlockingNullByte(bytes, out var nullByteOffset)) - throw new FileIndexer.BinaryFileSkippedException($"{relativePath}: binary file skipped because it contains NULL byte at byte offset {nullByteOffset}"); + throw new FileIndexer.BinaryFileSkippedException( + relativePath, + nullByteOffset, + $"{relativePath}: binary file skipped because it contains NULL byte at byte offset {nullByteOffset}"); if (isUtf16Encoded) { diff --git a/src/CodeIndex/Indexer/Scanning/FileIndexer.cs b/src/CodeIndex/Indexer/Scanning/FileIndexer.cs index 56766e423e..8bc88fe3a0 100644 --- a/src/CodeIndex/Indexer/Scanning/FileIndexer.cs +++ b/src/CodeIndex/Indexer/Scanning/FileIndexer.cs @@ -4239,7 +4239,14 @@ internal static bool TryDetectUtf16Encoding( out bool hasBom) => FileContentLoader.TryDetectUtf16Encoding(rawBytes, allowHeuristic, out bigEndian, out hasBom); - internal sealed class BinaryFileSkippedException(string message) : InvalidOperationException(message); + internal sealed class BinaryFileSkippedException( + string relativePath, + long nullByteOffset, + string message) : InvalidOperationException(message) + { + public string RelativePath { get; } = relativePath; + public long NullByteOffset { get; } = nullByteOffset; + } internal sealed class FileTooLargeSkippedException( string relativePath, diff --git a/src/CodeIndex/Mcp/McpToolHandlers.cs b/src/CodeIndex/Mcp/McpToolHandlers.cs index 41f98c89a3..02d286c982 100644 --- a/src/CodeIndex/Mcp/McpToolHandlers.cs +++ b/src/CodeIndex/Mcp/McpToolHandlers.cs @@ -6031,23 +6031,24 @@ static string FormatDiagnosticPath(string projectRoot, string path) txn.Commit(); McpIndexFileCommittedForTesting?.Invoke(record.Path); } - catch (FileIndexer.BinaryFileSkippedException) + catch (FileIndexer.BinaryFileSkippedException ex) { try { - var relativePath = FileIndexer.NormalizePathSeparators(Path.GetRelativePath(projectPath, filePath)); - if (writer.HasFileAtPath(relativePath)) - { - using var txn = writer.BeginTransaction(); - writer.DeleteFileByPath(relativePath); - WriteProjectRootOnce(); - txn.Commit(); - } + var skippedRecord = indexer.BuildSkippedFileRecord(filePath); + using var txn = writer.BeginTransaction(); + var fileId = writer.UpsertFile(skippedRecord); + writer.InsertChunks([]); + writer.InsertSymbols([]); + writer.InsertReferences([]); + writer.InsertIssues(fileId, [IndexCommandRunner.BuildNullByteIssue(ex)]); + WriteProjectRootOnce(); + txn.Commit(); } - catch (Exception ex) + catch (Exception cleanupEx) { errors++; - failures.Add(BuildIndexFileFailure(projectPath, filePath, ex, "delete_skipped_binary")); + failures.Add(BuildIndexFileFailure(projectPath, filePath, cleanupEx, "record_skipped_binary")); } } catch (Exception ex) when (ex is FileNotFoundException or DirectoryNotFoundException) diff --git a/tests/CodeIndex.Tests/FileIndexerTests.cs b/tests/CodeIndex.Tests/FileIndexerTests.cs index 77dabd01b1..9ef6419196 100644 --- a/tests/CodeIndex.Tests/FileIndexerTests.cs +++ b/tests/CodeIndex.Tests/FileIndexerTests.cs @@ -4986,6 +4986,8 @@ public void BuildRecord_NonUtf16NullByte_ThrowsOffsetDiagnostic() var ex = Assert.Throws(() => indexer.BuildRecordWithRawBytes(filePath)); Assert.Contains("NULL byte at byte offset 6", ex.Message, StringComparison.Ordinal); + Assert.Equal("binary.cs", ex.RelativePath); + Assert.Equal(6, ex.NullByteOffset); } finally { diff --git a/tests/CodeIndex.Tests/IndexCommandRunnerTests.cs b/tests/CodeIndex.Tests/IndexCommandRunnerTests.cs index 72c18275b9..1d404350fb 100644 --- a/tests/CodeIndex.Tests/IndexCommandRunnerTests.cs +++ b/tests/CodeIndex.Tests/IndexCommandRunnerTests.cs @@ -6419,7 +6419,7 @@ public void Run_Rebuild_CancelledAfterReadinessDemotion_PreservesExistingIndex() } [Fact] - public void Run_Rebuild_WhenIndexedFileBecomesBinary_RemovesStaleRow() + public void Run_Rebuild_WhenIndexedFileBecomesBinary_PersistsNullByteIssue() { var projectRoot = CreateTempProject(); try @@ -6438,7 +6438,9 @@ public void Run_Rebuild_WhenIndexedFileBecomesBinary_RemovesStaleRow() var rebuildExitCode = IndexCommandRunner.Run([projectRoot, "--rebuild", "--yes", "--json"], _jsonOptions); Assert.Equal(CommandExitCodes.Success, rebuildExitCode); - Assert.DoesNotContain("app.py", ReadIndexedPaths(dbPath)); + Assert.Contains("app.py", ReadIndexedPaths(dbPath)); + var issue = Assert.Single(ReadFileIssues(dbPath).Where(issue => issue.Path == "app.py" && issue.Kind == "null_byte")); + Assert.Contains("byte offset 0", issue.Message, StringComparison.Ordinal); } finally { @@ -11859,6 +11861,32 @@ private static HashSet ReadIndexedPaths(string dbPath) .ToHashSet(StringComparer.Ordinal); } + private static List ReadFileIssues(string dbPath) + { + using var connection = OpenNonPoolingConnection(dbPath); + connection.Open(); + using var command = connection.CreateCommand(); + command.CommandText = """ + SELECT f.path, i.kind, i.line, i.message + FROM file_issues i + JOIN files f ON f.id = i.file_id + ORDER BY f.path, i.kind, i.line, i.message + """; + using var reader = command.ExecuteReader(); + var issues = new List(); + while (reader.Read()) + { + issues.Add(new FileIssue + { + Path = reader.GetString(0), + Kind = reader.GetString(1), + Line = reader.GetInt32(2), + Message = reader.GetString(3), + }); + } + return issues; + } + private static Dictionary ReadSymbolKindCounts(string dbPath) { using var connection = OpenNonPoolingConnection(dbPath); From 4ffc9050891e8629cfed1e875dd5e014494b4c0b Mon Sep 17 00:00:00 2001 From: Widthdom Date: Sat, 20 Jun 2026 14:56:01 +0900 Subject: [PATCH 14/18] Capture symbol regex timeouts (#3800) --- changelog.d/unreleased/3800.fixed.md | 5 +- .../Cli/IndexCommandRunner.FullScan.cs | 90 +++++++++++++++---- .../Cli/IndexCommandRunner.Update.cs | 25 ++++-- .../Indexer/Symbols/SymbolExtractionWorker.cs | 24 ++++- src/CodeIndex/Mcp/McpToolHandlers.cs | 15 +++- 5 files changed, 125 insertions(+), 34 deletions(-) diff --git a/changelog.d/unreleased/3800.fixed.md b/changelog.d/unreleased/3800.fixed.md index 3a53106b84..92e2e16fbe 100644 --- a/changelog.d/unreleased/3800.fixed.md +++ b/changelog.d/unreleased/3800.fixed.md @@ -4,6 +4,7 @@ issues: - 3800 affected: - src/CodeIndex/Indexer/BoundedRegex.cs + - src/CodeIndex/Indexer/Symbols/SymbolExtractionWorker.cs - src/CodeIndex/Cli/IndexCommandRunner.FullScan.cs - src/CodeIndex/Cli/IndexCommandRunner.Update.cs - src/CodeIndex/Mcp/McpToolHandlers.cs @@ -11,8 +12,8 @@ affected: ## English -- **Regex timeout fallbacks are now visible in file issues (#3800)** — reference extraction records bounded `regex_timeout` diagnostics with hashed pattern metadata when a guarded regex falls back to no-match behavior, so skipped matches no longer disappear silently. +- **Regex timeout fallbacks are now visible in file issues (#3800)** — symbol and reference extraction record bounded `regex_timeout` diagnostics with hashed pattern metadata when a guarded regex falls back to no-match behavior, so skipped matches no longer disappear silently. ## 日本語 -- **regex timeout fallback が file issue に表示されるようになりました (#3800)** — reference 抽出中に保護された regex が no-match fallback へ切り替わった場合、ハッシュ化した pattern メタデータ付きの bounded な `regex_timeout` 診断を記録し、match の欠落が無言で起きないようにしました。 +- **regex timeout fallback が file issue に表示されるようになりました (#3800)** — symbol 抽出と reference 抽出中に保護された regex が no-match fallback へ切り替わった場合、ハッシュ化した pattern メタデータ付きの bounded な `regex_timeout` 診断を記録し、match の欠落が無言で起きないようにしました。 diff --git a/src/CodeIndex/Cli/IndexCommandRunner.FullScan.cs b/src/CodeIndex/Cli/IndexCommandRunner.FullScan.cs index 1a23282610..a9b8bd8844 100644 --- a/src/CodeIndex/Cli/IndexCommandRunner.FullScan.cs +++ b/src/CodeIndex/Cli/IndexCommandRunner.FullScan.cs @@ -65,22 +65,38 @@ internal static FileIssue BuildNullByteIssue(FileIndexer.BinaryFileSkippedExcept Message = ex.Message, }; - internal static FileIssue? BuildRegexTimeoutIssue(string path, BoundedRegex.RegexTimeoutCaptureScope capture) + internal static FileIssue? BuildRegexTimeoutIssue(string path, BoundedRegex.RegexTimeoutCaptureScope capture) => + BuildRegexTimeoutIssue( + path, + capture.Language, + capture.PatternFamily, + capture.TimeoutCount, + capture.Diagnostics, + capture.DiagnosticsTruncated); + + internal static FileIssue? BuildRegexTimeoutIssue( + string path, + string? language, + string patternFamily, + int timeoutCount, + IReadOnlyList diagnostics, + bool diagnosticsTruncated) { - if (!capture.HasTimeouts) + if (timeoutCount <= 0) return null; - var samples = capture.Diagnostics.Count == 0 + var normalizedLanguage = string.IsNullOrWhiteSpace(language) ? "unknown" : language; + var samples = diagnostics.Count == 0 ? "none" - : string.Join(", ", capture.Diagnostics.Select(static diagnostic => + : string.Join(", ", diagnostics.Select(static diagnostic => $"{diagnostic.Operation}:{diagnostic.PatternHash} len={diagnostic.PatternLength} timeout={diagnostic.TimeoutMs:0.###}ms")); - var truncationSuffix = capture.DiagnosticsTruncated ? "; additional timeout diagnostics omitted" : string.Empty; + var truncationSuffix = diagnosticsTruncated ? "; additional timeout diagnostics omitted" : string.Empty; return new FileIssue { Path = path, Kind = "regex_timeout", Line = 0, - Message = $"Regex timeout fallback occurred during {capture.PatternFamily} for language {capture.Language} ({capture.TimeoutCount:N0} timeout(s); samples {samples}{truncationSuffix}); extraction used a safe no-match fallback and may be incomplete for this file.", + Message = $"Regex timeout fallback occurred during {patternFamily} for language {normalizedLanguage} ({timeoutCount:N0} timeout(s); samples {samples}{truncationSuffix}); extraction used a safe no-match fallback and may be incomplete for this file.", }; } @@ -166,19 +182,26 @@ private static void ThrowIfFullScanExtractionStalled( throw new IndexExtractionStalledException(filesProcessed, filesTotal, timeout, activePath); } - private static List ExtractSymbolsWithStallTimeout( + private sealed record SymbolExtractionResult(List Symbols, FileIssue? RegexTimeoutIssue); + + private static SymbolExtractionResult ExtractSymbolsWithStallTimeout( long fileId, string? lang, string content, string filePath, string projectRoot, + string issuePath, string phasePath, SymbolExtractionWorkerClient worker, CancellationToken cancellationToken) { var timeout = IndexExtractionStallTimeoutForTesting?.Invoke() ?? IndexExtractionStallTimeout; if (timeout <= TimeSpan.Zero) - return SymbolExtractor.Extract(fileId, lang, content, filePath, projectRoot, cancellationToken); + { + using var regexTimeouts = BoundedRegex.CaptureTimeouts(lang, "symbol_extraction"); + var symbols = SymbolExtractor.Extract(fileId, lang, content, filePath, projectRoot, cancellationToken); + return new SymbolExtractionResult(symbols, BuildRegexTimeoutIssue(issuePath, regexTimeouts)); + } cancellationToken.ThrowIfCancellationRequested(); var result = worker.Invoke(fileId, lang, content, filePath, projectRoot, timeout, cancellationToken); @@ -188,7 +211,14 @@ private static List ExtractSymbolsWithStallTimeout( if (!result.Success) throw new InvalidOperationException(result.WorkerError ?? "isolated symbol extraction worker failed."); - return result.Symbols ?? []; + var regexTimeoutIssue = BuildRegexTimeoutIssue( + issuePath, + lang, + "symbol_extraction", + result.RegexTimeoutCount, + result.RegexTimeoutDiagnostics ?? [], + result.RegexTimeoutDiagnosticsTruncated); + return new SymbolExtractionResult(result.Symbols ?? [], regexTimeoutIssue); } private static string CollapseLineBreaks(string value) @@ -1143,25 +1173,31 @@ void StopJsonHeartbeat() continue; } activeJsonExtractionPhases[workerIndex] = FormatIndexPhasePath(record.Path, "symbols"); - symbols = ExtractSymbolsWithStallTimeout( + var symbolExtraction = ExtractSymbolsWithStallTimeout( 0, record.Lang, content, filePath, Path.GetFullPath(options.ProjectPath!), + record.Path, activeJsonExtractionPhases[workerIndex], workerSymbolExtractionWorker, extractionCancellationToken); + symbols = symbolExtraction.Symbols; + var symbolRegexTimeoutIssue = symbolExtraction.RegexTimeoutIssue; if (symbols.Count > options.MaxSymbolsPerFile) { var issue = BuildSymbolCountExceededIssue(record.Path, symbols.Count, options.MaxSymbolsPerFile); + IReadOnlyList capIssues = symbolRegexTimeoutIssue == null + ? [issue] + : AppendIssue([symbolRegexTimeoutIssue], issue); extractionResults.Add( - FullScanFileWorkItem.Success(filePath, record, string.Empty, rawBytes, issue.Message, [], [], [], [issue]), + FullScanFileWorkItem.Success(filePath, record, string.Empty, rawBytes, issue.Message, [], [], [], capIssues), extractionCancellationToken); continue; } SymbolExtractor.ApplyFamilyScope(symbols, indexer.GetFamilyScopeKey(filePath, record.Lang)); - FileIssue? regexTimeoutIssue = null; + FileIssue? referenceRegexTimeoutIssue = null; if (options.SymbolsOnly) { references = []; @@ -1179,12 +1215,14 @@ void StopJsonHeartbeat() record.Lang == "csharp" ? csharpWorkspace.Symbols : null, extractionCancellationToken, maxReferenceCount: options.MaxReferencesPerFile + 1); - regexTimeoutIssue = BuildRegexTimeoutIssue(record.Path, regexTimeouts); + referenceRegexTimeoutIssue = BuildRegexTimeoutIssue(record.Path, regexTimeouts); } activeJsonExtractionPhases[workerIndex] = FormatIndexPhasePath(record.Path, "validating"); issues = FileIndexer.ValidateContent(record.Path, rawBytes, content, record.Lang); - if (regexTimeoutIssue != null) - issues = AppendIssue(issues, regexTimeoutIssue); + if (symbolRegexTimeoutIssue != null) + issues = AppendIssue(issues, symbolRegexTimeoutIssue); + if (referenceRegexTimeoutIssue != null) + issues = AppendIssue(issues, referenceRegexTimeoutIssue); if (references.Count > options.MaxReferencesPerFile) { var issue = BuildReferenceCountExceededIssue(record.Path, references.Count, options.MaxReferencesPerFile); @@ -1416,23 +1454,29 @@ void StopJsonHeartbeat() continue; } currentJsonIndexFile = FormatIndexPhasePath(record.Path, "symbols"); + SymbolExtractionResult? symbolExtraction = null; var symbols = item.Symbols == null - ? ExtractSymbolsWithStallTimeout( + ? (symbolExtraction = ExtractSymbolsWithStallTimeout( fileId, record.Lang, item.Content!, item.FilePath, Path.GetFullPath(options.ProjectPath!), + record.Path, currentJsonIndexFile, mainSymbolExtractionWorker, - cancellationToken) + cancellationToken)).Symbols : ReassignSymbolFileIds(item.Symbols, fileId); + var symbolRegexTimeoutIssue = symbolExtraction?.RegexTimeoutIssue; if (symbols.Count > options.MaxSymbolsPerFile) { var issue = BuildSymbolCountExceededIssue(record.Path, symbols.Count, options.MaxSymbolsPerFile); + IReadOnlyList capIssues = symbolRegexTimeoutIssue == null + ? [issue] + : AppendIssue([symbolRegexTimeoutIssue], issue); writer.InsertSymbols([]); writer.InsertReferences([]); - writer.InsertIssues(fileId, [issue]); + writer.InsertIssues(fileId, capIssues); if (options.Verbose) WriteIndexVerboseStatus($" [SKIP] {record.Path} ({issue.Message})"); txn.Commit(); @@ -1457,9 +1501,12 @@ void StopJsonHeartbeat() if (symbols.Count > options.MaxSymbolsPerFile) { var issue = BuildSymbolCountExceededIssue(record.Path, symbols.Count, options.MaxSymbolsPerFile); + IReadOnlyList capIssues = symbolRegexTimeoutIssue == null + ? [issue] + : AppendIssue([symbolRegexTimeoutIssue], issue); writer.InsertSymbols([]); writer.InsertReferences([]); - writer.InsertIssues(fileId, [issue]); + writer.InsertIssues(fileId, capIssues); if (options.Verbose) WriteIndexVerboseStatus($" [SKIP] {record.Path} ({issue.Message})"); txn.Commit(); @@ -1477,6 +1524,11 @@ void StopJsonHeartbeat() writer.InsertChunks(chunks); FileIndexer.ValidateSymbolLineRanges(record, symbols); writer.InsertSymbols(symbols); + if (symbolRegexTimeoutIssue != null) + { + var baseIssues = item.Issues ?? FileIndexer.ValidateContent(record.Path, item.RawBytes!, item.Content!, record.Lang); + item = item with { Issues = AppendIssue(baseIssues, symbolRegexTimeoutIssue) }; + } currentJsonIndexFile = FormatIndexPhasePath(record.Path, "references"); IReadOnlyList references; if (options.SymbolsOnly) diff --git a/src/CodeIndex/Cli/IndexCommandRunner.Update.cs b/src/CodeIndex/Cli/IndexCommandRunner.Update.cs index ada4a58f09..7f7269ce8b 100644 --- a/src/CodeIndex/Cli/IndexCommandRunner.Update.cs +++ b/src/CodeIndex/Cli/IndexCommandRunner.Update.cs @@ -692,21 +692,27 @@ void ThrowIfUpdateCancelled() continue; } currentUpdatePath = FormatIndexPhasePath(relPath, "symbols"); - var symbols = ExtractSymbolsWithStallTimeout( + var symbolExtraction = ExtractSymbolsWithStallTimeout( fileId, record.Lang, content, absPath, Path.GetFullPath(options.ProjectPath!), + record.Path, currentUpdatePath, symbolExtractionWorker, cancellationToken); + var symbols = symbolExtraction.Symbols; + var symbolRegexTimeoutIssue = symbolExtraction.RegexTimeoutIssue; if (symbols.Count > options.MaxSymbolsPerFile) { var issue = BuildSymbolCountExceededIssue(record.Path, symbols.Count, options.MaxSymbolsPerFile); + IReadOnlyList capIssues = symbolRegexTimeoutIssue == null + ? [issue] + : AppendIssue([symbolRegexTimeoutIssue], issue); writer.InsertSymbols([]); writer.InsertReferences([]); - writer.InsertIssues(fileId, [issue]); + writer.InsertIssues(fileId, capIssues); writer.ClearBatchInProgress(); txn.Commit(); fileBatchMarked = false; @@ -722,9 +728,12 @@ void ThrowIfUpdateCancelled() if (symbols.Count > options.MaxSymbolsPerFile) { var issue = BuildSymbolCountExceededIssue(record.Path, symbols.Count, options.MaxSymbolsPerFile); + IReadOnlyList capIssues = symbolRegexTimeoutIssue == null + ? [issue] + : AppendIssue([symbolRegexTimeoutIssue], issue); writer.InsertSymbols([]); writer.InsertReferences([]); - writer.InsertIssues(fileId, [issue]); + writer.InsertIssues(fileId, capIssues); writer.ClearBatchInProgress(); txn.Commit(); fileBatchMarked = false; @@ -738,7 +747,7 @@ void ThrowIfUpdateCancelled() writer.InsertSymbols(symbols); currentUpdatePath = FormatIndexPhasePath(relPath, "references"); List references; - FileIssue? regexTimeoutIssue; + FileIssue? referenceRegexTimeoutIssue; using (var regexTimeouts = BoundedRegex.CaptureTimeouts(record.Lang, "reference_extraction")) { references = ReferenceExtractor.Extract( @@ -750,7 +759,7 @@ void ThrowIfUpdateCancelled() record.Lang == "csharp" ? csharpWorkspace.Symbols : null, cancellationToken, maxReferenceCount: options.MaxReferencesPerFile + 1); - regexTimeoutIssue = BuildRegexTimeoutIssue(record.Path, regexTimeouts); + referenceRegexTimeoutIssue = BuildRegexTimeoutIssue(record.Path, regexTimeouts); } postExtractionHooks.OnReferencesExtracted(fileContext, references); FileIssue? referenceCapIssue = null; @@ -763,8 +772,10 @@ void ThrowIfUpdateCancelled() // Validate content for encoding issues / エンコーディング問題を検証 currentUpdatePath = FormatIndexPhasePath(relPath, "validating"); IReadOnlyList issues = FileIndexer.ValidateContent(record.Path, rawBytes, content, record.Lang); - if (regexTimeoutIssue != null) - issues = AppendIssue(issues, regexTimeoutIssue); + if (symbolRegexTimeoutIssue != null) + issues = AppendIssue(issues, symbolRegexTimeoutIssue); + if (referenceRegexTimeoutIssue != null) + issues = AppendIssue(issues, referenceRegexTimeoutIssue); if (referenceCapIssue != null) issues = AppendIssue(issues, referenceCapIssue); writer.InsertIssues(fileId, issues); diff --git a/src/CodeIndex/Indexer/Symbols/SymbolExtractionWorker.cs b/src/CodeIndex/Indexer/Symbols/SymbolExtractionWorker.cs index fdc7ae79f1..7a32773094 100644 --- a/src/CodeIndex/Indexer/Symbols/SymbolExtractionWorker.cs +++ b/src/CodeIndex/Indexer/Symbols/SymbolExtractionWorker.cs @@ -15,7 +15,10 @@ internal sealed record SymbolExtractionWorkerResult( bool TimedOut, string? WorkerError, long DurationMs, - List? Symbols); + List? Symbols, + int RegexTimeoutCount = 0, + List? RegexTimeoutDiagnostics = null, + bool RegexTimeoutDiagnosticsTruncated = false); internal sealed class SymbolExtractionWorkerClient : IDisposable { @@ -141,7 +144,10 @@ internal SymbolExtractionWorkerResult Invoke( TimedOut: false, WorkerError: null, DurationMs: stopwatch.ElapsedMilliseconds, - Symbols: response.Symbols); + Symbols: response.Symbols, + RegexTimeoutCount: response.RegexTimeoutCount, + RegexTimeoutDiagnostics: response.RegexTimeoutDiagnostics, + RegexTimeoutDiagnosticsTruncated: response.RegexTimeoutDiagnosticsTruncated); } } @@ -583,6 +589,7 @@ private static WorkerResponse InvokeInsideWorker(WorkerRequest request, WorkerOp Console.SetError(capturedError); WriteConsoleOutputForTestingIfRequested(options); DelayForTestingIfRequested(options, cancellationToken); + using var regexTimeouts = BoundedRegex.CaptureTimeouts(request.Lang, "symbol_extraction"); var symbols = SymbolExtractor.Extract( request.FileId, request.Lang, @@ -590,7 +597,13 @@ private static WorkerResponse InvokeInsideWorker(WorkerRequest request, WorkerOp request.FilePath, request.ProjectRoot, cancellationToken); - return new WorkerResponse(symbols, null, capturedError.GetCapturedText()); + return new WorkerResponse( + symbols, + null, + capturedError.GetCapturedText(), + regexTimeouts.TimeoutCount, + regexTimeouts.Diagnostics.ToList(), + regexTimeouts.DiagnosticsTruncated); } catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) { @@ -768,7 +781,10 @@ internal sealed record WorkerRequest( internal sealed record WorkerResponse( List? Symbols, string? WorkerError, - string? CapturedStderr); + string? CapturedStderr, + int RegexTimeoutCount = 0, + List? RegexTimeoutDiagnostics = null, + bool RegexTimeoutDiagnosticsTruncated = false); private sealed record WorkerOptions( int MaxProtocolLineCharacters, diff --git a/src/CodeIndex/Mcp/McpToolHandlers.cs b/src/CodeIndex/Mcp/McpToolHandlers.cs index 02d286c982..d4286b1549 100644 --- a/src/CodeIndex/Mcp/McpToolHandlers.cs +++ b/src/CodeIndex/Mcp/McpToolHandlers.cs @@ -5978,7 +5978,13 @@ static string FormatDiagnosticPath(string projectRoot, string path) McpIndexFileCommittedForTesting?.Invoke(record.Path); continue; } - var symbols = SymbolExtractor.Extract(fileId, record.Lang, content, filePath, projectPath, requestToken).ToList(); + List symbols; + FileIssue? symbolRegexTimeoutIssue; + using (var regexTimeouts = BoundedRegex.CaptureTimeouts(record.Lang, "symbol_extraction")) + { + symbols = SymbolExtractor.Extract(fileId, record.Lang, content, filePath, projectPath, requestToken).ToList(); + symbolRegexTimeoutIssue = IndexCommandRunner.BuildRegexTimeoutIssue(record.Path, regexTimeouts); + } SymbolExtractor.ApplyFamilyScope(symbols, indexer.GetFamilyScopeKey(filePath, record.Lang)); var fileContext = new FileContext(projectPath, record.Path, filePath, record.Lang); postExtractionHooks.OnSymbolsExtracted(fileContext, symbols); @@ -5986,9 +5992,12 @@ static string FormatDiagnosticPath(string projectRoot, string path) if (symbols.Count > maxSymbolsPerFile) { var issue = BuildMcpSymbolCountExceededIssue(record.Path, symbols.Count, maxSymbolsPerFile); + IReadOnlyList capIssues = symbolRegexTimeoutIssue == null + ? [issue] + : IndexCommandRunner.AppendIssue([symbolRegexTimeoutIssue], issue); writer.InsertSymbols([]); writer.InsertReferences([]); - writer.InsertIssues(fileId, [issue]); + writer.InsertIssues(fileId, capIssues); } else { @@ -6020,6 +6029,8 @@ static string FormatDiagnosticPath(string projectRoot, string path) // Keep MCP index parity with CLI index: persist file-level validation issues too. // MCPインデックスもCLIインデックスと同等に、ファイル検証issueを保存する。 IReadOnlyList issues = FileIndexer.ValidateContent(record.Path, rawBytes, content, record.Lang); + if (symbolRegexTimeoutIssue != null) + issues = IndexCommandRunner.AppendIssue(issues, symbolRegexTimeoutIssue); if (regexTimeoutIssue != null) issues = IndexCommandRunner.AppendIssue(issues, regexTimeoutIssue); if (referenceCapIssue != null) From 415e395e4b1fa2077b150883dc0f60c2e6cb76bb Mon Sep 17 00:00:00 2001 From: Widthdom Date: Sat, 20 Jun 2026 15:12:35 +0900 Subject: [PATCH 15/18] Align null-byte issue tests (#3835) --- tests/CodeIndex.Tests/FileIndexerTests.cs | 19 ++++++++++++++++++- .../IndexCommandRunnerTests.cs | 10 +++++++--- 2 files changed, 25 insertions(+), 4 deletions(-) diff --git a/tests/CodeIndex.Tests/FileIndexerTests.cs b/tests/CodeIndex.Tests/FileIndexerTests.cs index 9ef6419196..e4f46ff9bc 100644 --- a/tests/CodeIndex.Tests/FileIndexerTests.cs +++ b/tests/CodeIndex.Tests/FileIndexerTests.cs @@ -4379,7 +4379,8 @@ public void IndexFilesUpdate_UsesOriginalUnicodePathForIoAndNfcPathForDb() File.WriteAllBytes(Path.Combine(tempDir, nfdPath), [0, 1, 2, 3]); Assert.Equal(CommandExitCodes.Success, IndexCommandRunner.Run([tempDir, "--files", nfdPath, "--json", "--quiet"], jsonOptions)); - Assert.False(HasIndexedFile(dbPath, "Caf\u00e9.cs")); + Assert.True(HasIndexedFile(dbPath, "Caf\u00e9.cs")); + Assert.True(HasFileIssue(dbPath, "Caf\u00e9.cs", "null_byte")); } finally { @@ -5950,4 +5951,20 @@ private static bool HasIndexedFile(string dbPath, string filePath) cmd.Parameters.AddWithValue("@path", filePath); return cmd.ExecuteScalar() != null; } + + private static bool HasFileIssue(string dbPath, string filePath, string kind) + { + using var db = new DbContext(dbPath); + using var cmd = db.Connection.CreateCommand(); + cmd.CommandText = """ + SELECT 1 + FROM file_issues i + JOIN files f ON f.id = i.file_id + WHERE f.path = @path + AND i.kind = @kind + """; + cmd.Parameters.AddWithValue("@path", filePath); + cmd.Parameters.AddWithValue("@kind", kind); + return cmd.ExecuteScalar() != null; + } } diff --git a/tests/CodeIndex.Tests/IndexCommandRunnerTests.cs b/tests/CodeIndex.Tests/IndexCommandRunnerTests.cs index 1d404350fb..7eab1708eb 100644 --- a/tests/CodeIndex.Tests/IndexCommandRunnerTests.cs +++ b/tests/CodeIndex.Tests/IndexCommandRunnerTests.cs @@ -1043,7 +1043,7 @@ public void LoadScanCheckpoint_JsonDepthOutsideBounds_ReturnsEmpty() } [Fact] - public void Run_NullByteFile_SkipsWithoutPersistingPartialRows() + public void Run_NullByteFile_PersistsNullByteIssueWithoutPartialRows() { var projectRoot = CreateTempProject(); try @@ -1059,13 +1059,17 @@ public void Run_NullByteFile_SkipsWithoutPersistingPartialRows() Assert.Equal(CommandExitCodes.Success, exitCode); Assert.Equal(0, json.GetProperty("summary").GetProperty("errors").GetInt32()); - Assert.Equal(1, json.GetProperty("summary").GetProperty("warnings").GetInt32()); + Assert.Equal(0, json.GetProperty("summary").GetProperty("warnings").GetInt32()); var dbPath = Path.Combine(projectRoot, ".cdidx", "codeindex.db"); - Assert.Equal(0, CountRows(dbPath, "files")); + Assert.Equal(1, CountRows(dbPath, "files")); Assert.Equal(0, CountRows(dbPath, "chunks")); Assert.Equal(0, CountRows(dbPath, "symbols")); Assert.Equal(0, CountRows(dbPath, "symbol_references")); + var issue = Assert.Single(ReadFileIssues(dbPath, "null_byte")); + Assert.Equal("binary.cs", issue.Path); + Assert.Equal(0, issue.Line); + Assert.Contains("byte offset", issue.Message); } finally { From e82bef1cd6269da04f4ff1ce099d3a5b8e92b566 Mon Sep 17 00:00:00 2001 From: Widthdom Date: Sat, 20 Jun 2026 16:19:11 +0900 Subject: [PATCH 16/18] Forward project marker budget warnings (#3762) --- changelog.d/unreleased/3762.fixed.md | 5 ++-- src/CodeIndex/Cli/IndexCommandRunner.cs | 1 + .../IndexCommandRunnerTests.cs | 30 +++++++++++++++++++ 3 files changed, 34 insertions(+), 2 deletions(-) diff --git a/changelog.d/unreleased/3762.fixed.md b/changelog.d/unreleased/3762.fixed.md index 8692995026..03ff31d4aa 100644 --- a/changelog.d/unreleased/3762.fixed.md +++ b/changelog.d/unreleased/3762.fixed.md @@ -3,6 +3,7 @@ category: fixed issues: - 3762 affected: + - src/CodeIndex/Cli/IndexCommandRunner.cs - src/CodeIndex/Indexer/Scanning/FileIndexer.cs - tests/CodeIndex.Tests/FileIndexerTests.cs - tests/CodeIndex.Tests/IndexCommandRunnerTests.cs @@ -10,8 +11,8 @@ affected: ## English -- **Project discovery budget diagnostics now name the budget that fired (#3762)** — project marker fingerprint truncation records directory/marker-file budget reasons in warnings and fingerprint salt, and solution fallback discovery has explicit directory-budget coverage. +- **Project discovery budget diagnostics now name the budget that fired (#3762)** — project marker fingerprint truncation records directory/marker-file budget reasons in warnings and fingerprint salt, forwards truncation warnings through CLI JSON output, and solution fallback discovery has explicit directory-budget coverage. ## 日本語 -- **project discovery budget 診断が発火した budget 名を示すようになりました (#3762)** — project marker fingerprint の truncation は directory / marker-file budget の理由を warning と fingerprint salt に記録し、solution fallback discovery には directory budget の明示テストを追加しました。 +- **project discovery budget 診断が発火した budget 名を示すようになりました (#3762)** — project marker fingerprint の truncation は directory / marker-file budget の理由を warning と fingerprint salt に記録し、truncation warning を CLI JSON 出力にも転送し、solution fallback discovery には directory budget の明示テストを追加しました。 diff --git a/src/CodeIndex/Cli/IndexCommandRunner.cs b/src/CodeIndex/Cli/IndexCommandRunner.cs index fd614a31a6..b49f1bd976 100644 --- a/src/CodeIndex/Cli/IndexCommandRunner.cs +++ b/src/CodeIndex/Cli/IndexCommandRunner.cs @@ -666,6 +666,7 @@ private static int AddProjectMarkerFingerprintWarnings( private static bool IsProjectMarkerFingerprintWarning(FileIndexer.ScanError warning) => warning.Message.StartsWith("Project marker discovery skipped", StringComparison.Ordinal) + || warning.Message.StartsWith("Project marker discovery truncated", StringComparison.Ordinal) || warning.Message.StartsWith("Skipped .gitmodules", StringComparison.Ordinal); private static void RestampHotspotFamilyTrustForUpdate( diff --git a/tests/CodeIndex.Tests/IndexCommandRunnerTests.cs b/tests/CodeIndex.Tests/IndexCommandRunnerTests.cs index 7eab1708eb..398299ebe5 100644 --- a/tests/CodeIndex.Tests/IndexCommandRunnerTests.cs +++ b/tests/CodeIndex.Tests/IndexCommandRunnerTests.cs @@ -1078,6 +1078,36 @@ public void Run_NullByteFile_PersistsNullByteIssueWithoutPartialRows() } } + [Fact] + public void Run_FullScanJson_ProjectMarkerBudgetWarningIncludesTruncatedWarning() + { + var projectRoot = CreateTempProject(); + var previousEnumerator = FileIndexer.EnumerateProjectMarkerDirectoriesForTesting; + try + { + var childDir = Path.Combine(projectRoot, "nested"); + Directory.CreateDirectory(childDir); + File.WriteAllText(Path.Combine(projectRoot, "App.cs"), "public class App { }\n"); + FileIndexer.EnumerateProjectMarkerDirectoriesForTesting = + _ => Enumerable.Repeat(childDir, 8192); + + var (exitCode, json, _) = RunAndCaptureJsonWithStderr([projectRoot, "--json"]); + + Assert.Equal(CommandExitCodes.Success, exitCode); + Assert.Contains( + json.GetProperty("warnings").EnumerateArray(), + warning => + warning.GetProperty("message").GetString()!.Contains("Project marker discovery truncated", StringComparison.Ordinal) + && warning.GetProperty("message").GetString()!.Contains("directory budget", StringComparison.Ordinal)); + } + finally + { + FileIndexer.EnumerateProjectMarkerDirectoriesForTesting = previousEnumerator; + SqliteConnection.ClearAllPools(); + DeleteDirectory(projectRoot); + } + } + [Fact] public void Run_FileAboveMaxFileBytes_PersistsFileTooLargeIssue() { From 763a6e948b02f9cac1bd30d5144bf52d665c52cc Mon Sep 17 00:00:00 2001 From: Widthdom Date: Sat, 20 Jun 2026 16:56:35 +0900 Subject: [PATCH 17/18] Cover MCP null-byte indexing (#3835) --- changelog.d/unreleased/3835.fixed.md | 1 + tests/CodeIndex.Tests/McpServerTests.cs | 48 +++++++++++++++++++++++++ 2 files changed, 49 insertions(+) diff --git a/changelog.d/unreleased/3835.fixed.md b/changelog.d/unreleased/3835.fixed.md index cd90566c09..8d46ac8486 100644 --- a/changelog.d/unreleased/3835.fixed.md +++ b/changelog.d/unreleased/3835.fixed.md @@ -9,6 +9,7 @@ affected: - src/CodeIndex/Indexer/Scanning/FileIndexer.cs - src/CodeIndex/Mcp/McpToolHandlers.cs - tests/CodeIndex.Tests/FileIndexerTests.cs + - tests/CodeIndex.Tests/McpServerTests.cs - USER_GUIDE.md --- diff --git a/tests/CodeIndex.Tests/McpServerTests.cs b/tests/CodeIndex.Tests/McpServerTests.cs index d96f5916af..8b5e247613 100644 --- a/tests/CodeIndex.Tests/McpServerTests.cs +++ b/tests/CodeIndex.Tests/McpServerTests.cs @@ -11385,6 +11385,54 @@ public void ToolsCall_Index_SuccessfulNoOpBackfillsMissingIndexedProjectRootMeta } } + [Fact] + public void ToolsCall_Index_NullByteFilePersistsNullByteIssue_Issue3835() + { + var fixtureDir = Path.Combine(Path.GetFullPath("."), $"mcp_index_null_byte_{Guid.NewGuid():N}"); + var dbPath = Path.Combine(Path.GetTempPath(), $"cdidx_mcp_index_null_byte_{Guid.NewGuid():N}.db"); + try + { + Directory.CreateDirectory(fixtureDir); + var prefix = Encoding.UTF8.GetBytes("public class Polluted { public void Run() { } }\n"); + var bytes = new byte[prefix.Length + 1]; + Array.Copy(prefix, bytes, prefix.Length); + bytes[^1] = 0; + File.WriteAllBytes(Path.Combine(fixtureDir, "binary.cs"), bytes); + + using var server = new McpServer(dbPath, ConsoleUi.LoadVersion()); + var request = new JsonObject + { + ["jsonrpc"] = "2.0", + ["id"] = 1, + ["method"] = "tools/call", + ["params"] = new JsonObject + { + ["name"] = "index", + ["arguments"] = new JsonObject + { + ["path"] = fixtureDir + } + } + }; + var response = server.HandleMessage(request)!; + + Assert.False(response["result"]!["isError"]?.GetValue() ?? false, response.ToJsonString()); + using var db = new DbContext(dbPath); + db.TryMigrateForRead(); + var reader = new DbReader(db.Connection, db.IsReadOnly); + var issue = Assert.Single(reader.GetIssues("null_byte")); + Assert.Equal("binary.cs", issue.Path); + Assert.Equal(0, issue.Line); + Assert.Contains("byte offset", issue.Message); + } + finally + { + if (Directory.Exists(fixtureDir)) + TestProjectHelper.DeleteDirectory(fixtureDir); + DeleteFileRobust(dbPath); + } + } + [Fact] public void ToolsCall_BackfillFold_StampsFoldReady() { From b4d85d98b043bbd598256f6bc30c1070dbc0cbef Mon Sep 17 00:00:00 2001 From: Widthdom Date: Sat, 20 Jun 2026 17:08:23 +0900 Subject: [PATCH 18/18] Cover MCP reference cap indexing (#3719) --- changelog.d/unreleased/3719.fixed.md | 1 + tests/CodeIndex.Tests/McpServerTests.cs | 66 +++++++++++++++++++++++++ 2 files changed, 67 insertions(+) diff --git a/changelog.d/unreleased/3719.fixed.md b/changelog.d/unreleased/3719.fixed.md index 978732d81a..c6779cae3a 100644 --- a/changelog.d/unreleased/3719.fixed.md +++ b/changelog.d/unreleased/3719.fixed.md @@ -12,6 +12,7 @@ affected: - src/CodeIndex/Mcp/McpToolHandlers.cs - src/CodeIndex/Mcp/McpToolDefinitions.cs - src/CodeIndex/Mcp/McpToolArgumentContracts.cs + - tests/CodeIndex.Tests/McpServerTests.cs --- ## English diff --git a/tests/CodeIndex.Tests/McpServerTests.cs b/tests/CodeIndex.Tests/McpServerTests.cs index 8b5e247613..e502526a15 100644 --- a/tests/CodeIndex.Tests/McpServerTests.cs +++ b/tests/CodeIndex.Tests/McpServerTests.cs @@ -270,6 +270,29 @@ private static JsonNode CallIndex(McpServer server, string path, Action " Target.Ping();")); + return $$""" +namespace DenseReferences; + +public static class Target +{ + public static void Ping() + { + } +} + +public sealed class Caller +{ + public void Run() + { +{{calls}} + } +} +"""; + } + private static Dictionary ReadSymbolKindCounts(string dbPath) { var counts = new Dictionary(StringComparer.Ordinal); @@ -11433,6 +11456,49 @@ public void ToolsCall_Index_NullByteFilePersistsNullByteIssue_Issue3835() } } + [Fact] + public void ToolsCall_Index_MaxReferencesPerFilePersistsReferenceCountExceededIssue_Issue3719() + { + var fixtureDir = Path.Combine(Path.GetFullPath("."), $"mcp_index_reference_cap_{Guid.NewGuid():N}"); + var dbPath = Path.Combine(Path.GetTempPath(), $"cdidx_mcp_index_reference_cap_{Guid.NewGuid():N}.db"); + try + { + Directory.CreateDirectory(fixtureDir); + File.WriteAllText(Path.Combine(fixtureDir, "DenseReferences.cs"), BuildDenseReferenceCSharpSource(8)); + + using var server = new McpServer(dbPath, ConsoleUi.LoadVersion()); + var response = CallIndex(server, fixtureDir, args => args["maxReferencesPerFile"] = 2); + + Assert.False(response["result"]?["isError"]?.GetValue() ?? false, response.ToJsonString()); + using var db = new DbContext(dbPath); + db.TryMigrateForRead(); + var reader = new DbReader(db.Connection, db.IsReadOnly); + var issue = Assert.Single(reader.GetIssues("reference_count_exceeded")); + Assert.Equal("DenseReferences.cs", issue.Path); + Assert.Equal(0, issue.Line); + Assert.Contains("maxReferencesPerFile", issue.Message); + + using var command = db.Connection.CreateCommand(); + command.CommandText = """ + SELECT + (SELECT COUNT(*) FROM chunks), + (SELECT COUNT(*) FROM symbols), + (SELECT COUNT(*) FROM symbol_references) + """; + using var row = command.ExecuteReader(); + Assert.True(row.Read()); + Assert.True(row.GetInt64(0) > 0); + Assert.True(row.GetInt64(1) > 0); + Assert.Equal(0, row.GetInt64(2)); + } + finally + { + if (Directory.Exists(fixtureDir)) + TestProjectHelper.DeleteDirectory(fixtureDir); + DeleteSqliteDatabaseFiles(dbPath); + } + } + [Fact] public void ToolsCall_BackfillFold_StampsFoldReady() {