diff --git a/README.md b/README.md index 18c5db2ed1..f02f8a7e9a 100644 --- a/README.md +++ b/README.md @@ -73,6 +73,7 @@ After the first command, use these cues and follow-up commands: | Edits or branch switches | Refresh incrementally with `--files`, `--commits`, or `--changed-between ` instead of rebuilding. See [Quick Start](USER_GUIDE.md#quick-start) and [Incremental update reliability](USER_GUIDE.md#incremental-update-reliability). | | Intentional rebuilds | Interactive terminals ask before deleting the DB. Scripts and CI must pass `--yes` or `--force`. | | Long-lived DB compaction | Run `cdidx optimize` or `cdidx index --optimize` to compact FTS5 segments immediately. Incremental refreshes also optimize opportunistically. | +| Pathological generated files | `--max-symbols-per-file ` skips indexing file content, symbols, and references when one file emits too many symbols, leaving a `symbol_count_exceeded` issue for audit. | | Maintenance rollback | Run `cdidx db checkpoint ` before risky DB maintenance and `cdidx db restore ` to roll back. `backfill-fold` creates an automatic checkpoint unless `--no-checkpoint` is passed. | | Permission or I/O scan errors | `cdidx` records the scan error, continues other directories, and writes `.cdidx/scan-checkpoint.json` so same-HEAD retries can skip completed directories. | @@ -337,6 +338,7 @@ extractor fixture を確認できます。詳細は | 編集後やブランチ切り替え後 | 再構築ではなく `--files`、`--commits`、`--changed-between ` で差分更新します。詳細は [クイックスタート](USER_GUIDE.md#クイックスタート) と [インクリメンタル更新の信頼性](USER_GUIDE.md#インクリメンタル更新の信頼性) を参照してください。 | | 意図的な再構築 | interactive terminal では既存 DB 削除前に確認を求めます。script / CI では `--yes` または `--force` が必要です。 | | 長期間使っている DB の compact | `cdidx optimize` または `cdidx index --optimize` で FTS5 segment をすぐに compact できます。差分更新中も必要に応じて自動 optimize します。 | +| 病的な generated file | 1 ファイルが過剰な symbol を出す場合、`--max-symbols-per-file ` は file content / symbols / references を保存せず、監査用の `symbol_count_exceeded` issue を残します。 | | 保守作業の rollback | risky な DB 保守の前に `cdidx db checkpoint `、戻す場合は `cdidx db restore ` を使います。`backfill-fold` は `--no-checkpoint` を渡さない限り自動 checkpoint を作成します。 | | 権限や I/O の scan error | `cdidx` は scan error を記録し、他のディレクトリの走査を続けます。同じ HEAD の再実行では `.cdidx/scan-checkpoint.json` により成功済みディレクトリを読み飛ばせます。 | diff --git a/changelog.d/unreleased/1604.fixed.md b/changelog.d/unreleased/1604.fixed.md new file mode 100644 index 0000000000..2a411ebfe0 --- /dev/null +++ b/changelog.d/unreleased/1604.fixed.md @@ -0,0 +1,24 @@ +--- +category: fixed +issues: + - 1604 +affected: + - src/CodeIndex/Cli/IndexCommandRunner.cs + - src/CodeIndex/Cli/IndexCommandRunner.Parse.cs + - src/CodeIndex/Cli/IndexCommandRunner.FullScan.cs + - src/CodeIndex/Cli/IndexCommandRunner.Update.cs + - src/CodeIndex/Cli/ConsoleUi.cs + - src/CodeIndex/Cli/IndexWatchRunner.cs + - src/CodeIndex/Cli/CliFlagSchema.cs + - src/CodeIndex/Database/DbWriter.cs + - README.md + - tests/CodeIndex.Tests/IndexCommandRunnerTests.cs +--- + +## English + +- **Indexing now caps per-file symbol output (#1604)** - `cdidx index --max-symbols-per-file N` skips file content, symbols, and references when one file emits too many symbols, leaving a `symbol_count_exceeded` issue instead of ballooning the database. + +## 日本語 + +- **index が 1 ファイルあたりの symbol 出力数を制限できるようになりました (#1604)** - `cdidx index --max-symbols-per-file N` は 1 ファイルが過剰な symbol を出す場合に file content / symbols / references を保存せず、DB を肥大化させる代わりに `symbol_count_exceeded` issue を残します。 diff --git a/changelog.d/unreleased/2744.fixed.md b/changelog.d/unreleased/2744.fixed.md new file mode 100644 index 0000000000..4f77f4144f --- /dev/null +++ b/changelog.d/unreleased/2744.fixed.md @@ -0,0 +1,15 @@ +--- +category: fixed +issues: + - 2744 +affected: + - src/CodeIndex/Cli/IndexCommandRunner.FullScan.cs +--- + +## English + +- **Extraction-stall diagnostics now point at bounded recovery for pathological symbol output (#2744)** - `E013_INDEX_EXTRACTION_STALLED` keeps the active file/phase and now suggests using `--max-symbols-per-file` or excluding the reported file when full refresh stalls on symbol extraction. + +## 日本語 + +- **extraction stall 診断が病的な symbol 出力への bounded recovery を示すようになりました (#2744)** - `E013_INDEX_EXTRACTION_STALLED` は active file / phase を保持したまま、full refresh が symbol extraction で止まる場合に `--max-symbols-per-file` または該当ファイルの除外を案内します。 diff --git a/src/CodeIndex/Cli/CliFlagSchema.cs b/src/CodeIndex/Cli/CliFlagSchema.cs index 25a301eb49..19f7bf3082 100644 --- a/src/CodeIndex/Cli/CliFlagSchema.cs +++ b/src/CodeIndex/Cli/CliFlagSchema.cs @@ -283,6 +283,7 @@ private static IReadOnlyList BuildAll() new() { Name = "--force", Description = "Bypass the per-database index lock", Commands = Set("index") }, new() { Name = "--duration-format", ValuePlaceholder = "", Description = "Index elapsed time display format", Commands = Set("index") }, new() { Name = "--max-file-bytes", ValuePlaceholder = "", Description = "Override the per-file indexing size limit", Commands = Set("index") }, + new() { Name = "--max-symbols-per-file", ValuePlaceholder = "", Description = "Skip file content, symbols, and references when one file emits too many symbols", Commands = Set("index") }, new() { Name = "--parallelism", ValuePlaceholder = "", Description = "Full-scan extraction worker count (default: CPU count capped at 16; also honors CDIDX_INDEX_PARALLELISM)", Commands = Set("index") }, new() { Name = "--memory-trace", Description = "Include phase memory samples in index JSON output", Commands = Set("index") }, new() { Name = "--commits", ValuePlaceholder = "", Description = "Update files changed in given git commits", Commands = Set("index") }, diff --git a/src/CodeIndex/Cli/ConsoleUi.cs b/src/CodeIndex/Cli/ConsoleUi.cs index 9b4cb30f8a..bd55501b1a 100644 --- a/src/CodeIndex/Cli/ConsoleUi.cs +++ b/src/CodeIndex/Cli/ConsoleUi.cs @@ -68,7 +68,7 @@ public static class ConsoleUi private static readonly (string Command, string Usage)[] CommandUsageLines = [ - ("index", "cdidx index [--db ] [--rebuild] [--optimize] [--verbose] [--dry-run] [--force] [--quiet] [--json] [--memory-trace] [--duration-format ] [--notify ] [--max-file-bytes ] [--follow-symlinks ] [--include-symbol-kind [,]] [--exclude-symbol-kind [,]] [--watch [--debounce ]]"), + ("index", "cdidx index [--db ] [--rebuild] [--optimize] [--verbose] [--dry-run] [--force] [--quiet] [--json] [--memory-trace] [--duration-format ] [--notify ] [--max-file-bytes ] [--max-symbols-per-file ] [--follow-symlinks ] [--include-symbol-kind [,]] [--exclude-symbol-kind [,]] [--watch [--debounce ]]"), ("hooks", "cdidx hooks [--project ] [--force] [--json]"), ("backfill-fold", "cdidx backfill-fold [--db ] [--dry-run] [--no-checkpoint] [--json]"), ("optimize", "cdidx optimize [--db ] [--json]"), @@ -875,6 +875,7 @@ private static void PrintFlagReference(Action WriteHelpLine) Console.WriteLine(" --duration-format Index elapsed time format: `auto` (default), `seconds`, or `hms`; JSON keeps raw elapsed_ms"); WriteHelpLine(" --notify Long index completion signal: auto, bell, osc9, desktop, or none (also honors CDIDX_NOTIFY; quiet/json suppress it)"); WriteHelpLine(" --max-file-bytes Index only files up to this size (default: 4MiB; also honors CDIDX_MAX_FILE_BYTES; accepts K/M/G suffixes)"); + WriteHelpLine(" --max-symbols-per-file Skip file content, symbols, and references when one file emits too many symbols (default: 5000)"); WriteHelpLine(" --parallelism Full-scan extraction workers (default: CPU count capped at 16; also honors CDIDX_INDEX_PARALLELISM)"); WriteHelpLine(" --follow-symlinks Directory symlink policy: none (default), internal, or all"); WriteHelpLine(" --include-symbol-kind [,] Keep only matching symbol kinds during indexing"); diff --git a/src/CodeIndex/Cli/IndexCommandRunner.FullScan.cs b/src/CodeIndex/Cli/IndexCommandRunner.FullScan.cs index 2000e45cbc..01346ef5c2 100644 --- a/src/CodeIndex/Cli/IndexCommandRunner.FullScan.cs +++ b/src/CodeIndex/Cli/IndexCommandRunner.FullScan.cs @@ -33,6 +33,15 @@ private static string FormatExtractionStalledMessage(IndexExtractionStalledExcep return $"Index extraction made no progress for {ConsoleUi.FormatDuration(ex.Timeout)}.{pathSuffix}"; } + private static FileIssue BuildSymbolCountExceededIssue(string path, int symbolCount, int maxSymbolsPerFile) => + new() + { + Path = path, + Kind = "symbol_count_exceeded", + Line = 0, + Message = $"Symbol extraction produced {symbolCount:N0} symbols, exceeding the --max-symbols-per-file limit of {maxSymbolsPerFile:N0}; file content, symbols, and references were not indexed. Exclude the generated/pathological file or raise --max-symbols-per-file if this is expected.", + }; + internal static string FormatIndexPhasePath(string path, string phase) => $"{path} ({phase})"; @@ -191,7 +200,7 @@ private static int WriteExtractionStalledResult(bool json, JsonSerializerOptions jsonOptions, $"Index extraction made no progress for {ConsoleUi.FormatDuration(ex.Timeout)} ({ex.FilesProcessed:N0}{totalSuffix} files processed).{pathSuffix}", CommandExitCodes.CancelledBySignal, - "Rerun with `--verbose` to inspect progress, lower `--parallelism`, or file a bug with the reported active phase.", + "Rerun with `--verbose` to inspect progress, lower `--parallelism`, exclude the reported file, or lower `--max-symbols-per-file` to skip pathological symbol output.", CommandErrorCodes.IndexExtractionStalled); } @@ -865,6 +874,14 @@ void StopJsonHeartbeat() Path.GetFullPath(options.ProjectPath!), activeJsonExtractionPhases[workerIndex], extractionCancellationToken); + if (symbols.Count > options.MaxSymbolsPerFile) + { + var issue = BuildSymbolCountExceededIssue(record.Path, symbols.Count, options.MaxSymbolsPerFile); + extractionResults.Add( + FullScanFileWorkItem.Success(filePath, record, string.Empty, rawBytes, issue.Message, [], [], [], [issue]), + extractionCancellationToken); + continue; + } SymbolExtractor.ApplyFamilyScope(symbols, indexer.GetFamilyScopeKey(filePath, record.Lang)); activeJsonExtractionPhases[workerIndex] = FormatIndexPhasePath(record.Path, "references"); references = ReferenceExtractor.Extract( @@ -1025,6 +1042,14 @@ void StopJsonHeartbeat() && AllowReuseWithCurrentHotspotFamilyTrust(record.Lang, hotspotFamilyTrustMatchesCurrent)); } if (existingId != null) + { + if (writer.CountSymbolsForFile(existingId.Value) > options.MaxSymbolsPerFile + || writer.HasIssueForFile(existingId.Value, "symbol_count_exceeded")) + { + existingId = null; + } + } + if (existingId != null) { writer.PurgeStaleFilesSharingChecksum(projectRoot, record.Path, record.Checksum); skipped++; @@ -1058,7 +1083,6 @@ void StopJsonHeartbeat() var chunks = item.Chunks == null ? ChunkSplitter.Split(fileId, item.Content!) : ReassignChunkFileIds(item.Chunks, fileId); - writer.InsertChunks(chunks); currentJsonIndexFile = FormatIndexPhasePath(record.Path, "symbols"); var symbols = item.Symbols == null ? ExtractSymbolsWithStallTimeout( @@ -1070,6 +1094,26 @@ void StopJsonHeartbeat() currentJsonIndexFile, cancellationToken) : ReassignSymbolFileIds(item.Symbols, fileId); + if (symbols.Count > options.MaxSymbolsPerFile) + { + var issue = BuildSymbolCountExceededIssue(record.Path, symbols.Count, options.MaxSymbolsPerFile); + writer.InsertSymbols([]); + writer.InsertReferences([]); + writer.InsertIssues(fileId, [issue]); + if (options.Verbose) + WriteIndexVerboseStatus($" [SKIP] {record.Path} ({issue.Message})"); + txn.Commit(); + processed++; + if (!options.Json && !options.Quiet) + { + PauseIndexSpinnerForConsoleWrite(); + ConsoleUi.PrintProgress(processed, files.Count); + ResumeIndexSpinnerAfterConsoleWrite(); + } + ReportJsonIndexProgressIfNeeded(); + currentJsonIndexFile = null; + continue; + } if (item.Symbols == null) SymbolExtractor.ApplyFamilyScope(symbols, indexer.GetFamilyScopeKey(item.FilePath, record.Lang)); var fileContext = new FileContext(projectRoot, record.Path, item.FilePath, record.Lang); @@ -1077,6 +1121,27 @@ void StopJsonHeartbeat() postExtractionHooks.OnSymbolsExtracted(fileContext, mutableSymbols); symbolsDroppedByKindFilter += options.SymbolKindFilter.Apply(mutableSymbols); symbols = (IReadOnlyList)mutableSymbols; + if (symbols.Count > options.MaxSymbolsPerFile) + { + var issue = BuildSymbolCountExceededIssue(record.Path, symbols.Count, options.MaxSymbolsPerFile); + writer.InsertSymbols([]); + writer.InsertReferences([]); + writer.InsertIssues(fileId, [issue]); + if (options.Verbose) + WriteIndexVerboseStatus($" [SKIP] {record.Path} ({issue.Message})"); + txn.Commit(); + processed++; + if (!options.Json && !options.Quiet) + { + PauseIndexSpinnerForConsoleWrite(); + ConsoleUi.PrintProgress(processed, files.Count); + ResumeIndexSpinnerAfterConsoleWrite(); + } + ReportJsonIndexProgressIfNeeded(); + currentJsonIndexFile = null; + continue; + } + writer.InsertChunks(chunks); FileIndexer.ValidateSymbolLineRanges(record, symbols); writer.InsertSymbols(symbols); currentJsonIndexFile = FormatIndexPhasePath(record.Path, "references"); diff --git a/src/CodeIndex/Cli/IndexCommandRunner.Parse.cs b/src/CodeIndex/Cli/IndexCommandRunner.Parse.cs index 4b96f65ade..ef7ef415f5 100644 --- a/src/CodeIndex/Cli/IndexCommandRunner.Parse.cs +++ b/src/CodeIndex/Cli/IndexCommandRunner.Parse.cs @@ -13,7 +13,7 @@ public static partial class IndexCommandRunner private static readonly string[] AcceptedIndexFlags = [ "--db", "--data-dir", "--rebuild", "--verbose", "--json", "--quiet", "--dry-run", "--force", - "--yes", "--watch", "--debounce", "--duration-format", "--max-file-bytes", + "--yes", "--watch", "--debounce", "--duration-format", "--max-file-bytes", "--max-symbols-per-file", "--notify", "--parallelism", "--memory-trace", "--follow-symlinks", "--commits", "--changed-between", "--files", "--solution", "--project", @@ -43,6 +43,7 @@ public static IndexCommandOptions ParseArgs(string[] args) var durationFormat = DurationOutputFormat.Auto; var notifyMode = ReadCompletionNotificationModeFromEnvironment(); long? maxFileSizeBytes = ReadMaxFileSizeBytesFromEnvironment(); + var maxSymbolsPerFile = DefaultMaxSymbolsPerFile; var parallelism = ReadIndexParallelismFromEnvironment(); var symlinkPolicy = FileIndexer.SymlinkPolicy.None; string? easterEgg = null; @@ -151,6 +152,12 @@ public static IndexCommandOptions ParseArgs(string[] args) case var option when option.StartsWith("--max-file-bytes=", StringComparison.Ordinal): maxFileSizeBytes = ParseMaxFileBytes(option["--max-file-bytes=".Length..], maxFileSizeBytes); break; + case "--max-symbols-per-file" when i + 1 < args.Length: + maxSymbolsPerFile = ParseMaxSymbolsPerFile(args[++i], maxSymbolsPerFile, "--max-symbols-per-file"); + break; + case var option when option.StartsWith("--max-symbols-per-file=", StringComparison.Ordinal): + maxSymbolsPerFile = ParseMaxSymbolsPerFile(option["--max-symbols-per-file=".Length..], maxSymbolsPerFile, "--max-symbols-per-file"); + break; case "--parallelism" when i + 1 < args.Length: parallelism = ParseIndexParallelism(args[++i], parallelism, "--parallelism"); break; @@ -309,6 +316,7 @@ public static IndexCommandOptions ParseArgs(string[] args) DurationFormat = durationFormat, NotifyMode = notifyMode, MaxFileSizeBytes = maxFileSizeBytes, + MaxSymbolsPerFile = maxSymbolsPerFile, Parallelism = parallelism, SymlinkPolicy = symlinkPolicy, SymbolKindFilter = SymbolKindFilter.Create(includeSymbolKinds, excludeSymbolKinds, symbolKindFilterError), @@ -407,6 +415,15 @@ private static int ParseIndexParallelism(string value, int fallback, string sour return fallback; } + private static int ParseMaxSymbolsPerFile(string value, int fallback, string source) + { + if (int.TryParse(value, System.Globalization.NumberStyles.Integer, System.Globalization.CultureInfo.InvariantCulture, out var parsed) && parsed > 0) + return parsed; + + Console.Error.WriteLine($"Warning: invalid {source} value '{value}' (ignored; use a positive integer) / 不正な {source} 値 '{value}'（無視。正の整数を指定）"); + return fallback; + } + private static DurationOutputFormat ParseDurationFormat(string value, DurationOutputFormat fallback) { return value.Trim().ToLowerInvariant() switch diff --git a/src/CodeIndex/Cli/IndexCommandRunner.Update.cs b/src/CodeIndex/Cli/IndexCommandRunner.Update.cs index 6a8722111a..d83aacdff5 100644 --- a/src/CodeIndex/Cli/IndexCommandRunner.Update.cs +++ b/src/CodeIndex/Cli/IndexCommandRunner.Update.cs @@ -701,6 +701,14 @@ void ThrowIfUpdateCancelled() && (statReusableLanguage != "csharp" || !csharpWorkspace.HasStaticInterfaceContracts) && (statReusableLanguage != "sql" || sqlGraphContractMatchesCurrent)); if (statMatchedId != null) + { + if (writer.CountSymbolsForFile(statMatchedId.Value) > options.MaxSymbolsPerFile + || writer.HasIssueForFile(statMatchedId.Value, "symbol_count_exceeded")) + { + statMatchedId = null; + } + } + if (statMatchedId != null) { skipped++; if (options.Verbose && !options.Json && !options.Quiet) @@ -735,6 +743,14 @@ void ThrowIfUpdateCancelled() && (record.Lang != "csharp" || !csharpWorkspace.HasStaticInterfaceContracts) && (record.Lang != "sql" || sqlGraphContractMatchesCurrent)); if (existingId != null) + { + if (writer.CountSymbolsForFile(existingId.Value) > options.MaxSymbolsPerFile + || writer.HasIssueForFile(existingId.Value, "symbol_count_exceeded")) + { + existingId = null; + } + } + if (existingId != null) { using var purgeTxn = writer.BeginTransaction(); var purged = writer.PurgeStaleFilesSharingChecksum(projectRoot, record.Path, record.Checksum) @@ -772,7 +788,6 @@ void ThrowIfUpdateCancelled() var fileId = writer.UpsertFile(record); currentUpdatePath = FormatIndexPhasePath(relPath, "chunking"); var chunks = ChunkSplitter.Split(fileId, content); - writer.InsertChunks(chunks); currentUpdatePath = FormatIndexPhasePath(relPath, "symbols"); var symbols = ExtractSymbolsWithStallTimeout( fileId, @@ -782,10 +797,39 @@ void ThrowIfUpdateCancelled() Path.GetFullPath(options.ProjectPath!), currentUpdatePath, cancellationToken); + if (symbols.Count > options.MaxSymbolsPerFile) + { + var issue = BuildSymbolCountExceededIssue(record.Path, symbols.Count, options.MaxSymbolsPerFile); + writer.InsertSymbols([]); + writer.InsertReferences([]); + writer.InsertIssues(fileId, [issue]); + writer.ClearBatchInProgress(); + txn.Commit(); + fileBatchMarked = false; + updated++; + ftsMutated = true; + WriteUpdateVerboseStatus($" [SKIP] {relPath} ({issue.Message})"); + continue; + } SymbolExtractor.ApplyFamilyScope(symbols, indexer.GetFamilyScopeKey(absPath, record.Lang)); var fileContext = new FileContext(projectRoot, record.Path, absPath, record.Lang); postExtractionHooks.OnSymbolsExtracted(fileContext, symbols); symbolsDroppedByKindFilter += options.SymbolKindFilter.Apply(symbols); + if (symbols.Count > options.MaxSymbolsPerFile) + { + var issue = BuildSymbolCountExceededIssue(record.Path, symbols.Count, options.MaxSymbolsPerFile); + writer.InsertSymbols([]); + writer.InsertReferences([]); + writer.InsertIssues(fileId, [issue]); + writer.ClearBatchInProgress(); + txn.Commit(); + fileBatchMarked = false; + updated++; + ftsMutated = true; + WriteUpdateVerboseStatus($" [SKIP] {relPath} ({issue.Message})"); + continue; + } + writer.InsertChunks(chunks); FileIndexer.ValidateSymbolLineRanges(record, symbols); writer.InsertSymbols(symbols); currentUpdatePath = FormatIndexPhasePath(relPath, "references"); diff --git a/src/CodeIndex/Cli/IndexCommandRunner.cs b/src/CodeIndex/Cli/IndexCommandRunner.cs index af87acaaec..c9f522f31d 100644 --- a/src/CodeIndex/Cli/IndexCommandRunner.cs +++ b/src/CodeIndex/Cli/IndexCommandRunner.cs @@ -19,6 +19,7 @@ public static partial class IndexCommandRunner { internal const string IncludeSymbolKindsEnvironmentVariable = "CDIDX_INDEX_INCLUDE_SYMBOL_KINDS"; internal const string ExcludeSymbolKindsEnvironmentVariable = "CDIDX_INDEX_EXCLUDE_SYMBOL_KINDS"; + internal const int DefaultMaxSymbolsPerFile = 5000; private const string SymbolKindFilterMetaKey = "index_symbol_kind_filter"; private const int ScanCheckpointVersion = 1; private const string ScanCheckpointFileName = "scan-checkpoint.json"; @@ -1270,6 +1271,7 @@ public sealed class IndexCommandOptions public DurationOutputFormat DurationFormat { get; init; } = DurationOutputFormat.Auto; public CompletionNotificationMode NotifyMode { get; init; } = CompletionNotificationMode.Auto; public long? MaxFileSizeBytes { get; init; } + public int MaxSymbolsPerFile { get; init; } = IndexCommandRunner.DefaultMaxSymbolsPerFile; public int Parallelism { get; init; } = IndexCommandRunner.DefaultIndexParallelism(); public bool MemoryTrace { get; init; } public FileIndexer.SymlinkPolicy SymlinkPolicy { get; init; } = FileIndexer.SymlinkPolicy.None; diff --git a/src/CodeIndex/Cli/IndexWatchRunner.cs b/src/CodeIndex/Cli/IndexWatchRunner.cs index df04b773a3..2189e33a4b 100644 --- a/src/CodeIndex/Cli/IndexWatchRunner.cs +++ b/src/CodeIndex/Cli/IndexWatchRunner.cs @@ -201,6 +201,11 @@ private static List BuildSubRunArgs(IndexCommandOptions baseOptions) args.Add("--max-file-bytes"); args.Add(maxFileSizeBytes.ToString(CultureInfo.InvariantCulture)); } + if (baseOptions.MaxSymbolsPerFile != IndexCommandRunner.DefaultMaxSymbolsPerFile) + { + args.Add("--max-symbols-per-file"); + args.Add(baseOptions.MaxSymbolsPerFile.ToString(CultureInfo.InvariantCulture)); + } if (baseOptions.SymlinkPolicy != FileIndexer.SymlinkPolicy.None) { args.Add("--follow-symlinks"); diff --git a/src/CodeIndex/Database/DbWriter.cs b/src/CodeIndex/Database/DbWriter.cs index d09d9573f9..6fb8d2e8b0 100644 --- a/src/CodeIndex/Database/DbWriter.cs +++ b/src/CodeIndex/Database/DbWriter.cs @@ -541,6 +541,23 @@ public bool HasAnyFilesWithLanguage(string lang) return cmd.ExecuteScalar() != null; } + public int CountSymbolsForFile(long fileId) + { + using var cmd = _conn.CreateCommand(); + cmd.CommandText = "SELECT COUNT(*) FROM symbols WHERE file_id = @file_id"; + cmd.Parameters.AddWithValue("@file_id", fileId); + return Convert.ToInt32(cmd.ExecuteScalar()); + } + + public bool HasIssueForFile(long fileId, string kind) + { + using var cmd = _conn.CreateCommand(); + cmd.CommandText = "SELECT 1 FROM file_issues WHERE file_id = @file_id AND kind = @kind LIMIT 1"; + cmd.Parameters.AddWithValue("@file_id", fileId); + cmd.Parameters.AddWithValue("@kind", kind); + return cmd.ExecuteScalar() != null; + } + public IReadOnlyList GetIndexedLanguages() { var languages = new List(); diff --git a/tests/CodeIndex.Tests/ConsoleUiTests.cs b/tests/CodeIndex.Tests/ConsoleUiTests.cs index 38daccbd27..df66a0d4b6 100644 --- a/tests/CodeIndex.Tests/ConsoleUiTests.cs +++ b/tests/CodeIndex.Tests/ConsoleUiTests.cs @@ -103,7 +103,7 @@ public void PrintUsage_WithoutBanner_HidesAsciiArtAndEasterEggFlags() Assert.DoesNotContain("██████╗", output); Assert.Contains("Usage:", output); - Assert.Contains("cdidx index [--db ] [--rebuild] [--optimize] [--verbose] [--dry-run] [--force] [--quiet] [--json] [--memory-trace] [--duration-format ] [--notify ] [--max-file-bytes ] [--follow-symlinks ]", output); + Assert.Contains("cdidx index [--db ] [--rebuild] [--optimize] [--verbose] [--dry-run] [--force] [--quiet] [--json] [--memory-trace] [--duration-format ] [--notify ] [--max-file-bytes ] [--max-symbols-per-file ] [--follow-symlinks ]", output); Assert.Contains("cdidx hooks [--project ] [--force] [--json]", output); Assert.Contains("cdidx index --commits [id ...] [--db ] [--verbose] [--dry-run] [--json] [--memory-trace] [--duration-format ]", output); Assert.Contains("cdidx index --files [path ...] [--db ] [--verbose] [--dry-run] [--json] [--memory-trace] [--duration-format ]", output); diff --git a/tests/CodeIndex.Tests/IndexCommandRunnerTests.cs b/tests/CodeIndex.Tests/IndexCommandRunnerTests.cs index c875bc4b11..40279d1428 100644 --- a/tests/CodeIndex.Tests/IndexCommandRunnerTests.cs +++ b/tests/CodeIndex.Tests/IndexCommandRunnerTests.cs @@ -251,6 +251,53 @@ public void Run_FileAboveMaxFileBytes_PersistsFileTooLargeIssue() } } + [Fact] + public void Run_FileAboveMaxSymbolsPerFile_PersistsSymbolCountExceededIssueOnly() + { + var projectRoot = CreateTempProject(); + try + { + var filePath = Path.Combine(projectRoot, "generated.py"); + File.WriteAllText(filePath, string.Join('\n', Enumerable.Range(0, 4).Select(i => $"def f{i}(): pass"))); + + var (initialExitCode, _) = RunAndCaptureJson([projectRoot, "--max-symbols-per-file", "10", "--json"]); + Assert.Equal(CommandExitCodes.Success, initialExitCode); + + var (exitCode, json) = RunAndCaptureJson([projectRoot, "--max-symbols-per-file", "2", "--json"]); + + Assert.Equal(CommandExitCodes.Success, exitCode); + Assert.Equal("success", json.GetProperty("status").GetString()); + Assert.Equal(0, json.GetProperty("summary").GetProperty("errors").GetInt32()); + + var dbPath = Path.Combine(projectRoot, ".cdidx", "codeindex.db"); + Assert.Equal(1, CountRows(dbPath, "files")); + Assert.Equal(0, CountRows(dbPath, "chunks")); + Assert.Equal(0, CountRows(dbPath, "symbols")); + Assert.Equal(0, CountRows(dbPath, "symbol_references")); + + using var db = new DbContext(dbPath); + db.TryMigrateForRead(); + var reader = new DbReader(db.Connection, db.IsReadOnly); + var issue = Assert.Single(reader.GetIssues("symbol_count_exceeded")); + Assert.Equal("generated.py", issue.Path); + Assert.Equal(0, issue.Line); + Assert.Contains("--max-symbols-per-file", issue.Message); + + var (raisedExitCode, raisedJson) = RunAndCaptureJson([projectRoot, "--max-symbols-per-file", "10", "--json"]); + + Assert.Equal(CommandExitCodes.Success, raisedExitCode); + Assert.Equal("success", raisedJson.GetProperty("status").GetString()); + Assert.True(CountRows(dbPath, "chunks") > 0); + Assert.True(CountRows(dbPath, "symbols") > 0); + Assert.Empty(reader.GetIssues("symbol_count_exceeded")); + } + finally + { + SqliteConnection.ClearAllPools(); + DeleteDirectory(projectRoot); + } + } + [Fact] public void RunFiles_FileAboveMaxFileBytes_PersistsFileTooLargeIssue() { @@ -1137,6 +1184,22 @@ public void ParseArgs_MaxFileBytesInlineFlag_ParsesBytesValue() Assert.Equal(12345, options.MaxFileSizeBytes); } + [Fact] + public void ParseArgs_MaxSymbolsPerFileFlag_ParsesPositiveValue() + { + var options = IndexCommandRunner.ParseArgs([".", "--max-symbols-per-file", "42"]); + + Assert.Equal(42, options.MaxSymbolsPerFile); + } + + [Fact] + public void ParseArgs_MaxSymbolsPerFileInlineFlag_ParsesPositiveValue() + { + var options = IndexCommandRunner.ParseArgs([".", "--max-symbols-per-file=43"]); + + Assert.Equal(43, options.MaxSymbolsPerFile); + } + [Fact] public void ParseArgs_MaxFileBytesInvalidValue_IsIgnored() { diff --git a/tests/CodeIndex.Tests/IndexWatchRunnerTests.cs b/tests/CodeIndex.Tests/IndexWatchRunnerTests.cs index dbd12fb704..ae36651f3f 100644 --- a/tests/CodeIndex.Tests/IndexWatchRunnerTests.cs +++ b/tests/CodeIndex.Tests/IndexWatchRunnerTests.cs @@ -157,6 +157,26 @@ public void BuildSubRunArgs_MaxFileBytes_PreservesWatchOverride() Assert.Equal((50L * 1024L * 1024L).ToString(System.Globalization.CultureInfo.InvariantCulture), args[flagIndex + 1]); } + [Fact] + public void BuildSubRunArgs_MaxSymbolsPerFile_PreservesWatchOverride() + { + var options = new IndexCommandOptions + { + ProjectPath = "/repo", + Json = true, + Watch = true, + MaxSymbolsPerFile = 42, + }; + var method = typeof(IndexWatchRunner).GetMethod("BuildSubRunArgs", BindingFlags.NonPublic | BindingFlags.Static); + + Assert.NotNull(method); + var args = Assert.IsType>(method.Invoke(null, [options])); + + var flagIndex = args.IndexOf("--max-symbols-per-file"); + Assert.True(flagIndex >= 0); + Assert.Equal("42", args[flagIndex + 1]); + } + [Fact] public void RunCore_CancellationToken_StopsImmediately() {