From edc63386df773a3fecefc848a30bbef83a3b2f74 Mon Sep 17 00:00:00 2001 From: Widthdom Date: Mon, 1 Jun 2026 00:06:46 +0900 Subject: [PATCH 1/5] Fix per-file symbol count cap (#1604) --- README.md | 2 + changelog.d/unreleased/1604.fixed.md | 23 +++++++ src/CodeIndex/Cli/ConsoleUi.cs | 3 +- .../Cli/IndexCommandRunner.FullScan.cs | 66 ++++++++++++++++++- src/CodeIndex/Cli/IndexCommandRunner.Parse.cs | 19 +++++- .../Cli/IndexCommandRunner.Update.cs | 44 ++++++++++++- src/CodeIndex/Cli/IndexCommandRunner.cs | 2 + src/CodeIndex/Cli/IndexWatchRunner.cs | 5 ++ src/CodeIndex/Database/DbWriter.cs | 8 +++ tests/CodeIndex.Tests/ConsoleUiTests.cs | 2 +- .../IndexCommandRunnerTests.cs | 55 ++++++++++++++++ .../CodeIndex.Tests/IndexWatchRunnerTests.cs | 20 ++++++ 12 files changed, 244 insertions(+), 5 deletions(-) create mode 100644 changelog.d/unreleased/1604.fixed.md diff --git a/README.md b/README.md index 89d760cda5..6821be052e 100644 --- a/README.md +++ b/README.md @@ -72,6 +72,7 @@ After the first command, use these cues and follow-up commands: | Edits or branch switches | Refresh incrementally with `--files`, `--commits`, or `--changed-between ` instead of rebuilding. See [Quick Start](USER_GUIDE.md#quick-start) and [Incremental update reliability](USER_GUIDE.md#incremental-update-reliability). | | Intentional rebuilds | Interactive terminals ask before deleting the DB. Scripts and CI must pass `--yes` or `--force`. | | Long-lived DB compaction | Run `cdidx optimize` or `cdidx index --optimize` to compact FTS5 segments immediately. Incremental refreshes also optimize opportunistically. | +| Pathological generated files | `--max-symbols-per-file ` skips indexing file content, symbols, and references when one file emits too many symbols, leaving a `symbol_count_exceeded` issue for audit. | | Maintenance rollback | Run `cdidx db checkpoint ` before risky DB maintenance and `cdidx db restore ` to roll back. `backfill-fold` creates an automatic checkpoint unless `--no-checkpoint` is passed. | | Permission or I/O scan errors | `cdidx` records the scan error, continues other directories, and writes `.cdidx/scan-checkpoint.json` so same-HEAD retries can skip completed directories. | @@ -334,6 +335,7 @@ extractor fixture を確認できます。詳細は | 編集後やブランチ切り替え後 | 再構築ではなく `--files`、`--commits`、`--changed-between ` で差分更新します。詳細は [クイックスタート](USER_GUIDE.md#クイックスタート) と [インクリメンタル更新の信頼性](USER_GUIDE.md#インクリメンタル更新の信頼性) を参照してください。 | | 意図的な再構築 | interactive terminal では既存 DB 削除前に確認を求めます。script / CI では `--yes` または `--force` が必要です。 | | 長期間使っている DB の compact | `cdidx optimize` または `cdidx index --optimize` で FTS5 segment をすぐに compact できます。差分更新中も必要に応じて自動 optimize します。 | +| 病的な generated file | 1 ファイルが過剰な symbol を出す場合、`--max-symbols-per-file ` は file content / symbols / references を保存せず、監査用の `symbol_count_exceeded` issue を残します。 | | 保守作業の rollback | risky な DB 保守の前に `cdidx db checkpoint `、戻す場合は `cdidx db restore ` を使います。`backfill-fold` は `--no-checkpoint` を渡さない限り自動 checkpoint を作成します。 | | 権限や I/O の scan error | `cdidx` は scan error を記録し、他のディレクトリの走査を続けます。同じ HEAD の再実行では `.cdidx/scan-checkpoint.json` により成功済みディレクトリを読み飛ばせます。 | diff --git a/changelog.d/unreleased/1604.fixed.md b/changelog.d/unreleased/1604.fixed.md new file mode 100644 index 0000000000..e5b9450948 --- /dev/null +++ b/changelog.d/unreleased/1604.fixed.md @@ -0,0 +1,23 @@ +--- +category: fixed +issues: + - 1604 +affected: + - src/CodeIndex/Cli/IndexCommandRunner.cs + - src/CodeIndex/Cli/IndexCommandRunner.Parse.cs + - src/CodeIndex/Cli/IndexCommandRunner.FullScan.cs + - src/CodeIndex/Cli/IndexCommandRunner.Update.cs + - src/CodeIndex/Cli/ConsoleUi.cs + - src/CodeIndex/Cli/IndexWatchRunner.cs + - src/CodeIndex/Database/DbWriter.cs + - README.md + - tests/CodeIndex.Tests/IndexCommandRunnerTests.cs +--- + +## English + +- **Indexing now caps per-file symbol output (#1604)** - `cdidx index --max-symbols-per-file N` skips file content, symbols, and references when one file emits too many symbols, leaving a `symbol_count_exceeded` issue instead of ballooning the database. + +## 日本語 + +- **index が 1 ファイルあたりの symbol 出力数を制限できるようになりました (#1604)** - `cdidx index --max-symbols-per-file N` は 1 ファイルが過剰な symbol を出す場合に file content / symbols / references を保存せず、DB を肥大化させる代わりに `symbol_count_exceeded` issue を残します。 diff --git a/src/CodeIndex/Cli/ConsoleUi.cs b/src/CodeIndex/Cli/ConsoleUi.cs index cee7629406..a39065cc44 100644 --- a/src/CodeIndex/Cli/ConsoleUi.cs +++ b/src/CodeIndex/Cli/ConsoleUi.cs @@ -68,7 +68,7 @@ public static class ConsoleUi private static readonly (string Command, string Usage)[] CommandUsageLines = [ - ("index", "cdidx index [--db ] [--rebuild] [--optimize] [--verbose] [--dry-run] [--force] [--quiet] [--json] [--memory-trace] [--duration-format ] [--notify ] [--max-file-bytes ] [--follow-symlinks ] [--include-symbol-kind [,]] [--exclude-symbol-kind [,]] [--watch [--debounce ]]"), + ("index", "cdidx index [--db ] [--rebuild] [--optimize] [--verbose] [--dry-run] [--force] [--quiet] [--json] [--memory-trace] [--duration-format ] [--notify ] [--max-file-bytes ] [--max-symbols-per-file ] [--follow-symlinks ] [--include-symbol-kind [,]] [--exclude-symbol-kind [,]] [--watch [--debounce ]]"), ("hooks", "cdidx hooks [--project ] [--force] [--json]"), ("backfill-fold", "cdidx backfill-fold [--db ] [--dry-run] [--no-checkpoint] [--json]"), ("optimize", "cdidx optimize [--db ] [--json]"), @@ -872,6 +872,7 @@ private static void PrintFlagReference(Action WriteHelpLine) Console.WriteLine(" --duration-format Index elapsed time format: `auto` (default), `seconds`, or `hms`; JSON keeps raw elapsed_ms"); WriteHelpLine(" --notify Long index completion signal: auto, bell, osc9, desktop, or none (also honors CDIDX_NOTIFY; quiet/json suppress it)"); WriteHelpLine(" --max-file-bytes Index only files up to this size (default: 4MiB; also honors CDIDX_MAX_FILE_BYTES; accepts K/M/G suffixes)"); + WriteHelpLine(" --max-symbols-per-file Skip file content, symbols, and references when one file emits too many symbols (default: 5000)"); WriteHelpLine(" --parallelism Full-scan extraction workers (default: CPU count capped at 16; also honors CDIDX_INDEX_PARALLELISM)"); WriteHelpLine(" --follow-symlinks Directory symlink policy: none (default), internal, or all"); WriteHelpLine(" --include-symbol-kind [,] Keep only matching symbol kinds during indexing"); diff --git a/src/CodeIndex/Cli/IndexCommandRunner.FullScan.cs b/src/CodeIndex/Cli/IndexCommandRunner.FullScan.cs index 2000e45cbc..ced4ec15c0 100644 --- a/src/CodeIndex/Cli/IndexCommandRunner.FullScan.cs +++ b/src/CodeIndex/Cli/IndexCommandRunner.FullScan.cs @@ -33,6 +33,15 @@ private static string FormatExtractionStalledMessage(IndexExtractionStalledExcep return $"Index extraction made no progress for {ConsoleUi.FormatDuration(ex.Timeout)}.{pathSuffix}"; } + private static FileIssue BuildSymbolCountExceededIssue(string path, int symbolCount, int maxSymbolsPerFile) => + new() + { + Path = path, + Kind = "symbol_count_exceeded", + Line = 0, + Message = $"Symbol extraction produced {symbolCount:N0} symbols, exceeding the --max-symbols-per-file limit of {maxSymbolsPerFile:N0}; file content, symbols, and references were not indexed. Exclude the generated/pathological file or raise --max-symbols-per-file if this is expected.", + }; + internal static string FormatIndexPhasePath(string path, string phase) => $"{path} ({phase})"; @@ -865,6 +874,14 @@ void StopJsonHeartbeat() Path.GetFullPath(options.ProjectPath!), activeJsonExtractionPhases[workerIndex], extractionCancellationToken); + if (symbols.Count > options.MaxSymbolsPerFile) + { + var issue = BuildSymbolCountExceededIssue(record.Path, symbols.Count, options.MaxSymbolsPerFile); + extractionResults.Add( + FullScanFileWorkItem.Success(filePath, record, string.Empty, rawBytes, issue.Message, [], [], [], [issue]), + extractionCancellationToken); + continue; + } SymbolExtractor.ApplyFamilyScope(symbols, indexer.GetFamilyScopeKey(filePath, record.Lang)); activeJsonExtractionPhases[workerIndex] = FormatIndexPhasePath(record.Path, "references"); references = ReferenceExtractor.Extract( @@ -1025,6 +1042,13 @@ void StopJsonHeartbeat() && AllowReuseWithCurrentHotspotFamilyTrust(record.Lang, hotspotFamilyTrustMatchesCurrent)); } if (existingId != null) + { + if (writer.CountSymbolsForFile(existingId.Value) > options.MaxSymbolsPerFile) + { + existingId = null; + } + } + if (existingId != null) { writer.PurgeStaleFilesSharingChecksum(projectRoot, record.Path, record.Checksum); skipped++; @@ -1058,7 +1082,6 @@ void StopJsonHeartbeat() var chunks = item.Chunks == null ? ChunkSplitter.Split(fileId, item.Content!) : ReassignChunkFileIds(item.Chunks, fileId); - writer.InsertChunks(chunks); currentJsonIndexFile = FormatIndexPhasePath(record.Path, "symbols"); var symbols = item.Symbols == null ? ExtractSymbolsWithStallTimeout( @@ -1070,6 +1093,26 @@ void StopJsonHeartbeat() currentJsonIndexFile, cancellationToken) : ReassignSymbolFileIds(item.Symbols, fileId); + if (symbols.Count > options.MaxSymbolsPerFile) + { + var issue = BuildSymbolCountExceededIssue(record.Path, symbols.Count, options.MaxSymbolsPerFile); + writer.InsertSymbols([]); + writer.InsertReferences([]); + writer.InsertIssues(fileId, [issue]); + if (options.Verbose) + WriteIndexVerboseStatus($" [SKIP] {record.Path} ({issue.Message})"); + txn.Commit(); + processed++; + if (!options.Json && !options.Quiet) + { + PauseIndexSpinnerForConsoleWrite(); + ConsoleUi.PrintProgress(processed, files.Count); + ResumeIndexSpinnerAfterConsoleWrite(); + } + ReportJsonIndexProgressIfNeeded(); + currentJsonIndexFile = null; + continue; + } if (item.Symbols == null) SymbolExtractor.ApplyFamilyScope(symbols, indexer.GetFamilyScopeKey(item.FilePath, record.Lang)); var fileContext = new FileContext(projectRoot, record.Path, item.FilePath, record.Lang); @@ -1077,6 +1120,27 @@ void StopJsonHeartbeat() postExtractionHooks.OnSymbolsExtracted(fileContext, mutableSymbols); symbolsDroppedByKindFilter += options.SymbolKindFilter.Apply(mutableSymbols); symbols = (IReadOnlyList)mutableSymbols; + if (symbols.Count > options.MaxSymbolsPerFile) + { + var issue = BuildSymbolCountExceededIssue(record.Path, symbols.Count, options.MaxSymbolsPerFile); + writer.InsertSymbols([]); + writer.InsertReferences([]); + writer.InsertIssues(fileId, [issue]); + if (options.Verbose) + WriteIndexVerboseStatus($" [SKIP] {record.Path} ({issue.Message})"); + txn.Commit(); + processed++; + if (!options.Json && !options.Quiet) + { + PauseIndexSpinnerForConsoleWrite(); + ConsoleUi.PrintProgress(processed, files.Count); + ResumeIndexSpinnerAfterConsoleWrite(); + } + ReportJsonIndexProgressIfNeeded(); + currentJsonIndexFile = null; + continue; + } + writer.InsertChunks(chunks); FileIndexer.ValidateSymbolLineRanges(record, symbols); writer.InsertSymbols(symbols); currentJsonIndexFile = FormatIndexPhasePath(record.Path, "references"); diff --git a/src/CodeIndex/Cli/IndexCommandRunner.Parse.cs b/src/CodeIndex/Cli/IndexCommandRunner.Parse.cs index 4b96f65ade..ef7ef415f5 100644 --- a/src/CodeIndex/Cli/IndexCommandRunner.Parse.cs +++ b/src/CodeIndex/Cli/IndexCommandRunner.Parse.cs @@ -13,7 +13,7 @@ public static partial class IndexCommandRunner private static readonly string[] AcceptedIndexFlags = [ "--db", "--data-dir", "--rebuild", "--verbose", "--json", "--quiet", "--dry-run", "--force", - "--yes", "--watch", "--debounce", "--duration-format", "--max-file-bytes", + "--yes", "--watch", "--debounce", "--duration-format", "--max-file-bytes", "--max-symbols-per-file", "--notify", "--parallelism", "--memory-trace", "--follow-symlinks", "--commits", "--changed-between", "--files", "--solution", "--project", @@ -43,6 +43,7 @@ public static IndexCommandOptions ParseArgs(string[] args) var durationFormat = DurationOutputFormat.Auto; var notifyMode = ReadCompletionNotificationModeFromEnvironment(); long? maxFileSizeBytes = ReadMaxFileSizeBytesFromEnvironment(); + var maxSymbolsPerFile = DefaultMaxSymbolsPerFile; var parallelism = ReadIndexParallelismFromEnvironment(); var symlinkPolicy = FileIndexer.SymlinkPolicy.None; string? easterEgg = null; @@ -151,6 +152,12 @@ public static IndexCommandOptions ParseArgs(string[] args) case var option when option.StartsWith("--max-file-bytes=", StringComparison.Ordinal): maxFileSizeBytes = ParseMaxFileBytes(option["--max-file-bytes=".Length..], maxFileSizeBytes); break; + case "--max-symbols-per-file" when i + 1 < args.Length: + maxSymbolsPerFile = ParseMaxSymbolsPerFile(args[++i], maxSymbolsPerFile, "--max-symbols-per-file"); + break; + case var option when option.StartsWith("--max-symbols-per-file=", StringComparison.Ordinal): + maxSymbolsPerFile = ParseMaxSymbolsPerFile(option["--max-symbols-per-file=".Length..], maxSymbolsPerFile, "--max-symbols-per-file"); + break; case "--parallelism" when i + 1 < args.Length: parallelism = ParseIndexParallelism(args[++i], parallelism, "--parallelism"); break; @@ -309,6 +316,7 @@ public static IndexCommandOptions ParseArgs(string[] args) DurationFormat = durationFormat, NotifyMode = notifyMode, MaxFileSizeBytes = maxFileSizeBytes, + MaxSymbolsPerFile = maxSymbolsPerFile, Parallelism = parallelism, SymlinkPolicy = symlinkPolicy, SymbolKindFilter = SymbolKindFilter.Create(includeSymbolKinds, excludeSymbolKinds, symbolKindFilterError), @@ -407,6 +415,15 @@ private static int ParseIndexParallelism(string value, int fallback, string sour return fallback; } + private static int ParseMaxSymbolsPerFile(string value, int fallback, string source) + { + if (int.TryParse(value, System.Globalization.NumberStyles.Integer, System.Globalization.CultureInfo.InvariantCulture, out var parsed) && parsed > 0) + return parsed; + + Console.Error.WriteLine($"Warning: invalid {source} value '{value}' (ignored; use a positive integer) / 不正な {source} 値 '{value}'（無視。正の整数を指定）"); + return fallback; + } + private static DurationOutputFormat ParseDurationFormat(string value, DurationOutputFormat fallback) { return value.Trim().ToLowerInvariant() switch diff --git a/src/CodeIndex/Cli/IndexCommandRunner.Update.cs b/src/CodeIndex/Cli/IndexCommandRunner.Update.cs index 95631c05c9..e3563ba4a4 100644 --- a/src/CodeIndex/Cli/IndexCommandRunner.Update.cs +++ b/src/CodeIndex/Cli/IndexCommandRunner.Update.cs @@ -701,6 +701,13 @@ void ThrowIfUpdateCancelled() && (statReusableLanguage != "csharp" || !csharpWorkspace.HasStaticInterfaceContracts) && (statReusableLanguage != "sql" || sqlGraphContractMatchesCurrent)); if (statMatchedId != null) + { + if (writer.CountSymbolsForFile(statMatchedId.Value) > options.MaxSymbolsPerFile) + { + statMatchedId = null; + } + } + if (statMatchedId != null) { skipped++; if (options.Verbose && !options.Json && !options.Quiet) @@ -735,6 +742,13 @@ void ThrowIfUpdateCancelled() && (record.Lang != "csharp" || !csharpWorkspace.HasStaticInterfaceContracts) && (record.Lang != "sql" || sqlGraphContractMatchesCurrent)); if (existingId != null) + { + if (writer.CountSymbolsForFile(existingId.Value) > options.MaxSymbolsPerFile) + { + existingId = null; + } + } + if (existingId != null) { using var purgeTxn = writer.BeginTransaction(); var purged = writer.PurgeStaleFilesSharingChecksum(projectRoot, record.Path, record.Checksum) @@ -772,7 +786,6 @@ void ThrowIfUpdateCancelled() var fileId = writer.UpsertFile(record); currentUpdatePath = FormatIndexPhasePath(relPath, "chunking"); var chunks = ChunkSplitter.Split(fileId, content); - writer.InsertChunks(chunks); currentUpdatePath = FormatIndexPhasePath(relPath, "symbols"); var symbols = ExtractSymbolsWithStallTimeout( fileId, @@ -782,10 +795,39 @@ void ThrowIfUpdateCancelled() Path.GetFullPath(options.ProjectPath!), currentUpdatePath, cancellationToken); + if (symbols.Count > options.MaxSymbolsPerFile) + { + var issue = BuildSymbolCountExceededIssue(record.Path, symbols.Count, options.MaxSymbolsPerFile); + writer.InsertSymbols([]); + writer.InsertReferences([]); + writer.InsertIssues(fileId, [issue]); + writer.ClearBatchInProgress(); + txn.Commit(); + fileBatchMarked = false; + updated++; + ftsMutated = true; + WriteUpdateVerboseStatus($" [SKIP] {relPath} ({issue.Message})"); + continue; + } SymbolExtractor.ApplyFamilyScope(symbols, indexer.GetFamilyScopeKey(absPath, record.Lang)); var fileContext = new FileContext(projectRoot, record.Path, absPath, record.Lang); postExtractionHooks.OnSymbolsExtracted(fileContext, symbols); symbolsDroppedByKindFilter += options.SymbolKindFilter.Apply(symbols); + if (symbols.Count > options.MaxSymbolsPerFile) + { + var issue = BuildSymbolCountExceededIssue(record.Path, symbols.Count, options.MaxSymbolsPerFile); + writer.InsertSymbols([]); + writer.InsertReferences([]); + writer.InsertIssues(fileId, [issue]); + writer.ClearBatchInProgress(); + txn.Commit(); + fileBatchMarked = false; + updated++; + ftsMutated = true; + WriteUpdateVerboseStatus($" [SKIP] {relPath} ({issue.Message})"); + continue; + } + writer.InsertChunks(chunks); FileIndexer.ValidateSymbolLineRanges(record, symbols); writer.InsertSymbols(symbols); currentUpdatePath = FormatIndexPhasePath(relPath, "references"); diff --git a/src/CodeIndex/Cli/IndexCommandRunner.cs b/src/CodeIndex/Cli/IndexCommandRunner.cs index c08796f17d..11b07e63d4 100644 --- a/src/CodeIndex/Cli/IndexCommandRunner.cs +++ b/src/CodeIndex/Cli/IndexCommandRunner.cs @@ -19,6 +19,7 @@ public static partial class IndexCommandRunner { internal const string IncludeSymbolKindsEnvironmentVariable = "CDIDX_INDEX_INCLUDE_SYMBOL_KINDS"; internal const string ExcludeSymbolKindsEnvironmentVariable = "CDIDX_INDEX_EXCLUDE_SYMBOL_KINDS"; + internal const int DefaultMaxSymbolsPerFile = 5000; private const string SymbolKindFilterMetaKey = "index_symbol_kind_filter"; private const int ScanCheckpointVersion = 1; private const string ScanCheckpointFileName = "scan-checkpoint.json"; @@ -1270,6 +1271,7 @@ public sealed class IndexCommandOptions public DurationOutputFormat DurationFormat { get; init; } = DurationOutputFormat.Auto; public CompletionNotificationMode NotifyMode { get; init; } = CompletionNotificationMode.Auto; public long? MaxFileSizeBytes { get; init; } + public int MaxSymbolsPerFile { get; init; } = IndexCommandRunner.DefaultMaxSymbolsPerFile; public int Parallelism { get; init; } = IndexCommandRunner.DefaultIndexParallelism(); public bool MemoryTrace { get; init; } public FileIndexer.SymlinkPolicy SymlinkPolicy { get; init; } = FileIndexer.SymlinkPolicy.None; diff --git a/src/CodeIndex/Cli/IndexWatchRunner.cs b/src/CodeIndex/Cli/IndexWatchRunner.cs index df04b773a3..2189e33a4b 100644 --- a/src/CodeIndex/Cli/IndexWatchRunner.cs +++ b/src/CodeIndex/Cli/IndexWatchRunner.cs @@ -201,6 +201,11 @@ private static List BuildSubRunArgs(IndexCommandOptions baseOptions) args.Add("--max-file-bytes"); args.Add(maxFileSizeBytes.ToString(CultureInfo.InvariantCulture)); } + if (baseOptions.MaxSymbolsPerFile != IndexCommandRunner.DefaultMaxSymbolsPerFile) + { + args.Add("--max-symbols-per-file"); + args.Add(baseOptions.MaxSymbolsPerFile.ToString(CultureInfo.InvariantCulture)); + } if (baseOptions.SymlinkPolicy != FileIndexer.SymlinkPolicy.None) { args.Add("--follow-symlinks"); diff --git a/src/CodeIndex/Database/DbWriter.cs b/src/CodeIndex/Database/DbWriter.cs index a12d5c2913..45800759b7 100644 --- a/src/CodeIndex/Database/DbWriter.cs +++ b/src/CodeIndex/Database/DbWriter.cs @@ -541,6 +541,14 @@ public bool HasAnyFilesWithLanguage(string lang) return cmd.ExecuteScalar() != null; } + public int CountSymbolsForFile(long fileId) + { + using var cmd = _conn.CreateCommand(); + cmd.CommandText = "SELECT COUNT(*) FROM symbols WHERE file_id = @file_id"; + cmd.Parameters.AddWithValue("@file_id", fileId); + return Convert.ToInt32(cmd.ExecuteScalar()); + } + public IReadOnlyList GetIndexedLanguages() { var languages = new List(); diff --git a/tests/CodeIndex.Tests/ConsoleUiTests.cs b/tests/CodeIndex.Tests/ConsoleUiTests.cs index 38daccbd27..df66a0d4b6 100644 --- a/tests/CodeIndex.Tests/ConsoleUiTests.cs +++ b/tests/CodeIndex.Tests/ConsoleUiTests.cs @@ -103,7 +103,7 @@ public void PrintUsage_WithoutBanner_HidesAsciiArtAndEasterEggFlags() Assert.DoesNotContain("██████╗", output); Assert.Contains("Usage:", output); - Assert.Contains("cdidx index [--db ] [--rebuild] [--optimize] [--verbose] [--dry-run] [--force] [--quiet] [--json] [--memory-trace] [--duration-format ] [--notify ] [--max-file-bytes ] [--follow-symlinks ]", output); + Assert.Contains("cdidx index [--db ] [--rebuild] [--optimize] [--verbose] [--dry-run] [--force] [--quiet] [--json] [--memory-trace] [--duration-format ] [--notify ] [--max-file-bytes ] [--max-symbols-per-file ] [--follow-symlinks ]", output); Assert.Contains("cdidx hooks [--project ] [--force] [--json]", output); Assert.Contains("cdidx index --commits [id ...] [--db ] [--verbose] [--dry-run] [--json] [--memory-trace] [--duration-format ]", output); Assert.Contains("cdidx index --files [path ...] [--db ] [--verbose] [--dry-run] [--json] [--memory-trace] [--duration-format ]", output); diff --git a/tests/CodeIndex.Tests/IndexCommandRunnerTests.cs b/tests/CodeIndex.Tests/IndexCommandRunnerTests.cs index c875bc4b11..0b67f2b33e 100644 --- a/tests/CodeIndex.Tests/IndexCommandRunnerTests.cs +++ b/tests/CodeIndex.Tests/IndexCommandRunnerTests.cs @@ -251,6 +251,45 @@ public void Run_FileAboveMaxFileBytes_PersistsFileTooLargeIssue() } } + [Fact] + public void Run_FileAboveMaxSymbolsPerFile_PersistsSymbolCountExceededIssueOnly() + { + var projectRoot = CreateTempProject(); + try + { + var filePath = Path.Combine(projectRoot, "generated.py"); + File.WriteAllText(filePath, string.Join('\n', Enumerable.Range(0, 4).Select(i => $"def f{i}(): pass"))); + + var (initialExitCode, _) = RunAndCaptureJson([projectRoot, "--max-symbols-per-file", "10", "--json"]); + Assert.Equal(CommandExitCodes.Success, initialExitCode); + + var (exitCode, json) = RunAndCaptureJson([projectRoot, "--max-symbols-per-file", "2", "--json"]); + + Assert.Equal(CommandExitCodes.Success, exitCode); + Assert.Equal("success", json.GetProperty("status").GetString()); + Assert.Equal(0, json.GetProperty("summary").GetProperty("errors").GetInt32()); + + var dbPath = Path.Combine(projectRoot, ".cdidx", "codeindex.db"); + Assert.Equal(1, CountRows(dbPath, "files")); + Assert.Equal(0, CountRows(dbPath, "chunks")); + Assert.Equal(0, CountRows(dbPath, "symbols")); + Assert.Equal(0, CountRows(dbPath, "symbol_references")); + + using var db = new DbContext(dbPath); + db.TryMigrateForRead(); + var reader = new DbReader(db.Connection, db.IsReadOnly); + var issue = Assert.Single(reader.GetIssues("symbol_count_exceeded")); + Assert.Equal("generated.py", issue.Path); + Assert.Equal(0, issue.Line); + Assert.Contains("--max-symbols-per-file", issue.Message); + } + finally + { + SqliteConnection.ClearAllPools(); + DeleteDirectory(projectRoot); + } + } + [Fact] public void RunFiles_FileAboveMaxFileBytes_PersistsFileTooLargeIssue() { @@ -1137,6 +1176,22 @@ public void ParseArgs_MaxFileBytesInlineFlag_ParsesBytesValue() Assert.Equal(12345, options.MaxFileSizeBytes); } + [Fact] + public void ParseArgs_MaxSymbolsPerFileFlag_ParsesPositiveValue() + { + var options = IndexCommandRunner.ParseArgs([".", "--max-symbols-per-file", "42"]); + + Assert.Equal(42, options.MaxSymbolsPerFile); + } + + [Fact] + public void ParseArgs_MaxSymbolsPerFileInlineFlag_ParsesPositiveValue() + { + var options = IndexCommandRunner.ParseArgs([".", "--max-symbols-per-file=43"]); + + Assert.Equal(43, options.MaxSymbolsPerFile); + } + [Fact] public void ParseArgs_MaxFileBytesInvalidValue_IsIgnored() { diff --git a/tests/CodeIndex.Tests/IndexWatchRunnerTests.cs b/tests/CodeIndex.Tests/IndexWatchRunnerTests.cs index dbd12fb704..ae36651f3f 100644 --- a/tests/CodeIndex.Tests/IndexWatchRunnerTests.cs +++ b/tests/CodeIndex.Tests/IndexWatchRunnerTests.cs @@ -157,6 +157,26 @@ public void BuildSubRunArgs_MaxFileBytes_PreservesWatchOverride() Assert.Equal((50L * 1024L * 1024L).ToString(System.Globalization.CultureInfo.InvariantCulture), args[flagIndex + 1]); } + [Fact] + public void BuildSubRunArgs_MaxSymbolsPerFile_PreservesWatchOverride() + { + var options = new IndexCommandOptions + { + ProjectPath = "/repo", + Json = true, + Watch = true, + MaxSymbolsPerFile = 42, + }; + var method = typeof(IndexWatchRunner).GetMethod("BuildSubRunArgs", BindingFlags.NonPublic | BindingFlags.Static); + + Assert.NotNull(method); + var args = Assert.IsType>(method.Invoke(null, [options])); + + var flagIndex = args.IndexOf("--max-symbols-per-file"); + Assert.True(flagIndex >= 0); + Assert.Equal("42", args[flagIndex + 1]); + } + [Fact] public void RunCore_CancellationToken_StopsImmediately() { From 78e17f2ef7c380adf40418a6a46c64a97c83ab17 Mon Sep 17 00:00:00 2001 From: Widthdom Date: Mon, 1 Jun 2026 00:08:27 +0900 Subject: [PATCH 2/5] Improve extraction stall recovery hint (#2744) --- changelog.d/unreleased/2744.fixed.md | 15 +++++++++++++++ src/CodeIndex/Cli/IndexCommandRunner.FullScan.cs | 2 +- 2 files changed, 16 insertions(+), 1 deletion(-) create mode 100644 changelog.d/unreleased/2744.fixed.md diff --git a/changelog.d/unreleased/2744.fixed.md b/changelog.d/unreleased/2744.fixed.md new file mode 100644 index 0000000000..4f77f4144f --- /dev/null +++ b/changelog.d/unreleased/2744.fixed.md @@ -0,0 +1,15 @@ +--- +category: fixed +issues: + - 2744 +affected: + - src/CodeIndex/Cli/IndexCommandRunner.FullScan.cs +--- + +## English + +- **Extraction-stall diagnostics now point at bounded recovery for pathological symbol output (#2744)** - `E013_INDEX_EXTRACTION_STALLED` keeps the active file/phase and now suggests using `--max-symbols-per-file` or excluding the reported file when full refresh stalls on symbol extraction. + +## 日本語 + +- **extraction stall 診断が病的な symbol 出力への bounded recovery を示すようになりました (#2744)** - `E013_INDEX_EXTRACTION_STALLED` は active file / phase を保持したまま、full refresh が symbol extraction で止まる場合に `--max-symbols-per-file` または該当ファイルの除外を案内します。 diff --git a/src/CodeIndex/Cli/IndexCommandRunner.FullScan.cs b/src/CodeIndex/Cli/IndexCommandRunner.FullScan.cs index ced4ec15c0..44742925c9 100644 --- a/src/CodeIndex/Cli/IndexCommandRunner.FullScan.cs +++ b/src/CodeIndex/Cli/IndexCommandRunner.FullScan.cs @@ -200,7 +200,7 @@ private static int WriteExtractionStalledResult(bool json, JsonSerializerOptions jsonOptions, $"Index extraction made no progress for {ConsoleUi.FormatDuration(ex.Timeout)} ({ex.FilesProcessed:N0}{totalSuffix} files processed).{pathSuffix}", CommandExitCodes.CancelledBySignal, - "Rerun with `--verbose` to inspect progress, lower `--parallelism`, or file a bug with the reported active phase.", + "Rerun with `--verbose` to inspect progress, lower `--parallelism`, exclude the reported file, or lower `--max-symbols-per-file` to skip pathological symbol output.", CommandErrorCodes.IndexExtractionStalled); } From 1c2b21316412362cfd062fdd701c6564d3118e53 Mon Sep 17 00:00:00 2001 From: Widthdom Date: Mon, 1 Jun 2026 00:24:18 +0900 Subject: [PATCH 3/5] Register symbol cap flag schema (#1604) --- changelog.d/unreleased/1604.fixed.md | 1 + src/CodeIndex/Cli/CliFlagSchema.cs | 1 + 2 files changed, 2 insertions(+) diff --git a/changelog.d/unreleased/1604.fixed.md b/changelog.d/unreleased/1604.fixed.md index e5b9450948..2a411ebfe0 100644 --- a/changelog.d/unreleased/1604.fixed.md +++ b/changelog.d/unreleased/1604.fixed.md @@ -9,6 +9,7 @@ affected: - src/CodeIndex/Cli/IndexCommandRunner.Update.cs - src/CodeIndex/Cli/ConsoleUi.cs - src/CodeIndex/Cli/IndexWatchRunner.cs + - src/CodeIndex/Cli/CliFlagSchema.cs - src/CodeIndex/Database/DbWriter.cs - README.md - tests/CodeIndex.Tests/IndexCommandRunnerTests.cs diff --git a/src/CodeIndex/Cli/CliFlagSchema.cs b/src/CodeIndex/Cli/CliFlagSchema.cs index 25a301eb49..19f7bf3082 100644 --- a/src/CodeIndex/Cli/CliFlagSchema.cs +++ b/src/CodeIndex/Cli/CliFlagSchema.cs @@ -283,6 +283,7 @@ private static IReadOnlyList BuildAll() new() { Name = "--force", Description = "Bypass the per-database index lock", Commands = Set("index") }, new() { Name = "--duration-format", ValuePlaceholder = "", Description = "Index elapsed time display format", Commands = Set("index") }, new() { Name = "--max-file-bytes", ValuePlaceholder = "", Description = "Override the per-file indexing size limit", Commands = Set("index") }, + new() { Name = "--max-symbols-per-file", ValuePlaceholder = "", Description = "Skip file content, symbols, and references when one file emits too many symbols", Commands = Set("index") }, new() { Name = "--parallelism", ValuePlaceholder = "", Description = "Full-scan extraction worker count (default: CPU count capped at 16; also honors CDIDX_INDEX_PARALLELISM)", Commands = Set("index") }, new() { Name = "--memory-trace", Description = "Include phase memory samples in index JSON output", Commands = Set("index") }, new() { Name = "--commits", ValuePlaceholder = "", Description = "Update files changed in given git commits", Commands = Set("index") }, From 00303637dfb5ccf0e56813d1fb5fb0ead8ee91b5 Mon Sep 17 00:00:00 2001 From: Widthdom Date: Mon, 1 Jun 2026 00:42:44 +0900 Subject: [PATCH 4/5] Reindex symbol-capped files when limit changes (#1604) --- src/CodeIndex/Cli/IndexCommandRunner.FullScan.cs | 3 ++- src/CodeIndex/Cli/IndexCommandRunner.Update.cs | 6 ++++-- src/CodeIndex/Database/DbWriter.cs | 9 +++++++++ tests/CodeIndex.Tests/IndexCommandRunnerTests.cs | 8 ++++++++ 4 files changed, 23 insertions(+), 3 deletions(-) diff --git a/src/CodeIndex/Cli/IndexCommandRunner.FullScan.cs b/src/CodeIndex/Cli/IndexCommandRunner.FullScan.cs index 44742925c9..01346ef5c2 100644 --- a/src/CodeIndex/Cli/IndexCommandRunner.FullScan.cs +++ b/src/CodeIndex/Cli/IndexCommandRunner.FullScan.cs @@ -1043,7 +1043,8 @@ void StopJsonHeartbeat() } if (existingId != null) { - if (writer.CountSymbolsForFile(existingId.Value) > options.MaxSymbolsPerFile) + if (writer.CountSymbolsForFile(existingId.Value) > options.MaxSymbolsPerFile + || writer.HasIssueForFile(existingId.Value, "symbol_count_exceeded")) { existingId = null; } diff --git a/src/CodeIndex/Cli/IndexCommandRunner.Update.cs b/src/CodeIndex/Cli/IndexCommandRunner.Update.cs index e3563ba4a4..9d040f3267 100644 --- a/src/CodeIndex/Cli/IndexCommandRunner.Update.cs +++ b/src/CodeIndex/Cli/IndexCommandRunner.Update.cs @@ -702,7 +702,8 @@ void ThrowIfUpdateCancelled() && (statReusableLanguage != "sql" || sqlGraphContractMatchesCurrent)); if (statMatchedId != null) { - if (writer.CountSymbolsForFile(statMatchedId.Value) > options.MaxSymbolsPerFile) + if (writer.CountSymbolsForFile(statMatchedId.Value) > options.MaxSymbolsPerFile + || writer.HasIssueForFile(statMatchedId.Value, "symbol_count_exceeded")) { statMatchedId = null; } @@ -743,7 +744,8 @@ void ThrowIfUpdateCancelled() && (record.Lang != "sql" || sqlGraphContractMatchesCurrent)); if (existingId != null) { - if (writer.CountSymbolsForFile(existingId.Value) > options.MaxSymbolsPerFile) + if (writer.CountSymbolsForFile(existingId.Value) > options.MaxSymbolsPerFile + || writer.HasIssueForFile(existingId.Value, "symbol_count_exceeded")) { existingId = null; } diff --git a/src/CodeIndex/Database/DbWriter.cs b/src/CodeIndex/Database/DbWriter.cs index 45800759b7..2594c1f878 100644 --- a/src/CodeIndex/Database/DbWriter.cs +++ b/src/CodeIndex/Database/DbWriter.cs @@ -549,6 +549,15 @@ public int CountSymbolsForFile(long fileId) return Convert.ToInt32(cmd.ExecuteScalar()); } + public bool HasIssueForFile(long fileId, string kind) + { + using var cmd = _conn.CreateCommand(); + cmd.CommandText = "SELECT 1 FROM file_issues WHERE file_id = @file_id AND kind = @kind LIMIT 1"; + cmd.Parameters.AddWithValue("@file_id", fileId); + cmd.Parameters.AddWithValue("@kind", kind); + return cmd.ExecuteScalar() != null; + } + public IReadOnlyList GetIndexedLanguages() { var languages = new List(); diff --git a/tests/CodeIndex.Tests/IndexCommandRunnerTests.cs b/tests/CodeIndex.Tests/IndexCommandRunnerTests.cs index 0b67f2b33e..40279d1428 100644 --- a/tests/CodeIndex.Tests/IndexCommandRunnerTests.cs +++ b/tests/CodeIndex.Tests/IndexCommandRunnerTests.cs @@ -282,6 +282,14 @@ public void Run_FileAboveMaxSymbolsPerFile_PersistsSymbolCountExceededIssueOnly( Assert.Equal("generated.py", issue.Path); Assert.Equal(0, issue.Line); Assert.Contains("--max-symbols-per-file", issue.Message); + + var (raisedExitCode, raisedJson) = RunAndCaptureJson([projectRoot, "--max-symbols-per-file", "10", "--json"]); + + Assert.Equal(CommandExitCodes.Success, raisedExitCode); + Assert.Equal("success", raisedJson.GetProperty("status").GetString()); + Assert.True(CountRows(dbPath, "chunks") > 0); + Assert.True(CountRows(dbPath, "symbols") > 0); + Assert.Empty(reader.GetIssues("symbol_count_exceeded")); } finally { From df0037bf80d747f3294c059928cdbc1da08a1317 Mon Sep 17 00:00:00 2001 From: Widthdom Date: Mon, 1 Jun 2026 01:48:32 +0900 Subject: [PATCH 5/5] Format update merge resolution --- .../Cli/IndexCommandRunner.Update.cs | 374 +++++++++--------- 1 file changed, 187 insertions(+), 187 deletions(-) diff --git a/src/CodeIndex/Cli/IndexCommandRunner.Update.cs b/src/CodeIndex/Cli/IndexCommandRunner.Update.cs index f9d9cc969e..d83aacdff5 100644 --- a/src/CodeIndex/Cli/IndexCommandRunner.Update.cs +++ b/src/CodeIndex/Cli/IndexCommandRunner.Update.cs @@ -655,226 +655,226 @@ void ThrowIfUpdateCancelled() continue; } - if (FileIndexer.TryGetFileIdentity(absPath, out var identity) && !visitedFileIdentities.Add(identity)) - { - var message = "Skipped hardlinked file because the same file content was already indexed from another path."; - warnings++; - warningList.Add(new CliJsonMessage(relPath, message)); - if (!options.Json && !options.Quiet) + if (FileIndexer.TryGetFileIdentity(absPath, out var identity) && !visitedFileIdentities.Add(identity)) { - PauseUpdateSpinnerForConsoleWrite(); - ConsoleUi.PrintWarning($"{relPath}: {message}"); - ResumeUpdateSpinnerAfterConsoleWrite(); - } + var message = "Skipped hardlinked file because the same file content was already indexed from another path."; + warnings++; + warningList.Add(new CliJsonMessage(relPath, message)); + if (!options.Json && !options.Quiet) + { + PauseUpdateSpinnerForConsoleWrite(); + ConsoleUi.PrintWarning($"{relPath}: {message}"); + ResumeUpdateSpinnerAfterConsoleWrite(); + } - if (!writer.HasFileAtPath(dbPath)) - { - skipped++; + if (!writer.HasFileAtPath(dbPath)) + { + skipped++; + continue; + } + + DemoteReadinessOnce(); + using var deleteTxn = writer.BeginTransaction(); + if (writer.DeleteFileByPath(dbPath)) + { + WriteProjectRootOnce(); + deleteTxn.Commit(); + removed++; + ftsMutated = true; + } + else + { + skipped++; + } continue; } - DemoteReadinessOnce(); - using var deleteTxn = writer.BeginTransaction(); - if (writer.DeleteFileByPath(dbPath)) + var statReusableLanguage = TryDetectStatReusableLanguage(absPath); + var statMatchedId = TryGetUnchangedFileIdFromStat( + writer, + projectRoot, + absPath, + statReusableLanguage, + allowReuse: symbolKindFilterMatchesPrior + && statReusableLanguage is not ("javascript" or "typescript") + && (statReusableLanguage != "csharp" || csharpSymbolNameContractMatchesCurrent) + && (statReusableLanguage != "csharp" || !csharpWorkspace.HasStaticInterfaceContracts) + && (statReusableLanguage != "sql" || sqlGraphContractMatchesCurrent)); + if (statMatchedId != null) { - WriteProjectRootOnce(); - deleteTxn.Commit(); - removed++; - ftsMutated = true; + if (writer.CountSymbolsForFile(statMatchedId.Value) > options.MaxSymbolsPerFile + || writer.HasIssueForFile(statMatchedId.Value, "symbol_count_exceeded")) + { + statMatchedId = null; + } } - else + if (statMatchedId != null) { skipped++; + if (options.Verbose && !options.Json && !options.Quiet) + { + PauseUpdateSpinnerForConsoleWrite(); + Console.WriteLine($" [SKIP] {relPath} (unchanged)"); + ResumeUpdateSpinnerAfterConsoleWrite(); + } + continue; } - continue; - } - var statReusableLanguage = TryDetectStatReusableLanguage(absPath); - var statMatchedId = TryGetUnchangedFileIdFromStat( - writer, - projectRoot, - absPath, - statReusableLanguage, - allowReuse: symbolKindFilterMatchesPrior - && statReusableLanguage is not ("javascript" or "typescript") - && (statReusableLanguage != "csharp" || csharpSymbolNameContractMatchesCurrent) - && (statReusableLanguage != "csharp" || !csharpWorkspace.HasStaticInterfaceContracts) - && (statReusableLanguage != "sql" || sqlGraphContractMatchesCurrent)); - if (statMatchedId != null) - { - if (writer.CountSymbolsForFile(statMatchedId.Value) > options.MaxSymbolsPerFile - || writer.HasIssueForFile(statMatchedId.Value, "symbol_count_exceeded")) - { - statMatchedId = null; - } - } - if (statMatchedId != null) - { - skipped++; - if (options.Verbose && !options.Json && !options.Quiet) + var (record, content, rawBytes, warning) = indexer.BuildRecordWithRawBytes(absPath, cancellationToken); + + if (warning != null && !options.Json && !options.Quiet) { PauseUpdateSpinnerForConsoleWrite(); - Console.WriteLine($" [SKIP] {relPath} (unchanged)"); + ConsoleUi.PrintWarning(warning); ResumeUpdateSpinnerAfterConsoleWrite(); } - continue; - } - - var (record, content, rawBytes, warning) = indexer.BuildRecordWithRawBytes(absPath, cancellationToken); - if (warning != null && !options.Json && !options.Quiet) - { - PauseUpdateSpinnerForConsoleWrite(); - ConsoleUi.PrintWarning(warning); - ResumeUpdateSpinnerAfterConsoleWrite(); - } - - var existingId = writer.GetUnchangedFileId( - record.Path, - record.Modified, - record.Checksum, - size: record.Size, - lines: record.Lines, - language: record.Lang, - generated: record.Generated, - allowReuse: symbolKindFilterMatchesPrior - && record.Lang is not ("javascript" or "typescript") - && (record.Lang != "csharp" || csharpSymbolNameContractMatchesCurrent) - && (record.Lang != "csharp" || !csharpWorkspace.HasStaticInterfaceContracts) - && (record.Lang != "sql" || sqlGraphContractMatchesCurrent)); - if (existingId != null) - { - if (writer.CountSymbolsForFile(existingId.Value) > options.MaxSymbolsPerFile - || writer.HasIssueForFile(existingId.Value, "symbol_count_exceeded")) + var existingId = writer.GetUnchangedFileId( + record.Path, + record.Modified, + record.Checksum, + size: record.Size, + lines: record.Lines, + language: record.Lang, + generated: record.Generated, + allowReuse: symbolKindFilterMatchesPrior + && record.Lang is not ("javascript" or "typescript") + && (record.Lang != "csharp" || csharpSymbolNameContractMatchesCurrent) + && (record.Lang != "csharp" || !csharpWorkspace.HasStaticInterfaceContracts) + && (record.Lang != "sql" || sqlGraphContractMatchesCurrent)); + if (existingId != null) + { + if (writer.CountSymbolsForFile(existingId.Value) > options.MaxSymbolsPerFile + || writer.HasIssueForFile(existingId.Value, "symbol_count_exceeded")) + { + existingId = null; + } + } + if (existingId != null) { - existingId = null; + using var purgeTxn = writer.BeginTransaction(); + var purged = writer.PurgeStaleFilesSharingChecksum(projectRoot, record.Path, record.Checksum) + + (projectRootWritten + ? writer.PurgeStaleFilesSharingDirectoryAndStem(projectRoot, record.Path) + : 0); + if (purged > 0) + { + DemoteReadinessOnce(); + WriteProjectRootOnce(); + purgeTxn.Commit(); + removed += purged; + ftsMutated = true; + } + skipped++; + if (options.Verbose && !options.Json && !options.Quiet) + { + PauseUpdateSpinnerForConsoleWrite(); + Console.WriteLine(purged > 0 + ? $" [SKIP] {relPath} (unchanged; purged {purged:N0} stale renamed path(s))" + : $" [SKIP] {relPath} (unchanged)"); + ResumeUpdateSpinnerAfterConsoleWrite(); + } + continue; } - } - if (existingId != null) - { - using var purgeTxn = writer.BeginTransaction(); - var purged = writer.PurgeStaleFilesSharingChecksum(projectRoot, record.Path, record.Checksum) - + (projectRootWritten - ? writer.PurgeStaleFilesSharingDirectoryAndStem(projectRoot, record.Path) - : 0); - if (purged > 0) + + DemoteReadinessOnce(); + writer.MarkBatchInProgress(); + fileBatchMarked = true; + using var txn = writer.BeginTransaction(); + writer.PurgeStaleFilesSharingChecksum(projectRoot, record.Path, record.Checksum); + if (projectRootWritten) + writer.PurgeStaleFilesSharingDirectoryAndStem(projectRoot, record.Path); + WriteProjectRootOnce(); + var fileId = writer.UpsertFile(record); + currentUpdatePath = FormatIndexPhasePath(relPath, "chunking"); + var chunks = ChunkSplitter.Split(fileId, content); + currentUpdatePath = FormatIndexPhasePath(relPath, "symbols"); + var symbols = ExtractSymbolsWithStallTimeout( + fileId, + record.Lang, + content, + absPath, + Path.GetFullPath(options.ProjectPath!), + currentUpdatePath, + cancellationToken); + if (symbols.Count > options.MaxSymbolsPerFile) { - DemoteReadinessOnce(); - WriteProjectRootOnce(); - purgeTxn.Commit(); - removed += purged; + var issue = BuildSymbolCountExceededIssue(record.Path, symbols.Count, options.MaxSymbolsPerFile); + writer.InsertSymbols([]); + writer.InsertReferences([]); + writer.InsertIssues(fileId, [issue]); + writer.ClearBatchInProgress(); + txn.Commit(); + fileBatchMarked = false; + updated++; ftsMutated = true; + WriteUpdateVerboseStatus($" [SKIP] {relPath} ({issue.Message})"); + continue; } - skipped++; - if (options.Verbose && !options.Json && !options.Quiet) + SymbolExtractor.ApplyFamilyScope(symbols, indexer.GetFamilyScopeKey(absPath, record.Lang)); + var fileContext = new FileContext(projectRoot, record.Path, absPath, record.Lang); + postExtractionHooks.OnSymbolsExtracted(fileContext, symbols); + symbolsDroppedByKindFilter += options.SymbolKindFilter.Apply(symbols); + if (symbols.Count > options.MaxSymbolsPerFile) { - PauseUpdateSpinnerForConsoleWrite(); - Console.WriteLine(purged > 0 - ? $" [SKIP] {relPath} (unchanged; purged {purged:N0} stale renamed path(s))" - : $" [SKIP] {relPath} (unchanged)"); - ResumeUpdateSpinnerAfterConsoleWrite(); + var issue = BuildSymbolCountExceededIssue(record.Path, symbols.Count, options.MaxSymbolsPerFile); + writer.InsertSymbols([]); + writer.InsertReferences([]); + writer.InsertIssues(fileId, [issue]); + writer.ClearBatchInProgress(); + txn.Commit(); + fileBatchMarked = false; + updated++; + ftsMutated = true; + WriteUpdateVerboseStatus($" [SKIP] {relPath} ({issue.Message})"); + continue; } - continue; - } - - DemoteReadinessOnce(); - writer.MarkBatchInProgress(); - fileBatchMarked = true; - using var txn = writer.BeginTransaction(); - writer.PurgeStaleFilesSharingChecksum(projectRoot, record.Path, record.Checksum); - if (projectRootWritten) - writer.PurgeStaleFilesSharingDirectoryAndStem(projectRoot, record.Path); - WriteProjectRootOnce(); - var fileId = writer.UpsertFile(record); - currentUpdatePath = FormatIndexPhasePath(relPath, "chunking"); - var chunks = ChunkSplitter.Split(fileId, content); - currentUpdatePath = FormatIndexPhasePath(relPath, "symbols"); - var symbols = ExtractSymbolsWithStallTimeout( - fileId, - record.Lang, - content, - absPath, - Path.GetFullPath(options.ProjectPath!), - currentUpdatePath, - cancellationToken); - if (symbols.Count > options.MaxSymbolsPerFile) - { - var issue = BuildSymbolCountExceededIssue(record.Path, symbols.Count, options.MaxSymbolsPerFile); - writer.InsertSymbols([]); - writer.InsertReferences([]); - writer.InsertIssues(fileId, [issue]); + writer.InsertChunks(chunks); + FileIndexer.ValidateSymbolLineRanges(record, symbols); + writer.InsertSymbols(symbols); + currentUpdatePath = FormatIndexPhasePath(relPath, "references"); + var references = ReferenceExtractor.Extract( + fileId, + record.Lang, + content, + symbols, + record.Path, + record.Lang == "csharp" ? csharpWorkspace.Symbols : null, + cancellationToken); + postExtractionHooks.OnReferencesExtracted(fileContext, references); + writer.InsertReferences(references); + // Validate content for encoding issues / エンコーディング問題を検証 + currentUpdatePath = FormatIndexPhasePath(relPath, "validating"); + var issues = FileIndexer.ValidateContent(record.Path, rawBytes, content); + writer.InsertIssues(fileId, issues); + currentUpdatePath = FormatIndexPhasePath(relPath, "committing"); writer.ClearBatchInProgress(); txn.Commit(); - fileBatchMarked = false; + updated++; ftsMutated = true; - WriteUpdateVerboseStatus($" [SKIP] {relPath} ({issue.Message})"); - continue; + ThrowIfUpdateCancelled(); + WriteUpdateVerboseStatus($" [OK ] {relPath} ({chunks.Count} chunks, {symbols.Count} symbols, {references.Count} refs)"); } - SymbolExtractor.ApplyFamilyScope(symbols, indexer.GetFamilyScopeKey(absPath, record.Lang)); - var fileContext = new FileContext(projectRoot, record.Path, absPath, record.Lang); - postExtractionHooks.OnSymbolsExtracted(fileContext, symbols); - symbolsDroppedByKindFilter += options.SymbolKindFilter.Apply(symbols); - if (symbols.Count > options.MaxSymbolsPerFile) + catch (IndexExtractionStalledException) { - var issue = BuildSymbolCountExceededIssue(record.Path, symbols.Count, options.MaxSymbolsPerFile); - writer.InsertSymbols([]); - writer.InsertReferences([]); - writer.InsertIssues(fileId, [issue]); - writer.ClearBatchInProgress(); - txn.Commit(); - fileBatchMarked = false; - updated++; - ftsMutated = true; - WriteUpdateVerboseStatus($" [SKIP] {relPath} ({issue.Message})"); - continue; + throw; } - writer.InsertChunks(chunks); - FileIndexer.ValidateSymbolLineRanges(record, symbols); - writer.InsertSymbols(symbols); - currentUpdatePath = FormatIndexPhasePath(relPath, "references"); - var references = ReferenceExtractor.Extract( - fileId, - record.Lang, - content, - symbols, - record.Path, - record.Lang == "csharp" ? csharpWorkspace.Symbols : null, - cancellationToken); - postExtractionHooks.OnReferencesExtracted(fileContext, references); - writer.InsertReferences(references); - // Validate content for encoding issues / エンコーディング問題を検証 - currentUpdatePath = FormatIndexPhasePath(relPath, "validating"); - var issues = FileIndexer.ValidateContent(record.Path, rawBytes, content); - writer.InsertIssues(fileId, issues); - currentUpdatePath = FormatIndexPhasePath(relPath, "committing"); - writer.ClearBatchInProgress(); - txn.Commit(); - - updated++; - ftsMutated = true; - ThrowIfUpdateCancelled(); - WriteUpdateVerboseStatus($" [OK ] {relPath} ({chunks.Count} chunks, {symbols.Count} symbols, {references.Count} refs)"); - } - catch (IndexExtractionStalledException) - { - throw; - } - catch (Exception ex) - { - if (ex is FileIndexer.BinaryFileSkippedException) + catch (Exception ex) { - warnings++; - warningList.Add(new CliJsonMessage(relPath, ex.Message)); - if (!options.Json && !options.Quiet) + if (ex is FileIndexer.BinaryFileSkippedException) { - PauseUpdateSpinnerForConsoleWrite(); - ConsoleUi.PrintWarning(ex.Message); - ResumeUpdateSpinnerAfterConsoleWrite(); - } + warnings++; + warningList.Add(new CliJsonMessage(relPath, ex.Message)); + if (!options.Json && !options.Quiet) + { + PauseUpdateSpinnerForConsoleWrite(); + ConsoleUi.PrintWarning(ex.Message); + ResumeUpdateSpinnerAfterConsoleWrite(); + } - if (writer.HasFileAtPath(dbPath)) - { + if (writer.HasFileAtPath(dbPath)) + { DemoteReadinessOnce(); using var deleteTxn = writer.BeginTransaction(); if (writer.DeleteFileByPath(dbPath))