From a12646a3240550c86049ce20318514cb617fe764 Mon Sep 17 00:00:00 2001 From: Widthdom Date: Mon, 15 Jun 2026 10:10:19 +0900 Subject: [PATCH 1/6] Optimize cdidx full scan after HEAD changes --- .../unreleased/+head-change-index-skip.fixed.md | 14 ++++++++++++++ src/CodeIndex/Cli/IndexCommandRunner.FullScan.cs | 8 ++++++-- tests/CodeIndex.Tests/IndexCommandRunnerTests.cs | 14 ++++++++------ 3 files changed, 28 insertions(+), 8 deletions(-) create mode 100644 changelog.d/unreleased/+head-change-index-skip.fixed.md diff --git a/changelog.d/unreleased/+head-change-index-skip.fixed.md b/changelog.d/unreleased/+head-change-index-skip.fixed.md new file mode 100644 index 0000000000..b9292c3369 --- /dev/null +++ b/changelog.d/unreleased/+head-change-index-skip.fixed.md @@ -0,0 +1,14 @@ +--- +category: fixed +affected: + - src/CodeIndex/Cli/IndexCommandRunner.FullScan.cs + - tests/CodeIndex.Tests/IndexCommandRunnerTests.cs +--- + +## English + +- **Reduced `cdidx .` time after a branch or HEAD change.** Default incremental full scans no longer pre-extract every file solely because the stored full-scan HEAD differs from the current worktree HEAD. Unchanged files can now be reused before symbol/reference extraction, while the existing HEAD-change warning, stale-file purge, readiness stamping, and current-HEAD metadata update remain intact. + +## 日本語 + +- **branch / HEAD 変更後の `cdidx .` 時間を短縮しました。** 既定の incremental full scan は、保存済み full-scan HEAD と現在の worktree HEAD が違うという理由だけで全ファイルを先行抽出しなくなりました。既存の HEAD 変更警告、stale file purge、readiness stamp、現在 HEAD metadata 更新は維持しつつ、未変更ファイルを symbol / reference 抽出前に再利用できます。 diff --git a/src/CodeIndex/Cli/IndexCommandRunner.FullScan.cs b/src/CodeIndex/Cli/IndexCommandRunner.FullScan.cs index 63d7dbcb0a..683155d0a9 100644 --- a/src/CodeIndex/Cli/IndexCommandRunner.FullScan.cs +++ b/src/CodeIndex/Cli/IndexCommandRunner.FullScan.cs @@ -786,12 +786,16 @@ void ThrowIfFullScanCancelled(int filesProcessed, int? filesTotal) using var postExtractionHooks = PostExtractionHookRunner.DiscoverDefault(options.MaxFileSizeBytes); var extractionParallelism = Math.Max(1, options.Parallelism); var hasPostExtractionHooks = postExtractionHooks.Hooks.Count > 0; - var parallelizeExtraction = (options.Rebuild || writer.GetCounts().files == 0 || headChangeDetected) + var existingFileCount = writer.GetCounts().files; + var parallelizeExtraction = (options.Rebuild || existingFileCount == 0) && !options.SymbolKindFilter.IsActive && !hasPostExtractionHooks; + var parallelizeExtractionReason = parallelizeExtraction + ? options.Rebuild ? "rebuild" : "empty_index" + : null; FullScanExtractionSchedulingForTesting?.Invoke( parallelizeExtraction, - headChangeDetected ? "head_changed" : null); + parallelizeExtractionReason); void StartIndexSpinnerIfNeeded() { diff --git a/tests/CodeIndex.Tests/IndexCommandRunnerTests.cs b/tests/CodeIndex.Tests/IndexCommandRunnerTests.cs index 334e795eac..8a9e44e6d0 100644 --- a/tests/CodeIndex.Tests/IndexCommandRunnerTests.cs +++ b/tests/CodeIndex.Tests/IndexCommandRunnerTests.cs @@ -1658,9 +1658,9 @@ def helper(): } [Fact] - public void Run_FullScanAfterHeadChange_ParallelizesExtraction() + public void Run_FullScanAfterHeadChange_DoesNotPreExtractUnchangedFiles() { - var projectRoot = TestProjectHelper.CreateTempProject("cdidx_head_changed_parallel_extract"); + var projectRoot = TestProjectHelper.CreateTempProject("cdidx_head_changed_skip_before_extract"); bool? parallelized = null; string? reason = null; try @@ -1673,8 +1673,8 @@ public void Run_FullScanAfterHeadChange_ParallelizesExtraction() var (initialExitCode, _) = RunAndCaptureJson([projectRoot, "--json"]); Assert.Equal(CommandExitCodes.Success, initialExitCode); - File.AppendAllText(Path.Combine(projectRoot, "app.cs"), "public class Next { public void Run() { } }\n"); - RunGit(projectRoot, "add", "app.cs"); + File.WriteAllText(Path.Combine(projectRoot, "feature.cs"), "public class Feature { public void Run() { } }\n"); + RunGit(projectRoot, "add", "feature.cs"); RunGit(projectRoot, "commit", "-m", "next"); IndexCommandRunner.FullScanExtractionSchedulingForTesting = (enabled, why) => @@ -1687,8 +1687,10 @@ public void Run_FullScanAfterHeadChange_ParallelizesExtraction() Assert.Equal(CommandExitCodes.Success, refreshExitCode); Assert.Equal("success", refreshJson.GetProperty("status").GetString()); - Assert.True(parallelized); - Assert.Equal("head_changed", reason); + Assert.True(refreshJson.GetProperty("head_changed").GetBoolean()); + Assert.False(parallelized); + Assert.Null(reason); + Assert.Equal(1, refreshJson.GetProperty("summary").GetProperty("files_skipped").GetInt32()); } finally { From 34e6979ff5c4afb9ba983e4efddcf05c4ec0be51 Mon Sep 17 00:00:00 2001 From: Widthdom Date: Mon, 15 Jun 2026 10:56:55 +0900 Subject: [PATCH 2/6] Optimize fresh full-scan indexing --- .../+fresh-index-reference-refresh.fixed.md | 15 +++++++ .../Cli/IndexCommandRunner.FullScan.cs | 29 ++++++++++--- src/CodeIndex/Database/DbWriter.cs | 22 ++++++---- .../IndexCommandRunnerTests.cs | 41 +++++++++++++++++++ 4 files changed, 94 insertions(+), 13 deletions(-) create mode 100644 changelog.d/unreleased/+fresh-index-reference-refresh.fixed.md diff --git a/changelog.d/unreleased/+fresh-index-reference-refresh.fixed.md b/changelog.d/unreleased/+fresh-index-reference-refresh.fixed.md new file mode 100644 index 0000000000..6d52b39d18 --- /dev/null +++ b/changelog.d/unreleased/+fresh-index-reference-refresh.fixed.md @@ -0,0 +1,15 @@ +--- +category: fixed +affected: + - src/CodeIndex/Cli/IndexCommandRunner.FullScan.cs + - src/CodeIndex/Database/DbWriter.cs + - tests/CodeIndex.Tests/IndexCommandRunnerTests.cs +--- + +## English + +- **Reduced first-time `cdidx .` indexing time.** Fresh full scans now defer mutual-recursion reference finalization until after bulk reference insertion instead of refreshing it after every file, and skip empty-database cleanup probes that cannot match existing rows. + +## 日本語 + +- **初回の `cdidx .` インデックス時間を短縮しました。** fresh full scan では、相互再帰参照の確定処理をファイルごとではなく参照の一括挿入後にまとめて行い、空DBでは既存行に一致しない cleanup probe も省略します。 diff --git a/src/CodeIndex/Cli/IndexCommandRunner.FullScan.cs b/src/CodeIndex/Cli/IndexCommandRunner.FullScan.cs index 683155d0a9..69cece5ca3 100644 --- a/src/CodeIndex/Cli/IndexCommandRunner.FullScan.cs +++ b/src/CodeIndex/Cli/IndexCommandRunner.FullScan.cs @@ -772,6 +772,7 @@ void ThrowIfFullScanCancelled(int filesProcessed, int? filesTotal) CancellationTokenSource? indexCts = null; int processed = 0, skipped = 0, warnings = warningList.Count, errors = errorList.Count; var symbolsDroppedByKindFilter = 0; + var mutualRecursionRefreshNeeded = false; var interactiveIndexSpinner = !options.Json && !options.Quiet && ConsoleUi.ShouldUseInteractiveConsole(); var redirectedIndexingMessagePrinted = false; @@ -787,7 +788,8 @@ void ThrowIfFullScanCancelled(int filesProcessed, int? filesTotal) var extractionParallelism = Math.Max(1, options.Parallelism); var hasPostExtractionHooks = postExtractionHooks.Hooks.Count > 0; var existingFileCount = writer.GetCounts().files; - var parallelizeExtraction = (options.Rebuild || existingFileCount == 0) + var startedWithNoIndexedFiles = existingFileCount == 0; + var parallelizeExtraction = (options.Rebuild || startedWithNoIndexedFiles) && !options.SymbolKindFilter.IsActive && !hasPostExtractionHooks; var parallelizeExtractionReason = parallelizeExtraction @@ -1156,7 +1158,7 @@ void StopJsonHeartbeat() } long? existingId = null; - if (!options.Rebuild) + if (!options.Rebuild && !startedWithNoIndexedFiles) { existingId = writer.GetUnchangedFileId( record.Path, @@ -1209,8 +1211,9 @@ void StopJsonHeartbeat() } using var txn = writer.BeginTransaction(); - writer.PurgeStaleFilesSharingChecksum(projectRoot, record.Path, record.Checksum); - var fileId = writer.UpsertFile(record); + if (!startedWithNoIndexedFiles) + writer.PurgeStaleFilesSharingChecksum(projectRoot, record.Path, record.Checksum); + var fileId = writer.UpsertFile(record, cleanExistingData: !startedWithNoIndexedFiles); currentJsonIndexFile = FormatIndexPhasePath(record.Path, "chunking"); var chunks = item.Chunks == null ? ChunkSplitter.Split(fileId, item.Content!) @@ -1289,7 +1292,9 @@ void StopJsonHeartbeat() cancellationToken) : ReassignReferenceFileIds(item.References, fileId); postExtractionHooks.OnReferencesExtracted(fileContext, AsMutableList(references)); - writer.InsertReferences(references); + writer.InsertReferences(references, refreshMutualRecursionFlags: false); + if (references.Count > 0) + mutualRecursionRefreshNeeded = true; currentJsonIndexFile = FormatIndexPhasePath(record.Path, "validating"); var issues = item.Issues ?? FileIndexer.ValidateContent(record.Path, item.RawBytes!, item.Content!, record.Lang); writer.InsertIssues(fileId, issues); @@ -1339,6 +1344,20 @@ void StopJsonHeartbeat() PauseIndexSpinnerForConsoleWrite(); + ThrowIfFullScanCancelled(processed, files.Count); + if (mutualRecursionRefreshNeeded) + { + WriteFullScanJsonLiveness(options, "finalizing reference graph..."); + var referenceGraphHeartbeat = StartFullScanJsonPhaseHeartbeat(options, "finalizing reference graph"); + try + { + writer.RefreshMutualRecursionFlags(); + } + finally + { + StopFullScanJsonPhaseHeartbeat(referenceGraphHeartbeat); + } + } ThrowIfFullScanCancelled(processed, files.Count); WriteFullScanJsonLiveness(options, "optimizing index..."); var optimizeHeartbeat = StartFullScanJsonPhaseHeartbeat(options, "optimizing index"); diff --git a/src/CodeIndex/Database/DbWriter.cs b/src/CodeIndex/Database/DbWriter.cs index d8970544e1..22126992fd 100644 --- a/src/CodeIndex/Database/DbWriter.cs +++ b/src/CodeIndex/Database/DbWriter.cs @@ -965,17 +965,22 @@ public bool HasFileAtPath(string relativePath) /// Upsert a file record and return its ID. /// Uses ON CONFLICT DO UPDATE to preserve the existing file ID (avoids /// unnecessary AUTOINCREMENT growth from INSERT OR REPLACE's delete+insert). - /// Cleans up old chunks/symbols before re-indexing. + /// Cleans up old chunks/symbols before re-indexing unless the caller knows + /// the path cannot already exist in the current database. /// ファイルレコードをUPSERTしてIDを返す。 /// ON CONFLICT DO UPDATEで既存IDを保持する（INSERT OR REPLACEの /// delete+insertによる不要なAUTOINCREMENT増加を回避）。 + /// 呼び出し元が現在のDBに同じ path が存在しないと保証できる場合を除き、 /// 再インデックス前に古いチャンク/シンボルをクリーンアップする。 /// - public long UpsertFile(FileRecord file) + public long UpsertFile(FileRecord file, bool cleanExistingData = true) { - // Clean up old chunks/symbols so new ones can be inserted - // 新しいチャンク/シンボル挿入のため古いデータをクリーンアップ - CleanExistingFileData(file.Path); + if (cleanExistingData) + { + // Clean up old chunks/symbols so new ones can be inserted + // 新しいチャンク/シンボル挿入のため古いデータをクリーンアップ + CleanExistingFileData(file.Path); + } // ON CONFLICT DO UPDATE preserves the existing row ID // ON CONFLICT DO UPDATEで既存の行IDを保持する @@ -1360,7 +1365,7 @@ AND s.signature LIKE '%static%' /// Insert indexed references in batches. /// インデックス済み参照をバッチ挿入する。 /// - public void InsertReferences(IReadOnlyList references) + public void InsertReferences(IReadOnlyList references, bool refreshMutualRecursionFlags = true) { if (references.Count == 0) return; @@ -1421,7 +1426,8 @@ INSERT INTO symbol_references ( transaction.Commit(); } - RefreshMutualRecursionFlags(); + if (refreshMutualRecursionFlags) + RefreshMutualRecursionFlags(); } private static void ValidateSymbolKinds(SymbolRecord symbol) @@ -1511,7 +1517,7 @@ FROM reference_lines return lineIds; } - private void RefreshMutualRecursionFlags() + internal void RefreshMutualRecursionFlags() { using var cmd = _conn.CreateCommand(); cmd.CommandText = @" diff --git a/tests/CodeIndex.Tests/IndexCommandRunnerTests.cs b/tests/CodeIndex.Tests/IndexCommandRunnerTests.cs index 8a9e44e6d0..d8effb318b 100644 --- a/tests/CodeIndex.Tests/IndexCommandRunnerTests.cs +++ b/tests/CodeIndex.Tests/IndexCommandRunnerTests.cs @@ -1567,6 +1567,38 @@ def helper(): } } + [Fact] + public void Run_FullScan_FinalizesMutualRecursionAfterBulkReferenceInsert() + { + var projectRoot = CreateTempProject(); + try + { + File.WriteAllText(Path.Combine(projectRoot, "cycle_a.cs"), """ + public static class FullScanCycleA + { + public static void CrossCycleA() { CrossCycleB(); } + } + """); + File.WriteAllText(Path.Combine(projectRoot, "cycle_b.cs"), """ + public static class FullScanCycleB + { + public static void CrossCycleB() { CrossCycleA(); } + } + """); + + var (exitCode, json) = RunAndCaptureJson([projectRoot, "--json"]); + + Assert.Equal(CommandExitCodes.Success, exitCode); + Assert.Equal("success", json.GetProperty("status").GetString()); + Assert.True(CountMutualRecursionReferences(Path.Combine(projectRoot, ".cdidx", "codeindex.db")) >= 2); + } + finally + { + SqliteConnection.ClearAllPools(); + DeleteDirectory(projectRoot); + } + } + [Fact] public void Run_FullScan_SkipsOversizedGitExclude() { @@ -11053,6 +11085,15 @@ private static int CountRows(string dbPath, string tableName) return Convert.ToInt32(command.ExecuteScalar()); } + private static int CountMutualRecursionReferences(string dbPath) + { + using var connection = new SqliteConnection($"Data Source={dbPath}"); + connection.Open(); + using var command = connection.CreateCommand(); + command.CommandText = "SELECT COUNT(*) FROM symbol_references WHERE is_mutual_recursion = 1"; + return Convert.ToInt32(command.ExecuteScalar()); + } + private static List ReadFileIssues(string dbPath, string kind) { using var connection = new SqliteConnection($"Data Source={dbPath}"); From e9c819342600a76b3323c4445c91cb8eaa959b86 Mon Sep 17 00:00:00 2001 From: Widthdom Date: Mon, 15 Jun 2026 11:45:07 +0900 Subject: [PATCH 3/6] Limit reference line lookups to insert batch --- .../+reference-line-lookup.fixed.md | 13 +++++++++++ src/CodeIndex/Database/DbWriter.cs | 22 ++++++++++--------- 2 files changed, 25 insertions(+), 10 deletions(-) create mode 100644 changelog.d/unreleased/+reference-line-lookup.fixed.md diff --git a/changelog.d/unreleased/+reference-line-lookup.fixed.md b/changelog.d/unreleased/+reference-line-lookup.fixed.md new file mode 100644 index 0000000000..03626cdb36 --- /dev/null +++ b/changelog.d/unreleased/+reference-line-lookup.fixed.md @@ -0,0 +1,13 @@ +--- +category: fixed +affected: + - src/CodeIndex/Database/DbWriter.cs +--- + +## English + +- **Reduced first-time reference indexing work.** Reference-line ID lookups now query only the current insert batch's exact `(file, line, context)` keys instead of rereading every reference line for the file on each batch. + +## 日本語 + +- **初回の reference indexing 処理量を削減しました。** reference line ID の取得時に、各 batch でファイル全体の reference line を読み直さず、現在の insert batch の `(file, line, context)` だけを正確に取得します。 diff --git a/src/CodeIndex/Database/DbWriter.cs b/src/CodeIndex/Database/DbWriter.cs index 22126992fd..8718bc74f3 100644 --- a/src/CodeIndex/Database/DbWriter.cs +++ b/src/CodeIndex/Database/DbWriter.cs @@ -1482,25 +1482,27 @@ private static void ValidateReferenceKinds(ReferenceRecord reference) cmd.ExecuteNonQuery(); } - var fileIds = contextsByLine.Keys.Select(key => key.FileId).Distinct().ToArray(); var lineIds = new Dictionary<(long FileId, int Line, string Context), long>(); - int fileIdsPerStatement = GetRowsPerInsertStatement(columnCount: 1); - for (int i = 0; i < fileIds.Length; i += fileIdsPerStatement) + int keysPerStatement = GetRowsPerInsertStatement(columnCount: 3); + for (int i = 0; i < rows.Length; i += keysPerStatement) { - int fileEnd = Math.Min(i + fileIdsPerStatement, fileIds.Length); + int keyEnd = Math.Min(i + keysPerStatement, rows.Length); using var cmd = _conn.CreateCommand(); - var parameters = new List(fileEnd - i); - for (int j = i; j < fileEnd; j++) + var predicates = new List(keyEnd - i); + for (int j = i; j < keyEnd; j++) { - var parameterName = $"@fid{j - i}"; - parameters.Add(parameterName); - cmd.Parameters.Add(parameterName, SqliteType.Integer).Value = fileIds[j]; + var suffix = j - i; + var ((fileId, line, _), context) = rows[j]; + predicates.Add($"(file_id = @lookupFid{suffix} AND line = @lookupLine{suffix} AND context = @lookupContext{suffix})"); + cmd.Parameters.Add($"@lookupFid{suffix}", SqliteType.Integer).Value = fileId; + cmd.Parameters.Add($"@lookupLine{suffix}", SqliteType.Integer).Value = line; + cmd.Parameters.Add($"@lookupContext{suffix}", SqliteType.Text).Value = context; } cmd.CommandText = $@" SELECT id, file_id, line, context FROM reference_lines - WHERE file_id IN ({string.Join(", ", parameters)})"; + WHERE {string.Join(" OR ", predicates)}"; using var reader = cmd.ExecuteReader(); while (reader.Read()) { From b12b0480289218b15765fac199b0bf138e313eca Mon Sep 17 00:00:00 2001 From: Widthdom Date: Mon, 15 Jun 2026 11:53:59 +0900 Subject: [PATCH 4/6] Optimize C# reference line-state scanning --- .../+csharp-reference-line-state.fixed.md | 14 ++ .../ReferenceExtractor.Preparation.cs | 11 +- .../ReferenceExtractor.TypeReferences.cs | 152 ++---------------- 3 files changed, 29 insertions(+), 148 deletions(-) create mode 100644 changelog.d/unreleased/+csharp-reference-line-state.fixed.md diff --git a/changelog.d/unreleased/+csharp-reference-line-state.fixed.md b/changelog.d/unreleased/+csharp-reference-line-state.fixed.md new file mode 100644 index 0000000000..81c7d3dec0 --- /dev/null +++ b/changelog.d/unreleased/+csharp-reference-line-state.fixed.md @@ -0,0 +1,14 @@ +--- +category: fixed +affected: + - src/CodeIndex/Indexer/References/ReferenceExtractor.Preparation.cs + - src/CodeIndex/Indexer/References/ReferenceExtractor.TypeReferences.cs +--- + +## English + +- **C# reference extraction now reuses one line-state scan** — Fresh full indexes avoid a duplicate pass over C# source lines when masking multiline string content and block comments, shaving CPU from first-time `cdidx .` runs. + +## 日本語 + +- **C# reference extraction が 1 回の行状態スキャンを再利用するようになりました** — 初回 `cdidx .` で multiline string content と block comment のマスク用に C# ソース行を二重走査しないようにし、CPU 使用量を削減しました。 diff --git a/src/CodeIndex/Indexer/References/ReferenceExtractor.Preparation.cs b/src/CodeIndex/Indexer/References/ReferenceExtractor.Preparation.cs index 8c40ab7e19..aa63dadf03 100644 --- a/src/CodeIndex/Indexer/References/ReferenceExtractor.Preparation.cs +++ b/src/CodeIndex/Indexer/References/ReferenceExtractor.Preparation.cs @@ -76,12 +76,11 @@ private static bool TryPrepareReferenceLines( : content; var lines = maskedContent.Split('\n'); var structuralLines = StructuralLineMasker.MaskLines(language, lines, out var jsTaggedTemplateHits); - var csharpLinesInsideMultilineStringContent = language == "csharp" - ? BuildCSharpMultilineStringContentLines(lines) - : null; - var csharpLinesInsideBlockComment = language == "csharp" - ? BuildCSharpBlockCommentLines(lines) - : null; + var csharpLineState = language == "csharp" + ? BuildCSharpLineStateMasks(lines) + : (MultilineStringContent: null, BlockComment: null); + var csharpLinesInsideMultilineStringContent = csharpLineState.MultilineStringContent; + var csharpLinesInsideBlockComment = csharpLineState.BlockComment; var referenceStructuralLines = language == "pascal" ? MaskPascalBlockCommentLines(structuralLines) : language == "haskell" diff --git a/src/CodeIndex/Indexer/References/ReferenceExtractor.TypeReferences.cs b/src/CodeIndex/Indexer/References/ReferenceExtractor.TypeReferences.cs index 8fee38f424..873c7c62f8 100644 --- a/src/CodeIndex/Indexer/References/ReferenceExtractor.TypeReferences.cs +++ b/src/CodeIndex/Indexer/References/ReferenceExtractor.TypeReferences.cs @@ -867,8 +867,9 @@ private static bool IsAtCSharpXmlDocAttachmentDepth( && !topLevelArrowExpressionContinuation); } - private static bool[] BuildCSharpBlockCommentLines(string[] lines) + private static (bool[] MultilineStringContent, bool[] BlockComment) BuildCSharpLineStateMasks(string[] lines) { + var insideStringContent = new bool[lines.Length]; var insideBlockComment = new bool[lines.Length]; var inBlockComment = false; var inVerbatimString = false; @@ -877,6 +878,7 @@ private static bool[] BuildCSharpBlockCommentLines(string[] lines) for (var i = 0; i < lines.Length; i++) { var line = lines[i]; + insideStringContent[i] = inVerbatimString || rawStringDelimiterLength > 0; insideBlockComment[i] = inBlockComment; var index = 0; @@ -1004,9 +1006,15 @@ private static bool[] BuildCSharpBlockCommentLines(string[] lines) } } - return insideBlockComment; + return (insideStringContent, insideBlockComment); } + private static bool[] BuildCSharpBlockCommentLines(string[] lines) => + BuildCSharpLineStateMasks(lines).BlockComment; + + private static bool[] BuildCSharpMultilineStringContentLines(string[] lines) => + BuildCSharpLineStateMasks(lines).MultilineStringContent; + private static bool IsCSharpTopLevelAssignmentOperator(string line, int index) { if (index < 0 || index >= line.Length || line[index] != '=') @@ -1506,146 +1514,6 @@ private static int FindSignatureOccurrenceStartColumn(string structuralLine, str return -1; } - private static bool[] BuildCSharpMultilineStringContentLines(string[] lines) - { - var insideStringContent = new bool[lines.Length]; - var inBlockComment = false; - var inVerbatimString = false; - var rawStringDelimiterLength = 0; - - for (var i = 0; i < lines.Length; i++) - { - var line = lines[i]; - insideStringContent[i] = inVerbatimString || rawStringDelimiterLength > 0; - - var index = 0; - while (index < line.Length) - { - if (inBlockComment) - { - var closeIndex = line.IndexOf("*/", index, StringComparison.Ordinal); - if (closeIndex < 0) - break; - - index = closeIndex + 2; - inBlockComment = false; - continue; - } - - if (rawStringDelimiterLength > 0) - { - var closeCandidateIndex = index; - while (closeCandidateIndex < line.Length && char.IsWhiteSpace(line[closeCandidateIndex])) - closeCandidateIndex++; - - var closeLength = CountCharacterRun(line, closeCandidateIndex, '"'); - if (closeLength >= rawStringDelimiterLength - && closeLength > 0) - { - rawStringDelimiterLength = 0; - index = closeCandidateIndex + closeLength; - continue; - } - - break; - } - - if (inVerbatimString) - { - if (line[index] == '"' && index + 1 < line.Length && line[index + 1] == '"') - { - index += 2; - continue; - } - - if (line[index] == '"') - { - index++; - inVerbatimString = false; - continue; - } - - index++; - continue; - } - - if (StartsWithOrdinal(line, index, "//")) - break; - - if (StartsWithOrdinal(line, index, "/*")) - { - inBlockComment = true; - index += 2; - continue; - } - - if (TryStartCSharpRawString(line, index, out var rawOpeningLength, out var rawDelimiterLength)) - { - rawStringDelimiterLength = rawDelimiterLength; - index += rawOpeningLength; - continue; - } - - if (TryStartCSharpVerbatimString(line, index, out var verbatimOpeningLength)) - { - inVerbatimString = true; - index += verbatimOpeningLength; - continue; - } - - if (TryStartCSharpRegularString(line, index, out var regularOpeningLength)) - { - index += regularOpeningLength; - while (index < line.Length) - { - if (line[index] == '\\') - { - index += Math.Min(2, line.Length - index); - continue; - } - - if (line[index] == '"') - { - index++; - break; - } - - index++; - } - - continue; - } - - if (line[index] == '\'') - { - index++; - while (index < line.Length) - { - if (line[index] == '\\') - { - index += Math.Min(2, line.Length - index); - continue; - } - - if (line[index] == '\'') - { - index++; - break; - } - - index++; - } - - continue; - } - - index++; - } - } - - return insideStringContent; - } - private static bool TryStartCSharpRawString( string line, int startIndex, From 77e87b8a7a0b87c67870de3c2cfe71f8d42f5f42 Mon Sep 17 00:00:00 2001 From: Widthdom Date: Mon, 15 Jun 2026 12:18:04 +0900 Subject: [PATCH 5/6] Add symbols-only indexing mode (#3607) --- README.md | 8 + USER_GUIDE.md | 2 + .../unreleased/+symbols-only-index.added.md | 20 +++ src/CodeIndex/Cli/CliFlagSchema.cs | 1 + src/CodeIndex/Cli/ConsoleUi.cs | 3 +- .../Cli/IndexCommandRunner.FullScan.cs | 139 ++++++++++++------ src/CodeIndex/Cli/IndexCommandRunner.Parse.cs | 7 +- .../Cli/IndexCommandRunner.Update.cs | 1 + .../Cli/IndexCommandRunner.Validation.cs | 11 ++ src/CodeIndex/Cli/IndexCommandRunner.cs | 3 +- src/CodeIndex/Database/DbWriter.cs | 22 +++ tests/CodeIndex.Tests/ConsoleUiTests.cs | 2 +- .../IndexCommandRunnerTests.cs | 109 ++++++++++++++ 13 files changed, 275 insertions(+), 53 deletions(-) create mode 100644 changelog.d/unreleased/+symbols-only-index.added.md diff --git a/README.md b/README.md index d764dadccb..44315c860f 100644 --- a/README.md +++ b/README.md @@ -85,6 +85,10 @@ CI checkouts, refresh with `cdidx .`, `--files`, `--commits`, or and [incremental update reliability](USER_GUIDE.md#incremental-update-reliability) for the full workflow. +For a faster first pass when you only need text search, `definition`, `symbols`, +or `map`, run `cdidx . --symbols-only`. Reference graph commands remain degraded +until you rerun `cdidx .` without that flag. + ## Highlights | Area | What to use | @@ -255,6 +259,10 @@ cdidx lsp --db .cdidx/codeindex.db 全体の流れは [ユーザーガイドのクイックスタート](USER_GUIDE.md#クイックスタート) と [インクリメンタル更新の信頼性](USER_GUIDE.md#インクリメンタル更新の信頼性) を参照してください。 +まず text search、`definition`、`symbols`、`map` だけを速く使いたい場合は +`cdidx . --symbols-only` を使えます。reference graph 系コマンドは、このフラグなしで +`cdidx .` を再実行するまで degraded のままです。 + ## 特長 | 分野 | 使うもの | diff --git a/USER_GUIDE.md b/USER_GUIDE.md index 4e0c6579c9..9b81ddcea8 100644 --- a/USER_GUIDE.md +++ b/USER_GUIDE.md @@ -1395,6 +1395,7 @@ same source location. | `--duration-format ` | `index` | Choose human elapsed-time display for index summaries. `auto` (default) uses unit labels; `seconds` emits decimal seconds; `hms` keeps `HH:MM:SS`. JSON always keeps raw `elapsed_ms`. | | `--max-file-bytes ` | `index` | Override the per-file indexing limit for this run. Defaults to 4MiB, or `CDIDX_MAX_FILE_BYTES` when set. Values accept raw bytes or `K` / `M` / `G` suffixes such as `50M`. | | `--max-symbols-per-file ` | `index` | Skip file content, symbols, and references when one file emits too many symbols. Defaults to `5000`; values above `50000` are rejected. | +| `--symbols-only` | `index` | Full-scan only. Build chunks, symbols, and issues while skipping reference extraction and graph finalization for a faster first pass. `search`, `definition`, `symbols`, and `map` are available; reference graph commands remain degraded until a normal `cdidx index ` run. | | `--parallelism ` | `index` | Set full-scan extraction worker count. Defaults to CPU count capped at 16, or `CDIDX_INDEX_PARALLELISM` when set. SQLite writes stay single-consumer. | | `--watch` | `index` | After the initial scan completes, stay running and reindex incrementally as files change (FileSystemWatcher / inotify / FSEvents). Rejects `--commits`, `--changed-between`, `--files`, and `--dry-run` because the loop already drives continuous incremental updates. | | `--debounce ` | `index` (watch only) | Coalesce bursts of file events into a single update after `` of quiet (non-negative integer; default: 500). Invalid values emit a warning and are ignored. | @@ -3881,6 +3882,7 @@ raw match density を正確に測る、といった理由で全 raw chunk hit | `--force` | `index` | 同一 DB に対する index ロックを bypass する。他の `cdidx index` が走っていないと確信できる場合のみ使う。並行実行は schema を破壊し得る。 | | `--duration-format ` | `index` | index summary の human 経過時間表示を選ぶ。`auto`（既定）は単位付き、`seconds` は小数秒、`hms` は `HH:MM:SS` を維持。JSON は常に raw の `elapsed_ms` を返す。 | | `--max-file-bytes ` | `index` | この実行で使うファイル単位の索引サイズ上限を上書きする。既定は 4MiB、または `CDIDX_MAX_FILE_BYTES` 設定値。値は raw byte 数、または `50M` のような `K` / `M` / `G` 接尾辞を受け付ける。 | +| `--symbols-only` | `index` | フルスキャン専用。参照抽出と graph finalization を省き、chunks、symbols、issues だけを作ることで初回利用を速くする。`search`、`definition`、`symbols`、`map` は使えるが、reference graph 系コマンドは通常の `cdidx index ` を実行するまで degraded のまま。 | | `--parallelism ` | `index` | フルスキャンの抽出 worker 数を指定する。既定は CPU 数を最大 16 に丸めた値、または `CDIDX_INDEX_PARALLELISM` 設定値。SQLite 書き込みは単一 consumer のまま。 | | `--watch` | `index` | 初回スキャン完了後もプロセスを残し、ファイル変更を検知して差分更新を繰り返す（FileSystemWatcher / inotify / FSEvents）。連続的な差分更新を内蔵しているため `--commits` / `--changed-between` / `--files` / `--dry-run` との併用は拒否する。 | | `--debounce ` | `index`（`--watch` 専用） | 一連のイベントを `` の静止後に 1 つの更新へ集約する（0 以上の整数。既定: 500）。不正な値は警告を出して無視する。 | diff --git a/changelog.d/unreleased/+symbols-only-index.added.md b/changelog.d/unreleased/+symbols-only-index.added.md new file mode 100644 index 0000000000..c0fe1e5aa6 --- /dev/null +++ b/changelog.d/unreleased/+symbols-only-index.added.md @@ -0,0 +1,20 @@ +--- +category: added +affected: + - src/CodeIndex/Cli/IndexCommandRunner.FullScan.cs + - src/CodeIndex/Cli/IndexCommandRunner.Parse.cs + - src/CodeIndex/Cli/IndexCommandRunner.Validation.cs + - src/CodeIndex/Cli/ConsoleUi.cs + - src/CodeIndex/Cli/CliFlagSchema.cs + - src/CodeIndex/Database/DbWriter.cs + - README.md + - USER_GUIDE.md +--- + +## English + +- **Added `cdidx index --symbols-only` for fast first-pass indexing** — full scans can now build chunks, symbols, and issues while skipping reference graph extraction, so search, definition, symbols, and map workflows become usable sooner; graph commands stay degraded until a normal index run. + +## 日本語 + +- **高速な初回 index 向けに `cdidx index --symbols-only` を追加しました** — フルスキャンで chunks、symbols、issues だけを作り、reference graph 抽出を省けるため、search、definition、symbols、map をより早く使い始められます。graph 系コマンドは通常の index 実行まで degraded のままです。 diff --git a/src/CodeIndex/Cli/CliFlagSchema.cs b/src/CodeIndex/Cli/CliFlagSchema.cs index 4914f5edcc..e531d0dcda 100644 --- a/src/CodeIndex/Cli/CliFlagSchema.cs +++ b/src/CodeIndex/Cli/CliFlagSchema.cs @@ -348,6 +348,7 @@ private static IReadOnlyList BuildAll() new() { Name = "--integrity-check", Description = "Run PRAGMA integrity_check on the database", Commands = Set("db") }, new() { Name = "--rebuild", Description = "Delete existing DB and rebuild from scratch", Commands = Set("index") }, new() { Name = "--optimize", Description = "Optimize the existing FTS5 table without scanning files", Commands = Set("index") }, + new() { Name = "--symbols-only", Description = "Build chunks and symbols while skipping reference graph extraction", Commands = Set("index") }, new() { Name = "--dry-run", Description = "Preview without writing", Commands = Set("index", "backfill-fold", "vacuum") }, new() { Name = "--no-checkpoint", Description = "Skip the automatic DB checkpoint before maintenance", Commands = Set("backfill-fold") }, new() { Name = "--force", Description = "Bypass the per-database index lock", Commands = Set("index") }, diff --git a/src/CodeIndex/Cli/ConsoleUi.cs b/src/CodeIndex/Cli/ConsoleUi.cs index a76101a56c..e8d406f9df 100644 --- a/src/CodeIndex/Cli/ConsoleUi.cs +++ b/src/CodeIndex/Cli/ConsoleUi.cs @@ -80,7 +80,7 @@ public static class ConsoleUi private static readonly (string Command, string Usage)[] CommandUsageLines = [ - ("index", "cdidx index [--db ] [--rebuild] [--optimize] [--verbose] [--dry-run] [--force] [--quiet] [--json] [--memory-trace] [--duration-format ] [--notify ] [--max-file-bytes ] [--max-symbols-per-file ] [--follow-symlinks ] [--include-symbol-kind [,]] [--exclude-symbol-kind [,]] [--watch [--debounce ]]"), + ("index", "cdidx index [--db ] [--rebuild] [--optimize] [--symbols-only] [--verbose] [--dry-run] [--force] [--quiet] [--json] [--memory-trace] [--duration-format ] [--notify ] [--max-file-bytes ] [--max-symbols-per-file ] [--follow-symlinks ] [--include-symbol-kind [,]] [--exclude-symbol-kind [,]] [--watch [--debounce ]]"), ("hooks", "cdidx hooks [--project ] [--force] [--json]"), ("backfill-fold", "cdidx backfill-fold [--db ] [--dry-run] [--no-checkpoint] [--json]"), ("optimize", "cdidx optimize [--db ] [--json]"), @@ -982,6 +982,7 @@ private static void PrintFlagReference(Action WriteHelpLine) Console.WriteLine(" --verbose Show per-file status ([OK ]/[SKIP]/[DEL ]/[ERR ])"); Console.WriteLine(" --dry-run Scan files without writing to the database"); Console.WriteLine(" --force Bypass the per-database index lock; only use when no other cdidx index is active"); + WriteHelpLine(" --symbols-only Build chunks and symbols but skip reference extraction; graph queries stay degraded until a normal index run"); Console.WriteLine(" --json Output results as JSON (for AI/machine use)"); Console.WriteLine(" --memory-trace Include phase memory samples in index JSON output"); Console.WriteLine(" --quiet, -q, --silent Suppress informational stderr output; errors still print (also honors CDIDX_QUIET=1)"); diff --git a/src/CodeIndex/Cli/IndexCommandRunner.FullScan.cs b/src/CodeIndex/Cli/IndexCommandRunner.FullScan.cs index 69cece5ca3..b7cdb98887 100644 --- a/src/CodeIndex/Cli/IndexCommandRunner.FullScan.cs +++ b/src/CodeIndex/Cli/IndexCommandRunner.FullScan.cs @@ -578,6 +578,7 @@ private static int RunFullScan( DateTime runStartedAtUtc, string[] spinnerFrames, JsonSerializerOptions jsonOptions, + int priorReadiness, string? priorFoldVersion, string? priorFoldFingerprint, bool priorSymbolExtractorVersionsMatchCurrent, @@ -612,6 +613,7 @@ private static int RunFullScan( var csharpSymbolNameContractMatchesCurrent = priorCSharpSymbolNameContractVersion == currentCSharpSymbolNameContractVersion; var currentSqlGraphContractVersion = DbContext.SqlGraphContractVersion.ToString(System.Globalization.CultureInfo.InvariantCulture); var sqlGraphContractMatchesCurrent = priorSqlGraphContractVersion == currentSqlGraphContractVersion; + var priorGraphReady = (priorReadiness & DbContext.GraphReadyFlag) != 0; var hotspotFamilyTrustMatchesCurrent = GetHotspotFamilyTrustMatchesCurrent( priorHotspotFamilyVersions, priorHotspotFamilyMarkerFingerprints, @@ -682,6 +684,8 @@ void ThrowIfFullScanCancelled(int filesProcessed, int? filesTotal) writer.MarkBatchInProgress(); writer.ClearReadyFlags(); writer.ClearHotspotFamilyReady(); + if (options.SymbolsOnly) + writer.ClearSqlGraphContractReady(); writer.ClearMetadataTargetReady(); FullScanWritePhaseStartedForTesting?.Invoke(); ThrowIfFullScanCancelled(0, files.Count); @@ -763,11 +767,19 @@ void ThrowIfFullScanCancelled(int filesProcessed, int? filesTotal) ConsoleUi.PrintWarning("Skipped authoritative purge outside directories whose file listing completed successfully because some paths could not be scanned."); } - // Purge references for languages no longer graph-supported / グラフ非対応になった言語の参照をパージ + // Purge references for languages no longer graph-supported, or all references in + // symbols-only mode so old graph rows cannot survive behind degraded readiness. + // グラフ非対応になった言語の参照をパージする。symbols-only では古い graph 行が + // degraded readiness の裏に残らないよう全参照を消す。 ThrowIfFullScanCancelled(0, files.Count); - var purgedRefs = writer.PurgeUnsupportedReferences(ReferenceExtractor.GetSupportedLanguages()); + var purgedRefs = options.SymbolsOnly + ? writer.PurgeAllReferences() + : writer.PurgeUnsupportedReferences(ReferenceExtractor.GetSupportedLanguages()); if (purgedRefs > 0 && !options.Json && !options.Quiet) - Console.WriteLine($" Purged {purgedRefs:N0} stale references (unsupported language)"); + { + var reason = options.SymbolsOnly ? "symbols-only mode" : "unsupported language"; + Console.WriteLine($" Purged {purgedRefs:N0} stale references ({reason})"); + } CancellationTokenSource? indexCts = null; int processed = 0, skipped = 0, warnings = warningList.Count, errors = errorList.Count; @@ -928,31 +940,38 @@ void StopJsonHeartbeat() jsonHeartbeatTask = null; } - WriteFullScanJsonLiveness(options, "preparing C# workspace symbols..."); - string? currentCSharpWorkspaceFile = null; - var csharpWorkspaceHeartbeat = StartFullScanJsonPhaseHeartbeat( - options, - "preparing C# workspace symbols", - () => currentCSharpWorkspaceFile); CSharpStaticInterfaceWorkspaceSymbols csharpWorkspace; - try - { - csharpWorkspace = BuildCSharpStaticInterfaceWorkspaceSymbols( - writer, - indexer, - projectRoot, - files, - path => currentCSharpWorkspaceFile = path, - cancellationToken); - } - catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + if (options.SymbolsOnly) { - throw new IndexInterruptedException(0, files.Count); + csharpWorkspace = new CSharpStaticInterfaceWorkspaceSymbols([], false); } - finally + else { - currentCSharpWorkspaceFile = null; - StopFullScanJsonPhaseHeartbeat(csharpWorkspaceHeartbeat); + WriteFullScanJsonLiveness(options, "preparing C# workspace symbols..."); + string? currentCSharpWorkspaceFile = null; + var csharpWorkspaceHeartbeat = StartFullScanJsonPhaseHeartbeat( + options, + "preparing C# workspace symbols", + () => currentCSharpWorkspaceFile); + try + { + csharpWorkspace = BuildCSharpStaticInterfaceWorkspaceSymbols( + writer, + indexer, + projectRoot, + files, + path => currentCSharpWorkspaceFile = path, + cancellationToken); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw new IndexInterruptedException(0, files.Count); + } + finally + { + currentCSharpWorkspaceFile = null; + StopFullScanJsonPhaseHeartbeat(csharpWorkspaceHeartbeat); + } } EnsureIndexingActivityVisible(); @@ -1017,15 +1036,22 @@ void StopJsonHeartbeat() continue; } SymbolExtractor.ApplyFamilyScope(symbols, indexer.GetFamilyScopeKey(filePath, record.Lang)); - activeJsonExtractionPhases[workerIndex] = FormatIndexPhasePath(record.Path, "references"); - references = ReferenceExtractor.Extract( - 0, - record.Lang, - content, - symbols, - record.Path, - record.Lang == "csharp" ? csharpWorkspace.Symbols : null, - extractionCancellationToken); + if (options.SymbolsOnly) + { + references = []; + } + else + { + activeJsonExtractionPhases[workerIndex] = FormatIndexPhasePath(record.Path, "references"); + references = ReferenceExtractor.Extract( + 0, + record.Lang, + content, + symbols, + record.Path, + record.Lang == "csharp" ? csharpWorkspace.Symbols : null, + extractionCancellationToken); + } activeJsonExtractionPhases[workerIndex] = FormatIndexPhasePath(record.Path, "validating"); issues = FileIndexer.ValidateContent(record.Path, rawBytes, content, record.Lang); } @@ -1158,7 +1184,7 @@ void StopJsonHeartbeat() } long? existingId = null; - if (!options.Rebuild && !startedWithNoIndexedFiles) + if (!options.Rebuild && !startedWithNoIndexedFiles && !options.SymbolsOnly) { existingId = writer.GetUnchangedFileId( record.Path, @@ -1169,6 +1195,7 @@ void StopJsonHeartbeat() language: record.Lang, generated: record.Generated, allowReuse: symbolKindFilterMatchesPrior + && priorGraphReady && record.Lang is not ("javascript" or "typescript") && (record.Lang != "csharp" || csharpSymbolNameContractMatchesCurrent) && (record.Lang != "csharp" || !csharpWorkspace.HasStaticInterfaceContracts) @@ -1281,17 +1308,25 @@ void StopJsonHeartbeat() FileIndexer.ValidateSymbolLineRanges(record, symbols); writer.InsertSymbols(symbols); currentJsonIndexFile = FormatIndexPhasePath(record.Path, "references"); - var references = item.References == null - ? ReferenceExtractor.Extract( - fileId, - record.Lang, - item.Content!, - symbols, - record.Path, - record.Lang == "csharp" ? csharpWorkspace.Symbols : null, - cancellationToken) - : ReassignReferenceFileIds(item.References, fileId); - postExtractionHooks.OnReferencesExtracted(fileContext, AsMutableList(references)); + IReadOnlyList references; + if (options.SymbolsOnly) + { + references = []; + } + else + { + references = item.References == null + ? ReferenceExtractor.Extract( + fileId, + record.Lang, + item.Content!, + symbols, + record.Path, + record.Lang == "csharp" ? csharpWorkspace.Symbols : null, + cancellationToken) + : ReassignReferenceFileIds(item.References, fileId); + postExtractionHooks.OnReferencesExtracted(fileContext, AsMutableList(references)); + } writer.InsertReferences(references, refreshMutualRecursionFlags: false); if (references.Count > 0) mutualRecursionRefreshNeeded = true; @@ -1388,9 +1423,12 @@ void StopJsonHeartbeat() // backfill verification below because incremental-by-default full scans skip // unchanged legacy files whose folded columns remain NULL. // full-scan は全repo をカバーするため、Graph / Issues は常に stamp。Fold のみ条件付き。 - writer.MarkGraphReady(); writer.MarkIssuesReady(); - writer.MarkSqlGraphContractReady(); + if (!options.SymbolsOnly) + { + writer.MarkGraphReady(); + writer.MarkSqlGraphContractReady(); + } writer.MarkCSharpSymbolNameContractReady(); // Issue #435: resolve every C# class-like row and stamp readiness. Full-scan // touches the entire repo, so the resolver output is authoritative regardless @@ -1407,10 +1445,13 @@ void StopJsonHeartbeat() { csharpMetadataTargetReadyAfter = true; } - graphTableAvailableAfter = true; + graphTableAvailableAfter = !options.SymbolsOnly; issuesTableAvailableAfter = true; csharpSymbolNameReadyAfter = true; - writer.RebuildTypeScriptAugmentationReferences(projectRoot); + if (!options.SymbolsOnly) + { + writer.RebuildTypeScriptAugmentationReferences(projectRoot); + } RestampHotspotFamilyTrustForFullScan( writer, reusedHotspotFamilyLanguages, diff --git a/src/CodeIndex/Cli/IndexCommandRunner.Parse.cs b/src/CodeIndex/Cli/IndexCommandRunner.Parse.cs index 3efcd213f8..009c78ff9b 100644 --- a/src/CodeIndex/Cli/IndexCommandRunner.Parse.cs +++ b/src/CodeIndex/Cli/IndexCommandRunner.Parse.cs @@ -15,7 +15,7 @@ public static partial class IndexCommandRunner "--db", "--data-dir", "--rebuild", "--verbose", "--json", "--quiet", "--dry-run", "--force", "--yes", "--watch", "--debounce", "--duration-format", "--max-file-bytes", "--max-symbols-per-file", "--notify", - "--parallelism", "--memory-trace", "--follow-symlinks", + "--parallelism", "--memory-trace", "--follow-symlinks", "--symbols-only", "--commits", "--changed-between", "--files", "--solution", "--project", "--include-symbol-kind", "--exclude-symbol-kind", "--optimize", "--help", "--read-only", "--immutable", @@ -42,6 +42,7 @@ public static IndexCommandOptions ParseArgs(string[] args) bool yes = false; bool watch = false; bool optimizeOnly = false; + bool symbolsOnly = false; bool memoryTrace = false; int? watchDebounceMs = null; var durationFormat = DurationOutputFormat.Auto; @@ -123,6 +124,9 @@ public static IndexCommandOptions ParseArgs(string[] args) case "--optimize": optimizeOnly = true; break; + case "--symbols-only": + symbolsOnly = true; + break; case "--memory-trace": memoryTrace = true; break; @@ -317,6 +321,7 @@ public static IndexCommandOptions ParseArgs(string[] args) Yes = yes, Watch = watch, OptimizeOnly = optimizeOnly, + SymbolsOnly = symbolsOnly, MemoryTrace = memoryTrace, WatchDebounceMs = watchDebounceMs, DurationFormat = durationFormat, diff --git a/src/CodeIndex/Cli/IndexCommandRunner.Update.cs b/src/CodeIndex/Cli/IndexCommandRunner.Update.cs index 0d654a038e..4e10cba8ca 100644 --- a/src/CodeIndex/Cli/IndexCommandRunner.Update.cs +++ b/src/CodeIndex/Cli/IndexCommandRunner.Update.cs @@ -94,6 +94,7 @@ private static int RunUpdateMode( runStartedAtUtc, spinnerFrames, jsonOptions, + priorReadiness, priorFoldVersion, priorFoldFingerprint, priorSymbolExtractorVersionsMatchCurrent, diff --git a/src/CodeIndex/Cli/IndexCommandRunner.Validation.cs b/src/CodeIndex/Cli/IndexCommandRunner.Validation.cs index 234fbf11eb..8df10c8f85 100644 --- a/src/CodeIndex/Cli/IndexCommandRunner.Validation.cs +++ b/src/CodeIndex/Cli/IndexCommandRunner.Validation.cs @@ -30,6 +30,17 @@ public static partial class IndexCommandRunner CommandErrorCodes.UsageError); } + if (options.SymbolsOnly && (options.DryRun || options.Watch || isUpdateMode || options.OptimizeOnly)) + { + return WriteCommandError( + options.Json, + jsonOptions, + "--symbols-only can only be combined with a full index scan; it cannot be used with --dry-run, --watch, --commits, --changed-between, --files, or --optimize", + CommandExitCodes.UsageError, + "Use `cdidx index --symbols-only` for a fast symbol/search-only bootstrap, then rerun `cdidx index ` when reference graph queries are needed.", + CommandErrorCodes.UsageError); + } + if (options.Rebuild && isUpdateMode) return WriteRebuildUpdateModeConflict(options, jsonOptions); diff --git a/src/CodeIndex/Cli/IndexCommandRunner.cs b/src/CodeIndex/Cli/IndexCommandRunner.cs index ea02e440a5..403622014e 100644 --- a/src/CodeIndex/Cli/IndexCommandRunner.cs +++ b/src/CodeIndex/Cli/IndexCommandRunner.cs @@ -277,7 +277,7 @@ internal static int Run(string[] indexArgs, JsonSerializerOptions jsonOptions, C initialExitCode = isUpdateMode ? RunUpdateMode(writer, indexer, projectRoot, resolvedDbPath, options, stopwatch, runStartedAtUtc, spinnerFrames, jsonOptions, priorReadiness, priorFoldVersion, priorFoldFingerprint, priorSymbolExtractorVersionsMatchCurrent, priorCSharpSymbolNameContractVersion, priorMetadataTargetCsharp, priorSqlGraphContractVersion, priorHotspotFamilyVersions, priorHotspotFamilyMarkerFingerprints, currentHotspotFamilyMarkerFingerprints, priorIndexedProjectRoot, priorIndexedHeadCommit, currentHeadCommit, priorSymbolKindFilterSignature, initialCwd, indexRunDiagnostics, indexCancellation.Token) - : RunFullScan(writer, indexer, projectRoot, resolvedDbPath, options, stopwatch, runStartedAtUtc, spinnerFrames, jsonOptions, priorFoldVersion, priorFoldFingerprint, priorSymbolExtractorVersionsMatchCurrent, priorCSharpSymbolNameContractVersion, priorMetadataTargetCsharp, priorSqlGraphContractVersion, priorHotspotFamilyVersions, priorHotspotFamilyMarkerFingerprints, currentHotspotFamilyMarkerFingerprints, priorIndexedProjectRoot, priorIndexedHeadCommit, currentHeadCommit, priorSymbolKindFilterSignature, initialCwd, indexRunDiagnostics, showNextSteps: !databaseExistedBeforeIndex, indexCancellation.Token); + : RunFullScan(writer, indexer, projectRoot, resolvedDbPath, options, stopwatch, runStartedAtUtc, spinnerFrames, jsonOptions, priorReadiness, priorFoldVersion, priorFoldFingerprint, priorSymbolExtractorVersionsMatchCurrent, priorCSharpSymbolNameContractVersion, priorMetadataTargetCsharp, priorSqlGraphContractVersion, priorHotspotFamilyVersions, priorHotspotFamilyMarkerFingerprints, currentHotspotFamilyMarkerFingerprints, priorIndexedProjectRoot, priorIndexedHeadCommit, currentHeadCommit, priorSymbolKindFilterSignature, initialCwd, indexRunDiagnostics, showNextSteps: !databaseExistedBeforeIndex, indexCancellation.Token); if (initialExitCode == CommandExitCodes.Success) db.RunPlannerStatisticsMaintenance(forceAnalyze: !databaseExistedBeforeIndex); } @@ -1582,6 +1582,7 @@ public sealed class IndexCommandOptions public bool Yes { get; init; } public bool Watch { get; init; } public bool OptimizeOnly { get; init; } + public bool SymbolsOnly { get; init; } public int? WatchDebounceMs { get; init; } public DurationOutputFormat DurationFormat { get; init; } = DurationOutputFormat.Auto; public CompletionNotificationMode NotifyMode { get; init; } = CompletionNotificationMode.Auto; diff --git a/src/CodeIndex/Database/DbWriter.cs b/src/CodeIndex/Database/DbWriter.cs index 8718bc74f3..2175e773b3 100644 --- a/src/CodeIndex/Database/DbWriter.cs +++ b/src/CodeIndex/Database/DbWriter.cs @@ -2042,6 +2042,23 @@ AND f.lang NOT IN ({string.Join(", ", inParams)}) return cmd.ExecuteNonQuery(); } + /// + /// Delete all reference graph rows while preserving files, chunks, symbols, and issues. + /// files / chunks / symbols / issues は残し、参照グラフ行だけを全削除する。 + /// + public int PurgeAllReferences() + { + using var referenceCmd = _conn.CreateCommand(); + referenceCmd.CommandText = "DELETE FROM symbol_references"; + var deletedReferences = referenceCmd.ExecuteNonQuery(); + + using var lineCmd = _conn.CreateCommand(); + lineCmd.CommandText = "DELETE FROM reference_lines"; + lineCmd.ExecuteNonQuery(); + + return deletedReferences; + } + /// /// Get total counts for the summary output. /// サマリー出力用の合計件数を取得する。 @@ -2232,6 +2249,11 @@ public void MarkSqlGraphContractReady() DbContext.SqlGraphContractVersion.ToString(System.Globalization.CultureInfo.InvariantCulture)); } + public void ClearSqlGraphContractReady() + { + SetMeta(DbContext.SqlGraphContractVersionMetaKey, null); + } + /// /// Stamp the current authoritative version for hotspot family grouping semantics. /// Only fully authoritative DB states should call this; mixed legacy/current DBs must diff --git a/tests/CodeIndex.Tests/ConsoleUiTests.cs b/tests/CodeIndex.Tests/ConsoleUiTests.cs index 9e50f7720f..795f0ea7d8 100644 --- a/tests/CodeIndex.Tests/ConsoleUiTests.cs +++ b/tests/CodeIndex.Tests/ConsoleUiTests.cs @@ -103,7 +103,7 @@ public void PrintUsage_WithoutBanner_HidesAsciiArtAndEasterEggFlags() Assert.DoesNotContain("██████╗", output); Assert.Contains("Usage:", output); - Assert.Contains("cdidx index [--db ] [--rebuild] [--optimize] [--verbose] [--dry-run] [--force] [--quiet] [--json] [--memory-trace] [--duration-format ] [--notify ] [--max-file-bytes ] [--max-symbols-per-file ] [--follow-symlinks ]", output); + Assert.Contains("cdidx index [--db ] [--rebuild] [--optimize] [--symbols-only] [--verbose] [--dry-run] [--force] [--quiet] [--json] [--memory-trace] [--duration-format ] [--notify ] [--max-file-bytes ] [--max-symbols-per-file ] [--follow-symlinks ]", output); Assert.Contains("cdidx hooks [--project ] [--force] [--json]", output); Assert.Contains("cdidx index --commits [commit-ref ...] [--db ] [--verbose] [--dry-run] [--json] [--memory-trace] [--duration-format ]", output); Assert.Contains("cdidx index --files [path ...] [--db ] [--verbose] [--dry-run] [--json] [--memory-trace] [--duration-format ]", output); diff --git a/tests/CodeIndex.Tests/IndexCommandRunnerTests.cs b/tests/CodeIndex.Tests/IndexCommandRunnerTests.cs index d8effb318b..44fdab716e 100644 --- a/tests/CodeIndex.Tests/IndexCommandRunnerTests.cs +++ b/tests/CodeIndex.Tests/IndexCommandRunnerTests.cs @@ -1126,6 +1126,89 @@ public void Run_FileAboveMaxSymbolsPerFile_PersistsSymbolCountExceededIssueOnly( } } + [Fact] + public void Run_SymbolsOnly_FullScanSkipsReferenceGraphUntilNormalIndex() + { + var projectRoot = CreateTempProject(); + try + { + File.WriteAllText( + Path.Combine(projectRoot, "app.cs"), + "public class App { public void Run() { Helper(); } private void Helper() { } }\n"); + File.WriteAllText( + Path.Combine(projectRoot, "query.sql"), + "CREATE TABLE users (id INTEGER PRIMARY KEY);\nSELECT id FROM users;\n"); + + var (symbolsOnlyExitCode, symbolsOnlyJson) = RunAndCaptureJson([projectRoot, "--symbols-only", "--json"]); + + Assert.Equal(CommandExitCodes.Success, symbolsOnlyExitCode); + Assert.Equal("success", symbolsOnlyJson.GetProperty("status").GetString()); + Assert.False(symbolsOnlyJson.GetProperty("graph_table_available").GetBoolean()); + Assert.True(symbolsOnlyJson.GetProperty("issues_table_available").GetBoolean()); + Assert.False(symbolsOnlyJson.GetProperty("sql_graph_contract_ready").GetBoolean()); + Assert.True(symbolsOnlyJson.GetProperty("hotspot_family_ready").GetBoolean()); + + var dbPath = Path.Combine(projectRoot, ".cdidx", "codeindex.db"); + Assert.Equal(2, CountRows(dbPath, "files")); + Assert.True(CountRows(dbPath, "chunks") > 0); + Assert.True(CountRows(dbPath, "symbols") > 0); + Assert.Equal(0, CountRows(dbPath, "symbol_references")); + Assert.Equal(0, CountRows(dbPath, "reference_lines")); + + var (normalExitCode, normalJson) = RunAndCaptureJson([projectRoot, "--json"]); + + Assert.Equal(CommandExitCodes.Success, normalExitCode); + Assert.True(normalJson.GetProperty("graph_table_available").GetBoolean()); + Assert.True(normalJson.GetProperty("sql_graph_contract_ready").GetBoolean()); + Assert.True(normalJson.GetProperty("hotspot_family_ready").GetBoolean()); + Assert.True(CountRows(dbPath, "symbol_references") > 0); + Assert.True(CountRows(dbPath, "reference_lines") > 0); + } + finally + { + SqliteConnection.ClearAllPools(); + DeleteDirectory(projectRoot); + } + } + + [Fact] + public void Run_SymbolsOnly_OnGraphReadyDbDemotesReferencesAndSqlContract() + { + var projectRoot = CreateTempProject(); + try + { + File.WriteAllText( + Path.Combine(projectRoot, "app.cs"), + "public class App { public void Run() { Helper(); } private void Helper() { } }\n"); + File.WriteAllText( + Path.Combine(projectRoot, "query.sql"), + "CREATE TABLE users (id INTEGER PRIMARY KEY);\nSELECT id FROM users;\n"); + + var (normalExitCode, normalJson) = RunAndCaptureJson([projectRoot, "--json"]); + + var dbPath = Path.Combine(projectRoot, ".cdidx", "codeindex.db"); + Assert.Equal(CommandExitCodes.Success, normalExitCode); + Assert.True(normalJson.GetProperty("graph_table_available").GetBoolean()); + Assert.True(normalJson.GetProperty("sql_graph_contract_ready").GetBoolean()); + Assert.True(CountRows(dbPath, "symbol_references") > 0); + Assert.True(CountRows(dbPath, "reference_lines") > 0); + + var (symbolsOnlyExitCode, symbolsOnlyJson) = RunAndCaptureJson([projectRoot, "--symbols-only", "--json"]); + + Assert.Equal(CommandExitCodes.Success, symbolsOnlyExitCode); + Assert.False(symbolsOnlyJson.GetProperty("graph_table_available").GetBoolean()); + Assert.False(symbolsOnlyJson.GetProperty("sql_graph_contract_ready").GetBoolean()); + Assert.True(symbolsOnlyJson.GetProperty("hotspot_family_ready").GetBoolean()); + Assert.Equal(0, CountRows(dbPath, "symbol_references")); + Assert.Equal(0, CountRows(dbPath, "reference_lines")); + } + finally + { + SqliteConnection.ClearAllPools(); + DeleteDirectory(projectRoot); + } + } + [Fact] public void RunFiles_FileAboveMaxFileBytes_PersistsFileTooLargeIssue() { @@ -2624,6 +2707,14 @@ public void ParseArgs_MaxSymbolsPerFileInlineFlag_ParsesPositiveValue() Assert.Equal(43, options.MaxSymbolsPerFile); } + [Fact] + public void ParseArgs_SymbolsOnlyFlag_SetsOption() + { + var options = IndexCommandRunner.ParseArgs([".", "--symbols-only"]); + + Assert.True(options.SymbolsOnly); + } + [Fact] public void ParseArgs_MaxSymbolsPerFileFlag_AcceptsMaximum_Issue3172() { @@ -3121,6 +3212,24 @@ public void Run_WatchWithFiles_PrintsActionableHint() } } + [Fact] + public void Run_SymbolsOnlyWithFiles_ReturnsUsageError() + { + var projectRoot = CreateTempProject(); + try + { + var (exitCode, _, stderr) = RunAndCaptureStreams([projectRoot, "--symbols-only", "--files", "app.cs"]); + + Assert.Equal(CommandExitCodes.UsageError, exitCode); + Assert.Contains("--symbols-only can only be combined with a full index scan", stderr); + Assert.Contains("fast symbol/search-only bootstrap", stderr); + } + finally + { + DeleteDirectory(projectRoot); + } + } + [Fact] public void Run_WatchWithDryRun_JsonIncludesHint() { From 79c5dc0cdbf20f3ba7d0a532b7a35979bc1b0846 Mon Sep 17 00:00:00 2001 From: Widthdom Date: Thu, 18 Jun 2026 19:48:26 +0900 Subject: [PATCH 6/6] Fix symbols-only graph reuse marker (#3655) --- src/CodeIndex/Cli/IndexCommandRunner.FullScan.cs | 9 +++++++-- src/CodeIndex/Cli/IndexCommandRunner.Update.cs | 2 ++ src/CodeIndex/Cli/IndexCommandRunner.cs | 8 ++++++-- src/CodeIndex/Database/DbContext.cs | 1 + 4 files changed, 16 insertions(+), 4 deletions(-) diff --git a/src/CodeIndex/Cli/IndexCommandRunner.FullScan.cs b/src/CodeIndex/Cli/IndexCommandRunner.FullScan.cs index b7cdb98887..72301d7463 100644 --- a/src/CodeIndex/Cli/IndexCommandRunner.FullScan.cs +++ b/src/CodeIndex/Cli/IndexCommandRunner.FullScan.cs @@ -579,6 +579,7 @@ private static int RunFullScan( string[] spinnerFrames, JsonSerializerOptions jsonOptions, int priorReadiness, + bool priorSymbolsOnlyGraphOmitted, string? priorFoldVersion, string? priorFoldFingerprint, bool priorSymbolExtractorVersionsMatchCurrent, @@ -613,7 +614,6 @@ private static int RunFullScan( var csharpSymbolNameContractMatchesCurrent = priorCSharpSymbolNameContractVersion == currentCSharpSymbolNameContractVersion; var currentSqlGraphContractVersion = DbContext.SqlGraphContractVersion.ToString(System.Globalization.CultureInfo.InvariantCulture); var sqlGraphContractMatchesCurrent = priorSqlGraphContractVersion == currentSqlGraphContractVersion; - var priorGraphReady = (priorReadiness & DbContext.GraphReadyFlag) != 0; var hotspotFamilyTrustMatchesCurrent = GetHotspotFamilyTrustMatchesCurrent( priorHotspotFamilyVersions, priorHotspotFamilyMarkerFingerprints, @@ -1195,7 +1195,7 @@ void StopJsonHeartbeat() language: record.Lang, generated: record.Generated, allowReuse: symbolKindFilterMatchesPrior - && priorGraphReady + && !priorSymbolsOnlyGraphOmitted && record.Lang is not ("javascript" or "typescript") && (record.Lang != "csharp" || csharpSymbolNameContractMatchesCurrent) && (record.Lang != "csharp" || !csharpWorkspace.HasStaticInterfaceContracts) @@ -1428,6 +1428,11 @@ void StopJsonHeartbeat() { writer.MarkGraphReady(); writer.MarkSqlGraphContractReady(); + writer.SetMeta(DbContext.SymbolsOnlyGraphOmittedMetaKey, null); + } + else + { + writer.SetMeta(DbContext.SymbolsOnlyGraphOmittedMetaKey, "true"); } writer.MarkCSharpSymbolNameContractReady(); // Issue #435: resolve every C# class-like row and stamp readiness. Full-scan diff --git a/src/CodeIndex/Cli/IndexCommandRunner.Update.cs b/src/CodeIndex/Cli/IndexCommandRunner.Update.cs index 4e10cba8ca..9362c10904 100644 --- a/src/CodeIndex/Cli/IndexCommandRunner.Update.cs +++ b/src/CodeIndex/Cli/IndexCommandRunner.Update.cs @@ -20,6 +20,7 @@ private static int RunUpdateMode( string[] spinnerFrames, JsonSerializerOptions jsonOptions, int priorReadiness, + bool priorSymbolsOnlyGraphOmitted, string? priorFoldVersion, string? priorFoldFingerprint, bool priorSymbolExtractorVersionsMatchCurrent, @@ -95,6 +96,7 @@ private static int RunUpdateMode( spinnerFrames, jsonOptions, priorReadiness, + priorSymbolsOnlyGraphOmitted, priorFoldVersion, priorFoldFingerprint, priorSymbolExtractorVersionsMatchCurrent, diff --git a/src/CodeIndex/Cli/IndexCommandRunner.cs b/src/CodeIndex/Cli/IndexCommandRunner.cs index 403622014e..1c79e1e445 100644 --- a/src/CodeIndex/Cli/IndexCommandRunner.cs +++ b/src/CodeIndex/Cli/IndexCommandRunner.cs @@ -246,6 +246,10 @@ internal static int Run(string[] indexArgs, JsonSerializerOptions jsonOptions, C var priorCSharpSymbolNameContractVersion = db.GetMetaString(DbContext.CSharpSymbolNameContractVersionMetaKey); var priorMetadataTargetCsharp = db.GetMetaString(DbContext.GetMetadataTargetVersionMetaKey("csharp")); var priorSqlGraphContractVersion = db.GetMetaString(DbContext.SqlGraphContractVersionMetaKey); + var priorSymbolsOnlyGraphOmitted = string.Equals( + db.GetMetaString(DbContext.SymbolsOnlyGraphOmittedMetaKey), + "true", + StringComparison.OrdinalIgnoreCase); var priorHotspotFamilyVersions = GetHotspotFamilyMetaSnapshot(db, DbContext.GetHotspotFamilyVersionMetaKey); var priorHotspotFamilyMarkerFingerprints = GetHotspotFamilyMetaSnapshot(db, DbContext.GetHotspotFamilyMarkerFingerprintMetaKey); var priorIndexedProjectRoot = db.GetMetaString(DbContext.IndexedProjectRootMetaKey); @@ -276,8 +280,8 @@ internal static int Run(string[] indexArgs, JsonSerializerOptions jsonOptions, C var projectRoot = Path.GetFullPath(options.ProjectPath!); initialExitCode = isUpdateMode - ? RunUpdateMode(writer, indexer, projectRoot, resolvedDbPath, options, stopwatch, runStartedAtUtc, spinnerFrames, jsonOptions, priorReadiness, priorFoldVersion, priorFoldFingerprint, priorSymbolExtractorVersionsMatchCurrent, priorCSharpSymbolNameContractVersion, priorMetadataTargetCsharp, priorSqlGraphContractVersion, priorHotspotFamilyVersions, priorHotspotFamilyMarkerFingerprints, currentHotspotFamilyMarkerFingerprints, priorIndexedProjectRoot, priorIndexedHeadCommit, currentHeadCommit, priorSymbolKindFilterSignature, initialCwd, indexRunDiagnostics, indexCancellation.Token) - : RunFullScan(writer, indexer, projectRoot, resolvedDbPath, options, stopwatch, runStartedAtUtc, spinnerFrames, jsonOptions, priorReadiness, priorFoldVersion, priorFoldFingerprint, priorSymbolExtractorVersionsMatchCurrent, priorCSharpSymbolNameContractVersion, priorMetadataTargetCsharp, priorSqlGraphContractVersion, priorHotspotFamilyVersions, priorHotspotFamilyMarkerFingerprints, currentHotspotFamilyMarkerFingerprints, priorIndexedProjectRoot, priorIndexedHeadCommit, currentHeadCommit, priorSymbolKindFilterSignature, initialCwd, indexRunDiagnostics, showNextSteps: !databaseExistedBeforeIndex, indexCancellation.Token); + ? RunUpdateMode(writer, indexer, projectRoot, resolvedDbPath, options, stopwatch, runStartedAtUtc, spinnerFrames, jsonOptions, priorReadiness, priorSymbolsOnlyGraphOmitted, priorFoldVersion, priorFoldFingerprint, priorSymbolExtractorVersionsMatchCurrent, priorCSharpSymbolNameContractVersion, priorMetadataTargetCsharp, priorSqlGraphContractVersion, priorHotspotFamilyVersions, priorHotspotFamilyMarkerFingerprints, currentHotspotFamilyMarkerFingerprints, priorIndexedProjectRoot, priorIndexedHeadCommit, currentHeadCommit, priorSymbolKindFilterSignature, initialCwd, indexRunDiagnostics, indexCancellation.Token) + : RunFullScan(writer, indexer, projectRoot, resolvedDbPath, options, stopwatch, runStartedAtUtc, spinnerFrames, jsonOptions, priorReadiness, priorSymbolsOnlyGraphOmitted, priorFoldVersion, priorFoldFingerprint, priorSymbolExtractorVersionsMatchCurrent, priorCSharpSymbolNameContractVersion, priorMetadataTargetCsharp, priorSqlGraphContractVersion, priorHotspotFamilyVersions, priorHotspotFamilyMarkerFingerprints, currentHotspotFamilyMarkerFingerprints, priorIndexedProjectRoot, priorIndexedHeadCommit, currentHeadCommit, priorSymbolKindFilterSignature, initialCwd, indexRunDiagnostics, showNextSteps: !databaseExistedBeforeIndex, indexCancellation.Token); if (initialExitCode == CommandExitCodes.Success) db.RunPlannerStatisticsMaintenance(forceAnalyze: !databaseExistedBeforeIndex); } diff --git a/src/CodeIndex/Database/DbContext.cs b/src/CodeIndex/Database/DbContext.cs index 2e2737883d..f26dd41b9e 100644 --- a/src/CodeIndex/Database/DbContext.cs +++ b/src/CodeIndex/Database/DbContext.cs @@ -1287,6 +1287,7 @@ public static string BuildIncompleteHotspotFamilyMarkerFingerprint(string? finge public const string CSharpSymbolNameContractVersionMetaKey = "csharp_symbol_name_contract_version"; public const int SqlGraphContractVersion = 1; public const string SqlGraphContractVersionMetaKey = "sql_graph_contract_version"; + public const string SymbolsOnlyGraphOmittedMetaKey = "symbols_only_graph_omitted"; public const string IndexedProjectRootMetaKey = "indexed_project_root"; // Git HEAD commit captured at the end of the most recent full-scan index run (`--rebuild` or // the default incremental full scan). Reading this back lets the CLI detect that a user