From 68c031ce9a41901f9ef58a82d6358ba05fb60499 Mon Sep 17 00:00:00 2001 From: Widthdom Date: Mon, 25 May 2026 12:32:09 +0900 Subject: [PATCH 1/3] Fix dangling reference line cleanup (#1781) --- changelog.d/unreleased/1781.fixed.md | 16 ++++++ src/CodeIndex/Database/DbContext.cs | 75 ++++++++++++++++++++++++-- tests/CodeIndex.Tests/DatabaseTests.cs | 51 ++++++++++++++++++ 3 files changed, 138 insertions(+), 4 deletions(-) create mode 100644 changelog.d/unreleased/1781.fixed.md diff --git a/changelog.d/unreleased/1781.fixed.md b/changelog.d/unreleased/1781.fixed.md new file mode 100644 index 0000000000..a6b3259a6b --- /dev/null +++ b/changelog.d/unreleased/1781.fixed.md @@ -0,0 +1,16 @@ +--- +category: fixed +issues: + - 1781 +affected: + - src/CodeIndex/Database/DbContext.cs + - tests/CodeIndex.Tests/DatabaseTests.cs +--- + +## English + +- **Reference context links are cleared instead of dangling after reference-line deletion (#1781)** — `symbol_references.reference_line_id` now uses `ON DELETE SET NULL`, and existing indexes are migrated by nulling already-dangling line-context pointers before rebuilding the table constraint. + +## 日本語 + +- **reference-line 削除後の参照コンテキストリンクが dangling ではなく NULL 化されるようになりました (#1781)** — `symbol_references.reference_line_id` は `ON DELETE SET NULL` を使い、既存 index は既に dangling になっている line-context pointer を NULL 化してからテーブル制約を再構築します。 diff --git a/src/CodeIndex/Database/DbContext.cs b/src/CodeIndex/Database/DbContext.cs index 9fec4db852..594960d8b8 100644 --- a/src/CodeIndex/Database/DbContext.cs +++ b/src/CodeIndex/Database/DbContext.cs @@ -1350,7 +1350,7 @@ file_id INTEGER NOT NULL REFERENCES files(id) ON DELETE CASCADE, line INTEGER, column_number INTEGER, context TEXT, - reference_line_id INTEGER REFERENCES reference_lines(id), + reference_line_id INTEGER REFERENCES reference_lines(id) ON DELETE SET NULL, container_kind TEXT, container_name TEXT )"); @@ -1399,7 +1399,7 @@ value TEXT EnsureColumn( "symbol_references", "reference_line_id", - rebuildsSymbolReferences ? "INTEGER" : "INTEGER REFERENCES reference_lines(id)"); + rebuildsSymbolReferences ? "INTEGER" : "INTEGER REFERENCES reference_lines(id) ON DELETE SET NULL"); // #86: Unicode-aware folded name columns for `--exact` name matching across all // `--exact` command variants. Populated by the writer via NameFold.Fold; NULL on // legacy rows until a full reindex, in which case the reader falls back to the @@ -1411,6 +1411,7 @@ value TEXT EnsureColumn("symbol_references", "is_self_reference", "INTEGER NOT NULL DEFAULT 0"); EnsureColumn("symbol_references", "is_mutual_recursion", "INTEGER NOT NULL DEFAULT 0"); EnforceRequiredFileIdConstraints(); + EnforceReferenceLineSetNullConstraint(); EnsureReferenceLinesContextKey(); // Indexes / インデックス @@ -1605,7 +1606,7 @@ file_id INTEGER NOT NULL REFERENCES files(id) ON DELETE CASCADE, line INTEGER, column_number INTEGER, context TEXT, - reference_line_id INTEGER REFERENCES reference_lines(id), + reference_line_id INTEGER REFERENCES reference_lines(id) ON DELETE SET NULL, container_kind TEXT, container_name TEXT, symbol_name_folded TEXT, @@ -1632,6 +1633,72 @@ is_mutual_recursion INTEGER NOT NULL DEFAULT 0 Execute($"DROP TABLE {oldReferenceLines}"); } + private void EnforceReferenceLineSetNullConstraint() + { + if (SymbolReferencesReferenceLineDeletesSetNull()) + return; + + const string symbolReferencesCreateSql = + """ + CREATE TABLE symbol_references ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + file_id INTEGER NOT NULL REFERENCES files(id) ON DELETE CASCADE, + symbol_name TEXT, + reference_kind TEXT, + line INTEGER, + column_number INTEGER, + context TEXT, + reference_line_id INTEGER REFERENCES reference_lines(id) ON DELETE SET NULL, + container_kind TEXT, + container_name TEXT, + symbol_name_folded TEXT, + container_name_folded TEXT, + is_self_reference INTEGER NOT NULL DEFAULT 0, + is_mutual_recursion INTEGER NOT NULL DEFAULT 0 + ) + """; + const string symbolReferencesColumns = "id, file_id, symbol_name, reference_kind, line, column_number, context, reference_line_id, container_kind, container_name, symbol_name_folded, container_name_folded, is_self_reference, is_mutual_recursion"; + const string oldSymbolReferences = "_symbol_references_reference_line_delete"; + + Execute($"DROP TABLE IF EXISTS {oldSymbolReferences}"); + Execute(@" + UPDATE symbol_references + SET reference_line_id = NULL + WHERE reference_line_id IS NOT NULL + AND NOT EXISTS ( + SELECT 1 + FROM reference_lines + WHERE reference_lines.id = symbol_references.reference_line_id + )"); + Execute($"ALTER TABLE symbol_references RENAME TO {oldSymbolReferences}"); + Execute(symbolReferencesCreateSql); + Execute($"INSERT INTO symbol_references ({symbolReferencesColumns}) SELECT {symbolReferencesColumns} FROM {oldSymbolReferences}"); + Execute($"DROP TABLE {oldSymbolReferences}"); + } + + private bool SymbolReferencesReferenceLineDeletesSetNull() + { + using var cmd = _connection.CreateCommand(); + if (_activeMigrationTransaction != null) + cmd.Transaction = _activeMigrationTransaction; + cmd.CommandText = "PRAGMA foreign_key_list('symbol_references')"; + + using var reader = cmd.ExecuteTrackedReader(); + while (reader.TrackedRead()) + { + var table = reader.GetString(2); + var from = reader.GetString(3); + var onDelete = reader.GetString(6); + if (string.Equals(table, "reference_lines", StringComparison.OrdinalIgnoreCase) + && string.Equals(from, "reference_line_id", StringComparison.OrdinalIgnoreCase)) + { + return string.Equals(onDelete, "SET NULL", StringComparison.OrdinalIgnoreCase); + } + } + + return false; + } + private void EnsureReferenceLinesContextKey() { if (ReferenceLinesHasContextUniqueKey()) @@ -1658,7 +1725,7 @@ file_id INTEGER NOT NULL REFERENCES files(id) ON DELETE CASCADE, line INTEGER, column_number INTEGER, context TEXT, - reference_line_id INTEGER REFERENCES reference_lines(id), + reference_line_id INTEGER REFERENCES reference_lines(id) ON DELETE SET NULL, container_kind TEXT, container_name TEXT, symbol_name_folded TEXT, diff --git a/tests/CodeIndex.Tests/DatabaseTests.cs b/tests/CodeIndex.Tests/DatabaseTests.cs index 21a447b928..9f43be7508 100644 --- a/tests/CodeIndex.Tests/DatabaseTests.cs +++ b/tests/CodeIndex.Tests/DatabaseTests.cs @@ -98,6 +98,46 @@ public void InsertReferences_UsesFoldedNamesForMutualRecursion() Assert.Equal(2L, (long)cmd.ExecuteScalar()!); } + [Fact] + public void DeleteFileData_WhenReferencedLineIsDeleted_PreservesReferenceWithNullLineContext() + { + var callerFileId = UpsertTestFile("src/caller.cs", checksum: "caller"); + var lineOwnerFileId = UpsertTestFile("src/line-owner.cs", checksum: "line-owner"); + + long referenceLineId; + using (var cmd = _db.Connection.CreateCommand()) + { + cmd.CommandText = @" + INSERT INTO reference_lines (file_id, line, context) + VALUES (@fileId, 3, 'Target();') + RETURNING id"; + cmd.Parameters.AddWithValue("@fileId", lineOwnerFileId); + referenceLineId = (long)cmd.ExecuteScalar()!; + } + + using (var cmd = _db.Connection.CreateCommand()) + { + cmd.CommandText = @" + INSERT INTO symbol_references ( + file_id, symbol_name, reference_kind, line, column_number, context, reference_line_id + ) + VALUES (@fileId, 'Target', 'call', 1, 1, NULL, @referenceLineId)"; + cmd.Parameters.AddWithValue("@fileId", callerFileId); + cmd.Parameters.AddWithValue("@referenceLineId", referenceLineId); + cmd.ExecuteNonQuery(); + } + + _writer.DeleteFileData(lineOwnerFileId); + + using var readCmd = _db.Connection.CreateCommand(); + readCmd.CommandText = "SELECT COUNT(*), COUNT(reference_line_id) FROM symbol_references WHERE file_id = @fileId"; + readCmd.Parameters.AddWithValue("@fileId", callerFileId); + using var reader = readCmd.ExecuteReader(); + Assert.True(reader.Read()); + Assert.Equal(1L, reader.GetInt64(0)); + Assert.Equal(0L, reader.GetInt64(1)); + } + [Fact] public void OptimizeFts_ResetsIncrementalWriteCounterAndStampsTime() { @@ -113,6 +153,17 @@ public void OptimizeFts_ResetsIncrementalWriteCounterAndStampsTime() Assert.False(string.IsNullOrWhiteSpace(_db.GetMetaString(DbWriter.FtsLastOptimizedAtMetaKey))); } + private long UpsertTestFile(string path, string checksum) + => _writer.UpsertFile(new FileRecord + { + Path = path, + Lang = "csharp", + Size = 100, + Lines = 4, + Modified = new DateTime(2025, 1, 1, 0, 0, 0, DateTimeKind.Utc), + Checksum = checksum, + }); + [Fact] public void OptimizeFtsIfIncrementalWriteThresholdReached_RunsOnlyAtThreshold() { From 2de10deffa9662568a3302050707f72691265a25 Mon Sep 17 00:00:00 2001 From: Widthdom Date: Mon, 25 May 2026 12:32:38 +0900 Subject: [PATCH 2/3] Batch stale file deletes (#1826) --- changelog.d/unreleased/1826.fixed.md | 15 +++++++ src/CodeIndex/Database/DbWriter.cs | 58 ++++++++++++++++------------ 2 files changed, 49 insertions(+), 24 deletions(-) create mode 100644 changelog.d/unreleased/1826.fixed.md diff --git a/changelog.d/unreleased/1826.fixed.md b/changelog.d/unreleased/1826.fixed.md new file mode 100644 index 0000000000..2885e9c1fa --- /dev/null +++ b/changelog.d/unreleased/1826.fixed.md @@ -0,0 +1,15 @@ +--- +category: fixed +issues: + - 1826 +affected: + - src/CodeIndex/Database/DbWriter.cs +--- + +## English + +- **Stale file purges delete file rows in chunked batches (#1826)** — purge paths now issue chunked `DELETE ... IN (...)` statements instead of one delete statement per stale file. + +## 日本語 + +- **stale file purge が file row を chunked batch で削除するようになりました (#1826)** — purge 経路は stale file ごとに個別の DELETE を発行せず、chunked `DELETE ... IN (...)` を使います。 diff --git a/src/CodeIndex/Database/DbWriter.cs b/src/CodeIndex/Database/DbWriter.cs index 2c7e2f85ca..c43f113772 100644 --- a/src/CodeIndex/Database/DbWriter.cs +++ b/src/CodeIndex/Database/DbWriter.cs @@ -21,6 +21,7 @@ public class DbWriter internal static Action? FoldBackfillRowUpdatedForTesting { get; set; } internal static Action? BatchRowSkipWarningForTesting { get; set; } private const int BatchSize = 500; + private const int DeleteFilesBatchSize = 500; private const int MaxSqlVariables = 999; private const int SqliteConstraintErrorCode = 19; private int _rowSkipSavepointCounter; @@ -507,22 +508,7 @@ public int PurgeStaleFilesSharingChecksum(string projectRoot, string retainedRel ReleaseCommand(cmd); } - if (staleIds.Count == 0) - return 0; - - using var txn = !IsInTransaction() ? BeginTransaction() : null; - using var deleteCmd = _conn.CreateCommand(); - deleteCmd.CommandText = "DELETE FROM files WHERE id = @id"; - var pId = deleteCmd.Parameters.Add("@id", SqliteType.Integer); - deleteCmd.Prepare(); - foreach (var id in staleIds) - { - pId.Value = id; - deleteCmd.ExecuteNonQuery(); - } - txn?.Commit(); - - return staleIds.Count; + return DeleteStaleFileIds(staleIds); } /// @@ -567,18 +553,42 @@ private int DeleteStaleFileIds(IReadOnlyCollection staleIds) return 0; using var txn = !IsInTransaction() ? BeginTransaction() : null; + DeleteFilesByIdBatched(staleIds); + txn?.Commit(); + + return staleIds.Count; + } + + private void DeleteFilesByIdBatched(IEnumerable fileIds, int batchSize = DeleteFilesBatchSize) + { + var batch = new List(batchSize); + foreach (var id in fileIds) + { + batch.Add(id); + if (batch.Count == batchSize) + { + DeleteFileIdBatch(batch); + batch.Clear(); + } + } + + if (batch.Count > 0) + DeleteFileIdBatch(batch); + } + + private void DeleteFileIdBatch(IReadOnlyList fileIds) + { using var deleteCmd = _conn.CreateCommand(); - deleteCmd.CommandText = "DELETE FROM files WHERE id = @id"; - var pId = deleteCmd.Parameters.Add("@id", SqliteType.Integer); - deleteCmd.Prepare(); - foreach (var id in staleIds) + var parameters = new List(fileIds.Count); + for (var i = 0; i < fileIds.Count; i++) { - pId.Value = id; - deleteCmd.ExecuteNonQuery(); + var parameterName = $"@id{i}"; + parameters.Add(parameterName); + deleteCmd.Parameters.Add(parameterName, SqliteType.Integer).Value = fileIds[i]; } - txn?.Commit(); - return staleIds.Count; + deleteCmd.CommandText = $"DELETE FROM files WHERE id IN ({string.Join(", ", parameters)})"; + deleteCmd.ExecuteNonQuery(); } private static string GetRelativeDirectory(string relativePath) From 3c3bab0ef70f712a61151da326892b17bd2d9183 Mon Sep 17 00:00:00 2001 From: Widthdom Date: Mon, 25 May 2026 12:33:18 +0900 Subject: [PATCH 3/3] Clean stale symbol references (#1785) --- changelog.d/unreleased/1785.fixed.md | 16 +++++++++ src/CodeIndex/Database/DbWriter.cs | 34 +++++++++++++++++++ tests/CodeIndex.Tests/DatabaseTests.cs | 46 ++++++++++++++++++++++++++ 3 files changed, 96 insertions(+) create mode 100644 changelog.d/unreleased/1785.fixed.md diff --git a/changelog.d/unreleased/1785.fixed.md b/changelog.d/unreleased/1785.fixed.md new file mode 100644 index 0000000000..94aa64fb03 --- /dev/null +++ b/changelog.d/unreleased/1785.fixed.md @@ -0,0 +1,16 @@ +--- +category: fixed +issues: + - 1785 +affected: + - src/CodeIndex/Database/DbWriter.cs + - tests/CodeIndex.Tests/DatabaseTests.cs +--- + +## English + +- **Stale file purges remove cross-file references to symbols defined only by the purged files (#1785)** — cleanup now drops phantom symbol edges when a deleted file was the only remaining definition for the referenced name. + +## 日本語 + +- **stale file purge が purge 対象ファイルにしか定義が残っていないシンボルへの cross-file reference を削除するようになりました (#1785)** — 削除済みファイルが参照名の唯一の定義だった場合、phantom symbol edge を cleanup で取り除きます。 diff --git a/src/CodeIndex/Database/DbWriter.cs b/src/CodeIndex/Database/DbWriter.cs index c43f113772..b686c59757 100644 --- a/src/CodeIndex/Database/DbWriter.cs +++ b/src/CodeIndex/Database/DbWriter.cs @@ -578,6 +578,8 @@ private void DeleteFilesByIdBatched(IEnumerable fileIds, int batchSize = D private void DeleteFileIdBatch(IReadOnlyList fileIds) { + DeleteCrossFileReferencesToSymbolsDefinedOnlyByFiles(fileIds); + using var deleteCmd = _conn.CreateCommand(); var parameters = new List(fileIds.Count); for (var i = 0; i < fileIds.Count; i++) @@ -591,6 +593,38 @@ private void DeleteFileIdBatch(IReadOnlyList fileIds) deleteCmd.ExecuteNonQuery(); } + private void DeleteCrossFileReferencesToSymbolsDefinedOnlyByFiles(IReadOnlyList fileIds) + { + using var deleteCmd = _conn.CreateCommand(); + var parameters = new List(fileIds.Count); + for (var i = 0; i < fileIds.Count; i++) + { + var parameterName = $"@id{i}"; + parameters.Add(parameterName); + deleteCmd.Parameters.Add(parameterName, SqliteType.Integer).Value = fileIds[i]; + } + + var idList = string.Join(", ", parameters); + deleteCmd.CommandText = $@" + DELETE FROM symbol_references + WHERE file_id NOT IN ({idList}) + AND symbol_name IS NOT NULL + AND symbol_name <> '' + AND EXISTS ( + SELECT 1 + FROM symbols deleted_symbols + WHERE deleted_symbols.file_id IN ({idList}) + AND deleted_symbols.name = symbol_references.symbol_name + ) + AND NOT EXISTS ( + SELECT 1 + FROM symbols retained_symbols + WHERE retained_symbols.file_id NOT IN ({idList}) + AND retained_symbols.name = symbol_references.symbol_name + )"; + deleteCmd.ExecuteNonQuery(); + } + private static string GetRelativeDirectory(string relativePath) { var normalized = relativePath.Replace('\\', '/'); diff --git a/tests/CodeIndex.Tests/DatabaseTests.cs b/tests/CodeIndex.Tests/DatabaseTests.cs index 9f43be7508..5c58473b03 100644 --- a/tests/CodeIndex.Tests/DatabaseTests.cs +++ b/tests/CodeIndex.Tests/DatabaseTests.cs @@ -138,6 +138,52 @@ INSERT INTO symbol_references ( Assert.Equal(0L, reader.GetInt64(1)); } + [Fact] + public void PurgeStaleFiles_RemovesCrossFileReferencesToSymbolsDefinedOnlyByDeletedFiles() + { + var projectRoot = TestProjectHelper.CreateTempProject("purge-stale-symbol-ref"); + try + { + Directory.CreateDirectory(Path.Combine(projectRoot, "src")); + File.WriteAllText(Path.Combine(projectRoot, "src", "target.py"), "# retained rename target"); + + var callerFileId = UpsertTestFile("src/caller.cs", checksum: "caller"); + var staleTargetFileId = UpsertTestFile("src/target.cs", checksum: "target"); + _ = UpsertTestFile("src/target.py", checksum: "target"); + _writer.InsertSymbols([ + new SymbolRecord + { + FileId = staleTargetFileId, + Kind = "function", + Name = "DeletedTarget", + Line = 1, + }, + ]); + _writer.InsertReferences([ + new ReferenceRecord + { + FileId = callerFileId, + SymbolName = "DeletedTarget", + ReferenceKind = "call", + Line = 1, + Column = 1, + Context = "DeletedTarget();", + }, + ]); + + var purged = _writer.PurgeStaleFilesSharingDirectoryAndStem(projectRoot, "src/target.py"); + + Assert.Equal(1, purged); + using var cmd = _db.Connection.CreateCommand(); + cmd.CommandText = "SELECT COUNT(*) FROM symbol_references WHERE symbol_name = 'DeletedTarget'"; + Assert.Equal(0L, (long)cmd.ExecuteScalar()!); + } + finally + { + TestProjectHelper.DeleteDirectory(projectRoot); + } + } + [Fact] public void OptimizeFts_ResetsIncrementalWriteCounterAndStampsTime() {