diff --git a/changelog.d/unreleased/1781.fixed.md b/changelog.d/unreleased/1781.fixed.md new file mode 100644 index 0000000000..a6b3259a6b --- /dev/null +++ b/changelog.d/unreleased/1781.fixed.md @@ -0,0 +1,16 @@ +--- +category: fixed +issues: + - 1781 +affected: + - src/CodeIndex/Database/DbContext.cs + - tests/CodeIndex.Tests/DatabaseTests.cs +--- + +## English + +- **Reference context links are cleared instead of dangling after reference-line deletion (#1781)** — `symbol_references.reference_line_id` now uses `ON DELETE SET NULL`, and existing indexes are migrated by nulling already-dangling line-context pointers before rebuilding the table constraint. + +## 日本語 + +- **reference-line 削除後の参照コンテキストリンクが dangling ではなく NULL 化されるようになりました (#1781)** — `symbol_references.reference_line_id` は `ON DELETE SET NULL` を使い、既存 index は既に dangling になっている line-context pointer を NULL 化してからテーブル制約を再構築します。 diff --git a/changelog.d/unreleased/1785.fixed.md b/changelog.d/unreleased/1785.fixed.md new file mode 100644 index 0000000000..94aa64fb03 --- /dev/null +++ b/changelog.d/unreleased/1785.fixed.md @@ -0,0 +1,16 @@ +--- +category: fixed +issues: + - 1785 +affected: + - src/CodeIndex/Database/DbWriter.cs + - tests/CodeIndex.Tests/DatabaseTests.cs +--- + +## English + +- **Stale file purges remove cross-file references to symbols defined only by the purged files (#1785)** — cleanup now drops phantom symbol edges when a deleted file was the only remaining definition for the referenced name. + +## 日本語 + +- **stale file purge が purge 対象ファイルにしか定義が残っていないシンボルへの cross-file reference を削除するようになりました (#1785)** — 削除済みファイルが参照名の唯一の定義だった場合、phantom symbol edge を cleanup で取り除きます。 diff --git a/changelog.d/unreleased/1826.fixed.md b/changelog.d/unreleased/1826.fixed.md new file mode 100644 index 0000000000..2885e9c1fa --- /dev/null +++ b/changelog.d/unreleased/1826.fixed.md @@ -0,0 +1,15 @@ +--- +category: fixed +issues: + - 1826 +affected: + - src/CodeIndex/Database/DbWriter.cs +--- + +## English + +- **Stale file purges delete file rows in chunked batches (#1826)** — purge paths now issue chunked `DELETE ... IN (...)` statements instead of one delete statement per stale file. + +## 日本語 + +- **stale file purge が file row を chunked batch で削除するようになりました (#1826)** — purge 経路は stale file ごとに個別の DELETE を発行せず、chunked `DELETE ... IN (...)` を使います。 diff --git a/src/CodeIndex/Database/DbContext.cs b/src/CodeIndex/Database/DbContext.cs index 72449202f9..533a54e043 100644 --- a/src/CodeIndex/Database/DbContext.cs +++ b/src/CodeIndex/Database/DbContext.cs @@ -1354,7 +1354,7 @@ reference_kind TEXT CHECK (reference_kind IN (" + referenceKindCheck + @")), line INTEGER, column_number INTEGER, context TEXT, - reference_line_id INTEGER REFERENCES reference_lines(id), + reference_line_id INTEGER REFERENCES reference_lines(id) ON DELETE SET NULL, container_kind TEXT CHECK (container_kind IS NULL OR container_kind IN (" + symbolKindCheck + @")), container_name TEXT )"); @@ -1403,7 +1403,7 @@ value TEXT EnsureColumn( "symbol_references", "reference_line_id", - rebuildsSymbolReferences ? "INTEGER" : "INTEGER REFERENCES reference_lines(id)"); + rebuildsSymbolReferences ? "INTEGER" : "INTEGER REFERENCES reference_lines(id) ON DELETE SET NULL"); // #86: Unicode-aware folded name columns for `--exact` name matching across all // `--exact` command variants. Populated by the writer via NameFold.Fold; NULL on // legacy rows until a full reindex, in which case the reader falls back to the @@ -1415,6 +1415,7 @@ value TEXT EnsureColumn("symbol_references", "is_self_reference", "INTEGER NOT NULL DEFAULT 0"); EnsureColumn("symbol_references", "is_mutual_recursion", "INTEGER NOT NULL DEFAULT 0"); EnforceRequiredFileIdConstraints(); + EnforceReferenceLineSetNullConstraint(); EnsureReferenceLinesContextKey(); // Indexes / インデックス @@ -1513,6 +1514,7 @@ CREATE TRIGGER IF NOT EXISTS fts_chunks_au AFTER UPDATE ON chunks BEGIN private void EnforceRequiredFileIdConstraints() { + var symbolKindCheck = SymbolKindCatalog.ToSqlCheckInList(SymbolKindCatalog.SymbolKinds); Execute("PRAGMA foreign_keys=OFF"); var legacyAlterTable = ExecuteScalar("PRAGMA legacy_alter_table"); Execute("PRAGMA legacy_alter_table=ON"); @@ -1534,11 +1536,11 @@ file_id INTEGER NOT NULL REFERENCES files(id) ON DELETE CASCADE, "id, file_id, chunk_index, start_line, end_line, content"); RebuildTableWithRequiredFileId( "symbols", - """ + $""" CREATE TABLE symbols ( id INTEGER PRIMARY KEY AUTOINCREMENT, file_id INTEGER NOT NULL REFERENCES files(id) ON DELETE CASCADE, - kind TEXT, + kind TEXT CHECK (kind IN ({symbolKindCheck})), sub_kind TEXT, name TEXT, line INTEGER, @@ -1548,7 +1550,7 @@ file_id INTEGER NOT NULL REFERENCES files(id) ON DELETE CASCADE, body_start_line INTEGER, body_end_line INTEGER, signature TEXT, - container_kind TEXT, + container_kind TEXT CHECK (container_kind IS NULL OR container_kind IN ({symbolKindCheck})), container_name TEXT, container_qualified_name TEXT, family_key TEXT, @@ -1588,6 +1590,8 @@ private void RebuildReferenceLineTablesWithRequiredFileId() return; } + var symbolKindCheck = SymbolKindCatalog.ToSqlCheckInList(SymbolKindCatalog.SymbolKinds); + var referenceKindCheck = SymbolKindCatalog.ToSqlCheckInList(SymbolKindCatalog.ReferenceKinds); const string referenceLinesCreateSql = """ CREATE TABLE reference_lines ( @@ -1599,18 +1603,18 @@ file_id INTEGER NOT NULL REFERENCES files(id) ON DELETE CASCADE, ) """; const string referenceLinesColumns = "id, file_id, line, context"; - const string symbolReferencesCreateSql = - """ + var symbolReferencesCreateSql = + $""" CREATE TABLE symbol_references ( id INTEGER PRIMARY KEY AUTOINCREMENT, file_id INTEGER NOT NULL REFERENCES files(id) ON DELETE CASCADE, symbol_name TEXT, - reference_kind TEXT, + reference_kind TEXT CHECK (reference_kind IN ({referenceKindCheck})), line INTEGER, column_number INTEGER, context TEXT, - reference_line_id INTEGER REFERENCES reference_lines(id), - container_kind TEXT, + reference_line_id INTEGER REFERENCES reference_lines(id) ON DELETE SET NULL, + container_kind TEXT CHECK (container_kind IS NULL OR container_kind IN ({symbolKindCheck})), container_name TEXT, symbol_name_folded TEXT, container_name_folded TEXT, @@ -1636,11 +1640,81 @@ is_mutual_recursion INTEGER NOT NULL DEFAULT 0 Execute($"DROP TABLE {oldReferenceLines}"); } + private void EnforceReferenceLineSetNullConstraint() + { + if (SymbolReferencesReferenceLineDeletesSetNull()) + return; + + var symbolKindCheck = SymbolKindCatalog.ToSqlCheckInList(SymbolKindCatalog.SymbolKinds); + var referenceKindCheck = SymbolKindCatalog.ToSqlCheckInList(SymbolKindCatalog.ReferenceKinds); + var symbolReferencesCreateSql = + $""" + CREATE TABLE symbol_references ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + file_id INTEGER NOT NULL REFERENCES files(id) ON DELETE CASCADE, + symbol_name TEXT, + reference_kind TEXT CHECK (reference_kind IN ({referenceKindCheck})), + line INTEGER, + column_number INTEGER, + context TEXT, + reference_line_id INTEGER REFERENCES reference_lines(id) ON DELETE SET NULL, + container_kind TEXT CHECK (container_kind IS NULL OR container_kind IN ({symbolKindCheck})), + container_name TEXT, + symbol_name_folded TEXT, + container_name_folded TEXT, + is_self_reference INTEGER NOT NULL DEFAULT 0, + is_mutual_recursion INTEGER NOT NULL DEFAULT 0 + ) + """; + const string symbolReferencesColumns = "id, file_id, symbol_name, reference_kind, line, column_number, context, reference_line_id, container_kind, container_name, symbol_name_folded, container_name_folded, is_self_reference, is_mutual_recursion"; + const string oldSymbolReferences = "_symbol_references_reference_line_delete"; + + Execute($"DROP TABLE IF EXISTS {oldSymbolReferences}"); + Execute(@" + UPDATE symbol_references + SET reference_line_id = NULL + WHERE reference_line_id IS NOT NULL + AND NOT EXISTS ( + SELECT 1 + FROM reference_lines + WHERE reference_lines.id = symbol_references.reference_line_id + )"); + Execute($"ALTER TABLE symbol_references RENAME TO {oldSymbolReferences}"); + Execute(symbolReferencesCreateSql); + Execute($"INSERT INTO symbol_references ({symbolReferencesColumns}) SELECT {symbolReferencesColumns} FROM {oldSymbolReferences}"); + Execute($"DROP TABLE {oldSymbolReferences}"); + } + + private bool SymbolReferencesReferenceLineDeletesSetNull() + { + using var cmd = _connection.CreateCommand(); + if (_activeMigrationTransaction != null) + cmd.Transaction = _activeMigrationTransaction; + cmd.CommandText = "PRAGMA foreign_key_list('symbol_references')"; + + using var reader = cmd.ExecuteTrackedReader(); + while (reader.TrackedRead()) + { + var table = reader.GetString(2); + var from = reader.GetString(3); + var onDelete = reader.GetString(6); + if (string.Equals(table, "reference_lines", StringComparison.OrdinalIgnoreCase) + && string.Equals(from, "reference_line_id", StringComparison.OrdinalIgnoreCase)) + { + return string.Equals(onDelete, "SET NULL", StringComparison.OrdinalIgnoreCase); + } + } + + return false; + } + private void EnsureReferenceLinesContextKey() { if (ReferenceLinesHasContextUniqueKey()) return; + var symbolKindCheck = SymbolKindCatalog.ToSqlCheckInList(SymbolKindCatalog.SymbolKinds); + var referenceKindCheck = SymbolKindCatalog.ToSqlCheckInList(SymbolKindCatalog.ReferenceKinds); const string referenceLinesCreateSql = """ CREATE TABLE reference_lines ( @@ -1652,18 +1726,18 @@ file_id INTEGER NOT NULL REFERENCES files(id) ON DELETE CASCADE, ) """; const string referenceLinesColumns = "id, file_id, line, context"; - const string symbolReferencesCreateSql = - """ + var symbolReferencesCreateSql = + $""" CREATE TABLE symbol_references ( id INTEGER PRIMARY KEY AUTOINCREMENT, file_id INTEGER NOT NULL REFERENCES files(id) ON DELETE CASCADE, symbol_name TEXT, - reference_kind TEXT, + reference_kind TEXT CHECK (reference_kind IN ({referenceKindCheck})), line INTEGER, column_number INTEGER, context TEXT, - reference_line_id INTEGER REFERENCES reference_lines(id), - container_kind TEXT, + reference_line_id INTEGER REFERENCES reference_lines(id) ON DELETE SET NULL, + container_kind TEXT CHECK (container_kind IS NULL OR container_kind IN ({symbolKindCheck})), container_name TEXT, symbol_name_folded TEXT, container_name_folded TEXT, @@ -1969,14 +2043,14 @@ file_id INTEGER NOT NULL REFERENCES files(id) ON DELETE CASCADE, line INTEGER, column_number INTEGER, context TEXT, - reference_line_id INTEGER REFERENCES reference_lines(id), + reference_line_id INTEGER REFERENCES reference_lines(id) ON DELETE SET NULL, container_kind TEXT, container_name TEXT, is_self_reference INTEGER NOT NULL DEFAULT 0, is_mutual_recursion INTEGER NOT NULL DEFAULT 0 )")); yield return ("EnsureColumn symbol_references.reference_line_id", - () => EnsureColumn("symbol_references", "reference_line_id", "INTEGER REFERENCES reference_lines(id)")); + () => EnsureColumn("symbol_references", "reference_line_id", "INTEGER REFERENCES reference_lines(id) ON DELETE SET NULL")); yield return ("EnsureColumn symbol_references.is_self_reference", () => EnsureColumn("symbol_references", "is_self_reference", "INTEGER NOT NULL DEFAULT 0")); yield return ("EnsureColumn symbol_references.is_mutual_recursion", diff --git a/src/CodeIndex/Database/DbWriter.cs b/src/CodeIndex/Database/DbWriter.cs index 1c30a97194..72f638ad05 100644 --- a/src/CodeIndex/Database/DbWriter.cs +++ b/src/CodeIndex/Database/DbWriter.cs @@ -28,6 +28,7 @@ public class DbWriter private readonly SemaphoreSlim _transactionGate = new(1, 1); private readonly AsyncLocal _currentTransactionGateToken = new(); private const int BatchSize = 500; + private const int DeleteFilesBatchSize = 500; private const int MaxSqlVariables = 999; private const int SqliteConstraintErrorCode = 19; private int _rowSkipSavepointCounter; @@ -617,22 +618,7 @@ public int PurgeStaleFilesSharingChecksum(string projectRoot, string retainedRel ReleaseCommand(cmd); } - if (staleIds.Count == 0) - return 0; - - using var txn = !IsInTransaction() ? BeginTransaction() : null; - using var deleteCmd = _conn.CreateCommand(); - deleteCmd.CommandText = "DELETE FROM files WHERE id = @id"; - var pId = deleteCmd.Parameters.Add("@id", SqliteType.Integer); - deleteCmd.Prepare(); - foreach (var id in staleIds) - { - pId.Value = id; - deleteCmd.ExecuteNonQuery(); - } - txn?.Commit(); - - return staleIds.Count; + return DeleteStaleFileIds(staleIds); } /// @@ -677,18 +663,76 @@ private int DeleteStaleFileIds(IReadOnlyCollection staleIds) return 0; using var txn = !IsInTransaction() ? BeginTransaction() : null; + DeleteFilesByIdBatched(staleIds); + txn?.Commit(); + + return staleIds.Count; + } + + private void DeleteFilesByIdBatched(IEnumerable fileIds, int batchSize = DeleteFilesBatchSize) + { + var batch = new List(batchSize); + foreach (var id in fileIds) + { + batch.Add(id); + if (batch.Count == batchSize) + { + DeleteFileIdBatch(batch); + batch.Clear(); + } + } + + if (batch.Count > 0) + DeleteFileIdBatch(batch); + } + + private void DeleteFileIdBatch(IReadOnlyList fileIds) + { + DeleteCrossFileReferencesToSymbolsDefinedOnlyByFiles(fileIds); + using var deleteCmd = _conn.CreateCommand(); - deleteCmd.CommandText = "DELETE FROM files WHERE id = @id"; - var pId = deleteCmd.Parameters.Add("@id", SqliteType.Integer); - deleteCmd.Prepare(); - foreach (var id in staleIds) + var parameters = new List(fileIds.Count); + for (var i = 0; i < fileIds.Count; i++) { - pId.Value = id; - deleteCmd.ExecuteNonQuery(); + var parameterName = $"@id{i}"; + parameters.Add(parameterName); + deleteCmd.Parameters.Add(parameterName, SqliteType.Integer).Value = fileIds[i]; } - txn?.Commit(); - return staleIds.Count; + deleteCmd.CommandText = $"DELETE FROM files WHERE id IN ({string.Join(", ", parameters)})"; + deleteCmd.ExecuteNonQuery(); + } + + private void DeleteCrossFileReferencesToSymbolsDefinedOnlyByFiles(IReadOnlyList fileIds) + { + using var deleteCmd = _conn.CreateCommand(); + var parameters = new List(fileIds.Count); + for (var i = 0; i < fileIds.Count; i++) + { + var parameterName = $"@id{i}"; + parameters.Add(parameterName); + deleteCmd.Parameters.Add(parameterName, SqliteType.Integer).Value = fileIds[i]; + } + + var idList = string.Join(", ", parameters); + deleteCmd.CommandText = $@" + DELETE FROM symbol_references + WHERE file_id NOT IN ({idList}) + AND symbol_name IS NOT NULL + AND symbol_name <> '' + AND EXISTS ( + SELECT 1 + FROM symbols deleted_symbols + WHERE deleted_symbols.file_id IN ({idList}) + AND deleted_symbols.name = symbol_references.symbol_name + ) + AND NOT EXISTS ( + SELECT 1 + FROM symbols retained_symbols + WHERE retained_symbols.file_id NOT IN ({idList}) + AND retained_symbols.name = symbol_references.symbol_name + )"; + deleteCmd.ExecuteNonQuery(); } private static string GetRelativeDirectory(string relativePath) diff --git a/tests/CodeIndex.Tests/DatabaseTests.cs b/tests/CodeIndex.Tests/DatabaseTests.cs index 6a48080274..cf49bae761 100644 --- a/tests/CodeIndex.Tests/DatabaseTests.cs +++ b/tests/CodeIndex.Tests/DatabaseTests.cs @@ -98,6 +98,92 @@ public void InsertReferences_UsesFoldedNamesForMutualRecursion() Assert.Equal(2L, (long)cmd.ExecuteScalar()!); } + [Fact] + public void DeleteFileData_WhenReferencedLineIsDeleted_PreservesReferenceWithNullLineContext() + { + var callerFileId = UpsertTestFile("src/caller.cs", checksum: "caller"); + var lineOwnerFileId = UpsertTestFile("src/line-owner.cs", checksum: "line-owner"); + + long referenceLineId; + using (var cmd = _db.Connection.CreateCommand()) + { + cmd.CommandText = @" + INSERT INTO reference_lines (file_id, line, context) + VALUES (@fileId, 3, 'Target();') + RETURNING id"; + cmd.Parameters.AddWithValue("@fileId", lineOwnerFileId); + referenceLineId = (long)cmd.ExecuteScalar()!; + } + + using (var cmd = _db.Connection.CreateCommand()) + { + cmd.CommandText = @" + INSERT INTO symbol_references ( + file_id, symbol_name, reference_kind, line, column_number, context, reference_line_id + ) + VALUES (@fileId, 'Target', 'call', 1, 1, NULL, @referenceLineId)"; + cmd.Parameters.AddWithValue("@fileId", callerFileId); + cmd.Parameters.AddWithValue("@referenceLineId", referenceLineId); + cmd.ExecuteNonQuery(); + } + + _writer.DeleteFileData(lineOwnerFileId); + + using var readCmd = _db.Connection.CreateCommand(); + readCmd.CommandText = "SELECT COUNT(*), COUNT(reference_line_id) FROM symbol_references WHERE file_id = @fileId"; + readCmd.Parameters.AddWithValue("@fileId", callerFileId); + using var reader = readCmd.ExecuteReader(); + Assert.True(reader.Read()); + Assert.Equal(1L, reader.GetInt64(0)); + Assert.Equal(0L, reader.GetInt64(1)); + } + + [Fact] + public void PurgeStaleFiles_RemovesCrossFileReferencesToSymbolsDefinedOnlyByDeletedFiles() + { + var projectRoot = TestProjectHelper.CreateTempProject("purge-stale-symbol-ref"); + try + { + Directory.CreateDirectory(Path.Combine(projectRoot, "src")); + File.WriteAllText(Path.Combine(projectRoot, "src", "target.py"), "# retained rename target"); + + var callerFileId = UpsertTestFile("src/caller.cs", checksum: "caller"); + var staleTargetFileId = UpsertTestFile("src/target.cs", checksum: "target"); + _ = UpsertTestFile("src/target.py", checksum: "target"); + _writer.InsertSymbols([ + new SymbolRecord + { + FileId = staleTargetFileId, + Kind = "function", + Name = "DeletedTarget", + Line = 1, + }, + ]); + _writer.InsertReferences([ + new ReferenceRecord + { + FileId = callerFileId, + SymbolName = "DeletedTarget", + ReferenceKind = "call", + Line = 1, + Column = 1, + Context = "DeletedTarget();", + }, + ]); + + var purged = _writer.PurgeStaleFilesSharingDirectoryAndStem(projectRoot, "src/target.py"); + + Assert.Equal(1, purged); + using var cmd = _db.Connection.CreateCommand(); + cmd.CommandText = "SELECT COUNT(*) FROM symbol_references WHERE symbol_name = 'DeletedTarget'"; + Assert.Equal(0L, (long)cmd.ExecuteScalar()!); + } + finally + { + TestProjectHelper.DeleteDirectory(projectRoot); + } + } + [Fact] public void InsertSymbols_UnknownKind_ThrowsBeforePersisting() { @@ -238,6 +324,17 @@ public void OptimizeFts_ResetsIncrementalWriteCounterAndStampsTime() Assert.False(string.IsNullOrWhiteSpace(_db.GetMetaString(DbWriter.FtsLastOptimizedAtMetaKey))); } + private long UpsertTestFile(string path, string checksum) + => _writer.UpsertFile(new FileRecord + { + Path = path, + Lang = "csharp", + Size = 100, + Lines = 4, + Modified = new DateTime(2025, 1, 1, 0, 0, 0, DateTimeKind.Utc), + Checksum = checksum, + }); + [Fact] public void OptimizeFtsIfIncrementalWriteThresholdReached_RunsOnlyAtThreshold() {