diff --git a/changelog.d/unreleased/1755.fixed.md b/changelog.d/unreleased/1755.fixed.md new file mode 100644 index 0000000000..b99aca5631 --- /dev/null +++ b/changelog.d/unreleased/1755.fixed.md @@ -0,0 +1,17 @@ +--- +category: fixed +issues: + - 1755 +affected: + - src/CodeIndex/Database/DbContext.cs + - src/CodeIndex/Database/DbWriter.cs + - tests/CodeIndex.Tests/DatabaseTests.cs +--- + +## English + +- **Reference-line writes preserve distinct same-line contexts (#1755)** — `reference_lines` now keys rows by file, line, and context and inserts idempotently, so overlapping writer batches no longer overwrite a different reference context for the same file and line. + +## 日本語 + +- **同一行の異なる参照コンテキストを保持するようになりました (#1755)** — `reference_lines` は file / line / context をキーにし、冪等な insert を行うため、重なった writer batch が同じ file / line の別コンテキストを上書きしなくなりました。 diff --git a/src/CodeIndex/Database/DbContext.cs b/src/CodeIndex/Database/DbContext.cs index f13fd61648..862092cd20 100644 --- a/src/CodeIndex/Database/DbContext.cs +++ b/src/CodeIndex/Database/DbContext.cs @@ -1294,7 +1294,7 @@ CREATE TABLE IF NOT EXISTS reference_lines ( file_id INTEGER NOT NULL REFERENCES files(id) ON DELETE CASCADE, line INTEGER NOT NULL, context TEXT NOT NULL, - UNIQUE(file_id, line) + UNIQUE(file_id, line, context) )"); // Symbols table / シンボルテーブル @@ -1391,6 +1391,7 @@ value TEXT EnsureColumn("symbol_references", "is_self_reference", "INTEGER NOT NULL DEFAULT 0"); EnsureColumn("symbol_references", "is_mutual_recursion", "INTEGER NOT NULL DEFAULT 0"); EnforceRequiredFileIdConstraints(); + EnsureReferenceLinesContextKey(); // Indexes / インデックス Execute("CREATE INDEX IF NOT EXISTS idx_files_lang ON files(lang)"); @@ -1570,7 +1571,7 @@ CREATE TABLE reference_lines ( file_id INTEGER NOT NULL REFERENCES files(id) ON DELETE CASCADE, line INTEGER NOT NULL, context TEXT NOT NULL, - UNIQUE(file_id, line) + UNIQUE(file_id, line, context) ) """; const string referenceLinesColumns = "id, file_id, line, context"; @@ -1611,6 +1612,97 @@ is_mutual_recursion INTEGER NOT NULL DEFAULT 0 Execute($"DROP TABLE {oldReferenceLines}"); } + private void EnsureReferenceLinesContextKey() + { + if (ReferenceLinesHasContextUniqueKey()) + return; + + const string referenceLinesCreateSql = + """ + CREATE TABLE reference_lines ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + file_id INTEGER NOT NULL REFERENCES files(id) ON DELETE CASCADE, + line INTEGER NOT NULL, + context TEXT NOT NULL, + UNIQUE(file_id, line, context) + ) + """; + const string referenceLinesColumns = "id, file_id, line, context"; + const string symbolReferencesCreateSql = + """ + CREATE TABLE symbol_references ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + file_id INTEGER NOT NULL REFERENCES files(id) ON DELETE CASCADE, + symbol_name TEXT, + reference_kind TEXT, + line INTEGER, + column_number INTEGER, + context TEXT, + reference_line_id INTEGER REFERENCES reference_lines(id), + container_kind TEXT, + container_name TEXT, + symbol_name_folded TEXT, + container_name_folded TEXT, + is_self_reference INTEGER NOT NULL DEFAULT 0, + is_mutual_recursion INTEGER NOT NULL DEFAULT 0 + ) + """; + const string symbolReferencesColumns = "id, file_id, symbol_name, reference_kind, line, column_number, context, reference_line_id, container_kind, container_name, symbol_name_folded, container_name_folded, is_self_reference, is_mutual_recursion"; + + const string oldReferenceLines = "_reference_lines_file_line_key"; + const string oldSymbolReferences = "_symbol_references_file_line_key"; + var foreignKeys = ReadPragmaLong("foreign_keys"); + Execute("PRAGMA foreign_keys=OFF"); + try + { + Execute($"DROP TABLE IF EXISTS {oldSymbolReferences}"); + Execute($"DROP TABLE IF EXISTS {oldReferenceLines}"); + Execute($"ALTER TABLE symbol_references RENAME TO {oldSymbolReferences}"); + Execute($"ALTER TABLE reference_lines RENAME TO {oldReferenceLines}"); + Execute(referenceLinesCreateSql); + Execute($"INSERT INTO reference_lines ({referenceLinesColumns}) SELECT {referenceLinesColumns} FROM {oldReferenceLines}"); + Execute(symbolReferencesCreateSql); + Execute($"INSERT INTO symbol_references ({symbolReferencesColumns}) SELECT {symbolReferencesColumns} FROM {oldSymbolReferences}"); + Execute($"DROP TABLE {oldSymbolReferences}"); + Execute($"DROP TABLE {oldReferenceLines}"); + } + finally + { + Execute($"PRAGMA foreign_keys={foreignKeys}"); + } + + _schemaCache?.Refresh(); + } + + private bool ReferenceLinesHasContextUniqueKey() + { + using var listCmd = _connection.CreateCommand(); + listCmd.CommandText = "PRAGMA index_list('reference_lines')"; + using var indexReader = listCmd.ExecuteReader(); + var indexNames = new List(); + while (indexReader.Read()) + { + var isUnique = indexReader.GetInt32(2) == 1; + if (isUnique) + indexNames.Add(indexReader.GetString(1)); + } + + foreach (var indexName in indexNames) + { + using var infoCmd = _connection.CreateCommand(); + infoCmd.CommandText = $"PRAGMA index_info('{indexName.Replace("'", "''")}')"; + using var infoReader = infoCmd.ExecuteReader(); + var columns = new List(); + while (infoReader.Read()) + columns.Add(infoReader.GetString(2)); + + if (columns.SequenceEqual(["file_id", "line", "context"], StringComparer.Ordinal)) + return true; + } + + return false; + } + private void RebuildTableWithRequiredFileId(string tableName, string createSql, string columns) { if (ColumnIsNotNull(tableName, "file_id")) @@ -1830,7 +1922,7 @@ CREATE TABLE IF NOT EXISTS reference_lines ( file_id INTEGER NOT NULL REFERENCES files(id) ON DELETE CASCADE, line INTEGER NOT NULL, context TEXT NOT NULL, - UNIQUE(file_id, line) + UNIQUE(file_id, line, context) )")); yield return ("CREATE TABLE symbol_references", () => Execute(@" CREATE TABLE IF NOT EXISTS symbol_references ( diff --git a/src/CodeIndex/Database/DbWriter.cs b/src/CodeIndex/Database/DbWriter.cs index fcf8a4bcd8..36483f343b 100644 --- a/src/CodeIndex/Database/DbWriter.cs +++ b/src/CodeIndex/Database/DbWriter.cs @@ -1018,7 +1018,7 @@ INSERT INTO symbol_references ( for (int j = i; j < end; j++) { var reference = references[j]; - var referenceLineId = referenceLineIds[(reference.FileId, reference.Line)]; + var referenceLineId = referenceLineIds[(reference.FileId, reference.Line, reference.Context)]; if (j > i) sql.Append(", "); @@ -1052,13 +1052,13 @@ INSERT INTO symbol_references ( RefreshMutualRecursionFlags(); } - private Dictionary<(long FileId, int Line), long> UpsertReferenceLines(IReadOnlyList references, int start, int end) + private Dictionary<(long FileId, int Line, string Context), long> UpsertReferenceLines(IReadOnlyList references, int start, int end) { - var contextsByLine = new Dictionary<(long FileId, int Line), string>(); + var contextsByLine = new Dictionary<(long FileId, int Line, string Context), string>(); for (int i = start; i < end; i++) { var reference = references[i]; - contextsByLine[(reference.FileId, reference.Line)] = reference.Context; + contextsByLine[(reference.FileId, reference.Line, reference.Context)] = reference.Context; } var rows = contextsByLine.ToArray(); @@ -1074,20 +1074,20 @@ INSERT INTO symbol_references ( if (j > i) sql.Append(", "); var suffix = j - i; - var ((fileId, line), context) = rows[j]; + var ((fileId, line, _), context) = rows[j]; sql.Append($"(@fid{suffix}, @line{suffix}, @context{suffix})"); cmd.Parameters.Add($"@fid{suffix}", SqliteType.Integer).Value = fileId; cmd.Parameters.Add($"@line{suffix}", SqliteType.Integer).Value = line; cmd.Parameters.Add($"@context{suffix}", SqliteType.Text).Value = context; } - sql.Append(" ON CONFLICT(file_id, line) DO UPDATE SET context = excluded.context"); + sql.Append(" ON CONFLICT(file_id, line, context) DO NOTHING"); cmd.CommandText = sql.ToString(); cmd.ExecuteNonQuery(); } var fileIds = contextsByLine.Keys.Select(key => key.FileId).Distinct().ToArray(); - var lineIds = new Dictionary<(long FileId, int Line), long>(); + var lineIds = new Dictionary<(long FileId, int Line, string Context), long>(); int fileIdsPerStatement = GetRowsPerInsertStatement(columnCount: 1); for (int i = 0; i < fileIds.Length; i += fileIdsPerStatement) { @@ -1102,7 +1102,7 @@ INSERT INTO symbol_references ( } cmd.CommandText = $@" - SELECT id, file_id, line + SELECT id, file_id, line, context FROM reference_lines WHERE file_id IN ({string.Join(", ", parameters)})"; using var reader = cmd.ExecuteReader(); @@ -1111,7 +1111,8 @@ FROM reference_lines var id = reader.GetInt64(0); var fileId = reader.GetInt64(1); var line = reader.GetInt32(2); - var key = (fileId, line); + var context = reader.GetString(3); + var key = (fileId, line, context); if (contextsByLine.ContainsKey(key)) lineIds[key] = id; } diff --git a/tests/CodeIndex.Tests/DatabaseTests.cs b/tests/CodeIndex.Tests/DatabaseTests.cs index 2cda50b230..21a447b928 100644 --- a/tests/CodeIndex.Tests/DatabaseTests.cs +++ b/tests/CodeIndex.Tests/DatabaseTests.cs @@ -1090,7 +1090,7 @@ public void InsertReferences_ChunksLargeInputAndDeduplicatesReferenceLines() ReferenceKind = "call", Line = i % 10 + 1, Column = 4, - Context = $"callee_{i}()", + Context = $"line_{i % 10}()", ContainerKind = "function", ContainerName = "caller", }) @@ -1269,6 +1269,115 @@ public void InsertReferences_DeduplicatesReferenceLinesByFileAndLine() Assert.Equal("return authenticate(user, password)", (string)cmd.ExecuteScalar()!); } + [Fact] + public void InsertReferences_PreservesDistinctReferenceLineContextsForSameFileAndLine() + { + var fileId = _writer.UpsertFile(new FileRecord + { + Path = "src/concurrent_ref_lines.py", Lang = "python", Size = 80, Lines = 5, + Modified = new DateTime(2025, 6, 1, 0, 0, 0, DateTimeKind.Utc), + }); + + _writer.InsertReferences([ + new ReferenceRecord { FileId = fileId, SymbolName = "authenticate", ReferenceKind = "call", Line = 2, Column = 4, Context = "return authenticate(user, password)", ContainerKind = "function", ContainerName = "login" }, + ]); + _writer.InsertReferences([ + new ReferenceRecord { FileId = fileId, SymbolName = "authorize", ReferenceKind = "call", Line = 2, Column = 11, Context = "return authorize(user)", ContainerKind = "function", ContainerName = "login" }, + ]); + + using var cmd = _db.Connection.CreateCommand(); + cmd.Parameters.AddWithValue("@fileId", fileId); + + cmd.CommandText = "SELECT COUNT(*) FROM reference_lines WHERE file_id = @fileId AND line = 2"; + Assert.Equal(2L, (long)cmd.ExecuteScalar()!); + + cmd.CommandText = """ + SELECT r.symbol_name, rl.context + FROM symbol_references r + JOIN reference_lines rl ON rl.id = r.reference_line_id + WHERE r.file_id = @fileId + ORDER BY r.symbol_name + """; + var rows = new List<(string SymbolName, string Context)>(); + using var reader = cmd.ExecuteReader(); + while (reader.Read()) + rows.Add((reader.GetString(0), reader.GetString(1))); + + Assert.Equal([ + ("authenticate", "return authenticate(user, password)"), + ("authorize", "return authorize(user)"), + ], rows); + } + + [Fact] + public void InitializeSchema_MigratesReferenceLinesToContextKey() + { + var dbPath = Path.Combine(Path.GetTempPath(), $"codeindex_ref_line_context_key_{Guid.NewGuid():N}.db"); + try + { + using (var connection = new SqliteConnection(new SqliteConnectionStringBuilder { DataSource = dbPath }.ConnectionString)) + { + connection.Open(); + using var seed = connection.CreateCommand(); + seed.CommandText = """ + CREATE TABLE files ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + path TEXT NOT NULL UNIQUE, + lang TEXT, + size INTEGER, + lines INTEGER, + checksum TEXT, + modified DATETIME, + generated INTEGER NOT NULL DEFAULT 0, + indexed_at DATETIME DEFAULT CURRENT_TIMESTAMP + ); + CREATE TABLE reference_lines ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + file_id INTEGER NOT NULL REFERENCES files(id) ON DELETE CASCADE, + line INTEGER NOT NULL, + context TEXT NOT NULL, + UNIQUE(file_id, line) + ); + CREATE TABLE symbol_references ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + file_id INTEGER NOT NULL REFERENCES files(id) ON DELETE CASCADE, + symbol_name TEXT, + reference_kind TEXT, + line INTEGER, + column_number INTEGER, + context TEXT, + reference_line_id INTEGER REFERENCES reference_lines(id), + container_kind TEXT, + container_name TEXT + ); + INSERT INTO files (id, path) VALUES (1, 'src/legacy.py'); + INSERT INTO reference_lines (id, file_id, line, context) VALUES (1, 1, 2, 'return authenticate(user, password)'); + INSERT INTO symbol_references (file_id, symbol_name, reference_kind, line, column_number, reference_line_id) + VALUES (1, 'authenticate', 'call', 2, 4, 1); + """; + seed.ExecuteNonQuery(); + } + + using var migrated = new DbContext(dbPath); + migrated.InitializeSchema(); + var writer = new DbWriter(migrated.Connection); + writer.InsertReferences([ + new ReferenceRecord { FileId = 1, SymbolName = "authorize", ReferenceKind = "call", Line = 2, Column = 11, Context = "return authorize(user)", ContainerKind = "function", ContainerName = "login" }, + ]); + + using var cmd = migrated.Connection.CreateCommand(); + cmd.CommandText = "SELECT COUNT(*) FROM reference_lines WHERE file_id = 1 AND line = 2"; + Assert.Equal(2L, (long)cmd.ExecuteScalar()!); + + cmd.CommandText = "SELECT COUNT(*) FROM symbol_references WHERE reference_line_id IS NOT NULL"; + Assert.Equal(2L, (long)cmd.ExecuteScalar()!); + } + finally + { + DeleteDbFiles(dbPath); + } + } + [Fact] public void InsertReferences_TypeScriptConstAssertion_RoundTripsThroughSql() {