Fix reference line context upserts

changelog.d/unreleased/1755.fixed.md

@@ -0,0 +1,17 @@
+---
+category: fixed
+issues:
+  - 1755
+affected:
+  - src/CodeIndex/Database/DbContext.cs
+  - src/CodeIndex/Database/DbWriter.cs
+  - tests/CodeIndex.Tests/DatabaseTests.cs
+---
+
+## English
+
+- **Reference-line writes preserve distinct same-line contexts (#1755)** — `reference_lines` now keys rows by file, line, and context and inserts idempotently, so overlapping writer batches no longer overwrite a different reference context for the same file and line.
+
+## 日本語
+
+- **同一行の異なる参照コンテキストを保持するようになりました (#1755)** — `reference_lines` は file / line / context をキーにし、冪等な insert を行うため、重なった writer batch が同じ file / line の別コンテキストを上書きしなくなりました。

src/CodeIndex/Database/DbContext.cs

@@ -1294,7 +1294,7 @@ CREATE TABLE IF NOT EXISTS reference_lines (
                 file_id     INTEGER NOT NULL REFERENCES files(id) ON DELETE CASCADE,
                 line        INTEGER NOT NULL,
                 context     TEXT NOT NULL,
-                UNIQUE(file_id, line)
+                UNIQUE(file_id, line, context)
             )");
 
         // Symbols table / シンボルテーブル
@@ -1391,6 +1391,7 @@ value  TEXT
         EnsureColumn("symbol_references", "is_self_reference", "INTEGER NOT NULL DEFAULT 0");
         EnsureColumn("symbol_references", "is_mutual_recursion", "INTEGER NOT NULL DEFAULT 0");
         EnforceRequiredFileIdConstraints();
+        EnsureReferenceLinesContextKey();
 
         // Indexes / インデックス
         Execute("CREATE INDEX IF NOT EXISTS idx_files_lang     ON files(lang)");
@@ -1570,7 +1571,7 @@ CREATE TABLE reference_lines (
                 file_id     INTEGER NOT NULL REFERENCES files(id) ON DELETE CASCADE,
                 line        INTEGER NOT NULL,
                 context     TEXT NOT NULL,
-                UNIQUE(file_id, line)
+                UNIQUE(file_id, line, context)
             )
             """;
         const string referenceLinesColumns = "id, file_id, line, context";
@@ -1611,6 +1612,97 @@ is_mutual_recursion INTEGER NOT NULL DEFAULT 0
         Execute($"DROP TABLE {oldReferenceLines}");
     }
 
+    private void EnsureReferenceLinesContextKey()
+    {
+        if (ReferenceLinesHasContextUniqueKey())
+            return;
+
+        const string referenceLinesCreateSql =
+            """
+            CREATE TABLE reference_lines (
+                id          INTEGER PRIMARY KEY AUTOINCREMENT,
+                file_id     INTEGER NOT NULL REFERENCES files(id) ON DELETE CASCADE,
+                line        INTEGER NOT NULL,
+                context     TEXT NOT NULL,
+                UNIQUE(file_id, line, context)
+            )
+            """;
+        const string referenceLinesColumns = "id, file_id, line, context";
+        const string symbolReferencesCreateSql =
+            """
+            CREATE TABLE symbol_references (
+                id              INTEGER PRIMARY KEY AUTOINCREMENT,
+                file_id         INTEGER NOT NULL REFERENCES files(id) ON DELETE CASCADE,
+                symbol_name     TEXT,
+                reference_kind  TEXT,
+                line            INTEGER,
+                column_number   INTEGER,
+                context         TEXT,
+                reference_line_id INTEGER REFERENCES reference_lines(id),
+                container_kind  TEXT,
+                container_name  TEXT,
+                symbol_name_folded TEXT,
+                container_name_folded TEXT,
+                is_self_reference INTEGER NOT NULL DEFAULT 0,
+                is_mutual_recursion INTEGER NOT NULL DEFAULT 0
+            )
+            """;
+        const string symbolReferencesColumns = "id, file_id, symbol_name, reference_kind, line, column_number, context, reference_line_id, container_kind, container_name, symbol_name_folded, container_name_folded, is_self_reference, is_mutual_recursion";
+
+        const string oldReferenceLines = "_reference_lines_file_line_key";
+        const string oldSymbolReferences = "_symbol_references_file_line_key";
+        var foreignKeys = ReadPragmaLong("foreign_keys");
+        Execute("PRAGMA foreign_keys=OFF");
+        try
+        {
+            Execute($"DROP TABLE IF EXISTS {oldSymbolReferences}");
+            Execute($"DROP TABLE IF EXISTS {oldReferenceLines}");
+            Execute($"ALTER TABLE symbol_references RENAME TO {oldSymbolReferences}");
+            Execute($"ALTER TABLE reference_lines RENAME TO {oldReferenceLines}");
+            Execute(referenceLinesCreateSql);
+            Execute($"INSERT INTO reference_lines ({referenceLinesColumns}) SELECT {referenceLinesColumns} FROM {oldReferenceLines}");
+            Execute(symbolReferencesCreateSql);
+            Execute($"INSERT INTO symbol_references ({symbolReferencesColumns}) SELECT {symbolReferencesColumns} FROM {oldSymbolReferences}");
+            Execute($"DROP TABLE {oldSymbolReferences}");
+            Execute($"DROP TABLE {oldReferenceLines}");
+        }
+        finally
+        {
+            Execute($"PRAGMA foreign_keys={foreignKeys}");
+        }
+
+        _schemaCache?.Refresh();
+    }
+
+    private bool ReferenceLinesHasContextUniqueKey()
+    {
+        using var listCmd = _connection.CreateCommand();
+        listCmd.CommandText = "PRAGMA index_list('reference_lines')";
+        using var indexReader = listCmd.ExecuteReader();
+        var indexNames = new List<string>();
+        while (indexReader.Read())
+        {
+            var isUnique = indexReader.GetInt32(2) == 1;
+            if (isUnique)
+                indexNames.Add(indexReader.GetString(1));
+        }
+
+        foreach (var indexName in indexNames)
+        {
+            using var infoCmd = _connection.CreateCommand();
+            infoCmd.CommandText = $"PRAGMA index_info('{indexName.Replace("'", "''")}')";
+            using var infoReader = infoCmd.ExecuteReader();
+            var columns = new List<string>();
+            while (infoReader.Read())
+                columns.Add(infoReader.GetString(2));
+
+            if (columns.SequenceEqual(["file_id", "line", "context"], StringComparer.Ordinal))
+                return true;
+        }
+
+        return false;
+    }
+
     private void RebuildTableWithRequiredFileId(string tableName, string createSql, string columns)
     {
         if (ColumnIsNotNull(tableName, "file_id"))
@@ -1830,7 +1922,7 @@ CREATE TABLE IF NOT EXISTS reference_lines (
                 file_id     INTEGER NOT NULL REFERENCES files(id) ON DELETE CASCADE,
                 line        INTEGER NOT NULL,
                 context     TEXT NOT NULL,
-                UNIQUE(file_id, line)
+                UNIQUE(file_id, line, context)
             )"));
         yield return ("CREATE TABLE symbol_references", () => Execute(@"
             CREATE TABLE IF NOT EXISTS symbol_references (

src/CodeIndex/Database/DbWriter.cs

@@ -1018,7 +1018,7 @@ INSERT INTO symbol_references (
             for (int j = i; j < end; j++)
             {
                 var reference = references[j];
-                var referenceLineId = referenceLineIds[(reference.FileId, reference.Line)];
+                var referenceLineId = referenceLineIds[(reference.FileId, reference.Line, reference.Context)];
 
                 if (j > i)
                     sql.Append(", ");
@@ -1052,13 +1052,13 @@ INSERT INTO symbol_references (
         RefreshMutualRecursionFlags();
     }
 
-    private Dictionary<(long FileId, int Line), long> UpsertReferenceLines(IReadOnlyList<ReferenceRecord> references, int start, int end)
+    private Dictionary<(long FileId, int Line, string Context), long> UpsertReferenceLines(IReadOnlyList<ReferenceRecord> references, int start, int end)
     {
-        var contextsByLine = new Dictionary<(long FileId, int Line), string>();
+        var contextsByLine = new Dictionary<(long FileId, int Line, string Context), string>();
         for (int i = start; i < end; i++)
         {
             var reference = references[i];
-            contextsByLine[(reference.FileId, reference.Line)] = reference.Context;
+            contextsByLine[(reference.FileId, reference.Line, reference.Context)] = reference.Context;
         }
 
         var rows = contextsByLine.ToArray();
@@ -1074,20 +1074,20 @@ INSERT INTO symbol_references (
                 if (j > i)
                     sql.Append(", ");
                 var suffix = j - i;
-                var ((fileId, line), context) = rows[j];
+                var ((fileId, line, _), context) = rows[j];
                 sql.Append($"(@fid{suffix}, @line{suffix}, @context{suffix})");
                 cmd.Parameters.Add($"@fid{suffix}", SqliteType.Integer).Value = fileId;
                 cmd.Parameters.Add($"@line{suffix}", SqliteType.Integer).Value = line;
                 cmd.Parameters.Add($"@context{suffix}", SqliteType.Text).Value = context;
             }
 
-            sql.Append(" ON CONFLICT(file_id, line) DO UPDATE SET context = excluded.context");
+            sql.Append(" ON CONFLICT(file_id, line, context) DO NOTHING");
             cmd.CommandText = sql.ToString();
             cmd.ExecuteNonQuery();
         }
 
         var fileIds = contextsByLine.Keys.Select(key => key.FileId).Distinct().ToArray();
-        var lineIds = new Dictionary<(long FileId, int Line), long>();
+        var lineIds = new Dictionary<(long FileId, int Line, string Context), long>();
         int fileIdsPerStatement = GetRowsPerInsertStatement(columnCount: 1);
         for (int i = 0; i < fileIds.Length; i += fileIdsPerStatement)
         {
@@ -1102,7 +1102,7 @@ INSERT INTO symbol_references (
             }
 
             cmd.CommandText = $@"
-                SELECT id, file_id, line
+                SELECT id, file_id, line, context
                 FROM reference_lines
                 WHERE file_id IN ({string.Join(", ", parameters)})";
             using var reader = cmd.ExecuteReader();
@@ -1111,7 +1111,8 @@ FROM reference_lines
                 var id = reader.GetInt64(0);
                 var fileId = reader.GetInt64(1);
                 var line = reader.GetInt32(2);
-                var key = (fileId, line);
+                var context = reader.GetString(3);
+                var key = (fileId, line, context);
                 if (contextsByLine.ContainsKey(key))
                     lineIds[key] = id;
             }

tests/CodeIndex.Tests/DatabaseTests.cs

@@ -1090,7 +1090,7 @@ public void InsertReferences_ChunksLargeInputAndDeduplicatesReferenceLines()
                 ReferenceKind = "call",
                 Line = i % 10 + 1,
                 Column = 4,
-                Context = $"callee_{i}()",
+                Context = $"line_{i % 10}()",
                 ContainerKind = "function",
                 ContainerName = "caller",
             })
@@ -1269,6 +1269,115 @@ public void InsertReferences_DeduplicatesReferenceLinesByFileAndLine()
         Assert.Equal("return authenticate(user, password)", (string)cmd.ExecuteScalar()!);
     }
 
+    [Fact]
+    public void InsertReferences_PreservesDistinctReferenceLineContextsForSameFileAndLine()
+    {
+        var fileId = _writer.UpsertFile(new FileRecord
+        {
+            Path = "src/concurrent_ref_lines.py", Lang = "python", Size = 80, Lines = 5,
+            Modified = new DateTime(2025, 6, 1, 0, 0, 0, DateTimeKind.Utc),
+        });
+
+        _writer.InsertReferences([
+            new ReferenceRecord { FileId = fileId, SymbolName = "authenticate", ReferenceKind = "call", Line = 2, Column = 4, Context = "return authenticate(user, password)", ContainerKind = "function", ContainerName = "login" },
+        ]);
+        _writer.InsertReferences([
+            new ReferenceRecord { FileId = fileId, SymbolName = "authorize", ReferenceKind = "call", Line = 2, Column = 11, Context = "return authorize(user)", ContainerKind = "function", ContainerName = "login" },
+        ]);
+
+        using var cmd = _db.Connection.CreateCommand();
+        cmd.Parameters.AddWithValue("@fileId", fileId);
+
+        cmd.CommandText = "SELECT COUNT(*) FROM reference_lines WHERE file_id = @fileId AND line = 2";
+        Assert.Equal(2L, (long)cmd.ExecuteScalar()!);
+
+        cmd.CommandText = """
+            SELECT r.symbol_name, rl.context
+            FROM symbol_references r
+            JOIN reference_lines rl ON rl.id = r.reference_line_id
+            WHERE r.file_id = @fileId
+            ORDER BY r.symbol_name
+            """;
+        var rows = new List<(string SymbolName, string Context)>();
+        using var reader = cmd.ExecuteReader();
+        while (reader.Read())
+            rows.Add((reader.GetString(0), reader.GetString(1)));
+
+        Assert.Equal([
+            ("authenticate", "return authenticate(user, password)"),
+            ("authorize", "return authorize(user)"),
+        ], rows);
+    }
+
+    [Fact]
+    public void InitializeSchema_MigratesReferenceLinesToContextKey()
+    {
+        var dbPath = Path.Combine(Path.GetTempPath(), $"codeindex_ref_line_context_key_{Guid.NewGuid():N}.db");
+        try
+        {
+            using (var connection = new SqliteConnection(new SqliteConnectionStringBuilder { DataSource = dbPath }.ConnectionString))
+            {
+                connection.Open();
+                using var seed = connection.CreateCommand();
+                seed.CommandText = """
+                    CREATE TABLE files (
+                        id INTEGER PRIMARY KEY AUTOINCREMENT,
+                        path TEXT NOT NULL UNIQUE,
+                        lang TEXT,
+                        size INTEGER,
+                        lines INTEGER,
+                        checksum TEXT,
+                        modified DATETIME,
+                        generated INTEGER NOT NULL DEFAULT 0,
+                        indexed_at DATETIME DEFAULT CURRENT_TIMESTAMP
+                    );
+                    CREATE TABLE reference_lines (
+                        id INTEGER PRIMARY KEY AUTOINCREMENT,
+                        file_id INTEGER NOT NULL REFERENCES files(id) ON DELETE CASCADE,
+                        line INTEGER NOT NULL,
+                        context TEXT NOT NULL,
+                        UNIQUE(file_id, line)
+                    );
+                    CREATE TABLE symbol_references (
+                        id INTEGER PRIMARY KEY AUTOINCREMENT,
+                        file_id INTEGER NOT NULL REFERENCES files(id) ON DELETE CASCADE,
+                        symbol_name TEXT,
+                        reference_kind TEXT,
+                        line INTEGER,
+                        column_number INTEGER,
+                        context TEXT,
+                        reference_line_id INTEGER REFERENCES reference_lines(id),
+                        container_kind TEXT,
+                        container_name TEXT
+                    );
+                    INSERT INTO files (id, path) VALUES (1, 'src/legacy.py');
+                    INSERT INTO reference_lines (id, file_id, line, context) VALUES (1, 1, 2, 'return authenticate(user, password)');
+                    INSERT INTO symbol_references (file_id, symbol_name, reference_kind, line, column_number, reference_line_id)
+                    VALUES (1, 'authenticate', 'call', 2, 4, 1);
+                    """;
+                seed.ExecuteNonQuery();
+            }
+
+            using var migrated = new DbContext(dbPath);
+            migrated.InitializeSchema();
+            var writer = new DbWriter(migrated.Connection);
+            writer.InsertReferences([
+                new ReferenceRecord { FileId = 1, SymbolName = "authorize", ReferenceKind = "call", Line = 2, Column = 11, Context = "return authorize(user)", ContainerKind = "function", ContainerName = "login" },
+            ]);
+
+            using var cmd = migrated.Connection.CreateCommand();
+            cmd.CommandText = "SELECT COUNT(*) FROM reference_lines WHERE file_id = 1 AND line = 2";
+            Assert.Equal(2L, (long)cmd.ExecuteScalar()!);
+
+            cmd.CommandText = "SELECT COUNT(*) FROM symbol_references WHERE reference_line_id IS NOT NULL";
+            Assert.Equal(2L, (long)cmd.ExecuteScalar()!);
+        }
+        finally
+        {
+            DeleteDbFiles(dbPath);
+        }
+    }
+
     [Fact]
     public void InsertReferences_TypeScriptConstAssertion_RoundTripsThroughSql()
     {