From 21af0a27c546f1b98fca724a1b324eb7953b4c42 Mon Sep 17 00:00:00 2001 From: Widthdom Date: Mon, 25 May 2026 05:33:14 +0900 Subject: [PATCH 1/2] Fix database metadata and validation issues (#1988 #2025 #2026 #2037) --- changelog.d/unreleased/1988.fixed.md | 16 +++ changelog.d/unreleased/2025.fixed.md | 16 +++ changelog.d/unreleased/2026.fixed.md | 16 +++ changelog.d/unreleased/2037.fixed.md | 17 ++++ src/CodeIndex/Cli/QueryCommandRunner.cs | 12 +++ src/CodeIndex/Database/DbContext.cs | 99 ++++++++++++++++++- src/CodeIndex/Database/DbWriter.cs | 5 + .../LegacySchemaMigrationTests.cs | 97 ++++++++++++++++++ .../QueryCommandRunnerTests.cs | 54 ++++++++++ 9 files changed, 331 insertions(+), 1 deletion(-) create mode 100644 changelog.d/unreleased/1988.fixed.md create mode 100644 changelog.d/unreleased/2025.fixed.md create mode 100644 changelog.d/unreleased/2026.fixed.md create mode 100644 changelog.d/unreleased/2037.fixed.md diff --git a/changelog.d/unreleased/1988.fixed.md b/changelog.d/unreleased/1988.fixed.md new file mode 100644 index 0000000000..d9fb2d7046 --- /dev/null +++ b/changelog.d/unreleased/1988.fixed.md @@ -0,0 +1,16 @@ +--- +category: fixed +issues: + - 1988 +affected: + - src/CodeIndex/Database/DbContext.cs + - tests/CodeIndex.Tests/LegacySchemaMigrationTests.cs +--- + +## English + +- **Column migrations re-check after taking the write lock (#1988)** — standalone `ALTER TABLE ADD COLUMN` migrations now re-read column state after `BEGIN IMMEDIATE`, avoiding duplicate concurrent DDL attempts when another process completed the migration first. + +## 日本語 + +- **列 migration が write lock 取得後に再確認するようになりました (#1988)** — 単独の `ALTER TABLE ADD COLUMN` migration は `BEGIN IMMEDIATE` 後に列状態を再確認し、別プロセスが先に migration を完了した場合の重複 DDL 試行を避けます。 diff --git a/changelog.d/unreleased/2025.fixed.md b/changelog.d/unreleased/2025.fixed.md new file mode 100644 index 0000000000..38d80ed4b0 --- /dev/null +++ b/changelog.d/unreleased/2025.fixed.md @@ -0,0 +1,16 @@ +--- +category: fixed +issues: + - 2025 +affected: + - src/CodeIndex/Database/DbWriter.cs + - tests/CodeIndex.Tests/LegacySchemaMigrationTests.cs +--- + +## English + +- **Metadata stamps now skip DBs without `codeindex_meta` (#2025)** — writer metadata updates are best-effort when the metadata table is absent, preventing missing-table failures from turning indexing cleanup into an unhelpful crash. + +## 日本語 + +- **`codeindex_meta` が無い DB では metadata stamp をスキップするようになりました (#2025)** — metadata table が存在しない場合の writer metadata 更新を best-effort にし、欠落テーブルによる indexing cleanup の分かりにくいクラッシュを防ぎました。 diff --git a/changelog.d/unreleased/2026.fixed.md b/changelog.d/unreleased/2026.fixed.md new file mode 100644 index 0000000000..9a35085a53 --- /dev/null +++ b/changelog.d/unreleased/2026.fixed.md @@ -0,0 +1,16 @@ +--- +category: fixed +issues: + - 2026 +affected: + - src/CodeIndex/Database/DbContext.cs + - tests/CodeIndex.Tests/LegacySchemaMigrationTests.cs +--- + +## English + +- **`codeindex_meta` now has a metadata-policy schema stamp (#2026)** — schema initialization records the metadata-key policy version and prunes known deprecated null keys while preserving unknown future contract stamps for forward-compatibility checks. + +## 日本語 + +- **`codeindex_meta` に metadata policy の schema stamp を追加しました (#2026)** — schema 初期化時に metadata key policy version を記録し、既知の廃止済み null key を削除しつつ、forward-compatibility check 用の未知の将来 contract stamp は保持します。 diff --git a/changelog.d/unreleased/2037.fixed.md b/changelog.d/unreleased/2037.fixed.md new file mode 100644 index 0000000000..0f64ed4895 --- /dev/null +++ b/changelog.d/unreleased/2037.fixed.md @@ -0,0 +1,17 @@ +--- +category: fixed +issues: + - 2037 +affected: + - src/CodeIndex/Cli/QueryCommandRunner.cs + - src/CodeIndex/Database/DbContext.cs + - tests/CodeIndex.Tests/QueryCommandRunnerTests.cs +--- + +## English + +- **Query commands now reject non-CodeIndex SQLite DBs early (#2037)** — `--db` query paths validate the minimal CodeIndex table set before opening a reader and print a direct rebuild hint for empty or wrong-schema SQLite files. + +## 日本語 + +- **query command が CodeIndex ではない SQLite DB を早期に拒否するようになりました (#2037)** — `--db` query 経路は reader を開く前に最小限の CodeIndex table set を検証し、空または別 schema の SQLite file に対して直接的な rebuild hint を表示します。 diff --git a/src/CodeIndex/Cli/QueryCommandRunner.cs b/src/CodeIndex/Cli/QueryCommandRunner.cs index d7a6afc92f..bdb6876c2b 100644 --- a/src/CodeIndex/Cli/QueryCommandRunner.cs +++ b/src/CodeIndex/Cli/QueryCommandRunner.cs @@ -220,6 +220,9 @@ public static int RunBatch(string[] cmdArgs, JsonSerializerOptions jsonOptions) try { using var db = new DbContext(dbPath); + if (!db.TryValidateIsCodeIndexDb(out var validationReason)) + return WriteInvalidCodeIndexDbError(dbPath, validationReason); + db.TryMigrateForRead(); s_batchReader = new DbReader(db); var firstFailure = CommandExitCodes.Success; @@ -4683,6 +4686,8 @@ private static int WithDb(QueryCommandOptions options, JsonSerializerOptions jso else { db = new DbContext(dbPath); + if (!db.TryValidateIsCodeIndexDb(out var validationReason)) + return WriteInvalidCodeIndexDbError(dbPath, validationReason); db.TryMigrateForRead(); reader = new DbReader(db); } @@ -4752,6 +4757,13 @@ private static int WithDb(QueryCommandOptions options, JsonSerializerOptions jso } } + private static int WriteInvalidCodeIndexDbError(string dbPath, string? validationReason) + { + Console.Error.WriteLine($"Error [{CommandErrorCodes.DbError}]: {dbPath} does not appear to be a valid CodeIndex database ({validationReason})."); + Console.Error.WriteLine("Hint: rebuild with `cdidx index --db ` to create a fresh database."); + return CommandExitCodes.DatabaseError; + } + private static string? GetDataDirectoryPath(string? dbPath) { if (string.IsNullOrWhiteSpace(dbPath) || diff --git a/src/CodeIndex/Database/DbContext.cs b/src/CodeIndex/Database/DbContext.cs index 862092cd20..ffec3443de 100644 --- a/src/CodeIndex/Database/DbContext.cs +++ b/src/CodeIndex/Database/DbContext.cs @@ -1035,6 +1035,8 @@ private static void RegisterConnectionFunctionsWithRetry( // bit 2 (FoldReadyFlag, #86): name_folded 列の完全バックフィル完了を示す。 public const int FoldReadyFlag = 4; public const int CurrentSchemaVersion = GraphReadyFlag | IssuesReadyFlag | FoldReadyFlag; // 7 — full CLI readiness + public const int CodeIndexMetaSchemaVersion = 1; + public const string CodeIndexMetaSchemaVersionMetaKey = "codeindex_meta_schema_version"; // Query-semantic readiness for hotspot family grouping. Stored in codeindex_meta instead of // PRAGMA user_version because this guards a higher-level interpretation contract // (`family_key` / `container_qualified_name` are authoritative for the whole DB), not @@ -1243,6 +1245,22 @@ public void ClearReadyFlags() return raw is string s ? s : null; } + public bool TryValidateIsCodeIndexDb(out string? reason) + { + var requiredTables = new[] { "files", "symbols" }; + foreach (var table in requiredTables) + { + if (!TableExists(table)) + { + reason = $"missing required table `{table}`"; + return false; + } + } + + reason = null; + return true; + } + private bool TableExists(string name) { using var cmd = _connection.CreateCommand(); @@ -1355,6 +1373,7 @@ CREATE TABLE IF NOT EXISTS codeindex_meta ( key TEXT PRIMARY KEY NOT NULL, value TEXT )"); + NormalizeCodeIndexMetaKeys(); // Schema migrations for existing DBs / 既存DB向けスキーマ移行 EnsureColumn("files", "checksum", "TEXT"); @@ -2072,8 +2091,19 @@ private static void EmitMigrationFailureWarning(DbMigrationFailure failure) private void EnsureColumn(string tableName, string columnName, string definition) { + if (_activeMigrationTransaction != null) + { + DbColumnEnsurer.EnsureColumn( + () => ColumnExists(tableName, columnName), + () => Execute($"ALTER TABLE {tableName} ADD COLUMN {columnName} {definition}")); + return; + } + DbColumnEnsurer.EnsureColumn( () => ColumnExists(tableName, columnName), + beginImmediate: () => Execute("BEGIN IMMEDIATE"), + commit: () => Execute("COMMIT"), + rollback: () => Execute("ROLLBACK"), () => Execute($"ALTER TABLE {tableName} ADD COLUMN {columnName} {definition}")); } @@ -2102,6 +2132,33 @@ private string ExecuteScalar(string sql) return cmd.ExecuteScalar()?.ToString() ?? ""; } + private void NormalizeCodeIndexMetaKeys() + { + if (!TableExists("codeindex_meta")) + return; + + using (var delete = _connection.CreateCommand()) + { + if (_activeMigrationTransaction != null) + delete.Transaction = _activeMigrationTransaction; + + delete.CommandText = @" + DELETE FROM codeindex_meta + WHERE key IN ('hotspot_family_version', 'hotspot_family_marker_fingerprint') + AND value IS NULL"; + delete.ExecuteNonQuery(); + } + + using var stamp = _connection.CreateCommand(); + if (_activeMigrationTransaction != null) + stamp.Transaction = _activeMigrationTransaction; + stamp.CommandText = @" + INSERT INTO codeindex_meta (key, value) VALUES ('codeindex_meta_schema_version', @version) + ON CONFLICT(key) DO UPDATE SET value = excluded.value"; + stamp.Parameters.AddWithValue("@version", CodeIndexMetaSchemaVersion.ToString(CultureInfo.InvariantCulture)); + stamp.ExecuteNonQuery(); + } + internal void MarkWriteWork() { if (!_isReadOnly && !_suppressWriteWorkTracking) @@ -2167,14 +2224,38 @@ public sealed record DbMigrationFailure( internal static class DbColumnEnsurer { - internal static void EnsureColumn(Func columnExists, Action alterColumn) + internal static void EnsureColumn( + Func columnExists, + Action? beginImmediate, + Action? commit, + Action? rollback, + Action alterColumn) { if (columnExists()) return; + var hasTransactionHooks = beginImmediate != null && commit != null && rollback != null; + var transactionStarted = false; try { + if (hasTransactionHooks) + { + beginImmediate!(); + transactionStarted = true; + if (columnExists()) + { + commit!(); + transactionStarted = false; + return; + } + } + alterColumn(); + if (transactionStarted) + { + commit!(); + transactionStarted = false; + } } catch (SqliteException ex) when (IsDuplicateColumnRace(ex, columnExists)) { @@ -2184,9 +2265,25 @@ internal static void EnsureColumn(Func columnExists, Action alterColumn) // or future wording changes still recover (#1532, #1690). // 列存在を PRAGMA 相当の状態で再確認し、SQLite の英語メッセージに依存せず // 「移行済み」を判定する (#1532)。 + if (transactionStarted) + { + try { rollback!(); } catch (SqliteException) { } + transactionStarted = false; + } + } + catch + { + if (transactionStarted) + { + try { rollback!(); } catch (SqliteException) { } + } + throw; } } + internal static void EnsureColumn(Func columnExists, Action alterColumn) + => EnsureColumn(columnExists, beginImmediate: null, commit: null, rollback: null, alterColumn); + private static bool IsDuplicateColumnRace(SqliteException exception, Func columnExists) { if (!IsDuplicateColumnAddError(exception)) diff --git a/src/CodeIndex/Database/DbWriter.cs b/src/CodeIndex/Database/DbWriter.cs index 36483f343b..d98741a7ed 100644 --- a/src/CodeIndex/Database/DbWriter.cs +++ b/src/CodeIndex/Database/DbWriter.cs @@ -2827,6 +2827,9 @@ private bool ColumnExists(string table, string column) /// public void SetMeta(string key, string? value) { + if (!HasMetaTable()) + return; + using var cmd = _conn.CreateCommand(); cmd.CommandText = @"INSERT INTO codeindex_meta (key, value) VALUES (@key, @value) ON CONFLICT(key) DO UPDATE SET value = excluded.value"; @@ -2844,6 +2847,8 @@ public void SetMeta(string key, string? value) } public void ClearReadyFlags() => Execute("PRAGMA user_version = 0"); + public bool HasMetaTable() => TableExists("codeindex_meta"); + private bool TableExists(string name) { using var cmd = _conn.CreateCommand(); diff --git a/tests/CodeIndex.Tests/LegacySchemaMigrationTests.cs b/tests/CodeIndex.Tests/LegacySchemaMigrationTests.cs index 4dbcefcbb3..e6eac69e62 100644 --- a/tests/CodeIndex.Tests/LegacySchemaMigrationTests.cs +++ b/tests/CodeIndex.Tests/LegacySchemaMigrationTests.cs @@ -1,3 +1,4 @@ +using System.Globalization; using System.Reflection; using CodeIndex.Cli; using CodeIndex.Database; @@ -1458,6 +1459,102 @@ public void EnsureColumn_AlterFailureWithoutColumnPresent_PropagatesInsteadOfBei } } + [Fact] + public void EnsureColumn_RechecksAfterImmediateLockBeforeAlter_Issue1988() + { + var columnPresent = false; + var beginCalled = false; + var alterCalled = false; + + DbColumnEnsurer.EnsureColumn( + () => columnPresent, + beginImmediate: () => + { + beginCalled = true; + columnPresent = true; + }, + commit: () => { }, + rollback: () => { }, + alterColumn: () => alterCalled = true); + + Assert.True(beginCalled); + Assert.False(alterCalled); + } + + [Fact] + public void SetMeta_MissingMetaTable_IsBestEffortNoOp_Issue2025() + { + var dir = Path.Combine(Path.GetTempPath(), $"codeindex_missing_meta_{Guid.NewGuid():N}"); + Directory.CreateDirectory(dir); + var dbPath = Path.Combine(dir, "codeindex.db"); + try + { + using var connection = new SqliteConnection(new SqliteConnectionStringBuilder { DataSource = dbPath }.ConnectionString); + connection.Open(); + using (var cmd = connection.CreateCommand()) + { + cmd.CommandText = "CREATE TABLE files (id INTEGER PRIMARY KEY)"; + cmd.ExecuteNonQuery(); + } + + var writer = new DbWriter(connection); + writer.SetMeta("fold_key_version", "1"); + + Assert.False(writer.HasMetaTable()); + } + finally + { + SqliteConnection.ClearAllPools(); + try { Directory.Delete(dir, recursive: true); } catch { } + } + } + + [Fact] + public void InitializeSchema_PrunesKnownDeprecatedNullMetaKeysAndStampsMetaSchemaVersion_Issue2026() + { + var dir = Path.Combine(Path.GetTempPath(), $"codeindex_meta_schema_{Guid.NewGuid():N}"); + Directory.CreateDirectory(dir); + var dbPath = Path.Combine(dir, "codeindex.db"); + try + { + using (var connection = new SqliteConnection(new SqliteConnectionStringBuilder { DataSource = dbPath }.ConnectionString)) + { + connection.Open(); + using var cmd = connection.CreateCommand(); + cmd.CommandText = @" + CREATE TABLE codeindex_meta (key TEXT PRIMARY KEY NOT NULL, value TEXT); + INSERT INTO codeindex_meta (key, value) VALUES ('fold_key_version', '1'); + INSERT INTO codeindex_meta (key, value) VALUES ('hotspot_family_version', NULL); + INSERT INTO codeindex_meta (key, value) VALUES ('future_contract_version', '99');"; + cmd.ExecuteNonQuery(); + } + + using (var db = new DbContext(dbPath)) + { + db.InitializeSchema(); + } + + using var verify = new SqliteConnection(new SqliteConnectionStringBuilder { DataSource = dbPath }.ConnectionString); + verify.Open(); + using var check = verify.CreateCommand(); + check.CommandText = "SELECT key, value FROM codeindex_meta ORDER BY key"; + using var reader = check.ExecuteReader(); + var values = new Dictionary(StringComparer.Ordinal); + while (reader.Read()) + values[reader.GetString(0)] = reader.IsDBNull(1) ? null : reader.GetString(1); + + Assert.Equal("1", values["fold_key_version"]); + Assert.Equal("99", values["future_contract_version"]); + Assert.Equal(DbContext.CodeIndexMetaSchemaVersion.ToString(CultureInfo.InvariantCulture), values[DbContext.CodeIndexMetaSchemaVersionMetaKey]); + Assert.DoesNotContain("hotspot_family_version", values.Keys); + } + finally + { + SqliteConnection.ClearAllPools(); + try { Directory.Delete(dir, recursive: true); } catch { } + } + } + [Fact] public async Task TryMigrateForRead_ConcurrentLegacyMigrations_SerializeAndComplete() { diff --git a/tests/CodeIndex.Tests/QueryCommandRunnerTests.cs b/tests/CodeIndex.Tests/QueryCommandRunnerTests.cs index 210c2a4833..9c3eaad44d 100644 --- a/tests/CodeIndex.Tests/QueryCommandRunnerTests.cs +++ b/tests/CodeIndex.Tests/QueryCommandRunnerTests.cs @@ -650,6 +650,33 @@ public void Authenticate() { } } } + [Fact] + public void RunBatch_EmptySqliteFileRejectedBeforeQuery_Issue2037() + { + var projectRoot = TestProjectHelper.CreateTempProject("cdidx_issue2037_batch_empty_sqlite"); + try + { + var dbPath = Path.Combine(projectRoot, "empty.db"); + using (var connection = new SqliteConnection(new SqliteConnectionStringBuilder { DataSource = dbPath }.ConnectionString)) + { + connection.Open(); + } + + var (exitCode, _, stderr) = CaptureConsoleWithInput( + "[\"status\",\"--json\"]\n", + () => QueryCommandRunner.RunBatch(["--db", dbPath], _jsonOptions)); + + Assert.Equal(CommandExitCodes.DatabaseError, exitCode); + Assert.Contains("does not appear to be a valid CodeIndex database", stderr); + Assert.Contains("missing required table `files`", stderr); + Assert.Contains("Hint: rebuild with `cdidx index --db `", stderr); + } + finally + { + TestProjectHelper.DeleteDirectory(projectRoot); + } + } + [Fact] public void ParseArgs_ImpactDepthZeroIsRetainedWhenExplicit() { @@ -2880,6 +2907,33 @@ public void WithDb_InvalidSqliteFileSurfacesSqliteCategory_Issue2072() } } + [Fact] + public void WithDb_EmptySqliteFileRejectedBeforeQuery_Issue2037() + { + var projectRoot = TestProjectHelper.CreateTempProject("cdidx_issue2037_empty_sqlite"); + try + { + var dbPath = Path.Combine(projectRoot, "empty.db"); + using (var connection = new SqliteConnection(new SqliteConnectionStringBuilder { DataSource = dbPath }.ConnectionString)) + { + connection.Open(); + } + + var (exitCode, _, stderr) = CaptureConsole(() => QueryCommandRunner.RunStatus( + ["--db", dbPath], + _jsonOptions)); + + Assert.Equal(CommandExitCodes.DatabaseError, exitCode); + Assert.Contains("does not appear to be a valid CodeIndex database", stderr); + Assert.Contains("missing required table `files`", stderr); + Assert.Contains("Hint: rebuild with `cdidx index --db `", stderr); + } + finally + { + TestProjectHelper.DeleteDirectory(projectRoot); + } + } + [Fact] public void WithDb_MalformedFileUriSurfacesDbPathParseError_Issue1990() { From 409b523ecad1008285220127497e92ab588ed703 Mon Sep 17 00:00:00 2001 From: Widthdom Date: Mon, 25 May 2026 09:45:04 +0900 Subject: [PATCH 2/2] Fix read migration inside external transactions (#1988) --- src/CodeIndex/Database/DbContext.cs | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/src/CodeIndex/Database/DbContext.cs b/src/CodeIndex/Database/DbContext.cs index ffec3443de..9fec4db852 100644 --- a/src/CodeIndex/Database/DbContext.cs +++ b/src/CodeIndex/Database/DbContext.cs @@ -31,6 +31,7 @@ public class DbContext : IDisposable private readonly bool _isReadOnly; private readonly string? _schemaCacheKey; private SqliteTransaction? _activeMigrationTransaction; + private bool _readMigrationInsideExternalTransaction; private DbSchemaCache? _schemaCache; private PreparedCommandCache? _preparedCommands; private bool _suppressWriteWorkTracking = true; @@ -1837,14 +1838,12 @@ public void TryMigrateForRead() } catch (SqliteException ex) when (IsNestedTransactionError(ex)) { - if (RunReadMigrationSteps()) - EnsureForeignKeysEnabled(); + RunReadMigrationStepsInsideExternalTransaction(); return; } catch (InvalidOperationException ex) when (IsNestedTransactionError(ex)) { - if (RunReadMigrationSteps()) - EnsureForeignKeysEnabled(); + RunReadMigrationStepsInsideExternalTransaction(); return; } catch (SqliteException ex) when (IsReadOnlyOpenError(ex)) @@ -1874,6 +1873,20 @@ public void TryMigrateForRead() } } + private void RunReadMigrationStepsInsideExternalTransaction() + { + _readMigrationInsideExternalTransaction = true; + try + { + if (RunReadMigrationSteps()) + EnsureForeignKeysEnabled(); + } + finally + { + _readMigrationInsideExternalTransaction = false; + } + } + private bool RunReadMigrationSteps() { try @@ -2091,7 +2104,7 @@ private static void EmitMigrationFailureWarning(DbMigrationFailure failure) private void EnsureColumn(string tableName, string columnName, string definition) { - if (_activeMigrationTransaction != null) + if (_activeMigrationTransaction != null || _readMigrationInsideExternalTransaction) { DbColumnEnsurer.EnsureColumn( () => ColumnExists(tableName, columnName),