diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index e37d469404..147011f506 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -46,7 +46,7 @@ Development contracts: |---|---|---| | Read-only database queries | `cdidx status --db /artifacts/codeindex.db --read-only --json`; `cdidx search AuthService --db /artifacts/codeindex.db --immutable` | Query commands accept `--read-only` (alias `--immutable`) to open an existing CodeIndex database through SQLite's immutable read-only URI mode. Use this for CI artifacts, mounted caches, and sandboxes where creating or updating `codeindex.db-wal` / `codeindex.db-shm` sidecars is not allowed. | | Mutating commands | `index`, `backfill-fold`, `optimize`, `vacuum` | These require writable storage and reject read-only database opens. | -| Reusable index artifact | `cdidx export codeindex.cdidx.zip`; `cdidx import codeindex.cdidx.zip --db ` | Run export after indexing and upload the archive. Consumers import before query commands. Use `--prune-paths` when the archive comes from another checkout and the restored DB should advertise the current workspace root. The archive contains `manifest.json` plus `codeindex.db`; import validates manifest format, manifest `user_version`, `database_sha256`, and the embedded SQLite file as a CodeIndex database before replacing the destination DB. Import rejects archive `codeindex.db` entries whose compressed or uncompressed metadata exceeds 8 GiB, and the extraction stream is also capped at 8 GiB. | +| Reusable index artifact | `cdidx export codeindex.cdidx.zip`; `cdidx import codeindex.cdidx.zip --db `; `cdidx import codeindex.cdidx.zip --dry-run --json` | Run export after indexing and upload the archive. Consumers import before query commands, or use `--dry-run` / `--check` to validate the archive without replacing the destination DB. Use `--prune-paths` when the archive comes from another checkout and the restored DB should advertise the current workspace root. The archive contains `manifest.json` plus `codeindex.db`; the manifest carries bounded summary/readiness metadata including row counts, readiness bits, writer/indexed-head metadata, schema contract stamps, and unknown-extension summary when available. Import validates manifest format, manifest `user_version`, `database_sha256`, present summary counts, and the embedded SQLite file as a CodeIndex database before replacing the destination DB. Import rejects archive `codeindex.db` entries whose compressed or uncompressed metadata exceeds 8 GiB, and the extraction stream is also capped at 8 GiB. | | Maintenance checkpoint | `cdidx db checkpoint `; `cdidx db restore ` | Checkpoint snapshots `codeindex.db` plus existing WAL/SHM sidecars before risky maintenance. Restore rolls back and keeps pre-restore files under `.restore-backup-/`. Checkpoints live next to the DB under `.checkpoints//`. `backfill-fold` creates an automatic checkpoint before it mutates rows unless `--no-checkpoint` is passed. | | Binary compatibility | [COMPATIBILITY.md](COMPATIBILITY.md) | Database compatibility across `cdidx` binary upgrades and downgrades is documented there. Keep that policy updated whenever readiness bits, `codeindex_meta` contract stamps, or rebuild requirements change. | | Fold backfill preview and recovery | `backfill-fold --dry-run`; MCP `backfill_fold` with `dry_run: true` or `force: true` | Dry-run previews folded-key rows without mutating the DB or stamping FoldReady. MCP accepts the same preview and can force rewriting all folded keys when an operator needs to recover from suspicious fold metadata or row state even though the stored version/fingerprint appears current. Non-dry-run row rewrites are resumable after interruption: completed row updates remain durable, and final FoldReady metadata is stamped only after verification succeeds. MCP responses include `progress.rows_done`, `progress.rows_total`, and `progress.fraction` so clients can report and retry long backfills. | @@ -2154,7 +2154,7 @@ net9 CI lane に合わせる場合は `FRAMEWORK=net9.0 make test` を使いま |---|---|---| | read-only database query | `cdidx status --db /artifacts/codeindex.db --read-only --json`; `cdidx search AuthService --db /artifacts/codeindex.db --immutable` | query コマンドは `--read-only`（alias: `--immutable`）を受け付け、既存の CodeIndex database を SQLite の immutable read-only URI mode で開けます。CI artifact、mounted cache、`codeindex.db-wal` / `codeindex.db-shm` sidecar を作成・更新できない sandbox で使います。 | | 変更系コマンド | `index`、`backfill-fold`、`optimize`、`vacuum` | 書き込み可能な storage を必要とし、read-only database open を拒否します。 | -| 再利用可能な index artifact | `cdidx export codeindex.cdidx.zip`; `cdidx import codeindex.cdidx.zip --db ` | CI job では index 後に export して archive を upload します。利用側は query コマンドの前に import できます。別 checkout 由来の archive を現在の workspace root として扱いたい場合は `--prune-paths` を使います。archive は `manifest.json` と `codeindex.db` を含み、import は manifest format、manifest `user_version`、`database_sha256`、embedded SQLite file が CodeIndex database であることを検証してから destination DB を置き換えます。archive の `codeindex.db` entry は compressed / uncompressed metadata と extraction stream の双方で 8 GiB を上限に拒否されます。 | +| 再利用可能な index artifact | `cdidx export codeindex.cdidx.zip`; `cdidx import codeindex.cdidx.zip --db `; `cdidx import codeindex.cdidx.zip --dry-run --json` | CI job では index 後に export して archive を upload します。利用側は query コマンドの前に import でき、`--dry-run` / `--check` で destination DB を置き換えず archive を検証できます。別 checkout 由来の archive を現在の workspace root として扱いたい場合は `--prune-paths` を使います。archive は `manifest.json` と `codeindex.db` を含み、manifest は row count、readiness bit、writer / indexed-head metadata、schema contract stamp、利用可能な unknown-extension summary などの bounded summary/readiness metadata を持ちます。import は manifest format、manifest `user_version`、`database_sha256`、存在する summary count、embedded SQLite file が CodeIndex database であることを検証してから destination DB を置き換えます。archive の `codeindex.db` entry は compressed / uncompressed metadata と extraction stream の双方で 8 GiB を上限に拒否されます。 | | maintenance checkpoint | `cdidx db checkpoint `; `cdidx db restore ` | 危険な maintenance の前に `codeindex.db` と既存 WAL/SHM sidecar の filesystem snapshot を作成し、restore で戻します。checkpoint は DB の隣の `.checkpoints//` に置かれ、restore は pre-restore file を `.restore-backup-/` に保持します。`backfill-fold` は `--no-checkpoint` を渡さない限り、row mutation 前に automatic checkpoint を作ります。 | | binary compatibility | [COMPATIBILITY.md](COMPATIBILITY.md) | `cdidx` binary の upgrade / downgrade をまたぐ database compatibility を記載します。readiness bit、`codeindex_meta` contract stamp、rebuild requirement を変える場合は、この policy も更新してください。 | | Fold backfill の preview / recovery | `backfill-fold --dry-run`; MCP `backfill_fold` の `dry_run: true` または `force: true` | dry-run は DB を変更せず FoldReady stamp も書かずに、rewrite 対象の folded-key row をプレビューします。MCP も同じ preview を受け付け、stored version / fingerprint が current に見える場合でも suspicious な fold metadata や row state を復旧するため `force: true` を受け付けます。non-dry-run rewrite は中断後に resume でき、完了済み row update は durable に残り、最終 FoldReady metadata は verification 成功後にだけ stamp されます。MCP response は `progress.rows_done`、`progress.rows_total`、`progress.fraction` を含みます。 | diff --git a/changelog.d/unreleased/3548.fixed.md b/changelog.d/unreleased/3548.fixed.md new file mode 100644 index 0000000000..c54d97a199 --- /dev/null +++ b/changelog.d/unreleased/3548.fixed.md @@ -0,0 +1,17 @@ +--- +category: fixed +issues: + - 3548 +affected: + - src/CodeIndex/Cli/ExportImportCommandRunner.cs + - src/CodeIndex/Cli/JsonOutputContracts.cs + - tests/CodeIndex.Tests/ExportImportCommandRunnerTests.cs +--- + +## English + +- **Export/import JSON errors now include stable validation phases (#3548)** — `cdidx export --json` and `cdidx import --json` now return structured error payloads with `phase` and `error_code` fields for expected parse, manifest, archive, checksum, SQLite validation, prune, replacement, and archive-write failures. + +## 日本語 + +- **export/import の JSON エラーに安定した検証 phase が含まれるようになりました (#3548)** — `cdidx export --json` と `cdidx import --json` は、想定される parse、manifest、archive、checksum、SQLite 検証、prune、置換、archive 書き込み失敗について `phase` と `error_code` を含む構造化エラー payload を返します。 diff --git a/changelog.d/unreleased/3549.changed.md b/changelog.d/unreleased/3549.changed.md new file mode 100644 index 0000000000..40fc4ab638 --- /dev/null +++ b/changelog.d/unreleased/3549.changed.md @@ -0,0 +1,17 @@ +--- +category: changed +issues: + - 3549 +affected: + - src/CodeIndex/Cli/ExportImportCommandRunner.cs + - tests/CodeIndex.Tests/ProgramCliTests.cs + - DEVELOPER_GUIDE.md +--- + +## English + +- **Export manifests now include index readiness and summary metadata (#3549)** — `cdidx export` writes bounded file/chunk/symbol/reference counts, readiness flags, writer and indexed-head metadata, schema contract stamps, and unknown-extension summary when available, and `cdidx import` validates present summary counts before replacing the destination DB. + +## 日本語 + +- **export manifest に index readiness と summary metadata が含まれるようになりました (#3549)** — `cdidx export` は bounded な file / chunk / symbol / reference count、readiness flag、writer と indexed-head metadata、schema contract stamp、利用可能な unknown-extension summary を書き込み、`cdidx import` は destination DB を置き換える前に存在する summary count を検証します。 diff --git a/changelog.d/unreleased/3550.added.md b/changelog.d/unreleased/3550.added.md new file mode 100644 index 0000000000..04e8d11def --- /dev/null +++ b/changelog.d/unreleased/3550.added.md @@ -0,0 +1,19 @@ +--- +category: added +issues: + - 3550 +affected: + - src/CodeIndex/Cli/ExportImportCommandRunner.cs + - src/CodeIndex/Cli/JsonOutputContracts.cs + - src/CodeIndex/Cli/ConsoleUi.cs + - tests/CodeIndex.Tests/ProgramCliTests.cs + - DEVELOPER_GUIDE.md +--- + +## English + +- **Import can now dry-run archive validation (#3550)** — `cdidx import --dry-run` and `--check` validate the archive through the same manifest, size, checksum, SQLite, compatibility, and optional prune-path phases without replacing the destination DB; JSON output reports each phase and whether replacement would be allowed. + +## 日本語 + +- **import で archive validation を dry-run できるようになりました (#3550)** — `cdidx import --dry-run` と `--check` は、destination DB を置き換えずに同じ manifest、size、checksum、SQLite、compatibility、任意の prune-path phase で archive を検証し、JSON output では各 phase と置換可能かどうかを報告します。 diff --git a/src/CodeIndex/Cli/ConsoleUi.cs b/src/CodeIndex/Cli/ConsoleUi.cs index ba31188742..cc7ec20314 100644 --- a/src/CodeIndex/Cli/ConsoleUi.cs +++ b/src/CodeIndex/Cli/ConsoleUi.cs @@ -116,7 +116,7 @@ private static readonly (string Command, string Usage)[] CommandUsageLines = ("suggestions", "cdidx suggestions [id] [--db ] [--json] [--status ] [--language ] [--category ] [--since ] [--agent ] [--limit ] [--offset ] [--format ] [--open-issues ]"), ("export", "cdidx export [--db ] [--json]"), ("export", "cdidx export ctags [--output ] [--db ]"), - ("import", "cdidx import [--db ] [--prune-paths] [--json]"), + ("import", "cdidx import [--db ] [--prune-paths] [--dry-run|--check] [--json]"), ("languages", "cdidx languages [--db ] [--json] [--indexed-only] [--capability ]"), ("batch", "cdidx batch [--db ] # reads JSON string arrays from stdin, one query command per line; max 1,048,576 chars/line and 256 arguments"), ("mcp", "cdidx mcp [--db ] [--transport stdio|http] [--http-listen ] [--audit-log ] [--audit-log-include-values] [--audit-log-max-bytes ] [--suggestion-dedup-threshold <0..1>]"), @@ -1091,6 +1091,7 @@ private static void PrintExamples() Console.WriteLine(" cdidx export ctags --output tags Export editor tags for Vim, Emacs, and Sublime"); Console.WriteLine(" cdidx export codeindex.cdidx.zip Export a portable CodeIndex archive"); Console.WriteLine(" cdidx import codeindex.cdidx.zip Import a portable CodeIndex archive"); + Console.WriteLine(" cdidx import codeindex.cdidx.zip --dry-run Validate an archive without replacing the DB"); Console.WriteLine(" cdidx search \"authenticate\" Full-text search"); Console.WriteLine(" cdidx search \"auth*\" Prefix shorthand in literal-safe mode"); Console.WriteLine(" cdidx search --query --path --path README.md Search for a literal option token"); diff --git a/src/CodeIndex/Cli/ExportImportCommandRunner.cs b/src/CodeIndex/Cli/ExportImportCommandRunner.cs index de137fbf30..57daf69d62 100644 --- a/src/CodeIndex/Cli/ExportImportCommandRunner.cs +++ b/src/CodeIndex/Cli/ExportImportCommandRunner.cs @@ -3,6 +3,7 @@ using System.Security.Cryptography; using System.Text; using System.Text.Json; +using System.Text.Json.Serialization; using CodeIndex.Database; using Microsoft.Data.Sqlite; @@ -16,7 +17,21 @@ internal static class ExportImportCommandRunner internal const int MaxImportManifestJsonDepth = 16; internal const long MaxImportDatabaseBytes = 8L * 1024 * 1024 * 1024; private const int ImportCopyBufferSize = 81920; + private const int ManifestUnknownExtensionFileLimit = DbContext.UnknownExtensionFilePathSampleLimit; + private const int ManifestUnknownExtensionPathCharLimit = 4096; private static readonly DateTimeOffset DeterministicZipTimestamp = new(1980, 1, 1, 0, 0, 0, TimeSpan.Zero); + private const string ExportCommandName = "export"; + private const string ImportCommandName = "import"; + private const string PhaseParseArgs = "parse_args"; + private const string PhaseOpenArchive = "open_archive"; + private const string PhaseManifest = "manifest"; + private const string PhaseDatabaseEntry = "database_entry"; + private const string PhaseSha256 = "sha256"; + private const string PhaseSqliteValidate = "sqlite_validate"; + private const string PhasePrunePaths = "prune_paths"; + private const string PhaseReplaceDb = "replace_db"; + private const string PhaseWriteArchive = "write_archive"; + private const string ImportUsage = "cdidx import [--db ] [--prune-paths] [--dry-run|--check] [--json]"; public static int RunExport(string[] args, JsonSerializerOptions jsonOptions, string appVersion) { @@ -30,8 +45,9 @@ public static int RunImport(string[] args, JsonSerializerOptions jsonOptions) { string? archivePath = null; string? dbPath = null; - var wantsJson = false; + var wantsJson = Array.Exists(args, arg => arg == "--json"); var prunePaths = false; + var dryRun = false; for (var i = 0; i < args.Length; i++) { @@ -46,68 +62,114 @@ public static int RunImport(string[] args, JsonSerializerOptions jsonOptions) prunePaths = true; continue; } + if (arg is "--dry-run" or "--check") + { + dryRun = true; + continue; + } if (TryReadValueOption(args, ref i, "--db", arg, out var dbValue, out var dbError)) { if (dbError != null) - return WriteError(dbError, "use `cdidx import --db `.", "cdidx import [--db ] [--json]"); + return WriteImportError(wantsJson, jsonOptions, PhaseParseArgs, "import_db_requires_value", dbError, "use `cdidx import --db `.", ImportUsage); dbPath = dbValue; continue; } if (arg.StartsWith("-", StringComparison.Ordinal)) - return WriteError($"unknown import option `{arg}`.", "use `cdidx import [--db ]`.", "cdidx import [--db ] [--prune-paths] [--json]"); + return WriteImportError(wantsJson, jsonOptions, PhaseParseArgs, "import_unknown_option", $"unknown import option `{arg}`.", "use `cdidx import [--db ]`.", ImportUsage); if (archivePath != null) - return WriteError($"import accepts exactly one archive path, got extra `{arg}`.", "remove the extra argument.", "cdidx import [--db ] [--json]"); + return WriteImportError(wantsJson, jsonOptions, PhaseParseArgs, "import_extra_archive_path", $"import accepts exactly one archive path, got extra `{arg}`.", "remove the extra argument.", ImportUsage); archivePath = arg; } if (string.IsNullOrWhiteSpace(archivePath)) - return WriteError("import requires an archive path.", "pass an archive produced by `cdidx export `.", "cdidx import [--db ] [--prune-paths] [--json]"); + return WriteImportError(wantsJson, jsonOptions, PhaseParseArgs, "import_archive_required", "import requires an archive path.", "pass an archive produced by `cdidx export `.", ImportUsage); dbPath ??= DbPathResolver.ResolveForQuery(Environment.CurrentDirectory, explicitDbPath: null, explicitDataDir: null).DbPath; var fullDbPath = Path.GetFullPath(DbPathResolver.NormalizeDbPath(dbPath)); var dbDirectory = Path.GetDirectoryName(fullDbPath); if (string.IsNullOrWhiteSpace(dbDirectory)) - return WriteError($"could not resolve destination DB directory for `{dbPath}`.", "pass an explicit `--db `.", "cdidx import [--db ] [--json]"); + return WriteImportError(wantsJson, jsonOptions, PhaseParseArgs, "import_db_directory_unresolved", $"could not resolve destination DB directory for `{dbPath}`.", "pass an explicit `--db `.", ImportUsage); - var tempPath = Path.Combine(dbDirectory, $".codeindex-import-{Guid.NewGuid():N}.db"); + var tempDirectory = dryRun ? Path.GetTempPath() : dbDirectory; + var tempPath = Path.Combine(tempDirectory, $".codeindex-import-{Guid.NewGuid():N}.db"); + var validationPhases = new List(); + var phase = PhaseOpenArchive; try { - Directory.CreateDirectory(dbDirectory); + if (!dryRun) + Directory.CreateDirectory(dbDirectory); using (var archive = ZipFile.OpenRead(archivePath)) { + AddImportValidationPhase(validationPhases, PhaseOpenArchive); + phase = PhaseManifest; var manifestEntry = archive.GetEntry(ManifestEntryName); if (manifestEntry == null) - return WriteError("archive is missing manifest.json.", "use an archive produced by `cdidx export `.", "cdidx import [--db ] [--json]"); + return WriteImportError(wantsJson, jsonOptions, PhaseManifest, "import_manifest_missing", "archive is missing manifest.json.", "use an archive produced by `cdidx export `.", ImportUsage); if (!TryReadManifest(manifestEntry, jsonOptions, out var manifest, out var manifestError)) - return WriteError($"archive manifest is invalid: {manifestError}.", "use an archive produced by `cdidx export `.", "cdidx import [--db ] [--json]"); + return WriteImportError(wantsJson, jsonOptions, PhaseManifest, "import_manifest_invalid", $"archive manifest is invalid: {manifestError}.", "use an archive produced by `cdidx export `.", ImportUsage); if (!TryValidateManifestHeader(manifest, out var manifestHeaderError)) - return WriteError($"archive manifest is invalid: {manifestHeaderError}.", "re-export from a compatible CodeIndex database.", "cdidx import [--db ] [--json]"); + return WriteImportError(wantsJson, jsonOptions, PhaseManifest, "import_manifest_incompatible", $"archive manifest is invalid: {manifestHeaderError}.", "re-export from a compatible CodeIndex database.", ImportUsage); + AddImportValidationPhase(validationPhases, PhaseManifest); + phase = PhaseDatabaseEntry; var dbEntry = archive.GetEntry(DatabaseEntryName); if (dbEntry == null) - return WriteError("archive is missing codeindex.db.", "use an archive produced by `cdidx export `.", "cdidx import [--db ] [--json]"); + return WriteImportError(wantsJson, jsonOptions, PhaseDatabaseEntry, "import_database_entry_missing", "archive is missing codeindex.db.", "use an archive produced by `cdidx export `.", ImportUsage); if (!TryValidateDatabaseEntrySize(dbEntry.Length, dbEntry.CompressedLength, out var sizeValidationMessage)) - return WriteError(sizeValidationMessage, "re-export a smaller CodeIndex database or rebuild a smaller index.", "cdidx import [--db ] [--prune-paths] [--json]"); + return WriteImportError(wantsJson, jsonOptions, PhaseDatabaseEntry, "import_database_entry_too_large", sizeValidationMessage, "re-export a smaller CodeIndex database or rebuild a smaller index.", ImportUsage); ExtractDatabaseEntryToFile(dbEntry, tempPath); + AddImportValidationPhase(validationPhases, PhaseDatabaseEntry); - if (!TryValidateImportedManifest(manifest, tempPath, out var manifestValidationMessage)) - return WriteError($"archive manifest mismatch: {manifestValidationMessage}.", "re-export from a compatible CodeIndex database.", "cdidx import [--db ] [--prune-paths] [--json]"); + phase = PhaseSha256; + if (!TryValidateImportedManifest(manifest, tempPath, out var manifestValidationMessage, out var manifestValidationPhase)) + return WriteImportError(wantsJson, jsonOptions, manifestValidationPhase, "import_manifest_mismatch", $"archive manifest mismatch: {manifestValidationMessage}.", "re-export from a compatible CodeIndex database.", ImportUsage); + AddImportValidationPhase(validationPhases, PhaseSha256); } + phase = PhaseSqliteValidate; if (!DbContext.TryValidateExistingCodeIndexDb(tempPath, out var validationMessage, out _)) - return WriteError($"archive database is invalid: {validationMessage}.", "re-export from a compatible CodeIndex database.", "cdidx import [--db ] [--prune-paths] [--json]"); + return WriteImportError(wantsJson, jsonOptions, PhaseSqliteValidate, "import_database_invalid", $"archive database is invalid: {validationMessage}.", "re-export from a compatible CodeIndex database.", ImportUsage); + AddImportValidationPhase(validationPhases, PhaseSqliteValidate); SqliteConnection.ClearAllPools(); if (prunePaths) { + phase = PhasePrunePaths; RewriteImportedProjectRoot(tempPath, Environment.CurrentDirectory); + AddImportValidationPhase(validationPhases, PhasePrunePaths); SqliteConnection.ClearAllPools(); } + if (dryRun) + { + AddImportValidationPhase(validationPhases, PhaseReplaceDb, "skipped", "dry-run does not replace the destination database"); + if (wantsJson) + { + Console.WriteLine(JsonSerializer.Serialize( + new ImportDryRunResult( + "1", + "success", + Path.GetFullPath(archivePath), + fullDbPath, + dryRun, + prunePaths, + ReplacementWouldBeAllowed: true, + validationPhases), + CliJsonSerializerContextFactory.Create(jsonOptions).ImportDryRunResult)); + } + else + { + Console.WriteLine($"Validated CodeIndex archive {Path.GetFullPath(archivePath)}; replacement would be allowed for {fullDbPath}"); + } + + return CommandExitCodes.Success; + } + + phase = PhaseReplaceDb; ReplaceImportedDatabase(tempPath, fullDbPath); if (wantsJson) { @@ -121,7 +183,7 @@ public static int RunImport(string[] args, JsonSerializerOptions jsonOptions) } catch (Exception ex) when (ex is IOException or InvalidDataException or UnauthorizedAccessException or SqliteException) { - return WriteError($"import failed ({CommandErrorWriter.FormatSanitizedException(ex)}).", "check the archive path and destination database permissions.", "cdidx import [--db ] [--prune-paths] [--json]"); + return WriteImportError(wantsJson, jsonOptions, phase, "import_failed", $"import failed ({CommandErrorWriter.FormatSanitizedException(ex)}).", "check the archive path and destination database permissions.", ImportUsage); } finally { @@ -134,7 +196,7 @@ private static int RunExportArchive(string[] args, JsonSerializerOptions jsonOpt { string? outputPath = null; string? dbPath = null; - var wantsJson = false; + var wantsJson = Array.Exists(args, arg => arg == "--json"); for (var i = 0; i < args.Length; i++) { @@ -148,41 +210,43 @@ private static int RunExportArchive(string[] args, JsonSerializerOptions jsonOpt if (TryReadValueOption(args, ref i, "--db", arg, out var dbValue, out var dbError)) { if (dbError != null) - return WriteError(dbError, "use `cdidx export --db `.", "cdidx export [--db ] [--json]"); + return WriteExportError(wantsJson, jsonOptions, PhaseParseArgs, "export_db_requires_value", dbError, "use `cdidx export --db `.", "cdidx export [--db ] [--json]"); dbPath = dbValue; continue; } if (arg.StartsWith("-", StringComparison.Ordinal)) - return WriteError($"unknown export option `{arg}`.", "use `cdidx export [--db ]` or `cdidx export ctags`.", "cdidx export [--db ] [--json]"); + return WriteExportError(wantsJson, jsonOptions, PhaseParseArgs, "export_unknown_option", $"unknown export option `{arg}`.", "use `cdidx export [--db ]` or `cdidx export ctags`.", "cdidx export [--db ] [--json]"); if (outputPath != null) - return WriteError($"export accepts exactly one archive path, got extra `{arg}`.", "remove the extra argument.", "cdidx export [--db ] [--json]"); + return WriteExportError(wantsJson, jsonOptions, PhaseParseArgs, "export_extra_archive_path", $"export accepts exactly one archive path, got extra `{arg}`.", "remove the extra argument.", "cdidx export [--db ] [--json]"); outputPath = arg; } if (string.IsNullOrWhiteSpace(outputPath)) - return WriteError("export requires an output archive path.", "pass a destination such as `codeindex.cdidx.zip`, or use `cdidx export ctags`.", "cdidx export [--db ] [--json]"); + return WriteExportError(wantsJson, jsonOptions, PhaseParseArgs, "export_archive_required", "export requires an output archive path.", "pass a destination such as `codeindex.cdidx.zip`, or use `cdidx export ctags`.", "cdidx export [--db ] [--json]"); dbPath ??= DbPathResolver.ResolveForQuery(Environment.CurrentDirectory, explicitDbPath: null, explicitDataDir: null).DbPath; var normalizedDbPath = DbPathResolver.NormalizeDbPath(dbPath); if (!DbContext.TryValidateExistingCodeIndexDb(normalizedDbPath, out var validationMessage, out _)) - return WriteError(validationMessage, "run `cdidx index ` first or pass `--db `.", "cdidx export [--db ] [--json]"); + return WriteExportError(wantsJson, jsonOptions, PhaseSqliteValidate, "export_database_invalid", validationMessage, "run `cdidx index ` first or pass `--db `.", "cdidx export [--db ] [--json]"); var fullSourceDbPath = Path.GetFullPath(normalizedDbPath); var fullOutputPath = Path.GetFullPath(outputPath); if (IsDatabaseOrSqliteSidecarPath(fullOutputPath, fullSourceDbPath)) { - return WriteError("export archive path must not be the source database or a SQLite sidecar.", "choose a separate archive path, for example `codeindex.cdidx.zip`.", "cdidx export [--db ] [--json]"); + return WriteExportError(wantsJson, jsonOptions, PhaseParseArgs, "export_archive_overlaps_database", "export archive path must not be the source database or a SQLite sidecar.", "choose a separate archive path, for example `codeindex.cdidx.zip`.", "cdidx export [--db ] [--json]"); } var snapshotPath = Path.Combine(Path.GetTempPath(), $"codeindex-export-{Guid.NewGuid():N}.db"); + var phase = PhaseWriteArchive; try { var outputDirectory = Path.GetDirectoryName(fullOutputPath); if (!string.IsNullOrWhiteSpace(outputDirectory)) Directory.CreateDirectory(outputDirectory); + phase = PhaseSqliteValidate; CreateDatabaseSnapshot(normalizedDbPath, snapshotPath); ExportManifest manifest; using (var snapshotConnection = new SqliteConnection(CreateUnpooledConnectionString(snapshotPath))) @@ -191,7 +255,9 @@ private static int RunExportArchive(string[] args, JsonSerializerOptions jsonOpt manifest = BuildManifest(snapshotConnection, appVersion); } SqliteConnection.ClearAllPools(); + phase = PhaseSha256; manifest = manifest with { DatabaseSha256 = ComputeSha256(snapshotPath) }; + phase = PhaseWriteArchive; WriteExportArchiveFile(fullOutputPath, snapshotPath, manifest, jsonOptions); if (wantsJson) @@ -202,7 +268,7 @@ private static int RunExportArchive(string[] args, JsonSerializerOptions jsonOpt } catch (Exception ex) { - return WriteError($"export failed ({CommandErrorWriter.FormatSanitizedException(ex)}).", "check the database and output archive paths.", "cdidx export [--db ] [--json]"); + return WriteExportError(wantsJson, jsonOptions, phase, "export_failed", $"export failed ({CommandErrorWriter.FormatSanitizedException(ex)}).", "check the database and output archive paths.", "cdidx export [--db ] [--json]"); } finally { @@ -292,14 +358,34 @@ FROM symbols s private static ExportManifest BuildManifest(SqliteConnection connection, string appVersion) { - using var cmd = connection.CreateCommand(); - cmd.CommandText = "PRAGMA user_version"; - var userVersion = Convert.ToInt32(cmd.ExecuteScalar(), System.Globalization.CultureInfo.InvariantCulture); - cmd.CommandText = "SELECT value FROM codeindex_meta WHERE key = 'indexed_project_root' LIMIT 1"; - var projectRoot = cmd.ExecuteScalar() as string; - cmd.CommandText = "SELECT value FROM codeindex_meta WHERE key = 'indexed_head_sha' LIMIT 1"; - var indexedHead = cmd.ExecuteScalar() as string; - return new ExportManifest("1", appVersion, userVersion, projectRoot, indexedHead, string.Empty); + var userVersion = ReadSqliteUserVersion(connection); + var projectRoot = ReadMetaString(connection, DbContext.IndexedProjectRootMetaKey); + var indexedHead = ReadMetaString(connection, DbContext.IndexedHeadShaMetaKey); + return new ExportManifest( + "1", + appVersion, + userVersion, + projectRoot, + indexedHead, + string.Empty, + FileCount: ReadTableCount(connection, "files"), + ChunkCount: ReadTableCount(connection, "chunks"), + SymbolCount: ReadTableCount(connection, "symbols"), + ReferenceCount: ReadTableCount(connection, "symbol_references"), + GraphReady: (userVersion & DbContext.GraphReadyFlag) != 0, + IssuesReady: (userVersion & DbContext.IssuesReadyFlag) != 0, + FoldReady: (userVersion & DbContext.FoldReadyFlag) != 0, + IndexWriterVersion: ReadMetaString(connection, DbContext.CdidxWriterVersionMetaKey), + IndexedHeadBranch: ReadMetaString(connection, DbContext.IndexedHeadBranchMetaKey), + IndexedHeadTimestamp: ReadMetaString(connection, DbContext.IndexedHeadTimestampMetaKey), + CodeIndexMetaSchemaVersion: ReadMetaInt(connection, DbContext.CodeIndexMetaSchemaVersionMetaKey), + CSharpSymbolNameContractVersion: ReadMetaInt(connection, DbContext.CSharpSymbolNameContractVersionMetaKey), + SqlGraphContractVersion: ReadMetaInt(connection, DbContext.SqlGraphContractVersionMetaKey), + HotspotFamilyVersion: ReadMetaInt(connection, DbContext.HotspotFamilyVersionMetaKey), + UnknownExtensionFileCount: ReadMetaLong(connection, DbContext.UnknownExtensionFileCountMetaKey), + UnknownExtensionFiles: ReadUnknownExtensionFiles(connection), + UnknownExtensionFilesTruncated: ReadMetaBool(connection, DbContext.UnknownExtensionFilesTruncatedMetaKey), + UnknownExtensionFilePathLimit: ReadMetaInt(connection, DbContext.UnknownExtensionFilePathLimitMetaKey)); } private static void AddTextEntry(ZipArchive archive, string name, string content) @@ -449,12 +535,49 @@ private static bool TryValidateManifestHeader(ExportManifest manifest, out strin return false; } + if (!ValidateNonNegativeManifestLong(manifest.FileCount, "file_count", out message) + || !ValidateNonNegativeManifestLong(manifest.ChunkCount, "chunk_count", out message) + || !ValidateNonNegativeManifestLong(manifest.SymbolCount, "symbol_count", out message) + || !ValidateNonNegativeManifestLong(manifest.ReferenceCount, "reference_count", out message) + || !ValidateNonNegativeManifestLong(manifest.UnknownExtensionFileCount, "unknown_extension_file_count", out message)) + { + return false; + } + + if (!ValidateNonNegativeManifestInt(manifest.CodeIndexMetaSchemaVersion, "codeindex_meta_schema_version", out message) + || !ValidateNonNegativeManifestInt(manifest.CSharpSymbolNameContractVersion, "csharp_symbol_name_contract_version", out message) + || !ValidateNonNegativeManifestInt(manifest.SqlGraphContractVersion, "sql_graph_contract_version", out message) + || !ValidateNonNegativeManifestInt(manifest.HotspotFamilyVersion, "hotspot_family_version", out message) + || !ValidateNonNegativeManifestInt(manifest.UnknownExtensionFilePathLimit, "unknown_extension_file_path_limit", out message)) + { + return false; + } + + if (manifest.UnknownExtensionFiles is { Length: > ManifestUnknownExtensionFileLimit }) + { + message = $"unknown_extension_files exceeds the manifest limit of {ManifestUnknownExtensionFileLimit}"; + return false; + } + + if (manifest.UnknownExtensionFiles != null) + { + foreach (var path in manifest.UnknownExtensionFiles) + { + if (path.Length > ManifestUnknownExtensionPathCharLimit) + { + message = $"unknown_extension_files contains a path longer than {ManifestUnknownExtensionPathCharLimit} characters"; + return false; + } + } + } + message = string.Empty; return true; } - private static bool TryValidateImportedManifest(ExportManifest manifest, string dbPath, out string message) + private static bool TryValidateImportedManifest(ExportManifest manifest, string dbPath, out string message, out string phase) { + phase = PhaseSha256; var actualSha256 = ComputeSha256(dbPath); if (!string.Equals(manifest.DatabaseSha256, actualSha256, StringComparison.OrdinalIgnoreCase)) { @@ -462,26 +585,156 @@ private static bool TryValidateImportedManifest(ExportManifest manifest, string return false; } - var actualUserVersion = ReadSqliteUserVersion(dbPath); + phase = PhaseSqliteValidate; + int actualUserVersion; + try + { + using var connection = new SqliteConnection(CreateUnpooledConnectionString(dbPath)); + connection.Open(); + actualUserVersion = ReadSqliteUserVersion(connection); + if (!TryValidateManifestCount(manifest.FileCount, connection, "files", "file_count", out message) + || !TryValidateManifestCount(manifest.ChunkCount, connection, "chunks", "chunk_count", out message) + || !TryValidateManifestCount(manifest.SymbolCount, connection, "symbols", "symbol_count", out message) + || !TryValidateManifestCount(manifest.ReferenceCount, connection, "symbol_references", "reference_count", out message)) + { + return false; + } + } + catch (SqliteException ex) + { + message = $"could not validate codeindex.db manifest metadata ({CommandErrorWriter.FormatSanitizedException(ex)})"; + return false; + } + if (actualUserVersion != manifest.UserVersion) { message = $"manifest user_version `{manifest.UserVersion}` does not match codeindex.db user_version `{actualUserVersion}`"; return false; } + phase = string.Empty; message = string.Empty; return true; } - private static int ReadSqliteUserVersion(string dbPath) + private static int ReadSqliteUserVersion(SqliteConnection connection) { - using var connection = new SqliteConnection(CreateUnpooledConnectionString(dbPath)); - connection.Open(); using var cmd = connection.CreateCommand(); cmd.CommandText = "PRAGMA user_version"; return Convert.ToInt32(cmd.ExecuteScalar(), CultureInfo.InvariantCulture); } + private static bool TryValidateManifestCount(long? expected, SqliteConnection connection, string tableName, string fieldName, out string message) + { + if (expected == null) + { + message = string.Empty; + return true; + } + + var actual = ReadTableCount(connection, tableName); + if (actual != expected.Value) + { + message = $"manifest {fieldName} `{expected.Value}` does not match codeindex.db {tableName} count `{actual}`"; + return false; + } + + message = string.Empty; + return true; + } + + private static bool ValidateNonNegativeManifestLong(long? value, string fieldName, out string message) + { + if (value is < 0) + { + message = $"{fieldName} must be non-negative"; + return false; + } + + message = string.Empty; + return true; + } + + private static bool ValidateNonNegativeManifestInt(int? value, string fieldName, out string message) + { + if (value is < 0) + { + message = $"{fieldName} must be non-negative"; + return false; + } + + message = string.Empty; + return true; + } + + private static long ReadTableCount(SqliteConnection connection, string tableName) + { + using var cmd = connection.CreateCommand(); + cmd.CommandText = tableName switch + { + "files" => "SELECT COUNT(*) FROM files", + "chunks" => "SELECT COUNT(*) FROM chunks", + "symbols" => "SELECT COUNT(*) FROM symbols", + "symbol_references" => "SELECT COUNT(*) FROM symbol_references", + _ => throw new ArgumentOutOfRangeException(nameof(tableName), tableName, "Unsupported manifest count table."), + }; + return Convert.ToInt64(cmd.ExecuteScalar(), CultureInfo.InvariantCulture); + } + + private static string? ReadMetaString(SqliteConnection connection, string key) + { + using var cmd = connection.CreateCommand(); + cmd.CommandText = "SELECT value FROM codeindex_meta WHERE key = @key LIMIT 1"; + cmd.Parameters.AddWithValue("@key", key); + return cmd.ExecuteScalar() as string; + } + + private static int? ReadMetaInt(SqliteConnection connection, string key) + { + var value = ReadMetaString(connection, key); + return int.TryParse(value, NumberStyles.Integer, CultureInfo.InvariantCulture, out var parsed) && parsed >= 0 + ? parsed + : null; + } + + private static long? ReadMetaLong(SqliteConnection connection, string key) + { + var value = ReadMetaString(connection, key); + return long.TryParse(value, NumberStyles.Integer, CultureInfo.InvariantCulture, out var parsed) && parsed >= 0 + ? parsed + : null; + } + + private static bool? ReadMetaBool(SqliteConnection connection, string key) + { + var value = ReadMetaString(connection, key); + return bool.TryParse(value, out var parsed) ? parsed : null; + } + + private static string[]? ReadUnknownExtensionFiles(SqliteConnection connection) + { + var json = ReadMetaString(connection, DbContext.UnknownExtensionFilePathsMetaKey); + if (string.IsNullOrWhiteSpace(json) || json.Length > MaxImportManifestBytes) + return null; + + try + { + var files = JsonSerializer.Deserialize(json); + if (files == null || files.Length == 0) + return null; + + return files + .Where(path => !string.IsNullOrWhiteSpace(path)) + .Take(ManifestUnknownExtensionFileLimit) + .Select(path => path.Length <= ManifestUnknownExtensionPathCharLimit ? path : path[..ManifestUnknownExtensionPathCharLimit]) + .ToArray(); + } + catch (JsonException) + { + return null; + } + } + private static bool IsSha256Hex(string? value) { if (value == null || value.Length != 64) @@ -655,7 +908,125 @@ private static bool TryReadValueOption(string[] args, ref int index, string opti private static int WriteError(string message, string hint, string usage) => CommandErrorWriter.Write(message, CommandExitCodes.UsageError, hint, usage); - internal sealed record ExportManifest(string FormatVersion, string CdidxVersion, int UserVersion, string? ProjectRoot, string? IndexedHeadSha, string DatabaseSha256); + private static int WriteImportError( + bool json, + JsonSerializerOptions jsonOptions, + string phase, + string errorCode, + string message, + string hint, + string usage) + => WriteStructuredError(json, jsonOptions, ImportCommandName, phase, errorCode, message, hint, usage); + + private static int WriteExportError( + bool json, + JsonSerializerOptions jsonOptions, + string phase, + string errorCode, + string message, + string hint, + string usage) + => WriteStructuredError(json, jsonOptions, ExportCommandName, phase, errorCode, message, hint, usage); + + private static int WriteStructuredError( + bool json, + JsonSerializerOptions jsonOptions, + string command, + string phase, + string errorCode, + string message, + string hint, + string usage) + { + if (json) + { + Console.WriteLine(JsonSerializer.Serialize( + new ExportImportErrorResult("1", "error", command, phase, errorCode, message, hint, usage), + CliJsonSerializerContextFactory.Create(jsonOptions).ExportImportErrorResult)); + return CommandExitCodes.UsageError; + } + + return WriteError(message, hint, usage); + } + + private static void AddImportValidationPhase( + List validationPhases, + string phase, + string status = "success", + string? message = null) + => validationPhases.Add(new ImportValidationPhaseResult(phase, status, message)); + + internal sealed record ExportManifest( + [property: JsonPropertyName("format_version")] + string FormatVersion, + [property: JsonPropertyName("cdidx_version")] + string CdidxVersion, + [property: JsonPropertyName("user_version")] + int UserVersion, + [property: JsonPropertyName("project_root")] + string? ProjectRoot, + [property: JsonPropertyName("indexed_head_sha")] + string? IndexedHeadSha, + [property: JsonPropertyName("database_sha256")] + string DatabaseSha256, + [property: JsonPropertyName("file_count")] + long? FileCount = null, + [property: JsonPropertyName("chunk_count")] + long? ChunkCount = null, + [property: JsonPropertyName("symbol_count")] + long? SymbolCount = null, + [property: JsonPropertyName("reference_count")] + long? ReferenceCount = null, + [property: JsonPropertyName("graph_ready")] + bool? GraphReady = null, + [property: JsonPropertyName("issues_ready")] + bool? IssuesReady = null, + [property: JsonPropertyName("fold_ready")] + bool? FoldReady = null, + [property: JsonPropertyName("index_writer_version")] + string? IndexWriterVersion = null, + [property: JsonPropertyName("indexed_head_branch")] + string? IndexedHeadBranch = null, + [property: JsonPropertyName("indexed_head_timestamp")] + string? IndexedHeadTimestamp = null, + [property: JsonPropertyName("codeindex_meta_schema_version")] + int? CodeIndexMetaSchemaVersion = null, + [property: JsonPropertyName("csharp_symbol_name_contract_version")] + int? CSharpSymbolNameContractVersion = null, + [property: JsonPropertyName("sql_graph_contract_version")] + int? SqlGraphContractVersion = null, + [property: JsonPropertyName("hotspot_family_version")] + int? HotspotFamilyVersion = null, + [property: JsonPropertyName("unknown_extension_file_count")] + long? UnknownExtensionFileCount = null, + [property: JsonPropertyName("unknown_extension_files")] + string[]? UnknownExtensionFiles = null, + [property: JsonPropertyName("unknown_extension_files_truncated")] + bool? UnknownExtensionFilesTruncated = null, + [property: JsonPropertyName("unknown_extension_file_path_limit")] + int? UnknownExtensionFilePathLimit = null); + internal sealed record ExportImportErrorResult( + [property: JsonPropertyName("api_version")] string ApiVersion, + [property: JsonPropertyName("status")] string Status, + [property: JsonPropertyName("command")] string Command, + [property: JsonPropertyName("phase")] string Phase, + [property: JsonPropertyName("error_code")] string ErrorCode, + [property: JsonPropertyName("message")] string Message, + [property: JsonPropertyName("hint")] string Hint, + [property: JsonPropertyName("usage")] string Usage); + internal sealed record ImportValidationPhaseResult( + [property: JsonPropertyName("phase")] string Phase, + [property: JsonPropertyName("status")] string Status, + [property: JsonPropertyName("message")] string? Message); + internal sealed record ImportDryRunResult( + [property: JsonPropertyName("api_version")] string ApiVersion, + [property: JsonPropertyName("status")] string Status, + [property: JsonPropertyName("archive_path")] string ArchivePath, + [property: JsonPropertyName("db_path")] string DbPath, + [property: JsonPropertyName("dry_run")] bool DryRun, + [property: JsonPropertyName("pruned_paths")] bool PrunedPaths, + [property: JsonPropertyName("replacement_would_be_allowed")] bool ReplacementWouldBeAllowed, + [property: JsonPropertyName("validation_phases")] IReadOnlyList ValidationPhases); internal sealed record ExportArchiveResult(string ApiVersion, string ArchivePath, string DbPath); internal sealed record ImportResult(string ApiVersion, string DbPath, bool PrunedPaths); } diff --git a/src/CodeIndex/Cli/JsonOutputContracts.cs b/src/CodeIndex/Cli/JsonOutputContracts.cs index bb5066b1ee..ea1c00b694 100644 --- a/src/CodeIndex/Cli/JsonOutputContracts.cs +++ b/src/CodeIndex/Cli/JsonOutputContracts.cs @@ -402,7 +402,10 @@ internal sealed record VersionInfoJsonResult( [JsonSerializable(typeof(DiffSummaryJsonResult))] [JsonSerializable(typeof(ExactZeroHintResult))] [JsonSerializable(typeof(ExportImportCommandRunner.ExportArchiveResult))] +[JsonSerializable(typeof(ExportImportCommandRunner.ExportImportErrorResult))] [JsonSerializable(typeof(ExportImportCommandRunner.ExportManifest))] +[JsonSerializable(typeof(ExportImportCommandRunner.ImportDryRunResult))] +[JsonSerializable(typeof(ExportImportCommandRunner.ImportValidationPhaseResult))] [JsonSerializable(typeof(ExcerptSemanticToken))] [JsonSerializable(typeof(FileDependencyResult))] [JsonSerializable(typeof(FileExcerptResult))] diff --git a/tests/CodeIndex.Tests/ExportImportCommandRunnerTests.cs b/tests/CodeIndex.Tests/ExportImportCommandRunnerTests.cs index 2cf904d5cc..32db80009b 100644 --- a/tests/CodeIndex.Tests/ExportImportCommandRunnerTests.cs +++ b/tests/CodeIndex.Tests/ExportImportCommandRunnerTests.cs @@ -1,4 +1,5 @@ using System.IO.Compression; +using System.Security.Cryptography; using System.Text.Json; using CodeIndex.Cli; @@ -7,6 +8,74 @@ namespace CodeIndex.Tests; [Collection("SQLite pool sensitive")] public class ExportImportCommandRunnerTests { + [Fact] + public void RunImport_JsonParseErrorIncludesPhaseAndCode_Issue3548() + { + var jsonOptions = new JsonSerializerOptions { PropertyNamingPolicy = JsonNamingPolicy.SnakeCaseLower }; + + var (exitCode, stdout, stderr) = ConsoleCapture.Capture(() => + ExportImportCommandRunner.RunImport(["--json", "--unknown"], jsonOptions)); + + Assert.Equal(CommandExitCodes.UsageError, exitCode); + Assert.Equal(string.Empty, stderr); + AssertExportImportError(stdout, "import", "parse_args", "import_unknown_option"); + } + + [Fact] + public void RunImport_JsonManifestErrorIncludesPhaseAndCode_Issue3548() + { + var workDir = Path.Combine(Path.GetTempPath(), $"cdidx_manifest_json_error_{Guid.NewGuid():N}"); + Directory.CreateDirectory(workDir); + try + { + var archivePath = CreateArchiveWithManifest(workDir, "{"); + var dbPath = Path.Combine(workDir, "codeindex.db"); + var jsonOptions = new JsonSerializerOptions { PropertyNamingPolicy = JsonNamingPolicy.SnakeCaseLower }; + + var (exitCode, stdout, stderr) = ConsoleCapture.Capture(() => + ExportImportCommandRunner.RunImport([archivePath, "--db", dbPath, "--json"], jsonOptions)); + + Assert.Equal(CommandExitCodes.UsageError, exitCode); + Assert.Equal(string.Empty, stderr); + AssertExportImportError(stdout, "import", "manifest", "import_manifest_invalid"); + Assert.False(File.Exists(dbPath)); + } + finally + { + Directory.Delete(workDir, recursive: true); + } + } + + [Fact] + public void RunImport_JsonSqliteValidationErrorIncludesPhaseAndCode_Issue3548() + { + var workDir = Path.Combine(Path.GetTempPath(), $"cdidx_import_sqlite_json_error_{Guid.NewGuid():N}"); + Directory.CreateDirectory(workDir); + try + { + var databaseBytes = new byte[] { 1, 2, 3, 4 }; + var sha256 = Convert.ToHexString(SHA256.HashData(databaseBytes)).ToLowerInvariant(); + var manifest = $$""" + {"format_version":"1","cdidx_version":"test","user_version":0,"database_sha256":"{{sha256}}"} + """; + var archivePath = CreateArchiveWithManifestAndDatabase(workDir, manifest, databaseBytes); + var dbPath = Path.Combine(workDir, "codeindex.db"); + var jsonOptions = new JsonSerializerOptions { PropertyNamingPolicy = JsonNamingPolicy.SnakeCaseLower }; + + var (exitCode, stdout, stderr) = ConsoleCapture.Capture(() => + ExportImportCommandRunner.RunImport([archivePath, "--db", dbPath, "--json"], jsonOptions)); + + Assert.Equal(CommandExitCodes.UsageError, exitCode); + Assert.Equal(string.Empty, stderr); + AssertExportImportError(stdout, "import", "sqlite_validate", "import_manifest_mismatch"); + Assert.False(File.Exists(dbPath)); + } + finally + { + Directory.Delete(workDir, recursive: true); + } + } + [Fact] public void RunImport_RejectsOversizedManifestBeforeDatabaseEntry() { @@ -590,4 +659,18 @@ private static string CreateArchiveWithManifestAndDatabase(string workDir, strin return archivePath; } + + private static void AssertExportImportError(string stdout, string command, string phase, string errorCode) + { + using var document = JsonDocument.Parse(stdout); + var root = document.RootElement; + Assert.Equal("1", root.GetProperty("api_version").GetString()); + Assert.Equal("error", root.GetProperty("status").GetString()); + Assert.Equal(command, root.GetProperty("command").GetString()); + Assert.Equal(phase, root.GetProperty("phase").GetString()); + Assert.Equal(errorCode, root.GetProperty("error_code").GetString()); + Assert.False(string.IsNullOrWhiteSpace(root.GetProperty("message").GetString())); + Assert.False(string.IsNullOrWhiteSpace(root.GetProperty("hint").GetString())); + Assert.False(string.IsNullOrWhiteSpace(root.GetProperty("usage").GetString())); + } } diff --git a/tests/CodeIndex.Tests/ProgramCliTests.cs b/tests/CodeIndex.Tests/ProgramCliTests.cs index 711d88c452..8781173b7e 100644 --- a/tests/CodeIndex.Tests/ProgramCliTests.cs +++ b/tests/CodeIndex.Tests/ProgramCliTests.cs @@ -6,6 +6,7 @@ using System.Globalization; using System.IO.Compression; using System.Text.Json; +using System.Text.Json.Nodes; namespace CodeIndex.Tests; @@ -394,6 +395,200 @@ public void ExportImportArchive_RestoresCodeIndexDatabase() } } + [Fact] + public void ExportArchive_ManifestIncludesReadinessAndSummaryMetadata_Issue3549() + { + var projectRoot = TestProjectHelper.CreateTempProject("cdidx_export_manifest_metadata"); + try + { + var sourceDbPath = TestProjectHelper.CreateProjectDb(projectRoot); + TestProjectHelper.InsertIndexedFile(sourceDbPath, "src/app.cs", "csharp", "class App { void Run() {} }\n"); + using (var db = new DbContext(sourceDbPath)) + { + using var cmd = db.Connection.CreateCommand(); + cmd.CommandText = $"PRAGMA user_version = {DbContext.CurrentSchemaVersion.ToString(CultureInfo.InvariantCulture)}"; + cmd.ExecuteNonQuery(); + var writer = new DbWriter(db.Connection); + writer.SetMeta(DbContext.CdidxWriterVersionMetaKey, "test-writer"); + writer.SetMeta(DbContext.IndexedHeadBranchMetaKey, "main"); + writer.SetMeta(DbContext.IndexedHeadTimestampMetaKey, "2026-06-11T00:00:00Z"); + writer.SetMeta(DbContext.CodeIndexMetaSchemaVersionMetaKey, "1"); + writer.SetMeta(DbContext.CSharpSymbolNameContractVersionMetaKey, "2"); + writer.SetMeta(DbContext.SqlGraphContractVersionMetaKey, "1"); + writer.SetMeta(DbContext.HotspotFamilyVersionMetaKey, "2"); + writer.SetMeta(DbContext.UnknownExtensionFileCountMetaKey, "2"); + writer.SetMeta(DbContext.UnknownExtensionFilePathsMetaKey, "[\"tools/custom.foo\",\"docs/archive.bar\"]"); + writer.SetMeta(DbContext.UnknownExtensionFilesTruncatedMetaKey, "false"); + writer.SetMeta(DbContext.UnknownExtensionFilePathLimitMetaKey, "50"); + } + + var archivePath = Path.Combine(projectRoot, "codeindex.cdidx.zip"); + + var (exportExit, _, exportStderr) = RunCliInSubprocess(["export", archivePath, "--db", sourceDbPath]); + + Assert.True(exportExit == 0, exportStderr); + Assert.Equal(string.Empty, exportStderr); + using var archive = ZipFile.OpenRead(archivePath); + var manifestEntry = archive.GetEntry("manifest.json") + ?? throw new InvalidOperationException("manifest.json entry was not found"); + using var document = JsonDocument.Parse(manifestEntry.Open()); + var root = document.RootElement; + Assert.Equal(1, root.GetProperty("file_count").GetInt64()); + Assert.True(root.GetProperty("chunk_count").GetInt64() >= 1); + Assert.True(root.GetProperty("symbol_count").GetInt64() >= 1); + Assert.True(root.GetProperty("reference_count").GetInt64() >= 0); + Assert.Equal("test-writer", root.GetProperty("index_writer_version").GetString()); + Assert.Equal("main", root.GetProperty("indexed_head_branch").GetString()); + Assert.Equal("2026-06-11T00:00:00Z", root.GetProperty("indexed_head_timestamp").GetString()); + Assert.Equal(1, root.GetProperty("codeindex_meta_schema_version").GetInt32()); + Assert.Equal(2, root.GetProperty("csharp_symbol_name_contract_version").GetInt32()); + Assert.Equal(1, root.GetProperty("sql_graph_contract_version").GetInt32()); + Assert.Equal(2, root.GetProperty("hotspot_family_version").GetInt32()); + Assert.Equal(2, root.GetProperty("unknown_extension_file_count").GetInt64()); + Assert.False(root.GetProperty("unknown_extension_files_truncated").GetBoolean()); + Assert.Equal(50, root.GetProperty("unknown_extension_file_path_limit").GetInt32()); + Assert.Equal("tools/custom.foo", root.GetProperty("unknown_extension_files")[0].GetString()); + Assert.Equal(JsonValueKind.True, root.GetProperty("graph_ready").ValueKind); + Assert.Equal(JsonValueKind.True, root.GetProperty("issues_ready").ValueKind); + Assert.Equal(JsonValueKind.True, root.GetProperty("fold_ready").ValueKind); + } + finally + { + TestProjectHelper.DeleteDirectory(projectRoot); + } + } + + [Fact] + public void ImportArchive_RejectsManifestFileCountMismatch_Issue3549() + { + var projectRoot = TestProjectHelper.CreateTempProject("cdidx_import_manifest_count_mismatch"); + try + { + var sourceDbPath = TestProjectHelper.CreateProjectDb(projectRoot); + TestProjectHelper.InsertIndexedFile(sourceDbPath, "src/app.cs", "csharp", "class App { void Run() {} }\n"); + var archivePath = Path.Combine(projectRoot, "codeindex.cdidx.zip"); + var importedDbPath = Path.Combine(projectRoot, "imported", "codeindex.db"); + + var (exportExit, _, exportStderr) = RunCliInSubprocess(["export", archivePath, "--db", sourceDbPath]); + ReplaceManifestNumber(archivePath, "file_count", 999); + var (importExit, importStdout, importStderr) = RunCliInSubprocess(["import", archivePath, "--db", importedDbPath, "--json"]); + + Assert.True(exportExit == 0, exportStderr); + Assert.Equal(CommandExitCodes.UsageError, importExit); + Assert.Equal(string.Empty, importStderr); + using var document = JsonDocument.Parse(importStdout); + Assert.Equal("sqlite_validate", document.RootElement.GetProperty("phase").GetString()); + Assert.Equal("import_manifest_mismatch", document.RootElement.GetProperty("error_code").GetString()); + Assert.Contains("file_count", document.RootElement.GetProperty("message").GetString(), StringComparison.Ordinal); + Assert.False(File.Exists(importedDbPath)); + } + finally + { + TestProjectHelper.DeleteDirectory(projectRoot); + } + } + + [Fact] + public void ImportArchive_AcceptsOlderManifestWithoutSummaryMetadata_Issue3549() + { + var projectRoot = TestProjectHelper.CreateTempProject("cdidx_import_old_manifest"); + try + { + var sourceDbPath = TestProjectHelper.CreateProjectDb(projectRoot); + TestProjectHelper.InsertIndexedFile(sourceDbPath, "src/app.cs", "csharp", "class App { void Run() {} }\n"); + var archivePath = Path.Combine(projectRoot, "codeindex.cdidx.zip"); + var importedDbPath = Path.Combine(projectRoot, "imported", "codeindex.db"); + + var (exportExit, _, exportStderr) = RunCliInSubprocess(["export", archivePath, "--db", sourceDbPath]); + RemoveManifestProperties( + archivePath, + "file_count", + "chunk_count", + "symbol_count", + "reference_count", + "graph_ready", + "issues_ready", + "fold_ready", + "index_writer_version", + "indexed_head_branch", + "indexed_head_timestamp", + "codeindex_meta_schema_version", + "csharp_symbol_name_contract_version", + "sql_graph_contract_version", + "hotspot_family_version", + "unknown_extension_file_count", + "unknown_extension_files", + "unknown_extension_files_truncated", + "unknown_extension_file_path_limit"); + var (importExit, importStdout, importStderr) = RunCliInSubprocess(["import", archivePath, "--db", importedDbPath, "--json"]); + + Assert.True(exportExit == 0, exportStderr); + Assert.Equal(CommandExitCodes.Success, importExit); + Assert.Equal(string.Empty, importStderr); + Assert.True(File.Exists(importedDbPath)); + using var document = JsonDocument.Parse(importStdout); + Assert.Equal("1", document.RootElement.GetProperty("api_version").GetString()); + Assert.Equal(Path.GetFullPath(importedDbPath), document.RootElement.GetProperty("db_path").GetString()); + } + finally + { + TestProjectHelper.DeleteDirectory(projectRoot); + } + } + + [Fact] + public void ImportArchive_DryRunJsonValidatesWithoutReplacingDestination_Issue3550() + { + var projectRoot = TestProjectHelper.CreateTempProject("cdidx_import_dry_run"); + try + { + var sourceDbPath = TestProjectHelper.CreateProjectDb(projectRoot); + TestProjectHelper.InsertIndexedFile(sourceDbPath, "src/app.cs", "csharp", "class App { void Run() {} }\n"); + var archivePath = Path.Combine(projectRoot, "codeindex.cdidx.zip"); + var destinationDbPath = Path.Combine(projectRoot, "destination", "codeindex.db"); + Directory.CreateDirectory(Path.GetDirectoryName(destinationDbPath)!); + File.WriteAllText(destinationDbPath, "existing db"); + File.WriteAllText(destinationDbPath + "-wal", "existing wal"); + File.WriteAllText(destinationDbPath + "-shm", "existing shm"); + + var (exportExit, _, exportStderr) = RunCliInSubprocess(["export", archivePath, "--db", sourceDbPath]); + var (dryRunExit, dryRunStdout, dryRunStderr) = RunCliInSubprocess([ + "import", archivePath, "--db", destinationDbPath, "--prune-paths", "--dry-run", "--json" + ]); + + Assert.True(exportExit == 0, exportStderr); + Assert.Equal(CommandExitCodes.Success, dryRunExit); + Assert.Equal(string.Empty, dryRunStderr); + Assert.Equal("existing db", File.ReadAllText(destinationDbPath)); + Assert.Equal("existing wal", File.ReadAllText(destinationDbPath + "-wal")); + Assert.Equal("existing shm", File.ReadAllText(destinationDbPath + "-shm")); + + using var document = JsonDocument.Parse(dryRunStdout); + var root = document.RootElement; + Assert.Equal("success", root.GetProperty("status").GetString()); + Assert.True(root.GetProperty("dry_run").GetBoolean()); + Assert.True(root.GetProperty("pruned_paths").GetBoolean()); + Assert.True(root.GetProperty("replacement_would_be_allowed").GetBoolean()); + var phases = root.GetProperty("validation_phases") + .EnumerateArray() + .ToDictionary( + phase => phase.GetProperty("phase").GetString()!, + phase => phase.GetProperty("status").GetString()!, + StringComparer.Ordinal); + Assert.Equal("success", phases["open_archive"]); + Assert.Equal("success", phases["manifest"]); + Assert.Equal("success", phases["database_entry"]); + Assert.Equal("success", phases["sha256"]); + Assert.Equal("success", phases["sqlite_validate"]); + Assert.Equal("success", phases["prune_paths"]); + Assert.Equal("skipped", phases["replace_db"]); + } + finally + { + TestProjectHelper.DeleteDirectory(projectRoot); + } + } + [Fact] public void ImportArchive_RejectsDatabaseHashMismatch() { @@ -1088,6 +1283,53 @@ private static void ReplaceManifestUserVersion(string archivePath, int newUserVe writer.Write(updatedManifestJson); } + private static void ReplaceManifestNumber(string archivePath, string propertyName, long newValue) + { + using var archive = ZipFile.Open(archivePath, ZipArchiveMode.Update); + var entry = archive.GetEntry("manifest.json") + ?? throw new InvalidOperationException("manifest.json entry was not found"); + + string manifestJson; + using (var reader = new StreamReader(entry.Open())) + { + manifestJson = reader.ReadToEnd(); + } + + using var document = JsonDocument.Parse(manifestJson); + var oldValue = document.RootElement.GetProperty(propertyName).GetRawText(); + var updatedManifestJson = manifestJson.Replace( + $"\"{propertyName}\":{oldValue}", + $"\"{propertyName}\":{newValue.ToString(CultureInfo.InvariantCulture)}", + StringComparison.Ordinal); + + entry.Delete(); + var replacementEntry = archive.CreateEntry("manifest.json", CompressionLevel.SmallestSize); + using var writer = new StreamWriter(replacementEntry.Open()); + writer.Write(updatedManifestJson); + } + + private static void RemoveManifestProperties(string archivePath, params string[] propertyNames) + { + using var archive = ZipFile.Open(archivePath, ZipArchiveMode.Update); + var entry = archive.GetEntry("manifest.json") + ?? throw new InvalidOperationException("manifest.json entry was not found"); + + JsonObject manifest; + using (var reader = new StreamReader(entry.Open())) + { + manifest = JsonNode.Parse(reader.ReadToEnd())?.AsObject() + ?? throw new InvalidOperationException("manifest.json did not contain an object"); + } + + foreach (var propertyName in propertyNames) + manifest.Remove(propertyName); + + entry.Delete(); + var replacementEntry = archive.CreateEntry("manifest.json", CompressionLevel.SmallestSize); + using var writer = new StreamWriter(replacementEntry.Open()); + writer.Write(manifest.ToJsonString()); + } + private sealed class SuggestionFixture : IDisposable { private readonly string _root;