From f568d47673e1ef544726b583e4b2868686ba1e20 Mon Sep 17 00:00:00 2001 From: Widthdom Date: Tue, 28 Jul 2026 12:30:22 +0900 Subject: [PATCH 1/3] Fix indexed HEAD provenance in query envelopes (#4854) --- DEVELOPER_GUIDE.md | 14 ++-- USER_GUIDE.md | 4 +- changelog.d/unreleased/4854.fixed.md | 20 ++++++ src/CodeIndex/Cli/JsonEnvelopeWrapper.cs | 4 +- .../IndexCommandRunnerTests.cs | 38 +++++++++-- .../JsonEnvelopeWrapperTests.cs | 67 +++++++++++++++++++ .../McpServerToolsCallTests.cs | 5 +- 7 files changed, 139 insertions(+), 13 deletions(-) create mode 100644 changelog.d/unreleased/4854.fixed.md diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index 2d416f878..4988b9987 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -1579,7 +1579,7 @@ access. | `outline` / `unused` cursor binding | `outline --json` accepts `--kind `, `--limit` / `--top`, opaque `--cursor `, and `--outline-fields ` for bounded machine output. Controlled outline responses keep the normal envelope and add `total_symbol_count`, `returned_symbol_count`, `cursor_offset`, `next_cursor`, `has_more`, and `result_stable_at`, plus `kind_filter` and `selected_fields` when active. `outline` and `unused` cursors bind their offset to the normalized path/scope, filters, ordering, and index generation; reuse after changing those inputs or refreshing the index fails with explicit restart-required guidance. Legacy `outline:` / `unused:` inputs remain accepted for transition, but every newly emitted cursor is opaque and bound. | | `hotspots --json` grouping semantics | `hotspots` and MCP `symbol_hotspots` emit `grouped_by`, `grouping_unit`, `count_kind`, `limit_applies_to`, `score_fields`, `ranking_fields`, and matching `query_context` fields. `--limit` applies to returned symbols, files, name/kind groups, or SQL statements; `--count` ignores `--limit` and reports total groups. Explicit `statement` grouping is SQL-only (`--lang sql` / `lang: "sql"`). | | `--json-envelope` commands | Applies to `search`, `definition`, `references`, `callers`, `callees`, `symbols`, `files`, `find`, `excerpt`, `map`, `inspect`, `outline`, `status`, `validate`, `languages`, `impact`, `deps`, `unused`, and `hotspots`. | -| `--json-envelope` shape | Wraps the per-line `--json` stream into a single `{"metadata": {...}, "results": [...]}` document. Stream terminal records are excluded from `results` and preserved as `metadata.stream_terminal`, while zero-result prelude/control records are preserved as `metadata.stream_control_records`; therefore `result_count` counts result rows only. A `find --all --count` object is both the count result and terminal scan metadata, so it remains in `results` and is also copied to `metadata.stream_terminal`. `metadata` also carries `api_version`, `command`, `cdidx_version`, `elapsed_ms`, `db_path`, `exit_code`, and, when applicable, `query_normalized` and `indexed_at_head_sha`. The bounded high-volume commands above support `--json-envelope --max-json-bytes` by measuring the final serialized document; other envelope/byte-cap combinations remain rejected. | +| `--json-envelope` shape | Wraps the per-line `--json` stream into a single `{"metadata": {...}, "results": [...]}` document. Stream terminal records are excluded from `results` and preserved as `metadata.stream_terminal`, while zero-result prelude/control records are preserved as `metadata.stream_control_records`; therefore `result_count` counts result rows only. A `find --all --count` object is both the count result and terminal scan metadata, so it remains in `results` and is also copied to `metadata.stream_terminal`. `metadata` also carries `api_version`, `command`, `cdidx_version`, `elapsed_ms`, `db_path`, `exit_code`, and, when applicable, `query_normalized` and `indexed_at_head_sha`. `indexed_at_head_sha` maps to the persisted latest-successful `indexed_head_sha` used by status and MCP output after full, `--files`, `--commits`, and `--changed-between` refreshes; failed/rolled-back refreshes do not advance it, and legacy DBs without that key fall back to full-scan-only `indexed_head_commit`. The bounded high-volume commands above support `--json-envelope --max-json-bytes` by measuring the final serialized document; other envelope/byte-cap combinations remain rejected. | | Envelope migration | `--json-envelope` implies `--json`, so callers do not need to pass both. The default output remains the legacy NDJSON / array form for one release; the envelope will become the default in the next major release, when the flat form becomes opt-in via `--json-flat`. | | `find --all` scan summary | `find` requires either repeatable `--path ` filters or explicit `--all`. `--all` cannot be combined with `--path`; safe case-insensitive ASCII literals of at least three characters use the external-content trigram FTS index to select files, then re-run the established line matcher over every selected file. Regex, `--exact` normalization, short/non-ASCII literals, legacy databases without the trigram table, missing trigram synchronization triggers, and active FTS bulk-load rebuilds use the bounded line-scan fallback, preventing false negatives from unsupported tokenization, normalization, or stale index state. Writable initialization rebuilds an existing trigram table when any synchronization trigger is missing, repairing artifacts retained across older-writer rebuilds. Both JSON and human summaries expose `search_strategy` plus optional `search_fallback_reason`; `candidate_files` remains the total scoped file count, while `files_scanned` / `lines_scanned` count post-index verification work. Default JSON rows end with a terminal record carrying `scan_complete`, `authoritative_rows`, returned/scanned counts, active caps, truncation/continuation fields, and recovery guidance. Count JSON carries the same terminal scan state in its single object and uses `authoritative_count`. Row formats that cannot represent the terminal metadata (JSON array, compact, CSV/TSV, LSP, quickfix, and SARIF) are rejected with `--all`; conflicting JSON/text flags are rejected or normalized to NDJSON independent of option order. Candidate-file or line-scan truncation exits with partial-result code `11` unless `--allow-partial` opts into `0`; ordinary result-limit early stops remain successful but set `scan_complete=false` and `result_limit_reached=true`. Human stderr summaries include the strategy, active caps, scan/authority state, continuation action, and recovery guidance and use the same partial exit semantics. | @@ -2354,8 +2354,12 @@ This exercises the entire stack end-to-end. unknown or no longer reachable from `HEAD` (force-push / divergent history) so consumers do not misread a divergent worktree as fresh. Unlike `indexed_head_commit` (#1508 / #1512, full-scan only), the #1509 - triple updates on every successful run so cross-session drift is always - detectable regardless of update mode. + triple updates on every successful full, `--files`, `--commits`, or + `--changed-between` run so cross-session drift is always detectable + regardless of update mode. Failed or rolled-back runs keep the prior triple. + Query-envelope `metadata.indexed_at_head_sha` and MCP/status + `indexed_head_sha` read this same latest-successful stamp, with a + full-scan-only `indexed_head_commit` fallback for legacy databases. 4. **Open SQLite.** `IndexCommandRunner` constructs `new DbContext(dbPath)`, which calls `new SqliteConnection(...)`. @@ -4815,7 +4819,7 @@ help はすべてこのレジストリを参照します。field 名は大文字 | `outline` / `unused` cursor の束縛 | `outline --json` は bounded な機械向け出力として `--kind `、`--limit` / `--top`、opaque な `--cursor `、`--outline-fields ` を受け付けます。制御付き outline 応答は通常の envelope を維持し、`total_symbol_count`、`returned_symbol_count`、`cursor_offset`、`next_cursor`、`has_more`、`result_stable_at` を追加し、該当時は `kind_filter` と `selected_fields` も返します。`outline` と `unused` の cursor は offset を正規化済み path/scope、filter、ordering、index generation に束縛するため、条件変更後または index 更新後の再利用は restart-required の明示案内付きで失敗します。移行用に legacy の `outline:` / `unused:` 入力は受理しますが、新しく出力する cursor はすべて opaque かつ束縛済みです。 | | `hotspots --json` grouping semantics | `hotspots` と MCP `symbol_hotspots` は `grouped_by`、`grouping_unit`、`count_kind`、`limit_applies_to`、`score_fields`、`ranking_fields` と、対応する `query_context` field を返します。`--limit` は返却される symbol、file、name/kind group、SQL statement に適用されます。`--count` は `--limit` を無視し、total group 数を返します。明示的な `statement` grouping は SQL 専用です（`--lang sql` / `lang: "sql"`）。 | | `--json-envelope` 対象 command | `search`、`definition`、`references`、`callers`、`callees`、`symbols`、`files`、`find`、`excerpt`、`map`、`inspect`、`outline`、`status`、`validate`、`languages`、`impact`、`deps`、`unused`、`hotspots`。 | -| `--json-envelope` shape | per-line `--json` stream を単一の `{"metadata": {...}, "results": [...]}` document に包みます。stream 終端レコードは `results` から除外して `metadata.stream_terminal` に保持し、0 件時の prelude / control record は `metadata.stream_control_records` に保持するため、`result_count` は result row だけを数えます。`find --all --count` object は count result であると同時に終端 scan metadata でもあるため、`results` に残しつつ `metadata.stream_terminal` にも複製します。`metadata` は `api_version`、`command`、`cdidx_version`、`elapsed_ms`、`db_path`、`exit_code`、該当時は `query_normalized` と `indexed_at_head_sha` も持ちます。上記の bounded 高ボリューム command は最終 document を測定することで `--json-envelope --max-json-bytes` を許可し、それ以外の envelope / byte-cap 組み合わせは引き続き拒否します。 | +| `--json-envelope` shape | per-line `--json` stream を単一の `{"metadata": {...}, "results": [...]}` document に包みます。stream 終端レコードは `results` から除外して `metadata.stream_terminal` に保持し、0 件時の prelude / control record は `metadata.stream_control_records` に保持するため、`result_count` は result row だけを数えます。`find --all --count` object は count result であると同時に終端 scan metadata でもあるため、`results` に残しつつ `metadata.stream_terminal` にも複製します。`metadata` は `api_version`、`command`、`cdidx_version`、`elapsed_ms`、`db_path`、`exit_code`、該当時は `query_normalized` と `indexed_at_head_sha` も持ちます。`indexed_at_head_sha` は full、`--files`、`--commits`、`--changed-between` refresh 後に status / MCP output が使う永続化済みの最新成功 `indexed_head_sha` に対応します。失敗または rollback された refresh では進まず、この key を持たない legacy DB では full-scan 限定 `indexed_head_commit` に fallback します。上記の bounded 高ボリューム command は最終 document を測定することで `--json-envelope --max-json-bytes` を許可し、それ以外の envelope / byte-cap 組み合わせは引き続き拒否します。 | | envelope migration | `--json-envelope` は `--json` を imply するため、caller は両方を指定する必要がありません。既定 output は 1 release の間 legacy NDJSON / array form のままです。次の major release では envelope が既定になり、flat form は `--json-flat` による opt-in になります。 | | `find --all` scan summary | `find` は repeatable な `--path ` か明示的な `--all` のどちらかを要求し、`--all` と `--path` は併用できません。3 文字以上の安全な大文字小文字を区別しない ASCII literal は external-content trigram FTS index で file を選び、選択した全 file に既存の行 matcher を再適用します。regex、`--exact` normalization、短い literal、非 ASCII literal、trigram table のない旧 database、trigram 同期 trigger の欠落、FTS bulk-load による再構築中は上限付き line-scan fallback を使い、未対応の tokenization、normalization、古い index 状態による false negative を防ぎます。writable initialization は同期 trigger が 1 つでも欠けた既存 trigram table を再構築し、旧 writer の rebuild 後に残った artifact も修復します。JSON と human summary は `search_strategy` と任意の `search_fallback_reason` を返します。`candidate_files` は scope 内の総 file 数を維持し、`files_scanned` / `lines_scanned` は index 適用後の検証量を数えます。既定 JSON row は `scan_complete`、`authoritative_rows`、返却 / 走査件数、有効な cap、切り詰め / continuation field、復旧案内を持つ終端レコードで終了します。count JSON は単一 object に同じ終端 scan 状態を持ち、`authoritative_count` を使います。終端 metadata を表現できない JSON array、compact、CSV/TSV、LSP、quickfix、SARIF は `--all` との組み合わせを拒否し、競合する JSON / text flag は option 順序にかかわらず拒否するか NDJSON に正規化します。candidate-file または line-scan による切り詰めは、`--allow-partial` で `0` を opt-in しない限り partial-result 終了コード `11` を返します。通常の result limit による早期停止は成功のままですが、`scan_complete=false` と `result_limit_reached=true` を設定します。human stderr summary は strategy、有効な cap、scan / authority 状態、continuation action、復旧案内を含み、同じ partial exit semantics を使います。 | @@ -5400,7 +5404,7 @@ flowchart TD 1. **バイナリ起動。** 自己完結型ホストがマネージエントリポイント（`Program.Main`）を解決。 2. **CLI ルーティング。** `Program.cs` が `IndexCommandRunner.Run(args, jsonOptions)` に振り分け。 -3. **DB パス解決。** `DbPathResolver` が `--db` 指定が無い限り `/.cdidx/codeindex.db` を算出し、`.cdidx/` ディレクトリを作成する。同じヘルパーは `status` / `map` / `inspect` の query-time workspace root 解決も担う。`--db` を付けない query は既定の `.cdidx/codeindex.db` sibling path をそのまま正とし、explicit DB は `codeindex_meta.indexed_project_root` を読む。保存済み root metadata を持たない legacy explicit DB は、明示パス自体が `.../.cdidx/codeindex.db` でも `project_root` / `git_head` / `git_is_dirty` / `indexed_head_commit` / `worktree_head_changed` を未設定のまま返す。`WorkspaceMetadataEnricher` は利用可能な場合、最新の成功 index stamp である `indexed_head_sha` と runtime HEAD を比較し、legacy DB だけで従来の full-scan 限定 `indexed_head_commit` に fallback する。index 構築後に worktree の branch / HEAD が切り替わったときは `worktree_head_changed=true` を surface して `status` が WARN を出せるようにする (issues #1512 and #3367)。加えて index 成功時 (full scan / partial update 問わず) に `IndexCommandRunner` が `codeindex_meta.indexed_head_sha` / `indexed_head_branch` / `indexed_head_timestamp` も best-effort で stamp する（`git` 失敗は index 成功を妨げない）。`status` はこれらを `indexed_head_sha` / `indexed_head_branch` / `indexed_head_timestamp` として返し、query 時に `git merge-base --is-ancestor` + `git rev-list --count` で算出する `commits_ahead_of_indexed_head` も付与する。indexed SHA が未知、または force-push / divergent history で現 `HEAD` の祖先でなくなった場合は `null` を返し、consumer が divergent worktree を「最新」と誤読しないようにする。`indexed_head_commit` (#1508 / #1512、full scan 限定) と異なり、#1509 のトリプルは partial update でも更新されるため cross-session のドリフト検出が常に機能する。 +3. **DB パス解決。** `DbPathResolver` が `--db` 指定が無い限り `/.cdidx/codeindex.db` を算出し、`.cdidx/` ディレクトリを作成する。同じヘルパーは `status` / `map` / `inspect` の query-time workspace root 解決も担う。`--db` を付けない query は既定の `.cdidx/codeindex.db` sibling path をそのまま正とし、explicit DB は `codeindex_meta.indexed_project_root` を読む。保存済み root metadata を持たない legacy explicit DB は、明示パス自体が `.../.cdidx/codeindex.db` でも `project_root` / `git_head` / `git_is_dirty` / `indexed_head_commit` / `worktree_head_changed` を未設定のまま返す。`WorkspaceMetadataEnricher` は利用可能な場合、最新の成功 index stamp である `indexed_head_sha` と runtime HEAD を比較し、legacy DB だけで従来の full-scan 限定 `indexed_head_commit` に fallback する。index 構築後に worktree の branch / HEAD が切り替わったときは `worktree_head_changed=true` を surface して `status` が WARN を出せるようにする (issues #1512 and #3367)。加えて index 成功時 (full scan / partial update 問わず) に `IndexCommandRunner` が `codeindex_meta.indexed_head_sha` / `indexed_head_branch` / `indexed_head_timestamp` も best-effort で stamp する（`git` 失敗は index 成功を妨げない）。`status` はこれらを `indexed_head_sha` / `indexed_head_branch` / `indexed_head_timestamp` として返し、query 時に `git merge-base --is-ancestor` + `git rev-list --count` で算出する `commits_ahead_of_indexed_head` も付与する。indexed SHA が未知、または force-push / divergent history で現 `HEAD` の祖先でなくなった場合は `null` を返し、consumer が divergent worktree を「最新」と誤読しないようにする。`indexed_head_commit` (#1508 / #1512、full scan 限定) と異なり、#1509 のトリプルは成功した full、`--files`、`--commits`、`--changed-between` run ごとに更新され、失敗または rollback された run では直前値を維持する。query envelope の `metadata.indexed_at_head_sha` と MCP / status の `indexed_head_sha` は同じ最新成功 stamp を読み、この key が無い legacy database だけ full-scan 限定 `indexed_head_commit` に fallback するため、update mode によらず cross-session のドリフト検出と response 間の整合性が保たれる。 4. **SQLite オープン。** `IndexCommandRunner` が `new DbContext(dbPath)` を構築し、内部で `new SqliteConnection(...)` が呼ばれる。**ネイティブライブラリの解決はこの時点で行われる。** `SqliteConnection` の静的コンストラクタが `SQLitePCL.Batteries_V2.Init()` を呼び、それが `SQLite3Provider_e_sqlite3` 上で `sqlite3_libversion_number()` を起動し、`e_sqlite3` への P/Invoke に到達する。Linux の .NET 動的ローダは次の順で探す（失敗時のエラーメッセージを参照）: - `${apphost_dir}/libe_sqlite3.so` - `${apphost_dir}/e_sqlite3.so`（および `lib` プレフィックスなしのバリエーション） diff --git a/USER_GUIDE.md b/USER_GUIDE.md index 70a3d6027..6e5ce75e3 100644 --- a/USER_GUIDE.md +++ b/USER_GUIDE.md @@ -1586,7 +1586,7 @@ Use `--json` for machine-readable output (AI agents): {"path":"src/Auth/TokenService.cs","lang":"csharp","chunk_start_line":1,"chunk_end_line":80,"snippet_start_line":40,"snippet_end_line":47,"snippet":"if (claims.Count == 0)\\n throw new InvalidOperationException();\\nreturn GenerateToken(claims);","match_lines":[42,47],"highlights":[{"line":47,"text":"return GenerateToken(claims);","terms":["GenerateToken"]}],"context_before":2,"context_after":3,"score":9.8} ``` -Add `--json-envelope` to wrap the per-line stream into a single document with a `metadata` block (command, `cdidx_version`, `elapsed_ms`, `db_path`, `result_count`, `exit_code`, optional `query_normalized` / `indexed_at_head_sha`) and a `results` array. The flag implies `--json` and works on every query command (`search`, `definition`, `references`, `callers`, `callees`, `symbols`, `files`, `find`, `excerpt`, `map`, `inspect`, `outline`, `status`, `validate`, `languages`, `impact`, `deps`, `unused`, `hotspots`). Wrapped commands can capture up to 10,485,760 output characters; if the budget is exceeded, cdidx returns a JSON envelope with empty `results`, non-zero `metadata.exit_code`, and `metadata.error`, and suggests using `--limit` / `--top` or streaming `--json`. The flat NDJSON / array output stays the default for one release; the envelope will become the default in the next major release, at which point the flat form will be opt-in via `--json-flat`. +Add `--json-envelope` to wrap the per-line stream into a single document with a `metadata` block (command, `cdidx_version`, `elapsed_ms`, `db_path`, `result_count`, `exit_code`, optional `query_normalized` / `indexed_at_head_sha`) and a `results` array. `indexed_at_head_sha` has the same meaning as status `indexed_head_sha`: it identifies the checkout captured by the latest successful full scan, `--files`, `--commits`, or `--changed-between` refresh. It does not advance after a failed or rolled-back refresh; databases created before `indexed_head_sha` fall back to the legacy full-scan-only `indexed_head_commit`. The flag implies `--json` and works on every query command (`search`, `definition`, `references`, `callers`, `callees`, `symbols`, `files`, `find`, `excerpt`, `map`, `inspect`, `outline`, `status`, `validate`, `languages`, `impact`, `deps`, `unused`, `hotspots`). Wrapped commands can capture up to 10,485,760 output characters; if the budget is exceeded, cdidx returns a JSON envelope with empty `results`, non-zero `metadata.exit_code`, and `metadata.error`, and suggests using `--limit` / `--top` or streaming `--json`. The flat NDJSON / array output stays the default for one release; the envelope will become the default in the next major release, at which point the flat form will be opt-in via `--json-flat`. Add `--profile` to any read command when debugging slow queries. It appends one JSON object after the normal result with `profile.phases` (`name`, `elapsed_ms`, `rows_scanned`), `profile.query_plan` (`EXPLAIN QUERY PLAN` rows), and `profile.queries` (the SQL text). Add `--slow-query-ms ` to log profiled SQL statements that meet the threshold to the persistent tool log. @@ -4745,7 +4745,7 @@ src/Auth/TokenService.cs:42-58 {"path":"src/Auth/TokenService.cs","start_line":42,"end_line":58,"content":"public string GenerateToken(...)...","lang":"csharp","score":9.8} ``` -`--json-envelope` を追加すると、1 行ごとの stream を `metadata`（command、`cdidx_version`、`elapsed_ms`、`db_path`、`result_count`、`exit_code`、任意の `query_normalized` / `indexed_at_head_sha`）と `results` 配列を持つ 1 つの JSON document に包みます。この flag は `--json` を暗黙に有効化し、各 query command で使えます。wrapped command の捕捉出力は最大 10,485,760 文字です。超過した場合は、空の `results`、非 0 の `metadata.exit_code`、`metadata.error` を持つ JSON envelope を返し、`--limit` / `--top` または streaming `--json` の利用を促します。 +`--json-envelope` を追加すると、1 行ごとの stream を `metadata`（command、`cdidx_version`、`elapsed_ms`、`db_path`、`result_count`、`exit_code`、任意の `query_normalized` / `indexed_at_head_sha`）と `results` 配列を持つ 1 つの JSON document に包みます。`indexed_at_head_sha` の意味は status の `indexed_head_sha` と同じで、最後に成功した full scan、`--files`、`--commits`、`--changed-between` refresh が取り込んだ checkout を示します。失敗または rollback された refresh では進まず、`indexed_head_sha` より前の database では legacy の full-scan 限定 `indexed_head_commit` に fallback します。この flag は `--json` を暗黙に有効化し、各 query command で使えます。wrapped command の捕捉出力は最大 10,485,760 文字です。超過した場合は、空の `results`、非 0 の `metadata.exit_code`、`metadata.error` を持つ JSON envelope を返し、`--limit` / `--top` または streaming `--json` の利用を促します。 ### シンボル検索（関数、クラスなど） diff --git a/changelog.d/unreleased/4854.fixed.md b/changelog.d/unreleased/4854.fixed.md new file mode 100644 index 000000000..2360cbf51 --- /dev/null +++ b/changelog.d/unreleased/4854.fixed.md @@ -0,0 +1,20 @@ +--- +category: fixed +issues: + - 4854 +affected: + - src/CodeIndex/Cli/JsonEnvelopeWrapper.cs + - tests/CodeIndex.Tests/JsonEnvelopeWrapperTests.cs + - tests/CodeIndex.Tests/IndexCommandRunnerTests.cs + - tests/CodeIndex.Tests/McpServerToolsCallTests.cs + - USER_GUIDE.md + - DEVELOPER_GUIDE.md +--- + +## English + +- **Query envelopes now agree with status on the latest indexed HEAD (#4854)** — `metadata.indexed_at_head_sha` now uses the latest successful full or partial refresh stamp established by #1509, #1527, and #4313, while retaining the legacy full-scan HEAD fallback for older databases and preserving the prior stamp after failed or rolled-back refreshes. + +## 日本語 + +- **query envelope と status が最新の indexed HEAD で一致するようになりました (#4854)** — `metadata.indexed_at_head_sha` は #1509、#1527、#4313 で確立された最新の成功 full / partial refresh stamp を使うようになり、古い database 向けの legacy full-scan HEAD fallback と、失敗または rollback された refresh 後の直前 stamp を維持します。 diff --git a/src/CodeIndex/Cli/JsonEnvelopeWrapper.cs b/src/CodeIndex/Cli/JsonEnvelopeWrapper.cs index f4830a54c..80c544326 100644 --- a/src/CodeIndex/Cli/JsonEnvelopeWrapper.cs +++ b/src/CodeIndex/Cli/JsonEnvelopeWrapper.cs @@ -262,7 +262,9 @@ private static JsonObject BuildEnvelope( var resolvedPath = dbPath; if (!dbPathExplicit && !File.Exists(LongPath.EnsureWindowsPrefix(resolvedPath))) return null; - return DbPathResolver.TryReadIndexedHeadCommit(DbPathResolver.NormalizeDbPath(resolvedPath)); + var normalizedPath = DbPathResolver.NormalizeDbPath(resolvedPath); + return DbPathResolver.TryReadIndexedHeadSha(normalizedPath) + ?? DbPathResolver.TryReadIndexedHeadCommit(normalizedPath); } catch { diff --git a/tests/CodeIndex.Tests/IndexCommandRunnerTests.cs b/tests/CodeIndex.Tests/IndexCommandRunnerTests.cs index 2df866e91..504a37be6 100644 --- a/tests/CodeIndex.Tests/IndexCommandRunnerTests.cs +++ b/tests/CodeIndex.Tests/IndexCommandRunnerTests.cs @@ -6800,12 +6800,16 @@ public void RunBackfillFold_LegacyDbWithoutCodeIndexMeta_Succeeds() [Fact] - public void Run_Rebuild_CancelledAfterReadinessDemotion_PreservesExistingIndex() + public void Run_Rebuild_CancelledAfterReadinessDemotion_PreservesExistingIndex_Issue4854() { var projectRoot = CreateTempProject(); try { File.WriteAllText(Path.Combine(projectRoot, "app.cs"), "public class App { public void Run() { } }\n"); + RunGit(projectRoot, "init"); + RunGit(projectRoot, "add", "."); + RunGit(projectRoot, "commit", "-m", "initial"); + var initialHead = RunGitCaptureStdOut(projectRoot, "rev-parse", "HEAD").Trim(); var initialExitCode = IndexCommandRunner.Run([projectRoot, "--json"], _jsonOptions); Assert.Equal(CommandExitCodes.Success, initialExitCode); @@ -6818,6 +6822,10 @@ public void Run_Rebuild_CancelledAfterReadinessDemotion_PreservesExistingIndex() Assert.Contains("app.cs", ReadIndexedPaths(dbPath)); File.WriteAllText(Path.Combine(projectRoot, "later.cs"), "public class Later { }\n"); + RunGit(projectRoot, "add", "."); + RunGit(projectRoot, "commit", "-m", "add later"); + var laterHead = RunGitCaptureStdOut(projectRoot, "rev-parse", "HEAD").Trim(); + Assert.NotEqual(initialHead, laterHead); using var cancellation = new CancellationTokenSource(); var hookInvoked = false; IndexCommandRunner.FullScanWritePhaseStartedForTesting = () => @@ -6849,6 +6857,7 @@ public void Run_Rebuild_CancelledAfterReadinessDemotion_PreservesExistingIndex() { using var db = new DbContext(DbOpenIntent.WriteIndex, dbPath); Assert.Equal(initialReadiness, db.GetUserVersion()); + Assert.Equal(initialHead, db.GetMetaString(DbContext.IndexedHeadShaMetaKey)); }); Assert.DoesNotContain("Last batch did not complete", reopenWarning); Assert.DoesNotContain("later.cs", ReadIndexedPaths(dbPath)); @@ -7770,7 +7779,7 @@ public void RunStatusCheck_FilesRefreshStaysStaleUntilCommitScopedRefreshAtHead( } [Fact] - public void RunStatusCheck_AfterChangedBetweenRefreshAtHead_TreatsCurrentIndexedHeadShaAsFresh_2808() + public void RunStatusCheck_AfterChangedBetweenRefreshAtHead_TreatsCurrentIndexedHeadShaAsFresh_2808_Issue4854() { var projectRoot = CreateTempProject(); try @@ -7813,6 +7822,14 @@ public void RunStatusCheck_AfterChangedBetweenRefreshAtHead_TreatsCurrentIndexed Assert.Equal(initialHead, check.GetProperty("indexed_head_commit").GetString()); Assert.Equal(currentHead, check.GetProperty("workspace_head_commit").GetString()); + var (queryExitCode, queryEnvelope) = RunProgramAndCaptureJson( + ["search", "Run", "--db", dbPath, "--json-envelope"], + projectRoot); + Assert.Equal(CommandExitCodes.Success, queryExitCode); + Assert.Equal( + statusJson.GetProperty("indexed_head_sha").GetString(), + queryEnvelope.GetProperty("metadata").GetProperty("indexed_at_head_sha").GetString()); + var (dotExitCode, dotJson) = RunProgramAndCaptureJson([projectRoot, "--json"]); Assert.Equal(CommandExitCodes.Success, dotExitCode); Assert.Equal("success", dotJson.GetProperty("status").GetString()); @@ -7826,6 +7843,13 @@ public void RunStatusCheck_AfterChangedBetweenRefreshAtHead_TreatsCurrentIndexed var (postDotStatusExitCode, postDotStatusJson) = RunStatusAndCaptureJson(["--db", dbPath, "--check", "--json"]); Assert.Equal(CommandExitCodes.Success, postDotStatusExitCode); Assert.True(postDotStatusJson.GetProperty("workspace_check").GetProperty("matches_workspace").GetBoolean()); + var (postDotQueryExitCode, postDotQueryEnvelope) = RunProgramAndCaptureJson( + ["search", "Run", "--db", dbPath, "--json-envelope"], + projectRoot); + Assert.Equal(CommandExitCodes.Success, postDotQueryExitCode); + Assert.Equal( + postDotStatusJson.GetProperty("indexed_head_sha").GetString(), + postDotQueryEnvelope.GetProperty("metadata").GetProperty("indexed_at_head_sha").GetString()); } finally { @@ -7857,7 +7881,9 @@ public void RunStatusCheck_AfterChangedBetweenRefreshAtHead_TreatsCurrentIndexed } } - private (int ExitCode, JsonElement Json) RunProgramAndCaptureJson(string[] args) + private (int ExitCode, JsonElement Json) RunProgramAndCaptureJson( + string[] args, + string? configStartDirectory = null) { lock (TestConsoleLock.Gate) { @@ -7870,7 +7896,11 @@ public void RunStatusCheck_AfterChangedBetweenRefreshAtHead_TreatsCurrentIndexed { Console.SetOut(stdout); Console.SetError(stderr); - var exitCode = ProgramRunner.Run(args, _jsonOptions, appVersion: "1.0.0-test", configStartDirectory: args[0]); + var exitCode = ProgramRunner.Run( + args, + _jsonOptions, + appVersion: "1.0.0-test", + configStartDirectory: configStartDirectory ?? args[0]); using var document = JsonDocument.Parse(stdout.ToString()); return (exitCode, document.RootElement.Clone()); } diff --git a/tests/CodeIndex.Tests/JsonEnvelopeWrapperTests.cs b/tests/CodeIndex.Tests/JsonEnvelopeWrapperTests.cs index 9d9580c6e..2dec9c5f4 100644 --- a/tests/CodeIndex.Tests/JsonEnvelopeWrapperTests.cs +++ b/tests/CodeIndex.Tests/JsonEnvelopeWrapperTests.cs @@ -1,6 +1,7 @@ using System.Text; using System.Text.Json; using CodeIndex.Cli; +using CodeIndex.Database; namespace CodeIndex.Tests; @@ -148,6 +149,72 @@ public void Status_WithEnvelope_WrapsSingleObjectIntoResultsArray() } } + [Fact] + public void Search_WithEnvelope_UsesLatestIndexedHeadAndFallsBackForLegacyDatabase_Issue4854() + { + var projectRoot = TestProjectHelper.CreateTempProject("envelope_indexed_head"); + try + { + var dbPath = TestProjectHelper.CreateProjectDb(projectRoot); + TestProjectHelper.InsertIndexedFile( + dbPath, + "src/App.cs", + "csharp", + "class App { void Authenticate() {} }\n"); + const string fullScanHead = "1111111111111111111111111111111111111111"; + const string latestHead = "2222222222222222222222222222222222222222"; + using (var db = new DbContext(DbOpenIntent.WriteIndex, dbPath)) + { + var writer = new DbWriter(db.Connection); + writer.SetMetaValues( + (DbContext.IndexedHeadCommitMetaKey, fullScanHead), + (DbContext.IndexedHeadShaMetaKey, latestHead)); + } + + var (latestExitCode, latestStdout, latestStderr) = CaptureConsole(() => ProgramRunner.Run( + ["search", "Authenticate", "--db", dbPath, "--json-envelope"], + _jsonOptions, + "1.0.0")); + + Assert.Equal(CommandExitCodes.Success, latestExitCode); + Assert.Equal(string.Empty, latestStderr); + using (var latestDocument = JsonDocument.Parse(latestStdout)) + { + Assert.Equal( + latestHead, + latestDocument.RootElement + .GetProperty("metadata") + .GetProperty("indexed_at_head_sha") + .GetString()); + } + + using (var db = new DbContext(DbOpenIntent.WriteIndex, dbPath)) + { + var writer = new DbWriter(db.Connection); + writer.SetMetaValues((DbContext.IndexedHeadShaMetaKey, null)); + } + + var (legacyExitCode, legacyStdout, legacyStderr) = CaptureConsole(() => ProgramRunner.Run( + ["search", "Authenticate", "--db", dbPath, "--json-envelope"], + _jsonOptions, + "1.0.0")); + + Assert.Equal(CommandExitCodes.Success, legacyExitCode); + Assert.Equal(string.Empty, legacyStderr); + using var legacyDocument = JsonDocument.Parse(legacyStdout); + Assert.Equal( + fullScanHead, + legacyDocument.RootElement + .GetProperty("metadata") + .GetProperty("indexed_at_head_sha") + .GetString()); + } + finally + { + TestProjectHelper.DeleteDirectory(projectRoot); + } + } + [Fact] public void Search_WithoutEnvelope_StillEmitsLegacyNdjson() { diff --git a/tests/CodeIndex.Tests/McpServerToolsCallTests.cs b/tests/CodeIndex.Tests/McpServerToolsCallTests.cs index 85a87cd0a..814c63f12 100644 --- a/tests/CodeIndex.Tests/McpServerToolsCallTests.cs +++ b/tests/CodeIndex.Tests/McpServerToolsCallTests.cs @@ -3749,14 +3749,16 @@ public void ToolsCall_Map_DepthReaggregatesPathScopedModules_Issue4573() } [Fact] - public void ToolsCall_Map_HeadMetadataUsesMapSnapshot_Issue4573() + public void ToolsCall_Map_HeadMetadataUsesMapSnapshot_Issue4573_Issue4854() { + var legacyFullScanHead = new string('0', 40); var initialHead = new string('a', 40); var nextHead = new string('b', 40); var initialTimestamp = DateTimeOffset.Parse("2026-07-17T01:02:03Z", CultureInfo.InvariantCulture); var nextTimestamp = initialTimestamp.AddMinutes(1); var writer = new DbWriter(_db.Connection); writer.SetMetaValues( + (DbContext.IndexedHeadCommitMetaKey, legacyFullScanHead), (DbContext.IndexedHeadShaMetaKey, initialHead), (DbContext.IndexedHeadTimestampMetaKey, initialTimestamp.ToString("O", CultureInfo.InvariantCulture))); RepoMapBuilder.HeadMetadataCapturedForTesting.Value = () => writer.SetMetaValues( @@ -3770,6 +3772,7 @@ public void ToolsCall_Map_HeadMetadataUsesMapSnapshot_Issue4573() var response = _server.HandleMessage(request)!; var structured = response["result"]!["structuredContent"]!; + Assert.NotEqual(legacyFullScanHead, structured["indexed_head_sha"]!.GetValue()); Assert.Equal(initialHead, structured["indexed_head_sha"]!.GetValue()); Assert.Equal(initialTimestamp, structured["indexed_head_timestamp"]!.GetValue()); Assert.Equal(initialHead, structured["head_freshness"]!["indexed_head"]!.GetValue()); From 865c2d5bfebb1ba8ae141747e83e1b6b7f71d773 Mon Sep 17 00:00:00 2001 From: Widthdom Date: Tue, 28 Jul 2026 14:30:53 +0900 Subject: [PATCH 2/3] Address indexed HEAD snapshot review findings (#4854) --- DEVELOPER_GUIDE.md | 2 + USER_GUIDE.md | 4 +- changelog.d/unreleased/4854.fixed.md | 7 +- src/CodeIndex/Cli/DbPathResolver.cs | 14 ++++ .../Cli/JsonEnvelopeWrapper.Bounded.cs | 13 ++- src/CodeIndex/Cli/JsonEnvelopeWrapper.cs | 10 ++- src/CodeIndex/Database/DbReader.cs | 39 +++++++++ .../JsonEnvelopeWrapperTests.cs | 80 +++++++++++++++++++ 8 files changed, 157 insertions(+), 12 deletions(-) diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index 4988b9987..68880e990 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -1580,6 +1580,7 @@ access. | `hotspots --json` grouping semantics | `hotspots` and MCP `symbol_hotspots` emit `grouped_by`, `grouping_unit`, `count_kind`, `limit_applies_to`, `score_fields`, `ranking_fields`, and matching `query_context` fields. `--limit` applies to returned symbols, files, name/kind groups, or SQL statements; `--count` ignores `--limit` and reports total groups. Explicit `statement` grouping is SQL-only (`--lang sql` / `lang: "sql"`). | | `--json-envelope` commands | Applies to `search`, `definition`, `references`, `callers`, `callees`, `symbols`, `files`, `find`, `excerpt`, `map`, `inspect`, `outline`, `status`, `validate`, `languages`, `impact`, `deps`, `unused`, and `hotspots`. | | `--json-envelope` shape | Wraps the per-line `--json` stream into a single `{"metadata": {...}, "results": [...]}` document. Stream terminal records are excluded from `results` and preserved as `metadata.stream_terminal`, while zero-result prelude/control records are preserved as `metadata.stream_control_records`; therefore `result_count` counts result rows only. A `find --all --count` object is both the count result and terminal scan metadata, so it remains in `results` and is also copied to `metadata.stream_terminal`. `metadata` also carries `api_version`, `command`, `cdidx_version`, `elapsed_ms`, `db_path`, `exit_code`, and, when applicable, `query_normalized` and `indexed_at_head_sha`. `indexed_at_head_sha` maps to the persisted latest-successful `indexed_head_sha` used by status and MCP output after full, `--files`, `--commits`, and `--changed-between` refreshes; failed/rolled-back refreshes do not advance it, and legacy DBs without that key fall back to full-scan-only `indexed_head_commit`. The bounded high-volume commands above support `--json-envelope --max-json-bytes` by measuring the final serialized document; other envelope/byte-cap combinations remain rejected. | +| Indexed-HEAD envelope snapshot | A present `indexed_head_sha` row is authoritative even when its value is NULL, so current-format databases with an unavailable Git HEAD omit `indexed_at_head_sha` instead of falling back to the legacy baseline. Bounded responses capture the resolved value in `ResponseSnapshot` and reuse it after generation validation, keeping metadata, projected rows, `result_stable_at`, and cursors on one database generation. | | Envelope migration | `--json-envelope` implies `--json`, so callers do not need to pass both. The default output remains the legacy NDJSON / array form for one release; the envelope will become the default in the next major release, when the flat form becomes opt-in via `--json-flat`. | | `find --all` scan summary | `find` requires either repeatable `--path ` filters or explicit `--all`. `--all` cannot be combined with `--path`; safe case-insensitive ASCII literals of at least three characters use the external-content trigram FTS index to select files, then re-run the established line matcher over every selected file. Regex, `--exact` normalization, short/non-ASCII literals, legacy databases without the trigram table, missing trigram synchronization triggers, and active FTS bulk-load rebuilds use the bounded line-scan fallback, preventing false negatives from unsupported tokenization, normalization, or stale index state. Writable initialization rebuilds an existing trigram table when any synchronization trigger is missing, repairing artifacts retained across older-writer rebuilds. Both JSON and human summaries expose `search_strategy` plus optional `search_fallback_reason`; `candidate_files` remains the total scoped file count, while `files_scanned` / `lines_scanned` count post-index verification work. Default JSON rows end with a terminal record carrying `scan_complete`, `authoritative_rows`, returned/scanned counts, active caps, truncation/continuation fields, and recovery guidance. Count JSON carries the same terminal scan state in its single object and uses `authoritative_count`. Row formats that cannot represent the terminal metadata (JSON array, compact, CSV/TSV, LSP, quickfix, and SARIF) are rejected with `--all`; conflicting JSON/text flags are rejected or normalized to NDJSON independent of option order. Candidate-file or line-scan truncation exits with partial-result code `11` unless `--allow-partial` opts into `0`; ordinary result-limit early stops remain successful but set `scan_complete=false` and `result_limit_reached=true`. Human stderr summaries include the strategy, active caps, scan/authority state, continuation action, and recovery guidance and use the same partial exit semantics. | @@ -4820,6 +4821,7 @@ help はすべてこのレジストリを参照します。field 名は大文字 | `hotspots --json` grouping semantics | `hotspots` と MCP `symbol_hotspots` は `grouped_by`、`grouping_unit`、`count_kind`、`limit_applies_to`、`score_fields`、`ranking_fields` と、対応する `query_context` field を返します。`--limit` は返却される symbol、file、name/kind group、SQL statement に適用されます。`--count` は `--limit` を無視し、total group 数を返します。明示的な `statement` grouping は SQL 専用です（`--lang sql` / `lang: "sql"`）。 | | `--json-envelope` 対象 command | `search`、`definition`、`references`、`callers`、`callees`、`symbols`、`files`、`find`、`excerpt`、`map`、`inspect`、`outline`、`status`、`validate`、`languages`、`impact`、`deps`、`unused`、`hotspots`。 | | `--json-envelope` shape | per-line `--json` stream を単一の `{"metadata": {...}, "results": [...]}` document に包みます。stream 終端レコードは `results` から除外して `metadata.stream_terminal` に保持し、0 件時の prelude / control record は `metadata.stream_control_records` に保持するため、`result_count` は result row だけを数えます。`find --all --count` object は count result であると同時に終端 scan metadata でもあるため、`results` に残しつつ `metadata.stream_terminal` にも複製します。`metadata` は `api_version`、`command`、`cdidx_version`、`elapsed_ms`、`db_path`、`exit_code`、該当時は `query_normalized` と `indexed_at_head_sha` も持ちます。`indexed_at_head_sha` は full、`--files`、`--commits`、`--changed-between` refresh 後に status / MCP output が使う永続化済みの最新成功 `indexed_head_sha` に対応します。失敗または rollback された refresh では進まず、この key を持たない legacy DB では full-scan 限定 `indexed_head_commit` に fallback します。上記の bounded 高ボリューム command は最終 document を測定することで `--json-envelope --max-json-bytes` を許可し、それ以外の envelope / byte-cap 組み合わせは引き続き拒否します。 | +| indexed HEAD envelope snapshot | `indexed_head_sha` row が存在する場合は値が NULL でも authoritative とし、Git HEAD を解決できない current-format database では legacy baseline に fallback せず `indexed_at_head_sha` を省略します。bounded response は解決済みの値を `ResponseSnapshot` に保持し、generation 検証後も再利用することで metadata、projection 済み row、`result_stable_at`、cursor を同じ database generation に固定します。 | | envelope migration | `--json-envelope` は `--json` を imply するため、caller は両方を指定する必要がありません。既定 output は 1 release の間 legacy NDJSON / array form のままです。次の major release では envelope が既定になり、flat form は `--json-flat` による opt-in になります。 | | `find --all` scan summary | `find` は repeatable な `--path ` か明示的な `--all` のどちらかを要求し、`--all` と `--path` は併用できません。3 文字以上の安全な大文字小文字を区別しない ASCII literal は external-content trigram FTS index で file を選び、選択した全 file に既存の行 matcher を再適用します。regex、`--exact` normalization、短い literal、非 ASCII literal、trigram table のない旧 database、trigram 同期 trigger の欠落、FTS bulk-load による再構築中は上限付き line-scan fallback を使い、未対応の tokenization、normalization、古い index 状態による false negative を防ぎます。writable initialization は同期 trigger が 1 つでも欠けた既存 trigram table を再構築し、旧 writer の rebuild 後に残った artifact も修復します。JSON と human summary は `search_strategy` と任意の `search_fallback_reason` を返します。`candidate_files` は scope 内の総 file 数を維持し、`files_scanned` / `lines_scanned` は index 適用後の検証量を数えます。既定 JSON row は `scan_complete`、`authoritative_rows`、返却 / 走査件数、有効な cap、切り詰め / continuation field、復旧案内を持つ終端レコードで終了します。count JSON は単一 object に同じ終端 scan 状態を持ち、`authoritative_count` を使います。終端 metadata を表現できない JSON array、compact、CSV/TSV、LSP、quickfix、SARIF は `--all` との組み合わせを拒否し、競合する JSON / text flag は option 順序にかかわらず拒否するか NDJSON に正規化します。candidate-file または line-scan による切り詰めは、`--allow-partial` で `0` を opt-in しない限り partial-result 終了コード `11` を返します。通常の result limit による早期停止は成功のままですが、`scan_complete=false` と `result_limit_reached=true` を設定します。human stderr summary は strategy、有効な cap、scan / authority 状態、continuation action、復旧案内を含み、同じ partial exit semantics を使います。 | diff --git a/USER_GUIDE.md b/USER_GUIDE.md index 6e5ce75e3..868d425a5 100644 --- a/USER_GUIDE.md +++ b/USER_GUIDE.md @@ -1586,7 +1586,7 @@ Use `--json` for machine-readable output (AI agents): {"path":"src/Auth/TokenService.cs","lang":"csharp","chunk_start_line":1,"chunk_end_line":80,"snippet_start_line":40,"snippet_end_line":47,"snippet":"if (claims.Count == 0)\\n throw new InvalidOperationException();\\nreturn GenerateToken(claims);","match_lines":[42,47],"highlights":[{"line":47,"text":"return GenerateToken(claims);","terms":["GenerateToken"]}],"context_before":2,"context_after":3,"score":9.8} ``` -Add `--json-envelope` to wrap the per-line stream into a single document with a `metadata` block (command, `cdidx_version`, `elapsed_ms`, `db_path`, `result_count`, `exit_code`, optional `query_normalized` / `indexed_at_head_sha`) and a `results` array. `indexed_at_head_sha` has the same meaning as status `indexed_head_sha`: it identifies the checkout captured by the latest successful full scan, `--files`, `--commits`, or `--changed-between` refresh. It does not advance after a failed or rolled-back refresh; databases created before `indexed_head_sha` fall back to the legacy full-scan-only `indexed_head_commit`. The flag implies `--json` and works on every query command (`search`, `definition`, `references`, `callers`, `callees`, `symbols`, `files`, `find`, `excerpt`, `map`, `inspect`, `outline`, `status`, `validate`, `languages`, `impact`, `deps`, `unused`, `hotspots`). Wrapped commands can capture up to 10,485,760 output characters; if the budget is exceeded, cdidx returns a JSON envelope with empty `results`, non-zero `metadata.exit_code`, and `metadata.error`, and suggests using `--limit` / `--top` or streaming `--json`. The flat NDJSON / array output stays the default for one release; the envelope will become the default in the next major release, at which point the flat form will be opt-in via `--json-flat`. +Add `--json-envelope` to wrap the per-line stream into a single document with a `metadata` block (command, `cdidx_version`, `elapsed_ms`, `db_path`, `result_count`, `exit_code`, optional `query_normalized` / `indexed_at_head_sha`) and a `results` array. `indexed_at_head_sha` has the same meaning as status `indexed_head_sha`: it identifies the checkout captured by the latest successful full scan, `--files`, `--commits`, or `--changed-between` refresh. It does not advance after a failed or rolled-back refresh; databases created before `indexed_head_sha` fall back to the legacy full-scan-only `indexed_head_commit`. If the latest-head key exists but its value is unavailable because Git HEAD could not be resolved, the envelope omits `indexed_at_head_sha` instead of reporting the legacy baseline. Bounded responses bind this stamp to the same validated index snapshot as their rows and cursor. The flag implies `--json` and works on every query command (`search`, `definition`, `references`, `callers`, `callees`, `symbols`, `files`, `find`, `excerpt`, `map`, `inspect`, `outline`, `status`, `validate`, `languages`, `impact`, `deps`, `unused`, `hotspots`). Wrapped commands can capture up to 10,485,760 output characters; if the budget is exceeded, cdidx returns a JSON envelope with empty `results`, non-zero `metadata.exit_code`, and `metadata.error`, and suggests using `--limit` / `--top` or streaming `--json`. The flat NDJSON / array output stays the default for one release; the envelope will become the default in the next major release, at which point the flat form will be opt-in via `--json-flat`. Add `--profile` to any read command when debugging slow queries. It appends one JSON object after the normal result with `profile.phases` (`name`, `elapsed_ms`, `rows_scanned`), `profile.query_plan` (`EXPLAIN QUERY PLAN` rows), and `profile.queries` (the SQL text). Add `--slow-query-ms ` to log profiled SQL statements that meet the threshold to the persistent tool log. @@ -4745,7 +4745,7 @@ src/Auth/TokenService.cs:42-58 {"path":"src/Auth/TokenService.cs","start_line":42,"end_line":58,"content":"public string GenerateToken(...)...","lang":"csharp","score":9.8} ``` -`--json-envelope` を追加すると、1 行ごとの stream を `metadata`（command、`cdidx_version`、`elapsed_ms`、`db_path`、`result_count`、`exit_code`、任意の `query_normalized` / `indexed_at_head_sha`）と `results` 配列を持つ 1 つの JSON document に包みます。`indexed_at_head_sha` の意味は status の `indexed_head_sha` と同じで、最後に成功した full scan、`--files`、`--commits`、`--changed-between` refresh が取り込んだ checkout を示します。失敗または rollback された refresh では進まず、`indexed_head_sha` より前の database では legacy の full-scan 限定 `indexed_head_commit` に fallback します。この flag は `--json` を暗黙に有効化し、各 query command で使えます。wrapped command の捕捉出力は最大 10,485,760 文字です。超過した場合は、空の `results`、非 0 の `metadata.exit_code`、`metadata.error` を持つ JSON envelope を返し、`--limit` / `--top` または streaming `--json` の利用を促します。 +`--json-envelope` を追加すると、1 行ごとの stream を `metadata`（command、`cdidx_version`、`elapsed_ms`、`db_path`、`result_count`、`exit_code`、任意の `query_normalized` / `indexed_at_head_sha`）と `results` 配列を持つ 1 つの JSON document に包みます。`indexed_at_head_sha` の意味は status の `indexed_head_sha` と同じで、最後に成功した full scan、`--files`、`--commits`、`--changed-between` refresh が取り込んだ checkout を示します。失敗または rollback された refresh では進まず、`indexed_head_sha` より前の database では legacy の full-scan 限定 `indexed_head_commit` に fallback します。最新 HEAD key が存在していても Git HEAD を解決できず値が未設定の場合は、legacy baseline を返さず `indexed_at_head_sha` を省略します。bounded response では、この stamp を row と cursor と同じ検証済み index snapshot に固定します。この flag は `--json` を暗黙に有効化し、各 query command で使えます。wrapped command の捕捉出力は最大 10,485,760 文字です。超過した場合は、空の `results`、非 0 の `metadata.exit_code`、`metadata.error` を持つ JSON envelope を返し、`--limit` / `--top` または streaming `--json` の利用を促します。 ### シンボル検索（関数、クラスなど） diff --git a/changelog.d/unreleased/4854.fixed.md b/changelog.d/unreleased/4854.fixed.md index 2360cbf51..8db3ae8ff 100644 --- a/changelog.d/unreleased/4854.fixed.md +++ b/changelog.d/unreleased/4854.fixed.md @@ -3,7 +3,10 @@ category: fixed issues: - 4854 affected: + - src/CodeIndex/Cli/DbPathResolver.cs + - src/CodeIndex/Cli/JsonEnvelopeWrapper.Bounded.cs - src/CodeIndex/Cli/JsonEnvelopeWrapper.cs + - src/CodeIndex/Database/DbReader.cs - tests/CodeIndex.Tests/JsonEnvelopeWrapperTests.cs - tests/CodeIndex.Tests/IndexCommandRunnerTests.cs - tests/CodeIndex.Tests/McpServerToolsCallTests.cs @@ -13,8 +16,8 @@ affected: ## English -- **Query envelopes now agree with status on the latest indexed HEAD (#4854)** — `metadata.indexed_at_head_sha` now uses the latest successful full or partial refresh stamp established by #1509, #1527, and #4313, while retaining the legacy full-scan HEAD fallback for older databases and preserving the prior stamp after failed or rolled-back refreshes. +- **Query envelopes now agree with status on the latest indexed HEAD (#4854)** — `metadata.indexed_at_head_sha` now uses the latest successful full or partial refresh stamp established by #1509, #1527, and #4313, while retaining the legacy full-scan HEAD fallback only for older databases without the latest key, omitting unresolved current stamps, binding bounded-response metadata to its validated snapshot, and preserving the prior stamp after failed or rolled-back refreshes. ## 日本語 -- **query envelope と status が最新の indexed HEAD で一致するようになりました (#4854)** — `metadata.indexed_at_head_sha` は #1509、#1527、#4313 で確立された最新の成功 full / partial refresh stamp を使うようになり、古い database 向けの legacy full-scan HEAD fallback と、失敗または rollback された refresh 後の直前 stamp を維持します。 +- **query envelope と status が最新の indexed HEAD で一致するようになりました (#4854)** — `metadata.indexed_at_head_sha` は #1509、#1527、#4313 で確立された最新の成功 full / partial refresh stamp を使うようになり、最新 key のない古い database だけを legacy full-scan HEAD へ fallback し、解決不能な current stamp を省略し、bounded response metadata を検証済み snapshot に固定しつつ、失敗または rollback された refresh 後の直前 stamp を維持します。 diff --git a/src/CodeIndex/Cli/DbPathResolver.cs b/src/CodeIndex/Cli/DbPathResolver.cs index 37704c7e6..18600c340 100644 --- a/src/CodeIndex/Cli/DbPathResolver.cs +++ b/src/CodeIndex/Cli/DbPathResolver.cs @@ -334,6 +334,20 @@ private static void SeedPathCasingFromWorkspaceStamp(string? indexedProjectRoot, public static string? TryReadIndexedHeadSha(string dbPath) => TryReadMetaString(dbPath, CodeIndex.Database.DbContext.IndexedHeadShaMetaKey); + public static string? TryReadIndexedHeadForResponse(string dbPath) + { + try + { + using var connection = OpenMetadataConnection(dbPath); + connection.Open(); + return CodeIndex.Database.DbReader.TryGetIndexedHeadForResponse(connection); + } + catch (Exception ex) when (IsMetadataProbeException(ex)) + { + return null; + } + } + public static DateTimeOffset? TryReadIndexedHeadTimestamp(string dbPath) { var raw = TryReadMetaString(dbPath, CodeIndex.Database.DbContext.IndexedHeadTimestampMetaKey); diff --git a/src/CodeIndex/Cli/JsonEnvelopeWrapper.Bounded.cs b/src/CodeIndex/Cli/JsonEnvelopeWrapper.Bounded.cs index 0a0d05132..5c22459fa 100644 --- a/src/CodeIndex/Cli/JsonEnvelopeWrapper.Bounded.cs +++ b/src/CodeIndex/Cli/JsonEnvelopeWrapper.Bounded.cs @@ -22,6 +22,7 @@ internal static partial class JsonEnvelopeWrapper private const string LegacyResponseCursorPrefix = "response:v1:"; private const string ResponseCursorPrefix = "response:v2:"; private static readonly AsyncLocal BoundedExecution = new(); + internal static Action? ResponseSnapshotValidatedForTesting { get; set; } private static readonly HashSet BoundedResponseCommands = ProjectionFieldRegistry.SupportedCommands.ToHashSet(StringComparer.Ordinal); @@ -274,6 +275,7 @@ private static int RunBoundedResponse( "The index generation changed while this page was being read.", "Restart pagination without --cursor after the active index refresh completes."); } + ResponseSnapshotValidatedForTesting?.Invoke(); var envelope = BuildBoundedEnvelopeWithinBudget( command, queryNormalized, @@ -351,7 +353,8 @@ JsonObject BuildCandidate(int count) exitCode, error: commandError is null ? null : (JsonObject)commandError.DeepClone(), streamTerminal: adjustedStreamTerminal, - streamControlRecords: streamControlRecords); + streamControlRecords: streamControlRecords, + responseSnapshot: snapshot); var metadata = (JsonObject)envelope["metadata"]!; metadata["result_stable_at"] = snapshot.ResultStableAt; if (commandError is not null) @@ -1454,7 +1457,8 @@ private readonly record struct ResponseCount( private readonly record struct ResponseSnapshot( string GenerationFingerprint, - string? ResultStableAt); + string? ResultStableAt, + string? IndexedHead); private static ResponseSnapshot SafeReadResponseSnapshot( string dbPath, @@ -1486,11 +1490,12 @@ private static ResponseSnapshot BuildResponseSnapshot(DbReader reader) var generation = reader.GetPaginationGeneration(); return new( BuildResponseValueFingerprint(generation.Identity), - generation.StableAt); + generation.StableAt, + reader.GetIndexedHeadForResponse()); } private static ResponseSnapshot BuildFallbackResponseSnapshot(string appVersion) - => new(BuildResponseValueFingerprint("catalog\0" + appVersion), null); + => new(BuildResponseValueFingerprint("catalog\0" + appVersion), null, null); private static string BuildResponseValueFingerprint(string value) { diff --git a/src/CodeIndex/Cli/JsonEnvelopeWrapper.cs b/src/CodeIndex/Cli/JsonEnvelopeWrapper.cs index 80c544326..41fb24714 100644 --- a/src/CodeIndex/Cli/JsonEnvelopeWrapper.cs +++ b/src/CodeIndex/Cli/JsonEnvelopeWrapper.cs @@ -222,7 +222,8 @@ private static JsonObject BuildEnvelope( int exitCode, JsonObject? error = null, JsonObject? streamTerminal = null, - JsonArray? streamControlRecords = null) + JsonArray? streamControlRecords = null, + ResponseSnapshot? responseSnapshot = null) { var metadata = new JsonObject { @@ -244,7 +245,9 @@ private static JsonObject BuildEnvelope( if (streamControlRecords is { Count: > 0 }) metadata["stream_control_records"] = streamControlRecords.DeepClone(); - var indexedHead = SafeReadIndexedHead(dbPath, dbPathExplicit); + var indexedHead = responseSnapshot.HasValue + ? responseSnapshot.Value.IndexedHead + : SafeReadIndexedHead(dbPath, dbPathExplicit); if (!string.IsNullOrEmpty(indexedHead)) metadata["indexed_at_head_sha"] = indexedHead; @@ -263,8 +266,7 @@ private static JsonObject BuildEnvelope( if (!dbPathExplicit && !File.Exists(LongPath.EnsureWindowsPrefix(resolvedPath))) return null; var normalizedPath = DbPathResolver.NormalizeDbPath(resolvedPath); - return DbPathResolver.TryReadIndexedHeadSha(normalizedPath) - ?? DbPathResolver.TryReadIndexedHeadCommit(normalizedPath); + return DbPathResolver.TryReadIndexedHeadForResponse(normalizedPath); } catch { diff --git a/src/CodeIndex/Database/DbReader.cs b/src/CodeIndex/Database/DbReader.cs index 6a9a34dd0..52b1f42a8 100644 --- a/src/CodeIndex/Database/DbReader.cs +++ b/src/CodeIndex/Database/DbReader.cs @@ -1091,6 +1091,45 @@ AND COUNT(s.family_key) < COUNT(*) /// internal string? GetMetaString(string key) => TryGetMetaString(_conn, key); + /// + /// Read the latest indexed HEAD used by response metadata. A present + /// indexed_head_sha row is authoritative even when its value is NULL; + /// only databases without that key fall back to the legacy full-scan stamp. + /// response metadata 用の最新 indexed HEAD を読む。indexed_head_sha row が + /// 存在する場合は NULL 値でも優先し、その key がない legacy DB だけ full-scan stamp に fallback する。 + /// + internal string? GetIndexedHeadForResponse() + => TryGetIndexedHeadForResponse(_conn); + + internal static string? TryGetIndexedHeadForResponse(SqliteConnection conn) + { + try + { + using var cmd = conn.CreateCommand(); + cmd.CommandText = """ + SELECT value + FROM codeindex_meta + WHERE key = CASE + WHEN EXISTS ( + SELECT 1 + FROM codeindex_meta + WHERE key = @latestHead + ) + THEN @latestHead + ELSE @legacyHead + END + LIMIT 1 + """; + SqliteCommandPolicy.Add(cmd, "@latestHead", DbContext.IndexedHeadShaMetaKey); + SqliteCommandPolicy.Add(cmd, "@legacyHead", DbContext.IndexedHeadCommitMetaKey); + return cmd.ExecuteScalar() as string; + } + catch (SqliteException) + { + return null; + } + } + private static string? TryGetMetaString(SqliteConnection conn, string key) { // Inline the codeindex_meta lookup to avoid creating a DbContext here. diff --git a/tests/CodeIndex.Tests/JsonEnvelopeWrapperTests.cs b/tests/CodeIndex.Tests/JsonEnvelopeWrapperTests.cs index 2dec9c5f4..4f150f256 100644 --- a/tests/CodeIndex.Tests/JsonEnvelopeWrapperTests.cs +++ b/tests/CodeIndex.Tests/JsonEnvelopeWrapperTests.cs @@ -194,6 +194,29 @@ public void Search_WithEnvelope_UsesLatestIndexedHeadAndFallsBackForLegacyDataba writer.SetMetaValues((DbContext.IndexedHeadShaMetaKey, null)); } + var (nullExitCode, nullStdout, nullStderr) = CaptureConsole(() => ProgramRunner.Run( + ["search", "Authenticate", "--db", dbPath, "--json-envelope"], + _jsonOptions, + "1.0.0")); + + Assert.Equal(CommandExitCodes.Success, nullExitCode); + Assert.Equal(string.Empty, nullStderr); + using (var nullDocument = JsonDocument.Parse(nullStdout)) + { + Assert.False( + nullDocument.RootElement + .GetProperty("metadata") + .TryGetProperty("indexed_at_head_sha", out _)); + } + + using (var db = new DbContext(DbOpenIntent.WriteIndex, dbPath)) + { + using var command = db.Connection.CreateCommand(); + command.CommandText = "DELETE FROM codeindex_meta WHERE key = @key"; + command.Parameters.AddWithValue("@key", DbContext.IndexedHeadShaMetaKey); + Assert.Equal(1, command.ExecuteNonQuery()); + } + var (legacyExitCode, legacyStdout, legacyStderr) = CaptureConsole(() => ProgramRunner.Run( ["search", "Authenticate", "--db", dbPath, "--json-envelope"], _jsonOptions, @@ -215,6 +238,63 @@ public void Search_WithEnvelope_UsesLatestIndexedHeadAndFallsBackForLegacyDataba } } + [Fact] + public void Search_BoundedEnvelope_KeepsIndexedHeadFromValidatedResponseSnapshot_Issue4854() + { + var projectRoot = TestProjectHelper.CreateTempProject("envelope_indexed_head_snapshot"); + try + { + var dbPath = TestProjectHelper.CreateProjectDb(projectRoot); + TestProjectHelper.InsertIndexedFile( + dbPath, + "src/App.cs", + "csharp", + "class App { void Authenticate() {} }\n"); + const string legacyHead = "1111111111111111111111111111111111111111"; + const string snapshotHead = "2222222222222222222222222222222222222222"; + const string nextHead = "3333333333333333333333333333333333333333"; + using (var db = new DbContext(DbOpenIntent.WriteIndex, dbPath)) + { + var writer = new DbWriter(db.Connection); + writer.SetMetaValues( + (DbContext.IndexedHeadCommitMetaKey, legacyHead), + (DbContext.IndexedHeadShaMetaKey, snapshotHead)); + } + + var hookInvoked = false; + JsonEnvelopeWrapper.ResponseSnapshotValidatedForTesting = () => + { + hookInvoked = true; + using var db = new DbContext(DbOpenIntent.WriteIndex, dbPath); + var writer = new DbWriter(db.Connection); + writer.SetMetaValues((DbContext.IndexedHeadShaMetaKey, nextHead)); + }; + + var (exitCode, stdout, stderr) = CaptureConsole(() => ProgramRunner.Run( + ["search", "Authenticate", "--db", dbPath, "--fields", "path", "--json"], + _jsonOptions, + "1.0.0")); + + Assert.True(hookInvoked); + Assert.Equal(CommandExitCodes.Success, exitCode); + Assert.Equal(string.Empty, stderr); + using var document = JsonDocument.Parse(stdout); + Assert.Equal( + snapshotHead, + document.RootElement + .GetProperty("metadata") + .GetProperty("indexed_at_head_sha") + .GetString()); + using var verificationDb = new DbContext(DbOpenIntent.WriteIndex, dbPath); + Assert.Equal(nextHead, verificationDb.GetMetaString(DbContext.IndexedHeadShaMetaKey)); + } + finally + { + JsonEnvelopeWrapper.ResponseSnapshotValidatedForTesting = null; + TestProjectHelper.DeleteDirectory(projectRoot); + } + } + [Fact] public void Search_WithoutEnvelope_StillEmitsLegacyNdjson() { From 29e85f162107018fa38fab0eeecbdf7ecba50249 Mon Sep 17 00:00:00 2001 From: Widthdom Date: Tue, 28 Jul 2026 15:25:26 +0900 Subject: [PATCH 3/3] Bind regular envelopes to indexed snapshots (#4854) --- DEVELOPER_GUIDE.md | 4 +- USER_GUIDE.md | 4 +- changelog.d/unreleased/4854.fixed.md | 4 +- src/CodeIndex/Cli/JsonEnvelopeWrapper.cs | 60 ++++++++++- .../JsonEnvelopeWrapperTests.cs | 101 ++++++++++++++++++ 5 files changed, 165 insertions(+), 8 deletions(-) diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index 68880e990..69a5602c3 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -1580,7 +1580,7 @@ access. | `hotspots --json` grouping semantics | `hotspots` and MCP `symbol_hotspots` emit `grouped_by`, `grouping_unit`, `count_kind`, `limit_applies_to`, `score_fields`, `ranking_fields`, and matching `query_context` fields. `--limit` applies to returned symbols, files, name/kind groups, or SQL statements; `--count` ignores `--limit` and reports total groups. Explicit `statement` grouping is SQL-only (`--lang sql` / `lang: "sql"`). | | `--json-envelope` commands | Applies to `search`, `definition`, `references`, `callers`, `callees`, `symbols`, `files`, `find`, `excerpt`, `map`, `inspect`, `outline`, `status`, `validate`, `languages`, `impact`, `deps`, `unused`, and `hotspots`. | | `--json-envelope` shape | Wraps the per-line `--json` stream into a single `{"metadata": {...}, "results": [...]}` document. Stream terminal records are excluded from `results` and preserved as `metadata.stream_terminal`, while zero-result prelude/control records are preserved as `metadata.stream_control_records`; therefore `result_count` counts result rows only. A `find --all --count` object is both the count result and terminal scan metadata, so it remains in `results` and is also copied to `metadata.stream_terminal`. `metadata` also carries `api_version`, `command`, `cdidx_version`, `elapsed_ms`, `db_path`, `exit_code`, and, when applicable, `query_normalized` and `indexed_at_head_sha`. `indexed_at_head_sha` maps to the persisted latest-successful `indexed_head_sha` used by status and MCP output after full, `--files`, `--commits`, and `--changed-between` refreshes; failed/rolled-back refreshes do not advance it, and legacy DBs without that key fall back to full-scan-only `indexed_head_commit`. The bounded high-volume commands above support `--json-envelope --max-json-bytes` by measuring the final serialized document; other envelope/byte-cap combinations remain rejected. | -| Indexed-HEAD envelope snapshot | A present `indexed_head_sha` row is authoritative even when its value is NULL, so current-format databases with an unavailable Git HEAD omit `indexed_at_head_sha` instead of falling back to the legacy baseline. Bounded responses capture the resolved value in `ResponseSnapshot` and reuse it after generation validation, keeping metadata, projected rows, `result_stable_at`, and cursors on one database generation. | +| Indexed-HEAD envelope snapshot | A present `indexed_head_sha` row is authoritative even when its value is NULL, so current-format databases with an unavailable Git HEAD omit `indexed_at_head_sha` instead of falling back to the legacy baseline. Regular and bounded envelopes capture the resolved value in `ResponseSnapshot`, verify the generation again after the inner query, and reuse the snapshot while serializing; a generation change returns restart guidance instead of mismatched rows. Bounded metadata, projected rows, `result_stable_at`, and cursors therefore remain on one database generation. | | Envelope migration | `--json-envelope` implies `--json`, so callers do not need to pass both. The default output remains the legacy NDJSON / array form for one release; the envelope will become the default in the next major release, when the flat form becomes opt-in via `--json-flat`. | | `find --all` scan summary | `find` requires either repeatable `--path ` filters or explicit `--all`. `--all` cannot be combined with `--path`; safe case-insensitive ASCII literals of at least three characters use the external-content trigram FTS index to select files, then re-run the established line matcher over every selected file. Regex, `--exact` normalization, short/non-ASCII literals, legacy databases without the trigram table, missing trigram synchronization triggers, and active FTS bulk-load rebuilds use the bounded line-scan fallback, preventing false negatives from unsupported tokenization, normalization, or stale index state. Writable initialization rebuilds an existing trigram table when any synchronization trigger is missing, repairing artifacts retained across older-writer rebuilds. Both JSON and human summaries expose `search_strategy` plus optional `search_fallback_reason`; `candidate_files` remains the total scoped file count, while `files_scanned` / `lines_scanned` count post-index verification work. Default JSON rows end with a terminal record carrying `scan_complete`, `authoritative_rows`, returned/scanned counts, active caps, truncation/continuation fields, and recovery guidance. Count JSON carries the same terminal scan state in its single object and uses `authoritative_count`. Row formats that cannot represent the terminal metadata (JSON array, compact, CSV/TSV, LSP, quickfix, and SARIF) are rejected with `--all`; conflicting JSON/text flags are rejected or normalized to NDJSON independent of option order. Candidate-file or line-scan truncation exits with partial-result code `11` unless `--allow-partial` opts into `0`; ordinary result-limit early stops remain successful but set `scan_complete=false` and `result_limit_reached=true`. Human stderr summaries include the strategy, active caps, scan/authority state, continuation action, and recovery guidance and use the same partial exit semantics. | @@ -4821,7 +4821,7 @@ help はすべてこのレジストリを参照します。field 名は大文字 | `hotspots --json` grouping semantics | `hotspots` と MCP `symbol_hotspots` は `grouped_by`、`grouping_unit`、`count_kind`、`limit_applies_to`、`score_fields`、`ranking_fields` と、対応する `query_context` field を返します。`--limit` は返却される symbol、file、name/kind group、SQL statement に適用されます。`--count` は `--limit` を無視し、total group 数を返します。明示的な `statement` grouping は SQL 専用です（`--lang sql` / `lang: "sql"`）。 | | `--json-envelope` 対象 command | `search`、`definition`、`references`、`callers`、`callees`、`symbols`、`files`、`find`、`excerpt`、`map`、`inspect`、`outline`、`status`、`validate`、`languages`、`impact`、`deps`、`unused`、`hotspots`。 | | `--json-envelope` shape | per-line `--json` stream を単一の `{"metadata": {...}, "results": [...]}` document に包みます。stream 終端レコードは `results` から除外して `metadata.stream_terminal` に保持し、0 件時の prelude / control record は `metadata.stream_control_records` に保持するため、`result_count` は result row だけを数えます。`find --all --count` object は count result であると同時に終端 scan metadata でもあるため、`results` に残しつつ `metadata.stream_terminal` にも複製します。`metadata` は `api_version`、`command`、`cdidx_version`、`elapsed_ms`、`db_path`、`exit_code`、該当時は `query_normalized` と `indexed_at_head_sha` も持ちます。`indexed_at_head_sha` は full、`--files`、`--commits`、`--changed-between` refresh 後に status / MCP output が使う永続化済みの最新成功 `indexed_head_sha` に対応します。失敗または rollback された refresh では進まず、この key を持たない legacy DB では full-scan 限定 `indexed_head_commit` に fallback します。上記の bounded 高ボリューム command は最終 document を測定することで `--json-envelope --max-json-bytes` を許可し、それ以外の envelope / byte-cap 組み合わせは引き続き拒否します。 | -| indexed HEAD envelope snapshot | `indexed_head_sha` row が存在する場合は値が NULL でも authoritative とし、Git HEAD を解決できない current-format database では legacy baseline に fallback せず `indexed_at_head_sha` を省略します。bounded response は解決済みの値を `ResponseSnapshot` に保持し、generation 検証後も再利用することで metadata、projection 済み row、`result_stable_at`、cursor を同じ database generation に固定します。 | +| indexed HEAD envelope snapshot | `indexed_head_sha` row が存在する場合は値が NULL でも authoritative とし、Git HEAD を解決できない current-format database では legacy baseline に fallback せず `indexed_at_head_sha` を省略します。通常および bounded envelope は解決済みの値を `ResponseSnapshot` に保持し、inner query 後に generation を再検証して serialization 中も同じ snapshot を再利用します。generation が変わった場合は不整合な row を返さず再実行案内を返すため、bounded metadata、projection 済み row、`result_stable_at`、cursor も同じ database generation に固定されます。 | | envelope migration | `--json-envelope` は `--json` を imply するため、caller は両方を指定する必要がありません。既定 output は 1 release の間 legacy NDJSON / array form のままです。次の major release では envelope が既定になり、flat form は `--json-flat` による opt-in になります。 | | `find --all` scan summary | `find` は repeatable な `--path ` か明示的な `--all` のどちらかを要求し、`--all` と `--path` は併用できません。3 文字以上の安全な大文字小文字を区別しない ASCII literal は external-content trigram FTS index で file を選び、選択した全 file に既存の行 matcher を再適用します。regex、`--exact` normalization、短い literal、非 ASCII literal、trigram table のない旧 database、trigram 同期 trigger の欠落、FTS bulk-load による再構築中は上限付き line-scan fallback を使い、未対応の tokenization、normalization、古い index 状態による false negative を防ぎます。writable initialization は同期 trigger が 1 つでも欠けた既存 trigram table を再構築し、旧 writer の rebuild 後に残った artifact も修復します。JSON と human summary は `search_strategy` と任意の `search_fallback_reason` を返します。`candidate_files` は scope 内の総 file 数を維持し、`files_scanned` / `lines_scanned` は index 適用後の検証量を数えます。既定 JSON row は `scan_complete`、`authoritative_rows`、返却 / 走査件数、有効な cap、切り詰め / continuation field、復旧案内を持つ終端レコードで終了します。count JSON は単一 object に同じ終端 scan 状態を持ち、`authoritative_count` を使います。終端 metadata を表現できない JSON array、compact、CSV/TSV、LSP、quickfix、SARIF は `--all` との組み合わせを拒否し、競合する JSON / text flag は option 順序にかかわらず拒否するか NDJSON に正規化します。candidate-file または line-scan による切り詰めは、`--allow-partial` で `0` を opt-in しない限り partial-result 終了コード `11` を返します。通常の result limit による早期停止は成功のままですが、`scan_complete=false` と `result_limit_reached=true` を設定します。human stderr summary は strategy、有効な cap、scan / authority 状態、continuation action、復旧案内を含み、同じ partial exit semantics を使います。 | diff --git a/USER_GUIDE.md b/USER_GUIDE.md index 868d425a5..80a7d11e2 100644 --- a/USER_GUIDE.md +++ b/USER_GUIDE.md @@ -1586,7 +1586,7 @@ Use `--json` for machine-readable output (AI agents): {"path":"src/Auth/TokenService.cs","lang":"csharp","chunk_start_line":1,"chunk_end_line":80,"snippet_start_line":40,"snippet_end_line":47,"snippet":"if (claims.Count == 0)\\n throw new InvalidOperationException();\\nreturn GenerateToken(claims);","match_lines":[42,47],"highlights":[{"line":47,"text":"return GenerateToken(claims);","terms":["GenerateToken"]}],"context_before":2,"context_after":3,"score":9.8} ``` -Add `--json-envelope` to wrap the per-line stream into a single document with a `metadata` block (command, `cdidx_version`, `elapsed_ms`, `db_path`, `result_count`, `exit_code`, optional `query_normalized` / `indexed_at_head_sha`) and a `results` array. `indexed_at_head_sha` has the same meaning as status `indexed_head_sha`: it identifies the checkout captured by the latest successful full scan, `--files`, `--commits`, or `--changed-between` refresh. It does not advance after a failed or rolled-back refresh; databases created before `indexed_head_sha` fall back to the legacy full-scan-only `indexed_head_commit`. If the latest-head key exists but its value is unavailable because Git HEAD could not be resolved, the envelope omits `indexed_at_head_sha` instead of reporting the legacy baseline. Bounded responses bind this stamp to the same validated index snapshot as their rows and cursor. The flag implies `--json` and works on every query command (`search`, `definition`, `references`, `callers`, `callees`, `symbols`, `files`, `find`, `excerpt`, `map`, `inspect`, `outline`, `status`, `validate`, `languages`, `impact`, `deps`, `unused`, `hotspots`). Wrapped commands can capture up to 10,485,760 output characters; if the budget is exceeded, cdidx returns a JSON envelope with empty `results`, non-zero `metadata.exit_code`, and `metadata.error`, and suggests using `--limit` / `--top` or streaming `--json`. The flat NDJSON / array output stays the default for one release; the envelope will become the default in the next major release, at which point the flat form will be opt-in via `--json-flat`. +Add `--json-envelope` to wrap the per-line stream into a single document with a `metadata` block (command, `cdidx_version`, `elapsed_ms`, `db_path`, `result_count`, `exit_code`, optional `query_normalized` / `indexed_at_head_sha`) and a `results` array. `indexed_at_head_sha` has the same meaning as status `indexed_head_sha`: it identifies the checkout captured by the latest successful full scan, `--files`, `--commits`, or `--changed-between` refresh. It does not advance after a failed or rolled-back refresh; databases created before `indexed_head_sha` fall back to the legacy full-scan-only `indexed_head_commit`. If the latest-head key exists but its value is unavailable because Git HEAD could not be resolved, the envelope omits `indexed_at_head_sha` instead of reporting the legacy baseline. Every envelope binds this stamp to the validated index generation used for its rows and rejects the response with restart guidance if that generation changes during execution; bounded responses also bind their cursor to that snapshot. The flag implies `--json` and works on every query command (`search`, `definition`, `references`, `callers`, `callees`, `symbols`, `files`, `find`, `excerpt`, `map`, `inspect`, `outline`, `status`, `validate`, `languages`, `impact`, `deps`, `unused`, `hotspots`). Wrapped commands can capture up to 10,485,760 output characters; if the budget is exceeded, cdidx returns a JSON envelope with empty `results`, non-zero `metadata.exit_code`, and `metadata.error`, and suggests using `--limit` / `--top` or streaming `--json`. The flat NDJSON / array output stays the default for one release; the envelope will become the default in the next major release, at which point the flat form will be opt-in via `--json-flat`. Add `--profile` to any read command when debugging slow queries. It appends one JSON object after the normal result with `profile.phases` (`name`, `elapsed_ms`, `rows_scanned`), `profile.query_plan` (`EXPLAIN QUERY PLAN` rows), and `profile.queries` (the SQL text). Add `--slow-query-ms ` to log profiled SQL statements that meet the threshold to the persistent tool log. @@ -4745,7 +4745,7 @@ src/Auth/TokenService.cs:42-58 {"path":"src/Auth/TokenService.cs","start_line":42,"end_line":58,"content":"public string GenerateToken(...)...","lang":"csharp","score":9.8} ``` -`--json-envelope` を追加すると、1 行ごとの stream を `metadata`（command、`cdidx_version`、`elapsed_ms`、`db_path`、`result_count`、`exit_code`、任意の `query_normalized` / `indexed_at_head_sha`）と `results` 配列を持つ 1 つの JSON document に包みます。`indexed_at_head_sha` の意味は status の `indexed_head_sha` と同じで、最後に成功した full scan、`--files`、`--commits`、`--changed-between` refresh が取り込んだ checkout を示します。失敗または rollback された refresh では進まず、`indexed_head_sha` より前の database では legacy の full-scan 限定 `indexed_head_commit` に fallback します。最新 HEAD key が存在していても Git HEAD を解決できず値が未設定の場合は、legacy baseline を返さず `indexed_at_head_sha` を省略します。bounded response では、この stamp を row と cursor と同じ検証済み index snapshot に固定します。この flag は `--json` を暗黙に有効化し、各 query command で使えます。wrapped command の捕捉出力は最大 10,485,760 文字です。超過した場合は、空の `results`、非 0 の `metadata.exit_code`、`metadata.error` を持つ JSON envelope を返し、`--limit` / `--top` または streaming `--json` の利用を促します。 +`--json-envelope` を追加すると、1 行ごとの stream を `metadata`（command、`cdidx_version`、`elapsed_ms`、`db_path`、`result_count`、`exit_code`、任意の `query_normalized` / `indexed_at_head_sha`）と `results` 配列を持つ 1 つの JSON document に包みます。`indexed_at_head_sha` の意味は status の `indexed_head_sha` と同じで、最後に成功した full scan、`--files`、`--commits`、`--changed-between` refresh が取り込んだ checkout を示します。失敗または rollback された refresh では進まず、`indexed_head_sha` より前の database では legacy の full-scan 限定 `indexed_head_commit` に fallback します。最新 HEAD key が存在していても Git HEAD を解決できず値が未設定の場合は、legacy baseline を返さず `indexed_at_head_sha` を省略します。すべての envelope はこの stamp を row と同じ検証済み index generation に固定し、実行中に generation が変わった場合は response を拒否して再実行案内を返します。bounded response では cursor も同じ snapshot に固定します。この flag は `--json` を暗黙に有効化し、各 query command で使えます。wrapped command の捕捉出力は最大 10,485,760 文字です。超過した場合は、空の `results`、非 0 の `metadata.exit_code`、`metadata.error` を持つ JSON envelope を返し、`--limit` / `--top` または streaming `--json` の利用を促します。 ### シンボル検索（関数、クラスなど） diff --git a/changelog.d/unreleased/4854.fixed.md b/changelog.d/unreleased/4854.fixed.md index 8db3ae8ff..2c5ae23a7 100644 --- a/changelog.d/unreleased/4854.fixed.md +++ b/changelog.d/unreleased/4854.fixed.md @@ -16,8 +16,8 @@ affected: ## English -- **Query envelopes now agree with status on the latest indexed HEAD (#4854)** — `metadata.indexed_at_head_sha` now uses the latest successful full or partial refresh stamp established by #1509, #1527, and #4313, while retaining the legacy full-scan HEAD fallback only for older databases without the latest key, omitting unresolved current stamps, binding bounded-response metadata to its validated snapshot, and preserving the prior stamp after failed or rolled-back refreshes. +- **Query envelopes now agree with status on the latest indexed HEAD (#4854)** — `metadata.indexed_at_head_sha` now uses the latest successful full or partial refresh stamp established by #1509, #1527, and #4313, while retaining the legacy full-scan HEAD fallback only for older databases without the latest key, omitting unresolved current stamps, binding regular and bounded responses to a validated generation, rejecting concurrent refresh races, and preserving the prior stamp after failed or rolled-back refreshes. ## 日本語 -- **query envelope と status が最新の indexed HEAD で一致するようになりました (#4854)** — `metadata.indexed_at_head_sha` は #1509、#1527、#4313 で確立された最新の成功 full / partial refresh stamp を使うようになり、最新 key のない古い database だけを legacy full-scan HEAD へ fallback し、解決不能な current stamp を省略し、bounded response metadata を検証済み snapshot に固定しつつ、失敗または rollback された refresh 後の直前 stamp を維持します。 +- **query envelope と status が最新の indexed HEAD で一致するようになりました (#4854)** — `metadata.indexed_at_head_sha` は #1509、#1527、#4313 で確立された最新の成功 full / partial refresh stamp を使うようになり、最新 key のない古い database だけを legacy full-scan HEAD へ fallback し、解決不能な current stamp を省略し、通常および bounded response を検証済み generation に固定して同時 refresh の競合を拒否しつつ、失敗または rollback された refresh 後の直前 stamp を維持します。 diff --git a/src/CodeIndex/Cli/JsonEnvelopeWrapper.cs b/src/CodeIndex/Cli/JsonEnvelopeWrapper.cs index 41fb24714..43eb05444 100644 --- a/src/CodeIndex/Cli/JsonEnvelopeWrapper.cs +++ b/src/CodeIndex/Cli/JsonEnvelopeWrapper.cs @@ -102,6 +102,7 @@ internal static int RunWrapped( var innerArgs = PrepareInnerArgs(args); var queryNormalized = ExtractQueryArg(args); var (resolvedDbPath, dbPathExplicit) = ResolveQueryDbPath(args); + var responseSnapshot = SafeReadResponseSnapshot(resolvedDbPath, dbPathExplicit, appVersion); using var captured = new BoundedStringWriter(MaxCapturedOutputChars); var stopwatch = Stopwatch.StartNew(); @@ -143,7 +144,8 @@ internal static int RunWrapped( stopwatch.Elapsed.TotalMilliseconds, new JsonArray(), exitCode, - envelopeError); + envelopeError, + responseSnapshot: responseSnapshot); Console.WriteLine(overflowEnvelope.ToJsonString(jsonOptions)); return exitCode; @@ -190,6 +192,24 @@ internal static int RunWrapped( }; results = []; } + + var completedSnapshot = SafeReadResponseSnapshot(resolvedDbPath, dbPathExplicit, appVersion); + if (!string.Equals( + responseSnapshot.GenerationFingerprint, + completedSnapshot.GenerationFingerprint, + StringComparison.Ordinal)) + { + return WriteResponseSnapshotChangedError( + command, + queryNormalized, + resolvedDbPath, + dbPathExplicit, + appVersion, + stopwatch.Elapsed.TotalMilliseconds, + jsonOptions, + responseSnapshot); + } + ResponseSnapshotValidatedForTesting?.Invoke(); var envelope = BuildEnvelope( command, queryNormalized, @@ -201,12 +221,48 @@ internal static int RunWrapped( exitCode, parseError, streamTerminal, - streamControlRecords); + streamControlRecords, + responseSnapshot); Console.WriteLine(envelope.ToJsonString(jsonOptions)); return exitCode; } + private static int WriteResponseSnapshotChangedError( + string command, + string? queryNormalized, + string dbPath, + bool dbPathExplicit, + string appVersion, + double elapsedMs, + JsonSerializerOptions jsonOptions, + ResponseSnapshot responseSnapshot) + { + const string message = "The index generation changed while this response was being read."; + const string hint = "Restart the command after the active index refresh completes."; + CommandErrorWriter.WriteStderr($"Error [{CommandErrorCodes.UsageError}]: {message}"); + CommandErrorWriter.WriteStderr($"Hint: {hint}"); + var error = new JsonObject + { + ["message"] = message, + ["hint"] = hint, + ["error_code"] = CommandErrorCodes.UsageError, + }; + var envelope = BuildEnvelope( + command, + queryNormalized, + dbPath, + dbPathExplicit, + appVersion, + elapsedMs, + [], + CommandExitCodes.UsageError, + error, + responseSnapshot: responseSnapshot); + Console.WriteLine(envelope.ToJsonString(jsonOptions)); + return CommandExitCodes.UsageError; + } + private static bool HasArgument(string[] args, string option) => args.Any(arg => string.Equals(arg, option, StringComparison.Ordinal) || arg.StartsWith(option + "=", StringComparison.Ordinal)); diff --git a/tests/CodeIndex.Tests/JsonEnvelopeWrapperTests.cs b/tests/CodeIndex.Tests/JsonEnvelopeWrapperTests.cs index 4f150f256..aa9a47568 100644 --- a/tests/CodeIndex.Tests/JsonEnvelopeWrapperTests.cs +++ b/tests/CodeIndex.Tests/JsonEnvelopeWrapperTests.cs @@ -238,6 +238,107 @@ public void Search_WithEnvelope_UsesLatestIndexedHeadAndFallsBackForLegacyDataba } } + [Fact] + public void Search_UnboundedEnvelope_KeepsIndexedHeadFromValidatedResponseSnapshot_Issue4854() + { + var projectRoot = TestProjectHelper.CreateTempProject("envelope_unbounded_indexed_head_snapshot"); + try + { + var dbPath = TestProjectHelper.CreateProjectDb(projectRoot); + TestProjectHelper.InsertIndexedFile( + dbPath, + "src/App.cs", + "csharp", + "class App { void Authenticate() {} }\n"); + const string snapshotHead = "2222222222222222222222222222222222222222"; + const string nextHead = "3333333333333333333333333333333333333333"; + using (var db = new DbContext(DbOpenIntent.WriteIndex, dbPath)) + { + var writer = new DbWriter(db.Connection); + writer.SetMetaValues((DbContext.IndexedHeadShaMetaKey, snapshotHead)); + } + + var hookInvoked = false; + JsonEnvelopeWrapper.ResponseSnapshotValidatedForTesting = () => + { + hookInvoked = true; + using var db = new DbContext(DbOpenIntent.WriteIndex, dbPath); + var writer = new DbWriter(db.Connection); + writer.SetMetaValues((DbContext.IndexedHeadShaMetaKey, nextHead)); + }; + + var (exitCode, stdout, stderr) = CaptureConsole(() => ProgramRunner.Run( + ["search", "Authenticate", "--db", dbPath, "--json-envelope"], + _jsonOptions, + "1.0.0")); + + Assert.True(hookInvoked); + Assert.Equal(CommandExitCodes.Success, exitCode); + Assert.Equal(string.Empty, stderr); + using var document = JsonDocument.Parse(stdout); + Assert.Equal( + snapshotHead, + document.RootElement + .GetProperty("metadata") + .GetProperty("indexed_at_head_sha") + .GetString()); + using var verificationDb = new DbContext(DbOpenIntent.WriteIndex, dbPath); + Assert.Equal(nextHead, verificationDb.GetMetaString(DbContext.IndexedHeadShaMetaKey)); + } + finally + { + JsonEnvelopeWrapper.ResponseSnapshotValidatedForTesting = null; + TestProjectHelper.DeleteDirectory(projectRoot); + } + } + + [Fact] + public void RunWrapped_UnboundedEnvelope_RejectsRowsWhenIndexGenerationChanges_Issue4854() + { + var projectRoot = TestProjectHelper.CreateTempProject("envelope_unbounded_generation_change"); + try + { + var dbPath = TestProjectHelper.CreateProjectDb(projectRoot); + const string snapshotHead = "2222222222222222222222222222222222222222"; + const string nextHead = "3333333333333333333333333333333333333333"; + using (var db = new DbContext(DbOpenIntent.WriteIndex, dbPath)) + { + var writer = new DbWriter(db.Connection); + writer.SetMetaValues((DbContext.IndexedHeadShaMetaKey, snapshotHead)); + } + + var (exitCode, stdout, stderr) = CaptureConsole(() => JsonEnvelopeWrapper.RunWrapped( + "search", + ["Authenticate", "--db", dbPath, "--json-envelope"], + "1.0.0", + _jsonOptions, + _ => + { + Console.WriteLine("""{"path":"src/App.cs"}"""); + using var db = new DbContext(DbOpenIntent.WriteIndex, dbPath); + var writer = new DbWriter(db.Connection); + writer.SetMetaValues((DbContext.IndexedHeadShaMetaKey, nextHead)); + return CommandExitCodes.Success; + })); + + Assert.Equal(CommandExitCodes.UsageError, exitCode); + Assert.Contains("index generation changed", stderr, StringComparison.Ordinal); + using var document = JsonDocument.Parse(stdout); + var root = document.RootElement; + Assert.Empty(root.GetProperty("results").EnumerateArray()); + Assert.Equal( + CommandErrorCodes.UsageError, + root.GetProperty("metadata").GetProperty("error").GetProperty("error_code").GetString()); + Assert.Equal( + snapshotHead, + root.GetProperty("metadata").GetProperty("indexed_at_head_sha").GetString()); + } + finally + { + TestProjectHelper.DeleteDirectory(projectRoot); + } + } + [Fact] public void Search_BoundedEnvelope_KeepsIndexedHeadFromValidatedResponseSnapshot_Issue4854() {