diff --git a/AGENT_GUIDE.md b/AGENT_GUIDE.md index 13f611a93..5784332c9 100644 --- a/AGENT_GUIDE.md +++ b/AGENT_GUIDE.md @@ -136,6 +136,7 @@ CI watching must be bounded. Do not loop indefinitely. ## Status Contract - `status --json` and related JSON/MCP payloads currently expose the trust fields documented in `README.md` and `DEVELOPER_GUIDE.md`, including `fold_ready`, `fold_ready_reason`, `graph_table_available`, `graph_data_current`, `index_complete`, `index_incomplete_reasons`, `issues_table_available`, `file_issues_data_current`, `migration_in_progress`, `sql_graph_contract_ready`, `sql_graph_contract_degraded_reason`, `hotspot_family_ready`, `hotspot_family_degraded_reason`, `language_readiness`, `csharp_symbol_name_ready`, `csharp_metadata_target_ready`, `csharp_metadata_target_degraded_reason`, `indexed_head_commit`, `worktree_head_changed`, `indexed_head_sha`, `indexed_head_branch`, `indexed_head_timestamp`, `commits_ahead_of_indexed_head`, `head_freshness`, `index_writer_version`, `index_newer_than_reader`, `index_newer_than_reader_reason`, `unknown_extension_file_count`, `unknown_extension_files`, `unknown_extension_files_truncated`, `unknown_extension_file_path_limit`, `unknown_extension_extension_counts`, `unknown_extension_category_counts`, `unknown_extension_groups`, `extractors`, `git_executable`, `path_case_sensitive`, `data_dir`, `data_dir_source`, `data_dir_mode`, `db_file_mode`, `database_permission_policy`, `database_permission_diagnostics`, `mac_profile`, `mac_profile_diagnostics`, `db_size_bytes`, `wal_size_bytes`, `db_pragma_settings` (`journal_mode`, `synchronous`, `wal_autocheckpoint`, `busy_timeout_ms`, `page_count`, `freelist_count`, `page_size`, `auto_vacuum`), `prepared_command_cache` (`count`, `capacity`, `hit_count`, `miss_count`, `eviction_count`), `maintenance_guidance`, WAL checkpoint diagnostics (`read_only_fallback`, `wal_checkpoint_attempted`, `wal_checkpoint_succeeded`, `wal_checkpoint_skipped_reason`, `wal_checkpoint_failure_reason`, `wal_checkpoint_busy`, `wal_checkpoint_log_page_count`, `wal_checkpoint_checkpointed_page_count`, `wal_checkpoint_remaining_page_count`, `read_only_immutable_fallback`, `wal_stale_snapshot_risk`, `wal_stale_snapshot_reason`), `symbol_kinds`, `symbols_by_language`, status kind cap metadata (`symbol_kind_limit`, `symbol_kind_name_limit`, `symbol_kind_total_count`, `symbol_kind_omitted_count`, `symbol_kind_names_truncated`, `symbols_by_language_kind_total_counts`, `symbols_by_language_kind_omitted_counts`, `symbols_by_language_kind_names_truncated`), `process`, `last_index_run`, `last_failed_or_partial_index_run`, `last_failed_or_partial_index_run.progress_persisted`, `last_failed_or_partial_index_run.recovery_hint`, `last_failed_or_partial_index_run.file_errors`, `last_workspace_freshened_at`, `hooks`, `hook_diagnostics`, `trust_overrides`, MCP-only `mcp_session`, `mcp.rate_limit.bucket_limit`, `mcp.rate_limit.bucket_limit_rejection_count`, and the `status --check`-only `stale_after_seconds` / `index_age_seconds` threshold audit fields and `repair_commands`. +- `maintenance_guidance.fts_optimization` is the shared, read-only recommendation contract for status, explain, optimize preview, and optimize execution. Keep `recommended`, `action`, `reason`, `threshold_writes`, `observed_writes`, and `state` synchronized; stale or unavailable snapshots must not recommend mutation. - A valid CLI `status --stale-after ` implies the workspace check. Check-mode JSON includes `query_context.check_mode` (`explicit` or `implied_by_stale_after`) and `query_context.stale_after_seconds`; ordinary status JSON omits `query_context`. - `database_size_attribution` is part of the synchronized status contract. Preserve its read-only main/WAL/SHM separation; exact logical reconciliation across object, freelist, and unexplained-residual bytes; table/index and page-type subtotals; 20-object/128-character sanitized bounds; and explicit `available=false` / stable `unavailable_reason` behavior without zero-valued unavailable object metrics. - Explicit WAL truncate-checkpoint diagnostics must preserve SQLite's `(busy, log, checkpointed)` result, treat non-zero `busy` or positive remaining pages as unsuccessful with bounded machine reasons, accept `(0, -1, -1)` as the successful non-WAL no-op, and never expose raw exception text or paths. diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index 8276b3760..a10bb1476 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -969,11 +969,12 @@ Current stable codes and triggers: | Schema discovery cache | `DbReader` schema discovery uses a process-level cache keyed by the normalized DB path. Column and index results are stored and returned as immutable `FrozenSet` snapshots, so callers cannot mutate schema decisions shared by other readers. Path states are reference-counted by live `DbSchemaCache` owners, are removed when the final owning `DbContext` is disposed, and are never evicted while an owner remains active. The cache checks `PRAGMA schema_version` before serving a lookup so SQLite DDL performed by cdidx or an external `sqlite3` session invalidates stale snapshots. Manual schema edits outside cdidx are still unsupported operationally; run `cdidx validate` after such edits before trusting query output. | | Batch trust marker | Index write batches stamp `codeindex_meta.batch_in_progress=true` before starting a mutation transaction and clear it inside the transaction that commits the matching rows and readiness metadata. If the indexer crashes after the marker is written but before the commit clears it, every later open reports `Last batch did not complete; run cdidx index --rebuild to re-index from a known clean state.` without changing readiness metadata. The explicit `index --rebuild` repair path alone demotes readiness before rebuilding. Gracefully handled per-file errors clear the marker after rollback; orphaned markers are reserved for interrupted or crashed batches whose trust metadata should not be treated as clean. | | Read-only opens and fallback | Query-only commands open with SQLite `Mode=ReadOnly` from the first attempt, retain WAL visibility, and never use writable setup or opportunistic migrations. A write-capable intent may still fall back to read-only when writable journal/WAL setup fails; an explicitly supplied `immutable=1` URI is the opt-in stale-snapshot escape hatch. If a WAL is present and must be observed from storage that cannot expose its sidecars, copy `.db`, `.db-wal`, and `.db-shm` together to a readable location or use a SQLite backup from an environment that can open the full WAL set. | -| Status pragma diagnostics | `status --json` exposes the selected read-only connection under `sqlite_connection_policy` (`active_mode=read_only`, `open_mode=read_only`) and resolved connection values under `db_pragma_settings` (`journal_mode`, `synchronous`, `wal_autocheckpoint`, `busy_timeout_ms`, `page_count`, `freelist_count`, `page_size`, `auto_vacuum`). It also exposes prepared-command cache counters under `prepared_command_cache` (`count`, `capacity`, `hit_count`, `miss_count`, `eviction_count`) for automation and support diagnostics. `maintenance_guidance` derives `wal_state`, `freelist_ratio`, `freelist_state`, `estimated_*_reclaimable`, `auto_vacuum_mode(_name)`, `recommended_command`, and `post_maintenance_follow_up` from those raw metrics without changing the raw values. `status --check --json` adds `repair_commands[]` entries with `name`, `args`, `reason`, and `safety_notes` so clients do not parse prose remediation strings. `last_failed_or_partial_index_run` exposes bounded failed/partial index context (`status`, `mode`, timings, counts, stable error code, reason, `progress_persisted`, and bounded `recovery_hint`) and must not include raw exception text or file paths. | +| Status pragma diagnostics | `status --json` exposes the selected read-only connection under `sqlite_connection_policy` (`active_mode=read_only`, `open_mode=read_only`) and resolved connection values under `db_pragma_settings` (`journal_mode`, `synchronous`, `wal_autocheckpoint`, `busy_timeout_ms`, `page_count`, `freelist_count`, `page_size`, `auto_vacuum`). It also exposes prepared-command cache counters under `prepared_command_cache` (`count`, `capacity`, `hit_count`, `miss_count`, `eviction_count`) for automation and support diagnostics. `maintenance_guidance` derives `wal_state`, `freelist_ratio`, `freelist_state`, `estimated_*_reclaimable`, `auto_vacuum_mode(_name)`, `recommended_command`, and `post_maintenance_follow_up` from those raw metrics without changing the raw values. Its nested `fts_optimization` uses the same pure evaluator as optimize preview and execution, exposing `recommended`, `action`, `reason`, `threshold_writes`, `observed_writes`, and `state` without writing to the database. `status --check --json` adds `repair_commands[]` entries with `name`, `args`, `reason`, and `safety_notes` so clients do not parse prose remediation strings. `last_failed_or_partial_index_run` exposes bounded failed/partial index context (`status`, `mode`, timings, counts, stable error code, reason, `progress_persisted`, and bounded `recovery_hint`) and must not include raw exception text or file paths. | | Maintenance thresholds | WAL guidance flips to `checkpoint_recommended` at `CDIDX_MAINTENANCE_WAL_WARN_BYTES` (default 64 MiB). Freelist guidance flips to `vacuum_recommended` at `CDIDX_MAINTENANCE_FREELIST_WARN_RATIO` (default `0.20`). Invalid or out-of-range env values fall back to defaults. | +| Maintenance command precedence | `maintenance_guidance.recommended_command` preserves the existing vacuum-then-checkpoint precedence. It returns `cdidx optimize --db ` only when WAL and freelist states are both exactly `ok` and the trusted FTS write snapshot reaches its threshold; an `unknown` higher-priority state or a stale/unavailable FTS snapshot never selects an optimize command. | | Page attribution | `status --json` reads SQLite page ownership without mutating the source. It prefers `dbstat` page bytes and otherwise traverses a bounded b-tree/WAL snapshot (at most 1,000,000 pages and 100,000 schema objects); when a live WAL connection is not already backed by a stable detached file set, the fallback first makes a cancellation-aware private backup of that connection's active read snapshot so a concurrent commit cannot mix generations. `allocated_object_bytes + freelist_bytes + unexplained_residual_bytes` equals `logical_database_bytes`; table/index and internal/leaf/overflow/other page subtotals each reconcile to `allocated_object_bytes`. Payload, unused space, and structural overhead form a second reconciliation. Physical main/WAL/SHM bytes are reported separately. Output is capped at 20 object names, each support-sanitized to at most 128 characters. A failed or inconsistent probe returns `available=false`, a stable `unavailable_reason`, and null/omitted attribution values rather than zeros. | | Vacuum | `cdidx vacuum` runs `PRAGMA incremental_vacuum` against writable incremental-auto-vacuum DBs, and performs a one-time `PRAGMA auto_vacuum=INCREMENTAL` plus full `VACUUM` conversion for legacy no-autovacuum DBs. `cdidx vacuum --dry-run --json` estimates reclaimable pages/bytes and returns the same maintenance guidance without executing vacuum pragmas. Real `cdidx vacuum --json` also reports before/after DB and WAL byte samples; `wal_checkpoint_timing_note` explains that `wal_size_bytes_after` is measured before connection cleanup, so later `status --json` output may show a smaller WAL after checkpoint/truncation. | -| FTS optimize preview | `cdidx optimize --dry-run` opens a `QueryOnly` snapshot, probes an existing lockfile without creating or acquiring it, and never runs write PRAGMAs, schema setup, FTS control inserts, or metadata writes against the source DB/WAL/SHM set. JSON reports size/freelist/readiness indicators, the write-threshold recommendation, and planned operations, including the real command's repair-mode schema initialization or migration check. Object sizes use `dbstat` page bytes when available and a labeled logical-payload fallback otherwise. A real optimize records its elapsed milliseconds so later previews can expose `estimated_duration_ms`. | +| FTS optimize preview | `cdidx optimize --dry-run` and its `cdidx index --optimize --dry-run` alias open a `QueryOnly` snapshot, probe an existing lockfile without creating or acquiring it, and never run write PRAGMAs, schema setup, FTS control inserts, or metadata writes against the source DB/WAL/SHM set. Both entry points preserve an explicitly supplied `file:` URI, including `immutable=1`, for the query connection while filesystem probes use its normalized local path, so status and either preview apply identical stale-snapshot semantics even with a hot WAL. JSON reports size/freelist/readiness indicators, planned operations, and the same `fts_optimization` recommendation object as status, including the exact threshold and observed writes. A stale batch or unavailable legacy counter/page snapshot suppresses recommendation with a stable reason and state. Real execution reports the same object before and after optimize, including the reset counter, and performs the repair-mode schema initialization or migration check. Object sizes use `dbstat` page bytes when available and a labeled logical-payload fallback otherwise. A real optimize records its elapsed milliseconds so later previews can expose `estimated_duration_ms`. | | Size and process diagnostics | `status --json` also reports `db_size_bytes`, `wal_size_bytes`, capped `symbol_kinds` / `symbols_by_language` kind maps with `symbol_kind_*` and `symbols_by_language_kind_*` overflow metadata when caps apply, current `process` heap/GC/working-set metrics, `last_index_run` metadata from successful CLI and MCP index runs, and `last_workspace_freshened_at` as the latest successful index/update timestamp. `last_index_run.bytes_read_skipped_file_count` and `bytes_read_incomplete` report whether unreadable files were omitted from the `bytes_read` total, while `last_index_run.diagnostics`, `diagnostic_count`, and `diagnostics_truncated` carry bounded warnings for best-effort index metadata writes that failed after the index data itself was successfully written. `indexed_at` still comes from indexed file rows, so partial or no-op updates can freshen the workspace without moving `indexed_at`. | | Memory tracing | `index --json --memory-trace` adds a `memory_timeline` block to the CLI index result and persists peak working-set MB into `last_index_run`; dry-run results also emit live `start`, `snapshot`, `scan`, and `finalize` samples but never persist run metadata. `index --dry-run --rebuild` bypasses destructive confirmation because it does not delete or rewrite the index. `CDIDX_MEM_WARN_MB=` prints a warning when the sampled working set crosses that threshold. | | Newer schema protection | Writable opens reject databases whose `PRAGMA user_version` contains readiness bits outside the current binary's `CurrentSchemaVersion` mask. Read-only status/query paths may still surface `index_newer_than_reader=true` as a degraded audit signal, but write-capable paths must fail with `E003_SCHEMA_TOO_NEW` so an older cdidx cannot silently rewrite a DB stamped by a newer one. | @@ -4362,11 +4363,12 @@ apply 時は `PRAGMA optimize` を実行します。 | schema discovery cache | `DbReader` の schema discovery は正規化済み DB path を key にした process-level cache を使います。column / index 結果は immutable な `FrozenSet` snapshot として保存・返却されるため、caller が他の reader と共有する schema 判定を変更することはできません。path state は有効な `DbSchemaCache` owner により参照カウントされ、最後の owner `DbContext` が dispose されると削除され、owner が active な間は退避されません。lookup 前に `PRAGMA schema_version` を確認するため、cdidx や外部 `sqlite3` session による SQLite DDL は stale snapshot を invalidate します。cdidx 外での手動 schema edit は運用上 unsupported であり、その後は query output を信頼する前に `cdidx validate` を実行してください。 | | batch trust marker | index write batch は mutation transaction を始める前に `codeindex_meta.batch_in_progress=true` を stamp し、対応する row と readiness metadata を commit する transaction 内で clear します。marker が書かれた後、clear される前に indexer が crash した場合、その後のすべての open は readiness metadata を変更せずに `Last batch did not complete; run cdidx index --rebuild to re-index from a known clean state.` と警告します。readiness を degrade するのは、明示的な `index --rebuild` repair path だけです。file ごとの error が graceful に処理された場合は rollback 後に marker を clear するため、orphaned marker は interrupted / crashed batch の trust metadata を clean と扱わないための signal です。 | | read-only open / fallback | query-only command は最初の試行から SQLite `Mode=ReadOnly` で開き、WAL の可視性を保ちながら writable setup と opportunistic migration を実行しません。write-capable intent は journal/WAL setup に失敗した場合に read-only へ fallback することがあります。明示的な `immutable=1` URI は stale snapshot を許容する opt-in escape hatch です。sidecar を公開できない storage 上の WAL を観測する必要がある場合は、`.db` / `.db-wal` / `.db-shm` をまとめて readable location に copy するか、full WAL set を open できる環境で SQLite backup を使います。 | -| status pragma diagnostics | `status --json` は選択された read-only connection を `sqlite_connection_policy` (`active_mode=read_only`, `open_mode=read_only`) で、解決済みの接続値を `db_pragma_settings` (`journal_mode`, `synchronous`, `wal_autocheckpoint`, `busy_timeout_ms`, `page_count`, `freelist_count`, `page_size`, `auto_vacuum`) で公開します。また、prepared command cache counter を `prepared_command_cache` (`count`, `capacity`, `hit_count`, `miss_count`, `eviction_count`) で公開します。`maintenance_guidance` は raw 値を変えずに `wal_state`、`freelist_ratio`、`freelist_state`、`estimated_*_reclaimable`、`auto_vacuum_mode(_name)`、`recommended_command`、`post_maintenance_follow_up` を派生します。`status --check --json` は `repair_commands[]` に `name`、`args`、`reason`、`safety_notes` を返し、client が prose remediation を parse しなくてよいようにします。`last_failed_or_partial_index_run` は bounded な failed / partial index context (`status`、`mode`、timing、count、stable error code、reason、`progress_persisted`、bounded な `recovery_hint`) のみを公開し、raw exception text や file path を含めてはいけません。 | +| status pragma diagnostics | `status --json` は選択された read-only connection を `sqlite_connection_policy` (`active_mode=read_only`, `open_mode=read_only`) で、解決済みの接続値を `db_pragma_settings` (`journal_mode`, `synchronous`, `wal_autocheckpoint`, `busy_timeout_ms`, `page_count`, `freelist_count`, `page_size`, `auto_vacuum`) で公開します。また、prepared command cache counter を `prepared_command_cache` (`count`, `capacity`, `hit_count`, `miss_count`, `eviction_count`) で公開します。`maintenance_guidance` は raw 値を変えずに `wal_state`、`freelist_ratio`、`freelist_state`、`estimated_*_reclaimable`、`auto_vacuum_mode(_name)`、`recommended_command`、`post_maintenance_follow_up` を派生します。nested な `fts_optimization` は optimize preview / execution と同じ純粋 evaluator を使い、database に書き込まず `recommended`、`action`、`reason`、`threshold_writes`、`observed_writes`、`state` を公開します。`status --check --json` は `repair_commands[]` に `name`、`args`、`reason`、`safety_notes` を返し、client が prose remediation を parse しなくてよいようにします。`last_failed_or_partial_index_run` は bounded な failed / partial index context (`status`、`mode`、timing、count、stable error code、reason、`progress_persisted`、bounded な `recovery_hint`) のみを公開し、raw exception text や file path を含めてはいけません。 | | maintenance threshold | WAL guidance は `CDIDX_MAINTENANCE_WAL_WARN_BYTES` (既定 64 MiB) 以上で `checkpoint_recommended` になります。freelist guidance は `CDIDX_MAINTENANCE_FREELIST_WARN_RATIO` (既定 `0.20`) 以上で `vacuum_recommended` になります。不正・範囲外の環境変数値は既定値へ戻します。 | +| maintenance command の優先順位 | `maintenance_guidance.recommended_command` は既存の vacuum、checkpoint の順序を維持します。WAL と freelist の state が両方とも厳密に `ok` で、信頼できる FTS write snapshot が threshold に達した場合だけ `cdidx optimize --db ` を返します。上位 state が `unknown` の場合や FTS snapshot が stale / unavailable の場合は optimize command を選択しません。 | | page attribution | `status --json` は source を変更せずに SQLite page ownership を読み取ります。`dbstat` page byte を優先し、利用できない場合は件数上限付きの b-tree / WAL snapshot traversal（最大1,000,000 page、100,000 schema object）へ fallback します。live WAL connection が安定した detached file set に基づいていない場合、fallback は先にその connection の active read snapshot を cancellation 対応の private backup に固定し、並行 commit による世代混在を防ぎます。`allocated_object_bytes + freelist_bytes + unexplained_residual_bytes` は `logical_database_bytes` と一致し、table/index と internal/leaf/overflow/other page の小計はそれぞれ `allocated_object_bytes` と一致します。payload、unused space、structural overhead も別に再照合されます。物理 main/WAL/SHM byte は分離して報告します。出力する object 名は最大20件で、各名称は support-safe sanitizer により最大128文字になります。probe の失敗・不整合時は `available=false`、安定した `unavailable_reason`、null / 省略された attribution 値を返し、ゼロとして偽装しません。 | | vacuum | `cdidx vacuum` は incremental-auto-vacuum DB では `PRAGMA incremental_vacuum` を実行し、legacy no-autovacuum DB では初回のみ `PRAGMA auto_vacuum=INCREMENTAL` と full `VACUUM` で変換します。`cdidx vacuum --dry-run --json` は vacuum pragma を実行せず、回収可能 page/byte の推定と同じ maintenance guidance を返します。実行系 `cdidx vacuum --json` は DB / WAL byte の before / after sample も返します。`wal_checkpoint_timing_note` は `wal_size_bytes_after` が connection cleanup 前の計測であり、checkpoint / truncation 後の `status --json` では WAL が小さく見える場合があることを示します。 | -| FTS optimize preview | `cdidx optimize --dry-run` は `QueryOnly` snapshot を開き、既存 lockfile を作成も取得もせずに probe し、source DB/WAL/SHM set に対する write PRAGMA、schema setup、FTS control insert、metadata write を一切実行しません。JSON は size/freelist/readiness 指標、write threshold に基づく推奨、実行系の repair mode による schema 初期化または migration の確認を含む planned operation を返します。object size は利用可能なら `dbstat` page byte を使い、利用できない場合は明示した logical-payload fallback を使います。実際の optimize は所要 millisecond を記録し、後続 preview が `estimated_duration_ms` として返せるようにします。 | +| FTS optimize preview | `cdidx optimize --dry-run` とその alias である `cdidx index --optimize --dry-run` は `QueryOnly` snapshot を開き、既存 lockfile を作成も取得もせずに probe し、source DB/WAL/SHM set に対する write PRAGMA、schema setup、FTS control insert、metadata write を一切実行しません。どちらの entry point も query connection では明示的に指定された `file:` URI と `immutable=1` を保持し、filesystem probe だけが正規化済み local path を使うため、hot WAL がある場合も status と両方の preview は同じ stale-snapshot semantics を適用します。JSON は size/freelist/readiness 指標、planned operation、および正確な threshold と observed write を含む status と同じ `fts_optimization` recommendation object を返します。stale batch または legacy counter / page snapshot が利用できない場合は、安定した reason と state で recommendation を抑止します。実行系は optimize 前後で同じ object と reset 後の counter を返し、repair mode の schema 初期化または migration 確認を実行します。object size は利用可能なら `dbstat` page byte を使い、利用できない場合は明示した logical-payload fallback を使います。実際の optimize は所要 millisecond を記録し、後続 preview が `estimated_duration_ms` として返せるようにします。 | | size / process diagnostics | `status --json` は `db_size_bytes`、`wal_size_bytes`、上限付きの `symbol_kinds` / `symbols_by_language` kind map と、上限適用時の `symbol_kind_*` / `symbols_by_language_kind_*` overflow metadata、現在の `process` heap / GC / working-set metrics、成功した CLI / MCP index 実行由来の `last_index_run` metadata、最新の成功 index/update 時刻を示す `last_workspace_freshened_at` も公開します。`last_index_run.bytes_read_skipped_file_count` と `bytes_read_incomplete` は、読み取り不能な file が `bytes_read` 合計から除外されたかどうかを報告します。`last_index_run.diagnostics`、`diagnostic_count`、`diagnostics_truncated` は、index data 自体の書き込みが成功した後に best-effort index metadata write が失敗した場合の上限付き warning を保持します。`indexed_at` は引き続き indexed file row 由来なので、partial / no-op update は `indexed_at` を動かさずに workspace 鮮度だけを更新することがあります。 | | memory tracing | `index --json --memory-trace` は CLI index 結果に `memory_timeline` block を追加し、peak working-set MB を `last_index_run` に保存します。dry-run 結果も live な `start`、`snapshot`、`scan`、`finalize` sample を返しますが、run metadata は保存しません。`index --dry-run --rebuild` は index を削除も rewrite もしないため destructive confirmation を bypass します。`CDIDX_MEM_WARN_MB=` は sampled working set がしきい値を超えたときに warning を出します。 | | newer schema protection | writable open は、`PRAGMA user_version` に current binary の `CurrentSchemaVersion` mask 外の readiness bit が含まれる database も拒否します。read-only status/query path は degraded audit signal として `index_newer_than_reader=true` を表示できますが、write-capable path は古い cdidx が新しい binary で stamp された DB を黙って rewrite しないよう `E003_SCHEMA_TOO_NEW` で失敗しなければなりません。 | diff --git a/README.md b/README.md index 2a96d3d84..c4dfa5061 100644 --- a/README.md +++ b/README.md @@ -238,6 +238,8 @@ fields, including readiness fields and runtime diagnostics such as `path_case_sensitive`. `cdidx status --explain sqlite_connection_policy` describes the active SQLite open mode, immutable-URI choice, timeout, cancellation, and WAL snapshot-risk diagnostics. +`cdidx status --explain maintenance_guidance` describes the shared FTS +optimization recommendation used by status and optimize. | Field group | Fields | |---|---| @@ -245,7 +247,7 @@ cancellation, and WAL snapshot-risk diagnostics. | Workspace and HEAD freshness | `indexed_head_commit`, `worktree_head_changed`, `indexed_head_sha`, `indexed_head_branch`, `indexed_head_timestamp`, `commits_ahead_of_indexed_head`, `head_freshness`. | | Version and forward compatibility | `index_writer_version`, `index_newer_than_reader`, `index_newer_than_reader_reason`. | | Unknown-extension and runtime diagnostics | `unknown_extension_file_count`, `unknown_extension_files`, `unknown_extension_files_truncated`, `unknown_extension_file_path_limit`, `unknown_extension_extension_counts`, `unknown_extension_category_counts`, `unknown_extension_groups`, `extractors`, `hooks`, `hook_diagnostics`, `trust_overrides`, `git_executable`, `path_case_sensitive`, `data_dir_mode`, `db_file_mode`, `database_permission_policy`, `database_permission_diagnostics`, `mac_profile`, `mac_profile_diagnostics`, `stale_after_seconds`, `index_age_seconds`, `query_context.check_mode`, `query_context.stale_after_seconds`, `process`, `last_index_run`, `last_workspace_freshened_at`, `last_index_run.bytes_read_skipped_file_count`, `last_index_run.bytes_read_incomplete`, `last_index_run.diagnostics`, `last_index_run.diagnostic_count`, `last_index_run.diagnostics_truncated`, `last_index_run.reference_extraction_cap_hits`, `last_failed_or_partial_index_run`, `last_failed_or_partial_index_run.progress_persisted`, `last_failed_or_partial_index_run.recovery_hint`, `last_failed_or_partial_index_run.file_errors`. | -| Database maintenance | `sqlite_connection_policy` (`active_mode`, `open_mode`, `immutable_uri`, WAL checkpoint/fallback fields), `db_size_bytes`, `wal_size_bytes`, `db_pragma_settings` (`journal_mode`, `synchronous`, `wal_autocheckpoint`, `busy_timeout_ms`, `page_count`, `freelist_count`, `page_size`, `auto_vacuum`), `prepared_command_cache` (`count`, `capacity`, `hit_count`, `miss_count`, `eviction_count`), `maintenance_guidance`. Query-only status reports `pooling=false` and either `read_only` or `immutable_read_only_uri`; checkpointed WAL databases use an immutable private snapshot, while non-empty WAL databases use a stable private main/WAL snapshot so source sidecars remain unchanged. Persistent private-snapshot copy failures report `query_only_snapshot_copy_failed` with temporary-storage capacity and permission guidance instead of being misreported as WAL churn. | +| Database maintenance | `sqlite_connection_policy` (`active_mode`, `open_mode`, `immutable_uri`, WAL checkpoint/fallback fields), `db_size_bytes`, `wal_size_bytes`, `db_pragma_settings` (`journal_mode`, `synchronous`, `wal_autocheckpoint`, `busy_timeout_ms`, `page_count`, `freelist_count`, `page_size`, `auto_vacuum`), `prepared_command_cache` (`count`, `capacity`, `hit_count`, `miss_count`, `eviction_count`), `maintenance_guidance`, `maintenance_guidance.fts_optimization` (`recommended`, `action`, `reason`, `threshold_writes`, `observed_writes`, `state`). Query-only status reports `pooling=false` and either `read_only` or `immutable_read_only_uri`; checkpointed WAL databases use an immutable private snapshot, while non-empty WAL databases use a stable private main/WAL snapshot so source sidecars remain unchanged. Persistent private-snapshot copy failures report `query_only_snapshot_copy_failed` with temporary-storage capacity and permission guidance instead of being misreported as WAL churn. | | WAL checkpoint diagnostics | `read_only_fallback`, `wal_checkpoint_attempted`, `wal_checkpoint_succeeded`, `wal_checkpoint_skipped_reason`, `wal_checkpoint_failure_reason`, `wal_checkpoint_busy`, `wal_checkpoint_log_page_count`, `wal_checkpoint_checkpointed_page_count`, `wal_checkpoint_remaining_page_count`, `read_only_immutable_fallback`, `wal_stale_snapshot_risk`, `wal_stale_snapshot_reason`. | | Database size attribution | `database_size_attribution` separates main DB, WAL, and SHM file bytes and reconciles logical pages across tables, indexes, freelist pages, internal/leaf/overflow page types, payload, unused space, structural overhead, and `unexplained_residual_bytes`. It emits at most 20 redacted/truncated object names. `available=false` with `unavailable_reason` means page attribution was not measurable; omitted object-byte fields must not be interpreted as zero. | | Remediation fields | `degraded_root_cause`, `degraded_reason`, `recommended_action`, `alternative_action`, `readiness_degradations`, `repair_commands`. | @@ -689,6 +691,8 @@ visible な status field の簡潔な説明は `cdidx status --explain ` readiness field に加えて、`path_case_sensitive` などの runtime diagnostic field も対象です。 `cdidx status --explain sqlite_connection_policy` は、有効な SQLite open mode、 immutable URI の選択、timeout、cancellation、WAL snapshot risk の diagnostic を説明します。 +`cdidx status --explain maintenance_guidance` は、status と optimize が共有する +FTS optimization recommendation を説明します。 | field group | fields | |---|---| @@ -696,7 +700,7 @@ immutable URI の選択、timeout、cancellation、WAL snapshot risk の diagnos | workspace / HEAD freshness | `indexed_head_commit`, `worktree_head_changed`, `indexed_head_sha`, `indexed_head_branch`, `indexed_head_timestamp`, `commits_ahead_of_indexed_head`, `head_freshness`。 | | version / forward compatibility | `index_writer_version`, `index_newer_than_reader`, `index_newer_than_reader_reason`。 | | unknown-extension / runtime diagnostics | `unknown_extension_file_count`, `unknown_extension_files`, `unknown_extension_files_truncated`, `unknown_extension_file_path_limit`, `unknown_extension_extension_counts`, `unknown_extension_category_counts`, `unknown_extension_groups`, `extractors`, `hooks`, `hook_diagnostics`, `trust_overrides`, `git_executable`, `path_case_sensitive`, `data_dir_mode`, `db_file_mode`, `database_permission_policy`, `database_permission_diagnostics`, `mac_profile`, `mac_profile_diagnostics`, `stale_after_seconds`, `index_age_seconds`, `query_context.check_mode`, `query_context.stale_after_seconds`, `process`, `last_index_run`, `last_workspace_freshened_at`, `last_index_run.bytes_read_skipped_file_count`, `last_index_run.bytes_read_incomplete`, `last_index_run.diagnostics`, `last_index_run.diagnostic_count`, `last_index_run.diagnostics_truncated`, `last_index_run.reference_extraction_cap_hits`, `last_failed_or_partial_index_run`, `last_failed_or_partial_index_run.progress_persisted`, `last_failed_or_partial_index_run.recovery_hint`, `last_failed_or_partial_index_run.file_errors`。 | -| database maintenance | `sqlite_connection_policy` (`active_mode`、`open_mode`、`immutable_uri`、WAL checkpoint / fallback field)、`db_size_bytes`、`wal_size_bytes`、`db_pragma_settings` (`journal_mode`, `synchronous`, `wal_autocheckpoint`, `busy_timeout_ms`, `page_count`, `freelist_count`, `page_size`, `auto_vacuum`)、`prepared_command_cache` (`count`, `capacity`, `hit_count`, `miss_count`, `eviction_count`)、`maintenance_guidance`。query-only status は `pooling=false` と、`read_only` または `immutable_read_only_uri` の mode を返します。checkpoint 済み WAL database は immutable な private snapshot、non-empty WAL database は安定した private main/WAL snapshot から読むため source sidecar は変化しません。private snapshot の永続的な copy failure は WAL churn と誤報せず、temporary storage の容量・権限を案内する `query_only_snapshot_copy_failed` を返します。 | +| database maintenance | `sqlite_connection_policy` (`active_mode`、`open_mode`、`immutable_uri`、WAL checkpoint / fallback field)、`db_size_bytes`、`wal_size_bytes`、`db_pragma_settings` (`journal_mode`, `synchronous`, `wal_autocheckpoint`, `busy_timeout_ms`, `page_count`, `freelist_count`, `page_size`, `auto_vacuum`)、`prepared_command_cache` (`count`, `capacity`, `hit_count`, `miss_count`, `eviction_count`)、`maintenance_guidance`、`maintenance_guidance.fts_optimization` (`recommended`, `action`, `reason`, `threshold_writes`, `observed_writes`, `state`)。query-only status は `pooling=false` と、`read_only` または `immutable_read_only_uri` の mode を返します。checkpoint 済み WAL database は immutable な private snapshot、non-empty WAL database は安定した private main/WAL snapshot から読むため source sidecar は変化しません。private snapshot の永続的な copy failure は WAL churn と誤報せず、temporary storage の容量・権限を案内する `query_only_snapshot_copy_failed` を返します。 | | WAL checkpoint diagnostics | `read_only_fallback`、`wal_checkpoint_attempted`、`wal_checkpoint_succeeded`、`wal_checkpoint_skipped_reason`、`wal_checkpoint_failure_reason`、`wal_checkpoint_busy`、`wal_checkpoint_log_page_count`、`wal_checkpoint_checkpointed_page_count`、`wal_checkpoint_remaining_page_count`、`read_only_immutable_fallback`、`wal_stale_snapshot_risk`、`wal_stale_snapshot_reason`。 | | database size attribution | `database_size_attribution` は main DB / WAL / SHM の file byte を分離し、論理 page を table、index、freelist、internal / leaf / overflow page、payload、unused space、structural overhead、`unexplained_residual_bytes` に再照合します。object 名は伏字・切り詰めを適用して最大20件だけ返します。`available=false` と `unavailable_reason` がある場合は page attribution を計測できなかったことを示し、省略された object-byte field をゼロとして解釈してはいけません。 | | remediation fields | `degraded_root_cause`, `degraded_reason`, `recommended_action`, `alternative_action`, `readiness_degradations`, `repair_commands`。 | diff --git a/TESTING_GUIDE.md b/TESTING_GUIDE.md index 92a747212..699a6e84f 100644 --- a/TESTING_GUIDE.md +++ b/TESTING_GUIDE.md @@ -27,6 +27,7 @@ Use the full suite by default. Use targeted filters only while iterating locally - `FsCheck.Xunit` is reserved for property-based tests that assert universal invariants (never-throws contracts, idempotence, "output is parseable by downstream consumer") across randomly generated inputs. Use it to complement, not replace, the example-based `[Fact]` / `[Theory]` tests — pick FsCheck when the property is a universally quantified claim, and an example test when a specific concrete case is the contract. - Test parallelism: enabled by default across independent test classes. Tests that touch process-global state such as SQLite pool resets, environment variables, or current-directory overrides must use an explicit non-parallel collection. Console-sensitive classes share one non-parallel xUnit collection, so they remain serial with each other and do not run beside independent classes that may write request-id or global diagnostics to the process console. Use `ConsoleCapture` for ordinary capture and keep every direct `Console.Out` / `Console.Error` swap under `TestConsoleLock.Gate`. Snapshot and assert global console writers under the same gate so another test cannot replace a writer between capture completion and the assertion. That gate aliases the production `ConsoleStreamOwnership` gate so console synchronization and scoped production redirects cannot retain a test writer after its capture ends. - Markdown unused-audit coverage indexes one real Markdown fixture containing common backtick and tilde fence-language markers. Keep default suppression, `documentation_surface` totals, reason tags, and `--all` recovery in that shared fixture. +- FTS optimization recommendation coverage keeps the shared evaluator exact at one write below, at, and one write above the 25-write threshold. Status, explain, optimize dry-run, optimize execution, and vacuum maintenance guidance must expose the same `recommended`, `action`, `reason`, `threshold_writes`, `observed_writes`, and `state`; stale batches, known WAL-stale snapshots, forward-incompatible schema stamps, and unavailable legacy counters/page snapshots suppress the recommendation, query-only status performs no source writes, and execution uses the focused counter/page/forward-contract/freshness snapshot instead of full status scans. A hot-WAL fixture opened through an explicit `immutable=1` URI must prove that status, standalone optimize dry-run, and the `index --optimize` dry-run alias preserve the same stale recommendation. A WAL or freelist state of `unknown` cannot select the optimize command, and a successful optimize reports the reset counter afterward. - Full-scan CLI and MCP no-op coverage treats one repository-wide reusable-stat snapshot read and one folded-readiness verification as performance contracts. Keep assertions for one snapshot read, one stat lookup per candidate, one folded verification, and no content load for unchanged files when changing incremental indexing. - Reference-graph refresh coverage treats graph-neutral indexing as a performance contract across incremental full scan, scoped `--files` update, and MCP indexing. Keep zero-refresh assertions for new and modified source files without symbols/references, plus a single batched refresh assertion when existing or new graph identity rows change. A healthy incremental generation must restrict identity/candidate/recursion work to transaction-committed dirty files, old and new `(language, folded name)` dependencies, and their old/new reciprocal edges; retain C#/Python language-transition and unchanged-target parity with a subsequent full refresh, rolled-back file batches, cancellation/retry, orphan-candidate cleanup, and the controlled 4,100-of-4,100 broad-scope fallback. Fresh/rebuild runs, missing identity contracts, and dirty sets of at least 4,096 references covering at least 50% of the graph must keep the full-refresh path. Query-plan coverage must keep all four scoped update phases and all ten candidate inserts on dirty-table-driven reference primary-key seeks, keep C# instantiate grouping on lookup names plus `idx_symbols_name_folded`, and prove that a sub-4,096 dirty set does not count the whole reference table without an explicit diagnostic hook. - Index-generation completeness coverage uses a table-driven full, symbols-only, max-file-byte, max-symbol, and max-reference matrix, with extractor failure kept separate because it uses a mutable hook. Assert that index-command JSON, immediate status, and workspace health expose identical index/graph booleans and reason arrays where available; MCP cap coverage must match the persisted status snapshot as well. Remove the additive completeness metadata in healthy and capped fixtures to preserve legacy fallback coverage, and clear issue readiness before a scoped capped update to prove current omission evidence survives degraded prior metadata. Lowering or raising the file-size policy must reprocess unchanged files in CLI and MCP indexing so a prior `file_too_large` issue cannot be reused. Structured remediation must distinguish symbols-only / missing-graph causes from reference safety caps and must not label an incomplete index as fold-only. Human output must identify incomplete generations instead of printing a complete summary. @@ -965,6 +966,7 @@ dotnet test --filter "FullyQualifiedName~GitHelperTests" - `FsCheck.Xunit` はランダム生成入力に対する普遍的不変条件（never-throws、idempotence、"出力が downstream consumer で parse 可能" 等）を表明する property-based テスト専用です。例ベースの `[Fact]` / `[Theory]` を置き換えるのではなく補完するもので、普遍量化された主張なら FsCheck、特定の具体ケースが契約なら例ベースという形で使い分けてください。 - テスト並列実行: 独立したテストクラス間ではデフォルトで有効です。SQLite pool の解放、環境変数の変更、カレントディレクトリの上書きのような process-global 状態を触るテストは、明示的な non-parallel collection に入れてください。console-sensitive class は同じ non-parallel な xUnit collection を共有するため、互いに直列実行され、request-id や global diagnostics を process console へ書く可能性がある独立 class とも並列実行されません。通常の capture には `ConsoleCapture` を使い、`Console.Out` / `Console.Error` を直接差し替える場合は `TestConsoleLock.Gate` で保護してください。global console writer の snapshot 取得と assertion も同じ gate 内で行い、capture 完了から assertion までの間に別のテストが writer を差し替えないようにします。この gate は本番の `ConsoleStreamOwnership` gate と同一なので、console 同期処理や scoped redirect が capture 終了後も test writer を保持することを防ぎます。 - Markdown の unused audit coverage は、一般的な backtick / tilde fence の language marker を含む実 Markdown fixture を1回 index します。同じ fixture で既定抑制、`documentation_surface` totals、reason tag、`--all` による復元を維持してください。 +- FTS optimization recommendation coverage は、25 write threshold の1つ下、ちょうど、1つ上で shared evaluator の境界を固定します。status、explain、optimize dry-run、optimize execution、vacuum maintenance guidance は同じ `recommended`、`action`、`reason`、`threshold_writes`、`observed_writes`、`state` を公開し、stale batch、既知の WAL-stale snapshot、forward-incompatible な schema stamp、利用できない legacy counter / page snapshot は recommendation を抑止します。query-only status は source に書き込まず、execution は full status scan ではなく counter / page / forward-contract / freshness に限定した snapshot を使います。hot WAL fixture を明示的な `immutable=1` URI で開き、status、standalone optimize dry-run、`index --optimize` dry-run alias が同じ stale recommendation を保持することも証明します。WAL または freelist の state が `unknown` の場合は optimize command を選択せず、成功した optimize は reset 後の counter を返す必要があります。 - full-scan CLI と MCP の no-op coverage は、リポジトリ全体の reusable-stat snapshot read と folded-readiness verification がそれぞれ 1 回であることを performance contract とします。incremental indexing を変更するときは、snapshot read が 1 回、候補ごとの stat lookup が 1 回、folded verification が 1 回、unchanged file の content load が 0 回という assertion を維持してください。 - reference-graph refresh coverage は、incremental full scan、scoped `--files` update、MCP indexing を横断する graph-neutral indexing を performance contract とします。symbol/reference を持たない新規・変更 source file では refresh 0 回を維持し、既存または新規の graph identity 行が変化する場合は batch 全体で refresh 1 回を assertion してください。健全な incremental generation では identity / candidate / recursion 処理を transaction commit 済みの dirty file、旧・新の `(language, folded name)` 依存、旧・新の逆辺に限定します。C# / Python の言語遷移、未変更targetを参照する新規callerと後続full refreshのparity、rollback file batch、cancel後retry、孤立candidate cleanup、4,100件中4,100件をdirtyにする制御broad-scope fallbackを維持してください。fresh/rebuild、identity契約欠落、または4,096件以上かつgraphの50%以上を占めるdirty集合ではfull-refresh経路を維持します。query-plan coverageでは、scoped updateの4 phaseとcandidate INSERT 10本をdirty table起点のreference主キーseekに保ち、C# instantiate groupingをlookup nameと`idx_symbols_name_folded`起点にし、明示的なdiagnostic hookがない4,096件未満のdirty集合ではreference table全件COUNTを行わないことを検証してください。 - index generation の completeness coverage は full、symbols-only、max-file-byte、max-symbol、max-reference を table-driven matrix で検証し、mutable hook を使う extractor failure は別 case に保ちます。index command JSON、直後の status、workspace health で、利用可能な index/graph の boolean と reason array が完全に一致すること、MCP の cap case も persisted status snapshot と一致することを assertion してください。healthy / capped fixture から additive completeness metadata を削除して legacy fallback coverage を維持し、scoped capped update の前に issue readiness を clear して、prior metadata が degraded でも今回の omission evidence が失われないことを検証します。file-size policy を下げた場合も上げた場合も、CLI / MCP indexing は unchanged file を再処理し、以前の `file_too_large` issue を再利用してはいけません。structured remediation は symbols-only / missing-graph 原因と reference safety cap を区別し、incomplete index を fold-only と表示しない必要があります。human output は complete summary ではなく incomplete generation を明示する必要があります。 diff --git a/changelog.d/unreleased/4887.fixed.md b/changelog.d/unreleased/4887.fixed.md new file mode 100644 index 000000000..a7a928e57 --- /dev/null +++ b/changelog.d/unreleased/4887.fixed.md @@ -0,0 +1,24 @@ +--- +category: fixed +issues: + - 4887 +affected: + - src/CodeIndex/Database/MaintenanceGuidanceBuilder.cs + - src/CodeIndex/Database/DbContext.cs + - src/CodeIndex/Database/DbContext.ConnectionLifecycle.cs + - src/CodeIndex/Database/DbReader.Status.cs + - src/CodeIndex/Cli/IndexCommandRunner.Maintenance.cs + - src/CodeIndex/Cli/JsonOutputContracts.cs + - src/CodeIndex/Cli/QueryCommandRunner.StatusFields.cs + - README.md + - DEVELOPER_GUIDE.md + - TESTING_GUIDE.md +--- + +## English + +- **Status and optimize now share one FTS recommendation (#4887)** — `status`, `status --explain maintenance_guidance`, standalone and `index --optimize` dry-run previews, optimize execution, and vacuum maintenance output use one read-only evaluator and report the same exact threshold, observed-write, WAL/batch/forward-schema freshness, legacy-schema, and post-optimize state, including matching stale results for explicit immutable URIs with a hot WAL; trusted threshold hits select `cdidx optimize` only when higher-priority WAL and freelist states are both known healthy. + +## 日本語 + +- **status と optimize が1つの FTS recommendation を共有するようになりました (#4887)** — `status`、`status --explain maintenance_guidance`、standalone と `index --optimize` の dry-run preview、optimize execution、vacuum maintenance output は1つの read-only evaluator を使い、正確なしきい値、observed write、WAL / batch / forward-incompatible schema の freshness、legacy schema、optimize 後の state を同じ形式で報告し、hot WAL がある明示的な immutable URI でも同じ stale 結果を返します。信頼できる snapshot が threshold に達し、上位の WAL / freelist state が両方とも既知の healthy state である場合だけ maintenance command に `cdidx optimize` を選択します。 diff --git a/src/CodeIndex/Cli/IndexCommandRunner.Maintenance.cs b/src/CodeIndex/Cli/IndexCommandRunner.Maintenance.cs index 1cb2d5f35..5855ce4aa 100644 --- a/src/CodeIndex/Cli/IndexCommandRunner.Maintenance.cs +++ b/src/CodeIndex/Cli/IndexCommandRunner.Maintenance.cs @@ -89,7 +89,8 @@ internal static int RunOptimizeFts( options.DryRun, forceLogicalObjectSizeFallbackForTesting, options.ShowPaths, - diagnosticDbPath: options.DbPath); + diagnosticDbPath: options.DbPath, + queryOnlyDbPath: options.DryRun ? options.DbPath : null); } private static int RunOptimizeFtsForDb( @@ -100,12 +101,14 @@ private static int RunOptimizeFtsForDb( bool dryRun = false, bool forceLogicalObjectSizeFallbackForTesting = false, bool showPaths = false, - string? diagnosticDbPath = null) + string? diagnosticDbPath = null, + string? queryOnlyDbPath = null) { var errorDbPath = diagnosticDbPath ?? dbPath; if (dryRun) return RunOptimizeFtsPreviewForDb( dbPath, + queryOnlyDbPath ?? dbPath, json, jsonOptions, forceLogicalObjectSizeFallbackForTesting, @@ -139,16 +142,25 @@ private static int RunOptimizeFtsForDb( using var db = new DbContext(DbOpenIntent.Repair, dbPath); db.InitializeSchema(); var writer = new DbWriter(db); - var before = writer.GetFtsIncrementalWritesSinceOptimize(); + var beforeRecommendation = db.GetFtsOptimizationRecommendation(); + var before = checked((int)Math.Min(beforeRecommendation.ObservedWrites, int.MaxValue)); writer.OptimizeFts(); stopwatch.Stop(); - var after = writer.GetFtsIncrementalWritesSinceOptimize(); + var afterRecommendation = db.GetFtsOptimizationRecommendation(); + var after = checked((int)Math.Min(afterRecommendation.ObservedWrites, int.MaxValue)); if (json) { var jsonContext = CliJsonSerializerContextFactory.Create(jsonOptions); CommandOutputWriter.WriteLine(JsonSerializer.Serialize( - new OptimizeFtsJsonResult("success", dbPath, before, after, stopwatch.ElapsedMilliseconds), + new OptimizeFtsJsonResult( + "success", + dbPath, + before, + after, + beforeRecommendation, + afterRecommendation, + stopwatch.ElapsedMilliseconds), jsonContext.OptimizeFtsJsonResult)); } else @@ -157,6 +169,14 @@ private static int RunOptimizeFtsForDb( CommandOutputWriter.WriteLine(ConsoleUi.FormatSummaryLine("DB", dbPath, indent: " ")); CommandOutputWriter.WriteLine(ConsoleUi.FormatSummaryLine("Writes before", before.ToString("N0", System.Globalization.CultureInfo.InvariantCulture), indent: " ")); CommandOutputWriter.WriteLine(ConsoleUi.FormatSummaryLine("Writes after", after.ToString("N0", System.Globalization.CultureInfo.InvariantCulture), indent: " ")); + CommandOutputWriter.WriteLine(ConsoleUi.FormatSummaryLine( + "Recommended before", + FormatFtsOptimizationRecommendation(beforeRecommendation), + indent: " ")); + CommandOutputWriter.WriteLine(ConsoleUi.FormatSummaryLine( + "Recommended after", + FormatFtsOptimizationRecommendation(afterRecommendation), + indent: " ")); CommandOutputWriter.WriteLine(ConsoleUi.FormatSummaryLine("Elapsed", ConsoleUi.FormatDuration(stopwatch.Elapsed), indent: " ")); } @@ -195,6 +215,7 @@ private static int RunOptimizeFtsForDb( private static int RunOptimizeFtsPreviewForDb( string dbPath, + string queryOnlyDbPath, bool json, JsonSerializerOptions jsonOptions, bool forceLogicalObjectSizeFallbackForTesting, @@ -217,7 +238,7 @@ private static int RunOptimizeFtsPreviewForDb( try { var (lockState, lockHolder) = IndexLock.ProbeReadOnly(IndexLock.GetLockPath(dbPath)); - using var db = new DbContext(DbOpenIntent.QueryOnly, dbPath); + using var db = new DbContext(DbOpenIntent.QueryOnly, queryOnlyDbPath); if (!db.TryValidateIsCodeIndexDb(out _)) { return MaintenanceDatabaseErrorWriter.Write( @@ -236,8 +257,6 @@ private static int RunOptimizeFtsPreviewForDb( forceLogicalObjectSizeFallbackForTesting, out var objectSizesMeasurement, out var objectSizesUnavailableReason); - var writesSinceOptimize = ParseNonNegativeLong( - db.GetMetaString(DbWriter.FtsIncrementalWritesSinceOptimizeMetaKey)); var estimatedDurationMs = ParseNullableNonNegativeLong( db.GetMetaString(DbWriter.FtsLastOptimizeDurationMsMetaKey)); var coreTableSizeBytes = SumObjectSizes( @@ -252,7 +271,8 @@ private static int RunOptimizeFtsPreviewForDb( var ftsSizeBytes = SumObjectSizes( objectSizes, OptimizeFtsObjectNames); - var optimizationRecommended = writesSinceOptimize >= DbWriter.DefaultFtsOptimizeIncrementalWriteThreshold; + var ftsOptimization = status.MaintenanceGuidance.FtsOptimization; + var writesSinceOptimize = ftsOptimization.ObservedWrites; stopwatch.Stop(); var result = new OptimizeFtsPreviewJsonResult @@ -280,10 +300,9 @@ private static int RunOptimizeFtsPreviewForDb( LockState = lockState, LockHolderVerification = lockHolder?.Verification.ToString().ToLowerInvariant(), WouldAcquireExclusiveIndexLock = true, - OptimizationRecommended = optimizationRecommended, - RecommendationReason = optimizationRecommended - ? "incremental_write_threshold_reached" - : "incremental_write_threshold_not_reached", + OptimizationRecommended = ftsOptimization.Recommended, + RecommendationReason = ftsOptimization.Reason, + FtsOptimization = ftsOptimization, Readiness = new OptimizeFtsReadinessJsonResult { FoldReady = status.FoldReady, @@ -336,7 +355,10 @@ result.EstimatedDurationMs is { } durationEstimateMs ? ConsoleUi.FormatDuration(TimeSpan.FromMilliseconds(durationEstimateMs)) : "unavailable", indent: " ")); - CommandOutputWriter.WriteLine(ConsoleUi.FormatSummaryLine("Recommended", result.OptimizationRecommended ? "yes" : "not yet", indent: " ")); + CommandOutputWriter.WriteLine(ConsoleUi.FormatSummaryLine( + "Recommended", + FormatFtsOptimizationRecommendation(result.FtsOptimization), + indent: " ")); CommandOutputWriter.WriteLine(" Planned operations:"); foreach (var operation in result.PlannedOperations ?? []) CommandOutputWriter.WriteLine($" - {operation}"); @@ -467,15 +489,14 @@ private static long SumObjectSizes( return total; } - private static long ParseNonNegativeLong(string? value) - => long.TryParse( - value, - System.Globalization.NumberStyles.Integer, - System.Globalization.CultureInfo.InvariantCulture, - out var parsed) - && parsed > 0 - ? parsed - : 0; + private static string FormatFtsOptimizationRecommendation(FtsOptimizationRecommendation recommendation) + { + var culture = System.Globalization.CultureInfo.InvariantCulture; + return $"{(recommendation.Recommended ? "yes" : "no")} " + + $"(action={recommendation.Action}; reason={recommendation.Reason}; " + + $"threshold={recommendation.ThresholdWrites.ToString("N0", culture)}; " + + $"observed={recommendation.ObservedWrites.ToString("N0", culture)}; state={recommendation.State})"; + } private static long? ParseNullableNonNegativeLong(string? value) => long.TryParse( diff --git a/src/CodeIndex/Cli/IndexCommandRunner.cs b/src/CodeIndex/Cli/IndexCommandRunner.cs index 881a4840b..cecfb4e81 100644 --- a/src/CodeIndex/Cli/IndexCommandRunner.cs +++ b/src/CodeIndex/Cli/IndexCommandRunner.cs @@ -192,6 +192,7 @@ internal static int Run( if (validationExitCode != null) return validationExitCode.Value; + var requestedDbPath = dbPath; dbPath = DbPathResolver.NormalizeDbPath(dbPath); var resolvedDbPath = Path.GetFullPath(dbPath); var databaseExistedBeforeIndex = File.Exists(LongPath.EnsureWindowsPrefix(resolvedDbPath)); @@ -223,7 +224,8 @@ internal static int Run( jsonOptions, options.ProjectPath, options.DryRun, - showPaths: options.ShowPaths); + showPaths: options.ShowPaths, + queryOnlyDbPath: options.DryRun ? requestedDbPath : null); bool ignoreCase; string ignoreRuleRoot; diff --git a/src/CodeIndex/Cli/JsonOutputContracts.cs b/src/CodeIndex/Cli/JsonOutputContracts.cs index fc04905be..14b2641a0 100644 --- a/src/CodeIndex/Cli/JsonOutputContracts.cs +++ b/src/CodeIndex/Cli/JsonOutputContracts.cs @@ -33,6 +33,8 @@ internal sealed record OptimizeFtsJsonResult( [property: JsonPropertyName("db_path")] string DbPath, [property: JsonPropertyName("writes_since_optimize_before")] int WritesSinceOptimizeBefore, [property: JsonPropertyName("writes_since_optimize_after")] int WritesSinceOptimizeAfter, + [property: JsonPropertyName("fts_optimization_before")] FtsOptimizationRecommendation FtsOptimizationBefore, + [property: JsonPropertyName("fts_optimization_after")] FtsOptimizationRecommendation FtsOptimizationAfter, [property: JsonPropertyName("elapsed_ms")] long ElapsedMs, [property: JsonPropertyName("api_version")] string ApiVersion = JsonOutputContract.ApiVersion) : IVersionedJsonResult; @@ -64,6 +66,7 @@ internal sealed class OptimizeFtsPreviewJsonResult : IVersionedJsonResult public bool WouldAcquireExclusiveIndexLock { get; init; } public bool OptimizationRecommended { get; init; } public string RecommendationReason { get; init; } = string.Empty; + public FtsOptimizationRecommendation FtsOptimization { get; init; } = new(); public OptimizeFtsReadinessJsonResult? Readiness { get; init; } public List? PlannedOperations { get; init; } public bool SourceDatabaseUnchanged { get; init; } @@ -1340,6 +1343,7 @@ internal sealed record ValidateConfigJsonResult( [JsonSerializable(typeof(StatusDbPragmaSettings))] [JsonSerializable(typeof(StatusLastIndexRun))] [JsonSerializable(typeof(StatusMaintenanceGuidance))] +[JsonSerializable(typeof(FtsOptimizationRecommendation))] [JsonSerializable(typeof(StatusProcessMetrics))] [JsonSerializable(typeof(StatusRepairCommand))] [JsonSerializable(typeof(StatusUnknownExtensionGroup))] diff --git a/src/CodeIndex/Cli/QueryCommandRunner.StatusFields.cs b/src/CodeIndex/Cli/QueryCommandRunner.StatusFields.cs index e85bbe4d4..654b1bb61 100644 --- a/src/CodeIndex/Cli/QueryCommandRunner.StatusFields.cs +++ b/src/CodeIndex/Cli/QueryCommandRunner.StatusFields.cs @@ -180,6 +180,12 @@ private sealed record StatusFieldExplanation( "the object reports the active/open modes, pooling and immutable-URI choices, command timeout, cancellation requirement, and WAL snapshot-risk diagnostics used by this status read.", "read-only fallback or stale-snapshot risk fields identify when the preferred query-only connection path could not be used safely or may omit hot WAL content.", "Inspect the nested policy and WAL diagnostics; avoid explicit immutable mode for a hot WAL database, or rerun after the writer checkpoints and closes."), + new( + "maintenance_guidance", + "Database maintenance guidance", + "`fts_optimization` reports one shared `recommended`, `action`, `reason`, `threshold_writes`, `observed_writes`, and `state` decision used by status and optimize.", + "`state=stale` or `state=unavailable` suppresses an optimize recommendation because the persisted write counter or database-page snapshot is not trustworthy.", + "Wait for active indexing to finish and rerun `cdidx status --json`; run `cdidx optimize --dry-run --json` to inspect the same decision before mutation."), new( "unknown_extension_file_count", "Unknown extension inventory", diff --git a/src/CodeIndex/Database/DbContext.ConnectionLifecycle.cs b/src/CodeIndex/Database/DbContext.ConnectionLifecycle.cs index 30db62725..0a85bc969 100644 --- a/src/CodeIndex/Database/DbContext.ConnectionLifecycle.cs +++ b/src/CodeIndex/Database/DbContext.ConnectionLifecycle.cs @@ -419,6 +419,40 @@ private void ConfigureAutoVacuumForEmptyDatabase() public VacuumResult RunIncrementalVacuum(bool dryRun = false) => RunIncrementalVacuum(dryRun, CancellationToken.None); + internal FtsOptimizationRecommendation GetFtsOptimizationRecommendation() + { + using var transaction = _connection.BeginTransaction(deferred: true); + var metadata = GetMetaStrings( + [ + DbWriter.FtsIncrementalWritesSinceOptimizeMetaKey, + BatchInProgressMetaKey, + ]); + var incrementalWrites = long.TryParse( + metadata[DbWriter.FtsIncrementalWritesSinceOptimizeMetaKey], + NumberStyles.Integer, + CultureInfo.InvariantCulture, + out var parsedWrites) + ? parsedWrites + : (long?)null; + var batchInProgress = bool.TryParse( + metadata[BatchInProgressMetaKey], + out var parsedBatchInProgress) + && parsedBatchInProgress; + var userVersion = checked((int)ReadPragmaLong("user_version")); + var indexNewerThanReader = DbReader.DetectNewerThanReaderContracts( + _connection, + userVersion).Newer; + var recommendation = FtsOptimizationRecommendationEvaluator.Evaluate( + new FtsOptimizationMetrics( + incrementalWrites, + ReadPragmaLong("page_count"), + SnapshotCurrent: !batchInProgress + && !indexNewerThanReader + && !WalStaleSnapshotRisk)); + transaction.Commit(); + return recommendation; + } + public VacuumResult RunIncrementalVacuum(bool dryRun, CancellationToken cancellationToken) { if (_isReadOnly && !dryRun) @@ -456,13 +490,15 @@ public VacuumResult RunIncrementalVacuum(bool dryRun, CancellationToken cancella var bytesReclaimed = pagesReclaimed * after.PageSize; var estimatedPagesReclaimable = Math.Max(0, before.FreelistCount); var estimatedBytesReclaimable = estimatedPagesReclaimable * before.PageSize; + var ftsOptimization = GetFtsOptimizationRecommendation(); var guidance = MaintenanceGuidanceBuilder.Build(new MaintenanceMetrics( after.PageCount, after.FreelistCount, after.PageSize, after.WalSizeBytes, after.DbSizeBytes, - after.AutoVacuumMode)); + after.AutoVacuumMode), + ftsOptimization: ftsOptimization); return new VacuumResult( Status: dryRun ? "dry_run" : "ok", DryRun: dryRun, diff --git a/src/CodeIndex/Database/DbContext.cs b/src/CodeIndex/Database/DbContext.cs index 17f8b7e03..f65e6a38e 100644 --- a/src/CodeIndex/Database/DbContext.cs +++ b/src/CodeIndex/Database/DbContext.cs @@ -361,6 +361,8 @@ internal static Func? ReadMigrationTransact public bool ReadOnlyImmutableFallback => _readOnlyImmutableFallback; internal bool ImmutableReadOnly => _immutableReadOnly; internal bool ImmutableReadOnlyWalRisk => _immutableReadOnlyWalRisk; + internal bool WalStaleSnapshotRisk => + (_immutableReadOnlyWalRisk || _readOnlyImmutableFallback) && !_walCheckpointSucceeded; internal bool ConnectionPooling => _connectionPooling; internal bool QueryOnlySnapshotRequiresRefresh => _queryOnlySnapshotRequiresRefresh; internal DbConnectionFactory.QueryOnlySnapshotSourceState? QueryOnlySnapshotSourceState diff --git a/src/CodeIndex/Database/DbReader.Status.cs b/src/CodeIndex/Database/DbReader.Status.cs index 75a1fb08b..4f1d3863f 100644 --- a/src/CodeIndex/Database/DbReader.Status.cs +++ b/src/CodeIndex/Database/DbReader.Status.cs @@ -150,6 +150,17 @@ GROUP BY COALESCE(f.lang, 'unknown'), s.kind var preparedCommandCache = GetPreparedCommandCacheStatus(); var dbSizeBytes = TryGetDatabaseFileSize(); var walSizeBytes = TryGetWalFileSize(); + var ftsIncrementalWritesSinceOptimize = ParseMetaLong( + TryGetMetaStringInternal(DbWriter.FtsIncrementalWritesSinceOptimizeMetaKey)); + var maintenanceSnapshotCurrent = ParseMetaBool( + TryGetMetaStringInternal(DbContext.BatchInProgressMetaKey)) != true + && !_indexNewerThanReader + && !WalStaleSnapshotRisk; + var ftsOptimization = FtsOptimizationRecommendationEvaluator.Evaluate( + new FtsOptimizationMetrics( + ftsIncrementalWritesSinceOptimize, + dbPragmaSettings.PageCount, + maintenanceSnapshotCurrent)); var databaseSizeAttribution = includeDatabaseSizeAttribution ? ReadDatabaseSizeAttribution( dbPragmaSettings, @@ -169,7 +180,8 @@ GROUP BY COALESCE(f.lang, 'unknown'), s.kind dbPragmaSettings.PageSize, walSizeBytes, dbSizeBytes, - dbPragmaSettings.AutoVacuum)); + dbPragmaSettings.AutoVacuum), + ftsOptimization: ftsOptimization); var lastIndexRun = GetLastIndexRun(); var referenceExtractionCapHits = GetReferenceExtractionCapHits(); var persistedReadiness = GetPersistedIndexGenerationReadiness( diff --git a/src/CodeIndex/Database/DbReader.cs b/src/CodeIndex/Database/DbReader.cs index 668894317..92df067cf 100644 --- a/src/CodeIndex/Database/DbReader.cs +++ b/src/CodeIndex/Database/DbReader.cs @@ -752,7 +752,7 @@ private static void SetParameter(SqliteCommand cmd, string name, object? value) cmd.Parameters[name].Value = value ?? DBNull.Value; } - private static (bool Newer, string? Reason) DetectNewerThanReaderContracts(SqliteConnection conn, int userVersion) + internal static (bool Newer, string? Reason) DetectNewerThanReaderContracts(SqliteConnection conn, int userVersion) { var newerContracts = new List(); // Numeric contract stamps. Each pair maps the persisted meta key to the binary's diff --git a/src/CodeIndex/Database/MaintenanceGuidanceBuilder.cs b/src/CodeIndex/Database/MaintenanceGuidanceBuilder.cs index 360f6789e..9287cf5a5 100644 --- a/src/CodeIndex/Database/MaintenanceGuidanceBuilder.cs +++ b/src/CodeIndex/Database/MaintenanceGuidanceBuilder.cs @@ -28,6 +28,8 @@ public sealed class StatusMaintenanceGuidance [JsonPropertyName("auto_vacuum_mode_name")] [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public string? AutoVacuumModeName { get; set; } + [JsonPropertyName("fts_optimization")] + public FtsOptimizationRecommendation FtsOptimization { get; set; } = new(); [JsonPropertyName("recommended_command")] public string RecommendedCommand { get; set; } = "none"; [JsonPropertyName("post_maintenance_follow_up")] @@ -43,6 +45,108 @@ internal readonly record struct MaintenanceMetrics( long? DbSizeBytes, long? AutoVacuumMode); +public sealed class FtsOptimizationRecommendation +{ + [JsonPropertyName("recommended")] + public bool Recommended { get; init; } + [JsonPropertyName("action")] + public string Action { get; init; } = FtsOptimizationRecommendationEvaluator.NoAction; + [JsonPropertyName("reason")] + public string Reason { get; init; } = FtsOptimizationRecommendationEvaluator.IncrementalWriteCountUnavailableReason; + [JsonPropertyName("threshold_writes")] + public int ThresholdWrites { get; init; } = DbWriter.DefaultFtsOptimizeIncrementalWriteThreshold; + [JsonPropertyName("observed_writes")] + public long ObservedWrites { get; init; } + [JsonPropertyName("state")] + public string State { get; init; } = FtsOptimizationRecommendationEvaluator.UnavailableState; +} + +internal readonly record struct FtsOptimizationMetrics( + long? IncrementalWritesSinceOptimize, + long? PageCount, + bool SnapshotCurrent); + +internal static class FtsOptimizationRecommendationEvaluator +{ + public const string OptimizeAction = "optimize"; + public const string NoAction = "none"; + public const string IncrementalWriteThresholdReachedReason = "incremental_write_threshold_reached"; + public const string IncrementalWriteThresholdNotReachedReason = "incremental_write_threshold_not_reached"; + public const string IncrementalWriteCountUnavailableReason = "incremental_write_count_unavailable"; + public const string PageCountUnavailableReason = "page_count_unavailable"; + public const string MaintenanceSnapshotStaleReason = "maintenance_snapshot_stale"; + public const string CurrentState = "current"; + public const string StaleState = "stale"; + public const string UnavailableState = "unavailable"; + + public static FtsOptimizationRecommendation Evaluate( + FtsOptimizationMetrics metrics, + int thresholdWrites = DbWriter.DefaultFtsOptimizeIncrementalWriteThreshold) + { + if (thresholdWrites <= 0) + throw new ArgumentOutOfRangeException(nameof(thresholdWrites)); + + var observedWrites = metrics.IncrementalWritesSinceOptimize is >= 0 + ? metrics.IncrementalWritesSinceOptimize.Value + : 0; + + if (!metrics.SnapshotCurrent) + return Build( + recommended: false, + NoAction, + MaintenanceSnapshotStaleReason, + thresholdWrites, + observedWrites, + StaleState); + + if (metrics.PageCount is null or <= 0) + return Build( + recommended: false, + NoAction, + PageCountUnavailableReason, + thresholdWrites, + observedWrites, + UnavailableState); + + if (metrics.IncrementalWritesSinceOptimize is null or < 0) + return Build( + recommended: false, + NoAction, + IncrementalWriteCountUnavailableReason, + thresholdWrites, + observedWrites, + UnavailableState); + + var recommended = observedWrites >= thresholdWrites; + return Build( + recommended, + recommended ? OptimizeAction : NoAction, + recommended + ? IncrementalWriteThresholdReachedReason + : IncrementalWriteThresholdNotReachedReason, + thresholdWrites, + observedWrites, + CurrentState); + } + + private static FtsOptimizationRecommendation Build( + bool recommended, + string action, + string reason, + int thresholdWrites, + long observedWrites, + string state) => + new() + { + Recommended = recommended, + Action = action, + Reason = reason, + ThresholdWrites = thresholdWrites, + ObservedWrites = observedWrites, + State = state, + }; +} + internal static class MaintenanceGuidanceBuilder { public const string WalWarnBytesEnvironmentVariable = "CDIDX_MAINTENANCE_WAL_WARN_BYTES"; @@ -54,7 +158,9 @@ internal static class MaintenanceGuidanceBuilder public static StatusMaintenanceGuidance Build( MaintenanceMetrics metrics, string vacuumCommand = "cdidx vacuum --db ", - string checkpointCommand = "sqlite3 \"PRAGMA wal_checkpoint(TRUNCATE);\"") + string checkpointCommand = "sqlite3 \"PRAGMA wal_checkpoint(TRUNCATE);\"", + string optimizeCommand = "cdidx optimize --db ", + FtsOptimizationRecommendation? ftsOptimization = null) { var walThresholdBytes = ReadPositiveLongEnvironment(WalWarnBytesEnvironmentVariable, DefaultWalWarnBytes); var freelistThresholdRatio = ReadRatioEnvironment(FreelistWarnRatioEnvironmentVariable, DefaultFreelistWarnRatio); @@ -68,12 +174,21 @@ public static StatusMaintenanceGuidance Build( ? "vacuum_recommended" : "ok" : "unknown"; + ftsOptimization ??= FtsOptimizationRecommendationEvaluator.Evaluate( + new FtsOptimizationMetrics( + IncrementalWritesSinceOptimize: null, + PageCount: metrics.PageCount, + SnapshotCurrent: true)); var recommendedCommand = freelistState == "vacuum_recommended" ? vacuumCommand : walState == "checkpoint_recommended" ? checkpointCommand - : "none"; + : walState == "ok" + && freelistState == "ok" + && ftsOptimization.Recommended + ? optimizeCommand + : "none"; return new StatusMaintenanceGuidance { @@ -86,6 +201,7 @@ public static StatusMaintenanceGuidance Build( EstimatedBytesReclaimable = estimatedBytes, AutoVacuumMode = metrics.AutoVacuumMode, AutoVacuumModeName = FormatAutoVacuumMode(metrics.AutoVacuumMode), + FtsOptimization = ftsOptimization, RecommendedCommand = recommendedCommand, PostMaintenanceFollowUp = BuildFollowUp(walState, freelistState, checkpointCommand), }; diff --git a/tests/CodeIndex.Tests/DatabaseTests.cs b/tests/CodeIndex.Tests/DatabaseTests.cs index cb5a93429..134ecd3d1 100644 --- a/tests/CodeIndex.Tests/DatabaseTests.cs +++ b/tests/CodeIndex.Tests/DatabaseTests.cs @@ -3984,6 +3984,161 @@ public void MaintenanceGuidance_OverflowEstimateReportsUnknown_Issue3964() Assert.Null(guidance.EstimatedBytesReclaimable); } + [Fact] + public void MaintenanceGuidance_FtsThresholdRecommendsOptimizeWhenHigherPriorityMaintenanceIsHealthy_Issue4887() + { + var ftsOptimization = FtsOptimizationRecommendationEvaluator.Evaluate( + new FtsOptimizationMetrics( + DbWriter.DefaultFtsOptimizeIncrementalWriteThreshold, + PageCount: 100, + SnapshotCurrent: true)); + var guidance = MaintenanceGuidanceBuilder.Build( + new MaintenanceMetrics( + PageCount: 100, + FreelistCount: 0, + PageSize: 4096, + WalSizeBytes: 0, + DbSizeBytes: 409_600, + AutoVacuumMode: 2), + ftsOptimization: ftsOptimization); + + Assert.True(guidance.FtsOptimization.Recommended); + Assert.Equal("cdidx optimize --db ", guidance.RecommendedCommand); + } + + [Theory] + [InlineData(null, 0L, "unknown", "ok")] + [InlineData(0L, null, "ok", "unknown")] + public void MaintenanceGuidance_FtsThresholdDoesNotRecommendWhenHigherPriorityStateIsUnknown_Issue4887( + long? walSizeBytes, + long? freelistCount, + string expectedWalState, + string expectedFreelistState) + { + var ftsOptimization = FtsOptimizationRecommendationEvaluator.Evaluate( + new FtsOptimizationMetrics( + DbWriter.DefaultFtsOptimizeIncrementalWriteThreshold, + PageCount: 100, + SnapshotCurrent: true)); + var guidance = MaintenanceGuidanceBuilder.Build( + new MaintenanceMetrics( + PageCount: 100, + FreelistCount: freelistCount, + PageSize: 4096, + WalSizeBytes: walSizeBytes, + DbSizeBytes: 409_600, + AutoVacuumMode: 2), + ftsOptimization: ftsOptimization); + + Assert.True(guidance.FtsOptimization.Recommended); + Assert.Equal(expectedWalState, guidance.WalState); + Assert.Equal(expectedFreelistState, guidance.FreelistState); + Assert.Equal("none", guidance.RecommendedCommand); + } + + [Theory] + [InlineData(24, false, "none", "incremental_write_threshold_not_reached")] + [InlineData(25, true, "optimize", "incremental_write_threshold_reached")] + [InlineData(26, true, "optimize", "incremental_write_threshold_reached")] + public void FtsOptimizationRecommendation_UsesExactWriteThreshold_Issue4887( + long observedWrites, + bool expectedRecommended, + string expectedAction, + string expectedReason) + { + var recommendation = FtsOptimizationRecommendationEvaluator.Evaluate( + new FtsOptimizationMetrics( + observedWrites, + PageCount: 100, + SnapshotCurrent: true)); + + Assert.Equal(expectedRecommended, recommendation.Recommended); + Assert.Equal(expectedAction, recommendation.Action); + Assert.Equal(expectedReason, recommendation.Reason); + Assert.Equal(DbWriter.DefaultFtsOptimizeIncrementalWriteThreshold, recommendation.ThresholdWrites); + Assert.Equal(observedWrites, recommendation.ObservedWrites); + Assert.Equal("current", recommendation.State); + } + + [Theory] + [InlineData(null, 100L, true, "incremental_write_count_unavailable", "unavailable")] + [InlineData(57L, null, true, "page_count_unavailable", "unavailable")] + [InlineData(57L, 100L, false, "maintenance_snapshot_stale", "stale")] + public void FtsOptimizationRecommendation_UntrustedStateIsNeverRecommended_Issue4887( + long? observedWrites, + long? pageCount, + bool snapshotCurrent, + string expectedReason, + string expectedState) + { + var recommendation = FtsOptimizationRecommendationEvaluator.Evaluate( + new FtsOptimizationMetrics(observedWrites, pageCount, snapshotCurrent)); + + Assert.False(recommendation.Recommended); + Assert.Equal("none", recommendation.Action); + Assert.Equal(expectedReason, recommendation.Reason); + Assert.Equal(DbWriter.DefaultFtsOptimizeIncrementalWriteThreshold, recommendation.ThresholdWrites); + Assert.Equal(Math.Max(0, observedWrites ?? 0), recommendation.ObservedWrites); + Assert.Equal(expectedState, recommendation.State); + } + + [Fact] + public void StatusFtsOptimization_ReadOnlySnapshotUsesCounterAndRejectsStaleBatch_Issue4887() + { + _writer.SetMeta( + DbWriter.FtsIncrementalWritesSinceOptimizeMetaKey, + DbWriter.DefaultFtsOptimizeIncrementalWriteThreshold.ToString(CultureInfo.InvariantCulture)); + + var dbBytesBeforeStatus = ReadDatabaseBytesWithSqliteSharing(_dbPath); + using (var readOnlyDb = new DbContext(DbOpenIntent.QueryOnly, _dbPath)) + { + var currentGuidance = new DbReader(readOnlyDb).GetStatus().MaintenanceGuidance; + var current = currentGuidance.FtsOptimization; + + Assert.True(current.Recommended); + Assert.Equal("optimize", current.Action); + Assert.Equal("incremental_write_threshold_reached", current.Reason); + Assert.Equal(DbWriter.DefaultFtsOptimizeIncrementalWriteThreshold, current.ThresholdWrites); + Assert.Equal(DbWriter.DefaultFtsOptimizeIncrementalWriteThreshold, current.ObservedWrites); + Assert.Equal("current", current.State); + Assert.Equal("cdidx optimize --db ", currentGuidance.RecommendedCommand); + } + Assert.Equal(dbBytesBeforeStatus, ReadDatabaseBytesWithSqliteSharing(_dbPath)); + + _writer.MarkBatchInProgress(); + try + { + var dbBytesBeforeStaleStatus = ReadDatabaseBytesWithSqliteSharing(_dbPath); + using var staleDb = new DbContext(DbOpenIntent.QueryOnly, _dbPath); + var staleGuidance = new DbReader(staleDb).GetStatus().MaintenanceGuidance; + var stale = staleGuidance.FtsOptimization; + Assert.False(stale.Recommended); + Assert.Equal("none", stale.Action); + Assert.Equal("maintenance_snapshot_stale", stale.Reason); + Assert.Equal(DbWriter.DefaultFtsOptimizeIncrementalWriteThreshold, stale.ThresholdWrites); + Assert.Equal(DbWriter.DefaultFtsOptimizeIncrementalWriteThreshold, stale.ObservedWrites); + Assert.Equal("stale", stale.State); + Assert.Equal("none", staleGuidance.RecommendedCommand); + Assert.Equal(dbBytesBeforeStaleStatus, ReadDatabaseBytesWithSqliteSharing(_dbPath)); + } + finally + { + _writer.ClearBatchInProgress(); + } + } + + private static byte[] ReadDatabaseBytesWithSqliteSharing(string path) + { + using var stream = new FileStream( + path, + FileMode.Open, + FileAccess.Read, + FileShare.ReadWrite | FileShare.Delete); + using var buffer = new MemoryStream(); + stream.CopyTo(buffer); + return buffer.ToArray(); + } + private static int GetTransactionDepth(DbWriter writer) { var field = typeof(DbWriter).GetField("_transactionDepth", System.Reflection.BindingFlags.Instance | System.Reflection.BindingFlags.NonPublic) diff --git a/tests/CodeIndex.Tests/DbReaderTests.cs b/tests/CodeIndex.Tests/DbReaderTests.cs index e66fea0f1..26dc79581 100644 --- a/tests/CodeIndex.Tests/DbReaderTests.cs +++ b/tests/CodeIndex.Tests/DbReaderTests.cs @@ -6092,9 +6092,12 @@ public void GetStatus_FlagsIndexNewerThanReaderWhenCSharpMetadataVersionExceedsC // that names the offending contract so `status` can WARN loudly instead of pretending // the DB is merely "degraded due to stale stamp". // Issue #1515: stored > current の数値 contract を「未来 DB」として明示する。 + _writer.SetMeta( + DbWriter.FtsIncrementalWritesSinceOptimizeMetaKey, + DbWriter.DefaultFtsOptimizeIncrementalWriteThreshold.ToString(CultureInfo.InvariantCulture)); _writer.SetMeta( DbContext.GetMetadataTargetVersionMetaKey("csharp"), - (DbContext.MetadataTargetVersion + 1).ToString(System.Globalization.CultureInfo.InvariantCulture)); + (DbContext.MetadataTargetVersion + 1).ToString(CultureInfo.InvariantCulture)); _writer.WriteCdidxWriterVersion("9.99.0"); var freshReader = new DbReader(_db.Connection); @@ -6104,6 +6107,20 @@ public void GetStatus_FlagsIndexNewerThanReaderWhenCSharpMetadataVersionExceedsC Assert.NotNull(status.IndexNewerThanReaderReason); Assert.Contains("metadata_target_version_csharp", status.IndexNewerThanReaderReason); Assert.Equal("9.99.0", status.IndexWriterVersion); + var statusRecommendation = status.MaintenanceGuidance.FtsOptimization; + Assert.False(statusRecommendation.Recommended); + Assert.Equal( + FtsOptimizationRecommendationEvaluator.MaintenanceSnapshotStaleReason, + statusRecommendation.Reason); + + using var focusedDb = new DbContext(DbOpenIntent.QueryOnly, _dbPath); + var focusedRecommendation = focusedDb.GetFtsOptimizationRecommendation(); + Assert.Equal(statusRecommendation.Recommended, focusedRecommendation.Recommended); + Assert.Equal(statusRecommendation.Action, focusedRecommendation.Action); + Assert.Equal(statusRecommendation.Reason, focusedRecommendation.Reason); + Assert.Equal(statusRecommendation.ThresholdWrites, focusedRecommendation.ThresholdWrites); + Assert.Equal(statusRecommendation.ObservedWrites, focusedRecommendation.ObservedWrites); + Assert.Equal(statusRecommendation.State, focusedRecommendation.State); } [Theory] @@ -6138,6 +6155,9 @@ public void GetStatus_FlagsIndexNewerThanReaderWhenUserVersionCarriesUnknownRead // mask therefore indicate the DB was written by a newer binary, even if every numeric // meta contract still equals the older binary's compiled max. // Issue #1515: CurrentSchemaVersion マスク外の bit も「未来 DB」シグナルにする。 + _writer.SetMeta( + DbWriter.FtsIncrementalWritesSinceOptimizeMetaKey, + DbWriter.DefaultFtsOptimizeIncrementalWriteThreshold.ToString(CultureInfo.InvariantCulture)); var unknownBit = (DbContext.CurrentSchemaVersion + 1) | DbContext.CurrentSchemaVersion; using (var cmd = _db.Connection.CreateCommand()) { @@ -6151,6 +6171,29 @@ public void GetStatus_FlagsIndexNewerThanReaderWhenUserVersionCarriesUnknownRead Assert.True(status.IndexNewerThanReader); Assert.NotNull(status.IndexNewerThanReaderReason); Assert.Contains("user_version_bits", status.IndexNewerThanReaderReason); + Assert.False(status.MaintenanceGuidance.FtsOptimization.Recommended); + Assert.Equal( + FtsOptimizationRecommendationEvaluator.NoAction, + status.MaintenanceGuidance.FtsOptimization.Action); + Assert.Equal( + FtsOptimizationRecommendationEvaluator.MaintenanceSnapshotStaleReason, + status.MaintenanceGuidance.FtsOptimization.Reason); + Assert.Equal( + FtsOptimizationRecommendationEvaluator.StaleState, + status.MaintenanceGuidance.FtsOptimization.State); + Assert.Equal( + DbWriter.DefaultFtsOptimizeIncrementalWriteThreshold, + status.MaintenanceGuidance.FtsOptimization.ObservedWrites); + + using var focusedDb = new DbContext(DbOpenIntent.QueryOnly, _dbPath); + var focusedRecommendation = focusedDb.GetFtsOptimizationRecommendation(); + Assert.False(focusedRecommendation.Recommended); + Assert.Equal( + FtsOptimizationRecommendationEvaluator.MaintenanceSnapshotStaleReason, + focusedRecommendation.Reason); + Assert.Equal( + FtsOptimizationRecommendationEvaluator.StaleState, + focusedRecommendation.State); } [Fact] diff --git a/tests/CodeIndex.Tests/DocumentationStatusContractTests.cs b/tests/CodeIndex.Tests/DocumentationStatusContractTests.cs index 773be1a0f..b3ad9013b 100644 --- a/tests/CodeIndex.Tests/DocumentationStatusContractTests.cs +++ b/tests/CodeIndex.Tests/DocumentationStatusContractTests.cs @@ -52,6 +52,9 @@ public class DocumentationStatusContractTests "db_pragma_settings", "prepared_command_cache", "maintenance_guidance", + "fts_optimization", + "threshold_writes", + "observed_writes", "process", "last_index_run", "last_workspace_freshened_at", diff --git a/tests/CodeIndex.Tests/IndexCommandRunnerTests.cs b/tests/CodeIndex.Tests/IndexCommandRunnerTests.cs index fe890a23a..7feb53bbe 100644 --- a/tests/CodeIndex.Tests/IndexCommandRunnerTests.cs +++ b/tests/CodeIndex.Tests/IndexCommandRunnerTests.cs @@ -5616,8 +5616,8 @@ public void RunOptimizeFts_DryRunPreviewsWithoutWritingThenOptimizeMutates_Issue Content = "public sealed class Preview; // 日本語", }, ]); - writer.RecordFtsIncrementalWrite(); - writer.RecordFtsIncrementalWrite(); + for (var i = 0; i < DbWriter.DefaultFtsOptimizeIncrementalWriteThreshold; i++) + writer.RecordFtsIncrementalWrite(); } SqliteConnection.ClearAllPools(); @@ -5648,8 +5648,23 @@ public void RunOptimizeFts_DryRunPreviewsWithoutWritingThenOptimizeMutates_Issue Assert.Equal("dry_run", previewJson.GetProperty("status").GetString()); Assert.True(previewJson.GetProperty("dry_run").GetBoolean()); Assert.Equal(dbPath, previewJson.GetProperty("db_path").GetString()); - Assert.Equal(2, previewJson.GetProperty("writes_since_optimize_before").GetInt32()); - Assert.Equal(2, previewJson.GetProperty("writes_since_optimize_after").GetInt32()); + Assert.Equal( + DbWriter.DefaultFtsOptimizeIncrementalWriteThreshold, + previewJson.GetProperty("writes_since_optimize_before").GetInt32()); + Assert.Equal( + DbWriter.DefaultFtsOptimizeIncrementalWriteThreshold, + previewJson.GetProperty("writes_since_optimize_after").GetInt32()); + var previewRecommendation = previewJson.GetProperty("fts_optimization"); + Assert.True(previewRecommendation.GetProperty("recommended").GetBoolean()); + Assert.Equal("optimize", previewRecommendation.GetProperty("action").GetString()); + Assert.Equal("incremental_write_threshold_reached", previewRecommendation.GetProperty("reason").GetString()); + Assert.Equal( + DbWriter.DefaultFtsOptimizeIncrementalWriteThreshold, + previewRecommendation.GetProperty("threshold_writes").GetInt32()); + Assert.Equal( + DbWriter.DefaultFtsOptimizeIncrementalWriteThreshold, + previewRecommendation.GetProperty("observed_writes").GetInt64()); + Assert.Equal("current", previewRecommendation.GetProperty("state").GetString()); Assert.True(previewJson.GetProperty("db_size_bytes").GetInt64() > 0); Assert.True(previewJson.GetProperty("page_count").GetInt64() > 0); Assert.True(previewJson.GetProperty("object_sizes_available").GetBoolean()); @@ -5672,6 +5687,17 @@ public void RunOptimizeFts_DryRunPreviewsWithoutWritingThenOptimizeMutates_Issue Assert.False(File.Exists(IndexLock.GetLockPath(dbPath))); Assert.False(File.Exists(IndexLock.GetInfoPath(IndexLock.GetLockPath(dbPath)))); + using (var statusDb = new DbContext(DbOpenIntent.QueryOnly, dbPath)) + { + var statusRecommendation = new DbReader(statusDb).GetStatus().MaintenanceGuidance.FtsOptimization; + Assert.Equal(previewRecommendation.GetProperty("recommended").GetBoolean(), statusRecommendation.Recommended); + Assert.Equal(previewRecommendation.GetProperty("action").GetString(), statusRecommendation.Action); + Assert.Equal(previewRecommendation.GetProperty("reason").GetString(), statusRecommendation.Reason); + Assert.Equal(previewRecommendation.GetProperty("threshold_writes").GetInt32(), statusRecommendation.ThresholdWrites); + Assert.Equal(previewRecommendation.GetProperty("observed_writes").GetInt64(), statusRecommendation.ObservedWrites); + Assert.Equal(previewRecommendation.GetProperty("state").GetString(), statusRecommendation.State); + } + int humanPreviewExitCode; string humanPreviewOutput; lock (TestConsoleLock.Gate) @@ -5697,6 +5723,11 @@ public void RunOptimizeFts_DryRunPreviewsWithoutWritingThenOptimizeMutates_Issue Assert.Contains("Core size", humanPreviewOutput, StringComparison.Ordinal); Assert.Contains("Readiness", humanPreviewOutput, StringComparison.Ordinal); Assert.Contains("Est. duration", humanPreviewOutput, StringComparison.Ordinal); + Assert.Contains("action=optimize", humanPreviewOutput, StringComparison.Ordinal); + Assert.Contains("reason=incremental_write_threshold_reached", humanPreviewOutput, StringComparison.Ordinal); + Assert.Contains("threshold=25", humanPreviewOutput, StringComparison.Ordinal); + Assert.Contains("observed=25", humanPreviewOutput, StringComparison.Ordinal); + Assert.Contains("state=current", humanPreviewOutput, StringComparison.Ordinal); Assert.Contains("Planned operations", humanPreviewOutput, StringComparison.Ordinal); Assert.Contains("initialize_or_migrate_schema", humanPreviewOutput, StringComparison.Ordinal); Assert.Equal(dbBytesBeforePreview, File.ReadAllBytes(dbPath)); @@ -5722,11 +5753,32 @@ public void RunOptimizeFts_DryRunPreviewsWithoutWritingThenOptimizeMutates_Issue Assert.Equal(CommandExitCodes.Success, exitCode); Assert.Equal("success", json.GetProperty("status").GetString()); - Assert.Equal(2, json.GetProperty("writes_since_optimize_before").GetInt32()); + Assert.Equal( + DbWriter.DefaultFtsOptimizeIncrementalWriteThreshold, + json.GetProperty("writes_since_optimize_before").GetInt32()); Assert.Equal(0, json.GetProperty("writes_since_optimize_after").GetInt32()); - Assert.Equal(6, json.EnumerateObject().Count()); + Assert.Equal(8, json.EnumerateObject().Count()); Assert.False(json.TryGetProperty("dry_run", out _)); Assert.False(json.TryGetProperty("planned_operations", out _)); + var beforeRecommendation = json.GetProperty("fts_optimization_before"); + Assert.True(beforeRecommendation.GetProperty("recommended").GetBoolean()); + Assert.Equal("optimize", beforeRecommendation.GetProperty("action").GetString()); + Assert.Equal("incremental_write_threshold_reached", beforeRecommendation.GetProperty("reason").GetString()); + Assert.Equal( + DbWriter.DefaultFtsOptimizeIncrementalWriteThreshold, + beforeRecommendation.GetProperty("threshold_writes").GetInt32()); + Assert.Equal( + DbWriter.DefaultFtsOptimizeIncrementalWriteThreshold, + beforeRecommendation.GetProperty("observed_writes").GetInt64()); + var afterRecommendation = json.GetProperty("fts_optimization_after"); + Assert.False(afterRecommendation.GetProperty("recommended").GetBoolean()); + Assert.Equal("none", afterRecommendation.GetProperty("action").GetString()); + Assert.Equal("incremental_write_threshold_not_reached", afterRecommendation.GetProperty("reason").GetString()); + Assert.Equal( + DbWriter.DefaultFtsOptimizeIncrementalWriteThreshold, + afterRecommendation.GetProperty("threshold_writes").GetInt32()); + Assert.Equal(0, afterRecommendation.GetProperty("observed_writes").GetInt64()); + Assert.Equal("current", afterRecommendation.GetProperty("state").GetString()); using var verifyDb = new DbContext(DbOpenIntent.WriteIndex, dbPath); Assert.Equal("0", verifyDb.GetMetaString(DbWriter.FtsIncrementalWritesSinceOptimizeMetaKey)); @@ -5897,6 +5949,15 @@ INSERT INTO symbols (file_id, kind, name, line, signature) Assert.Contains( json.GetProperty("planned_operations").EnumerateArray().Select(item => item.GetString()), operation => operation == "initialize_or_migrate_schema"); + var recommendation = json.GetProperty("fts_optimization"); + Assert.False(recommendation.GetProperty("recommended").GetBoolean()); + Assert.Equal("none", recommendation.GetProperty("action").GetString()); + Assert.Equal("incremental_write_count_unavailable", recommendation.GetProperty("reason").GetString()); + Assert.Equal( + DbWriter.DefaultFtsOptimizeIncrementalWriteThreshold, + recommendation.GetProperty("threshold_writes").GetInt32()); + Assert.Equal(0, recommendation.GetProperty("observed_writes").GetInt64()); + Assert.Equal("unavailable", recommendation.GetProperty("state").GetString()); Assert.Equal(bytesBefore, File.ReadAllBytes(dbPath)); Assert.False(File.Exists(IndexLock.GetLockPath(dbPath))); } @@ -6051,7 +6112,7 @@ public void Run_IndexOptimizeMissingDatabase_RedactsHumanPreambleUnlessEnabled_I } [Fact] - public void RunOptimizeFts_ReadOnlyUri_ReturnsDbNotWritable() + public void RunOptimizeFts_ReadOnlyUri_PreservesImmutableFreshnessAndReturnsDbNotWritable_Issue4887() { var dbPath = CreateTempDbPath("cdidx_optimize_readonly"); try @@ -6059,61 +6120,175 @@ public void RunOptimizeFts_ReadOnlyUri_ReturnsDbNotWritable() using (var db = new DbContext(DbOpenIntent.WriteIndex, dbPath)) { db.InitializeSchema(); - var writer = new DbWriter(db); - writer.RecordFtsIncrementalWrite(); + using var checkpoint = db.Connection.CreateCommand(); + checkpoint.CommandText = "PRAGMA wal_checkpoint(TRUNCATE);"; + checkpoint.ExecuteNonQuery(); } + SqliteConnection.ClearAllPools(); var dbUri = new Uri(dbPath).AbsoluteUri + "?immutable=1"; - int previewExitCode; - JsonElement previewJson; - lock (TestConsoleLock.Gate) + using (var hotWalConnection = new SqliteConnection( + new SqliteConnectionStringBuilder + { + DataSource = dbPath, + Mode = SqliteOpenMode.ReadWrite, + Pooling = false, + }.ToString())) { - var originalOut = Console.Out; - try + hotWalConnection.Open(); + using (var command = hotWalConnection.CreateCommand()) { - using var stdout = new StringWriter(); - Console.SetOut(stdout); - previewExitCode = IndexCommandRunner.RunOptimizeFts(["--db", dbUri, "--dry-run", "--json"], _jsonOptions); - using var document = JsonDocument.Parse(stdout.ToString()); - previewJson = document.RootElement.Clone(); + command.CommandText = """ + PRAGMA journal_mode=WAL; + PRAGMA wal_autocheckpoint=0; + INSERT INTO codeindex_meta(key, value) + VALUES (@key, @value) + ON CONFLICT(key) DO UPDATE SET value = excluded.value; + """; + command.Parameters.AddWithValue( + "@key", + DbWriter.FtsIncrementalWritesSinceOptimizeMetaKey); + command.Parameters.AddWithValue( + "@value", + DbWriter.DefaultFtsOptimizeIncrementalWriteThreshold.ToString(CultureInfo.InvariantCulture)); + command.ExecuteNonQuery(); } - finally + Assert.True(File.Exists(dbPath + "-wal")); + + FtsOptimizationRecommendation statusRecommendation; + using (var statusDb = new DbContext(DbOpenIntent.QueryOnly, dbUri)) + statusRecommendation = new DbReader(statusDb).GetStatus().MaintenanceGuidance.FtsOptimization; + Assert.False(statusRecommendation.Recommended); + Assert.Equal( + FtsOptimizationRecommendationEvaluator.MaintenanceSnapshotStaleReason, + statusRecommendation.Reason); + Assert.Equal( + FtsOptimizationRecommendationEvaluator.StaleState, + statusRecommendation.State); + + int previewExitCode; + JsonElement previewJson; + lock (TestConsoleLock.Gate) { - Console.SetOut(originalOut); + var originalOut = Console.Out; + try + { + using var stdout = new StringWriter(); + Console.SetOut(stdout); + previewExitCode = IndexCommandRunner.RunOptimizeFts(["--db", dbUri, "--dry-run", "--json"], _jsonOptions); + using var document = JsonDocument.Parse(stdout.ToString()); + previewJson = document.RootElement.Clone(); + } + finally + { + Console.SetOut(originalOut); + } } - } - Assert.Equal(CommandExitCodes.Success, previewExitCode); - Assert.Equal("dry_run", previewJson.GetProperty("status").GetString()); - Assert.True(previewJson.GetProperty("source_database_unchanged").GetBoolean()); + Assert.Equal(CommandExitCodes.Success, previewExitCode); + Assert.Equal("dry_run", previewJson.GetProperty("status").GetString()); + Assert.True(previewJson.GetProperty("source_database_unchanged").GetBoolean()); + var previewRecommendation = previewJson.GetProperty("fts_optimization"); + Assert.Equal( + statusRecommendation.Recommended, + previewRecommendation.GetProperty("recommended").GetBoolean()); + Assert.Equal( + statusRecommendation.Action, + previewRecommendation.GetProperty("action").GetString()); + Assert.Equal( + statusRecommendation.Reason, + previewRecommendation.GetProperty("reason").GetString()); + Assert.Equal( + statusRecommendation.ThresholdWrites, + previewRecommendation.GetProperty("threshold_writes").GetInt32()); + Assert.Equal( + statusRecommendation.ObservedWrites, + previewRecommendation.GetProperty("observed_writes").GetInt64()); + Assert.Equal( + statusRecommendation.State, + previewRecommendation.GetProperty("state").GetString()); - int exitCode; - JsonElement json; - lock (TestConsoleLock.Gate) - { - var originalOut = Console.Out; - try + int aliasPreviewExitCode; + JsonElement aliasPreviewJson; + lock (TestConsoleLock.Gate) { - using var stdout = new StringWriter(); - Console.SetOut(stdout); - exitCode = IndexCommandRunner.RunOptimizeFts(["--db", dbUri, "--json"], _jsonOptions); - using var document = JsonDocument.Parse(stdout.ToString()); - json = document.RootElement.Clone(); + var originalOut = Console.Out; + try + { + using var stdout = new StringWriter(); + Console.SetOut(stdout); + aliasPreviewExitCode = IndexCommandRunner.Run( + [ + Path.GetDirectoryName(dbPath)!, + "--optimize", + "--dry-run", + "--db", + dbUri, + "--json", + ], + _jsonOptions); + using var document = JsonDocument.Parse(stdout.ToString()); + aliasPreviewJson = document.RootElement.Clone(); + } + finally + { + Console.SetOut(originalOut); + } } - finally + + Assert.Equal(CommandExitCodes.Success, aliasPreviewExitCode); + Assert.Equal("dry_run", aliasPreviewJson.GetProperty("status").GetString()); + Assert.True(aliasPreviewJson.GetProperty("source_database_unchanged").GetBoolean()); + var aliasPreviewRecommendation = aliasPreviewJson.GetProperty("fts_optimization"); + Assert.Equal( + statusRecommendation.Recommended, + aliasPreviewRecommendation.GetProperty("recommended").GetBoolean()); + Assert.Equal( + statusRecommendation.Action, + aliasPreviewRecommendation.GetProperty("action").GetString()); + Assert.Equal( + statusRecommendation.Reason, + aliasPreviewRecommendation.GetProperty("reason").GetString()); + Assert.Equal( + statusRecommendation.ThresholdWrites, + aliasPreviewRecommendation.GetProperty("threshold_writes").GetInt32()); + Assert.Equal( + statusRecommendation.ObservedWrites, + aliasPreviewRecommendation.GetProperty("observed_writes").GetInt64()); + Assert.Equal( + statusRecommendation.State, + aliasPreviewRecommendation.GetProperty("state").GetString()); + + int exitCode; + JsonElement json; + lock (TestConsoleLock.Gate) { - Console.SetOut(originalOut); + var originalOut = Console.Out; + try + { + using var stdout = new StringWriter(); + Console.SetOut(stdout); + exitCode = IndexCommandRunner.RunOptimizeFts(["--db", dbUri, "--json"], _jsonOptions); + using var document = JsonDocument.Parse(stdout.ToString()); + json = document.RootElement.Clone(); + } + finally + { + Console.SetOut(originalOut); + } } - } - Assert.Equal(CommandExitCodes.DatabaseError, exitCode); - Assert.Equal("error", json.GetProperty("status").GetString()); - Assert.Equal(CommandErrorCodes.DbNotWritable, json.GetProperty("error_code").GetString()); - Assert.Equal("database_not_writable", json.GetProperty("category").GetString()); - Assert.Equal("database is not writable", json.GetProperty("message").GetString()); + Assert.Equal(CommandExitCodes.DatabaseError, exitCode); + Assert.Equal("error", json.GetProperty("status").GetString()); + Assert.Equal(CommandErrorCodes.DbNotWritable, json.GetProperty("error_code").GetString()); + Assert.Equal("database_not_writable", json.GetProperty("category").GetString()); + Assert.Equal("database is not writable", json.GetProperty("message").GetString()); + } using var verifyDb = new DbContext(DbOpenIntent.WriteIndex, dbPath); - Assert.Equal("1", verifyDb.GetMetaString(DbWriter.FtsIncrementalWritesSinceOptimizeMetaKey)); + Assert.Equal( + DbWriter.DefaultFtsOptimizeIncrementalWriteThreshold.ToString(CultureInfo.InvariantCulture), + verifyDb.GetMetaString(DbWriter.FtsIncrementalWritesSinceOptimizeMetaKey)); } finally { diff --git a/tests/CodeIndex.Tests/QueryCommandRunnerFilesTests.cs b/tests/CodeIndex.Tests/QueryCommandRunnerFilesTests.cs index 2bf36717d..d7e13dd20 100644 --- a/tests/CodeIndex.Tests/QueryCommandRunnerFilesTests.cs +++ b/tests/CodeIndex.Tests/QueryCommandRunnerFilesTests.cs @@ -1804,6 +1804,23 @@ public void RunStatus_Explain_PrintsVisibleStatusFieldDescriptionWithoutDatabase Assert.Contains("cdidx index ", stdout); } + [Fact] + public void RunStatus_Explain_MaintenanceGuidanceDescribesSharedOptimizationDecision_Issue4887() + { + var (exitCode, stdout, stderr) = CaptureConsole(() => QueryCommandRunner.RunStatus( + ["--explain", "maintenance_guidance"], + _jsonOptions)); + + Assert.Equal(CommandExitCodes.Success, exitCode); + Assert.Equal(string.Empty, stderr); + Assert.Contains("Database maintenance guidance (maintenance_guidance)", stdout); + Assert.Contains("fts_optimization", stdout); + Assert.Contains("threshold_writes", stdout); + Assert.Contains("observed_writes", stdout); + Assert.Contains("state=stale", stdout); + Assert.Contains("optimize --dry-run", stdout); + } + [Fact] public void RunStatus_Explain_PrintsHeadFreshnessFieldDescriptionWithoutDatabase_Issue3911() { @@ -1888,6 +1905,7 @@ public void RunStatus_ExplainJson_PrintsMachineReadableDescription() Assert.Contains("path_case_sensitive", json.GetProperty("known_fields").EnumerateArray().Select(item => item.GetString())); Assert.Contains("sqlite_connection_policy", json.GetProperty("known_fields").EnumerateArray().Select(item => item.GetString())); Assert.Contains("git_executable", json.GetProperty("known_fields").EnumerateArray().Select(item => item.GetString())); + Assert.Contains("maintenance_guidance", json.GetProperty("known_fields").EnumerateArray().Select(item => item.GetString())); var (policyExitCode, policyStdout, policyStderr) = CaptureConsole(() => QueryCommandRunner.RunStatus( ["--explain", "sqlite_connection_policy", "--json"], @@ -4059,6 +4077,14 @@ public void RunStatus_ReadOnlyFlagReportsImmutableRiskAcrossJsonCommands_Issue45 { var dbPath = TestProjectHelper.CreateProjectDb(projectRoot); TestProjectHelper.InsertIndexedFile(dbPath, "src/app.cs", "csharp", "class App {}\n"); + using (var db = new DbContext(DbOpenIntent.WriteIndex, dbPath)) + { + new DbWriter(db).SetMeta( + DbWriter.FtsIncrementalWritesSinceOptimizeMetaKey, + DbWriter.DefaultFtsOptimizeIncrementalWriteThreshold.ToString( + System.Globalization.CultureInfo.InvariantCulture)); + } + SqliteConnection.ClearAllPools(); var options = QueryCommandRunner.ParseArgs( ["--db", dbPath, "--read-only", "--json"], @@ -4086,6 +4112,16 @@ public void RunStatus_ReadOnlyFlagReportsImmutableRiskAcrossJsonCommands_Issue45 Assert.True(policy.GetProperty("immutable_uri").GetBoolean()); Assert.True(document.RootElement.GetProperty("wal_stale_snapshot_risk").GetBoolean()); Assert.Equal("explicit_immutable_read_only", document.RootElement.GetProperty("wal_stale_snapshot_reason").GetString()); + var ftsOptimization = document.RootElement + .GetProperty("maintenance_guidance") + .GetProperty("fts_optimization"); + Assert.False(ftsOptimization.GetProperty("recommended").GetBoolean()); + Assert.Equal("none", ftsOptimization.GetProperty("action").GetString()); + Assert.Equal("maintenance_snapshot_stale", ftsOptimization.GetProperty("reason").GetString()); + Assert.Equal( + DbWriter.DefaultFtsOptimizeIncrementalWriteThreshold, + ftsOptimization.GetProperty("observed_writes").GetInt64()); + Assert.Equal("stale", ftsOptimization.GetProperty("state").GetString()); Assert.Equal(SqliteConnectionPolicy.DefaultCommandTimeoutSeconds, policy.GetProperty("command_timeout_seconds").GetInt32()); Assert.True(policy.GetProperty("long_running_commands_require_cancellation").GetBoolean()); Assert.False(policy.GetProperty("read_only_fallback").GetBoolean()); diff --git a/tests/CodeIndex.Tests/QueryCommandRunnerTests.cs b/tests/CodeIndex.Tests/QueryCommandRunnerTests.cs index ef17253b1..94a148eb7 100644 --- a/tests/CodeIndex.Tests/QueryCommandRunnerTests.cs +++ b/tests/CodeIndex.Tests/QueryCommandRunnerTests.cs @@ -991,6 +991,10 @@ public void RunVacuum_DryRunJsonReportsMaintenanceEstimate_Issue3564() var dbPath = TestProjectHelper.CreateProjectDb(project.Root); using (var db = new DbContext(DbOpenIntent.WriteIndex, dbPath)) { + new DbWriter(db).SetMeta( + DbWriter.FtsIncrementalWritesSinceOptimizeMetaKey, + DbWriter.DefaultFtsOptimizeIncrementalWriteThreshold.ToString( + System.Globalization.CultureInfo.InvariantCulture)); using var command = db.Connection.CreateCommand(); command.CommandText = @" CREATE TABLE vacuum_payload (id INTEGER PRIMARY KEY, payload BLOB); @@ -1023,6 +1027,14 @@ INSERT INTO vacuum_payload (payload) var guidance = root.GetProperty("maintenance_guidance"); Assert.Equal("vacuum_recommended", guidance.GetProperty("freelist_state").GetString()); Assert.Equal("cdidx vacuum --db ", guidance.GetProperty("recommended_command").GetString()); + var ftsOptimization = guidance.GetProperty("fts_optimization"); + Assert.True(ftsOptimization.GetProperty("recommended").GetBoolean()); + Assert.Equal("optimize", ftsOptimization.GetProperty("action").GetString()); + Assert.Equal("incremental_write_threshold_reached", ftsOptimization.GetProperty("reason").GetString()); + Assert.Equal( + DbWriter.DefaultFtsOptimizeIncrementalWriteThreshold, + ftsOptimization.GetProperty("observed_writes").GetInt64()); + Assert.Equal("current", ftsOptimization.GetProperty("state").GetString()); } [Fact] diff --git a/tests/CodeIndex.Tests/golden/status.json b/tests/CodeIndex.Tests/golden/status.json index 13c03744a..f428da15a 100644 --- a/tests/CodeIndex.Tests/golden/status.json +++ b/tests/CodeIndex.Tests/golden/status.json @@ -302,6 +302,14 @@ "estimated_bytes_reclaimable": 0, "auto_vacuum_mode": 2, "auto_vacuum_mode_name": "incremental", + "fts_optimization": { + "recommended": false, + "action": "none", + "reason": "incremental_write_count_unavailable", + "threshold_writes": 25, + "observed_writes": 0, + "state": "unavailable" + }, "recommended_command": "none" }, "db_size_bytes": "\u003CCOUNT\u003E",