diff --git a/AGENT_GUIDE.md b/AGENT_GUIDE.md index e3efa113f7..ad7ea033a2 100644 --- a/AGENT_GUIDE.md +++ b/AGENT_GUIDE.md @@ -131,13 +131,13 @@ CI watching must be bounded. Do not loop indefinitely. ## Status Contract -- `status --json` and related JSON/MCP payloads currently expose the trust fields documented in `README.md` and `DEVELOPER_GUIDE.md`, including `fold_ready`, `fold_ready_reason`, `graph_table_available`, `issues_table_available`, `file_issues_data_current`, `migration_in_progress`, `sql_graph_contract_ready`, `sql_graph_contract_degraded_reason`, `hotspot_family_ready`, `hotspot_family_degraded_reason`, `language_readiness`, `csharp_symbol_name_ready`, `csharp_metadata_target_ready`, `csharp_metadata_target_degraded_reason`, `indexed_head_commit`, `worktree_head_changed`, `index_writer_version`, `index_newer_than_reader`, `index_newer_than_reader_reason`, `unknown_extension_file_count`, `unknown_extension_files`, `unknown_extension_files_truncated`, `unknown_extension_file_path_limit`, `extractors`, `path_case_sensitive`, `data_dir`, `data_dir_source`, `data_dir_mode`, `mac_profile`, `db_size_bytes`, `wal_size_bytes`, `db_pragma_settings` (`journal_mode`, `synchronous`, `wal_autocheckpoint`, `page_count`, `freelist_count`, `page_size`), `symbol_kinds`, `symbols_by_language`, status kind cap metadata (`symbol_kind_limit`, `symbol_kind_name_limit`, `symbol_kind_total_count`, `symbol_kind_omitted_count`, `symbol_kind_names_truncated`, `symbols_by_language_kind_total_counts`, `symbols_by_language_kind_omitted_counts`, `symbols_by_language_kind_names_truncated`), `process`, `last_index_run`, `last_workspace_freshened_at`, `hooks`, MCP-only `mcp_session`, and the `status --check`-only `stale_after_seconds` / `index_age_seconds` threshold audit fields. +- `status --json` and related JSON/MCP payloads currently expose the trust fields documented in `README.md` and `DEVELOPER_GUIDE.md`, including `fold_ready`, `fold_ready_reason`, `graph_table_available`, `issues_table_available`, `file_issues_data_current`, `migration_in_progress`, `sql_graph_contract_ready`, `sql_graph_contract_degraded_reason`, `hotspot_family_ready`, `hotspot_family_degraded_reason`, `language_readiness`, `csharp_symbol_name_ready`, `csharp_metadata_target_ready`, `csharp_metadata_target_degraded_reason`, `indexed_head_commit`, `worktree_head_changed`, `index_writer_version`, `index_newer_than_reader`, `index_newer_than_reader_reason`, `unknown_extension_file_count`, `unknown_extension_files`, `unknown_extension_files_truncated`, `unknown_extension_file_path_limit`, `unknown_extension_extension_counts`, `unknown_extension_category_counts`, `unknown_extension_groups`, `extractors`, `path_case_sensitive`, `data_dir`, `data_dir_source`, `data_dir_mode`, `mac_profile`, `db_size_bytes`, `wal_size_bytes`, `db_pragma_settings` (`journal_mode`, `synchronous`, `wal_autocheckpoint`, `page_count`, `freelist_count`, `page_size`, `auto_vacuum`), `maintenance_guidance`, `symbol_kinds`, `symbols_by_language`, status kind cap metadata (`symbol_kind_limit`, `symbol_kind_name_limit`, `symbol_kind_total_count`, `symbol_kind_omitted_count`, `symbol_kind_names_truncated`, `symbols_by_language_kind_total_counts`, `symbols_by_language_kind_omitted_counts`, `symbols_by_language_kind_names_truncated`), `process`, `last_index_run`, `last_failed_or_partial_index_run`, `last_workspace_freshened_at`, `hooks`, MCP-only `mcp_session`, and the `status --check`-only `stale_after_seconds` / `index_age_seconds` threshold audit fields and `repair_commands`. - When any readiness field is degraded, the CLI adds `degraded_root_cause`, `degraded_reason`, `recommended_action`, `alternative_action`, and `readiness_degradations[]`. `degraded_root_cause` is the primary stable machine code; `readiness_degradations[]` lists every degraded field with `root_cause`, human reason, and remediation strings. - `hotspot_family_degraded_reason` currently uses `hotspot_family_support_not_indexed`, `hotspot_family_metadata_stale`, `hotspot_family_disabled_at_index_time`, `partial_family_key_population`, and `hotspot_family_marker_fingerprint_incomplete`; the incomplete marker fingerprint code means marker traversal hit safety caps and should stay synchronized with README / developer-guide recovery notes. - `issues_table_available` reports physical `file_issues` table presence only. `file_issues_data_current` reports whether the table is also stamped current for the active index generation. - `index_writer_version` records the `cdidx` version that last wrote to the DB (stamped into `codeindex_meta` as `cdidx_writer_version` on every full scan, update, and MCP index). `index_newer_than_reader` flips to `true` whenever any persisted numeric contract stamp in `codeindex_meta` (or unknown `PRAGMA user_version` readiness bits) exceeds the current binary's compiled maximum, so an older CLI re-opening a DB written by a newer CLI degrades loudly with an audit trail instead of silently dropping back to text-search fallbacks. `index_newer_than_reader_reason` enumerates the specific newer-than-reader stamps. - `status` also surfaces indexed-HEAD freshness via `indexed_head_sha`, `indexed_head_branch`, `indexed_head_timestamp`, and `commits_ahead_of_indexed_head`. They are stamped by `cdidx index` on every successful run (full scan AND partial update, distinct from `indexed_head_commit` which is full-scan only) on a best-effort basis (never blocks an otherwise-successful index) and omitted on non-git workspaces, detached HEAD (branch only), or legacy DBs created before this contract. -- `status` also surfaces unknown-extension scan coverage via `unknown_extension_file_count`, stamped by successful full-repository index runs (`cdidx index ` and MCP `index_project`) as the number of non-indexed files with non-empty extensions that do not map to a known language. Current scans also stamp `unknown_extension_files` as a path sample bounded by `unknown_extension_file_path_limit` items and the string-list decoded-character budget, `unknown_extension_files_truncated` when more paths existed than were emitted for either bound, and `unknown_extension_file_path_limit` as the item cap rather than a guarantee that that many paths are returned. These fields are omitted on legacy DBs or before a current full scan has stamped them. +- `status` also surfaces unknown-extension scan coverage via `unknown_extension_file_count`, stamped by successful full-repository index runs (`cdidx index ` and MCP `index_project`) as the number of non-indexed files with non-empty extensions that do not map to a known language. Current scans also stamp `unknown_extension_files` as a path sample bounded by `unknown_extension_file_path_limit` items and the string-list decoded-character budget, `unknown_extension_files_truncated` when more paths existed than were emitted for either bound, and `unknown_extension_file_path_limit` as the item cap rather than a guarantee that that many paths are returned. Newer scans also expose `unknown_extension_extension_counts`, `unknown_extension_category_counts`, and `unknown_extension_groups`; groups classify common non-code buckets such as repository metadata, licenses, binary assets, configuration, structural metadata, and language-support candidates, and include `recommended_action` values of `ignore_configuration`, `first_class_structural_extraction`, or `language_support`. These fields are omitted on legacy DBs or before a current full scan has stamped them. - `status` also surfaces extractor plugin and pattern-config runtime diagnostics via `extractors`, including loaded counts, skipped file counts, and a bounded diagnostics list for incompatible or malformed plugin/pattern files. Diagnostic paths and messages are sanitized before output. - `status` also surfaces metadata-only post-extraction hook candidates and callback budgets through `hooks[]` / `hooks[].callback_budget_ms` without loading hook assemblies. Index runs still enforce `CDIDX_HOOK_CALLBACK_BUDGET_MS` (default: 5000 ms) on scratch copies, discard timed-out mutations, and disable timed-out hooks for the remainder of the current run. - `status` also surfaces `.cdidx` data-directory permissions via `data_dir_mode` on POSIX filesystems. New `.cdidx` data directories are forced to `0700`; the field is omitted on Windows, URI DBs, or when the directory mode cannot be inspected. diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index e37d469404..b16269fe4b 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -380,8 +380,9 @@ Current stable codes and triggers: | Schema discovery cache | `DbReader` schema discovery uses a process-level cache keyed by the normalized DB path. The cache stores `PRAGMA table_info`, `PRAGMA index_list`, and `sqlite_master` table-existence results, and checks `PRAGMA schema_version` before serving a lookup so SQLite DDL performed by cdidx or an external `sqlite3` session invalidates stale snapshots. Manual schema edits outside cdidx are still unsupported operationally; run `cdidx validate` after such edits before trusting query output. | | Batch trust marker | Index write batches stamp `codeindex_meta.batch_in_progress=true` before starting a mutation transaction and clear it inside the transaction that commits the matching rows and readiness metadata. If the indexer crashes after the marker is written but before the commit clears it, the next writable DB open demotes readiness bits and warns: `Last batch did not complete; run cdidx index --rebuild to re-index from a known clean state.` Gracefully handled per-file errors clear the marker after rollback; orphaned markers are reserved for interrupted or crashed batches whose trust metadata should not be treated as clean. | | Read-only fallback | When the normal writable open cannot create or lock journal/WAL side files, read-only fallback uses an immutable SQLite URI so query commands can still read a DB from read-only or sandboxed storage. The fallback intentionally skips writable pragmas, migrations, and WAL recovery writes. If a WAL is present and must be observed, copy `.db`, `.db-wal`, and `.db-shm` together to a writable location or use a SQLite backup from an environment that can open the full WAL set. | -| Status pragma diagnostics | `status --json` exposes resolved connection values under `db_pragma_settings` (`journal_mode`, `synchronous`, `wal_autocheckpoint`, `page_count`, `freelist_count`, `page_size`) for automation and support diagnostics. | -| Vacuum | `cdidx vacuum` runs `PRAGMA incremental_vacuum` against writable incremental-auto-vacuum DBs, and performs a one-time `PRAGMA auto_vacuum=INCREMENTAL` plus full `VACUUM` conversion for legacy no-autovacuum DBs. | +| Status pragma diagnostics | `status --json` exposes resolved connection values under `db_pragma_settings` (`journal_mode`, `synchronous`, `wal_autocheckpoint`, `page_count`, `freelist_count`, `page_size`, `auto_vacuum`) for automation and support diagnostics. `maintenance_guidance` derives `wal_state`, `freelist_ratio`, `freelist_state`, `estimated_*_reclaimable`, `auto_vacuum_mode(_name)`, `recommended_command`, and `post_maintenance_follow_up` from those raw metrics without changing the raw values. `status --check --json` adds `repair_commands[]` entries with `name`, `args`, `reason`, and `safety_notes` so clients do not parse prose remediation strings. `last_failed_or_partial_index_run` exposes bounded failed/partial index context (`status`, `mode`, timings, counts, stable error code, reason) and must not include raw exception text or file paths. | +| Maintenance thresholds | WAL guidance flips to `checkpoint_recommended` at `CDIDX_MAINTENANCE_WAL_WARN_BYTES` (default 64 MiB). Freelist guidance flips to `vacuum_recommended` at `CDIDX_MAINTENANCE_FREELIST_WARN_RATIO` (default `0.20`). Invalid or out-of-range env values fall back to defaults. | +| Vacuum | `cdidx vacuum` runs `PRAGMA incremental_vacuum` against writable incremental-auto-vacuum DBs, and performs a one-time `PRAGMA auto_vacuum=INCREMENTAL` plus full `VACUUM` conversion for legacy no-autovacuum DBs. `cdidx vacuum --dry-run --json` estimates reclaimable pages/bytes and returns the same maintenance guidance without executing vacuum pragmas. | | Size and process diagnostics | `status --json` also reports `db_size_bytes`, `wal_size_bytes`, capped `symbol_kinds` / `symbols_by_language` kind maps with `symbol_kind_*` and `symbols_by_language_kind_*` overflow metadata when caps apply, current `process` heap/GC/working-set metrics, `last_index_run` metadata from successful CLI and MCP index runs, and `last_workspace_freshened_at` as the latest successful index/update timestamp. `indexed_at` still comes from indexed file rows, so partial or no-op updates can freshen the workspace without moving `indexed_at`. | | Memory tracing | `index --json --memory-trace` adds a `memory_timeline` block to the CLI index result and persists peak working-set MB into `last_index_run`; `CDIDX_MEM_WARN_MB=` prints a warning when the sampled working set crosses that threshold. | | Newer schema protection | Writable opens reject databases whose `PRAGMA user_version` contains readiness bits outside the current binary's `CurrentSchemaVersion` mask. Read-only status/query paths may still surface `index_newer_than_reader=true` as a degraded audit signal, but write-capable paths must fail with `E003_SCHEMA_TOO_NEW` so an older cdidx cannot silently rewrite a DB stamped by a newer one. | @@ -974,8 +975,9 @@ For the AI agent search-rule template, see [AI Integration](USER_GUIDE.md#ai-int | Readiness and graph trust | `fold_ready`, `fold_ready_reason`, `graph_table_available`, `issues_table_available`, `file_issues_data_current`, `migration_in_progress`, `sql_graph_contract_ready`, `sql_graph_contract_degraded_reason`, `hotspot_family_ready`, `hotspot_family_degraded_reason`, `language_readiness`, `csharp_symbol_name_ready`, `csharp_metadata_target_ready`, `csharp_metadata_target_degraded_reason`. | | Workspace and HEAD freshness | `indexed_head_commit`, `worktree_head_changed`, `indexed_head_sha`, `indexed_head_branch`, `indexed_head_timestamp`, `commits_ahead_of_indexed_head`. | | Version and forward compatibility | `index_writer_version`, `index_newer_than_reader`, `index_newer_than_reader_reason`. | -| Unknown-extension and runtime diagnostics | `unknown_extension_file_count`, `unknown_extension_files`, `unknown_extension_files_truncated`, `unknown_extension_file_path_limit`, `extractors`, `path_case_sensitive`, `data_dir_mode`, `mac_profile`, `stale_after_seconds`, `index_age_seconds`. | -| Remediation fields | `degraded_root_cause`, `degraded_reason`, `recommended_action`, `alternative_action`, `readiness_degradations`. | +| Unknown-extension and runtime diagnostics | `unknown_extension_file_count`, `unknown_extension_files`, `unknown_extension_files_truncated`, `unknown_extension_file_path_limit`, `unknown_extension_extension_counts`, `unknown_extension_category_counts`, `unknown_extension_groups`, `extractors`, `path_case_sensitive`, `data_dir_mode`, `mac_profile`, `stale_after_seconds`, `index_age_seconds`, `last_failed_or_partial_index_run`. | +| Database maintenance | `db_size_bytes`, `wal_size_bytes`, `db_pragma_settings` (`journal_mode`, `synchronous`, `wal_autocheckpoint`, `page_count`, `freelist_count`, `page_size`, `auto_vacuum`), `maintenance_guidance`. | +| Remediation fields | `degraded_root_cause`, `degraded_reason`, `recommended_action`, `alternative_action`, `readiness_degradations`, `repair_commands`. | | MCP-only session diagnostics | `mcp_session`, which is session-scoped diagnostics rather than persisted DB state. It contains `log_level`, bounded `roots`, optional `client_info`, and bounded optional `client_capabilities`. When advertised roots are capped, `roots_truncated`, `root_count`, `root_limit`, and `root_uri_length_limit` describe the truncation. When client capabilities are capped, `client_capabilities_truncated`, `client_capabilities_truncation_reason`, `client_capabilities_serialized_bytes`, `client_capabilities_byte_limit`, and `client_capabilities_depth_limit` describe the retained diagnostic subset. | | Documentation sync | Keep this list synchronized with `README.md` and `AGENT_GUIDE.md`; `DocumentationStatusContractTests` fails when any required field is missing from one of those docs. | @@ -2540,8 +2542,9 @@ alternative action を同じ場所へ追加してください。 | schema discovery cache | `DbReader` の schema discovery は正規化済み DB path を key にした process-level cache を使います。この cache は `PRAGMA table_info`、`PRAGMA index_list`、`sqlite_master` の table existence 結果を保持し、lookup 前に `PRAGMA schema_version` を確認するため、cdidx や外部 `sqlite3` session による SQLite DDL は stale snapshot を invalidate します。cdidx 外での手動 schema edit は運用上 unsupported であり、その後は query output を信頼する前に `cdidx validate` を実行してください。 | | batch trust marker | index write batch は mutation transaction を始める前に `codeindex_meta.batch_in_progress=true` を stamp し、対応する row と readiness metadata を commit する transaction 内で clear します。marker が書かれた後、clear される前に indexer が crash した場合、次の writable DB open は readiness bit を degrade し、`Last batch did not complete; run cdidx index --rebuild to re-index from a known clean state.` と警告します。file ごとの error が graceful に処理された場合は rollback 後に marker を clear するため、orphaned marker は interrupted / crashed batch の trust metadata を clean と扱わないための signal です。 | | read-only fallback | 通常の writable open が journal/WAL side file を作成または lock できない場合、read-only fallback は immutable SQLite URI を使うため、query command は read-only / sandboxed storage 上の DB でも読み取りを継続できます。この fallback は意図的に writable pragma、migration、WAL recovery write を skip します。WAL が存在し、その内容を観測する必要がある場合は、`.db` / `.db-wal` / `.db-shm` をまとめて writable location に copy するか、full WAL set を open できる環境で SQLite backup を使います。 | -| status pragma diagnostics | `status --json` は automation / support diagnostics 用に、解決済みの接続値を `db_pragma_settings` (`journal_mode`, `synchronous`, `wal_autocheckpoint`, `page_count`, `freelist_count`, `page_size`) で公開します。 | -| vacuum | `cdidx vacuum` は incremental-auto-vacuum DB では `PRAGMA incremental_vacuum` を実行し、legacy no-autovacuum DB では初回のみ `PRAGMA auto_vacuum=INCREMENTAL` と full `VACUUM` で変換します。 | +| status pragma diagnostics | `status --json` は automation / support diagnostics 用に、解決済みの接続値を `db_pragma_settings` (`journal_mode`, `synchronous`, `wal_autocheckpoint`, `page_count`, `freelist_count`, `page_size`, `auto_vacuum`) で公開します。`maintenance_guidance` は raw 値を変えずに `wal_state`、`freelist_ratio`、`freelist_state`、`estimated_*_reclaimable`、`auto_vacuum_mode(_name)`、`recommended_command`、`post_maintenance_follow_up` を派生します。`status --check --json` は `repair_commands[]` に `name`、`args`、`reason`、`safety_notes` を返し、client が prose remediation を parse しなくてよいようにします。`last_failed_or_partial_index_run` は bounded な failed / partial index context (`status`、`mode`、timing、count、stable error code、reason) のみを公開し、raw exception text や file path を含めてはいけません。 | +| maintenance threshold | WAL guidance は `CDIDX_MAINTENANCE_WAL_WARN_BYTES` (既定 64 MiB) 以上で `checkpoint_recommended` になります。freelist guidance は `CDIDX_MAINTENANCE_FREELIST_WARN_RATIO` (既定 `0.20`) 以上で `vacuum_recommended` になります。不正・範囲外の環境変数値は既定値へ戻します。 | +| vacuum | `cdidx vacuum` は incremental-auto-vacuum DB では `PRAGMA incremental_vacuum` を実行し、legacy no-autovacuum DB では初回のみ `PRAGMA auto_vacuum=INCREMENTAL` と full `VACUUM` で変換します。`cdidx vacuum --dry-run --json` は vacuum pragma を実行せず、回収可能 page/byte の推定と同じ maintenance guidance を返します。 | | size / process diagnostics | `status --json` は `db_size_bytes`、`wal_size_bytes`、上限付きの `symbol_kinds` / `symbols_by_language` kind map と、上限適用時の `symbol_kind_*` / `symbols_by_language_kind_*` overflow metadata、現在の `process` heap / GC / working-set metrics、成功した CLI / MCP index 実行由来の `last_index_run` metadata、最新の成功 index/update 時刻を示す `last_workspace_freshened_at` も公開します。`indexed_at` は引き続き indexed file row 由来なので、partial / no-op update は `indexed_at` を動かさずに workspace 鮮度だけを更新することがあります。 | | memory tracing | `index --json --memory-trace` は CLI index 結果に `memory_timeline` block を追加し、peak working-set MB を `last_index_run` に保存します。`CDIDX_MEM_WARN_MB=` は sampled working set がしきい値を超えたときに warning を出します。 | | newer schema protection | writable open は、`PRAGMA user_version` に current binary の `CurrentSchemaVersion` mask 外の readiness bit が含まれる database も拒否します。read-only status/query path は degraded audit signal として `index_newer_than_reader=true` を表示できますが、write-capable path は古い cdidx が新しい binary で stamp された DB を黙って rewrite しないよう `E003_SCHEMA_TOO_NEW` で失敗しなければなりません。 | @@ -3108,8 +3111,9 @@ AI エージェント向け検索ルールのテンプレートについては | readiness / graph trust | `fold_ready`, `fold_ready_reason`, `graph_table_available`, `issues_table_available`, `file_issues_data_current`, `migration_in_progress`, `sql_graph_contract_ready`, `sql_graph_contract_degraded_reason`, `hotspot_family_ready`, `hotspot_family_degraded_reason`, `language_readiness`, `csharp_symbol_name_ready`, `csharp_metadata_target_ready`, `csharp_metadata_target_degraded_reason`。 | | workspace / HEAD freshness | `indexed_head_commit`, `worktree_head_changed`, `indexed_head_sha`, `indexed_head_branch`, `indexed_head_timestamp`, `commits_ahead_of_indexed_head`。 | | version / forward compatibility | `index_writer_version`, `index_newer_than_reader`, `index_newer_than_reader_reason`。 | -| unknown-extension / runtime diagnostics | `unknown_extension_file_count`, `unknown_extension_files`, `unknown_extension_files_truncated`, `unknown_extension_file_path_limit`, `extractors`, `path_case_sensitive`, `data_dir_mode`, `mac_profile`, `stale_after_seconds`, `index_age_seconds`。 | -| remediation fields | `degraded_root_cause`, `degraded_reason`, `recommended_action`, `alternative_action`, `readiness_degradations`。 | +| unknown-extension / runtime diagnostics | `unknown_extension_file_count`, `unknown_extension_files`, `unknown_extension_files_truncated`, `unknown_extension_file_path_limit`, `unknown_extension_extension_counts`, `unknown_extension_category_counts`, `unknown_extension_groups`, `extractors`, `path_case_sensitive`, `data_dir_mode`, `mac_profile`, `stale_after_seconds`, `index_age_seconds`, `last_failed_or_partial_index_run`。 | +| database maintenance | `db_size_bytes`, `wal_size_bytes`, `db_pragma_settings` (`journal_mode`, `synchronous`, `wal_autocheckpoint`, `page_count`, `freelist_count`, `page_size`, `auto_vacuum`), `maintenance_guidance`。 | +| remediation fields | `degraded_root_cause`, `degraded_reason`, `recommended_action`, `alternative_action`, `readiness_degradations`, `repair_commands`。 | | MCP-only session diagnostics | `mcp_session`。これは persisted DB state ではなく session-scoped diagnostics で、`log_level`、上限付きの `roots`、任意の `client_info`、上限付きの任意の `client_capabilities` を含みます。advertised root が切り詰められた場合は `roots_truncated`、`root_count`、`root_limit`、`root_uri_length_limit` が切り詰め内容を示します。client capabilities が切り詰められた場合は `client_capabilities_truncated`、`client_capabilities_truncation_reason`、`client_capabilities_serialized_bytes`、`client_capabilities_byte_limit`、`client_capabilities_depth_limit` が保持された診断 subset を示します。 | | documentation sync | この一覧は `README.md` と `AGENT_GUIDE.md` と同期してください。必須 field がそれらの docs から欠けると `DocumentationStatusContractTests` が失敗します。 | diff --git a/README.md b/README.md index 06911415bc..5081ef5822 100644 --- a/README.md +++ b/README.md @@ -132,8 +132,9 @@ names visible so documentation and tests stay synchronized. | Readiness and graph trust | `fold_ready`, `fold_ready_reason`, `graph_table_available`, `issues_table_available`, `file_issues_data_current`, `migration_in_progress`, `sql_graph_contract_ready`, `sql_graph_contract_degraded_reason`, `hotspot_family_ready`, `hotspot_family_degraded_reason`, `language_readiness`, `csharp_symbol_name_ready`, `csharp_metadata_target_ready`, `csharp_metadata_target_degraded_reason`. | | Workspace and HEAD freshness | `indexed_head_commit`, `worktree_head_changed`, `indexed_head_sha`, `indexed_head_branch`, `indexed_head_timestamp`, `commits_ahead_of_indexed_head`. | | Version and forward compatibility | `index_writer_version`, `index_newer_than_reader`, `index_newer_than_reader_reason`. | -| Unknown-extension and runtime diagnostics | `unknown_extension_file_count`, `unknown_extension_files`, `unknown_extension_files_truncated`, `unknown_extension_file_path_limit`, `extractors`, `path_case_sensitive`, `data_dir_mode`, `mac_profile`, `stale_after_seconds`, `index_age_seconds`. | -| Remediation fields | `degraded_root_cause`, `degraded_reason`, `recommended_action`, `alternative_action`, `readiness_degradations`. | +| Unknown-extension and runtime diagnostics | `unknown_extension_file_count`, `unknown_extension_files`, `unknown_extension_files_truncated`, `unknown_extension_file_path_limit`, `unknown_extension_extension_counts`, `unknown_extension_category_counts`, `unknown_extension_groups`, `extractors`, `path_case_sensitive`, `data_dir_mode`, `mac_profile`, `stale_after_seconds`, `index_age_seconds`, `last_failed_or_partial_index_run`. | +| Database maintenance | `db_size_bytes`, `wal_size_bytes`, `db_pragma_settings` (`journal_mode`, `synchronous`, `wal_autocheckpoint`, `page_count`, `freelist_count`, `page_size`, `auto_vacuum`), `maintenance_guidance`. | +| Remediation fields | `degraded_root_cause`, `degraded_reason`, `recommended_action`, `alternative_action`, `readiness_degradations`, `repair_commands`. | | MCP-only session diagnostics | `mcp_session`. | ## Verifying Releases @@ -288,8 +289,9 @@ freshness、compatibility、remediation field を返します。詳細な意味 | readiness / graph trust | `fold_ready`, `fold_ready_reason`, `graph_table_available`, `issues_table_available`, `file_issues_data_current`, `migration_in_progress`, `sql_graph_contract_ready`, `sql_graph_contract_degraded_reason`, `hotspot_family_ready`, `hotspot_family_degraded_reason`, `language_readiness`, `csharp_symbol_name_ready`, `csharp_metadata_target_ready`, `csharp_metadata_target_degraded_reason`。 | | workspace / HEAD freshness | `indexed_head_commit`, `worktree_head_changed`, `indexed_head_sha`, `indexed_head_branch`, `indexed_head_timestamp`, `commits_ahead_of_indexed_head`。 | | version / forward compatibility | `index_writer_version`, `index_newer_than_reader`, `index_newer_than_reader_reason`。 | -| unknown-extension / runtime diagnostics | `unknown_extension_file_count`, `unknown_extension_files`, `unknown_extension_files_truncated`, `unknown_extension_file_path_limit`, `extractors`, `path_case_sensitive`, `data_dir_mode`, `mac_profile`, `stale_after_seconds`, `index_age_seconds`。 | -| remediation fields | `degraded_root_cause`, `degraded_reason`, `recommended_action`, `alternative_action`, `readiness_degradations`。 | +| unknown-extension / runtime diagnostics | `unknown_extension_file_count`, `unknown_extension_files`, `unknown_extension_files_truncated`, `unknown_extension_file_path_limit`, `unknown_extension_extension_counts`, `unknown_extension_category_counts`, `unknown_extension_groups`, `extractors`, `path_case_sensitive`, `data_dir_mode`, `mac_profile`, `stale_after_seconds`, `index_age_seconds`, `last_failed_or_partial_index_run`。 | +| database maintenance | `db_size_bytes`, `wal_size_bytes`, `db_pragma_settings` (`journal_mode`, `synchronous`, `wal_autocheckpoint`, `page_count`, `freelist_count`, `page_size`, `auto_vacuum`), `maintenance_guidance`。 | +| remediation fields | `degraded_root_cause`, `degraded_reason`, `recommended_action`, `alternative_action`, `readiness_degradations`, `repair_commands`。 | | MCP-only session diagnostics | `mcp_session`。 | ## リリース成果物の検証 diff --git a/USER_GUIDE.md b/USER_GUIDE.md index 4c49a86911..91f0730f2f 100644 --- a/USER_GUIDE.md +++ b/USER_GUIDE.md @@ -3156,6 +3156,17 @@ cdidx db --integrity-check --json # 機械可読な結果 DB を read-only で開いて SQLite の `PRAGMA integrity_check` を実行し、`ok` か、検出された破損行の一覧を出力します。終了コードは安定しており、`0` = 健全、`2` (NotFound) = ファイル無し、`3` (DatabaseError) = 破損検出です。SQLite には汎用的な修復プリミティブが無いため、チェックが失敗した場合は `cdidx index --rebuild` で再構築するのが推奨復旧手段です。 +`--json` の診断出力は自動化向けに安定した `severity` と `diagnostic_code` を含みます。`db --integrity-check --json` は `integrity_ok` / `integrity_failed` を返し、`db schema --json` は `schema_ok` / `schema_truncated` に加えて `object_type_counts` と `object_type_omitted_counts` で SQLite の table / index / trigger / view 件数と省略数を返します。 + +DB / WAL の肥大や空き page を確認したい場合は `status --json` の `maintenance_guidance` を見ます。既定では WAL が 64 MiB 以上で `checkpoint_recommended`、`freelist_count / page_count` が 0.20 以上で `vacuum_recommended` になり、`recommended_command` と `post_maintenance_follow_up` が返ります。しきい値は `CDIDX_MAINTENANCE_WAL_WARN_BYTES` と `CDIDX_MAINTENANCE_FREELIST_WARN_RATIO` で調整できます。 + +`status --check --json` は failed check ごとに `repair_commands` を返します。各 entry は `name`、`args`、`reason`、`safety_notes` を持つため、自動化は `recommended_action` の文章を分解せずに修復コマンドを組み立てられます。前回の index が中断・失敗した情報が DB に残っている場合は、`last_failed_or_partial_index_run` に bounded metadata だけを返し、例外本文や file path は含めません。 + +```bash +cdidx vacuum --dry-run --json # 回収見積もりと maintenance guidance だけを確認 +cdidx vacuum --json # incremental vacuum / 初回変換を実行 +``` + ### コード検索 ```bash diff --git a/changelog.d/unreleased/3523.fixed.md b/changelog.d/unreleased/3523.fixed.md new file mode 100644 index 0000000000..a09894fbff --- /dev/null +++ b/changelog.d/unreleased/3523.fixed.md @@ -0,0 +1,18 @@ +--- +category: fixed +issues: + - 3523 +affected: + - src/CodeIndex/Database/UnknownExtensionClassifier.cs + - src/CodeIndex/Database/DbReader.FilesStatus.cs + - src/CodeIndex/Database/DbWriter.cs + - src/CodeIndex/Models/QueryResults.cs +--- + +## English + +- **Status now classifies unknown-extension files (#3523)** — `status --json` keeps the existing bounded sample and now adds extension counts, category counts, grouped classifications, and recommended next actions for language support, ignore configuration, or structural extraction. + +## 日本語 + +- **`status` が未知拡張子ファイルを分類するようになりました (#3523)** — `status --json` は既存の上限付き sample を維持しつつ、拡張子別件数、カテゴリ別件数、分類グループ、language support / ignore configuration / structural extraction の推奨 next action を追加します。 diff --git a/changelog.d/unreleased/3564.fixed.md b/changelog.d/unreleased/3564.fixed.md new file mode 100644 index 0000000000..1c21ca6795 --- /dev/null +++ b/changelog.d/unreleased/3564.fixed.md @@ -0,0 +1,28 @@ +--- +category: fixed +issues: + - 3564 +affected: + - src/CodeIndex/Database/MaintenanceGuidanceBuilder.cs + - src/CodeIndex/Database/DbContext.cs + - src/CodeIndex/Database/DbReader.FilesStatus.cs + - src/CodeIndex/Models/QueryResults.cs + - src/CodeIndex/Cli/QueryCommandRunner.cs + - src/CodeIndex/Cli/CliFlagSchema.cs + - src/CodeIndex/Cli/ConsoleUi.cs + - src/CodeIndex/Cli/JsonOutputContracts.cs + - tests/CodeIndex.Tests/DatabaseTests.cs + - tests/CodeIndex.Tests/QueryCommandRunnerTests.cs + - README.md + - AGENT_GUIDE.md + - DEVELOPER_GUIDE.md + - USER_GUIDE.md +--- + +## English + +- **Status and vacuum now provide DB maintenance guidance (#3564)** — `status --json` reports `maintenance_guidance` with WAL/freelist states, thresholds, reclaim estimates, auto-vacuum mode, recommended commands, and follow-up text; `vacuum --dry-run --json` returns the same guidance without running vacuum pragmas. + +## 日本語 + +- **status と vacuum が DB maintenance guidance を返すようになりました (#3564)** — `status --json` は WAL/freelist 状態、しきい値、回収見積もり、auto-vacuum mode、推奨コマンド、follow-up を含む `maintenance_guidance` を返し、`vacuum --dry-run --json` は vacuum pragma を実行せず同じ guidance を返します。 diff --git a/changelog.d/unreleased/3565.fixed.md b/changelog.d/unreleased/3565.fixed.md new file mode 100644 index 0000000000..304b575c85 --- /dev/null +++ b/changelog.d/unreleased/3565.fixed.md @@ -0,0 +1,18 @@ +--- +category: fixed +issues: + - 3565 +affected: + - src/CodeIndex/Cli/DbCommandRunner.cs + - src/CodeIndex/Cli/JsonOutputContracts.cs + - tests/CodeIndex.Tests/DbCommandRunnerTests.cs + - USER_GUIDE.md +--- + +## English + +- **DB diagnostic JSON now exposes stable severity and schema object coverage (#3565)** — `db --integrity-check --json` and `db schema --json` include `severity` and `diagnostic_code`, while schema JSON also reports total and omitted object counts for SQLite tables, indexes, triggers, and views. + +## 日本語 + +- **DB diagnostic JSON が安定した severity と schema object coverage を返すようになりました (#3565)** — `db --integrity-check --json` と `db schema --json` は `severity` と `diagnostic_code` を含み、schema JSON は SQLite の table / index / trigger / view の総数と省略数も返します。 diff --git a/changelog.d/unreleased/3567.fixed.md b/changelog.d/unreleased/3567.fixed.md new file mode 100644 index 0000000000..6b19c7ba7c --- /dev/null +++ b/changelog.d/unreleased/3567.fixed.md @@ -0,0 +1,28 @@ +--- +category: fixed +issues: + - 3567 +affected: + - src/CodeIndex/Cli/IndexCommandRunner.cs + - src/CodeIndex/Cli/JsonOutputContracts.cs + - src/CodeIndex/Cli/QueryCommandRunner.cs + - src/CodeIndex/Database/DbContext.cs + - src/CodeIndex/Database/DbReader.FilesStatus.cs + - src/CodeIndex/Database/DbWriter.cs + - src/CodeIndex/Mcp/McpToolHandlers.cs + - src/CodeIndex/Models/QueryResults.cs + - tests/CodeIndex.Tests/DocumentationStatusContractTests.cs + - tests/CodeIndex.Tests/QueryCommandRunnerFilesTests.cs + - README.md + - AGENT_GUIDE.md + - DEVELOPER_GUIDE.md + - USER_GUIDE.md +--- + +## English + +- **status --check JSON now exposes structured repair commands (#3567)** — failed checks include `repair_commands` entries with command names, args, reasons, and safety notes, and status can surface bounded `last_failed_or_partial_index_run` metadata without raw exception text or file paths. + +## 日本語 + +- **status --check JSON が構造化 repair command を返すようになりました (#3567)** — failed check は command name、args、reason、safety notes を持つ `repair_commands` を含み、status は raw exception text や file path を含めず bounded な `last_failed_or_partial_index_run` metadata を返せます。 diff --git a/src/CodeIndex/Cli/CliFlagSchema.cs b/src/CodeIndex/Cli/CliFlagSchema.cs index 9593377625..3f4679cac3 100644 --- a/src/CodeIndex/Cli/CliFlagSchema.cs +++ b/src/CodeIndex/Cli/CliFlagSchema.cs @@ -329,7 +329,7 @@ private static IReadOnlyList BuildAll() new() { Name = "--integrity-check", Description = "Run PRAGMA integrity_check on the database", Commands = Set("db") }, new() { Name = "--rebuild", Description = "Delete existing DB and rebuild from scratch", Commands = Set("index") }, new() { Name = "--optimize", Description = "Optimize the existing FTS5 table without scanning files", Commands = Set("index") }, - new() { Name = "--dry-run", Description = "Preview without writing", Commands = Set("index", "backfill-fold") }, + new() { Name = "--dry-run", Description = "Preview without writing", Commands = Set("index", "backfill-fold", "vacuum") }, new() { Name = "--no-checkpoint", Description = "Skip the automatic DB checkpoint before maintenance", Commands = Set("backfill-fold") }, new() { Name = "--force", Description = "Bypass the per-database index lock", Commands = Set("index") }, new() { Name = "--duration-format", ValuePlaceholder = "", Description = "Index elapsed time display format", Commands = Set("index") }, diff --git a/src/CodeIndex/Cli/ConsoleUi.cs b/src/CodeIndex/Cli/ConsoleUi.cs index ba31188742..5b1b3c8d01 100644 --- a/src/CodeIndex/Cli/ConsoleUi.cs +++ b/src/CodeIndex/Cli/ConsoleUi.cs @@ -83,7 +83,7 @@ private static readonly (string Command, string Usage)[] CommandUsageLines = ("hooks", "cdidx hooks [--project ] [--force] [--json]"), ("backfill-fold", "cdidx backfill-fold [--db ] [--dry-run] [--no-checkpoint] [--json]"), ("optimize", "cdidx optimize [--db ] [--json]"), - ("vacuum", "cdidx vacuum [--db ] [--json]"), + ("vacuum", "cdidx vacuum [--db ] [--dry-run] [--json]"), ("index-commits", "cdidx index --commits [commit-ref ...] [--db ] [--verbose] [--dry-run] [--json] [--memory-trace] [--duration-format ] [--max-file-bytes ] [--include-symbol-kind [,]] [--exclude-symbol-kind [,]]"), ("index-changed-between", "cdidx index --changed-between [--db ] [--verbose] [--dry-run] [--json] [--memory-trace] [--duration-format ] [--max-file-bytes ] [--include-symbol-kind [,]] [--exclude-symbol-kind [,]]"), ("index-files", "cdidx index --files [path ...] [--db ] [--verbose] [--dry-run] [--json] [--memory-trace] [--duration-format ] [--max-file-bytes ] [--include-symbol-kind [,]] [--exclude-symbol-kind [,]]"), @@ -1080,7 +1080,7 @@ private static void PrintExamples() Console.WriteLine(" cdidx ./myproject Index a project"); Console.WriteLine(" cdidx backfill-fold Upgrade folded-name columns in an existing DB"); Console.WriteLine(" cdidx optimize Optimize FTS5 segments in an existing DB"); - Console.WriteLine(" cdidx vacuum --json Reclaim DB free pages and report page metrics"); + Console.WriteLine(" cdidx vacuum --dry-run --json Estimate DB free pages and maintenance guidance"); Console.WriteLine(" cdidx index ./myproject --commits abc123 Update DB from one commit"); Console.WriteLine(" cdidx index ./myproject --commits abc123 def456"); Console.WriteLine(" Update DB from multiple commits"); diff --git a/src/CodeIndex/Cli/DbCommandRunner.cs b/src/CodeIndex/Cli/DbCommandRunner.cs index dc66f34066..c97246b0da 100644 --- a/src/CodeIndex/Cli/DbCommandRunner.cs +++ b/src/CodeIndex/Cli/DbCommandRunner.cs @@ -126,10 +126,13 @@ private static int RunIntegrityCheck(DbCommandOptions options, JsonSerializerOpt if (options.Json) { + var severity = ok ? "ok" : "error"; Console.WriteLine(JsonSerializer.Serialize( new DbIntegrityCheckJsonResult( displayDbPath, ok, + severity, + ok ? "integrity_ok" : "integrity_failed", ok ? new List() : issues, result.Truncated, result.RowsTruncated, @@ -181,11 +184,16 @@ private static int RunSchema(DbCommandOptions options, JsonSerializerOptions jso var fullPath = DbPathResolver.FormatDbPathForDisplay(dbPath); if (options.Json) { + var severity = schema.Truncated ? "warn" : "ok"; var jsonContext = CliJsonSerializerContextFactory.Create(jsonOptions); Console.WriteLine(JsonSerializer.Serialize( new DbSchemaJsonResult( fullPath, schema.UserVersion, + severity, + schema.Truncated ? "schema_truncated" : "schema_ok", + schema.ObjectTypeCounts, + schema.ObjectTypeOmittedCounts, schema.Entries, schema.Truncated, schema.EntriesTruncated, @@ -468,6 +476,7 @@ private static DbSchemaReadResult ReadSchema(string dbPath) versionCmd.CommandText = "PRAGMA user_version"; var rawVersion = versionCmd.ExecuteScalar(); var userVersion = rawVersion is long l ? (int)l : (rawVersion is int i ? i : 0); + var objectTypeCounts = ReadSchemaObjectTypeCounts(connection); using var cmd = connection.CreateCommand(); cmd.CommandText = @" @@ -500,7 +509,42 @@ WHERE type IN ('table', 'index', 'trigger', 'view') boundedSql.Text)); } - return new DbSchemaReadResult(userVersion, entries, entriesTruncated, sqlTruncated); + var emittedTypeCounts = entries + .GroupBy(entry => entry.Type, StringComparer.Ordinal) + .ToDictionary(group => group.Key, group => group.Count(), StringComparer.Ordinal); + var omittedTypeCounts = objectTypeCounts.ToDictionary( + kv => kv.Key, + kv => Math.Max(0, kv.Value - (emittedTypeCounts.TryGetValue(kv.Key, out var emitted) ? emitted : 0)), + StringComparer.Ordinal); + + return new DbSchemaReadResult(userVersion, entries, objectTypeCounts, omittedTypeCounts, entriesTruncated, sqlTruncated); + } + + private static Dictionary ReadSchemaObjectTypeCounts(SqliteConnection connection) + { + var counts = new Dictionary(StringComparer.Ordinal) + { + ["table"] = 0, + ["index"] = 0, + ["trigger"] = 0, + ["view"] = 0, + }; + + using var cmd = connection.CreateCommand(); + cmd.CommandText = @" + SELECT type, COUNT(*) + FROM sqlite_master + WHERE type IN ('table', 'index', 'trigger', 'view') + GROUP BY type"; + using var reader = cmd.ExecuteReader(); + while (reader.Read()) + { + var type = reader.GetString(0); + if (counts.ContainsKey(type)) + counts[type] = Convert.ToInt32(reader.GetInt64(1), System.Globalization.CultureInfo.InvariantCulture); + } + + return counts; } private static DbIntegrityCheckReadResult BoundIntegrityRows(IEnumerable rawRows) @@ -1048,6 +1092,8 @@ internal sealed record DbIntegrityCheckReadResult(List Rows, bool RowsTr internal sealed record DbSchemaReadResult( int UserVersion, List Entries, + Dictionary ObjectTypeCounts, + Dictionary ObjectTypeOmittedCounts, bool EntriesTruncated, bool SqlTruncated) { diff --git a/src/CodeIndex/Cli/DiffCommandRunner.cs b/src/CodeIndex/Cli/DiffCommandRunner.cs index 51c4fa1611..84f61f8259 100644 --- a/src/CodeIndex/Cli/DiffCommandRunner.cs +++ b/src/CodeIndex/Cli/DiffCommandRunner.cs @@ -234,6 +234,9 @@ OR key LIKE 'metadata_target_version_%' OR key = 'unknown_extension_file_paths_json' OR key = 'unknown_extension_files_truncated' OR key = 'unknown_extension_file_path_limit' + OR key = 'unknown_extension_extension_counts_json' + OR key = 'unknown_extension_category_counts_json' + OR key = 'unknown_extension_groups_json' OR key = 'cdidx_writer_version' ORDER BY key, diff --git a/src/CodeIndex/Cli/IndexCommandRunner.cs b/src/CodeIndex/Cli/IndexCommandRunner.cs index 98179b75ef..f23051bc2a 100644 --- a/src/CodeIndex/Cli/IndexCommandRunner.cs +++ b/src/CodeIndex/Cli/IndexCommandRunner.cs @@ -138,6 +138,16 @@ internal static int Run(string[] indexArgs, JsonSerializerOptions jsonOptions, C } catch (OperationCanceledException) when (indexCancellation.IsCancellationRequested) { + TryStampLastFailedIndexRun( + resolvedDbPath, + status: "partial", + mode, + runStartedAtUtc, + stopwatch.ElapsedMilliseconds, + filesProcessed: 0, + filesTotal: null, + CommandErrorCodes.Interrupted, + reason: "cancelled"); return WriteInterruptedResult(options.Json, jsonOptions, filesProcessed: 0, filesTotal: null); } @@ -268,18 +278,58 @@ internal static int Run(string[] indexArgs, JsonSerializerOptions jsonOptions, C } catch (IndexInterruptedException ex) { + TryStampLastFailedIndexRun( + resolvedDbPath, + status: "partial", + mode, + runStartedAtUtc, + stopwatch.ElapsedMilliseconds, + ex.FilesProcessed, + ex.FilesTotal, + CommandErrorCodes.Interrupted, + reason: "interrupted"); return WriteInterruptedResult(options.Json, jsonOptions, ex.FilesProcessed, ex.FilesTotal); } catch (OperationCanceledException) when (indexCancellation.IsCancellationRequested) { + TryStampLastFailedIndexRun( + resolvedDbPath, + status: "partial", + mode, + runStartedAtUtc, + stopwatch.ElapsedMilliseconds, + filesProcessed: 0, + filesTotal: null, + CommandErrorCodes.Interrupted, + reason: "cancelled"); return WriteInterruptedResult(options.Json, jsonOptions, filesProcessed: 0, filesTotal: null); } catch (IndexExtractionStalledException ex) { + TryStampLastFailedIndexRun( + resolvedDbPath, + status: "failed", + mode, + runStartedAtUtc, + stopwatch.ElapsedMilliseconds, + ex.FilesProcessed, + ex.FilesTotal, + CommandErrorCodes.IndexExtractionStalled, + reason: "extraction_stalled"); return WriteExtractionStalledResult(options.Json, jsonOptions, ex); } catch (Exception ex) when (IsDatabaseFilesystemError(ex)) { + TryStampLastFailedIndexRun( + resolvedDbPath, + status: "failed", + mode, + runStartedAtUtc, + stopwatch.ElapsedMilliseconds, + filesProcessed: null, + filesTotal: null, + CommandErrorCodes.DbError, + reason: "database_filesystem_error"); return WriteDatabaseFilesystemError(options.Json, jsonOptions, resolvedDbPath, ex); } @@ -376,6 +426,44 @@ private static void StampLastIndexRunMetadata( writer.SetMeta(DbContext.LastIndexRunPeakMemoryMbMetaKey, memoryTimeline == null ? null : (memoryTimeline.PeakWorkingSetBytes / (1024 * 1024)).ToString(System.Globalization.CultureInfo.InvariantCulture)); + writer.ClearLastFailedIndexRunMetadata(); + } + + private static void TryStampLastFailedIndexRun( + string dbPath, + string status, + string mode, + DateTime startedAtUtc, + long durationMs, + long? filesProcessed, + long? filesTotal, + string errorCode, + string reason) + { + if (string.IsNullOrWhiteSpace(dbPath) + || dbPath.StartsWith("file:", StringComparison.OrdinalIgnoreCase) + || !File.Exists(dbPath)) + { + return; + } + + try + { + using var db = new DbContext(dbPath); + db.InitializeSchema(); + var writer = new DbWriter(db); + writer.SetMeta(DbContext.LastFailedIndexRunStatusMetaKey, status); + writer.SetMeta(DbContext.LastFailedIndexRunModeMetaKey, mode); + writer.SetMeta(DbContext.LastFailedIndexRunStartedAtMetaKey, startedAtUtc.ToString("o", System.Globalization.CultureInfo.InvariantCulture)); + writer.SetMeta(DbContext.LastFailedIndexRunDurationMsMetaKey, durationMs.ToString(System.Globalization.CultureInfo.InvariantCulture)); + writer.SetMeta(DbContext.LastFailedIndexRunFilesProcessedMetaKey, filesProcessed?.ToString(System.Globalization.CultureInfo.InvariantCulture)); + writer.SetMeta(DbContext.LastFailedIndexRunFilesTotalMetaKey, filesTotal?.ToString(System.Globalization.CultureInfo.InvariantCulture)); + writer.SetMeta(DbContext.LastFailedIndexRunErrorCodeMetaKey, errorCode); + writer.SetMeta(DbContext.LastFailedIndexRunReasonMetaKey, reason); + } + catch (Exception ex) when (ex is CodeIndexException or IOException or UnauthorizedAccessException or NotSupportedException or SqliteException) + { + } } private static long SumReadableFileBytes(IEnumerable paths) diff --git a/src/CodeIndex/Cli/JsonOutputContracts.cs b/src/CodeIndex/Cli/JsonOutputContracts.cs index bb5066b1ee..dc75240e02 100644 --- a/src/CodeIndex/Cli/JsonOutputContracts.cs +++ b/src/CodeIndex/Cli/JsonOutputContracts.cs @@ -52,6 +52,8 @@ internal sealed record UpgradeJsonResult( internal sealed record DbIntegrityCheckJsonResult( [property: JsonPropertyName("db_path")] string DbPath, [property: JsonPropertyName("ok")] bool Ok, + [property: JsonPropertyName("severity")] string Severity, + [property: JsonPropertyName("diagnostic_code")] string DiagnosticCode, [property: JsonPropertyName("issues")] List Issues, [property: JsonPropertyName("truncated")] bool Truncated = false, [property: JsonPropertyName("rows_truncated")] bool RowsTruncated = false, @@ -98,6 +100,10 @@ internal sealed record DbSchemaEntryJsonResult( internal sealed record DbSchemaJsonResult( [property: JsonPropertyName("db_path")] string DbPath, [property: JsonPropertyName("user_version")] int UserVersion, + [property: JsonPropertyName("severity")] string Severity, + [property: JsonPropertyName("diagnostic_code")] string DiagnosticCode, + [property: JsonPropertyName("object_type_counts")] Dictionary ObjectTypeCounts, + [property: JsonPropertyName("object_type_omitted_counts")] Dictionary ObjectTypeOmittedCounts, [property: JsonPropertyName("entries")] List Entries, [property: JsonPropertyName("truncated")] bool Truncated = false, [property: JsonPropertyName("entries_truncated")] bool EntriesTruncated = false, @@ -484,10 +490,14 @@ internal sealed record VersionInfoJsonResult( [JsonSerializable(typeof(ExtractorRegistryDiagnostic))] [JsonSerializable(typeof(ExtractorRegistryStatus))] [JsonSerializable(typeof(StatusResult))] +[JsonSerializable(typeof(StatusFailedOrPartialIndexRun))] [JsonSerializable(typeof(StatusReadinessDegradation))] [JsonSerializable(typeof(StatusDbPragmaSettings))] [JsonSerializable(typeof(StatusLastIndexRun))] +[JsonSerializable(typeof(StatusMaintenanceGuidance))] [JsonSerializable(typeof(StatusProcessMetrics))] +[JsonSerializable(typeof(StatusRepairCommand))] +[JsonSerializable(typeof(StatusUnknownExtensionGroup))] [JsonSerializable(typeof(SuggestionDetailJsonResult))] [JsonSerializable(typeof(SuggestionExportJsonResult))] [JsonSerializable(typeof(SuggestionIssueDraftDuplicateMatchJsonResult))] diff --git a/src/CodeIndex/Cli/QueryCommandRunner.cs b/src/CodeIndex/Cli/QueryCommandRunner.cs index 3b0a8f6536..92bb173c8d 100644 --- a/src/CodeIndex/Cli/QueryCommandRunner.cs +++ b/src/CodeIndex/Cli/QueryCommandRunner.cs @@ -253,6 +253,7 @@ private sealed record StatusReadinessField( "--list-recipes", "--read-only", "--immutable", + "--dry-run", "--pretty", "--compact", "--body-only", @@ -3828,7 +3829,10 @@ public static int RunStatus( ? BuildStatusCheckFailures(status, options.StatusCheckScopes) : Array.Empty(); if (options.CheckWorkspace) + { status.FailedChecks = checkFailures.Select(f => f.Name).ToList(); + status.RepairCommands = BuildStatusRepairCommands(status, checkFailures, options); + } if (options.Json) { @@ -4175,7 +4179,7 @@ public static int RunVacuum(string[] cmdArgs, JsonSerializerOptions jsonOptions) } using var db = new DbContext(options.DbPath); - var result = db.RunIncrementalVacuum(); + var result = db.RunIncrementalVacuum(options.DryRun); if (options.Json) { Console.WriteLine(JsonSerializer.Serialize( @@ -4184,10 +4188,17 @@ public static int RunVacuum(string[] cmdArgs, JsonSerializerOptions jsonOptions) } else { - Console.WriteLine($"Vacuum complete: reclaimed {result.PagesReclaimed:N0} page(s) ({result.BytesReclaimed:N0} bytes)."); + Console.WriteLine(result.DryRun + ? $"Vacuum dry run: estimated reclaimable {result.EstimatedPagesReclaimable:N0} page(s) ({result.EstimatedBytesReclaimable:N0} bytes)." + : $"Vacuum complete: reclaimed {result.PagesReclaimed:N0} page(s) ({result.BytesReclaimed:N0} bytes)."); Console.WriteLine(ConsoleUi.FormatSummaryLine("Page size", $"{result.PageSize:N0} bytes")); Console.WriteLine(ConsoleUi.FormatSummaryLine("Pages", $"{result.PageCountBefore:N0} -> {result.PageCountAfter:N0}")); Console.WriteLine(ConsoleUi.FormatSummaryLine("Freelist", $"{result.FreelistCountBefore:N0} -> {result.FreelistCountAfter:N0}")); + Console.WriteLine(ConsoleUi.FormatSummaryLine("AutoVac", $"{result.AutoVacuumModeBeforeName} -> {result.AutoVacuumModeAfterName}")); + if (result.MaintenanceGuidance.RecommendedCommand != "none") + Console.WriteLine(ConsoleUi.FormatSummaryLine("Recommend", result.MaintenanceGuidance.RecommendedCommand)); + if (!string.IsNullOrWhiteSpace(result.MaintenanceGuidance.PostMaintenanceFollowUp)) + Console.WriteLine(ConsoleUi.FormatSummaryLine("Follow-up", result.MaintenanceGuidance.PostMaintenanceFollowUp)); } return CommandExitCodes.Success; @@ -6145,6 +6156,7 @@ public static QueryCommandOptions ParseArgs( bool exactSubstring = false; bool dbPathExplicit = false; bool readOnly = false; + bool dryRun = false; bool checkWorkspace = false; TimeSpan? staleAfter = null; HashSet? statusCheckScopes = null; @@ -6295,6 +6307,9 @@ void WarnIfDuplicateSingleValueOption(string canonicalName, string newValue) case "--immutable": readOnly = true; break; + case "--dry-run": + dryRun = true; + break; case "--pretty": break; case "--compact": @@ -7013,6 +7028,7 @@ void WarnIfDuplicateSingleValueOption(string canonicalName, string newValue) DbPath = resolvedDbPath, DbPathExplicit = dbPathExplicit, ReadOnly = readOnly, + DryRun = dryRun, DataDir = dbResolution.DataDir, DataDirSource = dbResolution.DataDirSource, Json = json ?? jsonDefault, @@ -8896,6 +8912,127 @@ private static IReadOnlyList BuildStatusCheckFailures(Status return failures; } + private static List? BuildStatusRepairCommands( + StatusResult status, + IReadOnlyList failures, + QueryCommandOptions options) + { + if (failures.Count == 0) + return null; + + var commands = new List(); + foreach (var failure in failures) + { + var command = failure.Name switch + { + "workspace_stale" or "workspace_unavailable" => BuildIndexRepairCommand( + status, + options, + failure.Name, + rebuild: false, + "Re-runs indexing for the current workspace snapshot."), + "graph_table_available" or "issues_table_available" or "file_issues_data_current" + or "sql_graph_contract_ready" or "csharp_symbol_name_ready" or "csharp_metadata_target_ready" + => BuildIndexRepairCommand( + status, + options, + failure.Name, + rebuild: false, + "Rewrites stale or missing index metadata before query results are trusted."), + "hotspot_family_ready" or "index_newer_than_reader" => BuildIndexRepairCommand( + status, + options, + failure.Name, + rebuild: true, + "Performs a full rebuild because partial updates cannot prove every indexed row was restamped."), + "fold_ready" => BuildBackfillFoldRepairCommand(options, failure.Name), + "migration_in_progress" => BuildStatusCheckRepairCommand(options, failure.Name), + _ => null, + }; + if (command != null) + commands.Add(command); + } + + return commands.Count == 0 ? null : commands; + } + + private static StatusRepairCommand BuildIndexRepairCommand( + StatusResult status, + QueryCommandOptions options, + string reason, + bool rebuild, + string safetyNote) + { + var args = new List + { + "index", + string.IsNullOrWhiteSpace(status.ProjectRoot) ? "." : status.ProjectRoot!, + }; + if (options.DbPathExplicit) + { + args.Add("--db"); + args.Add(ResolveWritableDbPathOrPlaceholder(options.DbPath)); + } + if (rebuild) + args.Add("--rebuild"); + + return new StatusRepairCommand + { + Name = "cdidx", + Args = args, + Reason = reason, + SafetyNotes = + [ + safetyNote, + "Avoid running concurrently with another cdidx index writer for the same database.", + ], + }; + } + + private static StatusRepairCommand BuildBackfillFoldRepairCommand(QueryCommandOptions options, string reason) + { + var args = new List { "backfill-fold" }; + if (options.DbPathExplicit) + { + args.Add("--db"); + args.Add(ResolveWritableDbPathOrPlaceholder(options.DbPath)); + } + + return new StatusRepairCommand + { + Name = "cdidx", + Args = args, + Reason = reason, + SafetyNotes = + [ + "Restamps folded-name columns in place without reparsing source files.", + "Use a full index rebuild instead if the database must be regenerated from source.", + ], + }; + } + + private static StatusRepairCommand BuildStatusCheckRepairCommand(QueryCommandOptions options, string reason) + { + var args = new List { "status", "--check", "--json" }; + if (options.DbPathExplicit) + { + args.Add("--db"); + args.Add(options.DbPath); + } + + return new StatusRepairCommand + { + Name = "cdidx", + Args = args, + Reason = reason, + SafetyNotes = + [ + "Wait for the active index or migration writer to finish before rerunning status.", + "Do not start a second writer unless the existing writer is known to be gone.", + ], + }; + } + private static void WriteStatusCheckDiagnostics(IReadOnlyList failures) { foreach (var failure in failures) @@ -9996,6 +10133,7 @@ public sealed class QueryCommandOptions public string DbPath { get; init; } = Path.Combine(".cdidx", "codeindex.db"); public bool DbPathExplicit { get; init; } public bool ReadOnly { get; init; } + public bool DryRun { get; init; } public string? DataDir { get; init; } public string? DataDirSource { get; init; } public bool Json { get; init; } diff --git a/src/CodeIndex/Database/DbContext.cs b/src/CodeIndex/Database/DbContext.cs index 04c5e5b0b7..d96d553d35 100644 --- a/src/CodeIndex/Database/DbContext.cs +++ b/src/CodeIndex/Database/DbContext.cs @@ -591,9 +591,9 @@ private void ConfigureAutoVacuumForEmptyDatabase() Execute("PRAGMA auto_vacuum=INCREMENTAL"); } - public VacuumResult RunIncrementalVacuum() + public VacuumResult RunIncrementalVacuum(bool dryRun = false) { - if (_isReadOnly) + if (_isReadOnly && !dryRun) { throw new CodeIndexException( code: CommandErrorCodes.DbNotWritable, @@ -604,31 +604,94 @@ public VacuumResult RunIncrementalVacuum() } var before = ReadVacuumMetrics(); - if (ReadAutoVacuumMode() == 2) + if (!dryRun && before.AutoVacuumMode == 2) { Execute($"PRAGMA incremental_vacuum({before.FreelistCount})"); } - else + else if (!dryRun) { Execute("PRAGMA auto_vacuum=INCREMENTAL"); Execute("VACUUM"); } - var after = ReadVacuumMetrics(); + var after = dryRun ? before : ReadVacuumMetrics(); + var pagesReclaimed = dryRun ? 0 : Math.Max(0, before.PageCount - after.PageCount); + var bytesReclaimed = pagesReclaimed * after.PageSize; + var estimatedPagesReclaimable = Math.Max(0, before.FreelistCount); + var estimatedBytesReclaimable = estimatedPagesReclaimable * before.PageSize; + var guidance = MaintenanceGuidanceBuilder.Build(new MaintenanceMetrics( + after.PageCount, + after.FreelistCount, + after.PageSize, + after.WalSizeBytes, + after.DbSizeBytes, + after.AutoVacuumMode)); return new VacuumResult( - Status: "ok", + Status: dryRun ? "dry_run" : "ok", + DryRun: dryRun, PageSize: after.PageSize, PageCountBefore: before.PageCount, FreelistCountBefore: before.FreelistCount, PageCountAfter: after.PageCount, FreelistCountAfter: after.FreelistCount, - PagesReclaimed: Math.Max(0, before.PageCount - after.PageCount), - BytesReclaimed: Math.Max(0, before.PageCount - after.PageCount) * after.PageSize); + PagesReclaimed: pagesReclaimed, + BytesReclaimed: bytesReclaimed, + EstimatedPagesReclaimable: estimatedPagesReclaimable, + EstimatedBytesReclaimable: estimatedBytesReclaimable, + DbSizeBytesBefore: before.DbSizeBytes, + WalSizeBytesBefore: before.WalSizeBytes, + DbSizeBytesAfter: after.DbSizeBytes, + WalSizeBytesAfter: after.WalSizeBytes, + AutoVacuumModeBefore: before.AutoVacuumMode, + AutoVacuumModeBeforeName: MaintenanceGuidanceBuilder.FormatAutoVacuumMode(before.AutoVacuumMode) ?? "unknown", + AutoVacuumModeAfter: after.AutoVacuumMode, + AutoVacuumModeAfterName: MaintenanceGuidanceBuilder.FormatAutoVacuumMode(after.AutoVacuumMode) ?? "unknown", + MaintenanceGuidance: guidance); + } + + private VacuumMetrics ReadVacuumMetrics() + => new( + ReadPragmaLong("page_count"), + ReadPragmaLong("freelist_count"), + ReadPragmaLong("page_size"), + ReadAutoVacuumMode(), + TryGetDatabaseFileSize(), + TryGetWalFileSize()); + + private long ReadAutoVacuumMode() => ReadPragmaLong("auto_vacuum"); + + private long? TryGetDatabaseFileSize() + { + var path = _connection.DataSource; + if (string.IsNullOrWhiteSpace(path) || path.StartsWith("file:", StringComparison.OrdinalIgnoreCase)) + return null; + + try + { + var info = new FileInfo(path); + return info.Exists ? info.Length : null; + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException or NotSupportedException or ArgumentException) + { + return null; + } } - private (long PageCount, long FreelistCount, long PageSize) ReadVacuumMetrics() - => (ReadPragmaLong("page_count"), ReadPragmaLong("freelist_count"), ReadPragmaLong("page_size")); + private long? TryGetWalFileSize() + { + var path = _connection.DataSource; + if (string.IsNullOrWhiteSpace(path) || path.StartsWith("file:", StringComparison.OrdinalIgnoreCase)) + return null; - private long ReadAutoVacuumMode() => ReadPragmaLong("auto_vacuum"); + try + { + var info = new FileInfo(path + "-wal"); + return info.Exists ? info.Length : 0; + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException or NotSupportedException or ArgumentException) + { + return null; + } + } private long ReadPragmaLong(string name) { @@ -637,6 +700,14 @@ private long ReadPragmaLong(string name) return Convert.ToInt64(cmd.ExecuteScalar(), CultureInfo.InvariantCulture); } + private readonly record struct VacuumMetrics( + long PageCount, + long FreelistCount, + long PageSize, + long AutoVacuumMode, + long? DbSizeBytes, + long? WalSizeBytes); + private void EnsureWritableUserVersionSupported(string dbPath) { var userVersion = GetUserVersion(); @@ -1433,6 +1504,14 @@ public static string BuildIncompleteHotspotFamilyMarkerFingerprint(string? finge public const string LastIndexRunRowsUpsertedMetaKey = "last_index_run_rows_upserted"; public const string LastIndexRunRowsDeletedMetaKey = "last_index_run_rows_deleted"; public const string LastIndexRunPeakMemoryMbMetaKey = "last_index_run_peak_memory_mb"; + public const string LastFailedIndexRunStatusMetaKey = "last_failed_index_run_status"; + public const string LastFailedIndexRunModeMetaKey = "last_failed_index_run_mode"; + public const string LastFailedIndexRunStartedAtMetaKey = "last_failed_index_run_started_at"; + public const string LastFailedIndexRunDurationMsMetaKey = "last_failed_index_run_duration_ms"; + public const string LastFailedIndexRunFilesProcessedMetaKey = "last_failed_index_run_files_processed"; + public const string LastFailedIndexRunFilesTotalMetaKey = "last_failed_index_run_files_total"; + public const string LastFailedIndexRunErrorCodeMetaKey = "last_failed_index_run_error_code"; + public const string LastFailedIndexRunReasonMetaKey = "last_failed_index_run_reason"; // Issue #1585: count of files seen by the most recent successful full-repository scan // whose non-empty extension did not map to a known language. This is a scan coverage // signal, not an indexed-file count, and is omitted by readers until a current index pass @@ -1444,6 +1523,9 @@ public static string BuildIncompleteHotspotFamilyMarkerFingerprint(string? finge public const string UnknownExtensionFilePathsMetaKey = "unknown_extension_file_paths_json"; public const string UnknownExtensionFilesTruncatedMetaKey = "unknown_extension_files_truncated"; public const string UnknownExtensionFilePathLimitMetaKey = "unknown_extension_file_path_limit"; + public const string UnknownExtensionExtensionCountsMetaKey = "unknown_extension_extension_counts_json"; + public const string UnknownExtensionCategoryCountsMetaKey = "unknown_extension_category_counts_json"; + public const string UnknownExtensionGroupsMetaKey = "unknown_extension_groups_json"; public const int UnknownExtensionFilePathSampleLimit = 50; public const string BatchInProgressMetaKey = "batch_in_progress"; // Issue #1546: case-sensitivity of the workspace filesystem the most recent successful diff --git a/src/CodeIndex/Database/DbReader.FilesStatus.cs b/src/CodeIndex/Database/DbReader.FilesStatus.cs index f2eddd1997..b90208a627 100644 --- a/src/CodeIndex/Database/DbReader.FilesStatus.cs +++ b/src/CodeIndex/Database/DbReader.FilesStatus.cs @@ -656,11 +656,21 @@ GROUP BY COALESCE(f.lang, 'unknown'), s.kind var unknownExtensionFiles = ParseMetaStringList(TryGetMetaStringInternal(DbContext.UnknownExtensionFilePathsMetaKey)); var unknownExtensionFilesTruncated = ParseMetaBool(TryGetMetaStringInternal(DbContext.UnknownExtensionFilesTruncatedMetaKey)); var unknownExtensionFilePathLimit = ParseMetaLong(TryGetMetaStringInternal(DbContext.UnknownExtensionFilePathLimitMetaKey)); + var unknownExtensionExtensionCounts = UnknownExtensionClassifier.DeserializeCounts(TryGetMetaStringInternal(DbContext.UnknownExtensionExtensionCountsMetaKey)); + var unknownExtensionCategoryCounts = UnknownExtensionClassifier.DeserializeCounts(TryGetMetaStringInternal(DbContext.UnknownExtensionCategoryCountsMetaKey)); + var unknownExtensionGroups = UnknownExtensionClassifier.DeserializeGroups(TryGetMetaStringInternal(DbContext.UnknownExtensionGroupsMetaKey)); if (unknownExtensionFiles != null) { unknownExtensionFilesTruncated ??= unknownExtensionFileCount.HasValue && unknownExtensionFileCount.Value > unknownExtensionFiles.Count; unknownExtensionFilePathLimit ??= unknownExtensionFiles.Count; + if (unknownExtensionExtensionCounts == null || unknownExtensionCategoryCounts == null || unknownExtensionGroups == null) + { + var fallbackClassification = UnknownExtensionClassifier.Classify(unknownExtensionFiles); + unknownExtensionExtensionCounts ??= fallbackClassification.ExtensionCounts; + unknownExtensionCategoryCounts ??= fallbackClassification.CategoryCounts; + unknownExtensionGroups ??= fallbackClassification.Groups; + } } // #1546: workspace case-sensitivity stamp. Read inside the SHARED snapshot for // consistency with the other freshness signals; missing on legacy DBs. @@ -669,11 +679,19 @@ GROUP BY COALESCE(f.lang, 'unknown'), s.kind var dbPragmaSettings = GetDbPragmaSettings(); var dbSizeBytes = TryGetDatabaseFileSize(); var walSizeBytes = TryGetWalFileSize(); + var maintenanceGuidance = MaintenanceGuidanceBuilder.Build(new MaintenanceMetrics( + dbPragmaSettings.PageCount, + dbPragmaSettings.FreelistCount, + dbPragmaSettings.PageSize, + walSizeBytes, + dbSizeBytes, + dbPragmaSettings.AutoVacuum)); var lastIndexRun = GetLastIndexRun(); var batchInProgress = string.Equals( TryGetMetaStringInternal(DbContext.BatchInProgressMetaKey), "true", StringComparison.OrdinalIgnoreCase); + var lastFailedOrPartialIndexRun = GetLastFailedOrPartialIndexRun(batchInProgress); var result = new StatusResult { @@ -685,6 +703,9 @@ GROUP BY COALESCE(f.lang, 'unknown'), s.kind UnknownExtensionFiles = unknownExtensionFiles, UnknownExtensionFilesTruncated = unknownExtensionFilesTruncated, UnknownExtensionFilePathLimit = unknownExtensionFilePathLimit, + UnknownExtensionExtensionCounts = unknownExtensionExtensionCounts, + UnknownExtensionCategoryCounts = unknownExtensionCategoryCounts, + UnknownExtensionGroups = unknownExtensionGroups, IndexedAt = freshness.IndexedAt, LastWorkspaceFreshenedAt = lastIndexRun?.StartedAt ?? indexedHeadTimestamp, LatestModified = freshness.LatestModified, @@ -712,10 +733,12 @@ GROUP BY COALESCE(f.lang, 'unknown'), s.kind IndexNewerThanReaderReason = _indexNewerThanReaderReason, PathCaseSensitive = pathCaseSensitive, DbPragmaSettings = dbPragmaSettings, + MaintenanceGuidance = maintenanceGuidance, DbSizeBytes = dbSizeBytes, WalSizeBytes = walSizeBytes, Process = StatusProcessMetrics.Capture(), LastIndexRun = lastIndexRun, + LastFailedOrPartialIndexRun = lastFailedOrPartialIndexRun, ReadOnlyFallback = _readOnlyFallback, WalCheckpointAttempted = _walCheckpointAttempted, WalCheckpointSucceeded = _walCheckpointSucceeded, @@ -751,6 +774,7 @@ private long ExecuteScalar(string sql) PageCount = ExecuteNullableLong("PRAGMA page_count"), FreelistCount = ExecuteNullableLong("PRAGMA freelist_count"), PageSize = ExecuteNullableLong("PRAGMA page_size"), + AutoVacuum = ExecuteNullableLong("PRAGMA auto_vacuum"), }; private long? TryGetDatabaseFileSize() @@ -820,6 +844,41 @@ private long ExecuteScalar(string sql) }; } + private StatusFailedOrPartialIndexRun? GetLastFailedOrPartialIndexRun(bool batchInProgress) + { + var status = TryGetMetaStringInternal(DbContext.LastFailedIndexRunStatusMetaKey); + var mode = TryGetMetaStringInternal(DbContext.LastFailedIndexRunModeMetaKey); + var startedAt = ParseMetaDateTime(TryGetMetaStringInternal(DbContext.LastFailedIndexRunStartedAtMetaKey)); + var durationMs = ParseMetaLong(TryGetMetaStringInternal(DbContext.LastFailedIndexRunDurationMsMetaKey)); + var filesProcessed = ParseMetaLong(TryGetMetaStringInternal(DbContext.LastFailedIndexRunFilesProcessedMetaKey)); + var filesTotal = ParseMetaLong(TryGetMetaStringInternal(DbContext.LastFailedIndexRunFilesTotalMetaKey)); + var errorCode = TryGetMetaStringInternal(DbContext.LastFailedIndexRunErrorCodeMetaKey); + var reason = TryGetMetaStringInternal(DbContext.LastFailedIndexRunReasonMetaKey); + if (status == null && mode == null && startedAt == null && durationMs == null && filesProcessed == null + && filesTotal == null && errorCode == null && reason == null) + { + return batchInProgress + ? new StatusFailedOrPartialIndexRun + { + Status = "partial", + Reason = "batch_in_progress", + } + : null; + } + + return new StatusFailedOrPartialIndexRun + { + Status = status, + Mode = mode, + StartedAt = startedAt, + DurationMs = durationMs, + FilesProcessed = filesProcessed, + FilesTotal = filesTotal, + ErrorCode = errorCode, + Reason = reason, + }; + } + private string? ExecuteScalarString(string sql) { using var cmd = _conn.CreateCommand(); diff --git a/src/CodeIndex/Database/DbWriter.cs b/src/CodeIndex/Database/DbWriter.cs index a6515c7d35..d93205a6a2 100644 --- a/src/CodeIndex/Database/DbWriter.cs +++ b/src/CodeIndex/Database/DbWriter.cs @@ -2178,6 +2178,18 @@ public void WriteCdidxWriterVersion(string version) SetMeta(DbContext.CdidxWriterVersionMetaKey, version); } + public void ClearLastFailedIndexRunMetadata() + { + SetMeta(DbContext.LastFailedIndexRunStatusMetaKey, null); + SetMeta(DbContext.LastFailedIndexRunModeMetaKey, null); + SetMeta(DbContext.LastFailedIndexRunStartedAtMetaKey, null); + SetMeta(DbContext.LastFailedIndexRunDurationMsMetaKey, null); + SetMeta(DbContext.LastFailedIndexRunFilesProcessedMetaKey, null); + SetMeta(DbContext.LastFailedIndexRunFilesTotalMetaKey, null); + SetMeta(DbContext.LastFailedIndexRunErrorCodeMetaKey, null); + SetMeta(DbContext.LastFailedIndexRunReasonMetaKey, null); + } + /// /// Stamp unknown-extension scan coverage from the latest successful full-worktree scan. /// Stores the total count plus a bounded path sample so status callers can identify the @@ -2191,6 +2203,7 @@ public void WriteUnknownExtensionFileMetadata(IReadOnlyList paths) var sample = JsonStringListCodec.TakeSerializableSample( paths, DbContext.UnknownExtensionFilePathSampleLimit); + var classification = UnknownExtensionClassifier.Classify(paths); SetMeta( DbContext.UnknownExtensionFileCountMetaKey, paths.Count.ToString(System.Globalization.CultureInfo.InvariantCulture)); @@ -2203,6 +2216,15 @@ public void WriteUnknownExtensionFileMetadata(IReadOnlyList paths) SetMeta( DbContext.UnknownExtensionFilePathLimitMetaKey, DbContext.UnknownExtensionFilePathSampleLimit.ToString(System.Globalization.CultureInfo.InvariantCulture)); + SetMeta( + DbContext.UnknownExtensionExtensionCountsMetaKey, + UnknownExtensionClassifier.SerializeCounts(classification.ExtensionCounts)); + SetMeta( + DbContext.UnknownExtensionCategoryCountsMetaKey, + UnknownExtensionClassifier.SerializeCounts(classification.CategoryCounts)); + SetMeta( + DbContext.UnknownExtensionGroupsMetaKey, + UnknownExtensionClassifier.SerializeGroups(classification.Groups)); } /// diff --git a/src/CodeIndex/Database/MaintenanceGuidanceBuilder.cs b/src/CodeIndex/Database/MaintenanceGuidanceBuilder.cs new file mode 100644 index 0000000000..dba6a6b9c1 --- /dev/null +++ b/src/CodeIndex/Database/MaintenanceGuidanceBuilder.cs @@ -0,0 +1,147 @@ +using System.Globalization; +using System.Text.Json.Serialization; + +namespace CodeIndex.Database; + +public sealed class StatusMaintenanceGuidance +{ + [JsonPropertyName("wal_state")] + public string WalState { get; set; } = "unknown"; + [JsonPropertyName("wal_threshold_bytes")] + public long WalThresholdBytes { get; set; } + [JsonPropertyName("freelist_state")] + public string FreelistState { get; set; } = "unknown"; + [JsonPropertyName("freelist_ratio")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public double? FreelistRatio { get; set; } + [JsonPropertyName("freelist_threshold_ratio")] + public double FreelistThresholdRatio { get; set; } + [JsonPropertyName("estimated_pages_reclaimable")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public long? EstimatedPagesReclaimable { get; set; } + [JsonPropertyName("estimated_bytes_reclaimable")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public long? EstimatedBytesReclaimable { get; set; } + [JsonPropertyName("auto_vacuum_mode")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public long? AutoVacuumMode { get; set; } + [JsonPropertyName("auto_vacuum_mode_name")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? AutoVacuumModeName { get; set; } + [JsonPropertyName("recommended_command")] + public string RecommendedCommand { get; set; } = "none"; + [JsonPropertyName("post_maintenance_follow_up")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? PostMaintenanceFollowUp { get; set; } +} + +internal readonly record struct MaintenanceMetrics( + long? PageCount, + long? FreelistCount, + long? PageSize, + long? WalSizeBytes, + long? DbSizeBytes, + long? AutoVacuumMode); + +internal static class MaintenanceGuidanceBuilder +{ + public const string WalWarnBytesEnvironmentVariable = "CDIDX_MAINTENANCE_WAL_WARN_BYTES"; + public const string FreelistWarnRatioEnvironmentVariable = "CDIDX_MAINTENANCE_FREELIST_WARN_RATIO"; + public const long DefaultWalWarnBytes = 64L * 1024 * 1024; + public const double DefaultFreelistWarnRatio = 0.20; + private const int RatioDecimalPlaces = 4; + + public static StatusMaintenanceGuidance Build( + MaintenanceMetrics metrics, + string vacuumCommand = "cdidx vacuum --db ", + string checkpointCommand = "sqlite3 \"PRAGMA wal_checkpoint(TRUNCATE);\"") + { + var walThresholdBytes = ReadPositiveLongEnvironment(WalWarnBytesEnvironmentVariable, DefaultWalWarnBytes); + var freelistThresholdRatio = ReadRatioEnvironment(FreelistWarnRatioEnvironmentVariable, DefaultFreelistWarnRatio); + var freelistRatio = CalculateFreelistRatio(metrics.PageCount, metrics.FreelistCount); + var estimatedBytes = EstimateBytes(metrics.FreelistCount, metrics.PageSize); + var walState = metrics.WalSizeBytes.HasValue + ? metrics.WalSizeBytes.Value >= walThresholdBytes ? "checkpoint_recommended" : "ok" + : "unknown"; + var freelistState = freelistRatio.HasValue + ? freelistRatio.Value >= freelistThresholdRatio && metrics.FreelistCount.GetValueOrDefault() > 0 + ? "vacuum_recommended" + : "ok" + : "unknown"; + + var recommendedCommand = freelistState == "vacuum_recommended" + ? vacuumCommand + : walState == "checkpoint_recommended" + ? checkpointCommand + : "none"; + + return new StatusMaintenanceGuidance + { + WalState = walState, + WalThresholdBytes = walThresholdBytes, + FreelistState = freelistState, + FreelistRatio = freelistRatio.HasValue ? Math.Round(freelistRatio.Value, RatioDecimalPlaces) : null, + FreelistThresholdRatio = freelistThresholdRatio, + EstimatedPagesReclaimable = metrics.FreelistCount, + EstimatedBytesReclaimable = estimatedBytes, + AutoVacuumMode = metrics.AutoVacuumMode, + AutoVacuumModeName = FormatAutoVacuumMode(metrics.AutoVacuumMode), + RecommendedCommand = recommendedCommand, + PostMaintenanceFollowUp = BuildFollowUp(walState, freelistState, checkpointCommand), + }; + } + + public static string? FormatAutoVacuumMode(long? mode) => mode switch + { + 0 => "none", + 1 => "full", + 2 => "incremental", + null => null, + _ => "unknown", + }; + + private static double? CalculateFreelistRatio(long? pageCount, long? freelistCount) + { + if (!pageCount.HasValue || !freelistCount.HasValue || pageCount.Value <= 0 || freelistCount.Value < 0) + return null; + return (double)freelistCount.Value / pageCount.Value; + } + + private static long? EstimateBytes(long? pages, long? pageSize) + { + if (!pages.HasValue || !pageSize.HasValue || pages.Value < 0 || pageSize.Value <= 0) + return null; + if (pages.Value > long.MaxValue / pageSize.Value) + return long.MaxValue; + return pages.Value * pageSize.Value; + } + + private static string? BuildFollowUp(string walState, string freelistState, string checkpointCommand) + { + if (freelistState == "vacuum_recommended" && walState == "checkpoint_recommended") + return $"After vacuum, rerun `cdidx status --json`; if wal_state remains checkpoint_recommended, close active writers and run `{checkpointCommand}`."; + if (freelistState == "vacuum_recommended") + return "After vacuum, rerun `cdidx status --json` to confirm freelist_count and wal_size_bytes."; + if (walState == "checkpoint_recommended") + return "Close active cdidx writers before checkpointing, then rerun `cdidx status --json` to confirm wal_size_bytes."; + return null; + } + + private static long ReadPositiveLongEnvironment(string name, long fallback) + { + var value = Environment.GetEnvironmentVariable(name); + return long.TryParse(value, NumberStyles.None, CultureInfo.InvariantCulture, out var parsed) && parsed > 0 + ? parsed + : fallback; + } + + private static double ReadRatioEnvironment(string name, double fallback) + { + var value = Environment.GetEnvironmentVariable(name); + return double.TryParse(value, NumberStyles.Float, CultureInfo.InvariantCulture, out var parsed) + && parsed > 0 + && parsed <= 1 + ? parsed + : fallback; + } +} diff --git a/src/CodeIndex/Database/UnknownExtensionClassifier.cs b/src/CodeIndex/Database/UnknownExtensionClassifier.cs new file mode 100644 index 0000000000..388a4cccdb --- /dev/null +++ b/src/CodeIndex/Database/UnknownExtensionClassifier.cs @@ -0,0 +1,333 @@ +using System.Text; +using System.Text.Json; + +namespace CodeIndex.Database; + +internal sealed record UnknownExtensionClassification( + Dictionary ExtensionCounts, + Dictionary CategoryCounts, + List Groups); + +internal static class UnknownExtensionClassifier +{ + internal const int MaxPersistedGroups = 128; + internal const int MaxSamplePathsPerGroup = 5; + private const int MaxDictionaryEntries = 256; + private const int MaxRawJsonCharacters = 512 * 1024; + private const int MaxDecodedStringCharacters = 64 * 1024; + private const int MaxJsonDepth = 8; + + private static readonly JsonReaderOptions ReaderOptions = new() + { + MaxDepth = MaxJsonDepth, + }; + + public static UnknownExtensionClassification Classify(IReadOnlyList paths) + { + ArgumentNullException.ThrowIfNull(paths); + + var extensionCounts = new Dictionary(StringComparer.Ordinal); + var categoryCounts = new Dictionary(StringComparer.Ordinal); + var groups = new Dictionary(StringComparer.Ordinal); + + foreach (var path in paths.OrderBy(p => p, StringComparer.Ordinal)) + { + if (string.IsNullOrWhiteSpace(path)) + continue; + + var extension = GetExtensionKey(path); + var (category, action) = ClassifyPath(path, extension); + Increment(extensionCounts, extension); + Increment(categoryCounts, category); + + var groupKey = extension + "\0" + category + "\0" + action; + if (!groups.TryGetValue(groupKey, out var group)) + { + group = new StatusUnknownExtensionGroup + { + Extension = extension, + Category = category, + RecommendedAction = action, + SamplePaths = [], + }; + groups[groupKey] = group; + } + + group.Count++; + if (group.SamplePaths.Count < MaxSamplePathsPerGroup) + group.SamplePaths.Add(path); + else + group.SamplePathsTruncated = true; + } + + var orderedGroups = groups.Values + .OrderByDescending(group => group.Count) + .ThenBy(group => group.Category, StringComparer.Ordinal) + .ThenBy(group => group.Extension, StringComparer.Ordinal) + .Take(MaxPersistedGroups) + .ToList(); + + return new UnknownExtensionClassification( + OrderCounts(extensionCounts), + OrderCounts(categoryCounts), + orderedGroups); + } + + public static string SerializeCounts(IReadOnlyDictionary counts) + { + using var buffer = new MemoryStream(); + using (var writer = new Utf8JsonWriter(buffer)) + { + writer.WriteStartObject(); + foreach (var (key, count) in counts.OrderBy(kv => kv.Key, StringComparer.Ordinal).Take(MaxDictionaryEntries)) + writer.WriteNumber(key, count); + writer.WriteEndObject(); + } + + return Encoding.UTF8.GetString(buffer.ToArray()); + } + + public static Dictionary? DeserializeCounts(string? raw) + { + if (string.IsNullOrWhiteSpace(raw) || raw.Length > MaxRawJsonCharacters) + return null; + + try + { + var utf8 = Encoding.UTF8.GetBytes(raw); + var reader = new Utf8JsonReader(utf8, ReaderOptions); + if (!reader.Read() || reader.TokenType != JsonTokenType.StartObject) + return null; + + var counts = new Dictionary(StringComparer.Ordinal); + var entries = 0; + while (reader.Read()) + { + if (reader.TokenType == JsonTokenType.EndObject) + return counts; + if (reader.TokenType != JsonTokenType.PropertyName) + return null; + + var key = reader.GetString(); + if (string.IsNullOrWhiteSpace(key) || key.Length > MaxDecodedStringCharacters || !reader.Read()) + return null; + if (reader.TokenType != JsonTokenType.Number || !reader.TryGetInt64(out var value) || value < 0) + { + if (reader.TokenType is JsonTokenType.StartArray or JsonTokenType.StartObject) + reader.Skip(); + continue; + } + + entries++; + if (entries > MaxDictionaryEntries) + return null; + counts[key] = value; + } + } + catch (JsonException) + { + return null; + } + + return null; + } + + public static string SerializeGroups(IReadOnlyList groups) + { + using var buffer = new MemoryStream(); + using (var writer = new Utf8JsonWriter(buffer)) + { + writer.WriteStartArray(); + foreach (var group in groups.Take(MaxPersistedGroups)) + { + writer.WriteStartObject(); + writer.WriteString("extension", group.Extension); + writer.WriteString("category", group.Category); + writer.WriteString("recommended_action", group.RecommendedAction); + writer.WriteNumber("count", group.Count); + writer.WriteStartArray("sample_paths"); + foreach (var samplePath in group.SamplePaths.Take(MaxSamplePathsPerGroup)) + writer.WriteStringValue(samplePath); + writer.WriteEndArray(); + writer.WriteBoolean("sample_paths_truncated", group.SamplePathsTruncated); + writer.WriteEndObject(); + } + writer.WriteEndArray(); + } + + return Encoding.UTF8.GetString(buffer.ToArray()); + } + + public static List? DeserializeGroups(string? raw) + { + if (string.IsNullOrWhiteSpace(raw) || raw.Length > MaxRawJsonCharacters) + return null; + + try + { + var utf8 = Encoding.UTF8.GetBytes(raw); + var reader = new Utf8JsonReader(utf8, ReaderOptions); + if (!reader.Read() || reader.TokenType != JsonTokenType.StartArray) + return null; + + var groups = new List(); + while (reader.Read()) + { + if (reader.TokenType == JsonTokenType.EndArray) + return groups; + if (reader.TokenType != JsonTokenType.StartObject) + { + reader.Skip(); + continue; + } + + var group = ReadGroup(ref reader); + if (group != null) + { + groups.Add(group); + if (groups.Count > MaxPersistedGroups) + return null; + } + } + } + catch (JsonException) + { + return null; + } + + return null; + } + + private static StatusUnknownExtensionGroup? ReadGroup(ref Utf8JsonReader reader) + { + string? extension = null; + string? category = null; + string? recommendedAction = null; + long? count = null; + var samplePaths = new List(); + var samplePathsTruncated = false; + + while (reader.Read()) + { + if (reader.TokenType == JsonTokenType.EndObject) + { + if (extension == null || category == null || recommendedAction == null || count == null) + return null; + return new StatusUnknownExtensionGroup + { + Extension = extension, + Category = category, + RecommendedAction = recommendedAction, + Count = count.Value, + SamplePaths = samplePaths, + SamplePathsTruncated = samplePathsTruncated, + }; + } + + if (reader.TokenType != JsonTokenType.PropertyName) + return null; + var propertyName = reader.GetString(); + if (!reader.Read()) + return null; + + switch (propertyName) + { + case "extension" when reader.TokenType == JsonTokenType.String: + extension = ReadBoundedString(ref reader); + break; + case "category" when reader.TokenType == JsonTokenType.String: + category = ReadBoundedString(ref reader); + break; + case "recommended_action" when reader.TokenType == JsonTokenType.String: + recommendedAction = ReadBoundedString(ref reader); + break; + case "count" when reader.TokenType == JsonTokenType.Number && reader.TryGetInt64(out var parsedCount) && parsedCount >= 0: + count = parsedCount; + break; + case "sample_paths" when reader.TokenType == JsonTokenType.StartArray: + samplePaths = ReadSamplePaths(ref reader); + break; + case "sample_paths_truncated" when reader.TokenType is JsonTokenType.True or JsonTokenType.False: + samplePathsTruncated = reader.GetBoolean(); + break; + default: + if (reader.TokenType is JsonTokenType.StartArray or JsonTokenType.StartObject) + reader.Skip(); + break; + } + } + + return null; + } + + private static List ReadSamplePaths(ref Utf8JsonReader reader) + { + var values = new List(); + while (reader.Read()) + { + if (reader.TokenType == JsonTokenType.EndArray) + return values; + if (reader.TokenType != JsonTokenType.String) + { + if (reader.TokenType is JsonTokenType.StartArray or JsonTokenType.StartObject) + reader.Skip(); + continue; + } + if (values.Count >= MaxSamplePathsPerGroup) + continue; + var value = ReadBoundedString(ref reader); + if (!string.IsNullOrWhiteSpace(value)) + values.Add(value); + } + + return values; + } + + private static string? ReadBoundedString(ref Utf8JsonReader reader) + { + var value = reader.GetString(); + return value is { Length: <= MaxDecodedStringCharacters } ? value : null; + } + + private static string GetExtensionKey(string path) + { + var fileName = Path.GetFileName(path.Replace('\\', '/')); + if (string.IsNullOrWhiteSpace(fileName)) + return ""; + var dotCount = fileName.Count(c => c == '.'); + if (fileName.StartsWith(".", StringComparison.Ordinal) && dotCount == 1) + return fileName.ToLowerInvariant(); + var extension = Path.GetExtension(fileName); + return string.IsNullOrWhiteSpace(extension) + ? "" + : extension.ToLowerInvariant(); + } + + private static (string Category, string RecommendedAction) ClassifyPath(string path, string extension) + { + var normalizedPath = path.Replace('\\', '/'); + var fileName = Path.GetFileName(normalizedPath).ToLowerInvariant(); + var lowerPath = normalizedPath.ToLowerInvariant(); + + if (fileName is ".git" or ".gitattributes" or ".gitkeep" || lowerPath.StartsWith(".git/", StringComparison.Ordinal)) + return ("repository_metadata", "ignore_configuration"); + if (lowerPath.StartsWith("licenses/", StringComparison.Ordinal) || fileName.Contains("license", StringComparison.Ordinal)) + return ("license", "ignore_configuration"); + if (extension is ".png" or ".jpg" or ".jpeg" or ".gif" or ".webp" or ".ico" or ".svg" or ".pdf" or ".zip" or ".gz" or ".tar" or ".woff" or ".woff2" or ".ttf") + return ("binary_asset", "ignore_configuration"); + if (extension is ".config" or ".runsettings" or ".rules" or ".props" or ".targets" || fileName.EndsWith(".config", StringComparison.Ordinal)) + return ("configuration", "ignore_configuration"); + if (extension is ".sln" or ".manifest" or ".lock" or ".assets") + return ("structural_metadata", "first_class_structural_extraction"); + return ("language_support_candidate", "language_support"); + } + + private static void Increment(Dictionary counts, string key) + => counts[key] = counts.TryGetValue(key, out var current) ? current + 1 : 1; + + private static Dictionary OrderCounts(Dictionary counts) + => counts + .OrderByDescending(kv => kv.Value) + .ThenBy(kv => kv.Key, StringComparer.Ordinal) + .ToDictionary(kv => kv.Key, kv => kv.Value, StringComparer.Ordinal); +} diff --git a/src/CodeIndex/Mcp/McpToolHandlers.cs b/src/CodeIndex/Mcp/McpToolHandlers.cs index 409970d778..21fedb9001 100644 --- a/src/CodeIndex/Mcp/McpToolHandlers.cs +++ b/src/CodeIndex/Mcp/McpToolHandlers.cs @@ -4159,6 +4159,7 @@ static long SumReadableFileBytes(IEnumerable paths) writer.SetMeta(DbContext.LastIndexRunBytesReadMetaKey, SumReadableFileBytes(files).ToString(System.Globalization.CultureInfo.InvariantCulture)); writer.SetMeta(DbContext.LastIndexRunRowsUpsertedMetaKey, processed.ToString(System.Globalization.CultureInfo.InvariantCulture)); writer.SetMeta(DbContext.LastIndexRunRowsDeletedMetaKey, purged.ToString(System.Globalization.CultureInfo.InvariantCulture)); + writer.ClearLastFailedIndexRunMetadata(); // Persist the current HEAD only after the run is fully successful (errors == 0). // Mirrors the CLI full-scan contract (Issue #1508) so MCP-driven re-indexes also // refresh `worktree_head_changed`; partial / failed runs leave the prior HEAD diff --git a/src/CodeIndex/Models/QueryResults.cs b/src/CodeIndex/Models/QueryResults.cs index 53caa83c84..56aeeb4068 100644 --- a/src/CodeIndex/Models/QueryResults.cs +++ b/src/CodeIndex/Models/QueryResults.cs @@ -543,6 +543,15 @@ public class StatusResult [JsonPropertyName("unknown_extension_file_path_limit")] [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public long? UnknownExtensionFilePathLimit { get; set; } + [JsonPropertyName("unknown_extension_extension_counts")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public Dictionary? UnknownExtensionExtensionCounts { get; set; } + [JsonPropertyName("unknown_extension_category_counts")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public Dictionary? UnknownExtensionCategoryCounts { get; set; } + [JsonPropertyName("unknown_extension_groups")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public List? UnknownExtensionGroups { get; set; } public DateTime? IndexedAt { get; set; } /// /// Timestamp of the most recent successful index/update run that freshened workspace @@ -701,6 +710,9 @@ public class StatusResult [JsonPropertyName("failed_checks")] [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public List? FailedChecks { get; set; } + [JsonPropertyName("repair_commands")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public List? RepairCommands { get; set; } /// /// Effective age threshold, in seconds, used by `status --check` to explain stale-index /// warnings. Null when `--check` was not requested. @@ -854,6 +866,8 @@ public class StatusResult /// [JsonPropertyName("db_pragma_settings")] public StatusDbPragmaSettings DbPragmaSettings { get; set; } = new(); + [JsonPropertyName("maintenance_guidance")] + public StatusMaintenanceGuidance MaintenanceGuidance { get; set; } = new(); [JsonPropertyName("db_size_bytes")] [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public long? DbSizeBytes { get; set; } @@ -865,6 +879,9 @@ public class StatusResult [JsonPropertyName("last_index_run")] [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public StatusLastIndexRun? LastIndexRun { get; set; } + [JsonPropertyName("last_failed_or_partial_index_run")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public StatusFailedOrPartialIndexRun? LastFailedOrPartialIndexRun { get; set; } } public sealed class StatusProcessMetrics @@ -920,6 +937,45 @@ public sealed class StatusLastIndexRun public long? PeakMemoryMb { get; set; } } +public sealed class StatusFailedOrPartialIndexRun +{ + public string? Status { get; set; } + public string? Mode { get; set; } + [JsonPropertyName("started_at")] + public DateTime? StartedAt { get; set; } + [JsonPropertyName("duration_ms")] + public long? DurationMs { get; set; } + [JsonPropertyName("files_processed")] + public long? FilesProcessed { get; set; } + [JsonPropertyName("files_total")] + public long? FilesTotal { get; set; } + [JsonPropertyName("error_code")] + public string? ErrorCode { get; set; } + public string? Reason { get; set; } +} + +public sealed class StatusRepairCommand +{ + public string Name { get; set; } = string.Empty; + public List Args { get; set; } = []; + public string Reason { get; set; } = string.Empty; + [JsonPropertyName("safety_notes")] + public List SafetyNotes { get; set; } = []; +} + +public sealed class StatusUnknownExtensionGroup +{ + public string Extension { get; set; } = string.Empty; + public string Category { get; set; } = string.Empty; + [JsonPropertyName("recommended_action")] + public string RecommendedAction { get; set; } = string.Empty; + public long Count { get; set; } + [JsonPropertyName("sample_paths")] + public List SamplePaths { get; set; } = []; + [JsonPropertyName("sample_paths_truncated")] + public bool SamplePathsTruncated { get; set; } +} + public class StatusReadinessDegradation { public string Field { get; set; } = string.Empty; @@ -941,17 +997,30 @@ public class StatusDbPragmaSettings public long? PageCount { get; set; } public long? FreelistCount { get; set; } public long? PageSize { get; set; } + public long? AutoVacuum { get; set; } } public sealed record VacuumResult( string Status, + bool DryRun, long PageSize, long PageCountBefore, long FreelistCountBefore, long PageCountAfter, long FreelistCountAfter, long PagesReclaimed, - long BytesReclaimed); + long BytesReclaimed, + long EstimatedPagesReclaimable, + long EstimatedBytesReclaimable, + long? DbSizeBytesBefore, + long? WalSizeBytesBefore, + long? DbSizeBytesAfter, + long? WalSizeBytesAfter, + long AutoVacuumModeBefore, + string AutoVacuumModeBeforeName, + long AutoVacuumModeAfter, + string AutoVacuumModeAfterName, + StatusMaintenanceGuidance MaintenanceGuidance); public class PostExtractionHookStatus { diff --git a/tests/CodeIndex.Tests/DatabaseTests.cs b/tests/CodeIndex.Tests/DatabaseTests.cs index b8ab138b49..bfb27535ef 100644 --- a/tests/CodeIndex.Tests/DatabaseTests.cs +++ b/tests/CodeIndex.Tests/DatabaseTests.cs @@ -819,6 +819,11 @@ INSERT INTO vacuum_payload (payload) Assert.True(result.FreelistCountAfter < result.FreelistCountBefore); Assert.True(result.PagesReclaimed > 0); Assert.True(result.BytesReclaimed > 0); + Assert.False(result.DryRun); + Assert.True(result.EstimatedBytesReclaimable > 0); + Assert.True(result.DbSizeBytesBefore > 0); + Assert.Equal(2, result.AutoVacuumModeAfter); + Assert.Equal("incremental", result.AutoVacuumModeAfterName); } finally { @@ -826,6 +831,43 @@ INSERT INTO vacuum_payload (payload) } } + [Fact] + public void MaintenanceGuidance_HighWalRecommendsCheckpoint_Issue3564() + { + var guidance = MaintenanceGuidanceBuilder.Build(new MaintenanceMetrics( + PageCount: 100, + FreelistCount: 0, + PageSize: 4096, + WalSizeBytes: MaintenanceGuidanceBuilder.DefaultWalWarnBytes + 1, + DbSizeBytes: 409_600, + AutoVacuumMode: 2)); + + Assert.Equal("checkpoint_recommended", guidance.WalState); + Assert.Equal("ok", guidance.FreelistState); + Assert.Contains("wal_checkpoint", guidance.RecommendedCommand, StringComparison.Ordinal); + Assert.Contains("status --json", guidance.PostMaintenanceFollowUp, StringComparison.Ordinal); + } + + [Fact] + public void MaintenanceGuidance_HighFreelistRecommendsVacuum_Issue3564() + { + var guidance = MaintenanceGuidanceBuilder.Build(new MaintenanceMetrics( + PageCount: 100, + FreelistCount: 25, + PageSize: 4096, + WalSizeBytes: 0, + DbSizeBytes: 409_600, + AutoVacuumMode: 2)); + + Assert.Equal("ok", guidance.WalState); + Assert.Equal("vacuum_recommended", guidance.FreelistState); + Assert.Equal(0.25, guidance.FreelistRatio); + Assert.Equal(25, guidance.EstimatedPagesReclaimable); + Assert.Equal(25 * 4096, guidance.EstimatedBytesReclaimable); + Assert.Equal("cdidx vacuum --db ", guidance.RecommendedCommand); + Assert.Equal("incremental", guidance.AutoVacuumModeName); + } + private static int GetTransactionDepth(DbWriter writer) { var field = typeof(DbWriter).GetField("_transactionDepth", System.Reflection.BindingFlags.Instance | System.Reflection.BindingFlags.NonPublic) diff --git a/tests/CodeIndex.Tests/DbCommandRunnerTests.cs b/tests/CodeIndex.Tests/DbCommandRunnerTests.cs index cbc33829f5..4d4202ee5f 100644 --- a/tests/CodeIndex.Tests/DbCommandRunnerTests.cs +++ b/tests/CodeIndex.Tests/DbCommandRunnerTests.cs @@ -265,6 +265,8 @@ public void Run_CleanDb_JsonReportsOkTrueAndEmptyIssues() Assert.Equal(CommandExitCodes.Success, exitCode); Assert.True(json.GetProperty("ok").GetBoolean()); + Assert.Equal("ok", json.GetProperty("severity").GetString()); + Assert.Equal("integrity_ok", json.GetProperty("diagnostic_code").GetString()); Assert.Equal(0, json.GetProperty("issues").GetArrayLength()); Assert.Equal(Path.GetFullPath(dbPath), json.GetProperty("db_path").GetString()); } @@ -276,6 +278,34 @@ public void Run_CleanDb_JsonReportsOkTrueAndEmptyIssues() } } + [Fact] + public void Run_IntegrityCheck_JsonReportsStableErrorSeverity() + { + var dbPath = Path.Combine(Path.GetTempPath(), $"cdidx_db_integrity_error_json_{Guid.NewGuid():N}.db"); + DbCommandRunner.IntegrityCheckRowsForTesting = () => ["simulated corruption"]; + try + { + using (var db = new DbContext(dbPath)) + db.InitializeSchema(); + SqliteConnection.ClearAllPools(); + + var (exitCode, json) = RunAndCaptureJson(["--integrity-check", "--db", dbPath, "--json"]); + + Assert.Equal(CommandExitCodes.DatabaseError, exitCode); + Assert.False(json.GetProperty("ok").GetBoolean()); + Assert.Equal("error", json.GetProperty("severity").GetString()); + Assert.Equal("integrity_failed", json.GetProperty("diagnostic_code").GetString()); + Assert.Equal("simulated corruption", json.GetProperty("issues")[0].GetString()); + } + finally + { + DbCommandRunner.IntegrityCheckRowsForTesting = null; + SqliteConnection.ClearAllPools(); + if (File.Exists(dbPath)) + File.Delete(dbPath); + } + } + [Fact] public void Run_Schema_JsonIncludesTablesAndUserVersion() { @@ -291,6 +321,10 @@ public void Run_Schema_JsonIncludesTablesAndUserVersion() Assert.Equal(CommandExitCodes.Success, exitCode); Assert.Equal(Path.GetFullPath(dbPath), json.GetProperty("db_path").GetString()); Assert.True(json.TryGetProperty("user_version", out _)); + Assert.Equal("ok", json.GetProperty("severity").GetString()); + Assert.Equal("schema_ok", json.GetProperty("diagnostic_code").GetString()); + Assert.True(json.GetProperty("object_type_counts").GetProperty("table").GetInt32() > 0); + Assert.Equal(0, json.GetProperty("object_type_omitted_counts").GetProperty("table").GetInt32()); Assert.Contains(json.GetProperty("entries").EnumerateArray(), entry => entry.GetProperty("type").GetString() == "table" && entry.GetProperty("name").GetString() == "files"); @@ -303,6 +337,37 @@ public void Run_Schema_JsonIncludesTablesAndUserVersion() } } + [Fact] + public void Run_Schema_JsonReportsObjectTypeOmissionsWhenEntryLimitTruncates() + { + var dbPath = Path.Combine(Path.GetTempPath(), $"cdidx_db_schema_truncated_{Guid.NewGuid():N}.db"); + try + { + using (var connection = new SqliteConnection(new SqliteConnectionStringBuilder { DataSource = dbPath }.ConnectionString)) + { + connection.Open(); + for (var i = 0; i < DbCommandRunner.SchemaEntryLimit + 5; i++) + Execute(connection, $"CREATE TABLE t_{i:D3}(id INTEGER PRIMARY KEY);"); + } + SqliteConnection.ClearAllPools(); + + var (exitCode, json) = RunAndCaptureJson(["schema", "--db", dbPath, "--json"]); + + Assert.Equal(CommandExitCodes.Success, exitCode); + Assert.Equal("warn", json.GetProperty("severity").GetString()); + Assert.Equal("schema_truncated", json.GetProperty("diagnostic_code").GetString()); + Assert.True(json.GetProperty("entries_truncated").GetBoolean()); + Assert.Equal(DbCommandRunner.SchemaEntryLimit + 5, json.GetProperty("object_type_counts").GetProperty("table").GetInt32()); + Assert.Equal(5, json.GetProperty("object_type_omitted_counts").GetProperty("table").GetInt32()); + } + finally + { + SqliteConnection.ClearAllPools(); + if (File.Exists(dbPath)) + File.Delete(dbPath); + } + } + [Fact] public void Run_Prune_DryRunCountsAndApplyDeletesOrphans() { diff --git a/tests/CodeIndex.Tests/DocumentationStatusContractTests.cs b/tests/CodeIndex.Tests/DocumentationStatusContractTests.cs index 9905042f4c..915b3795ad 100644 --- a/tests/CodeIndex.Tests/DocumentationStatusContractTests.cs +++ b/tests/CodeIndex.Tests/DocumentationStatusContractTests.cs @@ -28,14 +28,19 @@ public class DocumentationStatusContractTests "unknown_extension_files", "unknown_extension_files_truncated", "unknown_extension_file_path_limit", + "unknown_extension_extension_counts", + "unknown_extension_category_counts", + "unknown_extension_groups", "extractors", "path_case_sensitive", "mac_profile", "stale_after_seconds", "index_age_seconds", + "last_failed_or_partial_index_run", "degraded_reason", "recommended_action", "alternative_action", + "repair_commands", ]; [Theory] diff --git a/tests/CodeIndex.Tests/IndexCommandRunnerTests.cs b/tests/CodeIndex.Tests/IndexCommandRunnerTests.cs index 603bd6be72..6dd6775947 100644 --- a/tests/CodeIndex.Tests/IndexCommandRunnerTests.cs +++ b/tests/CodeIndex.Tests/IndexCommandRunnerTests.cs @@ -2630,6 +2630,19 @@ public void Run_VerboseReportsUnknownExtensionCountAndStatusJsonStampsCount() .Select(path => path.GetString()) .ToArray(); Assert.Equal(["data.unmapped", "notes.mystery"], paths); + var extensionCounts = statusJson.GetProperty("unknown_extension_extension_counts"); + Assert.Equal(1, extensionCounts.GetProperty(".mystery").GetInt64()); + Assert.Equal(1, extensionCounts.GetProperty(".unmapped").GetInt64()); + var categoryCounts = statusJson.GetProperty("unknown_extension_category_counts"); + Assert.Equal(2, categoryCounts.GetProperty("language_support_candidate").GetInt64()); + var groups = statusJson.GetProperty("unknown_extension_groups").EnumerateArray().ToArray(); + Assert.Equal(2, groups.Length); + Assert.All(groups, group => + { + Assert.Equal("language_support_candidate", group.GetProperty("category").GetString()); + Assert.Equal("language_support", group.GetProperty("recommended_action").GetString()); + Assert.Equal(1, group.GetProperty("count").GetInt64()); + }); } finally { diff --git a/tests/CodeIndex.Tests/QueryCommandRunnerFilesTests.cs b/tests/CodeIndex.Tests/QueryCommandRunnerFilesTests.cs index caedfde37a..3f30dac5c7 100644 --- a/tests/CodeIndex.Tests/QueryCommandRunnerFilesTests.cs +++ b/tests/CodeIndex.Tests/QueryCommandRunnerFilesTests.cs @@ -1421,6 +1421,15 @@ public void RunStatus_CheckJson_ReturnsStaleIndexWhenContentChecksumDiffers() Assert.Equal("changed_files", check.GetProperty("reason").GetString()); Assert.Equal(1, check.GetProperty("changed_file_count").GetInt32()); Assert.Equal("src/app.cs", check.GetProperty("changed_files")[0].GetString()); + var repairCommand = Assert.Single(json.GetProperty("repair_commands").EnumerateArray()); + Assert.Equal("cdidx", repairCommand.GetProperty("name").GetString()); + Assert.Equal("workspace_stale", repairCommand.GetProperty("reason").GetString()); + var repairArgs = repairCommand.GetProperty("args").EnumerateArray().Select(arg => arg.GetString()).ToArray(); + Assert.Contains("index", repairArgs); + Assert.Contains(projectRoot, repairArgs); + Assert.Contains("--db", repairArgs); + Assert.Contains(dbPath, repairArgs); + Assert.NotEmpty(repairCommand.GetProperty("safety_notes").EnumerateArray()); Assert.Contains("index stale", json.GetProperty("summary").GetString()); } finally @@ -1463,6 +1472,100 @@ public void RunStatus_CheckJson_DetectsMissingAndUnindexedFiles() } } + [Fact] + public void RunStatus_CheckJson_ReportsBackfillRepairCommandForFoldDegradation_Issue3567() + { + var projectRoot = TestProjectHelper.CreateTempProject("cdidx_query_runner_status_repair_fold"); + try + { + Directory.CreateDirectory(Path.Combine(projectRoot, "src")); + File.WriteAllText(Path.Combine(projectRoot, "src", "app.cs"), "class App {}\n"); + var dbPath = TestProjectHelper.CreateProjectDb(projectRoot); + TestProjectHelper.InsertIndexedFile(dbPath, "src/app.cs", "csharp", "class App {}\n"); + using (var db = new DbContext(dbPath)) + { + var writer = new DbWriter(db.Connection); + writer.MarkGraphReady(); + writer.MarkIssuesReady(); + writer.MarkCSharpSymbolNameContractReady(); + writer.MarkMetadataTargetReady("csharp"); + writer.MarkSqlGraphContractReady(); + writer.MarkHotspotFamilyReady("csharp", "test"); + } + + var (exitCode, stdout, stderr) = CaptureConsole(() => QueryCommandRunner.RunStatus( + ["--db", dbPath, "--check", "--json"], + _jsonOptions)); + + using var document = ParseJsonOutput(stdout); + var json = document.RootElement; + var repairCommand = Assert.Single(json.GetProperty("repair_commands").EnumerateArray()); + var repairArgs = repairCommand.GetProperty("args").EnumerateArray().Select(arg => arg.GetString()).ToArray(); + + Assert.Equal(2, exitCode); + Assert.Equal(string.Empty, stderr); + Assert.Equal("fold_ready", json.GetProperty("failed_checks")[0].GetString()); + Assert.Equal("cdidx", repairCommand.GetProperty("name").GetString()); + Assert.Equal("fold_ready", repairCommand.GetProperty("reason").GetString()); + Assert.Equal("backfill-fold", repairArgs[0]); + Assert.Contains("--db", repairArgs); + Assert.Contains(dbPath, repairArgs); + } + finally + { + TestProjectHelper.DeleteDirectory(projectRoot); + } + } + + [Fact] + public void RunStatus_CheckJson_ReportsLastFailedOrPartialIndexRun_Issue3567() + { + var projectRoot = TestProjectHelper.CreateTempProject("cdidx_query_runner_status_failed_run"); + try + { + Directory.CreateDirectory(Path.Combine(projectRoot, "src")); + File.WriteAllText(Path.Combine(projectRoot, "src", "app.cs"), "class App {}\n"); + var dbPath = TestProjectHelper.CreateProjectDb(projectRoot); + TestProjectHelper.InsertIndexedFile(dbPath, "src/app.cs", "csharp", "class App {}\n"); + MarkStatusReadinessReady(dbPath); + using (var db = new DbContext(dbPath)) + { + var writer = new DbWriter(db.Connection); + writer.SetMeta(DbContext.LastFailedIndexRunStatusMetaKey, "partial"); + writer.SetMeta(DbContext.LastFailedIndexRunModeMetaKey, "update"); + writer.SetMeta(DbContext.LastFailedIndexRunStartedAtMetaKey, "2026-06-11T00:00:00.0000000Z"); + writer.SetMeta(DbContext.LastFailedIndexRunDurationMsMetaKey, "1234"); + writer.SetMeta(DbContext.LastFailedIndexRunFilesProcessedMetaKey, "3"); + writer.SetMeta(DbContext.LastFailedIndexRunFilesTotalMetaKey, "9"); + writer.SetMeta(DbContext.LastFailedIndexRunErrorCodeMetaKey, CommandErrorCodes.Interrupted); + writer.SetMeta(DbContext.LastFailedIndexRunReasonMetaKey, "interrupted"); + } + + var (exitCode, stdout, stderr) = CaptureConsole(() => QueryCommandRunner.RunStatus( + ["--db", dbPath, "--check", "--json"], + _jsonOptions)); + + using var document = ParseJsonOutput(stdout); + var run = document.RootElement.GetProperty("last_failed_or_partial_index_run"); + + Assert.Equal(CommandExitCodes.Success, exitCode); + Assert.Equal(string.Empty, stderr); + Assert.Equal("partial", run.GetProperty("status").GetString()); + Assert.Equal("update", run.GetProperty("mode").GetString()); + Assert.Equal(1234, run.GetProperty("duration_ms").GetInt64()); + Assert.Equal(3, run.GetProperty("files_processed").GetInt64()); + Assert.Equal(9, run.GetProperty("files_total").GetInt64()); + Assert.Equal(CommandErrorCodes.Interrupted, run.GetProperty("error_code").GetString()); + Assert.Equal("interrupted", run.GetProperty("reason").GetString()); + Assert.False(run.TryGetProperty("exception", out _)); + Assert.False(run.TryGetProperty("active_path", out _)); + } + finally + { + TestProjectHelper.DeleteDirectory(projectRoot); + } + } + [Fact] public void RunStatus_CheckJson_MatchesNfcIndexedPathsAfterNfdWorkspaceSort() { diff --git a/tests/CodeIndex.Tests/QueryCommandRunnerTests.cs b/tests/CodeIndex.Tests/QueryCommandRunnerTests.cs index c8e7a8dda2..ecb241213d 100644 --- a/tests/CodeIndex.Tests/QueryCommandRunnerTests.cs +++ b/tests/CodeIndex.Tests/QueryCommandRunnerTests.cs @@ -646,6 +646,54 @@ public void RunVacuum_RejectsReadOnlyUriWithNeutralWritableMessage_Issue1631() } } + [Fact] + public void RunVacuum_DryRunJsonReportsMaintenanceEstimate_Issue3564() + { + var projectRoot = TestProjectHelper.CreateTempProject("cdidx_vacuum_dry_run"); + try + { + var dbPath = TestProjectHelper.CreateProjectDb(projectRoot); + using (var db = new DbContext(dbPath)) + { + using var command = db.Connection.CreateCommand(); + command.CommandText = @" + CREATE TABLE vacuum_payload (id INTEGER PRIMARY KEY, payload BLOB); + WITH RECURSIVE n(value) AS ( + SELECT 1 + UNION ALL + SELECT value + 1 FROM n WHERE value < 128 + ) + INSERT INTO vacuum_payload (payload) + SELECT randomblob(4096) FROM n; + DELETE FROM vacuum_payload;"; + command.ExecuteNonQuery(); + } + + var (exitCode, stdout, stderr) = CaptureConsole(() => QueryCommandRunner.RunVacuum( + ["--db", dbPath, "--dry-run", "--json"], + _jsonOptions)); + + Assert.Equal(CommandExitCodes.Success, exitCode); + Assert.Equal(string.Empty, stderr); + using var document = ParseJsonOutput(stdout); + var root = document.RootElement; + Assert.Equal("dry_run", root.GetProperty("status").GetString()); + Assert.True(root.GetProperty("dry_run").GetBoolean()); + Assert.True(root.GetProperty("estimated_pages_reclaimable").GetInt64() > 0); + Assert.True(root.GetProperty("estimated_bytes_reclaimable").GetInt64() > 0); + Assert.Equal(0, root.GetProperty("pages_reclaimed").GetInt64()); + Assert.Equal(root.GetProperty("page_count_before").GetInt64(), root.GetProperty("page_count_after").GetInt64()); + Assert.Equal("incremental", root.GetProperty("auto_vacuum_mode_after_name").GetString()); + var guidance = root.GetProperty("maintenance_guidance"); + Assert.Equal("vacuum_recommended", guidance.GetProperty("freelist_state").GetString()); + Assert.Equal("cdidx vacuum --db ", guidance.GetProperty("recommended_command").GetString()); + } + finally + { + TestProjectHelper.DeleteDirectory(projectRoot); + } + } + diff --git a/tests/CodeIndex.Tests/golden/status.json b/tests/CodeIndex.Tests/golden/status.json index 792e3c6d37..c53b85d00d 100644 --- a/tests/CodeIndex.Tests/golden/status.json +++ b/tests/CodeIndex.Tests/golden/status.json @@ -170,7 +170,20 @@ "wal_autocheckpoint": 1000, "page_count": "\u003CCOUNT\u003E", "freelist_count": 0, - "page_size": 4096 + "page_size": 4096, + "auto_vacuum": 2 + }, + "maintenance_guidance": { + "wal_state": "ok", + "wal_threshold_bytes": 67108864, + "freelist_state": "ok", + "freelist_ratio": 0, + "freelist_threshold_ratio": 0.2, + "estimated_pages_reclaimable": 0, + "estimated_bytes_reclaimable": 0, + "auto_vacuum_mode": 2, + "auto_vacuum_mode_name": "incremental", + "recommended_command": "none" }, "db_size_bytes": "\u003CCOUNT\u003E", "wal_size_bytes": "\u003CCOUNT\u003E",