diff --git a/AGENT_GUIDE.md b/AGENT_GUIDE.md index 1a1af2b5f6..0ae1579cfa 100644 --- a/AGENT_GUIDE.md +++ b/AGENT_GUIDE.md @@ -131,7 +131,7 @@ CI watching must be bounded. Do not loop indefinitely. ## Status Contract -- `status --json` and related JSON/MCP payloads currently expose the trust fields documented in `README.md` and `DEVELOPER_GUIDE.md`, including `fold_ready`, `fold_ready_reason`, `graph_table_available`, `issues_table_available`, `file_issues_data_current`, `migration_in_progress`, `sql_graph_contract_ready`, `sql_graph_contract_degraded_reason`, `hotspot_family_ready`, `hotspot_family_degraded_reason`, `language_readiness`, `csharp_symbol_name_ready`, `csharp_metadata_target_ready`, `csharp_metadata_target_degraded_reason`, `indexed_head_commit`, `worktree_head_changed`, `index_writer_version`, `index_newer_than_reader`, `index_newer_than_reader_reason`, `unknown_extension_file_count`, `unknown_extension_files`, `unknown_extension_files_truncated`, `unknown_extension_file_path_limit`, `unknown_extension_extension_counts`, `unknown_extension_category_counts`, `unknown_extension_groups`, `extractors`, `path_case_sensitive`, `data_dir`, `data_dir_source`, `data_dir_mode`, `mac_profile`, `mac_profile_diagnostics`, `db_size_bytes`, `wal_size_bytes`, `db_pragma_settings` (`journal_mode`, `synchronous`, `wal_autocheckpoint`, `page_count`, `freelist_count`, `page_size`, `auto_vacuum`), `maintenance_guidance`, `symbol_kinds`, `symbols_by_language`, status kind cap metadata (`symbol_kind_limit`, `symbol_kind_name_limit`, `symbol_kind_total_count`, `symbol_kind_omitted_count`, `symbol_kind_names_truncated`, `symbols_by_language_kind_total_counts`, `symbols_by_language_kind_omitted_counts`, `symbols_by_language_kind_names_truncated`), `process`, `last_index_run`, `last_failed_or_partial_index_run`, `last_workspace_freshened_at`, `hooks`, `hook_diagnostics`, `trust_overrides`, MCP-only `mcp_session`, and the `status --check`-only `stale_after_seconds` / `index_age_seconds` threshold audit fields and `repair_commands`. +- `status --json` and related JSON/MCP payloads currently expose the trust fields documented in `README.md` and `DEVELOPER_GUIDE.md`, including `fold_ready`, `fold_ready_reason`, `graph_table_available`, `issues_table_available`, `file_issues_data_current`, `migration_in_progress`, `sql_graph_contract_ready`, `sql_graph_contract_degraded_reason`, `hotspot_family_ready`, `hotspot_family_degraded_reason`, `language_readiness`, `csharp_symbol_name_ready`, `csharp_metadata_target_ready`, `csharp_metadata_target_degraded_reason`, `indexed_head_commit`, `worktree_head_changed`, `index_writer_version`, `index_newer_than_reader`, `index_newer_than_reader_reason`, `unknown_extension_file_count`, `unknown_extension_files`, `unknown_extension_files_truncated`, `unknown_extension_file_path_limit`, `unknown_extension_extension_counts`, `unknown_extension_category_counts`, `unknown_extension_groups`, `extractors`, `path_case_sensitive`, `data_dir`, `data_dir_source`, `data_dir_mode`, `mac_profile`, `mac_profile_diagnostics`, `db_size_bytes`, `wal_size_bytes`, `db_pragma_settings` (`journal_mode`, `synchronous`, `wal_autocheckpoint`, `busy_timeout_ms`, `page_count`, `freelist_count`, `page_size`, `auto_vacuum`), `prepared_command_cache` (`count`, `capacity`, `hit_count`, `miss_count`, `eviction_count`), `maintenance_guidance`, `symbol_kinds`, `symbols_by_language`, status kind cap metadata (`symbol_kind_limit`, `symbol_kind_name_limit`, `symbol_kind_total_count`, `symbol_kind_omitted_count`, `symbol_kind_names_truncated`, `symbols_by_language_kind_total_counts`, `symbols_by_language_kind_omitted_counts`, `symbols_by_language_kind_names_truncated`), `process`, `last_index_run`, `last_failed_or_partial_index_run`, `last_workspace_freshened_at`, `hooks`, `hook_diagnostics`, `trust_overrides`, MCP-only `mcp_session`, and the `status --check`-only `stale_after_seconds` / `index_age_seconds` threshold audit fields and `repair_commands`. - When any readiness field is degraded, the CLI adds `degraded_root_cause`, `degraded_reason`, `recommended_action`, `alternative_action`, and `readiness_degradations[]`. `degraded_root_cause` is the primary stable machine code; `readiness_degradations[]` lists every degraded field with `root_cause`, human reason, and remediation strings. - `hotspot_family_degraded_reason` currently uses `hotspot_family_support_not_indexed`, `hotspot_family_metadata_stale`, `hotspot_family_disabled_at_index_time`, `partial_family_key_population`, and `hotspot_family_marker_fingerprint_incomplete`; the incomplete marker fingerprint code means marker traversal hit safety caps and should stay synchronized with README / developer-guide recovery notes. - `issues_table_available` reports physical `file_issues` table presence only. `file_issues_data_current` reports whether the table is also stamped current for the active index generation. diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index d03115f17c..43523f6b86 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -461,7 +461,7 @@ Current stable codes and triggers: | Schema discovery cache | `DbReader` schema discovery uses a process-level cache keyed by the normalized DB path. The cache stores `PRAGMA table_info`, `PRAGMA index_list`, and `sqlite_master` table-existence results, and checks `PRAGMA schema_version` before serving a lookup so SQLite DDL performed by cdidx or an external `sqlite3` session invalidates stale snapshots. Manual schema edits outside cdidx are still unsupported operationally; run `cdidx validate` after such edits before trusting query output. | | Batch trust marker | Index write batches stamp `codeindex_meta.batch_in_progress=true` before starting a mutation transaction and clear it inside the transaction that commits the matching rows and readiness metadata. If the indexer crashes after the marker is written but before the commit clears it, the next writable DB open demotes readiness bits and warns: `Last batch did not complete; run cdidx index --rebuild to re-index from a known clean state.` Gracefully handled per-file errors clear the marker after rollback; orphaned markers are reserved for interrupted or crashed batches whose trust metadata should not be treated as clean. | | Read-only fallback | When the normal writable open cannot create or lock journal/WAL side files, read-only fallback uses an immutable SQLite URI so query commands can still read a DB from read-only or sandboxed storage. The fallback intentionally skips writable pragmas, migrations, and WAL recovery writes. If a WAL is present and must be observed, copy `.db`, `.db-wal`, and `.db-shm` together to a writable location or use a SQLite backup from an environment that can open the full WAL set. | -| Status pragma diagnostics | `status --json` exposes resolved connection values under `db_pragma_settings` (`journal_mode`, `synchronous`, `wal_autocheckpoint`, `page_count`, `freelist_count`, `page_size`, `auto_vacuum`) for automation and support diagnostics. `maintenance_guidance` derives `wal_state`, `freelist_ratio`, `freelist_state`, `estimated_*_reclaimable`, `auto_vacuum_mode(_name)`, `recommended_command`, and `post_maintenance_follow_up` from those raw metrics without changing the raw values. `status --check --json` adds `repair_commands[]` entries with `name`, `args`, `reason`, and `safety_notes` so clients do not parse prose remediation strings. `last_failed_or_partial_index_run` exposes bounded failed/partial index context (`status`, `mode`, timings, counts, stable error code, reason) and must not include raw exception text or file paths. | +| Status pragma diagnostics | `status --json` exposes resolved connection values under `db_pragma_settings` (`journal_mode`, `synchronous`, `wal_autocheckpoint`, `busy_timeout_ms`, `page_count`, `freelist_count`, `page_size`, `auto_vacuum`) and prepared-command cache counters under `prepared_command_cache` (`count`, `capacity`, `hit_count`, `miss_count`, `eviction_count`) for automation and support diagnostics. `maintenance_guidance` derives `wal_state`, `freelist_ratio`, `freelist_state`, `estimated_*_reclaimable`, `auto_vacuum_mode(_name)`, `recommended_command`, and `post_maintenance_follow_up` from those raw metrics without changing the raw values. `status --check --json` adds `repair_commands[]` entries with `name`, `args`, `reason`, and `safety_notes` so clients do not parse prose remediation strings. `last_failed_or_partial_index_run` exposes bounded failed/partial index context (`status`, `mode`, timings, counts, stable error code, reason) and must not include raw exception text or file paths. | | Maintenance thresholds | WAL guidance flips to `checkpoint_recommended` at `CDIDX_MAINTENANCE_WAL_WARN_BYTES` (default 64 MiB). Freelist guidance flips to `vacuum_recommended` at `CDIDX_MAINTENANCE_FREELIST_WARN_RATIO` (default `0.20`). Invalid or out-of-range env values fall back to defaults. | | Vacuum | `cdidx vacuum` runs `PRAGMA incremental_vacuum` against writable incremental-auto-vacuum DBs, and performs a one-time `PRAGMA auto_vacuum=INCREMENTAL` plus full `VACUUM` conversion for legacy no-autovacuum DBs. `cdidx vacuum --dry-run --json` estimates reclaimable pages/bytes and returns the same maintenance guidance without executing vacuum pragmas. | | Size and process diagnostics | `status --json` also reports `db_size_bytes`, `wal_size_bytes`, capped `symbol_kinds` / `symbols_by_language` kind maps with `symbol_kind_*` and `symbols_by_language_kind_*` overflow metadata when caps apply, current `process` heap/GC/working-set metrics, `last_index_run` metadata from successful CLI and MCP index runs, and `last_workspace_freshened_at` as the latest successful index/update timestamp. `last_index_run.bytes_read_skipped_file_count` and `bytes_read_incomplete` report whether unreadable files were omitted from the `bytes_read` total, while `last_index_run.diagnostics`, `diagnostic_count`, and `diagnostics_truncated` carry bounded warnings for best-effort index metadata writes that failed after the index data itself was successfully written. `indexed_at` still comes from indexed file rows, so partial or no-op updates can freshen the workspace without moving `indexed_at`. | @@ -489,6 +489,8 @@ Operators can override the defaults with environment variables: |---|---:|---| | `CDIDX_SQLITE_CACHE_KB` | `65536` | Positive cache size in KiB, up to `1048576`; cdidx applies it as a negative SQLite `cache_size` value so SQLite interprets it as KiB. Invalid or oversized values fall back to the default. | | `CDIDX_SQLITE_MMAP_BYTES` | `268435456` | Non-negative memory-map window in bytes on 64-bit processes, up to `1073741824`. Use `0` to disable mmap. Invalid or oversized values fall back to the default. | +| `CDIDX_SQLITE_BUSY_TIMEOUT_MS` | `5000` | Non-negative SQLite busy timeout in milliseconds, up to `3600000`. Use a higher value for slow disks or concurrent MCP/index workflows; invalid or oversized values fall back to the default. | +| `CDIDX_PREPARED_COMMAND_CACHE_CAPACITY` | `32` | Positive prepared SQLite command cache capacity per connection, up to `512`. Invalid or oversized values fall back to the default. | After a successful `cdidx index` run, the writer refreshes SQLite planner statistics so large repositories do not rely on default selectivity estimates for `search`, `references`, `callers`, and related joins. A brand-new index database runs full `ANALYZE` once after the initial population; later successful index runs use SQLite's lighter `PRAGMA optimize`. This maintenance is best-effort and never changes the schema contract. @@ -1059,7 +1061,7 @@ For the AI agent search-rule template, see [AI Integration](USER_GUIDE.md#ai-int | Workspace and HEAD freshness | `indexed_head_commit`, `worktree_head_changed`, `indexed_head_sha`, `indexed_head_branch`, `indexed_head_timestamp`, `commits_ahead_of_indexed_head`. | | Version and forward compatibility | `index_writer_version`, `index_newer_than_reader`, `index_newer_than_reader_reason`. | | Unknown-extension and runtime diagnostics | `unknown_extension_file_count`, `unknown_extension_files`, `unknown_extension_files_truncated`, `unknown_extension_file_path_limit`, `unknown_extension_extension_counts`, `unknown_extension_category_counts`, `unknown_extension_groups`, `extractors`, `hooks`, `hook_diagnostics`, `trust_overrides`, `path_case_sensitive`, `data_dir_mode`, `mac_profile`, `mac_profile_diagnostics`, `stale_after_seconds`, `index_age_seconds`, `last_failed_or_partial_index_run`. | -| Database maintenance | `db_size_bytes`, `wal_size_bytes`, `db_pragma_settings` (`journal_mode`, `synchronous`, `wal_autocheckpoint`, `page_count`, `freelist_count`, `page_size`, `auto_vacuum`), `maintenance_guidance`. | +| Database maintenance | `db_size_bytes`, `wal_size_bytes`, `db_pragma_settings` (`journal_mode`, `synchronous`, `wal_autocheckpoint`, `busy_timeout_ms`, `page_count`, `freelist_count`, `page_size`, `auto_vacuum`), `prepared_command_cache` (`count`, `capacity`, `hit_count`, `miss_count`, `eviction_count`), `maintenance_guidance`. | | Remediation fields | `degraded_root_cause`, `degraded_reason`, `recommended_action`, `alternative_action`, `readiness_degradations`, `repair_commands`. | | MCP-only session diagnostics | `mcp_session`, which is session-scoped diagnostics rather than persisted DB state. It contains `log_level`, bounded `roots`, optional `client_info`, and bounded optional `client_capabilities`. When advertised roots are capped, `roots_truncated`, `root_count`, `root_limit`, and `root_uri_length_limit` describe the truncation. When client capabilities are capped, `client_capabilities_truncated`, `client_capabilities_truncation_reason`, `client_capabilities_serialized_bytes`, `client_capabilities_byte_limit`, and `client_capabilities_depth_limit` describe the retained diagnostic subset. | | Documentation sync | Keep this list synchronized with `README.md` and `AGENT_GUIDE.md`; `DocumentationStatusContractTests` fails when any required field is missing from one of those docs. | @@ -2714,7 +2716,7 @@ alternative action を同じ場所へ追加してください。 | schema discovery cache | `DbReader` の schema discovery は正規化済み DB path を key にした process-level cache を使います。この cache は `PRAGMA table_info`、`PRAGMA index_list`、`sqlite_master` の table existence 結果を保持し、lookup 前に `PRAGMA schema_version` を確認するため、cdidx や外部 `sqlite3` session による SQLite DDL は stale snapshot を invalidate します。cdidx 外での手動 schema edit は運用上 unsupported であり、その後は query output を信頼する前に `cdidx validate` を実行してください。 | | batch trust marker | index write batch は mutation transaction を始める前に `codeindex_meta.batch_in_progress=true` を stamp し、対応する row と readiness metadata を commit する transaction 内で clear します。marker が書かれた後、clear される前に indexer が crash した場合、次の writable DB open は readiness bit を degrade し、`Last batch did not complete; run cdidx index --rebuild to re-index from a known clean state.` と警告します。file ごとの error が graceful に処理された場合は rollback 後に marker を clear するため、orphaned marker は interrupted / crashed batch の trust metadata を clean と扱わないための signal です。 | | read-only fallback | 通常の writable open が journal/WAL side file を作成または lock できない場合、read-only fallback は immutable SQLite URI を使うため、query command は read-only / sandboxed storage 上の DB でも読み取りを継続できます。この fallback は意図的に writable pragma、migration、WAL recovery write を skip します。WAL が存在し、その内容を観測する必要がある場合は、`.db` / `.db-wal` / `.db-shm` をまとめて writable location に copy するか、full WAL set を open できる環境で SQLite backup を使います。 | -| status pragma diagnostics | `status --json` は automation / support diagnostics 用に、解決済みの接続値を `db_pragma_settings` (`journal_mode`, `synchronous`, `wal_autocheckpoint`, `page_count`, `freelist_count`, `page_size`, `auto_vacuum`) で公開します。`maintenance_guidance` は raw 値を変えずに `wal_state`、`freelist_ratio`、`freelist_state`、`estimated_*_reclaimable`、`auto_vacuum_mode(_name)`、`recommended_command`、`post_maintenance_follow_up` を派生します。`status --check --json` は `repair_commands[]` に `name`、`args`、`reason`、`safety_notes` を返し、client が prose remediation を parse しなくてよいようにします。`last_failed_or_partial_index_run` は bounded な failed / partial index context (`status`、`mode`、timing、count、stable error code、reason) のみを公開し、raw exception text や file path を含めてはいけません。 | +| status pragma diagnostics | `status --json` は automation / support diagnostics 用に、解決済みの接続値を `db_pragma_settings` (`journal_mode`, `synchronous`, `wal_autocheckpoint`, `busy_timeout_ms`, `page_count`, `freelist_count`, `page_size`, `auto_vacuum`) で、prepared command cache counter を `prepared_command_cache` (`count`, `capacity`, `hit_count`, `miss_count`, `eviction_count`) で公開します。`maintenance_guidance` は raw 値を変えずに `wal_state`、`freelist_ratio`、`freelist_state`、`estimated_*_reclaimable`、`auto_vacuum_mode(_name)`、`recommended_command`、`post_maintenance_follow_up` を派生します。`status --check --json` は `repair_commands[]` に `name`、`args`、`reason`、`safety_notes` を返し、client が prose remediation を parse しなくてよいようにします。`last_failed_or_partial_index_run` は bounded な failed / partial index context (`status`、`mode`、timing、count、stable error code、reason) のみを公開し、raw exception text や file path を含めてはいけません。 | | maintenance threshold | WAL guidance は `CDIDX_MAINTENANCE_WAL_WARN_BYTES` (既定 64 MiB) 以上で `checkpoint_recommended` になります。freelist guidance は `CDIDX_MAINTENANCE_FREELIST_WARN_RATIO` (既定 `0.20`) 以上で `vacuum_recommended` になります。不正・範囲外の環境変数値は既定値へ戻します。 | | vacuum | `cdidx vacuum` は incremental-auto-vacuum DB では `PRAGMA incremental_vacuum` を実行し、legacy no-autovacuum DB では初回のみ `PRAGMA auto_vacuum=INCREMENTAL` と full `VACUUM` で変換します。`cdidx vacuum --dry-run --json` は vacuum pragma を実行せず、回収可能 page/byte の推定と同じ maintenance guidance を返します。 | | size / process diagnostics | `status --json` は `db_size_bytes`、`wal_size_bytes`、上限付きの `symbol_kinds` / `symbols_by_language` kind map と、上限適用時の `symbol_kind_*` / `symbols_by_language_kind_*` overflow metadata、現在の `process` heap / GC / working-set metrics、成功した CLI / MCP index 実行由来の `last_index_run` metadata、最新の成功 index/update 時刻を示す `last_workspace_freshened_at` も公開します。`last_index_run.bytes_read_skipped_file_count` と `bytes_read_incomplete` は、読み取り不能な file が `bytes_read` 合計から除外されたかどうかを報告します。`last_index_run.diagnostics`、`diagnostic_count`、`diagnostics_truncated` は、index data 自体の書き込みが成功した後に best-effort index metadata write が失敗した場合の上限付き warning を保持します。`indexed_at` は引き続き indexed file row 由来なので、partial / no-op update は `indexed_at` を動かさずに workspace 鮮度だけを更新することがあります。 | @@ -2742,6 +2744,8 @@ operator は environment variable で既定値を上書きできる。 |---|---:|---| | `CDIDX_SQLITE_CACHE_KB` | `65536` | KiB 単位の正の cache size。上限は `1048576`。cdidx は SQLite が KiB として解釈するよう負の `cache_size` 値として適用する。invalid / oversized value は既定値に戻る。 | | `CDIDX_SQLITE_MMAP_BYTES` | `268435456` | 64-bit process で使う memory-map window の byte 数。`0` 以上、上限 `1073741824`。`0` で mmap を無効化する。invalid / oversized value は既定値に戻る。 | +| `CDIDX_SQLITE_BUSY_TIMEOUT_MS` | `5000` | SQLite busy timeout の millisecond 値。`0` 以上、上限 `3600000`。低速 disk や concurrent MCP/index workflow では大きい値を使える。invalid / oversized value は既定値に戻る。 | +| `CDIDX_PREPARED_COMMAND_CACHE_CAPACITY` | `32` | connection ごとの prepared SQLite command cache capacity。正の整数、上限 `512`。invalid / oversized value は既定値に戻る。 | `cdidx index` が成功すると、writer は SQLite planner statistics を更新し、大規模 repository で `search`、`references`、`callers` などの join が default selectivity estimate に依存しないようにする。新規 index database は初回 population 後に full `ANALYZE` を一度実行し、それ以降の成功した index run では軽量な `PRAGMA optimize` を使う。この maintenance は best-effort であり、schema contract は変更しない。 @@ -3290,7 +3294,7 @@ AI エージェント向け検索ルールのテンプレートについては | workspace / HEAD freshness | `indexed_head_commit`, `worktree_head_changed`, `indexed_head_sha`, `indexed_head_branch`, `indexed_head_timestamp`, `commits_ahead_of_indexed_head`。 | | version / forward compatibility | `index_writer_version`, `index_newer_than_reader`, `index_newer_than_reader_reason`。 | | unknown-extension / runtime diagnostics | `unknown_extension_file_count`, `unknown_extension_files`, `unknown_extension_files_truncated`, `unknown_extension_file_path_limit`, `unknown_extension_extension_counts`, `unknown_extension_category_counts`, `unknown_extension_groups`, `extractors`, `hooks`, `hook_diagnostics`, `trust_overrides`, `path_case_sensitive`, `data_dir_mode`, `mac_profile`, `mac_profile_diagnostics`, `stale_after_seconds`, `index_age_seconds`, `last_failed_or_partial_index_run`。 | -| database maintenance | `db_size_bytes`, `wal_size_bytes`, `db_pragma_settings` (`journal_mode`, `synchronous`, `wal_autocheckpoint`, `page_count`, `freelist_count`, `page_size`, `auto_vacuum`), `maintenance_guidance`。 | +| database maintenance | `db_size_bytes`, `wal_size_bytes`, `db_pragma_settings` (`journal_mode`, `synchronous`, `wal_autocheckpoint`, `busy_timeout_ms`, `page_count`, `freelist_count`, `page_size`, `auto_vacuum`), `prepared_command_cache` (`count`, `capacity`, `hit_count`, `miss_count`, `eviction_count`), `maintenance_guidance`。 | | remediation fields | `degraded_root_cause`, `degraded_reason`, `recommended_action`, `alternative_action`, `readiness_degradations`, `repair_commands`。 | | MCP-only session diagnostics | `mcp_session`。これは persisted DB state ではなく session-scoped diagnostics で、`log_level`、上限付きの `roots`、任意の `client_info`、上限付きの任意の `client_capabilities` を含みます。advertised root が切り詰められた場合は `roots_truncated`、`root_count`、`root_limit`、`root_uri_length_limit` が切り詰め内容を示します。client capabilities が切り詰められた場合は `client_capabilities_truncated`、`client_capabilities_truncation_reason`、`client_capabilities_serialized_bytes`、`client_capabilities_byte_limit`、`client_capabilities_depth_limit` が保持された診断 subset を示します。 | | documentation sync | この一覧は `README.md` と `AGENT_GUIDE.md` と同期してください。必須 field がそれらの docs から欠けると `DocumentationStatusContractTests` が失敗します。 | diff --git a/README.md b/README.md index 9ecbfdb47e..aefa79fa45 100644 --- a/README.md +++ b/README.md @@ -142,7 +142,7 @@ names visible so documentation and tests stay synchronized. | Workspace and HEAD freshness | `indexed_head_commit`, `worktree_head_changed`, `indexed_head_sha`, `indexed_head_branch`, `indexed_head_timestamp`, `commits_ahead_of_indexed_head`. | | Version and forward compatibility | `index_writer_version`, `index_newer_than_reader`, `index_newer_than_reader_reason`. | | Unknown-extension and runtime diagnostics | `unknown_extension_file_count`, `unknown_extension_files`, `unknown_extension_files_truncated`, `unknown_extension_file_path_limit`, `unknown_extension_extension_counts`, `unknown_extension_category_counts`, `unknown_extension_groups`, `extractors`, `hooks`, `hook_diagnostics`, `trust_overrides`, `path_case_sensitive`, `data_dir_mode`, `mac_profile`, `mac_profile_diagnostics`, `stale_after_seconds`, `index_age_seconds`, `last_index_run.bytes_read_skipped_file_count`, `last_index_run.bytes_read_incomplete`, `last_index_run.diagnostics`, `last_index_run.diagnostic_count`, `last_index_run.diagnostics_truncated`, `last_failed_or_partial_index_run`. | -| Database maintenance | `db_size_bytes`, `wal_size_bytes`, `db_pragma_settings` (`journal_mode`, `synchronous`, `wal_autocheckpoint`, `page_count`, `freelist_count`, `page_size`, `auto_vacuum`), `maintenance_guidance`. | +| Database maintenance | `db_size_bytes`, `wal_size_bytes`, `db_pragma_settings` (`journal_mode`, `synchronous`, `wal_autocheckpoint`, `busy_timeout_ms`, `page_count`, `freelist_count`, `page_size`, `auto_vacuum`), `prepared_command_cache` (`count`, `capacity`, `hit_count`, `miss_count`, `eviction_count`), `maintenance_guidance`. | | Remediation fields | `degraded_root_cause`, `degraded_reason`, `recommended_action`, `alternative_action`, `readiness_degradations`, `repair_commands`. | | MCP-only session diagnostics | `mcp_session`. | @@ -323,7 +323,7 @@ freshness、compatibility、remediation field を返します。詳細な意味 | workspace / HEAD freshness | `indexed_head_commit`, `worktree_head_changed`, `indexed_head_sha`, `indexed_head_branch`, `indexed_head_timestamp`, `commits_ahead_of_indexed_head`。 | | version / forward compatibility | `index_writer_version`, `index_newer_than_reader`, `index_newer_than_reader_reason`。 | | unknown-extension / runtime diagnostics | `unknown_extension_file_count`, `unknown_extension_files`, `unknown_extension_files_truncated`, `unknown_extension_file_path_limit`, `unknown_extension_extension_counts`, `unknown_extension_category_counts`, `unknown_extension_groups`, `extractors`, `hooks`, `hook_diagnostics`, `trust_overrides`, `path_case_sensitive`, `data_dir_mode`, `mac_profile`, `mac_profile_diagnostics`, `stale_after_seconds`, `index_age_seconds`, `last_index_run.bytes_read_skipped_file_count`, `last_index_run.bytes_read_incomplete`, `last_index_run.diagnostics`, `last_index_run.diagnostic_count`, `last_index_run.diagnostics_truncated`, `last_failed_or_partial_index_run`。 | -| database maintenance | `db_size_bytes`, `wal_size_bytes`, `db_pragma_settings` (`journal_mode`, `synchronous`, `wal_autocheckpoint`, `page_count`, `freelist_count`, `page_size`, `auto_vacuum`), `maintenance_guidance`。 | +| database maintenance | `db_size_bytes`, `wal_size_bytes`, `db_pragma_settings` (`journal_mode`, `synchronous`, `wal_autocheckpoint`, `busy_timeout_ms`, `page_count`, `freelist_count`, `page_size`, `auto_vacuum`), `prepared_command_cache` (`count`, `capacity`, `hit_count`, `miss_count`, `eviction_count`), `maintenance_guidance`。 | | remediation fields | `degraded_root_cause`, `degraded_reason`, `recommended_action`, `alternative_action`, `readiness_degradations`, `repair_commands`。 | | MCP-only session diagnostics | `mcp_session`。 | diff --git a/changelog.d/unreleased/3702.fixed.md b/changelog.d/unreleased/3702.fixed.md new file mode 100644 index 0000000000..70a35ce02f --- /dev/null +++ b/changelog.d/unreleased/3702.fixed.md @@ -0,0 +1,19 @@ +--- +category: fixed +issues: + - 3702 +affected: + - src/CodeIndex/Database/SqliteDynamicSql.cs + - src/CodeIndex/Database/DbReader.cs + - src/CodeIndex/Database/DbSymbolReader.cs + - src/CodeIndex/Database/DbWriter.cs + - tests/CodeIndex.Tests/SqliteDynamicSqlTests.cs +--- + +## English + +- Centralize generated SQLite `IN` parameter lists and repeated path/visibility/language filter parameter binding behind an explicit 999-parameter budget. + +## 日本語 + +- SQLite の生成 `IN` parameter list と path / visibility / language filter の繰り返し parameter binding を、明示的な 999 parameter budget を持つ共通ヘルパへ集約しました。 diff --git a/changelog.d/unreleased/3716.fixed.md b/changelog.d/unreleased/3716.fixed.md new file mode 100644 index 0000000000..d19e00722f --- /dev/null +++ b/changelog.d/unreleased/3716.fixed.md @@ -0,0 +1,16 @@ +--- +category: fixed +issues: + - 3716 +affected: + - src/CodeIndex/Database/DbWriter.cs + - tests/CodeIndex.Tests/LegacySchemaMigrationTests.cs +--- + +## English + +- Centralize `PRAGMA user_version` ready-bit stamping so fold-ready and other readiness flags share the same read-modify-write helper across raw and tracked transactions. + +## 日本語 + +- `PRAGMA user_version` の ready-bit stamp を共通化し、fold-ready と他の readiness flag が raw transaction / tracked transaction の両方で同じ read-modify-write helper を使うようにしました。 diff --git a/changelog.d/unreleased/3718.fixed.md b/changelog.d/unreleased/3718.fixed.md new file mode 100644 index 0000000000..f6b5d769e2 --- /dev/null +++ b/changelog.d/unreleased/3718.fixed.md @@ -0,0 +1,18 @@ +--- +category: fixed +issues: + - 3718 +affected: + - src/CodeIndex/Database/DbContext.cs + - src/CodeIndex/Cli/IndexCommandRunner.cs + - src/CodeIndex/Mcp/McpToolHandlers.cs + - tests/CodeIndex.Tests/IndexCommandRunnerTests.cs +--- + +## English + +- Record best-effort planner statistics maintenance failures as bounded `last_index_run.diagnostics` entries without changing successful index exit codes. + +## 日本語 + +- best-effort の planner statistics maintenance 失敗を、index 成功の exit code を変えずに上限付きの `last_index_run.diagnostics` として記録するようにしました。 diff --git a/changelog.d/unreleased/3739.fixed.md b/changelog.d/unreleased/3739.fixed.md new file mode 100644 index 0000000000..fcf4cbef73 --- /dev/null +++ b/changelog.d/unreleased/3739.fixed.md @@ -0,0 +1,16 @@ +--- +category: fixed +issues: + - 3739 +affected: + - src/CodeIndex/Database/DbDebug.cs + - tests/CodeIndex.Tests/DbDebugTests.cs +--- + +## English + +- Keep `DbDebug` query-plan diagnostics bounded by row and detail length, and report explicit truncation markers when limits are reached. + +## 日本語 + +- `DbDebug` の query-plan diagnostics を行数と detail 長で上限管理し、上限到達時に明示的な truncation marker を報告するようにしました。 diff --git a/changelog.d/unreleased/3767.fixed.md b/changelog.d/unreleased/3767.fixed.md new file mode 100644 index 0000000000..5cc06bd826 --- /dev/null +++ b/changelog.d/unreleased/3767.fixed.md @@ -0,0 +1,24 @@ +--- +category: fixed +issues: + - 3767 +affected: + - src/CodeIndex/Database/DbContext.cs + - src/CodeIndex/Database/DbPragmaPolicy.cs + - src/CodeIndex/Database/DbReader.FilesStatus.cs + - src/CodeIndex/Models/QueryResults.cs + - README.md + - DEVELOPER_GUIDE.md + - AGENT_GUIDE.md + - tests/CodeIndex.Tests/QueryCommandRunnerTests.cs + - tests/CodeIndex.Tests/QueryCommandRunnerFilesTests.cs + - tests/CodeIndex.Tests/golden/status.json +--- + +## English + +- Make SQLite busy timeout configurable with `CDIDX_SQLITE_BUSY_TIMEOUT_MS` and expose the effective value as `db_pragma_settings.busy_timeout_ms` in `status --json`. + +## 日本語 + +- SQLite busy timeout を `CDIDX_SQLITE_BUSY_TIMEOUT_MS` で設定できるようにし、`status --json` の `db_pragma_settings.busy_timeout_ms` で有効値を確認できるようにしました。 diff --git a/changelog.d/unreleased/3795.fixed.md b/changelog.d/unreleased/3795.fixed.md new file mode 100644 index 0000000000..e948954d31 --- /dev/null +++ b/changelog.d/unreleased/3795.fixed.md @@ -0,0 +1,25 @@ +--- +category: fixed +issues: + - 3795 +affected: + - src/CodeIndex/Database/PreparedCommandCache.cs + - src/CodeIndex/Database/DbContext.cs + - src/CodeIndex/Database/DbReader.FilesStatus.cs + - src/CodeIndex/Database/DbWriter.cs + - src/CodeIndex/Models/QueryResults.cs + - README.md + - DEVELOPER_GUIDE.md + - AGENT_GUIDE.md + - tests/CodeIndex.Tests/PreparedCommandCacheTests.cs + - tests/CodeIndex.Tests/QueryCommandRunnerFilesTests.cs + - tests/CodeIndex.Tests/golden/status.json +--- + +## English + +- Expose prepared command cache count, capacity, hit, miss, and eviction counters through `status --json`, make capacity configurable with `CDIDX_PREPARED_COMMAND_CACHE_CAPACITY`, and use typed metadata parameters on a hot writer path. + +## 日本語 + +- prepared command cache の count / capacity / hit / miss / eviction counter を `status --json` で公開し、`CDIDX_PREPARED_COMMAND_CACHE_CAPACITY` で capacity を設定できるようにしたうえで、writer の hot metadata path を型付き parameter にしました。 diff --git a/changelog.d/unreleased/3834.fixed.md b/changelog.d/unreleased/3834.fixed.md new file mode 100644 index 0000000000..32fbfb34a5 --- /dev/null +++ b/changelog.d/unreleased/3834.fixed.md @@ -0,0 +1,18 @@ +--- +category: fixed +issues: + - 3834 +affected: + - src/CodeIndex/Cli/DiffCommandRunner.cs + - src/CodeIndex/Cli/QueryCommandRunner.cs + - tests/CodeIndex.Tests/DiffCommandRunnerTests.cs + - tests/CodeIndex.Tests/QueryCommandRunnerTests.cs +--- + +## English + +- Harden database diff and cross-database dependency reads with quoted PRAGMA identifiers, explicit diff comparison budgets, filesystem-case-aware workspace DB de-duplication, read-only attach fallback, and segment-aware `--exclude-tests` filtering. + +## 日本語 + +- database diff と cross-database dependency read について、PRAGMA identifier quoting、明示的な diff comparison budget、filesystem の case 感度に基づく workspace DB 重複排除、read-only attach fallback、segment-aware な `--exclude-tests` filter を追加しました。 diff --git a/src/CodeIndex/Cli/DiffCommandRunner.cs b/src/CodeIndex/Cli/DiffCommandRunner.cs index 24d6a59b81..da062f28a5 100644 --- a/src/CodeIndex/Cli/DiffCommandRunner.cs +++ b/src/CodeIndex/Cli/DiffCommandRunner.cs @@ -11,7 +11,11 @@ public static class DiffCommandRunner { private const int DefaultDiffLimit = 20; internal const int MaxDiffEncodedFieldSampleLength = 1024; + internal const int MaxDiffComparedRowsPerSide = 1_000_000; + internal const int MaxDiffComparedRowBytes = 4 * 1024 * 1024; internal static int MaxDiffLimit => QueryCommandRunner.NumericFlagUpperBounds["--limit"]; + internal static int? MaxDiffComparedRowsPerSideForTesting { get; set; } + internal static int? MaxDiffComparedRowBytesForTesting { get; set; } private const int DriftExitCode = 1; private const int SchemaMismatchExitCode = 2; private const int UnreadableExitCode = 3; @@ -397,8 +401,10 @@ private static OrderedRowsDiff DiffOrderedRows( var onlyInLeft = new List(limit); var onlyInRight = new List(limit); - var leftHasValue = TryReadRow(leftReader, out var leftValue); - var rightHasValue = TryReadRow(rightReader, out var rightValue); + var leftRowsRead = 0; + var rightRowsRead = 0; + var leftHasValue = TryReadRow(leftReader, out var leftValue, ref leftRowsRead, "left"); + var rightHasValue = TryReadRow(rightReader, out var rightValue, ref rightRowsRead, "right"); var equal = true; while (leftHasValue || rightHasValue) @@ -409,8 +415,8 @@ private static OrderedRowsDiff DiffOrderedRows( if (comparison == 0) { - leftHasValue = TryReadRow(leftReader, out leftValue); - rightHasValue = TryReadRow(rightReader, out rightValue); + leftHasValue = TryReadRow(leftReader, out leftValue, ref leftRowsRead, "left"); + rightHasValue = TryReadRow(rightReader, out rightValue, ref rightRowsRead, "right"); continue; } @@ -419,13 +425,13 @@ private static OrderedRowsDiff DiffOrderedRows( { if (onlyInLeft.Count < limit) onlyInLeft.Add(EncodeRow(leftValue.SortValues)); - leftHasValue = TryReadRow(leftReader, out leftValue); + leftHasValue = TryReadRow(leftReader, out leftValue, ref leftRowsRead, "left"); } else { if (onlyInRight.Count < limit) onlyInRight.Add(EncodeRow(rightValue.SortValues)); - rightHasValue = TryReadRow(rightReader, out rightValue); + rightHasValue = TryReadRow(rightReader, out rightValue, ref rightRowsRead, "right"); } if (onlyInLeft.Count >= limit && onlyInRight.Count >= limit) @@ -449,8 +455,10 @@ private static OrderedRowsDiff DiffOrderedStrings(SqliteConnection leftConnectio var onlyInLeft = new List(limit); var onlyInRight = new List(limit); - var leftHasValue = TryReadString(leftReader, out var leftValue); - var rightHasValue = TryReadString(rightReader, out var rightValue); + var leftRowsRead = 0; + var rightRowsRead = 0; + var leftHasValue = TryReadString(leftReader, out var leftValue, ref leftRowsRead, "left"); + var rightHasValue = TryReadString(rightReader, out var rightValue, ref rightRowsRead, "right"); var equal = true; while (leftHasValue || rightHasValue) @@ -461,8 +469,8 @@ private static OrderedRowsDiff DiffOrderedStrings(SqliteConnection leftConnectio if (comparison == 0) { - leftHasValue = TryReadString(leftReader, out leftValue); - rightHasValue = TryReadString(rightReader, out rightValue); + leftHasValue = TryReadString(leftReader, out leftValue, ref leftRowsRead, "left"); + rightHasValue = TryReadString(rightReader, out rightValue, ref rightRowsRead, "right"); continue; } @@ -471,13 +479,13 @@ private static OrderedRowsDiff DiffOrderedStrings(SqliteConnection leftConnectio { if (onlyInLeft.Count < limit) onlyInLeft.Add(leftValue); - leftHasValue = TryReadString(leftReader, out leftValue); + leftHasValue = TryReadString(leftReader, out leftValue, ref leftRowsRead, "left"); } else { if (onlyInRight.Count < limit) onlyInRight.Add(rightValue); - rightHasValue = TryReadString(rightReader, out rightValue); + rightHasValue = TryReadString(rightReader, out rightValue, ref rightRowsRead, "right"); } if (onlyInLeft.Count >= limit && onlyInRight.Count >= limit) @@ -499,14 +507,16 @@ private static bool RowsEqual(SqliteConnection leftConnection, SqliteConnection using var leftReader = leftCommand.ExecuteReader(); using var rightReader = rightCommand.ExecuteReader(); - var leftHasValue = TryReadRow(leftReader, out var leftValue); - var rightHasValue = TryReadRow(rightReader, out var rightValue); + var leftRowsRead = 0; + var rightRowsRead = 0; + var leftHasValue = TryReadRow(leftReader, out var leftValue, ref leftRowsRead, "left"); + var rightHasValue = TryReadRow(rightReader, out var rightValue, ref rightRowsRead, "right"); while (leftHasValue && rightHasValue) { if (CompareRows(leftValue, rightValue) != 0) return false; - leftHasValue = TryReadRow(leftReader, out leftValue); - rightHasValue = TryReadRow(rightReader, out rightValue); + leftHasValue = TryReadRow(leftReader, out leftValue, ref leftRowsRead, "left"); + rightHasValue = TryReadRow(rightReader, out rightValue, ref rightRowsRead, "right"); } return leftHasValue == rightHasValue; @@ -521,20 +531,22 @@ private static bool StringRowsEqual(SqliteConnection leftConnection, SqliteConne using var leftReader = leftCommand.ExecuteReader(); using var rightReader = rightCommand.ExecuteReader(); - var leftHasValue = TryReadString(leftReader, out var leftValue); - var rightHasValue = TryReadString(rightReader, out var rightValue); + var leftRowsRead = 0; + var rightRowsRead = 0; + var leftHasValue = TryReadString(leftReader, out var leftValue, ref leftRowsRead, "left"); + var rightHasValue = TryReadString(rightReader, out var rightValue, ref rightRowsRead, "right"); while (leftHasValue && rightHasValue) { if (!string.Equals(leftValue, rightValue, StringComparison.Ordinal)) return false; - leftHasValue = TryReadString(leftReader, out leftValue); - rightHasValue = TryReadString(rightReader, out rightValue); + leftHasValue = TryReadString(leftReader, out leftValue, ref leftRowsRead, "left"); + rightHasValue = TryReadString(rightReader, out rightValue, ref rightRowsRead, "right"); } return leftHasValue == rightHasValue; } - private static bool TryReadRow(SqliteDataReader reader, out DiffRow value) + private static bool TryReadRow(SqliteDataReader reader, out DiffRow value, ref int rowsRead, string side) { if (!reader.Read()) { @@ -542,9 +554,16 @@ private static bool TryReadRow(SqliteDataReader reader, out DiffRow value) return false; } + IncrementDiffRowsRead(ref rowsRead, side); var sortValues = new object?[reader.FieldCount]; + long rowBytes = 0; for (var i = 0; i < reader.FieldCount; i++) - sortValues[i] = reader.IsDBNull(i) ? null : reader.GetValue(i); + { + var fieldValue = reader.IsDBNull(i) ? null : reader.GetValue(i); + rowBytes += EstimateDiffValueBytes(fieldValue); + EnsureDiffRowByteBudget(rowBytes, side); + sortValues[i] = fieldValue; + } value = new DiffRow(sortValues); return true; @@ -612,7 +631,7 @@ private static int CompareBytes(byte[] left, byte[] right) return left.Length.CompareTo(right.Length); } - private static bool TryReadString(SqliteDataReader reader, out string value) + private static bool TryReadString(SqliteDataReader reader, out string value, ref int rowsRead, string side) { if (!reader.Read()) { @@ -620,10 +639,36 @@ private static bool TryReadString(SqliteDataReader reader, out string value) return false; } + IncrementDiffRowsRead(ref rowsRead, side); value = reader.IsDBNull(0) ? string.Empty : reader.GetString(0); + EnsureDiffRowByteBudget(EstimateDiffValueBytes(value), side); return true; } + private static void IncrementDiffRowsRead(ref int rowsRead, string side) + { + rowsRead++; + var maxRows = MaxDiffComparedRowsPerSideForTesting ?? MaxDiffComparedRowsPerSide; + if (rowsRead > maxRows) + throw new InvalidOperationException($"diff {side} row comparison exceeded the safety budget of {maxRows} rows."); + } + + private static void EnsureDiffRowByteBudget(long rowBytes, string side) + { + var maxBytes = MaxDiffComparedRowBytesForTesting ?? MaxDiffComparedRowBytes; + if (rowBytes > maxBytes) + throw new InvalidOperationException($"diff {side} row comparison exceeded the safety budget of {maxBytes} bytes per row."); + } + + private static long EstimateDiffValueBytes(object? value) + => value switch + { + null or DBNull => 0, + byte[] bytes => bytes.LongLength, + string text => Encoding.UTF8.GetByteCount(text), + _ => Encoding.UTF8.GetByteCount(Convert.ToString(value, System.Globalization.CultureInfo.InvariantCulture) ?? string.Empty), + }; + private static SqliteConnection OpenReadOnlyConnection(string dbPath) { var isUri = dbPath.StartsWith("file:", StringComparison.OrdinalIgnoreCase); @@ -675,7 +720,7 @@ private static long ExecuteCountIfTableExists(SqliteConnection connection, strin private static bool ColumnExists(SqliteConnection connection, string table, string column) { using var command = connection.CreateCommand(); - command.CommandText = $"PRAGMA table_info({table})"; + command.CommandText = $"PRAGMA table_info({SqliteIdentifier.Quote(table)})"; using var reader = command.ExecuteReader(); while (reader.Read()) { diff --git a/src/CodeIndex/Cli/IndexCommandRunner.cs b/src/CodeIndex/Cli/IndexCommandRunner.cs index 558fd14b3a..5e9e2de285 100644 --- a/src/CodeIndex/Cli/IndexCommandRunner.cs +++ b/src/CodeIndex/Cli/IndexCommandRunner.cs @@ -33,6 +33,7 @@ public static partial class IndexCommandRunner private const int MaxIndexRunDiagnosticLength = 512; private const int ScanCheckpointVersion = 1; private const string ScanCheckpointFileName = "scan-checkpoint.json"; + private static readonly System.Threading.AsyncLocal>?> ScopedPlannerStatisticsMaintenanceDiagnosticStampingForTesting = new(); private static readonly TimeSpan IndexExtractionStallTimeout = TimeSpan.FromMinutes(5); internal readonly record struct FileByteReadSummary(long BytesRead, long SkippedFileCount); @@ -302,7 +303,11 @@ internal static int Run(string[] indexArgs, JsonSerializerOptions jsonOptions, C ? RunUpdateMode(writer, indexer, projectRoot, resolvedDbPath, options, stopwatch, runStartedAtUtc, spinnerFrames, jsonOptions, priorReadiness, priorSymbolsOnlyGraphOmitted, priorFoldVersion, priorFoldFingerprint, priorSymbolExtractorVersionsMatchCurrent, priorCSharpSymbolNameContractVersion, priorMetadataTargetCsharp, priorSqlGraphContractVersion, priorHotspotFamilyVersions, priorHotspotFamilyMarkerFingerprints, currentHotspotFamilyMarkerFingerprints, priorIndexedProjectRoot, priorIndexedHeadCommit, currentHeadCommit, priorSymbolKindFilterSignature, initialCwd, indexRunDiagnostics, indexCancellation.Token) : RunFullScan(writer, indexer, projectRoot, resolvedDbPath, options, stopwatch, runStartedAtUtc, spinnerFrames, jsonOptions, priorReadiness, priorSymbolsOnlyGraphOmitted, priorFoldVersion, priorFoldFingerprint, priorSymbolExtractorVersionsMatchCurrent, priorCSharpSymbolNameContractVersion, priorMetadataTargetCsharp, priorSqlGraphContractVersion, priorHotspotFamilyVersions, priorHotspotFamilyMarkerFingerprints, currentHotspotFamilyMarkerFingerprints, priorIndexedProjectRoot, priorIndexedHeadCommit, currentHeadCommit, priorSymbolKindFilterSignature, initialCwd, indexRunDiagnostics, showNextSteps: !databaseExistedBeforeIndex, indexCancellation.Token); if (initialExitCode == CommandExitCodes.Success) - db.RunPlannerStatisticsMaintenance(forceAnalyze: !databaseExistedBeforeIndex); + { + var plannerMaintenanceFailure = db.RunPlannerStatisticsMaintenance(forceAnalyze: !databaseExistedBeforeIndex); + if (plannerMaintenanceFailure != null) + TryStampPlannerStatisticsMaintenanceDiagnostic(writer, indexRunDiagnostics, plannerMaintenanceFailure); + } } } catch (IndexInterruptedException ex) @@ -516,6 +521,31 @@ internal static void StampLastIndexRunDiagnostics(DbWriter writer, IReadOnlyList (total > sample.Count).ToString(System.Globalization.CultureInfo.InvariantCulture)); } + internal static Action>? PlannerStatisticsMaintenanceDiagnosticStampingForTesting + { + get => ScopedPlannerStatisticsMaintenanceDiagnosticStampingForTesting.Value; + set => ScopedPlannerStatisticsMaintenanceDiagnosticStampingForTesting.Value = value; + } + + internal static bool TryStampPlannerStatisticsMaintenanceDiagnostic( + DbWriter writer, + List indexRunDiagnostics, + DbContext.PlannerStatisticsMaintenanceFailure plannerMaintenanceFailure) + { + indexRunDiagnostics.Add(FormatPlannerStatisticsMaintenanceDiagnostic(plannerMaintenanceFailure)); + try + { + PlannerStatisticsMaintenanceDiagnosticStampingForTesting?.Invoke(writer, indexRunDiagnostics); + StampLastIndexRunDiagnostics(writer, indexRunDiagnostics); + return true; + } + catch (Exception ex) + { + GlobalToolLog.Error("planner_statistics_maintenance_diagnostic_persist_failed", ex, includeStacks: false); + return false; + } + } + internal static string FormatIndexRunDiagnostic(string code, Exception ex) { var raw = $"{code}: {ex.GetType().Name}: {CollapseLineBreaks(ex.Message)}"; @@ -535,6 +565,12 @@ internal static string FormatIndexRunDiagnostic(string code, string? target, Exc : raw[..MaxIndexRunDiagnosticLength] + "..."; } + internal static string FormatPlannerStatisticsMaintenanceDiagnostic(DbContext.PlannerStatisticsMaintenanceFailure failure) + => FormatIndexRunDiagnostic( + "planner_statistics_maintenance_failed", + failure.CommandText, + failure.Exception); + private static void RecordIndexRunDiagnostic(List? diagnostics, string code, Exception ex) { if (diagnostics == null) diff --git a/src/CodeIndex/Cli/QueryCommandRunner.cs b/src/CodeIndex/Cli/QueryCommandRunner.cs index 88b217d8b6..c991d74d67 100644 --- a/src/CodeIndex/Cli/QueryCommandRunner.cs +++ b/src/CodeIndex/Cli/QueryCommandRunner.cs @@ -7181,13 +7181,16 @@ private static List GetWorkspaceFileDependencies(DbReader .ToList(); } - private static List BuildWorkspaceDependencyDatabaseList(QueryCommandOptions options) + internal static List BuildWorkspaceDependencyDatabaseList(QueryCommandOptions options) { var primaryDb = Path.GetFullPath(DbPathResolver.NormalizeDbPath(options.DbPath)); + var comparer = PathCasing.IsIgnoreCase(primaryDb) + ? StringComparer.OrdinalIgnoreCase + : StringComparer.Ordinal; return options.WorkspaceDbPaths .Select(path => Path.GetFullPath(DbPathResolver.NormalizeDbPath(path))) .Prepend(primaryDb) - .Distinct(StringComparer.Ordinal) + .Distinct(comparer) .ToList(); } @@ -7211,17 +7214,10 @@ private static bool TryWriteWorkspaceDependencyFanOutError(QueryCommandOptions o private static List GetCrossDatabaseFileDependencies(string sourceDbPath, string targetDbPath, QueryCommandOptions options, bool reverse, int limit) { - var builder = new SqliteConnectionStringBuilder - { - DataSource = sourceDbPath, - Mode = SqliteOpenMode.ReadOnly, - }; - using var connection = new SqliteConnection(builder.ConnectionString); - connection.Open(); - using var attach = connection.CreateCommand(); - attach.CommandText = "ATTACH DATABASE @targetDb AS targetdb"; - attach.Parameters.AddWithValue("@targetDb", targetDbPath); - attach.ExecuteNonQuery(); + using var sourceDb = new DbContext(sourceDbPath); + sourceDb.TryMigrateForRead(); + var connection = sourceDb.Connection; + AttachCrossDatabaseTarget(connection, targetDbPath); using var cmd = connection.CreateCommand(); var sourcePathExpr = reverse ? "dst.path" : "src.path"; @@ -7247,8 +7243,8 @@ FROM symbol_references r AddCrossDatabaseExcludeFilters(cmd, "dst", options.ExcludePaths, include: reverse); if (options.ExcludeTests) cmd.CommandText += reverse - ? " AND dst.path NOT LIKE '%test%' COLLATE NOCASE" - : " AND src.path NOT LIKE '%test%' COLLATE NOCASE"; + ? $" AND NOT {BuildCrossDatabaseTestPathCondition("dst")}" + : $" AND NOT {BuildCrossDatabaseTestPathCondition("src")}"; cmd.CommandText += @" ), edge_totals AS ( @@ -7300,16 +7296,37 @@ LEFT JOIN ranked_edge_symbols return results; } + private static void AttachCrossDatabaseTarget(SqliteConnection connection, string targetDbPath) + { + try + { + AttachCrossDatabaseTargetCore(connection, targetDbPath); + } + catch (SqliteException) when (!SqliteFileUri.StartsWithFileScheme(targetDbPath) && File.Exists(LongPath.EnsureWindowsPrefix(targetDbPath))) + { + AttachCrossDatabaseTargetCore(connection, DbContext.ToReadOnlyUri(targetDbPath)); + } + } + + private static void AttachCrossDatabaseTargetCore(SqliteConnection connection, string targetDbPath) + { + using var attach = connection.CreateCommand(); + attach.CommandText = "ATTACH DATABASE @targetDb AS targetdb"; + attach.Parameters.Add("@targetDb", SqliteType.Text).Value = targetDbPath; + attach.ExecuteNonQuery(); + } + private static void AddCrossDatabasePathFilters(SqliteCommand cmd, string alias, IReadOnlyList patterns, bool include) { if (!include || patterns.Count == 0) return; + SqliteDynamicSql.EnsureParameterBudget(patterns.Count, "cross-database path filters"); var parts = new List(patterns.Count); for (var i = 0; i < patterns.Count; i++) { - var name = $"@crossPath{alias}{i}"; + var name = SqliteDynamicSql.BuildParameterName($"crossPath{alias}", i); parts.Add($"{alias}.path LIKE {name} ESCAPE '\\'"); - cmd.Parameters.AddWithValue(name, CrossDatabaseGlobToLikePattern(patterns[i])); + cmd.Parameters.Add(name, SqliteType.Text).Value = CrossDatabaseGlobToLikePattern(patterns[i]); } cmd.CommandText += " AND (" + string.Join(" OR ", parts) + ")"; } @@ -7318,14 +7335,21 @@ private static void AddCrossDatabaseExcludeFilters(SqliteCommand cmd, string ali { if (!include || patterns.Count == 0) return; + SqliteDynamicSql.EnsureParameterBudget(patterns.Count, "cross-database exclude path filters"); for (var i = 0; i < patterns.Count; i++) { - var name = $"@crossExclude{alias}{i}"; + var name = SqliteDynamicSql.BuildParameterName($"crossExclude{alias}", i); cmd.CommandText += $" AND {alias}.path NOT LIKE {name} ESCAPE '\\'"; - cmd.Parameters.AddWithValue(name, CrossDatabaseGlobToLikePattern(patterns[i])); + cmd.Parameters.Add(name, SqliteType.Text).Value = CrossDatabaseGlobToLikePattern(patterns[i]); } } + internal static string BuildCrossDatabaseTestPathConditionForTesting(string alias) + => BuildCrossDatabaseTestPathCondition(alias); + + private static string BuildCrossDatabaseTestPathCondition(string alias) + => DbReader.TestPathCondition.Replace("f.path", $"{alias}.path", StringComparison.Ordinal); + private static string CrossDatabaseGlobToLikePattern(string pattern) { var builder = new System.Text.StringBuilder(pattern.Length); diff --git a/src/CodeIndex/Database/DbContext.cs b/src/CodeIndex/Database/DbContext.cs index fbdbcc0971..a8a81d8863 100644 --- a/src/CodeIndex/Database/DbContext.cs +++ b/src/CodeIndex/Database/DbContext.cs @@ -19,6 +19,7 @@ public class DbContext : IDisposable public const long MaxMmapSizeBytes = 1073741824; public const string CacheSizeEnvironmentVariable = "CDIDX_SQLITE_CACHE_KB"; public const string MmapSizeEnvironmentVariable = "CDIDX_SQLITE_MMAP_BYTES"; + public const string BusyTimeoutEnvironmentVariable = "CDIDX_SQLITE_BUSY_TIMEOUT_MS"; public const int DefaultWalAutocheckpointPages = 1000; public const string DefaultSynchronousMode = "NORMAL"; public const string SymbolExtractorVersionMetaPrefix = "symbol_extractor_version_"; @@ -101,6 +102,7 @@ private static readonly (string Table, string Column)[] ReadMigrationRequiredCol private bool _rebuildFtsAfterSchemaMigration; private static readonly AsyncLocal?> ScopedOptimizePragmaExecutedForTesting = new(); + private static readonly AsyncLocal?> ScopedPlannerStatisticsCommandCreatedForTesting = new(); private static readonly AsyncLocal?> ScopedPlannerStatisticsCommandExecutedForTesting = new(); private static readonly AsyncLocal?> ScopedWalCheckpointTruncateExecutedForTesting = new(); @@ -110,6 +112,12 @@ internal static Action? OptimizePragmaExecutedForTesting set => ScopedOptimizePragmaExecutedForTesting.Value = value; } + internal static Action? PlannerStatisticsCommandCreatedForTesting + { + get => ScopedPlannerStatisticsCommandCreatedForTesting.Value; + set => ScopedPlannerStatisticsCommandCreatedForTesting.Value = value; + } + internal static Action? PlannerStatisticsCommandExecutedForTesting { get => ScopedPlannerStatisticsCommandExecutedForTesting.Value; @@ -154,7 +162,9 @@ public static string GetSymbolExtractorVersionMetaKey(string lang) /// ホットパス共有の prepared command LRU キャッシュ。Issue #1566. /// internal PreparedCommandCache PreparedCommands - => _preparedCommands ??= new PreparedCommandCache(_connection); + => _preparedCommands ??= new PreparedCommandCache( + _connection, + PreparedCommandCache.ReadCapacityFromEnvironment()); public static bool TryValidateExistingCodeIndexDb( string dbPath, @@ -291,7 +301,7 @@ public DbContext(string dbPath, CancellationToken cancellationToken = default) _connection = new SqliteConnection(DbPathResolver.BuildSqliteConnectionString(dbPath, SqliteOpenMode.ReadOnly)); cancellationToken.ThrowIfCancellationRequested(); _connection.Open(); - Execute("PRAGMA busy_timeout=5000"); + ApplyBusyTimeoutPragma(); ApplyConnectionPerformancePragmas(); RegisterConnectionFunctionsWithRetry(_connection, cancellationToken: cancellationToken); _isReadOnly = true; @@ -327,7 +337,7 @@ public DbContext(string dbPath, CancellationToken cancellationToken = default) static connection => connection.Open(), dbPath: dbPath, cancellationToken: cancellationToken); - Execute("PRAGMA busy_timeout=5000"); + ApplyBusyTimeoutPragma(); ApplyConnectionPerformancePragmas(); RegisterConnectionFunctionsWithRetry(_connection, cancellationToken: cancellationToken); EnsureWritableUserVersionSupported(dbPath); @@ -367,7 +377,7 @@ public DbContext(string dbPath, CancellationToken cancellationToken = default) static connection => connection.Open(), dbPath: dbPath, cancellationToken: cancellationToken); - Execute("PRAGMA busy_timeout=5000"); + ApplyBusyTimeoutPragma(); ApplyConnectionPerformancePragmas(); RegisterConnectionFunctionsWithRetry(_connection, cancellationToken: cancellationToken); EnsureWritableUserVersionSupported(dbPath); @@ -425,7 +435,7 @@ private void OpenReadOnlyFallback(string dbPath, CancellationToken cancellationT { cancellationToken.ThrowIfCancellationRequested(); _connection = OpenReadOnly(dbPath); - Execute("PRAGMA busy_timeout=5000"); + ApplyBusyTimeoutPragma(); ApplyConnectionPerformancePragmas(); RegisterConnectionFunctionsWithRetry(_connection, cancellationToken: cancellationToken); _isReadOnly = true; @@ -645,6 +655,12 @@ private VacuumMetrics ReadVacuumMetrics() private long ReadAutoVacuumMode() => ReadPragmaLong("auto_vacuum"); + private void ApplyBusyTimeoutPragma() + { + var busyTimeoutMs = DbPragmaPolicy.ReadBusyTimeoutMs(BusyTimeoutEnvironmentVariable); + Execute($"PRAGMA busy_timeout={busyTimeoutMs}"); + } + private long? TryGetDatabaseFileSize() { var path = _connection.DataSource; @@ -2716,26 +2732,31 @@ internal void MarkWriteWork(bool walCheckpointable = true) } } - internal void RunPlannerStatisticsMaintenance(bool forceAnalyze) + internal sealed record PlannerStatisticsMaintenanceFailure(string CommandText, SqliteException Exception); + + internal PlannerStatisticsMaintenanceFailure? RunPlannerStatisticsMaintenance(bool forceAnalyze) { if (_isReadOnly) - return; + return null; using var cmd = _connection.CreateCommand(); cmd.CommandText = forceAnalyze ? "ANALYZE" : "PRAGMA optimize"; try { + PlannerStatisticsCommandCreatedForTesting?.Invoke(cmd); cmd.ExecuteNonQuery(); PlannerStatisticsCommandExecutedForTesting?.Invoke(_connection.DataSource, cmd.CommandText); if (!forceAnalyze) OptimizePragmaExecutedForTesting?.Invoke(_connection.DataSource); _hasWriteWork = false; + return null; } - catch (SqliteException) + catch (SqliteException ex) { // Planner statistics are an index-performance aid. If SQLite rejects ANALYZE / // optimize during cleanup (read-only handoff, transient filesystem state), keep // the completed index usable instead of converting success into failure. + return new PlannerStatisticsMaintenanceFailure(cmd.CommandText, ex); } } diff --git a/src/CodeIndex/Database/DbPragmaPolicy.cs b/src/CodeIndex/Database/DbPragmaPolicy.cs index 8825714fb7..e200cf3114 100644 --- a/src/CodeIndex/Database/DbPragmaPolicy.cs +++ b/src/CodeIndex/Database/DbPragmaPolicy.cs @@ -5,6 +5,9 @@ namespace CodeIndex.Database; internal static class DbPragmaPolicy { + internal const int DefaultBusyTimeoutMs = 5000; + internal const int MaxBusyTimeoutMs = 3_600_000; + internal static DbConnectionPragmaSettings ReadConnectionPragmaSettings( string cacheSizeEnvironmentVariable, int defaultCacheSizeKb, @@ -57,6 +60,12 @@ internal static bool IsSafetyLevelTransactionError(SqliteException ex) => ex.SqliteErrorCode == 1 && ex.Message.Contains("Safety level may not be changed inside a transaction", StringComparison.OrdinalIgnoreCase); + internal static int ReadBusyTimeoutMs(string environmentVariable) + => ReadNonNegativeIntEnvironment( + environmentVariable, + DefaultBusyTimeoutMs, + MaxBusyTimeoutMs); + private static int ReadPositiveIntEnvironment(string name, int fallback, int maximum) { var value = Environment.GetEnvironmentVariable(name); @@ -67,6 +76,16 @@ private static int ReadPositiveIntEnvironment(string name, int fallback, int max : fallback; } + private static int ReadNonNegativeIntEnvironment(string name, int fallback, int maximum) + { + var value = Environment.GetEnvironmentVariable(name); + return int.TryParse(value, NumberStyles.None, CultureInfo.InvariantCulture, out var parsed) + && parsed >= 0 + && parsed <= maximum + ? parsed + : fallback; + } + private static long ReadNonNegativeLongEnvironment(string name, long fallback, long maximum) { var value = Environment.GetEnvironmentVariable(name); diff --git a/src/CodeIndex/Database/DbReader.FilesStatus.cs b/src/CodeIndex/Database/DbReader.FilesStatus.cs index 3ab7026165..f0bfd54161 100644 --- a/src/CodeIndex/Database/DbReader.FilesStatus.cs +++ b/src/CodeIndex/Database/DbReader.FilesStatus.cs @@ -818,6 +818,7 @@ GROUP BY COALESCE(f.lang, 'unknown'), s.kind // #1546: case-sensitivity stamp も同 snapshot で読む。stamp 無し旧 DB は null。 var pathCaseSensitive = ParseMetaBool(TryGetMetaStringInternal(DbContext.WorkspacePathCaseSensitiveMetaKey)); var dbPragmaSettings = GetDbPragmaSettings(); + var preparedCommandCache = GetPreparedCommandCacheStatus(); var dbSizeBytes = TryGetDatabaseFileSize(); var walSizeBytes = TryGetWalFileSize(); var maintenanceGuidance = MaintenanceGuidanceBuilder.Build(new MaintenanceMetrics( @@ -874,6 +875,7 @@ GROUP BY COALESCE(f.lang, 'unknown'), s.kind IndexNewerThanReaderReason = _indexNewerThanReaderReason, PathCaseSensitive = pathCaseSensitive, DbPragmaSettings = dbPragmaSettings, + PreparedCommandCache = preparedCommandCache, MaintenanceGuidance = maintenanceGuidance, DbSizeBytes = dbSizeBytes, WalSizeBytes = walSizeBytes, @@ -890,6 +892,22 @@ GROUP BY COALESCE(f.lang, 'unknown'), s.kind return result; } + private StatusPreparedCommandCache? GetPreparedCommandCacheStatus() + { + if (_commandCache == null) + return null; + + var diagnostics = _commandCache.GetDiagnostics(); + return new StatusPreparedCommandCache + { + Count = diagnostics.Count, + Capacity = diagnostics.Capacity, + HitCount = diagnostics.HitCount, + MissCount = diagnostics.MissCount, + EvictionCount = diagnostics.EvictionCount, + }; + } + /// /// Delegate to RepoMapBuilder for repo-level overview generation. /// RepoMapBuilderに委譲してリポジトリ俯瞰情報を生成する。 @@ -912,6 +930,7 @@ private long ExecuteScalar(string sql) JournalMode = ExecuteScalarString("PRAGMA journal_mode"), Synchronous = NormalizeSynchronousMode(ExecuteScalarString("PRAGMA synchronous")), WalAutocheckpoint = ExecuteNullableLong("PRAGMA wal_autocheckpoint"), + BusyTimeoutMs = ExecuteNullableLong("PRAGMA busy_timeout"), PageCount = ExecuteNullableLong("PRAGMA page_count"), FreelistCount = ExecuteNullableLong("PRAGMA freelist_count"), PageSize = ExecuteNullableLong("PRAGMA page_size"), diff --git a/src/CodeIndex/Database/DbReader.cs b/src/CodeIndex/Database/DbReader.cs index 55e81707e7..b661465172 100644 --- a/src/CodeIndex/Database/DbReader.cs +++ b/src/CodeIndex/Database/DbReader.cs @@ -1643,6 +1643,8 @@ private HashSet LoadColumns(string tableName) internal static void AppendPathFilters(ref string sql, IReadOnlyList? pathPatterns, IReadOnlyList? excludePathPatterns, bool excludeTests) { + EnsurePathFilterParameterBudget(pathPatterns, excludePathPatterns); + if (!IncludeGeneratedScope.Value && GeneratedColumnAvailableScope.Value) sql += " AND COALESCE(f.generated, 0) = 0"; @@ -1653,14 +1655,14 @@ internal static void AppendPathFilters(ref string sql, IReadOnlyList? pa // (`*` / `?`) are translated to SQL LIKE wildcards. var ors = new List(pathPatterns.Count); for (int i = 0; i < pathPatterns.Count; i++) - ors.Add($"f.path LIKE @pathPattern{i} ESCAPE '\\'"); + ors.Add($"f.path LIKE {SqliteDynamicSql.BuildParameterName("pathPattern", i)} ESCAPE '\\'"); sql += " AND (" + string.Join(" OR ", ors) + ")"; } if (excludePathPatterns != null) { for (int i = 0; i < excludePathPatterns.Count; i++) - sql += $" AND f.path NOT LIKE @excludePathPattern{i} ESCAPE '\\'"; + sql += $" AND f.path NOT LIKE {SqliteDynamicSql.BuildParameterName("excludePathPattern", i)} ESCAPE '\\'"; } if (excludeTests) @@ -1669,17 +1671,13 @@ internal static void AppendPathFilters(ref string sql, IReadOnlyList? pa internal static void AddPathFilterParameters(SqliteCommand cmd, IReadOnlyList? pathPatterns, IReadOnlyList? excludePathPatterns) { + EnsurePathFilterParameterBudget(pathPatterns, excludePathPatterns); + if (pathPatterns != null) - { - for (int i = 0; i < pathPatterns.Count; i++) - cmd.Parameters.AddWithValue($"@pathPattern{i}", BuildPathLikePattern(pathPatterns[i])); - } + SqliteDynamicSql.AddParameters(cmd, "pathPattern", pathPatterns, SqliteType.Text, "path filters", static pattern => BuildPathLikePattern(pattern)); if (excludePathPatterns != null) - { - for (int i = 0; i < excludePathPatterns.Count; i++) - cmd.Parameters.AddWithValue($"@excludePathPattern{i}", BuildPathLikePattern(excludePathPatterns[i])); - } + SqliteDynamicSql.AddParameters(cmd, "excludePathPattern", excludePathPatterns, SqliteType.Text, "path filters", static pattern => BuildPathLikePattern(pattern)); } private static void AppendAdditionalPathIncludeFilters(ref string sql, IReadOnlyList? pathPatterns, string parameterPrefix) @@ -1687,9 +1685,10 @@ private static void AppendAdditionalPathIncludeFilters(ref string sql, IReadOnly if (pathPatterns == null || pathPatterns.Count == 0) return; + SqliteDynamicSql.EnsureParameterBudget(pathPatterns.Count, "additional path include filters"); var ors = new List(pathPatterns.Count); for (int i = 0; i < pathPatterns.Count; i++) - ors.Add($"f.path LIKE @{parameterPrefix}{i} ESCAPE '\\'"); + ors.Add($"f.path LIKE {SqliteDynamicSql.BuildParameterName(parameterPrefix, i)} ESCAPE '\\'"); sql += " AND (" + string.Join(" OR ", ors) + ")"; } @@ -1698,12 +1697,13 @@ private static void AddPathIncludeFilterParameters(SqliteCommand cmd, IReadOnlyL if (pathPatterns == null) return; - for (int i = 0; i < pathPatterns.Count; i++) - cmd.Parameters.AddWithValue($"@{parameterPrefix}{i}", BuildPathLikePattern(pathPatterns[i])); + SqliteDynamicSql.AddParameters(cmd, parameterPrefix, pathPatterns, SqliteType.Text, "additional path include filters", static pattern => BuildPathLikePattern(pattern)); } internal static string BuildPathFiltersSql(string fileAlias, IReadOnlyList? pathPatterns, IReadOnlyList? excludePathPatterns, bool excludeTests) { + EnsurePathFilterParameterBudget(pathPatterns, excludePathPatterns); + var sql = string.Empty; if (!IncludeGeneratedScope.Value && GeneratedColumnAvailableScope.Value) sql += $" AND COALESCE({fileAlias}.generated, 0) = 0"; @@ -1715,14 +1715,14 @@ internal static string BuildPathFiltersSql(string fileAlias, IReadOnlyList(pathPatterns.Count); for (int i = 0; i < pathPatterns.Count; i++) - ors.Add($"{fileAlias}.path LIKE @pathPattern{i} ESCAPE '\\'"); + ors.Add($"{fileAlias}.path LIKE {SqliteDynamicSql.BuildParameterName("pathPattern", i)} ESCAPE '\\'"); sql += " AND (" + string.Join(" OR ", ors) + ")"; } if (excludePathPatterns != null) { for (int i = 0; i < excludePathPatterns.Count; i++) - sql += $" AND {fileAlias}.path NOT LIKE @excludePathPattern{i} ESCAPE '\\'"; + sql += $" AND {fileAlias}.path NOT LIKE {SqliteDynamicSql.BuildParameterName("excludePathPattern", i)} ESCAPE '\\'"; } if (excludeTests) @@ -1731,6 +1731,9 @@ internal static string BuildPathFiltersSql(string fileAlias, IReadOnlyList? pathPatterns, IReadOnlyCollection? excludePathPatterns) + => SqliteDynamicSql.EnsureParameterBudget((pathPatterns?.Count ?? 0) + (excludePathPatterns?.Count ?? 0), "path filters"); + internal static DateTime? GetNullableDateTime(SqliteDataReader reader, int ordinal) { if (reader.IsDBNull(ordinal)) diff --git a/src/CodeIndex/Database/DbSymbolReader.cs b/src/CodeIndex/Database/DbSymbolReader.cs index d3852b09f9..029ace7150 100644 --- a/src/CodeIndex/Database/DbSymbolReader.cs +++ b/src/CodeIndex/Database/DbSymbolReader.cs @@ -167,29 +167,32 @@ public NormalizedSymbolSearchQueryList(IEnumerable queries) private void AppendVisibilityFilters(ref string sql, IReadOnlyList? visibilityFilters, IReadOnlyList? excludeVisibilityFilters) { - if (visibilityFilters is { Count: > 0 }) - sql += $" AND lower({GetSymbolColumnSql("visibility", "''")}) IN ({string.Join(",", ExpandVisibilityFilterValues(visibilityFilters).Select((_, i) => $"@visibility{i}"))})"; - if (excludeVisibilityFilters is { Count: > 0 }) - sql += $" AND lower({GetSymbolColumnSql("visibility", "''")}) NOT IN ({string.Join(",", ExpandVisibilityFilterValues(excludeVisibilityFilters).Select((_, i) => $"@excludeVisibility{i}"))})"; + var expandedVisibilityFilters = visibilityFilters is { Count: > 0 } ? ExpandVisibilityFilterValues(visibilityFilters) : null; + var expandedExcludeVisibilityFilters = excludeVisibilityFilters is { Count: > 0 } ? ExpandVisibilityFilterValues(excludeVisibilityFilters) : null; + EnsureVisibilityFilterParameterBudget(expandedVisibilityFilters, expandedExcludeVisibilityFilters); + + if (expandedVisibilityFilters is { Count: > 0 }) + sql += $" AND lower({GetSymbolColumnSql("visibility", "''")}) IN ({SqliteDynamicSql.BuildParameterList("visibility", expandedVisibilityFilters.Count)})"; + if (expandedExcludeVisibilityFilters is { Count: > 0 }) + sql += $" AND lower({GetSymbolColumnSql("visibility", "''")}) NOT IN ({SqliteDynamicSql.BuildParameterList("excludeVisibility", expandedExcludeVisibilityFilters.Count)})"; } private static void AddVisibilityFilterParameters(SqliteCommand cmd, IReadOnlyList? visibilityFilters, IReadOnlyList? excludeVisibilityFilters) { - if (visibilityFilters is { Count: > 0 }) - { - var expanded = ExpandVisibilityFilterValues(visibilityFilters); - for (int i = 0; i < expanded.Count; i++) - cmd.Parameters.AddWithValue($"@visibility{i}", expanded[i]); - } + var expandedVisibilityFilters = visibilityFilters is { Count: > 0 } ? ExpandVisibilityFilterValues(visibilityFilters) : null; + var expandedExcludeVisibilityFilters = excludeVisibilityFilters is { Count: > 0 } ? ExpandVisibilityFilterValues(excludeVisibilityFilters) : null; + EnsureVisibilityFilterParameterBudget(expandedVisibilityFilters, expandedExcludeVisibilityFilters); - if (excludeVisibilityFilters is { Count: > 0 }) - { - var expanded = ExpandVisibilityFilterValues(excludeVisibilityFilters); - for (int i = 0; i < expanded.Count; i++) - cmd.Parameters.AddWithValue($"@excludeVisibility{i}", expanded[i]); - } + if (expandedVisibilityFilters is { Count: > 0 }) + SqliteDynamicSql.AddParameters(cmd, "visibility", expandedVisibilityFilters, SqliteType.Text, "visibility filters"); + + if (expandedExcludeVisibilityFilters is { Count: > 0 }) + SqliteDynamicSql.AddParameters(cmd, "excludeVisibility", expandedExcludeVisibilityFilters, SqliteType.Text, "visibility filters"); } + private static void EnsureVisibilityFilterParameterBudget(IReadOnlyCollection? visibilityFilters, IReadOnlyCollection? excludeVisibilityFilters) + => SqliteDynamicSql.EnsureParameterBudget((visibilityFilters?.Count ?? 0) + (excludeVisibilityFilters?.Count ?? 0), "visibility filters"); + private static bool HasVisibilityFilters(IReadOnlyList? visibilityFilters, IReadOnlyList? excludeVisibilityFilters) => visibilityFilters is { Count: > 0 } || excludeVisibilityFilters is { Count: > 0 }; diff --git a/src/CodeIndex/Database/DbWriter.cs b/src/CodeIndex/Database/DbWriter.cs index 7ef1af2af5..05b053490a 100644 --- a/src/CodeIndex/Database/DbWriter.cs +++ b/src/CodeIndex/Database/DbWriter.cs @@ -49,7 +49,6 @@ internal static Action? BatchRowSkipWarningForTesting private static readonly TimeSpan TransactionStateContentionWaitInterval = TimeSpan.FromMilliseconds(50); private const int BatchSize = 500; private const int DeleteFilesBatchSize = 500; - private const int MaxSqlVariables = 999; private const int SqliteConstraintErrorCode = 19; private const int TypeScriptModuleSyntaxFallbackMaxBytes = (int)FileIndexer.DefaultMaxFileSizeBytes; private const int TypeScriptModuleSyntaxFallbackMaxLines = 16384; @@ -889,14 +888,7 @@ private void DeleteFileIdBatch(IReadOnlyList fileIds) DeleteCrossFileReferencesToSymbolsDefinedOnlyByFiles(fileIds); using var deleteCmd = _conn.CreateCommand(); - var parameters = new List(fileIds.Count); - for (var i = 0; i < fileIds.Count; i++) - { - var parameterName = $"@id{i}"; - parameters.Add(parameterName); - deleteCmd.Parameters.Add(parameterName, SqliteType.Integer).Value = fileIds[i]; - } - + var parameters = SqliteDynamicSql.AddParameters(deleteCmd, "id", fileIds, SqliteType.Integer, "file id delete batch"); deleteCmd.CommandText = $"DELETE FROM files WHERE id IN ({string.Join(", ", parameters)})"; deleteCmd.ExecuteNonQuery(); } @@ -904,14 +896,7 @@ private void DeleteFileIdBatch(IReadOnlyList fileIds) private void DeleteCrossFileReferencesToSymbolsDefinedOnlyByFiles(IReadOnlyList fileIds) { using var deleteCmd = _conn.CreateCommand(); - var parameters = new List(fileIds.Count); - for (var i = 0; i < fileIds.Count; i++) - { - var parameterName = $"@id{i}"; - parameters.Add(parameterName); - deleteCmd.Parameters.Add(parameterName, SqliteType.Integer).Value = fileIds[i]; - } - + var parameters = SqliteDynamicSql.AddParameters(deleteCmd, "id", fileIds, SqliteType.Integer, "cross-file reference delete batch"); var idList = string.Join(", ", parameters); deleteCmd.CommandText = $@" DELETE FROM symbol_references @@ -2179,22 +2164,7 @@ public bool MarkFoldReady(bool stampCurrentSymbolExtractorVersions = false) return false; } - // Inline the SetReadyBit body. SetReadyBit opens its own BEGIN IMMEDIATE - // when not already in a DbWriter-tracked transaction, but our raw - // BEGIN IMMEDIATE above is not tracked in _transactionDepth, so a direct - // SetReadyBit call would attempt a nested BEGIN IMMEDIATE and fail. - // SetReadyBit は _transactionDepth ベースでしか外側 transaction を見ないため、 - // 生 BEGIN IMMEDIATE 内では呼べない。内容を inline 展開する。 - int current; - using (var read = _conn.CreateCommand()) - { - read.CommandText = "PRAGMA user_version"; - var raw = read.ExecuteScalar(); - current = raw is long l ? (int)l : (raw is int i ? i : 0); - } - int next = current | DbContext.FoldReadyFlag; - if (next != current) - Execute($"PRAGMA user_version = {next}"); + ApplyReadyBitToUserVersion(DbContext.FoldReadyFlag, ownTransaction ? null : _activeTransaction); SetMeta("fold_key_version", NameFold.Version.ToString(System.Globalization.CultureInfo.InvariantCulture)); SetMeta("fold_key_fingerprint", NameFold.Fingerprint()); @@ -3423,8 +3393,8 @@ private void SetMetaCore(string key, string? value) cmd.Transaction = _activeTransaction; cmd.CommandText = @"INSERT INTO codeindex_meta (key, value) VALUES (@key, @value) ON CONFLICT(key) DO UPDATE SET value = excluded.value"; - cmd.Parameters.AddWithValue("@key", key); - cmd.Parameters.AddWithValue("@value", (object?)value ?? DBNull.Value); + cmd.Parameters.Add("@key", SqliteType.Text).Value = key; + cmd.Parameters.Add("@value", SqliteType.Text).Value = (object?)value ?? DBNull.Value; cmd.ExecuteNonQuery(); } @@ -3828,7 +3798,7 @@ private static int GetRowsPerInsertStatement(int columnCount) if (columnCount <= 0) throw new ArgumentOutOfRangeException(nameof(columnCount)); - return Math.Max(1, Math.Min(BatchSize, MaxSqlVariables / columnCount)); + return Math.Max(1, Math.Min(BatchSize, SqliteDynamicSql.MaxSqlVariables / columnCount)); } private void SetReadyBit(int flag) @@ -3855,17 +3825,7 @@ private void SetReadyBit(int flag) var transaction = ownTransaction ? null : _activeTransaction; try { - int current; - using (var read = _conn.CreateCommand()) - { - read.Transaction = transaction; - read.CommandText = "PRAGMA user_version"; - var raw = read.ExecuteScalar(); - current = raw is long l ? (int)l : (raw is int i ? i : 0); - } - int next = current | flag; - if (next != current) - Execute($"PRAGMA user_version = {next}", transaction); + ApplyReadyBitToUserVersion(flag, transaction); if (ownTransaction) { Execute("COMMIT"); @@ -3887,17 +3847,26 @@ private void SetReadyBit(int flag) } } - private static List BuildSupportedLanguageParameters(SqliteCommand cmd, IReadOnlyCollection supportedLanguages) + private void ApplyReadyBitToUserVersion(int flag, SqliteTransaction? transaction) { - var inParams = new List(supportedLanguages.Count); - for (int i = 0; i < supportedLanguages.Count; i++) + int current; + using (var read = _conn.CreateCommand()) { - var paramName = $"@lang{i}"; - inParams.Add(paramName); - cmd.Parameters.AddWithValue(paramName, supportedLanguages.ElementAt(i)); + read.Transaction = transaction; + read.CommandText = "PRAGMA user_version"; + var raw = read.ExecuteScalar(); + current = raw is long l ? (int)l : (raw is int i ? i : 0); } - return inParams; + int next = current | flag; + if (next != current) + Execute($"PRAGMA user_version = {next}", transaction); + } + + private static List BuildSupportedLanguageParameters(SqliteCommand cmd, IReadOnlyCollection supportedLanguages) + { + var values = supportedLanguages as IReadOnlyList ?? supportedLanguages.ToList(); + return SqliteDynamicSql.AddParameters(cmd, "lang", values, SqliteType.Text, "supported language filters"); } private bool IsInTransaction() => _transactionDepth > 0; diff --git a/src/CodeIndex/Database/PreparedCommandCache.cs b/src/CodeIndex/Database/PreparedCommandCache.cs index 491d68c050..bf24186598 100644 --- a/src/CodeIndex/Database/PreparedCommandCache.cs +++ b/src/CodeIndex/Database/PreparedCommandCache.cs @@ -1,4 +1,5 @@ using Microsoft.Data.Sqlite; +using System.Globalization; namespace CodeIndex.Database; @@ -20,11 +21,16 @@ namespace CodeIndex.Database; internal sealed class PreparedCommandCache : IDisposable { internal const int DefaultCapacity = 32; + internal const int MaxCapacity = 512; + internal const string CapacityEnvironmentVariable = "CDIDX_PREPARED_COMMAND_CACHE_CAPACITY"; private readonly SqliteConnection _connection; private readonly int _capacity; private readonly LinkedList _lru = new(); private readonly Dictionary> _map = new(StringComparer.Ordinal); + private long _hitCount; + private long _missCount; + private long _evictionCount; private bool _disposed; public PreparedCommandCache(SqliteConnection connection, int capacity = DefaultCapacity) @@ -38,6 +44,22 @@ public PreparedCommandCache(SqliteConnection connection, int capacity = DefaultC public int Count => _lru.Count; public int Capacity => _capacity; + public long HitCount => _hitCount; + public long MissCount => _missCount; + public long EvictionCount => _evictionCount; + + internal static int ReadCapacityFromEnvironment() + { + var raw = Environment.GetEnvironmentVariable(CapacityEnvironmentVariable); + return int.TryParse(raw, NumberStyles.None, CultureInfo.InvariantCulture, out var parsed) + && parsed > 0 + && parsed <= MaxCapacity + ? parsed + : DefaultCapacity; + } + + internal PreparedCommandCacheDiagnostics GetDiagnostics() + => new(Count, Capacity, HitCount, MissCount, EvictionCount); /// /// Return a prepared for . On a miss @@ -61,9 +83,11 @@ public SqliteCommand GetOrAdd(string sql, Action configureSchema) // LRU touch: move to front so least-recently-used falls out of the tail. _lru.Remove(existing); _lru.AddFirst(existing); + _hitCount++; return existing.Value.Command; } + _missCount++; var cmd = _connection.CreateCommand(); try { @@ -86,6 +110,7 @@ public SqliteCommand GetOrAdd(string sql, Action configureSchema) var tail = _lru.Last!; _lru.RemoveLast(); _map.Remove(tail.Value.Sql); + _evictionCount++; tail.Value.Command.Dispose(); } @@ -113,4 +138,11 @@ public Entry(string sql, SqliteCommand command) public string Sql { get; } public SqliteCommand Command { get; } } + + internal sealed record PreparedCommandCacheDiagnostics( + int Count, + int Capacity, + long HitCount, + long MissCount, + long EvictionCount); } diff --git a/src/CodeIndex/Database/SqliteDynamicSql.cs b/src/CodeIndex/Database/SqliteDynamicSql.cs new file mode 100644 index 0000000000..c66414202c --- /dev/null +++ b/src/CodeIndex/Database/SqliteDynamicSql.cs @@ -0,0 +1,64 @@ +using Microsoft.Data.Sqlite; + +namespace CodeIndex.Database; + +internal static class SqliteDynamicSql +{ + internal const int MaxSqlVariables = 999; + + internal static void EnsureParameterBudget(int parameterCount, string context) + { + if (parameterCount < 0) + throw new ArgumentOutOfRangeException(nameof(parameterCount), parameterCount, "Parameter count cannot be negative."); + if (parameterCount > MaxSqlVariables) + throw new ArgumentOutOfRangeException( + nameof(parameterCount), + parameterCount, + $"{context} uses {parameterCount} SQLite parameters, exceeding the supported budget of {MaxSqlVariables}."); + } + + internal static string BuildParameterName(string prefix, int index) + { + if (index < 0) + throw new ArgumentOutOfRangeException(nameof(index), index, "Parameter index cannot be negative."); + + return $"@{prefix}{index}"; + } + + internal static List BuildParameterNames(string prefix, int count) + { + EnsureParameterBudget(count, prefix); + var names = new List(count); + for (var i = 0; i < count; i++) + names.Add(BuildParameterName(prefix, i)); + + return names; + } + + internal static string BuildParameterList(string prefix, int count) + => string.Join(", ", BuildParameterNames(prefix, count)); + + internal static List AddParameters( + SqliteCommand cmd, + string prefix, + IReadOnlyList values, + SqliteType sqliteType, + string? context = null, + Func? bindValue = null) + { + ArgumentNullException.ThrowIfNull(cmd); + ArgumentNullException.ThrowIfNull(values); + EnsureParameterBudget(values.Count, context ?? prefix); + + var names = new List(values.Count); + for (var i = 0; i < values.Count; i++) + { + var parameterName = BuildParameterName(prefix, i); + names.Add(parameterName); + var value = bindValue == null ? values[i] : bindValue(values[i]); + cmd.Parameters.Add(parameterName, sqliteType).Value = value ?? DBNull.Value; + } + + return names; + } +} diff --git a/src/CodeIndex/Mcp/McpToolHandlers.cs b/src/CodeIndex/Mcp/McpToolHandlers.cs index f1390d2957..33aba34117 100644 --- a/src/CodeIndex/Mcp/McpToolHandlers.cs +++ b/src/CodeIndex/Mcp/McpToolHandlers.cs @@ -6312,6 +6312,12 @@ static string FormatDiagnosticPath(string projectRoot, string path) writer.ClearBatchInProgress(); readinessTxn.Commit(); } + if (!scanResult.HadErrors && errors == 0) + { + var plannerMaintenanceFailure = db.RunPlannerStatisticsMaintenance(forceAnalyze: false); + if (plannerMaintenanceFailure != null) + IndexCommandRunner.TryStampPlannerStatisticsMaintenanceDiagnostic(writer, indexRunDiagnostics, plannerMaintenanceFailure); + } var (totalFiles, totalChunks, totalSymbols, totalReferences) = writer.GetCounts(); await EmitProgressNotificationAsync(progressToken, files.Count, files.Count, errors == 0 ? "Indexing complete." : "Indexing completed with errors.").ConfigureAwait(false); if (memorySamples != null) diff --git a/src/CodeIndex/Models/QueryResults.cs b/src/CodeIndex/Models/QueryResults.cs index 339cd13133..4ad79072b9 100644 --- a/src/CodeIndex/Models/QueryResults.cs +++ b/src/CodeIndex/Models/QueryResults.cs @@ -1108,6 +1108,9 @@ public class StatusResult /// [JsonPropertyName("db_pragma_settings")] public StatusDbPragmaSettings DbPragmaSettings { get; set; } = new(); + [JsonPropertyName("prepared_command_cache")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public StatusPreparedCommandCache? PreparedCommandCache { get; set; } [JsonPropertyName("maintenance_guidance")] public StatusMaintenanceGuidance MaintenanceGuidance { get; set; } = new(); [JsonPropertyName("db_size_bytes")] @@ -1156,6 +1159,18 @@ public static StatusProcessMetrics Capture() } } +public sealed class StatusPreparedCommandCache +{ + public int Count { get; set; } + public int Capacity { get; set; } + [JsonPropertyName("hit_count")] + public long HitCount { get; set; } + [JsonPropertyName("miss_count")] + public long MissCount { get; set; } + [JsonPropertyName("eviction_count")] + public long EvictionCount { get; set; } +} + public sealed class StatusLastIndexRun { public string? Mode { get; set; } @@ -1246,6 +1261,8 @@ public class StatusDbPragmaSettings public string? JournalMode { get; set; } public string? Synchronous { get; set; } public long? WalAutocheckpoint { get; set; } + [JsonPropertyName("busy_timeout_ms")] + public long? BusyTimeoutMs { get; set; } public long? PageCount { get; set; } public long? FreelistCount { get; set; } public long? PageSize { get; set; } diff --git a/tests/CodeIndex.Tests/DbDebugTests.cs b/tests/CodeIndex.Tests/DbDebugTests.cs index f15c42d258..df4a89dc87 100644 --- a/tests/CodeIndex.Tests/DbDebugTests.cs +++ b/tests/CodeIndex.Tests/DbDebugTests.cs @@ -72,7 +72,7 @@ public void ExecuteTrackedReader_SlowQueryThresholdIgnoresCurrentCulturePositive } [Fact] - public void ExecuteTrackedReader_ProfileCapsQueryPlanRows() + public void ExecuteTrackedReader_ProfileCapsQueryPlanRows_Issue3739() { DbDebug.ResetForTesting(); try @@ -89,8 +89,10 @@ public void ExecuteTrackedReader_ProfileCapsQueryPlanRows() } var entry = Assert.Single(DbDebug.EndProfile()); - Assert.True(entry.QueryPlan.Count <= DbDebug.MaxQueryPlanRows + 1); - Assert.Contains(entry.QueryPlan, row => row.Detail.Contains("truncated after", StringComparison.Ordinal)); + Assert.Equal(DbDebug.MaxQueryPlanRows + 1, entry.QueryPlan.Count); + Assert.Equal( + $"EXPLAIN QUERY PLAN rows truncated after {DbDebug.MaxQueryPlanRows} rows.", + entry.QueryPlan[^1].Detail); } finally { @@ -99,7 +101,7 @@ public void ExecuteTrackedReader_ProfileCapsQueryPlanRows() } [Fact] - public void ExecuteTrackedReader_ProfileTruncatesLongQueryPlanDetails() + public void ExecuteTrackedReader_ProfileTruncatesLongQueryPlanDetails_Issue3739() { DbDebug.ResetForTesting(); try diff --git a/tests/CodeIndex.Tests/DiffCommandRunnerTests.cs b/tests/CodeIndex.Tests/DiffCommandRunnerTests.cs index 1960de353f..17b76b6d72 100644 --- a/tests/CodeIndex.Tests/DiffCommandRunnerTests.cs +++ b/tests/CodeIndex.Tests/DiffCommandRunnerTests.cs @@ -1,6 +1,7 @@ using System.Security.Cryptography; using System.Text; using System.Text.Json; +using System.Reflection; using CodeIndex.Cli; using CodeIndex.Database; using Microsoft.Data.Sqlite; @@ -181,6 +182,54 @@ public void Run_DetailedJsonReportsLimitedSymbolRows_Issue2885() } } + [Fact] + public void Run_StopsWhenDiffRowBudgetIsExceeded_Issue3834() + { + var leftRoot = TestProjectHelper.CreateTempProject("cdidx_diff_row_budget_left"); + var rightRoot = TestProjectHelper.CreateTempProject("cdidx_diff_row_budget_right"); + var originalRowBudget = DiffCommandRunner.MaxDiffComparedRowsPerSideForTesting; + try + { + var leftDb = TestProjectHelper.CreateProjectDb(leftRoot); + var rightDb = TestProjectHelper.CreateProjectDb(rightRoot); + TestProjectHelper.InsertIndexedFile(leftDb, "src/A.cs", "csharp", "public class A { }"); + TestProjectHelper.InsertIndexedFile(leftDb, "src/B.cs", "csharp", "public class B { }"); + TestProjectHelper.InsertIndexedFile(rightDb, "src/A.cs", "csharp", "public class A { }"); + TestProjectHelper.InsertIndexedFile(rightDb, "src/B.cs", "csharp", "public class B { }"); + DiffCommandRunner.MaxDiffComparedRowsPerSideForTesting = 1; + + var (exitCode, stdout, stderr) = RunWithCapturedStreams([leftDb, rightDb]); + + Assert.Equal(3, exitCode); + Assert.Equal(string.Empty, stdout); + Assert.Contains("diff left row comparison exceeded the safety budget of 1 rows", stderr); + } + finally + { + DiffCommandRunner.MaxDiffComparedRowsPerSideForTesting = originalRowBudget; + TestProjectHelper.DeleteDirectory(leftRoot); + TestProjectHelper.DeleteDirectory(rightRoot); + } + } + + [Fact] + public void ColumnExists_QuotesTableIdentifiersForPragmaInfo_Issue3834() + { + using var connection = new SqliteConnection("Data Source=:memory:"); + connection.Open(); + using (var command = connection.CreateCommand()) + { + command.CommandText = "CREATE TABLE \"odd \"\" table\" (\"odd col\" INTEGER)"; + command.ExecuteNonQuery(); + } + var method = typeof(DiffCommandRunner).GetMethod("ColumnExists", BindingFlags.NonPublic | BindingFlags.Static) + ?? throw new MissingMethodException(nameof(DiffCommandRunner), "ColumnExists"); + + var exists = Assert.IsType(method.Invoke(null, new object?[] { connection, "odd \" table", "odd col" })); + + Assert.True(exists); + } + [Fact] public void Run_DetailedJsonTruncatesLargeEncodedSymbolFields_Issue3163() { diff --git a/tests/CodeIndex.Tests/IndexCommandRunnerTests.cs b/tests/CodeIndex.Tests/IndexCommandRunnerTests.cs index 620df80021..3b66db6459 100644 --- a/tests/CodeIndex.Tests/IndexCommandRunnerTests.cs +++ b/tests/CodeIndex.Tests/IndexCommandRunnerTests.cs @@ -1502,6 +1502,80 @@ public void Run_NewIndexDatabase_RunsAnalyzeAfterSuccessfulIndex() } } + [Fact] + public void Run_PlannerStatisticsMaintenanceFailure_AddsLastIndexRunDiagnostic_Issue3718() + { + var projectRoot = CreateTempProject(); + var dbPath = Path.Combine(projectRoot, ".cdidx", "codeindex.db"); + try + { + File.WriteAllText(Path.Combine(projectRoot, "app.py"), "print('hello')\n"); + DbContext.PlannerStatisticsCommandCreatedForTesting = command => + { + command.CommandText = "SELECT * FROM cdidx_missing_planner_statistics_table"; + }; + + var (exitCode, json) = RunAndCaptureJson([projectRoot, "--json", "--quiet"]); + + Assert.Equal(CommandExitCodes.Success, exitCode); + Assert.Equal("success", json.GetProperty("status").GetString()); + using var db = new DbContext(dbPath); + var status = new DbReader(db.Connection).GetStatus(); + Assert.NotNull(status.LastIndexRun); + var lastIndexRun = status.LastIndexRun!; + var diagnostic = Assert.Single(lastIndexRun.Diagnostics ?? []); + Assert.Contains("planner_statistics_maintenance_failed", diagnostic, StringComparison.Ordinal); + Assert.Contains("SELECT * FROM cdidx_missing_planner_statistics_table", diagnostic, StringComparison.Ordinal); + Assert.Contains(nameof(SqliteException), diagnostic, StringComparison.Ordinal); + Assert.Equal(1, lastIndexRun.DiagnosticCount); + Assert.False(lastIndexRun.DiagnosticsTruncated); + } + finally + { + DbContext.PlannerStatisticsCommandCreatedForTesting = null; + SqliteConnection.ClearAllPools(); + DeleteDirectory(projectRoot); + } + } + + [Fact] + public void Run_PlannerStatisticsMaintenanceDiagnosticStampFailure_DoesNotFailSuccessfulIndex_Issue3718() + { + var projectRoot = CreateTempProject(); + try + { + File.WriteAllText(Path.Combine(projectRoot, "app.py"), "print('hello')\n"); + DbContext.PlannerStatisticsCommandCreatedForTesting = command => + { + command.CommandText = "SELECT * FROM cdidx_missing_planner_statistics_table"; + }; + var stampCalls = 0; + string[] capturedDiagnostics = []; + IndexCommandRunner.PlannerStatisticsMaintenanceDiagnosticStampingForTesting = (_, diagnostics) => + { + stampCalls++; + capturedDiagnostics = diagnostics.ToArray(); + throw new IOException("metadata store became unavailable"); + }; + + var (exitCode, json) = RunAndCaptureJson([projectRoot, "--json", "--quiet"]); + + Assert.Equal(CommandExitCodes.Success, exitCode); + Assert.Equal("success", json.GetProperty("status").GetString()); + Assert.Equal(1, stampCalls); + var diagnostic = Assert.Single(capturedDiagnostics); + Assert.Contains("planner_statistics_maintenance_failed", diagnostic, StringComparison.Ordinal); + Assert.Contains("SELECT * FROM cdidx_missing_planner_statistics_table", diagnostic, StringComparison.Ordinal); + } + finally + { + DbContext.PlannerStatisticsCommandCreatedForTesting = null; + IndexCommandRunner.PlannerStatisticsMaintenanceDiagnosticStampingForTesting = null; + SqliteConnection.ClearAllPools(); + DeleteDirectory(projectRoot); + } + } + [Fact] public void Run_CancelDuringFreshIndex_ReturnsInterruptedJson() { diff --git a/tests/CodeIndex.Tests/LegacySchemaMigrationTests.cs b/tests/CodeIndex.Tests/LegacySchemaMigrationTests.cs index c0d1dfc979..564c623b63 100644 --- a/tests/CodeIndex.Tests/LegacySchemaMigrationTests.cs +++ b/tests/CodeIndex.Tests/LegacySchemaMigrationTests.cs @@ -1185,6 +1185,41 @@ public void CompletedIndex_MarkIndexCompleteStampsUserVersion_AndTablesAreTruste } } + [Fact] + public void ReadyBitStamps_ShareUserVersionHelperAcrossRawAndTrackedTransactions_Issue3716() + { + var dir = Path.Combine(Path.GetTempPath(), $"codeindex_ready_bits_{Guid.NewGuid():N}"); + Directory.CreateDirectory(dir); + var dbPath = Path.Combine(dir, "codeindex.db"); + try + { + using var db = new DbContext(dbPath); + db.InitializeSchema(); + var writer = new DbWriter(db.Connection); + + Assert.True(writer.MarkFoldReady()); + var afterFold = db.GetUserVersion(); + Assert.NotEqual(0, afterFold & DbContext.FoldReadyFlag); + Assert.Equal(0, afterFold & DbContext.GraphReadyFlag); + + using (var transaction = writer.BeginTransaction()) + { + writer.MarkGraphReady(); + transaction.Commit(); + } + + var afterGraph = db.GetUserVersion(); + Assert.NotEqual(0, afterGraph & DbContext.FoldReadyFlag); + Assert.NotEqual(0, afterGraph & DbContext.GraphReadyFlag); + Assert.Equal(0, afterGraph & DbContext.IssuesReadyFlag); + } + finally + { + SqliteConnection.ClearAllPools(); + try { Directory.Delete(dir, recursive: true); } catch { } + } + } + [Fact] public void ReadOnlyLegacyDb_MissingSignatureColumn_DepsAndImpactDoNotCrashOnCSharp() { diff --git a/tests/CodeIndex.Tests/PreparedCommandCacheTests.cs b/tests/CodeIndex.Tests/PreparedCommandCacheTests.cs index a0f49fa2fa..4d86733afb 100644 --- a/tests/CodeIndex.Tests/PreparedCommandCacheTests.cs +++ b/tests/CodeIndex.Tests/PreparedCommandCacheTests.cs @@ -80,6 +80,33 @@ public void GetOrAdd_EvictsLeastRecentlyUsedWhenOverCapacity() Assert.NotSame(first, rebuilt); } + [Fact] + public void GetOrAdd_TracksHitMissAndEvictionDiagnostics_Issue3795() + { + using var cache = new PreparedCommandCache(_db.Connection, capacity: 2); + + var first = cache.GetOrAdd( + "SELECT 1 FROM files WHERE path = @path", + c => c.Parameters.Add("@path", SqliteType.Text)); + var firstAgain = cache.GetOrAdd( + "SELECT 1 FROM files WHERE path = @path", + c => throw new InvalidOperationException("cache hit should not re-configure")); + cache.GetOrAdd( + "SELECT 1 FROM files WHERE lang = @lang", + c => c.Parameters.Add("@lang", SqliteType.Text)); + cache.GetOrAdd( + "SELECT 1 FROM files WHERE size = @size", + c => c.Parameters.Add("@size", SqliteType.Integer)); + + Assert.Same(first, firstAgain); + var diagnostics = cache.GetDiagnostics(); + Assert.Equal(2, diagnostics.Count); + Assert.Equal(2, diagnostics.Capacity); + Assert.Equal(1, diagnostics.HitCount); + Assert.Equal(3, diagnostics.MissCount); + Assert.Equal(1, diagnostics.EvictionCount); + } + [Fact] public void GetOrAdd_TouchOnHitDelaysEviction() { @@ -148,6 +175,43 @@ public void Ctor_RejectsNonPositiveCapacity() () => new PreparedCommandCache(_db.Connection, capacity: -1)); } + [Fact] + public void ReadCapacityFromEnvironment_UsesBoundedConfiguredCapacity_Issue3795() + { + using var env = EnvironmentVariableScope.Capture(PreparedCommandCache.CapacityEnvironmentVariable); + + env.Set(PreparedCommandCache.CapacityEnvironmentVariable, "4"); + Assert.Equal(4, PreparedCommandCache.ReadCapacityFromEnvironment()); + + env.Set(PreparedCommandCache.CapacityEnvironmentVariable, (PreparedCommandCache.MaxCapacity + 1).ToString()); + Assert.Equal(PreparedCommandCache.DefaultCapacity, PreparedCommandCache.ReadCapacityFromEnvironment()); + + env.Set(PreparedCommandCache.CapacityEnvironmentVariable, "not-a-number"); + Assert.Equal(PreparedCommandCache.DefaultCapacity, PreparedCommandCache.ReadCapacityFromEnvironment()); + } + + [Fact] + public void DbContext_UsesConfiguredPreparedCommandCacheCapacity_Issue3795() + { + using var env = EnvironmentVariableScope.Capture(PreparedCommandCache.CapacityEnvironmentVariable); + env.Set(PreparedCommandCache.CapacityEnvironmentVariable, "4"); + var dbPath = Path.Combine(Path.GetTempPath(), $"prepcache_capacity_{Guid.NewGuid():N}.db"); + DbContext? db = null; + try + { + db = new DbContext(dbPath); + db.InitializeSchema(); + + Assert.Equal(4, db.PreparedCommands.Capacity); + } + finally + { + db?.Dispose(); + SqliteConnection.ClearAllPools(); + try { File.Delete(dbPath); } catch { } + } + } + [Fact] public void DbWriter_WithCache_ReusesCommandsAcrossUpsertCalls() { @@ -185,6 +249,7 @@ public void DbWriter_WithCache_ReusesCommandsAcrossUpsertCalls() // The cache should hold prepared commands for the hot per-file SQLs. // ホットパス SQL に対応する prepared command が cache に積まれている。 Assert.True(_db.PreparedCommands.Count > 0); + Assert.True(_db.PreparedCommands.MissCount > 0); } [Fact] diff --git a/tests/CodeIndex.Tests/QueryCommandRunnerFilesTests.cs b/tests/CodeIndex.Tests/QueryCommandRunnerFilesTests.cs index 5810b76e91..16e78f9dc9 100644 --- a/tests/CodeIndex.Tests/QueryCommandRunnerFilesTests.cs +++ b/tests/CodeIndex.Tests/QueryCommandRunnerFilesTests.cs @@ -1278,6 +1278,13 @@ public void RunStatus_Json_UsesIndexedAndSourceFreshnessInsteadOfClockAge() Assert.Equal("wal", pragmas.GetProperty("journal_mode").GetString()); Assert.Equal(DbContext.DefaultSynchronousMode, pragmas.GetProperty("synchronous").GetString()); Assert.Equal(DbContext.DefaultWalAutocheckpointPages, pragmas.GetProperty("wal_autocheckpoint").GetInt32()); + Assert.Equal(DbPragmaPolicy.DefaultBusyTimeoutMs, pragmas.GetProperty("busy_timeout_ms").GetInt32()); + var preparedCommandCache = json.GetProperty("prepared_command_cache"); + Assert.Equal(PreparedCommandCache.DefaultCapacity, preparedCommandCache.GetProperty("capacity").GetInt32()); + Assert.True(preparedCommandCache.GetProperty("count").GetInt32() >= 0); + Assert.True(preparedCommandCache.GetProperty("miss_count").GetInt64() >= 0); + Assert.True(preparedCommandCache.GetProperty("hit_count").GetInt64() >= 0); + Assert.True(preparedCommandCache.GetProperty("eviction_count").GetInt64() >= 0); } finally { diff --git a/tests/CodeIndex.Tests/QueryCommandRunnerTests.cs b/tests/CodeIndex.Tests/QueryCommandRunnerTests.cs index d3131c09f1..71d39c87be 100644 --- a/tests/CodeIndex.Tests/QueryCommandRunnerTests.cs +++ b/tests/CodeIndex.Tests/QueryCommandRunnerTests.cs @@ -569,6 +569,39 @@ public void RunStatusJson_ReportsSqlitePageMetrics_Issue1631() Assert.True(settings.GetProperty("page_count").GetInt64() > 0); Assert.True(settings.GetProperty("page_size").GetInt64() > 0); Assert.True(settings.GetProperty("freelist_count").GetInt64() >= 0); + Assert.Equal(DbPragmaPolicy.DefaultBusyTimeoutMs, settings.GetProperty("busy_timeout_ms").GetInt64()); + } + finally + { + TestProjectHelper.DeleteDirectory(projectRoot); + } + } + + [Fact] + public void RunStatusJson_ReportsConfiguredBusyTimeout_Issue3767() + { + var projectRoot = TestProjectHelper.CreateTempProject("cdidx_status_busy_timeout_3767"); + using var env = EnvironmentVariableScope.Capture( + DbContext.BusyTimeoutEnvironmentVariable); + env.Set(DbContext.BusyTimeoutEnvironmentVariable, "12345"); + try + { + var dbPath = TestProjectHelper.CreateProjectDb(projectRoot); + TestProjectHelper.InsertIndexedFile( + dbPath, + "src/app.cs", + "csharp", + "public class App { }"); + + var (exitCode, stdout, stderr) = CaptureConsole(() => QueryCommandRunner.RunStatus( + ["--db", dbPath, "--json"], + _jsonOptions)); + + Assert.Equal(CommandExitCodes.Success, exitCode); + Assert.Equal(string.Empty, stderr); + using var document = ParseJsonOutput(stdout); + var settings = document.RootElement.GetProperty("db_pragma_settings"); + Assert.Equal(12345, settings.GetProperty("busy_timeout_ms").GetInt64()); } finally { @@ -4114,6 +4147,70 @@ public void RunDeps_WorkspaceDbJson_CapsCrossDatabaseSymbolSample_Issue3155() } } + [Fact] + public void RunDeps_WorkspaceDbExcludeTestsKeepsNonTestPathSegments_Issue3834() + { + var primaryRoot = TestProjectHelper.CreateTempProject("cdidx_deps_workspace_exclude_tests_primary"); + var memberRoot = TestProjectHelper.CreateTempProject("cdidx_deps_workspace_exclude_tests_member"); + try + { + var primaryDb = TestProjectHelper.CreateProjectDb(primaryRoot); + var memberDb = TestProjectHelper.CreateProjectDb(memberRoot); + InsertFileWithReference(primaryDb, "contest/PrimaryCaller.cs", "SharedTarget"); + InsertFileWithSymbol(memberDb, "src/SharedTarget.cs", "SharedTarget"); + + var (exitCode, stdout, stderr) = CaptureConsole(() => QueryCommandRunner.RunDeps( + ["--db", primaryDb, "--workspace-db", memberDb, "--json", "--limit", "10", "--lang", "csharp", "--exclude-tests"], + _jsonOptions)); + + using var document = ParseJsonOutput(stdout); + var edge = Assert.Single(document.RootElement.GetProperty("edges").EnumerateArray()); + + Assert.Equal(CommandExitCodes.Success, exitCode); + Assert.NotNull(stderr); + Assert.Equal("contest/PrimaryCaller.cs", edge.GetProperty("source_path").GetString()); + Assert.Equal("src/SharedTarget.cs", edge.GetProperty("target_path").GetString()); + Assert.DoesNotContain("LIKE '%test%'", QueryCommandRunner.BuildCrossDatabaseTestPathConditionForTesting("src")); + } + finally + { + TestProjectHelper.DeleteDirectory(primaryRoot); + TestProjectHelper.DeleteDirectory(memberRoot); + } + } + + [Fact] + public void BuildWorkspaceDependencyDatabaseList_DeduplicatesUsingFilesystemCaseSensitivity_Issue3834() + { + var previousProbe = PathCasing.IgnoreCaseProbeForTesting; + PathCasing.ResetCacheForTests(); + PathCasing.IgnoreCaseProbeForTesting = _ => true; + try + { + var tempDir = Path.GetTempPath(); + var dbPath = Path.Combine(tempDir, "CodeIndexCaseProbe.db"); + var options = new QueryCommandOptions + { + DbPath = dbPath, + WorkspaceDbPaths = + [ + Path.Combine(tempDir, "codeindexcaseprobe.db"), + Path.Combine(tempDir, "OtherCodeIndexCaseProbe.db"), + ], + }; + + var dbs = QueryCommandRunner.BuildWorkspaceDependencyDatabaseList(options); + + Assert.Equal(2, dbs.Count); + Assert.Equal(Path.GetFullPath(dbPath), dbs[0]); + } + finally + { + PathCasing.IgnoreCaseProbeForTesting = previousProbe; + PathCasing.ResetCacheForTests(); + } + } + [Fact] public void RunDeps_WorkspaceDbTooManyDistinctDatabases_ReturnsUsageError_Issue3154() { diff --git a/tests/CodeIndex.Tests/SqliteDynamicSqlTests.cs b/tests/CodeIndex.Tests/SqliteDynamicSqlTests.cs new file mode 100644 index 0000000000..35dd9d19e8 --- /dev/null +++ b/tests/CodeIndex.Tests/SqliteDynamicSqlTests.cs @@ -0,0 +1,104 @@ +using System.Reflection; +using CodeIndex.Database; +using Microsoft.Data.Sqlite; + +namespace CodeIndex.Tests; + +public class SqliteDynamicSqlTests +{ + [Fact] + public void BuildParameterList_AllowsMaximumSqliteVariables_Issue3702() + { + var list = SqliteDynamicSql.BuildParameterList("p", SqliteDynamicSql.MaxSqlVariables); + + Assert.StartsWith("@p0, @p1", list, StringComparison.Ordinal); + Assert.EndsWith("@p998", list, StringComparison.Ordinal); + } + + [Fact] + public void BuildParameterList_RejectsListsOverSqliteVariableBudget_Issue3702() + { + var ex = Assert.Throws( + () => SqliteDynamicSql.BuildParameterList("p", SqliteDynamicSql.MaxSqlVariables + 1)); + + Assert.Contains("exceeding the supported budget", ex.Message); + } + + [Fact] + public void PathFilterParameters_AllowLargeListNearSqliteVariableBudget_Issue3702() + { + var pathPatterns = Enumerable + .Range(0, SqliteDynamicSql.MaxSqlVariables) + .Select(i => $"src/{i}.cs") + .ToList(); + var sql = "SELECT 1 FROM files f WHERE 1 = 1"; + + DbReader.AppendPathFilters(ref sql, pathPatterns, excludePathPatterns: null, excludeTests: false); + using var connection = CreateInMemoryConnection(); + using var cmd = connection.CreateCommand(); + DbReader.AddPathFilterParameters(cmd, pathPatterns, excludePathPatterns: null); + + Assert.Contains("@pathPattern998", sql); + Assert.Equal(SqliteDynamicSql.MaxSqlVariables, cmd.Parameters.Count); + Assert.Equal("%src/998.cs%", cmd.Parameters["@pathPattern998"].Value); + Assert.Equal(SqliteType.Text, cmd.Parameters["@pathPattern998"].SqliteType); + } + + [Fact] + public void PathFilterParameters_RejectCombinedIncludeExcludeOverBudget_Issue3702() + { + var includePatterns = Enumerable.Range(0, 500).Select(i => $"src/{i}.cs").ToList(); + var excludePatterns = Enumerable.Range(0, 500).Select(i => $"tests/{i}.cs").ToList(); + var sql = "SELECT 1 FROM files f WHERE 1 = 1"; + + var ex = Assert.Throws( + () => DbReader.AppendPathFilters(ref sql, includePatterns, excludePatterns, excludeTests: false)); + + Assert.Contains("path filters", ex.Message); + } + + [Fact] + public void VisibilityFilterParameters_UseSharedBudgetAfterAliasExpansion_Issue3702() + { + using var connection = CreateInMemoryConnection(); + using var cmd = connection.CreateCommand(); + var method = typeof(DbReader).GetMethod( + "AddVisibilityFilterParameters", + BindingFlags.NonPublic | BindingFlags.Static) + ?? throw new MissingMethodException(nameof(DbReader), "AddVisibilityFilterParameters"); + + method.Invoke(null, new object?[] { cmd, new[] { "public" }, new[] { "private" } }); + + Assert.Equal(6, cmd.Parameters.Count); + Assert.Equal("public", cmd.Parameters["@visibility0"].Value); + Assert.Equal("export", cmd.Parameters["@visibility3"].Value); + Assert.Equal("private", cmd.Parameters["@excludeVisibility0"].Value); + Assert.Equal("fileprivate", cmd.Parameters["@excludeVisibility1"].Value); + Assert.Equal(SqliteType.Text, cmd.Parameters["@visibility0"].SqliteType); + } + + [Fact] + public void SupportedLanguageParameters_UseSharedDynamicInListBuilder_Issue3702() + { + using var connection = CreateInMemoryConnection(); + using var cmd = connection.CreateCommand(); + var method = typeof(DbWriter).GetMethod( + "BuildSupportedLanguageParameters", + BindingFlags.NonPublic | BindingFlags.Static) + ?? throw new MissingMethodException(nameof(DbWriter), "BuildSupportedLanguageParameters"); + + var names = Assert.IsType>(method.Invoke(null, new object?[] { cmd, new[] { "csharp", "python" } })); + + Assert.Equal(new[] { "@lang0", "@lang1" }, names); + Assert.Equal("csharp", cmd.Parameters["@lang0"].Value); + Assert.Equal("python", cmd.Parameters["@lang1"].Value); + Assert.Equal(SqliteType.Text, cmd.Parameters["@lang0"].SqliteType); + } + + private static SqliteConnection CreateInMemoryConnection() + { + var connection = new SqliteConnection("Data Source=:memory:"); + connection.Open(); + return connection; + } +} diff --git a/tests/CodeIndex.Tests/golden/status.json b/tests/CodeIndex.Tests/golden/status.json index bfb25f7c46..f2bfee676d 100644 --- a/tests/CodeIndex.Tests/golden/status.json +++ b/tests/CodeIndex.Tests/golden/status.json @@ -178,11 +178,19 @@ "journal_mode": "wal", "synchronous": "NORMAL", "wal_autocheckpoint": 1000, + "busy_timeout_ms": 5000, "page_count": "\u003CCOUNT\u003E", "freelist_count": 0, "page_size": 4096, "auto_vacuum": 2 }, + "prepared_command_cache": { + "count": 0, + "capacity": 32, + "hit_count": 0, + "miss_count": 0, + "eviction_count": 0 + }, "maintenance_guidance": { "wal_state": "ok", "wal_threshold_bytes": 67108864,