diff --git a/AGENT_GUIDE.md b/AGENT_GUIDE.md index 18fdbfadc..13f611a93 100644 --- a/AGENT_GUIDE.md +++ b/AGENT_GUIDE.md @@ -137,6 +137,7 @@ CI watching must be bounded. Do not loop indefinitely. - `status --json` and related JSON/MCP payloads currently expose the trust fields documented in `README.md` and `DEVELOPER_GUIDE.md`, including `fold_ready`, `fold_ready_reason`, `graph_table_available`, `graph_data_current`, `index_complete`, `index_incomplete_reasons`, `issues_table_available`, `file_issues_data_current`, `migration_in_progress`, `sql_graph_contract_ready`, `sql_graph_contract_degraded_reason`, `hotspot_family_ready`, `hotspot_family_degraded_reason`, `language_readiness`, `csharp_symbol_name_ready`, `csharp_metadata_target_ready`, `csharp_metadata_target_degraded_reason`, `indexed_head_commit`, `worktree_head_changed`, `indexed_head_sha`, `indexed_head_branch`, `indexed_head_timestamp`, `commits_ahead_of_indexed_head`, `head_freshness`, `index_writer_version`, `index_newer_than_reader`, `index_newer_than_reader_reason`, `unknown_extension_file_count`, `unknown_extension_files`, `unknown_extension_files_truncated`, `unknown_extension_file_path_limit`, `unknown_extension_extension_counts`, `unknown_extension_category_counts`, `unknown_extension_groups`, `extractors`, `git_executable`, `path_case_sensitive`, `data_dir`, `data_dir_source`, `data_dir_mode`, `db_file_mode`, `database_permission_policy`, `database_permission_diagnostics`, `mac_profile`, `mac_profile_diagnostics`, `db_size_bytes`, `wal_size_bytes`, `db_pragma_settings` (`journal_mode`, `synchronous`, `wal_autocheckpoint`, `busy_timeout_ms`, `page_count`, `freelist_count`, `page_size`, `auto_vacuum`), `prepared_command_cache` (`count`, `capacity`, `hit_count`, `miss_count`, `eviction_count`), `maintenance_guidance`, WAL checkpoint diagnostics (`read_only_fallback`, `wal_checkpoint_attempted`, `wal_checkpoint_succeeded`, `wal_checkpoint_skipped_reason`, `wal_checkpoint_failure_reason`, `wal_checkpoint_busy`, `wal_checkpoint_log_page_count`, `wal_checkpoint_checkpointed_page_count`, `wal_checkpoint_remaining_page_count`, `read_only_immutable_fallback`, `wal_stale_snapshot_risk`, `wal_stale_snapshot_reason`), `symbol_kinds`, `symbols_by_language`, status kind cap metadata (`symbol_kind_limit`, `symbol_kind_name_limit`, `symbol_kind_total_count`, `symbol_kind_omitted_count`, `symbol_kind_names_truncated`, `symbols_by_language_kind_total_counts`, `symbols_by_language_kind_omitted_counts`, `symbols_by_language_kind_names_truncated`), `process`, `last_index_run`, `last_failed_or_partial_index_run`, `last_failed_or_partial_index_run.progress_persisted`, `last_failed_or_partial_index_run.recovery_hint`, `last_failed_or_partial_index_run.file_errors`, `last_workspace_freshened_at`, `hooks`, `hook_diagnostics`, `trust_overrides`, MCP-only `mcp_session`, `mcp.rate_limit.bucket_limit`, `mcp.rate_limit.bucket_limit_rejection_count`, and the `status --check`-only `stale_after_seconds` / `index_age_seconds` threshold audit fields and `repair_commands`. - A valid CLI `status --stale-after ` implies the workspace check. Check-mode JSON includes `query_context.check_mode` (`explicit` or `implied_by_stale_after`) and `query_context.stale_after_seconds`; ordinary status JSON omits `query_context`. +- `database_size_attribution` is part of the synchronized status contract. Preserve its read-only main/WAL/SHM separation; exact logical reconciliation across object, freelist, and unexplained-residual bytes; table/index and page-type subtotals; 20-object/128-character sanitized bounds; and explicit `available=false` / stable `unavailable_reason` behavior without zero-valued unavailable object metrics. - Explicit WAL truncate-checkpoint diagnostics must preserve SQLite's `(busy, log, checkpointed)` result, treat non-zero `busy` or positive remaining pages as unsuccessful with bounded machine reasons, accept `(0, -1, -1)` as the successful non-WAL no-op, and never expose raw exception text or paths. - When any readiness field is degraded, the CLI adds `degraded_root_cause`, `degraded_reason`, `recommended_action`, `alternative_action`, and `readiness_degradations[]`. `degraded_root_cause` is the primary stable machine code; `readiness_degradations[]` lists every degraded field with `root_cause`, human reason, and remediation strings. - `hotspot_family_degraded_reason` currently uses `hotspot_family_support_not_indexed`, `hotspot_family_metadata_stale`, `hotspot_family_disabled_at_index_time`, `partial_family_key_population`, and `hotspot_family_marker_fingerprint_incomplete`; the incomplete marker fingerprint code means marker traversal hit safety caps and should stay synchronized with README / developer-guide recovery notes. diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index 47ec758c5..f4b2ad4ce 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -963,6 +963,7 @@ Current stable codes and triggers: | Read-only opens and fallback | Query-only commands open with SQLite `Mode=ReadOnly` from the first attempt, retain WAL visibility, and never use writable setup or opportunistic migrations. A write-capable intent may still fall back to read-only when writable journal/WAL setup fails; an explicitly supplied `immutable=1` URI is the opt-in stale-snapshot escape hatch. If a WAL is present and must be observed from storage that cannot expose its sidecars, copy `.db`, `.db-wal`, and `.db-shm` together to a readable location or use a SQLite backup from an environment that can open the full WAL set. | | Status pragma diagnostics | `status --json` exposes the selected read-only connection under `sqlite_connection_policy` (`active_mode=read_only`, `open_mode=read_only`) and resolved connection values under `db_pragma_settings` (`journal_mode`, `synchronous`, `wal_autocheckpoint`, `busy_timeout_ms`, `page_count`, `freelist_count`, `page_size`, `auto_vacuum`). It also exposes prepared-command cache counters under `prepared_command_cache` (`count`, `capacity`, `hit_count`, `miss_count`, `eviction_count`) for automation and support diagnostics. `maintenance_guidance` derives `wal_state`, `freelist_ratio`, `freelist_state`, `estimated_*_reclaimable`, `auto_vacuum_mode(_name)`, `recommended_command`, and `post_maintenance_follow_up` from those raw metrics without changing the raw values. `status --check --json` adds `repair_commands[]` entries with `name`, `args`, `reason`, and `safety_notes` so clients do not parse prose remediation strings. `last_failed_or_partial_index_run` exposes bounded failed/partial index context (`status`, `mode`, timings, counts, stable error code, reason, `progress_persisted`, and bounded `recovery_hint`) and must not include raw exception text or file paths. | | Maintenance thresholds | WAL guidance flips to `checkpoint_recommended` at `CDIDX_MAINTENANCE_WAL_WARN_BYTES` (default 64 MiB). Freelist guidance flips to `vacuum_recommended` at `CDIDX_MAINTENANCE_FREELIST_WARN_RATIO` (default `0.20`). Invalid or out-of-range env values fall back to defaults. | +| Page attribution | `status --json` reads SQLite page ownership without mutating the source. It prefers `dbstat` page bytes and otherwise traverses a bounded b-tree/WAL snapshot (at most 1,000,000 pages and 100,000 schema objects); when a live WAL connection is not already backed by a stable detached file set, the fallback first makes a cancellation-aware private backup of that connection's active read snapshot so a concurrent commit cannot mix generations. `allocated_object_bytes + freelist_bytes + unexplained_residual_bytes` equals `logical_database_bytes`; table/index and internal/leaf/overflow/other page subtotals each reconcile to `allocated_object_bytes`. Payload, unused space, and structural overhead form a second reconciliation. Physical main/WAL/SHM bytes are reported separately. Output is capped at 20 object names, each support-sanitized to at most 128 characters. A failed or inconsistent probe returns `available=false`, a stable `unavailable_reason`, and null/omitted attribution values rather than zeros. | | Vacuum | `cdidx vacuum` runs `PRAGMA incremental_vacuum` against writable incremental-auto-vacuum DBs, and performs a one-time `PRAGMA auto_vacuum=INCREMENTAL` plus full `VACUUM` conversion for legacy no-autovacuum DBs. `cdidx vacuum --dry-run --json` estimates reclaimable pages/bytes and returns the same maintenance guidance without executing vacuum pragmas. Real `cdidx vacuum --json` also reports before/after DB and WAL byte samples; `wal_checkpoint_timing_note` explains that `wal_size_bytes_after` is measured before connection cleanup, so later `status --json` output may show a smaller WAL after checkpoint/truncation. | | FTS optimize preview | `cdidx optimize --dry-run` opens a `QueryOnly` snapshot, probes an existing lockfile without creating or acquiring it, and never runs write PRAGMAs, schema setup, FTS control inserts, or metadata writes against the source DB/WAL/SHM set. JSON reports size/freelist/readiness indicators, the write-threshold recommendation, and planned operations, including the real command's repair-mode schema initialization or migration check. Object sizes use `dbstat` page bytes when available and a labeled logical-payload fallback otherwise. A real optimize records its elapsed milliseconds so later previews can expose `estimated_duration_ms`. | | Size and process diagnostics | `status --json` also reports `db_size_bytes`, `wal_size_bytes`, capped `symbol_kinds` / `symbols_by_language` kind maps with `symbol_kind_*` and `symbols_by_language_kind_*` overflow metadata when caps apply, current `process` heap/GC/working-set metrics, `last_index_run` metadata from successful CLI and MCP index runs, and `last_workspace_freshened_at` as the latest successful index/update timestamp. `last_index_run.bytes_read_skipped_file_count` and `bytes_read_incomplete` report whether unreadable files were omitted from the `bytes_read` total, while `last_index_run.diagnostics`, `diagnostic_count`, and `diagnostics_truncated` carry bounded warnings for best-effort index metadata writes that failed after the index data itself was successfully written. `indexed_at` still comes from indexed file rows, so partial or no-op updates can freshen the workspace without moving `indexed_at`. | @@ -1741,6 +1742,7 @@ access. | Version and forward compatibility | `index_writer_version`, `index_newer_than_reader`, `index_newer_than_reader_reason`. | | Unknown-extension and runtime diagnostics | `unknown_extension_file_count`, `unknown_extension_files`, `unknown_extension_files_truncated`, `unknown_extension_file_path_limit`, `unknown_extension_extension_counts`, `unknown_extension_category_counts`, `unknown_extension_groups`, `extractors`, `hooks`, `hook_diagnostics`, `trust_overrides`, `path_case_sensitive`, `data_dir_mode`, `mac_profile`, `mac_profile_diagnostics`, `stale_after_seconds`, `index_age_seconds`, `query_context.check_mode`, `query_context.stale_after_seconds`, `last_index_run.reference_extraction_cap_hits`, `last_failed_or_partial_index_run`, `last_failed_or_partial_index_run.progress_persisted`, `last_failed_or_partial_index_run.recovery_hint`, `last_failed_or_partial_index_run.file_errors`. | | Database maintenance | `db_size_bytes`, `wal_size_bytes`, `db_pragma_settings` (`journal_mode`, `synchronous`, `wal_autocheckpoint`, `busy_timeout_ms`, `page_count`, `freelist_count`, `page_size`, `auto_vacuum`), `prepared_command_cache` (`count`, `capacity`, `hit_count`, `miss_count`, `eviction_count`), `maintenance_guidance`. | +| Database size attribution | `database_size_attribution` (`available`, `measurement`, `unavailable_reason`, physical main/WAL/SHM sizes, logical/object/freelist/residual reconciliation, page-type and payload/overhead subtotals, and bounded `top_objects`). | | Remediation fields | `degraded_root_cause`, `degraded_reason`, `recommended_action`, `alternative_action`, `readiness_degradations`, `repair_commands`. | | MCP-only session diagnostics | `mcp_session`, `mcp_session.metrics`, `mcp_session.audit_log`, `mcp.rate_limit.bucket_limit`, and `mcp.rate_limit.bucket_limit_rejection_count`. `mcp_session` is session-scoped diagnostics rather than persisted DB state. It contains `log_level`, bounded `roots`, optional `client_info`, bounded optional `client_capabilities`, an always-present `metrics` object, and `audit_log` when audit emission is enabled. When advertised roots are capped, `roots_truncated`, `root_count`, `root_limit`, and `root_uri_length_limit` describe the truncation. When client capabilities are capped, `client_capabilities_truncated`, `client_capabilities_truncation_reason`, `client_capabilities_serialized_bytes`, `client_capabilities_byte_limit`, and `client_capabilities_depth_limit` describe the retained diagnostic subset. `mcp_session.metrics` is `{"enabled":false}` when unconfigured. An enabled metrics sink contains `enabled`, `path`, `max_bytes`, `bytes_written`, `disposed`, `degraded`, `queue_capacity`, `queue_depth`, `queued_event_count`, `written_event_count`, `dropped_event_count`, `queue_full_drop_count`, `serialization_failure_count`, `write_failure_count`, `rotation_failure_count`, `batch_flush_count`, `consecutive_failure_count`, and `recovery_count`, plus optional `next_retry_at`, `last_recovery_at`, and `last_failure`. MCP ping always mirrors the metrics object as `metrics`; metrics degradation is intentionally excluded from its top-level liveness result. The audit status fields and their health semantics are defined in [MCP audit log emission](#mcp-audit-log-emission). `mcp.rate_limit.bucket_limit` is the configured process-local cap across normalized `(partition, caller)` buckets: every direct call uses one fixed caller-wide coarse partition, canonical known tools additionally use secondary per-tool partitions, and unknown `batch_query` slots share one fixed invalid-slot partition per caller. `mcp.rate_limit.bucket_limit_rejection_count` counts calls denied because creating a new bucket would exceed that cap. | | Documentation sync | Keep this list synchronized with `README.md` and `AGENT_GUIDE.md`; `DocumentationStatusContractTests` fails when any required field is missing from one of those docs. | @@ -4346,6 +4348,7 @@ apply 時は `PRAGMA optimize` を実行します。 | read-only open / fallback | query-only command は最初の試行から SQLite `Mode=ReadOnly` で開き、WAL の可視性を保ちながら writable setup と opportunistic migration を実行しません。write-capable intent は journal/WAL setup に失敗した場合に read-only へ fallback することがあります。明示的な `immutable=1` URI は stale snapshot を許容する opt-in escape hatch です。sidecar を公開できない storage 上の WAL を観測する必要がある場合は、`.db` / `.db-wal` / `.db-shm` をまとめて readable location に copy するか、full WAL set を open できる環境で SQLite backup を使います。 | | status pragma diagnostics | `status --json` は選択された read-only connection を `sqlite_connection_policy` (`active_mode=read_only`, `open_mode=read_only`) で、解決済みの接続値を `db_pragma_settings` (`journal_mode`, `synchronous`, `wal_autocheckpoint`, `busy_timeout_ms`, `page_count`, `freelist_count`, `page_size`, `auto_vacuum`) で公開します。また、prepared command cache counter を `prepared_command_cache` (`count`, `capacity`, `hit_count`, `miss_count`, `eviction_count`) で公開します。`maintenance_guidance` は raw 値を変えずに `wal_state`、`freelist_ratio`、`freelist_state`、`estimated_*_reclaimable`、`auto_vacuum_mode(_name)`、`recommended_command`、`post_maintenance_follow_up` を派生します。`status --check --json` は `repair_commands[]` に `name`、`args`、`reason`、`safety_notes` を返し、client が prose remediation を parse しなくてよいようにします。`last_failed_or_partial_index_run` は bounded な failed / partial index context (`status`、`mode`、timing、count、stable error code、reason、`progress_persisted`、bounded な `recovery_hint`) のみを公開し、raw exception text や file path を含めてはいけません。 | | maintenance threshold | WAL guidance は `CDIDX_MAINTENANCE_WAL_WARN_BYTES` (既定 64 MiB) 以上で `checkpoint_recommended` になります。freelist guidance は `CDIDX_MAINTENANCE_FREELIST_WARN_RATIO` (既定 `0.20`) 以上で `vacuum_recommended` になります。不正・範囲外の環境変数値は既定値へ戻します。 | +| page attribution | `status --json` は source を変更せずに SQLite page ownership を読み取ります。`dbstat` page byte を優先し、利用できない場合は件数上限付きの b-tree / WAL snapshot traversal（最大1,000,000 page、100,000 schema object）へ fallback します。live WAL connection が安定した detached file set に基づいていない場合、fallback は先にその connection の active read snapshot を cancellation 対応の private backup に固定し、並行 commit による世代混在を防ぎます。`allocated_object_bytes + freelist_bytes + unexplained_residual_bytes` は `logical_database_bytes` と一致し、table/index と internal/leaf/overflow/other page の小計はそれぞれ `allocated_object_bytes` と一致します。payload、unused space、structural overhead も別に再照合されます。物理 main/WAL/SHM byte は分離して報告します。出力する object 名は最大20件で、各名称は support-safe sanitizer により最大128文字になります。probe の失敗・不整合時は `available=false`、安定した `unavailable_reason`、null / 省略された attribution 値を返し、ゼロとして偽装しません。 | | vacuum | `cdidx vacuum` は incremental-auto-vacuum DB では `PRAGMA incremental_vacuum` を実行し、legacy no-autovacuum DB では初回のみ `PRAGMA auto_vacuum=INCREMENTAL` と full `VACUUM` で変換します。`cdidx vacuum --dry-run --json` は vacuum pragma を実行せず、回収可能 page/byte の推定と同じ maintenance guidance を返します。実行系 `cdidx vacuum --json` は DB / WAL byte の before / after sample も返します。`wal_checkpoint_timing_note` は `wal_size_bytes_after` が connection cleanup 前の計測であり、checkpoint / truncation 後の `status --json` では WAL が小さく見える場合があることを示します。 | | FTS optimize preview | `cdidx optimize --dry-run` は `QueryOnly` snapshot を開き、既存 lockfile を作成も取得もせずに probe し、source DB/WAL/SHM set に対する write PRAGMA、schema setup、FTS control insert、metadata write を一切実行しません。JSON は size/freelist/readiness 指標、write threshold に基づく推奨、実行系の repair mode による schema 初期化または migration の確認を含む planned operation を返します。object size は利用可能なら `dbstat` page byte を使い、利用できない場合は明示した logical-payload fallback を使います。実際の optimize は所要 millisecond を記録し、後続 preview が `estimated_duration_ms` として返せるようにします。 | | size / process diagnostics | `status --json` は `db_size_bytes`、`wal_size_bytes`、上限付きの `symbol_kinds` / `symbols_by_language` kind map と、上限適用時の `symbol_kind_*` / `symbols_by_language_kind_*` overflow metadata、現在の `process` heap / GC / working-set metrics、成功した CLI / MCP index 実行由来の `last_index_run` metadata、最新の成功 index/update 時刻を示す `last_workspace_freshened_at` も公開します。`last_index_run.bytes_read_skipped_file_count` と `bytes_read_incomplete` は、読み取り不能な file が `bytes_read` 合計から除外されたかどうかを報告します。`last_index_run.diagnostics`、`diagnostic_count`、`diagnostics_truncated` は、index data 自体の書き込みが成功した後に best-effort index metadata write が失敗した場合の上限付き warning を保持します。`indexed_at` は引き続き indexed file row 由来なので、partial / no-op update は `indexed_at` を動かさずに workspace 鮮度だけを更新することがあります。 | @@ -5143,6 +5146,7 @@ help はすべてこのレジストリを参照します。field 名は大文字 | version / forward compatibility | `index_writer_version`, `index_newer_than_reader`, `index_newer_than_reader_reason`。 | | unknown-extension / runtime diagnostics | `unknown_extension_file_count`, `unknown_extension_files`, `unknown_extension_files_truncated`, `unknown_extension_file_path_limit`, `unknown_extension_extension_counts`, `unknown_extension_category_counts`, `unknown_extension_groups`, `extractors`, `hooks`, `hook_diagnostics`, `trust_overrides`, `path_case_sensitive`, `data_dir_mode`, `mac_profile`, `mac_profile_diagnostics`, `stale_after_seconds`, `index_age_seconds`, `query_context.check_mode`, `query_context.stale_after_seconds`, `last_index_run.reference_extraction_cap_hits`, `last_failed_or_partial_index_run`, `last_failed_or_partial_index_run.progress_persisted`, `last_failed_or_partial_index_run.recovery_hint`, `last_failed_or_partial_index_run.file_errors`。 | | database maintenance | `db_size_bytes`, `wal_size_bytes`, `db_pragma_settings` (`journal_mode`, `synchronous`, `wal_autocheckpoint`, `busy_timeout_ms`, `page_count`, `freelist_count`, `page_size`, `auto_vacuum`), `prepared_command_cache` (`count`, `capacity`, `hit_count`, `miss_count`, `eviction_count`), `maintenance_guidance`。 | +| database size attribution | `database_size_attribution`（`available`、`measurement`、`unavailable_reason`、物理 main/WAL/SHM size、論理/object/freelist/residual の再照合、page type と payload/overhead の小計、上限付き `top_objects`）。 | | remediation fields | `degraded_root_cause`, `degraded_reason`, `recommended_action`, `alternative_action`, `readiness_degradations`, `repair_commands`。 | | MCP-only session diagnostics | `mcp_session`、`mcp_session.metrics`、`mcp_session.audit_log`、`mcp.rate_limit.bucket_limit`、`mcp.rate_limit.bucket_limit_rejection_count`。`mcp_session` は persisted DB state ではなく session-scoped diagnostics で、`log_level`、上限付きの `roots`、任意の `client_info`、上限付きの任意の `client_capabilities`、常設の `metrics` object、audit 出力が有効な場合の `audit_log` を含みます。advertised root が切り詰められた場合は `roots_truncated`、`root_count`、`root_limit`、`root_uri_length_limit` が切り詰め内容を示します。client capabilities が切り詰められた場合は `client_capabilities_truncated`、`client_capabilities_truncation_reason`、`client_capabilities_serialized_bytes`、`client_capabilities_byte_limit`、`client_capabilities_depth_limit` が保持された診断 subset を示します。未設定時の `mcp_session.metrics` は `{"enabled":false}` です。有効な metrics sink は `enabled`、`path`、`max_bytes`、`bytes_written`、`disposed`、`degraded`、`queue_capacity`、`queue_depth`、`queued_event_count`、`written_event_count`、`dropped_event_count`、`queue_full_drop_count`、`serialization_failure_count`、`write_failure_count`、`rotation_failure_count`、`batch_flush_count`、`consecutive_failure_count`、`recovery_count` に加え、任意の `next_retry_at`、`last_recovery_at`、`last_failure` を追加します。MCP ping は常に metrics object を `metrics` として返し、metrics の degradation は意図的に top-level liveness result へ反映しません。audit status field と health semantics は [MCP 監査ログの出力](#mcp-監査ログの出力) に定義します。`mcp.rate_limit.bucket_limit` は normalized な `(partition, caller)` bucket 全体に対する process-local 上限で、direct call はすべて caller-wide の固定 coarse partition、canonical な既知 tool は追加の secondary per-tool partition、unknown な `batch_query` slot は caller ごとの 1 つの固定 invalid-slot partition を使います。`mcp.rate_limit.bucket_limit_rejection_count` は新規 bucket 作成がその上限を超えるため拒否された呼び出し数です。 | | documentation sync | この一覧は `README.md` と `AGENT_GUIDE.md` と同期してください。必須 field がそれらの docs から欠けると `DocumentationStatusContractTests` が失敗します。 | diff --git a/README.md b/README.md index 3e9e12ce8..2a96d3d84 100644 --- a/README.md +++ b/README.md @@ -247,6 +247,7 @@ cancellation, and WAL snapshot-risk diagnostics. | Unknown-extension and runtime diagnostics | `unknown_extension_file_count`, `unknown_extension_files`, `unknown_extension_files_truncated`, `unknown_extension_file_path_limit`, `unknown_extension_extension_counts`, `unknown_extension_category_counts`, `unknown_extension_groups`, `extractors`, `hooks`, `hook_diagnostics`, `trust_overrides`, `git_executable`, `path_case_sensitive`, `data_dir_mode`, `db_file_mode`, `database_permission_policy`, `database_permission_diagnostics`, `mac_profile`, `mac_profile_diagnostics`, `stale_after_seconds`, `index_age_seconds`, `query_context.check_mode`, `query_context.stale_after_seconds`, `process`, `last_index_run`, `last_workspace_freshened_at`, `last_index_run.bytes_read_skipped_file_count`, `last_index_run.bytes_read_incomplete`, `last_index_run.diagnostics`, `last_index_run.diagnostic_count`, `last_index_run.diagnostics_truncated`, `last_index_run.reference_extraction_cap_hits`, `last_failed_or_partial_index_run`, `last_failed_or_partial_index_run.progress_persisted`, `last_failed_or_partial_index_run.recovery_hint`, `last_failed_or_partial_index_run.file_errors`. | | Database maintenance | `sqlite_connection_policy` (`active_mode`, `open_mode`, `immutable_uri`, WAL checkpoint/fallback fields), `db_size_bytes`, `wal_size_bytes`, `db_pragma_settings` (`journal_mode`, `synchronous`, `wal_autocheckpoint`, `busy_timeout_ms`, `page_count`, `freelist_count`, `page_size`, `auto_vacuum`), `prepared_command_cache` (`count`, `capacity`, `hit_count`, `miss_count`, `eviction_count`), `maintenance_guidance`. Query-only status reports `pooling=false` and either `read_only` or `immutable_read_only_uri`; checkpointed WAL databases use an immutable private snapshot, while non-empty WAL databases use a stable private main/WAL snapshot so source sidecars remain unchanged. Persistent private-snapshot copy failures report `query_only_snapshot_copy_failed` with temporary-storage capacity and permission guidance instead of being misreported as WAL churn. | | WAL checkpoint diagnostics | `read_only_fallback`, `wal_checkpoint_attempted`, `wal_checkpoint_succeeded`, `wal_checkpoint_skipped_reason`, `wal_checkpoint_failure_reason`, `wal_checkpoint_busy`, `wal_checkpoint_log_page_count`, `wal_checkpoint_checkpointed_page_count`, `wal_checkpoint_remaining_page_count`, `read_only_immutable_fallback`, `wal_stale_snapshot_risk`, `wal_stale_snapshot_reason`. | +| Database size attribution | `database_size_attribution` separates main DB, WAL, and SHM file bytes and reconciles logical pages across tables, indexes, freelist pages, internal/leaf/overflow page types, payload, unused space, structural overhead, and `unexplained_residual_bytes`. It emits at most 20 redacted/truncated object names. `available=false` with `unavailable_reason` means page attribution was not measurable; omitted object-byte fields must not be interpreted as zero. | | Remediation fields | `degraded_root_cause`, `degraded_reason`, `recommended_action`, `alternative_action`, `readiness_degradations`, `repair_commands`. | | MCP-only session diagnostics | `mcp_session`, `mcp_session.metrics`, `mcp_session.audit_log`, `mcp.rate_limit.bucket_limit`, `mcp.rate_limit.bucket_limit_rejection_count`. | @@ -697,6 +698,7 @@ immutable URI の選択、timeout、cancellation、WAL snapshot risk の diagnos | unknown-extension / runtime diagnostics | `unknown_extension_file_count`, `unknown_extension_files`, `unknown_extension_files_truncated`, `unknown_extension_file_path_limit`, `unknown_extension_extension_counts`, `unknown_extension_category_counts`, `unknown_extension_groups`, `extractors`, `hooks`, `hook_diagnostics`, `trust_overrides`, `git_executable`, `path_case_sensitive`, `data_dir_mode`, `db_file_mode`, `database_permission_policy`, `database_permission_diagnostics`, `mac_profile`, `mac_profile_diagnostics`, `stale_after_seconds`, `index_age_seconds`, `query_context.check_mode`, `query_context.stale_after_seconds`, `process`, `last_index_run`, `last_workspace_freshened_at`, `last_index_run.bytes_read_skipped_file_count`, `last_index_run.bytes_read_incomplete`, `last_index_run.diagnostics`, `last_index_run.diagnostic_count`, `last_index_run.diagnostics_truncated`, `last_index_run.reference_extraction_cap_hits`, `last_failed_or_partial_index_run`, `last_failed_or_partial_index_run.progress_persisted`, `last_failed_or_partial_index_run.recovery_hint`, `last_failed_or_partial_index_run.file_errors`。 | | database maintenance | `sqlite_connection_policy` (`active_mode`、`open_mode`、`immutable_uri`、WAL checkpoint / fallback field)、`db_size_bytes`、`wal_size_bytes`、`db_pragma_settings` (`journal_mode`, `synchronous`, `wal_autocheckpoint`, `busy_timeout_ms`, `page_count`, `freelist_count`, `page_size`, `auto_vacuum`)、`prepared_command_cache` (`count`, `capacity`, `hit_count`, `miss_count`, `eviction_count`)、`maintenance_guidance`。query-only status は `pooling=false` と、`read_only` または `immutable_read_only_uri` の mode を返します。checkpoint 済み WAL database は immutable な private snapshot、non-empty WAL database は安定した private main/WAL snapshot から読むため source sidecar は変化しません。private snapshot の永続的な copy failure は WAL churn と誤報せず、temporary storage の容量・権限を案内する `query_only_snapshot_copy_failed` を返します。 | | WAL checkpoint diagnostics | `read_only_fallback`、`wal_checkpoint_attempted`、`wal_checkpoint_succeeded`、`wal_checkpoint_skipped_reason`、`wal_checkpoint_failure_reason`、`wal_checkpoint_busy`、`wal_checkpoint_log_page_count`、`wal_checkpoint_checkpointed_page_count`、`wal_checkpoint_remaining_page_count`、`read_only_immutable_fallback`、`wal_stale_snapshot_risk`、`wal_stale_snapshot_reason`。 | +| database size attribution | `database_size_attribution` は main DB / WAL / SHM の file byte を分離し、論理 page を table、index、freelist、internal / leaf / overflow page、payload、unused space、structural overhead、`unexplained_residual_bytes` に再照合します。object 名は伏字・切り詰めを適用して最大20件だけ返します。`available=false` と `unavailable_reason` がある場合は page attribution を計測できなかったことを示し、省略された object-byte field をゼロとして解釈してはいけません。 | | remediation fields | `degraded_root_cause`, `degraded_reason`, `recommended_action`, `alternative_action`, `readiness_degradations`, `repair_commands`。 | | MCP-only session diagnostics | `mcp_session`, `mcp_session.metrics`, `mcp_session.audit_log`, `mcp.rate_limit.bucket_limit`, `mcp.rate_limit.bucket_limit_rejection_count`。 | diff --git a/TESTING_GUIDE.md b/TESTING_GUIDE.md index c14191ddd..fd49c8cb2 100644 --- a/TESTING_GUIDE.md +++ b/TESTING_GUIDE.md @@ -103,6 +103,7 @@ Use `docs/test-doc-maintenance-plan.md` before moving oversized suites or adding C# qualified common-call coverage keeps static BCL, instance, LINQ extension, alias-qualified, current-instance, and unresolved-receiver cases together. Assert that extraction retains every row, default bare-name references/callers/callees and hotspot counts retain resolved evidence while excluding unresolved noise, the completeness option restores that noise deterministically, and dependency queries remain identity-scoped (#4867). Crystal, Groovy, Tcl, Prolog, and `ambiguous_pl` graph fixtures keep imports, parenthesized calls, conservative same-file command/predicate calls, caller containers, and keyword false-positive controls independently diagnosable; capability tests must keep their advertised symbol/reference/graph flags aligned with those extractor fixtures (#4746). Database status tests must also pin stale and current extractor-version stamps for those languages so pre-graph rows cannot report authoritative graph readiness. + Database page-attribution coverage keeps empty/schema-only, WAL-visible overflow, database-truncating WAL, and a connection-pinned read snapshot after a newer WAL commit, cancellation of both `dbstat` aggregation and WAL validation, the bounded large-page-count rejection, reconciliation, corrupt-file rejection, main/WAL/SHM separation, the 20-object and 128-character support-safe output bounds, explicit unavailable/not-requested values, and unchanged `total_changes()` / `PRAGMA query_only` in coordinated `DbReaderTests` fixtures. HDL graph coverage keeps Verilog, SystemVerilog, and VHDL in one coordinated extractor fixture for syntax-visible hierarchy/package/architecture edges, known same-file signal/type references, scope attribution, declaration/literal false-positive suppression, and comment masking. The paired `QueryCommandRunnerReferencesTests` fixture must persist and query one representative edge per HDL language so extractor-only assertions cannot hide database or CLI graph drift. `IndexCommandRunnerFullScanTests` must also clear the HDL graph contract and persisted edge, then prove that a normal full scan refreshes an unchanged HDL file before restoring readiness. JavaScript/TypeScript discriminant-guard coverage shares one JavaScript-compatible fixture across both languages, while `DatabaseTests` pins `type_tag` as an accepted persisted reference kind so extractor-only assertions cannot hide database kind-catalog drift. Dockerfile named-stage reference variants share one multi-stage fixture when ordinary, lowercase, platform-flagged, commented, hyphenated, and dotted forms can be distinguished by exact per-stage call counts; keep external base-image exclusions in that fixture as the negative control. @@ -1039,6 +1040,7 @@ dotnet test --filter "FullyQualifiedName~GitHelperTests" C# の修飾付き一般名 call の coverage は、static BCL、instance、LINQ extension、alias 修飾、current instance、未解決 receiver の各 case を同じ fixture に維持します。extraction が全 row を保持すること、無修飾名による references / callers / callees と hotspot count の既定動作が解決済み evidence を維持しつつ未解決 noise を除外すること、completeness option がその noise を決定的に復元すること、dependency query が identity scope のままであることを検証してください（#4867）。 Crystal、Groovy、Tcl、Prolog、`ambiguous_pl` の graph fixture では、import、括弧付き call、同一ファイルに限定した保守的な command / predicate call、caller container、keyword の false-positive control を個別に診断可能な状態で維持し、capability test の symbol / reference / graph 広告を extractor fixture と一致させてください（#4746）。 さらに database status test でこれらの言語の古い extractor-version stamp と現行 stamp を固定し、graph 対応前の row が authoritative な graph readiness を報告できないことを検証してください。 + database page-attribution coverage では、empty / schema-only、WAL から可視な overflow、database を縮小する WAL、後続 WAL commit 後も connection に固定された read snapshot の各 case、`dbstat` 集約と WAL 検証のキャンセル、large page count の上限付き拒否、再照合、破損 file の拒否、main/WAL/SHM の分離、20 object / 128文字の support-safe 出力上限、明示的な unavailable / not-requested 値、`total_changes()` / `PRAGMA query_only` が不変であることを、連携した `DbReaderTests` fixture で維持してください。 HDL graph coverage では Verilog、SystemVerilog、VHDL を1つの連携した extractor fixture にまとめ、構文上確認できる hierarchy/package/architecture edge、同一ファイル内の既知 signal/type reference、scope attribution、declaration/literal の false-positive 抑止、comment masking を検証します。対になる `QueryCommandRunnerReferencesTests` fixture では各 HDL 言語の代表 edge を1件ずつ永続化して query し、extractor-only assertion が database または CLI graph のずれを隠せないようにします。さらに `IndexCommandRunnerFullScanTests` では HDL graph contract と永続化 edge を消去し、通常の full scan が未変更 HDL file を更新してから readiness を復元することを検証します。 JavaScript / TypeScript の discriminant-guard coverage は両言語で JavaScript-compatible な fixture を共有し、`DatabaseTests` で `type_tag` が永続化可能な reference kind であることも固定して、extractor-only assertion が database kind catalog のずれを隠せないようにします。 Dockerfile の named-stage reference variant は、通常、小文字、platform flag、comment、hyphen、dot 形式を stage ごとの厳密な call 数で区別できる場合、1つの multi-stage fixture を共有します。外部 base image の除外も negative control として同じ fixture に残します。 diff --git a/changelog.d/unreleased/4888.added.md b/changelog.d/unreleased/4888.added.md new file mode 100644 index 000000000..2f6d5cde0 --- /dev/null +++ b/changelog.d/unreleased/4888.added.md @@ -0,0 +1,26 @@ +--- +category: added +issues: + - 4888 +affected: + - src/CodeIndex/Database/DbReader.DatabaseSizeAttribution.cs + - src/CodeIndex/Database/DbReader.cs + - src/CodeIndex/Database/SqlitePageAttributionReader.cs + - src/CodeIndex/Database/DbReader.Status.cs + - src/CodeIndex/Cli/QueryCommandRunner.Status.cs + - src/CodeIndex/Mcp/McpToolHandlers.Query.Status.cs + - src/CodeIndex/Models/QueryResults.cs + - tests/CodeIndex.Tests/DbReaderTests.cs + - README.md + - DEVELOPER_GUIDE.md + - TESTING_GUIDE.md + - AGENT_GUIDE.md +--- + +## English + +- **`status --json` now reconciles SQLite database bytes by object and page usage (#4888)** — The bounded, source-read-only `database_size_attribution` block separates main DB, WAL, and SHM bytes; attributes tables, indexes, freelist, internal/leaf/overflow pages, payload, unused space, structural overhead, and unexplained residual bytes; and emits at most 20 support-sanitized object names. A live WAL fallback pins the connection's active read snapshot before traversal so concurrent commits cannot mix status generations. Unsupported or inconsistent page probes report an explicit unavailable reason without presenting missing measurements as zero, while human/compact and unrelated internal status consumers skip the page scan. + +## 日本語 + +- **`status --json` が SQLite database byte を object / page usage 別に再照合できるようになりました (#4888)** — source を変更しない件数上限付きの `database_size_attribution` block が main DB、WAL、SHM の byte を分離し、table、index、freelist、internal/leaf/overflow page、payload、unused space、structural overhead、未説明の残差 byte を分類し、support-safe に処理した object 名を最大20件返します。live WAL fallback は traversal 前に connection の active read snapshot を固定し、並行 commit による status 世代の混在を防ぎます。page probe が非対応または不整合の場合は欠損値をゼロとして示さず、明示的な unavailable reason を返し、人間向け/compact output と無関係な内部 status 利用では page scan を省略します。 diff --git a/src/CodeIndex/Cli/IndexCommandRunner.Maintenance.cs b/src/CodeIndex/Cli/IndexCommandRunner.Maintenance.cs index be0d4ad58..247d5687d 100644 --- a/src/CodeIndex/Cli/IndexCommandRunner.Maintenance.cs +++ b/src/CodeIndex/Cli/IndexCommandRunner.Maintenance.cs @@ -230,7 +230,7 @@ private static int RunOptimizeFtsPreviewForDb( MaintenanceDatabaseFailureKind.NotDatabase)); } - var status = new DbReader(db).GetStatus(); + var status = new DbReader(db).GetStatus(includeDatabaseSizeAttribution: false); var objectSizes = ReadOptimizeObjectSizes( db, forceLogicalObjectSizeFallbackForTesting, diff --git a/src/CodeIndex/Cli/JsonOutputContracts.cs b/src/CodeIndex/Cli/JsonOutputContracts.cs index e9be53a4e..db771a750 100644 --- a/src/CodeIndex/Cli/JsonOutputContracts.cs +++ b/src/CodeIndex/Cli/JsonOutputContracts.cs @@ -1327,6 +1327,8 @@ internal sealed record ValidateConfigJsonResult( [JsonSerializable(typeof(LicenseTermsJsonResult))] [JsonSerializable(typeof(LicenseTrademarkJsonResult))] [JsonSerializable(typeof(StatusResult))] +[JsonSerializable(typeof(StatusDatabaseSizeAttribution))] +[JsonSerializable(typeof(StatusDatabaseObjectSize))] [JsonSerializable(typeof(StatusLogPathJsonResult))] [JsonSerializable(typeof(StatusHeadFreshness))] [JsonSerializable(typeof(StatusSqliteConnectionPolicy))] diff --git a/src/CodeIndex/Cli/QueryCommandRunner.Languages.cs b/src/CodeIndex/Cli/QueryCommandRunner.Languages.cs index a1bddaa54..9e2619d05 100644 --- a/src/CodeIndex/Cli/QueryCommandRunner.Languages.cs +++ b/src/CodeIndex/Cli/QueryCommandRunner.Languages.cs @@ -37,7 +37,7 @@ public static int RunLanguages(string[] cmdArgs, JsonSerializerOptions jsonOptio { return WithDb(options, jsonOptions, reader => { - var status = reader.GetStatus(); + var status = reader.GetStatus(includeDatabaseSizeAttribution: false); var catalog = BuildLanguageCatalog(reader.GetIndexedProjectRoot()); var indexedLanguageCounts = loadIndexedCounts ? status.Languages : null; return WriteLanguages( diff --git a/src/CodeIndex/Cli/QueryCommandRunner.QueryHints.cs b/src/CodeIndex/Cli/QueryCommandRunner.QueryHints.cs index 433b7451a..bfa2e8ca1 100644 --- a/src/CodeIndex/Cli/QueryCommandRunner.QueryHints.cs +++ b/src/CodeIndex/Cli/QueryCommandRunner.QueryHints.cs @@ -12,7 +12,7 @@ public static partial class QueryCommandRunner private static void WriteLangHint(string? lang, DbReader reader) { if (lang == null) return; - var status = reader.GetStatus(); + var status = reader.GetStatus(includeDatabaseSizeAttribution: false); if (status.Languages.Count > 0 && status.Languages.ContainsKey(lang)) return; @@ -51,7 +51,7 @@ private static void WriteSymbolExtractionCapabilityHint(string? lang, DbReader r if (SymbolExtractor.GetSupportedLanguages(reader.GetIndexedProjectRoot()).Contains(lang, StringComparer.Ordinal)) return; - var status = reader.GetStatus(); + var status = reader.GetStatus(includeDatabaseSizeAttribution: false); if (status.Languages.Count == 0 || !status.Languages.ContainsKey(lang)) return; diff --git a/src/CodeIndex/Cli/QueryCommandRunner.RepairCommands.cs b/src/CodeIndex/Cli/QueryCommandRunner.RepairCommands.cs index 831b36f9c..5f9f3e257 100644 --- a/src/CodeIndex/Cli/QueryCommandRunner.RepairCommands.cs +++ b/src/CodeIndex/Cli/QueryCommandRunner.RepairCommands.cs @@ -14,7 +14,7 @@ private static string BuildFoldBackfillCommand(string dbPath, bool dbPathExplici private static string BuildCSharpCanonicalNameRepairCommand(DbReader reader, QueryCommandOptions options) { - var status = reader.GetStatus(); + var status = reader.GetStatus(includeDatabaseSizeAttribution: false); WorkspaceMetadataEnricher.Enrich(status, options.DbPath, options.DbPathExplicit); return BuildCSharpCanonicalNameRepairCommand(status.ProjectRoot, options.DbPath, options.DbPathExplicit); } @@ -24,7 +24,7 @@ private static string BuildCSharpCanonicalNameRepairCommand(string? projectRoot, private static string BuildSqlGraphContractRepairCommand(DbReader reader, QueryCommandOptions options) { - var status = reader.GetStatus(); + var status = reader.GetStatus(includeDatabaseSizeAttribution: false); WorkspaceMetadataEnricher.Enrich(status, options.DbPath, options.DbPathExplicit); return BuildSqlGraphContractRepairCommand(status.ProjectRoot, options.DbPath, options.DbPathExplicit); } diff --git a/src/CodeIndex/Cli/QueryCommandRunner.Status.cs b/src/CodeIndex/Cli/QueryCommandRunner.Status.cs index 85a883e9a..919e807c4 100644 --- a/src/CodeIndex/Cli/QueryCommandRunner.Status.cs +++ b/src/CodeIndex/Cli/QueryCommandRunner.Status.cs @@ -110,7 +110,7 @@ public static int RunStatus( } } - var status = reader.GetStatus(); + var status = reader.GetStatus(includeDatabaseSizeAttribution: options.Json); WorkspaceMetadataEnricher.Enrich(status, options.DbPath, options.DbPathExplicit, cancellationToken); status.DataDir = options.DataDir; status.DataDirSource = options.DataDirSource; diff --git a/src/CodeIndex/Database/DbReader.DatabaseSizeAttribution.cs b/src/CodeIndex/Database/DbReader.DatabaseSizeAttribution.cs new file mode 100644 index 000000000..a7dc7f7aa --- /dev/null +++ b/src/CodeIndex/Database/DbReader.DatabaseSizeAttribution.cs @@ -0,0 +1,523 @@ +using CodeIndex.Diagnostics; +using Microsoft.Data.Sqlite; + +namespace CodeIndex.Database; + +public partial class DbReader +{ + internal const int DatabaseSizeAttributionTopObjectLimit = 20; + internal const int DatabaseSizeAttributionObjectNameLimit = 128; + internal const long DatabaseSizeAttributionPageLimit = 1_000_000; + internal const long DatabaseSizeAttributionObjectLimit = 100_000; + + private StatusDatabaseSizeAttribution ReadDatabaseSizeAttribution( + StatusDbPragmaSettings pragmaSettings, + long? mainFileBytes, + long? walFileBytes, + long? shmFileBytes) + { + var physicalFileSetBytes = TryAddNonNegative(mainFileBytes, walFileBytes, shmFileBytes); + if (!TryMultiplyNonNegative( + pragmaSettings.PageCount, + pragmaSettings.PageSize, + out var logicalDatabaseBytes) + || !TryMultiplyNonNegative( + pragmaSettings.FreelistCount, + pragmaSettings.PageSize, + out var freelistBytes)) + { + return BuildUnavailableDatabaseSizeAttribution( + "pragma_values_unavailable", + pragmaSettings, + logicalDatabaseBytes: null, + mainFileBytes, + walFileBytes, + shmFileBytes, + physicalFileSetBytes, + freelistBytes: null); + } + if (pragmaSettings.PageCount > DatabaseSizeAttributionPageLimit) + { + return BuildUnavailableDatabaseSizeAttribution( + "page_attribution_limit_exceeded", + pragmaSettings, + logicalDatabaseBytes, + mainFileBytes, + walFileBytes, + shmFileBytes, + physicalFileSetBytes, + freelistBytes); + } + + try + { + _cancellation.ThrowIfCancellationRequested(); + using var command = _conn.CreateCommand(); + command.CommandText = """ + WITH object_pages AS ( + SELECT + d.name AS object_name, + CASE + WHEN d.name = 'sqlite_schema' THEN 'table' + WHEN s.type = 'table' THEN 'table' + WHEN s.type = 'index' THEN 'index' + ELSE 'other' + END AS object_type, + SUM(d.pgsize) AS page_bytes, + SUM(d.payload) AS payload_bytes, + SUM(d.unused) AS unused_bytes, + SUM(CASE WHEN d.pagetype = 'internal' THEN d.pgsize ELSE 0 END) AS internal_page_bytes, + SUM(CASE WHEN d.pagetype = 'leaf' THEN d.pgsize ELSE 0 END) AS leaf_page_bytes, + SUM(CASE WHEN d.pagetype = 'overflow' THEN d.pgsize ELSE 0 END) AS overflow_page_bytes, + SUM(CASE + WHEN d.pagetype IN ('internal', 'leaf', 'overflow') THEN 0 + ELSE d.pgsize + END) AS other_page_bytes + FROM dbstat AS d + LEFT JOIN sqlite_schema AS s + ON s.name = d.name + AND s.type IN ('table', 'index') + GROUP BY + d.name, + CASE + WHEN d.name = 'sqlite_schema' THEN 'table' + WHEN s.type = 'table' THEN 'table' + WHEN s.type = 'index' THEN 'index' + ELSE 'other' + END + ), + ranked AS ( + SELECT + object_name, + object_type, + page_bytes, + payload_bytes, + unused_bytes, + internal_page_bytes, + leaf_page_bytes, + overflow_page_bytes, + other_page_bytes, + COUNT(*) OVER () AS object_count, + SUM(page_bytes) OVER () AS allocated_object_bytes, + SUM(payload_bytes) OVER () AS total_payload_bytes, + SUM(unused_bytes) OVER () AS total_unused_bytes, + SUM(internal_page_bytes) OVER () AS total_internal_page_bytes, + SUM(leaf_page_bytes) OVER () AS total_leaf_page_bytes, + SUM(overflow_page_bytes) OVER () AS total_overflow_page_bytes, + SUM(other_page_bytes) OVER () AS total_other_page_bytes, + SUM(CASE WHEN object_type = 'table' THEN page_bytes ELSE 0 END) OVER () AS table_bytes, + SUM(CASE WHEN object_type = 'index' THEN page_bytes ELSE 0 END) OVER () AS index_bytes, + SUM(CASE WHEN object_type = 'other' THEN page_bytes ELSE 0 END) OVER () AS other_object_bytes + FROM object_pages + ) + SELECT + object_name, + object_type, + page_bytes, + payload_bytes, + unused_bytes, + internal_page_bytes, + leaf_page_bytes, + overflow_page_bytes, + other_page_bytes, + object_count, + allocated_object_bytes, + total_payload_bytes, + total_unused_bytes, + total_internal_page_bytes, + total_leaf_page_bytes, + total_overflow_page_bytes, + total_other_page_bytes, + table_bytes, + index_bytes, + other_object_bytes + FROM ranked + ORDER BY page_bytes DESC, object_name COLLATE BINARY + LIMIT $limit + """; + command.Parameters.AddWithValue("$limit", DatabaseSizeAttributionTopObjectLimit); + + using var cancellationRegistration = RegisterSqliteInterruptForCancellation(); + using var reader = command.ExecuteReader(); + var topObjects = new List(DatabaseSizeAttributionTopObjectLimit); + long objectCount = 0; + long allocatedObjectBytes = 0; + long payloadBytes = 0; + long unusedBytes = 0; + long internalPageBytes = 0; + long leafPageBytes = 0; + long overflowPageBytes = 0; + long otherPageBytes = 0; + long tableBytes = 0; + long indexBytes = 0; + long otherObjectBytes = 0; + var first = true; + + while (reader.Read()) + { + _cancellation.ThrowIfCancellationRequested(); + var rawName = reader.GetString(0); + var safeName = DiagnosticSanitizer.ForMessage( + rawName, + DatabaseSizeAttributionObjectNameLimit - 3); + topObjects.Add(new StatusDatabaseObjectSize + { + Name = safeName, + ObjectType = reader.GetString(1), + NameRedactedOrTruncated = !string.Equals(rawName, safeName, StringComparison.Ordinal), + PageBytes = reader.GetInt64(2), + PayloadBytes = reader.GetInt64(3), + UnusedBytes = reader.GetInt64(4), + }); + + if (!first) + continue; + + objectCount = reader.GetInt64(9); + allocatedObjectBytes = reader.GetInt64(10); + payloadBytes = reader.GetInt64(11); + unusedBytes = reader.GetInt64(12); + internalPageBytes = reader.GetInt64(13); + leafPageBytes = reader.GetInt64(14); + overflowPageBytes = reader.GetInt64(15); + otherPageBytes = reader.GetInt64(16); + tableBytes = reader.GetInt64(17); + indexBytes = reader.GetInt64(18); + otherObjectBytes = reader.GetInt64(19); + first = false; + } + + if (objectCount > DatabaseSizeAttributionObjectLimit) + { + return BuildUnavailableDatabaseSizeAttribution( + "page_attribution_limit_exceeded", + pragmaSettings, + logicalDatabaseBytes, + mainFileBytes, + walFileBytes, + shmFileBytes, + physicalFileSetBytes, + freelistBytes); + } + + if (!TrySubtractNonNegative( + allocatedObjectBytes, + payloadBytes, + unusedBytes, + out var structuralOverheadBytes) + || !TrySubtractNonNegative( + logicalDatabaseBytes, + allocatedObjectBytes, + freelistBytes, + out var unexplainedResidualBytes) + || !Reconciles( + allocatedObjectBytes, + tableBytes, + indexBytes, + otherObjectBytes) + || !Reconciles( + allocatedObjectBytes, + internalPageBytes, + leafPageBytes, + overflowPageBytes, + otherPageBytes)) + { + return BuildUnavailableDatabaseSizeAttribution( + "page_attribution_inconsistent", + pragmaSettings, + logicalDatabaseBytes, + mainFileBytes, + walFileBytes, + shmFileBytes, + physicalFileSetBytes, + freelistBytes); + } + + return new StatusDatabaseSizeAttribution + { + Available = true, + Measurement = "dbstat_page_bytes", + PageSizeBytes = pragmaSettings.PageSize, + PageCount = pragmaSettings.PageCount, + LogicalDatabaseBytes = logicalDatabaseBytes, + MainFileBytes = mainFileBytes, + WalFileBytes = walFileBytes, + ShmFileBytes = shmFileBytes, + PhysicalFileSetBytes = physicalFileSetBytes, + AllocatedObjectBytes = allocatedObjectBytes, + TableBytes = tableBytes, + IndexBytes = indexBytes, + OtherObjectBytes = otherObjectBytes, + InternalPageBytes = internalPageBytes, + LeafPageBytes = leafPageBytes, + OverflowPageBytes = overflowPageBytes, + OtherPageBytes = otherPageBytes, + PayloadBytes = payloadBytes, + UnusedBytes = unusedBytes, + StructuralOverheadBytes = structuralOverheadBytes, + FreelistBytes = freelistBytes, + UnexplainedResidualBytes = unexplainedResidualBytes, + ObjectCount = objectCount, + TopObjectLimit = DatabaseSizeAttributionTopObjectLimit, + TopObjectsTruncated = objectCount > topObjects.Count, + TopObjects = topObjects, + }; + } + catch (SqliteException exception) when (IsSqliteInterruptCancellation(exception)) + { + throw new OperationCanceledException( + "The SQLite database-attribution scan was interrupted by cancellation.", + exception, + _cancellation); + } + catch (SqliteException) + { + return ReadFilePageAttribution( + pragmaSettings, + logicalDatabaseBytes, + mainFileBytes, + walFileBytes, + shmFileBytes, + physicalFileSetBytes, + freelistBytes); + } + } + + private StatusDatabaseSizeAttribution ReadFilePageAttribution( + StatusDbPragmaSettings pragmaSettings, + long logicalDatabaseBytes, + long? mainFileBytes, + long? walFileBytes, + long? shmFileBytes, + long? physicalFileSetBytes, + long freelistBytes) + { + var databasePath = TryGetLocalDatabasePath(); + if (databasePath == null + || pragmaSettings.PageCount is not { } pageCount + || pragmaSettings.PageSize is not { } pageSize) + { + return BuildUnavailableDatabaseSizeAttribution( + "database_file_unavailable", + pragmaSettings, + logicalDatabaseBytes, + mainFileBytes, + walFileBytes, + shmFileBytes, + physicalFileSetBytes, + freelistBytes); + } + + try + { + var attribution = + string.Equals(pragmaSettings.JournalMode, "wal", StringComparison.OrdinalIgnoreCase) + && !_databaseFileSnapshotStable + ? SqlitePageAttributionReader.ReadConnectionSnapshot( + _conn, + pageCount, + pageSize, + _cancellation) + : SqlitePageAttributionReader.Read( + _conn, + databasePath, + pageCount, + pageSize, + _cancellation); + if (!TrySubtractNonNegative( + attribution.AllocatedObjectBytes, + attribution.PayloadBytes, + attribution.UnusedBytes, + out var structuralOverheadBytes) + || !TrySubtractNonNegative( + logicalDatabaseBytes, + attribution.AllocatedObjectBytes, + freelistBytes, + out var unexplainedResidualBytes) + || !Reconciles( + attribution.AllocatedObjectBytes, + attribution.TableBytes, + attribution.IndexBytes, + attribution.OtherObjectBytes) + || !Reconciles( + attribution.AllocatedObjectBytes, + attribution.InternalPageBytes, + attribution.LeafPageBytes, + attribution.OverflowPageBytes, + attribution.OtherPageBytes)) + { + return BuildUnavailableDatabaseSizeAttribution( + "page_attribution_inconsistent", + pragmaSettings, + logicalDatabaseBytes, + mainFileBytes, + walFileBytes, + shmFileBytes, + physicalFileSetBytes, + freelistBytes); + } + + return new StatusDatabaseSizeAttribution + { + Available = true, + Measurement = "sqlite_file_btree_pages", + PageSizeBytes = pageSize, + PageCount = pageCount, + LogicalDatabaseBytes = logicalDatabaseBytes, + MainFileBytes = mainFileBytes, + WalFileBytes = walFileBytes, + ShmFileBytes = shmFileBytes, + PhysicalFileSetBytes = physicalFileSetBytes, + AllocatedObjectBytes = attribution.AllocatedObjectBytes, + TableBytes = attribution.TableBytes, + IndexBytes = attribution.IndexBytes, + OtherObjectBytes = attribution.OtherObjectBytes, + InternalPageBytes = attribution.InternalPageBytes, + LeafPageBytes = attribution.LeafPageBytes, + OverflowPageBytes = attribution.OverflowPageBytes, + OtherPageBytes = attribution.OtherPageBytes, + PayloadBytes = attribution.PayloadBytes, + UnusedBytes = attribution.UnusedBytes, + StructuralOverheadBytes = structuralOverheadBytes, + FreelistBytes = freelistBytes, + UnexplainedResidualBytes = unexplainedResidualBytes, + ObjectCount = attribution.ObjectCount, + TopObjectLimit = DatabaseSizeAttributionTopObjectLimit, + TopObjectsTruncated = attribution.ObjectCount > attribution.TopObjects.Count, + TopObjects = attribution.TopObjects + .Select(item => new StatusDatabaseObjectSize + { + Name = item.Name, + ObjectType = item.ObjectType, + NameRedactedOrTruncated = item.NameRedactedOrTruncated, + PageBytes = item.PageBytes, + PayloadBytes = item.PayloadBytes, + UnusedBytes = item.UnusedBytes, + }) + .ToList(), + }; + } + catch (Exception ex) when (ex is + SqliteException + or IOException + or UnauthorizedAccessException + or InvalidDataException + or OverflowException + or ArgumentException + or NotSupportedException) + { + return BuildUnavailableDatabaseSizeAttribution( + "file_page_attribution_unavailable", + pragmaSettings, + logicalDatabaseBytes, + mainFileBytes, + walFileBytes, + shmFileBytes, + physicalFileSetBytes, + freelistBytes); + } + } + + internal static StatusDatabaseSizeAttribution BuildUnavailableDatabaseSizeAttribution( + string reason, + StatusDbPragmaSettings pragmaSettings, + long? logicalDatabaseBytes, + long? mainFileBytes, + long? walFileBytes, + long? shmFileBytes, + long? physicalFileSetBytes, + long? freelistBytes) + => new() + { + Available = false, + Measurement = "unavailable", + UnavailableReason = reason, + PageSizeBytes = pragmaSettings.PageSize, + PageCount = pragmaSettings.PageCount, + LogicalDatabaseBytes = logicalDatabaseBytes, + MainFileBytes = mainFileBytes, + WalFileBytes = walFileBytes, + ShmFileBytes = shmFileBytes, + PhysicalFileSetBytes = physicalFileSetBytes, + FreelistBytes = freelistBytes, + TopObjectLimit = DatabaseSizeAttributionTopObjectLimit, + }; + + private static bool TryMultiplyNonNegative(long? left, long? right, out long result) + { + result = 0; + if (left is null or < 0 || right is null or <= 0) + return false; + + try + { + result = checked(left.Value * right.Value); + return true; + } + catch (OverflowException) + { + return false; + } + } + + private static long? TryAddNonNegative(params long?[] values) + { + try + { + long total = 0; + foreach (var value in values) + { + if (value is null or < 0) + return null; + total = checked(total + value.Value); + } + + return total; + } + catch (OverflowException) + { + return null; + } + } + + private static bool TrySubtractNonNegative( + long total, + long first, + long second, + out long residual) + { + residual = 0; + if (total < 0 || first < 0 || second < 0) + return false; + + try + { + residual = checked(total - first - second); + return residual >= 0; + } + catch (OverflowException) + { + return false; + } + } + + private static bool Reconciles(long expected, params long[] values) + { + try + { + long actual = 0; + foreach (var value in values) + { + if (value < 0) + return false; + actual = checked(actual + value); + } + + return actual == expected; + } + catch (OverflowException) + { + return false; + } + } +} diff --git a/src/CodeIndex/Database/DbReader.FilesStatus.cs b/src/CodeIndex/Database/DbReader.FilesStatus.cs index 612a56a0e..7300ffddd 100644 --- a/src/CodeIndex/Database/DbReader.FilesStatus.cs +++ b/src/CodeIndex/Database/DbReader.FilesStatus.cs @@ -2027,31 +2027,45 @@ private long ExecuteScalar(string sql) private long? TryGetDatabaseFileSize() { - var path = _conn.DataSource; - if (string.IsNullOrWhiteSpace(path) || path.StartsWith("file:", StringComparison.OrdinalIgnoreCase)) + var path = TryGetLocalDatabasePath(); + if (path == null) return null; - try - { - var info = new FileInfo(path); - return info.Exists ? info.Length : null; - } - catch (Exception ex) when (ex is IOException or UnauthorizedAccessException or NotSupportedException or ArgumentException) - { - return null; - } + return TryGetFileSize(path, missingValue: null); } private long? TryGetWalFileSize() + => TryGetDatabaseSidecarFileSize("-wal"); + + private long? TryGetShmFileSize() + => TryGetDatabaseSidecarFileSize("-shm"); + + private long? TryGetDatabaseSidecarFileSize(string suffix) + { + var path = TryGetLocalDatabasePath(); + if (path == null) + return null; + + return TryGetFileSize(path + suffix, missingValue: 0); + } + + private string? TryGetLocalDatabasePath() { var path = _conn.DataSource; - if (string.IsNullOrWhiteSpace(path) || path.StartsWith("file:", StringComparison.OrdinalIgnoreCase)) + if (string.IsNullOrWhiteSpace(path)) return null; + return path.StartsWith("file:", StringComparison.OrdinalIgnoreCase) + ? DbConnectionFactory.TryGetLocalPath(path) + : path; + } + + private static long? TryGetFileSize(string path, long? missingValue) + { try { - var info = new FileInfo(path + "-wal"); - return info.Exists ? info.Length : 0; + var info = new FileInfo(path); + return info.Exists ? info.Length : missingValue; } catch (Exception ex) when (ex is IOException or UnauthorizedAccessException or NotSupportedException or ArgumentException) { diff --git a/src/CodeIndex/Database/DbReader.Status.cs b/src/CodeIndex/Database/DbReader.Status.cs index 1100cc036..75a1fb08b 100644 --- a/src/CodeIndex/Database/DbReader.Status.cs +++ b/src/CodeIndex/Database/DbReader.Status.cs @@ -22,6 +22,20 @@ public partial class DbReader /// データベースの統計情報を取得する。 /// public StatusResult GetStatus() + => GetStatus(includeDatabaseSizeAttribution: true); + + /// + /// Get database statistics while allowing internal non-status consumers to skip the + /// bounded page scan. + /// 内部の非 status 利用で件数上限付き page scan を省略可能にして database 統計を取得する。 + /// + /// + /// Whether to run the bounded page scan; internal consumers that do not emit the + /// attribution block can skip it. + /// 件数上限付き page scan を実行するかどうか。attribution block を出力しない内部利用では + /// scan を省略できる。 + /// + internal StatusResult GetStatus(bool includeDatabaseSizeAttribution) { // Issue #180: wrap the multi-statement status read in one DEFERRED transaction so // every COUNT(*) / freshness / readiness query resolves against the same WAL @@ -136,6 +150,19 @@ GROUP BY COALESCE(f.lang, 'unknown'), s.kind var preparedCommandCache = GetPreparedCommandCacheStatus(); var dbSizeBytes = TryGetDatabaseFileSize(); var walSizeBytes = TryGetWalFileSize(); + var databaseSizeAttribution = includeDatabaseSizeAttribution + ? ReadDatabaseSizeAttribution( + dbPragmaSettings, + dbSizeBytes, + walSizeBytes, + TryGetShmFileSize()) + : new StatusDatabaseSizeAttribution + { + Available = false, + Measurement = "unavailable", + UnavailableReason = "not_requested", + TopObjectLimit = DatabaseSizeAttributionTopObjectLimit, + }; var maintenanceGuidance = MaintenanceGuidanceBuilder.Build(new MaintenanceMetrics( dbPragmaSettings.PageCount, dbPragmaSettings.FreelistCount, @@ -209,6 +236,7 @@ GROUP BY COALESCE(f.lang, 'unknown'), s.kind MaintenanceGuidance = maintenanceGuidance, DbSizeBytes = dbSizeBytes, WalSizeBytes = walSizeBytes, + DatabaseSizeAttribution = databaseSizeAttribution, Process = StatusProcessMetrics.Capture(), LastIndexRun = lastIndexRun, LastFailedOrPartialIndexRun = lastFailedOrPartialIndexRun, diff --git a/src/CodeIndex/Database/DbReader.cs b/src/CodeIndex/Database/DbReader.cs index f60c69182..668894317 100644 --- a/src/CodeIndex/Database/DbReader.cs +++ b/src/CodeIndex/Database/DbReader.cs @@ -65,6 +65,7 @@ public partial class DbReader : IDisposable private readonly bool _readOnlyImmutableFallback; private readonly bool _immutableReadOnly; private readonly bool _immutableReadOnlyWalRisk; + private readonly bool _databaseFileSnapshotStable; private readonly bool _connectionPooling; private readonly string? _walCheckpointSkippedReason; private readonly string? _walCheckpointFailureReason; @@ -475,7 +476,9 @@ public DbReader(DbContext context) context.WalCheckpointCheckpointedPageCount, context.WalCheckpointRemainingPageCount, context.DatabasePermissionPolicyName, - context.DatabasePermissionDiagnostics) + context.DatabasePermissionDiagnostics, + context.QueryOnlySnapshotRequiresRefresh + || (context.ImmutableReadOnly && !context.ImmutableReadOnlyWalRisk)) { } @@ -505,7 +508,9 @@ public DbReader(DbContext context, CancellationToken cancellation) context.WalCheckpointCheckpointedPageCount, context.WalCheckpointRemainingPageCount, context.DatabasePermissionPolicyName, - context.DatabasePermissionDiagnostics) + context.DatabasePermissionDiagnostics, + context.QueryOnlySnapshotRequiresRefresh + || (context.ImmutableReadOnly && !context.ImmutableReadOnlyWalRisk)) { } @@ -551,7 +556,8 @@ private DbReader( long? walCheckpointCheckpointedPageCount = null, long? walCheckpointRemainingPageCount = null, string databasePermissionPolicy = DatabasePermissionPolicy.BestEffortName, - IReadOnlyList? databasePermissionDiagnostics = null) + IReadOnlyList? databasePermissionDiagnostics = null, + bool databaseFileSnapshotStable = false) { _conn = connection; _commandCache = commandCache; @@ -568,6 +574,7 @@ private DbReader( _readOnlyImmutableFallback = readOnlyImmutableFallback; _immutableReadOnly = immutableReadOnly; _immutableReadOnlyWalRisk = immutableReadOnlyWalRisk; + _databaseFileSnapshotStable = databaseFileSnapshotStable; _connectionPooling = connectionPooling; _walCheckpointSkippedReason = walCheckpointSkippedReason; _walCheckpointFailureReason = walCheckpointFailureReason; diff --git a/src/CodeIndex/Database/SqlitePageAttributionReader.cs b/src/CodeIndex/Database/SqlitePageAttributionReader.cs new file mode 100644 index 000000000..80b68e395 --- /dev/null +++ b/src/CodeIndex/Database/SqlitePageAttributionReader.cs @@ -0,0 +1,847 @@ +using System.Buffers.Binary; +using CodeIndex.Cli; +using CodeIndex.Diagnostics; +using Microsoft.Data.Sqlite; + +namespace CodeIndex.Database; + +internal static class SqlitePageAttributionReader +{ + private const long MaxWalFrameCount = 1_000_000; + + internal sealed record ObjectSize( + string Name, + string ObjectType, + bool NameRedactedOrTruncated, + long PageBytes, + long PayloadBytes, + long UnusedBytes); + + internal sealed record Result( + long ObjectCount, + long AllocatedObjectBytes, + long TableBytes, + long IndexBytes, + long OtherObjectBytes, + long InternalPageBytes, + long LeafPageBytes, + long OverflowPageBytes, + long OtherPageBytes, + long PayloadBytes, + long UnusedBytes, + IReadOnlyList TopObjects); + + internal static Result Read( + SqliteConnection connection, + string databasePath, + long pageCount, + long pageSize, + CancellationToken cancellationToken) + { + if (pageCount is < 0 or > DbReader.DatabaseSizeAttributionPageLimit) + throw new InvalidDataException("SQLite page count exceeds the bounded attribution scan."); + if (pageSize is < 512 or > 65536 || (pageSize & (pageSize - 1)) != 0) + throw new InvalidDataException("SQLite page size is invalid."); + + using var source = new PageSource( + databasePath, + pageCount, + checked((int)pageSize), + cancellationToken); + var visitedPages = new HashSet(); + var aggregate = new MutableObjectSize(); + var topObjects = new List(DbReader.DatabaseSizeAttributionTopObjectLimit); + long objectCount = 0; + + AddObject( + "sqlite_schema", + "table", + rootPage: 1, + source, + pageCount, + pageSize, + visitedPages, + aggregate, + topObjects, + ref objectCount, + cancellationToken); + + using var command = connection.CreateCommand(); + command.CommandText = """ + SELECT name, type, rootpage + FROM sqlite_schema + WHERE type IN ('table', 'index') + AND rootpage > 0 + ORDER BY name COLLATE BINARY + """; + using var reader = command.ExecuteReader(); + while (reader.Read()) + { + cancellationToken.ThrowIfCancellationRequested(); + AddObject( + reader.GetString(0), + reader.GetString(1), + reader.GetInt64(2), + source, + pageCount, + pageSize, + visitedPages, + aggregate, + topObjects, + ref objectCount, + cancellationToken); + } + + topObjects.Sort(CompareObjectSizes); + return new Result( + objectCount, + aggregate.PageBytes, + aggregate.TableBytes, + aggregate.IndexBytes, + aggregate.OtherObjectBytes, + aggregate.InternalPageBytes, + aggregate.LeafPageBytes, + aggregate.OverflowPageBytes, + aggregate.OtherPageBytes, + aggregate.PayloadBytes, + aggregate.UnusedBytes, + topObjects); + } + + internal static Result ReadConnectionSnapshot( + SqliteConnection connection, + long pageCount, + long pageSize, + CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + var snapshotDirectory = DataDirectorySecurity.CreateSensitiveTempDirectory( + "cdidx-attribution-snapshot-"); + var snapshotPath = Path.Combine(snapshotDirectory.FullName, "snapshot.db"); + try + { + using (var destination = new SqliteConnection( + new SqliteConnectionStringBuilder + { + DataSource = snapshotPath, + Mode = SqliteOpenMode.ReadWriteCreate, + Pooling = false, + }.ToString())) + { + destination.Open(); + DataDirectorySecurity.ApplyPrivateFileMode(snapshotPath); + CopyConnectionSnapshot( + connection, + destination, + cancellationToken); + } + + cancellationToken.ThrowIfCancellationRequested(); + return Read( + connection, + snapshotPath, + pageCount, + pageSize, + cancellationToken); + } + finally + { + TryDeleteConnectionSnapshot(snapshotDirectory.FullName, snapshotPath); + } + } + + private static void CopyConnectionSnapshot( + SqliteConnection source, + SqliteConnection destination, + CancellationToken cancellationToken) + { + using var cancellationRegistration = cancellationToken.CanBeCanceled + ? cancellationToken.UnsafeRegister( + static state => + { + var connections = ((SqliteConnection Source, SqliteConnection Destination))state!; + SQLitePCL.raw.sqlite3_interrupt(connections.Source.Handle); + SQLitePCL.raw.sqlite3_interrupt(connections.Destination.Handle); + }, + (source, destination)) + : default; + using var backup = SQLitePCL.raw.sqlite3_backup_init( + destination.Handle, + "main", + source.Handle, + "main"); + if (backup.IsInvalid) + throw new InvalidDataException("SQLite snapshot backup could not be initialized."); + + while (true) + { + cancellationToken.ThrowIfCancellationRequested(); + var result = SQLitePCL.raw.sqlite3_backup_step(backup, 256); + if (result == SQLitePCL.raw.SQLITE_DONE) + break; + if (result != SQLitePCL.raw.SQLITE_OK) + { + cancellationToken.ThrowIfCancellationRequested(); + throw new InvalidDataException( + $"SQLite snapshot backup failed with result code {result}."); + } + } + + var finishResult = SQLitePCL.raw.sqlite3_backup_finish(backup); + if (finishResult != SQLitePCL.raw.SQLITE_OK) + { + cancellationToken.ThrowIfCancellationRequested(); + throw new InvalidDataException( + $"SQLite snapshot backup finalization failed with result code {finishResult}."); + } + } + + private static void TryDeleteConnectionSnapshot( + string snapshotDirectory, + string snapshotPath) + { + try + { + foreach (var path in new[] + { + snapshotPath, + snapshotPath + "-journal", + snapshotPath + "-wal", + snapshotPath + "-shm", + }) + { + if (File.Exists(path)) + File.Delete(path); + } + + if (Directory.Exists(snapshotDirectory)) + Directory.Delete(snapshotDirectory); + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + GlobalToolLog.Error( + $"database_attribution_snapshot_cleanup_failed error={ex.GetType().Name}"); + } + } + + private static void AddObject( + string rawName, + string objectType, + long rootPage, + PageSource source, + long pageCount, + long pageSize, + HashSet visitedPages, + MutableObjectSize aggregate, + List topObjects, + ref long objectCount, + CancellationToken cancellationToken) + { + objectCount = checked(objectCount + 1); + if (objectCount > DbReader.DatabaseSizeAttributionObjectLimit) + throw new InvalidDataException("SQLite object count exceeds the bounded attribution scan."); + + var size = ReadObject( + rootPage, + source, + pageCount, + pageSize, + visitedPages, + cancellationToken); + aggregate.Add(size, objectType); + + var safeName = DiagnosticSanitizer.ForMessage( + rawName, + DbReader.DatabaseSizeAttributionObjectNameLimit - 3); + topObjects.Add(new ObjectSize( + safeName, + NormalizeObjectType(objectType), + !string.Equals(rawName, safeName, StringComparison.Ordinal), + size.PageBytes, + size.PayloadBytes, + size.UnusedBytes)); + topObjects.Sort(CompareObjectSizes); + if (topObjects.Count > DbReader.DatabaseSizeAttributionTopObjectLimit) + topObjects.RemoveAt(topObjects.Count - 1); + } + + private static MutableObjectSize ReadObject( + long rootPage, + PageSource source, + long pageCount, + long pageSize, + HashSet visitedPages, + CancellationToken cancellationToken) + { + var size = new MutableObjectSize(); + var stack = new Stack(); + var page = new byte[checked((int)pageSize)]; + var overflowPageBuffer = new byte[checked((int)pageSize)]; + stack.Push(rootPage); + + while (stack.Count > 0) + { + cancellationToken.ThrowIfCancellationRequested(); + var pageNumber = stack.Pop(); + AddVisitedPage(pageNumber, pageCount, visitedPages); + source.ReadPage(pageNumber, page); + + var headerOffset = pageNumber == 1 ? 100 : 0; + if (headerOffset + 12 > source.UsableSize) + throw new InvalidDataException("SQLite b-tree page header is outside the usable page."); + + var pageType = page[headerOffset]; + var interior = pageType is 2 or 5; + var leaf = pageType is 10 or 13; + if (!interior && !leaf) + throw new InvalidDataException("SQLite b-tree page type is invalid."); + + size.PageBytes = checked(size.PageBytes + pageSize); + if (interior) + size.InternalPageBytes = checked(size.InternalPageBytes + pageSize); + else + size.LeafPageBytes = checked(size.LeafPageBytes + pageSize); + + var headerSize = interior ? 12 : 8; + var cellCount = ReadUInt16(page, headerOffset + 3); + var cellPointerEnd = checked(headerOffset + headerSize + (cellCount * 2)); + if (cellPointerEnd > source.UsableSize) + throw new InvalidDataException("SQLite cell pointer array is outside the usable page."); + + var cellContentStart = ReadUInt16(page, headerOffset + 5); + if (cellContentStart == 0 && pageSize == 65536) + cellContentStart = 65536; + if (cellContentStart < cellPointerEnd || cellContentStart > source.UsableSize) + throw new InvalidDataException("SQLite cell content boundary is invalid."); + + var freeblockBytes = ReadFreeblockBytes( + page, + ReadUInt16(page, headerOffset + 1), + source.UsableSize); + var fragmentedBytes = page[headerOffset + 7]; + size.UnusedBytes = checked( + size.UnusedBytes + + (cellContentStart - cellPointerEnd) + + freeblockBytes + + fragmentedBytes); + + List? childPages = interior ? new List(cellCount + 1) : null; + + for (var cellIndex = 0; cellIndex < cellCount; cellIndex++) + { + var pointerOffset = checked(headerOffset + headerSize + (cellIndex * 2)); + var cellOffset = ReadUInt16(page, pointerOffset); + if (cellOffset < cellContentStart || cellOffset >= source.UsableSize) + throw new InvalidDataException("SQLite cell offset is invalid."); + + var cursor = cellOffset; + if (interior) + { + childPages!.Add(ReadPageNumber(page, cursor, pageCount)); + cursor = checked(cursor + 4); + } + + if (pageType == 5) + continue; + + var payloadBytes = ReadVarint(page, cursor, source.UsableSize, out var payloadVarintBytes); + cursor = checked(cursor + payloadVarintBytes); + if (pageType == 13) + { + _ = ReadVarint(page, cursor, source.UsableSize, out var rowIdVarintBytes); + cursor = checked(cursor + rowIdVarintBytes); + } + + if (payloadBytes > checked(pageCount * source.UsableSize)) + throw new InvalidDataException("SQLite cell payload exceeds the bounded database size."); + + size.PayloadBytes = checked(size.PayloadBytes + payloadBytes); + var localPayloadBytes = ComputeLocalPayloadBytes( + payloadBytes, + source.UsableSize, + tableLeaf: pageType == 13); + var overflowPointerOffset = checked(cursor + checked((int)localPayloadBytes)); + if (overflowPointerOffset > source.UsableSize) + throw new InvalidDataException("SQLite local payload exceeds the usable page."); + + if (payloadBytes <= localPayloadBytes) + continue; + if (overflowPointerOffset + 4 > source.UsableSize) + throw new InvalidDataException("SQLite overflow pointer is outside the usable page."); + + var overflowPage = ReadPageNumber(page, overflowPointerOffset, pageCount); + ReadOverflowChain( + overflowPage, + payloadBytes - localPayloadBytes, + source, + pageCount, + pageSize, + visitedPages, + size, + overflowPageBuffer, + cancellationToken); + } + + if (childPages != null) + { + childPages.Add(ReadPageNumber(page, headerOffset + 8, pageCount)); + for (var childIndex = childPages.Count - 1; childIndex >= 0; childIndex--) + stack.Push(childPages[childIndex]); + } + } + + return size; + } + + private static void ReadOverflowChain( + long firstPage, + long payloadBytes, + PageSource source, + long pageCount, + long pageSize, + HashSet visitedPages, + MutableObjectSize size, + byte[] page, + CancellationToken cancellationToken) + { + var pageNumber = firstPage; + var remainingPayloadBytes = payloadBytes; + var overflowPayloadCapacity = source.UsableSize - 4L; + if (overflowPayloadCapacity <= 0) + throw new InvalidDataException("SQLite overflow payload capacity is invalid."); + + while (remainingPayloadBytes > 0) + { + cancellationToken.ThrowIfCancellationRequested(); + AddVisitedPage(pageNumber, pageCount, visitedPages); + source.ReadPage(pageNumber, page); + size.PageBytes = checked(size.PageBytes + pageSize); + size.OverflowPageBytes = checked(size.OverflowPageBytes + pageSize); + + var usedBytes = Math.Min(remainingPayloadBytes, overflowPayloadCapacity); + size.UnusedBytes = checked( + size.UnusedBytes + + (overflowPayloadCapacity - usedBytes)); + remainingPayloadBytes -= usedBytes; + + var nextPage = BinaryPrimitives.ReadUInt32BigEndian(page.AsSpan(0, 4)); + if (remainingPayloadBytes == 0) + { + if (nextPage != 0) + throw new InvalidDataException("SQLite overflow chain exceeds the declared payload."); + break; + } + + if (nextPage == 0) + throw new InvalidDataException("SQLite overflow chain ended before the declared payload."); + pageNumber = nextPage; + } + } + + private static long ComputeLocalPayloadBytes(long payloadBytes, int usableSize, bool tableLeaf) + { + var maxLocal = tableLeaf + ? usableSize - 35L + : ((usableSize - 12L) * 64 / 255) - 23; + if (payloadBytes <= maxLocal) + return payloadBytes; + + var minLocal = ((usableSize - 12L) * 32 / 255) - 23; + var candidate = minLocal + ((payloadBytes - minLocal) % (usableSize - 4L)); + return candidate <= maxLocal ? candidate : minLocal; + } + + private static long ReadVarint( + byte[] page, + int offset, + int usableSize, + out int bytesRead) + { + ulong value = 0; + for (var index = 0; index < 9; index++) + { + var position = checked(offset + index); + if (position >= usableSize) + throw new InvalidDataException("SQLite varint exceeds the usable page."); + + var current = page[position]; + if (index == 8) + { + value = (value << 8) | current; + bytesRead = 9; + return value <= long.MaxValue + ? checked((long)value) + : throw new InvalidDataException("SQLite varint exceeds Int64."); + } + + value = (value << 7) | (uint)(current & 0x7f); + if ((current & 0x80) == 0) + { + bytesRead = index + 1; + return checked((long)value); + } + } + + throw new InvalidDataException("SQLite varint is invalid."); + } + + private static int ReadFreeblockBytes(byte[] page, int firstOffset, int usableSize) + { + var total = 0; + var offset = firstOffset; + var previousOffset = 0; + while (offset != 0) + { + if (offset <= previousOffset + || offset + 4 > usableSize) + { + throw new InvalidDataException("SQLite freeblock chain is invalid."); + } + + var nextOffset = ReadUInt16(page, offset); + var blockSize = ReadUInt16(page, offset + 2); + if (blockSize < 4 || offset + blockSize > usableSize) + throw new InvalidDataException("SQLite freeblock size is invalid."); + total = checked(total + blockSize); + previousOffset = offset; + offset = nextOffset; + } + + return total; + } + + private static int ReadUInt16(byte[] page, int offset) + { + if (offset < 0 || offset + 2 > page.Length) + throw new InvalidDataException("SQLite UInt16 field is outside the page."); + return BinaryPrimitives.ReadUInt16BigEndian(page.AsSpan(offset, 2)); + } + + private static long ReadPageNumber(byte[] page, int offset, long pageCount) + { + if (offset < 0 || offset + 4 > page.Length) + throw new InvalidDataException("SQLite page number is outside the page."); + var pageNumber = BinaryPrimitives.ReadUInt32BigEndian(page.AsSpan(offset, 4)); + if (pageNumber == 0 || pageNumber > pageCount) + throw new InvalidDataException("SQLite page number is outside the database."); + return pageNumber; + } + + private static void AddVisitedPage( + long pageNumber, + long pageCount, + HashSet visitedPages) + { + if (pageNumber <= 0 || pageNumber > pageCount) + throw new InvalidDataException("SQLite page number is outside the database."); + if (!visitedPages.Add(pageNumber)) + throw new InvalidDataException("SQLite page is attributed more than once."); + if (visitedPages.Count > DbReader.DatabaseSizeAttributionPageLimit) + throw new InvalidDataException("SQLite attribution scan exceeded its page limit."); + } + + private static int CompareObjectSizes(ObjectSize left, ObjectSize right) + { + var sizeComparison = right.PageBytes.CompareTo(left.PageBytes); + return sizeComparison != 0 + ? sizeComparison + : StringComparer.Ordinal.Compare(left.Name, right.Name); + } + + private static string NormalizeObjectType(string objectType) + => objectType switch + { + "table" => "table", + "index" => "index", + _ => "other", + }; + + private sealed class MutableObjectSize + { + public long PageBytes { get; set; } + public long PayloadBytes { get; set; } + public long UnusedBytes { get; set; } + public long InternalPageBytes { get; set; } + public long LeafPageBytes { get; set; } + public long OverflowPageBytes { get; set; } + public long OtherPageBytes { get; set; } + public long TableBytes { get; set; } + public long IndexBytes { get; set; } + public long OtherObjectBytes { get; set; } + + public void Add(MutableObjectSize value, string objectType) + { + PageBytes = checked(PageBytes + value.PageBytes); + PayloadBytes = checked(PayloadBytes + value.PayloadBytes); + UnusedBytes = checked(UnusedBytes + value.UnusedBytes); + InternalPageBytes = checked(InternalPageBytes + value.InternalPageBytes); + LeafPageBytes = checked(LeafPageBytes + value.LeafPageBytes); + OverflowPageBytes = checked(OverflowPageBytes + value.OverflowPageBytes); + OtherPageBytes = checked(OtherPageBytes + value.OtherPageBytes); + switch (NormalizeObjectType(objectType)) + { + case "table": + TableBytes = checked(TableBytes + value.PageBytes); + break; + case "index": + IndexBytes = checked(IndexBytes + value.PageBytes); + break; + default: + OtherObjectBytes = checked(OtherObjectBytes + value.PageBytes); + break; + } + } + } + + private sealed class PageSource : IDisposable + { + private const uint WalMagicLittleEndianChecksums = 0x377f0682; + private const uint WalMagicBigEndianChecksums = 0x377f0683; + private const int WalHeaderSize = 32; + private const int WalFrameHeaderSize = 24; + + private readonly FileStream _main; + private readonly FileStream? _wal; + private readonly Dictionary _walPageOffsets; + private readonly int _pageSize; + + public PageSource( + string databasePath, + long pageCount, + int pageSize, + CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + _pageSize = pageSize; + var main = OpenReadStream(databasePath); + FileStream? wal = null; + try + { + ValidateDatabaseHeader(main, pageSize); + var walOverlay = OpenWalOverlay( + databasePath, + pageCount, + pageSize, + cancellationToken); + wal = walOverlay.Stream; + var usableSize = ReadUsableSize(main, pageSize); + _main = main; + _wal = wal; + _walPageOffsets = walOverlay.PageOffsets; + UsableSize = usableSize; + } + catch + { + wal?.Dispose(); + main.Dispose(); + throw; + } + } + + public int UsableSize { get; } + + public void ReadPage(long pageNumber, byte[] destination) + { + if (destination.Length != _pageSize) + throw new ArgumentException("SQLite page buffer size does not match.", nameof(destination)); + + if (_wal != null && _walPageOffsets.TryGetValue(pageNumber, out var walOffset)) + { + ReadExactly(_wal, walOffset, destination); + return; + } + + ReadExactly(_main, checked((pageNumber - 1) * _pageSize), destination); + } + + public void Dispose() + { + _wal?.Dispose(); + _main.Dispose(); + } + + private static FileStream OpenReadStream(string path) + => new( + path, + FileMode.Open, + FileAccess.Read, + FileShare.ReadWrite | FileShare.Delete, + bufferSize: 64 * 1024, + FileOptions.RandomAccess); + + private static void ValidateDatabaseHeader(FileStream stream, int expectedPageSize) + { + var header = new byte[100]; + ReadExactly(stream, 0, header); + if (!header.AsSpan(0, 16).SequenceEqual("SQLite format 3\0"u8)) + throw new InvalidDataException("SQLite database header is invalid."); + + var encodedPageSize = BinaryPrimitives.ReadUInt16BigEndian(header.AsSpan(16, 2)); + var pageSize = encodedPageSize == 1 ? 65536 : encodedPageSize; + if (pageSize != expectedPageSize) + throw new InvalidDataException("SQLite file and PRAGMA page sizes differ."); + } + + private static int ReadUsableSize(FileStream stream, int pageSize) + { + var header = new byte[21]; + ReadExactly(stream, 0, header); + var reservedBytes = header[20]; + var usableSize = pageSize - reservedBytes; + return usableSize >= 480 + ? usableSize + : throw new InvalidDataException("SQLite usable page size is invalid."); + } + + private static (FileStream? Stream, Dictionary PageOffsets) OpenWalOverlay( + string databasePath, + long pageCount, + int pageSize, + CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + var walPath = databasePath + "-wal"; + if (!File.Exists(walPath)) + return (null, []); + + var wal = OpenReadStream(walPath); + try + { + if (wal.Length < WalHeaderSize) + { + wal.Dispose(); + return (null, []); + } + + var header = new byte[WalHeaderSize]; + ReadExactly(wal, 0, header); + var magic = BinaryPrimitives.ReadUInt32BigEndian(header.AsSpan(0, 4)); + if (magic is not WalMagicBigEndianChecksums and not WalMagicLittleEndianChecksums) + throw new InvalidDataException("SQLite WAL header magic is invalid."); + var checksumWordsAreLittleEndian = magic == WalMagicLittleEndianChecksums; + uint checksum1 = 0; + uint checksum2 = 0; + AccumulateWalChecksum( + header.AsSpan(0, 24), + checksumWordsAreLittleEndian, + ref checksum1, + ref checksum2); + if (checksum1 != BinaryPrimitives.ReadUInt32BigEndian(header.AsSpan(24, 4)) + || checksum2 != BinaryPrimitives.ReadUInt32BigEndian(header.AsSpan(28, 4))) + { + throw new InvalidDataException("SQLite WAL header checksum is invalid."); + } + + var walPageSize = BinaryPrimitives.ReadUInt32BigEndian(header.AsSpan(8, 4)); + if (walPageSize != pageSize) + throw new InvalidDataException("SQLite WAL and database page sizes differ."); + + var salt1 = BinaryPrimitives.ReadUInt32BigEndian(header.AsSpan(16, 4)); + var salt2 = BinaryPrimitives.ReadUInt32BigEndian(header.AsSpan(20, 4)); + var frameSize = checked(WalFrameHeaderSize + pageSize); + var frameCount = (wal.Length - WalHeaderSize) / frameSize; + if (frameCount > MaxWalFrameCount) + throw new InvalidDataException("SQLite WAL exceeds the bounded attribution scan."); + + var frameHeader = new byte[WalFrameHeaderSize]; + var framePage = new byte[pageSize]; + long lastCommitFrame = -1; + long lastCommitPageCount = -1; + for (long frame = 0; frame < frameCount; frame++) + { + cancellationToken.ThrowIfCancellationRequested(); + var frameOffset = checked(WalHeaderSize + (frame * frameSize)); + ReadExactly(wal, frameOffset, frameHeader); + var frameSalt1 = BinaryPrimitives.ReadUInt32BigEndian(frameHeader.AsSpan(8, 4)); + var frameSalt2 = BinaryPrimitives.ReadUInt32BigEndian(frameHeader.AsSpan(12, 4)); + if (frameSalt1 != salt1 || frameSalt2 != salt2) + break; + + ReadExactly(wal, checked(frameOffset + WalFrameHeaderSize), framePage); + AccumulateWalChecksum( + frameHeader.AsSpan(0, 8), + checksumWordsAreLittleEndian, + ref checksum1, + ref checksum2); + AccumulateWalChecksum( + framePage, + checksumWordsAreLittleEndian, + ref checksum1, + ref checksum2); + if (checksum1 != BinaryPrimitives.ReadUInt32BigEndian(frameHeader.AsSpan(16, 4)) + || checksum2 != BinaryPrimitives.ReadUInt32BigEndian(frameHeader.AsSpan(20, 4))) + { + break; + } + + var commitPageCount = BinaryPrimitives.ReadUInt32BigEndian(frameHeader.AsSpan(4, 4)); + if (commitPageCount > 0) + { + lastCommitFrame = frame; + lastCommitPageCount = commitPageCount; + } + } + + if (lastCommitFrame < 0) + { + wal.Dispose(); + return (null, []); + } + if (lastCommitPageCount != pageCount) + throw new InvalidDataException("SQLite WAL commit and PRAGMA page counts differ."); + + var pageOffsets = new Dictionary(); + for (long frame = 0; frame <= lastCommitFrame; frame++) + { + cancellationToken.ThrowIfCancellationRequested(); + var frameOffset = checked(WalHeaderSize + (frame * frameSize)); + ReadExactly(wal, frameOffset, frameHeader); + var pageNumber = BinaryPrimitives.ReadUInt32BigEndian(frameHeader.AsSpan(0, 4)); + if (pageNumber == 0) + throw new InvalidDataException("SQLite WAL frame page number is invalid."); + if (pageNumber > pageCount) + continue; + pageOffsets[pageNumber] = checked(frameOffset + WalFrameHeaderSize); + } + + return (wal, pageOffsets); + } + catch + { + wal.Dispose(); + throw; + } + } + + private static void AccumulateWalChecksum( + ReadOnlySpan bytes, + bool littleEndianWords, + ref uint checksum1, + ref uint checksum2) + { + if ((bytes.Length & 7) != 0) + throw new InvalidDataException("SQLite WAL checksum input is not word-paired."); + + for (var offset = 0; offset < bytes.Length; offset += 8) + { + var first = littleEndianWords + ? BinaryPrimitives.ReadUInt32LittleEndian(bytes.Slice(offset, 4)) + : BinaryPrimitives.ReadUInt32BigEndian(bytes.Slice(offset, 4)); + var second = littleEndianWords + ? BinaryPrimitives.ReadUInt32LittleEndian(bytes.Slice(offset + 4, 4)) + : BinaryPrimitives.ReadUInt32BigEndian(bytes.Slice(offset + 4, 4)); + unchecked + { + checksum1 += first + checksum2; + checksum2 += second + checksum1; + } + } + } + + private static void ReadExactly(FileStream stream, long offset, byte[] destination) + { + stream.Position = offset; + stream.ReadExactly(destination); + } + } +} diff --git a/src/CodeIndex/Mcp/McpToolHandlers.Languages.cs b/src/CodeIndex/Mcp/McpToolHandlers.Languages.cs index 56dd902de..652f4acc5 100644 --- a/src/CodeIndex/Mcp/McpToolHandlers.Languages.cs +++ b/src/CodeIndex/Mcp/McpToolHandlers.Languages.cs @@ -239,7 +239,7 @@ JsonNode BuildResponse(HashSet? indexedLanguages, string? workspaceRoot) return WithDbReader(id, args, reader => { - var status = reader.GetStatus(); + var status = reader.GetStatus(includeDatabaseSizeAttribution: false); var indexedLanguages = indexedOnly ? new HashSet(status.Languages.Keys, StringComparer.Ordinal) : null; diff --git a/src/CodeIndex/Mcp/McpToolHandlers.Query.Status.cs b/src/CodeIndex/Mcp/McpToolHandlers.Query.Status.cs index a796fad55..7be1a0bdc 100644 --- a/src/CodeIndex/Mcp/McpToolHandlers.Query.Status.cs +++ b/src/CodeIndex/Mcp/McpToolHandlers.Query.Status.cs @@ -40,7 +40,11 @@ private JsonNode ExecuteStatus(JsonNode? id, JsonNode? args) var response = WithDbReader(id, args, reader => { var requestToken = _currentRequestToken.Value; - var status = reader.GetStatus(); + var includeDatabaseSizeAttribution = + format == "full" + && (projectionFields == null + || projectionFields.Contains("database_size_attribution", StringComparer.Ordinal)); + var status = reader.GetStatus(includeDatabaseSizeAttribution); QueryCommandRunner.ApplyStatusSymbolKindLimits(status, reader.GetSymbolKindCounts()); WorkspaceMetadataEnricher.Enrich(status, _dbPath, _dbPathExplicit, requestToken); status.DbFileMode = DbContext.GetUnixFileModeString( diff --git a/src/CodeIndex/Models/QueryResults.cs b/src/CodeIndex/Models/QueryResults.cs index 93008960b..08a3c8a30 100644 --- a/src/CodeIndex/Models/QueryResults.cs +++ b/src/CodeIndex/Models/QueryResults.cs @@ -1104,6 +1104,111 @@ public sealed record GitExecutableStatus( [property: JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] bool? AncestorDirectoriesTrusted); +/// +/// Bounded per-object page attribution for SQLite status diagnostics. +/// SQLite status 診断向けの件数上限付きオブジェクト別ページ内訳。 +/// +public sealed class StatusDatabaseObjectSize +{ + public string Name { get; set; } = string.Empty; + [JsonPropertyName("object_type")] + public string ObjectType { get; set; } = string.Empty; + [JsonPropertyName("name_redacted_or_truncated")] + public bool NameRedactedOrTruncated { get; set; } + [JsonPropertyName("page_bytes")] + public long PageBytes { get; set; } + [JsonPropertyName("payload_bytes")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public long? PayloadBytes { get; set; } + [JsonPropertyName("unused_bytes")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public long? UnusedBytes { get; set; } +} + +/// +/// Read-only reconciliation of SQLite logical pages and physical database files. +/// SQLite の論理ページと物理 database file を読み取り専用で再照合した結果。 +/// +public sealed class StatusDatabaseSizeAttribution +{ + public bool Available { get; set; } + public string Measurement { get; set; } = "unavailable"; + [JsonPropertyName("unavailable_reason")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? UnavailableReason { get; set; } + [JsonPropertyName("page_size_bytes")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public long? PageSizeBytes { get; set; } + [JsonPropertyName("page_count")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public long? PageCount { get; set; } + [JsonPropertyName("logical_database_bytes")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public long? LogicalDatabaseBytes { get; set; } + [JsonPropertyName("main_file_bytes")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public long? MainFileBytes { get; set; } + [JsonPropertyName("wal_file_bytes")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public long? WalFileBytes { get; set; } + [JsonPropertyName("shm_file_bytes")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public long? ShmFileBytes { get; set; } + [JsonPropertyName("physical_file_set_bytes")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public long? PhysicalFileSetBytes { get; set; } + [JsonPropertyName("allocated_object_bytes")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public long? AllocatedObjectBytes { get; set; } + [JsonPropertyName("table_bytes")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public long? TableBytes { get; set; } + [JsonPropertyName("index_bytes")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public long? IndexBytes { get; set; } + [JsonPropertyName("other_object_bytes")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public long? OtherObjectBytes { get; set; } + [JsonPropertyName("internal_page_bytes")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public long? InternalPageBytes { get; set; } + [JsonPropertyName("leaf_page_bytes")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public long? LeafPageBytes { get; set; } + [JsonPropertyName("overflow_page_bytes")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public long? OverflowPageBytes { get; set; } + [JsonPropertyName("other_page_bytes")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public long? OtherPageBytes { get; set; } + [JsonPropertyName("payload_bytes")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public long? PayloadBytes { get; set; } + [JsonPropertyName("unused_bytes")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public long? UnusedBytes { get; set; } + [JsonPropertyName("structural_overhead_bytes")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public long? StructuralOverheadBytes { get; set; } + [JsonPropertyName("freelist_bytes")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public long? FreelistBytes { get; set; } + [JsonPropertyName("unexplained_residual_bytes")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public long? UnexplainedResidualBytes { get; set; } + [JsonPropertyName("object_count")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public long? ObjectCount { get; set; } + [JsonPropertyName("top_object_limit")] + public int TopObjectLimit { get; set; } + [JsonPropertyName("top_objects_truncated")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public bool? TopObjectsTruncated { get; set; } + [JsonPropertyName("top_objects")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public List? TopObjects { get; set; } +} + public class StatusResult { internal const string SqliteConnectionPolicyJsonFieldName = "sqlite_connection_policy"; @@ -1545,6 +1650,8 @@ public class StatusResult [JsonPropertyName("wal_size_bytes")] [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public long? WalSizeBytes { get; set; } + [JsonPropertyName("database_size_attribution")] + public StatusDatabaseSizeAttribution DatabaseSizeAttribution { get; set; } = new(); [JsonPropertyName("process")] public StatusProcessMetrics Process { get; set; } = StatusProcessMetrics.Capture(); [JsonPropertyName("last_index_run")] diff --git a/tests/CodeIndex.Tests/DbReaderTests.cs b/tests/CodeIndex.Tests/DbReaderTests.cs index 5a5501040..e66fea0f1 100644 --- a/tests/CodeIndex.Tests/DbReaderTests.cs +++ b/tests/CodeIndex.Tests/DbReaderTests.cs @@ -167,6 +167,392 @@ public void GetStatus_ExposesOperationalMetrics() Assert.False(status.LastIndexRun.DiagnosticsTruncated); } + [Fact] + public void GetStatus_AttributesDatabasePagesWithoutMutatingSource_Issue4888() + { + var rawObjectName = + "/Users/example/private/token=database-secret-" + + new string('x', 180); + using (var command = _db.Connection.CreateCommand()) + { + command.CommandText = + $"CREATE TABLE {SqliteIdentifier.Quote(rawObjectName)} (payload BLOB NOT NULL);" + + $"INSERT INTO {SqliteIdentifier.Quote(rawObjectName)}(payload) VALUES (zeroblob(1048576));"; + command.ExecuteNonQuery(); + } + + long totalChangesBefore; + long queryOnlyBefore; + using (var command = _db.Connection.CreateCommand()) + { + command.CommandText = "SELECT total_changes()"; + totalChangesBefore = (long)command.ExecuteScalar()!; + command.CommandText = "PRAGMA query_only"; + queryOnlyBefore = (long)command.ExecuteScalar()!; + } + + var attribution = _reader.GetStatus().DatabaseSizeAttribution; + + Assert.True(attribution.Available); + Assert.Contains( + attribution.Measurement, + ["dbstat_page_bytes", "sqlite_file_btree_pages"]); + Assert.NotNull(attribution.LogicalDatabaseBytes); + Assert.Equal( + attribution.LogicalDatabaseBytes, + attribution.AllocatedObjectBytes + + attribution.FreelistBytes + + attribution.UnexplainedResidualBytes); + Assert.Equal( + attribution.AllocatedObjectBytes, + attribution.TableBytes + + attribution.IndexBytes + + attribution.OtherObjectBytes); + Assert.Equal( + attribution.AllocatedObjectBytes, + attribution.InternalPageBytes + + attribution.LeafPageBytes + + attribution.OverflowPageBytes + + attribution.OtherPageBytes); + Assert.Equal( + attribution.AllocatedObjectBytes, + attribution.PayloadBytes + + attribution.UnusedBytes + + attribution.StructuralOverheadBytes); + Assert.True(attribution.OverflowPageBytes > 0); + Assert.NotNull(attribution.TopObjects); + Assert.InRange( + attribution.TopObjects!.Count, + 1, + DbReader.DatabaseSizeAttributionTopObjectLimit); + Assert.Equal( + attribution.ObjectCount > attribution.TopObjects.Count, + attribution.TopObjectsTruncated); + Assert.All( + attribution.TopObjects, + item => Assert.InRange( + item.Name.Length, + 1, + DbReader.DatabaseSizeAttributionObjectNameLimit)); + var redactedObject = Assert.Single( + attribution.TopObjects, + item => item.NameRedactedOrTruncated); + Assert.DoesNotContain("database-secret", redactedObject.Name, StringComparison.Ordinal); + Assert.DoesNotContain("/Users/example", redactedObject.Name, StringComparison.Ordinal); + + if (attribution.PhysicalFileSetBytes is { } physicalFileSetBytes) + { + Assert.Equal( + physicalFileSetBytes, + attribution.MainFileBytes + + attribution.WalFileBytes + + attribution.ShmFileBytes); + } + + using (var command = _db.Connection.CreateCommand()) + { + command.CommandText = "SELECT total_changes()"; + Assert.Equal(totalChangesBefore, (long)command.ExecuteScalar()!); + command.CommandText = "PRAGMA query_only"; + Assert.Equal(queryOnlyBefore, (long)command.ExecuteScalar()!); + } + } + + [Fact] + public void DatabaseSizeAttribution_UnavailableDoesNotReportZeroObjectSizes_Issue4888() + { + var attribution = DbReader.BuildUnavailableDatabaseSizeAttribution( + "dbstat_unavailable", + new StatusDbPragmaSettings + { + PageSize = 4096, + PageCount = 10, + FreelistCount = 2, + }, + logicalDatabaseBytes: 40960, + mainFileBytes: 40960, + walFileBytes: 0, + shmFileBytes: 0, + physicalFileSetBytes: 40960, + freelistBytes: 8192); + + Assert.False(attribution.Available); + Assert.Equal("unavailable", attribution.Measurement); + Assert.Equal("dbstat_unavailable", attribution.UnavailableReason); + Assert.Null(attribution.AllocatedObjectBytes); + Assert.Null(attribution.TableBytes); + Assert.Null(attribution.IndexBytes); + Assert.Null(attribution.UnexplainedResidualBytes); + Assert.Null(attribution.TopObjects); + } + + [Fact] + public void DatabaseSizeAttribution_CorruptDatabaseHeaderFailsClosed_Issue4888() + { + var corruptPath = Path.Combine(_dbDir, "corrupt-attribution.db"); + File.WriteAllBytes(corruptPath, new byte[4096]); + + Assert.Throws(() => SqlitePageAttributionReader.Read( + _db.Connection, + corruptPath, + pageCount: 1, + pageSize: 4096, + CancellationToken.None)); + } + + [Fact] + public void DatabaseSizeAttribution_EmptyDatabaseAttributesSchemaPage_Issue4888() + { + var emptyPath = Path.Combine(_dbDir, "empty-attribution.db"); + using var connection = new SqliteConnection($"Data Source={emptyPath}"); + connection.Open(); + using var command = connection.CreateCommand(); + command.CommandText = "CREATE TABLE scratch (value INTEGER); DROP TABLE scratch;"; + command.ExecuteNonQuery(); + command.CommandText = "PRAGMA page_count"; + var pageCount = (long)command.ExecuteScalar()!; + command.CommandText = "PRAGMA page_size"; + var pageSize = (long)command.ExecuteScalar()!; + + var attribution = SqlitePageAttributionReader.Read( + connection, + emptyPath, + pageCount, + pageSize, + CancellationToken.None); + + Assert.Equal(1, attribution.ObjectCount); + Assert.Equal(pageSize, attribution.AllocatedObjectBytes); + Assert.Equal(pageSize, attribution.TableBytes); + Assert.Equal(0, attribution.IndexBytes); + Assert.Equal(pageSize, attribution.LeafPageBytes); + Assert.InRange( + attribution.PayloadBytes + attribution.UnusedBytes, + 0, + attribution.AllocatedObjectBytes); + var schemaObject = Assert.Single(attribution.TopObjects); + Assert.Equal("sqlite_schema", schemaObject.Name); + } + + [Fact] + public void DatabaseSizeAttribution_RejectsPageCountsBeyondScanLimit_Issue4888() + { + Assert.Throws(() => SqlitePageAttributionReader.Read( + _db.Connection, + _dbPath, + pageCount: DbReader.DatabaseSizeAttributionPageLimit + 1, + pageSize: 4096, + CancellationToken.None)); + } + + [Fact] + public void GetStatus_CancelsDbstatAggregationWithoutFallingBack_Issue4888() + { + using var cancellation = new CancellationTokenSource(); + _db.Connection.CreateFunction( + "cancel_database_attribution", + () => + { + cancellation.Cancel(); + return 0L; + }); + using (var command = _db.Connection.CreateCommand()) + { + command.CommandText = """ + CREATE TEMP VIEW dbstat AS + WITH RECURSIVE pages(value) AS ( + VALUES(1) + UNION ALL + SELECT value + 1 + FROM pages + WHERE value < 1000000 + ) + SELECT + 'sqlite_schema' AS name, + '/' AS path, + value AS pageno, + 'leaf' AS pagetype, + 0 AS ncell, + 0 AS payload, + cancel_database_attribution() AS unused, + 0 AS mx_payload, + (value - 1) * 4096 AS pgoffset, + 4096 AS pgsize + FROM pages + """; + command.ExecuteNonQuery(); + } + using var reader = new DbReader(_db, cancellation.Token); + + var exception = Assert.Throws(() => reader.GetStatus()); + + var sqliteException = Assert.IsType(exception.InnerException); + Assert.Equal(9, sqliteException.SqliteErrorCode); + } + + [Fact] + public void DatabaseSizeAttribution_TruncatingWalSkipsObsoleteFramesAndObservesCancellation_Issue4888() + { + var walPath = Path.Combine(_dbDir, "truncating-wal-attribution.db"); + using var connection = new SqliteConnection($"Data Source={walPath}"); + connection.Open(); + using var command = connection.CreateCommand(); + command.CommandText = """ + PRAGMA page_size=4096; + PRAGMA auto_vacuum=INCREMENTAL; + VACUUM; + PRAGMA journal_mode=WAL; + PRAGMA wal_autocheckpoint=0; + CREATE TABLE scratch (payload BLOB NOT NULL); + BEGIN; + """; + command.ExecuteNonQuery(); + command.CommandText = "INSERT INTO scratch(payload) VALUES (zeroblob(3000))"; + for (var row = 0; row < 64; row++) + command.ExecuteNonQuery(); + command.CommandText = "COMMIT"; + command.ExecuteNonQuery(); + command.CommandText = "PRAGMA page_count"; + var peakPageCount = (long)command.ExecuteScalar()!; + command.CommandText = "DROP TABLE scratch; PRAGMA incremental_vacuum(1000000);"; + command.ExecuteNonQuery(); + command.CommandText = "PRAGMA page_count"; + var pageCount = (long)command.ExecuteScalar()!; + command.CommandText = "PRAGMA page_size"; + var pageSize = (long)command.ExecuteScalar()!; + + Assert.True(peakPageCount > pageCount); + Assert.True(new FileInfo(walPath + "-wal").Length > 0); + Assert.Throws(() => SqlitePageAttributionReader.Read( + connection, + walPath, + pageCount, + pageSize, + new CancellationToken(canceled: true))); + + var attribution = SqlitePageAttributionReader.Read( + connection, + walPath, + pageCount, + pageSize, + CancellationToken.None); + + Assert.Equal(1, pageCount); + Assert.Equal(1, attribution.ObjectCount); + Assert.Equal(pageSize, attribution.AllocatedObjectBytes); + Assert.Equal(pageSize, attribution.TableBytes); + } + + [Fact] + public void DatabaseSizeAttribution_ConnectionSnapshotIgnoresNewerWalCommits_Issue4888() + { + var walPath = Path.Combine(_dbDir, "connection-snapshot-attribution.db"); + var connectionString = new SqliteConnectionStringBuilder + { + DataSource = walPath, + Pooling = false, + }.ToString(); + using var writer = new SqliteConnection(connectionString); + writer.Open(); + using (var command = writer.CreateCommand()) + { + command.CommandText = """ + PRAGMA page_size=4096; + PRAGMA journal_mode=WAL; + PRAGMA wal_autocheckpoint=0; + CREATE TABLE scratch (payload BLOB NOT NULL); + BEGIN; + """; + command.ExecuteNonQuery(); + command.CommandText = + "INSERT INTO scratch(payload) VALUES (zeroblob(3000))"; + for (var row = 0; row < 200; row++) + command.ExecuteNonQuery(); + command.CommandText = "COMMIT; PRAGMA wal_checkpoint(TRUNCATE);"; + command.ExecuteNonQuery(); + } + + using var connection = new SqliteConnection(connectionString); + connection.Open(); + using var transaction = connection.BeginTransaction(deferred: true); + using var readerCommand = connection.CreateCommand(); + readerCommand.CommandText = "SELECT COUNT(*) FROM scratch"; + Assert.Equal(200, (long)readerCommand.ExecuteScalar()!); + readerCommand.CommandText = "PRAGMA page_count"; + var pageCount = (long)readerCommand.ExecuteScalar()!; + readerCommand.CommandText = "PRAGMA page_size"; + var pageSize = (long)readerCommand.ExecuteScalar()!; + + var before = SqlitePageAttributionReader.ReadConnectionSnapshot( + connection, + pageCount, + pageSize, + CancellationToken.None); + using (var command = writer.CreateCommand()) + { + command.CommandText = "DELETE FROM scratch"; + command.ExecuteNonQuery(); + } + var after = SqlitePageAttributionReader.ReadConnectionSnapshot( + connection, + pageCount, + pageSize, + CancellationToken.None); + + Assert.Equal( + new + { + before.ObjectCount, + before.AllocatedObjectBytes, + before.TableBytes, + before.IndexBytes, + before.OtherObjectBytes, + before.InternalPageBytes, + before.LeafPageBytes, + before.OverflowPageBytes, + before.OtherPageBytes, + before.PayloadBytes, + before.UnusedBytes, + }, + new + { + after.ObjectCount, + after.AllocatedObjectBytes, + after.TableBytes, + after.IndexBytes, + after.OtherObjectBytes, + after.InternalPageBytes, + after.LeafPageBytes, + after.OverflowPageBytes, + after.OtherPageBytes, + after.PayloadBytes, + after.UnusedBytes, + }); + Assert.Equal(before.TopObjects.ToArray(), after.TopObjects.ToArray()); + Assert.True(before.AllocatedObjectBytes > pageSize); + readerCommand.CommandText = "SELECT COUNT(*) FROM scratch"; + Assert.Equal(200, (long)readerCommand.ExecuteScalar()!); + Assert.Throws(() => + SqlitePageAttributionReader.ReadConnectionSnapshot( + connection, + pageCount, + pageSize, + new CancellationToken(canceled: true))); + } + + [Fact] + public void GetStatus_CanSkipDatabasePageScanForInternalConsumers_Issue4888() + { + var attribution = _reader.GetStatus( + includeDatabaseSizeAttribution: false).DatabaseSizeAttribution; + + Assert.False(attribution.Available); + Assert.Equal("unavailable", attribution.Measurement); + Assert.Equal("not_requested", attribution.UnavailableReason); + Assert.Null(attribution.AllocatedObjectBytes); + Assert.Null(attribution.TopObjects); + } + [Fact] public void GetStatus_NormalizesIndexedHeadTimestampOffsetForMachineJson_Issue4321() { diff --git a/tests/CodeIndex.Tests/DocumentationStatusContractTests.cs b/tests/CodeIndex.Tests/DocumentationStatusContractTests.cs index f87dbf71a..773be1a0f 100644 --- a/tests/CodeIndex.Tests/DocumentationStatusContractTests.cs +++ b/tests/CodeIndex.Tests/DocumentationStatusContractTests.cs @@ -48,6 +48,7 @@ public class DocumentationStatusContractTests "mac_profile", "db_size_bytes", "wal_size_bytes", + "database_size_attribution", "db_pragma_settings", "prepared_command_cache", "maintenance_guidance", diff --git a/tests/CodeIndex.Tests/JsonOutputSnapshotHelper.cs b/tests/CodeIndex.Tests/JsonOutputSnapshotHelper.cs index 9a73f23cf..6338b1c3f 100644 --- a/tests/CodeIndex.Tests/JsonOutputSnapshotHelper.cs +++ b/tests/CodeIndex.Tests/JsonOutputSnapshotHelper.cs @@ -64,8 +64,29 @@ internal static class JsonOutputSnapshotHelper private static readonly HashSet VolatileCountKeys = new(StringComparer.Ordinal) { "page_count", + "page_size_bytes", "db_size_bytes", "wal_size_bytes", + "logical_database_bytes", + "main_file_bytes", + "wal_file_bytes", + "shm_file_bytes", + "physical_file_set_bytes", + "allocated_object_bytes", + "table_bytes", + "index_bytes", + "other_object_bytes", + "internal_page_bytes", + "leaf_page_bytes", + "overflow_page_bytes", + "other_page_bytes", + "payload_bytes", + "unused_bytes", + "structural_overhead_bytes", + "freelist_bytes", + "unexplained_residual_bytes", + "object_count", + "page_bytes", "heap_bytes", "gc_heap_size_bytes", "gc_gen0_count", diff --git a/tests/CodeIndex.Tests/golden/status.json b/tests/CodeIndex.Tests/golden/status.json index 8fb4c3983..13c03744a 100644 --- a/tests/CodeIndex.Tests/golden/status.json +++ b/tests/CodeIndex.Tests/golden/status.json @@ -306,6 +306,195 @@ }, "db_size_bytes": "\u003CCOUNT\u003E", "wal_size_bytes": "\u003CCOUNT\u003E", + "database_size_attribution": { + "available": true, + "measurement": "sqlite_file_btree_pages", + "page_size_bytes": "\u003CCOUNT\u003E", + "page_count": "\u003CCOUNT\u003E", + "logical_database_bytes": "\u003CCOUNT\u003E", + "main_file_bytes": "\u003CCOUNT\u003E", + "wal_file_bytes": "\u003CCOUNT\u003E", + "shm_file_bytes": "\u003CCOUNT\u003E", + "physical_file_set_bytes": "\u003CCOUNT\u003E", + "allocated_object_bytes": "\u003CCOUNT\u003E", + "table_bytes": "\u003CCOUNT\u003E", + "index_bytes": "\u003CCOUNT\u003E", + "other_object_bytes": "\u003CCOUNT\u003E", + "internal_page_bytes": "\u003CCOUNT\u003E", + "leaf_page_bytes": "\u003CCOUNT\u003E", + "overflow_page_bytes": "\u003CCOUNT\u003E", + "other_page_bytes": "\u003CCOUNT\u003E", + "payload_bytes": "\u003CCOUNT\u003E", + "unused_bytes": "\u003CCOUNT\u003E", + "structural_overhead_bytes": "\u003CCOUNT\u003E", + "freelist_bytes": "\u003CCOUNT\u003E", + "unexplained_residual_bytes": "\u003CCOUNT\u003E", + "object_count": "\u003CCOUNT\u003E", + "top_object_limit": 20, + "top_objects_truncated": true, + "top_objects": [ + { + "name": "sqlite_schema", + "object_type": "table", + "name_redacted_or_truncated": false, + "page_bytes": "\u003CCOUNT\u003E", + "payload_bytes": "\u003CCOUNT\u003E", + "unused_bytes": "\u003CCOUNT\u003E" + }, + { + "name": "chunks", + "object_type": "table", + "name_redacted_or_truncated": false, + "page_bytes": "\u003CCOUNT\u003E", + "payload_bytes": "\u003CCOUNT\u003E", + "unused_bytes": "\u003CCOUNT\u003E" + }, + { + "name": "codeindex_meta", + "object_type": "table", + "name_redacted_or_truncated": false, + "page_bytes": "\u003CCOUNT\u003E", + "payload_bytes": "\u003CCOUNT\u003E", + "unused_bytes": "\u003CCOUNT\u003E" + }, + { + "name": "file_issues", + "object_type": "table", + "name_redacted_or_truncated": false, + "page_bytes": "\u003CCOUNT\u003E", + "payload_bytes": "\u003CCOUNT\u003E", + "unused_bytes": "\u003CCOUNT\u003E" + }, + { + "name": "files", + "object_type": "table", + "name_redacted_or_truncated": false, + "page_bytes": "\u003CCOUNT\u003E", + "payload_bytes": "\u003CCOUNT\u003E", + "unused_bytes": "\u003CCOUNT\u003E" + }, + { + "name": "fts_chunks_config", + "object_type": "table", + "name_redacted_or_truncated": false, + "page_bytes": "\u003CCOUNT\u003E", + "payload_bytes": "\u003CCOUNT\u003E", + "unused_bytes": "\u003CCOUNT\u003E" + }, + { + "name": "fts_chunks_data", + "object_type": "table", + "name_redacted_or_truncated": false, + "page_bytes": "\u003CCOUNT\u003E", + "payload_bytes": "\u003CCOUNT\u003E", + "unused_bytes": "\u003CCOUNT\u003E" + }, + { + "name": "fts_chunks_docsize", + "object_type": "table", + "name_redacted_or_truncated": false, + "page_bytes": "\u003CCOUNT\u003E", + "payload_bytes": "\u003CCOUNT\u003E", + "unused_bytes": "\u003CCOUNT\u003E" + }, + { + "name": "fts_chunks_idx", + "object_type": "table", + "name_redacted_or_truncated": false, + "page_bytes": "\u003CCOUNT\u003E", + "payload_bytes": "\u003CCOUNT\u003E", + "unused_bytes": "\u003CCOUNT\u003E" + }, + { + "name": "fts_chunks_trigram_config", + "object_type": "table", + "name_redacted_or_truncated": false, + "page_bytes": "\u003CCOUNT\u003E", + "payload_bytes": "\u003CCOUNT\u003E", + "unused_bytes": "\u003CCOUNT\u003E" + }, + { + "name": "fts_chunks_trigram_data", + "object_type": "table", + "name_redacted_or_truncated": false, + "page_bytes": "\u003CCOUNT\u003E", + "payload_bytes": "\u003CCOUNT\u003E", + "unused_bytes": "\u003CCOUNT\u003E" + }, + { + "name": "fts_chunks_trigram_docsize", + "object_type": "table", + "name_redacted_or_truncated": false, + "page_bytes": "\u003CCOUNT\u003E", + "payload_bytes": "\u003CCOUNT\u003E", + "unused_bytes": "\u003CCOUNT\u003E" + }, + { + "name": "fts_chunks_trigram_idx", + "object_type": "table", + "name_redacted_or_truncated": false, + "page_bytes": "\u003CCOUNT\u003E", + "payload_bytes": "\u003CCOUNT\u003E", + "unused_bytes": "\u003CCOUNT\u003E" + }, + { + "name": "hotspot_reference_counts", + "object_type": "table", + "name_redacted_or_truncated": false, + "page_bytes": "\u003CCOUNT\u003E", + "payload_bytes": "\u003CCOUNT\u003E", + "unused_bytes": "\u003CCOUNT\u003E" + }, + { + "name": "idx_chunks_file", + "object_type": "index", + "name_redacted_or_truncated": false, + "page_bytes": "\u003CCOUNT\u003E", + "payload_bytes": "\u003CCOUNT\u003E", + "unused_bytes": "\u003CCOUNT\u003E" + }, + { + "name": "idx_chunks_file_end_start_nonnull", + "object_type": "index", + "name_redacted_or_truncated": false, + "page_bytes": "\u003CCOUNT\u003E", + "payload_bytes": "\u003CCOUNT\u003E", + "unused_bytes": "\u003CCOUNT\u003E" + }, + { + "name": "idx_chunks_file_start_chunk_nonnull", + "object_type": "index", + "name_redacted_or_truncated": false, + "page_bytes": "\u003CCOUNT\u003E", + "payload_bytes": "\u003CCOUNT\u003E", + "unused_bytes": "\u003CCOUNT\u003E" + }, + { + "name": "idx_file_issues_file_kind", + "object_type": "index", + "name_redacted_or_truncated": false, + "page_bytes": "\u003CCOUNT\u003E", + "payload_bytes": "\u003CCOUNT\u003E", + "unused_bytes": "\u003CCOUNT\u003E" + }, + { + "name": "idx_files_checksum", + "object_type": "index", + "name_redacted_or_truncated": false, + "page_bytes": "\u003CCOUNT\u003E", + "payload_bytes": "\u003CCOUNT\u003E", + "unused_bytes": "\u003CCOUNT\u003E" + }, + { + "name": "idx_files_generated", + "object_type": "index", + "name_redacted_or_truncated": false, + "page_bytes": "\u003CCOUNT\u003E", + "payload_bytes": "\u003CCOUNT\u003E", + "unused_bytes": "\u003CCOUNT\u003E" + } + ] + }, "process": { "heap_bytes": "\u003CCOUNT\u003E", "gc_heap_size_bytes": "\u003CCOUNT\u003E",