diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index eb7e44263..80e8efbf7 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -1388,13 +1388,14 @@ For the AI agent search-rule template, see [AI Integration](USER_GUIDE.md#ai-int | Map scope, depth, and freshness | `map --depth ` applies path, language, test, generated-code, and exclusion filters before aggregating modules by the requested prefix depth. Scoped map output excludes the workspace-global decomposition plan. Workspace HEAD metadata is read in one query from the same SQLite snapshot as the map and remains explicit under `head_freshness`: `scope=workspace`, `indexed_head_source=latest_index` for the current successful index stamp (or `legacy_full_scan` only when it is the fallback), and `legacy_full_scan_head` for the separately labeled compatibility stamp. `issue-drafts` evaluates every scoped file for its thresholds, so `candidate_source=evaluated_scoped_candidates`, candidate counts, group totals, omitted counts, and `truncation.issue_draft_candidates` are candidate-based even though candidate details remain bounded; `truncation.largest_files` is a labeled compatibility alias only. | | `test-extractor` JSON | Machine-readable `test-extractor` success uses a versioned `{"api_version":"1","symbols":[...]}` envelope; the nested symbol objects retain their established property names. `--json` failures use the shared versioned command-error contract. | | Compact location envelope | CLI `--format compact` location output uses a versioned envelope with `api_version`, returned `count`, conservative limit-based `truncated` / `truncation` metadata, applied `query_context`, and lightweight `results` rows. | -| Bounded high-volume responses | `definition`, `find`, `status`, `hotspots`, `references`, `callers`, `callees`, `impact`, and `map` accept shared `--fields ` and opaque `--cursor ` controls. `find`, `status`, and the direct graph commands also auto-select this envelope for `--max-json-bytes`; location commands plus `status`, `hotspots`, `impact`, and `map` auto-select it for `--compact` / `--format compact`. Existing compact location output preserves its top-level keys and `file` / `line` rows while adding the shared metadata and count/cursor aliases; `refs` / `stats` aliases and bounded read-only batch children are canonicalized through the same wrapper. The response metadata reports `returned_count`, `total_count`, `total_count_authoritative`, `omitted_count`, `remaining_count`, `cursor_offset`, `page_limit`, `has_more`, `next_cursor`, `pagination_window_limit`, and `pagination_window_exhausted`. The safety window is 10,000 rows; exhaustion suppresses `next_cursor` rather than returning a cursor that the next request would reject. A cursor is tied to the command, query, filters, and sort arguments and is rejected if reused with a different selection. Pageable commands pass the cursor offset into their database/scan layer instead of serializing an `offset + limit` prefix. `hotspots` and `impact` page their active primary nested collection as `results`, identify it with `metadata.primary_collection`, and retain scalar/container evidence in `metadata.response_context`; dotted fields such as `callers.path,callers.depth` select that collection and project its rows. The final newline is included in `--max-json-bytes`, and trailing whole rows are removed until the complete envelope fits. `definition` remains metadata-only by default; explicit `--body` content is retained for `body`, `body_content`, or `all`, and suppressed when the projection excludes it. `map --sections` remains its section-level projection, while dotted bounded fields page a selected array section with section-specific totals and scalar projections skip unused ranked arrays. | +| Grouped search totals | `search --format grouped` derives `total_matches` / `matched_count`, `total_groups`, and `total_files` from the complete bounded query rather than the displayed page. `grouped_match_count` counts rows supplied to returned groups, `emitted_match_count` counts rows left after per-file grouping limits, and `omitted_match_count`, `truncated`, `has_more`, and `continuation_action` describe incomplete output. | +| Bounded high-volume responses | `search`, `definition`, `find`, `status`, `hotspots`, `references`, `callers`, `callees`, `symbols`, `files`, `languages`, `impact`, and `map` accept shared bounded-response controls where their schema exposes them. Newly emitted opaque `--cursor ` values bind the offset to the command/query/filter selection and index generation; legacy `response:v1::` cursors remain accepted for transition. Reuse with changed selection or generation fails with restart-required guidance. `search --format compact`, `symbols --format compact`, and `files --format compact` auto-select the bounded contract, while `search --json=array --json-envelope` provides the opt-in array envelope and `languages --json` selects it when paging or `--max-json-bytes` is requested. Existing compact roots and location rows remain compatible while adding shared metadata. Metadata reports `returned_count`, authoritative `total_count` where available, `omitted_count`, `remaining_count`, `cursor_offset`, `page_limit`, `has_more`, `next_cursor`, `result_stable_at`, `pagination_window_limit`, and `pagination_window_exhausted`. The safety window is 10,000 rows; exhaustion suppresses `next_cursor` rather than returning a cursor that the next request would reject. Pageable commands pass the cursor offset into their database/scan layer instead of serializing an `offset + limit` prefix. `find --all` partial scans encode the next path/line in the opaque cursor so replay continues after the last scanned line. `hotspots` and `impact` page their active primary nested collection as `results`, identify it with `metadata.primary_collection`, and retain scalar/container evidence in `metadata.response_context`; dotted fields such as `callers.path,callers.depth` select that collection and project its rows. The final newline is included in `--max-json-bytes`, and trailing whole rows are removed until the complete envelope fits. `definition` remains metadata-only by default; explicit `--body` content is retained for `body`, `body_content`, or `all`, and suppressed when the projection excludes it. `map --sections` remains its section-level projection, while dotted bounded fields page a selected array section with section-specific totals and scalar projections skip unused ranked arrays. | | Bounded response edge cases | `impact` applies the cursor offset only to the selected nested collection so definition pages do not repeat or alter caller/fallback mode. Plain `map --compact` preserves its established section arrays and truncation payload; a collection projection is rejected when `--summary-only` or an excluding `--sections` filter would remove it. Explicit definition body fields override compact defaults. Profile and verbose records are moved into `metadata.stream_control_records`, and parser/capture failures emit an error envelope only when it fits the active hard byte cap. | | `--count --json` envelope | Count-only JSON for `search`, `definition`, `references`, `callers`, `callees`, `symbols`, `files`, `find`, `impact`, and `unused` is a single automation-oriented object. It always includes `count`, applied `query_context`, freshness metadata (`indexed_file_count`, `indexed_at`, `freshness_available`), and trust flags `degraded` / `authoritative_count`; commands with matched-file totals also include `files` and the older `file_count` compatibility alias. `file_count` carries the same value as `files`, remains for compatibility, and is not scheduled for removal before the next major release. `unused --count --json` also includes `returned_bucket_counts`, `returned_contract_domain_counts`, and `summary.by_bucket` / `summary.by_confidence` / `summary.by_contract_domain`. `authoritative_count=false` means a readiness or graph/exact trust signal made the count non-authoritative, while the freshness fields describe the indexed snapshot used for the count. | | Recipe SARIF | `search --recipe --format sarif` emits one result per bounded recipe result. Rule IDs use `recipe/query`; standard `fingerprints.cdidx/v1` values are derived from the normalized source location; result properties preserve recipe/query identity, severity, confidence, and per-query truncation; run properties preserve scope, applied result limits, aggregate counts, and conservative omitted-result metadata. Bound SARIF with `--limit` / `--total-limit`; row selectors such as `--sample`, `--first-per-file`, and `--per-file-limit` are rejected instead of being silently ignored. Recipe severity maps `critical` / `high` to `error`, `medium` to `warning`, and `low` / `info` to `note`. | | Recipe classifier output | Recipe run JSON may add `audit_classifications` to individual `CompactSearchResult` rows when a recipe classifier can classify the hit, and query/count payloads may add `classifier_counts` when classified rows are present. These fields are additive; use them to separate triage domains such as DTO/result-wrapper `.Result` properties versus Task/ValueTask blocking waits without changing the raw search query. | | NDJSON terminal records | Default NDJSON for `search`, `symbols`, and `files` appends one final `terminal_record` after result rows; search also emits it for zero-result responses, while raw `symbols` and `files` keep zero-result NDJSON empty. Recipe/audit search row streams share the same writer. Terminals report returned and observed total counts, `total_count_authoritative` / `total_count_lower_bound`, selection or interruption reason, applied limits, omitted rows, and recovery guidance. `--max-json-bytes` covers the complete stdout stream, including newlines and this terminal record; a cap that cannot fit the terminal fails before stdout. Capped output rejects `--profile`, `--verbose`, and `--json-envelope`. Byte-cap partial output exits with `CommandExitCodes.PartialResult` (`11`) unless `--allow-partial` explicitly opts into exit `0`. `--results-only` is the explicit terminal-record opt-out for these NDJSON row streams and is rejected with array, compact, summary, or count output. | -| `outline --json` controls | `outline --json` accepts `--kind `, `--limit` / `--top`, `--cursor `, and `--outline-fields ` for bounded machine output. Controlled responses keep the normal outline envelope and add `total_symbol_count`, `returned_symbol_count`, `cursor_offset`, `next_cursor`, `has_more`, plus `kind_filter` and `selected_fields` when those controls are active. These additive fields do not require an API version bump. | +| `outline` / `unused` cursor binding | `outline --json` accepts `--kind `, `--limit` / `--top`, opaque `--cursor `, and `--outline-fields ` for bounded machine output. Controlled outline responses keep the normal envelope and add `total_symbol_count`, `returned_symbol_count`, `cursor_offset`, `next_cursor`, `has_more`, and `result_stable_at`, plus `kind_filter` and `selected_fields` when active. `outline` and `unused` cursors bind their offset to the normalized path/scope, filters, ordering, and index generation; reuse after changing those inputs or refreshing the index fails with explicit restart-required guidance. Legacy `outline:` / `unused:` inputs remain accepted for transition, but every newly emitted cursor is opaque and bound. | | `hotspots --json` grouping semantics | `hotspots` and MCP `symbol_hotspots` emit `grouped_by`, `grouping_unit`, `count_kind`, `limit_applies_to`, `score_fields`, `ranking_fields`, and matching `query_context` fields. `--limit` applies to returned symbols, files, name/kind groups, or SQL statements; `--count` ignores `--limit` and reports total groups. Explicit `statement` grouping is SQL-only (`--lang sql` / `lang: "sql"`). | | `--json-envelope` commands | Applies to `search`, `definition`, `references`, `callers`, `callees`, `symbols`, `files`, `find`, `excerpt`, `map`, `inspect`, `outline`, `status`, `validate`, `languages`, `impact`, `deps`, `unused`, and `hotspots`. | | `--json-envelope` shape | Wraps the per-line `--json` stream into a single `{"metadata": {...}, "results": [...]}` document. Stream terminal records are excluded from `results` and preserved as `metadata.stream_terminal`, while zero-result prelude/control records are preserved as `metadata.stream_control_records`; therefore `result_count` counts result rows only. A `find --all --count` object is both the count result and terminal scan metadata, so it remains in `results` and is also copied to `metadata.stream_terminal`. `metadata` also carries `api_version`, `command`, `cdidx_version`, `elapsed_ms`, `db_path`, `exit_code`, and, when applicable, `query_normalized` and `indexed_at_head_sha`. The bounded high-volume commands above support `--json-envelope --max-json-bytes` by measuring the final serialized document; other envelope/byte-cap combinations remain rejected. | @@ -4417,13 +4418,14 @@ AI エージェント向け検索ルールのテンプレートについては | map の scope、depth、freshness | `map --depth ` は path、language、test、generated-code、除外条件を適用してから、指定した prefix depth で module を集計します。scope を絞った map output からは workspace 全体向けの decomposition plan を除外します。workspace HEAD metadata は map と同じ SQLite snapshot から 1 query で読み、`head_freshness` に `scope=workspace`、現在の成功 index stamp なら `indexed_head_source=latest_index`（fallback の場合だけ `legacy_full_scan`）、互換用 stamp は別名の `legacy_full_scan_head` として明示します。`issue-drafts` は scope 内の全 file を閾値評価するため、`candidate_source=evaluated_scoped_candidates`、candidate 件数、group 合計、省略数、`truncation.issue_draft_candidates` は candidate 基準になります。`truncation.largest_files` は明示的な互換 alias としてのみ残します。 | | `test-extractor` JSON | 機械可読な `test-extractor` success は versioned `{"api_version":"1","symbols":[...]}` envelope を使い、内側の symbol object は既存の property 名を維持します。`--json` failure は共通の versioned command-error 契約を使います。 | | compact location envelope | CLI の `--format compact` location output は、`api_version`、返却 `count`、limit 到達を基準にした保守的な `truncated` / `truncation` metadata、適用済み `query_context`、軽量な `results` row を持つ versioned envelope です。 | -| 高ボリューム応答の bounded 契約 | `definition`、`find`、`status`、`hotspots`、`references`、`callers`、`callees`、`impact`、`map` は、共通の `--fields ` と opaque な `--cursor ` を受け付けます。`find`、`status`、直接 graph command は `--max-json-bytes` で、location command と `status`、`hotspots`、`impact`、`map` は `--compact` / `--format compact` でもこの envelope を自動選択します。既存の compact location 出力はトップレベル key と `file` / `line` row を維持し、共通 metadata と count / cursor alias を追加します。`refs` / `stats` alias と bounded な read-only batch 子 command も canonicalize して同じ wrapper を通します。response metadata は `returned_count`、`total_count`、`total_count_authoritative`、`omitted_count`、`remaining_count`、`cursor_offset`、`page_limit`、`has_more`、`next_cursor`、`pagination_window_limit`、`pagination_window_exhausted` を返します。safety window は 10,000 row で、上限到達時は次の request が拒否する cursor を返さず `next_cursor` を抑止します。cursor は command、query、filter、sort 引数に結び付けられ、異なる選択条件で再利用すると拒否されます。pageable command は `offset + limit` 件を serialize せず、cursor offset を database / scan layer へ渡します。`hotspots` と `impact` は active な主要 nested collection を `results` としてページングし、`metadata.primary_collection` でその名前を示し、scalar / container evidence は `metadata.response_context` に保持します。`callers.path,callers.depth` のような dotted field で collection と row field を同時に選べます。`--max-json-bytes` は最後の改行を含み、完全な envelope が収まるまで末尾の完全な row を省略します。`definition` は既定で metadata-only のままで、明示的な `--body` は `body`、`body_content`、`all` で保持し、それ以外の projection では materialize 前に抑止します。`map --sections` は section-level projection として残り、dotted な bounded field は選択した array section を section 固有の総件数付きでページングし、scalar projection は不要な ranking array を構築しません。 | +| grouped search の総数 | `search --format grouped` の `total_matches` / `matched_count`、`total_groups`、`total_files` は、表示 page ではなく上限適用前の query 全体から算出します。`grouped_match_count` は返却 group に渡した row 数、`emitted_match_count` は file ごとの上限適用後に残った row 数を表し、`omitted_match_count`、`truncated`、`has_more`、`continuation_action` が未完了出力を示します。 | +| 高ボリューム応答の bounded 契約 | `search`、`definition`、`find`、`status`、`hotspots`、`references`、`callers`、`callees`、`symbols`、`files`、`languages`、`impact`、`map` は、それぞれの schema が公開する共通 bounded-response control に対応します。新しく発行する opaque な `--cursor ` は offset を command / query / filter と index generation に束縛し、移行用に legacy の `response:v1::` も受理します。選択条件または generation を変えて再利用すると restart-required の案内付きで失敗します。`search --format compact`、`symbols --format compact`、`files --format compact` は bounded 契約を自動選択し、`search --json=array --json-envelope` は opt-in の array envelope、`languages --json` は paging または `--max-json-bytes` 指定時に同じ契約を使います。既存 compact の root と location row は維持したまま共通 metadata を追加します。metadata は `returned_count`、取得可能な場合は authoritative な `total_count`、`omitted_count`、`remaining_count`、`cursor_offset`、`page_limit`、`has_more`、`next_cursor`、`result_stable_at`、`pagination_window_limit`、`pagination_window_exhausted` を返します。safety window は 10,000 row で、上限到達時は次の request が拒否する cursor を返さず `next_cursor` を抑止します。pageable command は `offset + limit` 件を serialize せず、cursor offset を database / scan layer へ渡します。`find --all` の partial scan cursor は次の path / line を保持し、再利用時は最後に scan した line の次から継続します。`hotspots` と `impact` は active な主要 nested collection を `results` としてページングし、`metadata.primary_collection` でその名前を示し、scalar / container evidence は `metadata.response_context` に保持します。`callers.path,callers.depth` のような dotted field で collection と row field を同時に選べます。`--max-json-bytes` は最後の改行を含み、完全な envelope が収まるまで末尾の完全な row を省略します。`definition` は既定で metadata-only のままで、明示的な `--body` は `body`、`body_content`、`all` で保持し、それ以外の projection では materialize 前に抑止します。`map --sections` は section-level projection として残り、dotted な bounded field は選択した array section を section 固有の総件数付きでページングし、scalar projection は不要な ranking array を構築しません。 | | bounded 応答の edge case | `impact` は選択された nested collection だけに cursor offset を適用するため、definition page の重複や caller / fallback mode の変化を防ぎます。通常の `map --compact` は既存の section array と truncation payload を維持し、collection projection が `--summary-only` または除外する `--sections` filter で失われる組み合わせは拒否します。明示的な definition body field は compact default より優先します。profile / verbose record は `metadata.stream_control_records` へ移し、parser / capture failure の error envelope は active な hard byte cap に収まる場合だけ出力します。 | | `--count --json` envelope | `search`、`definition`、`references`、`callers`、`callees`、`symbols`、`files`、`find`、`impact`、`unused` の count-only JSON は単一の自動化向け object です。常に `count`、適用済み `query_context`、freshness metadata（`indexed_file_count`、`indexed_at`、`freshness_available`）、trust flag の `degraded` / `authoritative_count` を含みます。matched-file total を持つ command は `files` と古い互換 alias の `file_count` も含みます。`file_count` は `files` と同じ値を持つ互換用 field として残り、少なくとも次の major release までは削除予定はありません。`unused --count --json` は `returned_bucket_counts`、`returned_contract_domain_counts`、`summary.by_bucket` / `summary.by_confidence` / `summary.by_contract_domain` も含みます。`authoritative_count=false` は readiness または graph/exact trust signal により count が authoritative ではないことを示し、freshness field は count に使った index snapshot を説明します。 | | Recipe SARIF | `search --recipe --format sarif` は、上限付き recipe result ごとに result を1件出力します。rule ID は `recipe/query` を使い、標準の `fingerprints.cdidx/v1` は正規化済み source location から導出します。result properties は recipe/query identity、severity、confidence、query ごとの truncation を保持し、run properties は scope、適用済み result limit、集計 count、保守的な omitted-result metadata を保持します。SARIF の上限には `--limit` / `--total-limit` を使い、`--sample`、`--first-per-file`、`--per-file-limit` のような row selector は黙って無視せず拒否します。recipe severity は `critical` / `high` を `error`、`medium` を `warning`、`low` / `info` を `note` に対応付けます。 | | Recipe classifier output | recipe classifier が hit を分類できる場合、recipe run JSON は個別の `CompactSearchResult` row に `audit_classifications` を追加することがあり、分類済み row がある query / count payload は `classifier_counts` を追加することがあります。これらは additive field です。raw search query を変えずに、DTO / result-wrapper の `.Result` property と Task / ValueTask の blocking wait などの triage domain を分離するために使います。 | | NDJSON terminal record | `search`、`symbols`、`files` の既定 NDJSON は result row の後に最後の `terminal_record` を 1 件追加します。`search` は 0 件応答にも終端を出力しますが、raw `symbols` / `files` の 0 件 NDJSON は空のままです。recipe / audit search の row stream も同じ writer を使います。終端は返却件数と観測済み総件数、`total_count_authoritative` / `total_count_lower_bound`、selection または中断理由、適用上限、省略行数、復旧案内を報告します。`--max-json-bytes` は改行と終端レコードを含む stdout stream 全体を対象にし、終端自体が収まらない cap は stdout 出力前に失敗します。上限付き出力は `--profile`、`--verbose`、`--json-envelope` を拒否します。byte cap による部分出力は、`--allow-partial` で終了コード `0` を明示許可しない限り `CommandExitCodes.PartialResult`（`11`）を返します。`--results-only` はこれらの NDJSON row stream から終端レコードを明示的に除外するための option であり、array / compact / summary / count 出力との組み合わせは拒否されます。 | -| `outline --json` controls | `outline --json` は bounded な機械向け出力として `--kind `、`--limit` / `--top`、`--cursor `、`--outline-fields ` を受け付けます。制御付き応答は通常の outline envelope を維持し、`total_symbol_count`、`returned_symbol_count`、`cursor_offset`、`next_cursor`、`has_more` を追加します。該当する場合は `kind_filter` と `selected_fields` も返します。これらは additive field なので API version bump は不要です。 | +| `outline` / `unused` cursor の束縛 | `outline --json` は bounded な機械向け出力として `--kind `、`--limit` / `--top`、opaque な `--cursor `、`--outline-fields ` を受け付けます。制御付き outline 応答は通常の envelope を維持し、`total_symbol_count`、`returned_symbol_count`、`cursor_offset`、`next_cursor`、`has_more`、`result_stable_at` を追加し、該当時は `kind_filter` と `selected_fields` も返します。`outline` と `unused` の cursor は offset を正規化済み path/scope、filter、ordering、index generation に束縛するため、条件変更後または index 更新後の再利用は restart-required の明示案内付きで失敗します。移行用に legacy の `outline:` / `unused:` 入力は受理しますが、新しく出力する cursor はすべて opaque かつ束縛済みです。 | | `hotspots --json` grouping semantics | `hotspots` と MCP `symbol_hotspots` は `grouped_by`、`grouping_unit`、`count_kind`、`limit_applies_to`、`score_fields`、`ranking_fields` と、対応する `query_context` field を返します。`--limit` は返却される symbol、file、name/kind group、SQL statement に適用されます。`--count` は `--limit` を無視し、total group 数を返します。明示的な `statement` grouping は SQL 専用です（`--lang sql` / `lang: "sql"`）。 | | `--json-envelope` 対象 command | `search`、`definition`、`references`、`callers`、`callees`、`symbols`、`files`、`find`、`excerpt`、`map`、`inspect`、`outline`、`status`、`validate`、`languages`、`impact`、`deps`、`unused`、`hotspots`。 | | `--json-envelope` shape | per-line `--json` stream を単一の `{"metadata": {...}, "results": [...]}` document に包みます。stream 終端レコードは `results` から除外して `metadata.stream_terminal` に保持し、0 件時の prelude / control record は `metadata.stream_control_records` に保持するため、`result_count` は result row だけを数えます。`find --all --count` object は count result であると同時に終端 scan metadata でもあるため、`results` に残しつつ `metadata.stream_terminal` にも複製します。`metadata` は `api_version`、`command`、`cdidx_version`、`elapsed_ms`、`db_path`、`exit_code`、該当時は `query_normalized` と `indexed_at_head_sha` も持ちます。上記の bounded 高ボリューム command は最終 document を測定することで `--json-envelope --max-json-bytes` を許可し、それ以外の envelope / byte-cap 組み合わせは引き続き拒否します。 | diff --git a/TESTING_GUIDE.md b/TESTING_GUIDE.md index e163eb959..b33a0c5a1 100644 --- a/TESTING_GUIDE.md +++ b/TESTING_GUIDE.md @@ -176,6 +176,9 @@ Use `docs/test-doc-maintenance-plan.md` before moving oversized suites or adding CLI parsing, command execution, and installer behavior. Index command coverage is split by run mode or feature area, and query command coverage is split by command family with partial test classes so shared console and fixture helpers stay centralized. Keep repeated query-result fixtures, such as overlapping chunk content used by multiple search deduplication tests, in narrow class-level helpers instead of duplicating local builders. `ProgramCliTests.cs` covers top-level entrypoint behavior that must be exercised through a subprocess, while `InstallScriptTests.cs` runs focused bash snippets against `install.sh` in library mode to lock in release-installer regressions without performing real network installs. Installer bundle-generation tests must also verify that `install.sh` is marked generated while every canonical `install_modules/` source remains unmarked. `ConsoleUiTests.cs` keeps each generated shell's complete long-flag catalog aligned with `CliFlagSchema` and checks every Fish command scope against the shared per-command completion projection. Add flags to the schema rather than weakening this Bash / Zsh / Fish / PowerShell parity contract. `ProgramRunnerTests.cs` enumerates `CliCommandCatalog.CommandSubcommands` and requires every valid nested verb to resolve to verb-specific usage with an example. It also pins destructive index confirmation and aggregate dependency-filter help, plus the read-only GitHub duplicate-preflight boundary for suggestion exports. + Grouped-search limit coverage must seed more matches and files than the returned page, then assert query-wide matched/group/file totals separately from grouped, emitted, and omitted row counts so a bounded page cannot report itself as complete. + Cursor-pagination coverage for `outline` and `unused` must replay the emitted opaque token successfully, then prove that changed filters/order and a changed index-generation stamp fail with restart-required diagnostics. + Discovery-pagination coverage must page compact search/symbol/file output and the opt-in search-array/languages envelopes, assert authoritative total/omitted/stable-at metadata, reject query or generation cursor mismatches, and prove that a partial `find --all` scan resumes after the last scanned line. Quiet-flag coverage in `ProgramCliTests.cs` reuses one seeded symbols database across text, NDJSON, and JSON-array modes and compares stdout with and without a trailing quiet alias, proving that quiet mode changes only informational stderr. Doctor full-inventory coverage keeps composed filter selection, filtered summary counts, exact UTF-8 byte-budget boundaries, and structured overflow errors together in `ProgramRunnerTests`; license JSON remains a subprocess contract in `ProgramCliTests` so immediate-command dispatch and the published field names are both exercised. Ctags export JSON coverage reuses one seeded database for default and `--include-generated` variants, asserts the fixed skip-reason keys sum to `skipped_count`, and keeps the missing-`files.generated` degradation in a separate legacy-schema fixture. @@ -1021,6 +1024,9 @@ dotnet test --filter "FullyQualifiedName~GitHelperTests" CLI の引数解析、コマンド実行、installer 挙動のテスト。Index command coverage は run mode または機能領域ごとの partial suite に分割し、Query command coverage は command family ごとの partial test class に分割して、共有 console / fixture helper は一箇所に保ちます。`ProgramCliTests.cs` はグローバル引数の解釈や完全な CLI 起動フローのように subprocess 経由で確認すべき Program エントリポイント挙動を扱い、`InstallScriptTests.cs` は `install.sh` を library mode で source した bash snippet を実行して、実ネットワーク install を行わずに release installer の回帰を固定する。installer bundle 生成テストでは、`install.sh` が generated と判定される一方、canonical な `install_modules/` source はすべて unmarked のままであることも検証してください。 `ConsoleUiTests.cs` は各生成シェルの long flag 全カタログを `CliFlagSchema` と同期させ、Fish の全 command scope を共有の command 別 completion 射影と照合します。この Bash / Zsh / Fish / PowerShell parity 契約を弱めず、新しい flag は schema に追加してください。 `ProgramRunnerTests.cs` は `CliCommandCatalog.CommandSubcommands` を列挙し、すべての有効な nested verb が例を含む verb 固有 usage に解決されることを必須とします。さらに destructive な index confirmation、aggregate dependency filter help、suggestion export の read-only GitHub duplicate-preflight 境界を固定します。 + grouped search の limit coverage では、返却 page より多い match と file を seed し、query 全体の match/group/file 総数と grouped、emitted、omitted row 数を別々に検証して、上限付き page が完了済みと報告できないようにしてください。 + `outline` と `unused` の cursor pagination coverage では、出力された opaque token の再利用成功に加え、filter/order の変更と index-generation stamp の変更が restart-required 診断で失敗することを検証してください。 + discovery pagination の coverage では compact search / symbol / file と opt-in の search array / languages envelope をページングし、authoritative な total / omitted / stable-at metadata、query / generation 不一致 cursor の拒否、partial な `find --all` scan が最後に scan した line の次から再開することを検証してください。 `ProgramCliTests.cs` の quiet flag coverage は1つの seeded symbols database を text、NDJSON、JSON array の各 mode で再利用し、末尾に quiet alias を付けた場合と付けない場合の stdout を比較して、quiet mode が informational stderr だけを変えることを固定します。 doctor full-inventory coverage では、合成 filter の選択、filtered summary 件数、UTF-8 byte budget の exact boundary、structured overflow error を `ProgramRunnerTests` にまとめます。license JSON は `ProgramCliTests` の subprocess contract として、immediate-command dispatch と公開 field 名を同時に検証します。 ctags export JSON coverage は1つの seeded database を既定と `--include-generated` variant で再利用し、固定された skip-reason key の合計が `skipped_count` と一致することを検証します。`files.generated` がない場合の縮退は別の legacy-schema fixture に保ってください。 diff --git a/USER_GUIDE.md b/USER_GUIDE.md index 7e8033b00..e80fb46a7 100644 --- a/USER_GUIDE.md +++ b/USER_GUIDE.md @@ -355,6 +355,23 @@ batch with the same filters and snippet bounds. Named batches emit one grouped JSON document, and `--format compact` keeps the per-result `CompactSearchResult` snippet/highlight context instead of reducing rows to file/line pairs. +`search --format grouped` reports query-wide `total_matches`, `total_groups`, +and `total_files` separately from `grouped_match_count` and +`emitted_match_count`. When `--limit` or `--per-file-limit` omits rows, +`omitted_match_count`, `truncated`, `has_more`, and `continuation_action` +describe the bounded output instead of presenting the displayed page as complete. +High-volume discovery output is resumable without increasing a global limit. +`search --format compact`, `symbols --format compact`, and `files --format +compact` add authoritative totals, omitted/truncated state, `result_stable_at`, +and an opaque `next_cursor` while retaining their existing compact roots. +`search --json=array --json-envelope` provides the same metadata around an +array. `languages --json` accepts `--limit` / `--top`, `--cursor`, and +`--max-json-bytes`; those controls select a bounded envelope without changing +the ordinary unbounded JSON shape. Pass each `next_cursor` back to the same +command and filters. A cursor is bound to that selection and index generation, +so changed inputs or a refreshed index require restarting the pagination. +When a bounded `find --all` scan exits partially, its terminal record includes +`next_cursor`; replaying it resumes after the last scanned line. For AI-oriented bounded payloads, `map`, `inspect`, and `outline` accept `--compact`. It implies JSON output, caps list sections to 5 items by default (or the explicit `--limit` / `--top` value), and adds `compact`, @@ -384,6 +401,9 @@ as a signal to inspect the accompanying readiness or graph/exact trust fields. ```bash cdidx search authenticate --json # ndjson stream, one result per line cdidx search authenticate --json=array # single JSON array +cdidx search authenticate --json=array --json-envelope --limit 50 +cdidx symbols --format compact --limit 50 +cdidx languages --json --limit 20 --max-json-bytes 65536 cdidx inspect QueryCommandRunner --json --pretty cdidx map --compact # capped JSON with truncation metadata cdidx inspect Compute --outline-only # file/definition/nearby symbol summary @@ -624,6 +644,10 @@ downstream audit tooling. Count-only JSON includes `returned_bucket_counts` and `summary.by_bucket` / `summary.by_confidence`, matching the full JSON summary. Use `--compact` for audit summaries that keep counts, confidence buckets, taxonomy, and filter context without returning the full `symbols` array. +When `unused` returns `next_cursor`, pass that opaque value back unchanged. +The cursor is bound to the effective audit scope, filters, ordering, and index +generation. Changing those inputs or refreshing the index requires restarting +without `--cursor`; JSON pages also expose `result_stable_at`. Public APIs, framework entrypoints, DTOs, serialization contracts, generated hooks, test-only hooks, Markdown headings and fenced-code language markers, reflection, and configuration-based usage can be false positives and are @@ -1583,13 +1607,13 @@ By default, `callers` and `callees` return only executable call, construction, a cdidx outline src/CodeIndex/Cli/GitHelper.cs cdidx outline src/CodeIndex/Cli/GitHelper.cs --json cdidx outline src/CodeIndex/Cli/GitHelper.cs --json --kind function --limit 20 --outline-fields name,line,kind,signature -cdidx outline src/CodeIndex/Cli/GitHelper.cs --json --cursor outline:20 --limit 20 --outline-fields name,line,kind,signature +cdidx outline src/CodeIndex/Cli/GitHelper.cs --json --cursor "$NEXT_CURSOR" --limit 20 --outline-fields name,line,kind,signature cdidx outline src/CodeIndex/Cli/QueryCommandRunner.cs --compact --kind function --sort size --limit 10 ``` Shows all symbols in a single file ordered deterministically by line, start column when available, kind, and name, with signature, visibility, and container nesting. Lets AI agents understand file structure in one call instead of reading the whole file or chaining `symbols` + `definition`. -For large files, `outline --json` supports `--kind `, `--sort `, `--limit` / `--top`, `--cursor `, and `--outline-fields ` so automation can request only the symbol page and fields it needs. Use `--sort size` (alias `span`) or `--sort complexity` to jump to large bodies first, and combine it with `--compact` for bounded giant-file triage. Controlled JSON output includes `total_symbol_count`, `returned_symbol_count`, `cursor_offset`, `next_cursor`, and `has_more`; it also reports `sort`, `kind_filter`, and `selected_fields` when those controls are used. Pass `--outline-fields all` to keep the full symbol payload while still opting into the paging metadata, or select `reference_count`, `size_lines`, `complexity_score`, and `sort_mode` for compact ranking evidence. +For large files, `outline --json` supports `--kind `, `--sort `, `--limit` / `--top`, opaque `--cursor `, and `--outline-fields ` so automation can request only the symbol page and fields it needs. Use `--sort size` (alias `span`) or `--sort complexity` to jump to large bodies first, and combine it with `--compact` for bounded giant-file triage. Controlled JSON output includes `total_symbol_count`, `returned_symbol_count`, `cursor_offset`, `next_cursor`, `has_more`, and `result_stable_at`; it also reports `sort`, `kind_filter`, and `selected_fields` when those controls are used. The cursor is bound to the file path, filters, ordering, and index generation, so changing them or refreshing the index requires restarting without `--cursor`. Pass `--outline-fields all` to keep the full symbol payload while still opting into the paging metadata, or select `reference_count`, `size_lines`, `complexity_score`, and `sort_mode` for compact ranking evidence. ### Reconstruct a file excerpt @@ -3571,6 +3595,23 @@ output modifier は 1 つの contract として検証されます。`--json` と ad hoc な grouped batch を実行します。名前付き batch は 1 つの grouped JSON document を 出力し、`--format compact` でも各 result の `CompactSearchResult` snippet / highlight context を維持し、file/line だけの行には縮約しません。 +`search --format grouped` は query 全体の `total_matches`、`total_groups`、 +`total_files` と、表示対象の `grouped_match_count`、実際に出力した +`emitted_match_count` を分けて報告します。`--limit` または `--per-file-limit` で +row を省略した場合は、`omitted_match_count`、`truncated`、`has_more`、 +`continuation_action` が上限付き出力であることを示します。 +高ボリュームな discovery 出力は global limit を増やさず再開できます。 +`search --format compact`、`symbols --format compact`、`files --format compact` +は既存の compact root を維持したまま、authoritative な総数、省略 / truncation 状態、 +`result_stable_at`、opaque な `next_cursor` を追加します。 +`search --json=array --json-envelope` は array を同じ metadata で包みます。 +`languages --json` は `--limit` / `--top`、`--cursor`、`--max-json-bytes` を +受け付け、これらを指定した場合だけ bounded envelope を選択するため、通常の上限なし +JSON 形状は変わりません。`next_cursor` は同じ command と filter に渡してください。 +cursor はその選択条件と index generation に束縛されるため、入力変更後または index +更新後は pagination を最初からやり直す必要があります。上限に達した +`find --all` scan が partial exit した場合、terminal record の `next_cursor` を +再利用すると最後に scan した line の次から継続します。 AI 向けに上限付き payload が必要な場合、`map`、`inspect`、`outline` は `--compact` に対応しています。これは JSON 出力を暗黙に有効化し、list section を 既定 5 件（明示した `--limit` / `--top` があればその値）に cap し、 @@ -3598,6 +3639,9 @@ graph/exact trust field を確認してください。 ```bash cdidx search authenticate --json # ndjson stream、1 行 1 result cdidx search authenticate --json=array # 単一 JSON array +cdidx search authenticate --json=array --json-envelope --limit 50 +cdidx symbols --format compact --limit 50 +cdidx languages --json --limit 20 --max-json-bytes 65536 cdidx inspect QueryCommandRunner --json --pretty cdidx map --compact # truncation metadata 付きの cap 済み JSON cdidx inspect Compute --outline-only # ファイル・定義・近傍シンボルの概要 @@ -3816,8 +3860,11 @@ preset です。JSON 出力には `query_context` も含まれるため、audit bucket と confidence filter を直接確認できます。count-only JSON には `returned_bucket_counts` と `summary.by_bucket` / `summary.by_confidence` も含まれ、 full JSON summary と同じ bucket totals を返します。count、confidence bucket、taxonomy、 -filter context だけが必要な場合は `--compact` を使ってください。Public API、framework entrypoint、DTO、 -serialization contract、generated hook、test-only hook、Markdown heading と fenced-code の +filter context だけが必要な場合は `--compact` を使ってください。 +`unused` が `next_cursor` を返した場合は、その opaque 値を変更せず次の呼び出しへ渡してください。 +cursor は有効な audit scope、filter、ordering、index generation に束縛されます。条件を変更した場合や +index を更新した場合は `--cursor` なしで再開する必要があり、JSON page は `result_stable_at` も返します。 +Public API、framework entrypoint、DTO、serialization contract、generated hook、test-only hook、Markdown heading と fenced-code の language marker、reflection、config 経由の使用は false positive になりうるため、 低 confidence bucket に寄せられます。 C# の `nameof(...)`、`typeof(...)`、`GetMethod("Foo")` のような @@ -4774,13 +4821,13 @@ cdidx callees AddToGitExclude --exclude-tests cdidx outline src/CodeIndex/Cli/GitHelper.cs cdidx outline src/CodeIndex/Cli/GitHelper.cs --json cdidx outline src/CodeIndex/Cli/GitHelper.cs --json --kind function --limit 20 --outline-fields name,line,kind,signature -cdidx outline src/CodeIndex/Cli/GitHelper.cs --json --cursor outline:20 --limit 20 --outline-fields name,line,kind,signature +cdidx outline src/CodeIndex/Cli/GitHelper.cs --json --cursor "$NEXT_CURSOR" --limit 20 --outline-fields name,line,kind,signature cdidx outline src/CodeIndex/Cli/QueryCommandRunner.cs --compact --kind function --sort size --limit 10 ``` 1ファイル内の全シンボルを行、利用可能な場合は開始列、種別、名前の決定的な順序で、シグネチャ・可視性・コンテナ深さに応じたネスト付きで表示します。ファイル全体を読んだり `symbols` + `definition` をチェーンしたりする代わりに、1回でファイル構造を把握できます。 -大きなファイル向けに、`outline --json` は `--kind `、`--sort `、`--limit` / `--top`、`--cursor `、`--outline-fields ` に対応します。自動化側は必要なシンボルページとフィールドだけを取得できます。`--sort size`（`span` alias）や `--sort complexity` を使うと大きい本体を先に確認でき、`--compact` と組み合わせると巨大ファイル調査向けの上限付きペイロードになります。制御付き JSON 出力には `total_symbol_count`、`returned_symbol_count`、`cursor_offset`、`next_cursor`、`has_more` が入り、sort、kind、field を指定した場合は `sort`、`kind_filter`、`selected_fields` も返します。`--outline-fields all` を渡すと、シンボルペイロードはフルのままページングメタデータだけを追加できます。ランキング根拠だけが必要な場合は `reference_count`、`size_lines`、`complexity_score`、`sort_mode` を選択できます。 +大きなファイル向けに、`outline --json` は `--kind `、`--sort `、`--limit` / `--top`、opaque な `--cursor `、`--outline-fields ` に対応します。自動化側は必要なシンボルページとフィールドだけを取得できます。`--sort size`（`span` alias）や `--sort complexity` を使うと大きい本体を先に確認でき、`--compact` と組み合わせると巨大ファイル調査向けの上限付きペイロードになります。制御付き JSON 出力には `total_symbol_count`、`returned_symbol_count`、`cursor_offset`、`next_cursor`、`has_more`、`result_stable_at` が入り、sort、kind、field を指定した場合は `sort`、`kind_filter`、`selected_fields` も返します。cursor は file path、filter、ordering、index generation に束縛されるため、それらを変更した場合や index を更新した場合は `--cursor` なしで再開してください。`--outline-fields all` を渡すと、シンボルペイロードはフルのままページングメタデータだけを追加できます。ランキング根拠だけが必要な場合は `reference_count`、`size_lines`、`complexity_score`、`sort_mode` を選択できます。 ### ファイル抜粋を再構成する diff --git a/changelog.d/unreleased/4730.fixed.md b/changelog.d/unreleased/4730.fixed.md new file mode 100644 index 000000000..568c27358 --- /dev/null +++ b/changelog.d/unreleased/4730.fixed.md @@ -0,0 +1,26 @@ +--- +category: fixed +issues: + - 4730 +affected: + - src/CodeIndex/Cli/JsonEnvelopeWrapper.cs + - src/CodeIndex/Cli/JsonEnvelopeWrapper.Bounded.cs + - src/CodeIndex/Cli/QueryCommandRunner.Discovery.cs + - src/CodeIndex/Cli/QueryCommandRunner.Find.cs + - src/CodeIndex/Cli/QueryCommandRunner.Languages.cs + - src/CodeIndex/Cli/QueryCommandRunner.SearchResults.cs + - src/CodeIndex/Database/DbReader.FilesStatus.cs + - tests/CodeIndex.Tests/JsonEnvelopeWrapperIssue4730Tests.cs + - tests/CodeIndex.Tests/QueryCommandRunnerTests.cs + - DEVELOPER_GUIDE.md + - USER_GUIDE.md + - TESTING_GUIDE.md +--- + +## English + +- **High-volume discovery output is resumable (#4730)** — compact search/symbol/file output, the opt-in search array envelope, and bounded language inventory now report authoritative total/omitted/truncated/stable-at metadata with opaque query- and generation-bound cursors; partial `find --all` scans return a cursor that resumes after the last scanned line. + +## 日本語 + +- **高ボリュームな discovery 出力を再開できるようになりました (#4730)** — compact search/symbol/file、opt-in の search array envelope、上限付き language inventory は authoritative な total / omitted / truncated / stable-at metadata と query / generation に束縛された opaque cursor を返し、partial な `find --all` scan は最後に scan した line の次から再開する cursor を返します。 diff --git a/changelog.d/unreleased/4734.fixed.md b/changelog.d/unreleased/4734.fixed.md new file mode 100644 index 000000000..ccb86abb2 --- /dev/null +++ b/changelog.d/unreleased/4734.fixed.md @@ -0,0 +1,23 @@ +--- +category: fixed +issues: + - 4734 +affected: + - src/CodeIndex/Cli/QueryCommandRunner.Outline.cs + - src/CodeIndex/Cli/QueryCommandRunner.Unused.cs + - src/CodeIndex/Cli/QueryCommandRunner.SearchRecipes.cs + - src/CodeIndex/Database/DbReader.FilesStatus.cs + - tests/CodeIndex.Tests/QueryCommandRunnerInspectTests.cs + - tests/CodeIndex.Tests/QueryCommandRunnerSymbolTests.cs + - DEVELOPER_GUIDE.md + - USER_GUIDE.md + - TESTING_GUIDE.md +--- + +## English + +- **Outline and unused cursors are bound to their result snapshot (#4734)** — newly emitted opaque cursors now bind the offset to normalized scope, filters, ordering, and index generation, and stale or mismatched reuse returns explicit restart-required guidance. + +## 日本語 + +- **Outline と unused の cursor を結果 snapshot に束縛しました (#4734)** — 新しく出力する opaque cursor は offset を正規化済み scope、filter、ordering、index generation に束縛し、古い cursor や条件不一致の再利用には明示的な restart-required 案内を返します。 diff --git a/changelog.d/unreleased/4735.fixed.md b/changelog.d/unreleased/4735.fixed.md new file mode 100644 index 000000000..f2dc5efc4 --- /dev/null +++ b/changelog.d/unreleased/4735.fixed.md @@ -0,0 +1,21 @@ +--- +category: fixed +issues: + - 4735 +affected: + - src/CodeIndex/Cli/QueryCommandRunner.Search.cs + - src/CodeIndex/Cli/QueryCommandRunner.SearchResults.cs + - src/CodeIndex/Cli/JsonOutputContracts.cs + - tests/CodeIndex.Tests/QueryCommandRunnerSearchTests.cs + - DEVELOPER_GUIDE.md + - USER_GUIDE.md + - TESTING_GUIDE.md +--- + +## English + +- **Grouped search reports complete query totals (#4735)** — bounded grouped output now distinguishes query-wide matched/group/file totals from grouped, emitted, and omitted rows, and reports truncation plus continuation guidance whenever output limits hide matches. + +## 日本語 + +- **Grouped search が query 全体の総数を報告するようになりました (#4735)** — 上限付き grouped 出力は query 全体の match/group/file 総数と grouped、emitted、omitted row 数を区別し、出力上限で match が隠れる場合は truncation と継続方法を報告します。 diff --git a/src/CodeIndex/Cli/CliFlagSchema.cs b/src/CodeIndex/Cli/CliFlagSchema.cs index 03196dd46..3bc66a27a 100644 --- a/src/CodeIndex/Cli/CliFlagSchema.cs +++ b/src/CodeIndex/Cli/CliFlagSchema.cs @@ -93,7 +93,7 @@ public static bool HasAuthoritativeHelpOptions(string command) => private static readonly string[] LimitCapableCommands = [ "search", "definition", "goto", "references", "callers", "callees", "symbols", - "files", "find", "map", "inspect", "outline", "deps", "impact", "unused", "hotspots", "validate", "audit", + "files", "find", "map", "inspect", "outline", "deps", "impact", "unused", "hotspots", "validate", "audit", "languages", ]; private static readonly string[] LangCapableCommands = @@ -117,7 +117,7 @@ public static bool HasAuthoritativeHelpOptions(string command) => private static readonly string[] CountCommands = [ "search", "definition", "references", "callers", "callees", "symbols", - "files", "find", "impact", "unused", "hotspots", "audit", + "files", "find", "impact", "unused", "hotspots", "audit", "languages", ]; private static readonly string[] StrictNotFoundCommands = [ @@ -142,7 +142,8 @@ public static bool HasAuthoritativeHelpOptions(string command) => private static readonly string[] UnusedFilterCommands = ["unused"]; private static readonly string[] BoundedProjectionCommands = [ - "definition", "find", "status", "hotspots", "references", "callers", "callees", "impact", "map", + "search", "definition", "find", "status", "hotspots", "references", "callers", "callees", + "symbols", "files", "languages", "impact", "map", ]; private static readonly string[] CursorCommands = ["search", "outline", "unused", "deps", .. BoundedProjectionCommands]; private static readonly string[] AllResultCommands = ["goto", "find", "unused"]; @@ -330,7 +331,7 @@ private static IReadOnlyList BuildAll() new() { Name = "--duplicate-threshold", ValuePlaceholder = "", Description = "Issue-drafts: explicit duplicate-preflight minimum score from 0 to 1", PrimaryCommands = Set("search", "suggestions") }, new() { Name = "--issue-title", ValuePlaceholder = "", Description = "Search issue-drafts: override the title for an ad hoc search draft", PrimaryCommands = Set("search") }, new() { Name = "--issue-label", ValuePlaceholder = "<label>", Description = "Search issue-drafts: add a label hint; repeat or comma-separate values", PrimaryCommands = Set("search") }, - new() { Name = "--cursor", ValuePlaceholder = "<cursor>", Description = "Pagination cursor returned as next_cursor; bounded query responses use response:v1 cursors", PrimaryCommands = Set(CursorCommands) }, + new() { Name = "--cursor", ValuePlaceholder = "<cursor>", Description = "Opaque pagination cursor returned as next_cursor and bound to its query and index generation", PrimaryCommands = Set(CursorCommands) }, new() { Name = "--status", ValuePlaceholder = "<all|draft|submitted_pending_triage|open_in_upstream|resolved_in_upstream|wont_fix|duplicate|superseded|submitted|unsubmitted>", Description = "Suggestions: filter by suggestion status", PrimaryCommands = Set("suggestions") }, new() { Name = "--category", ValuePlaceholder = "<symbol_extraction|reference_extraction|search_ranking|language_support|output_format|crash_report|unexpected_error|other>", Description = "Suggestions: filter by category", PrimaryCommands = Set("suggestions") }, new() { Name = "--agent", ValuePlaceholder = "<agent>", Description = "Suggestions: filter by agent", PrimaryCommands = Set("suggestions") }, @@ -376,7 +377,7 @@ private static IReadOnlyList<CliFlag> BuildAll() new() { Name = "--env-domain", ValuePlaceholder = "<domain>", Description = "Doctor full environment inventory: filter by exact domain", PrimaryCommands = Set("doctor") }, new() { Name = "--env-category", ValuePlaceholder = "<category>", Description = "Doctor full environment inventory: filter by exact category", PrimaryCommands = Set("doctor") }, new() { Name = "--env-sensitivity", ValuePlaceholder = "<sensitivity>", Description = "Doctor full environment inventory: filter by exact sensitivity", PrimaryCommands = Set("doctor") }, - new() { Name = "--max-json-bytes", ValuePlaceholder = "<n>", Description = "Bound emitted JSON bytes; bounded high-volume responses truncate projected rows with paging metadata", PrimaryCommands = Set("search", "definition", "find", "status", "references", "callers", "callees", "excerpt", "inspect", "impact", "recipes", "audit", "map", "files", "symbols", "deps", "hotspots", "doctor") }, + new() { Name = "--max-json-bytes", ValuePlaceholder = "<n>", Description = "Bound emitted JSON bytes; bounded high-volume responses truncate projected rows with paging metadata", PrimaryCommands = Set("search", "definition", "find", "status", "references", "callers", "callees", "excerpt", "inspect", "impact", "recipes", "audit", "map", "files", "symbols", "deps", "hotspots", "languages", "doctor") }, new() { Name = "--next-steps", Description = "Search: print inspect/excerpt follow-up commands for top hits", PrimaryCommands = Set("search") }, new() { Name = "--exclude-comments", Description = "Search: suppress comment-only matches after origin classification", PrimaryCommands = Set("search") }, new() { Name = "--exclude-strings", Description = "Search: suppress string, regex, and help-text matches after origin classification", PrimaryCommands = Set("search") }, diff --git a/src/CodeIndex/Cli/ConsoleUi.cs b/src/CodeIndex/Cli/ConsoleUi.cs index c00412c2d..4ccd7e55c 100644 --- a/src/CodeIndex/Cli/ConsoleUi.cs +++ b/src/CodeIndex/Cli/ConsoleUi.cs @@ -87,20 +87,20 @@ private static readonly (string Command, string Usage)[] CommandUsageLines = ("recipes", "cdidx recipes [list] [--query <filter>] [--names|--summary-only] [--json] [--pretty] [--format <text|json|compact>] [--max-json-bytes <n>]"), ("recipes-list", "cdidx recipes list [--query <filter>] [--names|--summary-only] [--json] [--pretty] [--format <text|json|compact>] [--max-json-bytes <n>]"), ("audit", "cdidx audit <recipe|recipe/query> [search filters] [--json[=ndjson]] [--format <text|json|count|compact|issue-drafts>] [--summary-only] [--limit <n>] [--total-limit <n>] [--results-only] [--search-fields <csv>] [--per-file-limit <n>] [--max-json-bytes <n>] [--snippet-lines <n>]"), - ("definition", "cdidx definition <query>|--query <query>|-- <query> [--db <path>] [--json] [--format <text|json|count|compact|csv|tsv|lsp|qf|sarif>] [--fields <csv>] [--cursor <response:v1:offset:fingerprint>] [--max-json-bytes <n>] [--verbose] [--limit <n>|--top <n>] [--lang <lang>] [--kind <kind>] [--visibility <v[,v]>] [--exclude-visibility <v[,v]>] [--path <glob>] [--exclude-path <glob>] [--exclude-tests] [--body] [--exact|--exact-name] [--count] [--group-partials] [--since <datetime>]"), + ("definition", "cdidx definition <query>|--query <query>|-- <query> [--db <path>] [--json] [--format <text|json|count|compact|csv|tsv|lsp|qf|sarif>] [--fields <csv>] [--cursor <next_cursor>] [--max-json-bytes <n>] [--verbose] [--limit <n>|--top <n>] [--lang <lang>] [--kind <kind>] [--visibility <v[,v]>] [--exclude-visibility <v[,v]>] [--path <glob>] [--exclude-path <glob>] [--exclude-tests] [--body] [--exact|--exact-name] [--count] [--group-partials] [--since <datetime>]"), ("goto", "cdidx goto <query>|--query <query>|-- <query> [--db <path>] [--json] [--limit <n>|--top <n>] [--lang <lang>] [--kind <kind>] [--path <glob>] [--exclude-path <glob>] [--exclude-tests] [--exact|--exact-name] [--all]"), - ("references", "cdidx references <query>|--query <query>|-- <query> [--db <path>] [--json] [--format <text|json|count|compact|csv|tsv|lsp|qf|sarif>] [--fields <csv>] [--cursor <response:v1:offset:fingerprint>] [--max-json-bytes <n>] [--verbose] [--limit <n>|--top <n>] [--lang <lang>] [--kind <kind>] [--path <glob>] [--exclude-path <glob>] [--exclude-tests] [--body] [--snippet-lines <n>] [--max-line-width <n>] [--exact|--exact-name] [--count]"), - ("callers", "cdidx callers <query>|--query <query>|-- <query> [--db <path>] [--json] [--format <text|json|count|compact|csv|tsv|lsp|qf|sarif>] [--fields <csv>] [--cursor <response:v1:offset:fingerprint>] [--max-json-bytes <n>] [--verbose] [--limit <n>|--top <n>] [--lang <lang>] [--kind <kind>] [--rank-by <weighted|count|kind>] [--raw-kinds] [--path <glob>] [--exclude-path <glob>] [--exclude-tests] [--body] [--snippet-lines <n>] [--max-line-width <n>] [--exact|--exact-name] [--count]"), - ("callees", "cdidx callees <query>|--query <query>|-- <query> [--db <path>] [--json] [--format <text|json|count|compact|csv|tsv|lsp|qf|sarif>] [--fields <csv>] [--cursor <response:v1:offset:fingerprint>] [--max-json-bytes <n>] [--verbose] [--limit <n>|--top <n>] [--lang <lang>] [--kind <kind>] [--rank-by <weighted|count|kind>] [--raw-kinds] [--path <glob>] [--exclude-path <glob>] [--exclude-tests] [--body] [--snippet-lines <n>] [--max-line-width <n>] [--exact|--exact-name] [--count]"), - ("symbols", "cdidx symbols [query|--query <query>|-- <query>] [--name <name>] [--db <path>] [--json[=ndjson|array]] [--compact] [--format <text|json|count|compact|lsp|qf|sarif>] [--summary-only] [--max-json-bytes <n>] [--allow-partial] [--verbose] [--limit <n>|--top <n>] [--sort <hotspot|references|size|complexity|path>] [--lang <lang>] [--kind <kind>] [--visibility <v[,v]>] [--exclude-visibility <v[,v]>] [--path <glob>] [--exclude-path <glob>] [--exclude-tests] [--exact|--exact-name] [--count] [--group-partials] [--since <datetime>]"), - ("files", "cdidx files [query|<glob>|--query <query>|-- <query>] [--db <path>] [--json[=ndjson|array]] [--format <text|json|count|compact>] [--summary-only] [--max-json-bytes <n>] [--allow-partial] [--verbose] [--limit <n>|--top <n>] [--lang <lang>] [--path <glob>] [--exclude-path <glob>] [--exclude-tests] [--count] [--since <datetime>] [--bytes]"), - ("find", "cdidx find <query> (--path <glob>|--all) [--db <path>] [--json] [--format <text|json|count|compact|csv|tsv|lsp|qf|sarif>] [--fields <csv>] [--cursor <response:v1:offset:fingerprint>] [--max-json-bytes <n>] [--verbose] [--limit <n>|--top <n>] [--lang <lang>] [--exclude-path <glob>] [--exclude-tests] [--context <n>] [--before <n>] [--after <n>] [--snippet-lines <n>] [--focus-line <line>] [--focus-column <n>] [--max-line-width <n>] [--line-scan-limit <n>] [--allow-partial] [--exact] [--regex] [--count]"), + ("references", "cdidx references <query>|--query <query>|-- <query> [--db <path>] [--json] [--format <text|json|count|compact|csv|tsv|lsp|qf|sarif>] [--fields <csv>] [--cursor <next_cursor>] [--max-json-bytes <n>] [--verbose] [--limit <n>|--top <n>] [--lang <lang>] [--kind <kind>] [--path <glob>] [--exclude-path <glob>] [--exclude-tests] [--body] [--snippet-lines <n>] [--max-line-width <n>] [--exact|--exact-name] [--count]"), + ("callers", "cdidx callers <query>|--query <query>|-- <query> [--db <path>] [--json] [--format <text|json|count|compact|csv|tsv|lsp|qf|sarif>] [--fields <csv>] [--cursor <next_cursor>] [--max-json-bytes <n>] [--verbose] [--limit <n>|--top <n>] [--lang <lang>] [--kind <kind>] [--rank-by <weighted|count|kind>] [--raw-kinds] [--path <glob>] [--exclude-path <glob>] [--exclude-tests] [--body] [--snippet-lines <n>] [--max-line-width <n>] [--exact|--exact-name] [--count]"), + ("callees", "cdidx callees <query>|--query <query>|-- <query> [--db <path>] [--json] [--format <text|json|count|compact|csv|tsv|lsp|qf|sarif>] [--fields <csv>] [--cursor <next_cursor>] [--max-json-bytes <n>] [--verbose] [--limit <n>|--top <n>] [--lang <lang>] [--kind <kind>] [--rank-by <weighted|count|kind>] [--raw-kinds] [--path <glob>] [--exclude-path <glob>] [--exclude-tests] [--body] [--snippet-lines <n>] [--max-line-width <n>] [--exact|--exact-name] [--count]"), + ("symbols", "cdidx symbols [query|--query <query>|-- <query>] [--name <name>] [--db <path>] [--json[=ndjson|array]] [--compact] [--format <text|json|count|compact|lsp|qf|sarif>] [--summary-only] [--cursor <next_cursor>] [--max-json-bytes <n>] [--allow-partial] [--verbose] [--limit <n>|--top <n>] [--sort <hotspot|references|size|complexity|path>] [--lang <lang>] [--kind <kind>] [--visibility <v[,v]>] [--exclude-visibility <v[,v]>] [--path <glob>] [--exclude-path <glob>] [--exclude-tests] [--exact|--exact-name] [--count] [--group-partials] [--since <datetime>]"), + ("files", "cdidx files [query|<glob>|--query <query>|-- <query>] [--db <path>] [--json[=ndjson|array]] [--format <text|json|count|compact>] [--summary-only] [--cursor <next_cursor>] [--max-json-bytes <n>] [--allow-partial] [--verbose] [--limit <n>|--top <n>] [--lang <lang>] [--path <glob>] [--exclude-path <glob>] [--exclude-tests] [--count] [--since <datetime>] [--bytes]"), + ("find", "cdidx find <query> (--path <glob>|--all) [--db <path>] [--json] [--format <text|json|count|compact|csv|tsv|lsp|qf|sarif>] [--fields <csv>] [--cursor <next_cursor>] [--max-json-bytes <n>] [--verbose] [--limit <n>|--top <n>] [--lang <lang>] [--exclude-path <glob>] [--exclude-tests] [--context <n>] [--before <n>] [--after <n>] [--snippet-lines <n>] [--focus-line <line>] [--focus-column <n>] [--max-line-width <n>] [--line-scan-limit <n>] [--allow-partial] [--exact] [--regex] [--count]"), ("excerpt", "cdidx excerpt <path[:line|:start-end]> [--line <line>|--start <line>|--start-line <line>] [--end <line>|--end-line <line>] [--context <n>|--before <n>|--after <n>] [--max-line-width <n>] [--focus-line <line>] [--focus-column <n>] [--focus-length <n>] [--db <path>] [--json] [--no-semantic-tokens] [--max-json-bytes <n>] [--verbose]"), - ("map", "cdidx map [--db <path>] [--json] [--format <text|json|compact|issue-drafts>] [--pretty] [--compact] [--fields <csv>] [--cursor <response:v1:offset:fingerprint>] [--summary-only] [--verbose] [--limit <n>|--top <n>] [--lang <lang>] [--path <glob>] [--exclude-path <glob>] [--exclude-tests] [--bytes] [--sections <summary,tree,languages,hotspots,metrics|list>] [--depth <n>] [--min-entrypoint-confidence <0.0..1.0>] [--max-json-bytes <n>]"), + ("map", "cdidx map [--db <path>] [--json] [--format <text|json|compact|issue-drafts>] [--pretty] [--compact] [--fields <csv>] [--cursor <next_cursor>] [--summary-only] [--verbose] [--limit <n>|--top <n>] [--lang <lang>] [--path <glob>] [--exclude-path <glob>] [--exclude-tests] [--bytes] [--sections <summary,tree,languages,hotspots,metrics|list>] [--depth <n>] [--min-entrypoint-confidence <0.0..1.0>] [--max-json-bytes <n>]"), ("inspect", "cdidx inspect <query>|--query <query>|-- <query> [--db <path>] [--json] [--format <text|json|compact>] [--pretty] [--compact] [--fields <csv>] [--outline-only] [--body-only] [--max-json-bytes <n>] [--verbose] [--limit <n>|--top <n>] [--lang <lang>] [--kind <kind>] [--path <glob>] [--exclude-path <glob>] [--exclude-tests] [--body] [--body-start <line>] [--body-lines <n>|--body-line-count <n>] [--context <n>|--before <n>|--after <n>] [--max-line-width <n>] [--exact|--exact-name] [--group-partials]"), ("inspect", "cdidx inspect --path <file> --line <line> [--end-line <line>] [--db <path>] [--json] [--format <text|json|compact>] [--pretty] [--compact] [--fields <csv>] [--outline-only] [--body-only] [--max-json-bytes <n>] [--body] [--body-start <line>] [--body-lines <n>|--body-line-count <n>] [--context <n>|--before <n>|--after <n>] [--max-line-width <n>]"), - ("outline", "cdidx outline <path> [--db <path>] [--json] [--pretty] [--compact] [--verbose] [--limit <n>|--top <n>] [--cursor <outline:offset>] [--sort <source|kind|references|size|complexity|path|name>] [--kind <kind[,kind]>] [--outline-fields <csv>]"), - ("status", "cdidx status [--db <path>] [--json] [--format <text|json|compact>] [--compact] [--fields <csv>] [--cursor <response:v1:offset:fingerprint>] [--max-json-bytes <n>] [--verbose] [--check[=workspace,fold,graph,issues,hotspot,csharp,sql,newer]] [--stale-after <duration>] [--explain <field>] [--log-path] [--config] [--check-updates]"), + ("outline", "cdidx outline <path> [--db <path>] [--json] [--pretty] [--compact] [--verbose] [--limit <n>|--top <n>] [--cursor <next_cursor>] [--sort <source|kind|references|size|complexity|path|name>] [--kind <kind[,kind]>] [--outline-fields <csv>]"), + ("status", "cdidx status [--db <path>] [--json] [--format <text|json|compact>] [--compact] [--fields <csv>] [--cursor <next_cursor>] [--max-json-bytes <n>] [--verbose] [--check[=workspace,fold,graph,issues,hotspot,csharp,sql,newer]] [--stale-after <duration>] [--explain <field>] [--log-path] [--config] [--check-updates]"), ("workspace", "cdidx workspace <list|status|use|current|clear|deactivate> [name-or-relative-path] [--json]"), ("workspace-list", "cdidx workspace list [--json]"), ("workspace-status", "cdidx workspace status [--json]"), @@ -129,10 +129,10 @@ private static readonly (string Command, string Usage)[] CommandUsageLines = ("diff", "cdidx diff <db1> <db2> [--json] [--summary-only] [--detailed] [--limit <n<=10000>] [--offset <n>]"), ("report", "cdidx report --output <bundle.tgz> [--db <path>] [--json] [--redact-paths] [--log-lines <n<=2000>] [--no-log] [--include-args]"), ("validate", "cdidx validate [--db <path>] [--json[=array]] [--format <text|json|count|compact|csv|tsv|lsp|qf|sarif>] [--verbose] [--limit <n>|--top <n>] [--kind <kind>] [--severity <info|warning|error>] [--path <glob>]"), - ("impact", "cdidx impact <query>|--query <query>|-- <query> [--db <path>] [--json] [--format <text|json|compact>] [--compact] [--fields <csv>] [--cursor <response:v1:offset:fingerprint>] [--max-json-bytes <n>] [--verbose] [--limit <n>|--top <n>] [--lang <lang>] [--path <glob>] [--exclude-path <glob>] [--exclude-tests] [--body] [--snippet-lines <n>] [--max-line-width <n>] [--max-hops <n>] [--exact-name] [--count] [--with-paths]"), + ("impact", "cdidx impact <query>|--query <query>|-- <query> [--db <path>] [--json] [--format <text|json|compact>] [--compact] [--fields <csv>] [--cursor <next_cursor>] [--max-json-bytes <n>] [--verbose] [--limit <n>|--top <n>] [--lang <lang>] [--path <glob>] [--exclude-path <glob>] [--exclude-tests] [--body] [--snippet-lines <n>] [--max-line-width <n>] [--max-hops <n>] [--exact-name] [--count] [--with-paths]"), ("deps", "cdidx deps [--db <path>] [--json] [--format <dot|graphml|json-graph|edgelist>] [--summary-only] [--max-json-bytes <n>] [--verbose] [--limit <n>|--top <n>] [--cursor <cursor>] [--graph-budget <n>] [--lang <lang>] [--path <glob>] [--exclude-path <glob>] [--exclude-tests] [--reverse] [--cycles] [--suppress-noise] [--symbol <name>] [--symbol-family <prefix>]"), - ("unused", "cdidx unused [--db <path>] [--json] [--compact] [--summary-only] [--verbose] [--limit <n>|--top <n>] [--cursor <unused:offset>] [--audit-scope <source|all>] [--kind <kind>] [--bucket <bucket>] [--min-confidence <medium|low>|--confidence <medium|low>] [--actionable] [--all] [--visibility <v[,v]>] [--exclude-visibility <v[,v]>] [--lang <lang>] [--path <glob>] [--exclude-path <glob>] [--exclude-tests] [--count] [--by-bucket]"), - ("hotspots", "cdidx hotspots [--db <path>] [--json] [--format <text|json|count|compact>] [--compact] [--fields <csv>] [--cursor <response:v1:offset:fingerprint>] [--summary-only] [--max-json-bytes <n>] [--verbose] [--limit <n>|--top <n>] [--kind <kind>] [--visibility <v[,v]>] [--exclude-visibility <v[,v]>] [--lang <lang>] [--path <glob>] [--exclude-path <glob>] [--exclude-tests] [--count] [--group-by <symbol|file|statement>] [--group-by-name]"), + ("unused", "cdidx unused [--db <path>] [--json] [--compact] [--summary-only] [--verbose] [--limit <n>|--top <n>] [--cursor <next_cursor>] [--audit-scope <source|all>] [--kind <kind>] [--bucket <bucket>] [--min-confidence <medium|low>|--confidence <medium|low>] [--actionable] [--all] [--visibility <v[,v]>] [--exclude-visibility <v[,v]>] [--lang <lang>] [--path <glob>] [--exclude-path <glob>] [--exclude-tests] [--count] [--by-bucket]"), + ("hotspots", "cdidx hotspots [--db <path>] [--json] [--format <text|json|count|compact>] [--compact] [--fields <csv>] [--cursor <next_cursor>] [--summary-only] [--max-json-bytes <n>] [--verbose] [--limit <n>|--top <n>] [--kind <kind>] [--visibility <v[,v]>] [--exclude-visibility <v[,v]>] [--lang <lang>] [--path <glob>] [--exclude-path <glob>] [--exclude-tests] [--count] [--group-by <symbol|file|statement>] [--group-by-name]"), ("suggestions", "cdidx suggestions [list|show|export|add|update|delete] [id|description] [--db <path>] [--json] [--description <text>] [--context <text>] [--title <text>] [--evidence-path <path>] [--status <all|draft|submitted_pending_triage|open_in_upstream|resolved_in_upstream|wont_fix|duplicate|superseded|submitted|unsubmitted>] [--actor <name>] [--reason <text>] [--language <lang>] [--category <category>] [--since <datetime>] [--agent <name>] [--limit <n>] [--offset <n>] [--format <json|markdown|issue-drafts>] [--output <path>] [--overwrite] [--open-issues <path|github|github:owner/name>] [--repo <owner/name>] [--issue-state <open|closed|all>] [--duplicate-confidence <low|medium|high>|--duplicate-threshold <score>]"), ("suggestions-list", "cdidx suggestions list [--status <state>] [--language <lang>] [--category <category>] [--since <datetime>] [--agent <name>] [--limit <n>] [--offset <n>] [--db <path>] [--json]"), ("suggestions-show", "cdidx suggestions show <id> [--status <state>] [--language <lang>] [--category <category>] [--since <datetime>] [--agent <name>] [--db <path>] [--json]"), @@ -145,7 +145,7 @@ private static readonly (string Command, string Usage)[] CommandUsageLines = ("export", "cdidx export ctags [--output <path>] [--db <path>] [--json] [--lang <lang>] [--path <glob>] [--exclude-path <glob>] [--exclude-tests] [--include-generated]"), ("export-ctags", "cdidx export ctags [--output <path>] [--db <path>] [--json] [--lang <lang>] [--path <glob>] [--exclude-path <glob>] [--exclude-tests] [--include-generated]"), ("import", "cdidx import <archive> [--db <path>] [--prune-paths] [--dry-run|--check] [--limit <n<=10000>] [--offset <n>] [--json]"), - ("languages", "cdidx languages [--db <path>] [--json] [--format <text|json|count>] [--summary-only] [--indexed-only] [--language <lang>|--extension <ext>|--alias <alias>] [--capability <all|none|graph|references|symbols|missing-any|missing-graph|missing-references|missing-symbols|search-only>]"), + ("languages", "cdidx languages [--db <path>] [--json] [--format <text|json|count>] [--summary-only] [--limit <n>|--top <n>] [--cursor <next_cursor>] [--max-json-bytes <n>] [--indexed-only] [--language <lang>|--extension <ext>|--alias <alias>] [--capability <all|none|graph|references|symbols|missing-any|missing-graph|missing-references|missing-symbols|search-only>]"), ("batch", "cdidx batch [--db <path>] [--json-summary] [--max-input-lines <n>] [--max-output-chars <n>] [--parallel <n>] # stdin is JSON Lines; --json-summary embeds typed child JSON plus a final summary"), ("hooks-install", "cdidx hooks install [--project <path>] [--force] [--dry-run] [--json]"), ("hooks-uninstall", "cdidx hooks uninstall [--project <path>] [--force] [--json]"), diff --git a/src/CodeIndex/Cli/JsonEnvelopeWrapper.Bounded.cs b/src/CodeIndex/Cli/JsonEnvelopeWrapper.Bounded.cs index d1e368790..fe25d45ad 100644 --- a/src/CodeIndex/Cli/JsonEnvelopeWrapper.Bounded.cs +++ b/src/CodeIndex/Cli/JsonEnvelopeWrapper.Bounded.cs @@ -3,6 +3,7 @@ using System.Text; using System.Text.Json; using System.Text.Json.Nodes; +using CodeIndex.Database; namespace CodeIndex.Cli; @@ -18,41 +19,47 @@ internal static partial class JsonEnvelopeWrapper { private const int DefaultPageLimit = 20; private const int MaxPageWindow = MaxRawJsonItems; - private const string ResponseCursorPrefix = "response:v1:"; + private const string LegacyResponseCursorPrefix = "response:v1:"; + private const string ResponseCursorPrefix = "response:v2:"; private static readonly AsyncLocal<BoundedExecutionContext?> BoundedExecution = new(); private static readonly HashSet<string> BoundedResponseCommands = new(StringComparer.Ordinal) { - "definition", "find", "status", "hotspots", "references", "callers", "callees", "impact", "map", + "search", "definition", "find", "status", "hotspots", "references", "callers", "callees", + "symbols", "files", "languages", "impact", "map", }; private static readonly HashSet<string> AutoWrapByteBudgetCommands = new(StringComparer.Ordinal) { - "find", "status", "references", "callers", "callees", + "find", "status", "references", "callers", "callees", "languages", }; private static readonly HashSet<string> AutoWrapCompactCommands = new(StringComparer.Ordinal) { - "definition", "find", "status", "hotspots", "references", "callers", "callees", "impact", "map", + "search", "definition", "find", "status", "hotspots", "references", "callers", "callees", + "symbols", "files", "impact", "map", }; private static readonly HashSet<string> LegacyLocationCompactCommands = new(StringComparer.Ordinal) { - "definition", "find", "references", "callers", "callees", + "search", "definition", "find", "references", "callers", "callees", "symbols", "files", }; private static readonly HashSet<string> PageableResponseCommands = new(StringComparer.Ordinal) { - "definition", "find", "hotspots", "references", "callers", "callees", "impact", "map", + "search", "definition", "find", "hotspots", "references", "callers", "callees", + "symbols", "files", "languages", "impact", "map", }; private static readonly HashSet<string> CountableResponseCommands = new(StringComparer.Ordinal) { - "definition", "find", "hotspots", "references", "callers", "callees", "impact", + "search", "definition", "find", "hotspots", "references", "callers", "callees", + "symbols", "files", "languages", "impact", }; private static readonly Dictionary<string, string[]> CompactFieldsByCommand = new(StringComparer.Ordinal) { + ["search"] = ["file", "line"], ["definition"] = ["file", "line", "column"], ["find"] = ["file", "line", "column"], ["references"] = ["file", "line", "column"], @@ -60,6 +67,9 @@ internal static partial class JsonEnvelopeWrapper ["callees"] = ["file", "line", "column"], ["hotspots"] = ["name", "kind", "path", "line", "reference_count", "reference_score", "ranking_score"], ["impact"] = ["path", "caller_name", "callee_name", "depth", "first_line", "reference_count", "result_kind"], + ["symbols"] = ["path", "line", "kind", "name"], + ["files"] = ["path", "lang", "lines"], + ["languages"] = ["lang", "extensions", "symbol_extraction", "reference_extraction", "graph_queries"], ["status"] = ["api_version", "files", "chunks", "symbols", "references", "indexed_at", "git_head", "git_is_dirty", "head_freshness", "version", "graph_table_available", "hotspot_family_ready", "summary"], ["map"] = ["api_version", "file_count", "total_lines", "total_symbols", "total_references", "indexed_at", "git_head", "git_is_dirty", "head_freshness", "graph_table_available", "sections"], }; @@ -68,8 +78,16 @@ internal static bool ShouldAutoWrapBoundedResponse(string command, string[] args { if (!BoundedResponseCommands.Contains(command)) return false; + if (command == "search" && IsSearchAggregateResponseRequest(args)) + return false; if (HasArgument(args, "--fields") || HasArgument(args, "--cursor")) return true; + if (command == "languages" + && HasJsonOutputSelection(args) + && (HasArgument(args, "--limit") || HasArgument(args, "--top"))) + { + return true; + } if (AutoWrapByteBudgetCommands.Contains(command) && HasArgument(args, "--max-json-bytes")) return true; return AutoWrapCompactCommands.Contains(command) && HasCompactOutputSelection(args); @@ -79,13 +97,33 @@ private static bool IsBoundedResponseRequest(string command, string[] args) { if (!BoundedResponseCommands.Contains(command)) return false; + if (command == "search" && IsSearchAggregateResponseRequest(args)) + return false; return HasArgument(args, "--fields") || HasArgument(args, "--cursor") - || (HasEnvelopeFlag(args) && HasArgument(args, "--max-json-bytes")) + || (command == "search" && HasEnvelopeFlag(args) && HasJsonArrayOutputSelection(args)) + || (command != "search" && HasEnvelopeFlag(args) && HasArgument(args, "--max-json-bytes")) || ShouldAutoWrapBoundedResponse(command, args); } + private static bool IsSearchAggregateResponseRequest(string[] args) + => HasArgument(args, "--recipe") + || HasArgument(args, "--list-recipes") + || HasArgument(args, "--named-query") + || HasArgument(args, "--count") + || HasArgument(args, "--group-by") + || HasArgument(args, "--unique") + || HasArgument(args, "--count-by") + || HasArgument(args, "--summary-only"); + + private static bool HasJsonOutputSelection(string[] args) + => args.Any(arg => string.Equals(arg, "--json", StringComparison.Ordinal) + || arg.StartsWith("--json=", StringComparison.Ordinal)); + + private static bool HasJsonArrayOutputSelection(string[] args) + => args.Any(arg => string.Equals(arg, "--json=array", StringComparison.OrdinalIgnoreCase)); + private static bool HasCompactOutputSelection(string[] args) { for (var i = 0; i < args.Length; i++) @@ -117,18 +155,23 @@ private static int RunBoundedResponse( return WriteBoundedResponseUsageError(mapProjectionError, "Remove the conflicting map filter, or select a collection enabled by --sections."); var queryNormalized = ExtractQueryArg(args); - var dbPathExplicit = TryExtractDbPath(args, out var explicitDbPath); - var resolvedDbPath = string.IsNullOrWhiteSpace(explicitDbPath) - ? Path.Combine(".cdidx", "codeindex.db") - : explicitDbPath!; - var fingerprint = BuildResponseFingerprint(command, args); - if (controls.CursorFingerprint is not null - && !string.Equals(controls.CursorFingerprint, fingerprint, StringComparison.Ordinal)) + var (resolvedDbPath, dbPathExplicit) = ResolveQueryDbPath(args); + var queryFingerprint = BuildResponseFingerprint(command, args); + var snapshot = SafeReadResponseSnapshot(resolvedDbPath, dbPathExplicit, appVersion); + if (controls.CursorQueryFingerprint is not null + && !string.Equals(controls.CursorQueryFingerprint, queryFingerprint, StringComparison.Ordinal)) { return WriteBoundedResponseUsageError( "--cursor does not match this command, query, or filter set.", "Use next_cursor from the preceding page without changing query, filter, or sort arguments."); } + if (controls.CursorGenerationFingerprint is not null + && !string.Equals(controls.CursorGenerationFingerprint, snapshot.GenerationFingerprint, StringComparison.Ordinal)) + { + return WriteBoundedResponseUsageError( + "--cursor is stale because the index generation changed.", + "Restart pagination without --cursor and use the next_cursor returned by the refreshed index."); + } if (controls.Offset > MaxPageWindow - controls.PageLimit) { return WriteBoundedResponseUsageError( @@ -141,10 +184,19 @@ private static int RunBoundedResponse( var stopwatch = System.Diagnostics.Stopwatch.StartNew(); int exitCode; JsonEnvelopeCaptureLimitExceededException? captureLimitExceeded = null; + BoundedExecutionContext? executionContext = null; try { using var outputScope = ScopedConsoleOutput.Redirect(captured); - using var executionScope = EnterBoundedExecution(command, controls.Offset, controls.PageLimit, controls.Fields, controls.Compact); + executionContext = new BoundedExecutionContext( + command, + controls.Offset, + controls.PageLimit, + controls.Fields, + controls.Compact, + controls.ResumePath, + controls.ResumeLine); + using var executionScope = EnterBoundedExecution(executionContext); exitCode = runInner(innerArgs); } catch (JsonEnvelopeCaptureLimitExceededException ex) @@ -199,9 +251,20 @@ private static int RunBoundedResponse( .Select(item => ProjectResponseItem(item, controls.EffectiveFields(command, extraction.PrimaryCollection))) .ToList(); - var count = ResolveTotalCount(command, args, runInner, extraction, availableItems.Count, controls.Offset, streamTerminal); + var count = executionContext?.ReportedTotalCount is { } reportedTotalCount + ? new ResponseCount( + reportedTotalCount, + executionContext.ReportedTotalCountAuthoritative) + : ResolveTotalCount(command, args, runInner, extraction, availableItems.Count, controls.Offset, streamTerminal); var totalCount = Math.Max(count.TotalCount, controls.Offset + pageItems.Count); var totalAuthoritative = count.Authoritative; + var completedSnapshot = SafeReadResponseSnapshot(resolvedDbPath, dbPathExplicit, appVersion); + if (!string.Equals(snapshot.GenerationFingerprint, completedSnapshot.GenerationFingerprint, StringComparison.Ordinal)) + { + return WriteBoundedResponseUsageError( + "The index generation changed while this page was being read.", + "Restart pagination without --cursor after the active index refresh completes."); + } var envelope = BuildBoundedEnvelopeWithinBudget( command, queryNormalized, @@ -211,7 +274,8 @@ private static int RunBoundedResponse( stopwatch.Elapsed.TotalMilliseconds, pageItems, controls, - fingerprint, + queryFingerprint, + snapshot, totalCount, totalAuthoritative, exitCode, @@ -243,7 +307,8 @@ private static int RunBoundedResponse( double elapsedMs, IReadOnlyList<JsonNode?> pageItems, BoundedResponseControls controls, - string fingerprint, + string queryFingerprint, + ResponseSnapshot snapshot, int totalCount, bool totalAuthoritative, int exitCode, @@ -275,6 +340,7 @@ JsonObject BuildCandidate(int count) streamTerminal: streamTerminal, streamControlRecords: streamControlRecords); var metadata = (JsonObject)envelope["metadata"]!; + metadata["result_stable_at"] = snapshot.ResultStableAt; if (commandError is not null) { metadata["returned_count"] = 0; @@ -294,7 +360,11 @@ JsonObject BuildCandidate(int count) } var nextOffset = controls.Offset + count; var paginationWindowExhausted = nextOffset < totalCount && nextOffset >= MaxPageWindow; - var hasMore = count > 0 && nextOffset < totalCount && !paginationWindowExhausted; + var scanCursor = ReadString(streamTerminal, "next_cursor"); + var emittedAllCapturedRows = count == pageItems.Count; + var selectedScanCursor = emittedAllCapturedRows ? scanCursor : null; + var hasMore = selectedScanCursor is not null + || count > 0 && nextOffset < totalCount && !paginationWindowExhausted; metadata["result_count"] = count; metadata["returned_count"] = count; metadata["total_count"] = totalCount; @@ -304,10 +374,11 @@ JsonObject BuildCandidate(int count) metadata["cursor_offset"] = controls.Offset; metadata["page_limit"] = controls.PageLimit; metadata["has_more"] = hasMore; - metadata["next_cursor"] = hasMore && count > 0 - ? FormatResponseCursor(nextOffset, fingerprint) - : null; - metadata["truncated"] = totalCount > count; + metadata["next_cursor"] = selectedScanCursor + ?? (hasMore && count > 0 + ? FormatResponseCursor(nextOffset, queryFingerprint, snapshot.GenerationFingerprint) + : null); + metadata["truncated"] = scanCursor is not null || totalCount > count; metadata["pagination_window_limit"] = MaxPageWindow; metadata["pagination_window_exhausted"] = paginationWindowExhausted; if (controls.Compact) @@ -345,7 +416,9 @@ JsonObject BuildCandidate(int count) nextOffset, hasMore, paginationWindowExhausted, - fingerprint); + queryFingerprint, + snapshot, + extraction.PrimaryCollection); } if (controls.Compact && command == "map" @@ -449,12 +522,18 @@ private static JsonObject BuildBackwardCompatibleCompactEnvelope( int nextOffset, bool hasMore, bool paginationWindowExhausted, - string fingerprint) + string queryFingerprint, + ResponseSnapshot snapshot, + string? primaryCollection) { var compatible = (JsonObject)sourcePayload.DeepClone(); - compatible["results"] = results.DeepClone(); + var collectionName = primaryCollection ?? "results"; + compatible[collectionName] = results.DeepClone(); compatible["metadata"] = envelope["metadata"]!.DeepClone(); - compatible["count"] = returnedCount; + if (collectionName == "results") + compatible["count"] = returnedCount; + else + compatible["emitted_count"] = returnedCount; compatible["returned_count"] = returnedCount; compatible["total_count"] = totalCount; compatible["total_count_authoritative"] = totalAuthoritative; @@ -464,8 +543,9 @@ private static JsonObject BuildBackwardCompatibleCompactEnvelope( compatible["page_limit"] = controls.PageLimit; compatible["has_more"] = hasMore; compatible["next_cursor"] = hasMore && returnedCount > 0 - ? FormatResponseCursor(nextOffset, fingerprint) + ? FormatResponseCursor(nextOffset, queryFingerprint, snapshot.GenerationFingerprint) : null; + compatible["result_stable_at"] = snapshot.ResultStableAt; compatible["truncated"] = totalCount > returnedCount; compatible["pagination_window_limit"] = MaxPageWindow; compatible["pagination_window_exhausted"] = paginationWindowExhausted; @@ -506,6 +586,12 @@ private static ResponseExtraction ExtractResponseItems(string command, JsonArray } if (command == "hotspots" && rawResults.FirstOrDefault() is JsonObject hotspotsPayload) return ExtractNestedCollection(hotspotsPayload, "hotspots"); + if (command == "symbols" && rawResults.FirstOrDefault() is JsonObject symbolsPayload) + return ExtractNestedCollection(symbolsPayload, "symbols"); + if (command == "files" && rawResults.FirstOrDefault() is JsonObject filesPayload) + return ExtractNestedCollection(filesPayload, "files"); + if (command == "languages" && rawResults.FirstOrDefault() is JsonObject languagesPayload) + return ExtractNestedCollection(languagesPayload, "languages"); if (command == "impact" && rawResults.FirstOrDefault() is JsonObject impactPayload) { var requestedCollection = SelectRequestedCollection(controls.Fields, "callers", "file_impacts", "definitions"); @@ -536,6 +622,14 @@ private static ResponseExtraction ExtractResponseItems(string command, JsonArray mapPayload); } } + if (rawResults.Count == 1 && rawResults[0] is JsonArray arrayPayload) + { + return new ResponseExtraction( + new JsonArray(arrayPayload.Select(item => item?.DeepClone()).ToArray()), + "results", + null, + null); + } var rows = new JsonArray(); foreach (var result in rawResults) @@ -647,12 +741,17 @@ private static ResponseCount ResolveTotalCount( if (TryReadInt(extraction.SourcePayload, "definition_count", out var definitionCount)) return new ResponseCount(definitionCount, true); } + ResponseCount? terminalCount = null; if (streamTerminal is not null && TryReadInt(streamTerminal, "total_count", out var terminalTotal) && TryReadBool(streamTerminal, "total_count_authoritative", out var terminalAuthoritative)) - return new ResponseCount(terminalTotal, terminalAuthoritative); + { + terminalCount = new ResponseCount(terminalTotal, terminalAuthoritative); + if (terminalAuthoritative) + return terminalCount.Value; + } if (!CountableResponseCommands.Contains(command)) - return new ResponseCount(offset + availableCount, false); + return terminalCount ?? new ResponseCount(offset + availableCount, false); var countArgs = PrepareCountArgs(command, args); using var captured = new BoundedStringWriter(MaxRawJsonItemChars); @@ -664,14 +763,14 @@ private static ResponseCount ResolveTotalCount( } catch { - return new ResponseCount(offset + availableCount, false); + return terminalCount ?? new ResponseCount(offset + availableCount, false); } try { var countItems = ParseRawJsonItems(command, captured.ToString(), out _, out _); var countPayload = countItems.OfType<JsonObject>().FirstOrDefault(obj => obj.ContainsKey("count")); if (countPayload is null || !TryReadInt(countPayload, "count", out var total)) - return new ResponseCount(offset + availableCount, false); + return terminalCount ?? new ResponseCount(offset + availableCount, false); var authoritative = TryReadBool(countPayload, "authoritative_count", out var explicitAuthority) ? explicitAuthority : countExitCode == CommandExitCodes.Success @@ -682,7 +781,7 @@ private static ResponseCount ResolveTotalCount( } catch { - return new ResponseCount(offset + availableCount, false); + return terminalCount ?? new ResponseCount(offset + availableCount, false); } } @@ -703,6 +802,11 @@ private static bool TryReadBool(JsonObject obj, string propertyName, out bool va private static bool ReadOptionalBool(JsonObject obj, string propertyName, bool defaultValue = false) => TryReadBool(obj, propertyName, out var value) ? value : defaultValue; + private static string? ReadString(JsonObject? obj, string propertyName) + => obj?[propertyName] is JsonValue value && value.TryGetValue<string>(out var text) + ? text + : null; + private static string[] PrepareBoundedInnerArgs(string command, string[] args, BoundedResponseControls controls) { var stripped = StripResponseOptions(args, stripLimit: PageableResponseCommands.Contains(command)); @@ -902,14 +1006,33 @@ private static bool TryParseBoundedResponseControls( } var offset = 0; - string? cursorFingerprint = null; - if (cursor is not null && !TryParseResponseCursor(cursor, out offset, out cursorFingerprint)) + string? cursorQueryFingerprint = null; + string? cursorGenerationFingerprint = null; + string? resumePath = null; + int? resumeLine = null; + if (cursor is not null + && !TryParseResponseCursor( + cursor, + out offset, + out cursorQueryFingerprint, + out cursorGenerationFingerprint, + out resumePath, + out resumeLine)) { controls = default!; - error = "--cursor must be a response:v1:<offset>:<fingerprint> cursor returned as next_cursor."; + error = "--cursor must be an opaque response:v2 cursor returned as next_cursor."; return false; } - controls = new BoundedResponseControls(fields, compact, maxJsonBytes, pageLimit, offset, cursorFingerprint); + controls = new BoundedResponseControls( + fields, + compact, + maxJsonBytes, + pageLimit, + offset, + cursorQueryFingerprint, + cursorGenerationFingerprint, + resumePath, + resumeLine); return true; } @@ -969,30 +1092,120 @@ private static string BuildResponseFingerprint(string command, string[] args) { var normalized = StripResponseOptions(args, stripLimit: true); normalized.RemoveAll(arg => string.Equals(arg, "--body", StringComparison.Ordinal)); + normalized.RemoveAll(arg => arg is "--allow-partial" or "--results-only" or "--verbose" or "--profile"); + RemoveOptionWithValue(normalized, "--line-scan-limit"); var input = command + "\0" + string.Join('\0', normalized); var hash = SHA256.HashData(Encoding.UTF8.GetBytes(input)); return Convert.ToHexString(hash.AsSpan(0, 8)).ToLowerInvariant(); } - private static string FormatResponseCursor(int offset, string fingerprint) - => $"{ResponseCursorPrefix}{offset.ToString(CultureInfo.InvariantCulture)}:{fingerprint}"; + private static void RemoveOptionWithValue(List<string> args, string option) + { + for (var i = args.Count - 1; i >= 0; i--) + { + if (args[i].StartsWith(option + "=", StringComparison.Ordinal)) + { + args.RemoveAt(i); + continue; + } + if (!string.Equals(args[i], option, StringComparison.Ordinal)) + continue; + args.RemoveAt(i); + if (i < args.Count) + args.RemoveAt(i); + } + } + + private static string FormatResponseCursor( + int offset, + string queryFingerprint, + string generationFingerprint, + string? resumePath = null, + int? resumeLine = null) + { + var payload = new JsonObject + { + ["offset"] = offset, + ["query"] = queryFingerprint, + ["generation"] = generationFingerprint, + }; + if (resumePath is not null) + payload["resume_path"] = resumePath; + if (resumeLine.HasValue) + payload["resume_line"] = resumeLine.Value; + var encoded = Convert.ToBase64String(Encoding.UTF8.GetBytes(payload.ToJsonString())) + .TrimEnd('=') + .Replace('+', '-') + .Replace('/', '_'); + return ResponseCursorPrefix + encoded; + } - private static bool TryParseResponseCursor(string cursor, out int offset, out string? fingerprint) + private static bool TryParseResponseCursor( + string cursor, + out int offset, + out string? queryFingerprint, + out string? generationFingerprint, + out string? resumePath, + out int? resumeLine) { offset = 0; - fingerprint = null; - if (!cursor.StartsWith(ResponseCursorPrefix, StringComparison.Ordinal)) + queryFingerprint = null; + generationFingerprint = null; + resumePath = null; + resumeLine = null; + if (cursor.StartsWith(LegacyResponseCursorPrefix, StringComparison.Ordinal)) + { + var remainder = cursor[LegacyResponseCursorPrefix.Length..]; + var separator = remainder.IndexOf(':'); + if (separator <= 0 || separator == remainder.Length - 1) + return false; + if (!int.TryParse(remainder[..separator], NumberStyles.None, CultureInfo.InvariantCulture, out offset) || offset < 0) + return false; + queryFingerprint = remainder[(separator + 1)..]; + return IsCursorFingerprint(queryFingerprint); + } + if (!cursor.StartsWith(ResponseCursorPrefix, StringComparison.Ordinal) + || cursor.Length > 16_384) + { return false; - var remainder = cursor[ResponseCursorPrefix.Length..]; - var separator = remainder.IndexOf(':'); - if (separator <= 0 || separator == remainder.Length - 1) + } + + var encoded = cursor[ResponseCursorPrefix.Length..] + .Replace('-', '+') + .Replace('_', '/'); + var paddingLength = (4 - encoded.Length % 4) % 4; + if (paddingLength > 0) + encoded += new string('=', paddingLength); + JsonObject? payload; + try + { + payload = JsonNode.Parse(Encoding.UTF8.GetString(Convert.FromBase64String(encoded))) as JsonObject; + } + catch (Exception ex) when (ex is FormatException or JsonException) + { return false; - if (!int.TryParse(remainder[..separator], NumberStyles.None, CultureInfo.InvariantCulture, out offset) || offset < 0) + } + if (payload is null + || !TryReadInt(payload, "offset", out offset) + || offset < 0) + { return false; - fingerprint = remainder[(separator + 1)..]; - return fingerprint.Length == 16 && fingerprint.All(Uri.IsHexDigit); + } + queryFingerprint = ReadString(payload, "query"); + generationFingerprint = ReadString(payload, "generation"); + resumePath = ReadString(payload, "resume_path"); + if (payload["resume_line"] is JsonValue resumeValue && resumeValue.TryGetValue<int>(out var parsedResumeLine)) + resumeLine = parsedResumeLine; + return IsCursorFingerprint(queryFingerprint) + && IsCursorFingerprint(generationFingerprint) + && (resumePath is null || resumePath.Length <= 4096) + && (!resumeLine.HasValue || resumeLine.Value > 0) + && (resumePath is null) == !resumeLine.HasValue; } + private static bool IsCursorFingerprint(string? fingerprint) + => fingerprint is { Length: 16 } && fingerprint.All(Uri.IsHexDigit); + private static int WriteBoundedResponseUsageError(string message, string hint) { CommandErrorWriter.WriteStderr($"Error [{CommandErrorCodes.UsageError}]: {message}"); @@ -1067,11 +1280,19 @@ private sealed record BoundedResponseControls( int? MaxJsonBytes, int PageLimit, int Offset, - string? CursorFingerprint) + string? CursorQueryFingerprint, + string? CursorGenerationFingerprint, + string? ResumePath, + int? ResumeLine) { public IReadOnlyList<string>? EffectiveFields(string command, string? primaryCollection) { - var selected = Fields ?? (CompactFieldsByCommand.TryGetValue(command, out var defaults) ? defaults : null); + var preserveFullDiscoveryRows = command is "search" or "languages"; + var selected = Fields + ?? ((!preserveFullDiscoveryRows || Compact) + && CompactFieldsByCommand.TryGetValue(command, out var defaults) + ? defaults + : null); if (selected is null || primaryCollection is null) return selected; var dotted = selected @@ -1094,6 +1315,69 @@ private sealed record ResponseExtraction( private readonly record struct ResponseCount(int TotalCount, bool Authoritative); + private readonly record struct ResponseSnapshot( + string GenerationFingerprint, + string? ResultStableAt); + + private static ResponseSnapshot SafeReadResponseSnapshot( + string dbPath, + bool dbPathExplicit, + string appVersion) + { + try + { + if (!dbPathExplicit + && !dbPath.StartsWith("file:", StringComparison.OrdinalIgnoreCase) + && !File.Exists(dbPath)) + { + return BuildFallbackResponseSnapshot(appVersion); + } + + using var db = new DbContext(DbOpenIntent.QueryOnly, dbPath); + if (!db.TryValidateIsCodeIndexDb(out _)) + return BuildFallbackResponseSnapshot(appVersion); + return BuildResponseSnapshot(new DbReader(db)); + } + catch + { + return BuildFallbackResponseSnapshot(appVersion); + } + } + + private static ResponseSnapshot BuildResponseSnapshot(DbReader reader) + { + var generation = reader.GetPaginationGeneration(); + return new( + BuildResponseValueFingerprint(generation.Identity), + generation.StableAt); + } + + private static ResponseSnapshot BuildFallbackResponseSnapshot(string appVersion) + => new(BuildResponseValueFingerprint("catalog\0" + appVersion), null); + + private static string BuildResponseValueFingerprint(string value) + { + var hash = SHA256.HashData(Encoding.UTF8.GetBytes(value)); + return Convert.ToHexString(hash.AsSpan(0, 8)).ToLowerInvariant(); + } + + internal static (string Cursor, string? ResultStableAt) BuildFindResumeCursor( + string[] args, + DbReader reader, + string resumePath, + int resumeLine) + { + var snapshot = BuildResponseSnapshot(reader); + return ( + FormatResponseCursor( + offset: 0, + BuildResponseFingerprint("find", args), + snapshot.GenerationFingerprint, + resumePath, + resumeLine), + snapshot.ResultStableAt); + } + internal static int GetBoundedResponseOffset(string command) { var execution = BoundedExecution.Value; @@ -1103,6 +1387,39 @@ internal static int GetBoundedResponseOffset(string command) : 0; } + internal static (string? Path, int? Line) GetBoundedFindResume() + { + var execution = BoundedExecution.Value; + return execution is not null && string.Equals(execution.Command, "find", StringComparison.Ordinal) + ? (execution.ResumePath, execution.ResumeLine) + : (null, null); + } + + internal static int? GetBoundedResponseLimit(string command) + { + var execution = BoundedExecution.Value; + return execution is not null + && string.Equals(execution.Command, CanonicalizeCommandName(command), StringComparison.Ordinal) + ? execution.Limit + : null; + } + + internal static void ReportBoundedResponseTotal( + string command, + int totalCount, + bool authoritative) + { + var execution = BoundedExecution.Value; + if (execution is null + || !string.Equals(execution.Command, CanonicalizeCommandName(command), StringComparison.Ordinal)) + { + return; + } + + execution.ReportedTotalCount = Math.Max(0, totalCount); + execution.ReportedTotalCountAuthoritative = authoritative; + } + internal static string? GetBoundedMapCollection() { var execution = BoundedExecution.Value; @@ -1136,15 +1453,10 @@ internal static bool IsBoundedMapScalarProjection() : null; } - private static IDisposable EnterBoundedExecution( - string command, - int offset, - int limit, - IReadOnlyList<string>? fields, - bool compact) + private static IDisposable EnterBoundedExecution(BoundedExecutionContext execution) { var previous = BoundedExecution.Value; - BoundedExecution.Value = new BoundedExecutionContext(command, offset, limit, fields, compact); + BoundedExecution.Value = execution; return new BoundedExecutionScope(previous); } @@ -1153,7 +1465,13 @@ private sealed record BoundedExecutionContext( int Offset, int Limit, IReadOnlyList<string>? Fields, - bool Compact); + bool Compact, + string? ResumePath, + int? ResumeLine) + { + public int? ReportedTotalCount { get; set; } + public bool ReportedTotalCountAuthoritative { get; set; } + } private sealed class BoundedExecutionScope(BoundedExecutionContext? previous) : IDisposable { diff --git a/src/CodeIndex/Cli/JsonEnvelopeWrapper.cs b/src/CodeIndex/Cli/JsonEnvelopeWrapper.cs index 7ac3af9ad..f4830a54c 100644 --- a/src/CodeIndex/Cli/JsonEnvelopeWrapper.cs +++ b/src/CodeIndex/Cli/JsonEnvelopeWrapper.cs @@ -101,10 +101,7 @@ internal static int RunWrapped( var innerArgs = PrepareInnerArgs(args); var queryNormalized = ExtractQueryArg(args); - var dbPathExplicit = TryExtractDbPath(args, out var explicitDbPath); - var resolvedDbPath = string.IsNullOrWhiteSpace(explicitDbPath) - ? Path.Combine(".cdidx", "codeindex.db") - : explicitDbPath!; + var (resolvedDbPath, dbPathExplicit) = ResolveQueryDbPath(args); using var captured = new BoundedStringWriter(MaxCapturedOutputChars); var stopwatch = Stopwatch.StartNew(); @@ -286,6 +283,37 @@ private static JsonArray ParseRawJsonItems( if (string.IsNullOrEmpty(raw)) return array; + var trimmed = raw.Trim(); + if (trimmed.Length <= MaxRawJsonItemChars) + { + try + { + var document = JsonFrameParser.ParseNode(trimmed, MaxRawJsonItemDepth); + if (document is not null) + { + rawJsonNodeCount = AddRawJsonNodeCount(rawJsonNodeCount, CountJsonNodes(document)); + if (IsJsonStreamTerminal(document)) + { + streamTerminal = (JsonObject)document.DeepClone(); + if (!IsTerminalResultRecord(command, document)) + return array; + } + else if (IsJsonStreamControlRecord(document)) + { + streamControlRecords.Add(document); + return array; + } + + array.Add(document); + return array; + } + } + catch (JsonException) + { + // Multiple NDJSON frames are parsed line by line below. + } + } + foreach (var line in EnumerateRawLines(raw)) { if (string.IsNullOrWhiteSpace(line)) @@ -443,7 +471,7 @@ private static bool IsTerminalResultRecord(string command, JsonNode node) // positional 引数（= query）と取り違えないようにする。 private static readonly HashSet<string> ValueConsumingOptions = new(StringComparer.Ordinal) { - "--db", "--limit", "--top", "--lang", "--kind", "--since", + "--db", "--data-dir", "--limit", "--top", "--lang", "--kind", "--since", "--start", "--end", "--before", "--after", "--name", "--snippet-lines", "--snippet-focus", "--path", "--exclude-path", "--max-hops", "--depth", "--focus-line", "--focus-column", "--focus-length", @@ -477,25 +505,42 @@ private static bool IsTerminalResultRecord(string command, JsonNode node) } private static bool TryExtractDbPath(string[] args, out string? dbPath) + => TryExtractOptionValue(args, "--db", out dbPath); + + private static bool TryExtractDataDir(string[] args, out string? dataDir) + => TryExtractOptionValue(args, "--data-dir", out dataDir); + + private static bool TryExtractOptionValue(string[] args, string option, out string? value) { for (var i = 0; i < args.Length; i++) { var arg = args[i]; - if (string.Equals(arg, "--db", StringComparison.Ordinal) && i + 1 < args.Length) + if (string.Equals(arg, option, StringComparison.Ordinal) && i + 1 < args.Length) { - dbPath = args[i + 1]; + value = args[i + 1]; return true; } - if (arg.StartsWith("--db=", StringComparison.Ordinal)) + if (arg.StartsWith(option + "=", StringComparison.Ordinal)) { - dbPath = arg["--db=".Length..]; + value = arg[(option.Length + 1)..]; return true; } } - dbPath = null; + value = null; return false; } + private static (string DbPath, bool DbPathExplicit) ResolveQueryDbPath(string[] args) + { + var dbPathExplicit = TryExtractDbPath(args, out var explicitDbPath); + TryExtractDataDir(args, out var explicitDataDir); + var resolution = DbPathResolver.ResolveForQuery( + Environment.CurrentDirectory, + explicitDbPath, + explicitDataDir); + return (resolution.DbPath, dbPathExplicit); + } + private sealed class BoundedStringWriter(int maxChars) : StringWriter { private int _writtenChars; diff --git a/src/CodeIndex/Cli/JsonOutputContracts.cs b/src/CodeIndex/Cli/JsonOutputContracts.cs index 02caa1495..3b1ba2b2b 100644 --- a/src/CodeIndex/Cli/JsonOutputContracts.cs +++ b/src/CodeIndex/Cli/JsonOutputContracts.cs @@ -537,10 +537,20 @@ internal sealed record SearchFileGroupedJsonResult( [property: JsonPropertyName("api_version")] string ApiVersion, [property: JsonPropertyName("query")] string Query, [property: JsonPropertyName("total_matches")] int TotalMatches, + [property: JsonPropertyName("matched_count")] int MatchedCount, + [property: JsonPropertyName("grouped_match_count")] int GroupedMatchCount, + [property: JsonPropertyName("emitted_match_count")] int EmittedMatchCount, + [property: JsonPropertyName("omitted_match_count")] int OmittedMatchCount, [property: JsonPropertyName("returned_groups")] int ReturnedGroups, + [property: JsonPropertyName("total_groups")] int TotalGroups, [property: JsonPropertyName("files")] int Files, + [property: JsonPropertyName("total_files")] int TotalFiles, [property: JsonPropertyName("per_file_limit")] int PerFileLimit, [property: JsonPropertyName("truncated")] bool Truncated, + [property: JsonPropertyName("has_more")] bool HasMore, + [property: JsonPropertyName("continuation_action")] + [property: JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + string? ContinuationAction, [property: JsonPropertyName("groups")] List<SearchFileGroupJsonResult> Groups); internal sealed record SearchFileGroupJsonResult( diff --git a/src/CodeIndex/Cli/QueryCommandRunner.ArgParsing.cs b/src/CodeIndex/Cli/QueryCommandRunner.ArgParsing.cs index a06de1802..c2e80ab8b 100644 --- a/src/CodeIndex/Cli/QueryCommandRunner.ArgParsing.cs +++ b/src/CodeIndex/Cli/QueryCommandRunner.ArgParsing.cs @@ -196,6 +196,7 @@ public static QueryCommandOptions ParseArgs( SearchCursor? searchCursor = null; int? unusedCursorOffset = null; int? outlineCursorOffset = null; + string? rawCursorValue = null; DependencyCycleCursor? dependencyCycleCursor = null; var namedSearchQueries = new List<SearchNamedQuery>(); bool languagesIndexedOnly = false; @@ -747,16 +748,21 @@ void WarnIfDuplicateSingleValueOption(string canonicalName, string newValue) if (TryReadStringOptionValue(args, ref i, "--cursor", inlineValue, allowSeparatedDashPrefixedLiteralValue: allowSeparatedDashPrefixedCursorValue, out var cursorValue, out var cursorError)) { WarnIfDuplicateSingleValueOption("--cursor", cursorValue!); - if (TryParseSearchCursor(cursorValue!, out var parsedCursor)) + var parsedCursorValue = cursorValue!; + if (TryParseSearchCursor(parsedCursorValue, out var parsedCursor)) searchCursor = parsedCursor; - else if (TryParseUnusedCursor(cursorValue!, out var parsedUnusedCursorOffset)) + else if (TryParseUnusedCursor(parsedCursorValue, out var parsedUnusedCursorOffset)) unusedCursorOffset = parsedUnusedCursorOffset; - else if (TryParseOutlineCursor(cursorValue!, out var parsedOutlineCursorOffset)) + else if (TryParseOutlineCursor(parsedCursorValue, out var parsedOutlineCursorOffset)) outlineCursorOffset = parsedOutlineCursorOffset; - else if (TryParseDependencyCycleCursor(cursorValue!, out var parsedDependencyCycleCursor)) + else if (TryParseDependencyCycleCursor(parsedCursorValue, out var parsedDependencyCycleCursor)) dependencyCycleCursor = parsedDependencyCycleCursor; else + { AddParseError("Error: --cursor must be a search, unused, outline, or dependency-cycle pagination cursor returned as `next_cursor`."); + break; + } + rawCursorValue = parsedCursorValue; } else { @@ -1771,6 +1777,7 @@ void WarnIfDuplicateSingleValueOption(string canonicalName, string newValue) SearchCursor = searchCursor, UnusedCursorOffset = unusedCursorOffset, OutlineCursorOffset = outlineCursorOffset, + CursorValue = rawCursorValue, DependencyCycleCursor = dependencyCycleCursor, NamedSearchQueries = namedSearchQueries, LanguagesIndexedOnly = languagesIndexedOnly, diff --git a/src/CodeIndex/Cli/QueryCommandRunner.Discovery.cs b/src/CodeIndex/Cli/QueryCommandRunner.Discovery.cs index abdd074f2..73fddb120 100644 --- a/src/CodeIndex/Cli/QueryCommandRunner.Discovery.cs +++ b/src/CodeIndex/Cli/QueryCommandRunner.Discovery.cs @@ -178,8 +178,8 @@ public static int RunSymbols(string[] cmdArgs, JsonSerializerOptions jsonOptions } var results = options.GroupPartials - ? reader.SearchSymbols(symbolQueries, options.Limit, options.Kind, options.Lang, options.PathPatterns, options.ExcludePaths, options.ExcludeTests, options.Since, exact, visibilityFilters: options.VisibilityFilters, excludeVisibilityFilters: options.ExcludeVisibilityFilters, sortMode: options.SymbolSortMode, groupPartials: true) - : reader.SearchSymbols(symbolQueries, options.Limit, options.Kind, options.Lang, options.PathPatterns, options.ExcludePaths, options.ExcludeTests, options.Since, exact, visibilityFilters: options.VisibilityFilters, excludeVisibilityFilters: options.ExcludeVisibilityFilters, sortMode: options.SymbolSortMode); + ? reader.SearchSymbols(symbolQueries, options.Limit, options.Kind, options.Lang, options.PathPatterns, options.ExcludePaths, options.ExcludeTests, options.Since, exact, visibilityFilters: options.VisibilityFilters, excludeVisibilityFilters: options.ExcludeVisibilityFilters, sortMode: options.SymbolSortMode, groupPartials: true, offset: JsonEnvelopeWrapper.GetBoundedResponseOffset("symbols")) + : reader.SearchSymbols(symbolQueries, options.Limit, options.Kind, options.Lang, options.PathPatterns, options.ExcludePaths, options.ExcludeTests, options.Since, exact, visibilityFilters: options.VisibilityFilters, excludeVisibilityFilters: options.ExcludeVisibilityFilters, sortMode: options.SymbolSortMode, offset: JsonEnvelopeWrapper.GetBoundedResponseOffset("symbols")); var hasExactPredicate = exact && symbolQueries is { Count: > 0 }; var exactSignal = reader.GetSymbolsExactQuerySignal(options.Lang, options.PathPatterns, options.ExcludePaths, options.ExcludeTests, options.Since); var multiNameExactHint = symbolQueries != null && symbolQueries.Count > 1; @@ -460,7 +460,16 @@ public static int RunFiles(string[] cmdArgs, JsonSerializerOptions jsonOptions) return CommandExitCodes.Success; } - var results = reader.ListFiles(options.Query, options.Limit, options.Lang, filesScope.PathPatterns, filesScope.ExcludePaths, filesScope.ExcludeTests, options.Since, orderBySize: options.RawBytes); + var results = reader.ListFiles( + options.Query, + options.Limit, + options.Lang, + filesScope.PathPatterns, + filesScope.ExcludePaths, + filesScope.ExcludeTests, + options.Since, + orderBySize: options.RawBytes, + offset: JsonEnvelopeWrapper.GetBoundedResponseOffset("files")); if (results.Count == 0) { if (IsDiscoveryNdjson(options)) diff --git a/src/CodeIndex/Cli/QueryCommandRunner.Find.cs b/src/CodeIndex/Cli/QueryCommandRunner.Find.cs index ba973c26e..38aa4c3ee 100644 --- a/src/CodeIndex/Cli/QueryCommandRunner.Find.cs +++ b/src/CodeIndex/Cli/QueryCommandRunner.Find.cs @@ -10,7 +10,7 @@ namespace CodeIndex.Cli; public static partial class QueryCommandRunner { internal const int MaxFindLineScanLimit = 10_000_000; - private const string FindUsage = "Usage: cdidx find <query> (--path <glob>|--all) [--db <path>] [--json] [--format <text|json|count|compact|csv|tsv|lsp|qf|sarif>] [--fields <csv>] [--cursor <response:v1:offset:fingerprint>] [--max-json-bytes <n>] [--verbose] [--limit <n>|--top <n>] [--lang <lang>] [--exclude-path <glob>] [--exclude-tests] [--context <n>] [--before <n>] [--after <n>] [--snippet-lines <n>] [--focus-line <line>] [--focus-column <n>] [--max-line-width <n>] [--line-scan-limit <n>] [--allow-partial] [--exact] [--regex] [--count]\n cdidx find --query <query> (--path <glob>|--all) [...]\n cdidx find [options] -- <query>"; + private const string FindUsage = "Usage: cdidx find <query> (--path <glob>|--all) [--db <path>] [--json] [--format <text|json|count|compact|csv|tsv|lsp|qf|sarif>] [--fields <csv>] [--cursor <next_cursor>] [--max-json-bytes <n>] [--verbose] [--limit <n>|--top <n>] [--lang <lang>] [--exclude-path <glob>] [--exclude-tests] [--context <n>] [--before <n>] [--after <n>] [--snippet-lines <n>] [--focus-line <line>] [--focus-column <n>] [--max-line-width <n>] [--line-scan-limit <n>] [--allow-partial] [--exact] [--regex] [--count]\n cdidx find --query <query> (--path <glob>|--all) [...]\n cdidx find [options] -- <query>"; public static int RunFind(string[] cmdArgs, JsonSerializerOptions jsonOptions) { @@ -158,7 +158,14 @@ public static int RunFind(string[] cmdArgs, JsonSerializerOptions jsonOptions) { AddFindScanJsonFields(payload, counts.Scan); if (options.All) - AddFindTerminalScanFields(payload, counts.Scan, counts.Count, countMode: true); + { + AddFindTerminalScanFields( + payload, + counts.Scan, + counts.Count, + countMode: true, + resultStableAt: reader.GetPaginationGeneration().StableAt); + } }); Console.WriteLine(payload.ToJsonString(jsonOptions)); } @@ -171,6 +178,7 @@ public static int RunFind(string[] cmdArgs, JsonSerializerOptions jsonOptions) } var (contextBefore, contextAfter, snippetLines) = ResolveFindContext(options, preparedFindArgs); + var (resumePath, resumeLine) = JsonEnvelopeWrapper.GetBoundedFindResume(); FindResults findResults; try { @@ -191,7 +199,9 @@ public static int RunFind(string[] cmdArgs, JsonSerializerOptions jsonOptions) candidateFileLimit, lineLimit, JsonEnvelopeWrapper.GetBoundedResponseOffset("find"), - useIndexedLiteralCandidates: options.All); + useIndexedLiteralCandidates: options.All, + resumePath: resumePath, + resumeLine: resumeLine); } catch (ArgumentException ex) when (options.Regex) { @@ -202,6 +212,7 @@ public static int RunFind(string[] cmdArgs, JsonSerializerOptions jsonOptions) return WriteFindRegexTimeoutError(ex, jsonOptions, options.Json); } var results = findResults.Results; + var findResume = BuildFindResumeCursor(cmdArgs, reader, findResults.Scan); if (results.Count == 0) { var candidateFileCount = findResults.Scan.CandidateFiles; @@ -233,7 +244,13 @@ public static int RunFind(string[] cmdArgs, JsonSerializerOptions jsonOptions) AddFindScanJsonFields(payload, findResults.Scan); if (IsFindAllNdjson(options)) { - AddFindTerminalScanFields(payload, findResults.Scan, returnedCount: 0, countMode: false); + AddFindTerminalScanFields( + payload, + findResults.Scan, + returnedCount: 0, + countMode: false, + nextCursor: findResume.Cursor, + resultStableAt: findResume.ResultStableAt); findTerminalLine = payload.ToJsonString(GetCompactJsonOptions(jsonOptions)); } else @@ -254,7 +271,7 @@ public static int RunFind(string[] cmdArgs, JsonSerializerOptions jsonOptions) WriteZeroResultHints(options, reader, filterHint: "try broadening --path or adding another --path value; --path is required for find."); } if (options.All) - WriteFindScanSummary(findResults.Scan); + WriteFindScanSummary(findResults.Scan, nextCursor: findResume.Cursor); } return FindScanExitCode(options, findResults.Scan, ZeroResultExitCode(options)); } @@ -299,7 +316,8 @@ public static int RunFind(string[] cmdArgs, JsonSerializerOptions jsonOptions) jsonOptions, findResults.Scan, results.Count, - resultLimitReached: results.Count >= options.Limit); + resultLimitReached: results.Count >= options.Limit, + commandArgs: cmdArgs); } } else @@ -312,7 +330,10 @@ public static int RunFind(string[] cmdArgs, JsonSerializerOptions jsonOptions) } var fileCount = results.Select(r => r.Path).Distinct().Count(); CommandErrorWriter.WriteStderr($"({results.Count} matches in {fileCount} files)"); - WriteFindScanSummary(findResults.Scan, resultLimitReached: results.Count >= options.Limit); + WriteFindScanSummary( + findResults.Scan, + resultLimitReached: results.Count >= options.Limit, + nextCursor: findResume.Cursor); } return FindScanExitCode(options, findResults.Scan); }, _ => @@ -603,8 +624,10 @@ private static string BuildFindScanTerminalLine( JsonSerializerOptions jsonOptions, FindScanSummary scan, int returnedCount, - bool resultLimitReached) + bool resultLimitReached, + string[] commandArgs) { + var findResume = BuildFindResumeCursor(commandArgs, reader, scan); var payload = new JsonObject { ["api_version"] = JsonOutputContract.ApiVersion, @@ -617,7 +640,9 @@ private static string BuildFindScanTerminalLine( returnedCount, countMode: false, appliedLimit: options.Limit, - resultLimitReached: resultLimitReached); + resultLimitReached: resultLimitReached, + nextCursor: findResume.Cursor, + resultStableAt: findResume.ResultStableAt); return payload.ToJsonString(GetCompactJsonOptions(jsonOptions)); } @@ -627,7 +652,9 @@ private static void AddFindTerminalScanFields( int returnedCount, bool countMode, int? appliedLimit = null, - bool resultLimitReached = false) + bool resultLimitReached = false, + string? nextCursor = null, + string? resultStableAt = null) { var scanComplete = !scan.Truncated && !resultLimitReached; payload["terminal_record"] = true; @@ -636,6 +663,8 @@ private static void AddFindTerminalScanFields( payload["partial_result"] = scan.Truncated; payload["scan_complete"] = scanComplete; payload["has_more"] = scan.Truncated || resultLimitReached; + payload["next_cursor"] = nextCursor; + payload["result_stable_at"] = resultStableAt; if (!countMode) payload["authoritative_rows"] = scanComplete; if (appliedLimit.HasValue) @@ -657,29 +686,34 @@ private static void AddFindTerminalScanFields( } private static string? FindScanContinuationAction(FindScanSummary scan, bool resultLimitReached) - => scan.TruncationReason switch - { - "line_scan_limit" => "increase_line_scan_limit_or_narrow_scope", - "candidate_file_limit" => "narrow_scope_with_path", - _ when scan.Truncated => "narrow_scope_with_path", - _ when resultLimitReached => "increase_result_limit_or_narrow_scope", - _ => null, - }; + => scan.NextPath is not null && scan.NextLine.HasValue + ? "resume_with_next_cursor" + : scan.TruncationReason switch + { + "line_scan_limit" => "increase_line_scan_limit_or_narrow_scope", + "candidate_file_limit" => "narrow_scope_with_path", + _ when scan.Truncated => "narrow_scope_with_path", + _ when resultLimitReached => "increase_result_limit_or_narrow_scope", + _ => null, + }; private static string? FindScanRecoveryGuidance(FindScanSummary scan, bool resultLimitReached) - => scan.TruncationReason switch - { - "line_scan_limit" => $"Increase --line-scan-limit up to {MaxFindLineScanLimit}, or replace --all with one or more --path filters. Pass --allow-partial only when exit code 0 is acceptable for an incomplete scan.", - "candidate_file_limit" => "Replace --all with one or more --path filters to scan fewer candidate files. Pass --allow-partial only when exit code 0 is acceptable for an incomplete scan.", - _ when scan.Truncated => "Replace --all with one or more --path filters and rerun. Pass --allow-partial only when exit code 0 is acceptable for an incomplete scan.", - _ when resultLimitReached => "Increase --limit or narrow the find query to retrieve additional matching rows.", - _ => null, - }; + => scan.NextPath is not null && scan.NextLine.HasValue + ? "Pass next_cursor back with --cursor to resume after the scan cap without rescanning completed lines; --line-scan-limit may be increased for the next page." + : scan.TruncationReason switch + { + "line_scan_limit" => $"Increase --line-scan-limit up to {MaxFindLineScanLimit}, or replace --all with one or more --path filters. Pass --allow-partial only when exit code 0 is acceptable for an incomplete scan.", + "candidate_file_limit" => "Replace --all with one or more --path filters to scan fewer candidate files. Pass --allow-partial only when exit code 0 is acceptable for an incomplete scan.", + _ when scan.Truncated => "Replace --all with one or more --path filters and rerun. Pass --allow-partial only when exit code 0 is acceptable for an incomplete scan.", + _ when resultLimitReached => "Increase --limit or narrow the find query to retrieve additional matching rows.", + _ => null, + }; private static void WriteFindScanSummary( FindScanSummary scan, bool resultLimitReached = false, - bool countMode = false) + bool countMode = false, + string? nextCursor = null) { var summary = $"scanned {scan.FilesScanned}/{scan.CandidateFiles} candidate files, {ConsoleUi.Counted(scan.LinesScanned, "line")}"; summary += $"; search_strategy={scan.SearchStrategy}"; @@ -700,8 +734,30 @@ private static void WriteFindScanSummary( if (continuationAction != null) summary += $"; continuation_action={continuationAction}"; CommandErrorWriter.WriteStderr($"({summary})"); + if (nextCursor != null) + CommandErrorWriter.WriteStderr($"next_cursor={nextCursor}"); var recoveryGuidance = FindScanRecoveryGuidance(scan, resultLimitReached); if (recoveryGuidance != null) CommandErrorWriter.WriteStderr($"Recovery: {recoveryGuidance}"); } + + private static (string? Cursor, string? ResultStableAt) BuildFindResumeCursor( + string[] commandArgs, + DbReader reader, + FindScanSummary scan) + { + if (!scan.Truncated + || scan.NextPath is null + || !scan.NextLine.HasValue) + { + return (null, reader.GetPaginationGeneration().StableAt); + } + + var cursor = JsonEnvelopeWrapper.BuildFindResumeCursor( + commandArgs, + reader, + scan.NextPath, + scan.NextLine.Value); + return (cursor.Cursor, cursor.ResultStableAt); + } } diff --git a/src/CodeIndex/Cli/QueryCommandRunner.Languages.cs b/src/CodeIndex/Cli/QueryCommandRunner.Languages.cs index 5d38080aa..a1bddaa54 100644 --- a/src/CodeIndex/Cli/QueryCommandRunner.Languages.cs +++ b/src/CodeIndex/Cli/QueryCommandRunner.Languages.cs @@ -154,7 +154,6 @@ int WriteLanguages( .Where(kv => options.LanguageCapabilities.All(capability => LanguageMatchesCapability(kv.Value, capability))) .OrderBy(kv => kv.Key, StringComparer.Ordinal) .ToList(); - if (json && (options.SummaryOnly || options.CountOnly || options.OutputFormat == OutputFormatCount)) { var payload = BuildLanguageSummaryPayload( @@ -171,6 +170,23 @@ int WriteLanguages( return CommandExitCodes.Success; } + var boundedLimit = JsonEnvelopeWrapper.GetBoundedResponseLimit("languages"); + if (!boundedLimit.HasValue + && cmdArgs.Any(arg => string.Equals(arg, "--limit", StringComparison.Ordinal) + || arg.StartsWith("--limit=", StringComparison.Ordinal) + || string.Equals(arg, "--top", StringComparison.Ordinal) + || arg.StartsWith("--top=", StringComparison.Ordinal))) + { + boundedLimit = options.Limit; + } + if (boundedLimit.HasValue) + { + filtered = filtered + .Skip(JsonEnvelopeWrapper.GetBoundedResponseOffset("languages")) + .Take(boundedLimit.Value) + .ToList(); + } + if (json) { var entries = filtered.Select(kv => new LanguageEntryJsonResult( diff --git a/src/CodeIndex/Cli/QueryCommandRunner.OptionReaders.cs b/src/CodeIndex/Cli/QueryCommandRunner.OptionReaders.cs index 58bf7f4a3..f138d8537 100644 --- a/src/CodeIndex/Cli/QueryCommandRunner.OptionReaders.cs +++ b/src/CodeIndex/Cli/QueryCommandRunner.OptionReaders.cs @@ -29,7 +29,7 @@ public static partial class QueryCommandRunner ["--repo"] = "pass a GitHub repository in owner/name form for `--open-issues github`, e.g. `--repo Widthdom/CodeIndex`.", ["--issue-title"] = "pass an issue title hint for ad hoc search issue-drafts, e.g. `--issue-title \"Thread.Yield audit\"`.", ["--issue-label"] = "pass an issue label hint for search issue-drafts, e.g. `--issue-label audit`; repeat or comma-separate values.", - ["--cursor"] = "pass the `next_cursor` returned by a prior paged response, such as `response:v1:<offset>:<fingerprint>`, a recipe search cursor, `outline:<offset>`, or `unused:<offset>`.", + ["--cursor"] = "pass the opaque `next_cursor` returned by a prior paged response, such as `response:v2:...`, a recipe search cursor, or the `page:v1:...` token returned by outline/unused.", ["--kind"] = "pass a kind identifier, e.g. `--kind function`. definition/symbols/outline/hotspots/unused take a symbol kind; references/callers/callees take a reference kind such as `call`, `instantiate`, or `subscribe`. Run the command's `--help` for the kind list.", ["--outline-fields"] = "pass outline symbol field names such as `name,line,signature`, or `all` for the full symbol payload.", ["--outline-only"] = "for inspect, return file, definitions, and nearby_symbols JSON only; add `--body` when definition body snippets are needed.", diff --git a/src/CodeIndex/Cli/QueryCommandRunner.Outline.cs b/src/CodeIndex/Cli/QueryCommandRunner.Outline.cs index 49747bd9c..0be0fc201 100644 --- a/src/CodeIndex/Cli/QueryCommandRunner.Outline.cs +++ b/src/CodeIndex/Cli/QueryCommandRunner.Outline.cs @@ -55,8 +55,26 @@ public static int RunOutline(string[] cmdArgs, JsonSerializerOptions jsonOptions : OutlineSortMode.Source; var includeReferenceCounts = OutlineNeedsReferenceCounts(options, outlineSortMode); var includeDerivedMetadata = OutlineNeedsDerivedMetadata(options, outlineSortMode); + var kindFilters = BuildOutlineKindFilters(options.Kind); return WithDb(options, jsonOptions, reader => { + var cursorComponents = new List<string?> + { + filePath, + FormatOutlineSortMode(outlineSortMode), + }; + cursorComponents.AddRange(kindFilters.Order(StringComparer.Ordinal)); + var cursorContext = BuildPaginationCursorContext(reader, "outline", cursorComponents); + var cursorValidationError = ValidateScopedOffsetCursor(options, "outline", cursorContext); + if (cursorValidationError != null) + { + WriteUsageError( + cursorValidationError, + GetUsageLineOrThrow("outline"), + "Restart outline pagination without --cursor and use the new next_cursor value."); + return CommandExitCodes.UsageError; + } + var outline = reader.GetOutline(filePath, includeReferenceCounts: includeReferenceCounts); if (outline == null) { @@ -67,20 +85,19 @@ public static int RunOutline(string[] cmdArgs, JsonSerializerOptions jsonOptions return CommandExitCodes.NotFound; } - var kindFilters = BuildOutlineKindFilters(options.Kind); var filteredSymbols = ApplyOutlineKindFilters(outline.Symbols, kindFilters); var displaySourceSymbols = ApplyOutlineSort(filteredSymbols, outlineSortMode, includeDerivedMetadata); if (options.Json) { if (options.Compact) { - var payload = BuildOutlineJsonPayload(outline, displaySourceSymbols, kindFilters, outlineSortMode, options, jsonOptions, compact: true); + var payload = BuildOutlineJsonPayload(outline, displaySourceSymbols, kindFilters, outlineSortMode, options, cursorContext, jsonOptions, compact: true); AddActiveSqliteDiagnostics(payload); Console.WriteLine(payload.ToJsonString(jsonOptions)); } else if (HasOutlineJsonControls(options, kindFilters)) { - var payload = BuildOutlineJsonPayload(outline, displaySourceSymbols, kindFilters, outlineSortMode, options, jsonOptions, compact: false); + var payload = BuildOutlineJsonPayload(outline, displaySourceSymbols, kindFilters, outlineSortMode, options, cursorContext, jsonOptions, compact: false); AddActiveSqliteDiagnostics(payload); Console.WriteLine(payload.ToJsonString(jsonOptions)); } @@ -337,6 +354,7 @@ private static JsonObject BuildOutlineJsonPayload( IReadOnlyList<string> kindFilters, OutlineSortMode sortMode, QueryCommandOptions options, + PaginationCursorContext cursorContext, JsonSerializerOptions jsonOptions, bool compact) { @@ -352,7 +370,7 @@ private static JsonObject BuildOutlineJsonPayload( var compactOutline = BuildOutlineView(outline, remainingSymbols, totalMatchingSymbols); var compactTruncation = ApplyOutlineCompactCaps(compactOutline, compactLimit); var payload = JsonSerializer.SerializeToNode(compactOutline, CliJsonSerializerContextFactory.Create(jsonOptions).OutlineResult)!.AsObject(); - AddOutlinePagingJsonFields(payload, kindFilters, sortMode, options.SortExplicit, totalMatchingSymbols, offset, compactOutline.Symbols.Count, jsonOptions); + AddOutlinePagingJsonFields(payload, kindFilters, sortMode, options.SortExplicit, totalMatchingSymbols, offset, compactOutline.Symbols.Count, cursorContext, jsonOptions); ApplyOutlineFieldSelection(payload, compactOutline.Symbols, options, jsonOptions); AddCompactJsonFields(payload, compactLimit, compactTruncation); return payload; @@ -364,7 +382,7 @@ private static JsonObject BuildOutlineJsonPayload( : remainingSymbols; var pagedOutline = BuildOutlineView(outline, pageSymbols, totalMatchingSymbols); var pagedPayload = JsonSerializer.SerializeToNode(pagedOutline, CliJsonSerializerContextFactory.Create(jsonOptions).OutlineResult)!.AsObject(); - AddOutlinePagingJsonFields(pagedPayload, kindFilters, sortMode, options.SortExplicit, totalMatchingSymbols, offset, pageSymbols.Count, jsonOptions); + AddOutlinePagingJsonFields(pagedPayload, kindFilters, sortMode, options.SortExplicit, totalMatchingSymbols, offset, pageSymbols.Count, cursorContext, jsonOptions); ApplyOutlineFieldSelection(pagedPayload, pageSymbols, options, jsonOptions); return pagedPayload; } @@ -387,6 +405,7 @@ private static void AddOutlinePagingJsonFields( int totalSymbolCount, int offset, int returnedSymbolCount, + PaginationCursorContext cursorContext, JsonSerializerOptions jsonOptions) { var nextOffset = offset + returnedSymbolCount; @@ -394,8 +413,10 @@ private static void AddOutlinePagingJsonFields( payload["total_symbol_count"] = totalSymbolCount; payload["returned_symbol_count"] = returnedSymbolCount; payload["cursor_offset"] = offset; - payload["next_cursor"] = hasMore ? JsonValue.Create(FormatOutlineCursor(nextOffset)) : null; + payload["next_cursor"] = hasMore ? JsonValue.Create(FormatOutlineCursor(nextOffset, cursorContext)) : null; payload["has_more"] = hasMore; + if (cursorContext.ResultStableAt != null) + payload["result_stable_at"] = cursorContext.ResultStableAt; if (kindFilters.Count > 0) payload["kind_filter"] = JsonSerializer.SerializeToNode(kindFilters.ToList(), CliJsonSerializerContextFactory.Create(jsonOptions).ListString); if (sortExplicit || sortMode != OutlineSortMode.Source) diff --git a/src/CodeIndex/Cli/QueryCommandRunner.ResultEnvelopes.cs b/src/CodeIndex/Cli/QueryCommandRunner.ResultEnvelopes.cs index e18a299f2..ebad701ec 100644 --- a/src/CodeIndex/Cli/QueryCommandRunner.ResultEnvelopes.cs +++ b/src/CodeIndex/Cli/QueryCommandRunner.ResultEnvelopes.cs @@ -229,7 +229,8 @@ private static JsonObject BuildQueryContextJson(QueryCommandOptions options, Jso query["result_kinds"] = JsonSerializer.SerializeToNode(options.ResultKinds, CliJsonSerializerContextFactory.Create(jsonOptions).ListString); if (options.UnusedCursorOffset.HasValue) { - query["cursor"] = FormatUnusedCursor(options.UnusedCursorOffset.Value); + query["cursor"] = options.CursorValue + ?? string.Create(System.Globalization.CultureInfo.InvariantCulture, $"unused:{options.UnusedCursorOffset.Value}"); query["offset"] = options.UnusedCursorOffset.Value; } if (options.RankMode != ReferenceRankMode.Weighted) diff --git a/src/CodeIndex/Cli/QueryCommandRunner.Search.cs b/src/CodeIndex/Cli/QueryCommandRunner.Search.cs index 47d623ede..d761aba8a 100644 --- a/src/CodeIndex/Cli/QueryCommandRunner.Search.cs +++ b/src/CodeIndex/Cli/QueryCommandRunner.Search.cs @@ -638,9 +638,7 @@ private static int RunSearchCore( if (options.CountOnly) { - var counts = HasSearchOriginFilters(options) - ? CountFilteredSearchResults(reader, options, exactSearch) - : reader.CountSearchResults(options.Query, options.Lang, options.RawFts, options.PathPatterns, options.ExcludePaths, options.ExcludeTests, !options.NoDedup, options.Since, exactSearch, options.Prefix, !options.NoVisibilityRank, options.GuardFilters, options.GuardWindow, options.GuardScope, options.TokenBoundary); + var counts = CountSearchMatches(reader, options, exactSearch); var queryDiagnostics = DbReader.AnalyzeFtsQuery(options.Query, options.RawFts, options.Prefix, options.Lang); if (counts.Count == 0) { @@ -693,6 +691,9 @@ private static int RunSearchCore( } var ftsQueryDiagnostics = DbReader.AnalyzeFtsQuery(options.Query, options.RawFts, options.Prefix, options.Lang); + var groupedCounts = options.OutputFormat == OutputFormatGrouped + ? CountSearchMatches(reader, options, exactSearch) + : default; var displayRows = ReadSearchDisplayRows(reader, options, exactSearch); var selection = ApplySearchOutputSelection(displayRows, options); displayRows = selection.Rows; @@ -705,7 +706,7 @@ private static int RunSearchCore( } if (options.Json && options.OutputFormat == OutputFormatGrouped) { - var groupedExitCode = WriteGroupedSearchResults([], options, jsonOptions); + var groupedExitCode = WriteGroupedSearchResults([], groupedCounts, options, jsonOptions); return groupedExitCode == CommandExitCodes.Success ? ZeroResultExitCode(options) : groupedExitCode; } if (options.Json && TryWriteEmptySearchJsonWithOptionalByteLimit(options, jsonOptions, out var emptyJsonExitCode)) @@ -797,7 +798,7 @@ selection.TruncationReason is "sample" or "first_per_file" } if (options.OutputFormat == OutputFormatGrouped) { - return WriteGroupedSearchResults(displayRows, options, jsonOptions); + return WriteGroupedSearchResults(displayRows, groupedCounts, options, jsonOptions); } if (options.OutputFormat == OutputFormatCsv || options.OutputFormat == OutputFormatTsv) { diff --git a/src/CodeIndex/Cli/QueryCommandRunner.SearchRecipes.cs b/src/CodeIndex/Cli/QueryCommandRunner.SearchRecipes.cs index 96d81e436..d422ce496 100644 --- a/src/CodeIndex/Cli/QueryCommandRunner.SearchRecipes.cs +++ b/src/CodeIndex/Cli/QueryCommandRunner.SearchRecipes.cs @@ -2738,11 +2738,130 @@ private static string FormatSearchSnippetFocusMode(SearchSnippetFocusMode mode) private static string FormatSearchCursor(SearchResult result) => string.Create(CultureInfo.InvariantCulture, $"{result.Score:R}:{result.ChunkId}:{result.NextOffset}"); - private static string FormatUnusedCursor(int offset) - => string.Create(CultureInfo.InvariantCulture, $"unused:{offset}"); + private const string ScopedOffsetCursorPrefix = "page:v1:"; - private static string FormatOutlineCursor(int offset) - => string.Create(CultureInfo.InvariantCulture, $"outline:{offset}"); + private readonly record struct ScopedOffsetCursor( + string Scope, + int Offset, + string QueryFingerprint, + string GenerationFingerprint); + + private readonly record struct PaginationCursorContext( + string QueryFingerprint, + string GenerationFingerprint, + string? ResultStableAt); + + private static PaginationCursorContext BuildPaginationCursorContext( + DbReader reader, + string scope, + IEnumerable<string?> queryComponents) + { + var generation = reader.GetPaginationGeneration(); + return new( + BuildScopedCursorFingerprint(queryComponents.Prepend(scope)), + BuildScopedCursorFingerprint([generation.Identity]), + generation.StableAt); + } + + private static string? ValidateScopedOffsetCursor( + QueryCommandOptions options, + string expectedScope, + PaginationCursorContext context) + { + if (options.CursorValue == null + || !options.CursorValue.StartsWith(ScopedOffsetCursorPrefix, StringComparison.Ordinal)) + { + return null; + } + + if (!TryParseScopedOffsetCursor(options.CursorValue, out var cursor) + || !string.Equals(cursor.Scope, expectedScope, StringComparison.Ordinal)) + { + return "The pagination cursor does not belong to this command; restart required."; + } + if (!string.Equals(cursor.QueryFingerprint, context.QueryFingerprint, StringComparison.Ordinal)) + return "The pagination cursor does not match the current query scope, filters, or ordering; restart required."; + if (!string.Equals(cursor.GenerationFingerprint, context.GenerationFingerprint, StringComparison.Ordinal)) + return "The pagination cursor is stale because the index generation changed; restart required."; + return null; + } + + private static string FormatScopedOffsetCursor(string scope, int offset, PaginationCursorContext context) + { + var payload = string.Create( + CultureInfo.InvariantCulture, + $"{scope}\n{offset}\n{context.QueryFingerprint}\n{context.GenerationFingerprint}"); + var encoded = Convert.ToBase64String(Encoding.UTF8.GetBytes(payload)) + .TrimEnd('=') + .Replace('+', '-') + .Replace('/', '_'); + return ScopedOffsetCursorPrefix + encoded; + } + + private static string BuildScopedCursorFingerprint(IEnumerable<string?> components) + { + var canonical = new StringBuilder(); + foreach (var component in components) + { + if (component == null) + { + canonical.Append("-1:"); + continue; + } + canonical + .Append(component.Length.ToString(CultureInfo.InvariantCulture)) + .Append(':') + .Append(component); + } + var hash = SHA256.HashData(Encoding.UTF8.GetBytes(canonical.ToString())); + return Convert.ToHexString(hash.AsSpan(0, 8)).ToLowerInvariant(); + } + + private static bool TryParseScopedOffsetCursor(string value, out ScopedOffsetCursor cursor) + { + cursor = default; + if (!value.StartsWith(ScopedOffsetCursorPrefix, StringComparison.Ordinal)) + return false; + + var encoded = value[ScopedOffsetCursorPrefix.Length..] + .Replace('-', '+') + .Replace('_', '/'); + var paddingLength = (4 - encoded.Length % 4) % 4; + if (paddingLength > 0) + encoded += new string('=', paddingLength); + + string payload; + try + { + payload = Encoding.UTF8.GetString(Convert.FromBase64String(encoded)); + } + catch (FormatException) + { + return false; + } + + var parts = payload.Split('\n'); + if (parts.Length != 4 + || parts[0] is not ("unused" or "outline") + || !int.TryParse(parts[1], NumberStyles.None, CultureInfo.InvariantCulture, out var offset) + || offset < 0 + || parts[2].Length != 16 + || parts[3].Length != 16 + || !parts[2].All(Uri.IsHexDigit) + || !parts[3].All(Uri.IsHexDigit)) + { + return false; + } + + cursor = new ScopedOffsetCursor(parts[0], offset, parts[2], parts[3]); + return true; + } + + private static string FormatUnusedCursor(int offset, PaginationCursorContext context) + => FormatScopedOffsetCursor("unused", offset, context); + + private static string FormatOutlineCursor(int offset, PaginationCursorContext context) + => FormatScopedOffsetCursor("outline", offset, context); private static bool TryParseSearchCursor(string value, out SearchCursor cursor) { @@ -2772,20 +2891,36 @@ private static bool TryParseUnusedCursor(string value, out int offset) { offset = 0; const string prefix = "unused:"; - if (!value.StartsWith(prefix, StringComparison.Ordinal)) - return false; - return int.TryParse(value[prefix.Length..], NumberStyles.Integer, CultureInfo.InvariantCulture, out offset) - && offset >= 0; + if (value.StartsWith(prefix, StringComparison.Ordinal)) + { + return int.TryParse(value[prefix.Length..], NumberStyles.Integer, CultureInfo.InvariantCulture, out offset) + && offset >= 0; + } + if (TryParseScopedOffsetCursor(value, out var cursor) + && string.Equals(cursor.Scope, "unused", StringComparison.Ordinal)) + { + offset = cursor.Offset; + return true; + } + return false; } private static bool TryParseOutlineCursor(string value, out int offset) { offset = 0; const string prefix = "outline:"; - if (!value.StartsWith(prefix, StringComparison.Ordinal)) - return false; - return int.TryParse(value[prefix.Length..], NumberStyles.Integer, CultureInfo.InvariantCulture, out offset) - && offset >= 0; + if (value.StartsWith(prefix, StringComparison.Ordinal)) + { + return int.TryParse(value[prefix.Length..], NumberStyles.Integer, CultureInfo.InvariantCulture, out offset) + && offset >= 0; + } + if (TryParseScopedOffsetCursor(value, out var cursor) + && string.Equals(cursor.Scope, "outline", StringComparison.Ordinal)) + { + offset = cursor.Offset; + return true; + } + return false; } private static string QuoteReplayShellArg(string arg) diff --git a/src/CodeIndex/Cli/QueryCommandRunner.SearchResults.cs b/src/CodeIndex/Cli/QueryCommandRunner.SearchResults.cs index f96b3a849..3dd1c9585 100644 --- a/src/CodeIndex/Cli/QueryCommandRunner.SearchResults.cs +++ b/src/CodeIndex/Cli/QueryCommandRunner.SearchResults.cs @@ -371,24 +371,11 @@ private static bool IsSupportedSearchAggregationValue(string value) private static SearchOutputSelection ApplySearchOutputSelection(List<SearchDisplayRow> rows, QueryCommandOptions options) { var originalCount = rows.Count; - var firstPerFileTruncated = false; - if (options.FirstPerFile) - { - var beforeFirstPerFile = rows.Count; - rows = rows - .GroupBy(row => row.Result.Path, StringComparer.Ordinal) - .Select(group => group.First()) - .ToList(); - firstPerFileTruncated = rows.Count < beforeFirstPerFile; - } - - var sampleTruncated = false; - if (options.SampleSize.HasValue && rows.Count > options.SampleSize.Value) - { - sampleTruncated = true; - rows = SampleSearchRows(rows, options.SampleSize.Value); - } - + rows = ApplySearchPostSelectors( + rows, + options, + out var firstPerFileTruncated, + out var sampleTruncated); var postSelectionCount = rows.Count; var limitTruncated = rows.Count > options.Limit; if (limitTruncated) @@ -410,6 +397,32 @@ private static SearchOutputSelection ApplySearchOutputSelection(List<SearchDispl Math.Max(0, originalCount - postSelectionCount)); } + private static List<SearchDisplayRow> ApplySearchPostSelectors( + List<SearchDisplayRow> rows, + QueryCommandOptions options, + out bool firstPerFileTruncated, + out bool sampleTruncated) + { + firstPerFileTruncated = false; + if (options.FirstPerFile) + { + var beforeFirstPerFile = rows.Count; + rows = rows + .GroupBy(row => row.Result.Path, StringComparer.Ordinal) + .Select(group => group.First()) + .ToList(); + firstPerFileTruncated = rows.Count < beforeFirstPerFile; + } + + sampleTruncated = false; + if (options.SampleSize.HasValue && rows.Count > options.SampleSize.Value) + { + sampleTruncated = true; + rows = SampleSearchRows(rows, options.SampleSize.Value); + } + return rows; + } + private static List<SearchDisplayRow> SampleSearchRows(List<SearchDisplayRow> rows, int sampleSize) { if (sampleSize <= 0 || rows.Count <= sampleSize) @@ -427,19 +440,34 @@ private static List<SearchDisplayRow> SampleSearchRows(List<SearchDisplayRow> ro return sampled; } - private static int WriteGroupedSearchResults(List<SearchDisplayRow> rows, QueryCommandOptions options, JsonSerializerOptions jsonOptions) + private static int WriteGroupedSearchResults( + List<SearchDisplayRow> rows, + QueryCountResult matchedCounts, + QueryCommandOptions options, + JsonSerializerOptions jsonOptions) { var groups = BuildSearchFileGroups(rows, options); - var totalMatches = rows.Count; + var groupedMatchCount = rows.Count; + var emittedMatchCount = groups.Sum(group => group.Results.Count); + var omittedMatchCount = Math.Max(0, matchedCounts.Count - emittedMatchCount); + var truncated = omittedMatchCount > 0 || groups.Any(group => group.Truncated); var json = JsonSerializer.Serialize( new SearchFileGroupedJsonResult( JsonOutputContract.ApiVersion, options.Query!, - totalMatches, + matchedCounts.Count, + matchedCounts.Count, + groupedMatchCount, + emittedMatchCount, + omittedMatchCount, groups.Count, + matchedCounts.FileCount, rows.Select(row => row.Result.Path).Distinct(StringComparer.Ordinal).Count(), + matchedCounts.FileCount, options.GroupedPerFileLimit, - groups.Any(group => group.Truncated), + truncated, + truncated, + truncated ? "Increase --limit or --per-file-limit, or use a resumable JSON envelope." : null, groups), CliJsonSerializerContextFactory.Create(jsonOptions).SearchFileGroupedJsonResult); return WriteJsonObjectWithOptionalByteLimit( @@ -449,6 +477,26 @@ private static int WriteGroupedSearchResults(List<SearchDisplayRow> rows, QueryC "Reduce --limit, --per-file-limit, or increase --max-json-bytes."); } + private static QueryCountResult CountSearchMatches(DbReader reader, QueryCommandOptions options, bool exact) + => HasSearchOriginFilters(options) + ? CountFilteredSearchResults(reader, options, exact) + : reader.CountSearchResults( + options.Query!, + options.Lang, + options.RawFts, + options.PathPatterns, + options.ExcludePaths, + options.ExcludeTests, + !options.NoDedup, + options.Since, + exact, + options.Prefix, + !options.NoVisibilityRank, + options.GuardFilters, + options.GuardWindow, + options.GuardScope, + options.TokenBoundary); + private static void WriteGroupedSearchResultsHuman(List<SearchDisplayRow> rows, QueryCommandOptions options) { foreach (var group in BuildSearchFileGroups(rows, options)) @@ -1030,15 +1078,80 @@ private static List<string> CombineExclusiveSearchFilters(IReadOnlyList<string> private static List<SearchDisplayRow> ReadSearchDisplayRows(DbReader reader, QueryCommandOptions options, bool exact) { + var responseOffset = JsonEnvelopeWrapper.GetBoundedResponseOffset("search"); + var boundedPageLimit = JsonEnvelopeWrapper.GetBoundedResponseLimit("search"); + if (boundedPageLimit.HasValue + && (options.FirstPerFile || options.SampleSize.HasValue)) + { + List<SearchDisplayRow> boundedRows; + if (!HasSearchOriginFilters(options)) + { + boundedRows = BuildSearchDisplayRows( + ReadSearchResults( + reader, + options, + exact, + SearchOriginFilterMaxCandidates, + guardRequestedLimit: options.Limit), + options, + exact); + } + else + { + boundedRows = ReadOriginFilteredSearchDisplayRows( + reader, + options, + exact, + SearchOriginFilterMaxCandidates); + } + + var rawWindowComplete = boundedRows.Count < SearchOriginFilterMaxCandidates; + var selectedRows = ApplySearchPostSelectors( + boundedRows, + options, + out _, + out _); + JsonEnvelopeWrapper.ReportBoundedResponseTotal( + "search", + selectedRows.Count, + rawWindowComplete); + return selectedRows + .Skip(responseOffset) + .Take(boundedPageLimit.Value == int.MaxValue + ? int.MaxValue + : boundedPageLimit.Value + 1) + .ToList(); + } + + var requestedLimit = GetSearchDisplayCandidateLimit(options); + var requestedThroughOffset = responseOffset > int.MaxValue - requestedLimit + ? int.MaxValue + : responseOffset + requestedLimit; + List<SearchDisplayRow> rows; if (!HasSearchOriginFilters(options)) - return BuildSearchDisplayRows(ReadSearchResults(reader, options, exact, GetSearchDisplayCandidateLimit(options)), options, exact); + { + rows = BuildSearchDisplayRows( + ReadSearchResults(reader, options, exact, requestedThroughOffset), + options, + exact); + } + else + { + rows = ReadOriginFilteredSearchDisplayRows(reader, options, exact, requestedThroughOffset); + } - return ReadOriginFilteredSearchDisplayRows(reader, options, exact); + return responseOffset == 0 + ? rows + : rows.Skip(responseOffset).ToList(); } - private static List<SearchDisplayRow> ReadOriginFilteredSearchDisplayRows(DbReader reader, QueryCommandOptions options, bool exact) + private static List<SearchDisplayRow> ReadOriginFilteredSearchDisplayRows( + DbReader reader, + QueryCommandOptions options, + bool exact, + int requestedLimit) { - var requestedLimit = Math.Max(0, GetSearchDisplayCandidateLimit(options)); + requestedLimit = Math.Max(0, requestedLimit); if (requestedLimit == 0) return []; diff --git a/src/CodeIndex/Cli/QueryCommandRunner.Unused.cs b/src/CodeIndex/Cli/QueryCommandRunner.Unused.cs index 084a44ebb..bee8dad1f 100644 --- a/src/CodeIndex/Cli/QueryCommandRunner.Unused.cs +++ b/src/CodeIndex/Cli/QueryCommandRunner.Unused.cs @@ -36,7 +36,7 @@ public static int RunUnused(string[] cmdArgs, JsonSerializerOptions jsonOptions) if (options.SearchCursor.HasValue) { WriteUsageError( - "--cursor for unused must use the `unused:<offset>` cursor returned by a previous unused response.", + "--cursor for unused must use the opaque cursor returned by a previous unused response.", GetUsageLineOrThrow("unused"), "Use the `next_cursor` value from `cdidx unused --json`."); return CommandExitCodes.UsageError; @@ -44,15 +44,15 @@ public static int RunUnused(string[] cmdArgs, JsonSerializerOptions jsonOptions) if (options.OutlineCursorOffset.HasValue) { WriteUsageError( - "--cursor for unused must use the `unused:<offset>` cursor returned by a previous unused response.", + "--cursor for unused must use the opaque cursor returned by a previous unused response.", GetUsageLineOrThrow("unused"), - "`outline:<offset>` cursors are for `cdidx outline <path>`."); + "Outline cursors are for `cdidx outline <path>`."); return CommandExitCodes.UsageError; } if (options.DependencyCycleCursor.HasValue) { WriteUsageError( - "--cursor for unused must use the `unused:<offset>` cursor returned by a previous unused response.", + "--cursor for unused must use the opaque cursor returned by a previous unused response.", GetUsageLineOrThrow("unused"), "Dependency-cycle cursors are for `cdidx deps --cycles`."); return CommandExitCodes.UsageError; @@ -89,6 +89,38 @@ public static int RunUnused(string[] cmdArgs, JsonSerializerOptions jsonOptions) baseSqlGraphSignal, reader.ScopeMayIncludeSqlSymbols(options.Kind, options.Lang, unusedScope.PathPatterns, unusedScope.ExcludePaths, unusedScope.ExcludeTests)); var applyDefaultSuppressions = ShouldApplyUnusedDefaultSuppressions(options, unusedScope); + var cursorComponents = new List<string?> + { + options.Kind, + options.Lang, + options.UnusedBucket, + options.MinUnusedConfidence, + options.UnusedActionable.ToString(CultureInfo.InvariantCulture), + options.All.ToString(CultureInfo.InvariantCulture), + applyDefaultSuppressions.ToString(CultureInfo.InvariantCulture), + unusedScope.ExcludeTests.ToString(CultureInfo.InvariantCulture), + unusedScope.AppliedSourceDefaults.ToString(CultureInfo.InvariantCulture), + "include_generated", + options.IncludeGenerated.ToString(CultureInfo.InvariantCulture), + "path", + }; + cursorComponents.AddRange(unusedScope.PathPatterns.Order(StringComparer.Ordinal)); + cursorComponents.Add("exclude_path"); + cursorComponents.AddRange(unusedScope.ExcludePaths.Order(StringComparer.Ordinal)); + cursorComponents.Add("visibility"); + cursorComponents.AddRange(unusedScope.VisibilityFilters.Order(StringComparer.Ordinal)); + cursorComponents.Add("exclude_visibility"); + cursorComponents.AddRange(unusedScope.ExcludeVisibilityFilters.Order(StringComparer.Ordinal)); + var cursorContext = BuildPaginationCursorContext(reader, "unused", cursorComponents); + var cursorValidationError = ValidateScopedOffsetCursor(options, "unused", cursorContext); + if (cursorValidationError != null) + { + WriteUsageError( + cursorValidationError, + GetUsageLineOrThrow("unused"), + "Restart unused pagination without --cursor and use the new next_cursor value."); + return CommandExitCodes.UsageError; + } UnusedCountResult CountUnusedSymbolsDetailedForCurrentQuery(Func<UnusedSymbolResult, bool>? resultFilter = null) { @@ -233,7 +265,7 @@ UnusedCountResult CountUnusedSymbolsDetailedForCurrentQuery(Func<UnusedSymbolRes var nextOffset = pageOffset + options.Limit; var nextCursor = pageableResults.Count > nextOffset && IsUnusedCursorOffsetWithinFetchCap(options.Limit, nextOffset) - ? FormatUnusedCursor(nextOffset) + ? FormatUnusedCursor(nextOffset, cursorContext) : null; var sqlGraphSignal = results.Count == 0 ? zeroResultSqlGraphSignal @@ -256,6 +288,7 @@ UnusedCountResult CountUnusedSymbolsDetailedForCurrentQuery(Func<UnusedSymbolRes options, unusedScope, nextCursor: nextCursor, + cursorContext: cursorContext, suppression: suppression)); } else if (suppression.Applied && GetUnusedSuppressedCount(suppression) > 0) @@ -281,7 +314,7 @@ UnusedCountResult CountUnusedSymbolsDetailedForCurrentQuery(Func<UnusedSymbolRes if (options.Json) { - Console.WriteLine(BuildUnusedJsonPayload(results, graphSupported, graphSupportReason, sqlGraphSignal, hdlGraphSignal, reader._hasReferencesTable, jsonOptions, options, unusedScope, byBucket: byBucket, nextCursor: nextCursor, suppression: suppression)); + Console.WriteLine(BuildUnusedJsonPayload(results, graphSupported, graphSupportReason, sqlGraphSignal, hdlGraphSignal, reader._hasReferencesTable, jsonOptions, options, unusedScope, byBucket: byBucket, nextCursor: nextCursor, cursorContext: cursorContext, suppression: suppression)); } else { @@ -712,7 +745,7 @@ internal static JsonObject BuildUnusedBucketTaxonomyJson() _ => "Unknown unused-symbol bucket.", }; - private static string BuildUnusedJsonPayload(IEnumerable<UnusedSymbolResult> results, bool? graphSupported, string? graphSupportReason, SqlGraphContractSignal sqlGraphSignal, HdlGraphContractSignal hdlGraphSignal, bool hasReferencesTable, JsonSerializerOptions jsonOptions, QueryCommandOptions? queryOptions = null, UnusedAuditScopeFilters? unusedScope = null, bool byBucket = false, string? nextCursor = null, UnusedDefaultSuppressionResult? suppression = null) + private static string BuildUnusedJsonPayload(IEnumerable<UnusedSymbolResult> results, bool? graphSupported, string? graphSupportReason, SqlGraphContractSignal sqlGraphSignal, HdlGraphContractSignal hdlGraphSignal, bool hasReferencesTable, JsonSerializerOptions jsonOptions, QueryCommandOptions? queryOptions = null, UnusedAuditScopeFilters? unusedScope = null, bool byBucket = false, string? nextCursor = null, PaginationCursorContext? cursorContext = null, UnusedDefaultSuppressionResult? suppression = null) { var resultList = results as List<UnusedSymbolResult> ?? results.ToList(); var payload = new JsonObject @@ -729,6 +762,8 @@ private static string BuildUnusedJsonPayload(IEnumerable<UnusedSymbolResult> res payload["default_suppression"] = BuildUnusedDefaultSuppressionJson(suppression, jsonOptions); if (nextCursor != null) payload["next_cursor"] = nextCursor; + if (cursorContext?.ResultStableAt != null) + payload["result_stable_at"] = cursorContext.Value.ResultStableAt; if (queryOptions?.Compact == true) { payload["compact"] = true; diff --git a/src/CodeIndex/Cli/QueryCommandRunner.cs b/src/CodeIndex/Cli/QueryCommandRunner.cs index 116fa2d47..ea6a1ccd8 100644 --- a/src/CodeIndex/Cli/QueryCommandRunner.cs +++ b/src/CodeIndex/Cli/QueryCommandRunner.cs @@ -203,6 +203,7 @@ public sealed class QueryCommandOptions public SearchCursor? SearchCursor { get; init; } public int? UnusedCursorOffset { get; init; } public int? OutlineCursorOffset { get; init; } + public string? CursorValue { get; init; } public DependencyCycleCursor? DependencyCycleCursor { get; init; } public List<SearchNamedQuery> NamedSearchQueries { get; init; } = []; public bool LanguagesIndexedOnly { get; init; } diff --git a/src/CodeIndex/Database/DbReader.FilesStatus.cs b/src/CodeIndex/Database/DbReader.FilesStatus.cs index c9c8415c0..40ae80702 100644 --- a/src/CodeIndex/Database/DbReader.FilesStatus.cs +++ b/src/CodeIndex/Database/DbReader.FilesStatus.cs @@ -118,7 +118,7 @@ internal static int? LegacyResourceReadSqliteVmStepLimitForTesting set => LegacyResourceReadSqliteVmStepLimitOverride.Value = value; } - public FindResults FindInFiles(string query, int limit, string? lang = null, IReadOnlyList<string>? pathPatterns = null, IReadOnlyList<string>? excludePathPatterns = null, bool excludeTests = false, int before = 0, int after = 0, bool exact = false, int maxLineWidth = LineWidthFormatter.DefaultMaxLineWidth, int? focusLine = null, int? focusColumn = null, bool regex = false, int? maxCandidateFiles = null, int? maxLinesScanned = null, int offset = 0, bool useIndexedLiteralCandidates = false) + public FindResults FindInFiles(string query, int limit, string? lang = null, IReadOnlyList<string>? pathPatterns = null, IReadOnlyList<string>? excludePathPatterns = null, bool excludeTests = false, int before = 0, int after = 0, bool exact = false, int maxLineWidth = LineWidthFormatter.DefaultMaxLineWidth, int? focusLine = null, int? focusColumn = null, bool regex = false, int? maxCandidateFiles = null, int? maxLinesScanned = null, int offset = 0, bool useIndexedLiteralCandidates = false, string? resumePath = null, int? resumeLine = null) { if (string.IsNullOrWhiteSpace(query) || limit <= 0) return new FindResults([], new FindScanSummary(0, 0, 0)); @@ -144,25 +144,41 @@ public FindResults FindInFiles(string query, int limit, string? lang = null, IRe var linesScanned = 0; var truncated = false; string? truncationReason = null; + string? nextPath = null; + int? nextLine = null; var matchesSkipped = 0; offset = Math.Max(0, offset); + var resumePending = resumePath is not null; var results = new List<FileFindResult>(); using var fileReader = fileCmd.ExecuteTrackedReader(); while (fileReader.TrackedRead()) { if (results.Count >= limit) break; + + var fileId = fileReader.GetInt64(0); + var path = fileReader.GetString(1); + var fileLang = GetNullableString(fileReader, 2); + var totalLines = fileReader.GetInt32(3); + if (resumePending) + { + if (!string.Equals(path, resumePath, StringComparison.Ordinal)) + continue; + resumePending = false; + } if (maxCandidateFiles.HasValue && filesScanned >= maxCandidateFiles.Value) { truncated = true; truncationReason ??= "candidate_file_limit"; + nextPath = path; + nextLine = 1; break; } - var fileId = fileReader.GetInt64(0); - var path = fileReader.GetString(1); - var fileLang = GetNullableString(fileReader, 2); - var totalLines = fileReader.GetInt32(3); + var firstEligibleLine = string.Equals(path, resumePath, StringComparison.Ordinal) + ? Math.Max(1, resumeLine ?? 1) + : 1; + var firstContextLine = Math.Max(1, firstEligibleLine - before); filesScanned++; if (totalLines <= 0) continue; @@ -178,18 +194,25 @@ public FindResults FindInFiles(string query, int limit, string? lang = null, IRe { if (indexedLine.Number > totalLines) break; + if (indexedLine.Number < firstContextLine) + continue; + var eligibleForMatch = indexedLine.Number >= firstEligibleLine; if (maxLinesScanned.HasValue && linesScanned >= maxLinesScanned.Value) { truncated = true; truncationReason ??= "line_scan_limit"; + nextPath = path; + nextLine = indexedLine.Number; stopScanning = true; break; } - linesScanned++; + if (eligibleForMatch) + linesScanned++; AddLineToFindWindow(indexedLine, snippetWindow, snippetLinesByNumber); if ((matchesSkipped < offset || acceptedMatches < limit) + && eligibleForMatch && (!focusLine.HasValue || indexedLine.Number == focusLine.Value)) { foreach (var lineMatch in EnumerateFindLineMatches( @@ -259,7 +282,9 @@ public FindResults FindInFiles(string query, int limit, string? lang = null, IRe maxCandidateFiles, maxLinesScanned, searchPlan.Strategy, - searchPlan.FallbackReason)); + searchPlan.FallbackReason, + nextPath, + nextLine)); } public int CountFindCandidateFiles(string? lang = null, IReadOnlyList<string>? pathPatterns = null, IReadOnlyList<string>? excludePathPatterns = null, bool excludeTests = false) @@ -1876,6 +1901,19 @@ public FreshnessHintResult GetFreshnessHint() }; } + internal (string Identity, string? StableAt) GetPaginationGeneration() + { + var freshness = GetFreshnessHint(); + var indexedHeadSha = TryGetMetaStringInternal(DbContext.IndexedHeadShaMetaKey); + var indexedHeadTimestamp = TryGetMetaStringInternal(DbContext.IndexedHeadTimestampMetaKey); + var indexedAt = freshness.IndexedAt?.ToUniversalTime().ToString("O", CultureInfo.InvariantCulture); + var stableAt = indexedHeadTimestamp ?? indexedAt; + var identity = string.Create( + CultureInfo.InvariantCulture, + $"{indexedHeadSha ?? "no-indexed-head"}\n{indexedHeadTimestamp ?? "no-indexed-head-timestamp"}\n{indexedAt ?? "no-indexed-at"}\n{freshness.FileCount}"); + return (identity, stableAt); + } + private (DateTime? IndexedAt, DateTime? LatestModified) GetWorkspaceFreshness() { return ( diff --git a/src/CodeIndex/Models/QueryResults.cs b/src/CodeIndex/Models/QueryResults.cs index f11eb7d4c..a1948474d 100644 --- a/src/CodeIndex/Models/QueryResults.cs +++ b/src/CodeIndex/Models/QueryResults.cs @@ -89,7 +89,9 @@ public readonly record struct FindScanSummary( int? CandidateFileLimit = null, int? LineLimit = null, string SearchStrategy = "line_scan", - string? SearchFallbackReason = null); + string? SearchFallbackReason = null, + string? NextPath = null, + int? NextLine = null); public readonly record struct FindCountResult(int Count, int FileCount, FindScanSummary Scan); diff --git a/tests/CodeIndex.Tests/ConsoleUiTests.cs b/tests/CodeIndex.Tests/ConsoleUiTests.cs index 234197921..935cd123d 100644 --- a/tests/CodeIndex.Tests/ConsoleUiTests.cs +++ b/tests/CodeIndex.Tests/ConsoleUiTests.cs @@ -385,11 +385,11 @@ public void PrintUsage_WithoutBanner_HidesAsciiArtAndEasterEggFlags() Assert.Contains("cdidx callers <query>|--query <query>|-- <query>", output); Assert.Contains("cdidx callees <query>|--query <query>|-- <query>", output); AssertSearchUsageFragments(output); - Assert.Contains("cdidx definition <query>|--query <query>|-- <query> [--db <path>] [--json] [--format <text|json|count|compact|csv|tsv|lsp|qf|sarif>] [--fields <csv>] [--cursor <response:v1:offset:fingerprint>] [--max-json-bytes <n>] [--verbose] [--limit <n>|--top <n>] [--lang <lang>] [--kind <kind>] [--visibility <v[,v]>] [--exclude-visibility <v[,v]>] [--path <glob>] [--exclude-path <glob>] [--exclude-tests] [--body] [--exact|--exact-name] [--count] [--group-partials] [--since <datetime>]", output); - Assert.Contains("cdidx references <query>|--query <query>|-- <query> [--db <path>] [--json] [--format <text|json|count|compact|csv|tsv|lsp|qf|sarif>] [--fields <csv>] [--cursor <response:v1:offset:fingerprint>] [--max-json-bytes <n>] [--verbose] [--limit <n>|--top <n>] [--lang <lang>] [--kind <kind>] [--path <glob>] [--exclude-path <glob>] [--exclude-tests] [--body] [--snippet-lines <n>] [--max-line-width <n>] [--exact|--exact-name] [--count]", output); + Assert.Contains("cdidx definition <query>|--query <query>|-- <query> [--db <path>] [--json] [--format <text|json|count|compact|csv|tsv|lsp|qf|sarif>] [--fields <csv>] [--cursor <next_cursor>] [--max-json-bytes <n>] [--verbose] [--limit <n>|--top <n>] [--lang <lang>] [--kind <kind>] [--visibility <v[,v]>] [--exclude-visibility <v[,v]>] [--path <glob>] [--exclude-path <glob>] [--exclude-tests] [--body] [--exact|--exact-name] [--count] [--group-partials] [--since <datetime>]", output); + Assert.Contains("cdidx references <query>|--query <query>|-- <query> [--db <path>] [--json] [--format <text|json|count|compact|csv|tsv|lsp|qf|sarif>] [--fields <csv>] [--cursor <next_cursor>] [--max-json-bytes <n>] [--verbose] [--limit <n>|--top <n>] [--lang <lang>] [--kind <kind>] [--path <glob>] [--exclude-path <glob>] [--exclude-tests] [--body] [--snippet-lines <n>] [--max-line-width <n>] [--exact|--exact-name] [--count]", output); Assert.Contains("cdidx inspect <query>|--query <query>|-- <query> [--db <path>] [--json] [--format <text|json|compact>] [--pretty] [--compact] [--fields <csv>] [--outline-only] [--body-only] [--max-json-bytes <n>] [--verbose] [--limit <n>|--top <n>] [--lang <lang>] [--kind <kind>] [--path <glob>] [--exclude-path <glob>] [--exclude-tests] [--body] [--body-start <line>] [--body-lines <n>|--body-line-count <n>] [--context <n>|--before <n>|--after <n>] [--max-line-width <n>] [--exact|--exact-name] [--group-partials]", output); Assert.Contains("cdidx inspect --path <file> --line <line> [--end-line <line>] [--db <path>] [--json] [--format <text|json|compact>] [--pretty] [--compact] [--fields <csv>] [--outline-only] [--body-only] [--max-json-bytes <n>] [--body] [--body-start <line>] [--body-lines <n>|--body-line-count <n>] [--context <n>|--before <n>|--after <n>] [--max-line-width <n>]", output); - Assert.Contains("cdidx outline <path> [--db <path>] [--json] [--pretty] [--compact] [--verbose] [--limit <n>|--top <n>] [--cursor <outline:offset>] [--sort <source|kind|references|size|complexity|path|name>] [--kind <kind[,kind]>] [--outline-fields <csv>]", output); + Assert.Contains("cdidx outline <path> [--db <path>] [--json] [--pretty] [--compact] [--verbose] [--limit <n>|--top <n>] [--cursor <next_cursor>] [--sort <source|kind|references|size|complexity|path|name>] [--kind <kind[,kind]>] [--outline-fields <csv>]", output); Assert.Contains("--snippet-lines <n> search/find snippet length (1-20, default: search 8; find 1)", output); Assert.Contains("--snippet-focus <mode> search only: long-line focus mode (leftmost|quality|proximity, default: quality)", output); Assert.Contains("--max-line-width <n> search/references/callers/callees/find/excerpt/impact/inspect only: clamp very long single-line snippet/context/excerpt payloads (`0` disables clamping; default: 512)", output); @@ -423,17 +423,17 @@ public void PrintUsage_WithoutBanner_HidesAsciiArtAndEasterEggFlags() Assert.Contains("--focus-column <n> find/excerpt: focus a specific 1-based column", output); Assert.Contains("--focus-line <line> find/excerpt: focus a line", output); Assert.Contains("excerpt keeps the leading window when no column is supplied", output); - Assert.Contains("cdidx map [--db <path>] [--json] [--format <text|json|compact|issue-drafts>] [--pretty] [--compact] [--fields <csv>] [--cursor <response:v1:offset:fingerprint>] [--summary-only] [--verbose] [--limit <n>|--top <n>] [--lang <lang>] [--path <glob>] [--exclude-path <glob>] [--exclude-tests] [--bytes] [--sections <summary,tree,languages,hotspots,metrics|list>] [--depth <n>] [--min-entrypoint-confidence <0.0..1.0>] [--max-json-bytes <n>]", output); - Assert.Contains("cdidx symbols [query|--query <query>|-- <query>] [--name <name>] [--db <path>] [--json[=ndjson|array]] [--compact] [--format <text|json|count|compact|lsp|qf|sarif>] [--summary-only] [--max-json-bytes <n>] [--allow-partial] [--verbose] [--limit <n>|--top <n>] [--sort <hotspot|references|size|complexity|path>] [--lang <lang>] [--kind <kind>] [--visibility <v[,v]>] [--exclude-visibility <v[,v]>] [--path <glob>] [--exclude-path <glob>] [--exclude-tests] [--exact|--exact-name] [--count] [--group-partials] [--since <datetime>]", output); + Assert.Contains("cdidx map [--db <path>] [--json] [--format <text|json|compact|issue-drafts>] [--pretty] [--compact] [--fields <csv>] [--cursor <next_cursor>] [--summary-only] [--verbose] [--limit <n>|--top <n>] [--lang <lang>] [--path <glob>] [--exclude-path <glob>] [--exclude-tests] [--bytes] [--sections <summary,tree,languages,hotspots,metrics|list>] [--depth <n>] [--min-entrypoint-confidence <0.0..1.0>] [--max-json-bytes <n>]", output); + Assert.Contains("cdidx symbols [query|--query <query>|-- <query>] [--name <name>] [--db <path>] [--json[=ndjson|array]] [--compact] [--format <text|json|count|compact|lsp|qf|sarif>] [--summary-only] [--cursor <next_cursor>] [--max-json-bytes <n>] [--allow-partial] [--verbose] [--limit <n>|--top <n>] [--sort <hotspot|references|size|complexity|path>] [--lang <lang>] [--kind <kind>] [--visibility <v[,v]>] [--exclude-visibility <v[,v]>] [--path <glob>] [--exclude-path <glob>] [--exclude-tests] [--exact|--exact-name] [--count] [--group-partials] [--since <datetime>]", output); Assert.Contains("--sort <mode> Symbols/outline: order audit output by a ranking", output); Assert.Contains("source, kind, references, size, complexity, path, and name", output); - Assert.Contains("cdidx files [query|<glob>|--query <query>|-- <query>] [--db <path>] [--json[=ndjson|array]] [--format <text|json|count|compact>] [--summary-only] [--max-json-bytes <n>] [--allow-partial] [--verbose] [--limit <n>|--top <n>] [--lang <lang>] [--path <glob>] [--exclude-path <glob>] [--exclude-tests] [--count] [--since <datetime>] [--bytes]", output); + Assert.Contains("cdidx files [query|<glob>|--query <query>|-- <query>] [--db <path>] [--json[=ndjson|array]] [--format <text|json|count|compact>] [--summary-only] [--cursor <next_cursor>] [--max-json-bytes <n>] [--allow-partial] [--verbose] [--limit <n>|--top <n>] [--lang <lang>] [--path <glob>] [--exclude-path <glob>] [--exclude-tests] [--count] [--since <datetime>] [--bytes]", output); Assert.Contains("cdidx validate [--db <path>] [--json[=array]] [--format <text|json|count|compact|csv|tsv|lsp|qf|sarif>] [--verbose] [--limit <n>|--top <n>] [--kind <kind>] [--severity <info|warning|error>] [--path <glob>]", output); Assert.Contains("Note: if a query itself starts with '-', pass it with --query <query> or -- <query>", output); Assert.DoesNotContain("cdidx validate [--db <path>] [--json] [--limit <n>] [--lang <lang>]", output); Assert.Contains("cdidx unused [--db <path>] [--json] [--compact]", output); - Assert.Contains("[--cursor <unused:offset>] [--audit-scope <source|all>]", output); - Assert.Contains("cdidx hotspots [--db <path>] [--json] [--format <text|json|count|compact>] [--compact] [--fields <csv>] [--cursor <response:v1:offset:fingerprint>] [--summary-only] [--max-json-bytes <n>] [--verbose] [--limit <n>|--top <n>] [--kind <kind>] [--visibility <v[,v]>] [--exclude-visibility <v[,v]>] [--lang <lang>] [--path <glob>] [--exclude-path <glob>] [--exclude-tests] [--count] [--group-by <symbol|file|statement>] [--group-by-name]", output); + Assert.Contains("[--cursor <next_cursor>] [--audit-scope <source|all>]", output); + Assert.Contains("cdidx hotspots [--db <path>] [--json] [--format <text|json|count|compact>] [--compact] [--fields <csv>] [--cursor <next_cursor>] [--summary-only] [--max-json-bytes <n>] [--verbose] [--limit <n>|--top <n>] [--kind <kind>] [--visibility <v[,v]>] [--exclude-visibility <v[,v]>] [--lang <lang>] [--path <glob>] [--exclude-path <glob>] [--exclude-tests] [--count] [--group-by <symbol|file|statement>] [--group-by-name]", output); Assert.Contains("--json Output as JSON (search/symbols/files stream", output); Assert.Contains("search/symbols/files/validate accept --json=array for one", output); Assert.Contains("--lang <lang> Filter by language (aliases: bat, cmd, cshtml, razor, ts, tsx, cts, mts)", output); @@ -546,11 +546,11 @@ public void PrintUsage_QueryLinesMatchImplementedOptions() var output = CaptureFullUsageOutput(showBanner: false); AssertSearchUsageFragments(output); - Assert.Contains("cdidx symbols [query|--query <query>|-- <query>] [--name <name>] [--db <path>] [--json[=ndjson|array]] [--compact] [--format <text|json|count|compact|lsp|qf|sarif>] [--summary-only] [--max-json-bytes <n>] [--allow-partial] [--verbose] [--limit <n>|--top <n>] [--sort <hotspot|references|size|complexity|path>] [--lang <lang>] [--kind <kind>] [--visibility <v[,v]>] [--exclude-visibility <v[,v]>] [--path <glob>] [--exclude-path <glob>] [--exclude-tests] [--exact|--exact-name] [--count] [--group-partials] [--since <datetime>]", output); - Assert.Contains("cdidx files [query|<glob>|--query <query>|-- <query>] [--db <path>] [--json[=ndjson|array]] [--format <text|json|count|compact>] [--summary-only] [--max-json-bytes <n>] [--allow-partial] [--verbose] [--limit <n>|--top <n>] [--lang <lang>] [--path <glob>] [--exclude-path <glob>] [--exclude-tests] [--count] [--since <datetime>] [--bytes]", output); - Assert.Contains("cdidx hotspots [--db <path>] [--json] [--format <text|json|count|compact>] [--compact] [--fields <csv>] [--cursor <response:v1:offset:fingerprint>] [--summary-only] [--max-json-bytes <n>] [--verbose] [--limit <n>|--top <n>] [--kind <kind>] [--visibility <v[,v]>] [--exclude-visibility <v[,v]>] [--lang <lang>] [--path <glob>] [--exclude-path <glob>] [--exclude-tests] [--count]", output); + Assert.Contains("cdidx symbols [query|--query <query>|-- <query>] [--name <name>] [--db <path>] [--json[=ndjson|array]] [--compact] [--format <text|json|count|compact|lsp|qf|sarif>] [--summary-only] [--cursor <next_cursor>] [--max-json-bytes <n>] [--allow-partial] [--verbose] [--limit <n>|--top <n>] [--sort <hotspot|references|size|complexity|path>] [--lang <lang>] [--kind <kind>] [--visibility <v[,v]>] [--exclude-visibility <v[,v]>] [--path <glob>] [--exclude-path <glob>] [--exclude-tests] [--exact|--exact-name] [--count] [--group-partials] [--since <datetime>]", output); + Assert.Contains("cdidx files [query|<glob>|--query <query>|-- <query>] [--db <path>] [--json[=ndjson|array]] [--format <text|json|count|compact>] [--summary-only] [--cursor <next_cursor>] [--max-json-bytes <n>] [--allow-partial] [--verbose] [--limit <n>|--top <n>] [--lang <lang>] [--path <glob>] [--exclude-path <glob>] [--exclude-tests] [--count] [--since <datetime>] [--bytes]", output); + Assert.Contains("cdidx hotspots [--db <path>] [--json] [--format <text|json|count|compact>] [--compact] [--fields <csv>] [--cursor <next_cursor>] [--summary-only] [--max-json-bytes <n>] [--verbose] [--limit <n>|--top <n>] [--kind <kind>] [--visibility <v[,v]>] [--exclude-visibility <v[,v]>] [--lang <lang>] [--path <glob>] [--exclude-path <glob>] [--exclude-tests] [--count]", output); Assert.Contains("cdidx unused", output); - Assert.Contains("[--cursor <unused:offset>] [--audit-scope <source|all>]", output); + Assert.Contains("[--cursor <next_cursor>] [--audit-scope <source|all>]", output); Assert.Contains("cdidx license", output); Assert.Contains("cdidx completions <shell>", output); Assert.Contains("cdidx --completions <shell>", output); diff --git a/tests/CodeIndex.Tests/JsonEnvelopeWrapperIssue4585Tests.cs b/tests/CodeIndex.Tests/JsonEnvelopeWrapperIssue4585Tests.cs index 0993332e0..5784dabf8 100644 --- a/tests/CodeIndex.Tests/JsonEnvelopeWrapperIssue4585Tests.cs +++ b/tests/CodeIndex.Tests/JsonEnvelopeWrapperIssue4585Tests.cs @@ -106,7 +106,7 @@ public void BoundedProjection_CursorIsStableAndByteBudgetTrimsProjectedRows_Issu Assert.True(firstMetadata.GetProperty("total_count_authoritative").GetBoolean()); Assert.Equal(40, firstMetadata.GetProperty("total_count").GetInt32()); var cursor = Assert.IsType<string>(firstMetadata.GetProperty("next_cursor").GetString()); - Assert.StartsWith("response:v1:1:", cursor, StringComparison.Ordinal); + Assert.StartsWith("response:v2:", cursor, StringComparison.Ordinal); var firstLine = firstDocument.RootElement.GetProperty("results")[0].GetProperty("line").GetInt32(); var secondArgs = firstArgs.Concat(["--cursor", cursor!]).ToArray(); @@ -418,9 +418,10 @@ public void Map_SelectedSectionPagesRowsAndScalarProjectionOmitsSections_Issue45 Assert.Equal(3, secondDocument.RootElement.GetProperty("metadata").GetProperty("total_count").GetInt32()); Assert.NotEqual(firstPath, secondDocument.RootElement.GetProperty("results")[0].GetProperty("path").GetString()); - var cursorFingerprint = cursor[cursor.LastIndexOf(':')..]; + var lastCursor = Assert.IsType<string>( + secondDocument.RootElement.GetProperty("metadata").GetProperty("next_cursor").GetString()); var (lastExitCode, lastStdout, lastStderr) = CaptureConsole(() => ProgramRunner.Run( - firstArgs.Concat(["--cursor", "response:v1:2" + cursorFingerprint]).ToArray(), + firstArgs.Concat(["--cursor", lastCursor]).ToArray(), _jsonOptions, "1.0.0-test")); diff --git a/tests/CodeIndex.Tests/JsonEnvelopeWrapperIssue4730Tests.cs b/tests/CodeIndex.Tests/JsonEnvelopeWrapperIssue4730Tests.cs new file mode 100644 index 000000000..ed8aa40e0 --- /dev/null +++ b/tests/CodeIndex.Tests/JsonEnvelopeWrapperIssue4730Tests.cs @@ -0,0 +1,496 @@ +using System.Text; +using System.Text.Json; +using CodeIndex.Cli; +using CodeIndex.Database; +using CodeIndex.Models; + +namespace CodeIndex.Tests; + +[Collection("Console sensitive")] +public sealed class JsonEnvelopeWrapperIssue4730Tests +{ + private readonly JsonSerializerOptions _jsonOptions = ProgramRunner.CreateDefaultJsonOptions(); + + [Fact] + public void Search_CompactAndArrayEnvelopeExposeAuthoritativeResumableMetadata_Issue4730() + { + var projectRoot = TestProjectHelper.CreateTempProject("discovery_search_paging_4730"); + try + { + var dbPath = TestProjectHelper.CreateProjectDb(projectRoot); + for (var index = 1; index <= 3; index++) + TestProjectHelper.InsertIndexedFile(dbPath, $"src/Needle{index}.txt", "text", $"Needle marker {index}\n"); + + var compactArgs = new[] + { + "search", "Needle", "--db", dbPath, "--format", "compact", "--limit", "1", + }; + var (compactExitCode, compactStdout, compactStderr) = CaptureConsole(() => + ProgramRunner.Run(compactArgs, _jsonOptions, "1.0.0-test")); + + Assert.Equal(CommandExitCodes.Success, compactExitCode); + Assert.Equal(string.Empty, compactStderr); + using var compactDocument = JsonDocument.Parse(compactStdout); + var compact = compactDocument.RootElement; + Assert.Equal(3, compact.GetProperty("total_count").GetInt32()); + Assert.Equal(2, compact.GetProperty("omitted_count").GetInt32()); + Assert.True(compact.GetProperty("truncated").GetBoolean()); + Assert.False(string.IsNullOrWhiteSpace(compact.GetProperty("result_stable_at").GetString())); + var compactCursor = Assert.IsType<string>(compact.GetProperty("next_cursor").GetString()); + Assert.StartsWith("response:v2:", compactCursor, StringComparison.Ordinal); + var firstFile = compact.GetProperty("results")[0].GetProperty("file").GetString(); + + var (secondExitCode, secondStdout, secondStderr) = CaptureConsole(() => + ProgramRunner.Run(compactArgs.Concat(["--cursor", compactCursor]).ToArray(), _jsonOptions, "1.0.0-test")); + + Assert.Equal(CommandExitCodes.Success, secondExitCode); + Assert.Equal(string.Empty, secondStderr); + using var secondDocument = JsonDocument.Parse(secondStdout); + Assert.Equal(1, secondDocument.RootElement.GetProperty("cursor_offset").GetInt32()); + Assert.NotEqual(firstFile, secondDocument.RootElement.GetProperty("results")[0].GetProperty("file").GetString()); + + var (arrayExitCode, arrayStdout, arrayStderr) = CaptureConsole(() => ProgramRunner.Run( + ["search", "Needle", "--db", dbPath, "--json=array", "--json-envelope", "--limit", "1"], + _jsonOptions, + "1.0.0-test")); + + Assert.Equal(CommandExitCodes.Success, arrayExitCode); + Assert.Equal(string.Empty, arrayStderr); + using var arrayDocument = JsonDocument.Parse(arrayStdout); + var arrayMetadata = arrayDocument.RootElement.GetProperty("metadata"); + Assert.Equal(3, arrayMetadata.GetProperty("total_count").GetInt32()); + Assert.True(arrayMetadata.GetProperty("total_count_authoritative").GetBoolean()); + Assert.Equal(2, arrayMetadata.GetProperty("omitted_count").GetInt32()); + Assert.StartsWith("response:v2:", arrayMetadata.GetProperty("next_cursor").GetString(), StringComparison.Ordinal); + Assert.Single(arrayDocument.RootElement.GetProperty("results").EnumerateArray()); + Assert.True(arrayDocument.RootElement.GetProperty("results")[0].TryGetProperty("snippet", out _)); + } + finally + { + TestProjectHelper.DeleteDirectory(projectRoot); + } + } + + [Fact] + public void Search_DataDirCursorUsesEffectiveDatabaseGeneration_Issue4730() + { + var projectRoot = TestProjectHelper.CreateTempProject("discovery_data_dir_cursor_4730"); + try + { + var dbPath = TestProjectHelper.CreateProjectDb(projectRoot); + TestProjectHelper.InsertIndexedFile(dbPath, "src/One.txt", "text", "DataDirNeedle one\n"); + TestProjectHelper.InsertIndexedFile(dbPath, "src/Two.txt", "text", "DataDirNeedle two\n"); + var dataDir = Path.GetDirectoryName(dbPath)!; + var args = new[] + { + "search", "DataDirNeedle", "--data-dir", dataDir, + "--format", "compact", "--limit", "1", + }; + + var (firstExitCode, firstStdout, firstStderr) = CaptureConsole(() => + ProgramRunner.Run(args, _jsonOptions, "1.0.0-test")); + + Assert.Equal(CommandExitCodes.Success, firstExitCode); + Assert.Equal(string.Empty, firstStderr); + using var firstDocument = JsonDocument.Parse(firstStdout); + Assert.Equal( + Path.GetFullPath(dbPath), + firstDocument.RootElement.GetProperty("metadata").GetProperty("db_path").GetString()); + var cursor = Assert.IsType<string>(firstDocument.RootElement.GetProperty("next_cursor").GetString()); + + using (var db = new DbContext(DbOpenIntent.WriteIndex, dbPath)) + { + var writer = new DbWriter(db.Connection); + writer.SetMeta(DbContext.IndexedHeadTimestampMetaKey, "2026-07-24T13:45:00.0000000+00:00"); + } + + var (staleExitCode, staleStdout, staleStderr) = CaptureConsole(() => + ProgramRunner.Run(args.Concat(["--cursor", cursor]).ToArray(), _jsonOptions, "1.0.0-test")); + + Assert.Equal(CommandExitCodes.UsageError, staleExitCode); + Assert.Equal(string.Empty, staleStdout); + Assert.Contains("index generation changed", staleStderr, StringComparison.Ordinal); + } + finally + { + TestProjectHelper.DeleteDirectory(projectRoot); + } + } + + [Fact] + public void Search_PostSelectorsPageTheGloballySelectedSequence_Issue4730() + { + var projectRoot = TestProjectHelper.CreateTempProject("discovery_search_selection_cursor_4730"); + try + { + var dbPath = TestProjectHelper.CreateProjectDb(projectRoot); + TestProjectHelper.InsertIndexedFile(dbPath, "src/a.cs", "csharp", "SelectorNeedle one\n"); + TestProjectHelper.InsertIndexedFile(dbPath, "src/b.cs", "csharp", "SelectorNeedle three\n"); + ReplaceChunks( + dbPath, + "src/a.cs", + new ChunkRecord { ChunkIndex = 0, StartLine = 1, EndLine = 1, Content = "SelectorNeedle one\n" }, + new ChunkRecord { ChunkIndex = 1, StartLine = 2, EndLine = 2, Content = "SelectorNeedle two\n" }); + + AssertPagedSearchSelection( + [ + "search", "SelectorNeedle", "--db", dbPath, "--exact-substring", + "--first-per-file", "--format", "compact", + ], + expectedTotal: 2); + AssertPagedSearchSelection( + [ + "search", "SelectorNeedle", "--db", dbPath, "--exact-substring", + "--sample", "2", "--format", "compact", + ], + expectedTotal: 2); + } + finally + { + TestProjectHelper.DeleteDirectory(projectRoot); + } + } + + [Fact] + public void Search_CompactAggregateModesKeepNativeContracts_Issue4730() + { + var (listExitCode, listStdout, listStderr) = CaptureConsole(() => + ProgramRunner.Run( + ["search", "--list-recipes", "--format", "compact"], + _jsonOptions, + "1.0.0-test")); + + Assert.Equal(CommandExitCodes.Success, listExitCode); + Assert.Equal(string.Empty, listStderr); + using (var listDocument = JsonDocument.Parse(listStdout)) + { + Assert.True(listDocument.RootElement.TryGetProperty("recipes", out var recipes)); + Assert.NotEmpty(recipes.EnumerateArray()); + Assert.False(listDocument.RootElement.TryGetProperty("results", out _)); + } + + var projectRoot = TestProjectHelper.CreateTempProject("discovery_search_aggregate_contract_4730"); + try + { + var dbPath = TestProjectHelper.CreateProjectDb(projectRoot); + TestProjectHelper.InsertIndexedFile(dbPath, "src/One.txt", "text", "AggregateNeedle\n"); + + var (batchExitCode, batchStdout, batchStderr) = CaptureConsole(() => + ProgramRunner.Run( + [ + "search", "--named-query", "one=AggregateNeedle", + "--db", dbPath, "--format", "compact", + ], + _jsonOptions, + "1.0.0-test")); + + Assert.Equal(CommandExitCodes.Success, batchExitCode); + Assert.Equal(string.Empty, batchStderr); + using var batchDocument = JsonDocument.Parse(batchStdout); + Assert.True(batchDocument.RootElement.TryGetProperty("queries", out var queries)); + Assert.NotEmpty(queries.EnumerateArray()); + Assert.False(batchDocument.RootElement.TryGetProperty("results", out _)); + } + finally + { + TestProjectHelper.DeleteDirectory(projectRoot); + } + } + + [Fact] + public void SymbolsFilesAndLanguages_PageWithGenerationBoundCursors_Issue4730() + { + var projectRoot = TestProjectHelper.CreateTempProject("discovery_catalog_paging_4730"); + try + { + var dbPath = TestProjectHelper.CreateProjectDb(projectRoot); + for (var index = 1; index <= 3; index++) + TestProjectHelper.InsertIndexedFile(dbPath, $"src/Type{index}.cs", "csharp", $"public sealed class Type{index} {{ }}\n"); + + var symbolsArgs = new[] + { + "symbols", "--db", dbPath, "--format", "compact", "--limit", "1", + }; + var symbolsCursor = AssertPagedCompactCommand(symbolsArgs, "symbols"); + + var filesArgs = new[] + { + "files", "--db", dbPath, "--format", "compact", "--limit", "1", + }; + AssertPagedCompactCommand(filesArgs, "files"); + + var languageArgs = new[] + { + "languages", "--db", dbPath, "--json", "--limit", "1", "--max-json-bytes", "8192", + }; + var (languageExitCode, languageStdout, languageStderr) = CaptureConsole(() => + ProgramRunner.Run(languageArgs, _jsonOptions, "1.0.0-test")); + + Assert.Equal(CommandExitCodes.Success, languageExitCode); + Assert.Equal(string.Empty, languageStderr); + Assert.True(Encoding.UTF8.GetByteCount(languageStdout) <= 8192); + using var languageDocument = JsonDocument.Parse(languageStdout); + var languageMetadata = languageDocument.RootElement.GetProperty("metadata"); + Assert.True(languageMetadata.GetProperty("total_count").GetInt32() > 1); + Assert.True(languageMetadata.GetProperty("has_more").GetBoolean()); + Assert.StartsWith("response:v2:", languageMetadata.GetProperty("next_cursor").GetString(), StringComparison.Ordinal); + Assert.False(string.IsNullOrWhiteSpace(languageMetadata.GetProperty("result_stable_at").GetString())); + Assert.Single(languageDocument.RootElement.GetProperty("results").EnumerateArray()); + Assert.True(languageDocument.RootElement.GetProperty("results")[0].TryGetProperty("exact_filenames", out _)); + var languageCursor = languageMetadata.GetProperty("next_cursor").GetString()!; + var firstLanguage = languageDocument.RootElement.GetProperty("results")[0].GetProperty("lang").GetString(); + + var (nextLanguageExitCode, nextLanguageStdout, nextLanguageStderr) = CaptureConsole(() => + ProgramRunner.Run(languageArgs.Concat(["--cursor", languageCursor]).ToArray(), _jsonOptions, "1.0.0-test")); + + Assert.Equal(CommandExitCodes.Success, nextLanguageExitCode); + Assert.Equal(string.Empty, nextLanguageStderr); + using (var nextLanguageDocument = JsonDocument.Parse(nextLanguageStdout)) + { + Assert.Equal(1, nextLanguageDocument.RootElement.GetProperty("metadata").GetProperty("cursor_offset").GetInt32()); + Assert.NotEqual(firstLanguage, nextLanguageDocument.RootElement.GetProperty("results")[0].GetProperty("lang").GetString()); + } + + var (mismatchExitCode, mismatchStdout, mismatchStderr) = CaptureConsole(() => + ProgramRunner.Run( + symbolsArgs.Concat(["--kind", "function", "--cursor", symbolsCursor]).ToArray(), + _jsonOptions, + "1.0.0-test")); + + Assert.Equal(CommandExitCodes.UsageError, mismatchExitCode); + Assert.Equal(string.Empty, mismatchStdout); + Assert.Contains("does not match this command, query, or filter set", mismatchStderr, StringComparison.Ordinal); + + using (var db = new DbContext(DbOpenIntent.WriteIndex, dbPath)) + { + var writer = new DbWriter(db.Connection); + writer.SetMeta(DbContext.IndexedHeadTimestampMetaKey, "2026-07-24T12:34:56.0000000+00:00"); + } + + var (staleExitCode, staleStdout, staleStderr) = CaptureConsole(() => + ProgramRunner.Run(symbolsArgs.Concat(["--cursor", symbolsCursor]).ToArray(), _jsonOptions, "1.0.0-test")); + + Assert.Equal(CommandExitCodes.UsageError, staleExitCode); + Assert.Equal(string.Empty, staleStdout); + Assert.Contains("index generation changed", staleStderr, StringComparison.Ordinal); + Assert.Contains("Restart pagination", staleStderr, StringComparison.Ordinal); + } + finally + { + TestProjectHelper.DeleteDirectory(projectRoot); + } + } + + [Fact] + public void FindAll_PartialScanCursorResumesAfterLastScannedLine_Issue4730() + { + var projectRoot = TestProjectHelper.CreateTempProject("find_scan_resume_4730"); + try + { + var dbPath = TestProjectHelper.CreateProjectDb(projectRoot); + var content = string.Join('\n', Enumerable.Range(1, 12).Select(index => $"Needle {index}")); + TestProjectHelper.InsertIndexedFile(dbPath, "src/Many.txt", "text", content); + var firstArgs = new[] + { + "find", "Needle", "--db", dbPath, "--all", "--exact", + "--json=ndjson", "--limit", "10", "--line-scan-limit", "3", + }; + + var (firstExitCode, firstStdout, firstStderr) = CaptureConsole(() => + ProgramRunner.Run(firstArgs, _jsonOptions, "1.0.0-test")); + + Assert.Equal(CommandExitCodes.PartialResult, firstExitCode); + Assert.Equal(string.Empty, firstStderr); + var firstLines = ParseNdjson(firstStdout); + Assert.Equal(new[] { 1, 2, 3 }, firstLines[..^1].Select(row => row.GetProperty("line").GetInt32()).ToArray()); + var terminal = firstLines[^1]; + Assert.Equal("resume_with_next_cursor", terminal.GetProperty("continuation_action").GetString()); + var cursor = Assert.IsType<string>(terminal.GetProperty("next_cursor").GetString()); + Assert.StartsWith("response:v2:", cursor, StringComparison.Ordinal); + + var (secondExitCode, secondStdout, secondStderr) = CaptureConsole(() => + ProgramRunner.Run(firstArgs.Concat(["--cursor", cursor]).ToArray(), _jsonOptions, "1.0.0-test")); + + Assert.Equal(CommandExitCodes.PartialResult, secondExitCode); + Assert.Equal(string.Empty, secondStderr); + using var secondDocument = JsonDocument.Parse(secondStdout); + Assert.Equal( + new[] { 4, 5, 6 }, + secondDocument.RootElement.GetProperty("results").EnumerateArray() + .Select(row => row.GetProperty("line").GetInt32()) + .ToArray()); + var secondCursor = secondDocument.RootElement.GetProperty("metadata").GetProperty("next_cursor").GetString(); + Assert.NotEqual(cursor, secondCursor); + } + finally + { + TestProjectHelper.DeleteDirectory(projectRoot); + } + } + + [Fact] + public void FindAll_ByteBudgetPagesCapturedRowsBeforeAdvancingScanCursor_Issue4730() + { + var projectRoot = TestProjectHelper.CreateTempProject("find_scan_byte_resume_4730"); + try + { + var dbPath = TestProjectHelper.CreateProjectDb(projectRoot); + var longPath = $"src/{new string('a', 180)}.txt"; + var content = string.Join('\n', Enumerable.Range(1, 12).Select(index => $"Needle {index}")); + TestProjectHelper.InsertIndexedFile(dbPath, longPath, "text", content); + var baseArgs = new[] + { + "find", "Needle", "--db", dbPath, "--all", "--exact", + "--json=ndjson", "--fields", "path,line", "--limit", "10", "--line-scan-limit", "8", + }; + + JsonDocument? firstDocument = null; + string? pageCursor = null; + var firstReturned = 0; + foreach (var byteBudget in new[] { 2_400, 2_800, 3_200, 3_600, 4_000, 4_800 }) + { + var (exitCode, stdout, _) = CaptureConsole(() => + ProgramRunner.Run( + baseArgs.Concat(["--max-json-bytes", byteBudget.ToString()]).ToArray(), + _jsonOptions, + "1.0.0-test")); + if (exitCode != CommandExitCodes.PartialResult || string.IsNullOrWhiteSpace(stdout)) + continue; + + var candidate = JsonDocument.Parse(stdout); + var metadata = candidate.RootElement.GetProperty("metadata"); + if (!metadata.TryGetProperty("byte_limit_reached", out var reached) || !reached.GetBoolean()) + { + candidate.Dispose(); + continue; + } + + firstDocument = candidate; + firstReturned = metadata.GetProperty("returned_count").GetInt32(); + pageCursor = metadata.GetProperty("next_cursor").GetString(); + break; + } + + using (firstDocument) + { + Assert.NotNull(firstDocument); + Assert.InRange(firstReturned, 1, 7); + Assert.StartsWith("response:v2:", pageCursor, StringComparison.Ordinal); + Assert.Equal( + Enumerable.Range(1, firstReturned), + firstDocument.RootElement.GetProperty("results").EnumerateArray() + .Select(row => row.GetProperty("line").GetInt32())); + } + + var (secondExitCode, secondStdout, secondStderr) = CaptureConsole(() => + ProgramRunner.Run( + baseArgs.Concat(["--max-json-bytes", "65536", "--cursor", pageCursor!]).ToArray(), + _jsonOptions, + "1.0.0-test")); + + Assert.Equal(CommandExitCodes.PartialResult, secondExitCode); + Assert.Equal(string.Empty, secondStderr); + using var secondDocument = JsonDocument.Parse(secondStdout); + Assert.Equal( + Enumerable.Range(firstReturned + 1, 8 - firstReturned), + secondDocument.RootElement.GetProperty("results").EnumerateArray() + .Select(row => row.GetProperty("line").GetInt32())); + Assert.StartsWith( + "response:v2:", + secondDocument.RootElement.GetProperty("metadata").GetProperty("next_cursor").GetString(), + StringComparison.Ordinal); + } + finally + { + TestProjectHelper.DeleteDirectory(projectRoot); + } + } + + private string AssertPagedCompactCommand(string[] args, string collectionName) + { + var (firstExitCode, firstStdout, firstStderr) = CaptureConsole(() => + ProgramRunner.Run(args, _jsonOptions, "1.0.0-test")); + + Assert.Equal(CommandExitCodes.Success, firstExitCode); + Assert.Equal(string.Empty, firstStderr); + using var firstDocument = JsonDocument.Parse(firstStdout); + var first = firstDocument.RootElement; + var metadata = first.GetProperty("metadata"); + Assert.True(metadata.GetProperty("total_count").GetInt32() > 1); + Assert.True(metadata.GetProperty("has_more").GetBoolean()); + Assert.False(string.IsNullOrWhiteSpace(metadata.GetProperty("result_stable_at").GetString())); + var cursor = Assert.IsType<string>(metadata.GetProperty("next_cursor").GetString()); + Assert.StartsWith("response:v2:", cursor, StringComparison.Ordinal); + var firstPath = first.GetProperty(collectionName)[0].GetProperty("path").GetString(); + + var (secondExitCode, secondStdout, secondStderr) = CaptureConsole(() => + ProgramRunner.Run(args.Concat(["--cursor", cursor]).ToArray(), _jsonOptions, "1.0.0-test")); + + Assert.Equal(CommandExitCodes.Success, secondExitCode); + Assert.Equal(string.Empty, secondStderr); + using var secondDocument = JsonDocument.Parse(secondStdout); + var second = secondDocument.RootElement; + Assert.Equal(1, second.GetProperty("cursor_offset").GetInt32()); + Assert.NotEqual(firstPath, second.GetProperty(collectionName)[0].GetProperty("path").GetString()); + return cursor; + } + + private void AssertPagedSearchSelection(string[] baseArgs, int expectedTotal) + { + var (allExitCode, allStdout, allStderr) = CaptureConsole(() => + ProgramRunner.Run(baseArgs.Concat(["--limit", "10"]).ToArray(), _jsonOptions, "1.0.0-test")); + + Assert.Equal(CommandExitCodes.Success, allExitCode); + Assert.Equal(string.Empty, allStderr); + using var allDocument = JsonDocument.Parse(allStdout); + var expectedFiles = allDocument.RootElement.GetProperty("results") + .EnumerateArray() + .Select(result => result.GetProperty("file").GetString()) + .ToArray(); + Assert.Equal(expectedTotal, expectedFiles.Length); + + var pageArgs = baseArgs.Concat(["--limit", "1"]).ToArray(); + var (firstExitCode, firstStdout, firstStderr) = CaptureConsole(() => + ProgramRunner.Run(pageArgs, _jsonOptions, "1.0.0-test")); + + Assert.Equal(CommandExitCodes.Success, firstExitCode); + Assert.Equal(string.Empty, firstStderr); + using var firstDocument = JsonDocument.Parse(firstStdout); + Assert.Equal(expectedTotal, firstDocument.RootElement.GetProperty("total_count").GetInt32()); + Assert.Equal(expectedFiles[0], firstDocument.RootElement.GetProperty("results")[0].GetProperty("file").GetString()); + var cursor = Assert.IsType<string>(firstDocument.RootElement.GetProperty("next_cursor").GetString()); + + var (secondExitCode, secondStdout, secondStderr) = CaptureConsole(() => + ProgramRunner.Run(pageArgs.Concat(["--cursor", cursor]).ToArray(), _jsonOptions, "1.0.0-test")); + + Assert.Equal(CommandExitCodes.Success, secondExitCode); + Assert.Equal(string.Empty, secondStderr); + using var secondDocument = JsonDocument.Parse(secondStdout); + Assert.Equal(expectedFiles[1], secondDocument.RootElement.GetProperty("results")[0].GetProperty("file").GetString()); + Assert.False(secondDocument.RootElement.GetProperty("has_more").GetBoolean()); + Assert.Equal(JsonValueKind.Null, secondDocument.RootElement.GetProperty("next_cursor").ValueKind); + } + + private static void ReplaceChunks(string dbPath, string path, params ChunkRecord[] chunks) + { + using var db = new DbContext(DbOpenIntent.WriteIndex, dbPath); + using var command = db.Connection.CreateCommand(); + command.CommandText = "SELECT id FROM files WHERE path = @path"; + command.Parameters.AddWithValue("@path", path); + var fileId = (long)(command.ExecuteScalar() + ?? throw new InvalidOperationException($"Missing indexed file {path}.")); + var writer = new DbWriter(db.Connection); + writer.DeleteFileData(fileId); + foreach (var chunk in chunks) + chunk.FileId = fileId; + writer.InsertChunks(chunks); + } + + private static JsonElement[] ParseNdjson(string stdout) + => stdout.Split(Environment.NewLine, StringSplitOptions.RemoveEmptyEntries) + .Select(line => JsonDocument.Parse(line).RootElement.Clone()) + .ToArray(); + + private static (int ExitCode, string Stdout, string Stderr) CaptureConsole(Func<int> action) + => ConsoleCapture.Capture(action); +} diff --git a/tests/CodeIndex.Tests/QueryCommandRunnerFindIssue4578Tests.cs b/tests/CodeIndex.Tests/QueryCommandRunnerFindIssue4578Tests.cs index bbb3ade4c..ac5f9181e 100644 --- a/tests/CodeIndex.Tests/QueryCommandRunnerFindIssue4578Tests.cs +++ b/tests/CodeIndex.Tests/QueryCommandRunnerFindIssue4578Tests.cs @@ -208,8 +208,8 @@ public void RunFind_AllScopeTextSummaryCarriesAuthorityCapsAndRecovery_Issue4578 Assert.Contains("line_scan_limit=1", rowStderr, StringComparison.Ordinal); Assert.Contains("scan_complete=false", rowStderr, StringComparison.Ordinal); Assert.Contains("authoritative_rows=false", rowStderr, StringComparison.Ordinal); - Assert.Contains("continuation_action=increase_line_scan_limit_or_narrow_scope", rowStderr, StringComparison.Ordinal); - Assert.Contains("--allow-partial", rowStderr, StringComparison.Ordinal); + Assert.Contains("continuation_action=resume_with_next_cursor", rowStderr, StringComparison.Ordinal); + Assert.Contains("next_cursor=response:v2:", rowStderr, StringComparison.Ordinal); var (countExitCode, _, countStderr) = CaptureConsole(() => QueryCommandRunner.RunFind( ["alpha", "--db", dbPath, "--all", "--count", "--line-scan-limit", "1"], @@ -251,14 +251,17 @@ private static void AssertPartialFindTerminal(JsonElement json, bool countMode) Assert.Equal("line_scan_limit", json.GetProperty("truncation_reason").GetString()); Assert.Equal(QueryCommandRunner.FindAllCandidateFileLimit, json.GetProperty("candidate_file_limit").GetInt32()); Assert.Equal(1, json.GetProperty("line_scan_limit").GetInt32()); - Assert.Equal("increase_line_scan_limit_or_narrow_scope", json.GetProperty("continuation_action").GetString()); - Assert.Contains("--allow-partial", json.GetProperty("recovery_guidance").GetString(), StringComparison.Ordinal); if (countMode) { + Assert.Equal("increase_line_scan_limit_or_narrow_scope", json.GetProperty("continuation_action").GetString()); + Assert.Contains("--allow-partial", json.GetProperty("recovery_guidance").GetString(), StringComparison.Ordinal); Assert.False(json.GetProperty("authoritative_count").GetBoolean()); } else { + Assert.Equal("resume_with_next_cursor", json.GetProperty("continuation_action").GetString()); + Assert.StartsWith("response:v2:", json.GetProperty("next_cursor").GetString(), StringComparison.Ordinal); + Assert.False(string.IsNullOrWhiteSpace(json.GetProperty("result_stable_at").GetString())); Assert.False(json.GetProperty("authoritative_rows").GetBoolean()); Assert.Equal(20, json.GetProperty("applied_limit").GetInt32()); } diff --git a/tests/CodeIndex.Tests/QueryCommandRunnerInspectTests.cs b/tests/CodeIndex.Tests/QueryCommandRunnerInspectTests.cs index ab743f7af..6dd06b4dd 100644 --- a/tests/CodeIndex.Tests/QueryCommandRunnerInspectTests.cs +++ b/tests/CodeIndex.Tests/QueryCommandRunnerInspectTests.cs @@ -1489,7 +1489,8 @@ public void M6() { } Assert.Equal(QueryCommandRunner.DefaultCompactSectionLimit + 3, json.GetProperty("total_symbol_count").GetInt32()); Assert.Equal(2, json.GetProperty("returned_symbol_count").GetInt32()); Assert.Equal(0, json.GetProperty("cursor_offset").GetInt32()); - Assert.Equal("outline:2", json.GetProperty("next_cursor").GetString()); + Assert.StartsWith("page:v1:", json.GetProperty("next_cursor").GetString(), StringComparison.Ordinal); + Assert.False(string.IsNullOrWhiteSpace(json.GetProperty("result_stable_at").GetString())); Assert.True(json.GetProperty("has_more").GetBoolean()); } finally @@ -1909,15 +1910,26 @@ public void M2() { } Assert.Equal(2, firstJson.GetProperty("returned_symbol_count").GetInt32()); Assert.Equal(0, firstJson.GetProperty("cursor_offset").GetInt32()); Assert.True(firstJson.GetProperty("has_more").GetBoolean()); - Assert.Equal("outline:2", firstJson.GetProperty("next_cursor").GetString()); + var nextCursor = firstJson.GetProperty("next_cursor").GetString(); + Assert.StartsWith("page:v1:", nextCursor, StringComparison.Ordinal); + Assert.False(string.IsNullOrWhiteSpace(firstJson.GetProperty("result_stable_at").GetString())); Assert.Equal(new[] { "function" }, firstJson.GetProperty("kind_filter").EnumerateArray().Select(item => item.GetString()).ToArray()); Assert.Equal(new[] { "name", "line", "kind" }, firstJson.GetProperty("selected_fields").EnumerateArray().Select(item => item.GetString()).ToArray()); Assert.Equal(new[] { "M0", "M1" }, firstSymbols.Select(symbol => symbol.GetProperty("name").GetString()).ToArray()); foreach (var symbol in firstSymbols) Assert.Equal(new[] { "name", "line", "kind" }, symbol.EnumerateObject().Select(property => property.Name).ToArray()); + var (mismatchedExitCode, mismatchedStdout, mismatchedStderr) = CaptureConsole(() => QueryCommandRunner.RunOutline( + ["src/many.cs", "--db", dbPath, "--json", "--kind", "function", "--sort", "name", "--limit", "2", "--cursor", nextCursor!, "--outline-fields", "name,line,kind"], + _jsonOptions)); + + Assert.Equal(CommandExitCodes.UsageError, mismatchedExitCode); + Assert.Equal(string.Empty, mismatchedStdout); + Assert.Contains("query scope, filters, or ordering", mismatchedStderr, StringComparison.Ordinal); + Assert.Contains("restart required", mismatchedStderr, StringComparison.Ordinal); + var (secondExitCode, secondStdout, secondStderr) = CaptureConsole(() => QueryCommandRunner.RunOutline( - ["src/many.cs", "--db", dbPath, "--json", "--kind", "function", "--limit", "2", "--cursor", "outline:2", "--outline-fields", "name,line,kind"], + ["src/many.cs", "--db", dbPath, "--json", "--kind", "function", "--limit", "2", "--cursor", nextCursor!, "--outline-fields", "name,line,kind"], _jsonOptions)); Assert.Equal(CommandExitCodes.Success, secondExitCode); @@ -1933,6 +1945,21 @@ public void M2() { } Assert.Equal(JsonValueKind.Null, secondJson.GetProperty("next_cursor").ValueKind); Assert.Single(secondSymbols); Assert.Equal("M2", secondSymbols[0].GetProperty("name").GetString()); + + using (var db = new DbContext(DbOpenIntent.WriteIndex, dbPath)) + { + var writer = new DbWriter(db.Connection); + writer.SetMeta(DbContext.IndexedHeadTimestampMetaKey, "2026-07-24T12:00:00.0000000+00:00"); + } + + var (staleExitCode, staleStdout, staleStderr) = CaptureConsole(() => QueryCommandRunner.RunOutline( + ["src/many.cs", "--db", dbPath, "--json", "--kind", "function", "--limit", "2", "--cursor", nextCursor!, "--outline-fields", "name,line,kind"], + _jsonOptions)); + + Assert.Equal(CommandExitCodes.UsageError, staleExitCode); + Assert.Equal(string.Empty, staleStdout); + Assert.Contains("index generation changed", staleStderr, StringComparison.Ordinal); + Assert.Contains("restart required", staleStderr, StringComparison.Ordinal); } finally { diff --git a/tests/CodeIndex.Tests/QueryCommandRunnerSearchTests.cs b/tests/CodeIndex.Tests/QueryCommandRunnerSearchTests.cs index 6e3cadb86..9b5a11af8 100644 --- a/tests/CodeIndex.Tests/QueryCommandRunnerSearchTests.cs +++ b/tests/CodeIndex.Tests/QueryCommandRunnerSearchTests.cs @@ -12382,6 +12382,60 @@ public void RunSearch_GroupedFormatBoundsRepresentativeMatches_Issue3788() } } + [Fact] + public void RunSearch_GroupedFormatReportsAuthoritativeTotalsWhenLimitBoundsRows_Issue4735() + { + var projectRoot = TestProjectHelper.CreateTempProject("cdidx_search_grouped_totals_4735"); + try + { + var dbPath = TestProjectHelper.CreateProjectDb(projectRoot); + TestProjectHelper.InsertIndexedFile( + dbPath, + "src/alpha.cs", + "csharp", + "GroupedTotalNeedle();\n"); + TestProjectHelper.InsertIndexedFile( + dbPath, + "src/beta.cs", + "csharp", + "GroupedTotalNeedle();\n"); + TestProjectHelper.InsertIndexedFile( + dbPath, + "src/gamma.cs", + "csharp", + "GroupedTotalNeedle();\n"); + TestProjectHelper.InsertIndexedFile( + dbPath, + "src/delta.cs", + "csharp", + "GroupedTotalNeedle();\n"); + + var (exitCode, stdout, stderr) = CaptureConsole(() => QueryCommandRunner.RunSearch( + ["GroupedTotalNeedle", "--db", dbPath, "--exact-substring", "--format", "grouped", "--limit", "2", "--per-file-limit", "1"], + _jsonOptions)); + + Assert.Equal(CommandExitCodes.Success, exitCode); + Assert.Equal(string.Empty, stderr); + using var document = ParseJsonOutput(stdout); + var root = document.RootElement; + + Assert.Equal(4, root.GetProperty("total_matches").GetInt32()); + Assert.Equal(4, root.GetProperty("matched_count").GetInt32()); + Assert.Equal(2, root.GetProperty("grouped_match_count").GetInt32()); + Assert.Equal(2, root.GetProperty("emitted_match_count").GetInt32()); + Assert.Equal(2, root.GetProperty("omitted_match_count").GetInt32()); + Assert.Equal(4, root.GetProperty("total_groups").GetInt32()); + Assert.Equal(4, root.GetProperty("total_files").GetInt32()); + Assert.True(root.GetProperty("truncated").GetBoolean()); + Assert.True(root.GetProperty("has_more").GetBoolean()); + Assert.Contains("Increase --limit", root.GetProperty("continuation_action").GetString(), StringComparison.Ordinal); + } + finally + { + TestProjectHelper.DeleteDirectory(projectRoot); + } + } + [Fact] public void RunSearch_SearchFieldsResultsOnlyOmitsDoneRecord_Issue3728() { diff --git a/tests/CodeIndex.Tests/QueryCommandRunnerSymbolTests.cs b/tests/CodeIndex.Tests/QueryCommandRunnerSymbolTests.cs index b50d7e074..dd868c9a8 100644 --- a/tests/CodeIndex.Tests/QueryCommandRunnerSymbolTests.cs +++ b/tests/CodeIndex.Tests/QueryCommandRunnerSymbolTests.cs @@ -1867,11 +1867,13 @@ public void RunUnused_DefaultAndAllJsonCursorsSharePaginationFixture_Issues3691A Assert.Equal(string.Empty, firstStderr); Assert.Single(firstSymbols); Assert.Equal("Hidden", firstSymbols[0].GetProperty("name").GetString()); - Assert.Equal("unused:1", firstJson.GetProperty("next_cursor").GetString()); + var firstCursor = firstJson.GetProperty("next_cursor").GetString(); + Assert.StartsWith("page:v1:", firstCursor, StringComparison.Ordinal); + Assert.False(string.IsNullOrWhiteSpace(firstJson.GetProperty("result_stable_at").GetString())); Assert.Equal(7, firstJson.GetProperty("default_suppression").GetProperty("suppressed_count").GetInt32()); var (secondExitCode, secondStdout, secondStderr) = CaptureConsole(() => QueryCommandRunner.RunUnused( - ["--db", dbPath, "--json", "--lang", "csharp", "--limit", "1", "--cursor", "unused:1"], + ["--db", dbPath, "--json", "--lang", "csharp", "--limit", "1", "--cursor", firstCursor!], _jsonOptions)); using var secondDocument = ParseJsonOutput(secondStdout); var secondJson = secondDocument.RootElement; @@ -1893,12 +1895,31 @@ public void RunUnused_DefaultAndAllJsonCursorsSharePaginationFixture_Issues3691A Assert.Equal(CommandExitCodes.Success, allFirstExitCode); Assert.Equal(string.Empty, allFirstStderr); Assert.Equal(2, allFirstJson.GetProperty("count").GetInt32()); - Assert.Equal("unused:2", allFirstJson.GetProperty("next_cursor").GetString()); + var allFirstCursor = allFirstJson.GetProperty("next_cursor").GetString(); + Assert.StartsWith("page:v1:", allFirstCursor, StringComparison.Ordinal); Assert.Equal("Hidden", allFirstSymbols[0].GetProperty("name").GetString()); Assert.Equal("InternalOnly", allFirstSymbols[1].GetProperty("name").GetString()); + var (mismatchedExitCode, mismatchedStdout, mismatchedStderr) = CaptureConsole(() => QueryCommandRunner.RunUnused( + ["--db", dbPath, "--json", "--all", "--lang", "csharp", "--kind", "function", "--limit", "2", "--cursor", allFirstCursor!], + _jsonOptions)); + + Assert.Equal(CommandExitCodes.UsageError, mismatchedExitCode); + Assert.Equal(string.Empty, mismatchedStdout); + Assert.Contains("query scope, filters, or ordering", mismatchedStderr, StringComparison.Ordinal); + Assert.Contains("restart required", mismatchedStderr, StringComparison.Ordinal); + + var (generatedMismatchExitCode, generatedMismatchStdout, generatedMismatchStderr) = CaptureConsole(() => QueryCommandRunner.RunUnused( + ["--db", dbPath, "--json", "--all", "--include-generated", "--lang", "csharp", "--limit", "2", "--cursor", allFirstCursor!], + _jsonOptions)); + + Assert.Equal(CommandExitCodes.UsageError, generatedMismatchExitCode); + Assert.Equal(string.Empty, generatedMismatchStdout); + Assert.Contains("query scope, filters, or ordering", generatedMismatchStderr, StringComparison.Ordinal); + Assert.Contains("restart required", generatedMismatchStderr, StringComparison.Ordinal); + var (allSecondExitCode, allSecondStdout, allSecondStderr) = CaptureConsole(() => QueryCommandRunner.RunUnused( - ["--db", dbPath, "--json", "--all", "--lang", "csharp", "--limit", "2", "--cursor", "unused:2"], + ["--db", dbPath, "--json", "--all", "--lang", "csharp", "--limit", "2", "--cursor", allFirstCursor!], _jsonOptions)); using var allSecondDocument = ParseJsonOutput(allSecondStdout); var allSecondJson = allSecondDocument.RootElement; @@ -1908,11 +1929,26 @@ public void RunUnused_DefaultAndAllJsonCursorsSharePaginationFixture_Issues3691A Assert.Equal(CommandExitCodes.Success, allSecondExitCode); Assert.Equal(string.Empty, allSecondStderr); Assert.Equal(2, allSecondJson.GetProperty("count").GetInt32()); - Assert.Equal("unused:4", allSecondJson.GetProperty("next_cursor").GetString()); - Assert.Equal("unused:2", allSecondQuery.GetProperty("cursor").GetString()); + Assert.StartsWith("page:v1:", allSecondJson.GetProperty("next_cursor").GetString(), StringComparison.Ordinal); + Assert.Equal(allFirstCursor, allSecondQuery.GetProperty("cursor").GetString()); Assert.Equal(2, allSecondQuery.GetProperty("offset").GetInt32()); Assert.Equal("PathResolver", allSecondSymbols[0].GetProperty("name").GetString()); Assert.Equal("ConnectionString", allSecondSymbols[1].GetProperty("name").GetString()); + + using (var db = new DbContext(DbOpenIntent.WriteIndex, dbPath)) + { + var writer = new DbWriter(db.Connection); + writer.SetMeta(DbContext.IndexedHeadTimestampMetaKey, "2026-07-24T12:00:00.0000000+00:00"); + } + + var (staleExitCode, staleStdout, staleStderr) = CaptureConsole(() => QueryCommandRunner.RunUnused( + ["--db", dbPath, "--json", "--all", "--lang", "csharp", "--limit", "2", "--cursor", allFirstCursor!], + _jsonOptions)); + + Assert.Equal(CommandExitCodes.UsageError, staleExitCode); + Assert.Equal(string.Empty, staleStdout); + Assert.Contains("index generation changed", staleStderr, StringComparison.Ordinal); + Assert.Contains("restart required", staleStderr, StringComparison.Ordinal); } finally { diff --git a/tests/CodeIndex.Tests/QueryCommandRunnerTests.cs b/tests/CodeIndex.Tests/QueryCommandRunnerTests.cs index ead43e28b..842ea09f6 100644 --- a/tests/CodeIndex.Tests/QueryCommandRunnerTests.cs +++ b/tests/CodeIndex.Tests/QueryCommandRunnerTests.cs @@ -2731,6 +2731,35 @@ public void RunLanguages_FormatCountReturnsCapabilitySummary_Issue4316() Assert.False(root.TryGetProperty("languages", out _)); } + [Fact] + public void RunLanguages_FormatCountIgnoresRowLimit_Issue4730() + { + using var project = TestProjectHelper.CreateTempProjectScope("cdidx_languages_count_limit_issue4730"); + var dbPath = TestProjectHelper.CreateProjectDb(project.Root); + TestProjectHelper.InsertIndexedFile(dbPath, "src/App.cs", "csharp", "class App { }\n"); + TestProjectHelper.InsertIndexedFile(dbPath, "src/main.adb", "ada", "procedure Main is begin null; end Main;\n"); + + var (baselineExitCode, baselineStdout, baselineStderr) = CaptureConsole(() => + QueryCommandRunner.RunLanguages(["--db", dbPath, "--indexed-only", "--format", "count"], _jsonOptions)); + var (limitedExitCode, limitedStdout, limitedStderr) = CaptureConsole(() => + QueryCommandRunner.RunLanguages(["--db", dbPath, "--indexed-only", "--format", "count", "--limit", "1"], _jsonOptions)); + + Assert.Equal(CommandExitCodes.Success, baselineExitCode); + Assert.Equal(CommandExitCodes.Success, limitedExitCode); + Assert.Equal(string.Empty, baselineStderr); + Assert.Equal(string.Empty, limitedStderr); + using var baselineDocument = ParseJsonOutput(baselineStdout); + using var limitedDocument = ParseJsonOutput(limitedStdout); + var baseline = baselineDocument.RootElement; + var limited = limitedDocument.RootElement; + + Assert.Equal(2, baseline.GetProperty("count").GetInt32()); + Assert.Equal(baseline.GetProperty("count").GetInt32(), limited.GetProperty("count").GetInt32()); + Assert.Equal( + baseline.GetProperty("capability_counts").GetRawText(), + limited.GetProperty("capability_counts").GetRawText()); + } + [Fact] public void RunLanguages_SummaryOnlyJsonReturnsCapabilitySummary_Issue4316() { @@ -4982,7 +5011,7 @@ public void RunDeps_CyclesUsesStableCompleteRankingAndCursorPagination_Issues318 Assert.Equal(CommandExitCodes.UsageError, outlineExitCode); Assert.Contains("outline --cursor must use an outline pagination cursor", outlineStderr, StringComparison.Ordinal); Assert.Equal(CommandExitCodes.UsageError, unusedExitCode); - Assert.Contains("cursor for unused must use the `unused:<offset>` cursor", unusedStderr, StringComparison.Ordinal); + Assert.Contains("cursor for unused must use the opaque cursor returned by a previous unused response", unusedStderr, StringComparison.Ordinal); } finally {