From fb3c5dee58c05c8cc04b8c1b257b2664f7b00563 Mon Sep 17 00:00:00 2001 From: Widthdom Date: Mon, 27 Jul 2026 05:24:00 +0900 Subject: [PATCH 1/3] Fix persisted index completeness reporting (#4826) --- AGENT_GUIDE.md | 1 + DEVELOPER_GUIDE.md | 27 +- README.md | 19 ++ TESTING_GUIDE.md | 2 + changelog.d/unreleased/4826.fixed.md | 20 ++ .../Cli/IndexCommandRunner.Diagnostics.cs | 8 +- .../Cli/IndexCommandRunner.FullScan.Output.cs | 43 +++- .../IndexCommandRunner.FullScan.Readiness.cs | 3 +- .../Cli/IndexCommandRunner.Update.Output.cs | 42 ++- .../Cli/IndexCommandRunner.Update.cs | 3 +- src/CodeIndex/Cli/JsonOutputContracts.cs | 4 + .../DbReader.IndexGenerationReadiness.cs | 172 +++++++++++++ src/CodeIndex/Database/DbReader.Status.cs | 69 ++--- .../Database/DbReader.WorkspaceHealth.cs | 26 +- src/CodeIndex/Database/DbReader.cs | 8 +- ...bWriter.ReferenceExtractionCompleteness.cs | 10 + .../Database/DbWriter.StatusMetadata.cs | 9 + src/CodeIndex/Mcp/McpToolHandlers.Graph.cs | 34 ++- .../Mcp/McpToolHandlers.Indexing.Execution.cs | 9 +- .../Mcp/McpToolHandlers.Indexing.Results.cs | 8 +- .../IndexCommandRunnerFullScanTests.cs | 243 +++++++++++++++++- .../IndexCommandRunnerUpdateTests.cs | 3 +- .../McpServerToolsCallTests.cs | 45 +++- .../QueryCommandRunnerFilesTests.cs | 6 +- tests/CodeIndex.Tests/golden/status.json | 3 +- 25 files changed, 699 insertions(+), 118 deletions(-) create mode 100644 changelog.d/unreleased/4826.fixed.md create mode 100644 src/CodeIndex/Database/DbReader.IndexGenerationReadiness.cs diff --git a/AGENT_GUIDE.md b/AGENT_GUIDE.md index be3c4ac9d3..18fdbfadc9 100644 --- a/AGENT_GUIDE.md +++ b/AGENT_GUIDE.md @@ -142,6 +142,7 @@ CI watching must be bounded. Do not loop indefinitely. - `hotspot_family_degraded_reason` currently uses `hotspot_family_support_not_indexed`, `hotspot_family_metadata_stale`, `hotspot_family_disabled_at_index_time`, `partial_family_key_population`, and `hotspot_family_marker_fingerprint_incomplete`; the incomplete marker fingerprint code means marker traversal hit safety caps and should stay synchronized with README / developer-guide recovery notes. - `issues_table_available` reports physical `file_issues` table presence only. `file_issues_data_current` reports whether the table is also stamped current for the active index generation. - `graph_table_available` reports a queryable persisted reference generation, while `graph_data_current`, `reference_graph_complete`, and `index_complete` report current-generation coverage. Reference extraction is bounded at 50,000 lookup symbols, 20,000 lookup lines, 512 names per line, and 20,000 container candidates; `reference_extraction_limits`, `reference_graph_incomplete_reasons`, and `reference_extraction_cap_hits` publish cap state, and `last_index_run.reference_extraction_cap_hits` snapshots it per run. Cap hits persist per file and propagate degraded, non-authoritative absence semantics to callers, callees, deps, and impact. Indexed Crystal, Groovy, Tcl, Prolog, or `ambiguous_pl` rows with a missing or stale extractor stamp add `dynamic_reference_graph_contract_stale` to `reference_graph_incomplete_reasons` and keep graph readiness false until a normal index refresh rewrites them. A per-file extraction failure keeps successful graph rows queryable, stamps completeness false with bounded `last_failed_or_partial_index_run.file_errors`, and returns exit `11` unless `index --allow-partial` explicitly opts into exit `0`. While such file failures remain unresolved, a later scoped update automatically uses the normal incremental full-scan path so unrelated targets cannot clear the failure and successful recovery can restore every workspace-wide readiness contract without `--rebuild`. +- Successful CLI full/update indexing, immediate status/workspace status, and MCP indexing/status must derive `index_complete`, `index_incomplete_reasons`, `reference_graph_complete`, and `reference_graph_incomplete_reasons` from the same persisted-readiness snapshot. Symbols-only runs and persisted file-size, symbol-count, reference-count, extractor-failure, or reference-cap evidence make the generation incomplete. Legacy databases keep the complete compatibility default only when persisted rows do not prove an omission. - `index_writer_version` records the `cdidx` version that last wrote to the DB (stamped into `codeindex_meta` as `cdidx_writer_version` on every full scan, update, and MCP index). `index_newer_than_reader` flips to `true` whenever any persisted numeric contract stamp in `codeindex_meta` (or unknown `PRAGMA user_version` readiness bits) exceeds the current binary's compiled maximum, so an older CLI re-opening a DB written by a newer CLI degrades loudly with an audit trail instead of silently dropping back to text-search fallbacks. `index_newer_than_reader_reason` enumerates the specific newer-than-reader stamps. - `status` also surfaces indexed-HEAD freshness via `indexed_head_sha`, `indexed_head_branch`, `indexed_head_timestamp`, `commits_ahead_of_indexed_head`, and the compact `head_freshness` summary. They are stamped by `cdidx index` on every successful run (full scan AND partial update, distinct from `indexed_head_commit` which is full-scan only) on a best-effort basis (never blocks an otherwise-successful index) and omitted on non-git workspaces, detached HEAD (branch only), or legacy DBs created before this contract. `worktree_head_changed` compares runtime HEAD with this latest stamp when available and falls back to `indexed_head_commit` only for legacy DBs. `head_freshness.state=fresh` requires `status --check` to match the workspace, `fresh_but_incomplete` keeps matching-workspace freshness distinct from incomplete extraction coverage, and `state=head_current` only means the runtime HEAD matches the `indexed_head` selected by `indexed_head_source`. - `status` also surfaces unknown-extension scan coverage via `unknown_extension_file_count`, stamped by successful full-repository index runs (`cdidx index ` and MCP `index_project`) as the number of non-indexed files with non-empty extensions that do not map to a known language. Current scans also stamp `unknown_extension_files` as a path sample bounded by `unknown_extension_file_path_limit` items and the string-list decoded-character budget, `unknown_extension_files_truncated` when more paths existed than were emitted for either bound, and `unknown_extension_file_path_limit` as the item cap rather than a guarantee that that many paths are returned. Newer scans also expose `unknown_extension_extension_counts`, `unknown_extension_category_counts`, and `unknown_extension_groups`; groups classify common non-code buckets such as repository metadata, licenses, binary assets, configuration, structural metadata, and language-support candidates, and include `recommended_action` values of `ignore_configuration`, `first_class_structural_extraction`, or `language_support`. These fields are omitted on legacy DBs or before a current full scan has stamped them. diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index 3303e8c2e3..def83b9b94 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -743,6 +743,17 @@ filters remain exact, so `--kind import` does not include local type declaration `status --json` emits structured readiness guidance whenever any trust field is degraded. The top-level `degraded_root_cause` is a stable machine-readable primary code, while `readiness_degradations[]` lists every degraded field with `root_cause`, human `degraded_reason`, `recommended_action`, and `alternative_action`. `migration_in_progress` is set from the active batch marker so clients can distinguish a temporary writer/migration window from a permanently degraded index. `issues_table_available` means the physical `file_issues` table exists; `file_issues_data_current` is the freshness/trust bit consumers should use before treating validate rows as authoritative. +Index-generation completeness is computed by one persisted-readiness reader and +reused by the successful full/update index response, immediate status and +workspace status, and MCP indexing/status responses. Persisted omission evidence +from symbols-only runs, `file_too_large`, `symbol_count_exceeded`, +`reference_count_exceeded`, extractor failures, and reference safety caps makes +`index_complete=false` with stable `index_incomplete_reasons`. +`reference_graph_complete` additionally requires an available, current graph +generation and repeats graph-specific stable reasons. A legacy database without +the completeness metadata keeps the compatibility default unless its persisted +rows prove that work was omitted. + Reference extraction publishes its fixed safety limits through CLI `languages --json` / `status --json` and the corresponding MCP responses: 50,000 lookup symbols, 20,000 lookup lines, 512 names per @@ -801,7 +812,8 @@ Current stable codes and triggers: | `hotspot_family_marker_fingerprint_incomplete` | hotspot-family marker fingerprint traversal hit a safety cap, so family trust was not stamped authoritative | reduce generated/ignored marker trees or raise the cap in code, then run `cdidx index --rebuild` | | `partial_family_key_population` | hotspot-family metadata is stamped but some indexed symbols still have NULL `family_key` values | `cdidx index --rebuild` | | `graph_table_available=false` | `symbol_references` is missing or not graph-ready | `cdidx index ` | -| `reference_graph_complete=false` | a reference-extraction safety cap was reached, or legacy storage cannot report cap state | narrow/exclude the reported generated or pathological files, then run `cdidx index ` | +| `reference_graph_complete=false` | the graph generation is unavailable/stale, a symbols-only run omitted it, or persisted file/extractor/cap evidence makes the index generation incomplete | address the reported stable reasons, then run `cdidx index ` | +| `index_complete=false` | a symbols-only run or persisted file-size, symbol-count, reference-count, extractor-failure, or safety-cap evidence proves that indexing work was omitted | address `index_incomplete_reasons`, then run `cdidx index ` | | `issues_table_available=false` | `file_issues` is missing or not issue-ready | `cdidx index ` | | `csharp_symbol_name_ready=false` | C# canonical symbol-name stamps are stale | `cdidx index ` | | `csharp_metadata_target_ready=false` | C# metadata-target stamps are stale | `cdidx index ` | @@ -3821,6 +3833,16 @@ filter、downstream JSON consumer が同じ値を理解できるようにして `status --json` は trust field のいずれかが degraded の場合に structured readiness guidance を出す。トップレベルの `degraded_root_cause` は primary の安定した machine-readable code で、`readiness_degradations[]` は degraded な各 field と `root_cause`、人間向け `degraded_reason`、`recommended_action`、`alternative_action` を列挙する。`migration_in_progress` は active batch marker から設定し、一時的な writer/migration window と恒久的な degraded index をクライアントが区別できるようにする。`issues_table_available` は物理的な `file_issues` table の存在を意味し、validate rows を authoritative として扱う前の freshness/trust bit は `file_issues_data_current` を使う。 +index generation の completeness は単一の persisted-readiness reader で計算し、 +成功した full/update index response、直後の status / workspace status、MCP の +indexing/status response で再利用します。symbols-only run、`file_too_large`、 +`symbol_count_exceeded`、`reference_count_exceeded`、extractor failure、 +reference safety cap の永続化済み省略証拠がある場合は +`index_complete=false` となり、安定した `index_incomplete_reasons` を返します。 +`reference_graph_complete` はさらに利用可能かつ current な graph generation を要求し、 +graph 固有の安定した理由を返します。completeness metadata を持たない legacy database は、 +永続化済み row が処理の省略を証明しない限り compatibility default を維持します。 + reference extraction の固定 safety limit は lookup symbol 50,000件、lookup line 20,000行、1行あたりの name 512件、container candidate 20,000件で、CLI の `languages --json` / `status --json` と対応する MCP response に公開します。cap diagnostic は file ごとの `file_issues` @@ -3898,7 +3920,8 @@ alternative action を同じ場所へ追加してください。 | `hotspot_family_marker_fingerprint_incomplete` | hotspot-family marker fingerprint traversal が safety cap に到達し、family trust が authoritative に stamp されなかった | generated / ignored marker tree を減らすか code 側の cap を上げてから `cdidx index --rebuild` | | `partial_family_key_population` | hotspot-family metadata は stamp 済みだが、一部の indexed symbol で `family_key` が NULL | `cdidx index --rebuild` | | `graph_table_available=false` | `symbol_references` が無い、または graph-ready ではない | `cdidx index ` | -| `reference_graph_complete=false` | reference-extraction safety cap に到達した、または legacy storage で cap state を報告できない | 報告された generated / pathological file を絞り込むか除外してから `cdidx index ` | +| `reference_graph_complete=false` | graph generation が unavailable/stale、symbols-only run で省略、または永続化済み file/extractor/cap 証拠により index generation が incomplete | 報告された安定理由に対処してから `cdidx index ` | +| `index_complete=false` | symbols-only run、または永続化済みの file-size / symbol-count / reference-count / extractor-failure / safety-cap 証拠により indexing work の省略が判明 | `index_incomplete_reasons` に対処してから `cdidx index ` | | `issues_table_available=false` | `file_issues` が無い、または issue-ready ではない | `cdidx index ` | | `csharp_symbol_name_ready=false` | C# canonical symbol-name stamp が stale | `cdidx index ` | | `csharp_metadata_target_ready=false` | C# metadata-target stamp が stale | `cdidx index ` | diff --git a/README.md b/README.md index 6e43b46dd0..c9afb16b1e 100644 --- a/README.md +++ b/README.md @@ -223,6 +223,16 @@ Check-mode JSON includes `query_context.check_mode` (`explicit` or `implied_by_stale_after`) and the effective `query_context.stale_after_seconds`; ordinary status JSON omits `query_context`. +Index-generation readiness is derived from persisted evidence and is shared by +the index command result, immediate `status` / workspace status, and MCP +responses. `index_complete=false` identifies omitted input or extraction work, +including symbols-only runs, file-size / symbol-count / reference-count limits, +and extractor failures or safety caps; `index_incomplete_reasons` contains the +stable reasons. `reference_graph_complete` additionally requires an available, +current graph generation and reports its own stable reasons. Legacy databases +without the newer metadata remain readable, while persisted omission evidence +still prevents a false complete result. + Reference extraction has fixed safety limits of 50,000 lookup symbols, 20,000 lookup lines, 512 names per line, and 20,000 container candidates. CLI `languages --json` / `status --json` and the corresponding MCP responses publish @@ -621,6 +631,15 @@ check mode の JSON は `query_context.check_mode`（`explicit` または `implied_by_stale_after`）と有効な `query_context.stale_after_seconds` を含み、 通常の status JSON では `query_context` を省略します。 +index generation の readiness は永続化済みの証拠から導出し、index command の結果、 +直後の `status` / workspace status、MCP response で同じ snapshot を共有します。 +symbols-only run、file size / symbol count / reference count の上限、extractor failure、 +safety cap などで入力または抽出処理を省略した場合は `index_complete=false` となり、 +`index_incomplete_reasons` に安定した理由を返します。 +`reference_graph_complete` はさらに利用可能かつ current な graph generation を要求し、 +専用の安定した理由を返します。新しい metadata を持たない legacy database も読み取り可能な +ままですが、永続化済みの省略証拠がある場合は誤って complete と報告しません。 + reference extraction の固定 safety limit は lookup symbol 50,000件、lookup line 20,000行、1行あたりの name 512件、container candidate 20,000件です。 CLI の `languages --json` / `status --json` と対応する MCP response は diff --git a/TESTING_GUIDE.md b/TESTING_GUIDE.md index cabd5c32b4..bd9da0c376 100644 --- a/TESTING_GUIDE.md +++ b/TESTING_GUIDE.md @@ -29,6 +29,7 @@ Use the full suite by default. Use targeted filters only while iterating locally - Markdown unused-audit coverage indexes one real Markdown fixture containing common backtick and tilde fence-language markers. Keep default suppression, `documentation_surface` totals, reason tags, and `--all` recovery in that shared fixture. - Full-scan CLI and MCP no-op coverage treats one repository-wide reusable-stat snapshot read and one folded-readiness verification as performance contracts. Keep assertions for one snapshot read, one stat lookup per candidate, one folded verification, and no content load for unchanged files when changing incremental indexing. - Reference-graph refresh coverage treats graph-neutral indexing as a performance contract across incremental full scan, scoped `--files` update, and MCP indexing. Keep zero-refresh assertions for new and modified source files without symbols/references, plus a single batched refresh assertion when existing or new graph identity rows change. A healthy incremental generation must restrict identity/candidate/recursion work to transaction-committed dirty files, old and new `(language, folded name)` dependencies, and their old/new reciprocal edges; retain C#/Python language-transition and unchanged-target parity with a subsequent full refresh, rolled-back file batches, cancellation/retry, orphan-candidate cleanup, and the controlled 4,100-of-4,100 broad-scope fallback. Fresh/rebuild runs, missing identity contracts, and dirty sets of at least 4,096 references covering at least 50% of the graph must keep the full-refresh path. Query-plan coverage must keep all four scoped update phases and all ten candidate inserts on dirty-table-driven reference primary-key seeks, keep C# instantiate grouping on lookup names plus `idx_symbols_name_folded`, and prove that a sub-4,096 dirty set does not count the whole reference table without an explicit diagnostic hook. +- Index-generation completeness coverage uses a table-driven full, symbols-only, max-file-byte, max-symbol, and max-reference matrix, with extractor failure kept separate because it uses a mutable hook. Assert that index-command JSON, immediate status, and workspace health expose identical index/graph booleans and reason arrays where available; MCP cap coverage must match the persisted status snapshot as well. Remove the additive completeness metadata in healthy and capped fixtures to preserve legacy fallback coverage. Human output must identify incomplete generations instead of printing a complete summary. - Reference-identity refresh coverage treats a stable graph rebuild as a physical-write performance contract. Keep NULL-safe changed-row predicates for source identity, the four-column target-resolution tuple, self-reference, and mutual-recursion updates; trigger audits must remain at zero on a stable rerun, repair each corrupted phase once, and prove a later-phase failure rolls back earlier identity writes. SQLite `changes()` must continue to report the final mutual-recursion phase. - C# metadata-target resolver coverage treats propagation work and stable reruns as performance contracts. Keep the reverse-ordered 8,000-class chain at exactly `n - 1` dependency edges and `n` queue visits instead of using a wall-clock threshold; retain cross-file partial fan-in, an unseeded cycle, pre-cancellation, rollback of an earlier row after an injected later update failure, zero trigger-audited writes on a stable rerun, and exactly one write when repairing a corrupted derived row. - Reference-insert transaction coverage keeps the public APIs' #1518 transaction/SAVEPOINT per 71-row batch, while the explicit atomic-file APIs must reject calls without a live caller-owned transaction and open zero reference-batch scopes. Atomic-file reference-line materialization may group only complete 71-row reference batches, stopping before the union of `(file_id, line, context)` keys would exceed 333 rows or after 32 batches; reference INSERT executions and their progress/cancellation checkpoints remain on the original 71-row boundaries. Preserve exact public/atomic statement counts, the 333-row and 32-batch stops, the unique `reference_lines` autoindex lookup plan, batch-two/three failure rollback for both normal and new-file reference-line paths, same/different contexts across a batch boundary, cancellation and empty-input ordering, and guarded multi-language integration coverage for full scan, scoped update, MCP indexing, and TypeScript augmentation rebuild. The controlled 321,352-reference/856-file performance contract retains the repository snapshot's five/six-batch large-file distribution and compares 5,009 public batch scopes with zero atomic-file batch scopes without using a wall-clock threshold. @@ -911,6 +912,7 @@ dotnet test --filter "FullyQualifiedName~GitHelperTests" - Markdown の unused audit coverage は、一般的な backtick / tilde fence の language marker を含む実 Markdown fixture を1回 index します。同じ fixture で既定抑制、`documentation_surface` totals、reason tag、`--all` による復元を維持してください。 - full-scan CLI と MCP の no-op coverage は、リポジトリ全体の reusable-stat snapshot read と folded-readiness verification がそれぞれ 1 回であることを performance contract とします。incremental indexing を変更するときは、snapshot read が 1 回、候補ごとの stat lookup が 1 回、folded verification が 1 回、unchanged file の content load が 0 回という assertion を維持してください。 - reference-graph refresh coverage は、incremental full scan、scoped `--files` update、MCP indexing を横断する graph-neutral indexing を performance contract とします。symbol/reference を持たない新規・変更 source file では refresh 0 回を維持し、既存または新規の graph identity 行が変化する場合は batch 全体で refresh 1 回を assertion してください。健全な incremental generation では identity / candidate / recursion 処理を transaction commit 済みの dirty file、旧・新の `(language, folded name)` 依存、旧・新の逆辺に限定します。C# / Python の言語遷移、未変更targetを参照する新規callerと後続full refreshのparity、rollback file batch、cancel後retry、孤立candidate cleanup、4,100件中4,100件をdirtyにする制御broad-scope fallbackを維持してください。fresh/rebuild、identity契約欠落、または4,096件以上かつgraphの50%以上を占めるdirty集合ではfull-refresh経路を維持します。query-plan coverageでは、scoped updateの4 phaseとcandidate INSERT 10本をdirty table起点のreference主キーseekに保ち、C# instantiate groupingをlookup nameと`idx_symbols_name_folded`起点にし、明示的なdiagnostic hookがない4,096件未満のdirty集合ではreference table全件COUNTを行わないことを検証してください。 +- index generation の completeness coverage は full、symbols-only、max-file-byte、max-symbol、max-reference を table-driven matrix で検証し、mutable hook を使う extractor failure は別 case に保ちます。index command JSON、直後の status、workspace health で、利用可能な index/graph の boolean と reason array が完全に一致すること、MCP の cap case も persisted status snapshot と一致することを assertion してください。healthy / capped fixture から additive completeness metadata を削除し、legacy fallback coverage も維持します。human output は complete summary ではなく incomplete generation を明示する必要があります。 - reference identity refresh coverage は、安定graphの再構築を物理writeのperformance contractとします。source identity、target resolutionの4列tuple、self reference、mutual recursionの更新にはNULL-safeなchanged-row predicateを維持し、安定rerunのtrigger auditは0、各corrupt phaseのrepairは1回、後段phaseの失敗で先行identity writeもrollbackされることを検証してください。SQLite `changes()` は引き続き最後のmutual-recursion phaseを表します。 - C# metadata-target resolver coverage は propagation work と安定 rerun を performance contract とします。逆順に保存した 8,000 class の chain では wall-clock threshold を使わず、dependency edge が厳密に `n - 1`、queue visit が `n` であることを維持してください。cross-file partial fan-in、seed を持たない cycle、事前 cancel、後段 update の注入失敗時に先行 row も rollback されること、安定 rerun の trigger audit が write 0 回、破損した derived row の修復が厳密に 1 write であることも残します。 - reference insert の transaction coverage は、public API の #1518 契約として71 row batchごとの transaction/SAVEPOINTを維持し、明示atomic-file APIは呼出元所有のlive transactionなしでは拒否され、reference batch scopeを0回に保つことを検証します。atomic-fileのreference-line materializationは完全な71 row reference batchだけをまとめ、`(file_id, line, context)` keyの和集合が333行を超える直前、または32 batchで停止します。reference INSERTの実行回数とprogress/cancellation checkpointは元の71 row境界に保ってください。public/atomicの正確なstatement数、333行/32 batch停止、`reference_lines` unique autoindexのlookup plan、通常/new-file両方のreference-line pathでbatch 2/3失敗時の全rollback、batch境界をまたぐ同一/異なるcontext、cancelとempty入力の順序、full scan・scoped update・MCP indexing・TypeScript augmentation rebuildのmulti-language guard付きintegrationを維持してください。321,352 refs / 856 filesの制御performance契約は自己snapshotの5/6 batch巨大file分布を保ち、wall-clock閾値を使わずpublic 5,009 scopeとatomic-file 0 scopeを比較します。 diff --git a/changelog.d/unreleased/4826.fixed.md b/changelog.d/unreleased/4826.fixed.md new file mode 100644 index 0000000000..63c1b5d4c6 --- /dev/null +++ b/changelog.d/unreleased/4826.fixed.md @@ -0,0 +1,20 @@ +--- +category: fixed +issues: + - 4826 +affected: + - src/CodeIndex/Database/DbReader.IndexGenerationReadiness.cs + - src/CodeIndex/Database/DbReader.WorkspaceHealth.cs + - src/CodeIndex/Cli/IndexCommandRunner.FullScan.Output.cs + - src/CodeIndex/Mcp/McpToolHandlers.Indexing.Results.cs + - README.md + - DEVELOPER_GUIDE.md +--- + +## English + +- **Index completeness now reflects persisted omissions (#4826)** — Full, symbols-only, capped, and failed extraction runs now expose one consistent set of index and reference-graph readiness fields and stable reasons across CLI index output, status/workspace status, and MCP responses. + +## 日本語 + +- **index completeness が永続化済みの省略を反映するようになりました (#4826)** — full、symbols-only、上限到達、抽出失敗の各 run で、CLI index output、status/workspace status、MCP response が同じ index / reference-graph readiness field と安定理由を返すようになりました。 diff --git a/src/CodeIndex/Cli/IndexCommandRunner.Diagnostics.cs b/src/CodeIndex/Cli/IndexCommandRunner.Diagnostics.cs index 36c9b8e75e..5a8d4d4de7 100644 --- a/src/CodeIndex/Cli/IndexCommandRunner.Diagnostics.cs +++ b/src/CodeIndex/Cli/IndexCommandRunner.Diagnostics.cs @@ -115,7 +115,8 @@ private static void StampLastIndexRunMetadata( rowsDeleted, memoryTimeline, diagnostics: null, - referenceExtractionCapHits: null); + referenceExtractionCapHits: null, + indexIncompleteReasons: null); private static void StampLastIndexRunMetadata( DbWriter writer, @@ -131,7 +132,8 @@ private static void StampLastIndexRunMetadata( long rowsDeleted, IndexMemoryTimelineJsonResult? memoryTimeline, IReadOnlyList? diagnostics, - ReferenceExtractionCapHitSummary? referenceExtractionCapHits) + ReferenceExtractionCapHitSummary? referenceExtractionCapHits, + IReadOnlyList? indexIncompleteReasons) { writer.SetMetaValues( (DbContext.LastIndexRunModeMetaKey, mode), @@ -152,7 +154,7 @@ private static void StampLastIndexRunMetadata( ? null : (memoryTimeline.PeakWorkingSetBytes / (1024 * 1024)).ToString(System.Globalization.CultureInfo.InvariantCulture))); StampLastIndexRunDiagnostics(writer, diagnostics); - writer.MarkIndexComplete(); + writer.MarkIndexCompleteness(indexIncompleteReasons ?? []); writer.ClearLastFailedIndexRunMetadata(); } diff --git a/src/CodeIndex/Cli/IndexCommandRunner.FullScan.Output.cs b/src/CodeIndex/Cli/IndexCommandRunner.FullScan.Output.cs index f3ab450c65..044f2e413d 100644 --- a/src/CodeIndex/Cli/IndexCommandRunner.FullScan.Output.cs +++ b/src/CodeIndex/Cli/IndexCommandRunner.FullScan.Output.cs @@ -83,7 +83,7 @@ private static int WriteFullScanFinalOutput(FullScanFinalOutputContext output) : output.Writer.GetCounts(); var signalReader = new DbReader(output.Writer.Connection); var referenceExtractionCapHitsAfter = signalReader.GetReferenceExtractionCapHits(); - var referenceGraphCompleteAfter = signalReader.IsReferenceGraphComplete( + var persistedReadinessAfter = signalReader.GetPersistedIndexGenerationReadiness( referenceExtractionCapHitsAfter); var sqlGraphContractSignalAfter = signalReader.GetSqlGraphContractSignal(lang: null); if (!output.HasSqlFilesAfter) @@ -110,7 +110,7 @@ private static int WriteFullScanFinalOutput(FullScanFinalOutputContext output) var hotspotFamilyDegradedReasonAfter = hotspotFamilySignalAfter.DegradedReason; var foldOnlyRemediation = BuildFoldOnlyReadinessRemediation( - output.GraphTableAvailableAfter, + persistedReadinessAfter.GraphTableAvailable, output.IssuesTableAvailableAfter, sqlGraphContractReadyAfter, hotspotFamilyReadyAfter, @@ -150,11 +150,17 @@ private static int WriteFullScanFinalOutput(FullScanFinalOutputContext output) SymbolsDroppedByKindFilter = output.SymbolsDroppedByKindFilter, }, SymbolKindFilter = output.Options.SymbolKindFilter.ToJsonResult(), - GraphTableAvailable = output.GraphTableAvailableAfter, - GraphDataCurrent = output.Errors == 0 && output.GraphTableAvailableAfter && referenceGraphCompleteAfter, - IndexComplete = output.Errors == 0, + GraphTableAvailable = persistedReadinessAfter.GraphTableAvailable, + GraphDataCurrent = persistedReadinessAfter.GraphDataCurrent, + IndexComplete = persistedReadinessAfter.IndexComplete, + IndexIncompleteReasons = persistedReadinessAfter.IndexComplete + ? null + : persistedReadinessAfter.IndexIncompleteReasons, ReferenceExtractionLimits = ReferenceExtractor.GetSafetyLimits(), - ReferenceGraphComplete = referenceGraphCompleteAfter, + ReferenceGraphComplete = persistedReadinessAfter.ReferenceGraphComplete, + ReferenceGraphIncompleteReasons = persistedReadinessAfter.ReferenceGraphComplete + ? null + : persistedReadinessAfter.ReferenceGraphIncompleteReasons, ReferenceExtractionCapHits = referenceExtractionCapHitsAfter, ErrorCode = output.Errors > 0 ? CommandErrorCodes.IndexPartial : null, IssuesTableAvailable = output.IssuesTableAvailableAfter, @@ -210,7 +216,14 @@ private static int WriteFullScanFinalOutput(FullScanFinalOutputContext output) if (output.Warnings > 0) CommandOutputWriter.WriteLine(ConsoleUi.FormatSummaryLine("Warnings", ConsoleUi.FormatNumber(output.Warnings), indent: " ")); if (output.Errors > 0) CommandOutputWriter.WriteLine(ConsoleUi.FormatSummaryLine("Errors", ConsoleUi.FormatNumber(output.Errors), indent: " ")); if (output.SymbolsDroppedByKindFilter > 0) CommandOutputWriter.WriteLine(ConsoleUi.FormatSummaryLine("Filtered symbols", ConsoleUi.FormatNumber(output.SymbolsDroppedByKindFilter), indent: " ")); - CommandOutputWriter.WriteLine(ConsoleUi.FormatSummaryLine("Graph", output.GraphTableAvailableAfter ? "ready" : "degraded", indent: " ")); + CommandOutputWriter.WriteLine(ConsoleUi.FormatSummaryLine( + "Index", + persistedReadinessAfter.IndexComplete ? "complete" : "incomplete", + indent: " ")); + CommandOutputWriter.WriteLine(ConsoleUi.FormatSummaryLine( + "Graph", + persistedReadinessAfter.ReferenceGraphComplete ? "ready" : "degraded", + indent: " ")); CommandOutputWriter.WriteLine(ConsoleUi.FormatSummaryLine("Issues", output.IssuesTableAvailableAfter ? "ready" : "degraded", indent: " ")); CommandOutputWriter.WriteLine(ConsoleUi.FormatSummaryLine("SQL graph", sqlGraphContractReadyAfter ? "ready" : "degraded", indent: " ")); CommandOutputWriter.WriteLine(ConsoleUi.FormatSummaryLine("Hotspots", hotspotFamilyReadyAfter ? "ready" : "degraded", indent: " ")); @@ -221,18 +234,26 @@ private static int WriteFullScanFinalOutput(FullScanFinalOutputContext output) CommandOutputWriter.WriteLine(); if (output.Errors > 0) ConsoleUi.PrintWarning($"Some files failed to index. Fix the reported files or permissions, then rerun `cdidx index \"{output.ProjectRoot}\"` to restore a fully ready index."); - if (!output.GraphTableAvailableAfter || !output.IssuesTableAvailableAfter || !sqlGraphContractReadyAfter || !hotspotFamilyReadyAfter || !output.CSharpSymbolNameReadyAfter || !output.CSharpMetadataTargetReadyAfter || !output.FoldReadyAfter) - ConsoleUi.PrintWarning(GetIndexReadinessWarning(output.GraphTableAvailableAfter, output.IssuesTableAvailableAfter, sqlGraphContractReadyAfter, hotspotFamilyReadyAfter, output.CSharpSymbolNameReadyAfter, output.CSharpMetadataTargetReadyAfter, output.FoldReadyAfter, output.FoldReadyReasonAfter, output.ProjectRoot, output.ResolvedDbPath)); + if (!persistedReadinessAfter.IndexComplete) + ConsoleUi.PrintWarning($"Index generation is incomplete: {string.Join(", ", persistedReadinessAfter.IndexIncompleteReasons)}."); + if (!persistedReadinessAfter.ReferenceGraphComplete) + ConsoleUi.PrintWarning($"Reference graph is incomplete: {string.Join(", ", persistedReadinessAfter.ReferenceGraphIncompleteReasons)}."); + if (!persistedReadinessAfter.GraphTableAvailable || !output.IssuesTableAvailableAfter || !sqlGraphContractReadyAfter || !hotspotFamilyReadyAfter || !output.CSharpSymbolNameReadyAfter || !output.CSharpMetadataTargetReadyAfter || !output.FoldReadyAfter) + ConsoleUi.PrintWarning(GetIndexReadinessWarning(persistedReadinessAfter.GraphTableAvailable, output.IssuesTableAvailableAfter, sqlGraphContractReadyAfter, hotspotFamilyReadyAfter, output.CSharpSymbolNameReadyAfter, output.CSharpMetadataTargetReadyAfter, output.FoldReadyAfter, output.FoldReadyReasonAfter, output.ProjectRoot, output.ResolvedDbPath)); if (cwdDriftDetected) ConsoleUi.PrintWarning(cwdDriftNotice!); - if (output.Errors == 0 && output.ShowNextSteps) + if (output.Errors == 0 + && persistedReadinessAfter.IndexComplete + && output.ShowNextSteps) ConsoleUi.PrintIndexCompleteSummary(output.ProjectRoot, output.ResolvedDbPath, incremental: !output.Options.Rebuild, output.FilesCount, output.LanguageCounts); } if (!output.Options.Json && !output.Options.Quiet && output.Stopwatch.Elapsed >= TimeSpan.FromSeconds(5)) ConsoleUi.EmitCompletionNotification( output.Options.NotifyMode, - $"cdidx index complete ({ConsoleUi.Counted(output.FilesCount, "file", format: "N0")})"); + persistedReadinessAfter.IndexComplete + ? $"cdidx index complete ({ConsoleUi.Counted(output.FilesCount, "file", format: "N0")})" + : $"cdidx index finished with omissions ({ConsoleUi.Counted(output.FilesCount, "file", format: "N0")})"); return output.Errors > 0 && !output.Options.AllowPartial ? CommandExitCodes.PartialResult diff --git a/src/CodeIndex/Cli/IndexCommandRunner.FullScan.Readiness.cs b/src/CodeIndex/Cli/IndexCommandRunner.FullScan.Readiness.cs index bd749dcc2c..72b9b6a15e 100644 --- a/src/CodeIndex/Cli/IndexCommandRunner.FullScan.Readiness.cs +++ b/src/CodeIndex/Cli/IndexCommandRunner.FullScan.Readiness.cs @@ -246,7 +246,8 @@ context.SkippedSymbolExtractorLanguages is null context.Purged, memoryTimelineForStamp, context.IndexRunDiagnostics, - writer.GetReferenceExtractionCapHits(issuesTableAvailableAfter)); + writer.GetReferenceExtractionCapHits(issuesTableAvailableAfter), + writer.GetPersistedIndexOmissionReasons(issuesTableAvailableAfter)); } return new FullScanReadinessResult( diff --git a/src/CodeIndex/Cli/IndexCommandRunner.Update.Output.cs b/src/CodeIndex/Cli/IndexCommandRunner.Update.Output.cs index 9e14ad5831..1f14d85add 100644 --- a/src/CodeIndex/Cli/IndexCommandRunner.Update.Output.cs +++ b/src/CodeIndex/Cli/IndexCommandRunner.Update.Output.cs @@ -58,7 +58,7 @@ private static int WriteUpdateFinalOutput(UpdateFinalOutputContext output) var (totalFiles, totalChunks, totalSymbols, totalReferences) = output.Writer.GetCounts(); var signalReader = new DbReader(output.Writer.Connection); var referenceExtractionCapHitsAfter = signalReader.GetReferenceExtractionCapHits(); - var referenceGraphCompleteAfter = signalReader.IsReferenceGraphComplete( + var persistedReadinessAfter = signalReader.GetPersistedIndexGenerationReadiness( referenceExtractionCapHitsAfter); var sqlGraphContractSignalAfter = signalReader.GetSqlGraphContractSignal(lang: null); var hdlGraphContractSignalAfter = signalReader.GetHdlGraphContractSignal(lang: null); @@ -69,7 +69,7 @@ private static int WriteUpdateFinalOutput(UpdateFinalOutputContext output) var hotspotFamilyDegradedReasonAfter = hotspotFamilySignalAfter.DegradedReason; var foldOnlyRemediation = BuildFoldOnlyReadinessRemediation( - output.GraphTableAvailableAfter, + persistedReadinessAfter.GraphTableAvailable, output.IssuesTableAvailableAfter, sqlGraphContractReadyAfter, hotspotFamilyReadyAfter, @@ -102,14 +102,17 @@ private static int WriteUpdateFinalOutput(UpdateFinalOutputContext output) FtsMergeRan = output.FtsMergeRan, }, SymbolKindFilter = output.Options.SymbolKindFilter.ToJsonResult(), - GraphTableAvailable = output.GraphTableAvailableAfter, - GraphDataCurrent = output.Errors == 0 - && output.GraphTableAvailableAfter - && referenceGraphCompleteAfter - && hdlGraphContractSignalAfter.Ready, - IndexComplete = output.Errors == 0, + GraphTableAvailable = persistedReadinessAfter.GraphTableAvailable, + GraphDataCurrent = persistedReadinessAfter.GraphDataCurrent, + IndexComplete = persistedReadinessAfter.IndexComplete, + IndexIncompleteReasons = persistedReadinessAfter.IndexComplete + ? null + : persistedReadinessAfter.IndexIncompleteReasons, ReferenceExtractionLimits = ReferenceExtractor.GetSafetyLimits(), - ReferenceGraphComplete = referenceGraphCompleteAfter, + ReferenceGraphComplete = persistedReadinessAfter.ReferenceGraphComplete, + ReferenceGraphIncompleteReasons = persistedReadinessAfter.ReferenceGraphComplete + ? null + : persistedReadinessAfter.ReferenceGraphIncompleteReasons, ReferenceExtractionCapHits = referenceExtractionCapHitsAfter, ErrorCode = output.Errors > 0 ? CommandErrorCodes.IndexPartial : null, IssuesTableAvailable = output.IssuesTableAvailableAfter, @@ -157,7 +160,14 @@ private static int WriteUpdateFinalOutput(UpdateFinalOutputContext output) if (output.Errors > 0) CommandOutputWriter.WriteLine(ConsoleUi.FormatSummaryLine("Errors", ConsoleUi.FormatNumber(output.Errors), indent: " ")); if (output.SymbolsDroppedByKindFilter > 0) CommandOutputWriter.WriteLine(ConsoleUi.FormatSummaryLine("Filtered symbols", ConsoleUi.FormatNumber(output.SymbolsDroppedByKindFilter), indent: " ")); if (output.FtsMergeRan) CommandOutputWriter.WriteLine(ConsoleUi.FormatSummaryLine("FTS merge", "completed", indent: " ")); - CommandOutputWriter.WriteLine(ConsoleUi.FormatSummaryLine("Graph", output.GraphTableAvailableAfter ? "ready" : "degraded", indent: " ")); + CommandOutputWriter.WriteLine(ConsoleUi.FormatSummaryLine( + "Index", + persistedReadinessAfter.IndexComplete ? "complete" : "incomplete", + indent: " ")); + CommandOutputWriter.WriteLine(ConsoleUi.FormatSummaryLine( + "Graph", + persistedReadinessAfter.ReferenceGraphComplete ? "ready" : "degraded", + indent: " ")); CommandOutputWriter.WriteLine(ConsoleUi.FormatSummaryLine("Issues", output.IssuesTableAvailableAfter ? "ready" : "degraded", indent: " ")); CommandOutputWriter.WriteLine(ConsoleUi.FormatSummaryLine("SQL graph", sqlGraphContractReadyAfter ? "ready" : "degraded", indent: " ")); CommandOutputWriter.WriteLine(ConsoleUi.FormatSummaryLine("Hotspots", hotspotFamilyReadyAfter ? "ready" : "degraded", indent: " ")); @@ -168,8 +178,12 @@ private static int WriteUpdateFinalOutput(UpdateFinalOutputContext output) CommandOutputWriter.WriteLine(); if (output.Errors > 0) ConsoleUi.PrintWarning($"Some files failed to update. Fix the reported files or permissions, then rerun `cdidx index \"{output.ProjectRoot}\"` to restore a fully ready index."); - if (!output.GraphTableAvailableAfter || !output.IssuesTableAvailableAfter || !sqlGraphContractReadyAfter || !hotspotFamilyReadyAfter || !output.CSharpSymbolNameReadyAfter || !output.CSharpMetadataTargetReadyAfter || !output.FoldReadyAfter) - ConsoleUi.PrintWarning(GetIndexReadinessWarning(output.GraphTableAvailableAfter, output.IssuesTableAvailableAfter, sqlGraphContractReadyAfter, hotspotFamilyReadyAfter, output.CSharpSymbolNameReadyAfter, output.CSharpMetadataTargetReadyAfter, output.FoldReadyAfter, output.FoldReadyReasonAfter, output.ProjectRoot, output.ResolvedDbPath)); + if (!persistedReadinessAfter.IndexComplete) + ConsoleUi.PrintWarning($"Index generation is incomplete: {string.Join(", ", persistedReadinessAfter.IndexIncompleteReasons)}."); + if (!persistedReadinessAfter.ReferenceGraphComplete) + ConsoleUi.PrintWarning($"Reference graph is incomplete: {string.Join(", ", persistedReadinessAfter.ReferenceGraphIncompleteReasons)}."); + if (!persistedReadinessAfter.GraphTableAvailable || !output.IssuesTableAvailableAfter || !sqlGraphContractReadyAfter || !hotspotFamilyReadyAfter || !output.CSharpSymbolNameReadyAfter || !output.CSharpMetadataTargetReadyAfter || !output.FoldReadyAfter) + ConsoleUi.PrintWarning(GetIndexReadinessWarning(persistedReadinessAfter.GraphTableAvailable, output.IssuesTableAvailableAfter, sqlGraphContractReadyAfter, hotspotFamilyReadyAfter, output.CSharpSymbolNameReadyAfter, output.CSharpMetadataTargetReadyAfter, output.FoldReadyAfter, output.FoldReadyReasonAfter, output.ProjectRoot, output.ResolvedDbPath)); if (cwdDriftDetected) ConsoleUi.PrintWarning(cwdDriftNotice!); } @@ -177,7 +191,9 @@ private static int WriteUpdateFinalOutput(UpdateFinalOutputContext output) if (!output.Options.Json && !output.Options.Quiet && output.Stopwatch.Elapsed >= TimeSpan.FromSeconds(5)) ConsoleUi.EmitCompletionNotification( output.Options.NotifyMode, - $"cdidx index update complete ({ConsoleUi.Counted(output.Updated + output.Removed + output.Skipped, "file", format: "N0")})"); + persistedReadinessAfter.IndexComplete + ? $"cdidx index update complete ({ConsoleUi.Counted(output.Updated + output.Removed + output.Skipped, "file", format: "N0")})" + : $"cdidx index update finished with omissions ({ConsoleUi.Counted(output.Updated + output.Removed + output.Skipped, "file", format: "N0")})"); return output.Errors > 0 && !output.Options.AllowPartial ? CommandExitCodes.PartialResult diff --git a/src/CodeIndex/Cli/IndexCommandRunner.Update.cs b/src/CodeIndex/Cli/IndexCommandRunner.Update.cs index d9d20b3a8a..ad1fbeced1 100644 --- a/src/CodeIndex/Cli/IndexCommandRunner.Update.cs +++ b/src/CodeIndex/Cli/IndexCommandRunner.Update.cs @@ -852,7 +852,8 @@ bool TryValidateCSharpWorkspaceInputSnapshot( removed, memoryTimelineForStamp, indexRunDiagnostics, - writer.GetReferenceExtractionCapHits(issuesTableAvailableAfter)); + writer.GetReferenceExtractionCapHits(issuesTableAvailableAfter), + writer.GetPersistedIndexOmissionReasons(issuesTableAvailableAfter)); } return WriteUpdateFinalOutput(new UpdateFinalOutputContext { diff --git a/src/CodeIndex/Cli/JsonOutputContracts.cs b/src/CodeIndex/Cli/JsonOutputContracts.cs index 3b1ba2b2b3..86343d961a 100644 --- a/src/CodeIndex/Cli/JsonOutputContracts.cs +++ b/src/CodeIndex/Cli/JsonOutputContracts.cs @@ -815,8 +815,10 @@ internal sealed class IndexUpdateJsonResult : IVersionedJsonResult public bool GraphTableAvailable { get; init; } public bool GraphDataCurrent { get; init; } public bool IndexComplete { get; init; } + public IReadOnlyList? IndexIncompleteReasons { get; init; } public ReferenceExtractionSafetyLimits ReferenceExtractionLimits { get; init; } = new(); public bool ReferenceGraphComplete { get; init; } + public IReadOnlyList? ReferenceGraphIncompleteReasons { get; init; } public ReferenceExtractionCapHitSummary ReferenceExtractionCapHits { get; init; } = new(); public string? ErrorCode { get; init; } public bool IssuesTableAvailable { get; init; } @@ -856,8 +858,10 @@ internal sealed class IndexFullScanJsonResult : IVersionedJsonResult public bool GraphTableAvailable { get; init; } public bool GraphDataCurrent { get; init; } public bool IndexComplete { get; init; } + public IReadOnlyList? IndexIncompleteReasons { get; init; } public ReferenceExtractionSafetyLimits ReferenceExtractionLimits { get; init; } = new(); public bool ReferenceGraphComplete { get; init; } + public IReadOnlyList? ReferenceGraphIncompleteReasons { get; init; } public ReferenceExtractionCapHitSummary ReferenceExtractionCapHits { get; init; } = new(); public string? ErrorCode { get; init; } public bool IssuesTableAvailable { get; init; } diff --git a/src/CodeIndex/Database/DbReader.IndexGenerationReadiness.cs b/src/CodeIndex/Database/DbReader.IndexGenerationReadiness.cs new file mode 100644 index 0000000000..51a82c59b2 --- /dev/null +++ b/src/CodeIndex/Database/DbReader.IndexGenerationReadiness.cs @@ -0,0 +1,172 @@ +using CodeIndex.Indexer; +using CodeIndex.Models; +using Microsoft.Data.Sqlite; + +namespace CodeIndex.Database; + +internal sealed record PersistedIndexGenerationReadiness( + bool GraphTableAvailable, + bool GraphDataCurrent, + bool IndexComplete, + IReadOnlyList IndexIncompleteReasons, + bool ReferenceGraphComplete, + IReadOnlyList ReferenceGraphIncompleteReasons, + ReferenceExtractionCapHitSummary ReferenceExtractionCapHits, + bool MigrationInProgress); + +public partial class DbReader +{ + internal const string SymbolsOnlyIndexIncompleteReason = "symbols_only_references_omitted"; + internal const string SymbolsOnlyReferenceGraphIncompleteReason = "symbols_only_graph_omitted"; + internal const string BatchInProgressIncompleteReason = "batch_in_progress"; + + private static readonly string[] IndexOmissionIssueKinds = + [ + "file_too_large", + "symbol_count_exceeded", + "reference_count_exceeded", + .. ReferenceExtractor.ReferenceSafetyCapDiagnosticKinds, + ]; + + internal PersistedIndexGenerationReadiness GetPersistedIndexGenerationReadiness( + ReferenceExtractionCapHitSummary? referenceExtractionCapHits = null, + IReadOnlyDictionary? indexedLanguages = null, + bool? hdlGraphContractReady = null, + SqliteTransaction? transaction = null) + { + var indexCompleteness = TryGetMetaStringInternal(DbContext.IndexCompletenessMetaKey); + var indexIncompleteReasons = MergeDistinctReasons( + ParseMetaStringList(TryGetMetaStringInternal(DbContext.IndexIncompleteReasonsMetaKey)), + ReadPersistedIndexOmissionReasons( + _conn, + _hasIssuesTable, + ParseMetaBool(TryGetMetaStringInternal(DbContext.SymbolsOnlyGraphOmittedMetaKey)) == true, + transaction)); + var migrationInProgress = string.Equals( + TryGetMetaStringInternal(DbContext.BatchInProgressMetaKey), + "true", + StringComparison.OrdinalIgnoreCase); + if (migrationInProgress) + AddDistinctReason(indexIncompleteReasons, BatchInProgressIncompleteReason); + + var explicitlyIncomplete = string.Equals( + indexCompleteness, + "incomplete", + StringComparison.OrdinalIgnoreCase); + if (explicitlyIncomplete && indexIncompleteReasons.Count == 0) + AddDistinctReason(indexIncompleteReasons, DegradationReasonCodes.IndexIncomplete); + var indexComplete = !migrationInProgress + && !explicitlyIncomplete + && indexIncompleteReasons.Count == 0; + + var capHits = referenceExtractionCapHits ?? GetReferenceExtractionCapHits(); + var languages = indexedLanguages ?? GetIndexedLanguageCounts(); + var dynamicReferenceGraphContractsCurrent = + AreDynamicReferenceGraphContractsCurrent(languages); + var symbolsOnlyGraphOmitted = ParseMetaBool( + TryGetMetaStringInternal(DbContext.SymbolsOnlyGraphOmittedMetaKey)) == true; + var referenceGraphIncompleteReasons = MergeDistinctReasons(capHits.Reasons); + if (!_hasReferencesTable) + { + AddDistinctReason( + referenceGraphIncompleteReasons, + symbolsOnlyGraphOmitted + ? SymbolsOnlyReferenceGraphIncompleteReason + : DegradationReasonCodes.GraphTableMissing); + } + foreach (var reason in indexIncompleteReasons) + { + AddDistinctReason( + referenceGraphIncompleteReasons, + reason == SymbolsOnlyIndexIncompleteReason + ? SymbolsOnlyReferenceGraphIncompleteReason + : reason); + } + if (!dynamicReferenceGraphContractsCurrent) + { + AddDistinctReason( + referenceGraphIncompleteReasons, + DynamicReferenceGraphContractStaleReason); + } + + var referenceGraphComplete = _hasReferencesTable + && indexComplete + && capHits.StateAvailable + && capHits.HitCount == 0 + && dynamicReferenceGraphContractsCurrent; + var hdlReady = hdlGraphContractReady + ?? GetHdlGraphContractSignal( + lang: null, + pathPatterns: null, + excludePathPatterns: null, + excludeTests: false).Ready; + + return new PersistedIndexGenerationReadiness( + _hasReferencesTable, + _hasReferencesTable && indexComplete && referenceGraphComplete && hdlReady, + indexComplete, + indexIncompleteReasons, + referenceGraphComplete, + referenceGraphIncompleteReasons, + capHits, + migrationInProgress); + } + + internal static IReadOnlyList ReadPersistedIndexOmissionReasons( + SqliteConnection connection, + bool hasIssuesTable, + bool symbolsOnlyGraphOmitted, + SqliteTransaction? transaction) + { + var reasons = new List(); + if (symbolsOnlyGraphOmitted) + reasons.Add(SymbolsOnlyIndexIncompleteReason); + if (!hasIssuesTable) + return reasons; + + using var command = connection.CreateCommand(); + command.Transaction = transaction; + var parameterNames = new string[IndexOmissionIssueKinds.Length]; + for (var index = 0; index < IndexOmissionIssueKinds.Length; index++) + { + var parameterName = $"@omissionKind{index}"; + parameterNames[index] = parameterName; + SqliteCommandPolicy.Add(command, parameterName, IndexOmissionIssueKinds[index]); + } + command.CommandText = $""" + SELECT kind + FROM file_issues + WHERE kind IN ({string.Join(", ", parameterNames)}) + GROUP BY kind + ORDER BY kind + """; + using var reader = command.ExecuteTrackedReader(); + while (reader.TrackedRead()) + AddDistinctReason(reasons, reader.GetString(0)); + return reasons; + } + + private static List MergeDistinctReasons( + IReadOnlyList? first, + IReadOnlyList? second = null) + { + var reasons = new List(); + if (first != null) + { + foreach (var reason in first) + AddDistinctReason(reasons, reason); + } + if (second != null) + { + foreach (var reason in second) + AddDistinctReason(reasons, reason); + } + return reasons; + } + + private static void AddDistinctReason(List reasons, string reason) + { + if (!reasons.Contains(reason, StringComparer.Ordinal)) + reasons.Add(reason); + } +} diff --git a/src/CodeIndex/Database/DbReader.Status.cs b/src/CodeIndex/Database/DbReader.Status.cs index 23a59f8b7b..1100cc0368 100644 --- a/src/CodeIndex/Database/DbReader.Status.cs +++ b/src/CodeIndex/Database/DbReader.Status.cs @@ -144,36 +144,14 @@ GROUP BY COALESCE(f.lang, 'unknown'), s.kind dbSizeBytes, dbPragmaSettings.AutoVacuum)); var lastIndexRun = GetLastIndexRun(); - var indexCompleteness = TryGetMetaStringInternal(DbContext.IndexCompletenessMetaKey); - var indexIncompleteReasons = ParseMetaStringList(TryGetMetaStringInternal(DbContext.IndexIncompleteReasonsMetaKey)); - var batchInProgress = string.Equals( - TryGetMetaStringInternal(DbContext.BatchInProgressMetaKey), - "true", - StringComparison.OrdinalIgnoreCase); - var lastFailedOrPartialIndexRun = GetLastFailedOrPartialIndexRun(batchInProgress); - var indexComplete = !batchInProgress - && !string.Equals(indexCompleteness, "incomplete", StringComparison.OrdinalIgnoreCase); var referenceExtractionCapHits = GetReferenceExtractionCapHits(); - var dynamicReferenceGraphContractsCurrent = - AreDynamicReferenceGraphContractsCurrent(langs); - var referenceGraphIncompleteReasons = - referenceExtractionCapHits.Reasons?.ToList() ?? []; - if (!dynamicReferenceGraphContractsCurrent - && !referenceGraphIncompleteReasons.Contains( - DynamicReferenceGraphContractStaleReason, - StringComparer.Ordinal)) - { - referenceGraphIncompleteReasons.Add(DynamicReferenceGraphContractStaleReason); - } - var referenceGraphComplete = referenceExtractionCapHits.StateAvailable - && referenceExtractionCapHits.HitCount == 0 - && dynamicReferenceGraphContractsCurrent; - if (batchInProgress) - { - indexIncompleteReasons ??= []; - if (!indexIncompleteReasons.Contains("batch_in_progress", StringComparer.Ordinal)) - indexIncompleteReasons.Add("batch_in_progress"); - } + var persistedReadiness = GetPersistedIndexGenerationReadiness( + referenceExtractionCapHits, + langs, + hdlGraphContractReady, + txn); + var batchInProgress = persistedReadiness.MigrationInProgress; + var lastFailedOrPartialIndexRun = GetLastFailedOrPartialIndexRun(batchInProgress); var result = new StatusResult { @@ -196,22 +174,21 @@ GROUP BY COALESCE(f.lang, 'unknown'), s.kind IndexedHeadTimestamp = indexedHeadTimestamp, Languages = langs, SymbolsByLanguage = symbolsByLanguage.Count > 0 ? symbolsByLanguage : null, - GraphTableAvailable = _hasReferencesTable, - GraphDataCurrent = _hasReferencesTable - && indexComplete - && referenceGraphComplete - && hdlGraphContractReady, + GraphTableAvailable = persistedReadiness.GraphTableAvailable, + GraphDataCurrent = persistedReadiness.GraphDataCurrent, ReferenceExtractionLimits = ReferenceExtractor.GetSafetyLimits(), - ReferenceGraphComplete = referenceGraphComplete, - ReferenceGraphIncompleteReasons = referenceGraphComplete + ReferenceGraphComplete = persistedReadiness.ReferenceGraphComplete, + ReferenceGraphIncompleteReasons = persistedReadiness.ReferenceGraphComplete ? null - : referenceGraphIncompleteReasons, + : persistedReadiness.ReferenceGraphIncompleteReasons.ToList(), ReferenceExtractionCapHits = referenceExtractionCapHits, IssuesTableAvailable = _hasIssuesPhysicalTable, FileIssuesDataCurrent = _hasIssuesTable, MigrationInProgress = batchInProgress, - IndexComplete = indexComplete, - IndexIncompleteReasons = indexComplete ? null : indexIncompleteReasons, + IndexComplete = persistedReadiness.IndexComplete, + IndexIncompleteReasons = persistedReadiness.IndexComplete + ? null + : persistedReadiness.IndexIncompleteReasons.ToList(), HotspotFamilyReady = hotspotFamilySignal.Ready, HotspotFamilyDegradedReason = hotspotFamilySignal.DegradedReason, LanguageReadiness = languageReadiness.Count > 0 ? languageReadiness : null, @@ -294,21 +271,13 @@ private bool AreDynamicReferenceGraphContractsCurrent( } internal bool IsReferenceGraphComplete(ReferenceExtractionCapHitSummary capHits) => - capHits.StateAvailable - && capHits.HitCount == 0 - && AreDynamicReferenceGraphContractsCurrent(GetIndexedLanguageCounts()); + GetPersistedIndexGenerationReadiness(capHits).ReferenceGraphComplete; internal IReadOnlyList GetReferenceGraphIncompleteReasons( ReferenceExtractionCapHitSummary capHits) { - var reasons = capHits.Reasons.ToList(); - if (!AreDynamicReferenceGraphContractsCurrent(GetIndexedLanguageCounts()) - && !reasons.Contains(DynamicReferenceGraphContractStaleReason, StringComparer.Ordinal)) - { - reasons.Add(DynamicReferenceGraphContractStaleReason); - } - - return reasons; + return GetPersistedIndexGenerationReadiness(capHits) + .ReferenceGraphIncompleteReasons; } internal Dictionary GetIndexedLanguageCounts() diff --git a/src/CodeIndex/Database/DbReader.WorkspaceHealth.cs b/src/CodeIndex/Database/DbReader.WorkspaceHealth.cs index 7156283d7a..6f416facf1 100644 --- a/src/CodeIndex/Database/DbReader.WorkspaceHealth.cs +++ b/src/CodeIndex/Database/DbReader.WorkspaceHealth.cs @@ -12,36 +12,28 @@ internal sealed record WorkspaceIndexHealthSnapshot( public partial class DbReader { internal WorkspaceIndexHealthSnapshot GetWorkspaceIndexHealth() - => RunInReadSnapshot(() => + => RunInReadSnapshot(transaction => { var freshness = GetWorkspaceFreshness(); - var batchInProgress = string.Equals( - TryGetMetaStringInternal(DbContext.BatchInProgressMetaKey), - "true", - StringComparison.OrdinalIgnoreCase); - var indexCompleteness = TryGetMetaStringInternal(DbContext.IndexCompletenessMetaKey); - var indexComplete = !batchInProgress - && !string.Equals(indexCompleteness, "incomplete", StringComparison.OrdinalIgnoreCase); var referenceExtractionCapHits = GetReferenceExtractionCapHits(); - var referenceGraphComplete = IsReferenceGraphComplete(referenceExtractionCapHits); var hdlGraphContractReady = !ScopeMayIncludeHdlFiles( lang: null, pathPatterns: null, excludePathPatterns: null, excludeTests: false) || _hdlGraphContractCurrent; - var graphDataCurrent = _hasReferencesTable - && indexComplete - && referenceGraphComplete - && hdlGraphContractReady; + var persistedReadiness = GetPersistedIndexGenerationReadiness( + referenceExtractionCapHits, + hdlGraphContractReady: hdlGraphContractReady, + transaction: transaction); return new WorkspaceIndexHealthSnapshot( freshness.IndexedAt, freshness.LatestModified, - _hasReferencesTable, - graphDataCurrent, - referenceGraphComplete, - indexComplete, + persistedReadiness.GraphTableAvailable, + persistedReadiness.GraphDataCurrent, + persistedReadiness.ReferenceGraphComplete, + persistedReadiness.IndexComplete, _indexNewerThanReader); }); } diff --git a/src/CodeIndex/Database/DbReader.cs b/src/CodeIndex/Database/DbReader.cs index ae69543fd4..ab9efb2e36 100644 --- a/src/CodeIndex/Database/DbReader.cs +++ b/src/CodeIndex/Database/DbReader.cs @@ -138,6 +138,12 @@ public T RunWithGeneratedScope(Func action) /// 複数 statement の読み取りを単一の deferred SQLite snapshot 上で実行する。 /// internal T RunInReadSnapshot(Func action) + { + ArgumentNullException.ThrowIfNull(action); + return RunInReadSnapshot(_ => action()); + } + + internal T RunInReadSnapshot(Func action) { ArgumentNullException.ThrowIfNull(action); using var cancellationRegistration = RegisterSqliteInterruptForCancellation(); @@ -146,7 +152,7 @@ internal T RunInReadSnapshot(Func action) _cancellation.ThrowIfCancellationRequested(); using var transaction = _conn.BeginTransaction(deferred: true); - var result = action(); + var result = action(transaction); _cancellation.ThrowIfCancellationRequested(); transaction.Commit(); return result; diff --git a/src/CodeIndex/Database/DbWriter.ReferenceExtractionCompleteness.cs b/src/CodeIndex/Database/DbWriter.ReferenceExtractionCompleteness.cs index b0bb612647..10d8e03973 100644 --- a/src/CodeIndex/Database/DbWriter.ReferenceExtractionCompleteness.cs +++ b/src/CodeIndex/Database/DbWriter.ReferenceExtractionCompleteness.cs @@ -19,4 +19,14 @@ internal ReferenceExtractionCapHitSummary GetReferenceExtractionCapHits(bool iss _conn, hasIssuesTable: issuesStateAvailable, _activeTransaction); + + internal IReadOnlyList GetPersistedIndexOmissionReasons(bool issuesStateAvailable) + => DbReader.ReadPersistedIndexOmissionReasons( + _conn, + hasIssuesTable: issuesStateAvailable, + symbolsOnlyGraphOmitted: string.Equals( + GetMetaString(DbContext.SymbolsOnlyGraphOmittedMetaKey), + "true", + StringComparison.OrdinalIgnoreCase), + _activeTransaction); } diff --git a/src/CodeIndex/Database/DbWriter.StatusMetadata.cs b/src/CodeIndex/Database/DbWriter.StatusMetadata.cs index 0244c821fc..3424b9e609 100644 --- a/src/CodeIndex/Database/DbWriter.StatusMetadata.cs +++ b/src/CodeIndex/Database/DbWriter.StatusMetadata.cs @@ -48,6 +48,15 @@ public void MarkIndexIncomplete(IReadOnlyList reasons) (DbContext.IndexIncompleteReasonsMetaKey, JsonStringListCodec.Serialize(reasons))); } + public void MarkIndexCompleteness(IReadOnlyList incompleteReasons) + { + ArgumentNullException.ThrowIfNull(incompleteReasons); + if (incompleteReasons.Count == 0) + MarkIndexComplete(); + else + MarkIndexIncomplete(incompleteReasons); + } + /// /// Stamp unknown-extension scan coverage from the latest successful full-worktree scan. /// Stores the total count plus a bounded path sample so status callers can identify the diff --git a/src/CodeIndex/Mcp/McpToolHandlers.Graph.cs b/src/CodeIndex/Mcp/McpToolHandlers.Graph.cs index 6d5ea5ac56..e4f78e4e51 100644 --- a/src/CodeIndex/Mcp/McpToolHandlers.Graph.cs +++ b/src/CodeIndex/Mcp/McpToolHandlers.Graph.cs @@ -672,19 +672,41 @@ private void AddReferenceGraphCompletenessSignal( DbReader reader, ReferenceExtractionCapHitSummary capHits) { - var complete = reader.IsReferenceGraphComplete(capHits); - var incompleteReasons = reader.GetReferenceGraphIncompleteReasons(capHits); + var readiness = reader.GetPersistedIndexGenerationReadiness(capHits); + AddReferenceGraphCompletenessSignal(payload, readiness); + } + + private void AddReferenceGraphCompletenessSignal( + JsonObject payload, + PersistedIndexGenerationReadiness readiness) + { payload["reference_extraction_limits"] = JsonSerializer.SerializeToNode( ReferenceExtractor.GetSafetyLimits(), _jsonOptions); - payload["reference_graph_complete"] = complete; + payload["reference_graph_complete"] = readiness.ReferenceGraphComplete; payload["reference_extraction_cap_hits"] = JsonSerializer.SerializeToNode( - capHits, + readiness.ReferenceExtractionCapHits, _jsonOptions); - if (!complete) + if (!readiness.ReferenceGraphComplete) { payload["reference_graph_incomplete_reasons"] = JsonSerializer.SerializeToNode( - incompleteReasons, + readiness.ReferenceGraphIncompleteReasons, + _jsonOptions); + payload["degraded"] = true; + } + } + + private void AddIndexGenerationReadinessSignal( + JsonObject payload, + PersistedIndexGenerationReadiness readiness) + { + payload["graph_table_available"] = readiness.GraphTableAvailable; + payload["graph_data_current"] = readiness.GraphDataCurrent; + payload["index_complete"] = readiness.IndexComplete; + if (!readiness.IndexComplete) + { + payload["index_incomplete_reasons"] = JsonSerializer.SerializeToNode( + readiness.IndexIncompleteReasons, _jsonOptions); payload["degraded"] = true; } diff --git a/src/CodeIndex/Mcp/McpToolHandlers.Indexing.Execution.cs b/src/CodeIndex/Mcp/McpToolHandlers.Indexing.Execution.cs index d6006584b0..d2ae823362 100644 --- a/src/CodeIndex/Mcp/McpToolHandlers.Indexing.Execution.cs +++ b/src/CodeIndex/Mcp/McpToolHandlers.Indexing.Execution.cs @@ -1173,10 +1173,10 @@ await EmitProgressNotificationAsync( var referenceExtractionCapHits = writer.GetReferenceExtractionCapHits( issuesStateAvailable: (indexSnapshot.Readiness & DbContext.IssuesReadyFlag) != 0); using var referenceSignalReader = new DbReader(writer.Connection, isReadOnly: true); - AddReferenceGraphCompletenessSignal( - structured, - referenceSignalReader, + var persistedReadiness = referenceSignalReader.GetPersistedIndexGenerationReadiness( referenceExtractionCapHits); + AddIndexGenerationReadinessSignal(structured, persistedReadiness); + AddReferenceGraphCompletenessSignal(structured, persistedReadiness); if (!sqlGraphContractReady) { AddSqlGraphContractSignal( @@ -2024,7 +2024,8 @@ await EmitProgressNotificationAsync( (DbContext.LastIndexRunReferenceExtractionCapHitsMetaKey, JsonSerializer.Serialize( referenceExtractionCapHits, StatusMetadataJsonContext.Default.ReferenceExtractionCapHitSummary))); - writer.MarkIndexComplete(); + writer.MarkIndexCompleteness( + writer.GetPersistedIndexOmissionReasons(issuesStateAvailable: true)); writer.ClearLastFailedIndexRunMetadata(); // Persist the current HEAD only after the run is fully successful (errors == 0). // Mirrors the CLI full-scan contract (Issue #1508) so MCP-driven re-indexes also diff --git a/src/CodeIndex/Mcp/McpToolHandlers.Indexing.Results.cs b/src/CodeIndex/Mcp/McpToolHandlers.Indexing.Results.cs index d36345e961..7f69ca393a 100644 --- a/src/CodeIndex/Mcp/McpToolHandlers.Indexing.Results.cs +++ b/src/CodeIndex/Mcp/McpToolHandlers.Indexing.Results.cs @@ -170,7 +170,9 @@ private JsonNode BuildIndexCompletionResult(JsonNode? id, IndexCompletionDetails } AddMcpIndexDiagnostics(structured, details.Failures, details.Diagnostics); using var signalReader = new DbReader(details.Writer.Connection); - AddReferenceGraphCompletenessSignal(structured, signalReader); + var persistedReadiness = signalReader.GetPersistedIndexGenerationReadiness(); + AddIndexGenerationReadinessSignal(structured, persistedReadiness); + AddReferenceGraphCompletenessSignal(structured, persistedReadiness); if (!details.SqlGraphContractReady) { var sqlGraphContractSignal = signalReader.GetSqlGraphContractSignal(); @@ -186,7 +188,9 @@ private JsonNode BuildIndexCompletionResult(JsonNode? id, IndexCompletionDetails return CreateToolResult( id, - details.Errors == 0 && !details.FoldReady + details.Errors == 0 && !persistedReadiness.IndexComplete + ? $"Indexing finished with persisted omissions: {string.Join(", ", persistedReadiness.IndexIncompleteReasons)}." + : details.Errors == 0 && !details.FoldReady ? details.FoldReadyReason switch { "stale_fold_key_version" => "Indexing complete. Note: --exact Unicode fold path not active because unchanged rows still carry an older fold-key version. Rewrite or purge those stale rows and rerun index, run backfill_fold, or do a full rebuild to upgrade.", diff --git a/tests/CodeIndex.Tests/IndexCommandRunnerFullScanTests.cs b/tests/CodeIndex.Tests/IndexCommandRunnerFullScanTests.cs index ebdfb15eef..b386cd3e38 100644 --- a/tests/CodeIndex.Tests/IndexCommandRunnerFullScanTests.cs +++ b/tests/CodeIndex.Tests/IndexCommandRunnerFullScanTests.cs @@ -266,6 +266,186 @@ def helper(): } } + [Theory] + [InlineData("full", true, true, true, null, null)] + [InlineData("symbols-only", false, false, false, "symbols_only_references_omitted", "symbols_only_graph_omitted")] + [InlineData("max-file-bytes", true, false, false, "file_too_large", "file_too_large")] + [InlineData("max-symbols", true, false, false, "symbol_count_exceeded", "symbol_count_exceeded")] + [InlineData("max-references", true, false, false, "reference_count_exceeded", "reference_count_exceeded")] + public void Run_FullScan_CompletenessMatrixMatchesImmediateStatus_Issue4826( + string scenario, + bool expectedGraphTableAvailable, + bool expectedIndexComplete, + bool expectedReferenceGraphComplete, + string? expectedIndexReason, + string? expectedReferenceReason) + { + var projectRoot = CreateTempProject(); + try + { + var args = new List { projectRoot, "--json", "--quiet" }; + switch (scenario) + { + case "full": + case "symbols-only": + File.WriteAllText( + Path.Combine(projectRoot, "App.cs"), + "public class App { public void Run() { Helper(); } private void Helper() { } }\n"); + if (scenario == "symbols-only") + args.Insert(1, "--symbols-only"); + break; + case "max-file-bytes": + File.WriteAllText( + Path.Combine(projectRoot, "large.py"), + "print('start')\n" + new string('a', 256)); + args.InsertRange(1, ["--max-file-bytes", "128"]); + break; + case "max-symbols": + File.WriteAllText( + Path.Combine(projectRoot, "generated.py"), + string.Join('\n', Enumerable.Range(0, 4).Select(i => $"def f{i}(): pass"))); + args.InsertRange(1, ["--max-symbols-per-file", "2"]); + break; + case "max-references": + File.WriteAllText( + Path.Combine(projectRoot, "DenseReferences.cs"), + BuildDenseReferenceCSharpSource(3)); + args.InsertRange(1, ["--max-references-per-file", "2"]); + break; + default: + throw new InvalidOperationException($"Unknown completeness scenario: {scenario}"); + } + + var (indexExitCode, indexJson) = RunAndCaptureJson([.. args]); + + Assert.Equal(CommandExitCodes.Success, indexExitCode); + Assert.Equal(expectedGraphTableAvailable, indexJson.GetProperty("graph_table_available").GetBoolean()); + Assert.Equal(expectedIndexComplete, indexJson.GetProperty("index_complete").GetBoolean()); + Assert.Equal(expectedReferenceGraphComplete, indexJson.GetProperty("reference_graph_complete").GetBoolean()); + Assert.Equal( + expectedGraphTableAvailable && expectedIndexComplete && expectedReferenceGraphComplete, + indexJson.GetProperty("graph_data_current").GetBoolean()); + AssertCompletenessReason(indexJson, "index_incomplete_reasons", expectedIndexReason); + AssertCompletenessReason(indexJson, "reference_graph_incomplete_reasons", expectedReferenceReason); + + var dbPath = Path.Combine(projectRoot, ".cdidx", "codeindex.db"); + var (statusExitCode, statusJson) = RunStatusAndCaptureJson(["--db", dbPath, "--json"]); + + Assert.Equal(CommandExitCodes.Success, statusExitCode); + AssertCompletenessSignalsEqual(indexJson, statusJson); + + using (var db = new DbContext(DbOpenIntent.QueryOnly, dbPath)) + { + var reader = new DbReader(db.Connection, db.IsReadOnly); + var workspaceHealth = reader.GetWorkspaceIndexHealth(); + Assert.Equal( + statusJson.GetProperty("graph_table_available").GetBoolean(), + workspaceHealth.GraphTableAvailable); + Assert.Equal( + statusJson.GetProperty("graph_data_current").GetBoolean(), + workspaceHealth.GraphDataCurrent); + Assert.Equal( + statusJson.GetProperty("index_complete").GetBoolean(), + workspaceHealth.IndexComplete); + Assert.Equal( + statusJson.GetProperty("reference_graph_complete").GetBoolean(), + workspaceHealth.ReferenceGraphComplete); + } + + if (scenario == "max-file-bytes") + { + var humanArgs = args + .Where(arg => arg is not "--json" and not "--quiet") + .ToArray(); + var (humanExitCode, stdout, stderr) = RunAndCaptureStreams(humanArgs); + Assert.Equal(CommandExitCodes.Success, humanExitCode); + Assert.Contains("Index", stdout, StringComparison.Ordinal); + Assert.Contains("incomplete", stdout, StringComparison.Ordinal); + Assert.Contains( + "Index generation is incomplete: file_too_large.", + stderr, + StringComparison.Ordinal); + Assert.Contains( + "Reference graph is incomplete: file_too_large.", + stderr, + StringComparison.Ordinal); + } + } + finally + { + SqliteConnection.ClearAllPools(); + DeleteDirectory(projectRoot); + } + } + + [Theory] + [InlineData(false, true, null)] + [InlineData(true, false, "file_too_large")] + public void PersistedReadiness_OlderDatabaseWithoutCompletenessMetadataUsesSafeFallback_Issue4826( + bool capFileBytes, + bool expectedComplete, + string? expectedReason) + { + var projectRoot = CreateTempProject(); + try + { + File.WriteAllText( + Path.Combine(projectRoot, "sample.py"), + capFileBytes + ? "print('start')\n" + new string('a', 256) + : "def ready(): return True\n"); + var args = capFileBytes + ? new[] { projectRoot, "--max-file-bytes", "128", "--json", "--quiet" } + : new[] { projectRoot, "--json", "--quiet" }; + var (indexExitCode, _) = RunAndCaptureJson(args); + Assert.Equal(CommandExitCodes.Success, indexExitCode); + + var dbPath = Path.Combine(projectRoot, ".cdidx", "codeindex.db"); + using (var connection = OpenNonPoolingConnection(dbPath)) + { + connection.Open(); + using var command = connection.CreateCommand(); + command.CommandText = """ + DELETE FROM codeindex_meta + WHERE key IN (@completeness, @reasons); + """; + command.Parameters.AddWithValue( + "@completeness", + DbContext.IndexCompletenessMetaKey); + command.Parameters.AddWithValue( + "@reasons", + DbContext.IndexIncompleteReasonsMetaKey); + command.ExecuteNonQuery(); + } + + using var db = new DbContext(DbOpenIntent.QueryOnly, dbPath); + var reader = new DbReader(db.Connection, db.IsReadOnly); + var status = reader.GetStatus(); + var workspaceHealth = reader.GetWorkspaceIndexHealth(); + + Assert.Equal(expectedComplete, status.IndexComplete); + Assert.Equal(expectedComplete, status.ReferenceGraphComplete); + Assert.Equal(expectedComplete, status.GraphDataCurrent); + Assert.Equal(expectedComplete, workspaceHealth.IndexComplete); + Assert.Equal(expectedComplete, workspaceHealth.ReferenceGraphComplete); + Assert.Equal(expectedComplete, workspaceHealth.GraphDataCurrent); + if (expectedReason == null) + { + Assert.Null(status.IndexIncompleteReasons); + } + else + { + Assert.Contains(expectedReason, status.IndexIncompleteReasons ?? []); + Assert.Contains(expectedReason, status.ReferenceGraphIncompleteReasons ?? []); + } + } + finally + { + SqliteConnection.ClearAllPools(); + DeleteDirectory(projectRoot); + } + } + [Fact] public void Run_FullScan_ExtractorFailurePersistsSuccessfulGraphAndTruthfulPartialState_Issue4609() { @@ -298,6 +478,9 @@ public void Run_FullScan_ExtractorFailurePersistsSuccessfulGraphAndTruthfulParti Assert.False(json.GetProperty("index_complete").GetBoolean()); Assert.True(json.GetProperty("graph_table_available").GetBoolean()); Assert.False(json.GetProperty("graph_data_current").GetBoolean()); + Assert.False(json.GetProperty("reference_graph_complete").GetBoolean()); + AssertCompletenessReason(json, "index_incomplete_reasons", "file_index_error"); + AssertCompletenessReason(json, "reference_graph_incomplete_reasons", "file_index_error"); Assert.False(json.GetProperty("sql_graph_contract_ready").GetBoolean()); var summary = json.GetProperty("summary"); @@ -329,6 +512,8 @@ public void Run_FullScan_ExtractorFailurePersistsSuccessfulGraphAndTruthfulParti Assert.True(statusJson.GetProperty("graph_table_available").GetBoolean()); Assert.False(statusJson.GetProperty("graph_data_current").GetBoolean()); Assert.False(statusJson.GetProperty("index_complete").GetBoolean()); + Assert.False(statusJson.GetProperty("reference_graph_complete").GetBoolean()); + AssertCompletenessSignalsEqual(json, statusJson); Assert.Equal(summary.GetProperty("references_total").GetInt64(), statusJson.GetProperty("references").GetInt64()); Assert.Contains("INCOMPLETE", statusJson.GetProperty("summary").GetString(), StringComparison.Ordinal); var lastFailure = statusJson.GetProperty("last_failed_or_partial_index_run"); @@ -356,6 +541,49 @@ public void Run_FullScan_ExtractorFailurePersistsSuccessfulGraphAndTruthfulParti } } + private static void AssertCompletenessSignalsEqual( + JsonElement indexJson, + JsonElement statusJson) + { + foreach (var propertyName in new[] + { + "graph_table_available", + "graph_data_current", + "index_complete", + "reference_graph_complete", + }) + { + Assert.Equal( + indexJson.GetProperty(propertyName).GetBoolean(), + statusJson.GetProperty(propertyName).GetBoolean()); + } + + Assert.Equal( + ReadCompletenessReasons(indexJson, "index_incomplete_reasons"), + ReadCompletenessReasons(statusJson, "index_incomplete_reasons")); + Assert.Equal( + ReadCompletenessReasons(indexJson, "reference_graph_incomplete_reasons"), + ReadCompletenessReasons(statusJson, "reference_graph_incomplete_reasons")); + } + + private static void AssertCompletenessReason( + JsonElement json, + string propertyName, + string? expectedReason) + { + var reasons = ReadCompletenessReasons(json, propertyName); + if (expectedReason == null) + Assert.Empty(reasons); + else + Assert.Contains(expectedReason, reasons); + } + + private static string[] ReadCompletenessReasons(JsonElement json, string propertyName) => + json.TryGetProperty(propertyName, out var reasons) + && reasons.ValueKind == JsonValueKind.Array + ? reasons.EnumerateArray().Select(reason => reason.GetString()!).ToArray() + : []; + [PublishedTrimmedCliFact] public void Run_FullScan_PublishedTrimmedBinary_IndexesJsonWorkerInputs_Issue4709() { @@ -2460,8 +2688,17 @@ public void Run_FullScan_PositiveCsharpNoOpLateOversizeContractRefreshesEveryCsh Assert.Equal(CommandExitCodes.Success, refreshExitCode); Assert.Equal("success", refreshJson.GetProperty("status").GetString()); - Assert.True(refreshJson.GetProperty("index_complete").GetBoolean()); - Assert.True(refreshJson.GetProperty("graph_data_current").GetBoolean()); + Assert.False(refreshJson.GetProperty("index_complete").GetBoolean()); + AssertCompletenessReason( + refreshJson, + "index_incomplete_reasons", + "file_too_large"); + Assert.False(refreshJson.GetProperty("reference_graph_complete").GetBoolean()); + AssertCompletenessReason( + refreshJson, + "reference_graph_incomplete_reasons", + "file_too_large"); + Assert.False(refreshJson.GetProperty("graph_data_current").GetBoolean()); Assert.Equal(["IParseable.cs", "Money.cs"], loadedPaths.Order(StringComparer.Ordinal)); Assert.Equal(0, CountMoneyParseImplicitImplementationReferences(projectRoot)); using var refreshDb = new DbContext( @@ -2775,6 +3012,8 @@ public void Run_FullScan_WithOversizedFile_PrintsSkipWarningWithoutRecoveryWarni Assert.Equal(CommandExitCodes.Success, exitCode); Assert.Contains("[WARN] File too large", stderr); + Assert.Contains("Index generation is incomplete: file_too_large.", stderr); + Assert.Contains("Reference graph is incomplete: file_too_large.", stderr); Assert.DoesNotContain("Some files failed to index", stderr); Assert.DoesNotContain("rerun `cdidx index", stderr); } diff --git a/tests/CodeIndex.Tests/IndexCommandRunnerUpdateTests.cs b/tests/CodeIndex.Tests/IndexCommandRunnerUpdateTests.cs index e5e700b335..cb6b291395 100644 --- a/tests/CodeIndex.Tests/IndexCommandRunnerUpdateTests.cs +++ b/tests/CodeIndex.Tests/IndexCommandRunnerUpdateTests.cs @@ -3379,7 +3379,8 @@ public void Run_UpdateMode_WithOversizedFile_PrintsSkipWarningWithoutRecoveryWar var (exitCode, _, stderr) = RunCliInSubprocess([projectRoot, "--files", "huge.py"], projectRoot); Assert.Equal(CommandExitCodes.Success, exitCode); - Assert.Equal(string.Empty, stderr); + Assert.Contains("Index generation is incomplete: file_too_large.", stderr); + Assert.Contains("Reference graph is incomplete: file_too_large.", stderr); Assert.DoesNotContain("Some files failed to update", stderr); Assert.DoesNotContain("rerun `cdidx index", stderr); } diff --git a/tests/CodeIndex.Tests/McpServerToolsCallTests.cs b/tests/CodeIndex.Tests/McpServerToolsCallTests.cs index 2252faf3d1..91531eed1c 100644 --- a/tests/CodeIndex.Tests/McpServerToolsCallTests.cs +++ b/tests/CodeIndex.Tests/McpServerToolsCallTests.cs @@ -8702,9 +8702,37 @@ public void ToolsCall_Index_MaxReferencesPerFilePersistsReferenceCountExceededIs var response = CallIndex(server, fixtureDir, args => args["maxReferencesPerFile"] = 2); Assert.False(response["result"]?["isError"]?.GetValue() ?? false, response.ToJsonString()); + var structured = response["result"]!["structuredContent"]!; + Assert.True(structured["graph_table_available"]!.GetValue()); + Assert.False(structured["graph_data_current"]!.GetValue()); + Assert.False(structured["index_complete"]!.GetValue()); + Assert.False(structured["reference_graph_complete"]!.GetValue()); + Assert.Contains( + "reference_count_exceeded", + structured["index_incomplete_reasons"]!.AsArray() + .Select(reason => reason!.GetValue())); + Assert.Contains( + "reference_count_exceeded", + structured["reference_graph_incomplete_reasons"]!.AsArray() + .Select(reason => reason!.GetValue())); using var db = new DbContext(DbOpenIntent.WriteIndex, dbPath); db.TryMigrateForRead(); var reader = new DbReader(db.Connection, db.IsReadOnly); + var status = reader.GetStatus(); + Assert.Equal(status.GraphTableAvailable, structured["graph_table_available"]!.GetValue()); + Assert.Equal(status.GraphDataCurrent, structured["graph_data_current"]!.GetValue()); + Assert.Equal(status.IndexComplete, structured["index_complete"]!.GetValue()); + Assert.Equal(status.ReferenceGraphComplete, structured["reference_graph_complete"]!.GetValue()); + Assert.Equal( + status.IndexIncompleteReasons ?? [], + structured["index_incomplete_reasons"]!.AsArray() + .Select(reason => reason!.GetValue()) + .ToArray()); + Assert.Equal( + status.ReferenceGraphIncompleteReasons ?? [], + structured["reference_graph_incomplete_reasons"]!.AsArray() + .Select(reason => reason!.GetValue()) + .ToArray()); var issue = Assert.Single(reader.GetIssues("reference_count_exceeded")); Assert.Equal("DenseReferences.cs", issue.Path); Assert.Equal(0, issue.Line); @@ -10019,13 +10047,26 @@ public void ToolsCall_Index_PositiveCsharpNoOpLateOversizeContractRefreshesEvery Assert.False(refreshResponse["result"]?["isError"]?.GetValue() ?? false, refreshResponse.ToJsonString()); var refreshStructured = refreshResponse["result"]!["structuredContent"]!; Assert.Equal(0, refreshStructured["summary"]!["errors"]!.GetValue()); + Assert.False(refreshStructured["index_complete"]!.GetValue()); + Assert.Contains( + "file_too_large", + refreshStructured["index_incomplete_reasons"]!.AsArray() + .Select(reason => reason!.GetValue())); + Assert.False(refreshStructured["reference_graph_complete"]!.GetValue()); + Assert.Contains( + "file_too_large", + refreshStructured["reference_graph_incomplete_reasons"]!.AsArray() + .Select(reason => reason!.GetValue())); Assert.Equal(["IParseable.cs", "Money.cs"], loadedPaths.Order(StringComparer.Ordinal)); Assert.Equal(0L, CountImplicitReferences()); using var refreshDb = new DbContext(DbOpenIntent.WriteIndex, dbPath); Assert.Equal(false, new DbWriter(refreshDb).GetCSharpStaticInterfaceSourceEvidence()); var refreshStatus = new DbReader(refreshDb.Connection).GetStatus(); - Assert.True(refreshStatus.IndexComplete); - Assert.True(refreshStatus.GraphDataCurrent); + Assert.False(refreshStatus.IndexComplete); + Assert.Contains("file_too_large", refreshStatus.IndexIncompleteReasons ?? []); + Assert.False(refreshStatus.ReferenceGraphComplete); + Assert.Contains("file_too_large", refreshStatus.ReferenceGraphIncompleteReasons ?? []); + Assert.False(refreshStatus.GraphDataCurrent); void WriteInterface(bool hasStaticContract) { diff --git a/tests/CodeIndex.Tests/QueryCommandRunnerFilesTests.cs b/tests/CodeIndex.Tests/QueryCommandRunnerFilesTests.cs index f83f6eee4b..f39200bc4f 100644 --- a/tests/CodeIndex.Tests/QueryCommandRunnerFilesTests.cs +++ b/tests/CodeIndex.Tests/QueryCommandRunnerFilesTests.cs @@ -2892,7 +2892,11 @@ public void RunStatus_CheckJsonScopedReadiness_ReportsOnlyRequestedSubsystem(str Assert.Equal(2, exitCode); Assert.Equal(string.Empty, stderr); - Assert.Equal([expectedFailure], failedChecks); + Assert.Equal( + scope == "graph" + ? ["graph_table_available", "reference_graph_complete"] + : [expectedFailure], + failedChecks); } finally { diff --git a/tests/CodeIndex.Tests/golden/status.json b/tests/CodeIndex.Tests/golden/status.json index b13e987dee..8fb4c3983c 100644 --- a/tests/CodeIndex.Tests/golden/status.json +++ b/tests/CodeIndex.Tests/golden/status.json @@ -177,7 +177,8 @@ }, "reference_graph_complete": false, "reference_graph_incomplete_reasons": [ - "reference_extraction_cap_state_unavailable" + "reference_extraction_cap_state_unavailable", + "graph_table_available=false" ], "reference_extraction_cap_hits": { "state_available": false, From 00e04c8cff7058f577203046eb8b2b631e7eee5d Mon Sep 17 00:00:00 2001 From: Widthdom Date: Mon, 27 Jul 2026 05:52:03 +0900 Subject: [PATCH 2/3] Address completeness review findings (#4826) --- DEVELOPER_GUIDE.md | 2 + TESTING_GUIDE.md | 4 +- .../IndexCommandRunner.FullScan.Readiness.cs | 2 +- .../Cli/IndexCommandRunner.Update.FileLoop.cs | 1 + .../Cli/IndexCommandRunner.Update.cs | 2 +- .../Cli/QueryCommandRunner.Status.cs | 31 ++++++++++-- .../DbReader.IndexGenerationReadiness.cs | 5 +- ...bWriter.ReferenceExtractionCompleteness.cs | 6 ++- .../Database/DegradationReasonCodes.cs | 13 +++-- .../Indexer/Scanning/IndexedFileStatReuse.cs | 3 +- .../Mcp/McpToolHandlers.Indexing.Execution.cs | 3 +- .../IndexCommandRunnerFullScanTests.cs | 20 ++++++++ .../IndexCommandRunnerUpdateTests.cs | 47 +++++++++++++++++++ .../QueryCommandRunnerFilesTests.cs | 16 +++++++ 14 files changed, 136 insertions(+), 19 deletions(-) diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index def83b9b94..7cf61f5f13 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -812,6 +812,7 @@ Current stable codes and triggers: | `hotspot_family_marker_fingerprint_incomplete` | hotspot-family marker fingerprint traversal hit a safety cap, so family trust was not stamped authoritative | reduce generated/ignored marker trees or raise the cap in code, then run `cdidx index --rebuild` | | `partial_family_key_population` | hotspot-family metadata is stamped but some indexed symbols still have NULL `family_key` values | `cdidx index --rebuild` | | `graph_table_available=false` | `symbol_references` is missing or not graph-ready | `cdidx index ` | +| `symbols_only_graph_omitted` | the last symbols-only generation intentionally omitted reference-graph rows | run `cdidx index ` without `--symbols-only` | | `reference_graph_complete=false` | the graph generation is unavailable/stale, a symbols-only run omitted it, or persisted file/extractor/cap evidence makes the index generation incomplete | address the reported stable reasons, then run `cdidx index ` | | `index_complete=false` | a symbols-only run or persisted file-size, symbol-count, reference-count, extractor-failure, or safety-cap evidence proves that indexing work was omitted | address `index_incomplete_reasons`, then run `cdidx index ` | | `issues_table_available=false` | `file_issues` is missing or not issue-ready | `cdidx index ` | @@ -3920,6 +3921,7 @@ alternative action を同じ場所へ追加してください。 | `hotspot_family_marker_fingerprint_incomplete` | hotspot-family marker fingerprint traversal が safety cap に到達し、family trust が authoritative に stamp されなかった | generated / ignored marker tree を減らすか code 側の cap を上げてから `cdidx index --rebuild` | | `partial_family_key_population` | hotspot-family metadata は stamp 済みだが、一部の indexed symbol で `family_key` が NULL | `cdidx index --rebuild` | | `graph_table_available=false` | `symbol_references` が無い、または graph-ready ではない | `cdidx index ` | +| `symbols_only_graph_omitted` | 直前の symbols-only generation が reference-graph row を意図的に省略した | `--symbols-only` を付けずに `cdidx index ` を実行 | | `reference_graph_complete=false` | graph generation が unavailable/stale、symbols-only run で省略、または永続化済み file/extractor/cap 証拠により index generation が incomplete | 報告された安定理由に対処してから `cdidx index ` | | `index_complete=false` | symbols-only run、または永続化済みの file-size / symbol-count / reference-count / extractor-failure / safety-cap 証拠により indexing work の省略が判明 | `index_incomplete_reasons` に対処してから `cdidx index ` | | `issues_table_available=false` | `file_issues` が無い、または issue-ready ではない | `cdidx index ` | diff --git a/TESTING_GUIDE.md b/TESTING_GUIDE.md index bd9da0c376..dc85c50b4a 100644 --- a/TESTING_GUIDE.md +++ b/TESTING_GUIDE.md @@ -29,7 +29,7 @@ Use the full suite by default. Use targeted filters only while iterating locally - Markdown unused-audit coverage indexes one real Markdown fixture containing common backtick and tilde fence-language markers. Keep default suppression, `documentation_surface` totals, reason tags, and `--all` recovery in that shared fixture. - Full-scan CLI and MCP no-op coverage treats one repository-wide reusable-stat snapshot read and one folded-readiness verification as performance contracts. Keep assertions for one snapshot read, one stat lookup per candidate, one folded verification, and no content load for unchanged files when changing incremental indexing. - Reference-graph refresh coverage treats graph-neutral indexing as a performance contract across incremental full scan, scoped `--files` update, and MCP indexing. Keep zero-refresh assertions for new and modified source files without symbols/references, plus a single batched refresh assertion when existing or new graph identity rows change. A healthy incremental generation must restrict identity/candidate/recursion work to transaction-committed dirty files, old and new `(language, folded name)` dependencies, and their old/new reciprocal edges; retain C#/Python language-transition and unchanged-target parity with a subsequent full refresh, rolled-back file batches, cancellation/retry, orphan-candidate cleanup, and the controlled 4,100-of-4,100 broad-scope fallback. Fresh/rebuild runs, missing identity contracts, and dirty sets of at least 4,096 references covering at least 50% of the graph must keep the full-refresh path. Query-plan coverage must keep all four scoped update phases and all ten candidate inserts on dirty-table-driven reference primary-key seeks, keep C# instantiate grouping on lookup names plus `idx_symbols_name_folded`, and prove that a sub-4,096 dirty set does not count the whole reference table without an explicit diagnostic hook. -- Index-generation completeness coverage uses a table-driven full, symbols-only, max-file-byte, max-symbol, and max-reference matrix, with extractor failure kept separate because it uses a mutable hook. Assert that index-command JSON, immediate status, and workspace health expose identical index/graph booleans and reason arrays where available; MCP cap coverage must match the persisted status snapshot as well. Remove the additive completeness metadata in healthy and capped fixtures to preserve legacy fallback coverage. Human output must identify incomplete generations instead of printing a complete summary. +- Index-generation completeness coverage uses a table-driven full, symbols-only, max-file-byte, max-symbol, and max-reference matrix, with extractor failure kept separate because it uses a mutable hook. Assert that index-command JSON, immediate status, and workspace health expose identical index/graph booleans and reason arrays where available; MCP cap coverage must match the persisted status snapshot as well. Remove the additive completeness metadata in healthy and capped fixtures to preserve legacy fallback coverage, and clear issue readiness before a scoped capped update to prove current omission evidence survives degraded prior metadata. Structured remediation must distinguish symbols-only / missing-graph causes from reference safety caps. Human output must identify incomplete generations instead of printing a complete summary. - Reference-identity refresh coverage treats a stable graph rebuild as a physical-write performance contract. Keep NULL-safe changed-row predicates for source identity, the four-column target-resolution tuple, self-reference, and mutual-recursion updates; trigger audits must remain at zero on a stable rerun, repair each corrupted phase once, and prove a later-phase failure rolls back earlier identity writes. SQLite `changes()` must continue to report the final mutual-recursion phase. - C# metadata-target resolver coverage treats propagation work and stable reruns as performance contracts. Keep the reverse-ordered 8,000-class chain at exactly `n - 1` dependency edges and `n` queue visits instead of using a wall-clock threshold; retain cross-file partial fan-in, an unseeded cycle, pre-cancellation, rollback of an earlier row after an injected later update failure, zero trigger-audited writes on a stable rerun, and exactly one write when repairing a corrupted derived row. - Reference-insert transaction coverage keeps the public APIs' #1518 transaction/SAVEPOINT per 71-row batch, while the explicit atomic-file APIs must reject calls without a live caller-owned transaction and open zero reference-batch scopes. Atomic-file reference-line materialization may group only complete 71-row reference batches, stopping before the union of `(file_id, line, context)` keys would exceed 333 rows or after 32 batches; reference INSERT executions and their progress/cancellation checkpoints remain on the original 71-row boundaries. Preserve exact public/atomic statement counts, the 333-row and 32-batch stops, the unique `reference_lines` autoindex lookup plan, batch-two/three failure rollback for both normal and new-file reference-line paths, same/different contexts across a batch boundary, cancellation and empty-input ordering, and guarded multi-language integration coverage for full scan, scoped update, MCP indexing, and TypeScript augmentation rebuild. The controlled 321,352-reference/856-file performance contract retains the repository snapshot's five/six-batch large-file distribution and compares 5,009 public batch scopes with zero atomic-file batch scopes without using a wall-clock threshold. @@ -912,7 +912,7 @@ dotnet test --filter "FullyQualifiedName~GitHelperTests" - Markdown の unused audit coverage は、一般的な backtick / tilde fence の language marker を含む実 Markdown fixture を1回 index します。同じ fixture で既定抑制、`documentation_surface` totals、reason tag、`--all` による復元を維持してください。 - full-scan CLI と MCP の no-op coverage は、リポジトリ全体の reusable-stat snapshot read と folded-readiness verification がそれぞれ 1 回であることを performance contract とします。incremental indexing を変更するときは、snapshot read が 1 回、候補ごとの stat lookup が 1 回、folded verification が 1 回、unchanged file の content load が 0 回という assertion を維持してください。 - reference-graph refresh coverage は、incremental full scan、scoped `--files` update、MCP indexing を横断する graph-neutral indexing を performance contract とします。symbol/reference を持たない新規・変更 source file では refresh 0 回を維持し、既存または新規の graph identity 行が変化する場合は batch 全体で refresh 1 回を assertion してください。健全な incremental generation では identity / candidate / recursion 処理を transaction commit 済みの dirty file、旧・新の `(language, folded name)` 依存、旧・新の逆辺に限定します。C# / Python の言語遷移、未変更targetを参照する新規callerと後続full refreshのparity、rollback file batch、cancel後retry、孤立candidate cleanup、4,100件中4,100件をdirtyにする制御broad-scope fallbackを維持してください。fresh/rebuild、identity契約欠落、または4,096件以上かつgraphの50%以上を占めるdirty集合ではfull-refresh経路を維持します。query-plan coverageでは、scoped updateの4 phaseとcandidate INSERT 10本をdirty table起点のreference主キーseekに保ち、C# instantiate groupingをlookup nameと`idx_symbols_name_folded`起点にし、明示的なdiagnostic hookがない4,096件未満のdirty集合ではreference table全件COUNTを行わないことを検証してください。 -- index generation の completeness coverage は full、symbols-only、max-file-byte、max-symbol、max-reference を table-driven matrix で検証し、mutable hook を使う extractor failure は別 case に保ちます。index command JSON、直後の status、workspace health で、利用可能な index/graph の boolean と reason array が完全に一致すること、MCP の cap case も persisted status snapshot と一致することを assertion してください。healthy / capped fixture から additive completeness metadata を削除し、legacy fallback coverage も維持します。human output は complete summary ではなく incomplete generation を明示する必要があります。 +- index generation の completeness coverage は full、symbols-only、max-file-byte、max-symbol、max-reference を table-driven matrix で検証し、mutable hook を使う extractor failure は別 case に保ちます。index command JSON、直後の status、workspace health で、利用可能な index/graph の boolean と reason array が完全に一致すること、MCP の cap case も persisted status snapshot と一致することを assertion してください。healthy / capped fixture から additive completeness metadata を削除して legacy fallback coverage を維持し、scoped capped update の前に issue readiness を clear して、prior metadata が degraded でも今回の omission evidence が失われないことを検証します。structured remediation は symbols-only / missing-graph 原因と reference safety cap を区別する必要があります。human output は complete summary ではなく incomplete generation を明示する必要があります。 - reference identity refresh coverage は、安定graphの再構築を物理writeのperformance contractとします。source identity、target resolutionの4列tuple、self reference、mutual recursionの更新にはNULL-safeなchanged-row predicateを維持し、安定rerunのtrigger auditは0、各corrupt phaseのrepairは1回、後段phaseの失敗で先行identity writeもrollbackされることを検証してください。SQLite `changes()` は引き続き最後のmutual-recursion phaseを表します。 - C# metadata-target resolver coverage は propagation work と安定 rerun を performance contract とします。逆順に保存した 8,000 class の chain では wall-clock threshold を使わず、dependency edge が厳密に `n - 1`、queue visit が `n` であることを維持してください。cross-file partial fan-in、seed を持たない cycle、事前 cancel、後段 update の注入失敗時に先行 row も rollback されること、安定 rerun の trigger audit が write 0 回、破損した derived row の修復が厳密に 1 write であることも残します。 - reference insert の transaction coverage は、public API の #1518 契約として71 row batchごとの transaction/SAVEPOINTを維持し、明示atomic-file APIは呼出元所有のlive transactionなしでは拒否され、reference batch scopeを0回に保つことを検証します。atomic-fileのreference-line materializationは完全な71 row reference batchだけをまとめ、`(file_id, line, context)` keyの和集合が333行を超える直前、または32 batchで停止します。reference INSERTの実行回数とprogress/cancellation checkpointは元の71 row境界に保ってください。public/atomicの正確なstatement数、333行/32 batch停止、`reference_lines` unique autoindexのlookup plan、通常/new-file両方のreference-line pathでbatch 2/3失敗時の全rollback、batch境界をまたぐ同一/異なるcontext、cancelとempty入力の順序、full scan・scoped update・MCP indexing・TypeScript augmentation rebuildのmulti-language guard付きintegrationを維持してください。321,352 refs / 856 filesの制御performance契約は自己snapshotの5/6 batch巨大file分布を保ち、wall-clock閾値を使わずpublic 5,009 scopeとatomic-file 0 scopeを比較します。 diff --git a/src/CodeIndex/Cli/IndexCommandRunner.FullScan.Readiness.cs b/src/CodeIndex/Cli/IndexCommandRunner.FullScan.Readiness.cs index 72b9b6a15e..375ec93d92 100644 --- a/src/CodeIndex/Cli/IndexCommandRunner.FullScan.Readiness.cs +++ b/src/CodeIndex/Cli/IndexCommandRunner.FullScan.Readiness.cs @@ -247,7 +247,7 @@ context.SkippedSymbolExtractorLanguages is null memoryTimelineForStamp, context.IndexRunDiagnostics, writer.GetReferenceExtractionCapHits(issuesTableAvailableAfter), - writer.GetPersistedIndexOmissionReasons(issuesTableAvailableAfter)); + writer.GetPersistedIndexOmissionReasons()); } return new FullScanReadinessResult( diff --git a/src/CodeIndex/Cli/IndexCommandRunner.Update.FileLoop.cs b/src/CodeIndex/Cli/IndexCommandRunner.Update.FileLoop.cs index 2b799c8fa5..8f6eca4d3f 100644 --- a/src/CodeIndex/Cli/IndexCommandRunner.Update.FileLoop.cs +++ b/src/CodeIndex/Cli/IndexCommandRunner.Update.FileLoop.cs @@ -472,6 +472,7 @@ void ThrowIfUpdateCancelled() absPath, dbPath, statReusableLanguage, + options.MaxFileSizeBytes ?? FileIndexer.DefaultMaxFileSizeBytes, options.MaxSymbolsPerFile, options.MaxReferencesPerFile, generatedExtractionSuppressed, diff --git a/src/CodeIndex/Cli/IndexCommandRunner.Update.cs b/src/CodeIndex/Cli/IndexCommandRunner.Update.cs index ad1fbeced1..8d39c2148d 100644 --- a/src/CodeIndex/Cli/IndexCommandRunner.Update.cs +++ b/src/CodeIndex/Cli/IndexCommandRunner.Update.cs @@ -853,7 +853,7 @@ bool TryValidateCSharpWorkspaceInputSnapshot( memoryTimelineForStamp, indexRunDiagnostics, writer.GetReferenceExtractionCapHits(issuesTableAvailableAfter), - writer.GetPersistedIndexOmissionReasons(issuesTableAvailableAfter)); + writer.GetPersistedIndexOmissionReasons()); } return WriteUpdateFinalOutput(new UpdateFinalOutputContext { diff --git a/src/CodeIndex/Cli/QueryCommandRunner.Status.cs b/src/CodeIndex/Cli/QueryCommandRunner.Status.cs index 491d9fcbbe..8a52cad7e7 100644 --- a/src/CodeIndex/Cli/QueryCommandRunner.Status.cs +++ b/src/CodeIndex/Cli/QueryCommandRunner.Status.cs @@ -554,11 +554,26 @@ private static string GetReferenceGraphDegradationRootCause(StatusResult status) return DegradationReasonCodes.DynamicReferenceGraphContractStale; } - return status.ReferenceGraphIncompleteReasons?.Contains( - DbReader.ReferenceExtractionCapStateUnavailableReason, - StringComparer.Ordinal) == true - ? DegradationReasonCodes.ReferenceExtractionCapStateUnavailable - : DegradationReasonCodes.ReferenceGraphIncomplete; + var reasons = status.ReferenceGraphIncompleteReasons ?? []; + if (reasons.Contains( + DbReader.ReferenceExtractionCapStateUnavailableReason, + StringComparer.Ordinal)) + { + return DegradationReasonCodes.ReferenceExtractionCapStateUnavailable; + } + if (reasons.Contains( + DbReader.SymbolsOnlyReferenceGraphIncompleteReason, + StringComparer.Ordinal)) + { + return DegradationReasonCodes.SymbolsOnlyGraphOmitted; + } + if (reasons.Contains(DegradationReasonCodes.GraphTableMissing, StringComparer.Ordinal)) + return DegradationReasonCodes.GraphTableMissing; + if (reasons.Any(ReferenceExtractor.IsSafetyCapDiagnosticKind)) + return DegradationReasonCodes.ReferenceGraphIncomplete; + return !status.IndexComplete + ? DegradationReasonCodes.IndexIncomplete + : DegradationReasonCodes.ReferenceGraphIncomplete; } private static StatusReadinessDegradation BuildStatusReadinessDegradation(string field, string rootCause, QueryCommandOptions options, StatusResult status) @@ -714,6 +729,12 @@ private static string GetReferenceGraphRepairSafetyNote(StatusResult status) "Refresh indexing to rewrite stale dynamic-language graph rows and extractor-version stamps.", DegradationReasonCodes.ReferenceExtractionCapStateUnavailable => "Refresh indexing to populate current per-file issue state before trusting reference-graph completeness.", + DegradationReasonCodes.SymbolsOnlyGraphOmitted => + "Rerun indexing without --symbols-only to generate reference-graph rows.", + DegradationReasonCodes.GraphTableMissing => + "Run normal indexing to create and stamp the reference-graph generation.", + DegradationReasonCodes.IndexIncomplete => + "Inspect index_incomplete_reasons and address the reported omitted input or extraction work.", _ => "Reduce or exclude the cap-hitting generated/pathological source before rerunning indexing.", }; diff --git a/src/CodeIndex/Database/DbReader.IndexGenerationReadiness.cs b/src/CodeIndex/Database/DbReader.IndexGenerationReadiness.cs index 51a82c59b2..0b0fd39673 100644 --- a/src/CodeIndex/Database/DbReader.IndexGenerationReadiness.cs +++ b/src/CodeIndex/Database/DbReader.IndexGenerationReadiness.cs @@ -17,7 +17,8 @@ internal sealed record PersistedIndexGenerationReadiness( public partial class DbReader { internal const string SymbolsOnlyIndexIncompleteReason = "symbols_only_references_omitted"; - internal const string SymbolsOnlyReferenceGraphIncompleteReason = "symbols_only_graph_omitted"; + internal const string SymbolsOnlyReferenceGraphIncompleteReason = + DegradationReasonCodes.SymbolsOnlyGraphOmitted; internal const string BatchInProgressIncompleteReason = "batch_in_progress"; private static readonly string[] IndexOmissionIssueKinds = @@ -39,7 +40,7 @@ internal PersistedIndexGenerationReadiness GetPersistedIndexGenerationReadiness( ParseMetaStringList(TryGetMetaStringInternal(DbContext.IndexIncompleteReasonsMetaKey)), ReadPersistedIndexOmissionReasons( _conn, - _hasIssuesTable, + _hasIssuesPhysicalTable, ParseMetaBool(TryGetMetaStringInternal(DbContext.SymbolsOnlyGraphOmittedMetaKey)) == true, transaction)); var migrationInProgress = string.Equals( diff --git a/src/CodeIndex/Database/DbWriter.ReferenceExtractionCompleteness.cs b/src/CodeIndex/Database/DbWriter.ReferenceExtractionCompleteness.cs index 10d8e03973..ba830b5c14 100644 --- a/src/CodeIndex/Database/DbWriter.ReferenceExtractionCompleteness.cs +++ b/src/CodeIndex/Database/DbWriter.ReferenceExtractionCompleteness.cs @@ -20,10 +20,12 @@ internal ReferenceExtractionCapHitSummary GetReferenceExtractionCapHits(bool iss hasIssuesTable: issuesStateAvailable, _activeTransaction); - internal IReadOnlyList GetPersistedIndexOmissionReasons(bool issuesStateAvailable) + internal IReadOnlyList GetPersistedIndexOmissionReasons() => DbReader.ReadPersistedIndexOmissionReasons( _conn, - hasIssuesTable: issuesStateAvailable, + // A writer always operates on the initialized current schema. Omission evidence + // remains relevant even when the workspace-wide IssuesReadyFlag is unset. + hasIssuesTable: true, symbolsOnlyGraphOmitted: string.Equals( GetMetaString(DbContext.SymbolsOnlyGraphOmittedMetaKey), "true", diff --git a/src/CodeIndex/Database/DegradationReasonCodes.cs b/src/CodeIndex/Database/DegradationReasonCodes.cs index 79c9857a5e..49c743043e 100644 --- a/src/CodeIndex/Database/DegradationReasonCodes.cs +++ b/src/CodeIndex/Database/DegradationReasonCodes.cs @@ -25,6 +25,7 @@ public static class DegradationReasonCodes public const string GraphDataNotCurrent = "graph_data_current=false"; public const string IndexIncomplete = "index_complete=false"; public const string ReferenceGraphIncomplete = "reference_graph_complete=false"; + public const string SymbolsOnlyGraphOmitted = "symbols_only_graph_omitted"; public const string ReferenceExtractionCapStateUnavailable = "reference_extraction_cap_state_unavailable"; public const string DynamicReferenceGraphContractStale = "dynamic_reference_graph_contract_stale"; public const string IssuesTableMissing = "issues_table_available=false"; @@ -55,6 +56,7 @@ public static class DegradationReasonCodes GraphDataNotCurrent, IndexIncomplete, ReferenceGraphIncomplete, + SymbolsOnlyGraphOmitted, ReferenceExtractionCapStateUnavailable, DynamicReferenceGraphContractStale, IssuesTableMissing, @@ -182,14 +184,19 @@ private static DegradationReasonMetadata CreateMetadata(string code) "Run `cdidx status --json` to inspect the incomplete files and stable reasons before changing index scope."), IndexIncomplete => new( code, - "One or more files failed while other file transactions were committed successfully.", - "Run `cdidx status --json`, fix `last_failed_or_partial_index_run.file_errors`, then rerun the same index command; a rebuild is not required.", - "Use `cdidx index --allow-partial` only when automation deliberately accepts an incomplete generation."), + "Required input or extraction work was omitted or failed, so persisted rows cover only a partial index generation.", + "Run `cdidx status --json`, address `index_incomplete_reasons`, then rerun the same index command; a rebuild is normally not required.", + "Use `cdidx status --json` and inspect `last_failed_or_partial_index_run.file_errors` for extractor failures or the stable omission reasons for capped and symbols-only runs."), ReferenceGraphIncomplete => new( code, "Reference extraction hit one or more hard safety caps, so missing callers, callees, dependencies, or impact edges are not authoritative absences.", "Inspect `reference_extraction_cap_hits.files`, reduce or exclude the generated/pathological source, then rerun `cdidx index `.", "Run `cdidx status --json` and use `reference_extraction_limits` and `reference_graph_incomplete_reasons` to identify the active cap before changing repository scope."), + SymbolsOnlyGraphOmitted => new( + code, + "The last symbols-only generation intentionally omitted reference-graph rows.", + "Run `cdidx index ` without `--symbols-only` to generate the reference graph.", + "Run `cdidx index --rebuild` without `--symbols-only` when a full rebuild is required."), ReferenceExtractionCapStateUnavailable => new( code, "Reference-extraction cap state is unavailable for this legacy or stale issue generation, so graph completeness cannot be established.", diff --git a/src/CodeIndex/Indexer/Scanning/IndexedFileStatReuse.cs b/src/CodeIndex/Indexer/Scanning/IndexedFileStatReuse.cs index 94f2a2f706..8fbc52a4fb 100644 --- a/src/CodeIndex/Indexer/Scanning/IndexedFileStatReuse.cs +++ b/src/CodeIndex/Indexer/Scanning/IndexedFileStatReuse.cs @@ -54,6 +54,7 @@ internal static Action? LookupForTesting string absolutePath, string relativePath, string? language, + long maxFileSizeBytes, int maxSymbolsPerFile, int maxReferencesPerFile, bool? generatedExtractionSuppressed, @@ -66,7 +67,7 @@ internal static Action? LookupForTesting try { var info = new FileInfo(absolutePath); - if (!info.Exists) + if (!info.Exists || info.Length > maxFileSizeBytes) return null; var fileId = writer.GetReusableUnchangedFileIdByStat( diff --git a/src/CodeIndex/Mcp/McpToolHandlers.Indexing.Execution.cs b/src/CodeIndex/Mcp/McpToolHandlers.Indexing.Execution.cs index d2ae823362..d88e9036c5 100644 --- a/src/CodeIndex/Mcp/McpToolHandlers.Indexing.Execution.cs +++ b/src/CodeIndex/Mcp/McpToolHandlers.Indexing.Execution.cs @@ -2024,8 +2024,7 @@ await EmitProgressNotificationAsync( (DbContext.LastIndexRunReferenceExtractionCapHitsMetaKey, JsonSerializer.Serialize( referenceExtractionCapHits, StatusMetadataJsonContext.Default.ReferenceExtractionCapHitSummary))); - writer.MarkIndexCompleteness( - writer.GetPersistedIndexOmissionReasons(issuesStateAvailable: true)); + writer.MarkIndexCompleteness(writer.GetPersistedIndexOmissionReasons()); writer.ClearLastFailedIndexRunMetadata(); // Persist the current HEAD only after the run is fully successful (errors == 0). // Mirrors the CLI full-scan contract (Issue #1508) so MCP-driven re-indexes also diff --git a/tests/CodeIndex.Tests/IndexCommandRunnerFullScanTests.cs b/tests/CodeIndex.Tests/IndexCommandRunnerFullScanTests.cs index b386cd3e38..ae5be07184 100644 --- a/tests/CodeIndex.Tests/IndexCommandRunnerFullScanTests.cs +++ b/tests/CodeIndex.Tests/IndexCommandRunnerFullScanTests.cs @@ -333,6 +333,26 @@ public void Run_FullScan_CompletenessMatrixMatchesImmediateStatus_Issue4826( Assert.Equal(CommandExitCodes.Success, statusExitCode); AssertCompletenessSignalsEqual(indexJson, statusJson); + if (scenario == "symbols-only") + { + var referenceDegradation = statusJson + .GetProperty("readiness_degradations") + .EnumerateArray() + .Single(degradation => + degradation.GetProperty("field").GetString() + == "reference_graph_complete"); + Assert.Equal( + DegradationReasonCodes.SymbolsOnlyGraphOmitted, + referenceDegradation.GetProperty("root_cause").GetString()); + Assert.Contains( + "symbols-only", + referenceDegradation.GetProperty("degraded_reason").GetString(), + StringComparison.Ordinal); + Assert.DoesNotContain( + "safety cap", + referenceDegradation.GetProperty("degraded_reason").GetString(), + StringComparison.OrdinalIgnoreCase); + } using (var db = new DbContext(DbOpenIntent.QueryOnly, dbPath)) { diff --git a/tests/CodeIndex.Tests/IndexCommandRunnerUpdateTests.cs b/tests/CodeIndex.Tests/IndexCommandRunnerUpdateTests.cs index cb6b291395..b89280428f 100644 --- a/tests/CodeIndex.Tests/IndexCommandRunnerUpdateTests.cs +++ b/tests/CodeIndex.Tests/IndexCommandRunnerUpdateTests.cs @@ -3390,6 +3390,53 @@ public void Run_UpdateMode_WithOversizedFile_PrintsSkipWarningWithoutRecoveryWar } } + [Fact] + public void Run_UpdateMode_CapPersistsIncompleteWhenIssueReadinessWasUnset_Issue4826() + { + var projectRoot = CreateTempProject(); + try + { + File.WriteAllText( + Path.Combine(projectRoot, "huge.py"), + "print('start')\n" + new string('a', 256)); + var initialExitCode = IndexCommandRunner.Run([projectRoot, "--json"], _jsonOptions); + Assert.Equal(CommandExitCodes.Success, initialExitCode); + + var dbPath = Path.Combine(projectRoot, ".cdidx", "codeindex.db"); + using (var connection = OpenNonPoolingConnection(dbPath)) + { + connection.Open(); + using var command = connection.CreateCommand(); + command.CommandText = + $"PRAGMA user_version = {DbContext.CurrentSchemaVersion & ~DbContext.IssuesReadyFlag}"; + command.ExecuteNonQuery(); + } + + var (updateExitCode, updateJson) = RunAndCaptureJson( + [projectRoot, "--files", "huge.py", "--max-file-bytes", "128", "--json", "--quiet"]); + + Assert.Equal(CommandExitCodes.Success, updateExitCode); + Assert.False(updateJson.GetProperty("issues_table_available").GetBoolean()); + Assert.False(updateJson.GetProperty("index_complete").GetBoolean()); + AssertCompletenessReason(updateJson, "index_incomplete_reasons", "file_too_large"); + Assert.False(updateJson.GetProperty("reference_graph_complete").GetBoolean()); + AssertCompletenessReason( + updateJson, + "reference_graph_incomplete_reasons", + "file_too_large"); + + var (statusExitCode, statusJson) = + RunStatusAndCaptureJson(["--db", dbPath, "--json"]); + Assert.Equal(CommandExitCodes.Success, statusExitCode); + AssertCompletenessSignalsEqual(updateJson, statusJson); + } + finally + { + SqliteConnection.ClearAllPools(); + DeleteDirectory(projectRoot); + } + } + [Fact] public void Run_UpdateMode_VerboseRedirectedOutput_DoesNotRepeatUpdatingBanner() { diff --git a/tests/CodeIndex.Tests/QueryCommandRunnerFilesTests.cs b/tests/CodeIndex.Tests/QueryCommandRunnerFilesTests.cs index f39200bc4f..2bf36717dd 100644 --- a/tests/CodeIndex.Tests/QueryCommandRunnerFilesTests.cs +++ b/tests/CodeIndex.Tests/QueryCommandRunnerFilesTests.cs @@ -2897,6 +2897,22 @@ public void RunStatus_CheckJsonScopedReadiness_ReportsOnlyRequestedSubsystem(str ? ["graph_table_available", "reference_graph_complete"] : [expectedFailure], failedChecks); + if (scope == "graph") + { + var referenceDegradation = document.RootElement + .GetProperty("readiness_degradations") + .EnumerateArray() + .Single(degradation => + degradation.GetProperty("field").GetString() + == "reference_graph_complete"); + Assert.Equal( + DegradationReasonCodes.GraphTableMissing, + referenceDegradation.GetProperty("root_cause").GetString()); + Assert.DoesNotContain( + "safety cap", + referenceDegradation.GetProperty("degraded_reason").GetString(), + StringComparison.OrdinalIgnoreCase); + } } finally { From f99cc828549c0e9860a4fe6d30fad5deda8e6a27 Mon Sep 17 00:00:00 2001 From: Widthdom Date: Mon, 27 Jul 2026 06:16:54 +0900 Subject: [PATCH 3/3] Handle final completeness review findings (#4826) --- TESTING_GUIDE.md | 4 +- ...xCommandRunner.FullScan.CSharpPreflight.cs | 4 +- ...ndexCommandRunner.FullScan.Finalization.cs | 12 ++- .../Cli/IndexCommandRunner.FullScan.Output.cs | 4 +- .../Cli/IndexCommandRunner.Update.Output.cs | 4 +- src/CodeIndex/Database/DbWriter.FileReuse.cs | 25 ++++- .../Indexer/Scanning/IndexedFileStatReuse.cs | 3 +- .../Mcp/McpToolHandlers.Indexing.Execution.cs | 4 +- .../IndexCommandRunnerFullScanTests.cs | 99 +++++++++++++++++++ .../McpServerToolsCallTests.cs | 54 ++++++++++ 10 files changed, 202 insertions(+), 11 deletions(-) diff --git a/TESTING_GUIDE.md b/TESTING_GUIDE.md index dc85c50b4a..cac1522953 100644 --- a/TESTING_GUIDE.md +++ b/TESTING_GUIDE.md @@ -29,7 +29,7 @@ Use the full suite by default. Use targeted filters only while iterating locally - Markdown unused-audit coverage indexes one real Markdown fixture containing common backtick and tilde fence-language markers. Keep default suppression, `documentation_surface` totals, reason tags, and `--all` recovery in that shared fixture. - Full-scan CLI and MCP no-op coverage treats one repository-wide reusable-stat snapshot read and one folded-readiness verification as performance contracts. Keep assertions for one snapshot read, one stat lookup per candidate, one folded verification, and no content load for unchanged files when changing incremental indexing. - Reference-graph refresh coverage treats graph-neutral indexing as a performance contract across incremental full scan, scoped `--files` update, and MCP indexing. Keep zero-refresh assertions for new and modified source files without symbols/references, plus a single batched refresh assertion when existing or new graph identity rows change. A healthy incremental generation must restrict identity/candidate/recursion work to transaction-committed dirty files, old and new `(language, folded name)` dependencies, and their old/new reciprocal edges; retain C#/Python language-transition and unchanged-target parity with a subsequent full refresh, rolled-back file batches, cancellation/retry, orphan-candidate cleanup, and the controlled 4,100-of-4,100 broad-scope fallback. Fresh/rebuild runs, missing identity contracts, and dirty sets of at least 4,096 references covering at least 50% of the graph must keep the full-refresh path. Query-plan coverage must keep all four scoped update phases and all ten candidate inserts on dirty-table-driven reference primary-key seeks, keep C# instantiate grouping on lookup names plus `idx_symbols_name_folded`, and prove that a sub-4,096 dirty set does not count the whole reference table without an explicit diagnostic hook. -- Index-generation completeness coverage uses a table-driven full, symbols-only, max-file-byte, max-symbol, and max-reference matrix, with extractor failure kept separate because it uses a mutable hook. Assert that index-command JSON, immediate status, and workspace health expose identical index/graph booleans and reason arrays where available; MCP cap coverage must match the persisted status snapshot as well. Remove the additive completeness metadata in healthy and capped fixtures to preserve legacy fallback coverage, and clear issue readiness before a scoped capped update to prove current omission evidence survives degraded prior metadata. Structured remediation must distinguish symbols-only / missing-graph causes from reference safety caps. Human output must identify incomplete generations instead of printing a complete summary. +- Index-generation completeness coverage uses a table-driven full, symbols-only, max-file-byte, max-symbol, and max-reference matrix, with extractor failure kept separate because it uses a mutable hook. Assert that index-command JSON, immediate status, and workspace health expose identical index/graph booleans and reason arrays where available; MCP cap coverage must match the persisted status snapshot as well. Remove the additive completeness metadata in healthy and capped fixtures to preserve legacy fallback coverage, and clear issue readiness before a scoped capped update to prove current omission evidence survives degraded prior metadata. Lowering or raising the file-size policy must reprocess unchanged files in CLI and MCP indexing so a prior `file_too_large` issue cannot be reused. Structured remediation must distinguish symbols-only / missing-graph causes from reference safety caps and must not label an incomplete index as fold-only. Human output must identify incomplete generations instead of printing a complete summary. - Reference-identity refresh coverage treats a stable graph rebuild as a physical-write performance contract. Keep NULL-safe changed-row predicates for source identity, the four-column target-resolution tuple, self-reference, and mutual-recursion updates; trigger audits must remain at zero on a stable rerun, repair each corrupted phase once, and prove a later-phase failure rolls back earlier identity writes. SQLite `changes()` must continue to report the final mutual-recursion phase. - C# metadata-target resolver coverage treats propagation work and stable reruns as performance contracts. Keep the reverse-ordered 8,000-class chain at exactly `n - 1` dependency edges and `n` queue visits instead of using a wall-clock threshold; retain cross-file partial fan-in, an unseeded cycle, pre-cancellation, rollback of an earlier row after an injected later update failure, zero trigger-audited writes on a stable rerun, and exactly one write when repairing a corrupted derived row. - Reference-insert transaction coverage keeps the public APIs' #1518 transaction/SAVEPOINT per 71-row batch, while the explicit atomic-file APIs must reject calls without a live caller-owned transaction and open zero reference-batch scopes. Atomic-file reference-line materialization may group only complete 71-row reference batches, stopping before the union of `(file_id, line, context)` keys would exceed 333 rows or after 32 batches; reference INSERT executions and their progress/cancellation checkpoints remain on the original 71-row boundaries. Preserve exact public/atomic statement counts, the 333-row and 32-batch stops, the unique `reference_lines` autoindex lookup plan, batch-two/three failure rollback for both normal and new-file reference-line paths, same/different contexts across a batch boundary, cancellation and empty-input ordering, and guarded multi-language integration coverage for full scan, scoped update, MCP indexing, and TypeScript augmentation rebuild. The controlled 321,352-reference/856-file performance contract retains the repository snapshot's five/six-batch large-file distribution and compares 5,009 public batch scopes with zero atomic-file batch scopes without using a wall-clock threshold. @@ -912,7 +912,7 @@ dotnet test --filter "FullyQualifiedName~GitHelperTests" - Markdown の unused audit coverage は、一般的な backtick / tilde fence の language marker を含む実 Markdown fixture を1回 index します。同じ fixture で既定抑制、`documentation_surface` totals、reason tag、`--all` による復元を維持してください。 - full-scan CLI と MCP の no-op coverage は、リポジトリ全体の reusable-stat snapshot read と folded-readiness verification がそれぞれ 1 回であることを performance contract とします。incremental indexing を変更するときは、snapshot read が 1 回、候補ごとの stat lookup が 1 回、folded verification が 1 回、unchanged file の content load が 0 回という assertion を維持してください。 - reference-graph refresh coverage は、incremental full scan、scoped `--files` update、MCP indexing を横断する graph-neutral indexing を performance contract とします。symbol/reference を持たない新規・変更 source file では refresh 0 回を維持し、既存または新規の graph identity 行が変化する場合は batch 全体で refresh 1 回を assertion してください。健全な incremental generation では identity / candidate / recursion 処理を transaction commit 済みの dirty file、旧・新の `(language, folded name)` 依存、旧・新の逆辺に限定します。C# / Python の言語遷移、未変更targetを参照する新規callerと後続full refreshのparity、rollback file batch、cancel後retry、孤立candidate cleanup、4,100件中4,100件をdirtyにする制御broad-scope fallbackを維持してください。fresh/rebuild、identity契約欠落、または4,096件以上かつgraphの50%以上を占めるdirty集合ではfull-refresh経路を維持します。query-plan coverageでは、scoped updateの4 phaseとcandidate INSERT 10本をdirty table起点のreference主キーseekに保ち、C# instantiate groupingをlookup nameと`idx_symbols_name_folded`起点にし、明示的なdiagnostic hookがない4,096件未満のdirty集合ではreference table全件COUNTを行わないことを検証してください。 -- index generation の completeness coverage は full、symbols-only、max-file-byte、max-symbol、max-reference を table-driven matrix で検証し、mutable hook を使う extractor failure は別 case に保ちます。index command JSON、直後の status、workspace health で、利用可能な index/graph の boolean と reason array が完全に一致すること、MCP の cap case も persisted status snapshot と一致することを assertion してください。healthy / capped fixture から additive completeness metadata を削除して legacy fallback coverage を維持し、scoped capped update の前に issue readiness を clear して、prior metadata が degraded でも今回の omission evidence が失われないことを検証します。structured remediation は symbols-only / missing-graph 原因と reference safety cap を区別する必要があります。human output は complete summary ではなく incomplete generation を明示する必要があります。 +- index generation の completeness coverage は full、symbols-only、max-file-byte、max-symbol、max-reference を table-driven matrix で検証し、mutable hook を使う extractor failure は別 case に保ちます。index command JSON、直後の status、workspace health で、利用可能な index/graph の boolean と reason array が完全に一致すること、MCP の cap case も persisted status snapshot と一致することを assertion してください。healthy / capped fixture から additive completeness metadata を削除して legacy fallback coverage を維持し、scoped capped update の前に issue readiness を clear して、prior metadata が degraded でも今回の omission evidence が失われないことを検証します。file-size policy を下げた場合も上げた場合も、CLI / MCP indexing は unchanged file を再処理し、以前の `file_too_large` issue を再利用してはいけません。structured remediation は symbols-only / missing-graph 原因と reference safety cap を区別し、incomplete index を fold-only と表示しない必要があります。human output は complete summary ではなく incomplete generation を明示する必要があります。 - reference identity refresh coverage は、安定graphの再構築を物理writeのperformance contractとします。source identity、target resolutionの4列tuple、self reference、mutual recursionの更新にはNULL-safeなchanged-row predicateを維持し、安定rerunのtrigger auditは0、各corrupt phaseのrepairは1回、後段phaseの失敗で先行identity writeもrollbackされることを検証してください。SQLite `changes()` は引き続き最後のmutual-recursion phaseを表します。 - C# metadata-target resolver coverage は propagation work と安定 rerun を performance contract とします。逆順に保存した 8,000 class の chain では wall-clock threshold を使わず、dependency edge が厳密に `n - 1`、queue visit が `n` であることを維持してください。cross-file partial fan-in、seed を持たない cycle、事前 cancel、後段 update の注入失敗時に先行 row も rollback されること、安定 rerun の trigger audit が write 0 回、破損した derived row の修復が厳密に 1 write であることも残します。 - reference insert の transaction coverage は、public API の #1518 契約として71 row batchごとの transaction/SAVEPOINTを維持し、明示atomic-file APIは呼出元所有のlive transactionなしでは拒否され、reference batch scopeを0回に保つことを検証します。atomic-fileのreference-line materializationは完全な71 row reference batchだけをまとめ、`(file_id, line, context)` keyの和集合が333行を超える直前、または32 batchで停止します。reference INSERTの実行回数とprogress/cancellation checkpointは元の71 row境界に保ってください。public/atomicの正確なstatement数、333行/32 batch停止、`reference_lines` unique autoindexのlookup plan、通常/new-file両方のreference-line pathでbatch 2/3失敗時の全rollback、batch境界をまたぐ同一/異なるcontext、cancelとempty入力の順序、full scan・scoped update・MCP indexing・TypeScript augmentation rebuildのmulti-language guard付きintegrationを維持してください。321,352 refs / 856 filesの制御performance契約は自己snapshotの5/6 batch巨大file分布を保ち、wall-clock閾値を使わずpublic 5,009 scopeとatomic-file 0 scopeを比較します。 diff --git a/src/CodeIndex/Cli/IndexCommandRunner.FullScan.CSharpPreflight.cs b/src/CodeIndex/Cli/IndexCommandRunner.FullScan.CSharpPreflight.cs index 80794508e6..b978c8ccfc 100644 --- a/src/CodeIndex/Cli/IndexCommandRunner.FullScan.CSharpPreflight.cs +++ b/src/CodeIndex/Cli/IndexCommandRunner.FullScan.CSharpPreflight.cs @@ -103,7 +103,9 @@ void ObservePersistedCSharpPath(string indexPath) context.StaleFilePurgePlan.FileIds, csharpPositiveNoOpPolicyCandidate ? ObservePersistedCSharpPath - : null) + : null, + maxFileSizeBytes: + options.MaxFileSizeBytes ?? FileIndexer.DefaultMaxFileSizeBytes) : null; Dictionary? csharpPrepassStatReuse = null; diff --git a/src/CodeIndex/Cli/IndexCommandRunner.FullScan.Finalization.cs b/src/CodeIndex/Cli/IndexCommandRunner.FullScan.Finalization.cs index 3f491697d3..b318c9672c 100644 --- a/src/CodeIndex/Cli/IndexCommandRunner.FullScan.Finalization.cs +++ b/src/CodeIndex/Cli/IndexCommandRunner.FullScan.Finalization.cs @@ -103,6 +103,8 @@ private static int AddPostExtractionHookWarnings(PostExtractionHookRunner? runne } private static FoldOnlyRemediation? BuildFoldOnlyReadinessRemediation( + bool indexComplete, + bool referenceGraphComplete, bool graphTableAvailable, bool issuesTableAvailable, bool sqlGraphContractReady, @@ -115,6 +117,8 @@ private static int AddPostExtractionHookWarnings(PostExtractionHookRunner? runne string resolvedDbPath) { if (!IsFoldOnlyReadinessDegraded( + indexComplete, + referenceGraphComplete, graphTableAvailable, issuesTableAvailable, sqlGraphContractReady, @@ -133,6 +137,8 @@ private static int AddPostExtractionHookWarnings(PostExtractionHookRunner? runne } private static bool IsFoldOnlyReadinessDegraded( + bool indexComplete, + bool referenceGraphComplete, bool graphTableAvailable, bool issuesTableAvailable, bool sqlGraphContractReady, @@ -141,6 +147,8 @@ private static bool IsFoldOnlyReadinessDegraded( bool csharpMetadataTargetReady, bool foldReady) => !foldReady + && indexComplete + && referenceGraphComplete && graphTableAvailable && issuesTableAvailable && sqlGraphContractReady @@ -148,9 +156,11 @@ private static bool IsFoldOnlyReadinessDegraded( && csharpSymbolNameReady && csharpMetadataTargetReady; - private static string GetIndexReadinessWarning(bool graphTableAvailable, bool issuesTableAvailable, bool sqlGraphContractReady, bool hotspotFamilyReady, bool csharpSymbolNameReady, bool csharpMetadataTargetReady, bool foldReady, string? foldReadyReason, string projectRoot, string resolvedDbPath) + private static string GetIndexReadinessWarning(bool indexComplete, bool referenceGraphComplete, bool graphTableAvailable, bool issuesTableAvailable, bool sqlGraphContractReady, bool hotspotFamilyReady, bool csharpSymbolNameReady, bool csharpMetadataTargetReady, bool foldReady, string? foldReadyReason, string projectRoot, string resolvedDbPath) { var foldOnlyRemediation = BuildFoldOnlyReadinessRemediation( + indexComplete, + referenceGraphComplete, graphTableAvailable, issuesTableAvailable, sqlGraphContractReady, diff --git a/src/CodeIndex/Cli/IndexCommandRunner.FullScan.Output.cs b/src/CodeIndex/Cli/IndexCommandRunner.FullScan.Output.cs index 044f2e413d..177d15909d 100644 --- a/src/CodeIndex/Cli/IndexCommandRunner.FullScan.Output.cs +++ b/src/CodeIndex/Cli/IndexCommandRunner.FullScan.Output.cs @@ -110,6 +110,8 @@ private static int WriteFullScanFinalOutput(FullScanFinalOutputContext output) var hotspotFamilyDegradedReasonAfter = hotspotFamilySignalAfter.DegradedReason; var foldOnlyRemediation = BuildFoldOnlyReadinessRemediation( + persistedReadinessAfter.IndexComplete, + persistedReadinessAfter.ReferenceGraphComplete, persistedReadinessAfter.GraphTableAvailable, output.IssuesTableAvailableAfter, sqlGraphContractReadyAfter, @@ -239,7 +241,7 @@ private static int WriteFullScanFinalOutput(FullScanFinalOutputContext output) if (!persistedReadinessAfter.ReferenceGraphComplete) ConsoleUi.PrintWarning($"Reference graph is incomplete: {string.Join(", ", persistedReadinessAfter.ReferenceGraphIncompleteReasons)}."); if (!persistedReadinessAfter.GraphTableAvailable || !output.IssuesTableAvailableAfter || !sqlGraphContractReadyAfter || !hotspotFamilyReadyAfter || !output.CSharpSymbolNameReadyAfter || !output.CSharpMetadataTargetReadyAfter || !output.FoldReadyAfter) - ConsoleUi.PrintWarning(GetIndexReadinessWarning(persistedReadinessAfter.GraphTableAvailable, output.IssuesTableAvailableAfter, sqlGraphContractReadyAfter, hotspotFamilyReadyAfter, output.CSharpSymbolNameReadyAfter, output.CSharpMetadataTargetReadyAfter, output.FoldReadyAfter, output.FoldReadyReasonAfter, output.ProjectRoot, output.ResolvedDbPath)); + ConsoleUi.PrintWarning(GetIndexReadinessWarning(persistedReadinessAfter.IndexComplete, persistedReadinessAfter.ReferenceGraphComplete, persistedReadinessAfter.GraphTableAvailable, output.IssuesTableAvailableAfter, sqlGraphContractReadyAfter, hotspotFamilyReadyAfter, output.CSharpSymbolNameReadyAfter, output.CSharpMetadataTargetReadyAfter, output.FoldReadyAfter, output.FoldReadyReasonAfter, output.ProjectRoot, output.ResolvedDbPath)); if (cwdDriftDetected) ConsoleUi.PrintWarning(cwdDriftNotice!); if (output.Errors == 0 diff --git a/src/CodeIndex/Cli/IndexCommandRunner.Update.Output.cs b/src/CodeIndex/Cli/IndexCommandRunner.Update.Output.cs index 1f14d85add..d88264ebe0 100644 --- a/src/CodeIndex/Cli/IndexCommandRunner.Update.Output.cs +++ b/src/CodeIndex/Cli/IndexCommandRunner.Update.Output.cs @@ -69,6 +69,8 @@ private static int WriteUpdateFinalOutput(UpdateFinalOutputContext output) var hotspotFamilyDegradedReasonAfter = hotspotFamilySignalAfter.DegradedReason; var foldOnlyRemediation = BuildFoldOnlyReadinessRemediation( + persistedReadinessAfter.IndexComplete, + persistedReadinessAfter.ReferenceGraphComplete, persistedReadinessAfter.GraphTableAvailable, output.IssuesTableAvailableAfter, sqlGraphContractReadyAfter, @@ -183,7 +185,7 @@ private static int WriteUpdateFinalOutput(UpdateFinalOutputContext output) if (!persistedReadinessAfter.ReferenceGraphComplete) ConsoleUi.PrintWarning($"Reference graph is incomplete: {string.Join(", ", persistedReadinessAfter.ReferenceGraphIncompleteReasons)}."); if (!persistedReadinessAfter.GraphTableAvailable || !output.IssuesTableAvailableAfter || !sqlGraphContractReadyAfter || !hotspotFamilyReadyAfter || !output.CSharpSymbolNameReadyAfter || !output.CSharpMetadataTargetReadyAfter || !output.FoldReadyAfter) - ConsoleUi.PrintWarning(GetIndexReadinessWarning(persistedReadinessAfter.GraphTableAvailable, output.IssuesTableAvailableAfter, sqlGraphContractReadyAfter, hotspotFamilyReadyAfter, output.CSharpSymbolNameReadyAfter, output.CSharpMetadataTargetReadyAfter, output.FoldReadyAfter, output.FoldReadyReasonAfter, output.ProjectRoot, output.ResolvedDbPath)); + ConsoleUi.PrintWarning(GetIndexReadinessWarning(persistedReadinessAfter.IndexComplete, persistedReadinessAfter.ReferenceGraphComplete, persistedReadinessAfter.GraphTableAvailable, output.IssuesTableAvailableAfter, sqlGraphContractReadyAfter, hotspotFamilyReadyAfter, output.CSharpSymbolNameReadyAfter, output.CSharpMetadataTargetReadyAfter, output.FoldReadyAfter, output.FoldReadyReasonAfter, output.ProjectRoot, output.ResolvedDbPath)); if (cwdDriftDetected) ConsoleUi.PrintWarning(cwdDriftNotice!); } diff --git a/src/CodeIndex/Database/DbWriter.FileReuse.cs b/src/CodeIndex/Database/DbWriter.FileReuse.cs index ab4e33412a..30edbaf54e 100644 --- a/src/CodeIndex/Database/DbWriter.FileReuse.cs +++ b/src/CodeIndex/Database/DbWriter.FileReuse.cs @@ -353,6 +353,12 @@ FROM file_issues WHERE file_id = f.id AND kind = 'reference_count_exceeded' ) + AND NOT EXISTS ( + SELECT 1 + FROM file_issues + WHERE file_id = f.id + AND kind = 'file_too_large' + ) AND ( @generated_suppressed IS NULL OR ( @@ -413,7 +419,8 @@ internal ReusableIndexedFileStatsSnapshot LoadReusableIndexedFileStats( int initialCapacity = 0, IReadOnlySet? includedPaths = null, IReadOnlyList? excludedFileIds = null, - Action? persistedCSharpPathObserver = null) + Action? persistedCSharpPathObserver = null, + long maxFileSizeBytes = FileIndexer.DefaultMaxFileSizeBytes) { cancellationToken.ThrowIfCancellationRequested(); ReusableStatSnapshotReadForTesting?.Invoke(); @@ -442,6 +449,10 @@ SELECT 1 FROM file_issues AND NOT EXISTS ( SELECT 1 FROM file_issues WHERE file_id = f.id AND kind = 'reference_count_exceeded' + ) + AND NOT EXISTS ( + SELECT 1 FROM file_issues + WHERE file_id = f.id AND kind = 'file_too_large' )" : string.Empty; var generatedSuppressionProjection = hasIssuesTable @@ -468,6 +479,7 @@ f.lang IS NOT NULL AND typeof(f.modified) = 'text' AND typeof(f.size) = 'integer' AND f.size >= 0 + AND f.size <= @max_file_size {staleIssuePredicate} AND NOT EXISTS ( SELECT 1 FROM symbols @@ -487,6 +499,7 @@ THEN 1 ELSE 0 END AS reusable_eligible { c.Parameters.Add("@max_symbols", SqliteType.Integer); c.Parameters.Add("@max_references", SqliteType.Integer); + c.Parameters.Add("@max_file_size", SqliteType.Integer); if (hasIssuesTable) c.Parameters.Add("@generated_issue_kind", SqliteType.Text); }); @@ -502,11 +515,14 @@ THEN 1 ELSE 0 END AS reusable_eligible ? initialCapacity : Math.Min(initialCapacity, includedPaths.Count); ReusableStatSnapshotInitialCapacityForTesting?.Invoke(reusableInitialCapacity); - var reusable = new ReusableIndexedFileStatsSnapshot(reusableInitialCapacity); + var reusable = new ReusableIndexedFileStatsSnapshot( + reusableInitialCapacity, + maxFileSizeBytes); try { cmd.Parameters["@max_symbols"].Value = maxSymbolsPerFile; cmd.Parameters["@max_references"].Value = maxReferencesPerFile; + cmd.Parameters["@max_file_size"].Value = maxFileSizeBytes; if (hasIssuesTable) cmd.Parameters["@generated_issue_kind"].Value = FileIndexer.GeneratedCodeExtractionSkippedIssueKind; cancellationToken.ThrowIfCancellationRequested(); @@ -1012,12 +1028,15 @@ internal sealed class ReusableIndexedFileStatsSnapshot : Dictionary? _nonReusablePersistedSizes; private HashSet? _unknownPersistedSizePaths; - internal ReusableIndexedFileStatsSnapshot(int capacity) + internal ReusableIndexedFileStatsSnapshot(int capacity, long maxFileSizeBytes) : base(0, StringComparer.Ordinal) { _reusableCapacity = capacity; + MaxFileSizeBytes = maxFileSizeBytes; } + internal long MaxFileSizeBytes { get; } + internal void RecordReusable(string path, ReusableIndexedFileStat stat) { if (!_reusableCapacityApplied) diff --git a/src/CodeIndex/Indexer/Scanning/IndexedFileStatReuse.cs b/src/CodeIndex/Indexer/Scanning/IndexedFileStatReuse.cs index 8fbc52a4fb..0d0cf1a971 100644 --- a/src/CodeIndex/Indexer/Scanning/IndexedFileStatReuse.cs +++ b/src/CodeIndex/Indexer/Scanning/IndexedFileStatReuse.cs @@ -94,7 +94,7 @@ internal static Action? LookupForTesting } internal static IndexedFileStatReuseResult? TryGetReusableUnchangedFile( - IReadOnlyDictionary reusableFiles, + ReusableIndexedFileStatsSnapshot reusableFiles, string absolutePath, string relativePath, string? language, @@ -114,6 +114,7 @@ internal static Action? LookupForTesting { var info = new FileInfo(absolutePath); if (!info.Exists + || info.Length > reusableFiles.MaxFileSizeBytes || info.Length != indexed.Size || info.LastWriteTimeUtc != indexed.ModifiedUtc) { diff --git a/src/CodeIndex/Mcp/McpToolHandlers.Indexing.Execution.cs b/src/CodeIndex/Mcp/McpToolHandlers.Indexing.Execution.cs index d88e9036c5..95bb0d473c 100644 --- a/src/CodeIndex/Mcp/McpToolHandlers.Indexing.Execution.cs +++ b/src/CodeIndex/Mcp/McpToolHandlers.Indexing.Execution.cs @@ -516,7 +516,9 @@ void ObservePersistedCSharpPath(string indexPath) staleFilePurgePlan.FileIds, csharpPositiveNoOpPolicyCandidate ? ObservePersistedCSharpPath - : null) + : null, + maxFileSizeBytes: + maxFileBytes ?? FileIndexer.DefaultMaxFileSizeBytes) : null; Dictionary? csharpPrepassStatReuse = null; var priorPositiveCSharpSourceNoOpCandidate = false; diff --git a/tests/CodeIndex.Tests/IndexCommandRunnerFullScanTests.cs b/tests/CodeIndex.Tests/IndexCommandRunnerFullScanTests.cs index ae5be07184..8a6718f371 100644 --- a/tests/CodeIndex.Tests/IndexCommandRunnerFullScanTests.cs +++ b/tests/CodeIndex.Tests/IndexCommandRunnerFullScanTests.cs @@ -398,6 +398,105 @@ public void Run_FullScan_CompletenessMatrixMatchesImmediateStatus_Issue4826( } } + [Theory] + [InlineData(false, true, false)] + [InlineData(true, false, true)] + public void Run_FullScan_FileSizePolicyTransitionReprocessesUnchangedFile_Issue4826( + bool initialCap, + bool nextCap, + bool expectedComplete) + { + var projectRoot = CreateTempProject(); + try + { + File.WriteAllText( + Path.Combine(projectRoot, "large.py"), + "print('start')\n" + new string('a', 256)); + var initialArgs = new List { projectRoot, "--json", "--quiet" }; + if (initialCap) + initialArgs.InsertRange(1, ["--max-file-bytes", "128"]); + var (initialExitCode, _) = RunAndCaptureJson([.. initialArgs]); + Assert.Equal(CommandExitCodes.Success, initialExitCode); + + var nextArgs = new List { projectRoot, "--json", "--quiet" }; + if (nextCap) + nextArgs.InsertRange(1, ["--max-file-bytes", "128"]); + var (nextExitCode, nextJson) = RunAndCaptureJson([.. nextArgs]); + + Assert.Equal(CommandExitCodes.Success, nextExitCode); + Assert.Equal(expectedComplete, nextJson.GetProperty("index_complete").GetBoolean()); + Assert.Equal( + expectedComplete, + nextJson.GetProperty("reference_graph_complete").GetBoolean()); + Assert.Equal( + 1, + nextJson.GetProperty("summary").GetProperty("files_extracted").GetInt64()); + Assert.Equal( + 1, + nextJson.GetProperty("summary").GetProperty("files_persisted").GetInt64()); + AssertCompletenessReason( + nextJson, + "index_incomplete_reasons", + expectedComplete ? null : "file_too_large"); + } + finally + { + SqliteConnection.ClearAllPools(); + DeleteDirectory(projectRoot); + } + } + + [Fact] + public void Run_FullScan_IncompleteGenerationDoesNotExposeFoldOnlyRemediation_Issue4826() + { + var projectRoot = CreateTempProject(); + try + { + File.WriteAllText(Path.Combine(projectRoot, "large.py"), "def ready(): return True\n"); + File.WriteAllText(Path.Combine(projectRoot, "other.py"), "def other(): return True\n"); + var initialExitCode = IndexCommandRunner.Run([projectRoot, "--json"], _jsonOptions); + Assert.Equal(CommandExitCodes.Success, initialExitCode); + + var dbPath = Path.Combine(projectRoot, ".cdidx", "codeindex.db"); + using (var connection = OpenNonPoolingConnection(dbPath)) + { + connection.Open(); + using var command = connection.CreateCommand(); + command.CommandText = """ + DELETE FROM codeindex_meta + WHERE key IN (@version, @fingerprint) + """; + command.Parameters.AddWithValue("@version", "fold_key_version"); + command.Parameters.AddWithValue("@fingerprint", "fold_key_fingerprint"); + command.ExecuteNonQuery(); + } + File.WriteAllText( + Path.Combine(projectRoot, "large.py"), + "print('start')\n" + new string('a', 256)); + + var (exitCode, json) = RunAndCaptureJson( + [projectRoot, "--max-file-bytes", "128", "--json", "--quiet"]); + + Assert.Equal(CommandExitCodes.Success, exitCode); + Assert.False(json.GetProperty("index_complete").GetBoolean()); + Assert.False(json.GetProperty("reference_graph_complete").GetBoolean()); + Assert.False(json.GetProperty("fold_ready").GetBoolean()); + Assert.Equal(JsonValueKind.Null, json.GetProperty("degraded_reason").ValueKind); + Assert.Equal(JsonValueKind.Null, json.GetProperty("recommended_action").ValueKind); + Assert.Equal(JsonValueKind.Null, json.GetProperty("alternative_action").ValueKind); + + var humanArgs = new[] { projectRoot, "--max-file-bytes", "128" }; + var (humanExitCode, _, stderr) = RunAndCaptureStreams(humanArgs); + Assert.Equal(CommandExitCodes.Success, humanExitCode); + Assert.DoesNotContain("fold-only", stderr, StringComparison.Ordinal); + } + finally + { + SqliteConnection.ClearAllPools(); + DeleteDirectory(projectRoot); + } + } + [Theory] [InlineData(false, true, null)] [InlineData(true, false, "file_too_large")] diff --git a/tests/CodeIndex.Tests/McpServerToolsCallTests.cs b/tests/CodeIndex.Tests/McpServerToolsCallTests.cs index 91531eed1c..7c6735ab73 100644 --- a/tests/CodeIndex.Tests/McpServerToolsCallTests.cs +++ b/tests/CodeIndex.Tests/McpServerToolsCallTests.cs @@ -8758,6 +8758,60 @@ public void ToolsCall_Index_MaxReferencesPerFilePersistsReferenceCountExceededIs } } + [Fact] + public void ToolsCall_Index_FileSizePolicyTransitionsReprocessUnchangedFile_Issue4826() + { + var fixtureDir = Path.Combine( + Path.GetFullPath("."), + $"mcp_index_file_size_transition_{Guid.NewGuid():N}"); + var dbPath = TestProjectHelper.CreateTempDbPath( + "cdidx_mcp_index_file_size_transition"); + try + { + Directory.CreateDirectory(fixtureDir); + File.WriteAllText( + Path.Combine(fixtureDir, "large.py"), + "print('start')\n" + new string('a', 256)); + + using var server = new McpServer(dbPath, ConsoleUi.LoadVersion()); + var initial = CallIndex(server, fixtureDir); + var capped = CallIndex( + server, + fixtureDir, + args => args["maxFileBytes"] = JsonNode.Parse("128")); + var recovered = CallIndex(server, fixtureDir); + + Assert.False( + initial["result"]?["isError"]?.GetValue() ?? false, + initial.ToJsonString()); + Assert.False( + capped["result"]?["isError"]?.GetValue() ?? false, + capped.ToJsonString()); + Assert.False( + recovered["result"]?["isError"]?.GetValue() ?? false, + recovered.ToJsonString()); + Assert.True( + initial["result"]!["structuredContent"]!["index_complete"]! + .GetValue()); + var cappedStructured = capped["result"]!["structuredContent"]!; + Assert.False(cappedStructured["index_complete"]!.GetValue()); + Assert.Contains( + "file_too_large", + cappedStructured["index_incomplete_reasons"]!.AsArray() + .Select(reason => reason!.GetValue())); + var recoveredStructured = recovered["result"]!["structuredContent"]!; + Assert.True(recoveredStructured["index_complete"]!.GetValue()); + Assert.True( + recoveredStructured["reference_graph_complete"]!.GetValue()); + Assert.Null(recoveredStructured["index_incomplete_reasons"]); + } + finally + { + TestProjectHelper.DeleteDirectory(fixtureDir); + TestProjectHelper.DeleteSqliteDatabaseFiles(dbPath); + } + } + [Fact] public void ToolsCall_Index_RefreshesMutualRecursionOnceAfterBulkReferenceInsert() {