diff --git a/AGENT_GUIDE.md b/AGENT_GUIDE.md index e50796feed..2cbbbf80db 100644 --- a/AGENT_GUIDE.md +++ b/AGENT_GUIDE.md @@ -131,12 +131,14 @@ CI watching must be bounded. Do not loop indefinitely. ## Status Contract -- `status --json` and related JSON/MCP payloads currently expose the trust fields documented in `README.md` and `DEVELOPER_GUIDE.md`, including `fold_ready`, `fold_ready_reason`, `graph_table_available`, `issues_table_available`, `file_issues_data_current`, `migration_in_progress`, `sql_graph_contract_ready`, `sql_graph_contract_degraded_reason`, `hotspot_family_ready`, `hotspot_family_degraded_reason`, `language_readiness`, `csharp_symbol_name_ready`, `csharp_metadata_target_ready`, `csharp_metadata_target_degraded_reason`, `indexed_head_commit`, `worktree_head_changed`, `index_writer_version`, `index_newer_than_reader`, `index_newer_than_reader_reason`, `unknown_extension_file_count`, `path_case_sensitive`, `data_dir`, `data_dir_source`, `data_dir_mode`, `mac_profile`, `db_size_bytes`, `wal_size_bytes`, `db_pragma_settings` (`journal_mode`, `synchronous`, `wal_autocheckpoint`, `page_count`, `freelist_count`, `page_size`), `symbols_by_language`, `process`, `last_index_run`, `hooks`, MCP-only `mcp_session`, and the `status --check`-only `stale_after_seconds` / `index_age_seconds` threshold audit fields. +- `status --json` and related JSON/MCP payloads currently expose the trust fields documented in `README.md` and `DEVELOPER_GUIDE.md`, including `fold_ready`, `fold_ready_reason`, `graph_table_available`, `issues_table_available`, `file_issues_data_current`, `migration_in_progress`, `sql_graph_contract_ready`, `sql_graph_contract_degraded_reason`, `hotspot_family_ready`, `hotspot_family_degraded_reason`, `language_readiness`, `csharp_symbol_name_ready`, `csharp_metadata_target_ready`, `csharp_metadata_target_degraded_reason`, `indexed_head_commit`, `worktree_head_changed`, `index_writer_version`, `index_newer_than_reader`, `index_newer_than_reader_reason`, `unknown_extension_file_count`, `unknown_extension_files`, `unknown_extension_files_truncated`, `unknown_extension_file_path_limit`, `extractors`, `path_case_sensitive`, `data_dir`, `data_dir_source`, `data_dir_mode`, `mac_profile`, `db_size_bytes`, `wal_size_bytes`, `db_pragma_settings` (`journal_mode`, `synchronous`, `wal_autocheckpoint`, `page_count`, `freelist_count`, `page_size`), `symbols_by_language`, `process`, `last_index_run`, `hooks`, MCP-only `mcp_session`, and the `status --check`-only `stale_after_seconds` / `index_age_seconds` threshold audit fields. - When any readiness field is degraded, the CLI adds `degraded_root_cause`, `degraded_reason`, `recommended_action`, `alternative_action`, and `readiness_degradations[]`. `degraded_root_cause` is the primary stable machine code; `readiness_degradations[]` lists every degraded field with `root_cause`, human reason, and remediation strings. - `issues_table_available` reports physical `file_issues` table presence only. `file_issues_data_current` reports whether the table is also stamped current for the active index generation. - `index_writer_version` records the `cdidx` version that last wrote to the DB (stamped into `codeindex_meta` as `cdidx_writer_version` on every full scan, update, and MCP index). `index_newer_than_reader` flips to `true` whenever any persisted numeric contract stamp in `codeindex_meta` (or unknown `PRAGMA user_version` readiness bits) exceeds the current binary's compiled maximum, so an older CLI re-opening a DB written by a newer CLI degrades loudly with an audit trail instead of silently dropping back to text-search fallbacks. `index_newer_than_reader_reason` enumerates the specific newer-than-reader stamps. - `status` also surfaces indexed-HEAD freshness via `indexed_head_sha`, `indexed_head_branch`, `indexed_head_timestamp`, and `commits_ahead_of_indexed_head`. They are stamped by `cdidx index` on every successful run (full scan AND partial update, distinct from `indexed_head_commit` which is full-scan only) on a best-effort basis (never blocks an otherwise-successful index) and omitted on non-git workspaces, detached HEAD (branch only), or legacy DBs created before this contract. -- `status` also surfaces unknown-extension scan coverage via `unknown_extension_file_count`, stamped by successful full-repository index runs (`cdidx index ` and MCP `index_project`) as the number of non-indexed files with non-empty extensions that do not map to a known language. It is omitted on legacy DBs or before a current full scan has stamped the value. +- `status` also surfaces unknown-extension scan coverage via `unknown_extension_file_count`, stamped by successful full-repository index runs (`cdidx index ` and MCP `index_project`) as the number of non-indexed files with non-empty extensions that do not map to a known language. Current scans also stamp `unknown_extension_files` as a bounded path sample, `unknown_extension_files_truncated` when the count exceeds the sample, and `unknown_extension_file_path_limit` as the cap. These fields are omitted on legacy DBs or before a current full scan has stamped them. +- `status` also surfaces extractor plugin and pattern-config runtime diagnostics via `extractors`, including loaded counts, skipped file counts, and a bounded diagnostics list for incompatible or malformed plugin/pattern files. +- `status` also surfaces post-extraction hook callback budgets through `hooks[].callback_budget_ms`. Index runs enforce `CDIDX_HOOK_CALLBACK_BUDGET_MS` (default: 5000 ms) on scratch copies, discard timed-out mutations, and disable timed-out hooks for the remainder of the current run. - `status` also surfaces `.cdidx` data-directory permissions via `data_dir_mode` on POSIX filesystems. New `.cdidx` data directories are forced to `0700`; the field is omitted on Windows, URI DBs, or when the directory mode cannot be inspected. - `status` also surfaces filesystem case-sensitivity via `path_case_sensitive`, stamped on every successful `cdidx index` run (full scan AND partial update, plus MCP-driven indexes) from `core.ignorecase` + a live filesystem probe. `true` means the volume is case-sensitive (`Foo.cs` and `foo.cs` are distinct); `false` means case-insensitive. Omitted on legacy DBs that predate the stamp. Use it to audit path-equality decisions on case-sensitive APFS, WSL NTFS / dev-drive, and ReFS mounts where the prior OS-keyed heuristic could mis-classify the workspace (#1546). - `status` also surfaces Linux mandatory-access-control context via `mac_profile` when `/proc/self/attr/current` or `/proc/self/attr/exec` indicates an AppArmor or SELinux profile. It is omitted on non-Linux hosts, unconstrained processes, or unreadable proc attributes (#1768). diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index a20e189ff8..c890b41b45 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -247,7 +247,7 @@ trust metadata before the dependent rows are written. Out-of-tree post-extraction hooks can implement `CodeIndex.Indexer.Hooks.IPostExtractionHook` in a `.dll` placed under `~/.config/cdidx/hooks/` (or the directory named by `CDIDX_HOOKS_DIR`). Hook assemblies are discovered in path order. Each concrete hook type is instantiated with a public parameterless constructor, then called after built-in symbol extraction and again after built-in reference extraction, before rows are persisted. Hooks receive a `FileContext` plus mutable `IList` / `IList` values, so they can annotate extracted records, add synthetic symbols, or add domain-specific references. -Hook failures are isolated to that hook invocation: assembly load, construction, and callback exceptions are captured as diagnostics and indexing continues. `status --json` and MCP `status` expose loaded hooks under `hooks` with `name`, `assembly_path`, and `type_name` so users can confirm which extensions are active. +Hook failures are isolated to that hook invocation: assembly load, construction, and callback exceptions are captured as diagnostics and indexing continues. Each callback runs against a scratch copy with a bounded wall-clock budget controlled by `CDIDX_HOOK_CALLBACK_BUDGET_MS` (default: 5000 ms). A timed-out callback contributes no mutations, emits an index warning, and disables that hook for the remainder of the current index run. `status --json` and MCP `status` expose loaded hooks under `hooks` with `name`, `assembly_path`, `type_name`, and `callback_budget_ms` so users can confirm which extensions are active and what timeout is being enforced. ### Ignore file parsing @@ -995,7 +995,11 @@ Adding `--json-envelope` to a query command (`search`, `definition`, `references Every top-level CLI/MCP JSON DTO (`StatusResult`, `RepoMapResult`, `SymbolAnalysisResult`, `ImpactAnalysisResult`, `OutlineResult`, `FileExcerptResult`, `CompactSearchResult`, `SymbolResult`, `DefinitionResult`, `UnusedSymbolResult`, `ReferenceResult`, `CallerResult`, `CalleeResult`, `FileResult`, `FileFindResult`) carries an `api_version` string field stamped from `JsonOutputContract.ApiVersion`. The same value is mirrored on the `--json-envelope` `metadata` block. This describes the JSON output contract, not the cdidx binary version (which is still surfaced via `version.json` and `cdidx --version`). Bump `JsonOutputContract.ApiVersion` only on **breaking** shape changes — renames, removals, or type changes of an existing field. Additive changes (new optional fields, new readiness flags, new enum values) keep the version stable so older consumers continue to parse the payload. Strict downstream consumers should pin against the major value and degrade gracefully when it changes. Issue #1555. -The documented `status --json` trust contract spans `fold_ready`, `fold_ready_reason`, `graph_table_available`, `issues_table_available`, `file_issues_data_current`, `migration_in_progress`, `sql_graph_contract_ready`, `sql_graph_contract_degraded_reason`, `hotspot_family_ready`, `hotspot_family_degraded_reason`, `language_readiness`, `csharp_symbol_name_ready`, `csharp_metadata_target_ready`, `csharp_metadata_target_degraded_reason`, `indexed_head_commit`, `worktree_head_changed`, `indexed_head_sha`, `indexed_head_branch`, `indexed_head_timestamp`, `commits_ahead_of_indexed_head`, `index_writer_version`, `index_newer_than_reader`, `index_newer_than_reader_reason`, `unknown_extension_file_count`, `path_case_sensitive`, `data_dir_mode`, `mac_profile`, `stale_after_seconds`, `index_age_seconds`, the remediation fields `degraded_root_cause`, `degraded_reason`, `recommended_action`, `alternative_action`, `readiness_degradations`, and MCP-only `mcp_session`. MCP `mcp_session` is session-scoped diagnostics, not persisted DB state, and contains `log_level`, `roots`, optional `client_info`, and optional `client_capabilities`. Keep this list synchronized with `README.md` and `AGENT_GUIDE.md`; `DocumentationStatusContractTests` fails when any required field is missing from one of those docs. +The documented `status --json` trust contract spans `fold_ready`, `fold_ready_reason`, `graph_table_available`, `issues_table_available`, `file_issues_data_current`, `migration_in_progress`, `sql_graph_contract_ready`, `sql_graph_contract_degraded_reason`, `hotspot_family_ready`, `hotspot_family_degraded_reason`, `language_readiness`, `csharp_symbol_name_ready`, `csharp_metadata_target_ready`, `csharp_metadata_target_degraded_reason`, `indexed_head_commit`, `worktree_head_changed`, `indexed_head_sha`, `indexed_head_branch`, `indexed_head_timestamp`, `commits_ahead_of_indexed_head`, `index_writer_version`, `index_newer_than_reader`, `index_newer_than_reader_reason`, `unknown_extension_file_count`, `unknown_extension_files`, `unknown_extension_files_truncated`, `unknown_extension_file_path_limit`, `extractors`, `path_case_sensitive`, `data_dir_mode`, `mac_profile`, `stale_after_seconds`, `index_age_seconds`, the remediation fields `degraded_root_cause`, `degraded_reason`, `recommended_action`, `alternative_action`, `readiness_degradations`, and MCP-only `mcp_session`. MCP `mcp_session` is session-scoped diagnostics, not persisted DB state, and contains `log_level`, `roots`, optional `client_info`, and optional `client_capabilities`. Keep this list synchronized with `README.md` and `AGENT_GUIDE.md`; `DocumentationStatusContractTests` fails when any required field is missing from one of those docs. + +`extractors` reports runtime extractor plugin and pattern-config health, including loaded plugin assembly and pattern counts, symbol/reference extractor counts, skipped file counts, and a bounded diagnostic list for incompatible or malformed files. + +`hooks[]` includes `callback_budget_ms`, mirroring the post-extraction callback budget enforced by `CDIDX_HOOK_CALLBACK_BUDGET_MS` (default: 5000 ms). Timed-out callback mutations are discarded because hooks run on a scratch copy before their results are applied. `references` already prefixes each human-readable row with `reference_kind`, and `callers` does the same for its grouped caller rows. When one grouped container mixes kinds (for example `call` and `subscribe` on the same event member), the human-readable label joins the distinct kinds with `+` (for example `call+subscribe`) instead of collapsing to a single preferred label, and the reference-kind column widens dynamically to fit the longest label in the batch so mixed rows do not overrun the neighbouring column. JSON output for `callers` and `callees` keeps the scalar `reference_kind` for back-compat (it reports the preferred summary kind `instantiate` > `subscribe` > `MIN(call)`) and adds a sorted `reference_kinds` array plus a `has_mixed_reference_kinds` bool so consumers can detect mixed containers without trusting a single collapsed label. This lets terminal users distinguish `call` / `instantiate` / `subscribe` / mixed without re-running the command with `--json` and lets AI clients answer mixed-kind questions without chasing a second `--exact` query. @@ -1162,7 +1166,7 @@ Process exit codes are coarse (`0` success including valid zero-row queries, `1` - **Path-aware narrowing and ranking** — `search`, `definition`, `references`, `callers`, `callees`, `symbols`, and `files` share path include/exclude filters plus `--exclude-tests`. Read queries prefer source files over tests/docs, and full-text search boosts exact symbol-name and path matches to surface likely implementation files first. - **Compact search snippets for AI** — `search --json` and MCP `search` return match-centered snippets with explicit snippet ranges, match lines, highlights, context counts, `truncated_line_count`, and `truncation_context` instead of whole chunks. `truncation_context.char_counts` and `truncation_context.total_chars` expose the omitted character counts behind each clamped snippet line, while truncated highlights also carry `truncated_char_counts`. `--snippet-lines` lets clients trade recall for smaller payloads, and `--max-line-width` (CLI) / `maxLineWidth` (MCP) routes each snippet line through the same `LineWidthFormatter.ClampLine` contract used by `find` / `references` / `excerpt` / `inspect` so hits inside minified / transpiled / generated single-line files no longer return hundreds of KB per result unless the caller explicitly sets `0`; clamped lines carry `...(+N)...` markers and `highlights[].truncated` / `highlights[].original_line_length`. - **Repo map for first-pass orientation** — `map` aggregates languages, modules, top files, file hot spots, and likely entrypoints from indexed data so AI clients can decide where to look before issuing precise queries. Entrypoint inference now falls back to known top-level entry files when symbol extraction does not produce an explicit `Main`-style symbol. -- **Freshness metadata for trust decisions** — `status` exposes whole-workspace freshness and git state, plus trust metadata such as `sql_graph_contract_ready` / `sql_graph_contract_degraded_reason`, `hotspot_family_ready` / `hotspot_family_degraded_reason`, forward-compatibility audit fields (`index_writer_version`, `index_newer_than_reader`, `index_newer_than_reader_reason` — see "Forward-compatibility readiness audit"), and fold remediation fields (`fold_ready_reason`, `degraded_reason`, `recommended_action`, `alternative_action`) so AI clients can tell up front whether SQL graph/dependency/impact answers, duplicate-name hotspot families, and Unicode `--exact` are authoritative. CLI `status --json` and MCP `status` both populate those fold remediation fields when `fold_ready=false`. It also carries `unknown_extension_file_count` after a current full-repository scan so extension-table coverage gaps are visible even when those files were excluded from indexing. When those fold remediation fields are derived from an explicit read-only `file:` DB URI, they are normalized back to a writable filesystem path for both absolute (`file:///...?...`) and relative (`file:codeindex.db?...`) forms instead of echoing the read-only URI into commands that would fail. `cdidx index` JSON/human readiness output also surfaces the same trust bits, keeping the post-index readiness summary aligned with `status`. `impact` / MCP `impact_analysis` also mirror the SQL graph-contract signal in JSON so stale SQL rows do not masquerade as authoritative zero-impact answers. `inspect` / MCP `analyze_symbol` and `references` / MCP `references` now mirror that same SQL graph-contract signal whenever SQL-backed graph reads contribute to their payloads, so stale SQL rows do not look like authoritative hits or zero-result answers there either. `map` keeps `indexed_at` / `latest_modified` scoped to the filtered result set and also exposes `workspace_indexed_at` / `workspace_latest_modified` for whole-workspace freshness. `inspect` mirrors those whole-workspace timestamps and git fields so symbol-oriented AI flows can make trust decisions without a separate `status` call. `files` exposes per-file checksum plus modified/indexed timestamps. File-column migrations are applied opportunistically for older DBs, and read paths are designed to avoid crashing if in-place migration is unavailable. CLI and MCP zero-result JSON responses for `search`, `files`, `symbols`, `definition`, `references`, `callers`, `callees`, `deps`, `unused`, `hotspots`, and `impact` include `indexed_file_count`, `indexed_at`, and `freshness_available`. `indexed_at:null` with `freshness_available=true` means the index is empty, while `freshness_available=false` means a legacy/read-only DB could not expose freshness timestamps and `freshness_degraded_reason` explains why. **HEAD-aware staleness signal**: every successful `cdidx index` full scan now stamps the captured `git HEAD` into `codeindex_meta` so subsequent runs can compare it against the workspace HEAD. When they differ and the user did not pass `--rebuild`, the CLI emits a `head_changed` warning recommending `cdidx index --rebuild` and exposes `head_changed` / `prior_indexed_head_commit` / `current_head_commit` / `head_change_notice` in `index --json`. `status --check` mirrors the same comparison through `workspace_check.head_changed` (alongside `indexed_head_commit` / `workspace_head_commit` when they differ), so AI clients that already gate on freshness can refuse to trust a default incremental scan after `git switch ` without a separate query. `--commits` / `--files` partial updates deliberately preserve the captured HEAD so the staleness signal survives until a real full scan reindexes the worktree. Non-git workspaces and legacy DBs that never captured a HEAD skip the comparison instead of false-positive flagging. +- **Freshness metadata for trust decisions** — `status` exposes whole-workspace freshness and git state, plus trust metadata such as `sql_graph_contract_ready` / `sql_graph_contract_degraded_reason`, `hotspot_family_ready` / `hotspot_family_degraded_reason`, forward-compatibility audit fields (`index_writer_version`, `index_newer_than_reader`, `index_newer_than_reader_reason` — see "Forward-compatibility readiness audit"), and fold remediation fields (`fold_ready_reason`, `degraded_reason`, `recommended_action`, `alternative_action`) so AI clients can tell up front whether SQL graph/dependency/impact answers, duplicate-name hotspot families, and Unicode `--exact` are authoritative. CLI `status --json` and MCP `status` both populate those fold remediation fields when `fold_ready=false`. It also carries `unknown_extension_file_count`, a capped `unknown_extension_files` path sample, `unknown_extension_files_truncated`, and `unknown_extension_file_path_limit` after a current full-repository scan so extension-table coverage gaps are visible and actionable even when those files were excluded from indexing. When those fold remediation fields are derived from an explicit read-only `file:` DB URI, they are normalized back to a writable filesystem path for both absolute (`file:///...?...`) and relative (`file:codeindex.db?...`) forms instead of echoing the read-only URI into commands that would fail. `cdidx index` JSON/human readiness output also surfaces the same trust bits, keeping the post-index readiness summary aligned with `status`. `impact` / MCP `impact_analysis` also mirror the SQL graph-contract signal in JSON so stale SQL rows do not masquerade as authoritative zero-impact answers. `inspect` / MCP `analyze_symbol` and `references` / MCP `references` now mirror that same SQL graph-contract signal whenever SQL-backed graph reads contribute to their payloads, so stale SQL rows do not look like authoritative hits or zero-result answers there either. `map` keeps `indexed_at` / `latest_modified` scoped to the filtered result set and also exposes `workspace_indexed_at` / `workspace_latest_modified` for whole-workspace freshness. `inspect` mirrors those whole-workspace timestamps and git fields so symbol-oriented AI flows can make trust decisions without a separate `status` call. `files` exposes per-file checksum plus modified/indexed timestamps. File-column migrations are applied opportunistically for older DBs, and read paths are designed to avoid crashing if in-place migration is unavailable. CLI and MCP zero-result JSON responses for `search`, `files`, `symbols`, `definition`, `references`, `callers`, `callees`, `deps`, `unused`, `hotspots`, and `impact` include `indexed_file_count`, `indexed_at`, and `freshness_available`. `indexed_at:null` with `freshness_available=true` means the index is empty, while `freshness_available=false` means a legacy/read-only DB could not expose freshness timestamps and `freshness_degraded_reason` explains why. **HEAD-aware staleness signal**: every successful `cdidx index` full scan now stamps the captured `git HEAD` into `codeindex_meta` so subsequent runs can compare it against the workspace HEAD. When they differ and the user did not pass `--rebuild`, the CLI emits a `head_changed` warning recommending `cdidx index --rebuild` and exposes `head_changed` / `prior_indexed_head_commit` / `current_head_commit` / `head_change_notice` in `index --json`. `status --check` mirrors the same comparison through `workspace_check.head_changed` (alongside `indexed_head_commit` / `workspace_head_commit` when they differ), so AI clients that already gate on freshness can refuse to trust a default incremental scan after `git switch ` without a separate query. `--commits` / `--files` partial updates deliberately preserve the captured HEAD so the staleness signal survives until a real full scan reindexes the worktree. Non-git workspaces and legacy DBs that never captured a HEAD skip the comparison instead of false-positive flagging. - **Folded-key upgrade without reparse** — `backfill-fold` and MCP `backfill_fold` recompute `name_folded` / `*_folded` directly from existing DB rows, then stamp `FoldReadyFlag` once verification confirms no required folded values remain NULL. This gives AI clients and users a low-cost upgrade path from pre-#86 DBs without re-reading every source file, and it also rewrites all folded rows when `fold_key_version` is missing or mismatched so future `NameFold.Version` bumps cannot silently restamp stale keys. - **Bundled symbol analysis** — `inspect` and MCP `analyze_symbol` return definition, nearby symbols, references, callers, callees, file metadata, workspace trust metadata, and graph-support metadata in one request so AI clients can answer common symbol questions with fewer round-trips. - **Language-aware reference extraction** — `references`, `callers`, `callees`, and `impact` are backed by an indexed reference table built only for languages where regex-based call/reference extraction is meaningful (30 of 46 languages). Unsupported languages intentionally fall back to text search instead of returning low-confidence pseudo-graph data. When a language is removed from graph support, `PurgeUnsupportedReferences` deletes its stale `symbol_references` rows on the next indexing run, and graph read paths additionally filter by supported languages to prevent stale edges from surviving between index runs. Shell is intentionally excluded because its command-style invocations (`foo arg1 arg2`) cannot be detected by the parenthesized-call regex. **Nested generic call sites**: C#/Java constructor calls like `new Dictionary>()` and C# generic method calls like `Helper.DoWork>()` are recovered by a depth-aware fallback scanner so the outer target still reaches the reference table even though the flat regex fast-path cannot balance `>>`. **JS/TS no-paren constructors**: JavaScript / TypeScript zero-argument constructor calls that legally omit `()` — for example `new Foo;`, `new Date;`, qualified targets like `new Demo.Provider;`, and one-level generic TypeScript forms like `new Box;` — are emitted as `instantiate` edges via a dedicated language-gated path, while next-line `.bar()` / `[0]` continuations are suppressed so a line-ended `new Foo` does not become a phantom standalone instantiation. **Constructor chain calls**: C# `: this(...)` / `: base(...)` initializers and Java `this(...)` / `super(...)` first-statement calls are detected separately from the generic call regex and rewritten so the reference target is the real constructor (enclosing class/record for `this`, the parsed base type from the class signature for `base` / `super`). Cross-line C# initializers are attributed to the owning constructor rather than the enclosing class. Base-type parsing strips generics, record primary-ctor args, `where` constraints, and `global::` / dotted namespace qualifiers; Java `super.method()` stays a normal method call. **Type-position dependency edges**: C#/Java base lists, declaration types, generic constraints, `throws`, `is`/`as`/`instanceof`, and real C# XML-doc `cref` sites are indexed as `type_reference` rows so `references` / `impact` can see compile-time rename dependencies without polluting the default dynamic call graph exposed by `callers` / `callees`. C# XML-doc `cref` extraction accepts declaration-attached XML-doc comments from both `///` lines and delimited `/** ... */` blocks, including declarations that begin later on the same physical line after the closing `*/` only when no unrelated same-line code or declaration intervenes, while ordinary `//` / `////` comments, non-documenting block comments, method-body XML-doc comments that merely precede a later declaration, brace-free field/property initializer continuations, brace-free expression lambdas, intervening top-level executable statements, same-line non-target code after `*/`, other nested executable continuations, and multiline raw/verbatim string content whose line happens to start with `/**` stay excluded. Non-doc code or string content after the closing `*/` on the same physical line is still outside the doc-comment slice. Even though the regex now runs against that narrower slice, the extractor preserves `symbol_references.column` relative to the original physical source line. On the C# read path, `using static` constant-pattern suppression is token-aware around `is` / `case`, reconstructs a small indexed multi-line window when the anchor lives on a previous line, and keeps trivia-bearing forms such as `value is/*comment*/Red`, `value is\n Red or Blue`, and `case\tRed:` filtered. Same-name type rescue also honors `file` visibility so file-local types only rescue references from the same physical file; inherited protected/public/internal nested types from real base classes rescue derived-class pattern heads only after the base reference is normalized through active type and namespace aliases, and alias-expanded constructed generic bases are canonicalized again before containing-type lookup so `AliasBase = Probe.Base` resolves the same way as `Probe.Base`; implemented interfaces do not contribute inherited nested-type rescue; and same-file `using Namespace;`, project-wide `global using Namespace;`, and active type aliases all participate in the rescue set. The extractor deliberately leaves ambiguous unqualified `using static` heads such as `value is Red` in the DB, because file-local parsing alone cannot know whether another file in the same namespace declares the real `Red` type; the workspace-aware read path is responsible for suppressing the pure constant-only cases. **SQL qualified-name alignment**: SQL definitions still persist their schema-qualified symbol name (`dbo.fn_X`), but graph/`deps`/unused/hotspot readers now resolve each SQL reference row through its stored source-line context, recorded call column, and enclosing container before they compare it to definitions, so qualified `references` / `callers` / `impact` queries stay schema-scoped even when one line contains multiple qualified calls or the lookup is non-exact. Those readers fall back to the bare leaf only when the source site itself is genuinely unqualified, which keeps `deps`, `unused`, and `hotspots` aligned with qualified SQL calls without regressing bare-call support or double-counting `EXEC dbo.fn_Target; EXEC sales.fn_Target;`. Once a row already has a recorded call column, those downstream readers no longer whole-line-upgrade that row to a later qualified token, so trailing comments, string literals, or a second qualified call cannot steal the earlier unqualified edge. Exact SQL graph/dependency readers also preserve the resolved segment count, so a quoted single identifier containing a dot such as `"sales.fn_Target"` stays distinct from the real qualified name `sales.fn_Target` across exact `references` / `callers` / `impact` and aggregate `deps` / `unused` / `hotspots`. SQL CTE body source rows use the raw `cte_body_reference` kind, so `references --kind cte_body_reference` can distinguish anchor/recursive-member internals from outer-query table references. Qualified SQL `callees` queries also keep leaf fallback disabled unless the caller query itself is unqualified, so `callees sales.Caller` no longer widens to `dbo.Caller`. SQL extractors also accept optional whitespace around qualified-name dots, so definitions/calls such as `[sales] . [fn_Target]` and `[dbo] . [fn_Target]` keep their full qualified identity instead of truncating at the first segment. The same SQL no-parens extractor now preserves ANSI / PostgreSQL double-quoted call targets such as `CALL "sales"."proc_name"` and `EXEC "dbo"."fn_Target"` instead of stripping them as string literals, while true single-quoted SQL string literals remain masked. Definition-oriented readers also canonicalize quoted qualified SQL names (`[dbo].[fn_X]` → `dbo.fn_X`) before matching, and they only fall back to the leaf identifier for unqualified queries so exact qualified lookups do not widen to sibling schemas that merely share the same leaf name. Exact SQL definition matching also preserves segment count, so a quoted single identifier that contains a dot (`"sales.fn_Target"`) does not collide with a real qualified name (`sales.fn_Target`). SQL exact graph leaf fallback also stays on the Unicode folded exact path, and both quoted qualified and unqualified Unicode exact definition lookups now use the folded normalized path, so queries such as `dbo.Äpfel` / `dbo.äpfel` and bare `Äpfel` / `äpfel` keep matching leaf call/reference rows such as `äpfel` plus stored definitions such as `[dbo].[Äpfel]` or `dbo.Äpfel` instead of silently degrading to ASCII-only `NOCASE`. Exact multi-name SQL `symbols --count` lookups also bind the folded leaf parameters on that same `_foldReady` path, so Unicode leaf query sets no longer fail with missing-parameter database errors. @@ -2765,7 +2769,13 @@ USER_GUIDEの[終了コード](USER_GUIDE.md#終了コード)セクションを - **AI向けの軽量検索スニペット** — `search --json` と MCP の `search` は、チャンク全文ではなく snippet range、match line、highlight、context count、`truncated_line_count`、`truncation_context` を持つ一致中心スニペットを返す。`truncation_context.char_counts` と `truncation_context.total_chars` はクランプされた各スニペット行の省略文字数を公開し、truncated な highlight も `truncated_char_counts` を持つ。`--snippet-lines` でペイロード量と文脈量のバランスを取れ、`--max-line-width`（CLI）/ `maxLineWidth`（MCP）は `find` / `references` / `excerpt` / `inspect` と同じ共有 `LineWidthFormatter.ClampLine` 契約で各スニペット行を最初のマッチトークン周辺にクランプするため、minified / transpiled / 生成された 1 行ファイル内の 1 ヒットで数百 KB を返さなくなる。クランプされた行はスニペットに `...(+N)...` マーカーが入り、`highlights[].truncated` と `highlights[].original_line_length` で AI クライアントがクランプを検出できる。 - **初動向けの repo map** — `map` は、インデックス済みデータから言語、モジュール、主要ファイル、ホットスポット、推定エントリポイントを集約し、AIクライアントが精密検索前に見るべき場所を決めやすくする。シンボル抽出が `Main` 系シンボルを出さない場合でも、既知のトップレベル実行ファイルへフォールバックして入口候補を補う。 - **信用判断のための鮮度メタデータ** — `status` はワークスペース全体の鮮度と git 状態を返す。`map` は `indexed_at` / `latest_modified` を絞り込み結果の鮮度として維持しつつ、`workspace_indexed_at` / `workspace_latest_modified` でワークスペース全体の鮮度も返す。`inspect` も同じワークスペース鮮度と git フィールドを返すため、シンボル中心の AI フローで `status` を別途呼ばずに済む。さらに `status` は `sql_graph_contract_ready` / `sql_graph_contract_degraded_reason`、`hotspot_family_ready` / `hotspot_family_degraded_reason` に加えて、forward-compatibility 監査 (`index_writer_version`、`index_newer_than_reader`、`index_newer_than_reader_reason`、詳細は「リーダー側の forward-compatibility 監査」を参照)、および fold-only remediation 用の `fold_ready_reason`、`degraded_reason`、`recommended_action`、`alternative_action` も返すため、AI クライアントは SQL graph/dependency/impact、duplicate-name hotspot family、Unicode `--exact` のどれが authoritative か、また DB が現在の binary より新しい `cdidx` で書かれていないかを最初に判断できる。現行の全体 scan 後は `unknown_extension_file_count` も返すため、未知拡張子で index 対象外になった件数を `status` から確認できる。これらの fold-only remediation field は、明示的な read-only `file:///...?...` DB URI から導出された場合でも、失敗する read-only URI をそのままコマンドへ埋め込まず、writable な filesystem path に正規化して返す。さらに `impact` / MCP `impact_analysis` に加えて、`inspect` / MCP `analyze_symbol`、`references` / `callers` / `callees`、`deps` / `unused` / `hotspots` 系も、SQL ベースの graph/dependency read が実際に結果へ関与したときだけ `sql_graph_contract_ready` / `sql_graph_contract_degraded_reason` を反映するため、stale な SQL 行が authoritative なヒットや 0 件応答に見えてしまうのを防ぎつつ、mixed-language index 内の純粋な非SQL結果を誤って degraded 扱いしない。`files` はファイルごとの checksum・modified・indexed timestamp を返す。古いDBに対する file 列の移行は可能なら自動で行い、その場移行できない場合でも読み取り経路がクラッシュしないようにする。CLI と MCP の 0 件 JSON レスポンスは `indexed_file_count`、`indexed_at`、`freshness_available` を含む。`freshness_available=true` で `indexed_at:null` なら空インデックス、`freshness_available=false` なら legacy/read-only DB で鮮度 timestamp を取得できず、理由は `freshness_degraded_reason` に入る。**HEAD 起点の stale 検知**: `cdidx index` の full scan が成功するたびに、現時点の `git HEAD` を `codeindex_meta` に stamp し、後続実行で workspace HEAD と比較できるようにする。`--rebuild` 指定なしに両者が異なる場合、CLI は `cdidx index --rebuild` を勧める `head_changed` 警告を表示し、`index --json` に `head_changed` / `prior_indexed_head_commit` / `current_head_commit` / `head_change_notice` を出力する。`status --check` も同じ比較を `workspace_check.head_changed` として公開し、差分時には `indexed_head_commit` / `workspace_head_commit` も併記するため、鮮度 gate ですでに `status --check` を通している AI クライアントは `git switch ` 後の既定の incremental scan を別クエリなしで拒否できる。`--commits` / `--files` の部分更新は意図的に記録 HEAD を維持し、次の full scan が worktree を再インデックスするまで stale 通知が継続する。非 Git workspace と HEAD を記録していない legacy DB は比較自体をスキップし、false-positive な警告を出さない。 -文書化された `status --json` trust contract は `fold_ready`、`fold_ready_reason`、`graph_table_available`、`issues_table_available`、`file_issues_data_current`、`migration_in_progress`、`sql_graph_contract_ready`、`sql_graph_contract_degraded_reason`、`hotspot_family_ready`、`hotspot_family_degraded_reason`、`csharp_symbol_name_ready`、`csharp_metadata_target_ready`、`csharp_metadata_target_degraded_reason`、`indexed_head_commit`、`worktree_head_changed`、`indexed_head_sha`、`indexed_head_branch`、`indexed_head_timestamp`、`commits_ahead_of_indexed_head`、`index_writer_version`、`index_newer_than_reader`、`index_newer_than_reader_reason`、`unknown_extension_file_count`、`path_case_sensitive`、`stale_after_seconds`、`index_age_seconds`、remediation field の `degraded_root_cause`、`degraded_reason`、`recommended_action`、`alternative_action`、`readiness_degradations`、および MCP 専用の `mcp_session` を対象にします。MCP `mcp_session` は永続化された DB 状態ではなく、セッション単位の診断情報で、`log_level`、`roots`、任意の `client_info`、任意の `client_capabilities` を含みます。この一覧は `README.md` と `AGENT_GUIDE.md` に同期してください。いずれかの必須 field がこれらの docs から漏れると `DocumentationStatusContractTests` が失敗します。 +`unknown_extension_files` は `unknown_extension_file_path_limit` 件までの未知拡張子 path sample で、`unknown_extension_files_truncated` は `unknown_extension_file_count` が sample 上限を超えたことを示します。 + +`extractors` は extractor plugin と pattern config の runtime health で、読み込み済み plugin assembly / pattern 件数、symbol/reference extractor 件数、skip されたファイル数、上限付き diagnostics list を返します。 + +`hooks[]` は `callback_budget_ms` を含み、`CDIDX_HOOK_CALLBACK_BUDGET_MS`（既定値: 5000 ms）で強制される post-extraction callback 予算を反映します。hook は結果反映前の scratch copy 上で実行されるため、timeout した callback の変更は破棄されます。 + +文書化された `status --json` trust contract は `fold_ready`、`fold_ready_reason`、`graph_table_available`、`issues_table_available`、`file_issues_data_current`、`migration_in_progress`、`sql_graph_contract_ready`、`sql_graph_contract_degraded_reason`、`hotspot_family_ready`、`hotspot_family_degraded_reason`、`csharp_symbol_name_ready`、`csharp_metadata_target_ready`、`csharp_metadata_target_degraded_reason`、`indexed_head_commit`、`worktree_head_changed`、`indexed_head_sha`、`indexed_head_branch`、`indexed_head_timestamp`、`commits_ahead_of_indexed_head`、`index_writer_version`、`index_newer_than_reader`、`index_newer_than_reader_reason`、`unknown_extension_file_count`、`unknown_extension_files`、`unknown_extension_files_truncated`、`unknown_extension_file_path_limit`、`extractors`、`path_case_sensitive`、`stale_after_seconds`、`index_age_seconds`、remediation field の `degraded_root_cause`、`degraded_reason`、`recommended_action`、`alternative_action`、`readiness_degradations`、および MCP 専用の `mcp_session` を対象にします。MCP `mcp_session` は永続化された DB 状態ではなく、セッション単位の診断情報で、`log_level`、`roots`、任意の `client_info`、任意の `client_capabilities` を含みます。この一覧は `README.md` と `AGENT_GUIDE.md` に同期してください。いずれかの必須 field がこれらの docs から漏れると `DocumentationStatusContractTests` が失敗します。 - **再解析不要の folded-key アップグレード** — `backfill-fold` と MCP `backfill_fold` は、既存 DB 行から `name_folded` / `*_folded` を直接再計算し、必要な folded 値に NULL が残っていないことを検証してから `FoldReadyFlag` を stamp する。これにより、pre-#86 DB から AI クライアントやユーザーが低コストで Unicode `--exact` へ上がれる。さらに `fold_key_version` が未記録または不一致なら全 folded 行を再生成するため、将来の `NameFold.Version` 変更後に古い key を silent に再 stamp してしまうことも防ぐ。 - **まとめて取るシンボル分析** — `inspect` と MCP の `analyze_symbol` は、定義、近傍シンボル、参照、caller、callee、ファイルメタデータ、ワークスペース信頼メタデータ、graph 対応メタデータを1回で返し、AIクライアントが一般的なシンボル調査を少ない往復で終えやすくする。 - **言語考慮の参照抽出** — `references`、`callers`、`callees` は、正規表現ベースの call/reference 抽出が意味を持つ言語だけに対してインデックス化された参照テーブルで支える。未対応言語では、低信頼な疑似グラフ結果を返す代わりにテキスト検索へ戻る前提で設計する。**nested generic 呼び出し**: `new Dictionary>()` のような C#/Java のコンストラクタ呼び出しと、`Helper.DoWork>()` のような C# generic method call は、平坦な regex fast-path で `>>` を釣り合わせられなくても depth-aware fallback scanner で拾い直し、外側 target を参照テーブルへ残す。**コンストラクタ連鎖呼び出し**: C# の `: this(...)` / `: base(...)` イニシャライザと、Java のコンストラクタ本体冒頭文 `this(...)` / `super(...)` は、汎用 call regex とは別に検出し、呼び先が実際のコンストラクタとなるように書き換える（`this` は外側の class/record、`base` / `super` は外側クラスのシグネチャから解析した基底型）。C# のクロス行イニシャライザは外側クラスではなく、そのコンストラクタに紐付ける。基底型の解析は generic 引数、record のプライマリコンストラクタ引数、`where` 制約、`global::` やドット付きの namespace 修飾を剥がす。Java の `super.method()` は通常のメソッド呼び出しのまま扱う。**型位置の依存エッジ**: C#/Java の継承リスト、宣言型、generic 制約、`throws`、`is` / `as` / `instanceof`、および実際の C# XML doc `///` `cref` は `type_reference` 行として索引し、既定の `callers` / `callees` が見せる動的 call graph を汚さずに、`references` / `impact` から compile-time rename 依存を辿れるようにする。**SQL qualified-name alignment**: SQL の graph/dependency reader は、各 reference 行の source-line context、記録済み call 列位置、enclosing container から SQL 参照名を復元して定義と照合するため、qualified な `references` / `callers` / `impact` query は exact / non-exact を問わず sibling schema へ widen しない。source 側が genuinely unqualified な場合にだけ bare leaf fallback を許可するので、qualified call を含む `deps` / `unused` / `hotspots` も schema 単位で整合し、`EXEC dbo.fn_Target; EXEC sales.fn_Target;` のような同一行 multi-call も二重計上しない。列位置が記録されている row は、その列に qualified token が見つからなければ whole-line の別 qualified token へ昇格させないため、行末コメント・文字列リテラル・後続の別 call が先頭の unqualified edge を横取りすることもない。qualified な `callees` query でも caller query 自体が unqualified なとき以外は leaf fallback を無効化したため、`callees sales.Caller` が `dbo.Caller` へ広がらない。SQL extractor は qualified-name の `.` 前後空白も許容し、definition 系 reader は quoted qualified SQL name (`[dbo].[fn_X]` → `dbo.fn_X`) を正規化してから照合する。さらに exact SQL 定義照合は segment 数を保持し、SQL の exact graph leaf fallback は Unicode folded exact path を維持する。SQL CTE 本体内の source 行は raw `cte_body_reference` kind を使うため、`references --kind cte_body_reference` で anchor/recursive member 内部を outer query の table reference と区別できる。そのため、quoted single identifier の衝突や Unicode exact lookup の ASCII-only `NOCASE` 退行も防ぐ。 diff --git a/README.md b/README.md index 3208e3dc05..f0f0be0957 100644 --- a/README.md +++ b/README.md @@ -193,16 +193,23 @@ The documented `status --json` trust contract covers these fields: language_readinesscsharp_symbol_name_readycsharp_metadata_target_readycsharp_metadata_target_degraded_reason indexed_head_commitworktree_head_changedindexed_head_shaindexed_head_branch indexed_head_timestampcommits_ahead_of_indexed_headindex_writer_versionindex_newer_than_reader -index_newer_than_reader_reasonunknown_extension_file_countpath_case_sensitivedata_dir -data_dir_sourcedata_dir_modemac_profiledb_size_bytes -wal_size_bytesdb_pragma_settingssymbols_by_languageprocess -last_index_runhooksstale_after_secondsindex_age_seconds -degraded_reasonrecommended_actionalternative_actionmcp_session +index_newer_than_reader_reasonunknown_extension_file_countunknown_extension_filesunknown_extension_files_truncated +unknown_extension_file_path_limitpath_case_sensitivedata_dirdata_dir_source +data_dir_modemac_profiledb_size_byteswal_size_bytes +db_pragma_settingssymbols_by_languageprocesslast_index_run +hooksstale_after_secondsindex_age_secondsdegraded_reason +recommended_actionalternative_actionmcp_sessionextractors When any readiness field is degraded, `degraded_root_cause` identifies the primary stable code and `readiness_degradations[]` lists every degraded field with `root_cause`, human `degraded_reason`, `recommended_action`, and `alternative_action`. `issues_table_available` reports physical table presence; use `file_issues_data_current` to decide whether `file_issues` rows are current for the index generation. +After a current full-repository scan, `unknown_extension_file_count` reports how many skipped files had unmapped non-empty extensions, while `unknown_extension_files` lists up to `unknown_extension_file_path_limit` paths and `unknown_extension_files_truncated` marks when more paths exist. + +`extractors` reports runtime extractor plugin and pattern-config diagnostics, including loaded counts, skipped file counts, and a bounded diagnostics list for load failures. + +`hooks[]` includes `callback_budget_ms`. `CDIDX_HOOK_CALLBACK_BUDGET_MS` bounds each post-extraction hook callback in milliseconds (default: 5000); callbacks that exceed the budget emit index warnings, drop timed-out mutations, and disable that hook for the current index run. + For MCP `status`, `mcp_session` is session-scoped diagnostic data rather than persisted index state. It includes `log_level`, `roots`, optional `client_info`, and optional `client_capabilities`. `process` is captured at status-call time and includes heap, GC collection, and working-set counters. `last_index_run` is persisted by successful CLI and MCP index runs with the run mode, duration, file counts, byte count, row-change counts, and optional peak-memory summary from CLI `--memory-trace`. @@ -447,16 +454,23 @@ upgrade / downgrade 後はインストール済み補完 script を再生成し language_readinesscsharp_symbol_name_readycsharp_metadata_target_readycsharp_metadata_target_degraded_reason indexed_head_commitworktree_head_changedindexed_head_shaindexed_head_branch indexed_head_timestampcommits_ahead_of_indexed_headindex_writer_versionindex_newer_than_reader -index_newer_than_reader_reasonunknown_extension_file_countpath_case_sensitivedata_dir -data_dir_sourcedata_dir_modemac_profiledb_size_bytes -wal_size_bytesdb_pragma_settingssymbols_by_languageprocess -last_index_runhooksstale_after_secondsindex_age_seconds -degraded_reasonrecommended_actionalternative_actionmcp_session +index_newer_than_reader_reasonunknown_extension_file_countunknown_extension_filesunknown_extension_files_truncated +unknown_extension_file_path_limitpath_case_sensitivedata_dirdata_dir_source +data_dir_modemac_profiledb_size_byteswal_size_bytes +db_pragma_settingssymbols_by_languageprocesslast_index_run +hooksstale_after_secondsindex_age_secondsdegraded_reason +recommended_actionalternative_actionmcp_sessionextractors readiness field のいずれかが degraded の場合、`degraded_root_cause` は primary の安定コードを示し、`readiness_degradations[]` は degraded な各 field と `root_cause`、人間向け `degraded_reason`、`recommended_action`、`alternative_action` を列挙します。`issues_table_available` は物理 table の有無を表し、`file_issues` 行が現在の index generation に対して current かどうかは `file_issues_data_current` を使って判定します。 +現行の全体 scan 後、`unknown_extension_file_count` は未知の非空拡張子で skip された件数を返し、`unknown_extension_files` は `unknown_extension_file_path_limit` 件までの path sample、`unknown_extension_files_truncated` は sample より多くの path があることを示します。 + +`extractors` は extractor plugin と pattern config の runtime 診断で、読み込み済み件数、skip されたファイル数、読み込み失敗の上限付き diagnostics list を含みます。 + +`hooks[]` は `callback_budget_ms` を含みます。`CDIDX_HOOK_CALLBACK_BUDGET_MS` は post-extraction hook callback ごとの上限ミリ秒を指定します（既定値: 5000）。上限を超えた callback は index warning を出し、timeout した変更を捨て、その index run 中は該当 hook を無効化します。 + MCP `status` の `mcp_session` は永続化された index 状態ではなく、セッション単位の診断情報です。`log_level`、`roots`、任意の `client_info`、任意の `client_capabilities` を含みます。 `process` は status 呼び出し時点の heap、GC collection、working-set counters です。`last_index_run` は成功した CLI / MCP index 実行が永続化し、run mode、duration、file counts、byte count、row-change counts、CLI `--memory-trace` 由来の任意の peak-memory summary を含みます。 diff --git a/changelog.d/unreleased/2822.fixed.md b/changelog.d/unreleased/2822.fixed.md new file mode 100644 index 0000000000..13aa2c2cf6 --- /dev/null +++ b/changelog.d/unreleased/2822.fixed.md @@ -0,0 +1,18 @@ +--- +category: fixed +issues: + - 2822 +affected: + - src/CodeIndex/Database/DbContext.cs + - src/CodeIndex/Database/DbReader.FilesStatus.cs + - src/CodeIndex/Database/DbWriter.cs + - src/CodeIndex/Models/QueryResults.cs +--- + +## English + +- **Status now exposes unknown-extension path samples (#2822)** — `status --json` and MCP `status` now include `unknown_extension_files`, `unknown_extension_files_truncated`, and `unknown_extension_file_path_limit` alongside the existing count so users can identify skipped files that need a language mapping or ignore rule. + +## 日本語 + +- **status が未知拡張子の path sample を返すようになりました (#2822)** — `status --json` と MCP `status` は既存の件数に加えて `unknown_extension_files`、`unknown_extension_files_truncated`、`unknown_extension_file_path_limit` を返すため、言語マッピングや ignore rule が必要な skip ファイルを特定できます。 diff --git a/changelog.d/unreleased/2865.fixed.md b/changelog.d/unreleased/2865.fixed.md new file mode 100644 index 0000000000..aac8b97cf8 --- /dev/null +++ b/changelog.d/unreleased/2865.fixed.md @@ -0,0 +1,18 @@ +--- +category: fixed +issues: + - 2865 +affected: + - src/CodeIndex/Indexer/Extensibility/ExtractorPluginRegistry.cs + - src/CodeIndex/Models/QueryResults.cs + - src/CodeIndex/Cli/QueryCommandRunner.cs + - src/CodeIndex/Mcp/McpToolHandlers.cs +--- + +## English + +- **Status now reports extractor plugin and pattern diagnostics (#2865)** — `status --json` and MCP `status` include an `extractors` object with plugin/pattern load counts, skipped file counts, and a bounded diagnostic summary for incompatible or malformed extractor files. + +## 日本語 + +- **status が extractor plugin / pattern 診断を返すようになりました (#2865)** — `status --json` と MCP `status` は `extractors` object を含み、plugin / pattern の読み込み件数、skip されたファイル数、互換性のないまたは壊れた extractor ファイルの上限付き diagnostics summary を返します。 diff --git a/changelog.d/unreleased/2877.fixed.md b/changelog.d/unreleased/2877.fixed.md new file mode 100644 index 0000000000..6f87104b31 --- /dev/null +++ b/changelog.d/unreleased/2877.fixed.md @@ -0,0 +1,18 @@ +--- +category: fixed +issues: + - 2877 +affected: + - src/CodeIndex/Indexer/Hooks/PostExtractionHooks.cs + - src/CodeIndex/Models/QueryResults.cs + - src/CodeIndex/Cli/QueryCommandRunner.cs + - src/CodeIndex/Mcp/McpToolHandlers.cs +--- + +## English + +- **Post-extraction hooks now have a bounded callback budget (#2877)** — `cdidx index` now enforces `CDIDX_HOOK_CALLBACK_BUDGET_MS` for each post-extraction hook callback, drops timed-out mutations, disables the timed-out hook for the current run, and reports `callback_budget_ms` in `status --json` and MCP `status`. + +## 日本語 + +- **post-extraction hook callback に上限時間を設けました (#2877)** — `cdidx index` は post-extraction hook callback ごとに `CDIDX_HOOK_CALLBACK_BUDGET_MS` を強制し、timeout した変更を破棄し、その run 中は timeout した hook を無効化し、`status --json` と MCP `status` で `callback_budget_ms` を報告します。 diff --git a/src/CodeIndex/Cli/DiffCommandRunner.cs b/src/CodeIndex/Cli/DiffCommandRunner.cs index de988006fc..672d65531e 100644 --- a/src/CodeIndex/Cli/DiffCommandRunner.cs +++ b/src/CodeIndex/Cli/DiffCommandRunner.cs @@ -228,6 +228,9 @@ OR key LIKE 'symbol_extractor_version_%' OR key LIKE 'metadata_target_version_%' OR key = 'workspace_path_case_sensitive' OR key = 'unknown_extension_file_count' + OR key = 'unknown_extension_file_paths_json' + OR key = 'unknown_extension_files_truncated' + OR key = 'unknown_extension_file_path_limit' OR key = 'cdidx_writer_version' ORDER BY key, diff --git a/src/CodeIndex/Cli/IndexCommandRunner.FullScan.cs b/src/CodeIndex/Cli/IndexCommandRunner.FullScan.cs index 01346ef5c2..05cd2b2895 100644 --- a/src/CodeIndex/Cli/IndexCommandRunner.FullScan.cs +++ b/src/CodeIndex/Cli/IndexCommandRunner.FullScan.cs @@ -1321,9 +1321,7 @@ void StopJsonHeartbeat() // metadata ahead of the success markers. // no-op full-scan の explicit DB root backfill は readiness stamp 後に限定する。 WriteProjectRootOnce(); - writer.SetMeta( - DbContext.UnknownExtensionFileCountMetaKey, - scanResult.UnknownExtensionFiles.Count.ToString(System.Globalization.CultureInfo.InvariantCulture)); + writer.WriteUnknownExtensionFileMetadata(scanResult.UnknownExtensionFiles); // Persist the current HEAD only after the run is fully successful (errors == 0). // We deliberately only stamp on full scans (rebuild or default incremental). Update // mode (`--commits` / `--files`) leaves the captured HEAD untouched so the next diff --git a/src/CodeIndex/Cli/JsonOutputContracts.cs b/src/CodeIndex/Cli/JsonOutputContracts.cs index 9745644871..b20db86ef1 100644 --- a/src/CodeIndex/Cli/JsonOutputContracts.cs +++ b/src/CodeIndex/Cli/JsonOutputContracts.cs @@ -2,6 +2,7 @@ using System.Text.Json; using System.Text.Json.Serialization; using CodeIndex.Database; +using CodeIndex.Indexer.Extensibility; using CodeIndex.Models; namespace CodeIndex.Cli; @@ -434,6 +435,8 @@ internal sealed record VersionInfoJsonResult( [JsonSerializable(typeof(SearchResult))] [JsonSerializable(typeof(SearchTermOccurrence))] [JsonSerializable(typeof(SearchTruncationContext))] +[JsonSerializable(typeof(ExtractorRegistryDiagnostic))] +[JsonSerializable(typeof(ExtractorRegistryStatus))] [JsonSerializable(typeof(StatusResult))] [JsonSerializable(typeof(StatusReadinessDegradation))] [JsonSerializable(typeof(StatusDbPragmaSettings))] diff --git a/src/CodeIndex/Cli/QueryCommandRunner.cs b/src/CodeIndex/Cli/QueryCommandRunner.cs index 87e3e5219c..fb7c493775 100644 --- a/src/CodeIndex/Cli/QueryCommandRunner.cs +++ b/src/CodeIndex/Cli/QueryCommandRunner.cs @@ -5,6 +5,7 @@ using System.Text.Json.Serialization.Metadata; using CodeIndex.Database; using CodeIndex.Indexer; +using CodeIndex.Indexer.Extensibility; using CodeIndex.Indexer.Hooks; using CodeIndex.Models; using Microsoft.Data.Sqlite; @@ -2910,7 +2911,10 @@ public static int RunStatus(string[] cmdArgs, JsonSerializerOptions jsonOptions, // Attach runtime metadata / ランタイムメタデータを付加 status.SymbolKinds = reader.GetSymbolKindCounts(); status.GraphSupportedLanguages = ReferenceExtractor.GetSupportedLanguages().OrderBy(l => l).ToList(); - var postExtractionHooks = PostExtractionHookRunner.DiscoverDefault().Hooks; + ExtractorPluginRegistry.LoadPatternConfigsForProjectRoot(status.ProjectRoot); + status.Extractors = ExtractorPluginRegistry.GetStatusSnapshot(); + using var postExtractionHookRunner = PostExtractionHookRunner.DiscoverDefault(); + var postExtractionHooks = postExtractionHookRunner.Hooks; if (postExtractionHooks.Count > 0) { status.Hooks = postExtractionHooks @@ -2919,6 +2923,7 @@ public static int RunStatus(string[] cmdArgs, JsonSerializerOptions jsonOptions, Name = hook.Name, AssemblyPath = hook.AssemblyPath, TypeName = hook.TypeName, + CallbackBudgetMs = (long)Math.Round(postExtractionHookRunner.CallbackBudget.TotalMilliseconds, MidpointRounding.AwayFromZero), }) .ToList(); } diff --git a/src/CodeIndex/Database/DbContext.cs b/src/CodeIndex/Database/DbContext.cs index 6a7395e6e0..97f0e80806 100644 --- a/src/CodeIndex/Database/DbContext.cs +++ b/src/CodeIndex/Database/DbContext.cs @@ -1273,6 +1273,10 @@ private static void RegisterConnectionFunctionsWithRetry( // ファイル数。index 済み件数ではなく scan coverage の信号であり、現行 index が stamp // するまでは reader 側で省略する。 public const string UnknownExtensionFileCountMetaKey = "unknown_extension_file_count"; + public const string UnknownExtensionFilePathsMetaKey = "unknown_extension_file_paths_json"; + public const string UnknownExtensionFilesTruncatedMetaKey = "unknown_extension_files_truncated"; + public const string UnknownExtensionFilePathLimitMetaKey = "unknown_extension_file_path_limit"; + public const int UnknownExtensionFilePathSampleLimit = 50; public const string BatchInProgressMetaKey = "batch_in_progress"; // Issue #1546: case-sensitivity of the workspace filesystem the most recent successful // index ran on, persisted as the string "true" / "false". Resolved via the probe in diff --git a/src/CodeIndex/Database/DbReader.FilesStatus.cs b/src/CodeIndex/Database/DbReader.FilesStatus.cs index d77e6bdf9c..78338e5f35 100644 --- a/src/CodeIndex/Database/DbReader.FilesStatus.cs +++ b/src/CodeIndex/Database/DbReader.FilesStatus.cs @@ -2,6 +2,7 @@ using Microsoft.Data.Sqlite; using System.Globalization; using System.Text; +using System.Text.Json; using System.Text.RegularExpressions; namespace CodeIndex.Database; @@ -541,6 +542,15 @@ GROUP BY COALESCE(f.lang, 'unknown'), s.kind var indexedHeadBranch = TryGetMetaStringInternal(DbContext.IndexedHeadBranchMetaKey); var indexedHeadTimestamp = ParseMetaDateTime(TryGetMetaStringInternal(DbContext.IndexedHeadTimestampMetaKey)); var unknownExtensionFileCount = ParseMetaLong(TryGetMetaStringInternal(DbContext.UnknownExtensionFileCountMetaKey)); + var unknownExtensionFiles = ParseMetaStringList(TryGetMetaStringInternal(DbContext.UnknownExtensionFilePathsMetaKey)); + var unknownExtensionFilesTruncated = ParseMetaBool(TryGetMetaStringInternal(DbContext.UnknownExtensionFilesTruncatedMetaKey)); + var unknownExtensionFilePathLimit = ParseMetaLong(TryGetMetaStringInternal(DbContext.UnknownExtensionFilePathLimitMetaKey)); + if (unknownExtensionFiles != null) + { + unknownExtensionFilesTruncated ??= unknownExtensionFileCount.HasValue + && unknownExtensionFileCount.Value > unknownExtensionFiles.Count; + unknownExtensionFilePathLimit ??= unknownExtensionFiles.Count; + } // #1546: workspace case-sensitivity stamp. Read inside the SHARED snapshot for // consistency with the other freshness signals; missing on legacy DBs. // #1546: case-sensitivity stamp も同 snapshot で読む。stamp 無し旧 DB は null。 @@ -561,6 +571,9 @@ GROUP BY COALESCE(f.lang, 'unknown'), s.kind Symbols = symbols, References = references, UnknownExtensionFileCount = unknownExtensionFileCount, + UnknownExtensionFiles = unknownExtensionFiles, + UnknownExtensionFilesTruncated = unknownExtensionFilesTruncated, + UnknownExtensionFilePathLimit = unknownExtensionFilePathLimit, IndexedAt = freshness.IndexedAt, LatestModified = freshness.LatestModified, IndexedHeadSha = indexedHeadSha, @@ -812,6 +825,26 @@ public FreshnessHintResult GetFreshnessHint() ? null : value; + private static List? ParseMetaStringList(string? raw) + { + if (string.IsNullOrWhiteSpace(raw)) + return null; + + try + { + var values = JsonSerializer.Deserialize>(raw); + return values == null + ? null + : values + .Where(static value => !string.IsNullOrWhiteSpace(value)) + .ToList(); + } + catch (JsonException) + { + return null; + } + } + private static long? ParseMetaLong(string? raw) => string.IsNullOrWhiteSpace(raw) || !long.TryParse( diff --git a/src/CodeIndex/Database/DbWriter.cs b/src/CodeIndex/Database/DbWriter.cs index 499c3921e9..1b1c50487d 100644 --- a/src/CodeIndex/Database/DbWriter.cs +++ b/src/CodeIndex/Database/DbWriter.cs @@ -3,6 +3,7 @@ using CodeIndex.Indexer; using CodeIndex.Models; using System.Text; +using System.Text.Json; namespace CodeIndex.Database; @@ -2068,6 +2069,33 @@ public void WriteCdidxWriterVersion(string version) SetMeta(DbContext.CdidxWriterVersionMetaKey, version); } + /// + /// Stamp unknown-extension scan coverage from the latest successful full-worktree scan. + /// Stores the total count plus a bounded path sample so status callers can identify the + /// first files that need a language mapping or ignore rule without unbounded metadata. + /// 未知拡張子の scan coverage を保存する。件数と上限付き path sample を status で返す。 + /// + public void WriteUnknownExtensionFileMetadata(IReadOnlyList paths) + { + ArgumentNullException.ThrowIfNull(paths); + + var sample = paths + .Take(DbContext.UnknownExtensionFilePathSampleLimit) + .ToArray(); + SetMeta( + DbContext.UnknownExtensionFileCountMetaKey, + paths.Count.ToString(System.Globalization.CultureInfo.InvariantCulture)); + SetMeta( + DbContext.UnknownExtensionFilePathsMetaKey, + JsonSerializer.Serialize(sample)); + SetMeta( + DbContext.UnknownExtensionFilesTruncatedMetaKey, + (paths.Count > sample.Length).ToString(System.Globalization.CultureInfo.InvariantCulture)); + SetMeta( + DbContext.UnknownExtensionFilePathLimitMetaKey, + DbContext.UnknownExtensionFilePathSampleLimit.ToString(System.Globalization.CultureInfo.InvariantCulture)); + } + /// /// Demote metadata-target trust for every known language. Called at the start of any /// indexing run that may leave the resolver output partially stale so readers fall back diff --git a/src/CodeIndex/Indexer/Extensibility/ExtractorPluginRegistry.cs b/src/CodeIndex/Indexer/Extensibility/ExtractorPluginRegistry.cs index 023c53aadd..b0b179e911 100644 --- a/src/CodeIndex/Indexer/Extensibility/ExtractorPluginRegistry.cs +++ b/src/CodeIndex/Indexer/Extensibility/ExtractorPluginRegistry.cs @@ -1,5 +1,6 @@ using System.Reflection; using System.Runtime.Loader; +using System.Text.Json.Serialization; using System.Text.RegularExpressions; namespace CodeIndex.Indexer.Extensibility; @@ -12,6 +13,12 @@ public static class ExtractorPluginRegistry private static readonly Dictionary SymbolExtractors = new(StringComparer.Ordinal); private static readonly Dictionary ReferenceExtractors = new(StringComparer.Ordinal); private static readonly HashSet LoadedPatternConfigPaths = new(StringComparer.OrdinalIgnoreCase); + private static readonly List Diagnostics = []; + private const int DiagnosticLimit = 20; + private static int pluginAssemblyCount; + private static int patternConfigCount; + private static int skippedFileCount; + private static int diagnosticTotalCount; private static bool pluginsLoaded; public static IReadOnlyCollection SymbolLanguages @@ -63,6 +70,26 @@ public static bool TryGetReferenceExtractor(string language, out IReferenceExtra return ReferenceExtractors.TryGetValue(language, out extractor!); } + internal static ExtractorRegistryStatus GetStatusSnapshot() + { + EnsurePluginsLoaded(); + lock (Gate) + { + return new ExtractorRegistryStatus + { + PluginAssemblyCount = pluginAssemblyCount, + PatternConfigCount = patternConfigCount, + SymbolExtractorCount = SymbolExtractors.Count, + ReferenceExtractorCount = ReferenceExtractors.Count, + SkippedFileCount = skippedFileCount, + DiagnosticCount = diagnosticTotalCount, + DiagnosticLimit = DiagnosticLimit, + DiagnosticsTruncated = diagnosticTotalCount > Diagnostics.Count, + Diagnostics = Diagnostics.Count == 0 ? null : Diagnostics.ToList(), + }; + } + } + public static void Register(ISymbolExtractor extractor) { ArgumentNullException.ThrowIfNull(extractor); @@ -86,6 +113,11 @@ internal static void ResetForTests() SymbolExtractors.Clear(); ReferenceExtractors.Clear(); LoadedPatternConfigPaths.Clear(); + Diagnostics.Clear(); + pluginAssemblyCount = 0; + patternConfigCount = 0; + skippedFileCount = 0; + diagnosticTotalCount = 0; pluginsLoaded = true; } } @@ -97,6 +129,11 @@ internal static void ReloadForTests() SymbolExtractors.Clear(); ReferenceExtractors.Clear(); LoadedPatternConfigPaths.Clear(); + Diagnostics.Clear(); + pluginAssemblyCount = 0; + patternConfigCount = 0; + skippedFileCount = 0; + diagnosticTotalCount = 0; pluginsLoaded = false; } } @@ -197,12 +234,13 @@ private static IEnumerable EnumeratePatternDirectories(string workspaceR private static void TryLoadPatternConfig(string path) { + var fullPath = path; try { - path = Path.GetFullPath(path); + fullPath = Path.GetFullPath(path); lock (Gate) { - if (!LoadedPatternConfigPaths.Add(path)) + if (!LoadedPatternConfigPaths.Add(fullPath)) return; } @@ -238,11 +276,31 @@ private static void TryLoadPatternConfig(string path) } if (language.Length > 0 && patterns.Count > 0) + { Register(new ConfiguredSymbolExtractor(language, extensions, patterns)); + lock (Gate) + patternConfigCount++; + } + else + { + RecordDiagnostic( + "pattern", + fullPath, + typeName: null, + severity: "skipped", + "Pattern config skipped: missing language or regex patterns.", + countsAsSkippedFile: true); + } } - catch + catch (Exception ex) { - // Pattern sidecars are best-effort like binary plugins. + RecordDiagnostic( + "pattern", + fullPath, + typeName: null, + severity: "error", + $"Failed to load pattern config: {ex.Message}", + countsAsSkippedFile: true); } } @@ -258,30 +316,59 @@ private static bool TryReadScalar(string line, string key, out string value) private static void TryLoadPlugin(string pluginPath) { + var fullPath = pluginPath; try { - var assembly = AssemblyLoadContext.Default.LoadFromAssemblyPath(Path.GetFullPath(pluginPath)); + fullPath = Path.GetFullPath(pluginPath); + var assembly = AssemblyLoadContext.Default.LoadFromAssemblyPath(fullPath); var attribute = assembly.GetCustomAttribute(); - if (attribute == null - || attribute.MinApiVersion > CurrentApiVersion + if (attribute == null) + { + RecordDiagnostic( + "plugin", + fullPath, + typeName: null, + severity: "skipped", + "Plugin assembly skipped: missing CdidxPluginAttribute.", + countsAsSkippedFile: true); + return; + } + + if (attribute.MinApiVersion > CurrentApiVersion || attribute.MaxApiVersion < CurrentApiVersion) { + RecordDiagnostic( + "plugin", + fullPath, + typeName: null, + severity: "skipped", + $"Plugin assembly skipped: API range {attribute.MinApiVersion}-{attribute.MaxApiVersion} does not include {CurrentApiVersion}.", + countsAsSkippedFile: true); return; } + lock (Gate) + pluginAssemblyCount++; + foreach (var type in assembly.GetTypes()) { if (type is { IsAbstract: false, IsInterface: false } && type.GetConstructor(Type.EmptyTypes) != null) - TryRegisterPluginType(type); + TryRegisterPluginType(type, fullPath); } } - catch + catch (Exception ex) { - // Plugin loading is best-effort so an incompatible DLL cannot prevent indexing. + RecordDiagnostic( + "plugin", + fullPath, + typeName: null, + severity: "error", + $"Failed to load plugin assembly: {ex.Message}", + countsAsSkippedFile: true); } } - private static void TryRegisterPluginType(Type type) + private static void TryRegisterPluginType(Type type, string pluginPath) { try { @@ -297,9 +384,33 @@ private static void TryRegisterPluginType(Type type) Register(referenceExtractor); } } - catch + catch (Exception ex) + { + RecordDiagnostic( + "plugin_type", + pluginPath, + type.FullName, + severity: "error", + $"Failed to instantiate plugin type: {ex.Message}", + countsAsSkippedFile: false); + } + } + + private static void RecordDiagnostic( + string kind, + string path, + string? typeName, + string severity, + string message, + bool countsAsSkippedFile) + { + lock (Gate) { - // Ignore broken plugin types and continue loading the rest of the assembly. + diagnosticTotalCount++; + if (countsAsSkippedFile) + skippedFileCount++; + if (Diagnostics.Count < DiagnosticLimit) + Diagnostics.Add(new ExtractorRegistryDiagnostic(kind, path, typeName, severity, message)); } } @@ -336,3 +447,32 @@ private static string NormalizePluginLanguage(string language) return extension.StartsWith(".", StringComparison.Ordinal) ? extension : "." + extension; } } + +public sealed class ExtractorRegistryStatus +{ + [JsonPropertyName("plugin_assembly_count")] + public int PluginAssemblyCount { get; init; } + [JsonPropertyName("pattern_config_count")] + public int PatternConfigCount { get; init; } + [JsonPropertyName("symbol_extractor_count")] + public int SymbolExtractorCount { get; init; } + [JsonPropertyName("reference_extractor_count")] + public int ReferenceExtractorCount { get; init; } + [JsonPropertyName("skipped_file_count")] + public int SkippedFileCount { get; init; } + [JsonPropertyName("diagnostic_count")] + public int DiagnosticCount { get; init; } + [JsonPropertyName("diagnostic_limit")] + public int DiagnosticLimit { get; init; } + [JsonPropertyName("diagnostics_truncated")] + public bool DiagnosticsTruncated { get; init; } + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public List? Diagnostics { get; init; } +} + +public sealed record ExtractorRegistryDiagnostic( + [property: JsonPropertyName("kind")] string Kind, + [property: JsonPropertyName("path")] string Path, + [property: JsonPropertyName("type_name")] string? TypeName, + [property: JsonPropertyName("severity")] string Severity, + [property: JsonPropertyName("message")] string Message); diff --git a/src/CodeIndex/Indexer/Hooks/PostExtractionHooks.cs b/src/CodeIndex/Indexer/Hooks/PostExtractionHooks.cs index ee5474d5e6..8786058ca0 100644 --- a/src/CodeIndex/Indexer/Hooks/PostExtractionHooks.cs +++ b/src/CodeIndex/Indexer/Hooks/PostExtractionHooks.cs @@ -1,4 +1,5 @@ using System.Collections.Concurrent; +using System.Diagnostics; using System.Reflection; using System.Runtime.Loader; using CodeIndex.Models; @@ -16,17 +17,29 @@ public sealed record FileContext(string ProjectRoot, string Path, string FullPat public sealed record PostExtractionHookInfo(string Name, string AssemblyPath, string TypeName); -public sealed record PostExtractionHookDiagnostic(string AssemblyPath, string? TypeName, string Message); +public sealed record PostExtractionHookDiagnostic( + string AssemblyPath, + string? TypeName, + string Message, + string? Callback = null, + long? DurationMs = null); public sealed class PostExtractionHookRunner : IDisposable { + public const string CallbackBudgetEnvironmentVariable = "CDIDX_HOOK_CALLBACK_BUDGET_MS"; + public static readonly TimeSpan DefaultCallbackBudget = TimeSpan.FromSeconds(5); + private readonly List hooks; private readonly ConcurrentQueue diagnostics = new(); + private readonly ConcurrentDictionary disabledHooks = new(StringComparer.Ordinal); + private readonly TimeSpan callbackBudget; private bool disposed; + internal static Func? CallbackBudgetForTesting { get; set; } - private PostExtractionHookRunner(List hooks) + private PostExtractionHookRunner(List hooks, TimeSpan callbackBudget) { this.hooks = hooks; + this.callbackBudget = callbackBudget; } public static PostExtractionHookRunner DiscoverDefault() @@ -35,7 +48,7 @@ public static PostExtractionHookRunner DiscoverDefault() public static PostExtractionHookRunner Discover(string? hooksDirectory) { var loaded = new List(); - var runner = new PostExtractionHookRunner(loaded); + var runner = new PostExtractionHookRunner(loaded, ResolveCallbackBudget()); if (string.IsNullOrWhiteSpace(hooksDirectory) || !Directory.Exists(hooksDirectory)) return runner; @@ -94,20 +107,21 @@ public static PostExtractionHookRunner Discover(string? hooksDirectory) public IReadOnlyList Diagnostics => diagnostics.ToList(); + public TimeSpan CallbackBudget => callbackBudget; + public void OnSymbolsExtracted(FileContext context, IList symbols) { ObjectDisposedException.ThrowIf(disposed, this); foreach (var hook in hooks) { - try + var workingSymbols = CloneSymbols(symbols); + if (InvokeHookWithBudget( + hook, + nameof(IPostExtractionHook.OnSymbolsExtracted), + () => hook.Instance.OnSymbolsExtracted(context, workingSymbols))) { - lock (hook.Instance) - hook.Instance.OnSymbolsExtracted(context, symbols); - } - catch (Exception ex) - { - diagnostics.Enqueue(new PostExtractionHookDiagnostic(hook.Info.AssemblyPath, hook.Info.TypeName, $"OnSymbolsExtracted failed: {ex.Message}")); + ReplaceList(symbols, workingSymbols); } } } @@ -117,17 +131,132 @@ public void OnReferencesExtracted(FileContext context, IList re ObjectDisposedException.ThrowIf(disposed, this); foreach (var hook in hooks) + { + var workingReferences = CloneReferences(references); + if (InvokeHookWithBudget( + hook, + nameof(IPostExtractionHook.OnReferencesExtracted), + () => hook.Instance.OnReferencesExtracted(context, workingReferences))) + { + ReplaceList(references, workingReferences); + } + } + } + + private bool InvokeHookWithBudget(LoadedPostExtractionHook hook, string callback, Action invoke) + { + if (disabledHooks.ContainsKey(hook.Info.TypeName)) + return false; + + var stopwatch = Stopwatch.StartNew(); + Exception? failure = null; + var task = Task.Run(() => { try { lock (hook.Instance) - hook.Instance.OnReferencesExtracted(context, references); + invoke(); } catch (Exception ex) { - diagnostics.Enqueue(new PostExtractionHookDiagnostic(hook.Info.AssemblyPath, hook.Info.TypeName, $"OnReferencesExtracted failed: {ex.Message}")); + failure = ex; } + }); + + if (!task.Wait(callbackBudget)) + { + stopwatch.Stop(); + disabledHooks.TryAdd(hook.Info.TypeName, 0); + diagnostics.Enqueue(new PostExtractionHookDiagnostic( + hook.Info.AssemblyPath, + hook.Info.TypeName, + $"{callback} exceeded the {callbackBudget.TotalMilliseconds:0} ms callback budget; hook disabled for this index run.", + callback, + stopwatch.ElapsedMilliseconds)); + return false; } + + stopwatch.Stop(); + if (failure != null) + { + diagnostics.Enqueue(new PostExtractionHookDiagnostic( + hook.Info.AssemblyPath, + hook.Info.TypeName, + $"{callback} failed: {failure.Message}", + callback, + stopwatch.ElapsedMilliseconds)); + } + + return true; + } + + private static TimeSpan ResolveCallbackBudget() + { + if (CallbackBudgetForTesting != null) + return NormalizeCallbackBudget(CallbackBudgetForTesting()); + + var raw = Environment.GetEnvironmentVariable(CallbackBudgetEnvironmentVariable); + return long.TryParse(raw, System.Globalization.NumberStyles.Integer, System.Globalization.CultureInfo.InvariantCulture, out var milliseconds) + ? NormalizeCallbackBudgetMilliseconds(milliseconds) + : DefaultCallbackBudget; + } + + private static TimeSpan NormalizeCallbackBudgetMilliseconds(long milliseconds) + => milliseconds <= 0 + ? DefaultCallbackBudget + : TimeSpan.FromMilliseconds(Math.Min(milliseconds, int.MaxValue)); + + private static TimeSpan NormalizeCallbackBudget(TimeSpan value) + => value <= TimeSpan.Zero + ? DefaultCallbackBudget + : TimeSpan.FromMilliseconds(Math.Min(value.TotalMilliseconds, int.MaxValue)); + + private static List CloneSymbols(IEnumerable symbols) + => symbols.Select(symbol => new SymbolRecord + { + Id = symbol.Id, + FileId = symbol.FileId, + Kind = symbol.Kind, + SubKind = symbol.SubKind, + Name = symbol.Name, + Line = symbol.Line, + StartLine = symbol.StartLine, + StartColumn = symbol.StartColumn, + EndLine = symbol.EndLine, + BodyStartLine = symbol.BodyStartLine, + BodyEndLine = symbol.BodyEndLine, + Signature = symbol.Signature, + ContainerKind = symbol.ContainerKind, + ContainerName = symbol.ContainerName, + ContainerQualifiedName = symbol.ContainerQualifiedName, + FamilyKey = symbol.FamilyKey, + Visibility = symbol.Visibility, + ReturnType = symbol.ReturnType, + IsMetadataTarget = symbol.IsMetadataTarget, + SameLineSignatureOccurrenceIndex = symbol.SameLineSignatureOccurrenceIndex, + }).ToList(); + + private static List CloneReferences(IEnumerable references) + => references.Select(reference => new ReferenceRecord + { + Id = reference.Id, + FileId = reference.FileId, + SymbolName = reference.SymbolName, + ReferenceKind = reference.ReferenceKind, + Line = reference.Line, + Column = reference.Column, + Context = reference.Context, + ContainerKind = reference.ContainerKind, + ContainerName = reference.ContainerName, + IsSelfReference = reference.IsSelfReference, + IsMutualRecursion = reference.IsMutualRecursion, + }).ToList(); + + private static void ReplaceList(IList target, IReadOnlyList replacement) + { + target.Clear(); + foreach (var item in replacement) + target.Add(item); } private static string? GetDefaultHooksDirectory() diff --git a/src/CodeIndex/Mcp/McpToolHandlers.cs b/src/CodeIndex/Mcp/McpToolHandlers.cs index dbf4f4cabb..3006ef2034 100644 --- a/src/CodeIndex/Mcp/McpToolHandlers.cs +++ b/src/CodeIndex/Mcp/McpToolHandlers.cs @@ -7,6 +7,7 @@ using CodeIndex.Cli; using CodeIndex.Database; using CodeIndex.Indexer; +using CodeIndex.Indexer.Extensibility; using CodeIndex.Indexer.Hooks; using CodeIndex.Models; @@ -1909,7 +1910,10 @@ private JsonNode ExecuteStatus(JsonNode? id) WorkspaceMetadataEnricher.Enrich(status, _dbPath, _dbPathExplicit); status.MacProfile = MacProfileDetector.DetectCurrent(); status.GraphSupportedLanguages = ReferenceExtractor.GetSupportedLanguages().OrderBy(l => l).ToList(); - var postExtractionHooks = PostExtractionHookRunner.DiscoverDefault().Hooks; + ExtractorPluginRegistry.LoadPatternConfigsForProjectRoot(status.ProjectRoot); + status.Extractors = ExtractorPluginRegistry.GetStatusSnapshot(); + using var postExtractionHookRunner = PostExtractionHookRunner.DiscoverDefault(); + var postExtractionHooks = postExtractionHookRunner.Hooks; if (postExtractionHooks.Count > 0) { status.Hooks = postExtractionHooks @@ -1918,6 +1922,7 @@ private JsonNode ExecuteStatus(JsonNode? id) Name = hook.Name, AssemblyPath = hook.AssemblyPath, TypeName = hook.TypeName, + CallbackBudgetMs = (long)Math.Round(postExtractionHookRunner.CallbackBudget.TotalMilliseconds, MidpointRounding.AwayFromZero), }) .ToList(); } @@ -3707,9 +3712,7 @@ static long SumReadableFileBytes(IEnumerable paths) // readiness is stamped, preserving the failure-path safety contract. // MCP の no-op full-scan root backfill も readiness stamp 後に限定する。 WriteProjectRootOnce(); - writer.SetMeta( - DbContext.UnknownExtensionFileCountMetaKey, - scanResult.UnknownExtensionFiles.Count.ToString(System.Globalization.CultureInfo.InvariantCulture)); + writer.WriteUnknownExtensionFileMetadata(scanResult.UnknownExtensionFiles); writer.SetMeta(DbContext.LastIndexRunModeMetaKey, rebuild ? "rebuild" : "mcp"); writer.SetMeta(DbContext.LastIndexRunStartedAtMetaKey, runStartedAtUtc.ToString("o", System.Globalization.CultureInfo.InvariantCulture)); writer.SetMeta(DbContext.LastIndexRunDurationMsMetaKey, runStopwatch.ElapsedMilliseconds.ToString(System.Globalization.CultureInfo.InvariantCulture)); diff --git a/src/CodeIndex/Models/QueryResults.cs b/src/CodeIndex/Models/QueryResults.cs index 5095fb788c..2f5c53f050 100644 --- a/src/CodeIndex/Models/QueryResults.cs +++ b/src/CodeIndex/Models/QueryResults.cs @@ -1,4 +1,5 @@ using System.Text.Json.Serialization; +using CodeIndex.Indexer.Extensibility; using CodeIndex.Models; namespace CodeIndex.Database; @@ -480,6 +481,15 @@ public class StatusResult [JsonPropertyName("unknown_extension_file_count")] [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public long? UnknownExtensionFileCount { get; set; } + [JsonPropertyName("unknown_extension_files")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public List? UnknownExtensionFiles { get; set; } + [JsonPropertyName("unknown_extension_files_truncated")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public bool? UnknownExtensionFilesTruncated { get; set; } + [JsonPropertyName("unknown_extension_file_path_limit")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public long? UnknownExtensionFilePathLimit { get; set; } public DateTime? IndexedAt { get; set; } public DateTime? LatestModified { get; set; } public string? ProjectRoot { get; set; } @@ -582,6 +592,9 @@ public class StatusResult public List? GraphSupportedLanguages { get; set; } [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public List? Hooks { get; set; } + [JsonPropertyName("extractors")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public ExtractorRegistryStatus? Extractors { get; set; } public string? Version { get; set; } [JsonPropertyName("update_check")] [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] @@ -857,6 +870,8 @@ public class PostExtractionHookStatus public string Name { get; set; } = string.Empty; public string AssemblyPath { get; set; } = string.Empty; public string TypeName { get; set; } = string.Empty; + [JsonPropertyName("callback_budget_ms")] + public long CallbackBudgetMs { get; set; } } public class RepoMapResult diff --git a/tests/CodeIndex.Tests/DocumentationStatusContractTests.cs b/tests/CodeIndex.Tests/DocumentationStatusContractTests.cs index e3c470cdf1..9905042f4c 100644 --- a/tests/CodeIndex.Tests/DocumentationStatusContractTests.cs +++ b/tests/CodeIndex.Tests/DocumentationStatusContractTests.cs @@ -25,6 +25,10 @@ public class DocumentationStatusContractTests "index_newer_than_reader", "index_newer_than_reader_reason", "unknown_extension_file_count", + "unknown_extension_files", + "unknown_extension_files_truncated", + "unknown_extension_file_path_limit", + "extractors", "path_case_sensitive", "mac_profile", "stale_after_seconds", diff --git a/tests/CodeIndex.Tests/ExtractorPluginRegistryTests.cs b/tests/CodeIndex.Tests/ExtractorPluginRegistryTests.cs new file mode 100644 index 0000000000..1c5f901472 --- /dev/null +++ b/tests/CodeIndex.Tests/ExtractorPluginRegistryTests.cs @@ -0,0 +1,49 @@ +using CodeIndex.Indexer.Extensibility; + +namespace CodeIndex.Tests; + +public class ExtractorPluginRegistryTests +{ + [Fact] + public void LoadPatternConfigs_BoundsDiagnosticsAndCountsSkippedFiles() + { + var projectRoot = TestProjectHelper.CreateTempProject("extractor_registry_diagnostics"); + lock (TestConsoleLock.Gate) + { + try + { + ExtractorPluginRegistry.ResetForTests(); + var patternsDir = Path.Combine(projectRoot, ".cdidx", "patterns"); + Directory.CreateDirectory(patternsDir); + for (var i = 0; i < 25; i++) + { + File.WriteAllText( + Path.Combine(patternsDir, $"broken-{i:D2}.yaml"), + "language: \"broken\"\npatterns:\n - kind: \"class\"\n regex: \"(?\"\n"); + } + + ExtractorPluginRegistry.LoadPatternConfigsForPath(Path.Combine(projectRoot, "sample.broken")); + var status = ExtractorPluginRegistry.GetStatusSnapshot(); + + Assert.Equal(0, status.PatternConfigCount); + Assert.Equal(25, status.SkippedFileCount); + Assert.Equal(25, status.DiagnosticCount); + Assert.Equal(20, status.DiagnosticLimit); + Assert.True(status.DiagnosticsTruncated); + Assert.NotNull(status.Diagnostics); + Assert.Equal(20, status.Diagnostics.Count); + Assert.All(status.Diagnostics, diagnostic => + { + Assert.Equal("pattern", diagnostic.Kind); + Assert.Equal("error", diagnostic.Severity); + Assert.EndsWith(".yaml", diagnostic.Path); + }); + } + finally + { + ExtractorPluginRegistry.ResetForTests(); + TestProjectHelper.DeleteDirectory(projectRoot); + } + } + } +} diff --git a/tests/CodeIndex.Tests/IndexCommandRunnerTests.cs b/tests/CodeIndex.Tests/IndexCommandRunnerTests.cs index 2b3abe3adf..2a06a0fc3c 100644 --- a/tests/CodeIndex.Tests/IndexCommandRunnerTests.cs +++ b/tests/CodeIndex.Tests/IndexCommandRunnerTests.cs @@ -6,6 +6,7 @@ using CodeIndex.Cli; using CodeIndex.Database; using CodeIndex.Indexer; +using CodeIndex.Indexer.Extensibility; using CodeIndex.Models; using Microsoft.Data.Sqlite; @@ -1610,6 +1611,43 @@ public void Run_VerboseReportsUnknownExtensionCountAndStatusJsonStampsCount() Assert.Contains("notes.mystery", stdout); Assert.Equal(CommandExitCodes.Success, statusExitCode); Assert.Equal(2, statusJson.GetProperty("unknown_extension_file_count").GetInt64()); + Assert.False(statusJson.GetProperty("unknown_extension_files_truncated").GetBoolean()); + Assert.Equal(50, statusJson.GetProperty("unknown_extension_file_path_limit").GetInt64()); + var paths = statusJson.GetProperty("unknown_extension_files") + .EnumerateArray() + .Select(path => path.GetString()) + .ToArray(); + Assert.Equal(["data.unmapped", "notes.mystery"], paths); + } + finally + { + DeleteDirectory(projectRoot); + } + } + + [Fact] + public void Run_StatusJsonCapsUnknownExtensionPathSample() + { + var projectRoot = CreateTempProject(); + try + { + File.WriteAllText(Path.Combine(projectRoot, "app.cs"), "class App { }\n"); + for (var i = 0; i < 52; i++) + File.WriteAllText(Path.Combine(projectRoot, $"unknown-{i:D2}.mystery"), "unknown extension\n"); + + var (exitCode, _) = RunAndCaptureJson([projectRoot, "--json"]); + var dbPath = Path.Combine(projectRoot, ".cdidx", "codeindex.db"); + var (statusExitCode, statusJson) = RunStatusAndCaptureJson(["--db", dbPath, "--json"]); + + Assert.Equal(CommandExitCodes.Success, exitCode); + Assert.Equal(CommandExitCodes.Success, statusExitCode); + Assert.Equal(52, statusJson.GetProperty("unknown_extension_file_count").GetInt64()); + Assert.True(statusJson.GetProperty("unknown_extension_files_truncated").GetBoolean()); + Assert.Equal(50, statusJson.GetProperty("unknown_extension_file_path_limit").GetInt64()); + var paths = statusJson.GetProperty("unknown_extension_files").EnumerateArray().ToArray(); + Assert.Equal(50, paths.Length); + Assert.Equal("unknown-00.mystery", paths[0].GetString()); + Assert.Equal("unknown-49.mystery", paths[^1].GetString()); } finally { @@ -1617,6 +1655,51 @@ public void Run_VerboseReportsUnknownExtensionCountAndStatusJsonStampsCount() } } + [Fact] + public void Run_StatusJsonIncludesExtractorPluginDiagnostics() + { + var projectRoot = CreateTempProject(); + lock (TestConsoleLock.Gate) + { + try + { + ExtractorPluginRegistry.ResetForTests(); + var patternsDir = Path.Combine(projectRoot, ".cdidx", "patterns"); + Directory.CreateDirectory(patternsDir); + File.WriteAllText( + Path.Combine(patternsDir, "toydsl.yaml"), + "language: \"toydsl\"\nextensions:\n - extension: \".toy\"\npatterns:\n - kind: \"class\"\n regex: \"^entity (?\\\\w+)\"\n"); + File.WriteAllText( + Path.Combine(patternsDir, "broken.yaml"), + "language: \"broken\"\npatterns:\n - kind: \"class\"\n regex: \"(?\"\n"); + File.WriteAllText(Path.Combine(projectRoot, "app.cs"), "class App { }\n"); + + var (exitCode, _) = RunAndCaptureJson([projectRoot, "--json"]); + var dbPath = Path.Combine(projectRoot, ".cdidx", "codeindex.db"); + var (statusExitCode, statusJson) = RunStatusAndCaptureJson(["--db", dbPath, "--json"]); + + Assert.Equal(CommandExitCodes.Success, exitCode); + Assert.Equal(CommandExitCodes.Success, statusExitCode); + var extractors = statusJson.GetProperty("extractors"); + Assert.True(extractors.GetProperty("pattern_config_count").GetInt32() >= 1); + Assert.True(extractors.GetProperty("skipped_file_count").GetInt32() >= 1); + Assert.True(extractors.GetProperty("diagnostic_count").GetInt32() >= 1); + Assert.Equal(20, extractors.GetProperty("diagnostic_limit").GetInt32()); + Assert.True(extractors.GetProperty("symbol_extractor_count").GetInt32() >= 1); + var diagnostic = Assert.Single( + extractors.GetProperty("diagnostics").EnumerateArray(), + item => item.GetProperty("path").GetString()?.EndsWith("broken.yaml", StringComparison.Ordinal) == true); + Assert.Equal("pattern", diagnostic.GetProperty("kind").GetString()); + Assert.Equal("error", diagnostic.GetProperty("severity").GetString()); + } + finally + { + ExtractorPluginRegistry.ResetForTests(); + DeleteDirectory(projectRoot); + } + } + } + [Fact] public void Run_GitRepo_PersistsIndexedHeadMetadata() { diff --git a/tests/CodeIndex.Tests/PostExtractionHookTests.cs b/tests/CodeIndex.Tests/PostExtractionHookTests.cs index 6f17e7f01d..9013bea407 100644 --- a/tests/CodeIndex.Tests/PostExtractionHookTests.cs +++ b/tests/CodeIndex.Tests/PostExtractionHookTests.cs @@ -6,6 +6,9 @@ namespace CodeIndex.Tests; public class PostExtractionHookTests { + internal const string SlowHookDelayEnvironmentVariable = "CDIDX_TEST_SLOW_POST_EXTRACTION_HOOK_MS"; + internal const string SlowHookCompletionPathEnvironmentVariable = "CDIDX_TEST_SLOW_POST_EXTRACTION_HOOK_DONE_PATH"; + [Fact] public void Discover_LoadsHooksAndAllowsSymbolAndReferenceMutation() { @@ -70,12 +73,97 @@ public void CallbackExceptions_AreDiagnosticsAndDoNotBlockOtherHooks() } } + [Fact] + public void CallbackBudgetExceeded_AddsDiagnosticAndSkipsTimedOutMutation() + { + var projectRoot = TestProjectHelper.CreateTempProject("post-extraction-hook-budget"); + lock (TestConsoleLock.Gate) + { + var originalDelay = Environment.GetEnvironmentVariable(SlowHookDelayEnvironmentVariable); + var originalCompletionPath = Environment.GetEnvironmentVariable(SlowHookCompletionPathEnvironmentVariable); + var originalBudget = PostExtractionHookRunner.CallbackBudgetForTesting; + try + { + Environment.SetEnvironmentVariable(SlowHookDelayEnvironmentVariable, "200"); + PostExtractionHookRunner.CallbackBudgetForTesting = () => TimeSpan.FromMilliseconds(50); + var hooksDir = Path.Combine(projectRoot, "hooks"); + var completionPath = Path.Combine(projectRoot, "slow-hook.done"); + Environment.SetEnvironmentVariable(SlowHookCompletionPathEnvironmentVariable, completionPath); + Directory.CreateDirectory(hooksDir); + File.Copy(Assembly.GetExecutingAssembly().Location, Path.Combine(hooksDir, "CodeIndex.Tests.dll")); + + { + using var runner = PostExtractionHookRunner.Discover(hooksDir); + var context = new FileContext(projectRoot, "src/App.cs", Path.Combine(projectRoot, "src", "App.cs"), "csharp"); + var symbols = new List(); + + runner.OnSymbolsExtracted(context, symbols); + WaitForSlowHookCompletion(completionPath); + + Assert.DoesNotContain(symbols, symbol => symbol.Name == "SlowHookTag"); + var diagnostic = Assert.Single( + runner.Diagnostics, + item => item.TypeName == typeof(SlowPostExtractionHook).FullName + && item.Callback == nameof(IPostExtractionHook.OnSymbolsExtracted)); + Assert.Contains("exceeded", diagnostic.Message, StringComparison.Ordinal); + Assert.True(diagnostic.DurationMs >= 50); + Assert.Equal(50, (long)Math.Round(runner.CallbackBudget.TotalMilliseconds, MidpointRounding.AwayFromZero)); + } + CollectUnloadedHookAssemblies(); + } + finally + { + PostExtractionHookRunner.CallbackBudgetForTesting = originalBudget; + Environment.SetEnvironmentVariable(SlowHookDelayEnvironmentVariable, originalDelay); + Environment.SetEnvironmentVariable(SlowHookCompletionPathEnvironmentVariable, originalCompletionPath); + TestProjectHelper.DeleteDirectory(projectRoot); + } + } + } + + [Fact] + public void CallbackBudget_NormalizesInvalidAndTooLargeValues() + { + lock (TestConsoleLock.Gate) + { + var originalBudget = PostExtractionHookRunner.CallbackBudgetForTesting; + try + { + PostExtractionHookRunner.CallbackBudgetForTesting = () => TimeSpan.Zero; + using (var defaulted = PostExtractionHookRunner.Discover(null)) + { + Assert.Equal(PostExtractionHookRunner.DefaultCallbackBudget, defaulted.CallbackBudget); + } + + PostExtractionHookRunner.CallbackBudgetForTesting = () => TimeSpan.FromMilliseconds((double)int.MaxValue + 1); + using var capped = PostExtractionHookRunner.Discover(null); + Assert.Equal(int.MaxValue, (long)Math.Round(capped.CallbackBudget.TotalMilliseconds, MidpointRounding.AwayFromZero)); + } + finally + { + PostExtractionHookRunner.CallbackBudgetForTesting = originalBudget; + } + } + } + private static void CollectUnloadedHookAssemblies() { GC.Collect(); GC.WaitForPendingFinalizers(); GC.Collect(); } + + private static void WaitForSlowHookCompletion(string completionPath) + { + var deadline = DateTimeOffset.UtcNow.AddSeconds(5); + while (!File.Exists(completionPath)) + { + if (DateTimeOffset.UtcNow >= deadline) + throw new TimeoutException("Timed out waiting for the slow post-extraction hook to finish."); + + Thread.Sleep(25); + } + } } public sealed class SamplePostExtractionHook : IPostExtractionHook @@ -116,3 +204,50 @@ public void OnSymbolsExtracted(FileContext context, IList symbols) public void OnReferencesExtracted(FileContext context, IList references) => throw new InvalidOperationException("boom"); } + +public sealed class SlowPostExtractionHook : IPostExtractionHook +{ + public void OnSymbolsExtracted(FileContext context, IList symbols) + { + if (!DelayWhenRequested()) + return; + + symbols.Add(new SymbolRecord + { + Kind = "domain_tag", + Name = "SlowHookTag", + Line = 1, + StartLine = 1, + EndLine = 1, + }); + } + + public void OnReferencesExtracted(FileContext context, IList references) + { + if (!DelayWhenRequested()) + return; + + references.Add(new ReferenceRecord + { + SymbolName = "SlowHookTag", + ReferenceKind = "domain_reference", + Line = 1, + Column = 1, + Context = context.Path, + }); + } + + private static bool DelayWhenRequested() + { + var raw = Environment.GetEnvironmentVariable(PostExtractionHookTests.SlowHookDelayEnvironmentVariable); + if (!int.TryParse(raw, out var milliseconds) || milliseconds <= 0) + return false; + + Thread.Sleep(milliseconds); + var completionPath = Environment.GetEnvironmentVariable(PostExtractionHookTests.SlowHookCompletionPathEnvironmentVariable); + if (!string.IsNullOrWhiteSpace(completionPath)) + File.WriteAllText(completionPath, "done"); + + return true; + } +} diff --git a/tests/CodeIndex.Tests/golden/status.json b/tests/CodeIndex.Tests/golden/status.json index d498c44408..a1e3a86416 100644 --- a/tests/CodeIndex.Tests/golden/status.json +++ b/tests/CodeIndex.Tests/golden/status.json @@ -75,6 +75,16 @@ "vue", "zig" ], + "extractors": { + "plugin_assembly_count": 0, + "pattern_config_count": 0, + "symbol_extractor_count": 0, + "reference_extractor_count": 0, + "skipped_file_count": 0, + "diagnostic_count": 0, + "diagnostic_limit": 20, + "diagnostics_truncated": false + }, "version": null, "summary": "1 files, 3 symbols, 0 refs across 1 languages (csharp); index fresh, DEGRADED", "graph_table_available": false,