diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index e56cb76df7..a1ac5bfbd8 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -153,6 +153,8 @@ git status --short -- '**/packages.lock.json' | DTOs | `Models/FileRecord.cs`, `Models/ChunkRecord.cs`, `Models/SymbolRecord.cs`, `Models/ReferenceRecord.cs` | Records shared by indexing, storage, query, and MCP layers. | | Tests | `tests/CodeIndex.Tests/*Tests.cs`, `TestProjectHelper.cs`, `TestConsoleLock.cs` | Focused unit/integration coverage for chunking, extraction, DB reads/writes, CLI behavior, MCP behavior, git helpers, and shared test harness utilities. | +Full-scan resume checkpoints live at `.cdidx/scan-checkpoint.json`. The internal JSON schema is versioned (`Version` is currently `1`) and binds `GitHead` to a bounded `Directories` array; malformed, stale, future-version, oversized, or over-depth payloads are ignored with a bounded warning and a full scan. + Large command and extractor files have a tracked decomposition plan in [docs/large-file-decomposition-plan.md](docs/large-file-decomposition-plan.md). Use that plan when splitting `QueryCommandRunner`, `SymbolExtractor`, diff --git a/README.md b/README.md index b01b9dc5c5..dde6050a36 100644 --- a/README.md +++ b/README.md @@ -90,6 +90,10 @@ For a faster first pass when you only need text search, `definition`, `symbols`, or `map`, run `cdidx . --symbols-only`. Reference graph commands remain degraded until you rerun `cdidx .` without that flag. +For generated or dense source that emits excessive reference rows, use +`cdidx . --max-references-per-file ` to keep text search and symbols indexed +while skipping references for only the over-limit file. + ## Highlights | Area | What to use | @@ -265,6 +269,10 @@ cdidx lsp --db .cdidx/codeindex.db `cdidx . --symbols-only` を使えます。reference graph 系コマンドは、このフラグなしで `cdidx .` を再実行するまで degraded のままです。 +生成コードや高密度なソースが過剰な reference 行を生成する場合は +`cdidx . --max-references-per-file ` を使うと、text search と symbols は保持しつつ +上限を超えたファイルだけ references をスキップできます。 + ## 特長 | 分野 | 使うもの | diff --git a/USER_GUIDE.md b/USER_GUIDE.md index 5d66e81fed..6c3fb7bab3 100644 --- a/USER_GUIDE.md +++ b/USER_GUIDE.md @@ -454,9 +454,10 @@ cdidx validate --json --limit 50 --path legacy/ ``` `validate` reports indexed files that are likely to produce misleading snippets -or symbol names: U+FFFD replacement characters, UTF-16 BOMs, null bytes, mixed or -CR-only line endings, likely non-UTF-8 content, Git LFS pointer placeholders, and -malformed or truncated Dockerfile JSON-form instruction payloads. +or symbol names: U+FFFD replacement characters, UTF-16 BOM and BOM-less +heuristic detections, null bytes, mixed or CR-only line endings, likely non-UTF-8 +content, Git LFS pointer placeholders, and malformed or truncated Dockerfile +JSON-form instruction payloads. For `replacement_char`, JSON and MCP responses include `origin` (`source_literal` or `decode_replacement`) and `severity` so agents can distinguish intentional U+FFFD literals from likely encoding damage. @@ -464,9 +465,9 @@ Use `--severity warning` to hide informational source literals and focus on findings that indicate likely encoding damage. Use `--json=array` when a pipeline expects a bare issue array instead of the default `{ "count": ..., "issues": [...] }` object. -LFS pointers are recorded as `lfs_pointer_skipped` and their placeholder body is -not indexed; run `git lfs pull` and then `cdidx index .` to index the real file -content. +LFS pointers are recorded as `lfs_pointer_skipped`; their placeholder body is +not indexed, and their checksum stays tied to the pointer identity until you run +`git lfs pull` and then `cdidx index .` to index the real file content. ### Find potentially unused symbols @@ -1706,7 +1707,8 @@ Supported schema (top-level keys are snake_case; nested indexing kind keys keep "suggestion_max_count": 5000, // → CDIDX_SUGGESTION_MAX_COUNT "indexing": { "includeKinds": ["class"], // → CDIDX_INDEX_INCLUDE_SYMBOL_KINDS - "excludeKinds": ["test_method"] // → CDIDX_INDEX_EXCLUDE_SYMBOL_KINDS + "excludeKinds": ["test_method"], // → CDIDX_INDEX_EXCLUDE_SYMBOL_KINDS + "generatedCodePatterns": ["src/generated/**", "*.client.ts"] // → CDIDX_INDEX_GENERATED_CODE_PATTERNS }, "mcp": { "tools": { @@ -1722,7 +1724,7 @@ Supported schema (top-level keys are snake_case; nested indexing kind keys keep } ``` -JSON5-style line comments (`//`) and trailing commas are accepted so the file stays human-editable. The optional `$schema` key is ignored at runtime; it is honored only so editors that recognize JSON Schema references can offer completion. Setting `disable_persistent_log` to `false` is a no-op (absence already means "logging enabled") — only `true` exports `CDIDX_DISABLE_PERSISTENT_LOG=1`. Config-sourced `metrics_path` and `global_tool_log_dir` values are resolved from the config workspace root and must stay inside that workspace; use the CLI flag or a real environment variable when you intentionally need an outside destination. `stale_after` uses the same compact duration format as `status --check --stale-after`: `30m`, `2h`, or `7d`, up to `30d`. `suggestion_dedup_threshold` sets the MCP suggestion fuzzy-deduplication cutoff as a number from `0` to `1`; the built-in default is `0.85`, and `cdidx mcp --suggestion-dedup-threshold <0..1>` overrides it for one MCP session. `suggestion_max_age_days` and `suggestion_max_count` bound the live `.cdidx/suggestions-*.json` store; pruned records are appended to `.cdidx/suggestions-*.archive.jsonl`, whose active file is capped at 8 MiB and rotates up to three retained generations (`.1` through `.3`). Defaults are 365 days and 5000 records, and config-file values may not exceed 3650 days or 100000 records. Matching environment variables above those caps fall back to the defaults. `mcp.rate_limit.bucket_idle_seconds` sets the same idle bucket TTL as `CDIDX_MCP_RATE_LIMIT_BUCKET_IDLE_SECONDS`; invalid runtime values fall back to the default with a warning. String-array settings such as `indexing.includeKinds`, `indexing.excludeKinds`, `mcp.tools.allow`, and `mcp.tools.deny` are capped at 128 entries and 256 characters per item before they are joined into environment variables. `indexing.includeKinds` and `indexing.excludeKinds` set the default symbol-kind filter for `cdidx index`; CLI flags `--include-symbol-kind [,]` and `--exclude-symbol-kind [,]` override those env-backed defaults for a single run. +JSON5-style line comments (`//`) and trailing commas are accepted so the file stays human-editable. The optional `$schema` key is ignored at runtime; it is honored only so editors that recognize JSON Schema references can offer completion. Setting `disable_persistent_log` to `false` is a no-op (absence already means "logging enabled") — only `true` exports `CDIDX_DISABLE_PERSISTENT_LOG=1`. Config-sourced `metrics_path` and `global_tool_log_dir` values are resolved from the config workspace root and must stay inside that workspace; use the CLI flag or a real environment variable when you intentionally need an outside destination. `stale_after` uses the same compact duration format as `status --check --stale-after`: `30m`, `2h`, or `7d`, up to `30d`. `suggestion_dedup_threshold` sets the MCP suggestion fuzzy-deduplication cutoff as a number from `0` to `1`; the built-in default is `0.85`, and `cdidx mcp --suggestion-dedup-threshold <0..1>` overrides it for one MCP session. `suggestion_max_age_days` and `suggestion_max_count` bound the live `.cdidx/suggestions-*.json` store; pruned records are appended to `.cdidx/suggestions-*.archive.jsonl`, whose active file is capped at 8 MiB and rotates up to three retained generations (`.1` through `.3`). Defaults are 365 days and 5000 records, and config-file values may not exceed 3650 days or 100000 records. Matching environment variables above those caps fall back to the defaults. `mcp.rate_limit.bucket_idle_seconds` sets the same idle bucket TTL as `CDIDX_MCP_RATE_LIMIT_BUCKET_IDLE_SECONDS`; invalid runtime values fall back to the default with a warning. String-array settings such as `indexing.includeKinds`, `indexing.excludeKinds`, `indexing.generatedCodePatterns`, `mcp.tools.allow`, and `mcp.tools.deny` are capped at 128 entries and 256 characters per item before they are joined into environment variables. `indexing.generatedCodePatterns` treats matching relative paths or basenames as extraction-suppressed generated-code sources. Matching files remain indexed for normal text search and chunk retrieval because the query-filtered `generated` flag is not set by this option; symbol/reference extraction is skipped and `file_issues` records `generated_code_extraction_skipped`. Patterns with a slash match slash-normalized relative paths, patterns without a slash match basenames, and `*`, `?`, and `**` are supported. `indexing.includeKinds` and `indexing.excludeKinds` set the default symbol-kind filter for `cdidx index`; CLI flags `--include-symbol-kind [,]` and `--exclude-symbol-kind [,]` override those env-backed defaults for a single run. ## How it works @@ -2305,7 +2307,7 @@ The MCP `tools/list` response includes an `examples` array for every registered | `unused_symbols` | Find symbols defined but never referenced, with confidence buckets for dead-code triage | | `symbol_hotspots` | Find high-impact hotspots. `groupBy` supports `symbol`, `file`, and `statement`; SQL scopes default to statement grouping while non-SQL scopes default to symbol grouping. | | `batch_query` | Execute multiple queries in a single call (MCP only, max 10). The response includes a top-level `metadata` object with `submitted`, `executed`, `errors`, `total_elapsed_ms`, `success_count`, and `failure_count`; every entry in `results` carries `request_index`, optional client `slot_id`, `ok`, `elapsed_ms`, `summary`, and compact `args_summary` fields so callers can correlate partial failures and slow inner queries without relying on positional guesses. Scalar values in `args_summary` are bounded before full JSON materialization, so huge numbers and strings cannot inflate diagnostics. | -| `validate` | Report encoding and file-content issues (U+FFFD with origin/severity, BOM, null bytes, mixed/CR-only line endings, UTF-16 BOM detection, likely non-UTF8 encodings, Dockerfile JSON-form diagnostics) | +| `validate` | Report encoding and file-content issues (U+FFFD with origin/severity, BOM, null bytes, mixed/CR-only line endings, UTF-16 BOM/heuristic detection, likely non-UTF8 encodings, Git LFS pointer placeholders, Dockerfile JSON-form diagnostics) | | `languages` | List all supported languages, file extensions, and capabilities | | `ping` | Lightweight connection check | | `index` | Index or re-index a project directory | @@ -2962,17 +2964,19 @@ cdidx validate --json --limit 50 --path legacy/ ``` `validate` は、snippet や symbol name を誤らせやすい indexed file を報告します。 -対象は U+FFFD replacement character、UTF-16 BOM、null byte、mixed / CR-only line -ending、likely non-UTF-8 content、Git LFS pointer placeholder、Dockerfile の -JSON-form instruction payload の parse / truncation 診断などです。 +対象は U+FFFD replacement character、UTF-16 BOM と BOM なし heuristic 検出、 +null byte、mixed / CR-only line ending、likely non-UTF-8 content、Git LFS pointer +placeholder、Dockerfile の JSON-form instruction payload の parse / truncation +診断などです。 `replacement_char` の JSON / MCP response には `origin` (`source_literal` / `decode_replacement`) と `severity` が入り、意図的な U+FFFD literal と エンコーディング破損の可能性を agent が区別できます。`--severity warning` を使うと、informational な source literal を隠して、エンコーディング破損の 可能性がある finding に集中できます。pipeline が既定の `{ "count": ..., "issues": [...] }` object ではなく bare issue array を期待する場合は `--json=array` を使えます。LFS pointer -は `lfs_pointer_skipped` として記録され、placeholder 本文は index されません。 -実体を index するには `git lfs pull` の後に `cdidx index .` を再実行してください。 +は `lfs_pointer_skipped` として記録され、placeholder 本文は index されず、checksum は +実体を取得するまで pointer identity に紐づきます。実体を index するには `git lfs pull` +の後に `cdidx index .` を再実行してください。 ### 未使用の可能性がある symbols を探す @@ -4224,7 +4228,8 @@ MCP のレスポンスサイズ上限は、環境変数 override で guard が "suggestion_max_count": 5000, // → CDIDX_SUGGESTION_MAX_COUNT "indexing": { "includeKinds": ["class"], // → CDIDX_INDEX_INCLUDE_SYMBOL_KINDS - "excludeKinds": ["test_method"] // → CDIDX_INDEX_EXCLUDE_SYMBOL_KINDS + "excludeKinds": ["test_method"], // → CDIDX_INDEX_EXCLUDE_SYMBOL_KINDS + "generatedCodePatterns": ["src/generated/**", "*.client.ts"] // → CDIDX_INDEX_GENERATED_CODE_PATTERNS }, "mcp": { "tools": { @@ -4240,7 +4245,7 @@ MCP のレスポンスサイズ上限は、環境変数 override で guard が } ``` -人手で編集しやすいよう JSON5 形式の行コメント（`//`）と末尾カンマを許容します。任意の `$schema` キーはランタイムでは無視され、JSON Schema 参照をサポートするエディタが補完を提供するためだけに認識されます。`disable_persistent_log` を `false` に設定しても何も起きません（不在のままで "ログ有効" が既定）— `true` の場合のみ `CDIDX_DISABLE_PERSISTENT_LOG=1` を export します。config 由来の `metrics_path` と `global_tool_log_dir` は設定ファイルの workspace root から解決され、その workspace 内に収まる必要があります。意図的に外部の出力先を使う場合は CLI フラグまたは実際の環境変数を使ってください。`stale_after` は `status --check --stale-after` と同じ compact duration 形式（`30m` / `2h` / `7d`、最大 `30d`）です。`suggestion_dedup_threshold` は MCP suggestion の fuzzy deduplication しきい値を `0` から `1` の数値で設定します。組み込み既定値は `0.85` で、`cdidx mcp --suggestion-dedup-threshold <0..1>` は 1 回の MCP session だけこの値を上書きします。`suggestion_max_age_days` と `suggestion_max_count` は live の `.cdidx/suggestions-*.json` store の上限を設定し、prune された record は `.cdidx/suggestions-*.archive.jsonl` に追記されます。この active archive は 8 MiB で上限管理され、最大 3 世代（`.1` から `.3`）までローテーションされます。既定値は 365 日と 5000 件で、config-file 値は 3650 日または 100000 件を超えられません。同じ環境変数がこの上限を超えた場合は既定値へ戻ります。`mcp.rate_limit.bucket_idle_seconds` は `CDIDX_MCP_RATE_LIMIT_BUCKET_IDLE_SECONDS` と同じ idle bucket TTL を設定します。不正な runtime 値は警告付きで既定値へ戻ります。`indexing.includeKinds`、`indexing.excludeKinds`、`mcp.tools.allow`、`mcp.tools.deny` のような string array 設定は、環境変数へ join される前に 128 件、1 要素 256 文字までに制限されます。`indexing.includeKinds` と `indexing.excludeKinds` は `cdidx index` の symbol-kind filter 既定値を設定し、CLI フラグ `--include-symbol-kind [,]` / `--exclude-symbol-kind [,]` はその env 経由の既定値を 1 回の実行だけ上書きします。 +人手で編集しやすいよう JSON5 形式の行コメント（`//`）と末尾カンマを許容します。任意の `$schema` キーはランタイムでは無視され、JSON Schema 参照をサポートするエディタが補完を提供するためだけに認識されます。`disable_persistent_log` を `false` に設定しても何も起きません（不在のままで "ログ有効" が既定）— `true` の場合のみ `CDIDX_DISABLE_PERSISTENT_LOG=1` を export します。config 由来の `metrics_path` と `global_tool_log_dir` は設定ファイルの workspace root から解決され、その workspace 内に収まる必要があります。意図的に外部の出力先を使う場合は CLI フラグまたは実際の環境変数を使ってください。`stale_after` は `status --check --stale-after` と同じ compact duration 形式（`30m` / `2h` / `7d`、最大 `30d`）です。`suggestion_dedup_threshold` は MCP suggestion の fuzzy deduplication しきい値を `0` から `1` の数値で設定します。組み込み既定値は `0.85` で、`cdidx mcp --suggestion-dedup-threshold <0..1>` は 1 回の MCP session だけこの値を上書きします。`suggestion_max_age_days` と `suggestion_max_count` は live の `.cdidx/suggestions-*.json` store の上限を設定し、prune された record は `.cdidx/suggestions-*.archive.jsonl` に追記されます。この active archive は 8 MiB で上限管理され、最大 3 世代（`.1` から `.3`）までローテーションされます。既定値は 365 日と 5000 件で、config-file 値は 3650 日または 100000 件を超えられません。同じ環境変数がこの上限を超えた場合は既定値へ戻ります。`mcp.rate_limit.bucket_idle_seconds` は `CDIDX_MCP_RATE_LIMIT_BUCKET_IDLE_SECONDS` と同じ idle bucket TTL を設定します。不正な runtime 値は警告付きで既定値へ戻ります。`indexing.includeKinds`、`indexing.excludeKinds`、`indexing.generatedCodePatterns`、`mcp.tools.allow`、`mcp.tools.deny` のような string array 設定は、環境変数へ join される前に 128 件、1 要素 256 文字までに制限されます。`indexing.generatedCodePatterns` は一致した相対パスまたはベース名を generated-code extraction の抑制対象として扱います。この設定では query filter 用の `generated` flag を立てないため、一致したファイルも通常の全文検索と chunk 取得用には引き続き index されます。symbol/reference 抽出はスキップされ、`file_issues` に `generated_code_extraction_skipped` が記録されます。スラッシュを含む pattern は slash-normalized relative path、スラッシュを含まない pattern は basename に一致し、`*`、`?`、`**` を利用できます。`indexing.includeKinds` と `indexing.excludeKinds` は `cdidx index` の symbol-kind filter 既定値を設定し、CLI フラグ `--include-symbol-kind [,]` / `--exclude-symbol-kind [,]` はその env 経由の既定値を 1 回の実行だけ上書きします。 ## 動作の仕組み @@ -4808,7 +4813,7 @@ OpenAI Codex CLI (`codex.json` または `~/.codex/config.json`): | `unused_symbols` | 定義されているが参照されていないシンボルを bucket 付きで検索（デッドコード検出向け） | | `symbol_hotspots` | 影響の大きい hotspot を検索。`groupBy` は `symbol` / `file` / `statement` を指定でき、SQL scope は statement grouping、非 SQL scope は symbol grouping が既定。 | | `batch_query` | 複数クエリを1回で実行（MCP専用、最大10件）。レスポンスにはトップレベル `metadata`（`submitted` / `executed` / `errors` / `total_elapsed_ms` / `success_count` / `failure_count`）と各 `results` エントリの `request_index`、任意の client `slot_id`、`ok`、`elapsed_ms`、`summary`、`args_summary` が含まれ、位置だけに依存せず部分失敗や遅い内部クエリを把握できます。`args_summary` の scalar 値は full JSON materialization の前に bounded 表示へ変換されるため、巨大な数値や文字列が診断を膨らませません。 | -| `validate` | エンコーディングと file-content の問題（origin/severity 付き U+FFFD、BOM、null バイト、改行混在 / CR-only 行末、UTF-16 BOM 検出、UTF-8 以外と推定されるエンコーディング、Dockerfile JSON-form 診断）を報告 | +| `validate` | エンコーディングと file-content の問題（origin/severity 付き U+FFFD、BOM、null バイト、改行混在 / CR-only 行末、UTF-16 BOM / heuristic 検出、UTF-8 以外と推定されるエンコーディング、Git LFS pointer placeholder、Dockerfile JSON-form 診断）を報告 | | `languages` | 対応言語一覧を拡張子・機能付きで表示。`--indexed-only` と `--capability graph|references|symbols|missing-graph|missing-references|missing-symbols|search-only` で現在の DB、機能別、または capability gap 別に絞り込み可能 | | `ping` | 軽量な接続確認 | | `index` | プロジェクトのインデックス作成・更新 | diff --git a/changelog.d/unreleased/3712.fixed.md b/changelog.d/unreleased/3712.fixed.md new file mode 100644 index 0000000000..9314188a52 --- /dev/null +++ b/changelog.d/unreleased/3712.fixed.md @@ -0,0 +1,18 @@ +--- +category: fixed +issues: + - 3712 +affected: + - src/CodeIndex/Cli/IndexCommandRunner.cs + - src/CodeIndex/Cli/IndexCommandRunner.FullScan.cs + - tests/CodeIndex.Tests/IndexCommandRunnerTests.cs + - DEVELOPER_GUIDE.md +--- + +## English + +- **Scan checkpoint recovery now reports schema/version diagnostics (#3712)** — malformed, stale, future-version, oversized, over-depth, or invalid-directory checkpoint payloads fall back to a full scan with a bounded `` warning instead of being ignored silently. + +## 日本語 + +- **scan checkpoint の復旧で schema/version 診断を出すようになりました (#3712)** — malformed、stale、future-version、oversized、over-depth、invalid-directory な checkpoint payload は、無言で無視される代わりに bounded な `` warning を出して full scan へフォールバックします。 diff --git a/changelog.d/unreleased/3719.fixed.md b/changelog.d/unreleased/3719.fixed.md new file mode 100644 index 0000000000..c6779cae3a --- /dev/null +++ b/changelog.d/unreleased/3719.fixed.md @@ -0,0 +1,24 @@ +--- +category: fixed +issues: + - 3719 +affected: + - src/CodeIndex/Cli/IndexCommandRunner.cs + - src/CodeIndex/Cli/IndexCommandRunner.FullScan.cs + - src/CodeIndex/Cli/IndexCommandRunner.Update.cs + - src/CodeIndex/Indexer/References/ReferenceExtractionContext.cs + - src/CodeIndex/Indexer/References/ReferenceExtractor.Core.cs + - src/CodeIndex/Indexer/References/ReferenceExtractor.cs + - src/CodeIndex/Mcp/McpToolHandlers.cs + - src/CodeIndex/Mcp/McpToolDefinitions.cs + - src/CodeIndex/Mcp/McpToolArgumentContracts.cs + - tests/CodeIndex.Tests/McpServerTests.cs +--- + +## English + +- **Reference extraction now has a per-file cap (#3719)** — `cdidx index` and MCP indexing accept `--max-references-per-file` / `maxReferencesPerFile`, stop extraction after the diagnostic sentinel count, preserve text search and symbols, and emit `reference_count_exceeded` file issues when a file's references are skipped. + +## 日本語 + +- **reference 抽出にファイル単位の上限を追加しました (#3719)** — `cdidx index` と MCP index が `--max-references-per-file` / `maxReferencesPerFile` を受け付け、診断用 sentinel 件数で抽出を止め、text search と symbols は保持しつつ、references をスキップしたファイルには `reference_count_exceeded` file issue を出します。 diff --git a/changelog.d/unreleased/3720.fixed.md b/changelog.d/unreleased/3720.fixed.md new file mode 100644 index 0000000000..ab7e2496e6 --- /dev/null +++ b/changelog.d/unreleased/3720.fixed.md @@ -0,0 +1,24 @@ +--- +category: fixed +issues: + - 3720 +affected: + - src/CodeIndex/Cli/CdidxConfigFile.cs + - src/CodeIndex/Cli/IndexCommandRunner.cs + - src/CodeIndex/Indexer/Scanning/FileIndexer.cs + - src/CodeIndex/Indexer/Scanning/GeneratedCodePatternMatcher.cs + - src/CodeIndex/Mcp/McpToolHandlers.cs + - tests/CodeIndex.Tests/CdidxConfigFileTests.cs + - tests/CodeIndex.Tests/FileIndexerTests.cs + - tests/CodeIndex.Tests/IndexCommandRunnerTests.cs + - tests/CodeIndex.Tests/McpServerTests.cs + - USER_GUIDE.md +--- + +## English + +- **Generated-code extraction suppression is now configurable (#3720)** — project config can suppress symbol/reference extraction for filename or path patterns while preserving normal text search visibility and recording `generated_code_extraction_skipped` in `file_issues`. + +## 日本語 + +- **generated-code extraction suppression を設定可能にしました (#3720)** — project config で filename/path pattern の symbol/reference 抽出を抑制でき、通常の text search での可視性は維持しつつ、`file_issues` に `generated_code_extraction_skipped` を記録します。 diff --git a/changelog.d/unreleased/3762.fixed.md b/changelog.d/unreleased/3762.fixed.md new file mode 100644 index 0000000000..03ff31d4aa --- /dev/null +++ b/changelog.d/unreleased/3762.fixed.md @@ -0,0 +1,18 @@ +--- +category: fixed +issues: + - 3762 +affected: + - src/CodeIndex/Cli/IndexCommandRunner.cs + - src/CodeIndex/Indexer/Scanning/FileIndexer.cs + - tests/CodeIndex.Tests/FileIndexerTests.cs + - tests/CodeIndex.Tests/IndexCommandRunnerTests.cs +--- + +## English + +- **Project discovery budget diagnostics now name the budget that fired (#3762)** — project marker fingerprint truncation records directory/marker-file budget reasons in warnings and fingerprint salt, forwards truncation warnings through CLI JSON output, and solution fallback discovery has explicit directory-budget coverage. + +## 日本語 + +- **project discovery budget 診断が発火した budget 名を示すようになりました (#3762)** — project marker fingerprint の truncation は directory / marker-file budget の理由を warning と fingerprint salt に記録し、truncation warning を CLI JSON 出力にも転送し、solution fallback discovery には directory budget の明示テストを追加しました。 diff --git a/changelog.d/unreleased/3763.fixed.md b/changelog.d/unreleased/3763.fixed.md new file mode 100644 index 0000000000..74868b9649 --- /dev/null +++ b/changelog.d/unreleased/3763.fixed.md @@ -0,0 +1,16 @@ +--- +category: fixed +issues: + - 3763 +affected: + - src/CodeIndex/Indexer/Scanning/FileIndexer.cs + - tests/CodeIndex.Tests/FileIndexerTests.cs +--- + +## English + +- **Dangling filesystem entry probes are now capped (#3763)** — per-directory dangling symlink checks stop after a bounded candidate count and emit a warning when additional diagnostics may be omitted, preserving normal scan correctness while avoiding unbounded probe work. + +## 日本語 + +- **dangling filesystem entry probe に上限を追加しました (#3763)** — ディレクトリ単位の dangling symlink 確認は bounded な candidate 数で停止し、追加診断が省略される可能性を warning として出すため、通常の scan correctness を保ちながら無制限な probe 作業を避けます。 diff --git a/changelog.d/unreleased/3785.fixed.md b/changelog.d/unreleased/3785.fixed.md new file mode 100644 index 0000000000..72894b6c43 --- /dev/null +++ b/changelog.d/unreleased/3785.fixed.md @@ -0,0 +1,17 @@ +--- +category: fixed +issues: + - 3785 +affected: + - src/CodeIndex/Indexer/References/ReferenceExtractor.Preparation.cs + - src/CodeIndex/Indexer/Scanning/ChunkSplitter.cs + - tests/CodeIndex.Tests/ChunkSplitterTests.cs +--- + +## English + +- **Chunk and reference preparation now avoid more full line-string materialization (#3785)** — chunk splitting tracks line start offsets and slices only the persisted chunk bodies, and reference preparation reuses structural line arrays when sanitizer output is unchanged, reducing duplicate allocations for large but valid files while preserving line ranges and trailing-newline behavior. + +## 日本語 + +- **chunk/reference preparation がより多くの全行 string 配列 materialization を避けるようになりました (#3785)** — chunk splitting は行開始 offset を追跡して永続化する chunk 本文だけを切り出し、reference preparation は sanitizer output が変わらない場合に structural line 配列を再利用するため、大きくても有効なファイルでの重複 allocation を減らしつつ、行範囲と末尾改行の挙動を維持します。 diff --git a/changelog.d/unreleased/3800.fixed.md b/changelog.d/unreleased/3800.fixed.md new file mode 100644 index 0000000000..92e2e16fbe --- /dev/null +++ b/changelog.d/unreleased/3800.fixed.md @@ -0,0 +1,19 @@ +--- +category: fixed +issues: + - 3800 +affected: + - src/CodeIndex/Indexer/BoundedRegex.cs + - src/CodeIndex/Indexer/Symbols/SymbolExtractionWorker.cs + - src/CodeIndex/Cli/IndexCommandRunner.FullScan.cs + - src/CodeIndex/Cli/IndexCommandRunner.Update.cs + - src/CodeIndex/Mcp/McpToolHandlers.cs +--- + +## English + +- **Regex timeout fallbacks are now visible in file issues (#3800)** — symbol and reference extraction record bounded `regex_timeout` diagnostics with hashed pattern metadata when a guarded regex falls back to no-match behavior, so skipped matches no longer disappear silently. + +## 日本語 + +- **regex timeout fallback が file issue に表示されるようになりました (#3800)** — symbol 抽出と reference 抽出中に保護された regex が no-match fallback へ切り替わった場合、ハッシュ化した pattern メタデータ付きの bounded な `regex_timeout` 診断を記録し、match の欠落が無言で起きないようにしました。 diff --git a/changelog.d/unreleased/3835.fixed.md b/changelog.d/unreleased/3835.fixed.md new file mode 100644 index 0000000000..8d46ac8486 --- /dev/null +++ b/changelog.d/unreleased/3835.fixed.md @@ -0,0 +1,22 @@ +--- +category: fixed +issues: + - 3835 +affected: + - src/CodeIndex/Cli/IndexCommandRunner.FullScan.cs + - src/CodeIndex/Cli/IndexCommandRunner.Update.cs + - src/CodeIndex/Indexer/Scanning/FileContentLoader.cs + - src/CodeIndex/Indexer/Scanning/FileIndexer.cs + - src/CodeIndex/Mcp/McpToolHandlers.cs + - tests/CodeIndex.Tests/FileIndexerTests.cs + - tests/CodeIndex.Tests/McpServerTests.cs + - USER_GUIDE.md +--- + +## English + +- **File scanning diagnostics now cover more edge cases (#3835)** — Git LFS pointer files keep pointer-identity checksums, BOM-less UTF-16 heuristic decoding reports the chosen encoding, NUL-byte binary skips include the byte offset and persist a `null_byte` file issue during indexing, out-of-policy symlink warnings no longer expose absolute external targets, and extensionless `env -S` shebangs resolve their interpreter. + +## 日本語 + +- **ファイル走査診断がより多くの端ケースを扱うようになりました (#3835)** — Git LFS pointer ファイルは pointer identity に基づく checksum を保持し、BOM なし UTF-16 heuristic decode は選択した encoding を報告し、NUL バイトによる binary skip は byte offset を含めて index 時にも `null_byte` file issue として永続化し、ポリシー外 symlink の warning は外部絶対 target を露出せず、拡張子なし `env -S` shebang は interpreter を解決するようになりました。 diff --git a/src/CodeIndex/Cli/CdidxConfigFile.cs b/src/CodeIndex/Cli/CdidxConfigFile.cs index 3b30c4d49a..6f9f2a2881 100644 --- a/src/CodeIndex/Cli/CdidxConfigFile.cs +++ b/src/CodeIndex/Cli/CdidxConfigFile.cs @@ -56,7 +56,7 @@ internal static class CdidxConfigFile "mcp", }; - private static readonly IReadOnlyList KnownIndexingKeys = new[] { "includeKinds", "excludeKinds" }; + private static readonly IReadOnlyList KnownIndexingKeys = new[] { "includeKinds", "excludeKinds", "generatedCodePatterns" }; private static readonly IReadOnlyList KnownSearchKeys = new[] { "limit", "snippet_lines", "max_line_width" }; private static readonly IReadOnlyList KnownOutputKeys = new[] { "format", "locale" }; private static readonly IReadOnlyList KnownGraphKeys = new[] { "max_hops" }; @@ -277,6 +277,14 @@ private static void AddIndexingEnvironmentSettings(JsonElement root, string path else if (value!.Length > 0) pending.Add((IndexCommandRunner.ExcludeSymbolKindsEnvironmentVariable, string.Join(",", value))); } + + if (indexing.TryGetProperty("generatedCodePatterns", out var generatedCodePatterns)) + { + if (!TryReadStringArray(generatedCodePatterns, "indexing.generatedCodePatterns", path, out var value, out var err)) + errors.Add(err!); + else if (value!.Length > 0) + pending.Add((IndexCommandRunner.GeneratedCodePatternsEnvironmentVariable, string.Join(",", value))); + } } private static void AddSearchEnvironmentSettings(JsonElement root, string path, List<(string EnvName, string Value)> pending, List errors) diff --git a/src/CodeIndex/Cli/CliFlagSchema.cs b/src/CodeIndex/Cli/CliFlagSchema.cs index 812e0c7a1c..de7d4cb2bd 100644 --- a/src/CodeIndex/Cli/CliFlagSchema.cs +++ b/src/CodeIndex/Cli/CliFlagSchema.cs @@ -367,6 +367,7 @@ private static IReadOnlyList BuildAll() new() { Name = "--duration-format", ValuePlaceholder = "", Description = "Index elapsed time display format", Commands = Set("index") }, new() { Name = "--max-file-bytes", ValuePlaceholder = "", Description = "Override the per-file indexing size limit", Commands = Set("index") }, new() { Name = "--max-symbols-per-file", ValuePlaceholder = "", Description = "Skip file content, symbols, and references when one file emits too many symbols (max 50000)", Commands = Set("index") }, + new() { Name = "--max-references-per-file", ValuePlaceholder = "", Description = "Skip references when one file emits too many references (max 1000000)", Commands = Set("index") }, new() { Name = "--parallelism", ValuePlaceholder = "", Description = "Full-scan extraction worker count (default: CPU count capped at 16; also honors CDIDX_INDEX_PARALLELISM)", Commands = Set("index") }, new() { Name = "--memory-trace", Description = "Include phase memory samples in index JSON output", Commands = Set("index") }, new() { Name = "--commits", ValuePlaceholder = "", Description = "Update files changed in given git commits", Commands = Set("index") }, diff --git a/src/CodeIndex/Cli/ConsoleUi.cs b/src/CodeIndex/Cli/ConsoleUi.cs index e220f14d3a..b309d04713 100644 --- a/src/CodeIndex/Cli/ConsoleUi.cs +++ b/src/CodeIndex/Cli/ConsoleUi.cs @@ -80,7 +80,7 @@ public static class ConsoleUi private static readonly (string Command, string Usage)[] CommandUsageLines = [ - ("index", "cdidx index [--db ] [--rebuild] [--optimize] [--symbols-only] [--verbose] [--dry-run] [--force] [--quiet] [--json] [--memory-trace] [--duration-format ] [--notify ] [--max-file-bytes ] [--max-symbols-per-file ] [--follow-symlinks ] [--include-symbol-kind [,]] [--exclude-symbol-kind [,]] [--watch [--debounce ]]"), + ("index", "cdidx index [--db ] [--rebuild] [--optimize] [--symbols-only] [--verbose] [--dry-run] [--force] [--quiet] [--json] [--memory-trace] [--duration-format ] [--notify ] [--max-file-bytes ] [--max-symbols-per-file ] [--max-references-per-file ] [--follow-symlinks ] [--include-symbol-kind [,]] [--exclude-symbol-kind [,]] [--watch [--debounce ]]"), ("hooks", "cdidx hooks [--project ] [--force] [--json]"), ("backfill-fold", "cdidx backfill-fold [--db ] [--dry-run] [--no-checkpoint] [--json]"), ("optimize", "cdidx optimize [--db ] [--json]"), @@ -990,6 +990,7 @@ private static void PrintFlagReference(Action WriteHelpLine) WriteHelpLine(" --notify Long index completion signal: auto, bell, osc9, desktop, or none (also honors CDIDX_NOTIFY; quiet/json suppress it)"); WriteHelpLine(" --max-file-bytes Index only files up to this size (default: 4MiB; also honors CDIDX_MAX_FILE_BYTES; accepts K/M/G suffixes)"); WriteHelpLine(" --max-symbols-per-file Skip file content, symbols, and references when one file emits too many symbols (default: 5000; max: 50000)"); + WriteHelpLine(" --max-references-per-file Skip references when one file emits too many references (default: 100000; max: 1000000)"); WriteHelpLine(" --parallelism Full-scan extraction workers (default: CPU count capped at 16; also honors CDIDX_INDEX_PARALLELISM)"); WriteHelpLine(" --follow-symlinks Symlink policy for directories and files: none (default), internal, or all"); WriteHelpLine(" --include-symbol-kind [,] Keep only matching symbol kinds during indexing"); diff --git a/src/CodeIndex/Cli/IndexCommandRunner.DryRun.cs b/src/CodeIndex/Cli/IndexCommandRunner.DryRun.cs index 0b5e62881d..609e394f25 100644 --- a/src/CodeIndex/Cli/IndexCommandRunner.DryRun.cs +++ b/src/CodeIndex/Cli/IndexCommandRunner.DryRun.cs @@ -20,7 +20,14 @@ private static int RunDryRun( CancellationToken cancellationToken) { var projectPath = options.ProjectPath!; - var dryIndexer = new FileIndexer(projectPath, ignoreCase, ignoreRuleRoot, options.MaxFileSizeBytes, directoryIgnoreCaseProbe: null, symlinkPolicy: options.SymlinkPolicy); + var dryIndexer = new FileIndexer( + projectPath, + ignoreCase, + ignoreRuleRoot, + options.MaxFileSizeBytes, + directoryIgnoreCaseProbe: null, + symlinkPolicy: options.SymlinkPolicy, + generatedCodePatterns: options.GeneratedCodePatterns); IReadOnlyList dryCandidates; IReadOnlyList dryDeleteCandidates; bool authoritativeFullScan; diff --git a/src/CodeIndex/Cli/IndexCommandRunner.FullScan.cs b/src/CodeIndex/Cli/IndexCommandRunner.FullScan.cs index 72301d7463..a9b8bd8844 100644 --- a/src/CodeIndex/Cli/IndexCommandRunner.FullScan.cs +++ b/src/CodeIndex/Cli/IndexCommandRunner.FullScan.cs @@ -47,6 +47,85 @@ private static FileIssue BuildSymbolCountExceededIssue(string path, int symbolCo Message = $"Symbol extraction produced {symbolCount:N0} symbols, exceeding the --max-symbols-per-file limit of {maxSymbolsPerFile:N0}; file content, symbols, and references were not indexed. Exclude the generated/pathological file or raise --max-symbols-per-file if this is expected.", }; + private static FileIssue BuildReferenceCountExceededIssue(string path, int referenceCount, int maxReferencesPerFile) => + new() + { + Path = path, + Kind = "reference_count_exceeded", + Line = 0, + Message = $"Reference extraction produced {referenceCount:N0} references, exceeding the --max-references-per-file limit of {maxReferencesPerFile:N0}; references were not indexed for this file. Exclude the generated/pathological file or raise --max-references-per-file if this is expected.", + }; + + internal static FileIssue BuildNullByteIssue(FileIndexer.BinaryFileSkippedException ex) => + new() + { + Path = ex.RelativePath, + Kind = "null_byte", + Line = 0, + Message = ex.Message, + }; + + internal static FileIssue? BuildRegexTimeoutIssue(string path, BoundedRegex.RegexTimeoutCaptureScope capture) => + BuildRegexTimeoutIssue( + path, + capture.Language, + capture.PatternFamily, + capture.TimeoutCount, + capture.Diagnostics, + capture.DiagnosticsTruncated); + + internal static FileIssue? BuildRegexTimeoutIssue( + string path, + string? language, + string patternFamily, + int timeoutCount, + IReadOnlyList diagnostics, + bool diagnosticsTruncated) + { + if (timeoutCount <= 0) + return null; + + var normalizedLanguage = string.IsNullOrWhiteSpace(language) ? "unknown" : language; + var samples = diagnostics.Count == 0 + ? "none" + : string.Join(", ", diagnostics.Select(static diagnostic => + $"{diagnostic.Operation}:{diagnostic.PatternHash} len={diagnostic.PatternLength} timeout={diagnostic.TimeoutMs:0.###}ms")); + var truncationSuffix = diagnosticsTruncated ? "; additional timeout diagnostics omitted" : string.Empty; + return new FileIssue + { + Path = path, + Kind = "regex_timeout", + Line = 0, + Message = $"Regex timeout fallback occurred during {patternFamily} for language {normalizedLanguage} ({timeoutCount:N0} timeout(s); samples {samples}{truncationSuffix}); extraction used a safe no-match fallback and may be incomplete for this file.", + }; + } + + private static bool ExistingFileViolatesExtractionCaps(DbWriter writer, long fileId, int maxSymbolsPerFile, int maxReferencesPerFile) => + writer.CountSymbolsForFile(fileId) > maxSymbolsPerFile + || writer.HasIssueForFile(fileId, "symbol_count_exceeded") + || writer.CountReferencesForFile(fileId) > maxReferencesPerFile + || writer.HasIssueForFile(fileId, "reference_count_exceeded"); + + internal static bool ExistingFileGeneratedSuppressionMismatch(DbWriter writer, long fileId, FileIssue? generatedSuppressionIssue) + => writer.HasIssueForFile(fileId, FileIndexer.GeneratedCodeExtractionSkippedIssueKind) != (generatedSuppressionIssue != null); + + internal static IReadOnlyList AppendIssue(IReadOnlyList issues, FileIssue issue) + { + if (issues.Count == 0) + return [issue]; + + var combined = issues.ToList(); + combined.Add(issue); + return combined; + } + + internal static IReadOnlyList AppendIssueIfMissing(IReadOnlyList issues, FileIssue issue) + { + if (issues.Any(existing => string.Equals(existing.Kind, issue.Kind, StringComparison.Ordinal))) + return issues; + return AppendIssue(issues, issue); + } + internal static string FormatIndexPhasePath(string path, string phase) => $"{path} ({phase})"; @@ -103,19 +182,26 @@ private static void ThrowIfFullScanExtractionStalled( throw new IndexExtractionStalledException(filesProcessed, filesTotal, timeout, activePath); } - private static List ExtractSymbolsWithStallTimeout( + private sealed record SymbolExtractionResult(List Symbols, FileIssue? RegexTimeoutIssue); + + private static SymbolExtractionResult ExtractSymbolsWithStallTimeout( long fileId, string? lang, string content, string filePath, string projectRoot, + string issuePath, string phasePath, SymbolExtractionWorkerClient worker, CancellationToken cancellationToken) { var timeout = IndexExtractionStallTimeoutForTesting?.Invoke() ?? IndexExtractionStallTimeout; if (timeout <= TimeSpan.Zero) - return SymbolExtractor.Extract(fileId, lang, content, filePath, projectRoot, cancellationToken); + { + using var regexTimeouts = BoundedRegex.CaptureTimeouts(lang, "symbol_extraction"); + var symbols = SymbolExtractor.Extract(fileId, lang, content, filePath, projectRoot, cancellationToken); + return new SymbolExtractionResult(symbols, BuildRegexTimeoutIssue(issuePath, regexTimeouts)); + } cancellationToken.ThrowIfCancellationRequested(); var result = worker.Invoke(fileId, lang, content, filePath, projectRoot, timeout, cancellationToken); @@ -125,7 +211,14 @@ private static List ExtractSymbolsWithStallTimeout( if (!result.Success) throw new InvalidOperationException(result.WorkerError ?? "isolated symbol extraction worker failed."); - return result.Symbols ?? []; + var regexTimeoutIssue = BuildRegexTimeoutIssue( + issuePath, + lang, + "symbol_extraction", + result.RegexTimeoutCount, + result.RegexTimeoutDiagnostics ?? [], + result.RegexTimeoutDiagnosticsTruncated); + return new SymbolExtractionResult(result.Symbols ?? [], regexTimeoutIssue); } private static string CollapseLineBreaks(string value) @@ -273,66 +366,109 @@ ex is UnauthorizedAccessException internal const int MaxScanCheckpointDirectories = 4096; internal const int MaxScanCheckpointDirectoryLength = 4096; - internal static IReadOnlySet LoadScanCheckpoint(string path, string? currentHead) + internal static IReadOnlySet LoadScanCheckpoint(string path, string? currentHead) => + LoadScanCheckpointDetailed(path, currentHead).Directories; + + internal static ScanCheckpointLoadResult LoadScanCheckpointDetailed(string path, string? currentHead) { try { - if (string.IsNullOrWhiteSpace(currentHead) || !File.Exists(path)) - return EmptyScanCheckpointDirectories(); + if (!File.Exists(path)) + return EmptyScanCheckpointLoadResult(); + if (string.IsNullOrWhiteSpace(currentHead)) + return IgnoredScanCheckpoint(path, "current Git HEAD is unavailable"); var text = DataDirectorySecurity.ReadTextWithinLimit(path, MaxScanCheckpointBytes, FileShare.ReadWrite); if (text is null) - return EmptyScanCheckpointDirectories(); + return IgnoredScanCheckpoint(path, $"file exceeds the scan checkpoint size limit of {MaxScanCheckpointBytes:N0} bytes"); var checkpoint = JsonSerializer.Deserialize( text, new JsonSerializerOptions { MaxDepth = MaxScanCheckpointJsonDepth }); - if (checkpoint is not { Version: ScanCheckpointVersion } - || !string.Equals(checkpoint.GitHead, currentHead, StringComparison.Ordinal) - || !TryBuildScanCheckpointDirectories(checkpoint.Directories, out var directories)) - { - return EmptyScanCheckpointDirectories(); - } - - return directories; + if (checkpoint is null) + return IgnoredScanCheckpoint(path, "JSON root is null or not a scan checkpoint object"); + if (checkpoint.Version != ScanCheckpointVersion) + return IgnoredScanCheckpoint(path, FormatScanCheckpointVersionMismatch(checkpoint.Version)); + if (!string.Equals(checkpoint.GitHead, currentHead, StringComparison.Ordinal)) + return IgnoredScanCheckpoint(path, "checkpoint GitHead does not match current HEAD; checkpoint is stale"); + if (!TryBuildScanCheckpointDirectories(checkpoint.Directories, out var directories, out var directoryFailureReason)) + return IgnoredScanCheckpoint(path, directoryFailureReason); + + return new ScanCheckpointLoadResult(directories, WarningMessage: null); } - catch (JsonException) + catch (JsonException ex) { - return EmptyScanCheckpointDirectories(); + return IgnoredScanCheckpoint( + path, + $"malformed checkpoint JSON or depth exceeds {MaxScanCheckpointJsonDepth:N0} ({CommandErrorWriter.FormatSanitizedException(ex)})"); } - catch (IOException) + catch (IOException ex) { - return EmptyScanCheckpointDirectories(); + return IgnoredScanCheckpoint(path, $"read failed ({CommandErrorWriter.FormatSanitizedException(ex)})"); } - catch (UnauthorizedAccessException) + catch (UnauthorizedAccessException ex) { - return EmptyScanCheckpointDirectories(); + return IgnoredScanCheckpoint(path, $"read failed ({CommandErrorWriter.FormatSanitizedException(ex)})"); } } - private static bool TryBuildScanCheckpointDirectories(IReadOnlyList? rawDirectories, out IReadOnlySet directories) + private static string FormatScanCheckpointVersionMismatch(int version) => + version > ScanCheckpointVersion + ? $"future checkpoint version {version:N0} exceeds supported version {ScanCheckpointVersion:N0}" + : $"unsupported checkpoint version {version:N0}; supported version is {ScanCheckpointVersion:N0}"; + + private static ScanCheckpointLoadResult EmptyScanCheckpointLoadResult() => + new(EmptyScanCheckpointDirectories(), WarningMessage: null); + + private static ScanCheckpointLoadResult IgnoredScanCheckpoint(string path, string reason) => + new( + EmptyScanCheckpointDirectories(), + $"scan checkpoint ignored for {ConsoleUi.FormatBoundedValue(path)}: {reason}; continuing with a full scan."); + + private static bool TryBuildScanCheckpointDirectories( + IReadOnlyList? rawDirectories, + out IReadOnlySet directories, + out string failureReason) { directories = EmptyScanCheckpointDirectories(); + failureReason = string.Empty; if (rawDirectories is not { Count: > 0 }) + { + failureReason = "Directories must be a non-empty JSON array"; return false; + } if (rawDirectories.Count > MaxScanCheckpointDirectories) + { + failureReason = + $"Directories contains {rawDirectories.Count:N0} entries, exceeding the limit of {MaxScanCheckpointDirectories:N0}"; return false; + } var result = new HashSet(StringComparer.Ordinal); foreach (var directory in rawDirectories) { if (directory is null) + { + failureReason = "Directories contains a null entry"; return false; + } if (directory.Length == 0) continue; if (directory.Length > MaxScanCheckpointDirectoryLength) + { + failureReason = + $"Directories contains an entry longer than {MaxScanCheckpointDirectoryLength:N0} characters"; return false; + } result.Add(directory); } if (result.Count == 0) + { + failureReason = "Directories contains only empty entries"; return false; + } directories = result; return true; @@ -459,7 +595,8 @@ void ThrowIfDiscoveryCancelled() } var scanCheckpointPath = Path.Combine(projectRoot, ".cdidx", ScanCheckpointFileName); - var checkpointedDirectories = LoadScanCheckpoint(scanCheckpointPath, currentHeadForCheckpoint); + var checkpointLoadResult = LoadScanCheckpointDetailed(scanCheckpointPath, currentHeadForCheckpoint); + var checkpointedDirectories = checkpointLoadResult.Directories; WriteFullScanJsonLiveness(options, "scanning files..."); var scanHeartbeat = StartFullScanJsonPhaseHeartbeat(options, "scanning files"); FileIndexer.ScanFilesResult scanResult; @@ -492,11 +629,15 @@ void ThrowIfDiscoveryCancelled() var warningList = warningScanErrors .Select(error => new CliJsonMessage(error.Path, error.Message)) .ToList(); + if (checkpointLoadResult.WarningMessage != null) + warningList.Add(new CliJsonMessage("", checkpointLoadResult.WarningMessage)); if (!options.Json && !options.Quiet) { Console.WriteLine($" Found {ConsoleUi.Counted(files.Count, "file", format: "N0")}"); foreach (var error in scanResult.Errors) ConsoleUi.PrintWarning($"{error.Path}: {error.Message}"); + if (checkpointLoadResult.WarningMessage != null) + ConsoleUi.PrintWarning(checkpointLoadResult.WarningMessage); Console.WriteLine(); } @@ -1013,29 +1154,50 @@ void StopJsonHeartbeat() IReadOnlyList? symbols = null; IReadOnlyList? references = null; IReadOnlyList? issues = null; + var generatedSuppressionIssue = indexer.BuildGeneratedCodeExtractionSkippedIssue(record.Path); if (parallelizeExtraction) { activeJsonExtractionPhases[workerIndex] = FormatIndexPhasePath(record.Path, "chunking"); chunks = ChunkSplitter.Split(0, content); + if (generatedSuppressionIssue != null) + { + symbols = []; + references = []; + activeJsonExtractionPhases[workerIndex] = FormatIndexPhasePath(record.Path, "validating"); + issues = AppendIssueIfMissing( + FileIndexer.ValidateContent(record.Path, rawBytes, content, record.Lang), + generatedSuppressionIssue); + extractionResults.Add( + FullScanFileWorkItem.Success(filePath, record, content, rawBytes, warning, chunks, symbols, references, issues), + extractionCancellationToken); + continue; + } activeJsonExtractionPhases[workerIndex] = FormatIndexPhasePath(record.Path, "symbols"); - symbols = ExtractSymbolsWithStallTimeout( + var symbolExtraction = ExtractSymbolsWithStallTimeout( 0, record.Lang, content, filePath, Path.GetFullPath(options.ProjectPath!), + record.Path, activeJsonExtractionPhases[workerIndex], workerSymbolExtractionWorker, extractionCancellationToken); + symbols = symbolExtraction.Symbols; + var symbolRegexTimeoutIssue = symbolExtraction.RegexTimeoutIssue; if (symbols.Count > options.MaxSymbolsPerFile) { var issue = BuildSymbolCountExceededIssue(record.Path, symbols.Count, options.MaxSymbolsPerFile); + IReadOnlyList capIssues = symbolRegexTimeoutIssue == null + ? [issue] + : AppendIssue([symbolRegexTimeoutIssue], issue); extractionResults.Add( - FullScanFileWorkItem.Success(filePath, record, string.Empty, rawBytes, issue.Message, [], [], [], [issue]), + FullScanFileWorkItem.Success(filePath, record, string.Empty, rawBytes, issue.Message, [], [], [], capIssues), extractionCancellationToken); continue; } SymbolExtractor.ApplyFamilyScope(symbols, indexer.GetFamilyScopeKey(filePath, record.Lang)); + FileIssue? referenceRegexTimeoutIssue = null; if (options.SymbolsOnly) { references = []; @@ -1043,6 +1205,7 @@ void StopJsonHeartbeat() else { activeJsonExtractionPhases[workerIndex] = FormatIndexPhasePath(record.Path, "references"); + using var regexTimeouts = BoundedRegex.CaptureTimeouts(record.Lang, "reference_extraction"); references = ReferenceExtractor.Extract( 0, record.Lang, @@ -1050,10 +1213,22 @@ void StopJsonHeartbeat() symbols, record.Path, record.Lang == "csharp" ? csharpWorkspace.Symbols : null, - extractionCancellationToken); + extractionCancellationToken, + maxReferenceCount: options.MaxReferencesPerFile + 1); + referenceRegexTimeoutIssue = BuildRegexTimeoutIssue(record.Path, regexTimeouts); } activeJsonExtractionPhases[workerIndex] = FormatIndexPhasePath(record.Path, "validating"); issues = FileIndexer.ValidateContent(record.Path, rawBytes, content, record.Lang); + if (symbolRegexTimeoutIssue != null) + issues = AppendIssue(issues, symbolRegexTimeoutIssue); + if (referenceRegexTimeoutIssue != null) + issues = AppendIssue(issues, referenceRegexTimeoutIssue); + if (references.Count > options.MaxReferencesPerFile) + { + var issue = BuildReferenceCountExceededIssue(record.Path, references.Count, options.MaxReferencesPerFile); + references = []; + issues = AppendIssue(issues, issue); + } } extractionResults.Add( FullScanFileWorkItem.Success(filePath, record, content, rawBytes, warning, chunks, symbols, references, issues), @@ -1065,7 +1240,11 @@ void StopJsonHeartbeat() } catch (FileIndexer.BinaryFileSkippedException ex) { - extractionResults.Add(FullScanFileWorkItem.Skipped(filePath, ex.Message), extractionCancellationToken); + var record = indexer.BuildSkippedFileRecord(filePath); + var issue = BuildNullByteIssue(ex); + extractionResults.Add( + FullScanFileWorkItem.Success(filePath, record, string.Empty, [], ex.Message, [], [], [], [issue]), + extractionCancellationToken); } catch (FileIndexer.FileTooLargeSkippedException ex) { @@ -1202,13 +1381,15 @@ void StopJsonHeartbeat() && (record.Lang != "sql" || sqlGraphContractMatchesCurrent) && AllowReuseWithCurrentHotspotFamilyTrust(record.Lang, hotspotFamilyTrustMatchesCurrent)); } - if (existingId != null) + if (existingId != null + && ExistingFileViolatesExtractionCaps(writer, existingId.Value, options.MaxSymbolsPerFile, options.MaxReferencesPerFile)) { - if (writer.CountSymbolsForFile(existingId.Value) > options.MaxSymbolsPerFile - || writer.HasIssueForFile(existingId.Value, "symbol_count_exceeded")) - { - existingId = null; - } + existingId = null; + } + if (existingId != null + && ExistingFileGeneratedSuppressionMismatch(writer, existingId.Value, indexer.BuildGeneratedCodeExtractionSkippedIssue(record.Path))) + { + existingId = null; } if (existingId != null) { @@ -1245,24 +1426,57 @@ void StopJsonHeartbeat() var chunks = item.Chunks == null ? ChunkSplitter.Split(fileId, item.Content!) : ReassignChunkFileIds(item.Chunks, fileId); + var generatedSuppressionIssue = indexer.BuildGeneratedCodeExtractionSkippedIssue(record.Path); + if (generatedSuppressionIssue != null) + { + writer.InsertChunks(chunks); + writer.InsertSymbols([]); + writer.InsertReferences([]); + var generatedIssues = AppendIssueIfMissing( + item.Issues ?? FileIndexer.ValidateContent(record.Path, item.RawBytes!, item.Content!, record.Lang), + generatedSuppressionIssue); + writer.InsertIssues(fileId, generatedIssues); + if (options.Verbose) + WriteIndexVerboseStatus($" [OK ] {record.Path} ({chunks.Count} chunks, generated-code extraction skipped)"); + currentJsonIndexFile = FormatIndexPhasePath(record.Path, "committing"); + WriteProjectRootOnce(); + txn.Commit(); + + processed++; + if (!options.Json && !options.Quiet) + { + PauseIndexSpinnerForConsoleWrite(); + ConsoleUi.PrintProgress(processed, files.Count); + ResumeIndexSpinnerAfterConsoleWrite(); + } + ReportJsonIndexProgressIfNeeded(); + currentJsonIndexFile = null; + continue; + } currentJsonIndexFile = FormatIndexPhasePath(record.Path, "symbols"); + SymbolExtractionResult? symbolExtraction = null; var symbols = item.Symbols == null - ? ExtractSymbolsWithStallTimeout( + ? (symbolExtraction = ExtractSymbolsWithStallTimeout( fileId, record.Lang, item.Content!, item.FilePath, Path.GetFullPath(options.ProjectPath!), + record.Path, currentJsonIndexFile, mainSymbolExtractionWorker, - cancellationToken) + cancellationToken)).Symbols : ReassignSymbolFileIds(item.Symbols, fileId); + var symbolRegexTimeoutIssue = symbolExtraction?.RegexTimeoutIssue; if (symbols.Count > options.MaxSymbolsPerFile) { var issue = BuildSymbolCountExceededIssue(record.Path, symbols.Count, options.MaxSymbolsPerFile); + IReadOnlyList capIssues = symbolRegexTimeoutIssue == null + ? [issue] + : AppendIssue([symbolRegexTimeoutIssue], issue); writer.InsertSymbols([]); writer.InsertReferences([]); - writer.InsertIssues(fileId, [issue]); + writer.InsertIssues(fileId, capIssues); if (options.Verbose) WriteIndexVerboseStatus($" [SKIP] {record.Path} ({issue.Message})"); txn.Commit(); @@ -1287,9 +1501,12 @@ void StopJsonHeartbeat() if (symbols.Count > options.MaxSymbolsPerFile) { var issue = BuildSymbolCountExceededIssue(record.Path, symbols.Count, options.MaxSymbolsPerFile); + IReadOnlyList capIssues = symbolRegexTimeoutIssue == null + ? [issue] + : AppendIssue([symbolRegexTimeoutIssue], issue); writer.InsertSymbols([]); writer.InsertReferences([]); - writer.InsertIssues(fileId, [issue]); + writer.InsertIssues(fileId, capIssues); if (options.Verbose) WriteIndexVerboseStatus($" [SKIP] {record.Path} ({issue.Message})"); txn.Commit(); @@ -1307,6 +1524,11 @@ void StopJsonHeartbeat() writer.InsertChunks(chunks); FileIndexer.ValidateSymbolLineRanges(record, symbols); writer.InsertSymbols(symbols); + if (symbolRegexTimeoutIssue != null) + { + var baseIssues = item.Issues ?? FileIndexer.ValidateContent(record.Path, item.RawBytes!, item.Content!, record.Lang); + item = item with { Issues = AppendIssue(baseIssues, symbolRegexTimeoutIssue) }; + } currentJsonIndexFile = FormatIndexPhasePath(record.Path, "references"); IReadOnlyList references; if (options.SymbolsOnly) @@ -1315,17 +1537,38 @@ void StopJsonHeartbeat() } else { - references = item.References == null - ? ReferenceExtractor.Extract( + FileIssue? regexTimeoutIssue = null; + if (item.References == null) + { + using var regexTimeouts = BoundedRegex.CaptureTimeouts(record.Lang, "reference_extraction"); + references = ReferenceExtractor.Extract( fileId, record.Lang, item.Content!, symbols, record.Path, record.Lang == "csharp" ? csharpWorkspace.Symbols : null, - cancellationToken) - : ReassignReferenceFileIds(item.References, fileId); + cancellationToken, + maxReferenceCount: options.MaxReferencesPerFile + 1); + regexTimeoutIssue = BuildRegexTimeoutIssue(record.Path, regexTimeouts); + } + else + { + references = ReassignReferenceFileIds(item.References, fileId); + } postExtractionHooks.OnReferencesExtracted(fileContext, AsMutableList(references)); + if (regexTimeoutIssue != null) + { + var baseIssues = item.Issues ?? FileIndexer.ValidateContent(record.Path, item.RawBytes!, item.Content!, record.Lang); + item = item with { Issues = AppendIssue(baseIssues, regexTimeoutIssue) }; + } + if (references.Count > options.MaxReferencesPerFile) + { + var issue = BuildReferenceCountExceededIssue(record.Path, references.Count, options.MaxReferencesPerFile); + references = []; + var baseIssues = item.Issues ?? FileIndexer.ValidateContent(record.Path, item.RawBytes!, item.Content!, record.Lang); + item = item with { Issues = AppendIssue(baseIssues, issue) }; + } } writer.InsertReferences(references, refreshMutualRecursionFlags: false); if (references.Count > 0) diff --git a/src/CodeIndex/Cli/IndexCommandRunner.Parse.cs b/src/CodeIndex/Cli/IndexCommandRunner.Parse.cs index 009c78ff9b..664156ee38 100644 --- a/src/CodeIndex/Cli/IndexCommandRunner.Parse.cs +++ b/src/CodeIndex/Cli/IndexCommandRunner.Parse.cs @@ -14,7 +14,7 @@ public static partial class IndexCommandRunner [ "--db", "--data-dir", "--rebuild", "--verbose", "--json", "--quiet", "--dry-run", "--force", "--yes", "--watch", "--debounce", "--duration-format", "--max-file-bytes", "--max-symbols-per-file", - "--notify", + "--max-references-per-file", "--notify", "--parallelism", "--memory-trace", "--follow-symlinks", "--symbols-only", "--commits", "--changed-between", "--files", "--solution", "--project", "--include-symbol-kind", "--exclude-symbol-kind", "--optimize", "--help", @@ -49,6 +49,7 @@ public static IndexCommandOptions ParseArgs(string[] args) var notifyMode = ReadCompletionNotificationModeFromEnvironment(); long? maxFileSizeBytes = ReadMaxFileSizeBytesFromEnvironment(); var maxSymbolsPerFile = DefaultMaxSymbolsPerFile; + var maxReferencesPerFile = DefaultMaxReferencesPerFile; var parallelism = ReadIndexParallelismFromEnvironment(); var symlinkPolicy = FileIndexer.SymlinkPolicy.None; string? easterEgg = null; @@ -67,6 +68,8 @@ public static IndexCommandOptions ParseArgs(string[] args) string? symbolKindFilterError = null; var includeSymbolKindsSpecifiedOnCli = false; var excludeSymbolKindsSpecifiedOnCli = false; + string? generatedCodePatternError = null; + var generatedCodePatterns = ReadGeneratedCodePatternsFromEnvironment(ref generatedCodePatternError); AddSymbolKindFilterValues( IncludeSymbolKindsEnvironmentVariable, @@ -170,6 +173,12 @@ public static IndexCommandOptions ParseArgs(string[] args) case var option when option.StartsWith("--max-symbols-per-file=", StringComparison.Ordinal): maxSymbolsPerFile = ParseMaxSymbolsPerFile(option["--max-symbols-per-file=".Length..], maxSymbolsPerFile, "--max-symbols-per-file", ref parseError); break; + case "--max-references-per-file" when i + 1 < args.Length: + maxReferencesPerFile = ParseMaxReferencesPerFile(args[++i], maxReferencesPerFile, "--max-references-per-file", ref parseError); + break; + case var option when option.StartsWith("--max-references-per-file=", StringComparison.Ordinal): + maxReferencesPerFile = ParseMaxReferencesPerFile(option["--max-references-per-file=".Length..], maxReferencesPerFile, "--max-references-per-file", ref parseError); + break; case "--parallelism" when i + 1 < args.Length: parallelism = ParseIndexParallelism(args[++i], parallelism, "--parallelism"); break; @@ -313,7 +322,7 @@ public static IndexCommandOptions ParseArgs(string[] args) ProjectFilters = projectFilters, SolutionPath = solutionPath, ProjectFilterError = projectFilterError, - ParseError = parseError, + ParseError = parseError ?? generatedCodePatternError, EasterEgg = easterEgg, DryRun = dryRun, Force = force, @@ -328,12 +337,43 @@ public static IndexCommandOptions ParseArgs(string[] args) NotifyMode = notifyMode, MaxFileSizeBytes = maxFileSizeBytes, MaxSymbolsPerFile = maxSymbolsPerFile, + MaxReferencesPerFile = maxReferencesPerFile, Parallelism = parallelism, SymlinkPolicy = symlinkPolicy, SymbolKindFilter = SymbolKindFilter.Create(includeSymbolKinds, excludeSymbolKinds, symbolKindFilterError), + GeneratedCodePatterns = generatedCodePatterns, }; } + internal static IReadOnlyList ReadGeneratedCodePatternsFromEnvironment() + { + string? parseError = null; + return ReadGeneratedCodePatternsFromEnvironment(ref parseError); + } + + private static IReadOnlyList ReadGeneratedCodePatternsFromEnvironment(ref string? parseError) + { + var value = CdidxEnvironment.GetEnvironmentVariable(GeneratedCodePatternsEnvironmentVariable); + if (string.IsNullOrWhiteSpace(value)) + return []; + if (!ValidateCsvBounds(GeneratedCodePatternsEnvironmentVariable, value, MaxGeneratedCodePatternCsvLength, MaxGeneratedCodePatternCount, ref parseError)) + return []; + + var patterns = new List(); + foreach (var raw in value.Split(',', StringSplitOptions.TrimEntries)) + { + if (raw.Length == 0) + { + parseError ??= $"{GeneratedCodePatternsEnvironmentVariable} contains an empty generated-code pattern"; + continue; + } + + patterns.Add(raw); + } + + return patterns; + } + private static FileIndexer.SymlinkPolicy ParseSymlinkPolicy(string value, FileIndexer.SymlinkPolicy fallback, ref string? parseError) { switch (value.Trim().ToLowerInvariant()) @@ -510,6 +550,22 @@ private static int ParseMaxSymbolsPerFile(string value, int fallback, string sou return fallback; } + private static int ParseMaxReferencesPerFile(string value, int fallback, string source, ref string? parseError) + { + if (int.TryParse(value, System.Globalization.NumberStyles.Integer, System.Globalization.CultureInfo.InvariantCulture, out var parsed) && parsed > 0) + { + if (parsed <= MaxReferencesPerFileLimit) + return parsed; + + parseError ??= $"{source} must be less than or equal to {MaxReferencesPerFileLimit}"; + return fallback; + } + + var displayValue = ConsoleUi.FormatBoundedValue(value); + Console.Error.WriteLine($"Warning: invalid {source} value '{displayValue}' (ignored; use a positive integer) / 不正な {source} 値 '{displayValue}'（無視。正の整数を指定）"); + return fallback; + } + private static DurationOutputFormat ParseDurationFormat(string value, DurationOutputFormat fallback) { return value.Trim().ToLowerInvariant() switch diff --git a/src/CodeIndex/Cli/IndexCommandRunner.Update.cs b/src/CodeIndex/Cli/IndexCommandRunner.Update.cs index 9362c10904..7f7269ce8b 100644 --- a/src/CodeIndex/Cli/IndexCommandRunner.Update.cs +++ b/src/CodeIndex/Cli/IndexCommandRunner.Update.cs @@ -579,13 +579,15 @@ void ThrowIfUpdateCancelled() && (statReusableLanguage != "csharp" || csharpSymbolNameContractMatchesCurrent) && (statReusableLanguage != "csharp" || !csharpWorkspace.HasStaticInterfaceContracts) && (statReusableLanguage != "sql" || sqlGraphContractMatchesCurrent)); - if (statMatchedId != null) + if (statMatchedId != null + && ExistingFileViolatesExtractionCaps(writer, statMatchedId.Value, options.MaxSymbolsPerFile, options.MaxReferencesPerFile)) { - if (writer.CountSymbolsForFile(statMatchedId.Value) > options.MaxSymbolsPerFile - || writer.HasIssueForFile(statMatchedId.Value, "symbol_count_exceeded")) - { - statMatchedId = null; - } + statMatchedId = null; + } + if (statMatchedId != null + && ExistingFileGeneratedSuppressionMismatch(writer, statMatchedId.Value, indexer.BuildGeneratedCodeExtractionSkippedIssue(dbPath))) + { + statMatchedId = null; } if (statMatchedId != null) { @@ -621,13 +623,15 @@ void ThrowIfUpdateCancelled() && (record.Lang != "csharp" || csharpSymbolNameContractMatchesCurrent) && (record.Lang != "csharp" || !csharpWorkspace.HasStaticInterfaceContracts) && (record.Lang != "sql" || sqlGraphContractMatchesCurrent)); - if (existingId != null) + if (existingId != null + && ExistingFileViolatesExtractionCaps(writer, existingId.Value, options.MaxSymbolsPerFile, options.MaxReferencesPerFile)) { - if (writer.CountSymbolsForFile(existingId.Value) > options.MaxSymbolsPerFile - || writer.HasIssueForFile(existingId.Value, "symbol_count_exceeded")) - { - existingId = null; - } + existingId = null; + } + if (existingId != null + && ExistingFileGeneratedSuppressionMismatch(writer, existingId.Value, indexer.BuildGeneratedCodeExtractionSkippedIssue(record.Path))) + { + existingId = null; } if (existingId != null) { @@ -667,22 +671,48 @@ void ThrowIfUpdateCancelled() var fileId = writer.UpsertFile(record); currentUpdatePath = FormatIndexPhasePath(relPath, "chunking"); var chunks = ChunkSplitter.Split(fileId, content); + var generatedSuppressionIssue = indexer.BuildGeneratedCodeExtractionSkippedIssue(record.Path); + if (generatedSuppressionIssue != null) + { + writer.InsertChunks(chunks); + writer.InsertSymbols([]); + writer.InsertReferences([]); + currentUpdatePath = FormatIndexPhasePath(relPath, "validating"); + var generatedIssues = AppendIssueIfMissing( + FileIndexer.ValidateContent(record.Path, rawBytes, content, record.Lang), + generatedSuppressionIssue); + writer.InsertIssues(fileId, generatedIssues); + currentUpdatePath = FormatIndexPhasePath(relPath, "committing"); + writer.ClearBatchInProgress(); + txn.Commit(); + fileBatchMarked = false; + updated++; + ftsMutated = true; + WriteUpdateVerboseStatus($" [OK ] {relPath} ({chunks.Count} chunks, generated-code extraction skipped)"); + continue; + } currentUpdatePath = FormatIndexPhasePath(relPath, "symbols"); - var symbols = ExtractSymbolsWithStallTimeout( + var symbolExtraction = ExtractSymbolsWithStallTimeout( fileId, record.Lang, content, absPath, Path.GetFullPath(options.ProjectPath!), + record.Path, currentUpdatePath, symbolExtractionWorker, cancellationToken); + var symbols = symbolExtraction.Symbols; + var symbolRegexTimeoutIssue = symbolExtraction.RegexTimeoutIssue; if (symbols.Count > options.MaxSymbolsPerFile) { var issue = BuildSymbolCountExceededIssue(record.Path, symbols.Count, options.MaxSymbolsPerFile); + IReadOnlyList capIssues = symbolRegexTimeoutIssue == null + ? [issue] + : AppendIssue([symbolRegexTimeoutIssue], issue); writer.InsertSymbols([]); writer.InsertReferences([]); - writer.InsertIssues(fileId, [issue]); + writer.InsertIssues(fileId, capIssues); writer.ClearBatchInProgress(); txn.Commit(); fileBatchMarked = false; @@ -698,9 +728,12 @@ void ThrowIfUpdateCancelled() if (symbols.Count > options.MaxSymbolsPerFile) { var issue = BuildSymbolCountExceededIssue(record.Path, symbols.Count, options.MaxSymbolsPerFile); + IReadOnlyList capIssues = symbolRegexTimeoutIssue == null + ? [issue] + : AppendIssue([symbolRegexTimeoutIssue], issue); writer.InsertSymbols([]); writer.InsertReferences([]); - writer.InsertIssues(fileId, [issue]); + writer.InsertIssues(fileId, capIssues); writer.ClearBatchInProgress(); txn.Commit(); fileBatchMarked = false; @@ -713,19 +746,38 @@ void ThrowIfUpdateCancelled() FileIndexer.ValidateSymbolLineRanges(record, symbols); writer.InsertSymbols(symbols); currentUpdatePath = FormatIndexPhasePath(relPath, "references"); - var references = ReferenceExtractor.Extract( - fileId, - record.Lang, - content, - symbols, - record.Path, - record.Lang == "csharp" ? csharpWorkspace.Symbols : null, - cancellationToken); + List references; + FileIssue? referenceRegexTimeoutIssue; + using (var regexTimeouts = BoundedRegex.CaptureTimeouts(record.Lang, "reference_extraction")) + { + references = ReferenceExtractor.Extract( + fileId, + record.Lang, + content, + symbols, + record.Path, + record.Lang == "csharp" ? csharpWorkspace.Symbols : null, + cancellationToken, + maxReferenceCount: options.MaxReferencesPerFile + 1); + referenceRegexTimeoutIssue = BuildRegexTimeoutIssue(record.Path, regexTimeouts); + } postExtractionHooks.OnReferencesExtracted(fileContext, references); + FileIssue? referenceCapIssue = null; + if (references.Count > options.MaxReferencesPerFile) + { + referenceCapIssue = BuildReferenceCountExceededIssue(record.Path, references.Count, options.MaxReferencesPerFile); + references = []; + } writer.InsertReferences(references); // Validate content for encoding issues / エンコーディング問題を検証 currentUpdatePath = FormatIndexPhasePath(relPath, "validating"); - var issues = FileIndexer.ValidateContent(record.Path, rawBytes, content, record.Lang); + IReadOnlyList issues = FileIndexer.ValidateContent(record.Path, rawBytes, content, record.Lang); + if (symbolRegexTimeoutIssue != null) + issues = AppendIssue(issues, symbolRegexTimeoutIssue); + if (referenceRegexTimeoutIssue != null) + issues = AppendIssue(issues, referenceRegexTimeoutIssue); + if (referenceCapIssue != null) + issues = AppendIssue(issues, referenceCapIssue); writer.InsertIssues(fileId, issues); currentUpdatePath = FormatIndexPhasePath(relPath, "committing"); writer.ClearBatchInProgress(); @@ -742,8 +794,11 @@ void ThrowIfUpdateCancelled() } catch (Exception ex) { - if (ex is FileIndexer.BinaryFileSkippedException) + if (ex is FileIndexer.BinaryFileSkippedException binaryFile) { + if (fileBatchMarked) + writer.ClearBatchInProgress(); + warnings++; warningList.Add(new CliJsonMessage(relPath, ex.Message)); if (!options.Json && !options.Quiet) @@ -753,22 +808,24 @@ void ThrowIfUpdateCancelled() ResumeUpdateSpinnerAfterConsoleWrite(); } - if (writer.HasFileAtPath(dbPath)) - { - DemoteReadinessOnce(); - using var deleteTxn = writer.BeginTransaction(); - if (writer.DeleteFileByPath(dbPath)) - { - WriteProjectRootOnce(); - deleteTxn.Commit(); - removed++; - ftsMutated = true; - } - } - else - { - skipped++; - } + DemoteReadinessOnce(); + writer.MarkBatchInProgress(); + using var txn = writer.BeginTransaction(); + var skippedRecord = indexer.BuildSkippedFileRecord(absPath); + writer.PurgeStaleFilesSharingChecksum(projectRoot, skippedRecord.Path, skippedRecord.Checksum); + if (projectRootWritten) + writer.PurgeStaleFilesSharingDirectoryAndStem(projectRoot, skippedRecord.Path); + WriteProjectRootOnce(); + var fileId = writer.UpsertFile(skippedRecord); + writer.InsertChunks([]); + writer.InsertSymbols([]); + writer.InsertReferences([]); + writer.InsertIssues(fileId, [BuildNullByteIssue(binaryFile)]); + writer.ClearBatchInProgress(); + txn.Commit(); + + updated++; + ftsMutated = true; continue; } diff --git a/src/CodeIndex/Cli/IndexCommandRunner.cs b/src/CodeIndex/Cli/IndexCommandRunner.cs index 1c79e1e445..b49f1bd976 100644 --- a/src/CodeIndex/Cli/IndexCommandRunner.cs +++ b/src/CodeIndex/Cli/IndexCommandRunner.cs @@ -19,10 +19,15 @@ public static partial class IndexCommandRunner { internal const string IncludeSymbolKindsEnvironmentVariable = "CDIDX_INDEX_INCLUDE_SYMBOL_KINDS"; internal const string ExcludeSymbolKindsEnvironmentVariable = "CDIDX_INDEX_EXCLUDE_SYMBOL_KINDS"; + internal const string GeneratedCodePatternsEnvironmentVariable = "CDIDX_INDEX_GENERATED_CODE_PATTERNS"; internal const int DefaultMaxSymbolsPerFile = 5000; internal const int MaxSymbolsPerFileLimit = 50_000; + internal const int DefaultMaxReferencesPerFile = 100_000; + internal const int MaxReferencesPerFileLimit = 1_000_000; internal const int MaxCommitRefCount = 64; internal const int MaxCommitRefLength = 256; + internal const int MaxGeneratedCodePatternCsvLength = 32_768; + internal const int MaxGeneratedCodePatternCount = 128; internal const int MaxGitExcludeBytes = 256 * 1024; internal const string SymbolKindFilterMetaKey = "index_symbol_kind_filter"; private const int MaxIndexRunDiagnosticLength = 512; @@ -37,6 +42,10 @@ private sealed record ScanCheckpoint( string? GitHead, IReadOnlyList Directories); + internal sealed record ScanCheckpointLoadResult( + IReadOnlySet Directories, + string? WarningMessage); + internal static Action? FullScanWritePhaseStartedForTesting { get; set; } internal static Action? FullScanExtractionSchedulingForTesting { get; set; } internal static Func? IndexExtractionStallTimeoutForTesting { get; set; } @@ -275,7 +284,14 @@ internal static int Run(string[] indexArgs, JsonSerializerOptions jsonOptions, C AddToGitExclude(options.ProjectPath, dbPath, indexRunDiagnostics); var writer = new DbWriter(db); - var indexer = new FileIndexer(options.ProjectPath, ignoreCase, ignoreRuleRoot, options.MaxFileSizeBytes, directoryIgnoreCaseProbe: null, symlinkPolicy: options.SymlinkPolicy); + var indexer = new FileIndexer( + options.ProjectPath, + ignoreCase, + ignoreRuleRoot, + options.MaxFileSizeBytes, + directoryIgnoreCaseProbe: null, + symlinkPolicy: options.SymlinkPolicy, + generatedCodePatterns: options.GeneratedCodePatterns); var currentHotspotFamilyMarkerFingerprints = GetHotspotFamilyMarkerFingerprints(indexer, indexCancellation.Token); var projectRoot = Path.GetFullPath(options.ProjectPath!); @@ -650,6 +666,7 @@ private static int AddProjectMarkerFingerprintWarnings( private static bool IsProjectMarkerFingerprintWarning(FileIndexer.ScanError warning) => warning.Message.StartsWith("Project marker discovery skipped", StringComparison.Ordinal) + || warning.Message.StartsWith("Project marker discovery truncated", StringComparison.Ordinal) || warning.Message.StartsWith("Skipped .gitmodules", StringComparison.Ordinal); private static void RestampHotspotFamilyTrustForUpdate( @@ -1088,6 +1105,8 @@ private static CSharpStaticInterfaceWorkspaceSymbols BuildCSharpStaticInterfaceW var (record, content, _, _) = indexer.BuildRecordWithRawBytes(absolutePath, cancellationToken); if (record.Lang != "csharp") continue; + if (indexer.BuildGeneratedCodeExtractionSkippedIssue(record.Path) != null) + continue; if (!MayContainCSharpStaticInterfaceContract(content)) continue; @@ -1592,10 +1611,12 @@ public sealed class IndexCommandOptions public CompletionNotificationMode NotifyMode { get; init; } = CompletionNotificationMode.Auto; public long? MaxFileSizeBytes { get; init; } public int MaxSymbolsPerFile { get; init; } = IndexCommandRunner.DefaultMaxSymbolsPerFile; + public int MaxReferencesPerFile { get; init; } = IndexCommandRunner.DefaultMaxReferencesPerFile; public int Parallelism { get; init; } = IndexCommandRunner.DefaultIndexParallelism(); public bool MemoryTrace { get; init; } public FileIndexer.SymlinkPolicy SymlinkPolicy { get; init; } = FileIndexer.SymlinkPolicy.None; public SymbolKindFilter SymbolKindFilter { get; init; } = SymbolKindFilter.Empty; + public IReadOnlyList GeneratedCodePatterns { get; init; } = []; } public sealed class SymbolKindFilter diff --git a/src/CodeIndex/Cli/IndexWatchRunner.cs b/src/CodeIndex/Cli/IndexWatchRunner.cs index ff423d16e3..b8eb3bb659 100644 --- a/src/CodeIndex/Cli/IndexWatchRunner.cs +++ b/src/CodeIndex/Cli/IndexWatchRunner.cs @@ -224,6 +224,11 @@ private static List BuildSubRunArgs(IndexCommandOptions baseOptions, str args.Add("--max-symbols-per-file"); args.Add(baseOptions.MaxSymbolsPerFile.ToString(CultureInfo.InvariantCulture)); } + if (baseOptions.MaxReferencesPerFile != IndexCommandRunner.DefaultMaxReferencesPerFile) + { + args.Add("--max-references-per-file"); + args.Add(baseOptions.MaxReferencesPerFile.ToString(CultureInfo.InvariantCulture)); + } if (baseOptions.Parallelism != IndexCommandRunner.DefaultIndexParallelism()) { args.Add("--parallelism"); diff --git a/src/CodeIndex/Database/DbWriter.cs b/src/CodeIndex/Database/DbWriter.cs index a14c2d5116..4053d8d9dc 100644 --- a/src/CodeIndex/Database/DbWriter.cs +++ b/src/CodeIndex/Database/DbWriter.cs @@ -720,6 +720,14 @@ public int CountSymbolsForFile(long fileId) return Convert.ToInt32(cmd.ExecuteScalar()); } + public int CountReferencesForFile(long fileId) + { + using var cmd = _conn.CreateCommand(); + cmd.CommandText = "SELECT COUNT(*) FROM symbol_references WHERE file_id = @file_id"; + cmd.Parameters.AddWithValue("@file_id", fileId); + return Convert.ToInt32(cmd.ExecuteScalar()); + } + public bool HasIssueForFile(long fileId, string kind) { using var cmd = _conn.CreateCommand(); diff --git a/src/CodeIndex/Indexer/BoundedRegex.cs b/src/CodeIndex/Indexer/BoundedRegex.cs index f92dc11fd5..5705cca2c5 100644 --- a/src/CodeIndex/Indexer/BoundedRegex.cs +++ b/src/CodeIndex/Indexer/BoundedRegex.cs @@ -1,4 +1,7 @@ +using System.Security.Cryptography; +using System.Text; using System.Text.RegularExpressions; +using System.Threading; using BclMatch = System.Text.RegularExpressions.Match; using BclRegex = System.Text.RegularExpressions.Regex; @@ -8,6 +11,57 @@ internal sealed class BoundedRegex : BclRegex { // Keep regex matches bounded, but leave enough scheduler headroom for full-suite CI contention. internal static readonly TimeSpan DefaultMatchTimeout = TimeSpan.FromSeconds(2); + private const int MaxCapturedTimeoutDiagnostics = 8; + private static readonly AsyncLocal TimeoutCaptureScope = new(); + + internal readonly record struct RegexTimeoutDiagnostic( + string Operation, + string PatternHash, + int PatternLength, + double TimeoutMs); + + internal sealed class RegexTimeoutCaptureScope : IDisposable + { + private readonly RegexTimeoutCaptureScope? _previous; + private readonly List _diagnostics = []; + + internal RegexTimeoutCaptureScope(string? language, string patternFamily) + { + _previous = TimeoutCaptureScope.Value; + Language = string.IsNullOrWhiteSpace(language) ? "unknown" : language; + PatternFamily = patternFamily; + TimeoutCaptureScope.Value = this; + } + + public string Language { get; } + public string PatternFamily { get; } + public int TimeoutCount { get; private set; } + public IReadOnlyList Diagnostics => _diagnostics; + public bool DiagnosticsTruncated => TimeoutCount > _diagnostics.Count; + public bool HasTimeouts => TimeoutCount > 0; + + internal void Record(string operation, string pattern, TimeSpan timeout) + { + TimeoutCount++; + if (_diagnostics.Count >= MaxCapturedTimeoutDiagnostics) + return; + + _diagnostics.Add(new RegexTimeoutDiagnostic( + operation, + HashPattern(pattern), + pattern.Length, + timeout.TotalMilliseconds)); + } + + public void Dispose() + { + if (ReferenceEquals(TimeoutCaptureScope.Value, this)) + TimeoutCaptureScope.Value = _previous; + } + } + + internal static RegexTimeoutCaptureScope CaptureTimeouts(string? language, string patternFamily) => + new(language, patternFamily); public BoundedRegex(string pattern) : base(pattern, RegexOptions.None, DefaultMatchTimeout) @@ -37,8 +91,9 @@ public BoundedRegex(string pattern, RegexOptions options, TimeSpan matchTimeout) { return BclRegex.Match(input, pattern, options, DefaultMatchTimeout); } - catch (RegexMatchTimeoutException) + catch (RegexMatchTimeoutException ex) { + RecordTimeout("match", pattern, ex); return BclMatch.Empty; } } @@ -49,8 +104,9 @@ public static BclMatch Match(BclRegex regex, string input) { return regex.Match(input); } - catch (RegexMatchTimeoutException) + catch (RegexMatchTimeoutException ex) { + RecordTimeout("match", regex.ToString(), ex); return BclMatch.Empty; } } @@ -66,8 +122,9 @@ public static BclMatch Match(BclRegex regex, string input) _ = matches.Count; return matches; } - catch (RegexMatchTimeoutException) + catch (RegexMatchTimeoutException ex) { + RecordTimeout("matches", pattern, ex); return EmptyMatches(); } } @@ -80,8 +137,9 @@ public static IEnumerable EnumerateMatches(BclRegex regex, string inpu matches = regex.Matches(input); _ = matches.Count; } - catch (RegexMatchTimeoutException) + catch (RegexMatchTimeoutException ex) { + RecordTimeout("matches", regex.ToString(), ex); yield break; } @@ -100,8 +158,9 @@ public static IEnumerable EnumerateMatches(string input, string patter matches = BclRegex.Matches(input, pattern, options, DefaultMatchTimeout); _ = matches.Count; } - catch (RegexMatchTimeoutException) + catch (RegexMatchTimeoutException ex) { + RecordTimeout("matches", pattern, ex); yield break; } @@ -118,8 +177,9 @@ public static IEnumerable EnumerateMatches(string input, string patter { return BclRegex.IsMatch(input, pattern, options, DefaultMatchTimeout); } - catch (RegexMatchTimeoutException) + catch (RegexMatchTimeoutException ex) { + RecordTimeout("is_match", pattern, ex); return false; } } @@ -133,8 +193,9 @@ public static IEnumerable EnumerateMatches(string input, string patter { return BclRegex.Replace(input, pattern, replacement, options, DefaultMatchTimeout); } - catch (RegexMatchTimeoutException) + catch (RegexMatchTimeoutException ex) { + RecordTimeout("replace", pattern, ex); return input; } } @@ -148,8 +209,9 @@ public static IEnumerable EnumerateMatches(string input, string patter { return BclRegex.Replace(input, pattern, evaluator, options, DefaultMatchTimeout); } - catch (RegexMatchTimeoutException) + catch (RegexMatchTimeoutException ex) { + RecordTimeout("replace", pattern, ex); return input; } } @@ -160,8 +222,9 @@ public static IEnumerable EnumerateMatches(string input, string patter { return base.Match(input); } - catch (RegexMatchTimeoutException) + catch (RegexMatchTimeoutException ex) { + RecordTimeout("match", ToString(), ex); return BclMatch.Empty; } } @@ -172,8 +235,9 @@ public static IEnumerable EnumerateMatches(string input, string patter { return base.Match(input, startat); } - catch (RegexMatchTimeoutException) + catch (RegexMatchTimeoutException ex) { + RecordTimeout("match", ToString(), ex); return BclMatch.Empty; } } @@ -184,8 +248,9 @@ public static IEnumerable EnumerateMatches(string input, string patter { return base.Match(input, beginning, length); } - catch (RegexMatchTimeoutException) + catch (RegexMatchTimeoutException ex) { + RecordTimeout("match", ToString(), ex); return BclMatch.Empty; } } @@ -198,8 +263,9 @@ public static IEnumerable EnumerateMatches(string input, string patter _ = matches.Count; return matches; } - catch (RegexMatchTimeoutException) + catch (RegexMatchTimeoutException ex) { + RecordTimeout("matches", ToString(), ex); return EmptyMatches(); } } @@ -212,8 +278,9 @@ public static IEnumerable EnumerateMatches(string input, string patter _ = matches.Count; return matches; } - catch (RegexMatchTimeoutException) + catch (RegexMatchTimeoutException ex) { + RecordTimeout("matches", ToString(), ex); return EmptyMatches(); } } @@ -224,8 +291,9 @@ public static IEnumerable EnumerateMatches(string input, string patter { return base.IsMatch(input); } - catch (RegexMatchTimeoutException) + catch (RegexMatchTimeoutException ex) { + RecordTimeout("is_match", ToString(), ex); return false; } } @@ -236,8 +304,9 @@ public static IEnumerable EnumerateMatches(string input, string patter { return base.IsMatch(input, startat); } - catch (RegexMatchTimeoutException) + catch (RegexMatchTimeoutException ex) { + RecordTimeout("is_match", ToString(), ex); return false; } } @@ -248,8 +317,9 @@ public static IEnumerable EnumerateMatches(string input, string patter { return base.Replace(input, replacement); } - catch (RegexMatchTimeoutException) + catch (RegexMatchTimeoutException ex) { + RecordTimeout("replace", ToString(), ex); return input; } } @@ -260,12 +330,22 @@ public static IEnumerable EnumerateMatches(string input, string patter { return base.Replace(input, evaluator); } - catch (RegexMatchTimeoutException) + catch (RegexMatchTimeoutException ex) { + RecordTimeout("replace", ToString(), ex); return input; } } + private static void RecordTimeout(string operation, string pattern, RegexMatchTimeoutException ex) => + TimeoutCaptureScope.Value?.Record(operation, pattern, ex.MatchTimeout); + + private static string HashPattern(string pattern) + { + var hash = SHA256.HashData(Encoding.UTF8.GetBytes(pattern)); + return Convert.ToHexString(hash.AsSpan(0, 8)).ToLowerInvariant(); + } + private static MatchCollection EmptyMatches() => BclRegex.Matches(string.Empty, @"\b\B", RegexOptions.None, DefaultMatchTimeout); } diff --git a/src/CodeIndex/Indexer/References/Languages/ElixirReferenceExtractor.cs b/src/CodeIndex/Indexer/References/Languages/ElixirReferenceExtractor.cs index 6c3cae88dc..e1c4129b97 100644 --- a/src/CodeIndex/Indexer/References/Languages/ElixirReferenceExtractor.cs +++ b/src/CodeIndex/Indexer/References/Languages/ElixirReferenceExtractor.cs @@ -89,7 +89,7 @@ void AddDefimplTypeReference(Group group, int column) if (!seen.Add(key)) return; - references.Add(new ReferenceRecord + ReferenceExtractor.TryAddReference(references, new ReferenceRecord { FileId = fileId, SymbolName = name, diff --git a/src/CodeIndex/Indexer/References/ReferenceExtractionContext.cs b/src/CodeIndex/Indexer/References/ReferenceExtractionContext.cs index 7f5a870665..c40a0da051 100644 --- a/src/CodeIndex/Indexer/References/ReferenceExtractionContext.cs +++ b/src/CodeIndex/Indexer/References/ReferenceExtractionContext.cs @@ -14,4 +14,5 @@ public sealed record ReferenceExtractionContext( string? Path = null, IReadOnlyList? WorkspaceSymbols = null, string? RequestedLanguage = null, - CancellationToken CancellationToken = default); + CancellationToken CancellationToken = default, + int? MaxReferenceCount = null); diff --git a/src/CodeIndex/Indexer/References/ReferenceExtractor.Core.cs b/src/CodeIndex/Indexer/References/ReferenceExtractor.Core.cs index 3bcce34f29..59c02fd00d 100644 --- a/src/CodeIndex/Indexer/References/ReferenceExtractor.Core.cs +++ b/src/CodeIndex/Indexer/References/ReferenceExtractor.Core.cs @@ -337,7 +337,7 @@ void EmitCSharpBclRegexWithoutTimeoutReferences(List references if (!seen.Add(dedupeKey)) continue; - references.Add(new ReferenceRecord + if (!TryAddReference(references, new ReferenceRecord { FileId = reference.FileId, SymbolName = reference.SymbolName, @@ -348,7 +348,10 @@ void EmitCSharpBclRegexWithoutTimeoutReferences(List references ContainerKind = reference.ContainerKind, ContainerName = reference.ContainerName, IsSelfReference = reference.IsSelfReference, - }); + })) + { + return; + } } } @@ -784,7 +787,7 @@ static string NormalizeCSharpBclRegexQualifiedName(string value) return normalized; } - var references = new List(); + var references = CreateReferenceList(request.MaxReferenceCount); var seen = new HashSet(StringComparer.Ordinal); if (language == "csharp") { @@ -839,6 +842,9 @@ static string NormalizeCSharpBclRegexQualifiedName(string value) for (int i = 0; i < lines.Length; i++) { + if (ReferenceLimitReached(references)) + break; + if ((i & 0x3f) == 0) request.CancellationToken.ThrowIfCancellationRequested(); @@ -2954,7 +2960,7 @@ void AddGradleDslReference(string name, int callIndex) } } - if (language == "csharp") + if (!ReferenceLimitReached(references) && language == "csharp") { CSharpReferenceExtractor.EmitSwitchExpressionTypePatternReferences( lines, @@ -2981,7 +2987,8 @@ void AddGradleDslReference(string name, int callIndex) } ApplyCSharpUsingAliasReferenceNames(references); - EmitCSharpBclRegexWithoutTimeoutReferences(references, seen); + if (!ReferenceLimitReached(references)) + EmitCSharpBclRegexWithoutTimeoutReferences(references, seen); MarkMutualRecursionReferences(references); return references; } diff --git a/src/CodeIndex/Indexer/References/ReferenceExtractor.Preparation.cs b/src/CodeIndex/Indexer/References/ReferenceExtractor.Preparation.cs index aa63dadf03..292086a336 100644 --- a/src/CodeIndex/Indexer/References/ReferenceExtractor.Preparation.cs +++ b/src/CodeIndex/Indexer/References/ReferenceExtractor.Preparation.cs @@ -91,9 +91,7 @@ private static bool TryPrepareReferenceLines( if (language == "python") referenceStructuralLines = MaskPythonFStrings(referenceStructuralLines); - var preparedLines = new string[lines.Length]; - for (var pi = 0; pi < lines.Length; pi++) - preparedLines[pi] = PrepareLine(language, referenceStructuralLines[pi]); + var preparedLines = PrepareReferenceLines(language, referenceStructuralLines); var goImportBlockLines = language == "go" ? GoReferenceExtractor.BuildImportBlockLineMap(lines) : null; @@ -139,6 +137,28 @@ private static bool TryPrepareReferenceLines( return true; } + private static string[] PrepareReferenceLines(string language, string[] referenceStructuralLines) + { + string[]? preparedLines = null; + for (var index = 0; index < referenceStructuralLines.Length; index++) + { + var structuralLine = referenceStructuralLines[index]; + var preparedLine = PrepareLine(language, structuralLine); + if (preparedLines == null) + { + if (string.Equals(preparedLine, structuralLine, StringComparison.Ordinal)) + continue; + + preparedLines = new string[referenceStructuralLines.Length]; + Array.Copy(referenceStructuralLines, preparedLines, index); + } + + preparedLines[index] = preparedLine; + } + + return preparedLines ?? referenceStructuralLines; + } + private static IReadOnlyDictionary>? GroupJsTaggedTemplatesByLine( IReadOnlyList? jsTaggedTemplateHits) { diff --git a/src/CodeIndex/Indexer/References/ReferenceExtractor.TypeReferences.cs b/src/CodeIndex/Indexer/References/ReferenceExtractor.TypeReferences.cs index ec892e96df..5c637d1bba 100644 --- a/src/CodeIndex/Indexer/References/ReferenceExtractor.TypeReferences.cs +++ b/src/CodeIndex/Indexer/References/ReferenceExtractor.TypeReferences.cs @@ -579,7 +579,7 @@ internal static void AddTypeReferenceSegment( if (!seen.Add(dedupeKey)) return; - references.Add(new ReferenceRecord + TryAddReference(references, new ReferenceRecord { FileId = fileId, SymbolName = segment, @@ -1664,7 +1664,7 @@ private static void AddChainReference( if (!seen.Add(dedupeKey)) return; - references.Add(new ReferenceRecord + TryAddReference(references, new ReferenceRecord { FileId = fileId, SymbolName = name, diff --git a/src/CodeIndex/Indexer/References/ReferenceExtractor.cs b/src/CodeIndex/Indexer/References/ReferenceExtractor.cs index 9a2ad52639..a9f138cb5b 100644 --- a/src/CodeIndex/Indexer/References/ReferenceExtractor.cs +++ b/src/CodeIndex/Indexer/References/ReferenceExtractor.cs @@ -893,7 +893,8 @@ public static List Extract( IReadOnlyList symbols, string? path = null, IReadOnlyList? workspaceSymbols = null, - CancellationToken cancellationToken = default) + CancellationToken cancellationToken = default, + int? maxReferenceCount = null) { cancellationToken.ThrowIfCancellationRequested(); var requestedLanguage = lang; @@ -916,6 +917,7 @@ public static List Extract( fileId, content, new ExtractionContext(pluginLanguage, path, symbols, workspaceSymbols)) + .Take(maxReferenceCount ?? int.MaxValue) .ToList(); } @@ -929,8 +931,33 @@ public static List Extract( path, workspaceSymbols, requestedLanguage, - cancellationToken)); + cancellationToken, + maxReferenceCount)); } + + private sealed class BoundedReferenceList(int maxReferenceCount) : List + { + internal int MaxReferenceCount { get; } = maxReferenceCount; + } + + internal static List CreateReferenceList(int? maxReferenceCount) + => maxReferenceCount is > 0 + ? new BoundedReferenceList(maxReferenceCount.Value) + : []; + + internal static bool ReferenceLimitReached(List references) + => references is BoundedReferenceList bounded + && bounded.Count >= bounded.MaxReferenceCount; + + internal static bool TryAddReference(List references, ReferenceRecord reference) + { + if (ReferenceLimitReached(references)) + return false; + + references.Add(reference); + return true; + } + private static Dictionary> BuildDefinitionNamesByLine( string language, IReadOnlyList symbols) @@ -1256,7 +1283,7 @@ internal static void AddReference( if (!seen.Add(dedupeKey)) return; - references.Add(new ReferenceRecord + TryAddReference(references, new ReferenceRecord { FileId = fileId, SymbolName = name, @@ -1819,17 +1846,22 @@ internal static void AddTypeReferenceSegments( var dedupeKey = BuildReferenceDedupeKey(fileId, language, lineNumber, column, "type_reference", normalizedSegment, container); if (seen.Add(dedupeKey)) { - references.Add(new ReferenceRecord + if (!TryAddReference( + references, + new ReferenceRecord + { + FileId = fileId, + SymbolName = normalizedSegment, + ReferenceKind = "type_reference", + Line = lineNumber, + Column = column, + Context = context, + ContainerKind = container?.Kind, + ContainerName = container?.Name, + })) { - FileId = fileId, - SymbolName = normalizedSegment, - ReferenceKind = "type_reference", - Line = lineNumber, - Column = column, - Context = context, - ContainerKind = container?.Kind, - ContainerName = container?.Name, - }); + return; + } } } diff --git a/src/CodeIndex/Indexer/References/Support/JvmMethodReferenceExtractor.cs b/src/CodeIndex/Indexer/References/Support/JvmMethodReferenceExtractor.cs index 9063f238ab..6002cf50d5 100644 --- a/src/CodeIndex/Indexer/References/Support/JvmMethodReferenceExtractor.cs +++ b/src/CodeIndex/Indexer/References/Support/JvmMethodReferenceExtractor.cs @@ -227,7 +227,7 @@ private static void AddChainReference( if (!seen.Add(dedupeKey)) return; - references.Add(new ReferenceRecord + ReferenceExtractor.TryAddReference(references, new ReferenceRecord { FileId = fileId, SymbolName = name, diff --git a/src/CodeIndex/Indexer/Scanning/ChunkSplitter.cs b/src/CodeIndex/Indexer/Scanning/ChunkSplitter.cs index 86d94427e4..bf61d09574 100644 --- a/src/CodeIndex/Indexer/Scanning/ChunkSplitter.cs +++ b/src/CodeIndex/Indexer/Scanning/ChunkSplitter.cs @@ -96,19 +96,29 @@ public static List Split(long fileId, string content) // 既存の issues 経路で観測できる。Closes #1542. if (HasOversizeLine(content)) return []; - // Remove trailing newline to avoid phantom empty line / 末尾改行による空行を除去 - var lines = content.EndsWith('\n') - ? content[..^1].Split('\n') - : content.Split('\n'); + // Track line start offsets instead of materializing every line string. Large + // source files can still be valid and under the file-size cap, and chunking + // should only allocate the persisted chunk bodies rather than a duplicate + // full-file string[] plus per-chunk slice arrays. A trailing newline is not + // treated as an extra phantom line, matching the previous Split('\n') path. + // 各行の string 配列を作らず、行開始 offset だけを保持する。大きな source file + // でも file-size 上限内なら有効なため、chunking では永続化する chunk 本文以外に + // ファイル全体分の string[] や chunk ごとの slice 配列を作らない。末尾改行を + // 余分な空行として扱わない点は従来の Split('\n') 経路と同じ。 + var lineStarts = GetLineStartOffsets(content); var chunks = new List(); int step = ChunkSize - Overlap; int chunkIndex = 0; + var effectiveContentLength = content.EndsWith('\n') ? content.Length - 1 : content.Length; - for (int start = 0; start < lines.Length; start += step) + for (int start = 0; start < lineStarts.Count; start += step) { - int end = Math.Min(start + ChunkSize, lines.Length); - var chunkLines = lines[start..end]; - var chunkContent = string.Join('\n', chunkLines); + int end = Math.Min(start + ChunkSize, lineStarts.Count); + var startOffset = lineStarts[start]; + var endOffset = end < lineStarts.Count + ? lineStarts[end] - 1 + : effectiveContentLength; + var chunkContent = content.Substring(startOffset, endOffset - startOffset); chunks.Add(new ChunkRecord { @@ -122,10 +132,22 @@ public static List Split(long fileId, string content) chunkIndex++; // Stop if we've reached the end / 末尾に到達したら終了 - if (end >= lines.Length) + if (end >= lineStarts.Count) break; } return chunks; } + + private static List GetLineStartOffsets(string content) + { + var lineStarts = new List { 0 }; + for (var i = 0; i < content.Length; i++) + { + if (content[i] == '\n' && i + 1 < content.Length) + lineStarts.Add(i + 1); + } + + return lineStarts; + } } diff --git a/src/CodeIndex/Indexer/Scanning/FileContentLoader.cs b/src/CodeIndex/Indexer/Scanning/FileContentLoader.cs index 1f435793a6..40898b2c6c 100644 --- a/src/CodeIndex/Indexer/Scanning/FileContentLoader.cs +++ b/src/CodeIndex/Indexer/Scanning/FileContentLoader.cs @@ -20,8 +20,10 @@ internal LoadedFileContent Load( var (content, warning) = DecodeIndexableContent(bytes, relativePath); content = NormalizeLineEndings(content); content = StripLineLeadingInvisibles(content); - if (IsGitLfsPointer(bytes)) + var isGitLfsPointer = IsGitLfsPointer(bytes); + if (isGitLfsPointer) content = string.Empty; + var checksumBytes = isGitLfsPointer ? bytes : Encoding.UTF8.GetBytes(content); return new LoadedFileContent( content, @@ -29,7 +31,7 @@ internal LoadedFileContent Load( sizeBytes, modifiedUtc, FileIndexer.CountPhysicalLines(content), - ComputeChecksum(Encoding.UTF8.GetBytes(content)), + ComputeChecksum(checksumBytes), warning); } @@ -105,14 +107,20 @@ internal LoadedFileContent Load( { var isUtf16Encoded = TryDetectUtf16Encoding(bytes, allowHeuristic: true, out var utf16BigEndian, out var hasUtf16Bom); - if (!isUtf16Encoded && ContainsIndexBlockingNullByte(bytes)) - throw new FileIndexer.BinaryFileSkippedException($"{relativePath}: binary file skipped because it contains NULL bytes"); + if (!isUtf16Encoded && TryFindIndexBlockingNullByte(bytes, out var nullByteOffset)) + throw new FileIndexer.BinaryFileSkippedException( + relativePath, + nullByteOffset, + $"{relativePath}: binary file skipped because it contains NULL byte at byte offset {nullByteOffset}"); if (isUtf16Encoded) { var content = new UnicodeEncoding(utf16BigEndian, byteOrderMark: hasUtf16Bom, throwOnInvalidBytes: false) .GetString(bytes); - return (content, null); + var warning = hasUtf16Bom + ? null + : $"{relativePath}: decoded as {(utf16BigEndian ? "UTF-16BE" : "UTF-16LE")} without BOM by NUL-byte heuristic"; + return (content, warning); } try @@ -274,7 +282,17 @@ private static bool IsGitLfsSizeLine(string line) internal static bool ContainsIndexBlockingNullByte(byte[] rawBytes) { - return !TryDetectUtf16Encoding(rawBytes, allowHeuristic: true, out _, out _) && rawBytes.Any(b => b == 0); + return TryFindIndexBlockingNullByte(rawBytes, out _); + } + + internal static bool TryFindIndexBlockingNullByte(byte[] rawBytes, out int offset) + { + offset = -1; + if (TryDetectUtf16Encoding(rawBytes, allowHeuristic: true, out _, out _)) + return false; + + offset = Array.IndexOf(rawBytes, (byte)0); + return offset >= 0; } internal static bool TryDetectUtf16Encoding( diff --git a/src/CodeIndex/Indexer/Scanning/FileIndexer.cs b/src/CodeIndex/Indexer/Scanning/FileIndexer.cs index c6b66dedc6..8bc88fe3a0 100644 --- a/src/CodeIndex/Indexer/Scanning/FileIndexer.cs +++ b/src/CodeIndex/Indexer/Scanning/FileIndexer.cs @@ -16,6 +16,7 @@ namespace CodeIndex.Indexer; /// public class FileIndexer { + internal const int MaxDanglingFileSystemEntryScanCandidates = 4096; internal static Func? FileSystemIgnoreCaseProbeForTesting { get; set; } internal static Func? ResolveDirectoryLinkTargetForTesting { get; set; } @@ -93,6 +94,7 @@ private sealed class ProjectMarkerFingerprintTraversalState public int DirectoriesVisited { get; set; } public int MarkerFilesCollected { get; set; } public bool Truncated { get; set; } + public string TruncationReason { get; set; } = "unknown"; } private readonly record struct ProjectMarkerFingerprintDirectory(string Path, IgnoreRuleSet IgnoreRules, bool IsProjectRoot); @@ -473,6 +475,8 @@ internal static bool IsDefaultExcludedFileName(string fileName) private readonly long _maxFileSizeBytes; private readonly FileContentLoader _contentLoader; private readonly SymlinkPolicy _symlinkPolicy; + private readonly int _maxDanglingFileSystemEntryScanCandidates; + private readonly GeneratedCodePatternMatcher _generatedCodePatterns; // Submodule working-tree paths declared in /.gitmodules, relative to // _projectRoot and slash-normalized. Used to override SkipDirs so that submodules // hosted under SkipDirs-named directories (e.g. vendor/foo) remain visible to the @@ -1006,8 +1010,13 @@ public FileIndexer(string projectRoot, bool ignoreCase) { } - public FileIndexer(string projectRoot, bool ignoreCase, string? ignoreRuleRoot, long? maxFileSizeBytes = null) - : this(projectRoot, ignoreCase, ignoreRuleRoot, maxFileSizeBytes, directoryIgnoreCaseProbe: null) + public FileIndexer( + string projectRoot, + bool ignoreCase, + string? ignoreRuleRoot, + long? maxFileSizeBytes = null, + IReadOnlyList? generatedCodePatterns = null) + : this(projectRoot, ignoreCase, ignoreRuleRoot, maxFileSizeBytes, directoryIgnoreCaseProbe: null, generatedCodePatterns: generatedCodePatterns) { } @@ -1018,7 +1027,9 @@ internal FileIndexer( long? maxFileSizeBytes, Func? directoryIgnoreCaseProbe, Func>? enumerateFiles = null, - SymlinkPolicy symlinkPolicy = SymlinkPolicy.None) + SymlinkPolicy symlinkPolicy = SymlinkPolicy.None, + int? maxDanglingFileSystemEntryScanCandidates = null, + IReadOnlyList? generatedCodePatterns = null) { _projectRoot = Path.GetFullPath(projectRoot); _ignoreRuleRoot = NormalizeIgnoreRuleRoot(ignoreRuleRoot); @@ -1030,6 +1041,10 @@ internal FileIndexer( _maxFileSizeBytes = ResolveMaxFileSizeBytes(maxFileSizeBytes); _contentLoader = new FileContentLoader(_maxFileSizeBytes); _symlinkPolicy = symlinkPolicy; + _maxDanglingFileSystemEntryScanCandidates = Math.Max( + 1, + maxDanglingFileSystemEntryScanCandidates ?? MaxDanglingFileSystemEntryScanCandidates); + _generatedCodePatterns = GeneratedCodePatternMatcher.FromPatterns(generatedCodePatterns, ignoreCase); ExtractorPluginRegistry.LoadPatternConfigsForProjectRoot(_projectRoot); var pathComparer = _ignoreCase ? StringComparer.OrdinalIgnoreCase : StringComparer.Ordinal; (_submodulePaths, _submoduleAncestorPaths, _submoduleLoadWarnings) = LoadGitSubmodulePaths(_ignoreRuleRoot, _projectRoot, pathComparer); @@ -1526,10 +1541,22 @@ private bool ShouldSkipDirectoryLink(string subDir, List errors, Hash if (_symlinkPolicy == SymlinkPolicy.Internal && IsPathEqualOrParent(_projectRoot, targetPath)) return false; - errors.Add(new ScanError(relative, $"Skipped symlinked directory outside the active symlink policy: {targetPath}", ScanIssueSeverity.Warning)); + errors.Add(new ScanError( + relative, + $"Skipped symlinked directory outside the active symlink policy: target {FormatSymlinkPolicyTargetForDiagnostic(targetPath)}", + ScanIssueSeverity.Warning)); return true; } + private string FormatSymlinkPolicyTargetForDiagnostic(string targetPath) + { + if (!IsPathEqualOrParent(_projectRoot, targetPath)) + return ""; + + var relative = NormalizePathSeparators(Path.GetRelativePath(_projectRoot, targetPath)); + return relative == "." ? "" : relative; + } + internal bool ShouldSkipDirectoryTraversal(string directory) => ShouldSkipDirectoryLink( directory, @@ -1561,7 +1588,7 @@ private static string GetDirectoryTraversalIdentity(string directory) { } - return directory; + return $"unresolved-reparse:{Path.GetFullPath(directory)}"; } internal static FileProbeStatus GetFileIndexability(string filePath) @@ -1762,7 +1789,7 @@ private ProjectMarkerFingerprintResult GetProjectMarkerFingerprintResult( if (traversalState.Truncated) { projectMarkers.Add( - $"__cdidx_project_marker_fingerprint_truncated__:directories={traversalState.DirectoriesVisited};markers={traversalState.MarkerFilesCollected}"); + $"__cdidx_project_marker_fingerprint_truncated__:reason={traversalState.TruncationReason};directories={traversalState.DirectoriesVisited};markers={traversalState.MarkerFilesCollected}"); } projectMarkers.Sort(StringComparer.Ordinal); @@ -1885,7 +1912,11 @@ private void CollectProjectMarkerFiles( if (traversalState.DirectoriesVisited >= maxDirectories) { - traversalState.Truncated = true; + TruncateProjectMarkerTraversal( + traversalState, + errors, + current.Path, + $"directory budget {maxDirectories:N0} exhausted after visiting {traversalState.DirectoriesVisited:N0} directories"); return; } @@ -1897,7 +1928,11 @@ private void CollectProjectMarkerFiles( var loadResult = LoadIgnoreRulesForDirectory(currentDirectory, current.IgnoreRules, errors, ref fullyScanned); if (!loadResult.IgnoreRulesAvailable) { - traversalState.Truncated = true; + TruncateProjectMarkerTraversal( + traversalState, + errors, + currentDirectory, + "ignore-rule loading failed"); return; } @@ -1910,7 +1945,11 @@ private void CollectProjectMarkerFiles( if (traversalState.MarkerFilesCollected >= maxMarkerFiles) { - traversalState.Truncated = true; + TruncateProjectMarkerTraversal( + traversalState, + errors, + currentDirectory, + $"marker file budget {maxMarkerFiles:N0} exhausted after collecting {traversalState.MarkerFilesCollected:N0} marker files"); return; } @@ -1934,7 +1973,11 @@ private void CollectProjectMarkerFiles( if (traversalState.DirectoriesVisited + pendingDirectories.Count >= maxDirectories) { - traversalState.Truncated = true; + TruncateProjectMarkerTraversal( + traversalState, + errors, + currentDirectory, + $"directory budget {maxDirectories:N0} would be exceeded while queuing subdirectories after visiting {traversalState.DirectoriesVisited:N0} directories"); return; } @@ -1944,16 +1987,39 @@ private void CollectProjectMarkerFiles( catch (UnauthorizedAccessException) { AddProjectMarkerTraversalWarning(errors, currentDirectory, nameof(UnauthorizedAccessException)); - traversalState.Truncated = true; + MarkProjectMarkerTraversalTruncated( + traversalState, + $"traversal failed with {nameof(UnauthorizedAccessException)}"); } catch (IOException) { AddProjectMarkerTraversalWarning(errors, currentDirectory, nameof(IOException)); - traversalState.Truncated = true; + MarkProjectMarkerTraversalTruncated( + traversalState, + $"traversal failed with {nameof(IOException)}"); } } } + private void TruncateProjectMarkerTraversal( + ProjectMarkerFingerprintTraversalState traversalState, + List errors, + string directory, + string reason) + { + MarkProjectMarkerTraversalTruncated(traversalState, reason); + AddProjectMarkerBudgetWarning(errors, directory, reason); + } + + private static void MarkProjectMarkerTraversalTruncated( + ProjectMarkerFingerprintTraversalState traversalState, + string reason) + { + if (!traversalState.Truncated) + traversalState.TruncationReason = reason; + traversalState.Truncated = true; + } + private static IEnumerable EnumerateProjectMarkerDirectories(string dir) => EnumerateProjectMarkerDirectoriesForTesting is { } enumerate ? enumerate(dir) @@ -1977,6 +2043,24 @@ private void AddProjectMarkerTraversalWarning(List errors, string dir ScanIssueSeverity.Warning)); } + private void AddProjectMarkerBudgetWarning(List errors, string directory, string reason) + { + if (errors.Count(static error => error.Message.StartsWith("Project marker discovery truncated", StringComparison.Ordinal)) + >= MaxProjectMarkerTraversalWarnings) + { + return; + } + + var relativePath = NormalizeIgnorePath(Path.GetRelativePath(_projectRoot, directory)); + if (string.IsNullOrEmpty(relativePath)) + relativePath = "."; + + errors.Add(new ScanError( + relativePath, + $"Project marker discovery truncated because {reason}.", + ScanIssueSeverity.Warning)); + } + private static IReadOnlyList? GetProjectMarkerPatterns(string? lang) => lang switch { "csharp" => ["*.csproj"], @@ -2528,9 +2612,22 @@ private void RecordDanglingFileSystemEntries( DirectoryScanState scanState, CancellationToken cancellationToken) { + var candidateLimit = _maxDanglingFileSystemEntryScanCandidates; + var candidateCount = 0; foreach (var enumeratedEntry in Directory.EnumerateFileSystemEntries(LongPath.EnsureWindowsPrefix(dir))) { cancellationToken.ThrowIfCancellationRequested(); + candidateCount++; + if (candidateCount > candidateLimit) + { + var relativeDir = ToRelativePath(dir); + scanState.Errors.Add(new ScanError( + relativeDir, + $"Dangling filesystem entry scan truncated after {candidateLimit:N0} candidate(s); additional dangling symlink diagnostics in this directory may be omitted.", + ScanIssueSeverity.Warning)); + return; + } + var entry = LongPath.RemoveWindowsPrefix(enumeratedEntry); if (!IsReparsePoint(entry) || ReparsePointTargetExists(entry)) continue; @@ -3472,6 +3569,21 @@ private static string EscapeControlCharacters(string value) internal static bool IsGeneratedCodeFile(string relativePath, string content) => HasGeneratedCodeFileName(relativePath) || HasGeneratedCodeHeader(content); + internal const string GeneratedCodeExtractionSkippedIssueKind = "generated_code_extraction_skipped"; + + internal FileIssue? BuildGeneratedCodeExtractionSkippedIssue(string relativePath) + => _generatedCodePatterns.TryMatch(relativePath, out _) + ? new FileIssue + { + Path = relativePath, + Kind = GeneratedCodeExtractionSkippedIssueKind, + Line = 0, + Message = "Generated-code extraction suppressed by project configuration; file content and chunks were indexed, but symbols and references were skipped.", + Origin = "generated_code_pattern", + Severity = FileIssue.SeverityInfo, + } + : null; + internal static int CountPhysicalLines(string content) { if (content.Length == 0) @@ -3612,8 +3724,13 @@ public static List ValidateContent(string relativePath, byte[] rawByt // 不正サロゲートペアに備え content 側 U+FFFD 走査は継続する。Closes #1540. var isUtf16 = TryDetectUtf16Encoding(rawBytes, allowHeuristic: true, out var utf16BigEndian, out var hasUtf16Bom); - if (isUtf16 && hasUtf16Bom) - AddUtf16BomIssue(issues, relativePath, utf16BigEndian); + if (isUtf16) + { + if (hasUtf16Bom) + AddUtf16BomIssue(issues, relativePath, utf16BigEndian); + else + AddUtf16HeuristicIssue(issues, relativePath, utf16BigEndian); + } if (TryGetConflictMarkerLine(content, out var conflictMarkerLine)) { @@ -3883,6 +4000,19 @@ private static void AddUtf16BomIssue(List issues, string relativePath }); } + private static void AddUtf16HeuristicIssue(List issues, string relativePath, bool utf16BigEndian) + { + issues.Add(new FileIssue + { + Path = relativePath, + Kind = "utf16_heuristic", + Line = 1, + Message = utf16BigEndian + ? "BOM-less UTF-16 BE detected by NUL-byte heuristic (decoded as UTF-16)" + : "BOM-less UTF-16 LE detected by NUL-byte heuristic (decoded as UTF-16)", + }); + } + private static void AddReplacementCharacterIssues( List issues, string relativePath, @@ -4109,7 +4239,14 @@ internal static bool TryDetectUtf16Encoding( out bool hasBom) => FileContentLoader.TryDetectUtf16Encoding(rawBytes, allowHeuristic, out bigEndian, out hasBom); - internal sealed class BinaryFileSkippedException(string message) : InvalidOperationException(message); + internal sealed class BinaryFileSkippedException( + string relativePath, + long nullByteOffset, + string message) : InvalidOperationException(message) + { + public string RelativePath { get; } = relativePath; + public long NullByteOffset { get; } = nullByteOffset; + } internal sealed class FileTooLargeSkippedException( string relativePath, @@ -4347,9 +4484,8 @@ private static LanguageDetectionResult TryDetectLanguageFromShebang( if (string.IsNullOrWhiteSpace(commandLine)) return new LanguageDetectionResult(FileProbeStatus.Unsupported, null); - var tokens = commandLine - .Split([' ', '\t'], StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries); - if (tokens.Length == 0) + var tokens = TokenizeShebangCommandLine(commandLine); + if (tokens.Count == 0) return new LanguageDetectionResult(FileProbeStatus.Unsupported, null); var interpreter = ResolveShebangInterpreter(tokens); @@ -4447,15 +4583,77 @@ private static int FindShebangLineEnd(ReadOnlySpan bytes, ShebangEncoding _ => new UTF8Encoding(false, throwOnInvalidBytes: true).GetString(bytes), }; + private static IReadOnlyList TokenizeShebangCommandLine(string commandLine) + { + var tokens = new List(); + var token = new StringBuilder(commandLine.Length); + char? quote = null; + var escaped = false; + + foreach (var ch in commandLine) + { + if (escaped) + { + token.Append(ch); + escaped = false; + continue; + } + + if (ch == '\\') + { + escaped = true; + continue; + } + + if (quote is { } activeQuote) + { + if (ch == activeQuote) + quote = null; + else + token.Append(ch); + continue; + } + + if (ch is '\'' or '"') + { + quote = ch; + continue; + } + + if (ch is ' ' or '\t') + { + if (token.Length > 0) + { + tokens.Add(token.ToString()); + token.Clear(); + } + continue; + } + + token.Append(ch); + } + + if (escaped) + token.Append('\\'); + if (token.Length > 0) + tokens.Add(token.ToString()); + + return tokens; + } + private static string? ResolveShebangInterpreter(IReadOnlyList tokens) { - var interpreter = Path.GetFileName(tokens[0]).ToLowerInvariant(); + var interpreter = NormalizeShebangInterpreterToken(tokens[0]); + if (interpreter == null) + return null; if (interpreter is not "env") return interpreter; for (var i = 1; i < tokens.Count; i++) { var token = tokens[i]; + if (token == "--") + continue; if (token.StartsWith("-", StringComparison.Ordinal)) continue; @@ -4464,12 +4662,29 @@ private static int FindShebangLineEnd(ReadOnlySpan bytes, ShebangEncoding if (token.Contains('=')) continue; - return Path.GetFileName(token).ToLowerInvariant(); + return NormalizeShebangInterpreterToken(token); } return null; } + private static string? NormalizeShebangInterpreterToken(string token) + { + var candidate = token; + if (token.IndexOfAny([' ', '\t']) >= 0) + { + var nestedTokens = TokenizeShebangCommandLine(token); + if (nestedTokens.Count == 0) + return null; + candidate = nestedTokens[0]; + } + + if (string.IsNullOrWhiteSpace(candidate)) + return null; + + return Path.GetFileName(candidate).ToLowerInvariant(); + } + private static string? MapShebangInterpreterToLanguage(string interpreter) => interpreter switch { "bash" or "sh" or "zsh" or "fish" or "dash" or "ksh" or "ash" => "shell", diff --git a/src/CodeIndex/Indexer/Scanning/GeneratedCodePatternMatcher.cs b/src/CodeIndex/Indexer/Scanning/GeneratedCodePatternMatcher.cs new file mode 100644 index 0000000000..a39eff9552 --- /dev/null +++ b/src/CodeIndex/Indexer/Scanning/GeneratedCodePatternMatcher.cs @@ -0,0 +1,135 @@ +using System.Text; +using System.Text.RegularExpressions; + +namespace CodeIndex.Indexer; + +internal sealed class GeneratedCodePatternMatcher +{ + internal static readonly GeneratedCodePatternMatcher Empty = new([]); + + private static readonly TimeSpan MatchTimeout = TimeSpan.FromMilliseconds(50); + + private readonly Rule[] _rules; + + private GeneratedCodePatternMatcher(Rule[] rules) + { + _rules = rules; + } + + internal static GeneratedCodePatternMatcher FromPatterns(IEnumerable? patterns, bool ignoreCase) + { + if (patterns == null) + return Empty; + + var rules = new List(); + foreach (var raw in patterns) + { + var pattern = NormalizePattern(raw); + if (pattern.Length == 0) + continue; + + rules.Add(new Rule( + pattern, + MatchBasenameOnly: !pattern.Contains('/', StringComparison.Ordinal), + BuildMatcher(pattern, ignoreCase))); + } + + return rules.Count == 0 ? Empty : new GeneratedCodePatternMatcher(rules.ToArray()); + } + + internal bool TryMatch(string relativePath, out string pattern) + { + pattern = string.Empty; + if (_rules.Length == 0) + return false; + + var normalizedPath = NormalizePath(relativePath); + var fileName = GetFileName(normalizedPath); + foreach (var rule in _rules) + { + var candidate = rule.MatchBasenameOnly ? fileName : normalizedPath; + if (rule.Matcher.IsMatch(candidate)) + { + pattern = rule.Pattern; + return true; + } + } + + return false; + } + + private static string NormalizePattern(string? raw) + { + if (string.IsNullOrWhiteSpace(raw)) + return string.Empty; + + var pattern = raw.Trim().Replace('\\', '/'); + while (pattern.StartsWith("./", StringComparison.Ordinal)) + pattern = pattern[2..]; + while (pattern.StartsWith("/", StringComparison.Ordinal)) + pattern = pattern[1..]; + return pattern; + } + + private static string NormalizePath(string path) + { + var normalized = path.Replace('\\', '/'); + while (normalized.StartsWith("./", StringComparison.Ordinal)) + normalized = normalized[2..]; + return normalized; + } + + private static string GetFileName(string normalizedPath) + { + var slash = normalizedPath.LastIndexOf('/'); + return slash < 0 ? normalizedPath : normalizedPath[(slash + 1)..]; + } + + private static Regex BuildMatcher(string pattern, bool ignoreCase) + { + var builder = new StringBuilder(pattern.Length * 2); + builder.Append('^'); + for (var i = 0; i < pattern.Length; i++) + { + var ch = pattern[i]; + if (ch == '*') + { + var isDoubleStar = i + 1 < pattern.Length && pattern[i + 1] == '*'; + if (isDoubleStar) + { + if (i + 2 < pattern.Length && pattern[i + 2] == '/') + { + builder.Append("(?:[^/]+/)*"); + i += 2; + } + else + { + builder.Append(".*"); + i++; + } + } + else + { + builder.Append("[^/]*"); + } + continue; + } + + if (ch == '?') + { + builder.Append("[^/]"); + continue; + } + + builder.Append(Regex.Escape(ch.ToString())); + } + builder.Append('$'); + + var options = RegexOptions.CultureInvariant | RegexOptions.NonBacktracking; + if (ignoreCase) + options |= RegexOptions.IgnoreCase; + return new Regex(builder.ToString(), options, MatchTimeout); + } + + private sealed record Rule(string Pattern, bool MatchBasenameOnly, Regex Matcher); +} diff --git a/src/CodeIndex/Indexer/Symbols/SymbolExtractionWorker.cs b/src/CodeIndex/Indexer/Symbols/SymbolExtractionWorker.cs index fdc7ae79f1..7a32773094 100644 --- a/src/CodeIndex/Indexer/Symbols/SymbolExtractionWorker.cs +++ b/src/CodeIndex/Indexer/Symbols/SymbolExtractionWorker.cs @@ -15,7 +15,10 @@ internal sealed record SymbolExtractionWorkerResult( bool TimedOut, string? WorkerError, long DurationMs, - List? Symbols); + List? Symbols, + int RegexTimeoutCount = 0, + List? RegexTimeoutDiagnostics = null, + bool RegexTimeoutDiagnosticsTruncated = false); internal sealed class SymbolExtractionWorkerClient : IDisposable { @@ -141,7 +144,10 @@ internal SymbolExtractionWorkerResult Invoke( TimedOut: false, WorkerError: null, DurationMs: stopwatch.ElapsedMilliseconds, - Symbols: response.Symbols); + Symbols: response.Symbols, + RegexTimeoutCount: response.RegexTimeoutCount, + RegexTimeoutDiagnostics: response.RegexTimeoutDiagnostics, + RegexTimeoutDiagnosticsTruncated: response.RegexTimeoutDiagnosticsTruncated); } } @@ -583,6 +589,7 @@ private static WorkerResponse InvokeInsideWorker(WorkerRequest request, WorkerOp Console.SetError(capturedError); WriteConsoleOutputForTestingIfRequested(options); DelayForTestingIfRequested(options, cancellationToken); + using var regexTimeouts = BoundedRegex.CaptureTimeouts(request.Lang, "symbol_extraction"); var symbols = SymbolExtractor.Extract( request.FileId, request.Lang, @@ -590,7 +597,13 @@ private static WorkerResponse InvokeInsideWorker(WorkerRequest request, WorkerOp request.FilePath, request.ProjectRoot, cancellationToken); - return new WorkerResponse(symbols, null, capturedError.GetCapturedText()); + return new WorkerResponse( + symbols, + null, + capturedError.GetCapturedText(), + regexTimeouts.TimeoutCount, + regexTimeouts.Diagnostics.ToList(), + regexTimeouts.DiagnosticsTruncated); } catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) { @@ -768,7 +781,10 @@ internal sealed record WorkerRequest( internal sealed record WorkerResponse( List? Symbols, string? WorkerError, - string? CapturedStderr); + string? CapturedStderr, + int RegexTimeoutCount = 0, + List? RegexTimeoutDiagnostics = null, + bool RegexTimeoutDiagnosticsTruncated = false); private sealed record WorkerOptions( int MaxProtocolLineCharacters, diff --git a/src/CodeIndex/Mcp/McpToolArgumentContracts.cs b/src/CodeIndex/Mcp/McpToolArgumentContracts.cs index 2f2f988952..7e98f77691 100644 --- a/src/CodeIndex/Mcp/McpToolArgumentContracts.cs +++ b/src/CodeIndex/Mcp/McpToolArgumentContracts.cs @@ -33,7 +33,7 @@ public partial class McpServer "validate" => new HashSet(StringComparer.Ordinal) { "kind", "severity", "limit", "path", "excludePaths", "excludeTests", "countOnly", "format", "project", "solution" }, "unused_symbols" => new HashSet(StringComparer.Ordinal) { "kind", "lang", "limit", "visibility", "excludeVisibility", "path", "excludePaths", "excludeTests", "bucket", "minConfidence", "byBucket", "project", "solution" }, "symbol_hotspots" => new HashSet(StringComparer.Ordinal) { "kind", "lang", "limit", "visibility", "excludeVisibility", "groupBy", "path", "excludePaths", "excludeTests", "project", "solution" }, - "index" => new HashSet(StringComparer.Ordinal) { "path", "rebuild", "dryRun", "dry_run", "maxFileBytes", "maxSymbolsPerFile", "followSymlinks", "includeSymbolKind", "excludeSymbolKind", "memoryTrace", "parallelism", "commits", "changedBetween", "files", "watch", "debounce" }, + "index" => new HashSet(StringComparer.Ordinal) { "path", "rebuild", "dryRun", "dry_run", "maxFileBytes", "maxSymbolsPerFile", "maxReferencesPerFile", "followSymlinks", "includeSymbolKind", "excludeSymbolKind", "memoryTrace", "parallelism", "commits", "changedBetween", "files", "watch", "debounce" }, "backfill_fold" => new HashSet(StringComparer.Ordinal) { "dry_run", "dryRun", "force" }, "suggest_improvement" => new HashSet(StringComparer.Ordinal) { "category", "language", "description", "context", "toolInvocationContext", "evidencePaths", "evidence_paths" }, _ => new HashSet(StringComparer.Ordinal), diff --git a/src/CodeIndex/Mcp/McpToolDefinitions.cs b/src/CodeIndex/Mcp/McpToolDefinitions.cs index 585ee31462..dc27e72c5e 100644 --- a/src/CodeIndex/Mcp/McpToolDefinitions.cs +++ b/src/CodeIndex/Mcp/McpToolDefinitions.cs @@ -478,6 +478,7 @@ private JsonNode HandleToolsList(JsonNode? id) ["dryRun"] = new JsonObject { ["type"] = "boolean", ["description"] = "Plan the index run without mutating the database. Reports scan counts, effective options, and unsupported MCP modes.", ["default"] = false }, ["maxFileBytes"] = new JsonObject { ["type"] = "integer", ["description"] = "Override the per-file indexing size limit for this run. Defaults to CDIDX_MAX_FILE_BYTES or 4MiB.", ["minimum"] = 1, ["maximum"] = int.MaxValue }, ["maxSymbolsPerFile"] = new JsonObject { ["type"] = "integer", ["description"] = "Skip symbol/reference indexing for files that produce more symbols than this limit, matching CLI --max-symbols-per-file.", ["default"] = IndexCommandRunner.DefaultMaxSymbolsPerFile, ["minimum"] = 1, ["maximum"] = IndexCommandRunner.MaxSymbolsPerFileLimit }, + ["maxReferencesPerFile"] = new JsonObject { ["type"] = "integer", ["description"] = "Skip references for files that produce more references than this limit, matching CLI --max-references-per-file.", ["default"] = IndexCommandRunner.DefaultMaxReferencesPerFile, ["minimum"] = 1, ["maximum"] = IndexCommandRunner.MaxReferencesPerFileLimit }, ["followSymlinks"] = new JsonObject { ["type"] = "string", ["enum"] = new JsonArray { "none", "internal", "all" }, ["description"] = "Directory and file symlink policy matching CLI --follow-symlinks.", ["default"] = "none" }, ["includeSymbolKind"] = new JsonObject { ["oneOf"] = new JsonArray { new JsonObject { ["type"] = "string" }, new JsonObject { ["type"] = "array", ["items"] = new JsonObject { ["type"] = "string" } } }, ["description"] = "Only index symbols with these kinds. Accepts a value, comma-separated string, or array." }, ["excludeSymbolKind"] = new JsonObject { ["oneOf"] = new JsonArray { new JsonObject { ["type"] = "string" }, new JsonObject { ["type"] = "array", ["items"] = new JsonObject { ["type"] = "string" } } }, ["description"] = "Drop symbols with these kinds before indexing. Accepts a value, comma-separated string, or array." }, diff --git a/src/CodeIndex/Mcp/McpToolHandlers.cs b/src/CodeIndex/Mcp/McpToolHandlers.cs index 9536910f8a..d4286b1549 100644 --- a/src/CodeIndex/Mcp/McpToolHandlers.cs +++ b/src/CodeIndex/Mcp/McpToolHandlers.cs @@ -857,8 +857,7 @@ private static bool TryGetExpectedJsonType(string toolName, string argumentName, { "limit" or "offset" or "snippetLines" or "maxLineWidth" or "before" or "after" or "focusLine" or "focusColumn" or "focusLength" or "startLine" or "endLine" or - "maxHops" or "maxDepth" or "depth" or "parallelism" or "maxFileBytes" or "maxSymbolsPerFile" or "debounce" or - "maxHops" or "maxDepth" or "depth" or "parallelism" or "maxFileBytes" or "maxSymbolsPerFile" or "debounce" or + "maxHops" or "maxDepth" or "depth" or "parallelism" or "maxFileBytes" or "maxSymbolsPerFile" or "maxReferencesPerFile" or "debounce" or "staleAfterSeconds" or "guardWindow" or "maxOutputBytes" or "maxResponseBytes" => "integer", "check" or "excludeTests" or "includeGenerated" or "indexedOnly" or "rawQuery" or "noDedup" or "exactSubstring" or @@ -866,7 +865,6 @@ private static bool TryGetExpectedJsonType(string toolName, string argumentName, "lspCompatible" or "regex" or "withPaths" or "rebuild" or "dryRun" or "dry_run" or "force" or "optimize" or "reverse" or "cycles" or "config" or "logPath" or "updateCheck" or - "optimize" or "reverse" or "cycles" or "config" or "logPath" or "updateCheck" or "rawKinds" or "orderBySize" or "rawBytes" or "byBucket" or "memoryTrace" or "watch" or "estimateOnly" or "listRecipes" => "boolean", "project" or "capability" or "scopes" or "visibility" or "excludeVisibility" or "includeSymbolKind" or "excludeSymbolKind" or @@ -5404,6 +5402,15 @@ private static FileIssue BuildMcpSymbolCountExceededIssue(string path, int symbo Message = $"Symbol extraction produced {symbolCount:N0} symbols, exceeding the maxSymbolsPerFile limit of {maxSymbolsPerFile:N0}; file content, symbols, and references were not indexed. Exclude the generated/pathological file or raise maxSymbolsPerFile if this is expected.", }; + private static FileIssue BuildMcpReferenceCountExceededIssue(string path, int referenceCount, int maxReferencesPerFile) => + new() + { + Path = path, + Kind = "reference_count_exceeded", + Line = 0, + Message = $"Reference extraction produced {referenceCount:N0} references, exceeding the maxReferencesPerFile limit of {maxReferencesPerFile:N0}; references were not indexed for this file. Exclude the generated/pathological file or raise maxReferencesPerFile if this is expected.", + }; + private static bool TryReadMcpIndexSymlinkPolicy(JsonNode? args, out FileIndexer.SymlinkPolicy symlinkPolicy, out string? error) { symlinkPolicy = FileIndexer.SymlinkPolicy.None; @@ -5494,6 +5501,7 @@ private static JsonObject BuildMcpIndexOptionsPayload( bool rebuild, long? maxFileBytes, int maxSymbolsPerFile, + int maxReferencesPerFile, FileIndexer.SymlinkPolicy symlinkPolicy, IReadOnlyList includeSymbolKinds, IReadOnlyList excludeSymbolKinds, @@ -5507,6 +5515,7 @@ private static JsonObject BuildMcpIndexOptionsPayload( ["rebuild"] = rebuild, ["maxFileBytes"] = maxFileBytes.HasValue ? JsonValue.Create(maxFileBytes.Value) : null, ["maxSymbolsPerFile"] = maxSymbolsPerFile, + ["maxReferencesPerFile"] = maxReferencesPerFile, ["followSymlinks"] = FormatMcpIndexSymlinkPolicy(symlinkPolicy), ["includeSymbolKind"] = ToJsonStringArray(includeSymbolKinds), ["excludeSymbolKind"] = ToJsonStringArray(excludeSymbolKinds), @@ -5587,6 +5596,9 @@ private async Task ExecuteIndexAsync(JsonNode? id, JsonNode? args, Jso var maxSymbolsPerFile = args?["maxSymbolsPerFile"]?.GetValue() ?? IndexCommandRunner.DefaultMaxSymbolsPerFile; if (maxSymbolsPerFile <= 0 || maxSymbolsPerFile > IndexCommandRunner.MaxSymbolsPerFileLimit) return CreateToolErrorResponse(id, $"maxSymbolsPerFile must be between 1 and {IndexCommandRunner.MaxSymbolsPerFileLimit}"); + var maxReferencesPerFile = args?["maxReferencesPerFile"]?.GetValue() ?? IndexCommandRunner.DefaultMaxReferencesPerFile; + if (maxReferencesPerFile <= 0 || maxReferencesPerFile > IndexCommandRunner.MaxReferencesPerFileLimit) + return CreateToolErrorResponse(id, $"maxReferencesPerFile must be between 1 and {IndexCommandRunner.MaxReferencesPerFileLimit}"); if (!TryReadMcpIndexSymlinkPolicy(args, out var symlinkPolicy, out var symlinkPolicyError)) return CreateToolErrorResponse(id, symlinkPolicyError!); var includeSymbolKinds = ReadStringOrCommaSeparatedList(args, "includeSymbolKind"); @@ -5620,6 +5632,7 @@ private async Task ExecuteIndexAsync(JsonNode? id, JsonNode? args, Jso rebuild, maxFileBytes, maxSymbolsPerFile, + maxReferencesPerFile, symlinkPolicy, includeSymbolKinds, excludeSymbolKinds, @@ -5650,7 +5663,8 @@ private async Task ExecuteIndexAsync(JsonNode? id, JsonNode? args, Jso GitHelper.TryGetRepositoryRoot(projectPath, _currentRequestToken.Value) ?? Path.GetFullPath(projectPath), maxFileBytes, directoryIgnoreCaseProbe: null, - symlinkPolicy: symlinkPolicy); + symlinkPolicy: symlinkPolicy, + generatedCodePatterns: IndexCommandRunner.ReadGeneratedCodePatternsFromEnvironment()); var scan = dryRunIndexer.ScanFilesDetailed(cancellationToken: _currentRequestToken.Value); if (memorySamples != null) memorySamples.Add(CaptureMcpIndexMemorySample("scan", runStopwatch)); @@ -5755,7 +5769,8 @@ private async Task ExecuteIndexAsync(JsonNode? id, JsonNode? args, Jso GitHelper.TryGetRepositoryRoot(projectPath, requestToken) ?? Path.GetFullPath(projectPath), maxFileBytes, directoryIgnoreCaseProbe: null, - symlinkPolicy: symlinkPolicy); + symlinkPolicy: symlinkPolicy, + generatedCodePatterns: IndexCommandRunner.ReadGeneratedCodePatternsFromEnvironment()); using var postExtractionHooks = PostExtractionHookRunner.DiscoverDefault(maxFileBytes); var currentHotspotFamilyMarkerFingerprints = GetHotspotFamilyMarkerFingerprints(indexer, requestToken); var currentCSharpSymbolNameContractVersion = DbContext.CSharpSymbolNameContractVersion.ToString(System.Globalization.CultureInfo.InvariantCulture); @@ -5906,7 +5921,9 @@ static string FormatDiagnosticPath(string projectRoot, string path) record.Modified, record.Checksum, size: record.Size, + lines: record.Lines, language: record.Lang, + generated: record.Generated, allowReuse: symbolKindFilterMatchesPrior && record.Lang is not ("javascript" or "typescript") && (record.Lang != "csharp" || csharpSymbolNameContractMatchesCurrent) @@ -5916,11 +5933,18 @@ static string FormatDiagnosticPath(string projectRoot, string path) if (existingId != null) { if (writer.CountSymbolsForFile(existingId.Value) > maxSymbolsPerFile - || writer.HasIssueForFile(existingId.Value, "symbol_count_exceeded")) + || writer.HasIssueForFile(existingId.Value, "symbol_count_exceeded") + || writer.CountReferencesForFile(existingId.Value) > maxReferencesPerFile + || writer.HasIssueForFile(existingId.Value, "reference_count_exceeded")) { existingId = null; } } + if (existingId != null + && IndexCommandRunner.ExistingFileGeneratedSuppressionMismatch(writer, existingId.Value, indexer.BuildGeneratedCodeExtractionSkippedIssue(record.Path))) + { + existingId = null; + } if (existingId != null) { skipped++; @@ -5936,7 +5960,31 @@ static string FormatDiagnosticPath(string projectRoot, string path) using var txn = writer.BeginTransaction(); var fileId = writer.UpsertFile(record); var chunks = ChunkSplitter.Split(fileId, content); - var symbols = SymbolExtractor.Extract(fileId, record.Lang, content, filePath, projectPath, requestToken).ToList(); + var generatedSuppressionIssue = indexer.BuildGeneratedCodeExtractionSkippedIssue(record.Path); + if (generatedSuppressionIssue != null) + { + writer.InsertChunks(chunks); + writer.InsertSymbols([]); + writer.InsertReferences([]); + var issues = IndexCommandRunner.AppendIssueIfMissing( + FileIndexer.ValidateContent(record.Path, rawBytes, content, record.Lang), + generatedSuppressionIssue); + writer.InsertIssues(fileId, issues); + WriteProjectRootOnce(); + writer.ClearBatchInProgress(); + txn.Commit(); + processed++; + await EmitProgressNotificationAsync(progressToken, processed, files.Count).ConfigureAwait(false); + McpIndexFileCommittedForTesting?.Invoke(record.Path); + continue; + } + List symbols; + FileIssue? symbolRegexTimeoutIssue; + using (var regexTimeouts = BoundedRegex.CaptureTimeouts(record.Lang, "symbol_extraction")) + { + symbols = SymbolExtractor.Extract(fileId, record.Lang, content, filePath, projectPath, requestToken).ToList(); + symbolRegexTimeoutIssue = IndexCommandRunner.BuildRegexTimeoutIssue(record.Path, regexTimeouts); + } SymbolExtractor.ApplyFamilyScope(symbols, indexer.GetFamilyScopeKey(filePath, record.Lang)); var fileContext = new FileContext(projectPath, record.Path, filePath, record.Lang); postExtractionHooks.OnSymbolsExtracted(fileContext, symbols); @@ -5944,27 +5992,49 @@ static string FormatDiagnosticPath(string projectRoot, string path) if (symbols.Count > maxSymbolsPerFile) { var issue = BuildMcpSymbolCountExceededIssue(record.Path, symbols.Count, maxSymbolsPerFile); + IReadOnlyList capIssues = symbolRegexTimeoutIssue == null + ? [issue] + : IndexCommandRunner.AppendIssue([symbolRegexTimeoutIssue], issue); writer.InsertSymbols([]); writer.InsertReferences([]); - writer.InsertIssues(fileId, [issue]); + writer.InsertIssues(fileId, capIssues); } else { writer.InsertChunks(chunks); writer.InsertSymbols(symbols); - var references = ReferenceExtractor.Extract( - fileId, - record.Lang, - content, - symbols, - record.Path, - record.Lang == "csharp" ? csharpWorkspace.Symbols : null, - requestToken); + List references; + FileIssue? regexTimeoutIssue; + using (var regexTimeouts = BoundedRegex.CaptureTimeouts(record.Lang, "reference_extraction")) + { + references = ReferenceExtractor.Extract( + fileId, + record.Lang, + content, + symbols, + record.Path, + record.Lang == "csharp" ? csharpWorkspace.Symbols : null, + requestToken, + maxReferenceCount: maxReferencesPerFile + 1); + regexTimeoutIssue = IndexCommandRunner.BuildRegexTimeoutIssue(record.Path, regexTimeouts); + } postExtractionHooks.OnReferencesExtracted(fileContext, references); + FileIssue? referenceCapIssue = null; + if (references.Count > maxReferencesPerFile) + { + referenceCapIssue = BuildMcpReferenceCountExceededIssue(record.Path, references.Count, maxReferencesPerFile); + references = []; + } writer.InsertReferences(references); // Keep MCP index parity with CLI index: persist file-level validation issues too. // MCPインデックスもCLIインデックスと同等に、ファイル検証issueを保存する。 - var issues = FileIndexer.ValidateContent(record.Path, rawBytes, content, record.Lang); + IReadOnlyList issues = FileIndexer.ValidateContent(record.Path, rawBytes, content, record.Lang); + if (symbolRegexTimeoutIssue != null) + issues = IndexCommandRunner.AppendIssue(issues, symbolRegexTimeoutIssue); + if (regexTimeoutIssue != null) + issues = IndexCommandRunner.AppendIssue(issues, regexTimeoutIssue); + if (referenceCapIssue != null) + issues = IndexCommandRunner.AppendIssue(issues, referenceCapIssue); writer.InsertIssues(fileId, issues); } WriteProjectRootOnce(); @@ -5972,23 +6042,24 @@ static string FormatDiagnosticPath(string projectRoot, string path) txn.Commit(); McpIndexFileCommittedForTesting?.Invoke(record.Path); } - catch (FileIndexer.BinaryFileSkippedException) + catch (FileIndexer.BinaryFileSkippedException ex) { try { - var relativePath = FileIndexer.NormalizePathSeparators(Path.GetRelativePath(projectPath, filePath)); - if (writer.HasFileAtPath(relativePath)) - { - using var txn = writer.BeginTransaction(); - writer.DeleteFileByPath(relativePath); - WriteProjectRootOnce(); - txn.Commit(); - } + var skippedRecord = indexer.BuildSkippedFileRecord(filePath); + using var txn = writer.BeginTransaction(); + var fileId = writer.UpsertFile(skippedRecord); + writer.InsertChunks([]); + writer.InsertSymbols([]); + writer.InsertReferences([]); + writer.InsertIssues(fileId, [IndexCommandRunner.BuildNullByteIssue(ex)]); + WriteProjectRootOnce(); + txn.Commit(); } - catch (Exception ex) + catch (Exception cleanupEx) { errors++; - failures.Add(BuildIndexFileFailure(projectPath, filePath, ex, "delete_skipped_binary")); + failures.Add(BuildIndexFileFailure(projectPath, filePath, cleanupEx, "record_skipped_binary")); } } catch (Exception ex) when (ex is FileNotFoundException or DirectoryNotFoundException) @@ -7319,6 +7390,8 @@ private static CSharpStaticInterfaceWorkspaceSymbols BuildMcpCSharpStaticInterfa var (record, content, _, _) = indexer.BuildRecordWithRawBytes(absolutePath, cancellationToken); if (record.Lang != "csharp") continue; + if (indexer.BuildGeneratedCodeExtractionSkippedIssue(record.Path) != null) + continue; pendingSymbols.AddRange(SymbolExtractor.Extract(0, record.Lang, content, record.Path, cancellationToken: cancellationToken)); } diff --git a/tests/CodeIndex.Tests/BoundedRegexTests.cs b/tests/CodeIndex.Tests/BoundedRegexTests.cs index 7ee6666f26..8c771336f3 100644 --- a/tests/CodeIndex.Tests/BoundedRegexTests.cs +++ b/tests/CodeIndex.Tests/BoundedRegexTests.cs @@ -44,6 +44,29 @@ public void InstanceMatch_Timeout_ReturnsEmpty() Assert.False(match.Success); } + [Fact] + public void CaptureTimeouts_RecordsHashedDiagnosticsForFileIssue() + { + const string pattern = "(a+)+$"; + var regex = new BoundedRegex(pattern, default, TimeSpan.FromMilliseconds(1)); + var input = new string('a', 10_000) + "!"; + + using var capture = BoundedRegex.CaptureTimeouts("csharp", "reference_extraction"); + var match = regex.Match(input); + var issue = IndexCommandRunner.BuildRegexTimeoutIssue("src/Pathological.cs", capture); + + Assert.False(match.Success); + Assert.True(capture.HasTimeouts); + Assert.NotNull(issue); + Assert.Equal("regex_timeout", issue.Kind); + Assert.Equal("src/Pathological.cs", issue.Path); + Assert.Contains("reference_extraction", issue.Message); + Assert.Contains(capture.Diagnostics[0].PatternHash, issue.Message); + Assert.Contains("len=6", issue.Message); + Assert.DoesNotContain(pattern, issue.Message); + Assert.DoesNotContain(input, issue.Message); + } + [Fact] public void InstanceMatches_Timeout_ReturnsEmpty() { diff --git a/tests/CodeIndex.Tests/CdidxConfigFileTests.cs b/tests/CodeIndex.Tests/CdidxConfigFileTests.cs index 9bc4a977be..5b9d7cd325 100644 --- a/tests/CodeIndex.Tests/CdidxConfigFileTests.cs +++ b/tests/CodeIndex.Tests/CdidxConfigFileTests.cs @@ -45,7 +45,8 @@ public void LoadAndApply_MaterializesKnownKeysIntoEnvironment() "suggestion_max_count": 250, "indexing": { "includeKinds": ["class"], - "excludeKinds": ["test_method", "generated_parser"] + "excludeKinds": ["test_method", "generated_parser"], + "generatedCodePatterns": ["src/generated/**", "*.client.ts"] }, "mcp": { "tools": { "allow": ["search", "definition"], "deny": ["index"] }, @@ -69,6 +70,7 @@ public void LoadAndApply_MaterializesKnownKeysIntoEnvironment() Assert.Equal("250", result.Settings["CDIDX_SUGGESTION_MAX_COUNT"]); Assert.Equal("class", result.Settings["CDIDX_INDEX_INCLUDE_SYMBOL_KINDS"]); Assert.Equal("test_method,generated_parser", result.Settings["CDIDX_INDEX_EXCLUDE_SYMBOL_KINDS"]); + Assert.Equal("src/generated/**,*.client.ts", result.Settings[IndexCommandRunner.GeneratedCodePatternsEnvironmentVariable]); Assert.Equal("search,definition", result.Settings["CDIDX_MCP_TOOLS_ALLOW"]); Assert.Equal("index", result.Settings["CDIDX_MCP_TOOLS_DENY"]); Assert.Equal("5", result.Settings[RateLimiterOptions.RpsEnvVar]); diff --git a/tests/CodeIndex.Tests/ChunkSplitterTests.cs b/tests/CodeIndex.Tests/ChunkSplitterTests.cs index c77a2ede5e..16065e8d34 100644 --- a/tests/CodeIndex.Tests/ChunkSplitterTests.cs +++ b/tests/CodeIndex.Tests/ChunkSplitterTests.cs @@ -49,6 +49,24 @@ public void Split_LargeFile_CreatesOverlappingChunks() Assert.Equal(71, chunks[1].StartLine); } + [Fact] + public void Split_ManyValidLines_KeepsAllocationBounded() + { + var content = string.Join('\n', Enumerable.Range(1, 5000).Select(i => $"line {i:D4} value")); + _ = ChunkSplitter.Split(1, "warmup\nline\n"); + + var before = GC.GetAllocatedBytesForCurrentThread(); + var chunks = ChunkSplitter.Split(1, content); + var allocated = GC.GetAllocatedBytesForCurrentThread() - before; + + Assert.Equal(72, chunks.Count); + Assert.Equal(1, chunks[0].StartLine); + Assert.Equal(80, chunks[0].EndLine); + Assert.Equal(4971, chunks[^1].StartLine); + Assert.Equal(5000, chunks[^1].EndLine); + Assert.True(allocated < 450_000, $"Chunk splitting allocated {allocated} bytes."); + } + [Fact] public void Split_EmptyFile_ReturnsNoChunks() { diff --git a/tests/CodeIndex.Tests/ConsoleUiTests.cs b/tests/CodeIndex.Tests/ConsoleUiTests.cs index e6fa58acbf..8996886eaa 100644 --- a/tests/CodeIndex.Tests/ConsoleUiTests.cs +++ b/tests/CodeIndex.Tests/ConsoleUiTests.cs @@ -103,7 +103,7 @@ public void PrintUsage_WithoutBanner_HidesAsciiArtAndEasterEggFlags() Assert.DoesNotContain("██████╗", output); Assert.Contains("Usage:", output); - Assert.Contains("cdidx index [--db ] [--rebuild] [--optimize] [--symbols-only] [--verbose] [--dry-run] [--force] [--quiet] [--json] [--memory-trace] [--duration-format ] [--notify ] [--max-file-bytes ] [--max-symbols-per-file ] [--follow-symlinks ]", output); + Assert.Contains("cdidx index [--db ] [--rebuild] [--optimize] [--symbols-only] [--verbose] [--dry-run] [--force] [--quiet] [--json] [--memory-trace] [--duration-format ] [--notify ] [--max-file-bytes ] [--max-symbols-per-file ] [--max-references-per-file ] [--follow-symlinks ]", output); Assert.Contains("cdidx hooks [--project ] [--force] [--json]", output); Assert.Contains("cdidx index --commits [commit-ref ...] [--db ] [--verbose] [--dry-run] [--json] [--memory-trace] [--duration-format ]", output); Assert.Contains("cdidx index --files [path ...] [--db ] [--verbose] [--dry-run] [--json] [--memory-trace] [--duration-format ]", output); diff --git a/tests/CodeIndex.Tests/FileIndexerTests.cs b/tests/CodeIndex.Tests/FileIndexerTests.cs index cd21c01ca7..e4f46ff9bc 100644 --- a/tests/CodeIndex.Tests/FileIndexerTests.cs +++ b/tests/CodeIndex.Tests/FileIndexerTests.cs @@ -48,6 +48,38 @@ public void ScanFilesDetailed_CancelledToken_ThrowsBeforeEnumeration() } } + [Fact] + public void ScanFilesDetailed_DanglingFileSystemEntryScanCapsCandidatesWithWarning() + { + var tempDir = Path.Combine(Path.GetTempPath(), $"cdidx-dangling-cap-{Guid.NewGuid():N}"); + Directory.CreateDirectory(tempDir); + try + { + for (var i = 0; i < 5; i++) + File.WriteAllText(Path.Combine(tempDir, $"file{i}.cs"), $"public class C{i} {{ }}\n"); + + var result = new FileIndexer( + tempDir, + ignoreCase: false, + ignoreRuleRoot: null, + maxFileSizeBytes: null, + directoryIgnoreCaseProbe: null, + maxDanglingFileSystemEntryScanCandidates: 3).ScanFilesDetailed(); + + Assert.Equal(5, result.Files.Count); + var warning = Assert.Single( + result.Errors, + error => error.Message.Contains("Dangling filesystem entry scan truncated", StringComparison.Ordinal)); + Assert.Equal(FileIndexer.ScanIssueSeverity.Warning, warning.Severity); + Assert.Contains("Dangling filesystem entry scan truncated after 3 candidate", warning.Message); + Assert.False(result.HadErrors); + } + finally + { + TestProjectHelper.DeleteDirectory(tempDir); + } + } + [Fact] public void Constructor_CaseProbeAvoidsRootProbeArtifacts_Issue3174() { @@ -829,6 +861,11 @@ public void GetProjectMarkerFingerprint_DirectoryCapReportsIncompleteTraversal() Assert.False(result.IsComplete); Assert.False(string.IsNullOrWhiteSpace(result.Fingerprint)); + var warning = Assert.Single( + result.Warnings, + error => error.Message.Contains("directory budget 1", StringComparison.Ordinal)); + Assert.Equal(FileIndexer.ScanIssueSeverity.Warning, warning.Severity); + Assert.Contains("Project marker discovery truncated", warning.Message, StringComparison.Ordinal); } finally { @@ -908,10 +945,15 @@ public void GetProjectMarkerFingerprint_FileCapTruncatesMarkerCollection() var fullFingerprint = indexer.GetProjectMarkerFingerprint("msbuild"); var cappedFingerprint = indexer.GetProjectMarkerFingerprintForTesting("msbuild", maxDirectories: 100, maxMarkerFiles: 1); + var cappedResult = indexer.GetProjectMarkerFingerprintResultForTesting("msbuild", maxDirectories: 100, maxMarkerFiles: 1); Assert.False(string.IsNullOrWhiteSpace(fullFingerprint)); Assert.False(string.IsNullOrWhiteSpace(cappedFingerprint)); Assert.NotEqual(fullFingerprint, cappedFingerprint); + Assert.False(cappedResult.IsComplete); + Assert.Contains( + cappedResult.Warnings, + error => error.Message.Contains("marker file budget 1", StringComparison.Ordinal)); } finally { @@ -1020,6 +1062,8 @@ public void DetectLanguage_UnknownExtensions_ReturnsNull(string filename) [InlineData("worker", "#!/usr/bin/python3\nprint('hi')\n", "python")] [InlineData("bundle", "#!/usr/bin/env ruby\nputs 'hi'\n", "ruby")] [InlineData("cli", "#!/usr/bin/env node\nconsole.log('hi')\n", "javascript")] + [InlineData("envsplit", "#!/usr/bin/env -S python -O\nprint('hi')\n", "python")] + [InlineData("envquoted", "#!/usr/bin/env -S \"python -O\"\nprint('hi')\n", "python")] [InlineData("script", "#!/usr/bin/env pwsh\nWrite-Host hi\n", "powershell")] public void DetectLanguage_ExtensionlessShebangScripts_ReturnCorrectLang(string fileName, string content, string expected) { @@ -4079,7 +4123,9 @@ public void ScanFiles_FollowSymlinksInternal_SkipsOutOfTreeDirectorySymlink() result.Errors, error => error.Path == "external" && error.Severity == FileIndexer.ScanIssueSeverity.Warning - && error.Message.Contains("symlinked directory", StringComparison.OrdinalIgnoreCase)); + && error.Message.Contains("symlinked directory", StringComparison.OrdinalIgnoreCase) + && error.Message.Contains("", StringComparison.Ordinal) + && !error.Message.Contains(externalDir, StringComparison.Ordinal)); } finally { @@ -4333,7 +4379,8 @@ public void IndexFilesUpdate_UsesOriginalUnicodePathForIoAndNfcPathForDb() File.WriteAllBytes(Path.Combine(tempDir, nfdPath), [0, 1, 2, 3]); Assert.Equal(CommandExitCodes.Success, IndexCommandRunner.Run([tempDir, "--files", nfdPath, "--json", "--quiet"], jsonOptions)); - Assert.False(HasIndexedFile(dbPath, "Caf\u00e9.cs")); + Assert.True(HasIndexedFile(dbPath, "Caf\u00e9.cs")); + Assert.True(HasFileIssue(dbPath, "Caf\u00e9.cs", "null_byte")); } finally { @@ -4840,7 +4887,9 @@ public void BuildRecord_Utf16LeWithoutBomFile_DecodedAsUtf16() var indexer = new FileIndexer(tempDir); var (_, content, _, warning) = indexer.BuildRecordWithRawBytes(filePath); - Assert.Null(warning); + Assert.NotNull(warning); + Assert.Contains("UTF-16LE without BOM", warning, StringComparison.Ordinal); + Assert.Contains("NUL-byte heuristic", warning, StringComparison.Ordinal); Assert.Contains("namespace Utf16LeNoBom;", content); Assert.False(FileIndexer.ContainsIndexBlockingNullByte(System.Text.Encoding.Unicode.GetBytes(payload))); } @@ -4868,7 +4917,9 @@ public void BuildRecord_Utf16BeWithoutBomFile_DecodedAsUtf16() var indexer = new FileIndexer(tempDir); var (_, content, _, warning) = indexer.BuildRecordWithRawBytes(filePath); - Assert.Null(warning); + Assert.NotNull(warning); + Assert.Contains("UTF-16BE without BOM", warning, StringComparison.Ordinal); + Assert.Contains("NUL-byte heuristic", warning, StringComparison.Ordinal); Assert.Contains("namespace Utf16BeNoBom;", content); Assert.False(FileIndexer.ContainsIndexBlockingNullByte(System.Text.Encoding.BigEndianUnicode.GetBytes(payload))); } @@ -4913,11 +4964,38 @@ public void ValidateContent_Utf16WithoutBom_DoesNotEmitNullByteIssue() var issues = FileIndexer.ValidateContent("utf16le-nobom.cs", rawBytes, payload); + var issue = Assert.Single(issues.Where(i => i.Kind == "utf16_heuristic")); + Assert.Equal(1, issue.Line); + Assert.Contains("UTF-16 LE", issue.Message, StringComparison.Ordinal); + Assert.Contains("NUL-byte heuristic", issue.Message, StringComparison.Ordinal); Assert.DoesNotContain(issues, i => i.Kind == "utf16_bom"); Assert.DoesNotContain(issues, i => i.Kind == "null_byte"); Assert.DoesNotContain(issues, i => i.Kind == "mixed_line_endings"); } + [Fact] + public void BuildRecord_NonUtf16NullByte_ThrowsOffsetDiagnostic() + { + var tempDir = Path.Combine(Path.GetTempPath(), $"codeindex_test_{Guid.NewGuid():N}"); + try + { + Directory.CreateDirectory(tempDir); + var filePath = Path.Combine(tempDir, "binary.cs"); + File.WriteAllBytes(filePath, [(byte)'c', (byte)'l', (byte)'a', (byte)'s', (byte)'s', (byte)' ', 0x00]); + + var indexer = new FileIndexer(tempDir); + var ex = Assert.Throws(() => indexer.BuildRecordWithRawBytes(filePath)); + + Assert.Contains("NULL byte at byte offset 6", ex.Message, StringComparison.Ordinal); + Assert.Equal("binary.cs", ex.RelativePath); + Assert.Equal(6, ex.NullByteOffset); + } + finally + { + Directory.Delete(tempDir, true); + } + } + [Fact] public void ValidateContent_HighFffdRatio_EmitsAggregateNonUtf8Likely() { @@ -5138,6 +5216,8 @@ size 12345 Assert.Equal(string.Empty, content); Assert.Equal(0, record.Lines); + Assert.Equal(FileIndexer.ComputeChecksum(rawBytes), record.Checksum); + Assert.NotEqual(FileIndexer.ComputeChecksum(System.Text.Encoding.UTF8.GetBytes(string.Empty)), record.Checksum); var issue = Assert.Single(issues, i => i.Kind == "lfs_pointer_skipped"); Assert.Equal("asset.cs", issue.Path); Assert.Equal(1, issue.Line); @@ -5148,6 +5228,40 @@ size 12345 } } + [Fact] + public void BuildRecord_ConfiguredGeneratedPatternBuildsExtractionIssueWithoutGeneratedFlag() + { + var tempDir = Path.Combine(Path.GetTempPath(), $"codeindex_test_{Guid.NewGuid():N}"); + try + { + var generatedDir = Path.Combine(tempDir, "src", "generated"); + Directory.CreateDirectory(generatedDir); + var filePath = Path.Combine(generatedDir, "Client.cs"); + File.WriteAllText(filePath, "public class Client { public string Lookup() => \"ok\"; }\n"); + + var indexer = new FileIndexer( + tempDir, + ignoreCase: false, + ignoreRuleRoot: null, + generatedCodePatterns: ["src/generated/**"]); + var (record, content, rawBytes, _) = indexer.BuildRecordWithRawBytes(filePath); + var issue = indexer.BuildGeneratedCodeExtractionSkippedIssue(record.Path); + + Assert.False(record.Generated); + Assert.Equal("src/generated/Client.cs", record.Path); + Assert.Contains("public class Client", content, StringComparison.Ordinal); + Assert.True(rawBytes.Length > 0); + Assert.NotNull(issue); + Assert.Equal(FileIndexer.GeneratedCodeExtractionSkippedIssueKind, issue.Kind); + Assert.Equal("src/generated/Client.cs", issue.Path); + Assert.Contains("symbols and references were skipped", issue.Message, StringComparison.Ordinal); + } + finally + { + Directory.Delete(tempDir, true); + } + } + [Fact] public void BuildRecord_GitLfsVersionLineWithoutPointerShapePreservesContent() { @@ -5837,4 +5951,20 @@ private static bool HasIndexedFile(string dbPath, string filePath) cmd.Parameters.AddWithValue("@path", filePath); return cmd.ExecuteScalar() != null; } + + private static bool HasFileIssue(string dbPath, string filePath, string kind) + { + using var db = new DbContext(dbPath); + using var cmd = db.Connection.CreateCommand(); + cmd.CommandText = """ + SELECT 1 + FROM file_issues i + JOIN files f ON f.id = i.file_id + WHERE f.path = @path + AND i.kind = @kind + """; + cmd.Parameters.AddWithValue("@path", filePath); + cmd.Parameters.AddWithValue("@kind", kind); + return cmd.ExecuteScalar() != null; + } } diff --git a/tests/CodeIndex.Tests/IndexCommandRunnerTests.cs b/tests/CodeIndex.Tests/IndexCommandRunnerTests.cs index 44fdab716e..398299ebe5 100644 --- a/tests/CodeIndex.Tests/IndexCommandRunnerTests.cs +++ b/tests/CodeIndex.Tests/IndexCommandRunnerTests.cs @@ -987,6 +987,39 @@ public void LoadScanCheckpoint_DirectoryPayloadOutsideBounds_ReturnsEmpty() } } + [Fact] + public void LoadScanCheckpointDetailed_InvalidPayloads_ReturnsReasonedWarning() + { + var projectRoot = CreateTempProject(); + var checkpointPath = Path.Combine(projectRoot, "scan-checkpoint.json"); + try + { + File.WriteAllText(checkpointPath, """{"Version":999,"GitHead":"abc123","Directories":["src"]}"""); + var futureVersion = IndexCommandRunner.LoadScanCheckpointDetailed(checkpointPath, "abc123"); + Assert.Empty(futureVersion.Directories); + Assert.Contains("future checkpoint version", futureVersion.WarningMessage); + + File.WriteAllText(checkpointPath, """{"Version":1,"GitHead":"old","Directories":["src"]}"""); + var stale = IndexCommandRunner.LoadScanCheckpointDetailed(checkpointPath, "abc123"); + Assert.Empty(stale.Directories); + Assert.Contains("checkpoint GitHead does not match current HEAD", stale.WarningMessage); + + File.WriteAllText(checkpointPath, """{"Version":1,"GitHead":"abc123","Directories":["src"]"""); + var malformed = IndexCommandRunner.LoadScanCheckpointDetailed(checkpointPath, "abc123"); + Assert.Empty(malformed.Directories); + Assert.Contains("malformed checkpoint JSON", malformed.WarningMessage); + + File.WriteAllText(checkpointPath, """{"Version":1,"GitHead":"abc123","Directories":[null]}"""); + var invalidDirectories = IndexCommandRunner.LoadScanCheckpointDetailed(checkpointPath, "abc123"); + Assert.Empty(invalidDirectories.Directories); + Assert.Contains("Directories contains a null entry", invalidDirectories.WarningMessage); + } + finally + { + DeleteDirectory(projectRoot); + } + } + [Fact] public void LoadScanCheckpoint_JsonDepthOutsideBounds_ReturnsEmpty() { @@ -1010,7 +1043,7 @@ public void LoadScanCheckpoint_JsonDepthOutsideBounds_ReturnsEmpty() } [Fact] - public void Run_NullByteFile_SkipsWithoutPersistingPartialRows() + public void Run_NullByteFile_PersistsNullByteIssueWithoutPartialRows() { var projectRoot = CreateTempProject(); try @@ -1026,13 +1059,17 @@ public void Run_NullByteFile_SkipsWithoutPersistingPartialRows() Assert.Equal(CommandExitCodes.Success, exitCode); Assert.Equal(0, json.GetProperty("summary").GetProperty("errors").GetInt32()); - Assert.Equal(1, json.GetProperty("summary").GetProperty("warnings").GetInt32()); + Assert.Equal(0, json.GetProperty("summary").GetProperty("warnings").GetInt32()); var dbPath = Path.Combine(projectRoot, ".cdidx", "codeindex.db"); - Assert.Equal(0, CountRows(dbPath, "files")); + Assert.Equal(1, CountRows(dbPath, "files")); Assert.Equal(0, CountRows(dbPath, "chunks")); Assert.Equal(0, CountRows(dbPath, "symbols")); Assert.Equal(0, CountRows(dbPath, "symbol_references")); + var issue = Assert.Single(ReadFileIssues(dbPath, "null_byte")); + Assert.Equal("binary.cs", issue.Path); + Assert.Equal(0, issue.Line); + Assert.Contains("byte offset", issue.Message); } finally { @@ -1041,6 +1078,36 @@ public void Run_NullByteFile_SkipsWithoutPersistingPartialRows() } } + [Fact] + public void Run_FullScanJson_ProjectMarkerBudgetWarningIncludesTruncatedWarning() + { + var projectRoot = CreateTempProject(); + var previousEnumerator = FileIndexer.EnumerateProjectMarkerDirectoriesForTesting; + try + { + var childDir = Path.Combine(projectRoot, "nested"); + Directory.CreateDirectory(childDir); + File.WriteAllText(Path.Combine(projectRoot, "App.cs"), "public class App { }\n"); + FileIndexer.EnumerateProjectMarkerDirectoriesForTesting = + _ => Enumerable.Repeat(childDir, 8192); + + var (exitCode, json, _) = RunAndCaptureJsonWithStderr([projectRoot, "--json"]); + + Assert.Equal(CommandExitCodes.Success, exitCode); + Assert.Contains( + json.GetProperty("warnings").EnumerateArray(), + warning => + warning.GetProperty("message").GetString()!.Contains("Project marker discovery truncated", StringComparison.Ordinal) + && warning.GetProperty("message").GetString()!.Contains("directory budget", StringComparison.Ordinal)); + } + finally + { + FileIndexer.EnumerateProjectMarkerDirectoriesForTesting = previousEnumerator; + SqliteConnection.ClearAllPools(); + DeleteDirectory(projectRoot); + } + } + [Fact] public void Run_FileAboveMaxFileBytes_PersistsFileTooLargeIssue() { @@ -1126,6 +1193,111 @@ public void Run_FileAboveMaxSymbolsPerFile_PersistsSymbolCountExceededIssueOnly( } } + [Fact] + public void Run_FileAboveMaxReferencesPerFile_FullScanPersistsReferenceCountExceededIssueOnly_Issue3719() + { + var projectRoot = CreateTempProject(); + try + { + var filePath = Path.Combine(projectRoot, "DenseReferences.cs"); + File.WriteAllText(filePath, BuildDenseReferenceCSharpSource(8)); + + var (exitCode, json) = RunAndCaptureJson([projectRoot, "--max-references-per-file", "2", "--json"]); + + Assert.Equal(CommandExitCodes.Success, exitCode); + Assert.Equal("success", json.GetProperty("status").GetString()); + Assert.Equal(0, json.GetProperty("summary").GetProperty("errors").GetInt32()); + + var dbPath = Path.Combine(projectRoot, ".cdidx", "codeindex.db"); + Assert.Equal(1, CountRows(dbPath, "files")); + Assert.True(CountRows(dbPath, "chunks") > 0); + Assert.True(CountRows(dbPath, "symbols") > 0); + Assert.Equal(0, CountRows(dbPath, "symbol_references")); + + using var db = new DbContext(dbPath); + db.TryMigrateForRead(); + var reader = new DbReader(db.Connection, db.IsReadOnly); + var issue = Assert.Single(reader.GetIssues("reference_count_exceeded")); + Assert.Equal("DenseReferences.cs", issue.Path); + Assert.Equal(0, issue.Line); + Assert.Contains("--max-references-per-file", issue.Message); + + var (raisedExitCode, raisedJson) = RunAndCaptureJson([projectRoot, "--max-references-per-file", "100", "--json"]); + + Assert.Equal(CommandExitCodes.Success, raisedExitCode); + Assert.Equal("success", raisedJson.GetProperty("status").GetString()); + Assert.True(CountRows(dbPath, "symbol_references") > 0); + Assert.Empty(reader.GetIssues("reference_count_exceeded")); + } + finally + { + SqliteConnection.ClearAllPools(); + DeleteDirectory(projectRoot); + } + } + + [Fact] + public void Run_FileAboveMaxReferencesPerFile_UpdatePersistsReferenceCountExceededIssueOnly_Issue3719() + { + var projectRoot = CreateTempProject(); + try + { + var filePath = Path.Combine(projectRoot, "DenseReferences.cs"); + File.WriteAllText(filePath, BuildDenseReferenceCSharpSource(8)); + + var (initialExitCode, _) = RunAndCaptureJson([projectRoot, "--max-references-per-file", "100", "--json"]); + Assert.Equal(CommandExitCodes.Success, initialExitCode); + + var dbPath = Path.Combine(projectRoot, ".cdidx", "codeindex.db"); + Assert.True(CountRows(dbPath, "symbol_references") > 0); + + var (exitCode, json) = RunAndCaptureJson([projectRoot, "--files", filePath, "--max-references-per-file", "2", "--json"]); + + Assert.Equal(CommandExitCodes.Success, exitCode); + Assert.Equal("success", json.GetProperty("status").GetString()); + Assert.Equal(0, json.GetProperty("summary").GetProperty("errors").GetInt32()); + Assert.Equal(1, CountRows(dbPath, "files")); + Assert.True(CountRows(dbPath, "chunks") > 0); + Assert.True(CountRows(dbPath, "symbols") > 0); + Assert.Equal(0, CountRows(dbPath, "symbol_references")); + + using var db = new DbContext(dbPath); + db.TryMigrateForRead(); + var reader = new DbReader(db.Connection, db.IsReadOnly); + var issue = Assert.Single(reader.GetIssues("reference_count_exceeded")); + Assert.Equal("DenseReferences.cs", issue.Path); + Assert.Contains("--max-references-per-file", issue.Message); + } + finally + { + SqliteConnection.ClearAllPools(); + DeleteDirectory(projectRoot); + } + } + + private static string BuildDenseReferenceCSharpSource(int callCount) + { + var calls = string.Join('\n', Enumerable.Range(0, callCount).Select(static _ => " Target.Ping();")); + return $$""" +namespace DenseReferences; + +public static class Target +{ + public static void Ping() + { + } +} + +public sealed class Caller +{ + public void Run() + { +{{calls}} + } +} +"""; + } + [Fact] public void Run_SymbolsOnly_FullScanSkipsReferenceGraphUntilNormalIndex() { @@ -2329,6 +2501,28 @@ public void ResolveProjects_RejectsFallbackDiscoveryFileTraversalLimit_Issue3213 } } + [Fact] + public void ResolveProjects_RejectsFallbackDiscoveryDirectoryTraversalLimit() + { + var projectRoot = TestProjectHelper.CreateTempProject("cdidx_solution_fallback_directory_limit"); + try + { + Directory.CreateDirectory(Path.Combine(projectRoot, "A")); + Directory.CreateDirectory(Path.Combine(projectRoot, "B")); + var limits = SolutionProjectResolverLimits.Default with { MaxFallbackDiscoveryDirectories = 1 }; + + var ex = Assert.Throws( + () => SolutionProjectResolver.ResolveProjects(projectRoot, solutionPath: null, limits)); + + Assert.Contains("fallback project discovery traversed more than 1 directories", ex.Message); + Assert.Contains("pass --solution ", ex.Message); + } + finally + { + TestProjectHelper.DeleteDirectory(projectRoot); + } + } + [Fact] public void ResolveProjectFiles_HonorsGitRootIgnoreRulesForNestedWorkspace_Issue2862() { @@ -2707,6 +2901,22 @@ public void ParseArgs_MaxSymbolsPerFileInlineFlag_ParsesPositiveValue() Assert.Equal(43, options.MaxSymbolsPerFile); } + [Fact] + public void ParseArgs_MaxReferencesPerFileFlag_ParsesPositiveValue_Issue3719() + { + var options = IndexCommandRunner.ParseArgs([".", "--max-references-per-file", "42"]); + + Assert.Equal(42, options.MaxReferencesPerFile); + } + + [Fact] + public void ParseArgs_MaxReferencesPerFileInlineFlag_ParsesPositiveValue_Issue3719() + { + var options = IndexCommandRunner.ParseArgs([".", "--max-references-per-file=43"]); + + Assert.Equal(43, options.MaxReferencesPerFile); + } + [Fact] public void ParseArgs_SymbolsOnlyFlag_SetsOption() { @@ -2734,6 +2944,16 @@ public void ParseArgs_MaxSymbolsPerFileFlag_RejectsValueAboveMaximum_Issue3172() Assert.Contains("--max-symbols-per-file must be less than or equal to 50000", options.ParseError); } + [Fact] + public void ParseArgs_MaxReferencesPerFileFlag_RejectsValueAboveMaximum_Issue3719() + { + var aboveMaximum = $"{IndexCommandRunner.MaxReferencesPerFileLimit + 1}"; + var options = IndexCommandRunner.ParseArgs([".", $"--max-references-per-file={aboveMaximum}"]); + + Assert.Equal(IndexCommandRunner.DefaultMaxReferencesPerFile, options.MaxReferencesPerFile); + Assert.Contains("--max-references-per-file must be less than or equal to 1000000", options.ParseError); + } + [Fact] public void ParseArgs_MaxFileBytesInvalidValue_IsIgnored() { @@ -5828,6 +6048,103 @@ FROM symbol_references r } } + [Fact] + public void Run_FullScan_ConfiguredGeneratedCodePatternsKeepChunksButSkipExtraction() + { + using var env = EnvironmentVariableScope.Capture(IndexCommandRunner.GeneratedCodePatternsEnvironmentVariable); + var projectRoot = CreateTempProject(); + try + { + env.Set(IndexCommandRunner.GeneratedCodePatternsEnvironmentVariable, "src/generated/**"); + Directory.CreateDirectory(Path.Combine(projectRoot, "src", "generated")); + File.WriteAllText( + Path.Combine(projectRoot, "src", "generated", "GeneratedClient.cs"), + """ + public class GeneratedClient + { + public string Lookup() => "generated"; + } + """); + File.WriteAllText( + Path.Combine(projectRoot, "NormalClient.cs"), + """ + public class NormalClient + { + public string Lookup() => "normal"; + } + """); + + var exitCode = IndexCommandRunner.Run([projectRoot, "--json", "--quiet"], _jsonOptions); + Assert.Equal(CommandExitCodes.Success, exitCode); + + var dbPath = Path.Combine(projectRoot, ".cdidx", "codeindex.db"); + using (var conn = OpenNonPoolingConnection(dbPath)) + { + conn.Open(); + using var generatedCmd = conn.CreateCommand(); + generatedCmd.CommandText = """ + SELECT f.generated, + (SELECT COUNT(*) FROM chunks c WHERE c.file_id = f.id AND c.content LIKE '%GeneratedClient%'), + (SELECT COUNT(*) FROM symbols s WHERE s.file_id = f.id), + (SELECT COUNT(*) FROM symbol_references r WHERE r.file_id = f.id), + (SELECT COUNT(*) FROM file_issues i WHERE i.file_id = f.id AND i.kind = @issueKind) + FROM files f + WHERE f.path = @path + """; + generatedCmd.Parameters.AddWithValue("@issueKind", FileIndexer.GeneratedCodeExtractionSkippedIssueKind); + generatedCmd.Parameters.AddWithValue("@path", "src/generated/GeneratedClient.cs"); + using (var reader = generatedCmd.ExecuteReader()) + { + Assert.True(reader.Read()); + Assert.Equal(0, reader.GetInt32(0)); + Assert.True(reader.GetInt32(1) > 0); + Assert.Equal(0, reader.GetInt32(2)); + Assert.Equal(0, reader.GetInt32(3)); + Assert.Equal(1, reader.GetInt32(4)); + } + + using var normalCmd = conn.CreateCommand(); + normalCmd.CommandText = """ + SELECT COUNT(*) + FROM symbols s + JOIN files f ON f.id = s.file_id + WHERE f.path = 'NormalClient.cs' + AND s.name = 'NormalClient' + """; + Assert.Equal(1L, (long)normalCmd.ExecuteScalar()!); + } + + env.Set(IndexCommandRunner.GeneratedCodePatternsEnvironmentVariable, null); + var updateExitCode = IndexCommandRunner.Run([projectRoot, "--files", "src/generated/GeneratedClient.cs", "--json", "--quiet"], _jsonOptions); + Assert.Equal(CommandExitCodes.Success, updateExitCode); + + using (var conn = OpenNonPoolingConnection(dbPath)) + { + conn.Open(); + using var updatedCmd = conn.CreateCommand(); + updatedCmd.CommandText = """ + SELECT f.generated, + (SELECT COUNT(*) FROM symbols s WHERE s.file_id = f.id AND s.name = 'GeneratedClient'), + (SELECT COUNT(*) FROM file_issues i WHERE i.file_id = f.id AND i.kind = @issueKind) + FROM files f + WHERE f.path = @path + """; + updatedCmd.Parameters.AddWithValue("@issueKind", FileIndexer.GeneratedCodeExtractionSkippedIssueKind); + updatedCmd.Parameters.AddWithValue("@path", "src/generated/GeneratedClient.cs"); + using var reader = updatedCmd.ExecuteReader(); + Assert.True(reader.Read()); + Assert.Equal(0, reader.GetInt32(0)); + Assert.Equal(1, reader.GetInt32(1)); + Assert.Equal(0, reader.GetInt32(2)); + } + } + finally + { + DeleteDirectory(projectRoot); + SqliteConnection.ClearAllPools(); + } + } + [Fact] public void Run_UpdateFiles_CsharpStaticInterfaceContractChange_ReindexesImplementers() { @@ -6136,7 +6453,7 @@ public void Run_Rebuild_CancelledAfterReadinessDemotion_PreservesExistingIndex() } [Fact] - public void Run_Rebuild_WhenIndexedFileBecomesBinary_RemovesStaleRow() + public void Run_Rebuild_WhenIndexedFileBecomesBinary_PersistsNullByteIssue() { var projectRoot = CreateTempProject(); try @@ -6155,7 +6472,9 @@ public void Run_Rebuild_WhenIndexedFileBecomesBinary_RemovesStaleRow() var rebuildExitCode = IndexCommandRunner.Run([projectRoot, "--rebuild", "--yes", "--json"], _jsonOptions); Assert.Equal(CommandExitCodes.Success, rebuildExitCode); - Assert.DoesNotContain("app.py", ReadIndexedPaths(dbPath)); + Assert.Contains("app.py", ReadIndexedPaths(dbPath)); + var issue = Assert.Single(ReadFileIssues(dbPath).Where(issue => issue.Path == "app.py" && issue.Kind == "null_byte")); + Assert.Contains("byte offset 0", issue.Message, StringComparison.Ordinal); } finally { @@ -7788,9 +8107,14 @@ public void Run_FullScan_IgnoresOversizedCheckpoint() padding.Append(' ', 1024).Append('\n'); File.WriteAllText(checkpointPath, checkpoint + padding); - var (exitCode, _) = RunAndCaptureJson([projectRoot, "--json"]); + var (exitCode, json) = RunAndCaptureJson([projectRoot, "--json"]); Assert.Equal(CommandExitCodes.Success, exitCode); + Assert.Contains( + json.GetProperty("warnings").EnumerateArray(), + warning => + warning.GetProperty("file").GetString() == "" + && warning.GetProperty("message").GetString()!.Contains("file exceeds the scan checkpoint size limit", StringComparison.Ordinal)); var indexedPaths = ReadIndexedPaths(Path.Combine(projectRoot, ".cdidx", "codeindex.db")); Assert.Contains("src/a.cs", indexedPaths); @@ -11571,6 +11895,32 @@ private static HashSet ReadIndexedPaths(string dbPath) .ToHashSet(StringComparer.Ordinal); } + private static List ReadFileIssues(string dbPath) + { + using var connection = OpenNonPoolingConnection(dbPath); + connection.Open(); + using var command = connection.CreateCommand(); + command.CommandText = """ + SELECT f.path, i.kind, i.line, i.message + FROM file_issues i + JOIN files f ON f.id = i.file_id + ORDER BY f.path, i.kind, i.line, i.message + """; + using var reader = command.ExecuteReader(); + var issues = new List(); + while (reader.Read()) + { + issues.Add(new FileIssue + { + Path = reader.GetString(0), + Kind = reader.GetString(1), + Line = reader.GetInt32(2), + Message = reader.GetString(3), + }); + } + return issues; + } + private static Dictionary ReadSymbolKindCounts(string dbPath) { using var connection = OpenNonPoolingConnection(dbPath); diff --git a/tests/CodeIndex.Tests/McpServerTests.cs b/tests/CodeIndex.Tests/McpServerTests.cs index c7b79eb376..e502526a15 100644 --- a/tests/CodeIndex.Tests/McpServerTests.cs +++ b/tests/CodeIndex.Tests/McpServerTests.cs @@ -270,6 +270,29 @@ private static JsonNode CallIndex(McpServer server, string path, Action " Target.Ping();")); + return $$""" +namespace DenseReferences; + +public static class Target +{ + public static void Ping() + { + } +} + +public sealed class Caller +{ + public void Run() + { +{{calls}} + } +} +"""; + } + private static Dictionary ReadSymbolKindCounts(string dbPath) { var counts = new Dictionary(StringComparer.Ordinal); @@ -10632,6 +10655,60 @@ public void ToolsCall_Index_ReprocessesAfterPartialSymbolKindFilterChange_Issue3 } } + [Fact] + public void ToolsCall_Index_GeneratedCodePatternCountsProcessedAndSkipsExtraction_Issue3720() + { + using var env = EnvironmentVariableScope.Capture(IndexCommandRunner.GeneratedCodePatternsEnvironmentVariable); + var fixtureDir = Path.Combine(Path.GetFullPath("."), $"mcp_index_generated_pattern_{Guid.NewGuid():N}"); + Directory.CreateDirectory(fixtureDir); + var dbPath = Path.Combine(Path.GetTempPath(), $"cdidx_mcp_index_generated_pattern_{Guid.NewGuid():N}.db"); + try + { + env.Set(IndexCommandRunner.GeneratedCodePatternsEnvironmentVariable, "generated/**"); + Directory.CreateDirectory(Path.Combine(fixtureDir, "generated")); + File.WriteAllText( + Path.Combine(fixtureDir, "generated", "Client.cs"), + "public class GeneratedClient { public string Lookup() => \"generated\"; }\n"); + using var server = new McpServer(dbPath, ConsoleUi.LoadVersion()); + + var response = CallIndex(server, fixtureDir); + + Assert.False(response["result"]?["isError"]?.GetValue() ?? false); + using var verifyDb = new DbContext(dbPath); + Assert.Equal("1", verifyDb.GetMetaString(DbContext.LastIndexRunRowsUpsertedMetaKey)); + var connectionString = new SqliteConnectionStringBuilder + { + DataSource = dbPath, + }.ToString(); + using var connection = new SqliteConnection(connectionString); + connection.Open(); + using var command = connection.CreateCommand(); + command.CommandText = """ + SELECT f.generated, + (SELECT COUNT(*) FROM chunks c WHERE c.file_id = f.id AND c.content LIKE '%GeneratedClient%'), + (SELECT COUNT(*) FROM symbols s WHERE s.file_id = f.id), + (SELECT COUNT(*) FROM symbol_references r WHERE r.file_id = f.id), + (SELECT COUNT(*) FROM file_issues i WHERE i.file_id = f.id AND i.kind = @issueKind) + FROM files f + WHERE f.path = @path + """; + command.Parameters.AddWithValue("@issueKind", FileIndexer.GeneratedCodeExtractionSkippedIssueKind); + command.Parameters.AddWithValue("@path", "generated/Client.cs"); + using var reader = command.ExecuteReader(); + Assert.True(reader.Read()); + Assert.Equal(0, reader.GetInt32(0)); + Assert.True(reader.GetInt32(1) > 0); + Assert.Equal(0, reader.GetInt32(2)); + Assert.Equal(0, reader.GetInt32(3)); + Assert.Equal(1, reader.GetInt32(4)); + } + finally + { + TestProjectHelper.DeleteDirectory(fixtureDir); + DeleteSqliteDatabaseFiles(dbPath); + } + } + [Fact] public void ToolsCall_UnknownArgumentName_TruncatesDisplay_Issue3117() { @@ -11331,6 +11408,97 @@ public void ToolsCall_Index_SuccessfulNoOpBackfillsMissingIndexedProjectRootMeta } } + [Fact] + public void ToolsCall_Index_NullByteFilePersistsNullByteIssue_Issue3835() + { + var fixtureDir = Path.Combine(Path.GetFullPath("."), $"mcp_index_null_byte_{Guid.NewGuid():N}"); + var dbPath = Path.Combine(Path.GetTempPath(), $"cdidx_mcp_index_null_byte_{Guid.NewGuid():N}.db"); + try + { + Directory.CreateDirectory(fixtureDir); + var prefix = Encoding.UTF8.GetBytes("public class Polluted { public void Run() { } }\n"); + var bytes = new byte[prefix.Length + 1]; + Array.Copy(prefix, bytes, prefix.Length); + bytes[^1] = 0; + File.WriteAllBytes(Path.Combine(fixtureDir, "binary.cs"), bytes); + + using var server = new McpServer(dbPath, ConsoleUi.LoadVersion()); + var request = new JsonObject + { + ["jsonrpc"] = "2.0", + ["id"] = 1, + ["method"] = "tools/call", + ["params"] = new JsonObject + { + ["name"] = "index", + ["arguments"] = new JsonObject + { + ["path"] = fixtureDir + } + } + }; + var response = server.HandleMessage(request)!; + + Assert.False(response["result"]!["isError"]?.GetValue() ?? false, response.ToJsonString()); + using var db = new DbContext(dbPath); + db.TryMigrateForRead(); + var reader = new DbReader(db.Connection, db.IsReadOnly); + var issue = Assert.Single(reader.GetIssues("null_byte")); + Assert.Equal("binary.cs", issue.Path); + Assert.Equal(0, issue.Line); + Assert.Contains("byte offset", issue.Message); + } + finally + { + if (Directory.Exists(fixtureDir)) + TestProjectHelper.DeleteDirectory(fixtureDir); + DeleteFileRobust(dbPath); + } + } + + [Fact] + public void ToolsCall_Index_MaxReferencesPerFilePersistsReferenceCountExceededIssue_Issue3719() + { + var fixtureDir = Path.Combine(Path.GetFullPath("."), $"mcp_index_reference_cap_{Guid.NewGuid():N}"); + var dbPath = Path.Combine(Path.GetTempPath(), $"cdidx_mcp_index_reference_cap_{Guid.NewGuid():N}.db"); + try + { + Directory.CreateDirectory(fixtureDir); + File.WriteAllText(Path.Combine(fixtureDir, "DenseReferences.cs"), BuildDenseReferenceCSharpSource(8)); + + using var server = new McpServer(dbPath, ConsoleUi.LoadVersion()); + var response = CallIndex(server, fixtureDir, args => args["maxReferencesPerFile"] = 2); + + Assert.False(response["result"]?["isError"]?.GetValue() ?? false, response.ToJsonString()); + using var db = new DbContext(dbPath); + db.TryMigrateForRead(); + var reader = new DbReader(db.Connection, db.IsReadOnly); + var issue = Assert.Single(reader.GetIssues("reference_count_exceeded")); + Assert.Equal("DenseReferences.cs", issue.Path); + Assert.Equal(0, issue.Line); + Assert.Contains("maxReferencesPerFile", issue.Message); + + using var command = db.Connection.CreateCommand(); + command.CommandText = """ + SELECT + (SELECT COUNT(*) FROM chunks), + (SELECT COUNT(*) FROM symbols), + (SELECT COUNT(*) FROM symbol_references) + """; + using var row = command.ExecuteReader(); + Assert.True(row.Read()); + Assert.True(row.GetInt64(0) > 0); + Assert.True(row.GetInt64(1) > 0); + Assert.Equal(0, row.GetInt64(2)); + } + finally + { + if (Directory.Exists(fixtureDir)) + TestProjectHelper.DeleteDirectory(fixtureDir); + DeleteSqliteDatabaseFiles(dbPath); + } + } + [Fact] public void ToolsCall_BackfillFold_StampsFoldReady() {