From 5caddea1a3a6d71266e7afdf56e2a55bdb804377 Mon Sep 17 00:00:00 2001 From: Widthdom Date: Fri, 31 Jul 2026 00:01:44 +0900 Subject: [PATCH 1/3] Add bounded MCP language catalog pagination (#4896) --- DEVELOPER_GUIDE.md | 2 + TESTING_GUIDE.md | 4 +- USER_GUIDE.md | 21 + changelog.d/unreleased/4896.added.md | 30 + src/CodeIndex/Cli/LanguageCatalog.cs | 146 +++++ .../Cli/QueryCommandRunner.Languages.cs | 167 +---- src/CodeIndex/Mcp/McpToolArgumentContracts.cs | 2 +- src/CodeIndex/Mcp/McpToolCatalog.cs | 8 +- .../Mcp/McpToolHandlers.ArgumentValidation.cs | 2 +- .../Mcp/McpToolHandlers.Languages.cs | 604 ++++++++++++------ .../McpServerIssue4896Tests.cs | 222 +++++++ .../McpServerToolsCallTests.cs | 8 +- tests/CodeIndex.Tests/McpToolContractTests.cs | 27 + 13 files changed, 887 insertions(+), 356 deletions(-) create mode 100644 changelog.d/unreleased/4896.added.md create mode 100644 src/CodeIndex/Cli/LanguageCatalog.cs create mode 100644 tests/CodeIndex.Tests/McpServerIssue4896Tests.cs diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index 0fa7f4a33..d50e51916 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -2036,6 +2036,7 @@ Process exit codes are coarse (`0` success including valid zero-row queries, `1` - **HTTP MCP aggregate request-body budget** — `CDIDX_MCP_HTTP_MAX_IN_FLIGHT_REQUEST_BYTES` bounds process-wide request-body reservations across reads, queued frames, MCP execution, and response completion. It defaults to 64 MiB, is capped at 1 GiB, must be at least the individual request cap, and rejects saturation with classified HTTP 429 instead of allowing queue depth multiplied by per-request size to become the memory bound (#4548). - **MCP bounded queues and concurrency gates** — HTTP request queue slots are acquired before `TryWrite`; once full, requests are rejected with HTTP 429, `Retry-After: 1`, `X-Cdidx-Mcp-Rejection: request_queue_limit`, and `http_request_queue_rejection_count` rather than blocking an HTTP handler. The request-log queue is best-effort and increments `http_request_log_queue_full_drop_count` / `http_request_log_dropped_count` on saturation. POST handlers and long-lived event streams use independent admission semaphores, report `concurrent_handler_limit` and `event_stream_limit` separately, and expose their effective capacities plus `http_separate_event_stream_handlers` through HTTP health. Limit environment variables use defaults only when absent; a present malformed or out-of-range value fails before listener startup. Transport-owned queue and handler gates are disposed only after bounded shutdown observes every acquired slot returned, because a late handler may still finish after listener teardown. Frame-loop gates are disposed only after EOF drain observes all request tasks, because bounded drain can intentionally leave late tasks running. - **MCP pagination offset cap** — `references`, `callers`, and `callees` clamp `offset` to 10,000 before executing SQL queries. `tools/list` advertises the maximum in each offset schema, and MCP `status` mirrors it under `mcp.limits.max_pagination_offset`. +- **MCP language-catalog pagination** — MCP `languages` builds rows through the same canonical `LanguageCatalog` used by the CLI, then applies exact normalized language/extension/alias filters and ordinal language sorting. Pages default to 20 rows and use opaque response-v2 cursors bound to every filter, `limit`, `maxBytes`, the fixed sort contract, and a fingerprint of the emitted catalog generation (including indexed-language membership for `indexedOnly`). Query changes return `cursor_query_mismatch`; catalog changes return retry-safe `cursor_stale`. `maxBytes` accepts 4,096 through 1,000,000 bytes and measures the complete UTF-8 JSON-RPC envelope, shrinking the row page before emission while keeping filtered totals separate from catalog and capability counts. - **MCP compact discovery and status projection** — `tools/list` keeps its complete response as both the default and explicit `format: "full"` contract. Opt-in `format: "compact"` replaces long descriptions, schemas, examples, and catalog metadata with bounded summaries and an on-demand full-definition recipe; `names` filters exact enabled tool names only and is capped at 24 names of 128 characters. Opaque continuation cursors preserve compact/name-filtered controls, and name-filtered full responses mark the returned list as scoped while deriving capability metadata from all enabled tools. The compact schema is deliberately non-authoritative and says so in `_meta`. MCP `status.fields` projects exact top-level fields after `format` and optional diagnostic attachments are built, while `api_version` remains part of every structured result. Projection inputs are capped at 32 names, 128 characters each, and 2,048 characters total; unknown names and nested paths fail as `invalid_argument`. Any new discovery or status mode must preserve the no-argument response byte-for-byte and add size/compatibility regression tests. - **MCP resource-list cursor stability** — `resources/list` emits a fixed-size opaque keyset cursor that binds the last consumed file id to a persisted indexed-file generation and the canonical discovery filters. The reader resolves that id back to the existing source/test/docs bucket plus path ordering inside the same SQLite snapshot. Any file insertion, deletion, or update changes the generation; a later page then returns `-32011` / `index_stale` with `restart_required: true`. Changing `path`, `lang`, or `includeGenerated` between pages returns `-32602` / `resources_list_filters_changed` with the same restart requirement. In either case, the client must omit `params.cursor` to restart instead of continuing across mixed snapshots or filters. Writable legacy databases install the generation row and triggers through the normal read migration before a cursor is issued. A mutable read-only legacy database that cannot prove generation tracking returns `resources_list_generation_unavailable` with `migration_required: true`; a canonical, unambiguous `immutable=1` legacy URI (optionally paired with `mode=ro`) may safely use connection-local generation zero because it cannot change between pages. Encoded, case-variant, whitespace-padded, duplicated, conflicting, or extra query parameters are not trusted as that immutable guarantee. The legacy decimal zero remains a first-page upgrade input, but nonzero decimal offsets cannot prove their source generation and therefore return the same restart-required error; decimal cursors are never emitted. Version-1 opaque cursors remain valid only with the default unfiltered view. - **MCP file resource discovery** — `resources/templates/list` advertises `cdidx://file-path/{path}` so a client that already knows an exact repository-relative path can construct a `resources/read` URI without paging the repository inventory. Simple URI-template expansion percent-encodes separators and reserved filename characters such as `?` and `#`; the template-only resolver decodes the value once, rejects absolute paths, traversal, backslashes, empty segments, queries, and fragments, then returns the canonical `cdidx://file/` identity. Canonical resource URIs continue to reject encoded separators. `resources/list` accepts `path` as one string or at most 100 strings of at most 1024 characters and 128 wildcard operators each, using the same anchored directory/glob semantics as file queries, plus an exact normalized `lang` filter and `includeGenerated` (default `false`). Generated files also require `includeGenerated: true` for direct reads. @@ -5464,6 +5465,7 @@ USER_GUIDEの[終了コード](USER_GUIDE.md#終了コード)セクションを - **HTTP MCP aggregate request-body budget** — `CDIDX_MCP_HTTP_MAX_IN_FLIGHT_REQUEST_BYTES` は read、queued frame、MCP 実行、response 完了にまたがる process-wide の request-body reservation を制限する。既定値は 64 MiB、最大値は 1 GiB で、request 単位上限以上でなければならない。飽和時は分類済み HTTP 429 で拒否し、queue depth と request 単位 size の積が memory 上限になることを防ぐ (#4548)。 - **MCP の有界 queue / concurrency gate** — HTTP request queue は `TryWrite` 前に slot を取り、満杯時は handler を block せず HTTP 429、`Retry-After: 1`、`X-Cdidx-Mcp-Rejection: request_queue_limit` で拒否する。POST handler と長寿命 event stream は独立した admission semaphore を使い、HTTP health は両方の有効 capacity と `http_separate_event_stream_handlers` を返す。limit 環境変数は未設定の場合だけ既定値を使い、設定済みの malformed 値または範囲外値は listener 起動前に失敗する。transport 所有 gate は bounded shutdown で全取得 slot の返却を確認できた場合だけ dispose する。 - **MCP pagination offset 上限** — `references`、`callers`、`callees` は SQL query 実行前に `offset` を 10,000 へクランプする。`tools/list` は各 offset schema に最大値を広告し、MCP `status` も `mcp.limits.max_pagination_offset` に同じ値を返す。 +- **MCP 言語 catalog の pagination** — MCP `languages` は CLI と共通の canonical `LanguageCatalog` から row を構築し、正規化した言語・拡張子・alias の完全一致 filter と言語名の ordinal sort を適用する。page は既定 20 行で、opaque な response-v2 cursor を全 filter、`limit`、`maxBytes`、固定 sort contract、出力 catalog generation の fingerprint（`indexedOnly` では indexed-language membership も含む）へ拘束する。query 変更は `cursor_query_mismatch`、catalog 変更は retry-safe な `cursor_stale` を返す。`maxBytes` は 4,096〜1,000,000 bytes を受け付け、UTF-8 JSON-RPC envelope 全体を計測して送信前に row page を縮小し、filtered total を catalog 件数および capability 件数から分離したまま返す。 - **MCP compact discovery と status projection** — `tools/list` は完全 response を既定値および明示的な `format: "full"` 契約として維持する。opt-in の `format: "compact"` は長い説明、schema、example、catalog metadata を上限付き要約と完全定義の on-demand 取得方法へ置き換える。`names` は有効化済み tool の exact name だけを filter し、128 文字の名前を最大 24 件まで受け付ける。opaque な continuation cursor は compact / name-filtered control を保持し、name-filtered な full response は返却 list を限定 scope と明示しつつ、有効な全 tool から capability metadata を構築する。compact schema は意図的に非 authoritative であり、その旨を `_meta` に明示する。MCP `status.fields` は `format` と任意 diagnostic attachment の構築後に exact な top-level field を project し、`api_version` はすべての structured result に残す。projection 入力は最大 32 件、各 128 文字、合計 2,048 文字で、未知名と nested path は `invalid_argument` とする。新しい discovery / status mode を追加する場合は、引数なし response を byte-for-byte で維持し、size と互換性の回帰テストを追加すること。 - **MCP ファイル resource discovery** — `resources/templates/list` は `cdidx://file-path/{path}` を公開し、正確なリポジトリ相対 path が既知の client は全 inventory をページングせず `resources/read` URI を構築できる。simple URI-template expansion は separator と `?` / `#` などの予約 filename 文字を percent encode する。template 専用 resolver は値を一度だけ decode し、absolute path、traversal、backslash、空 segment、query、fragment を拒否して canonical な `cdidx://file/` identity を返す。canonical resource URI は encoded separator を引き続き拒否する。`resources/list` は `path` に 1 文字列または各 1024 文字・wildcard operator 128 個以内かつ最大 100 件の文字列を受け付け、file query と同じ anchored directory / glob semantics を使うほか、正規化した完全一致の `lang` と `includeGenerated`（既定 `false`）も受け付ける。cursor は generation と canonical filter の両方に結び付き、ページ間の filter 変更は `-32602` / `resources_list_filters_changed` と `restart_required: true` を返す。generated file は discovery と direct read のどちらでも `includeGenerated: true` が必要である。 - **MCP resource list カーソルの安定性** — `resources/list` は、最後に消費した file id と永続化されたインデックス済みファイル世代を結び付ける固定長の不透明 keyset cursor を返す。reader は同じ SQLite snapshot 内でその id を既存の source/test/docs bucket と path の並び順へ解決する。ファイルの追加・削除・更新で世代が変わると、後続ページは `restart_required: true` 付きの `-32011` / `index_stale` を返し、混在 snapshot を続行せず `params.cursor` を省略して再開する必要がある。書き込み可能な legacy DB は cursor 発行前に通常の read migration で世代 row と trigger を導入する。世代追跡を証明できない変更可能な read-only legacy DB は `migration_required: true` 付きの `resources_list_generation_unavailable` を返すが、canonical かつ曖昧性のない `immutable=1` legacy URI（任意で `mode=ro` を併記）はページ間で変化しないため connection-local な世代 0 を安全に利用できる。encoded、case variant、空白付き、重複、競合、または余分な query parameter はこの immutable 保証として信頼しない。旧 decimal の 0 は先頭ページ用の移行入力として残すが、0 以外の decimal offset は発行時世代を検証できないため同じ再開必須 error を返し、decimal cursor は出力に使わない。 diff --git a/TESTING_GUIDE.md b/TESTING_GUIDE.md index 1f17e6363..9277741c5 100644 --- a/TESTING_GUIDE.md +++ b/TESTING_GUIDE.md @@ -200,7 +200,7 @@ Use `docs/test-doc-maintenance-plan.md` before moving oversized suites or adding Grouped-search limit coverage must seed more matches and files than the returned page, then assert query-wide matched/group/file totals separately from grouped, emitted, and omitted row counts so a bounded page cannot report itself as complete. `goto --all` coverage shares one 63-definition fixture across the default bare LSP array and the plain `--json` argument path; it must assert complete deterministic output without a limit and retain a separate explicit-limit assertion in the same setup. Cursor-pagination coverage for `outline` and `unused` must replay the emitted opaque token successfully, then prove that changed filters/order and a changed index-generation stamp fail with restart-required diagnostics. - Discovery-pagination coverage must page compact search/symbol/file output and the opt-in search-array/languages envelopes, assert authoritative total/omitted/stable-at metadata, reject query or generation cursor mismatches, and prove that a partial `find --all` scan resumes after the last scanned line. + Discovery-pagination coverage must page compact search/symbol/file output and the opt-in search-array/languages envelopes, assert authoritative total/omitted/stable-at metadata, reject query or generation cursor mismatches, and prove that a partial `find --all` scan resumes after the last scanned line. MCP `languages` coverage additionally keeps exact canonical/alias/extension matching (including ambiguity buckets and Unicode-empty lookups), gap-free full enumeration, catalog-generation invalidation, separate catalog/capability counts, and exact whole-envelope UTF-8 byte boundaries in one focused suite. Inspect graph-section coverage must compare name and path/line resolution through the same persisted candidate ID, keep ambiguous overload and partial-family bundles isolated, assert independent total/returned/truncated metadata for references, callers, and callees (including empty sections), and replay a query-, page-size-, and generation-bound cursor across the smallest two-row page boundary. Put inbound callers in another file to prove the location path is only a locator, seed equal-rank same-line callees to pin the complete identity tie-breakers, reject page-size changes before candidate lookup, reject inspect cursors in another command, and verify the same envelopes in MCP `analyze_symbol`. Quiet-flag coverage in `ProgramCliTests.cs` reuses one seeded symbols database across text, NDJSON, and JSON-array modes and compares stdout with and without a trailing quiet alias, proving that quiet mode changes only informational stderr. Doctor full-inventory coverage keeps composed filter selection, filtered summary counts, exact UTF-8 byte-budget boundaries, and structured overflow errors together in `ProgramRunnerTests`; license JSON remains a subprocess contract in `ProgramCliTests` so immediate-command dispatch and the published field names are both exercised. @@ -1144,7 +1144,7 @@ dotnet test --filter "FullyQualifiedName~GitHelperTests" grouped search の limit coverage では、返却 page より多い match と file を seed し、query 全体の match/group/file 総数と grouped、emitted、omitted row 数を別々に検証して、上限付き page が完了済みと報告できないようにしてください。 `goto --all` の coverage は、既定の bare LSP array と plain `--json` 引数経路で 1 つの 63 定義 fixture を共有し、limit 未指定時の完全かつ決定的な出力と、同じ setup 内での明示 limit の維持を検証してください。 `outline` と `unused` の cursor pagination coverage では、出力された opaque token の再利用成功に加え、filter/order の変更と index-generation stamp の変更が restart-required 診断で失敗することを検証してください。 - discovery pagination の coverage では compact search / symbol / file と opt-in の search array / languages envelope をページングし、authoritative な total / omitted / stable-at metadata、query / generation 不一致 cursor の拒否、partial な `find --all` scan が最後に scan した line の次から再開することを検証してください。 + discovery pagination の coverage では compact search / symbol / file と opt-in の search array / languages envelope をページングし、authoritative な total / omitted / stable-at metadata、query / generation 不一致 cursor の拒否、partial な `find --all` scan が最後に scan した line の次から再開することを検証してください。MCP `languages` の coverage ではさらに、canonical / alias / extension の完全一致（ambiguity bucket と Unicode の空 lookup を含む）、欠落のない全件列挙、catalog generation の失効、catalog 件数と capability 件数の分離、response envelope 全体の UTF-8 byte exact boundary を1つの focused suite にまとめます。 inspect graph-section coverage では、name と path/line resolution が同じ persisted candidate ID を通ること、曖昧な overload と partial-family bundle が分離されること、references / callers / callees の独立した total / returned / truncated metadata（空 section を含む）を検証してください。inbound caller は別ファイルに置いて location path が locator にすぎないことを証明し、最小の2行 page 境界で query / page size / generation に束縛された cursor を再利用します。同順位かつ同じ行の callee で完全な identity tie-breaker を固定し、candidate lookup より前に page-size 変更を拒否し、別 command では inspect cursor を拒否してください。MCP `analyze_symbol` でも同じ envelope を確認します。 `ProgramCliTests.cs` の quiet flag coverage は1つの seeded symbols database を text、NDJSON、JSON array の各 mode で再利用し、末尾に quiet alias を付けた場合と付けない場合の stdout を比較して、quiet mode が informational stderr だけを変えることを固定します。 doctor full-inventory coverage では、合成 filter の選択、filtered summary 件数、UTF-8 byte budget の exact boundary、structured overflow error を `ProgramRunnerTests` にまとめます。license JSON は `ProgramCliTests` の subprocess contract として、immediate-command dispatch と公開 field 名を同時に検証します。 diff --git a/USER_GUIDE.md b/USER_GUIDE.md index a896743b2..e7e2ae16e 100644 --- a/USER_GUIDE.md +++ b/USER_GUIDE.md @@ -2638,6 +2638,17 @@ Rows with unsupported references or graph queries include `unsupported_guidance` entries with the unsupported capability, an explanatory message, and `recommended_commands` for the next safe query. +MCP clients use the `languages` tool against the same canonical catalog and exact +`language`, `extension`, `alias`, and `capability` matching rules. The default +page contains at most 20 rows; follow the opaque `next_cursor` with unchanged +filters, `limit`, and `maxBytes` to enumerate the catalog without gaps. +`maxBytes` bounds the complete UTF-8 JSON-RPC response envelope from 4,096 to +1,000,000 bytes. Page metadata reports authoritative filtered totals separately +from `summary.catalog_language_count` and the symbol/reference capability counts, +plus whether the item or byte budget caused continuation. A changed filter or +page control returns a typed cursor mismatch, while a changed catalog generation +returns a typed stale-cursor error that requires restarting without `cursor`. + | Language family | Symbols | References / graph | Notes and example query | |---|---|---|---| | C# / Razor / Blazor | namespaces, types, members, properties, imports | calls, constructors, events, attributes, annotations, type references, metadata edges | Modern partial members and metadata targets are indexed. `cdidx inspect Run --lang csharp --exact-name` | @@ -5870,6 +5881,16 @@ indexing はファイル単位の SQLite transaction を commit します。長 参照抽出やグラフクエリが未対応の行では、`unsupported_guidance` に未対応の機能、説明メッセージ、 次に安全に使う `recommended_commands` が入ります。 +MCP client では、同じ canonical catalog と完全一致の `language`、`extension`、`alias`、 +`capability` 規則を `languages` tool から利用できます。既定 page は最大 20 行です。 +全 catalog を欠落なく列挙するには、filter、`limit`、`maxBytes` を変えずに不透明な +`next_cursor` を渡してください。`maxBytes` は UTF-8 JSON-RPC response envelope 全体を +4,096〜1,000,000 bytes に制限します。page metadata は authoritative な filtered total を +`summary.catalog_language_count` および symbol/reference capability 件数と分離し、 +item budget と byte budget のどちらが継続理由かも示します。filter または page control を +変えると typed cursor mismatch、catalog generation が変わると `cursor` なしでの再開を +要求する typed stale-cursor error を返します。 + | 言語ファミリ | Symbols | References / graph | メモと例 | |---|---|---|---| | C# / Razor / Blazor | namespace、type、member、property、import | call、constructor、event、attribute、annotation、type reference、metadata edge | modern partial member と metadata target を索引します。`cdidx inspect Run --lang csharp --exact-name` | diff --git a/changelog.d/unreleased/4896.added.md b/changelog.d/unreleased/4896.added.md new file mode 100644 index 000000000..0976d2d13 --- /dev/null +++ b/changelog.d/unreleased/4896.added.md @@ -0,0 +1,30 @@ +--- +category: added +issues: + - 4896 +affected: + - src/CodeIndex/Cli/LanguageCatalog.cs + - src/CodeIndex/Cli/QueryCommandRunner.Languages.cs + - src/CodeIndex/Mcp/McpToolArgumentContracts.cs + - src/CodeIndex/Mcp/McpToolHandlers.Languages.cs + - src/CodeIndex/Mcp/McpToolHandlers.ArgumentValidation.cs + - src/CodeIndex/Mcp/McpToolCatalog.cs + - tests/CodeIndex.Tests/McpServerIssue4896Tests.cs + - tests/CodeIndex.Tests/McpServerToolsCallTests.cs + - tests/CodeIndex.Tests/McpToolContractTests.cs + - USER_GUIDE.md + - DEVELOPER_GUIDE.md + - TESTING_GUIDE.md +--- + +## English + +- Make the MCP language catalog queryable by exact language, alias, extension, + and capability filters, with stable catalog-generation-bound pagination and a + UTF-8 byte budget for the complete JSON-RPC response envelope. + +## 日本語 + +- MCP の言語カタログを言語名・alias・拡張子・capability の完全一致で検索可能にし、 + catalog generation に拘束された安定 pagination と JSON-RPC response envelope 全体の + UTF-8 byte budget を追加しました。 diff --git a/src/CodeIndex/Cli/LanguageCatalog.cs b/src/CodeIndex/Cli/LanguageCatalog.cs new file mode 100644 index 000000000..59ca810ae --- /dev/null +++ b/src/CodeIndex/Cli/LanguageCatalog.cs @@ -0,0 +1,146 @@ +using System.Text; +using CodeIndex.Database; +using CodeIndex.Indexer; +using CodeIndex.Indexer.Extensibility; +using CodeIndex.Models; + +namespace CodeIndex.Cli; + +internal sealed record LanguageCatalogEntry( + List Extensions, + List ExactFilenames, + List FilenamePrefixPatterns, + List LegacyPatterns, + List PatternProvenance, + List Aliases, + bool Symbols, + bool References, + bool Graph, + List CapabilityGaps, + List UnsupportedGuidance); + +internal sealed record LanguageCatalogSnapshot( + IReadOnlyList> Languages, + int SymbolLanguageCount, + int ReferenceLanguageCount, + IReadOnlyList Diagnostics); + +internal static class LanguageCatalog +{ + internal static LanguageCatalogSnapshot Build(string? workspaceRoot) + { + ExtractorPluginRegistry.LoadPatternConfigsForProjectRoot(workspaceRoot); + var languagePatterns = FileIndexer.GetLanguagePatterns(workspaceRoot, out var diagnostics); + var symbolLanguages = SymbolExtractor.GetSupportedLanguages(workspaceRoot); + var referenceLanguages = ReferenceExtractor.GetSupportedLanguages(workspaceRoot); + var languages = new Dictionary(StringComparer.Ordinal); + + foreach (var pattern in languagePatterns) + { + var language = pattern.Language; + if (!languages.TryGetValue(language, out var entry)) + { + entry = CreateEntry(language, symbolLanguages, referenceLanguages); + languages[language] = entry; + } + + switch (pattern.Kind) + { + case FileIndexer.LanguagePatternKind.Extension: + entry.Extensions.Add(pattern.Pattern); + break; + case FileIndexer.LanguagePatternKind.ExactFilename: + entry.ExactFilenames.Add(pattern.Pattern); + break; + case FileIndexer.LanguagePatternKind.FilenamePrefixPattern: + entry.FilenamePrefixPatterns.Add(pattern.Pattern); + break; + } + + if (!entry.LegacyPatterns.Contains(pattern.Pattern, StringComparer.Ordinal)) + entry.LegacyPatterns.Add(pattern.Pattern); + entry.PatternProvenance.Add(pattern); + } + + foreach (var language in FileIndexer.GetContentDetectedLanguageBuckets()) + { + if (!languages.ContainsKey(language)) + languages[language] = CreateEntry(language, symbolLanguages, referenceLanguages); + } + + return new LanguageCatalogSnapshot( + languages.OrderBy(pair => pair.Key, StringComparer.Ordinal).ToList(), + symbolLanguages.Count, + referenceLanguages.Count, + diagnostics); + } + + internal static bool MatchesLanguage(string language, string lookup) + => string.Equals( + DbReader.NormalizeQueryLanguage(lookup), + language, + StringComparison.Ordinal); + + internal static bool MatchesExtension(LanguageCatalogEntry language, string lookup) + { + var normalized = NormalizeLookupKey(lookup); + return language.Extensions.Any(extension => + string.Equals(NormalizeLookupKey(extension), normalized, StringComparison.Ordinal)); + } + + internal static bool MatchesAlias(LanguageCatalogEntry language, string lookup) + { + var normalized = NormalizeLookupKey(lookup); + return language.Aliases.Any(alias => + string.Equals(NormalizeLookupKey(alias), normalized, StringComparison.Ordinal)); + } + + internal static bool MatchesCapability(LanguageCatalogEntry language, string capability) + => capability switch + { + "all" => language.Symbols && language.References && language.Graph, + "none" => !language.Symbols && !language.References && !language.Graph, + "symbols" => language.Symbols, + "references" => language.References, + "graph" => language.Graph, + "missing-any" => language.CapabilityGaps.Count > 0, + "missing-symbols" => !language.Symbols, + "missing-references" => !language.References, + "missing-graph" => !language.Graph, + "search-only" => !language.Symbols && !language.References && !language.Graph, + _ => false, + }; + + internal static string NormalizeLookupKey(string value) + { + var builder = new StringBuilder(value.Length); + foreach (var character in value.Trim()) + { + if (char.IsWhiteSpace(character) || character is '-' or '_' or '.') + continue; + builder.Append(char.ToLowerInvariant(character)); + } + return builder.ToString(); + } + + private static LanguageCatalogEntry CreateEntry( + string language, + IReadOnlyCollection symbolLanguages, + IReadOnlyCollection referenceLanguages) + { + var hasSymbols = symbolLanguages.Contains(language); + var hasReferences = referenceLanguages.Contains(language); + return new LanguageCatalogEntry( + [], + [], + [], + [], + [], + QueryCommandRunner.GetLanguageAliases(language).ToList(), + hasSymbols, + hasReferences, + hasReferences, + LanguageCapabilitySupport.BuildGaps(hasSymbols, hasReferences, hasReferences), + LanguageCapabilitySupport.BuildUnsupportedGuidance(language, hasSymbols, hasReferences, hasReferences)); + } +} diff --git a/src/CodeIndex/Cli/QueryCommandRunner.Languages.cs b/src/CodeIndex/Cli/QueryCommandRunner.Languages.cs index 9e2619d05..4ca781f6f 100644 --- a/src/CodeIndex/Cli/QueryCommandRunner.Languages.cs +++ b/src/CodeIndex/Cli/QueryCommandRunner.Languages.cs @@ -1,10 +1,8 @@ using System.Globalization; -using System.Text; using System.Text.Json; using System.Text.Json.Nodes; using CodeIndex.Database; using CodeIndex.Indexer; -using CodeIndex.Indexer.Extensibility; using CodeIndex.Models; namespace CodeIndex.Cli; @@ -38,7 +36,7 @@ public static int RunLanguages(string[] cmdArgs, JsonSerializerOptions jsonOptio return WithDb(options, jsonOptions, reader => { var status = reader.GetStatus(includeDatabaseSizeAttribution: false); - var catalog = BuildLanguageCatalog(reader.GetIndexedProjectRoot()); + var catalog = LanguageCatalog.Build(reader.GetIndexedProjectRoot()); var indexedLanguageCounts = loadIndexedCounts ? status.Languages : null; return WriteLanguages( SelectLanguages(catalog.Languages, indexedLanguageCounts), @@ -48,92 +46,15 @@ public static int RunLanguages(string[] cmdArgs, JsonSerializerOptions jsonOptio }); } - var defaultCatalog = BuildLanguageCatalog(workspaceRoot: null); + var defaultCatalog = LanguageCatalog.Build(workspaceRoot: null); return WriteLanguages( SelectLanguages(defaultCatalog.Languages, indexedLanguageCounts: null), defaultCatalog.Languages.Count, indexedLanguageCounts: null, defaultCatalog.Diagnostics); - (List> Languages, IReadOnlyList Diagnostics) - BuildLanguageCatalog(string? workspaceRoot) - { - ExtractorPluginRegistry.LoadPatternConfigsForProjectRoot(workspaceRoot); - var languagePatterns = FileIndexer.GetLanguagePatterns(workspaceRoot, out var diagnostics); - var symbolLangs = SymbolExtractor.GetSupportedLanguages(workspaceRoot); - var graphLangs = ReferenceExtractor.GetSupportedLanguages(workspaceRoot); - - // Build a consolidated view: language -> capability flags and gaps. - // 統合ビュー: 言語 -> capability flag と gap。 - var allLangs = new Dictionary(StringComparer.Ordinal); - foreach (var pattern in languagePatterns) - { - var lang = pattern.Language; - if (!allLangs.TryGetValue(lang, out var info)) - { - var hasSymbols = symbolLangs.Contains(lang); - var hasReferences = graphLangs.Contains(lang); - info = new LanguageSupportInfo( - [], - [], - [], - [], - [], - GetLanguageAliases(lang).ToList(), - hasSymbols, - hasReferences, - hasReferences, - LanguageCapabilitySupport.BuildGaps(hasSymbols, hasReferences, hasReferences), - LanguageCapabilitySupport.BuildUnsupportedGuidance(lang, hasSymbols, hasReferences, hasReferences)); - allLangs[lang] = info; - } - - switch (pattern.Kind) - { - case FileIndexer.LanguagePatternKind.Extension: - info.Extensions.Add(pattern.Pattern); - break; - case FileIndexer.LanguagePatternKind.ExactFilename: - info.ExactFilenames.Add(pattern.Pattern); - break; - case FileIndexer.LanguagePatternKind.FilenamePrefixPattern: - info.FilenamePrefixPatterns.Add(pattern.Pattern); - break; - } - if (!info.LegacyPatterns.Contains(pattern.Pattern, StringComparer.Ordinal)) - info.LegacyPatterns.Add(pattern.Pattern); - info.PatternProvenance.Add(new LanguagePatternProvenanceJsonResult( - pattern.Pattern, - GetLanguagePatternKindName(pattern.Kind), - pattern.Source)); - } - - foreach (var lang in FileIndexer.GetContentDetectedLanguageBuckets()) - { - if (allLangs.ContainsKey(lang)) - continue; - - var hasSymbols = symbolLangs.Contains(lang); - var hasReferences = graphLangs.Contains(lang); - allLangs[lang] = new LanguageSupportInfo( - [], - [], - [], - [], - [], - GetLanguageAliases(lang).ToList(), - hasSymbols, - hasReferences, - hasReferences, - LanguageCapabilitySupport.BuildGaps(hasSymbols, hasReferences, hasReferences), - LanguageCapabilitySupport.BuildUnsupportedGuidance(lang, hasSymbols, hasReferences, hasReferences)); - } - - return (allLangs.OrderBy(kv => kv.Key).ToList(), diagnostics); - } - - IEnumerable> SelectLanguages( - IEnumerable> languages, + IEnumerable> SelectLanguages( + IEnumerable> languages, IReadOnlyDictionary? indexedLanguageCounts) { var selected = languages; @@ -145,7 +66,7 @@ IEnumerable> SelectLanguages( } int WriteLanguages( - IEnumerable> languages, + IEnumerable> languages, int totalLanguageCount, IReadOnlyDictionary? indexedLanguageCounts, IReadOnlyList languageMapDiagnostics) @@ -196,8 +117,12 @@ int WriteLanguages( kv.Value.FilenamePrefixPatterns.OrderBy(value => value, StringComparer.Ordinal).ToList(), kv.Value.LegacyPatterns.OrderBy(value => value, StringComparer.Ordinal).ToList(), kv.Value.PatternProvenance - .OrderBy(value => value.Kind, StringComparer.Ordinal) + .OrderBy(value => GetLanguagePatternKindName(value.Kind), StringComparer.Ordinal) .ThenBy(value => value.Pattern, StringComparer.Ordinal) + .Select(value => new LanguagePatternProvenanceJsonResult( + value.Pattern, + GetLanguagePatternKindName(value.Kind), + value.Source)) .ToList(), kv.Value.Aliases.OrderBy(a => a).ToList(), kv.Value.Symbols, @@ -271,19 +196,6 @@ int WriteLanguages( } } - private sealed record LanguageSupportInfo( - List Extensions, - List ExactFilenames, - List FilenamePrefixPatterns, - List LegacyPatterns, - List PatternProvenance, - List Aliases, - bool Symbols, - bool References, - bool Graph, - List CapabilityGaps, - List UnsupportedGuidance); - private static string GetLanguagePatternKindName(FileIndexer.LanguagePatternKind kind) => kind switch { @@ -342,50 +254,13 @@ private static bool ShouldLoadLanguageIndexedCounts(QueryCommandOptions options) return indexedLanguageCounts.TryGetValue(lang, out var count) ? count : 0; } - private static bool LanguageMatchesLookup(string lang, LanguageSupportInfo language, QueryCommandOptions options) - => options.LanguageLookups.Any(lookup => string.Equals(DbReader.NormalizeQueryLanguage(lookup), lang, StringComparison.Ordinal)) - || options.LanguageExtensionLookups.Any(lookup => LanguageMatchesExtensionLookup(language, lookup)) - || options.LanguageAliasLookups.Any(lookup => LanguageMatchesAliasLookup(language, lookup)); - - private static bool LanguageMatchesExtensionLookup(LanguageSupportInfo language, string lookup) - { - var normalized = NormalizeLanguageLookupKey(lookup); - return language.Extensions.Any(ext => string.Equals(NormalizeLanguageLookupKey(ext), normalized, StringComparison.Ordinal)); - } - - private static bool LanguageMatchesAliasLookup(LanguageSupportInfo language, string lookup) - { - var normalized = NormalizeLanguageLookupKey(lookup); - return language.Aliases.Any(alias => string.Equals(NormalizeLanguageLookupKey(alias), normalized, StringComparison.Ordinal)); - } + private static bool LanguageMatchesLookup(string lang, LanguageCatalogEntry language, QueryCommandOptions options) + => options.LanguageLookups.Any(lookup => LanguageCatalog.MatchesLanguage(lang, lookup)) + || options.LanguageExtensionLookups.Any(lookup => LanguageCatalog.MatchesExtension(language, lookup)) + || options.LanguageAliasLookups.Any(lookup => LanguageCatalog.MatchesAlias(language, lookup)); - private static string NormalizeLanguageLookupKey(string value) - { - var builder = new StringBuilder(value.Length); - foreach (var ch in value.Trim()) - { - if (char.IsWhiteSpace(ch) || ch is '-' or '_' or '.') - continue; - builder.Append(char.ToLowerInvariant(ch)); - } - return builder.ToString(); - } - - private static bool LanguageMatchesCapability(LanguageSupportInfo language, string capability) - => capability switch - { - LanguageCapabilityAll => language.Symbols && language.References && language.Graph, - LanguageCapabilityNone => !language.Symbols && !language.References && !language.Graph, - LanguageCapabilitySymbols => language.Symbols, - LanguageCapabilityReferences => language.References, - LanguageCapabilityGraph => language.Graph, - LanguageCapabilityMissingAny => language.CapabilityGaps.Count > 0, - LanguageCapabilityMissingSymbols => !language.Symbols, - LanguageCapabilityMissingReferences => !language.References, - LanguageCapabilityMissingGraph => !language.Graph, - LanguageCapabilitySearchOnly => !language.Symbols && !language.References && !language.Graph, - _ => false, - }; + private static bool LanguageMatchesCapability(LanguageCatalogEntry language, string capability) + => LanguageCatalog.MatchesCapability(language, capability); private static bool TryNormalizeLanguageCapability(string value, out string capability) { @@ -404,7 +279,7 @@ LanguageCapabilityMissingSymbols or } private static JsonObject BuildLanguageSummaryPayload( - IReadOnlyList> languages, + IReadOnlyList> languages, int totalLanguageCount, IReadOnlyDictionary? indexedLanguageCounts, QueryCommandOptions options, @@ -453,13 +328,13 @@ private static JsonObject BuildLanguageSummaryPayload( return payload; } - private static JsonObject BuildLanguageCapabilityCounts(IReadOnlyList> languages) + private static JsonObject BuildLanguageCapabilityCounts(IReadOnlyList> languages) { - static bool HasAll(LanguageSupportInfo language) + static bool HasAll(LanguageCatalogEntry language) => language.Symbols && language.References && language.Graph; - static bool HasNone(LanguageSupportInfo language) + static bool HasNone(LanguageCatalogEntry language) => !language.Symbols && !language.References && !language.Graph; - static bool IsSymbolOnly(LanguageSupportInfo language) + static bool IsSymbolOnly(LanguageCatalogEntry language) => language.Symbols && !language.References && !language.Graph; return new JsonObject diff --git a/src/CodeIndex/Mcp/McpToolArgumentContracts.cs b/src/CodeIndex/Mcp/McpToolArgumentContracts.cs index 2147a523a..b5ac9e585 100644 --- a/src/CodeIndex/Mcp/McpToolArgumentContracts.cs +++ b/src/CodeIndex/Mcp/McpToolArgumentContracts.cs @@ -29,7 +29,7 @@ public partial class McpServer "batch_query" => new HashSet(StringComparer.Ordinal) { "queries", "maxResponseBytes", "estimateOnly" }, "deps" => new HashSet(StringComparer.Ordinal) { "path", "reverse", "format", "cycles", "lang", "limit", "graphBudget", "cursor", "excludePaths", "excludeTests", "includeGenerated", "project", "solution" }, "impact_analysis" => new HashSet(StringComparer.Ordinal) { "query", "lang", "maxHops", "maxDepth", "limit", "path", "excludePaths", "excludeTests", "includeGenerated", "withPaths", "countOnly", "project", "solution" }, - "languages" => new HashSet(StringComparer.Ordinal) { "indexedOnly", "capability", "extension", "alias" }, + "languages" => new HashSet(StringComparer.Ordinal) { "indexedOnly", "capability", "language", "extension", "alias", "limit", "cursor", "maxBytes" }, "validate" => new HashSet(StringComparer.Ordinal) { "kind", "severity", "limit", "cursor", "path", "excludePaths", "excludeTests", "countOnly", "format", "project", "solution" }, "unused_symbols" => new HashSet(StringComparer.Ordinal) { "kind", "lang", "limit", "visibility", "excludeVisibility", "path", "excludePaths", "excludeTests", "bucket", "minConfidence", "byBucket", "project", "solution" }, "symbol_hotspots" => new HashSet(StringComparer.Ordinal) { "kind", "lang", "limit", "visibility", "excludeVisibility", "groupBy", "path", "excludePaths", "excludeTests", "project", "solution" }, diff --git a/src/CodeIndex/Mcp/McpToolCatalog.cs b/src/CodeIndex/Mcp/McpToolCatalog.cs index 8932ef3a0..c4e6292ca 100644 --- a/src/CodeIndex/Mcp/McpToolCatalog.cs +++ b/src/CodeIndex/Mcp/McpToolCatalog.cs @@ -409,7 +409,7 @@ private static JsonArray CreateToolCatalog() ReadOnlyAnnotations()), CreateToolDefinition( "languages", - "List supported languages with extensions, aliases, capabilities, and unsupported_guidance fallback commands. Use `indexedOnly`, `capability`, `extension`, or `alias` to match CLI language filters and extension lookup. / 対応言語一覧を拡張子・別名・機能・`unsupported_guidance` の代替コマンド付きで返す。`indexedOnly` / `capability` / `extension` / `alias` で CLI の言語フィルタと拡張子 lookup に合わせて絞り込める。", + "List supported languages with exact CLI-compatible language, extension, alias, and capability filters. Results use stable catalog-generation-bound cursor pagination and a JSON-RPC envelope byte budget. / 対応言語一覧を CLI 互換の言語・拡張子・別名・機能の完全一致フィルタで返す。結果はカタログ世代に拘束された安定 cursor pagination と JSON-RPC envelope の byte budget を使用する。", new JsonObject { ["type"] = "object", @@ -417,8 +417,12 @@ private static JsonArray CreateToolCatalog() { ["indexedOnly"] = new JsonObject { ["type"] = "boolean", ["description"] = "Return only languages currently present in the index. Requires the configured database.", ["default"] = false }, ["capability"] = new JsonObject { ["oneOf"] = new JsonArray { new JsonObject { ["type"] = "string", ["enum"] = new JsonArray { "symbols", "graph", "references" } }, new JsonObject { ["type"] = "array", ["items"] = new JsonObject { ["type"] = "string", ["enum"] = new JsonArray { "symbols", "graph", "references" } } } }, ["description"] = "Filter by language capability. `graph` and `references` both require call-graph/reference extraction support. Accepts a single value or an array; all requested capabilities must match." }, + ["language"] = new JsonObject { ["type"] = "string", ["description"] = "Look up one canonical language using the same exact normalization as CLI `languages --language`, e.g. `csharp` or `cs`." }, ["extension"] = new JsonObject { ["type"] = "string", ["description"] = "Look up languages by file extension. Accepts `cs` or `.cs` style values." }, - ["alias"] = new JsonObject { ["type"] = "string", ["description"] = "Look up languages by canonical language name or CLI language alias, e.g. `cs` for `csharp`." } + ["alias"] = new JsonObject { ["type"] = "string", ["description"] = "Look up languages by exact CLI language alias; canonical language names remain accepted for backward compatibility." }, + ["limit"] = new JsonObject { ["type"] = "integer", ["minimum"] = 1, ["maximum"] = MaxLimit, ["description"] = "Maximum catalog entries to return per page.", ["default"] = QueryCommandRunner.DefaultQueryLimit }, + ["cursor"] = new JsonObject { ["type"] = "string", ["maxLength"] = MaxMcpQueryCursorCharacters, ["description"] = "Opaque next_cursor from the previous languages page. Keep every filter, limit, and maxBytes unchanged." }, + ["maxBytes"] = new JsonObject { ["type"] = "integer", ["minimum"] = MinLanguageCatalogMaxBytes, ["maximum"] = MaxLanguageCatalogMaxBytes, ["description"] = "Maximum UTF-8 bytes for the complete JSON-RPC response envelope.", ["default"] = DefaultLanguageCatalogMaxBytes } } }, ReadOnlyAnnotations()), diff --git a/src/CodeIndex/Mcp/McpToolHandlers.ArgumentValidation.cs b/src/CodeIndex/Mcp/McpToolHandlers.ArgumentValidation.cs index fbd7ae464..df32649ca 100644 --- a/src/CodeIndex/Mcp/McpToolHandlers.ArgumentValidation.cs +++ b/src/CodeIndex/Mcp/McpToolHandlers.ArgumentValidation.cs @@ -245,7 +245,7 @@ private static bool TryGetExpectedJsonType(string toolName, string argumentName, "focusLine" or "focusColumn" or "focusLength" or "startLine" or "endLine" or "maxHops" or "maxDepth" or "depth" or "parallelism" or "maxFileBytes" or "maxSymbolsPerFile" or "maxReferencesPerFile" or "debounce" or "staleAfterSeconds" or - "guardWindow" or "maxOutputBytes" or "maxResponseBytes" or "graphBudget" => "integer", + "guardWindow" or "maxOutputBytes" or "maxResponseBytes" or "maxBytes" or "graphBudget" => "integer", "check" or "excludeTests" or "includeGenerated" or "indexedOnly" or "rawQuery" or "noDedup" or "exactSubstring" or "tokenBoundary" or "exactName" or "exact" or "prefix" or "countOnly" or "includeBody" or "lsp_compatible" or "lspCompatible" or diff --git a/src/CodeIndex/Mcp/McpToolHandlers.Languages.cs b/src/CodeIndex/Mcp/McpToolHandlers.Languages.cs index 652f4acc5..aeab7932e 100644 --- a/src/CodeIndex/Mcp/McpToolHandlers.Languages.cs +++ b/src/CodeIndex/Mcp/McpToolHandlers.Languages.cs @@ -1,28 +1,40 @@ +using System.Globalization; using System.Text.Json; using System.Text.Json.Nodes; using CodeIndex.Cli; using CodeIndex.Indexer; -using CodeIndex.Indexer.Extensibility; -using CodeIndex.Models; namespace CodeIndex.Mcp; public partial class McpServer { + internal const int MinLanguageCatalogMaxBytes = MinResourceListMaxBytes; + internal const int DefaultLanguageCatalogMaxBytes = DefaultResourceListMaxBytes; + internal const int MaxLanguageCatalogMaxBytes = MaxResourceListMaxBytes; + private JsonNode ExecuteLanguages(JsonNode? id, JsonNode? args) { + var adjustments = new ArgumentAdjustmentCollector(); var indexedOnly = args?["indexedOnly"]?.GetValue() ?? false; var capabilities = ReadStringOrArrayList(args, "capability") .Select(value => value.Trim().ToLowerInvariant()) + .Distinct(StringComparer.Ordinal) + .OrderBy(value => value, StringComparer.Ordinal) .ToList(); - var extensionFilter = args?["extension"]?.GetValue()?.Trim(); - var normalizedExtension = string.IsNullOrWhiteSpace(extensionFilter) + var languageFilter = NormalizeOptionalLookup(args?["language"]?.GetValue()); + var extensionFilter = NormalizeOptionalLookup(args?["extension"]?.GetValue()); + var aliasFilter = NormalizeOptionalLookup(args?["alias"]?.GetValue()); + var normalizedLanguage = languageFilter is null + ? null + : CodeIndex.Database.DbReader.NormalizeQueryLanguage(languageFilter); + var normalizedExtension = extensionFilter is null ? null : extensionFilter.StartsWith(".", StringComparison.Ordinal) ? extensionFilter : "." + extensionFilter; - var aliasFilter = QueryCommandRunner.NormalizeLangFilterValue(args?["alias"]?.GetValue()); + var normalizedAlias = aliasFilter is null ? null : LanguageCatalog.NormalizeLookupKey(aliasFilter); + var limit = ReadLimit(args, QueryCommandRunner.DefaultQueryLimit, adjustments); if (args?["capability"] is JsonArray capabilityArray && capabilities.Count != capabilityArray.Count) - return CreateToolErrorResponse(id, "capability entries must be non-empty strings."); + return CreateToolErrorResponse(id, "capability entries must be non-empty, unique strings."); foreach (var capability in capabilities) { @@ -30,212 +42,151 @@ private JsonNode ExecuteLanguages(JsonNode? id, JsonNode? args) return CreateToolErrorResponse(id, $"Invalid language capability '{capability}'. Use one of: symbols, graph, references."); } - (Dictionary Languages, int SymbolLanguageCount, int ReferenceLanguageCount, IReadOnlyList Diagnostics) BuildCatalog(string? workspaceRoot) + var requestedMaxBytes = DefaultLanguageCatalogMaxBytes; + if (args?["maxBytes"] is JsonNode maxBytesNode + && (maxBytesNode is not JsonValue maxBytesValue + || !maxBytesValue.TryGetValue(out requestedMaxBytes) + || requestedMaxBytes < MinLanguageCatalogMaxBytes + || requestedMaxBytes > MaxLanguageCatalogMaxBytes)) { - ExtractorPluginRegistry.LoadPatternConfigsForProjectRoot(workspaceRoot); - var languagePatterns = FileIndexer.GetLanguagePatterns(workspaceRoot, out var diagnostics); - var symbolLangs = SymbolExtractor.GetSupportedLanguages(workspaceRoot); - var referenceLangs = ReferenceExtractor.GetSupportedLanguages(workspaceRoot); - var languages = new Dictionary(StringComparer.Ordinal); - foreach (var pattern in languagePatterns) - { - var lang = pattern.Language; - if (!languages.TryGetValue(lang, out var info)) - { - var hasSymbols = symbolLangs.Contains(lang); - var hasReferences = referenceLangs.Contains(lang); - info = new McpLanguageSupportInfo( - [], - [], - [], - [], - [], - QueryCommandRunner.GetLanguageAliases(lang).ToList(), - hasSymbols, - hasReferences, - hasReferences, - LanguageCapabilitySupport.BuildGaps(hasSymbols, hasReferences, hasReferences), - LanguageCapabilitySupport.BuildUnsupportedGuidance(lang, hasSymbols, hasReferences, hasReferences)); - languages[lang] = info; - } + return CreateLanguageCatalogMaxBytesError(id); + } - switch (pattern.Kind) - { - case FileIndexer.LanguagePatternKind.Extension: - info.Extensions.Add(pattern.Pattern); - break; - case FileIndexer.LanguagePatternKind.ExactFilename: - info.ExactFilenames.Add(pattern.Pattern); - break; - case FileIndexer.LanguagePatternKind.FilenamePrefixPattern: - info.FilenamePrefixPatterns.Add(pattern.Pattern); - break; - } - if (!info.LegacyPatterns.Contains(pattern.Pattern, StringComparer.Ordinal)) - info.LegacyPatterns.Add(pattern.Pattern); - info.PatternProvenance.Add(pattern); - } + var effectiveMaxBytes = Math.Min(requestedMaxBytes, GetMaxResponseBytes()); + var activeTransportMaxResponseBytes = Volatile.Read(ref _activeTransportMaxResponseBytes); + if (activeTransportMaxResponseBytes > 0) + effectiveMaxBytes = Math.Min(effectiveMaxBytes, activeTransportMaxResponseBytes); + if (_currentBatchResponseItemMaxBytes.Value is { } batchResponseItemMaxBytes) + effectiveMaxBytes = Math.Min(effectiveMaxBytes, batchResponseItemMaxBytes); - foreach (var lang in FileIndexer.GetContentDetectedLanguageBuckets()) + McpQueryCursor? cursor = null; + if (args?["cursor"] is JsonNode cursorNode) + { + if (cursorNode is not JsonValue cursorValue + || !cursorValue.TryGetValue(out var cursorText) + || !TryParseMcpQueryCursor(cursorText, out cursor)) { - if (languages.ContainsKey(lang)) - continue; - - var hasSymbols = symbolLangs.Contains(lang); - var hasReferences = referenceLangs.Contains(lang); - languages[lang] = new McpLanguageSupportInfo( - [], - [], - [], - [], - [], - QueryCommandRunner.GetLanguageAliases(lang).ToList(), - hasSymbols, - hasReferences, - hasReferences, - LanguageCapabilitySupport.BuildGaps(hasSymbols, hasReferences, hasReferences), - LanguageCapabilitySupport.BuildUnsupportedGuidance(lang, hasSymbols, hasReferences, hasReferences)); + return CreateMcpCursorError( + id, + "languages", + "cursor_malformed", + "cursor must be an opaque response:v2 next_cursor returned by languages.", + stale: false); } - - return (languages, symbolLangs.Count, referenceLangs.Count, diagnostics); } - JsonNode BuildResponse(HashSet? indexedLanguages, string? workspaceRoot) + JsonNode BuildResponse( + HashSet? indexedLanguages, + string? workspaceRoot, + string? stableAt) { - var catalog = BuildCatalog(workspaceRoot); - var sorted = catalog.Languages - .Where(kv => !indexedOnly || indexedLanguages?.Contains(kv.Key) == true) - .Where(kv => capabilities.All(capability => LanguageMatchesCapability(kv.Value.Symbols, kv.Value.References, kv.Value.Graph, capability))) - .Where(kv => normalizedExtension is null || kv.Value.Extensions.Contains(normalizedExtension, StringComparer.OrdinalIgnoreCase)) - .Where(kv => aliasFilter is null - || string.Equals(kv.Key, aliasFilter, StringComparison.OrdinalIgnoreCase) - || kv.Value.Aliases.Contains(aliasFilter, StringComparer.OrdinalIgnoreCase)) - .OrderBy(kv => kv.Key, StringComparer.Ordinal) + var catalog = LanguageCatalog.Build(workspaceRoot); + var filtered = catalog.Languages + .Where(pair => !indexedOnly || indexedLanguages?.Contains(pair.Key) == true) + .Where(pair => capabilities.All(capability => LanguageCatalog.MatchesCapability(pair.Value, capability))) + .Where(pair => normalizedLanguage is null + || string.Equals(pair.Key, normalizedLanguage, StringComparison.Ordinal)) + .Where(pair => normalizedExtension is null + || LanguageCatalog.MatchesExtension(pair.Value, normalizedExtension)) + .Where(pair => aliasFilter is null + || LanguageCatalog.MatchesAlias(pair.Value, aliasFilter) + || LanguageCatalog.MatchesLanguage(pair.Key, aliasFilter)) + .OrderBy(pair => pair.Key, StringComparer.Ordinal) .ToList(); - var languagesArray = new JsonArray(); - foreach (var (lang, info) in sorted) - { - var extArray = new JsonArray(); - foreach (var ext in info.Extensions.OrderBy(e => e, StringComparer.Ordinal)) - extArray.Add(ext); - - var exactFilenameArray = new JsonArray(); - foreach (var filename in info.ExactFilenames.OrderBy(value => value, StringComparer.Ordinal)) - exactFilenameArray.Add(filename); - - var filenamePrefixPatternArray = new JsonArray(); - foreach (var pattern in info.FilenamePrefixPatterns.OrderBy(value => value, StringComparer.Ordinal)) - filenamePrefixPatternArray.Add(pattern); + var queryFingerprint = BuildMcpQueryFingerprint( + "languages", + limit, + "catalog-v1", + [ + new("alias", normalizedAlias), + new("extension", normalizedExtension is null + ? null + : LanguageCatalog.NormalizeLookupKey(normalizedExtension)), + new("indexed-only", indexedOnly ? "true" : "false"), + new("language", normalizedLanguage), + new("max-bytes", requestedMaxBytes.ToString(CultureInfo.InvariantCulture)), + new("sort", "language-ordinal-v1"), + ], + ("capability", capabilities, PreserveOrder: false)); + var generation = ( + BuildLanguageCatalogGenerationFingerprint(catalog, indexedOnly ? indexedLanguages : null), + stableAt); - var legacyPatternArray = new JsonArray(); - foreach (var pattern in info.LegacyPatterns.OrderBy(value => value, StringComparer.Ordinal)) - legacyPatternArray.Add(pattern); - - var provenanceArray = new JsonArray(); - foreach (var pattern in info.PatternProvenance - .OrderBy(value => value.Kind) - .ThenBy(value => value.Pattern, StringComparer.Ordinal)) - { - provenanceArray.Add(new JsonObject - { - ["pattern"] = pattern.Pattern, - ["kind"] = pattern.Kind switch - { - FileIndexer.LanguagePatternKind.Extension => "extension", - FileIndexer.LanguagePatternKind.ExactFilename => "exact_filename", - FileIndexer.LanguagePatternKind.FilenamePrefixPattern => "filename_prefix_pattern", - _ => throw new ArgumentOutOfRangeException(nameof(pattern.Kind), pattern.Kind, null), - }, - ["source"] = pattern.Source, - }); - } + if (ValidateMcpQueryCursor( + id, + "languages", + cursor, + queryFingerprint, + generation.Item1, + filtered.Count) is JsonObject cursorError) + { + return cursorError; + } - var guidanceArray = new JsonArray(); - foreach (var guidance in info.UnsupportedGuidance) + var offset = cursor?.Offset ?? 0; + var availableCount = Math.Min(limit, filtered.Count - offset); + for (var returnedCount = availableCount; returnedCount >= 0; returnedCount--) + { + if (returnedCount == 0 && availableCount > 0) { - guidanceArray.Add(new JsonObject - { - ["capability"] = guidance.Capability, - ["message"] = guidance.Message, - ["recommended_commands"] = new JsonArray(guidance.RecommendedCommands.Select(command => JsonValue.Create(command)).ToArray()), - }); + return CreateLanguageCatalogEffectiveMaxBytesError( + id, + requestedMaxBytes, + effectiveMaxBytes); } - languagesArray.Add(new JsonObject - { - ["lang"] = lang, - ["extensions"] = extArray, - ["exact_filenames"] = exactFilenameArray, - ["filename_prefix_patterns"] = filenamePrefixPatternArray, - ["legacy_patterns"] = legacyPatternArray, - ["pattern_provenance"] = provenanceArray, - ["aliases"] = new JsonArray(info.Aliases.OrderBy(alias => alias, StringComparer.Ordinal).Select(alias => JsonValue.Create(alias)).ToArray()), - ["symbol_extraction"] = info.Symbols, - ["reference_extraction"] = info.References, - ["graph_queries"] = info.Graph, - ["capability_gaps"] = new JsonArray(info.CapabilityGaps.Select(gap => JsonValue.Create(gap)).ToArray()), - ["unsupported_guidance"] = guidanceArray, - }); - } + var page = filtered.Skip(offset).Take(returnedCount).ToList(); + var byteBudgetReached = returnedCount < availableCount; + var payload = BuildLanguageCatalogPayload( + page, + filtered, + catalog, + indexedOnly, + capabilities, + normalizedLanguage, + normalizedExtension, + aliasFilter, + requestedMaxBytes, + effectiveMaxBytes, + byteBudgetReached); + AddMcpPaginationEnvelope( + payload, + filtered.Count, + returnedCount, + offset, + limit, + queryFingerprint, + generation); + payload["continuation_reason"] = byteBudgetReached + ? "byte_budget" + : offset + returnedCount < filtered.Count ? "item_limit" : "complete"; + adjustments.ApplyTo(payload); - var payload = new JsonObject - { - ["languages"] = languagesArray, - ["detection_policy"] = new JsonObject - { - ["filename_case_policy"] = "filesystem", - ["filename_case_source"] = "path_case_sensitive", - ["extension_case_policy"] = "case_insensitive", - ["precedence"] = new JsonArray(QueryCommandRunner.LanguageDetectionPrecedence - .Select(value => JsonValue.Create(value)).ToArray()), - }, - ["language_map_diagnostics"] = new JsonArray(catalog.Diagnostics.Select(diagnostic => (JsonNode)new JsonObject - { - ["code"] = diagnostic.Code, - ["config"] = diagnostic.Config, - ["reason"] = diagnostic.Reason, - ["blocks_parent_fallback"] = diagnostic.BlocksParentFallback, - }).ToArray()), - ["reference_extraction_limits"] = JsonSerializer.SerializeToNode( - ReferenceExtractor.GetSafetyLimits(), - _jsonOptions), - ["filters"] = new JsonObject - { - ["indexedOnly"] = indexedOnly, - ["capability"] = new JsonArray(capabilities.Select(capability => JsonValue.Create(capability)).ToArray()), - ["extension"] = normalizedExtension, - ["alias"] = aliasFilter, - }, - }; - if (normalizedExtension is not null) - { - payload["extension_lookup"] = new JsonObject + var summary = $"{returnedCount} of {filtered.Count} matching languages returned; " + + $"{catalog.Languages.Count} languages are in the catalog."; + var response = CreateLanguageCatalogToolResult(id, summary, payload); + if (TryMeasureJsonUtf8BytesWithinLimit( + response, + _jsonOptions, + effectiveMaxBytes, + out _)) { - ["extension"] = normalizedExtension, - ["matched"] = sorted.Count, - ["languages"] = new JsonArray(sorted.Select(kv => JsonValue.Create(kv.Key)).ToArray()), - }; - } - if (aliasFilter is not null) - { - payload["alias_lookup"] = new JsonObject - { - ["alias"] = aliasFilter, - ["matched"] = sorted.Count, - ["languages"] = new JsonArray(sorted.Select(kv => JsonValue.Create(kv.Key)).ToArray()), - }; + return response; + } } - var summary = $"{sorted.Count} languages supported. {catalog.SymbolLanguageCount} with symbol extraction, {catalog.ReferenceLanguageCount} with reference extraction, {catalog.ReferenceLanguageCount} with call-graph queries."; - return CreateToolResult(id, summary, payload); + return CreateLanguageCatalogEffectiveMaxBytesError( + id, + requestedMaxBytes, + effectiveMaxBytes); } var configuredDatabaseAvailable = _dbPath.StartsWith("file:", StringComparison.OrdinalIgnoreCase) || string.Equals(_dbPath, ":memory:", StringComparison.Ordinal) || File.Exists(LongPath.EnsureWindowsPrefix(_dbPath)); if (!indexedOnly && !configuredDatabaseAvailable) - return BuildResponse(null, workspaceRoot: null); + return BuildResponse(null, workspaceRoot: null, stableAt: null); return WithDbReader(id, args, reader => { @@ -243,22 +194,275 @@ JsonNode BuildResponse(HashSet? indexedLanguages, string? workspaceRoot) var indexedLanguages = indexedOnly ? new HashSet(status.Languages.Keys, StringComparer.Ordinal) : null; - return BuildResponse(indexedLanguages, reader.GetIndexedProjectRoot()); + return BuildResponse( + indexedLanguages, + reader.GetIndexedProjectRoot(), + reader.GetPaginationGeneration().StableAt); }); } - private sealed record McpLanguageSupportInfo( - List Extensions, - List ExactFilenames, - List FilenamePrefixPatterns, - List LegacyPatterns, - List PatternProvenance, - List Aliases, - bool Symbols, - bool References, - bool Graph, - List CapabilityGaps, - List UnsupportedGuidance); + private JsonObject BuildLanguageCatalogPayload( + IReadOnlyList> page, + IReadOnlyList> filtered, + LanguageCatalogSnapshot catalog, + bool indexedOnly, + IReadOnlyList capabilities, + string? languageFilter, + string? extensionFilter, + string? aliasFilter, + int requestedMaxBytes, + int effectiveMaxBytes, + bool byteBudgetReached) + { + var payload = new JsonObject + { + ["languages"] = new JsonArray(page.Select(pair => BuildLanguageCatalogEntry(pair.Key, pair.Value)).ToArray()), + ["detection_policy"] = new JsonObject + { + ["filename_case_policy"] = "filesystem", + ["filename_case_source"] = "path_case_sensitive", + ["extension_case_policy"] = "case_insensitive", + ["precedence"] = new JsonArray(QueryCommandRunner.LanguageDetectionPrecedence + .Select(value => JsonValue.Create(value)).ToArray()), + }, + ["language_map_diagnostics"] = new JsonArray(catalog.Diagnostics.Select(diagnostic => (JsonNode)new JsonObject + { + ["code"] = diagnostic.Code, + ["config"] = diagnostic.Config, + ["reason"] = diagnostic.Reason, + ["blocks_parent_fallback"] = diagnostic.BlocksParentFallback, + }).ToArray()), + ["reference_extraction_limits"] = JsonSerializer.SerializeToNode( + ReferenceExtractor.GetSafetyLimits(), + _jsonOptions), + ["filters"] = new JsonObject + { + ["indexedOnly"] = indexedOnly, + ["capability"] = new JsonArray(capabilities.Select(capability => JsonValue.Create(capability)).ToArray()), + ["language"] = languageFilter, + ["extension"] = extensionFilter, + ["alias"] = aliasFilter, + }, + ["summary"] = new JsonObject + { + ["catalog_language_count"] = catalog.Languages.Count, + ["filtered_language_count"] = filtered.Count, + ["symbol_extraction_language_count"] = catalog.SymbolLanguageCount, + ["reference_extraction_language_count"] = catalog.ReferenceLanguageCount, + ["graph_query_language_count"] = catalog.ReferenceLanguageCount, + }, + ["response_budget"] = new JsonObject + { + ["scope"] = "json_rpc_envelope", + ["requested_max_bytes"] = requestedMaxBytes, + ["effective_max_bytes"] = effectiveMaxBytes, + ["byte_budget_reached"] = byteBudgetReached, + }, + }; + if (languageFilter is not null) + { + payload["language_lookup"] = BuildLanguageLookup( + "language", + languageFilter, + filtered); + } + if (extensionFilter is not null) + { + payload["extension_lookup"] = BuildLanguageLookup( + "extension", + extensionFilter, + filtered); + } + if (aliasFilter is not null) + { + payload["alias_lookup"] = BuildLanguageLookup( + "alias", + aliasFilter, + filtered); + } + + return payload; + } + + private static JsonObject BuildLanguageCatalogEntry(string language, LanguageCatalogEntry entry) + => new() + { + ["lang"] = language, + ["extensions"] = BuildLanguageStringArray(entry.Extensions), + ["exact_filenames"] = BuildLanguageStringArray(entry.ExactFilenames), + ["filename_prefix_patterns"] = BuildLanguageStringArray(entry.FilenamePrefixPatterns), + ["legacy_patterns"] = BuildLanguageStringArray(entry.LegacyPatterns), + ["pattern_provenance"] = new JsonArray(entry.PatternProvenance + .OrderBy(pattern => pattern.Kind) + .ThenBy(pattern => pattern.Pattern, StringComparer.Ordinal) + .Select(pattern => (JsonNode)new JsonObject + { + ["pattern"] = pattern.Pattern, + ["kind"] = pattern.Kind switch + { + FileIndexer.LanguagePatternKind.Extension => "extension", + FileIndexer.LanguagePatternKind.ExactFilename => "exact_filename", + FileIndexer.LanguagePatternKind.FilenamePrefixPattern => "filename_prefix_pattern", + _ => throw new ArgumentOutOfRangeException(nameof(pattern.Kind), pattern.Kind, null), + }, + ["source"] = pattern.Source, + }).ToArray()), + ["aliases"] = BuildLanguageStringArray(entry.Aliases), + ["symbol_extraction"] = entry.Symbols, + ["reference_extraction"] = entry.References, + ["graph_queries"] = entry.Graph, + ["capability_gaps"] = BuildLanguageStringArray(entry.CapabilityGaps), + ["unsupported_guidance"] = new JsonArray(entry.UnsupportedGuidance.Select(guidance => (JsonNode)new JsonObject + { + ["capability"] = guidance.Capability, + ["message"] = guidance.Message, + ["recommended_commands"] = new JsonArray(guidance.RecommendedCommands + .Select(command => JsonValue.Create(command)).ToArray()), + }).ToArray()), + }; + private static JsonObject BuildLanguageLookup( + string propertyName, + string value, + IReadOnlyList> matches) + => new() + { + [propertyName] = value, + ["matched"] = matches.Count, + ["languages"] = new JsonArray(matches.Select(pair => JsonValue.Create(pair.Key)).ToArray()), + }; + + private static JsonArray BuildLanguageStringArray(IEnumerable values) + => new(values + .OrderBy(value => value, StringComparer.Ordinal) + .Select(value => JsonValue.Create(value)) + .ToArray()); + + private JsonObject CreateLanguageCatalogToolResult( + JsonNode? id, + string summary, + JsonObject payload) + { + EnrichToolStructuredContent(payload); + var result = new JsonObject + { + ["content"] = new JsonArray + { + new JsonObject + { + ["type"] = "text", + ["mimeType"] = "application/json", + ["text"] = summary, + }, + }, + ["structuredContent"] = payload, + }; + return CreateSuccessResponse(hasId: true, id: id, result: result); + } + + private static string BuildLanguageCatalogGenerationFingerprint( + LanguageCatalogSnapshot catalog, + IReadOnlySet? indexedLanguages) + { + var components = new List + { + "mcp-language-catalog-generation:v1", + "symbol-language-count:" + catalog.SymbolLanguageCount.ToString(CultureInfo.InvariantCulture), + "reference-language-count:" + catalog.ReferenceLanguageCount.ToString(CultureInfo.InvariantCulture), + }; + foreach (var (language, entry) in catalog.Languages) + { + components.Add("language:" + language); + components.AddRange(entry.Extensions.OrderBy(value => value, StringComparer.Ordinal) + .Select(value => "extension:" + value)); + components.AddRange(entry.ExactFilenames.OrderBy(value => value, StringComparer.Ordinal) + .Select(value => "exact-filename:" + value)); + components.AddRange(entry.FilenamePrefixPatterns.OrderBy(value => value, StringComparer.Ordinal) + .Select(value => "filename-prefix:" + value)); + components.AddRange(entry.Aliases.OrderBy(value => value, StringComparer.Ordinal) + .Select(value => "alias:" + value)); + components.AddRange(entry.CapabilityGaps.OrderBy(value => value, StringComparer.Ordinal) + .Select(value => "capability-gap:" + value)); + components.Add("symbols:" + entry.Symbols); + components.Add("references:" + entry.References); + components.Add("graph:" + entry.Graph); + foreach (var pattern in entry.PatternProvenance + .OrderBy(value => value.Kind) + .ThenBy(value => value.Pattern, StringComparer.Ordinal)) + { + components.Add("pattern-kind:" + pattern.Kind); + components.Add("pattern:" + pattern.Pattern); + components.Add("pattern-source:" + pattern.Source); + } + foreach (var guidance in entry.UnsupportedGuidance + .OrderBy(value => value.Capability, StringComparer.Ordinal) + .ThenBy(value => value.Message, StringComparer.Ordinal)) + { + components.Add("guidance-capability:" + guidance.Capability); + components.Add("guidance-message:" + guidance.Message); + components.AddRange(guidance.RecommendedCommands + .Select(value => "guidance-command:" + value)); + } + } + + foreach (var diagnostic in catalog.Diagnostics + .OrderBy(value => value.Code, StringComparer.Ordinal) + .ThenBy(value => value.Config, StringComparer.Ordinal)) + { + components.Add("diagnostic-code:" + diagnostic.Code); + components.Add("diagnostic-config:" + diagnostic.Config); + components.Add("diagnostic-reason:" + diagnostic.Reason); + components.Add("diagnostic-blocks-parent:" + diagnostic.BlocksParentFallback); + } + + if (indexedLanguages is not null) + { + components.Add("indexed-only:true"); + components.AddRange(indexedLanguages + .OrderBy(value => value, StringComparer.Ordinal) + .Select(value => "indexed-language:" + value)); + } + + return InspectGraphCursorCodec.BuildQueryFingerprint(components); + } + + private static string? NormalizeOptionalLookup(string? value) + { + var trimmed = value?.Trim(); + return string.IsNullOrWhiteSpace(trimmed) ? null : trimmed; + } + + private JsonObject CreateLanguageCatalogMaxBytesError(JsonNode? id) + => CreateToolErrorResponse( + id, + $"languages maxBytes must be between {MinLanguageCatalogMaxBytes} and {MaxLanguageCatalogMaxBytes}.", + category: McpErrorEnvelope.CategoryInvalidArgument, + suggestion: "Choose a maxBytes value inside the advertised languages schema range.", + retrySafe: true, + extraData: new JsonObject + { + ["error_code"] = "invalid_max_bytes", + ["min_max_bytes"] = MinLanguageCatalogMaxBytes, + ["max_max_bytes"] = MaxLanguageCatalogMaxBytes, + ["default_max_bytes"] = DefaultLanguageCatalogMaxBytes, + }); + + private JsonObject CreateLanguageCatalogEffectiveMaxBytesError( + JsonNode? id, + int requestedMaxBytes, + int effectiveMaxBytes) + => CreateToolErrorResponse( + id, + "languages maxBytes is too small for the response metadata and one catalog entry.", + category: McpErrorEnvelope.CategoryInvalidArgument, + suggestion: "Increase maxBytes and restart languages pagination without cursor.", + retrySafe: true, + extraData: new JsonObject + { + ["error_code"] = "max_bytes_too_small", + ["requested_max_bytes"] = requestedMaxBytes, + ["effective_max_bytes"] = effectiveMaxBytes, + ["restart_required"] = true, + }); } diff --git a/tests/CodeIndex.Tests/McpServerIssue4896Tests.cs b/tests/CodeIndex.Tests/McpServerIssue4896Tests.cs new file mode 100644 index 000000000..e404cbd8d --- /dev/null +++ b/tests/CodeIndex.Tests/McpServerIssue4896Tests.cs @@ -0,0 +1,222 @@ +using System.Text.Json.Nodes; +using CodeIndex.Indexer.Extensibility; +using CodeIndex.Mcp; + +namespace CodeIndex.Tests; + +public partial class McpServerTests +{ + [Fact] + public void Languages_ExactFiltersReportSeparateCatalogAndMatchCounts_Issue4896() + { + var language = CallIssue4896Languages(new JsonObject + { + ["language"] = "cs", + }, id: 1); + var languageEntry = Assert.Single(language["languages"]!.AsArray())!; + Assert.Equal("csharp", languageEntry["lang"]!.GetValue()); + Assert.Equal(1, language["summary"]!["filtered_language_count"]!.GetValue()); + Assert.True(language["summary"]!["catalog_language_count"]!.GetValue() > 1); + Assert.True(language["summary"]!["symbol_extraction_language_count"]!.GetValue() > 1); + + var alias = CallIssue4896Languages(new JsonObject + { + ["alias"] = "F#", + }, id: 2); + Assert.Equal("fsharp", Assert.Single(alias["languages"]!.AsArray())!["lang"]!.GetValue()); + Assert.Equal(1, alias["alias_lookup"]!["matched"]!.GetValue()); + + var ambiguousExtension = CallIssue4896Languages(new JsonObject + { + ["extension"] = "m", + }, id: 3); + Assert.Equal( + "ambiguous_m", + Assert.Single(ambiguousExtension["languages"]!.AsArray())!["lang"]!.GetValue()); + Assert.Equal(1, ambiguousExtension["extension_lookup"]!["matched"]!.GetValue()); + + var emptyUnicodeLookup = CallIssue4896Languages(new JsonObject + { + ["language"] = "日本語", + }, id: 4); + Assert.Equal(0, emptyUnicodeLookup["total_count"]!.GetValue()); + Assert.Equal(0, emptyUnicodeLookup["returned_count"]!.GetValue()); + Assert.False(emptyUnicodeLookup["has_more"]!.GetValue()); + Assert.Null(emptyUnicodeLookup["next_cursor"]); + Assert.Equal("complete", emptyUnicodeLookup["continuation_reason"]!.GetValue()); + } + + [Fact] + public void Languages_CursorEnumeratesCatalogAndRejectsMismatchAndStaleGeneration_Issue4896() + { + var names = new List(); + string? cursor = null; + var totalCount = -1; + JsonObject? finalPage = null; + for (var pageNumber = 0; pageNumber < 100; pageNumber++) + { + var arguments = new JsonObject + { + ["limit"] = 7, + }; + if (cursor is not null) + arguments["cursor"] = cursor; + + var page = CallIssue4896Languages(arguments, id: 10 + pageNumber); + finalPage = page; + totalCount = totalCount < 0 ? page["total_count"]!.GetValue() : totalCount; + Assert.Equal(totalCount, page["total_count"]!.GetValue()); + Assert.Equal( + page["languages"]!.AsArray().Count, + page["returned_count"]!.GetValue()); + names.AddRange(page["languages"]!.AsArray() + .Select(entry => entry!["lang"]!.GetValue())); + + cursor = page["next_cursor"]?.GetValue(); + if (!page["has_more"]!.GetValue()) + break; + Assert.False(string.IsNullOrWhiteSpace(cursor)); + } + + Assert.NotNull(finalPage); + Assert.Equal(totalCount, names.Count); + Assert.Equal(names.Count, names.Distinct(StringComparer.Ordinal).Count()); + Assert.Equal(names.OrderBy(name => name, StringComparer.Ordinal), names); + Assert.False(finalPage!["has_more"]!.GetValue()); + Assert.Null(finalPage["next_cursor"]); + Assert.True(finalPage["returned_count"]!.GetValue() > 0); + Assert.Equal("complete", finalPage["continuation_reason"]!.GetValue()); + + var first = CallIssue4896Languages(new JsonObject + { + ["limit"] = 3, + }, id: 200); + var firstCursor = first["next_cursor"]!.GetValue(); + var malformed = CallIssue4853ToolError( + _server, + "languages", + new JsonObject + { + ["limit"] = 3, + ["cursor"] = "not-a-cursor", + }, + id: 201); + Assert.Equal("cursor_malformed", malformed["error_code"]!.GetValue()); + + var mismatch = CallIssue4853ToolError( + _server, + "languages", + new JsonObject + { + ["language"] = "csharp", + ["limit"] = 3, + ["cursor"] = firstCursor, + }, + id: 202); + Assert.Equal("invalid_argument", mismatch["category"]!.GetValue()); + Assert.Equal("cursor_query_mismatch", mismatch["error_code"]!.GetValue()); + + lock (TestConsoleLock.Gate) + { + try + { + ExtractorPluginRegistry.ResetForTests(); + var staleFirst = CallIssue4896Languages(new JsonObject + { + ["limit"] = 1, + }, id: 203); + var staleCursor = staleFirst["next_cursor"]!.GetValue(); + ExtractorPluginRegistry.RegisterForWorkspaceForTests( + _projectRoot, + new McpWorkspaceCatalogSymbolExtractor()); + + var stale = CallIssue4853ToolError( + _server, + "languages", + new JsonObject + { + ["limit"] = 1, + ["cursor"] = staleCursor, + }, + id: 204); + Assert.Equal("index_stale", stale["category"]!.GetValue()); + Assert.Equal("cursor_stale", stale["error_code"]!.GetValue()); + } + finally + { + ExtractorPluginRegistry.ResetForTests(); + } + } + } + + [Fact] + public void Languages_MaxBytesHonorsExactUtf8EnvelopeBoundary_Issue4896() + { + var exactBudget = McpServer.MaxLanguageCatalogMaxBytes; + JsonObject? exactResponse = null; + var exactBytes = -1; + for (var iteration = 0; iteration < 6; iteration++) + { + exactResponse = CallIssue4896LanguagesResponse(new JsonObject + { + ["limit"] = 20, + ["maxBytes"] = exactBudget, + }, id: 300); + Assert.True(_server.TrySerializeJsonNodeWithinByteLimitForTests( + exactResponse, + int.MaxValue, + out _, + out exactBytes)); + Assert.True(exactBytes <= exactBudget); + if (exactBytes == exactBudget) + break; + exactBudget = exactBytes; + } + + Assert.NotNull(exactResponse); + Assert.Equal(exactBudget, exactBytes); + var exactStructured = exactResponse!["result"]!["structuredContent"]!; + var exactReturnedCount = exactStructured["returned_count"]!.GetValue(); + Assert.True(exactReturnedCount > 1); + Assert.False(exactStructured["response_budget"]!["byte_budget_reached"]!.GetValue()); + + var oneByteLessResponse = CallIssue4896LanguagesResponse(new JsonObject + { + ["limit"] = 20, + ["maxBytes"] = exactBudget - 1, + }, id: 300); + Assert.True(_server.TrySerializeJsonNodeWithinByteLimitForTests( + oneByteLessResponse, + exactBudget - 1, + out _, + out var oneByteLessBytes)); + Assert.True(oneByteLessBytes <= exactBudget - 1); + Assert.True( + oneByteLessResponse["result"]!["structuredContent"]!["returned_count"]!.GetValue() + < exactReturnedCount); + } + + private JsonObject CallIssue4896Languages(JsonObject arguments, int id) + { + var response = CallIssue4896LanguagesResponse(arguments, id); + Assert.Null(response["error"]); + Assert.False(response["result"]?["isError"]?.GetValue() ?? false); + return response["result"]!["structuredContent"]!.AsObject(); + } + + private JsonObject CallIssue4896LanguagesResponse(JsonObject arguments, int id) + { + var request = new JsonObject + { + ["jsonrpc"] = "2.0", + ["id"] = id, + ["method"] = "tools/call", + ["params"] = new JsonObject + { + ["name"] = "languages", + ["arguments"] = arguments, + }, + }; + return _server.HandleMessage(request)!.AsObject(); + } +} diff --git a/tests/CodeIndex.Tests/McpServerToolsCallTests.cs b/tests/CodeIndex.Tests/McpServerToolsCallTests.cs index 236264dd8..073e5a7de 100644 --- a/tests/CodeIndex.Tests/McpServerToolsCallTests.cs +++ b/tests/CodeIndex.Tests/McpServerToolsCallTests.cs @@ -6354,11 +6354,11 @@ public void ToolsCall_BatchQuery_ArgsSummaryBoundsHugeScalarNumbers_Issue3816() [Fact] public void ToolsCall_Languages_ReturnsCapabilities() { - var request = JsonNode.Parse("""{"jsonrpc":"2.0","id":1,"method":"tools/call","params":{"name":"languages","arguments":{}}}""")!; + var request = JsonNode.Parse("""{"jsonrpc":"2.0","id":1,"method":"tools/call","params":{"name":"languages","arguments":{"limit":200}}}""")!; var response = _server.HandleMessage(request)!; var text = response["result"]!["content"]![0]!["text"]!.GetValue(); - Assert.Contains("languages supported", text); + Assert.Contains("matching languages returned", text); var structured = response["result"]!["structuredContent"]!; var precedence = structured["detection_policy"]!["precedence"]!.AsArray() .Select(value => value!.GetValue()) @@ -6461,7 +6461,7 @@ public void ToolsCall_Languages_ReturnsCapabilities() [Fact] public void ToolsCall_Languages_SeparatesFilenamePatternKindsAndKeepsLegacyPatterns_Issue4617() { - var request = JsonNode.Parse("""{"jsonrpc":"2.0","id":1,"method":"tools/call","params":{"name":"languages","arguments":{}}}""")!; + var request = JsonNode.Parse("""{"jsonrpc":"2.0","id":1,"method":"tools/call","params":{"name":"languages","arguments":{"limit":200}}}""")!; var response = _server.HandleMessage(request)!; var languages = response["result"]!["structuredContent"]!["languages"]!.AsArray(); @@ -6497,7 +6497,7 @@ public void ToolsCall_Languages_DefaultCatalogUsesIndexedWorkspaceSnapshot_Issue Assert.Contains("mcpcatalog", SymbolExtractor.GetSupportedLanguages(_projectRoot)); Assert.Contains("mcpcatalog", FileIndexer.GetLanguageExtensions(_projectRoot).Values); - var request = JsonNode.Parse("""{"jsonrpc":"2.0","id":1,"method":"tools/call","params":{"name":"languages","arguments":{}}}""")!; + var request = JsonNode.Parse("""{"jsonrpc":"2.0","id":1,"method":"tools/call","params":{"name":"languages","arguments":{"limit":200}}}""")!; var response = _server.HandleMessage(request)!; var language = Assert.Single( diff --git a/tests/CodeIndex.Tests/McpToolContractTests.cs b/tests/CodeIndex.Tests/McpToolContractTests.cs index e76d29ebe..02ff74948 100644 --- a/tests/CodeIndex.Tests/McpToolContractTests.cs +++ b/tests/CodeIndex.Tests/McpToolContractTests.cs @@ -130,6 +130,33 @@ public void ToolsList_HighVolumeDiscoveryCursorsHaveSharedArgumentContract_Issue StringComparison.Ordinal); } + [Fact] + public void ToolsList_LanguagesAdvertisesBoundedCatalogContract_Issue4896() + { + var properties = GetAdvertisedToolSchemas()["languages"]; + var allowed = GetAllowedToolArguments("languages"); + + foreach (var argumentName in new[] { "language", "extension", "alias", "limit", "cursor", "maxBytes" }) + { + Assert.True(properties.ContainsKey(argumentName)); + Assert.Contains(argumentName, allowed); + } + + Assert.Equal((true, "string"), TryGetExpectedJsonType("languages", "language")); + Assert.Equal((true, "integer"), TryGetExpectedJsonType("languages", "limit")); + Assert.Equal((true, "string"), TryGetExpectedJsonType("languages", "cursor")); + Assert.Equal((true, "integer"), TryGetExpectedJsonType("languages", "maxBytes")); + Assert.Equal( + McpServer.MaxMcpQueryCursorCharacters, + properties["cursor"]["maxLength"]!.GetValue()); + Assert.Equal( + McpServer.MinLanguageCatalogMaxBytes, + properties["maxBytes"]["minimum"]!.GetValue()); + Assert.Equal( + McpServer.MaxLanguageCatalogMaxBytes, + properties["maxBytes"]["maximum"]!.GetValue()); + } + [Fact] public void ToolsList_DepsArgumentsHaveSharedArgumentContract_Issue3196() { From e7aa9d89b5282dfe27b2db7317e404969bad9aa6 Mon Sep 17 00:00:00 2001 From: Widthdom Date: Fri, 31 Jul 2026 00:27:44 +0900 Subject: [PATCH 2/3] Preserve duplicate MCP capability filters (#4896) --- src/CodeIndex/Mcp/McpToolHandlers.Languages.cs | 3 --- tests/CodeIndex.Tests/McpServerIssue4896Tests.cs | 16 +++++++++++++--- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/src/CodeIndex/Mcp/McpToolHandlers.Languages.cs b/src/CodeIndex/Mcp/McpToolHandlers.Languages.cs index aeab7932e..d28cb317c 100644 --- a/src/CodeIndex/Mcp/McpToolHandlers.Languages.cs +++ b/src/CodeIndex/Mcp/McpToolHandlers.Languages.cs @@ -33,9 +33,6 @@ private JsonNode ExecuteLanguages(JsonNode? id, JsonNode? args) var normalizedAlias = aliasFilter is null ? null : LanguageCatalog.NormalizeLookupKey(aliasFilter); var limit = ReadLimit(args, QueryCommandRunner.DefaultQueryLimit, adjustments); - if (args?["capability"] is JsonArray capabilityArray && capabilities.Count != capabilityArray.Count) - return CreateToolErrorResponse(id, "capability entries must be non-empty, unique strings."); - foreach (var capability in capabilities) { if (!IsKnownLanguageCapability(capability)) diff --git a/tests/CodeIndex.Tests/McpServerIssue4896Tests.cs b/tests/CodeIndex.Tests/McpServerIssue4896Tests.cs index e404cbd8d..ddd0bdafd 100644 --- a/tests/CodeIndex.Tests/McpServerIssue4896Tests.cs +++ b/tests/CodeIndex.Tests/McpServerIssue4896Tests.cs @@ -19,17 +19,27 @@ public void Languages_ExactFiltersReportSeparateCatalogAndMatchCounts_Issue4896( Assert.True(language["summary"]!["catalog_language_count"]!.GetValue() > 1); Assert.True(language["summary"]!["symbol_extraction_language_count"]!.GetValue() > 1); + var duplicateCapability = CallIssue4896Languages(new JsonObject + { + ["language"] = "csharp", + ["capability"] = new JsonArray("graph", "graph"), + }, id: 2); + Assert.Equal( + "csharp", + Assert.Single(duplicateCapability["languages"]!.AsArray())!["lang"]!.GetValue()); + Assert.Single(duplicateCapability["filters"]!["capability"]!.AsArray()); + var alias = CallIssue4896Languages(new JsonObject { ["alias"] = "F#", - }, id: 2); + }, id: 3); Assert.Equal("fsharp", Assert.Single(alias["languages"]!.AsArray())!["lang"]!.GetValue()); Assert.Equal(1, alias["alias_lookup"]!["matched"]!.GetValue()); var ambiguousExtension = CallIssue4896Languages(new JsonObject { ["extension"] = "m", - }, id: 3); + }, id: 4); Assert.Equal( "ambiguous_m", Assert.Single(ambiguousExtension["languages"]!.AsArray())!["lang"]!.GetValue()); @@ -38,7 +48,7 @@ public void Languages_ExactFiltersReportSeparateCatalogAndMatchCounts_Issue4896( var emptyUnicodeLookup = CallIssue4896Languages(new JsonObject { ["language"] = "日本語", - }, id: 4); + }, id: 5); Assert.Equal(0, emptyUnicodeLookup["total_count"]!.GetValue()); Assert.Equal(0, emptyUnicodeLookup["returned_count"]!.GetValue()); Assert.False(emptyUnicodeLookup["has_more"]!.GetValue()); From 1f35740f7d035c215f6989d713c2c29a71f6a716 Mon Sep 17 00:00:00 2001 From: Widthdom Date: Fri, 31 Jul 2026 00:44:51 +0900 Subject: [PATCH 3/3] Align MCP language capabilities with CLI (#4896) --- src/CodeIndex/Cli/LanguageCatalog.cs | 17 +++++++++++++++++ src/CodeIndex/Mcp/McpToolCatalog.cs | 10 +++++++++- src/CodeIndex/Mcp/McpToolHandlers.Languages.cs | 6 +++++- src/CodeIndex/Mcp/McpToolHandlers.cs | 2 +- .../CodeIndex.Tests/McpServerIssue4896Tests.cs | 18 ++++++++++++++++++ tests/CodeIndex.Tests/McpToolContractTests.cs | 8 ++++++++ 6 files changed, 58 insertions(+), 3 deletions(-) diff --git a/src/CodeIndex/Cli/LanguageCatalog.cs b/src/CodeIndex/Cli/LanguageCatalog.cs index 59ca810ae..4c64533ad 100644 --- a/src/CodeIndex/Cli/LanguageCatalog.cs +++ b/src/CodeIndex/Cli/LanguageCatalog.cs @@ -27,6 +27,20 @@ internal sealed record LanguageCatalogSnapshot( internal static class LanguageCatalog { + internal static IReadOnlyList SupportedCapabilities { get; } = Array.AsReadOnly( + [ + "all", + "none", + "graph", + "references", + "symbols", + "missing-any", + "missing-graph", + "missing-references", + "missing-symbols", + "search-only", + ]); + internal static LanguageCatalogSnapshot Build(string? workspaceRoot) { ExtractorPluginRegistry.LoadPatternConfigsForProjectRoot(workspaceRoot); @@ -111,6 +125,9 @@ internal static bool MatchesCapability(LanguageCatalogEntry language, string cap _ => false, }; + internal static bool IsKnownCapability(string capability) + => SupportedCapabilities.Contains(capability, StringComparer.Ordinal); + internal static string NormalizeLookupKey(string value) { var builder = new StringBuilder(value.Length); diff --git a/src/CodeIndex/Mcp/McpToolCatalog.cs b/src/CodeIndex/Mcp/McpToolCatalog.cs index c4e6292ca..984f0f8b4 100644 --- a/src/CodeIndex/Mcp/McpToolCatalog.cs +++ b/src/CodeIndex/Mcp/McpToolCatalog.cs @@ -7,6 +7,14 @@ namespace CodeIndex.Mcp; public partial class McpServer { + private static JsonArray CreateLanguageCapabilityEnum() + { + var values = new JsonArray(); + foreach (var capability in LanguageCatalog.SupportedCapabilities) + values.Add(capability); + return values; + } + private static JsonArray CreateToolCatalog() { var tools = new JsonArray @@ -416,7 +424,7 @@ private static JsonArray CreateToolCatalog() ["properties"] = new JsonObject { ["indexedOnly"] = new JsonObject { ["type"] = "boolean", ["description"] = "Return only languages currently present in the index. Requires the configured database.", ["default"] = false }, - ["capability"] = new JsonObject { ["oneOf"] = new JsonArray { new JsonObject { ["type"] = "string", ["enum"] = new JsonArray { "symbols", "graph", "references" } }, new JsonObject { ["type"] = "array", ["items"] = new JsonObject { ["type"] = "string", ["enum"] = new JsonArray { "symbols", "graph", "references" } } } }, ["description"] = "Filter by language capability. `graph` and `references` both require call-graph/reference extraction support. Accepts a single value or an array; all requested capabilities must match." }, + ["capability"] = new JsonObject { ["oneOf"] = new JsonArray { new JsonObject { ["type"] = "string", ["enum"] = CreateLanguageCapabilityEnum() }, new JsonObject { ["type"] = "array", ["items"] = new JsonObject { ["type"] = "string", ["enum"] = CreateLanguageCapabilityEnum() } } }, ["description"] = "Filter by the same capability or capability-gap values as CLI `languages --capability`. Accepts a single value or an array; all requested capabilities must match." }, ["language"] = new JsonObject { ["type"] = "string", ["description"] = "Look up one canonical language using the same exact normalization as CLI `languages --language`, e.g. `csharp` or `cs`." }, ["extension"] = new JsonObject { ["type"] = "string", ["description"] = "Look up languages by file extension. Accepts `cs` or `.cs` style values." }, ["alias"] = new JsonObject { ["type"] = "string", ["description"] = "Look up languages by exact CLI language alias; canonical language names remain accepted for backward compatibility." }, diff --git a/src/CodeIndex/Mcp/McpToolHandlers.Languages.cs b/src/CodeIndex/Mcp/McpToolHandlers.Languages.cs index d28cb317c..899693c8e 100644 --- a/src/CodeIndex/Mcp/McpToolHandlers.Languages.cs +++ b/src/CodeIndex/Mcp/McpToolHandlers.Languages.cs @@ -36,7 +36,11 @@ private JsonNode ExecuteLanguages(JsonNode? id, JsonNode? args) foreach (var capability in capabilities) { if (!IsKnownLanguageCapability(capability)) - return CreateToolErrorResponse(id, $"Invalid language capability '{capability}'. Use one of: symbols, graph, references."); + { + return CreateToolErrorResponse( + id, + $"Invalid language capability '{capability}'. Use one of: {string.Join(", ", LanguageCatalog.SupportedCapabilities)}."); + } } var requestedMaxBytes = DefaultLanguageCatalogMaxBytes; diff --git a/src/CodeIndex/Mcp/McpToolHandlers.cs b/src/CodeIndex/Mcp/McpToolHandlers.cs index b58ad7997..1c2717ade 100644 --- a/src/CodeIndex/Mcp/McpToolHandlers.cs +++ b/src/CodeIndex/Mcp/McpToolHandlers.cs @@ -1346,7 +1346,7 @@ private JsonNode ExecutePing(JsonNode? id) } private static bool IsKnownLanguageCapability(string capability) => - capability is "symbols" or "graph" or "references"; + LanguageCatalog.IsKnownCapability(capability); private static bool LanguageMatchesCapability(bool symbols, bool references, bool graph, string capability) => capability switch diff --git a/tests/CodeIndex.Tests/McpServerIssue4896Tests.cs b/tests/CodeIndex.Tests/McpServerIssue4896Tests.cs index ddd0bdafd..33334b719 100644 --- a/tests/CodeIndex.Tests/McpServerIssue4896Tests.cs +++ b/tests/CodeIndex.Tests/McpServerIssue4896Tests.cs @@ -1,4 +1,5 @@ using System.Text.Json.Nodes; +using CodeIndex.Cli; using CodeIndex.Indexer.Extensibility; using CodeIndex.Mcp; @@ -56,6 +57,23 @@ public void Languages_ExactFiltersReportSeparateCatalogAndMatchCounts_Issue4896( Assert.Equal("complete", emptyUnicodeLookup["continuation_reason"]!.GetValue()); } + [Fact] + public void Languages_AcceptsEveryCliCapabilityFilter_Issue4896() + { + foreach (var capability in LanguageCatalog.SupportedCapabilities) + { + var response = CallIssue4896Languages(new JsonObject + { + ["capability"] = capability, + }, id: 6); + + Assert.Equal( + [capability], + response["filters"]!["capability"]!.AsArray() + .Select(value => value!.GetValue())); + } + } + [Fact] public void Languages_CursorEnumeratesCatalogAndRejectsMismatchAndStaleGeneration_Issue4896() { diff --git a/tests/CodeIndex.Tests/McpToolContractTests.cs b/tests/CodeIndex.Tests/McpToolContractTests.cs index 02ff74948..d22023b7a 100644 --- a/tests/CodeIndex.Tests/McpToolContractTests.cs +++ b/tests/CodeIndex.Tests/McpToolContractTests.cs @@ -1,5 +1,6 @@ using System.Reflection; using System.Text.Json.Nodes; +using CodeIndex.Cli; using CodeIndex.Mcp; namespace CodeIndex.Tests; @@ -155,6 +156,13 @@ public void ToolsList_LanguagesAdvertisesBoundedCatalogContract_Issue4896() Assert.Equal( McpServer.MaxLanguageCatalogMaxBytes, properties["maxBytes"]["maximum"]!.GetValue()); + var capabilitySchemas = properties["capability"]["oneOf"]!.AsArray(); + var scalarCapabilities = capabilitySchemas[0]!["enum"]!.AsArray() + .Select(value => value!.GetValue()); + var arrayCapabilities = capabilitySchemas[1]!["items"]!["enum"]!.AsArray() + .Select(value => value!.GetValue()); + Assert.Equal(LanguageCatalog.SupportedCapabilities, scalarCapabilities); + Assert.Equal(LanguageCatalog.SupportedCapabilities, arrayCapabilities); } [Fact]