diff --git a/README.md b/README.md index 3cc9e9a4b..8fc2940d7 100644 --- a/README.md +++ b/README.md @@ -78,6 +78,7 @@ cdidx references UserService --fields path,line,reference_kind --limit 20 --max- cdidx inspect QueryCommandRunner --outline-only cdidx outline src/CodeIndex/Cli/QueryCommandRunner.cs --json --kind function --limit 50 --max-json-bytes 16384 cdidx unused --compact --by-bucket --max-json-bytes 16384 +cdidx unused --json --by-bucket --max-json-bytes 16384 cdidx map --compact --max-json-bytes 65536 cdidx map --format issue-drafts --limit 10 cdidx search --recipe risky-code --format compact --max-json-bytes 65536 @@ -93,6 +94,16 @@ cdidx license --json cdidx validate ``` +For full `unused --json --by-bucket` output, `symbols` is the canonical row +collection and each `by_bucket` entry is a lightweight compatibility projection +with a zero-based `symbol_index` into that collection plus common locator +fields. `by_bucket_format` is `canonical_symbol_index_v1`; compact output uses +`summary_v1`. `--max-json-bytes` bounds the complete UTF-8 response and, when +needed, returns a canonical-row prefix with `truncated`, `omitted_count`, and +`next_cursor` so following pages neither duplicate nor skip symbols. +The MCP `unused_symbols` equivalent uses `symbols_by_bucket` and +`symbols_by_bucket_format` for the same indexed projection. + For C# explicit-interface implementations, symbol output and outlines keep the short display name (`Run`, `Value`, `Changed`, or `Item`). Qualified exact-name queries use the interface spelling, such as `IFoo.Run`, `IFoo.Value`, @@ -575,6 +586,7 @@ cdidx references UserService --fields path,line,reference_kind --limit 20 --max- cdidx inspect QueryCommandRunner --outline-only cdidx outline src/CodeIndex/Cli/QueryCommandRunner.cs --json --kind function --limit 50 --max-json-bytes 16384 cdidx unused --compact --by-bucket --max-json-bytes 16384 +cdidx unused --json --by-bucket --max-json-bytes 16384 cdidx map --compact --max-json-bytes 65536 cdidx map --format issue-drafts --limit 10 cdidx search --recipe risky-code --format compact --max-json-bytes 65536 @@ -590,6 +602,15 @@ cdidx license --json cdidx validate ``` +完全形式の `unused --json --by-bucket` 出力では、`symbols` が正規の行集合で、 +各 `by_bucket` 要素はその集合への 0 始まりの `symbol_index` と共通の位置情報を持つ +軽量な互換投影です。`by_bucket_format` は `canonical_symbol_index_v1`、compact +出力では `summary_v1` です。`--max-json-bytes` は UTF-8 応答全体を上限内に収め、 +必要な場合は正規行の prefix と `truncated`、`omitted_count`、`next_cursor` を返すため、 +後続ページで symbol の重複や欠落が発生しません。 +MCP の `unused_symbols` では、同じ index 付き投影を `symbols_by_bucket` と +`symbols_by_bucket_format` で公開します。 + C# の明示的 interface 実装では、symbol 出力と outline は短い表示名 （`Run`、`Value`、`Changed`、`Item`）を維持します。修飾した exact-name query には `IFoo.Run`、`IFoo.Value`、`IFoo.Changed`、`IFoo.Item` のような interface 表記を使い、 diff --git a/TESTING_GUIDE.md b/TESTING_GUIDE.md index 9fddebc2e..5758d5db5 100644 --- a/TESTING_GUIDE.md +++ b/TESTING_GUIDE.md @@ -28,6 +28,7 @@ Use the full suite by default. Use targeted filters only while iterating locally - Test parallelism: enabled by default across independent test classes. Tests that touch process-global state such as SQLite pool resets, environment variables, or current-directory overrides must use an explicit non-parallel collection. Console-sensitive classes share one non-parallel xUnit collection, so they remain serial with each other and do not run beside independent classes that may write request-id or global diagnostics to the process console. Use `ConsoleCapture` for ordinary capture and keep every direct `Console.Out` / `Console.Error` swap under `TestConsoleLock.Gate`. Snapshot and assert global console writers under the same gate so another test cannot replace a writer between capture completion and the assertion. That gate aliases the production `ConsoleStreamOwnership` gate so console synchronization and scoped production redirects cannot retain a test writer after its capture ends. - MCP structured-output schema coverage is split between the catalog assertions in `McpServerToolsListTests` and actual-result validation in `McpServerOutputSchemaTests`. It requires every full catalog entry to advertise an `outputSchema`, validates actual success and typed-error results for all 24 tools plus search-recipe, empty-excerpt, and truncated variants, and rejects incomplete, versionless-error, excessive-depth, and every cross-tool success pair with the deliberately small local evaluator. Keep the per-tool `tool` discriminator, required fields, bounds, and runtime field names/nesting in sync. Extend that evaluator only for JSON Schema keywords emitted by `McpToolOutputSchemas`; compact catalog entries intentionally omit complete schemas. - Markdown unused-audit coverage indexes one real Markdown fixture containing common backtick and tilde fence-language markers. Keep default suppression, `documentation_surface` totals, reason tags, and `--all` recovery in that shared fixture. +- Unused by-bucket JSON coverage keeps `symbols` as the only full-row collection and checks every lightweight bucket membership against its canonical `symbol_index`. Preserve empty, single-row, large multi-bucket-set, CLI full/compact, MCP, and bounded UTF-8 cursor fixtures; byte-limited pages must stay within the requested budget and reassemble without duplicate or skipped canonical rows. - FTS optimization recommendation coverage keeps the shared evaluator exact at one write below, at, and one write above the 25-write threshold. Status, explain, optimize dry-run, optimize execution, and vacuum maintenance guidance must expose the same `recommended`, `action`, `reason`, `threshold_writes`, `observed_writes`, and `state`; stale batches, known WAL-stale snapshots, forward-incompatible schema stamps, and unavailable legacy counters/page snapshots suppress the recommendation, query-only status performs no source writes, and execution uses the focused counter/page/forward-contract/freshness snapshot instead of full status scans. A hot-WAL fixture opened through an explicit `immutable=1` URI must prove that status, standalone optimize dry-run, and the `index --optimize` dry-run alias preserve the same stale recommendation. A WAL or freelist state of `unknown` cannot select the optimize command, and a successful optimize reports the reset counter afterward. - Full-scan CLI and MCP no-op coverage treats one repository-wide reusable-stat snapshot read and one folded-readiness verification as performance contracts. Keep assertions for one snapshot read, one stat lookup per candidate, one folded verification, and no content load for unchanged files when changing incremental indexing. - Reference-graph refresh coverage treats graph-neutral indexing as a performance contract across incremental full scan, scoped `--files` update, and MCP indexing. Keep zero-refresh assertions for new and modified source files without symbols/references, plus a single batched refresh assertion when existing or new graph identity rows change. A healthy incremental generation must restrict identity/candidate/recursion work to transaction-committed dirty files, old and new `(language, folded name)` dependencies, and their old/new reciprocal edges; retain C#/Python language-transition and unchanged-target parity with a subsequent full refresh, rolled-back file batches, cancellation/retry, orphan-candidate cleanup, and the controlled 4,100-of-4,100 broad-scope fallback. Fresh/rebuild runs, missing identity contracts, and dirty sets of at least 4,096 references covering at least 50% of the graph must keep the full-refresh path. Query-plan coverage must keep all four scoped update phases and all ten candidate inserts on dirty-table-driven reference primary-key seeks, keep C# instantiate grouping on lookup names plus `idx_symbols_name_folded`, and prove that a sub-4,096 dirty set does not count the whole reference table without an explicit diagnostic hook. @@ -976,6 +977,7 @@ dotnet test --filter "FullyQualifiedName~GitHelperTests" - テスト並列実行: 独立したテストクラス間ではデフォルトで有効です。SQLite pool の解放、環境変数の変更、カレントディレクトリの上書きのような process-global 状態を触るテストは、明示的な non-parallel collection に入れてください。console-sensitive class は同じ non-parallel な xUnit collection を共有するため、互いに直列実行され、request-id や global diagnostics を process console へ書く可能性がある独立 class とも並列実行されません。通常の capture には `ConsoleCapture` を使い、`Console.Out` / `Console.Error` を直接差し替える場合は `TestConsoleLock.Gate` で保護してください。global console writer の snapshot 取得と assertion も同じ gate 内で行い、capture 完了から assertion までの間に別のテストが writer を差し替えないようにします。この gate は本番の `ConsoleStreamOwnership` gate と同一なので、console 同期処理や scoped redirect が capture 終了後も test writer を保持することを防ぎます。 - MCP structured-output schema の coverage は `McpServerToolsListTests` の catalog assertion と `McpServerOutputSchemaTests` の actual-result validation に分けます。full catalog の全 entry が `outputSchema` を公開することを要求し、全24 tool の実際の success / typed-error result に加えて search recipe、empty excerpt、truncated variant を検証し、不完全な payload、version のない error、過剰な nesting、全 tool 間の success 組み合わせを意図的に小さく保った local evaluator で拒否します。tool ごとの `tool` discriminator、required field、bound、runtime の field 名 / nesting を同期してください。この evaluator は `McpToolOutputSchemas` が出力する JSON Schema keyword に必要な場合だけ拡張し、compact catalog entry は完全な schema を意図的に省略します。 - Markdown の unused audit coverage は、一般的な backtick / tilde fence の language marker を含む実 Markdown fixture を1回 index します。同じ fixture で既定抑制、`documentation_surface` totals、reason tag、`--all` による復元を維持してください。 +- unused の bucket 別 JSON coverage では、完全な行集合を `symbols` だけに保ち、各 lightweight bucket membership を正規の `symbol_index` と照合します。空集合、1行、大規模な複数 bucket 集合、CLI の full / compact、MCP、UTF-8 byte 上限付き cursor の fixture を維持してください。byte 制限された各 page は要求 budget 内に収まり、連結時に正規行の重複や欠落が発生してはいけません。 - FTS optimization recommendation coverage は、25 write threshold の1つ下、ちょうど、1つ上で shared evaluator の境界を固定します。status、explain、optimize dry-run、optimize execution、vacuum maintenance guidance は同じ `recommended`、`action`、`reason`、`threshold_writes`、`observed_writes`、`state` を公開し、stale batch、既知の WAL-stale snapshot、forward-incompatible な schema stamp、利用できない legacy counter / page snapshot は recommendation を抑止します。query-only status は source に書き込まず、execution は full status scan ではなく counter / page / forward-contract / freshness に限定した snapshot を使います。hot WAL fixture を明示的な `immutable=1` URI で開き、status、standalone optimize dry-run、`index --optimize` dry-run alias が同じ stale recommendation を保持することも証明します。WAL または freelist の state が `unknown` の場合は optimize command を選択せず、成功した optimize は reset 後の counter を返す必要があります。 - full-scan CLI と MCP の no-op coverage は、リポジトリ全体の reusable-stat snapshot read と folded-readiness verification がそれぞれ 1 回であることを performance contract とします。incremental indexing を変更するときは、snapshot read が 1 回、候補ごとの stat lookup が 1 回、folded verification が 1 回、unchanged file の content load が 0 回という assertion を維持してください。 - reference-graph refresh coverage は、incremental full scan、scoped `--files` update、MCP indexing を横断する graph-neutral indexing を performance contract とします。symbol/reference を持たない新規・変更 source file では refresh 0 回を維持し、既存または新規の graph identity 行が変化する場合は batch 全体で refresh 1 回を assertion してください。健全な incremental generation では identity / candidate / recursion 処理を transaction commit 済みの dirty file、旧・新の `(language, folded name)` 依存、旧・新の逆辺に限定します。C# / Python の言語遷移、未変更targetを参照する新規callerと後続full refreshのparity、rollback file batch、cancel後retry、孤立candidate cleanup、4,100件中4,100件をdirtyにする制御broad-scope fallbackを維持してください。fresh/rebuild、identity契約欠落、または4,096件以上かつgraphの50%以上を占めるdirty集合ではfull-refresh経路を維持します。query-plan coverageでは、scoped updateの4 phaseとcandidate INSERT 10本をdirty table起点のreference主キーseekに保ち、C# instantiate groupingをlookup nameと`idx_symbols_name_folded`起点にし、明示的なdiagnostic hookがない4,096件未満のdirty集合ではreference table全件COUNTを行わないことを検証してください。 diff --git a/changelog.d/unreleased/4904.fixed.md b/changelog.d/unreleased/4904.fixed.md new file mode 100644 index 000000000..a1ebc8cc1 --- /dev/null +++ b/changelog.d/unreleased/4904.fixed.md @@ -0,0 +1,24 @@ +--- +category: fixed +issues: + - 4904 +affected: + - README.md + - TESTING_GUIDE.md + - src/CodeIndex/Cli/CliFlagSchema.cs + - src/CodeIndex/Cli/ConsoleUi.cs + - src/CodeIndex/Cli/QueryCommandRunner.Unused.cs + - src/CodeIndex/Mcp/McpToolHandlers.SymbolAnalysis.cs + - src/CodeIndex/Mcp/McpToolHandlers.cs + - src/CodeIndex/Mcp/McpToolOutputSchemas.cs + - tests/CodeIndex.Tests/McpServerToolsCallTests.cs + - tests/CodeIndex.Tests/QueryCommandRunnerSymbolTests.cs +--- + +## English + +- **By-bucket unused JSON now serializes each full symbol row once (#4904)** — CLI and MCP responses keep full rows in the canonical `symbols` collection and expose bucket membership through lightweight indexed projections; CLI output also supports exact UTF-8 `--max-json-bytes` paging without duplicate or skipped canonical rows. + +## 日本語 + +- **bucket 別の unused JSON が各 symbol の完全な行を一度だけシリアライズするようになりました (#4904)** — CLI と MCP の応答では完全な行を正規の `symbols` 集合に保持し、bucket membership は軽量な index 付き投影で公開します。また CLI 出力は、正規行の重複や欠落を発生させずに UTF-8 byte 数を厳密に制限する `--max-json-bytes` paging に対応しました。 diff --git a/src/CodeIndex/Cli/CodeIndexExceptionFormatter.cs b/src/CodeIndex/Cli/CodeIndexExceptionFormatter.cs index 588521426..d535ffe74 100644 --- a/src/CodeIndex/Cli/CodeIndexExceptionFormatter.cs +++ b/src/CodeIndex/Cli/CodeIndexExceptionFormatter.cs @@ -13,7 +13,11 @@ namespace CodeIndex.Cli; /// internal static class CodeIndexExceptionFormatter { - public static void Write(CodeIndexException ex, string[] args, JsonSerializerOptions jsonOptions) + public static void Write( + CodeIndexException ex, + string[] args, + JsonSerializerOptions jsonOptions, + int? maxJsonBytes = null) { var message = CommandErrorWriter.FormatSanitizedExceptionMessage(ex); if (HasJsonFlag(args)) @@ -25,10 +29,15 @@ public static void Write(CodeIndexException ex, string[] args, JsonSerializerOpt ErrorCode: ex.Code, Path: ex.Path, Category: ex.Category); - Console.WriteLine(JsonSerializer.Serialize( + var json = JsonSerializer.Serialize( payload, - CliJsonSerializerContextFactory.Create(jsonOptions).CommandErrorJsonResult)); - return; + CliJsonSerializerContextFactory.Create(jsonOptions).CommandErrorJsonResult); + if (!maxJsonBytes.HasValue + || JsonEnvelopeWrapper.JsonFitsResponseBudget(json, maxJsonBytes.Value)) + { + Console.WriteLine(json); + return; + } } // Keep human output close to the existing `Error [Exxx]: ...` shape that diff --git a/src/CodeIndex/Cli/QueryCommandRunner.Database.cs b/src/CodeIndex/Cli/QueryCommandRunner.Database.cs index f74a438b2..db80d9ea1 100644 --- a/src/CodeIndex/Cli/QueryCommandRunner.Database.cs +++ b/src/CodeIndex/Cli/QueryCommandRunner.Database.cs @@ -79,7 +79,12 @@ private static int WithDb( { db = new DbContext(DbOpenIntent.QueryOnly, dbPath, cancellationToken); if (!db.TryValidateIsCodeIndexDb(out var validationReason)) - return WriteInvalidCodeIndexDbError(dbPath, validationReason, options.Json, jsonOptions); + return WriteInvalidCodeIndexDbError( + dbPath, + validationReason, + options.Json, + jsonOptions, + options.MaxJsonBytes); reader = new DbReader(db); databaseReadyForQueries = true; } @@ -158,7 +163,8 @@ private static int WithDb( CodeIndexExceptionFormatter.Write( codeIndexException, options.Json ? ["--json"] : [], - jsonOptions); + jsonOptions, + options.MaxJsonBytes); Database.DbDebug.DumpToStderr(ex); return ProgramRunner.MapCodeIndexExceptionExitCode(codeIndexException.Code); } @@ -188,7 +194,12 @@ private static int WithDb( } var jsonDatabaseOpenFailure = options.Json && !databaseReadyForQueries; - var databaseExitCode = WriteDatabaseOpenFailureJsonAware(ex, dbPath, jsonDatabaseOpenFailure, jsonOptions); + var databaseExitCode = WriteDatabaseOpenFailureJsonAware( + ex, + dbPath, + jsonDatabaseOpenFailure, + jsonOptions, + options.MaxJsonBytes); Database.DbDebug.DumpToStderr(ex); return databaseExitCode; } @@ -205,14 +216,16 @@ private static int WriteInvalidCodeIndexDbError( string dbPath, string? validationReason, bool json, - JsonSerializerOptions jsonOptions) + JsonSerializerOptions jsonOptions, + int? maxJsonBytes = null) { return WriteDatabaseCommandError( json, jsonOptions, $"{FormatDbDiagnosticValue(dbPath)} does not appear to be a valid CodeIndex database ({validationReason}).", "rebuild with `cdidx index --db ` to create a fresh database.", - "database"); + "database", + maxJsonBytes); } private static ProjectFilterRootResolution ResolveProjectRootForDbPath(string dbPath, bool dbPathExplicit) @@ -241,7 +254,8 @@ private static int WriteDatabaseOpenFailureJsonAware( Exception ex, string dbPath, bool json, - JsonSerializerOptions jsonOptions) + JsonSerializerOptions jsonOptions, + int? maxJsonBytes = null) { GlobalToolLog.Error($"database_open_failed db={FormatLogValue(dbPath)} exception={FormatLogValue(ex.ToString())}"); @@ -253,7 +267,8 @@ private static int WriteDatabaseOpenFailureJsonAware( jsonOptions, $"database access denied: {CommandErrorWriter.FormatSanitizedExceptionMessage(unauthorized)}", MacProfileDetector.BuildDatabaseHint(MacProfileDetector.DetectCurrent()), - DiagnosticRedactor.ClassifyException(unauthorized)); + DiagnosticRedactor.ClassifyException(unauthorized), + maxJsonBytes); } var io = FindException(ex); @@ -264,7 +279,8 @@ private static int WriteDatabaseOpenFailureJsonAware( jsonOptions, $"database I/O error: {CommandErrorWriter.FormatSanitizedExceptionMessage(io)}", MacProfileDetector.BuildDatabaseHint(MacProfileDetector.DetectCurrent()), - DiagnosticRedactor.ClassifyException(io)); + DiagnosticRedactor.ClassifyException(io), + maxJsonBytes); } var sqlite = FindException(ex); @@ -277,7 +293,8 @@ private static int WriteDatabaseOpenFailureJsonAware( jsonOptions, $"database access/open denied: {CommandErrorWriter.FormatSanitizedExceptionMessage(sqlite)}", MacProfileDetector.BuildDatabaseHint(MacProfileDetector.DetectCurrent()), - DiagnosticRedactor.ClassifyException(sqlite)); + DiagnosticRedactor.ClassifyException(sqlite), + maxJsonBytes); } if (sqlite.SqliteErrorCode == 11) @@ -287,7 +304,8 @@ private static int WriteDatabaseOpenFailureJsonAware( jsonOptions, $"SQLite reported database corruption: {CommandErrorWriter.FormatSanitizedExceptionMessage(sqlite)}", "rebuild the index with `cdidx index --rebuild`, or delete the broken `.cdidx/codeindex.db*` files and run `cdidx index ` again.", - DiagnosticRedactor.ClassifyException(sqlite)); + DiagnosticRedactor.ClassifyException(sqlite), + maxJsonBytes); } return WriteDatabaseCommandError( @@ -297,7 +315,8 @@ private static int WriteDatabaseOpenFailureJsonAware( MacProfileDetector.IsPermissionStyleSqliteError(sqlite) ? MacProfileDetector.BuildDatabaseHint(MacProfileDetector.DetectCurrent()) : "check `--db`, verify the index was written by a compatible cdidx version, or rebuild it with `cdidx index --rebuild`.", - DiagnosticRedactor.ClassifyException(sqlite)); + DiagnosticRedactor.ClassifyException(sqlite), + maxJsonBytes); } return WriteDatabaseCommandError( @@ -305,7 +324,8 @@ private static int WriteDatabaseOpenFailureJsonAware( jsonOptions, $"database error: {CommandErrorWriter.FormatSanitizedExceptionMessage(ex)}", "check `--db`, or rebuild the index with `cdidx index ` if the DB may be stale or corrupted.", - DiagnosticRedactor.ClassifyException(ex)); + DiagnosticRedactor.ClassifyException(ex), + maxJsonBytes); } private static int WriteDatabaseCommandError( @@ -313,15 +333,44 @@ private static int WriteDatabaseCommandError( JsonSerializerOptions jsonOptions, string message, string hint, - string? category) - => CommandErrorWriter.WriteJsonOrHuman( - json, + string? category, + int? maxJsonBytes = null) + { + var normalizedHint = TrimHintPrefix(hint); + if (!json || !maxJsonBytes.HasValue) + { + return CommandErrorWriter.WriteJsonOrHuman( + json, + jsonOptions, + message, + CommandExitCodes.DatabaseError, + normalizedHint, + errorCode: CommandErrorCodes.DbError, + category: category); + } + + var payload = CommandErrorWriter.BuildJsonPayload( jsonOptions, message, CommandExitCodes.DatabaseError, - TrimHintPrefix(hint), + normalizedHint, errorCode: CommandErrorCodes.DbError, category: category); + var payloadJson = payload.ToJsonString(jsonOptions); + if (JsonEnvelopeWrapper.JsonFitsResponseBudget(payloadJson, maxJsonBytes.Value)) + { + CommandErrorWriter.WriteStdout(payloadJson); + } + else + { + CommandErrorWriter.Write( + message, + normalizedHint, + errorCode: CommandErrorCodes.DbError); + } + + return CommandExitCodes.DatabaseError; + } private static string TrimHintPrefix(string hint) { diff --git a/src/CodeIndex/Cli/QueryCommandRunner.Unused.cs b/src/CodeIndex/Cli/QueryCommandRunner.Unused.cs index ca2e6c621..3976c3538 100644 --- a/src/CodeIndex/Cli/QueryCommandRunner.Unused.cs +++ b/src/CodeIndex/Cli/QueryCommandRunner.Unused.cs @@ -75,6 +75,16 @@ public static int RunUnused(string[] cmdArgs, JsonSerializerOptions jsonOptions) "Use `cdidx unused --json --summary-only` for compact unused totals."); return CommandExitCodes.UsageError; } + if (options.MaxJsonBytes.HasValue && !options.Json) + { + WriteUsageError( + "--max-json-bytes is only supported with unused JSON output.", + GetUsageLineOrThrow("unused"), + "Add --json or --compact, or remove --max-json-bytes."); + return CommandExitCodes.UsageError; + } + if (TryWriteCappedJsonDiagnosticsUsageError("unused", options)) + return CommandExitCodes.UsageError; var unusedScope = BuildUnusedAuditScopeFilters(options); return WithDb(options, jsonOptions, reader => @@ -196,6 +206,12 @@ UnusedCountResult CountUnusedSymbolsDetailedForCurrentQuery(Func nextOffset + var hasMoreAfterPage = pageableResults.Count > pageOffset + results.Count; + var nextOffset = pageOffset + results.Count; + var nextCursor = hasMoreAfterPage && IsUnusedCursorOffsetWithinFetchCap(options.Limit, nextOffset) ? FormatUnusedCursor(nextOffset, cursorContext) : null; @@ -280,8 +307,10 @@ UnusedCountResult CountUnusedSymbolsDetailedForCurrentQuery(Func(), + if (!TryWriteUnusedJsonPage( + results, + pageOffset, + hasMoreAfterPage, graphSupported, graphSupportReason, sqlGraphSignal, @@ -290,9 +319,12 @@ UnusedCountResult CountUnusedSymbolsDetailedForCurrentQuery(Func 0) { @@ -317,7 +349,24 @@ UnusedCountResult CountUnusedSymbolsDetailedForCurrentQuery(Func "Unknown unused-symbol bucket.", }; - private static string BuildUnusedJsonPayload(IEnumerable results, bool? graphSupported, string? graphSupportReason, SqlGraphContractSignal sqlGraphSignal, HdlGraphContractSignal hdlGraphSignal, bool hasReferencesTable, JsonSerializerOptions jsonOptions, QueryCommandOptions? queryOptions = null, UnusedAuditScopeFilters? unusedScope = null, bool byBucket = false, string? nextCursor = null, PaginationCursorContext? cursorContext = null, UnusedDefaultSuppressionResult? suppression = null) + private static bool TryWriteUnusedJsonPage( + List requestedResults, + int pageOffset, + bool hasMoreAfterPage, + bool? graphSupported, + string? graphSupportReason, + SqlGraphContractSignal sqlGraphSignal, + HdlGraphContractSignal hdlGraphSignal, + bool hasReferencesTable, + JsonSerializerOptions jsonOptions, + QueryCommandOptions options, + UnusedAuditScopeFilters unusedScope, + bool byBucket, + PaginationCursorContext cursorContext, + UnusedDefaultSuppressionResult suppression) + { + (string Json, bool HasUsableContinuation) BuildCandidate(int emittedCount, bool byteLimitReached) + { + var emittedResults = emittedCount == requestedResults.Count + ? requestedResults + : requestedResults.Take(emittedCount).ToList(); + var emittedSqlGraphSignal = emittedResults.Count == 0 + ? sqlGraphSignal + : NarrowSqlGraphContractSignalByLanguages( + sqlGraphSignal, + emittedResults.Select(result => result.Lang), + options.Lang); + var nextOffset = pageOffset + emittedCount; + var hasMore = emittedCount < requestedResults.Count || hasMoreAfterPage; + var nextCursor = hasMore + && emittedCount > 0 + && IsUnusedCursorOffsetWithinFetchCap(options.Limit, nextOffset) + ? FormatUnusedCursor(nextOffset, cursorContext) + : null; + return ( + BuildUnusedJsonPayload( + emittedResults, + graphSupported, + graphSupportReason, + emittedSqlGraphSignal, + hdlGraphSignal, + hasReferencesTable, + jsonOptions, + options, + unusedScope, + byBucket, + nextCursor, + cursorContext, + suppression, + options.MaxJsonBytes, + byteLimitReached, + requestedResults.Count - emittedCount), + CanEmitUnusedByteTruncatedPage( + hasMore, + options.Limit, + pageOffset, + emittedCount)); + } + + var fullCandidate = BuildCandidate(requestedResults.Count, byteLimitReached: false); + if (!options.MaxJsonBytes.HasValue + || JsonEnvelopeWrapper.JsonFitsResponseBudget(fullCandidate.Json, options.MaxJsonBytes.Value)) + { + Console.WriteLine(fullCandidate.Json); + return true; + } + + if (requestedResults.Count == 0) + { + WriteUsageError( + $"--max-json-bytes {options.MaxJsonBytes.Value.ToString(CultureInfo.InvariantCulture)} is too small for the empty unused response metadata.", + GetUsageLineOrThrow("unused"), + "Increase --max-json-bytes or remove it and rerun the same query."); + return false; + } + + string? bestJson = null; + var continuationBlocked = false; + var low = 1; + var high = requestedResults.Count - 1; + while (low <= high) + { + var count = low + ((high - low) / 2); + var candidate = BuildCandidate(count, byteLimitReached: true); + var fitsBudget = JsonEnvelopeWrapper.JsonFitsResponseBudget( + candidate.Json, + options.MaxJsonBytes.Value); + if (fitsBudget && candidate.HasUsableContinuation) + { + bestJson = candidate.Json; + low = count + 1; + } + else + { + continuationBlocked |= fitsBudget; + high = count - 1; + } + } + + if (bestJson != null) + { + Console.WriteLine(bestJson); + return true; + } + + if (continuationBlocked) + { + WriteUsageError( + $"--max-json-bytes {options.MaxJsonBytes.Value.ToString(CultureInfo.InvariantCulture)} cannot emit a byte-truncated unused page with a continuation cursor inside the pagination safety window.", + GetUsageLineOrThrow("unused"), + "Reduce --limit and retry the same cursor, or narrow the unused query before restarting pagination."); + return false; + } + + WriteUsageError( + $"--max-json-bytes {options.MaxJsonBytes.Value.ToString(CultureInfo.InvariantCulture)} is too small for the unused response metadata and one canonical symbol row.", + GetUsageLineOrThrow("unused"), + "Increase --max-json-bytes or narrow the unused query before restarting pagination."); + return false; + } + + private static bool CanEmitUnusedByteTruncatedPage( + bool hasMore, + int pageLimit, + int pageOffset, + int emittedCount) + => !hasMore + || emittedCount > 0 + && IsUnusedCursorOffsetWithinFetchCap(pageLimit, pageOffset + emittedCount); + + internal static bool CanEmitUnusedByteTruncatedPageForTests( + int pageLimit, + int pageOffset, + int emittedCount) + => CanEmitUnusedByteTruncatedPage( + hasMore: true, + pageLimit, + pageOffset, + emittedCount); + + private static string BuildUnusedJsonPayload( + IEnumerable results, + bool? graphSupported, + string? graphSupportReason, + SqlGraphContractSignal sqlGraphSignal, + HdlGraphContractSignal hdlGraphSignal, + bool hasReferencesTable, + JsonSerializerOptions jsonOptions, + QueryCommandOptions? queryOptions = null, + UnusedAuditScopeFilters? unusedScope = null, + bool byBucket = false, + string? nextCursor = null, + PaginationCursorContext? cursorContext = null, + UnusedDefaultSuppressionResult? suppression = null, + int? outputByteLimit = null, + bool byteLimitReached = false, + int omittedCount = 0) { var resultList = results as List ?? results.ToList(); var payload = new JsonObject @@ -761,6 +966,12 @@ private static string BuildUnusedJsonPayload(IEnumerable res ["summary"] = BuildUnusedSummaryJson(resultList, jsonOptions, suppression), ["bucket_taxonomy"] = BuildUnusedBucketTaxonomyJson(), }; + if (outputByteLimit.HasValue) + { + payload["output_byte_limit"] = outputByteLimit.Value; + payload["truncated"] = byteLimitReached; + payload["omitted_count"] = omittedCount; + } if (suppression is { Applied: true }) payload["default_suppression"] = BuildUnusedDefaultSuppressionJson(suppression, jsonOptions); if (nextCursor != null) @@ -775,6 +986,7 @@ private static string BuildUnusedJsonPayload(IEnumerable res if (byBucket) { payload["by_bucket"] = BuildUnusedBucketSummariesJson(resultList); + payload["by_bucket_format"] = "summary_v1"; omittedSections.Add("by_bucket.symbols"); } payload["omitted_sections"] = omittedSections; @@ -783,7 +995,10 @@ private static string BuildUnusedJsonPayload(IEnumerable res { payload["symbols"] = JsonSerializer.SerializeToNode(resultList, CliJsonSerializerContextFactory.Create(jsonOptions).ListUnusedSymbolResult); if (byBucket) - payload["by_bucket"] = BuildUnusedResultsByBucketJson(resultList, jsonOptions); + { + payload["by_bucket"] = BuildUnusedResultsByBucketJson(resultList); + payload["by_bucket_format"] = "canonical_symbol_index_v1"; + } } if (!hasReferencesTable) @@ -873,22 +1088,38 @@ private static JsonObject BuildUnusedQueryContextJson(QueryCommandOptions option return query; } - private static JsonObject BuildUnusedResultsByBucketJson(IEnumerable results, JsonSerializerOptions jsonOptions) + internal static JsonObject BuildUnusedResultsByBucketJson(IEnumerable results) { - var grouped = results - .GroupBy(result => result.UnusedBucket, StringComparer.Ordinal) - .ToDictionary(group => group.Key, group => group.ToList(), StringComparer.Ordinal); + var resultList = results as IReadOnlyList ?? results.ToList(); var byBucket = new JsonObject(); foreach (var bucket in OrderedUnusedBuckets) + byBucket[bucket] = new JsonArray(); + + for (var index = 0; index < resultList.Count; index++) { - if (grouped.TryGetValue(bucket, out var bucketResults)) - byBucket[bucket] = JsonSerializer.SerializeToNode(bucketResults, CliJsonSerializerContextFactory.Create(jsonOptions).ListUnusedSymbolResult); - else - byBucket[bucket] = new JsonArray(); + var result = resultList[index]; + if (!byBucket.TryGetPropertyValue(result.UnusedBucket, out var bucketNode) + || bucketNode is not JsonArray bucketRows) + { + bucketRows = new JsonArray(); + byBucket[result.UnusedBucket] = bucketRows; + } + bucketRows.Add(BuildUnusedBucketMembershipJson(result, index)); } + return byBucket; } + internal static JsonObject BuildUnusedBucketMembershipJson(UnusedSymbolResult result, int symbolIndex) + => new() + { + ["symbol_index"] = symbolIndex, + ["name"] = result.Name, + ["kind"] = result.Kind, + ["path"] = result.Path, + ["line"] = result.Line, + }; + private static string GetUnusedBucketHeading(string bucket) => bucket switch { "likely_unused_private" => "Likely unused private", diff --git a/src/CodeIndex/Mcp/McpToolHandlers.SymbolAnalysis.cs b/src/CodeIndex/Mcp/McpToolHandlers.SymbolAnalysis.cs index 67091f5b7..daa40e195 100644 --- a/src/CodeIndex/Mcp/McpToolHandlers.SymbolAnalysis.cs +++ b/src/CodeIndex/Mcp/McpToolHandlers.SymbolAnalysis.cs @@ -72,7 +72,10 @@ private JsonNode ExecuteUnusedSymbols(JsonNode? id, JsonNode? args) AddVisibilityFilterEcho(payload, visibilityFilters, excludeVisibilityFilters); payload["byBucket"] = byBucket; if (byBucket) + { payload["symbols_by_bucket"] = BuildUnusedSymbolsByBucket(results); + payload["symbols_by_bucket_format"] = "canonical_symbol_index_v1"; + } AddSqlGraphContractSignal(payload, sqlGraphSignal); AddHdlGraphContractSignal(payload, hdlGraphSignal); var summary = results.Count > 0 diff --git a/src/CodeIndex/Mcp/McpToolHandlers.cs b/src/CodeIndex/Mcp/McpToolHandlers.cs index dcf75f0e0..fb734bc7f 100644 --- a/src/CodeIndex/Mcp/McpToolHandlers.cs +++ b/src/CodeIndex/Mcp/McpToolHandlers.cs @@ -691,22 +691,25 @@ private static JsonArray BuildCompactCalleeRows(IEnumerable result return rows; } - private JsonObject BuildUnusedSymbolsByBucket(IEnumerable results) + private static JsonObject BuildUnusedSymbolsByBucket(IEnumerable results) { - var grouped = results - .GroupBy(result => result.UnusedBucket, StringComparer.Ordinal) - .ToDictionary(group => group.Key, group => group.ToList(), StringComparer.Ordinal); + var resultList = results as IReadOnlyList ?? results.ToList(); var buckets = new JsonObject(); foreach (var bucket in QueryCommandRunner.OrderedUnusedBuckets) + buckets[bucket] = new JsonArray(); + + for (var index = 0; index < resultList.Count; index++) { - if (grouped.TryGetValue(bucket, out var rows)) - buckets[bucket] = ToJsonArray(rows); - } - foreach (var (bucket, rows) in grouped.OrderBy(kv => kv.Key, StringComparer.Ordinal)) - { - if (!buckets.ContainsKey(bucket)) - buckets[bucket] = ToJsonArray(rows); + var result = resultList[index]; + if (!buckets.TryGetPropertyValue(result.UnusedBucket, out var bucketNode) + || bucketNode is not JsonArray bucketRows) + { + bucketRows = new JsonArray(); + buckets[result.UnusedBucket] = bucketRows; + } + bucketRows.Add(QueryCommandRunner.BuildUnusedBucketMembershipJson(result, index)); } + return buckets; } diff --git a/src/CodeIndex/Mcp/McpToolOutputSchemas.cs b/src/CodeIndex/Mcp/McpToolOutputSchemas.cs index 55d26cf1c..20309755c 100644 --- a/src/CodeIndex/Mcp/McpToolOutputSchemas.cs +++ b/src/CodeIndex/Mcp/McpToolOutputSchemas.cs @@ -488,6 +488,7 @@ private static JsonObject UnusedSymbolsProperties() ["summary"] = ObjectSchema(), ["symbols"] = Reference("rows"), ["symbols_by_bucket"] = ObjectSchema(), + ["symbols_by_bucket_format"] = StringSchema(), ["returned_bucket_counts"] = ObjectSchema(), ["returned_contract_domain_counts"] = ObjectSchema(), ["bucket_taxonomy"] = ObjectSchema(), diff --git a/tests/CodeIndex.Tests/McpServerToolsCallTests.cs b/tests/CodeIndex.Tests/McpServerToolsCallTests.cs index 249575af3..fceb98ba0 100644 --- a/tests/CodeIndex.Tests/McpServerToolsCallTests.cs +++ b/tests/CodeIndex.Tests/McpServerToolsCallTests.cs @@ -376,6 +376,16 @@ public void ToolsCall_SymbolGraphAndAnalyzeExposeCliQueryOptions_Issue3542() [Fact] public void ToolsCall_UnusedAndHotspotsExposeVisibilityAndBucketOptions_Issue3542() { + InsertIndexedFile( + "src/unused-bucket.cs", + "csharp", + """ + public sealed class UnusedBucketFixture + { + public void UnusedBucketMember() { } + } + """); + var hotspotsRequest = JsonNode.Parse( """{"jsonrpc":"2.0","id":1,"method":"tools/call","params":{"name":"symbol_hotspots","arguments":{"visibility":"public","excludeVisibility":["private"]}}}""")!; var hotspotsResponse = _server.HandleMessage(hotspotsRequest)!; @@ -389,6 +399,26 @@ public void ToolsCall_UnusedAndHotspotsExposeVisibilityAndBucketOptions_Issue354 var unusedStructured = unusedResponse["result"]!["structuredContent"]!; Assert.True(unusedStructured["byBucket"]!.GetValue()); Assert.NotNull(unusedStructured["symbols_by_bucket"]); + Assert.Equal( + "canonical_symbol_index_v1", + unusedStructured["symbols_by_bucket_format"]!.GetValue()); + var symbols = unusedStructured["symbols"]!.AsArray(); + var memberships = unusedStructured["symbols_by_bucket"]! + .AsObject() + .SelectMany(bucket => bucket.Value!.AsArray()) + .Select(row => row!.AsObject()) + .ToArray(); + Assert.Equal(symbols.Count, memberships.Length); + Assert.Equal( + Enumerable.Range(0, symbols.Count), + memberships.Select(row => row["symbol_index"]!.GetValue()).Order()); + Assert.All(memberships, membership => + { + var symbol = symbols[membership["symbol_index"]!.GetValue()]!; + Assert.Equal(symbol["name"]!.GetValue(), membership["name"]!.GetValue()); + Assert.Null(membership["unusedReason"]); + Assert.Equal(5, membership.Count); + }); Assert.Equal("public", Assert.Single(unusedStructured["visibility"]!.AsArray())!.GetValue()); } diff --git a/tests/CodeIndex.Tests/QueryCommandRunnerSymbolTests.cs b/tests/CodeIndex.Tests/QueryCommandRunnerSymbolTests.cs index 0aa100d67..f0bbf4499 100644 --- a/tests/CodeIndex.Tests/QueryCommandRunnerSymbolTests.cs +++ b/tests/CodeIndex.Tests/QueryCommandRunnerSymbolTests.cs @@ -1,3 +1,4 @@ +using System.Text; using System.Text.Json; using CodeIndex.Cli; using CodeIndex.Database; @@ -2192,13 +2193,39 @@ public void RunUnused_JsonAndCompactByBucketShareTaxonomyFixture_Issue4067() using var document = ParseJsonOutput(stdout); var json = document.RootElement; var byBucket = json.GetProperty("by_bucket"); + var symbols = json.GetProperty("symbols").EnumerateArray().ToArray(); Assert.Equal(CommandExitCodes.Success, exitCode); Assert.Equal(string.Empty, stderr); + Assert.Equal("canonical_symbol_index_v1", json.GetProperty("by_bucket_format").GetString()); Assert.Equal("Hidden", byBucket.GetProperty("likely_unused_private")[0].GetProperty("name").GetString()); Assert.Equal("InternalOnly", byBucket.GetProperty("maybe_unused_nonpublic")[0].GetProperty("name").GetString()); Assert.Equal(3, byBucket.GetProperty("public_or_exported_no_refs").GetArrayLength()); Assert.Equal("ConnectionString", byBucket.GetProperty("reflection_or_config_suspect")[0].GetProperty("name").GetString()); + var membershipRows = byBucket.EnumerateObject() + .SelectMany(bucket => bucket.Value.EnumerateArray()) + .ToArray(); + Assert.Equal(symbols.Length, membershipRows.Length); + Assert.Equal( + Enumerable.Range(0, symbols.Length), + membershipRows.Select(row => row.GetProperty("symbol_index").GetInt32()).Order()); + Assert.All(membershipRows, membership => + { + var symbol = symbols[membership.GetProperty("symbol_index").GetInt32()]; + Assert.Equal(symbol.GetProperty("name").GetString(), membership.GetProperty("name").GetString()); + Assert.Equal(symbol.GetProperty("path").GetString(), membership.GetProperty("path").GetString()); + Assert.False(membership.TryGetProperty("unused_reason", out _)); + Assert.Equal(5, membership.EnumerateObject().Count()); + }); + + var (canonicalExitCode, canonicalStdout, canonicalStderr) = CaptureConsole(() => QueryCommandRunner.RunUnused( + ["--db", dbPath, "--json", "--all", "--lang", "csharp"], + _jsonOptions)); + Assert.Equal(CommandExitCodes.Success, canonicalExitCode); + Assert.Equal(string.Empty, canonicalStderr); + Assert.True( + Encoding.UTF8.GetByteCount(stdout) < Encoding.UTF8.GetByteCount(canonicalStdout) * 3 / 2, + "The by-bucket compatibility projection should stay materially smaller than a second full symbol collection."); var (compactExitCode, compactStdout, compactStderr) = CaptureConsole(() => QueryCommandRunner.RunUnused( ["--db", dbPath, "--compact", "--by-bucket", "--lang", "csharp"], @@ -2217,6 +2244,7 @@ public void RunUnused_JsonAndCompactByBucketShareTaxonomyFixture_Issue4067() Assert.True(compactJson.GetProperty("compact").GetBoolean()); Assert.False(compactJson.TryGetProperty("symbols", out _)); + Assert.Equal("summary_v1", compactJson.GetProperty("by_bucket_format").GetString()); Assert.Equal(JsonValueKind.Object, likelyUnused.ValueKind); Assert.Equal(1, likelyUnused.GetProperty("count").GetInt32()); Assert.Equal("Hidden", likelyUnused.GetProperty("representative").GetProperty("name").GetString()); @@ -2229,6 +2257,341 @@ public void RunUnused_JsonAndCompactByBucketShareTaxonomyFixture_Issue4067() } } + [Fact] + public void BuildUnusedResultsByBucketJson_UsesCompleteCanonicalMembershipIndexes_Issue4904() + { + var empty = QueryCommandRunner.BuildUnusedResultsByBucketJson([]); + Assert.All( + QueryCommandRunner.OrderedUnusedBuckets, + bucket => Assert.Empty(empty[bucket]!.AsArray())); + + var results = Enumerable.Range(0, 500) + .Select(index => new UnusedSymbolResult + { + Name = $"Symbol{index}", + Kind = "function", + Path = $"src/File{index}.cs", + Line = index + 1, + UnusedBucket = QueryCommandRunner.OrderedUnusedBuckets[index % QueryCommandRunner.OrderedUnusedBuckets.Length], + UnusedConfidence = "low", + UnusedReason = "fixture", + }) + .ToList(); + + var one = QueryCommandRunner.BuildUnusedResultsByBucketJson(results.Take(1)); + var oneMembership = Assert.Single(one[QueryCommandRunner.OrderedUnusedBuckets[0]]!.AsArray())!.AsObject(); + Assert.Equal(0, oneMembership["symbol_index"]!.GetValue()); + Assert.Equal("Symbol0", oneMembership["name"]!.GetValue()); + + var byBucket = QueryCommandRunner.BuildUnusedResultsByBucketJson(results); + var memberships = byBucket + .SelectMany(bucket => bucket.Value!.AsArray()) + .Select(row => row!.AsObject()) + .ToArray(); + + Assert.Equal(results.Count, memberships.Length); + Assert.Equal( + Enumerable.Range(0, results.Count), + memberships.Select(row => row["symbol_index"]!.GetValue()).Order()); + Assert.All(memberships, row => Assert.Equal(5, row.Count)); + Assert.Equal("Symbol0", memberships.Single(row => row["symbol_index"]!.GetValue() == 0)["name"]!.GetValue()); + } + + [Fact] + public void RunUnused_MaxJsonBytesPagesCanonicalRowsWithoutGaps_Issue4904() + { + var (projectRoot, dbPath) = CreateUnusedFixtureDb(); + try + { + var (unboundedExitCode, unboundedStdout, unboundedStderr) = CaptureConsole(() => QueryCommandRunner.RunUnused( + ["--db", dbPath, "--json", "--all", "--lang", "csharp", "--by-bucket", "--limit", "100"], + _jsonOptions)); + using var unboundedDocument = ParseJsonOutput(unboundedStdout); + var expectedNames = unboundedDocument.RootElement + .GetProperty("symbols") + .EnumerateArray() + .Select(symbol => symbol.GetProperty("name").GetString()) + .ToArray(); + Assert.Equal(CommandExitCodes.Success, unboundedExitCode); + Assert.Equal(string.Empty, unboundedStderr); + Assert.True(expectedNames.Length > 1); + + var byteBudget = Encoding.UTF8.GetByteCount(unboundedStdout) - 1; + + var returnedNames = new List(); + string? cursor = null; + var sawTruncatedPage = false; + do + { + var args = new List + { + "--db", dbPath, + "--json", + "--all", + "--lang", "csharp", + "--by-bucket", + "--limit", "100", + "--max-json-bytes", byteBudget.ToString(), + }; + if (cursor != null) + { + args.Add("--cursor"); + args.Add(cursor); + } + + var (exitCode, stdout, stderr) = CaptureConsole(() => QueryCommandRunner.RunUnused( + args.ToArray(), + _jsonOptions)); + using var document = ParseJsonOutput(stdout); + var json = document.RootElement; + var pageNames = json.GetProperty("symbols") + .EnumerateArray() + .Select(symbol => symbol.GetProperty("name").GetString()) + .ToArray(); + + Assert.Equal(CommandExitCodes.Success, exitCode); + Assert.Equal(string.Empty, stderr); + Assert.NotEmpty(pageNames); + Assert.True(Encoding.UTF8.GetByteCount(stdout) <= byteBudget); + Assert.Equal(byteBudget, json.GetProperty("output_byte_limit").GetInt32()); + Assert.Equal(pageNames.Length, json.GetProperty("count").GetInt32()); + sawTruncatedPage |= json.GetProperty("truncated").GetBoolean(); + returnedNames.AddRange(pageNames); + cursor = json.TryGetProperty("next_cursor", out var cursorElement) + ? cursorElement.GetString() + : null; + } + while (cursor != null); + + Assert.True(sawTruncatedPage); + Assert.Equal(expectedNames, returnedNames); + + var (unboundedCompactExitCode, unboundedCompactStdout, unboundedCompactStderr) = CaptureConsole(() => QueryCommandRunner.RunUnused( + ["--db", dbPath, "--compact", "--all", "--lang", "csharp", "--by-bucket"], + _jsonOptions)); + Assert.Equal(CommandExitCodes.Success, unboundedCompactExitCode); + Assert.Equal(string.Empty, unboundedCompactStderr); + var compactByteBudget = Encoding.UTF8.GetByteCount(unboundedCompactStdout) + 128; + var (compactExitCode, compactStdout, compactStderr) = CaptureConsole(() => QueryCommandRunner.RunUnused( + ["--db", dbPath, "--compact", "--all", "--lang", "csharp", "--by-bucket", "--max-json-bytes", compactByteBudget.ToString()], + _jsonOptions)); + using var compactDocument = ParseJsonOutput(compactStdout); + Assert.Equal(CommandExitCodes.Success, compactExitCode); + Assert.Equal(string.Empty, compactStderr); + Assert.True(Encoding.UTF8.GetByteCount(compactStdout) <= compactByteBudget); + Assert.False(compactDocument.RootElement.TryGetProperty("symbols", out _)); + + var (countExitCode, countStdout, countStderr) = CaptureConsole(() => QueryCommandRunner.RunUnused( + ["--db", dbPath, "--json", "--count", "--all", "--max-json-bytes", "65536"], + _jsonOptions)); + using var countDocument = ParseJsonOutput(countStdout); + Assert.Equal(CommandExitCodes.Success, countExitCode); + Assert.Equal(string.Empty, countStderr); + Assert.Equal(65536, countDocument.RootElement.GetProperty("output_byte_limit").GetInt32()); + Assert.False(countDocument.RootElement.GetProperty("truncated").GetBoolean()); + Assert.Equal(0, countDocument.RootElement.GetProperty("omitted_count").GetInt32()); + + var (emptyExitCode, emptyStdout, emptyStderr) = CaptureConsole(() => QueryCommandRunner.RunUnused( + ["--db", dbPath, "--json", "--lang", "rust", "--max-json-bytes", "65536"], + _jsonOptions)); + using var emptyDocument = ParseJsonOutput(emptyStdout); + Assert.Equal(CommandExitCodes.Success, emptyExitCode); + Assert.Equal(string.Empty, emptyStderr); + Assert.Equal(0, emptyDocument.RootElement.GetProperty("count").GetInt32()); + Assert.Empty(emptyDocument.RootElement.GetProperty("symbols").EnumerateArray()); + + var (smallExitCode, smallStdout, smallStderr) = CaptureConsole(() => QueryCommandRunner.RunUnused( + ["--db", dbPath, "--json", "--all", "--max-json-bytes", "1"], + _jsonOptions)); + Assert.Equal(CommandExitCodes.UsageError, smallExitCode); + Assert.Equal(string.Empty, smallStdout); + Assert.Contains("one canonical symbol row", smallStderr); + } + finally + { + TestProjectHelper.DeleteDirectory(projectRoot); + } + } + + [Fact] + public void RunUnused_MaxJsonBytesWithoutJsonReturnsUsageError_Issue4904() + { + var (exitCode, stdout, stderr) = CaptureConsole(() => QueryCommandRunner.RunUnused( + ["--max-json-bytes", "1000"], + _jsonOptions)); + + Assert.Equal(CommandExitCodes.UsageError, exitCode); + Assert.Equal(string.Empty, stdout); + Assert.Contains("only supported with unused JSON output", stderr); + } + + [Theory] + [InlineData("--verbose")] + [InlineData("--profile")] + public void RunUnused_MaxJsonBytesRejectsSeparateJsonDiagnostics_Issue4904(string diagnosticsOption) + { + var (exitCode, stdout, stderr) = CaptureConsole(() => QueryCommandRunner.RunUnused( + ["--json", "--count", "--max-json-bytes", "3000", diagnosticsOption], + _jsonOptions)); + + Assert.Equal(CommandExitCodes.UsageError, exitCode); + Assert.Equal(string.Empty, stdout); + Assert.Contains("cannot be combined with --profile or --verbose", stderr); + } + + [Fact] + public void RunUnused_MaxJsonBytesCapsInvalidDatabaseJson_Issue4904() + { + using var project = TestProjectHelper.CreateTempProjectScope("cdidx_issue4904_invalid_db_budget"); + var dbPath = Path.Combine(project.Root, "empty.db"); + using (var connection = new SqliteConnection(new SqliteConnectionStringBuilder { DataSource = dbPath }.ConnectionString)) + { + connection.Open(); + } + + var (tinyExitCode, tinyStdout, tinyStderr) = CaptureConsole(() => QueryCommandRunner.RunUnused( + ["--db", dbPath, "--json", "--max-json-bytes", "1"], + _jsonOptions)); + + Assert.Equal(CommandExitCodes.DatabaseError, tinyExitCode); + Assert.Equal(string.Empty, tinyStdout); + Assert.Contains("does not appear to be a valid CodeIndex database", tinyStderr); + + const int fittingBudget = 1024; + var (fittingExitCode, fittingStdout, fittingStderr) = CaptureConsole(() => QueryCommandRunner.RunUnused( + ["--db", dbPath, "--json", "--max-json-bytes", "1024"], + _jsonOptions)); + + Assert.Equal(CommandExitCodes.DatabaseError, fittingExitCode); + Assert.Equal(string.Empty, fittingStderr); + Assert.True(Encoding.UTF8.GetByteCount(fittingStdout) <= fittingBudget); + using var document = ParseJsonOutput(fittingStdout); + Assert.Equal(CommandErrorCodes.DbError, document.RootElement.GetProperty("error_code").GetString()); + } + + [Fact] + public void RunUnused_MaxJsonBytesCapsDatabaseCodeIndexExceptionJson_Issue4904() + { + var (projectRoot, dbPath) = CreateUnusedFixtureDb(); + try + { + using var env = EnvironmentVariableScope.Capture(DatabasePermissionPolicy.EnvironmentVariable); + env.Set(DatabasePermissionPolicy.EnvironmentVariable, "invalid"); + + var (exitCode, stdout, stderr) = CaptureConsole(() => QueryCommandRunner.RunUnused( + ["--db", dbPath, "--json", "--all", "--max-json-bytes", "1"], + _jsonOptions)); + + Assert.Equal(CommandExitCodes.DatabaseError, exitCode); + Assert.Equal(string.Empty, stdout); + Assert.Contains("Invalid CDIDX_DB_PERMISSION_POLICY value", stderr); + } + finally + { + TestProjectHelper.DeleteDirectory(projectRoot); + } + } + + [Fact] + public void UnusedByteTruncatedFinalWindowRequiresUsableContinuationCursor_Issue4904() + { + Assert.True(QueryCommandRunner.CanEmitUnusedByteTruncatedPageForTests( + QueryCommandRunner.MaxQueryResultLimit, + QueryCommandRunner.MaxUnusedPaginationOffset - 1, + emittedCount: 1)); + Assert.False(QueryCommandRunner.CanEmitUnusedByteTruncatedPageForTests( + QueryCommandRunner.MaxQueryResultLimit, + QueryCommandRunner.MaxUnusedPaginationOffset, + emittedCount: 1)); + } + + [Fact] + public void RunUnused_MaxJsonBytesRenarrowsSqlReadinessToEmittedRows_Issue4904() + { + var projectRoot = TestProjectHelper.CreateTempProject("cdidx_issue4904_sql_page_budget"); + try + { + var dbPath = TestProjectHelper.CreateProjectDb(projectRoot); + TestProjectHelper.InsertIndexedFile( + dbPath, + "src/a.cs", + "csharp", + """ + public class C + { + private void AardvarkUnused() { } + public void Target() { } + public void Caller() { Target(); } + } + """); + TestProjectHelper.InsertIndexedFile( + dbPath, + "src/z.sql", + "sql", + """ + CREATE PROCEDURE dbo.SqlCaller + AS + BEGIN + EXEC dbo.Target; + END; + GO + """); + using (var db = new DbContext(DbOpenIntent.WriteIndex, dbPath)) + { + var writer = new DbWriter(db.Connection); + writer.MarkGraphReady(); + writer.MarkSqlGraphContractReady(); + } + DowngradeMixedSqlGraphContractCountRows(dbPath); + + var (unboundedExitCode, unboundedStdout, unboundedStderr) = CaptureConsole(() => QueryCommandRunner.RunUnused( + ["--db", dbPath, "--json", "--all", "--limit", "100"], + _jsonOptions)); + using var unboundedDocument = ParseJsonOutput(unboundedStdout); + var unboundedSymbols = unboundedDocument.RootElement.GetProperty("symbols").EnumerateArray().ToArray(); + + Assert.Equal(CommandExitCodes.Success, unboundedExitCode); + Assert.Equal(string.Empty, unboundedStderr); + Assert.Equal("csharp", unboundedSymbols[0].GetProperty("lang").GetString()); + Assert.Equal("sql", unboundedSymbols[1].GetProperty("lang").GetString()); + Assert.False(unboundedDocument.RootElement.GetProperty("sql_graph_contract_ready").GetBoolean()); + + var low = 1; + var high = Encoding.UTF8.GetByteCount(unboundedStdout); + (int ExitCode, string Stdout, string Stderr)? smallestSuccessfulPage = null; + while (low <= high) + { + var budget = low + ((high - low) / 2); + var page = CaptureConsole(() => QueryCommandRunner.RunUnused( + ["--db", dbPath, "--json", "--all", "--limit", "100", "--max-json-bytes", budget.ToString()], + _jsonOptions)); + if (page.Result == CommandExitCodes.Success) + { + smallestSuccessfulPage = page; + high = budget - 1; + } + else + { + low = budget + 1; + } + } + + Assert.NotNull(smallestSuccessfulPage); + Assert.Equal(string.Empty, smallestSuccessfulPage.Value.Stderr); + using var pageDocument = ParseJsonOutput(smallestSuccessfulPage.Value.Stdout); + var pageJson = pageDocument.RootElement; + var emitted = Assert.Single(pageJson.GetProperty("symbols").EnumerateArray()); + Assert.Equal("csharp", emitted.GetProperty("lang").GetString()); + Assert.True(pageJson.GetProperty("truncated").GetBoolean()); + Assert.False(pageJson.TryGetProperty("sql_graph_contract_ready", out _)); + Assert.False(pageJson.TryGetProperty("sql_graph_contract_degraded_reason", out _)); + } + finally + { + TestProjectHelper.DeleteDirectory(projectRoot); + } + } + [Fact] public void RunUnused_CompactJsonOmitsSymbolBodiesAndShowsFilters_Issue3395() { diff --git a/tests/CodeIndex.Tests/QueryCommandRunnerTests.cs b/tests/CodeIndex.Tests/QueryCommandRunnerTests.cs index 296a4ae4a..a1febd9ad 100644 --- a/tests/CodeIndex.Tests/QueryCommandRunnerTests.cs +++ b/tests/CodeIndex.Tests/QueryCommandRunnerTests.cs @@ -8261,7 +8261,7 @@ private void InvokeWriteDatabaseOpenFailureJsonAware(Exception exception) "WriteDatabaseOpenFailureJsonAware", BindingFlags.NonPublic | BindingFlags.Static); Assert.NotNull(method); - var exitCode = method.Invoke(null, [exception, "/tmp/db-open-failure-test.db", false, _jsonOptions]); + var exitCode = method.Invoke(null, [exception, "/tmp/db-open-failure-test.db", false, _jsonOptions, null]); Assert.Equal(CommandExitCodes.DatabaseError, exitCode); }