From b9cfe3925f9ba5d219f6ba7f69c6b86d25370b4a Mon Sep 17 00:00:00 2001 From: Widthdom Date: Sun, 14 Jun 2026 11:26:09 +0900 Subject: [PATCH 1/2] Fix excerpt semantic token coordinates (#3563) --- DEVELOPER_GUIDE.md | 4 +- changelog.d/unreleased/3563.fixed.md | 18 +++++++ src/CodeIndex/Cli/JsonOutputContracts.cs | 1 + src/CodeIndex/Cli/QueryCommandRunner.cs | 47 +++++++++++++---- .../Database/DbReader.FilesStatus.cs | 43 +++++++++++----- src/CodeIndex/Models/QueryResults.cs | 12 +++++ .../QueryCommandRunnerSearchTests.cs | 18 +++++-- tests/CodeIndex.Tests/golden/excerpt.json | 51 +++++++++++++++++++ 8 files changed, 167 insertions(+), 27 deletions(-) create mode 100644 changelog.d/unreleased/3563.fixed.md diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index cc4167bf7a..fc165eb73a 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -1073,7 +1073,7 @@ When the match line falls inside an indexed symbol range, `search --json` and MC `find --json` remains line-delimited for repeated matches and adds bounded match-span/truncation metadata to each row: `length` reports the 1-based `column` span length, `original_line_length` reports the source line length before any line-width clamp, and `snippet_truncation_context.line_count` / `char_counts` / `total_chars` / optional `reason` describe snippet clamping. `reason` is `line_width` when `--max-line-width` elides one or more snippet lines. -`excerpt --json` includes `semantic_tokens`, a lightweight range list with 1-based start/end positions, token `type`, and `modifiers`, so IDE and LLM clients can render or post-process excerpt spans without reparsing the raw `content` string. Excerpt rows also expose `requested_start_line`, `requested_end_line`, `effective_start_line`, `effective_end_line`, `content_truncation_reasons`, and optional `content_recovery` so clients can tell when `--max-line-width` caused `line_width_cap` and replay `cdidx excerpt ... --max-line-width 0 --json` for the omitted text. Body-bearing JSON rows use matching `body_requested_*`, `body_effective_*`, `body_content_truncation_reasons`, and `body_content_recovery` fields; body reasons include `body_line_cap` for snippet/body line caps and `body_byte_cap` for definition body byte caps. +`excerpt --json` includes `semantic_tokens`, a lightweight range list with 1-based source start/end positions, token `type`, and `modifiers`, so IDE and LLM clients can render or post-process excerpt spans without reparsing the raw `content` string. `semantic_token_coordinate_space` is `source`; when `--max-line-width` clamps returned content, `content_line_spans` maps each returned content line and visible content-column span back to the matching source line and source-column span, while clamp markers remain unmapped and are not emitted as semantic tokens. Excerpt rows also expose `requested_start_line`, `requested_end_line`, `effective_start_line`, `effective_end_line`, `content_truncation_reasons`, and optional `content_recovery` so clients can tell when `--max-line-width` caused `line_width_cap` and replay `cdidx excerpt ... --max-line-width 0 --json` for the omitted text. Body-bearing JSON rows use matching `body_requested_*`, `body_effective_*`, `body_content_truncation_reasons`, and `body_content_recovery` fields; body reasons include `body_line_cap` for snippet/body line caps and `body_byte_cap` for definition body byte caps. `inspect` and MCP `analyze_symbol` bundle the primary definition, nearby symbols from the same file, references, callers, callees, file metadata, workspace freshness/git metadata, and graph-support metadata into one response. When those bundled graph sections actually depend on SQL-backed reads, the payload also mirrors `sql_graph_contract_ready` / `sql_graph_contract_degraded_reason` (plus the existing camelCase aliases on MCP responses); mixed-language bundles that only return C# / JS / etc. graph rows omit the SQL trust signal entirely. This is intended for symbol-oriented AI workflows that would otherwise need several back-to-back calls. Call graph sections remain language-aware: for unsupported languages, clients can now distinguish "unsupported" from "no hits" via `graphSupported` / `graphSupportReason`, and should prefer `search` instead of assuming graph data will exist. @@ -3288,7 +3288,7 @@ literal-safe な `search` query は reader 層で FTS5 sanitization 前に 1000 `find --json` は繰り返し一致でも line-delimited のまま維持し、各 row に bounded な match span / truncation metadata を追加します。`length` は 1-based の `column` から始まる一致長、`original_line_length` は行幅クランプ前のソース行長、`snippet_truncation_context.line_count` / `char_counts` / `total_chars` / 任意の `reason` は snippet クランプを表します。`--max-line-width` によって snippet 行が省略された場合、`reason` は `line_width` になります。 -`excerpt --json` は 1-based の開始/終了位置、token `type`、`modifiers` を持つ軽量 range list の `semantic_tokens` を返すため、IDE や LLM クライアントは生の `content` 文字列を再パースせずに抜粋範囲を描画・後処理できます。excerpt row は `requested_start_line`、`requested_end_line`、`effective_start_line`、`effective_end_line`、`content_truncation_reasons`、任意の `content_recovery` も返すため、`--max-line-width` による `line_width_cap` を検出し、省略部分を `cdidx excerpt ... --max-line-width 0 --json` で再取得できます。body を持つ JSON row も対応する `body_requested_*`、`body_effective_*`、`body_content_truncation_reasons`、`body_content_recovery` を返します。body reason には snippet/body 行数上限の `body_line_cap` と definition body byte 上限の `body_byte_cap` があります。 +`excerpt --json` は 1-based の source 開始/終了位置、token `type`、`modifiers` を持つ軽量 range list の `semantic_tokens` を返すため、IDE や LLM クライアントは生の `content` 文字列を再パースせずに抜粋範囲を描画・後処理できます。`semantic_token_coordinate_space` は `source` です。`--max-line-width` で返却内容がクランプされた場合、`content_line_spans` は返却 content 行と可視 content column span を、対応する source 行と source column span に対応付けます。clamp marker は未対応領域として扱い、semantic token には含めません。excerpt row は `requested_start_line`、`requested_end_line`、`effective_start_line`、`effective_end_line`、`content_truncation_reasons`、任意の `content_recovery` も返すため、`--max-line-width` による `line_width_cap` を検出し、省略部分を `cdidx excerpt ... --max-line-width 0 --json` で再取得できます。body を持つ JSON row も対応する `body_requested_*`、`body_effective_*`、`body_content_truncation_reasons`、`body_content_recovery` を返します。body reason には snippet/body 行数上限の `body_line_cap` と definition body byte 上限の `body_byte_cap` があります。 `inspect` と MCP の `analyze_symbol` は、主定義、同一ファイル内の近傍シンボル、参照、caller、callee、ファイルメタデータ、さらにワークスペース鮮度/git メタデータと graph 対応メタデータを1レスポンスにまとめます。bundle 内の graph 節が実際に SQL ベースの read に依存する場合だけ、`sql_graph_contract_ready` / `sql_graph_contract_degraded_reason`（MCP では既存の camelCase alias も）も返します。mixed-language bundle で C# / JS などの graph row しか返っていない場合は SQL trust signal を出さないため、無関係なクエリが stale SQL state に引きずられません。複数の連続クエリを避けたい AI ワークフロー向けです。call graph 系の節は言語差分を考慮しており、未対応言語では `graphSupported` / `graphSupportReason` によって「未対応」と「ヒットなし」を区別できます。その場合は `search` を優先して使う前提です。 diff --git a/changelog.d/unreleased/3563.fixed.md b/changelog.d/unreleased/3563.fixed.md new file mode 100644 index 0000000000..32e46a3379 --- /dev/null +++ b/changelog.d/unreleased/3563.fixed.md @@ -0,0 +1,18 @@ +--- +category: fixed +issues: + - 3563 +affected: + - src/CodeIndex/Cli/QueryCommandRunner.cs + - src/CodeIndex/Database/DbReader.FilesStatus.cs + - src/CodeIndex/Models/QueryResults.cs + - DEVELOPER_GUIDE.md +--- + +## English + +- **Excerpt semantic tokens now use source coordinates for clamped content (#3563)** — `excerpt --json` reports `semantic_token_coordinate_space: "source"` and includes `content_line_spans` so clients can map visible clamped content back to original source columns without treating clamp markers as tokens. + +## 日本語 + +- **clamp された excerpt content の semantic token が source 座標を使うようになりました (#3563)** — `excerpt --json` は `semantic_token_coordinate_space: "source"` と `content_line_spans` を返し、クライアントが clamp marker を token として扱わずに可視 content を元の source column へ対応付けられるようになりました。 diff --git a/src/CodeIndex/Cli/JsonOutputContracts.cs b/src/CodeIndex/Cli/JsonOutputContracts.cs index fafa58ae30..8e91f9147f 100644 --- a/src/CodeIndex/Cli/JsonOutputContracts.cs +++ b/src/CodeIndex/Cli/JsonOutputContracts.cs @@ -472,6 +472,7 @@ internal sealed record VersionInfoJsonResult( [JsonSerializable(typeof(ExportImportCommandRunner.ExportManifest))] [JsonSerializable(typeof(ExportImportCommandRunner.ImportDryRunResult))] [JsonSerializable(typeof(ExportImportCommandRunner.ImportValidationPhaseResult))] +[JsonSerializable(typeof(ExcerptContentLineSpan))] [JsonSerializable(typeof(ExcerptRecoveryHint))] [JsonSerializable(typeof(ExcerptSemanticToken))] [JsonSerializable(typeof(FileDependencyResult))] diff --git a/src/CodeIndex/Cli/QueryCommandRunner.cs b/src/CodeIndex/Cli/QueryCommandRunner.cs index e4b0b6b767..38d4cd5db5 100644 --- a/src/CodeIndex/Cli/QueryCommandRunner.cs +++ b/src/CodeIndex/Cli/QueryCommandRunner.cs @@ -3858,11 +3858,19 @@ private static List BuildExcerptSemanticTokens(FileExcerpt { var tokens = new List(); var lines = excerpt.Content.Replace("\r\n", "\n").Split('\n'); - for (var lineIndex = 0; lineIndex < lines.Length; lineIndex++) + var spans = excerpt.ContentLineSpans.Count == 0 + ? BuildIdentityExcerptContentLineSpans(excerpt, lines) + : excerpt.ContentLineSpans; + foreach (var span in spans) { - var line = lines[lineIndex]; - var column = 0; - while (column < line.Length) + if (span.ContentLine <= 0 || span.ContentLine > lines.Length) + continue; + + var line = lines[span.ContentLine - 1]; + var startColumn = Math.Clamp(span.ContentStartColumn - 1, 0, line.Length); + var endColumn = Math.Clamp(span.ContentEndColumn - 1, startColumn, line.Length); + var column = startColumn; + while (column < endColumn) { if (!IsSemanticTokenStart(line[column])) { @@ -3872,16 +3880,18 @@ private static List BuildExcerptSemanticTokens(FileExcerpt var start = column; column++; - while (column < line.Length && IsSemanticTokenPart(line[column])) + while (column < endColumn && IsSemanticTokenPart(line[column])) column++; var tokenText = line[start..column]; + var sourceStartColumn = span.SourceStartColumn + ((start + 1) - span.ContentStartColumn); + var sourceEndColumn = span.SourceStartColumn + ((column + 1) - span.ContentStartColumn); tokens.Add(new ExcerptSemanticToken { - StartLine = excerpt.StartLine + lineIndex, - StartColumn = start + 1, - EndLine = excerpt.StartLine + lineIndex, - EndColumn = column + 1, + StartLine = span.SourceLine, + StartColumn = sourceStartColumn, + EndLine = span.SourceLine, + EndColumn = sourceEndColumn, Type = ClassifySemanticToken(tokenText), }); } @@ -3890,6 +3900,25 @@ private static List BuildExcerptSemanticTokens(FileExcerpt return tokens; } + private static List BuildIdentityExcerptContentLineSpans(FileExcerptResult excerpt, string[] lines) + { + var spans = new List(lines.Length); + for (var i = 0; i < lines.Length; i++) + { + spans.Add(new ExcerptContentLineSpan + { + ContentLine = i + 1, + SourceLine = excerpt.StartLine + i, + ContentStartColumn = 1, + ContentEndColumn = lines[i].Length + 1, + SourceStartColumn = 1, + SourceEndColumn = lines[i].Length + 1, + }); + } + + return spans; + } + private static bool IsSemanticTokenStart(char value) => char.IsLetter(value) || value == '_' || char.IsDigit(value); diff --git a/src/CodeIndex/Database/DbReader.FilesStatus.cs b/src/CodeIndex/Database/DbReader.FilesStatus.cs index dc8a1a0ef1..d65e27359b 100644 --- a/src/CodeIndex/Database/DbReader.FilesStatus.cs +++ b/src/CodeIndex/Database/DbReader.FilesStatus.cs @@ -561,14 +561,32 @@ FROM chunks c var focusLineIndex = focusLine.HasValue ? selectedLines.IndexOf(focusLine.Value) : -1; if (focusLineIndex >= 0 && focusColumn.HasValue && focusColumn.Value > contentLines[focusLineIndex].Length) return null; - var clampedContent = maxLineWidth.HasValue - ? LineWidthFormatter.ClampLines( - contentLines, - maxLineWidth.Value, - focusLineIndex >= 0 ? focusLineIndex : null, - focusLineIndex >= 0 ? focusColumn : null, - focusLength) - : new ClampedTextResult(string.Join("\n", contentLines), false); + var excerptLines = new string[contentLines.Count]; + var contentLineSpans = new List(contentLines.Count); + var contentTruncated = false; + for (var i = 0; i < contentLines.Count; i++) + { + var clampedLine = maxLineWidth.HasValue + ? LineWidthFormatter.ClampLine( + contentLines[i], + maxLineWidth.Value, + i == focusLineIndex ? focusColumn : null, + focusLength) + : ClampedTextResult.Unclamped(contentLines[i]); + + excerptLines[i] = clampedLine.Text; + contentTruncated |= clampedLine.Truncated; + var visibleLength = Math.Max(0, clampedLine.OriginalVisibleEndColumn - clampedLine.OriginalVisibleStartColumn + 1); + contentLineSpans.Add(new ExcerptContentLineSpan + { + ContentLine = i + 1, + SourceLine = selectedLines[i], + ContentStartColumn = clampedLine.TextVisibleStartColumn, + ContentEndColumn = clampedLine.TextVisibleStartColumn + visibleLength, + SourceStartColumn = clampedLine.OriginalVisibleStartColumn, + SourceEndColumn = clampedLine.OriginalVisibleStartColumn + visibleLength, + }); + } return new FileExcerptResult { @@ -580,12 +598,13 @@ FROM chunks c RequestedEndLine = requestedEndCeiling, EffectiveStartLine = selectedLines[0], EffectiveEndLine = selectedLines[^1], - Content = clampedContent.Text, - ContentTruncated = clampedContent.Truncated, - ContentTruncationReasons = clampedContent.Truncated ? ["line_width_cap"] : [], - ContentRecovery = clampedContent.Truncated + Content = string.Join("\n", excerptLines), + ContentTruncated = contentTruncated, + ContentTruncationReasons = contentTruncated ? ["line_width_cap"] : [], + ContentRecovery = contentTruncated ? FileExcerptResult.CreateRecoveryHint(path, selectedLines[0], selectedLines[^1]) : null, + ContentLineSpans = contentLineSpans, }; } diff --git a/src/CodeIndex/Models/QueryResults.cs b/src/CodeIndex/Models/QueryResults.cs index 4f3faf9079..63b7967749 100644 --- a/src/CodeIndex/Models/QueryResults.cs +++ b/src/CodeIndex/Models/QueryResults.cs @@ -262,6 +262,8 @@ public class FileExcerptResult public List ContentTruncationReasons { get; set; } = []; [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public ExcerptRecoveryHint? ContentRecovery { get; set; } + public string SemanticTokenCoordinateSpace { get; set; } = "source"; + public List ContentLineSpans { get; set; } = []; [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public List? SemanticTokens { get; set; } @@ -285,6 +287,16 @@ private static bool IsSafeCliArgumentChar(char c) => char.IsLetterOrDigit(c) || c is '/' or '.' or '_' or '-' or ':'; } +public class ExcerptContentLineSpan +{ + public int ContentLine { get; set; } + public int SourceLine { get; set; } + public int ContentStartColumn { get; set; } + public int ContentEndColumn { get; set; } + public int SourceStartColumn { get; set; } + public int SourceEndColumn { get; set; } +} + public class ExcerptSemanticToken { public int StartLine { get; set; } diff --git a/tests/CodeIndex.Tests/QueryCommandRunnerSearchTests.cs b/tests/CodeIndex.Tests/QueryCommandRunnerSearchTests.cs index f688bf414f..df358433a9 100644 --- a/tests/CodeIndex.Tests/QueryCommandRunnerSearchTests.cs +++ b/tests/CodeIndex.Tests/QueryCommandRunnerSearchTests.cs @@ -2990,11 +2990,12 @@ public void RunExcerpt_JsonClampsLongSingleLineContentAroundFocus() try { var dbPath = TestProjectHelper.CreateProjectDb(projectRoot); - var longLine = new string('a', 320) + "TARGET" + new string('b', 320); + var longLine = new string('a', 320) + " TARGET " + new string('b', 320); + var targetColumn = longLine.IndexOf("TARGET", StringComparison.Ordinal) + 1; TestProjectHelper.InsertIndexedFile(dbPath, "dist/data.txt", "text", longLine); var (exitCode, stdout, stderr) = CaptureConsole(() => QueryCommandRunner.RunExcerpt( - ["dist/data.txt", "--db", dbPath, "--start", "1", "--end", "1", "--json", "--max-line-width", "96", "--focus-column", (longLine.IndexOf("TARGET", StringComparison.Ordinal) + 1).ToString(), "--focus-length", "6"], + ["dist/data.txt", "--db", dbPath, "--start", "1", "--end", "1", "--json", "--max-line-width", "96", "--focus-column", targetColumn.ToString(), "--focus-length", "6"], _jsonOptions)); using var document = ParseJsonOutput(stdout); @@ -3023,11 +3024,20 @@ public void RunExcerpt_JsonClampsLongSingleLineContentAroundFocus() Assert.DoesNotContain(longLine, json.GetProperty("content").GetString()); Assert.Contains("TARGET", json.GetProperty("content").GetString()); Assert.True(json.GetProperty("content").GetString()!.Length <= 96); + Assert.Equal("source", json.GetProperty("semantic_token_coordinate_space").GetString()); + var span = Assert.Single(json.GetProperty("content_line_spans").EnumerateArray()); + Assert.Equal(1, span.GetProperty("content_line").GetInt32()); + Assert.Equal(1, span.GetProperty("source_line").GetInt32()); + Assert.True(span.GetProperty("content_start_column").GetInt32() > 1); + Assert.True(span.GetProperty("source_start_column").GetInt32() <= targetColumn); + Assert.True(span.GetProperty("source_end_column").GetInt32() >= targetColumn + "TARGET".Length); var semanticTokens = json.GetProperty("semantic_tokens").EnumerateArray().ToArray(); Assert.Contains(semanticTokens, token => - token.GetProperty("type").GetString() == "variable" && + token.GetProperty("type").GetString() == "type" && token.GetProperty("start_line").GetInt32() == 1 && - token.GetProperty("start_column").GetInt32() > 0); + token.GetProperty("start_column").GetInt32() == targetColumn && + token.GetProperty("end_column").GetInt32() == targetColumn + "TARGET".Length); + Assert.DoesNotContain(semanticTokens, token => token.GetProperty("type").GetString() == "number"); } finally { diff --git a/tests/CodeIndex.Tests/golden/excerpt.json b/tests/CodeIndex.Tests/golden/excerpt.json index 5e73b41ce6..c3f9d1dc33 100644 --- a/tests/CodeIndex.Tests/golden/excerpt.json +++ b/tests/CodeIndex.Tests/golden/excerpt.json @@ -11,6 +11,57 @@ "content": "namespace Demo;\n\npublic static class Lib\n{\n public static int Add(int a, int b) =\u003E a \u002B b;\n}", "content_truncated": false, "content_truncation_reasons": [], + "semantic_token_coordinate_space": "source", + "content_line_spans": [ + { + "content_line": 1, + "source_line": 1, + "content_start_column": 1, + "content_end_column": 16, + "source_start_column": 1, + "source_end_column": 16 + }, + { + "content_line": 2, + "source_line": 2, + "content_start_column": 1, + "content_end_column": 1, + "source_start_column": 1, + "source_end_column": 1 + }, + { + "content_line": 3, + "source_line": 3, + "content_start_column": 1, + "content_end_column": 24, + "source_start_column": 1, + "source_end_column": 24 + }, + { + "content_line": 4, + "source_line": 4, + "content_start_column": 1, + "content_end_column": 2, + "source_start_column": 1, + "source_end_column": 2 + }, + { + "content_line": 5, + "source_line": 5, + "content_start_column": 1, + "content_end_column": 50, + "source_start_column": 1, + "source_end_column": 50 + }, + { + "content_line": 6, + "source_line": 6, + "content_start_column": 1, + "content_end_column": 2, + "source_start_column": 1, + "source_end_column": 2 + } + ], "semantic_tokens": [ { "start_line": 1, From db57a0a2d90a5c6361fe5106277550f81ef5c48a Mon Sep 17 00:00:00 2001 From: Widthdom Date: Sun, 14 Jun 2026 11:39:57 +0900 Subject: [PATCH 2/2] Keep MCP excerpt metadata within budget (#3563) --- src/CodeIndex/Mcp/McpToolHandlers.cs | 88 ++++++++++++++++++++++++- tests/CodeIndex.Tests/McpServerTests.cs | 50 ++++++++++++++ 2 files changed, 137 insertions(+), 1 deletion(-) diff --git a/src/CodeIndex/Mcp/McpToolHandlers.cs b/src/CodeIndex/Mcp/McpToolHandlers.cs index db0364d979..aca3d7d9c4 100644 --- a/src/CodeIndex/Mcp/McpToolHandlers.cs +++ b/src/CodeIndex/Mcp/McpToolHandlers.cs @@ -3564,20 +3564,106 @@ internal static void ApplyExcerptOutputBudget(JsonObject payload, int maxOutputB return; var builder = new StringBuilder(); + var retainedLineCount = 0; + var firstRetainedLine = true; foreach (var line in content.Replace("\r\n", "\n").Split('\n')) { - var candidate = builder.Length == 0 ? line : builder.ToString() + "\n" + line; + var candidate = firstRetainedLine ? line : builder.ToString() + "\n" + line; if (Encoding.UTF8.GetByteCount(candidate) > maxOutputBytes) break; builder.Clear(); builder.Append(candidate); + retainedLineCount++; + firstRetainedLine = false; } payload[contentKey] = builder.ToString(); + TrimExcerptCoordinatePayload(payload, retainedLineCount); payload["contentTruncated"] = true; payload["truncated"] = true; payload["truncation_reason"] = "output_size_cap"; } + private static void TrimExcerptCoordinatePayload(JsonObject payload, int retainedLineCount) + { + var spansKey = FirstPayloadKey(payload, "contentLineSpans", "content_line_spans", "ContentLineSpans"); + var retainedSpans = new List(); + var hasSpanMapping = false; + if (spansKey is not null && payload[spansKey] is JsonArray spans) + { + hasSpanMapping = true; + var trimmedSpans = new JsonArray(); + foreach (var spanNode in spans) + { + if (spanNode is not JsonObject span) + continue; + var contentLine = GetPayloadInt(span, "contentLine", "content_line", "ContentLine"); + if (!contentLine.HasValue || contentLine.Value > retainedLineCount) + continue; + + trimmedSpans.Add(span.DeepClone()); + var sourceLine = GetPayloadInt(span, "sourceLine", "source_line", "SourceLine"); + var sourceStartColumn = GetPayloadInt(span, "sourceStartColumn", "source_start_column", "SourceStartColumn"); + var sourceEndColumn = GetPayloadInt(span, "sourceEndColumn", "source_end_column", "SourceEndColumn"); + if (sourceLine.HasValue && sourceStartColumn.HasValue && sourceEndColumn.HasValue) + retainedSpans.Add(new ExcerptPayloadSpan(sourceLine.Value, sourceStartColumn.Value, sourceEndColumn.Value)); + } + + payload[spansKey] = trimmedSpans; + } + + var tokensKey = FirstPayloadKey(payload, "semanticTokens", "semantic_tokens", "SemanticTokens"); + if (tokensKey is null || payload[tokensKey] is not JsonArray tokens) + return; + if (!hasSpanMapping) + { + if (retainedLineCount == 0) + payload[tokensKey] = new JsonArray(); + return; + } + + var trimmedTokens = new JsonArray(); + if (retainedLineCount > 0 && retainedSpans.Count > 0) + { + foreach (var tokenNode in tokens) + { + if (tokenNode is not JsonObject token) + continue; + var startLine = GetPayloadInt(token, "startLine", "start_line", "StartLine"); + var endLine = GetPayloadInt(token, "endLine", "end_line", "EndLine"); + var startColumn = GetPayloadInt(token, "startColumn", "start_column", "StartColumn"); + var endColumn = GetPayloadInt(token, "endColumn", "end_column", "EndColumn"); + if (!startLine.HasValue || !endLine.HasValue || !startColumn.HasValue || !endColumn.HasValue) + continue; + if (retainedSpans.Any(span => + startLine.Value == span.SourceLine && + endLine.Value == span.SourceLine && + startColumn.Value >= span.SourceStartColumn && + endColumn.Value <= span.SourceEndColumn)) + { + trimmedTokens.Add(token.DeepClone()); + } + } + } + + payload[tokensKey] = trimmedTokens; + } + + private static string? FirstPayloadKey(JsonObject payload, params string[] keys) + => keys.FirstOrDefault(payload.ContainsKey); + + private static int? GetPayloadInt(JsonObject obj, params string[] keys) + { + foreach (var key in keys) + { + if (obj[key] is JsonNode node) + return node.GetValue(); + } + + return null; + } + + private readonly record struct ExcerptPayloadSpan(int SourceLine, int SourceStartColumn, int SourceEndColumn); + private JsonNode ExecuteFindInFile(JsonNode? id, JsonNode? args) { if (!TryReadRequiredStringParameter(args, "query", out var query, out var requiredError)) diff --git a/tests/CodeIndex.Tests/McpServerTests.cs b/tests/CodeIndex.Tests/McpServerTests.cs index 9c9defb565..7b63494cae 100644 --- a/tests/CodeIndex.Tests/McpServerTests.cs +++ b/tests/CodeIndex.Tests/McpServerTests.cs @@ -9776,6 +9776,48 @@ public void ApplyExcerptOutputBudget_TruncatesAtLineBoundary_Issue1605() { ["content"] = "short\n" + new string('x', 200), ["contentTruncated"] = false, + ["contentLineSpans"] = new JsonArray + { + new JsonObject + { + ["contentLine"] = 1, + ["sourceLine"] = 10, + ["contentStartColumn"] = 1, + ["contentEndColumn"] = 6, + ["sourceStartColumn"] = 3, + ["sourceEndColumn"] = 8, + }, + new JsonObject + { + ["contentLine"] = 2, + ["sourceLine"] = 11, + ["contentStartColumn"] = 1, + ["contentEndColumn"] = 201, + ["sourceStartColumn"] = 1, + ["sourceEndColumn"] = 201, + }, + }, + ["semanticTokens"] = new JsonArray + { + new JsonObject + { + ["startLine"] = 10, + ["startColumn"] = 3, + ["endLine"] = 10, + ["endColumn"] = 8, + ["type"] = "variable", + ["modifiers"] = new JsonArray(), + }, + new JsonObject + { + ["startLine"] = 11, + ["startColumn"] = 1, + ["endLine"] = 11, + ["endColumn"] = 5, + ["type"] = "variable", + ["modifiers"] = new JsonArray(), + }, + }, }; McpServer.ApplyExcerptOutputBudget(payload, 20); @@ -9784,6 +9826,14 @@ public void ApplyExcerptOutputBudget_TruncatesAtLineBoundary_Issue1605() Assert.Equal("output_size_cap", payload["truncation_reason"]!.GetValue()); Assert.Equal("short", payload["content"]!.GetValue()); Assert.True(payload["contentTruncated"]!.GetValue()); + var spans = payload["contentLineSpans"]!.AsArray(); + var span = Assert.Single(spans); + Assert.Equal(1, span!["contentLine"]!.GetValue()); + Assert.Equal(10, span["sourceLine"]!.GetValue()); + var tokens = payload["semanticTokens"]!.AsArray(); + var token = Assert.Single(tokens); + Assert.Equal(10, token!["startLine"]!.GetValue()); + Assert.Equal(8, token["endColumn"]!.GetValue()); } [Fact]