From ac20647daa10de93bcdf821ccc2ebd2a06bef151 Mon Sep 17 00:00:00 2001 From: Widthdom Date: Tue, 28 Jul 2026 09:50:12 +0900 Subject: [PATCH 1/3] Preserve live LSP document symbol hierarchy (#4851) --- DEVELOPER_GUIDE.md | 24 ++- README.md | 4 +- USER_GUIDE.md | 28 ++-- changelog.d/unreleased/4851.fixed.md | 21 +++ src/CodeIndex/Lsp/LspLiveDocumentStore.cs | 19 ++- src/CodeIndex/Lsp/LspServer.SymbolRequests.cs | 55 ++++++- src/CodeIndex/Lsp/LspServer.cs | 14 +- tests/CodeIndex.Tests/LspServerTests.cs | 140 ++++++++++++++++++ 8 files changed, 280 insertions(+), 25 deletions(-) create mode 100644 changelog.d/unreleased/4851.fixed.md diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index 2ea044278f..822b4f5b59 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -427,9 +427,11 @@ Editor integrations can request standard location shapes directly. `definition`, The `cdidx lsp` server advertises full text document synchronization and keeps open document text in a bounded in-memory cache only. Position-based providers must read that live cache before disk so unsaved editor buffers can identify the -requested token, but provider results remain conservative and index-backed: -return empty arrays or null when the database cannot answer safely instead of -inventing language-server analysis. +requested token. Provider results remain conservative and index-backed except +that document symbols for an indexed document may be structurally re-extracted +from the bounded live buffer through the normal language extractor and container +pipeline. Other providers return empty arrays or null when the database cannot +answer safely instead of inventing language-server analysis. Disk-backed position-line caching must enforce its 4 MiB input limit while streaming, not only through a pre-read `Length` check. Bytes beyond the limit must never reach text decoding, including when a shared file grows concurrently, @@ -457,6 +459,10 @@ container names, container kinds, enclosing ranges, and same-line selection columns. Same-range members such as positional record properties therefore stay beneath their declaring type regardless of deterministic presentation order, while a later same-named container on the line cannot capture an earlier member. +Live document symbols use the same extractor, normalization, and hierarchy +builder as indexed symbols, so a full-text change updates both ranges and +containers together. Numeric document versions must increase; an older or equal +change cannot replace the newest accepted live text. The stdio reader and the single response worker are separated by a bounded queue, so `$/cancelRequest` can cancel an active or queued symbol request without making database-backed request processing concurrent. Cancellation IDs retain @@ -3624,9 +3630,11 @@ editor integration は標準的な location 形状を直接要求できる。`de `cdidx lsp` server は full text document synchronization を advertise し、open document text は 上限付きの in-memory cache にだけ保持する。position-based provider は未保存 editor buffer から -request token を特定できるよう disk より先に live cache を読む必要があるが、provider result は -保守的かつ index-backed のままにする。database が安全に答えられない場合は、language-server -analysis を作り上げず、空配列または null を返す。 +request token を特定できるよう disk より先に live cache を読む必要がある。provider result は +保守的かつ index-backed のままとするが、indexed document の document symbol だけは上限付きの +live buffer を通常の language extractor と container pipeline で構造的に再抽出できる。それ以外の +provider は database が安全に答えられない場合、language-server analysis を作り上げず、空配列 +または null を返す。 disk 上の position-line cache は、事前の `Length` check だけでなく streaming 中も 4 MiB の input 上限を強制する必要がある。共有 file が同時に増大する場合も上限超過 byte を text decode に 渡してはならず、bounded な failure reason は `position_file_too_large` のままとする。 @@ -3649,6 +3657,10 @@ integer `partialResultToken` と `workDoneToken` を処理する。partial resul container name・container kind・包含 range・同一行の selection column で親を解決するため、 positional record property のように同じ range を持つ member も決定的な表示順序に左右されず 宣言元 type の配下に留まり、行内で後にある同名 container が前の member を取り込まない。 +live document symbol は indexed symbol と同じ extractor、normalization、hierarchy builder を +使うため、full-text change では range と container が一緒に更新される。numeric document version +は増加する必要があり、古い、または同じ version の change は最後に受理した live text を +置き換えられない。 stdio reader と単一 response worker は上限付き queue で分離するため、database-backed request processing を並行化せずに `$/cancelRequest` で active または queued symbol request を cancel できる。cancellation ID は JSON 型を保持し、cancel diff --git a/README.md b/README.md index 8fb848a92c..e7be17ebbf 100644 --- a/README.md +++ b/README.md @@ -170,7 +170,7 @@ incomplete generation; JSON still reports `status: "partial"`. |---|---| | Search and navigation | `search`, `find`, `excerpt`, `symbols`, `definition`, `references`, `callers`, `callees`, `inspect`, `map`, `deps`, `impact`, `unused`, and `hotspots`. See the [command reference](USER_GUIDE.md#command-reference). | | AI integration | `cdidx mcp` exposes indexed search tools for Claude Code, Cursor, Windsurf, Copilot, Codex, and other MCP clients. See [AI Integration](USER_GUIDE.md#ai-integration). | -| Editor lookup | `cdidx lsp --db .cdidx/codeindex.db` starts a read-only LSP shim for editors that can launch an LSP command. Large document/workspace symbol requests support bounded partial-result chunks, work-done progress, and request cancellation. References use indexed symbol identity, symbol locations select identifiers, and type hints omit explicitly declared types. C# semantic tokens distinguish keywords, modifiers, namespace components, types, fields, methods, and declarations. | +| Editor lookup | `cdidx lsp --db .cdidx/codeindex.db` starts a read-only LSP shim for editors that can launch an LSP command. Large document/workspace symbol requests support bounded partial-result chunks, work-done progress, and request cancellation. Open-document symbols are re-extracted from the latest accepted full-text version, so hierarchy and ranges follow unsaved edits while stale document versions are ignored. References use indexed symbol identity, symbol locations select identifiers, and type hints omit explicitly declared types. C# semantic tokens distinguish keywords, modifiers, namespace components, types, fields, methods, and declarations. | | Freshness | `status --check`, `--files`, `--commits`, `--changed-between`, and `--watch` keep the DB aligned with the workspace. | | Validation | `cdidx validate` reports encoding and line-ending issues in indexed files. See [Validate indexed files](USER_GUIDE.md#validate-indexed-files). | | Language coverage | `cdidx languages --json` is the live capability probe; add `--format count`, `--summary-only`, `--capability `, `--language`, `--extension`, or `--alias` to narrow output. See [Supported languages](USER_GUIDE.md#supported-languages). | @@ -583,7 +583,7 @@ commit し、構造化 `file_errors` を返して partial-result 終了コード |---|---| | 検索とナビゲーション | `search`、`find`、`excerpt`、`symbols`、`definition`、`references`、`callers`、`callees`、`inspect`、`map`、`deps`、`impact`、`unused`、`hotspots`。詳細は [コマンドリファレンス](USER_GUIDE.md#コマンドリファレンス)。 | | AI 連携 | `cdidx mcp` は Claude Code、Cursor、Windsurf、Copilot、Codex などの MCP client に indexed search tool を提供します。詳細は [AIとの連携](USER_GUIDE.md#aiとの連携)。 | -| editor lookup | `cdidx lsp --db .cdidx/codeindex.db` は、LSP command を起動できる editor 向けの read-only LSP shim です。大きな document/workspace symbol request は上限付き partial-result chunk、work-done progress、request cancellation に対応します。reference は indexed symbol identity を使い、symbol location は identifier を選択し、type hint は明示的に宣言された型を省略します。C# semantic token は keyword、modifier、namespace component、type、field、method、declaration を区別します。 | +| editor lookup | `cdidx lsp --db .cdidx/codeindex.db` は、LSP command を起動できる editor 向けの read-only LSP shim です。大きな document/workspace symbol request は上限付き partial-result chunk、work-done progress、request cancellation に対応します。open document の symbol は最後に受理した full-text version から再抽出するため、古い document version を無視しながら未保存の編集に hierarchy と range を追従させます。reference は indexed symbol identity を使い、symbol location は identifier を選択し、type hint は明示的に宣言された型を省略します。C# semantic token は keyword、modifier、namespace component、type、field、method、declaration を区別します。 | | 鮮度管理 | `status --check`、`--files`、`--commits`、`--changed-between`、`--watch` で DB と workspace を揃えます。 | | validation | `cdidx validate` は indexed file の encoding / line-ending 問題を報告します。詳細は [Indexed files を validate する](USER_GUIDE.md#indexed-files-を-validate-する)。 | | 対応言語 | `cdidx languages --json` が live capability probe です。`--language`、`--extension`、`--alias` で 1 行を lookup できます。詳細は [対応言語](USER_GUIDE.md#対応言語)。 | diff --git a/USER_GUIDE.md b/USER_GUIDE.md index 7b2b197004..3d73b41bdd 100644 --- a/USER_GUIDE.md +++ b/USER_GUIDE.md @@ -2720,9 +2720,13 @@ capped at 4194304 bytes, the session holds at most 64 live documents and 16777216 aggregate live-document bytes, and older entries are evicted when a budget is exceeded. `textDocument/didChange` processes only the last 64 change entries in an oversized `contentChanges` array, preserving the latest full-text -update without retaining unbounded intermediate edits. Position-based requests -read the live buffer first, so unsaved edits can drive token lookup without -writing back to the CodeIndex database. +update without retaining unbounded intermediate edits. A full-text change whose +numeric document version is older than or equal to the latest accepted version +does not replace that live buffer. Position-based requests read the live buffer +first, so unsaved edits can drive token lookup without writing back to the +CodeIndex database. `textDocument/documentSymbol` also re-extracts the latest +accepted live text through the normal language extractor and container pipeline; +when no live buffer is available, it falls back to indexed symbols. Incoming `textDocument.uri` values must be strings, must be absolute `file:` URIs, and are rejected before URI parsing when they exceed 4096 characters, matching the MCP resource URI limit and keeping error responses bounded. LSP @@ -2745,9 +2749,9 @@ strings are capped at 1000 characters before symbol search runs. `workspace/symbol` accepts optional numeric `limit` / `maxResults` parameters and clamps them to 1000 results. `textDocument/documentSymbol` returns hierarchical `DocumentSymbol` children when container metadata is available, -returns at most 1000 indexed symbols, truncates each `detail` string to 512 -characters with `...`, and trims the tree before the result array exceeds -524288 JSON bytes. +returns at most 1000 symbols from the latest accepted live buffer or the index, +truncates each `detail` string to 512 characters with `...`, and trims the tree +before the result array exceeds 524288 JSON bytes. Both symbol providers advertise work-done progress. Requests may pass bounded string or integer `partialResultToken` / `workDoneToken` values. With a partial result token, the server sends deterministic `$/progress` notifications capped @@ -5828,8 +5832,11 @@ open buffer は上限付きの in-memory cache に保持されます。各 docum session 全体では最大 64 live documents / 16777216 aggregate live-document bytes に制限され、 budget を超えた場合は古い entry から evict されます。`textDocument/didChange` は過大な `contentChanges` array では最後の 64 change entries だけを処理し、unbounded な intermediate edit を -保持せずに最新の full-text update を維持します。position-based request は live buffer を先に読むため、 -未保存の編集内容でも CodeIndex database に書き戻さず token lookup に利用できます。 +保持せずに最新の full-text update を維持します。numeric document version が最後に受理した version +以下の full-text change は、その live buffer を置き換えません。position-based request は live buffer +を先に読むため、未保存の編集内容でも CodeIndex database に書き戻さず token lookup に利用できます。 +`textDocument/documentSymbol` も最後に受理した live text を通常の language extractor と +container pipeline で再抽出し、live buffer がない場合は indexed symbol に fallback します。 受信した `textDocument.uri` は string かつ absolute `file:` URI である必要があり、 4096 文字を超える場合は URI parse の前に拒否されます。これは MCP resource URI の上限と 揃えており、エラー応答が過大にならないようにします。 @@ -5849,8 +5856,9 @@ invalid request として拒否します。 `workspace/symbol` の query string は symbol search を実行する前に 1000 文字で上限をかけます。 `workspace/symbol` は任意の numeric `limit` / `maxResults` parameter を受け取り、1000 件までに clamp します。`textDocument/documentSymbol` は container metadata がある場合に階層化された -`DocumentSymbol` children を返し、最大 1000 件の indexed symbol を返し、各 `detail` string を -`...` 付きの 512 文字に切り詰め、result tree が 524288 JSON bytes を超える前に trim します。 +`DocumentSymbol` children を返し、最後に受理した live buffer または index から最大 1000 件の +symbol を返し、各 `detail` string を `...` 付きの 512 文字に切り詰め、result tree が +524288 JSON bytes を超える前に trim します。 両方の symbol provider は work-done progress を advertise します。request は上限付きの string または integer の `partialResultToken` / `workDoneToken` を渡せます。partial-result token が ある場合、server は決定的な順序の `$/progress` notification を1件あたり最大100 symbol・ diff --git a/changelog.d/unreleased/4851.fixed.md b/changelog.d/unreleased/4851.fixed.md new file mode 100644 index 0000000000..0ab1bcf61f --- /dev/null +++ b/changelog.d/unreleased/4851.fixed.md @@ -0,0 +1,21 @@ +--- +category: fixed +issues: + - 4851 +affected: + - src/CodeIndex/Lsp/LspLiveDocumentStore.cs + - src/CodeIndex/Lsp/LspServer.cs + - src/CodeIndex/Lsp/LspServer.SymbolRequests.cs + - tests/CodeIndex.Tests/LspServerTests.cs + - README.md + - USER_GUIDE.md + - DEVELOPER_GUIDE.md +--- + +## English + +- **Preserved live LSP document-symbol hierarchy after full document changes (#4851)** — `textDocument/documentSymbol` now re-extracts the latest accepted live buffer with the same container pipeline used for indexed symbols, preserving the hierarchy and ranges fixed in #4736 across successive full-text changes. Stale or duplicate numeric document versions no longer overwrite newer live text. + +## 日本語 + +- **document 全体の変更後も live LSP document-symbol hierarchy を維持しました（#4851）** — `textDocument/documentSymbol` は最後に受理した live buffer を indexed symbol と同じ container pipeline で再抽出するようになり、連続する full-text change の後も #4736 で修正した hierarchy と range を維持します。古い、または重複した numeric document version は新しい live text を上書きしません。 diff --git a/src/CodeIndex/Lsp/LspLiveDocumentStore.cs b/src/CodeIndex/Lsp/LspLiveDocumentStore.cs index e475e6cef2..36c0b2e89c 100644 --- a/src/CodeIndex/Lsp/LspLiveDocumentStore.cs +++ b/src/CodeIndex/Lsp/LspLiveDocumentStore.cs @@ -6,6 +6,7 @@ internal sealed class LspLiveDocumentStore { private readonly Dictionary _documents; private readonly Dictionary _documentByteCounts; + private readonly Dictionary _documentVersions; private readonly List _documentOrder = []; private readonly StringComparison _keyComparison; private readonly int _maxDocuments; @@ -19,6 +20,7 @@ internal LspLiveDocumentStore(StringComparer comparer, StringComparison keyCompa { _documents = new Dictionary(comparer); _documentByteCounts = new Dictionary(comparer); + _documentVersions = new Dictionary(comparer); _keyComparison = keyComparison; _maxDocuments = maxDocuments; _maxDocumentBytes = maxDocumentBytes; @@ -31,13 +33,21 @@ internal LspLiveDocumentStore(StringComparer comparer, StringComparison keyCompa internal long EvictedBytes => _evictedBytes; - internal void SetText(string key, string text) + internal bool SetText(string key, string text, int? version = null) { + if (version.HasValue + && _documentVersions.TryGetValue(key, out var previousVersion) + && previousVersion.HasValue + && version.Value <= previousVersion.Value) + { + return false; + } + var textBytes = Encoding.UTF8.GetByteCount(text); if (textBytes > _maxDocumentBytes || textBytes > _maxLiveBytes) { Remove(key); - return; + return false; } if (!_documents.ContainsKey(key)) @@ -47,8 +57,11 @@ internal void SetText(string key, string text) _documents[key] = text; _documentByteCounts[key] = textBytes; + _documentVersions[key] = version + ?? (_documentVersions.TryGetValue(key, out var currentVersion) ? currentVersion : null); _documentBytes += textBytes; EnsureCapacity(); + return _documents.ContainsKey(key); } internal bool TryGetText(string key, out string text) => _documents.TryGetValue(key, out text!); @@ -66,6 +79,7 @@ internal void Remove(string key, bool recordEviction = false) } _documents.Remove(key); + _documentVersions.Remove(key); _documentOrder.RemoveAll(existing => string.Equals(existing, key, _keyComparison)); } @@ -82,6 +96,7 @@ private void EnsureCapacity() { _documents.Clear(); _documentByteCounts.Clear(); + _documentVersions.Clear(); _documentOrder.Clear(); _documentBytes = 0; } diff --git a/src/CodeIndex/Lsp/LspServer.SymbolRequests.cs b/src/CodeIndex/Lsp/LspServer.SymbolRequests.cs index 0aa8e57ca9..63445428d2 100644 --- a/src/CodeIndex/Lsp/LspServer.SymbolRequests.cs +++ b/src/CodeIndex/Lsp/LspServer.SymbolRequests.cs @@ -8,6 +8,7 @@ using CodeIndex.Cli; using CodeIndex.Database; using CodeIndex.Diagnostics; +using CodeIndex.Indexer; using CodeIndex.Mcp; using CodeIndex.Models; using CodeIndex.Security; @@ -183,7 +184,7 @@ private SymbolResponse CreateDocumentSymbolResponse( if (!TryResolveIndexedDocument(root, out var document)) return new SymbolResponse([], [], 0, false); - var candidates = _reader.SearchSymbols((string?)null, MaxDocumentSymbolMaterialization + 1, pathPatterns: [document.IndexedPath]); + var candidates = GetDocumentSymbolCandidates(document, cancellationToken); var materializationTruncated = candidates.Count > MaxDocumentSymbolMaterialization; var materializedCount = Math.Min(candidates.Count, MaxDocumentSymbolMaterialization); Activity.Current?.SetTag("lsp.document_symbols.materialized_count", materializedCount); @@ -217,6 +218,58 @@ private SymbolResponse CreateDocumentSymbolResponse( materializationTruncated || tree.RemovedCount > 0); } + private IReadOnlyList GetDocumentSymbolCandidates( + IndexedDocumentContext document, + CancellationToken cancellationToken) + { + if (!_liveDocumentStore.TryGetText(document.ResolvedPath, out var liveText)) + { + return _reader.SearchSymbols( + (string?)null, + MaxDocumentSymbolMaterialization + 1, + pathPatterns: [document.IndexedPath]); + } + + var language = FileIndexer.DetectLanguage(document.ResolvedPath); + if (language == null) + { + return _reader.SearchSymbols( + (string?)null, + MaxDocumentSymbolMaterialization + 1, + pathPatterns: [document.IndexedPath]); + } + + return SymbolExtractor.Extract( + 0, + language, + liveText, + document.ResolvedPath, + _projectRoot ?? document.WorkspaceRoot, + cancellationToken) + .Take(MaxDocumentSymbolMaterialization + 1) + .Select(symbol => new SymbolResult + { + Path = document.IndexedPath, + Lang = language, + Kind = symbol.Kind, + SubKind = symbol.SubKind, + Name = symbol.Name, + Line = symbol.Line, + StartLine = symbol.StartLine, + StartColumn = symbol.StartColumn, + EndLine = symbol.EndLine, + BodyStartLine = symbol.BodyStartLine, + BodyEndLine = symbol.BodyEndLine, + Signature = symbol.Signature, + ContainerKind = symbol.ContainerKind, + ContainerName = symbol.ContainerName, + ContainerQualifiedName = symbol.ContainerQualifiedName, + Visibility = symbol.Visibility, + ReturnType = symbol.ReturnType, + }) + .ToList(); + } + private IEnumerable EnumerateDocumentSymbolItems( IndexedDocumentContext document, IReadOnlyList symbols, diff --git a/src/CodeIndex/Lsp/LspServer.cs b/src/CodeIndex/Lsp/LspServer.cs index f6954264c6..6509d8a26f 100644 --- a/src/CodeIndex/Lsp/LspServer.cs +++ b/src/CodeIndex/Lsp/LspServer.cs @@ -771,7 +771,7 @@ private JsonObject HandleInitialize(JsonNode? id, JsonElement root) { var uri = GetTextDocumentUri(root); if (TryGet(root, out var textElement, "params", "textDocument", "text") && textElement.ValueKind == JsonValueKind.String) - SetLiveDocumentText(uri, textElement.GetString() ?? string.Empty); + SetLiveDocumentText(uri, textElement.GetString() ?? string.Empty, GetTextDocumentVersion(root)); return null; } @@ -803,7 +803,7 @@ private JsonObject HandleInitialize(JsonNode? id, JsonElement root) } if (latestText != null) - SetLiveDocumentText(uri, latestText); + SetLiveDocumentText(uri, latestText, GetTextDocumentVersion(root)); return null; } @@ -815,16 +815,22 @@ private JsonObject HandleInitialize(JsonNode? id, JsonElement root) return null; } - private void SetLiveDocumentText(string uri, string text) + private void SetLiveDocumentText(string uri, string text, int? version) { if (!TryGetLiveDocumentKeyFromUri(uri, out var key)) return; - _liveDocumentStore.SetText(key, text); + _liveDocumentStore.SetText(key, text, version); Activity.Current?.SetTag("lsp.live_documents.bytes", _liveDocumentStore.Bytes); Activity.Current?.SetTag("lsp.live_documents.eviction_count", _liveDocumentStore.EvictionCount); } + private static int? GetTextDocumentVersion(JsonElement root) => + TryGet(root, out var versionElement, "params", "textDocument", "version") + && versionElement.TryGetInt32(out var version) + ? version + : null; + private bool TryGetLiveDocumentKeyFromUri(string uri, out string key) { key = string.Empty; diff --git a/tests/CodeIndex.Tests/LspServerTests.cs b/tests/CodeIndex.Tests/LspServerTests.cs index 945a156ed1..a9fecc7828 100644 --- a/tests/CodeIndex.Tests/LspServerTests.cs +++ b/tests/CodeIndex.Tests/LspServerTests.cs @@ -1995,6 +1995,123 @@ public void HandleMessage_DocumentSymbol_NestsMixedRecordMembersDeterministicall } } + [Fact] + public void HandleMessage_DocumentSymbol_PreservesLiveHierarchyAcrossFullChanges_Issue4851() + { + var projectRoot = TestProjectHelper.CreateTempProject("cdidx_lsp_document_symbol_live_hierarchy"); + try + { + var dbPath = TestProjectHelper.CreateProjectDb(projectRoot); + var initialSource = string.Join( + "\r\n", + "namespace Ω", + "{", + " public sealed class Outer", + " {", + " public readonly record struct Token(int Line, int 長さ)", + " {", + " public int Body => 長さ;", + " public sealed class Inner { }", + " }", + " }", + "}", + string.Empty); + var shiftedLfSource = string.Join( + "\n", + "// full-change line shift", + string.Empty, + "namespace Ω", + "{", + " public sealed class Outer", + " {", + " public readonly record struct Token(int Line, int 長さ)", + " {", + " public int Body => 長さ;", + " public sealed class Inner { }", + " }", + " }", + "}", + string.Empty); + var newerCrLfSource = string.Join( + "\r\n", + "// newer full change", + "namespace Ω", + "{", + " public sealed class Outer", + " {", + " public readonly record struct Token(int Line, int 長さ)", + " {", + " public int Body => 長さ;", + " public string 追加 => \"値\";", + " public sealed class Inner { }", + " }", + " }", + "}", + string.Empty); + var sourcePath = TestProjectHelper.WriteTextFile(projectRoot, "app.cs", initialSource); + TestProjectHelper.InsertIndexedFile(dbPath, "app.cs", "csharp", initialSource); + using var db = new DbContext(DbOpenIntent.WriteIndex, dbPath); + using var server = new LspServer(new DbReader(db), "1.2.3", ProgramRunner.CreateDefaultJsonOptions(), projectRoot); + + var indexedResponse = server.HandleMessage(CreateTextDocumentRequest("textDocument/documentSymbol", sourcePath, 48509)); + Assert.NotNull(indexedResponse); + var indexedRoots = indexedResponse!["result"]!.AsArray(); + var indexedStructure = GetDocumentSymbolStructure(indexedRoots); + var indexedRecursiveCount = FlattenDocumentSymbols(indexedRoots).Count(); + + Assert.Null(server.HandleMessage(CreateDidOpenRequest(sourcePath, initialSource, version: 1))); + var initialResponse = server.HandleMessage(CreateTextDocumentRequest("textDocument/documentSymbol", sourcePath, 48510)); + Assert.NotNull(initialResponse); + var initialRoots = initialResponse!["result"]!.AsArray(); + var initialStructure = GetDocumentSymbolStructure(initialRoots); + var initialRecursiveCount = FlattenDocumentSymbols(initialRoots).Count(); + Assert.Equal(indexedRoots.Count, initialRoots.Count); + Assert.Equal(indexedRecursiveCount, initialRecursiveCount); + Assert.Equal(indexedStructure, initialStructure); + var initialToken = Assert.Single( + FlattenDocumentSymbols(initialRoots) + .Where(symbol => symbol?["name"]?.GetValue() == "Token")); + var initialTokenLine = initialToken!["selectionRange"]!["start"]!["line"]!.GetValue(); + var initialTokenChildren = initialToken["children"]!.AsArray(); + Assert.Contains(initialTokenChildren, child => child?["name"]?.GetValue() == "Line"); + Assert.Contains(initialTokenChildren, child => child?["name"]?.GetValue() == "長さ"); + + Assert.Null(server.HandleMessage(CreateDidChangeRequest(sourcePath, shiftedLfSource, version: 2))); + var shiftedResponse = server.HandleMessage(CreateTextDocumentRequest("textDocument/documentSymbol", sourcePath, 48511)); + Assert.NotNull(shiftedResponse); + var shiftedRoots = shiftedResponse!["result"]!.AsArray(); + Assert.Equal(initialRoots.Count, shiftedRoots.Count); + Assert.Equal(initialRecursiveCount, FlattenDocumentSymbols(shiftedRoots).Count()); + Assert.Equal(initialStructure, GetDocumentSymbolStructure(shiftedRoots)); + var shiftedToken = Assert.Single( + FlattenDocumentSymbols(shiftedRoots) + .Where(symbol => symbol?["name"]?.GetValue() == "Token")); + Assert.Equal( + initialTokenLine + 2, + shiftedToken!["selectionRange"]!["start"]!["line"]!.GetValue()); + + Assert.Null(server.HandleMessage(CreateDidChangeRequest(sourcePath, newerCrLfSource, version: 3))); + var newerResponse = server.HandleMessage(CreateTextDocumentRequest("textDocument/documentSymbol", sourcePath, 48512)); + Assert.NotNull(newerResponse); + var newerRoots = newerResponse!["result"]!.AsArray(); + var newerStructure = GetDocumentSymbolStructure(newerRoots); + Assert.Equal(initialRoots.Count, newerRoots.Count); + Assert.Equal(initialRecursiveCount + 1, FlattenDocumentSymbols(newerRoots).Count()); + Assert.Contains( + FlattenDocumentSymbols(newerRoots), + symbol => symbol?["name"]?.GetValue() == "追加"); + + Assert.Null(server.HandleMessage(CreateDidChangeRequest(sourcePath, "class Stale { }\n", version: 2))); + var staleResponse = server.HandleMessage(CreateTextDocumentRequest("textDocument/documentSymbol", sourcePath, 48513)); + Assert.NotNull(staleResponse); + Assert.Equal(newerStructure, GetDocumentSymbolStructure(staleResponse!["result"]!.AsArray())); + } + finally + { + TestProjectHelper.DeleteDirectory(projectRoot); + } + } + [Fact] public void HandleMessage_DocumentSymbol_NestsSameStartLongerContainerBeforeChild_Issue3537() { @@ -3554,6 +3671,29 @@ private static void AssertSemanticToken( } } + private static IReadOnlyList GetDocumentSymbolStructure(JsonArray symbols) + { + var structure = new List(); + AddDocumentSymbolStructure(symbols, string.Empty, structure); + return structure; + } + + private static void AddDocumentSymbolStructure( + JsonArray symbols, + string parentIdentity, + List structure) + { + foreach (var symbol in symbols) + { + var name = symbol!["name"]!.GetValue(); + var kind = symbol["kind"]!.GetValue(); + var identity = $"{parentIdentity}/{name}:{kind}"; + structure.Add(identity); + if (symbol["children"] is JsonArray children) + AddDocumentSymbolStructure(children, identity, structure); + } + } + private static object? GetActivityTag(Activity activity, string key) => activity.TagObjects.FirstOrDefault(tag => tag.Key == key).Value; From a0838dcd90c236f274dd7ef7008f42c46170c68d Mon Sep 17 00:00:00 2001 From: Widthdom Date: Tue, 28 Jul 2026 10:33:18 +0900 Subject: [PATCH 2/3] Use indexed language for live LSP symbols (#4851) --- DEVELOPER_GUIDE.md | 12 ++++--- changelog.d/unreleased/4851.fixed.md | 4 +-- src/CodeIndex/Lsp/LspServer.SymbolRequests.cs | 4 +-- tests/CodeIndex.Tests/LspServerTests.cs | 31 +++++++++++++++++++ 4 files changed, 42 insertions(+), 9 deletions(-) diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index 822b4f5b59..f4612db370 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -430,8 +430,9 @@ must read that live cache before disk so unsaved editor buffers can identify the requested token. Provider results remain conservative and index-backed except that document symbols for an indexed document may be structurally re-extracted from the bounded live buffer through the normal language extractor and container -pipeline. Other providers return empty arrays or null when the database cannot -answer safely instead of inventing language-server analysis. +pipeline, using the indexed file's authoritative language rather than +re-detecting it from the path. Other providers return empty arrays or null when +the database cannot answer safely instead of inventing language-server analysis. Disk-backed position-line caching must enforce its 4 MiB input limit while streaming, not only through a pre-read `Length` check. Bytes beyond the limit must never reach text decoding, including when a shared file grows concurrently, @@ -3632,9 +3633,10 @@ editor integration は標準的な location 形状を直接要求できる。`de 上限付きの in-memory cache にだけ保持する。position-based provider は未保存 editor buffer から request token を特定できるよう disk より先に live cache を読む必要がある。provider result は 保守的かつ index-backed のままとするが、indexed document の document symbol だけは上限付きの -live buffer を通常の language extractor と container pipeline で構造的に再抽出できる。それ以外の -provider は database が安全に答えられない場合、language-server analysis を作り上げず、空配列 -または null を返す。 +live buffer を通常の language extractor と container pipeline で構造的に再抽出できる。このとき +path から再判定せず、indexed file の authoritative language を使う。それ以外の provider は +database が安全に答えられない場合、language-server analysis を作り上げず、空配列または null を +返す。 disk 上の position-line cache は、事前の `Length` check だけでなく streaming 中も 4 MiB の input 上限を強制する必要がある。共有 file が同時に増大する場合も上限超過 byte を text decode に 渡してはならず、bounded な failure reason は `position_file_too_large` のままとする。 diff --git a/changelog.d/unreleased/4851.fixed.md b/changelog.d/unreleased/4851.fixed.md index 0ab1bcf61f..bd51242831 100644 --- a/changelog.d/unreleased/4851.fixed.md +++ b/changelog.d/unreleased/4851.fixed.md @@ -14,8 +14,8 @@ affected: ## English -- **Preserved live LSP document-symbol hierarchy after full document changes (#4851)** — `textDocument/documentSymbol` now re-extracts the latest accepted live buffer with the same container pipeline used for indexed symbols, preserving the hierarchy and ranges fixed in #4736 across successive full-text changes. Stale or duplicate numeric document versions no longer overwrite newer live text. +- **Preserved live LSP document-symbol hierarchy after full document changes (#4851)** — `textDocument/documentSymbol` now re-extracts the latest accepted live buffer with the indexed file's authoritative language and the same container pipeline used for indexed symbols, preserving the hierarchy and ranges fixed in #4736 across successive full-text changes. Stale or duplicate numeric document versions no longer overwrite newer live text. ## 日本語 -- **document 全体の変更後も live LSP document-symbol hierarchy を維持しました（#4851）** — `textDocument/documentSymbol` は最後に受理した live buffer を indexed symbol と同じ container pipeline で再抽出するようになり、連続する full-text change の後も #4736 で修正した hierarchy と range を維持します。古い、または重複した numeric document version は新しい live text を上書きしません。 +- **document 全体の変更後も live LSP document-symbol hierarchy を維持しました（#4851）** — `textDocument/documentSymbol` は indexed file の authoritative language と indexed symbol と同じ container pipeline を使って、最後に受理した live buffer を再抽出するようになり、連続する full-text change の後も #4736 で修正した hierarchy と range を維持します。古い、または重複した numeric document version は新しい live text を上書きしません。 diff --git a/src/CodeIndex/Lsp/LspServer.SymbolRequests.cs b/src/CodeIndex/Lsp/LspServer.SymbolRequests.cs index 63445428d2..ea4e8b54c9 100644 --- a/src/CodeIndex/Lsp/LspServer.SymbolRequests.cs +++ b/src/CodeIndex/Lsp/LspServer.SymbolRequests.cs @@ -230,8 +230,8 @@ private IReadOnlyList GetDocumentSymbolCandidates( pathPatterns: [document.IndexedPath]); } - var language = FileIndexer.DetectLanguage(document.ResolvedPath); - if (language == null) + var language = _reader.GetFileByPath(document.IndexedPath)?.Lang; + if (string.IsNullOrWhiteSpace(language)) { return _reader.SearchSymbols( (string?)null, diff --git a/tests/CodeIndex.Tests/LspServerTests.cs b/tests/CodeIndex.Tests/LspServerTests.cs index a9fecc7828..e9e991ab7d 100644 --- a/tests/CodeIndex.Tests/LspServerTests.cs +++ b/tests/CodeIndex.Tests/LspServerTests.cs @@ -2112,6 +2112,37 @@ public void HandleMessage_DocumentSymbol_PreservesLiveHierarchyAcrossFullChanges } } + [Fact] + public void HandleMessage_DocumentSymbol_UsesIndexedLanguageForLiveContentSensitiveExtension_Issue4851() + { + var projectRoot = TestProjectHelper.CreateTempProject("cdidx_lsp_document_symbol_live_indexed_language"); + try + { + var dbPath = TestProjectHelper.CreateProjectDb(projectRoot); + const string indexedSource = "class IndexedType {\npublic:\n void indexed();\n};\n"; + const string liveSource = "class LiveType {\npublic:\n void live();\n};\n"; + var sourcePath = TestProjectHelper.WriteTextFile(projectRoot, "sample.h", indexedSource); + TestProjectHelper.InsertIndexedFile(dbPath, "sample.h", "cpp", indexedSource); + using var db = new DbContext(DbOpenIntent.WriteIndex, dbPath); + using var server = new LspServer(new DbReader(db), "1.2.3", ProgramRunner.CreateDefaultJsonOptions(), projectRoot); + + Assert.Null(server.HandleMessage(CreateDidOpenRequest(sourcePath, indexedSource, version: 1))); + Assert.Null(server.HandleMessage(CreateDidChangeRequest(sourcePath, liveSource, version: 2))); + + var response = server.HandleMessage(CreateTextDocumentRequest("textDocument/documentSymbol", sourcePath, 48514)); + + Assert.NotNull(response); + var liveType = Assert.Single(response!["result"]!.AsArray()); + Assert.Equal("LiveType", liveType!["name"]!.GetValue()); + var liveMethod = Assert.Single(liveType["children"]!.AsArray()); + Assert.Equal("live", liveMethod!["name"]!.GetValue()); + } + finally + { + TestProjectHelper.DeleteDirectory(projectRoot); + } + } + [Fact] public void HandleMessage_DocumentSymbol_NestsSameStartLongerContainerBeforeChild_Issue3537() { From 4a5311ea12ae435ac61e19a22335bb817a3a6d14 Mon Sep 17 00:00:00 2001 From: Widthdom Date: Tue, 28 Jul 2026 12:20:00 +0900 Subject: [PATCH 3/3] Bound live LSP symbol extraction (#4851) --- DEVELOPER_GUIDE.md | 18 +++-- changelog.d/unreleased/4851.fixed.md | 4 +- .../Symbols/SymbolExtractor.ExtractCore.cs | 41 +++++++----- .../Indexer/Symbols/SymbolExtractor.State.cs | 12 +++- .../Indexer/Symbols/SymbolExtractor.cs | 41 ++++++++++++ src/CodeIndex/Lsp/LspLiveDocumentStore.cs | 58 +++++++++++++---- src/CodeIndex/Lsp/LspServer.SymbolRequests.cs | 20 +++--- tests/CodeIndex.Tests/LspProtocolTests.cs | 38 +++++++++++ tests/CodeIndex.Tests/LspServerTests.cs | 65 +++++++++++++++++++ 9 files changed, 250 insertions(+), 47 deletions(-) diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index f4612db370..ceeb8cc404 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -431,8 +431,13 @@ requested token. Provider results remain conservative and index-backed except that document symbols for an indexed document may be structurally re-extracted from the bounded live buffer through the normal language extractor and container pipeline, using the indexed file's authoritative language rather than -re-detecting it from the path. Other providers return empty arrays or null when -the database cannot answer safely instead of inventing language-server analysis. +re-detecting it from the path. Live extraction stops at the document-symbol +materialization bound and falls back to indexed symbols when that bounded +extractor is unavailable. Numeric document-version tombstones remain bounded +across live-text eviction and are cleared by `didClose`, so an evicted newer +version cannot be replaced by a stale change. Other providers return empty +arrays or null when the database cannot answer safely instead of inventing +language-server analysis. Disk-backed position-line caching must enforce its 4 MiB input limit while streaming, not only through a pre-read `Length` check. Bytes beyond the limit must never reach text decoding, including when a shared file grows concurrently, @@ -3634,9 +3639,12 @@ editor integration は標準的な location 形状を直接要求できる。`de request token を特定できるよう disk より先に live cache を読む必要がある。provider result は 保守的かつ index-backed のままとするが、indexed document の document symbol だけは上限付きの live buffer を通常の language extractor と container pipeline で構造的に再抽出できる。このとき -path から再判定せず、indexed file の authoritative language を使う。それ以外の provider は -database が安全に答えられない場合、language-server analysis を作り上げず、空配列または null を -返す。 +path から再判定せず、indexed file の authoritative language を使う。live extraction は +document-symbol materialization 上限で停止し、その bounded extractor を利用できない場合は +indexed symbol に fallback する。numeric document-version tombstone は live text の eviction +後も上限付きで保持し、`didClose` で消去するため、evict 済みの新しい version を stale change が +置き換えることはない。それ以外の provider は database が安全に答えられない場合、 +language-server analysis を作り上げず、空配列または null を返す。 disk 上の position-line cache は、事前の `Length` check だけでなく streaming 中も 4 MiB の input 上限を強制する必要がある。共有 file が同時に増大する場合も上限超過 byte を text decode に 渡してはならず、bounded な failure reason は `position_file_too_large` のままとする。 diff --git a/changelog.d/unreleased/4851.fixed.md b/changelog.d/unreleased/4851.fixed.md index bd51242831..e5d54ebc67 100644 --- a/changelog.d/unreleased/4851.fixed.md +++ b/changelog.d/unreleased/4851.fixed.md @@ -14,8 +14,8 @@ affected: ## English -- **Preserved live LSP document-symbol hierarchy after full document changes (#4851)** — `textDocument/documentSymbol` now re-extracts the latest accepted live buffer with the indexed file's authoritative language and the same container pipeline used for indexed symbols, preserving the hierarchy and ranges fixed in #4736 across successive full-text changes. Stale or duplicate numeric document versions no longer overwrite newer live text. +- **Preserved live LSP document-symbol hierarchy after full document changes (#4851)** — `textDocument/documentSymbol` now re-extracts the latest accepted live buffer with the indexed file's authoritative language and the same container pipeline used for indexed symbols, preserving the hierarchy and ranges fixed in #4736 across successive full-text changes. Live extraction stops at the document-symbol materialization bound and safely falls back to indexed symbols when the extractor is unavailable. Bounded version tombstones prevent stale or duplicate numeric document versions from overwriting newer live text after cache eviction. ## 日本語 -- **document 全体の変更後も live LSP document-symbol hierarchy を維持しました（#4851）** — `textDocument/documentSymbol` は indexed file の authoritative language と indexed symbol と同じ container pipeline を使って、最後に受理した live buffer を再抽出するようになり、連続する full-text change の後も #4736 で修正した hierarchy と range を維持します。古い、または重複した numeric document version は新しい live text を上書きしません。 +- **document 全体の変更後も live LSP document-symbol hierarchy を維持しました（#4851）** — `textDocument/documentSymbol` は indexed file の authoritative language と indexed symbol と同じ container pipeline を使って、最後に受理した live buffer を再抽出するようになり、連続する full-text change の後も #4736 で修正した hierarchy と range を維持します。live extraction は document-symbol materialization 上限で停止し、extractor を利用できない場合は indexed symbol に安全に fallback します。上限付き version tombstone により、cache eviction 後も古い、または重複した numeric document version が新しい live text を上書きしません。 diff --git a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.ExtractCore.cs b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.ExtractCore.cs index 7921e911eb..cf545e537e 100644 --- a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.ExtractCore.cs +++ b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.ExtractCore.cs @@ -19,7 +19,8 @@ private static List ExtractCore( string? filePath = null, string? projectRoot = null, bool patternConfigsAlreadyLoaded = false, - CancellationToken cancellationToken = default) + CancellationToken cancellationToken = default, + int? maxSymbols = null) { var originalLang = lang; if (TryPrepareSymbolExtraction( @@ -124,7 +125,7 @@ private static List ExtractCore( var getCSharpSwitchExpressionLines = scanInputs.GetCSharpSwitchExpressionLines; var getCssQualifiedRuleAncestors = scanInputs.GetCssQualifiedRuleAncestors; var initialSymbolCapacity = EstimateSymbolListInitialCapacity(lines.Length); - var symbols = new SymbolExtractionList(initialSymbolCapacity); + var symbols = new SymbolExtractionList(initialSymbolCapacity, maxSymbols); var extractionState = symbols.ExtractionState; var scanState = new PatternScanState(); List? pendingRecordPrimaryComponents = null; @@ -137,6 +138,9 @@ private static List ExtractCore( : null; for (int i = 0; i < lines.Length; i++) { + if (symbols.IsAtCapacity) + break; + if ((i & 0x3f) == 0) cancellationToken.ThrowIfCancellationRequested(); @@ -1238,21 +1242,24 @@ private static List ExtractCore( } } - AddSupplementalSymbols( - fileId, - originalLang, - lang, - content, - filePath, - lines, - structuralLines, - symbols, - extractionState, - getPrivateScopeColumns, - GetJavaScriptTypeScriptSanitizedLines, - csharpMatchLines, - pythonModulePrefix, - prologMultilineHeads); + if (!symbols.IsAtCapacity) + { + AddSupplementalSymbols( + fileId, + originalLang, + lang, + content, + filePath, + lines, + structuralLines, + symbols, + extractionState, + getPrivateScopeColumns, + GetJavaScriptTypeScriptSanitizedLines, + csharpMatchLines, + pythonModulePrefix, + prologMultilineHeads); + } FinalizePatternSymbols( fileId, lang, diff --git a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.State.cs b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.State.cs index 850ca961c9..f9797903b9 100644 --- a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.State.cs +++ b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.State.cs @@ -47,13 +47,19 @@ public void Remove(SymbolRecord symbol) => private sealed class SymbolExtractionList : List { - public SymbolExtractionList(int initialCapacity) - : base(initialCapacity) + private readonly int? _maxSymbols; + + public SymbolExtractionList(int initialCapacity, int? maxSymbols = null) + : base(maxSymbols.HasValue ? Math.Min(initialCapacity, maxSymbols.Value) : initialCapacity) { - ExtractionState = new SymbolExtractionState(initialCapacity); + _maxSymbols = maxSymbols; + ExtractionState = new SymbolExtractionState( + maxSymbols.HasValue ? Math.Min(initialCapacity, maxSymbols.Value) : initialCapacity); } public SymbolExtractionState ExtractionState { get; } + + public bool IsAtCapacity => _maxSymbols.HasValue && Count >= _maxSymbols.Value; } private sealed class SymbolAddState diff --git a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.cs b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.cs index c6b9e84ef7..a2e6902e92 100644 --- a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.cs +++ b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.cs @@ -340,6 +340,44 @@ public static List Extract(long fileId, string? lang, string conte patternConfigsAlreadyLoaded: false, cancellationToken: cancellationToken); + internal static bool TryExtractBounded( + long fileId, + string? lang, + string content, + int maxSymbols, + string? filePath, + string? projectRoot, + CancellationToken cancellationToken, + out List symbols) + { + if (maxSymbols <= 0) + throw new ArgumentOutOfRangeException(nameof(maxSymbols)); + + ExtractorPluginRegistry.LoadPatternConfigsForProjectRoot(projectRoot); + var normalizedLanguage = NormalizeLanguage(lang); + if (normalizedLanguage == null + || normalizedLanguage is "commonlisp" or "racket" or "solidity" or "html" or "assembly" + || !PatternCache.ContainsKey(normalizedLanguage)) + { + symbols = []; + return false; + } + + symbols = ExtractCore( + fileId, + lang, + content, + contentIsNormalized: false, + hasOversizeLine: null, + conflictMarkerLine: null, + filePath, + projectRoot, + patternConfigsAlreadyLoaded: true, + cancellationToken: cancellationToken, + maxSymbols: maxSymbols); + return true; + } + internal static List ExtractWithPatternConfigsLoaded( long fileId, string? lang, @@ -782,6 +820,9 @@ private static void AddSymbolRecord( SymbolRecord symbol, string? rawLine = null) { + if (symbols is SymbolExtractionList extractionSymbols && extractionSymbols.IsAtCapacity) + return; + if (string.IsNullOrWhiteSpace(symbol.Name)) return; diff --git a/src/CodeIndex/Lsp/LspLiveDocumentStore.cs b/src/CodeIndex/Lsp/LspLiveDocumentStore.cs index 36c0b2e89c..ba29e9d25f 100644 --- a/src/CodeIndex/Lsp/LspLiveDocumentStore.cs +++ b/src/CodeIndex/Lsp/LspLiveDocumentStore.cs @@ -6,8 +6,9 @@ internal sealed class LspLiveDocumentStore { private readonly Dictionary _documents; private readonly Dictionary _documentByteCounts; - private readonly Dictionary _documentVersions; + private readonly Dictionary _documentVersions; private readonly List _documentOrder = []; + private readonly List _documentVersionOrder = []; private readonly StringComparison _keyComparison; private readonly int _maxDocuments; private readonly int _maxDocumentBytes; @@ -20,7 +21,7 @@ internal LspLiveDocumentStore(StringComparer comparer, StringComparison keyCompa { _documents = new Dictionary(comparer); _documentByteCounts = new Dictionary(comparer); - _documentVersions = new Dictionary(comparer); + _documentVersions = new Dictionary(comparer); _keyComparison = keyComparison; _maxDocuments = maxDocuments; _maxDocumentBytes = maxDocumentBytes; @@ -33,12 +34,14 @@ internal LspLiveDocumentStore(StringComparer comparer, StringComparison keyCompa internal long EvictedBytes => _evictedBytes; + internal int VersionTombstoneCount => + _documentVersions.Keys.Count(key => !_documents.ContainsKey(key)); + internal bool SetText(string key, string text, int? version = null) { if (version.HasValue && _documentVersions.TryGetValue(key, out var previousVersion) - && previousVersion.HasValue - && version.Value <= previousVersion.Value) + && version.Value <= previousVersion) { return false; } @@ -46,7 +49,9 @@ internal bool SetText(string key, string text, int? version = null) var textBytes = Encoding.UTF8.GetByteCount(text); if (textBytes > _maxDocumentBytes || textBytes > _maxLiveBytes) { - Remove(key); + RememberVersion(key, version); + Remove(key, preserveVersion: true); + TrimVersionTombstones(); return false; } @@ -57,16 +62,16 @@ internal bool SetText(string key, string text, int? version = null) _documents[key] = text; _documentByteCounts[key] = textBytes; - _documentVersions[key] = version - ?? (_documentVersions.TryGetValue(key, out var currentVersion) ? currentVersion : null); + RememberVersion(key, version); _documentBytes += textBytes; EnsureCapacity(); + TrimVersionTombstones(); return _documents.ContainsKey(key); } internal bool TryGetText(string key, out string text) => _documents.TryGetValue(key, out text!); - internal void Remove(string key, bool recordEviction = false) + internal void Remove(string key, bool recordEviction = false, bool preserveVersion = false) { if (_documentByteCounts.Remove(key, out var bytes)) { @@ -79,8 +84,12 @@ internal void Remove(string key, bool recordEviction = false) } _documents.Remove(key); - _documentVersions.Remove(key); _documentOrder.RemoveAll(existing => string.Equals(existing, key, _keyComparison)); + if (!preserveVersion) + { + _documentVersions.Remove(key); + _documentVersionOrder.RemoveAll(existing => string.Equals(existing, key, _keyComparison)); + } } private void EnsureCapacity() @@ -89,16 +98,43 @@ private void EnsureCapacity() && _documentOrder.Count > 0) { var oldestKey = _documentOrder[0]; - Remove(oldestKey, recordEviction: true); + Remove(oldestKey, recordEviction: true, preserveVersion: true); } if (_documents.Count > _maxDocuments || _documentBytes > _maxLiveBytes) { _documents.Clear(); _documentByteCounts.Clear(); - _documentVersions.Clear(); _documentOrder.Clear(); _documentBytes = 0; } } + + private void RememberVersion(string key, int? version) + { + if (!version.HasValue) + return; + + _documentVersions[key] = version.Value; + _documentVersionOrder.RemoveAll(existing => string.Equals(existing, key, _keyComparison)); + _documentVersionOrder.Add(key); + } + + private void TrimVersionTombstones() + { + var tombstoneCount = VersionTombstoneCount; + for (var index = 0; tombstoneCount > _maxDocuments && index < _documentVersionOrder.Count;) + { + var key = _documentVersionOrder[index]; + if (_documents.ContainsKey(key)) + { + index++; + continue; + } + + _documentVersionOrder.RemoveAt(index); + _documentVersions.Remove(key); + tombstoneCount--; + } + } } diff --git a/src/CodeIndex/Lsp/LspServer.SymbolRequests.cs b/src/CodeIndex/Lsp/LspServer.SymbolRequests.cs index ea4e8b54c9..ee1bf0b675 100644 --- a/src/CodeIndex/Lsp/LspServer.SymbolRequests.cs +++ b/src/CodeIndex/Lsp/LspServer.SymbolRequests.cs @@ -231,7 +231,16 @@ private IReadOnlyList GetDocumentSymbolCandidates( } var language = _reader.GetFileByPath(document.IndexedPath)?.Lang; - if (string.IsNullOrWhiteSpace(language)) + if (string.IsNullOrWhiteSpace(language) + || !SymbolExtractor.TryExtractBounded( + 0, + language, + liveText, + MaxDocumentSymbolMaterialization + 1, + document.ResolvedPath, + _projectRoot ?? document.WorkspaceRoot, + cancellationToken, + out var liveSymbols)) { return _reader.SearchSymbols( (string?)null, @@ -239,14 +248,7 @@ private IReadOnlyList GetDocumentSymbolCandidates( pathPatterns: [document.IndexedPath]); } - return SymbolExtractor.Extract( - 0, - language, - liveText, - document.ResolvedPath, - _projectRoot ?? document.WorkspaceRoot, - cancellationToken) - .Take(MaxDocumentSymbolMaterialization + 1) + return liveSymbols .Select(symbol => new SymbolResult { Path = document.IndexedPath, diff --git a/tests/CodeIndex.Tests/LspProtocolTests.cs b/tests/CodeIndex.Tests/LspProtocolTests.cs index a6f3c03ef9..d2823087bd 100644 --- a/tests/CodeIndex.Tests/LspProtocolTests.cs +++ b/tests/CodeIndex.Tests/LspProtocolTests.cs @@ -37,4 +37,42 @@ public void LiveDocumentStore_EvictsOldestDocumentWhenCapacityIsExceeded() Assert.Equal(1, store.EvictionCount); Assert.Equal(Encoding.UTF8.GetByteCount("first"), store.EvictedBytes); } + + [Fact] + public void LiveDocumentStore_PreservesBoundedVersionTombstonesAcrossEviction() + { + var store = new LspLiveDocumentStore( + StringComparer.Ordinal, + StringComparison.Ordinal, + maxDocuments: 1, + maxDocumentBytes: 100, + maxLiveBytes: 100); + + Assert.True(store.SetText("/first.cs", "first", version: 3)); + Assert.True(store.SetText("/second.cs", "second", version: 2)); + Assert.False(store.SetText("/first.cs", "stale", version: 2)); + Assert.Equal(1, store.VersionTombstoneCount); + + store.Remove("/first.cs"); + + Assert.True(store.SetText("/first.cs", "reopened", version: 2)); + Assert.True(store.VersionTombstoneCount <= 1); + } + + [Fact] + public void LiveDocumentStore_PreservesNewerVersionWhenOversizedTextIsDropped() + { + var store = new LspLiveDocumentStore( + StringComparer.Ordinal, + StringComparison.Ordinal, + maxDocuments: 1, + maxDocumentBytes: 5, + maxLiveBytes: 5); + + Assert.True(store.SetText("/app.cs", "small", version: 3)); + Assert.False(store.SetText("/app.cs", "oversized", version: 4)); + Assert.False(store.TryGetText("/app.cs", out _)); + Assert.False(store.SetText("/app.cs", "old", version: 3)); + Assert.Equal(1, store.VersionTombstoneCount); + } } diff --git a/tests/CodeIndex.Tests/LspServerTests.cs b/tests/CodeIndex.Tests/LspServerTests.cs index e9e991ab7d..aedc8bfac6 100644 --- a/tests/CodeIndex.Tests/LspServerTests.cs +++ b/tests/CodeIndex.Tests/LspServerTests.cs @@ -2143,6 +2143,71 @@ public void HandleMessage_DocumentSymbol_UsesIndexedLanguageForLiveContentSensit } } + [Fact] + public void HandleMessage_DocumentSymbol_BoundsDenseLiveExtraction_Issue4851() + { + var projectRoot = TestProjectHelper.CreateTempProject("cdidx_lsp_document_symbol_live_bounded"); + try + { + var dbPath = TestProjectHelper.CreateProjectDb(projectRoot); + const string indexedSource = "class IndexedType { }\n"; + var liveSource = new StringBuilder(); + for (var i = 0; i < 10_000; i++) + liveSource.Append("class Live").Append(i.ToString("D4", CultureInfo.InvariantCulture)).Append(" { }\n"); + + var sourcePath = TestProjectHelper.WriteTextFile(projectRoot, "dense.cs", indexedSource); + TestProjectHelper.InsertIndexedFile(dbPath, "dense.cs", "csharp", indexedSource); + using var db = new DbContext(DbOpenIntent.WriteIndex, dbPath); + using var server = new LspServer(new DbReader(db), "1.2.3", ProgramRunner.CreateDefaultJsonOptions(), projectRoot); + + Assert.Null(server.HandleMessage(CreateDidOpenRequest(sourcePath, liveSource.ToString(), version: 1))); + + var response = server.HandleMessage(CreateTextDocumentRequest("textDocument/documentSymbol", sourcePath, 48515)); + + Assert.NotNull(response); + var symbols = response!["result"]!.AsArray(); + Assert.Equal(LspServer.MaxDocumentSymbols, symbols.Count); + Assert.Equal("Live0000", symbols[0]!["name"]!.GetValue()); + Assert.Equal("Live0999", symbols[^1]!["name"]!.GetValue()); + } + finally + { + TestProjectHelper.DeleteDirectory(projectRoot); + } + } + + [Fact] + public void HandleMessage_DocumentSymbol_FallsBackWhenIndexedExtractorIsUnavailable_Issue4851() + { + var projectRoot = TestProjectHelper.CreateTempProject("cdidx_lsp_document_symbol_live_extractor_fallback"); + try + { + var dbPath = TestProjectHelper.CreateProjectDb(projectRoot); + const string indexedSource = "class IndexedType { }\n"; + var sourcePath = TestProjectHelper.WriteTextFile(projectRoot, "sample.custom", indexedSource); + TestProjectHelper.InsertIndexedFile(dbPath, "sample.custom", "csharp", indexedSource); + using var db = new DbContext(DbOpenIntent.WriteIndex, dbPath); + using (var command = db.Connection.CreateCommand()) + { + command.CommandText = "UPDATE files SET lang = 'unavailable_issue4851' WHERE path = 'sample.custom'"; + Assert.Equal(1, command.ExecuteNonQuery()); + } + using var server = new LspServer(new DbReader(db), "1.2.3", ProgramRunner.CreateDefaultJsonOptions(), projectRoot); + + Assert.Null(server.HandleMessage(CreateDidOpenRequest(sourcePath, indexedSource, version: 1))); + + var response = server.HandleMessage(CreateTextDocumentRequest("textDocument/documentSymbol", sourcePath, 48516)); + + Assert.NotNull(response); + var indexedType = Assert.Single(response!["result"]!.AsArray()); + Assert.Equal("IndexedType", indexedType!["name"]!.GetValue()); + } + finally + { + TestProjectHelper.DeleteDirectory(projectRoot); + } + } + [Fact] public void HandleMessage_DocumentSymbol_NestsSameStartLongerContainerBeforeChild_Issue3537() {