From 84e21a0dc11c374f9fb6ae4804844498ddd3b25b Mon Sep 17 00:00:00 2001 From: Widthdom Date: Sun, 31 May 2026 18:19:39 +0900 Subject: [PATCH 1/6] Fix Python multiline f-string masking (#1438) --- changelog.d/unreleased/1438.fixed.md | 17 ++ .../ReferenceExtractor.Preparation.cs | 3 + .../ReferenceExtractor.TypeReferences.cs | 217 +++++++++++++++++- .../ReferenceExtractorTests.cs | 24 ++ 4 files changed, 258 insertions(+), 3 deletions(-) create mode 100644 changelog.d/unreleased/1438.fixed.md diff --git a/changelog.d/unreleased/1438.fixed.md b/changelog.d/unreleased/1438.fixed.md new file mode 100644 index 0000000000..84bc4cb077 --- /dev/null +++ b/changelog.d/unreleased/1438.fixed.md @@ -0,0 +1,17 @@ +--- +category: fixed +issues: + - 1438 +affected: + - src/CodeIndex/Indexer/References/ReferenceExtractor.Preparation.cs + - src/CodeIndex/Indexer/References/ReferenceExtractor.TypeReferences.cs + - tests/CodeIndex.Tests/ReferenceExtractorTests.cs +--- + +## English + +- **Python multi-line f-strings no longer emit references from literal text (#1438)** — triple-quoted f-string bodies are masked across physical lines while interpolation expressions still contribute real reference edges. + +## 日本語 + +- **Python の複数行 f-string がリテラル本文から参照を出さなくなりました (#1438)** — 三重引用符の f-string 本文を物理行をまたいでマスクしつつ、補間式内の実参照は引き続き抽出します。 diff --git a/src/CodeIndex/Indexer/References/ReferenceExtractor.Preparation.cs b/src/CodeIndex/Indexer/References/ReferenceExtractor.Preparation.cs index 11b35f9d00..8c40ab7e19 100644 --- a/src/CodeIndex/Indexer/References/ReferenceExtractor.Preparation.cs +++ b/src/CodeIndex/Indexer/References/ReferenceExtractor.Preparation.cs @@ -89,6 +89,9 @@ private static bool TryPrepareReferenceLines( : UsesCStyleBlockComments(language) ? MaskCStyleBlockCommentLines(language, structuralLines) : structuralLines; + if (language == "python") + referenceStructuralLines = MaskPythonFStrings(referenceStructuralLines); + var preparedLines = new string[lines.Length]; for (var pi = 0; pi < lines.Length; pi++) preparedLines[pi] = PrepareLine(language, referenceStructuralLines[pi]); diff --git a/src/CodeIndex/Indexer/References/ReferenceExtractor.TypeReferences.cs b/src/CodeIndex/Indexer/References/ReferenceExtractor.TypeReferences.cs index 7c59b79428..c3ef2dc02a 100644 --- a/src/CodeIndex/Indexer/References/ReferenceExtractor.TypeReferences.cs +++ b/src/CodeIndex/Indexer/References/ReferenceExtractor.TypeReferences.cs @@ -2560,9 +2560,7 @@ private static string ReplaceRegexMatchesWithSpaces(Regex regex, string input) private static string PrepareLine(string lang, string line) { - var result = lang == "python" - ? MaskPythonSingleLineFStrings(line) - : line; + var result = line; if (lang == "rust") result = MaskRustLifetimeTokens(result); if (lang != "cobol") @@ -3188,6 +3186,162 @@ private static string MaskPythonSingleLineFStrings(string line) return new string(masked); } + private static string[] MaskPythonFStrings(IReadOnlyList lines) + { + var result = new string[lines.Count]; + for (var lineIndex = 0; lineIndex < lines.Count; lineIndex++) + result[lineIndex] = lines[lineIndex]; + + for (var lineIndex = 0; lineIndex < result.Length; lineIndex++) + { + var line = result[lineIndex]; + if (line.IndexOf('f') < 0 && line.IndexOf('F') < 0) + continue; + + var chars = line.ToCharArray(); + var changed = false; + for (var column = 0; column < line.Length; column++) + { + if (!TryOpenPythonString(line, column, out var prefixLength, out var quoteChar, out var isRaw, out var isFString, out var isTripleQuoted)) + continue; + + if (!isFString) + { + column += prefixLength; + continue; + } + + if (!isTripleQuoted) + { + result[lineIndex] = MaskPythonSingleLineFStrings(line); + chars = result[lineIndex].ToCharArray(); + changed = true; + break; + } + + MaskPythonTripleQuotedFString(result, lineIndex, column, prefixLength, quoteChar, isRaw, out var endLineIndex, out var endColumn); + lineIndex = endLineIndex; + line = result[lineIndex]; + chars = line.ToCharArray(); + column = endColumn; + changed = true; + } + + if (changed) + result[lineIndex] = new string(chars); + } + + return result; + } + + private static void MaskPythonTripleQuotedFString( + string[] lines, + int startLineIndex, + int startColumn, + int prefixLength, + char quoteChar, + bool isRaw, + out int endLineIndex, + out int endColumn) + { + var lineIndex = startLineIndex; + var column = startColumn; + var inExpression = false; + var expressionDepth = 0; + + endLineIndex = startLineIndex; + endColumn = startColumn; + + while (lineIndex < lines.Length) + { + var line = lines[lineIndex]; + var chars = line.ToCharArray(); + if (lineIndex == startLineIndex) + { + ReplaceWithSpaces(chars, startColumn, prefixLength + 3); + column = startColumn + prefixLength + 3; + } + else + { + column = 0; + } + + while (column < line.Length) + { + if (!inExpression) + { + if (!isRaw && line[column] == '\\' && column + 1 < line.Length) + { + ReplaceWithSpaces(chars, column, 2); + column += 2; + continue; + } + + if (line[column] == '{' && column + 1 < line.Length && line[column + 1] == '{') + { + ReplaceWithSpaces(chars, column, 2); + column += 2; + continue; + } + + if (line[column] == '}' && column + 1 < line.Length && line[column + 1] == '}') + { + ReplaceWithSpaces(chars, column, 2); + column += 2; + continue; + } + + if (line[column] == '{') + { + chars[column++] = ' '; + inExpression = true; + expressionDepth = 1; + continue; + } + + if (column + 2 < line.Length + && line[column] == quoteChar + && line[column + 1] == quoteChar + && line[column + 2] == quoteChar) + { + ReplaceWithSpaces(chars, column, 3); + lines[lineIndex] = new string(chars); + endLineIndex = lineIndex; + endColumn = column + 2; + return; + } + + chars[column++] = ' '; + continue; + } + + if (line[column] == '{') + { + expressionDepth++; + column++; + continue; + } + + if (line[column] == '}') + { + expressionDepth--; + chars[column++] = ' '; + if (expressionDepth == 0) + inExpression = false; + continue; + } + + column++; + } + + lines[lineIndex] = new string(chars); + lineIndex++; + } + + endLineIndex = Math.Max(startLineIndex, lines.Length - 1); + endColumn = 0; + } + private static void ReplaceWithSpaces(char[] buffer, int start, int length) { for (var i = start; i < start + length && i < buffer.Length; i++) @@ -3235,6 +3389,63 @@ private static bool TryOpenPythonSingleLineString( return true; } + private static bool TryOpenPythonString( + string line, + int startIndex, + out int prefixLength, + out char quoteChar, + out bool isRaw, + out bool isFString, + out bool isTripleQuoted) + { + isTripleQuoted = false; + if (!TryOpenPythonSingleOrTripleString(line, startIndex, out prefixLength, out quoteChar, out isRaw, out isFString, out isTripleQuoted)) + return false; + return true; + } + + private static bool TryOpenPythonSingleOrTripleString( + string line, + int startIndex, + out int prefixLength, + out char quoteChar, + out bool isRaw, + out bool isFString, + out bool isTripleQuoted) + { + prefixLength = 0; + quoteChar = '\0'; + isRaw = false; + isFString = false; + isTripleQuoted = false; + + if (startIndex < 0 || startIndex >= line.Length) + return false; + + if (startIndex > 0 && IsIdentifierChar(line[startIndex - 1])) + return false; + + var p = startIndex; + var prefixChars = 0; + while (p < line.Length && prefixChars < 2 && IsPythonStringPrefixChar(line[p])) + { + if (line[p] is 'r' or 'R') + isRaw = true; + if (line[p] is 'f' or 'F') + isFString = true; + p++; + prefixChars++; + } + + if (p >= line.Length || (line[p] != '\'' && line[p] != '"')) + return false; + + prefixLength = p - startIndex; + quoteChar = line[p]; + isTripleQuoted = p + 2 < line.Length && line[p + 1] == quoteChar && line[p + 2] == quoteChar; + return true; + } + private static bool IsIgnoredCallName(string language, string name) { if (LanguageSpecificCallNameKeeps.TryGetValue(language, out var languageSpecificKeepNames) diff --git a/tests/CodeIndex.Tests/ReferenceExtractorTests.cs b/tests/CodeIndex.Tests/ReferenceExtractorTests.cs index a34bdd6a0c..18ec0a6c70 100644 --- a/tests/CodeIndex.Tests/ReferenceExtractorTests.cs +++ b/tests/CodeIndex.Tests/ReferenceExtractorTests.cs @@ -5512,6 +5512,30 @@ def use(): Assert.Equal("use", runReference.ContainerName); } + [Fact] + public void Extract_PythonFString_MasksMultilineLiteralTextButKeepsInterpolationReferences() + { + const string content = """" + def run(): + return 42 + + def use(user_name): + value = f"""hello + {run()} + goodbye user_name + """ + return value + """"; + + var symbols = SymbolExtractor.Extract(1, "python", content); + var references = ReferenceExtractor.Extract(1, "python", content, symbols); + + var runReference = Assert.Single(references, reference => reference.SymbolName == "run"); + Assert.Equal("call", runReference.ReferenceKind); + Assert.Equal("use", runReference.ContainerName); + Assert.DoesNotContain(references, reference => reference.SymbolName is "hello" or "goodbye" or "user_name"); + } + [Fact] public void Extract_CsharpInterpolatedVerbatimString_WithEscapedBraces_DoesNotLeakPhantomReference() { From 8eed8c2fb271200f62da765fd690ab9a95c9c65e Mon Sep 17 00:00:00 2001 From: Widthdom Date: Sun, 31 May 2026 18:20:38 +0900 Subject: [PATCH 2/6] Fix Lisp quoted form symbol extraction (#2004) --- changelog.d/unreleased/2004.fixed.md | 16 ++++++ .../Indexer/Symbols/SymbolExtractor.Lisp.cs | 56 +++++++++++++++++++ tests/CodeIndex.Tests/SymbolExtractorTests.cs | 20 +++++++ 3 files changed, 92 insertions(+) create mode 100644 changelog.d/unreleased/2004.fixed.md diff --git a/changelog.d/unreleased/2004.fixed.md b/changelog.d/unreleased/2004.fixed.md new file mode 100644 index 0000000000..de663620d8 --- /dev/null +++ b/changelog.d/unreleased/2004.fixed.md @@ -0,0 +1,16 @@ +--- +category: fixed +issues: + - 2004 +affected: + - src/CodeIndex/Indexer/Symbols/SymbolExtractor.Lisp.cs + - tests/CodeIndex.Tests/SymbolExtractorTests.cs +--- + +## English + +- **Lisp reader macros no longer create phantom definitions (#2004)** — quoted and quasiquoted forms are excluded from symbol definition extraction so macro templates do not appear as real functions. + +## 日本語 + +- **Lisp の reader macro が phantom 定義を作らなくなりました (#2004)** — quote / quasiquote されたフォームをシンボル定義抽出から除外し、マクロテンプレートが実関数として現れないようにしました。 diff --git a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.Lisp.cs b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.Lisp.cs index cc2ce6de3f..bafa397d45 100644 --- a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.Lisp.cs +++ b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.Lisp.cs @@ -4,6 +4,19 @@ namespace CodeIndex.Indexer; public static partial class SymbolExtractor { + private static readonly HashSet LispNonDefinitionHeads = new(StringComparer.OrdinalIgnoreCase) + { + "quote", + "quasiquote", + "unquote", + "unquote-splicing", + "function", + "eval-when", + "let", + "let*", + "lambda", + }; + internal static string[] MaskLispCodeLines(IReadOnlyList lines) { var maskedLines = new string[lines.Count]; @@ -96,6 +109,12 @@ private static List ExtractLispSymbols(long fileId, string languag if (!TryReadLispListHead(maskedLine, cursor, out var head, out _, out var afterHead)) continue; + if (LispNonDefinitionHeads.Contains(head)) + { + MaskLispForm(maskedLines, lineIndex, cursor); + maskedLine = maskedLines[lineIndex]; + continue; + } if (!TryCreateLispSymbol(language, maskedLine, head, afterHead, out var kind, out var name, out var nameIndex)) continue; @@ -281,6 +300,43 @@ private static (int EndLine, int? BodyStartLine, int? BodyEndLine) FindLispFormR return BuildLispRange(startLineIndex, maskedLines.Count - 1, kind); } + private static void MaskLispForm(string[] maskedLines, int startLineIndex, int startColumn) + { + var depth = 0; + var started = false; + + for (var lineIndex = startLineIndex; lineIndex < maskedLines.Length; lineIndex++) + { + var chars = maskedLines[lineIndex].ToCharArray(); + var column = lineIndex == startLineIndex ? startColumn : 0; + for (; column < chars.Length; column++) + { + if (chars[column] == '(') + { + depth++; + started = true; + } + else if (chars[column] == ')' && started) + { + depth--; + chars[column] = ' '; + if (depth == 0) + { + maskedLines[lineIndex] = new string(chars); + return; + } + + continue; + } + + if (started) + chars[column] = ' '; + } + + maskedLines[lineIndex] = new string(chars); + } + } + private static (int EndLine, int? BodyStartLine, int? BodyEndLine) BuildLispRange( int startLineIndex, int endLineIndex, diff --git a/tests/CodeIndex.Tests/SymbolExtractorTests.cs b/tests/CodeIndex.Tests/SymbolExtractorTests.cs index 0501d8fb48..cee3ca956e 100644 --- a/tests/CodeIndex.Tests/SymbolExtractorTests.cs +++ b/tests/CodeIndex.Tests/SymbolExtractorTests.cs @@ -21469,6 +21469,26 @@ public void Extract_CommonLisp_DetectsTopLevelDefinitions() Assert.Contains(symbols, s => s.Kind == "function" && s.Name == "with-widget"); } + [Fact] + public void Extract_CommonLisp_DoesNotTreatQuotedReaderMacroFormsAsDefinitions() + { + var content = """ + '(defun quoted-function () nil) + `(let ((x 1)) ,x) + ,(expand-macro) + (quote (defun nested-quoted () nil)) + (defun real-function () nil) + """; + + var symbols = SymbolExtractor.Extract(1, "commonlisp", content); + + Assert.Contains(symbols, symbol => symbol.Kind == "function" && symbol.Name == "real-function"); + Assert.DoesNotContain(symbols, symbol => symbol.Name == "quoted-function"); + Assert.DoesNotContain(symbols, symbol => symbol.Name == "nested-quoted"); + Assert.DoesNotContain(symbols, symbol => symbol.Name == "let"); + Assert.DoesNotContain(symbols, symbol => symbol.Name == "expand-macro"); + } + [Fact] public void Extract_Racket_DetectsModuleAndDefinitions() { From b78a098d0f35352e08cee64fdf0489540ff19243 Mon Sep 17 00:00:00 2001 From: Widthdom Date: Sun, 31 May 2026 18:21:35 +0900 Subject: [PATCH 3/6] Fix Perl hash constant normalization (#2005) --- changelog.d/unreleased/2005.fixed.md | 16 ++++++ .../Indexer/Symbols/SymbolExtractor.Perl.cs | 52 ++++++++++++++++++- tests/CodeIndex.Tests/SymbolExtractorTests.cs | 24 +++++++++ 3 files changed, 90 insertions(+), 2 deletions(-) create mode 100644 changelog.d/unreleased/2005.fixed.md diff --git a/changelog.d/unreleased/2005.fixed.md b/changelog.d/unreleased/2005.fixed.md new file mode 100644 index 0000000000..e53ba3489d --- /dev/null +++ b/changelog.d/unreleased/2005.fixed.md @@ -0,0 +1,16 @@ +--- +category: fixed +issues: + - 2005 +affected: + - src/CodeIndex/Indexer/Symbols/SymbolExtractor.Perl.cs + - tests/CodeIndex.Tests/SymbolExtractorTests.cs +--- + +## English + +- **Perl hash constant keys are normalized and deduplicated (#2005)** — quoted keys are trimmed, escape-decoded, Unicode-normalized, and deduplicated against equivalent bareword constants. + +## 日本語 + +- **Perl の hash constant キーを正規化して重複排除するようになりました (#2005)** — quoted key を trim・escape decode・Unicode 正規化し、等価な bareword constant と重複しないようにしました。 diff --git a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.Perl.cs b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.Perl.cs index 79a5509e45..c8139225b2 100644 --- a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.Perl.cs +++ b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.Perl.cs @@ -1,3 +1,5 @@ +using System.Globalization; +using System.Text; using System.Text.RegularExpressions; using CodeIndex.Models; @@ -11,7 +13,7 @@ public static partial class SymbolExtractor @"^\s*use\s+constant\s+\{", RegexOptions.Compiled | RegexOptions.CultureInvariant); private static readonly Regex PerlHashConstantKeyRegex = new( - @"(?:^|,)\s*(?:""(?[\p{L}_][\p{L}\p{Nd}_]*)""|'(?[\p{L}_][\p{L}\p{Nd}_]*)'|(?[\p{L}_][\p{L}\p{Nd}_]*))\s*=>", + @"(?:^|,)\s*(?:""(?(?:\\x[0-9A-Fa-f]{2}|\\u[0-9A-Fa-f]{4}|\\.|[^""])*)""|'(?(?:\\x[0-9A-Fa-f]{2}|\\u[0-9A-Fa-f]{4}|\\.|[^'])*)'|(?[\p{L}_][\p{L}\p{Nd}_]*))\s*=>", RegexOptions.Compiled | RegexOptions.CultureInvariant); private static void ExtractPerlHashConstantSymbols(long fileId, string[] lines, List symbols) @@ -21,6 +23,7 @@ private static void ExtractPerlHashConstantSymbols(long fileId, string[] lines, if (!TryCollectPerlHashConstantBody(lines, i, out var body, out var lineSegments, out var endLineIndex, out var signature)) continue; + var seenConstantNames = new HashSet(StringComparer.Ordinal); foreach (Match keyMatch in PerlHashConstantKeyRegex.Matches(body)) { var nameGroup = keyMatch.Groups["bare"].Success @@ -28,6 +31,9 @@ private static void ExtractPerlHashConstantSymbols(long fileId, string[] lines, : keyMatch.Groups["quoted"]; if (!nameGroup.Success) continue; + var name = NormalizePerlConstantName(nameGroup.Value); + if (name.Length == 0 || !seenConstantNames.Add(name)) + continue; var (lineIndex, column) = ResolvePerlHashConstantBodyPosition(lineSegments, nameGroup.Index); AddSymbolRecord( @@ -38,7 +44,7 @@ private static void ExtractPerlHashConstantSymbols(long fileId, string[] lines, { FileId = fileId, Kind = "function", - Name = nameGroup.Value, + Name = name, Line = lineIndex + 1, StartLine = lineIndex + 1, EndLine = lineIndex + 1, @@ -51,6 +57,48 @@ private static void ExtractPerlHashConstantSymbols(long fileId, string[] lines, } } + private static string NormalizePerlConstantName(string name) + => DecodePerlQuotedConstantEscapes(name.Trim()).Normalize(NormalizationForm.FormC); + + private static string DecodePerlQuotedConstantEscapes(string value) + { + if (value.IndexOf('\\') < 0) + return value; + + var builder = new StringBuilder(value.Length); + for (var i = 0; i < value.Length; i++) + { + if (value[i] != '\\' || i + 1 >= value.Length) + { + builder.Append(value[i]); + continue; + } + + var marker = value[i + 1]; + if (marker == 'x' && i + 3 < value.Length && TryParseHexScalar(value.AsSpan(i + 2, 2), out var hexByte)) + { + builder.Append((char)hexByte); + i += 3; + continue; + } + + if (marker == 'u' && i + 5 < value.Length && TryParseHexScalar(value.AsSpan(i + 2, 4), out var unicodeScalar)) + { + builder.Append(char.ConvertFromUtf32(unicodeScalar)); + i += 5; + continue; + } + + builder.Append(marker); + i++; + } + + return builder.ToString(); + } + + private static bool TryParseHexScalar(ReadOnlySpan value, out int scalar) + => int.TryParse(value, NumberStyles.HexNumber, CultureInfo.InvariantCulture, out scalar); + private readonly record struct PerlBodyLineSegment(int BodyStartIndex, int LineIndex, int ColumnOffset); private static bool TryCollectPerlHashConstantBody( diff --git a/tests/CodeIndex.Tests/SymbolExtractorTests.cs b/tests/CodeIndex.Tests/SymbolExtractorTests.cs index cee3ca956e..aa3d196a36 100644 --- a/tests/CodeIndex.Tests/SymbolExtractorTests.cs +++ b/tests/CodeIndex.Tests/SymbolExtractorTests.cs @@ -25284,6 +25284,30 @@ fun normalize ($value) { Assert.Contains(symbols, s => s.Kind == "function" && s.Name == "normalize"); } + [Fact] + public void Extract_PerlHashConstants_NormalizesQuotedKeysAndDeduplicates() + { + var content = """ + use constant { + "foo " => 1, + foo => 2, + "naïve" => 3, + "nai\u0308ve" => 4, + "hex\xEF" => 5, + "hexï" => 6, + " " => 7, + }; + """; + + var symbols = SymbolExtractor.Extract(1, "perl", content); + + Assert.Equal(3, symbols.Count(symbol => symbol.Kind == "function")); + Assert.Single(symbols.Where(symbol => symbol.Kind == "function" && symbol.Name == "foo")); + Assert.Single(symbols.Where(symbol => symbol.Kind == "function" && symbol.Name == "naïve")); + Assert.Single(symbols.Where(symbol => symbol.Kind == "function" && symbol.Name == "hexï")); + Assert.DoesNotContain(symbols, symbol => symbol.Name == "foo "); + } + [Fact] public void Extract_Perl_CapturesPackageBlockNamespaces() { From 5d96734c32a5d43545bc3ec9a0aa6cc42f9e5395 Mon Sep 17 00:00:00 2001 From: Widthdom Date: Sun, 31 May 2026 18:50:06 +0900 Subject: [PATCH 4/6] Address Lisp wrapper definitions (#2004) --- .../Indexer/Symbols/SymbolExtractor.Lisp.cs | 4 ---- tests/CodeIndex.Tests/SymbolExtractorTests.cs | 13 +++++++++++++ 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.Lisp.cs b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.Lisp.cs index bafa397d45..d52bf45056 100644 --- a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.Lisp.cs +++ b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.Lisp.cs @@ -11,10 +11,6 @@ public static partial class SymbolExtractor "unquote", "unquote-splicing", "function", - "eval-when", - "let", - "let*", - "lambda", }; internal static string[] MaskLispCodeLines(IReadOnlyList lines) diff --git a/tests/CodeIndex.Tests/SymbolExtractorTests.cs b/tests/CodeIndex.Tests/SymbolExtractorTests.cs index aa3d196a36..5d912f3047 100644 --- a/tests/CodeIndex.Tests/SymbolExtractorTests.cs +++ b/tests/CodeIndex.Tests/SymbolExtractorTests.cs @@ -21489,6 +21489,19 @@ public void Extract_CommonLisp_DoesNotTreatQuotedReaderMacroFormsAsDefinitions() Assert.DoesNotContain(symbols, symbol => symbol.Name == "expand-macro"); } + [Fact] + public void Extract_CommonLisp_PreservesDefinitionsInsideEvalWhen() + { + var content = """ + (eval-when (:compile-toplevel :load-toplevel :execute) + (defun real-wrapper-function () nil)) + """; + + var symbols = SymbolExtractor.Extract(1, "commonlisp", content); + + Assert.Contains(symbols, symbol => symbol.Kind == "function" && symbol.Name == "real-wrapper-function"); + } + [Fact] public void Extract_Racket_DetectsModuleAndDefinitions() { From 211ce3ae0f920373c7d277bcf269861f244333ee Mon Sep 17 00:00:00 2001 From: Widthdom Date: Sun, 31 May 2026 19:32:35 +0900 Subject: [PATCH 5/6] Handle nested Python f-string expression strings (#1438) --- .../ReferenceExtractor.TypeReferences.cs | 49 +++++++++++++++++++ .../ReferenceExtractorTests.cs | 21 ++++++++ 2 files changed, 70 insertions(+) diff --git a/src/CodeIndex/Indexer/References/ReferenceExtractor.TypeReferences.cs b/src/CodeIndex/Indexer/References/ReferenceExtractor.TypeReferences.cs index c3ef2dc02a..817ac6df39 100644 --- a/src/CodeIndex/Indexer/References/ReferenceExtractor.TypeReferences.cs +++ b/src/CodeIndex/Indexer/References/ReferenceExtractor.TypeReferences.cs @@ -3247,7 +3247,10 @@ private static void MaskPythonTripleQuotedFString( var lineIndex = startLineIndex; var column = startColumn; var inExpression = false; + var inExpressionString = false; var expressionDepth = 0; + var expressionStringQuote = '\0'; + var expressionStringTripleQuoted = false; endLineIndex = startLineIndex; endColumn = startColumn; @@ -3315,6 +3318,52 @@ private static void MaskPythonTripleQuotedFString( continue; } + if (inExpressionString) + { + if (line[column] == '\\' && column + 1 < line.Length) + { + column += 2; + continue; + } + + if (expressionStringTripleQuoted) + { + if (column + 2 < line.Length + && line[column] == expressionStringQuote + && line[column + 1] == expressionStringQuote + && line[column + 2] == expressionStringQuote) + { + column += 3; + inExpressionString = false; + continue; + } + + column++; + continue; + } + + if (line[column] == expressionStringQuote) + { + column++; + inExpressionString = false; + continue; + } + + column++; + continue; + } + + if (line[column] == '\'' || line[column] == '"') + { + expressionStringQuote = line[column]; + expressionStringTripleQuoted = column + 2 < line.Length + && line[column + 1] == expressionStringQuote + && line[column + 2] == expressionStringQuote; + column += expressionStringTripleQuoted ? 3 : 1; + inExpressionString = true; + continue; + } + if (line[column] == '{') { expressionDepth++; diff --git a/tests/CodeIndex.Tests/ReferenceExtractorTests.cs b/tests/CodeIndex.Tests/ReferenceExtractorTests.cs index 18ec0a6c70..92ea866498 100644 --- a/tests/CodeIndex.Tests/ReferenceExtractorTests.cs +++ b/tests/CodeIndex.Tests/ReferenceExtractorTests.cs @@ -5536,6 +5536,27 @@ goodbye user_name Assert.DoesNotContain(references, reference => reference.SymbolName is "hello" or "goodbye" or "user_name"); } + [Fact] + public void Extract_PythonFString_KeepsReferencesAfterNestedExpressionStringBrace() + { + const string content = """" + def run(): + return 42 + + def use(format_value): + value = f"""{format_value("}") + run()}""" + return value + """"; + + var symbols = SymbolExtractor.Extract(1, "python", content); + var references = ReferenceExtractor.Extract(1, "python", content, symbols); + + Assert.Contains(references, reference => + reference.SymbolName == "run" + && reference.ReferenceKind == "call" + && reference.ContainerName == "use"); + } + [Fact] public void Extract_CsharpInterpolatedVerbatimString_WithEscapedBraces_DoesNotLeakPhantomReference() { From b99f4ea2b08efd4d24a235830c2f1170e2b3d172 Mon Sep 17 00:00:00 2001 From: Widthdom Date: Sun, 31 May 2026 19:32:47 +0900 Subject: [PATCH 6/6] Handle Perl braced hex constants (#2005) --- .../Indexer/Symbols/SymbolExtractor.Perl.cs | 77 +++++++++++++++++-- tests/CodeIndex.Tests/SymbolExtractorTests.cs | 9 ++- 2 files changed, 75 insertions(+), 11 deletions(-) diff --git a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.Perl.cs b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.Perl.cs index c8139225b2..e1b0586c06 100644 --- a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.Perl.cs +++ b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.Perl.cs @@ -13,7 +13,7 @@ public static partial class SymbolExtractor @"^\s*use\s+constant\s+\{", RegexOptions.Compiled | RegexOptions.CultureInvariant); private static readonly Regex PerlHashConstantKeyRegex = new( - @"(?:^|,)\s*(?:""(?(?:\\x[0-9A-Fa-f]{2}|\\u[0-9A-Fa-f]{4}|\\.|[^""])*)""|'(?(?:\\x[0-9A-Fa-f]{2}|\\u[0-9A-Fa-f]{4}|\\.|[^'])*)'|(?[\p{L}_][\p{L}\p{Nd}_]*))\s*=>", + @"(?:^|,)\s*(?:""(?(?:\\x\{[0-9A-Fa-f]+\}|\\x[0-9A-Fa-f]{2}|\\.|[^""])*)""|'(?(?:\\x\{[0-9A-Fa-f]+\}|\\x[0-9A-Fa-f]{2}|\\.|[^'])*)'|(?[\p{L}_][\p{L}\p{Nd}_]*))\s*=>", RegexOptions.Compiled | RegexOptions.CultureInvariant); private static void ExtractPerlHashConstantSymbols(long fileId, string[] lines, List symbols) @@ -75,17 +75,22 @@ private static string DecodePerlQuotedConstantEscapes(string value) } var marker = value[i + 1]; - if (marker == 'x' && i + 3 < value.Length && TryParseHexScalar(value.AsSpan(i + 2, 2), out var hexByte)) + if (marker == 'x' + && i + 3 < value.Length + && value[i + 2] == '{' + && TryFindPerlBracedHexEscapeEnd(value, i + 3, out var escapeEnd) + && TryParseHexScalar(value.AsSpan(i + 3, escapeEnd - (i + 3)), out var scalar) + && scalar <= 0x10FFFF) { - builder.Append((char)hexByte); - i += 3; + builder.Append(char.ConvertFromUtf32(scalar)); + i = escapeEnd; continue; } - if (marker == 'u' && i + 5 < value.Length && TryParseHexScalar(value.AsSpan(i + 2, 4), out var unicodeScalar)) + if (marker == 'x' && i + 3 < value.Length && TryParseHexScalar(value.AsSpan(i + 2, 2), out var hexByte)) { - builder.Append(char.ConvertFromUtf32(unicodeScalar)); - i += 5; + builder.Append((char)hexByte); + i += 3; continue; } @@ -99,6 +104,21 @@ private static string DecodePerlQuotedConstantEscapes(string value) private static bool TryParseHexScalar(ReadOnlySpan value, out int scalar) => int.TryParse(value, NumberStyles.HexNumber, CultureInfo.InvariantCulture, out scalar); + private static bool TryFindPerlBracedHexEscapeEnd(string value, int start, out int end) + { + for (var i = start; i < value.Length; i++) + { + if (value[i] == '}') + { + end = i; + return i > start; + } + } + + end = -1; + return false; + } + private readonly record struct PerlBodyLineSegment(int BodyStartIndex, int LineIndex, int ColumnOffset); private static bool TryCollectPerlHashConstantBody( @@ -124,11 +144,13 @@ private static bool TryCollectPerlHashConstantBody( return false; var builder = new System.Text.StringBuilder(); + var inQuotedKey = false; + var quotedKeyDelimiter = '\0'; for (var lineIndex = startLineIndex; lineIndex < lines.Length; lineIndex++) { var line = lines[lineIndex]; var segmentStart = lineIndex == startLineIndex ? openBraceIndex + 1 : 0; - var closeBraceIndex = line.IndexOf('}', segmentStart); + var closeBraceIndex = FindPerlHashConstantBlockCloseBrace(line, segmentStart, ref inQuotedKey, ref quotedKeyDelimiter); var segmentEnd = closeBraceIndex >= 0 ? closeBraceIndex : line.Length; if (segmentEnd > segmentStart) { @@ -149,6 +171,45 @@ private static bool TryCollectPerlHashConstantBody( return false; } + private static int FindPerlHashConstantBlockCloseBrace( + string line, + int start, + ref bool inQuotedKey, + ref char quotedKeyDelimiter) + { + for (var i = start; i < line.Length; i++) + { + if (inQuotedKey) + { + if (line[i] == '\\' && i + 1 < line.Length) + { + i++; + continue; + } + + if (line[i] == quotedKeyDelimiter) + { + inQuotedKey = false; + quotedKeyDelimiter = '\0'; + } + + continue; + } + + if (line[i] == '"' || line[i] == '\'') + { + inQuotedKey = true; + quotedKeyDelimiter = line[i]; + continue; + } + + if (line[i] == '}') + return i; + } + + return -1; + } + private static (int LineIndex, int Column) ResolvePerlHashConstantBodyPosition( IReadOnlyList lineSegments, int bodyIndex) diff --git a/tests/CodeIndex.Tests/SymbolExtractorTests.cs b/tests/CodeIndex.Tests/SymbolExtractorTests.cs index 5d912f3047..8503dfed2e 100644 --- a/tests/CodeIndex.Tests/SymbolExtractorTests.cs +++ b/tests/CodeIndex.Tests/SymbolExtractorTests.cs @@ -25305,19 +25305,22 @@ use constant { "foo " => 1, foo => 2, "naïve" => 3, - "nai\u0308ve" => 4, + "naïve" => 4, "hex\xEF" => 5, "hexï" => 6, - " " => 7, + "braced\x{00EF}" => 7, + "bracedï" => 8, + " " => 9, }; """; var symbols = SymbolExtractor.Extract(1, "perl", content); - Assert.Equal(3, symbols.Count(symbol => symbol.Kind == "function")); + Assert.Equal(4, symbols.Count(symbol => symbol.Kind == "function")); Assert.Single(symbols.Where(symbol => symbol.Kind == "function" && symbol.Name == "foo")); Assert.Single(symbols.Where(symbol => symbol.Kind == "function" && symbol.Name == "naïve")); Assert.Single(symbols.Where(symbol => symbol.Kind == "function" && symbol.Name == "hexï")); + Assert.Single(symbols.Where(symbol => symbol.Kind == "function" && symbol.Name == "bracedï")); Assert.DoesNotContain(symbols, symbol => symbol.Name == "foo "); }