Fix extractor false positives for Python, Lisp, and Perl

changelog.d/unreleased/1438.fixed.md

@@ -0,0 +1,17 @@
+---
+category: fixed
+issues:
+  - 1438
+affected:
+  - src/CodeIndex/Indexer/References/ReferenceExtractor.Preparation.cs
+  - src/CodeIndex/Indexer/References/ReferenceExtractor.TypeReferences.cs
+  - tests/CodeIndex.Tests/ReferenceExtractorTests.cs
+---
+
+## English
+
+- **Python multi-line f-strings no longer emit references from literal text (#1438)** — triple-quoted f-string bodies are masked across physical lines while interpolation expressions still contribute real reference edges.
+
+## 日本語
+
+- **Python の複数行 f-string がリテラル本文から参照を出さなくなりました (#1438)** — 三重引用符の f-string 本文を物理行をまたいでマスクしつつ、補間式内の実参照は引き続き抽出します。

changelog.d/unreleased/2004.fixed.md

@@ -0,0 +1,16 @@
+---
+category: fixed
+issues:
+  - 2004
+affected:
+  - src/CodeIndex/Indexer/Symbols/SymbolExtractor.Lisp.cs
+  - tests/CodeIndex.Tests/SymbolExtractorTests.cs
+---
+
+## English
+
+- **Lisp reader macros no longer create phantom definitions (#2004)** — quoted and quasiquoted forms are excluded from symbol definition extraction so macro templates do not appear as real functions.
+
+## 日本語
+
+- **Lisp の reader macro が phantom 定義を作らなくなりました (#2004)** — quote / quasiquote されたフォームをシンボル定義抽出から除外し、マクロテンプレートが実関数として現れないようにしました。

changelog.d/unreleased/2005.fixed.md

@@ -0,0 +1,16 @@
+---
+category: fixed
+issues:
+  - 2005
+affected:
+  - src/CodeIndex/Indexer/Symbols/SymbolExtractor.Perl.cs
+  - tests/CodeIndex.Tests/SymbolExtractorTests.cs
+---
+
+## English
+
+- **Perl hash constant keys are normalized and deduplicated (#2005)** — quoted keys are trimmed, escape-decoded, Unicode-normalized, and deduplicated against equivalent bareword constants.
+
+## 日本語
+
+- **Perl の hash constant キーを正規化して重複排除するようになりました (#2005)** — quoted key を trim・escape decode・Unicode 正規化し、等価な bareword constant と重複しないようにしました。

src/CodeIndex/Indexer/References/ReferenceExtractor.Preparation.cs

@@ -89,6 +89,9 @@ private static bool TryPrepareReferenceLines(
                 : UsesCStyleBlockComments(language)
                     ? MaskCStyleBlockCommentLines(language, structuralLines)
                     : structuralLines;
+        if (language == "python")
+            referenceStructuralLines = MaskPythonFStrings(referenceStructuralLines);
+
         var preparedLines = new string[lines.Length];
         for (var pi = 0; pi < lines.Length; pi++)
             preparedLines[pi] = PrepareLine(language, referenceStructuralLines[pi]);

src/CodeIndex/Indexer/References/ReferenceExtractor.TypeReferences.cs

@@ -2560,9 +2560,7 @@ private static string ReplaceRegexMatchesWithSpaces(Regex regex, string input)
 
     private static string PrepareLine(string lang, string line)
     {
-        var result = lang == "python"
-            ? MaskPythonSingleLineFStrings(line)
-            : line;
+        var result = line;
         if (lang == "rust")
             result = MaskRustLifetimeTokens(result);
         if (lang != "cobol")
@@ -3188,6 +3186,211 @@ private static string MaskPythonSingleLineFStrings(string line)
         return new string(masked);
     }
 
+    private static string[] MaskPythonFStrings(IReadOnlyList<string> lines)
+    {
+        var result = new string[lines.Count];
+        for (var lineIndex = 0; lineIndex < lines.Count; lineIndex++)
+            result[lineIndex] = lines[lineIndex];
+
+        for (var lineIndex = 0; lineIndex < result.Length; lineIndex++)
+        {
+            var line = result[lineIndex];
+            if (line.IndexOf('f') < 0 && line.IndexOf('F') < 0)
+                continue;
+
+            var chars = line.ToCharArray();
+            var changed = false;
+            for (var column = 0; column < line.Length; column++)
+            {
+                if (!TryOpenPythonString(line, column, out var prefixLength, out var quoteChar, out var isRaw, out var isFString, out var isTripleQuoted))
+                    continue;
+
+                if (!isFString)
+                {
+                    column += prefixLength;
+                    continue;
+                }
+
+                if (!isTripleQuoted)
+                {
+                    result[lineIndex] = MaskPythonSingleLineFStrings(line);
+                    chars = result[lineIndex].ToCharArray();
+                    changed = true;
+                    break;
+                }
+
+                MaskPythonTripleQuotedFString(result, lineIndex, column, prefixLength, quoteChar, isRaw, out var endLineIndex, out var endColumn);
+                lineIndex = endLineIndex;
+                line = result[lineIndex];
+                chars = line.ToCharArray();
+                column = endColumn;
+                changed = true;
+            }
+
+            if (changed)
+                result[lineIndex] = new string(chars);
+        }
+
+        return result;
+    }
+
+    private static void MaskPythonTripleQuotedFString(
+        string[] lines,
+        int startLineIndex,
+        int startColumn,
+        int prefixLength,
+        char quoteChar,
+        bool isRaw,
+        out int endLineIndex,
+        out int endColumn)
+    {
+        var lineIndex = startLineIndex;
+        var column = startColumn;
+        var inExpression = false;
+        var inExpressionString = false;
+        var expressionDepth = 0;
+        var expressionStringQuote = '\0';
+        var expressionStringTripleQuoted = false;
+
+        endLineIndex = startLineIndex;
+        endColumn = startColumn;
+
+        while (lineIndex < lines.Length)
+        {
+            var line = lines[lineIndex];
+            var chars = line.ToCharArray();
+            if (lineIndex == startLineIndex)
+            {
+                ReplaceWithSpaces(chars, startColumn, prefixLength + 3);
+                column = startColumn + prefixLength + 3;
+            }
+            else
+            {
+                column = 0;
+            }
+
+            while (column < line.Length)
+            {
+                if (!inExpression)
+                {
+                    if (!isRaw && line[column] == '\\' && column + 1 < line.Length)
+                    {
+                        ReplaceWithSpaces(chars, column, 2);
+                        column += 2;
+                        continue;
+                    }
+
+                    if (line[column] == '{' && column + 1 < line.Length && line[column + 1] == '{')
+                    {
+                        ReplaceWithSpaces(chars, column, 2);
+                        column += 2;
+                        continue;
+                    }
+
+                    if (line[column] == '}' && column + 1 < line.Length && line[column + 1] == '}')
+                    {
+                        ReplaceWithSpaces(chars, column, 2);
+                        column += 2;
+                        continue;
+                    }
+
+                    if (line[column] == '{')
+                    {
+                        chars[column++] = ' ';
+                        inExpression = true;
+                        expressionDepth = 1;
+                        continue;
+                    }
+
+                    if (column + 2 < line.Length
+                        && line[column] == quoteChar
+                        && line[column + 1] == quoteChar
+                        && line[column + 2] == quoteChar)
+                    {
+                        ReplaceWithSpaces(chars, column, 3);
+                        lines[lineIndex] = new string(chars);
+                        endLineIndex = lineIndex;
+                        endColumn = column + 2;
+                        return;
+                    }
+
+                    chars[column++] = ' ';
+                    continue;
+                }
+
+                if (inExpressionString)
+                {
+                    if (line[column] == '\\' && column + 1 < line.Length)
+                    {
+                        column += 2;
+                        continue;
+                    }
+
+                    if (expressionStringTripleQuoted)
+                    {
+                        if (column + 2 < line.Length
+                            && line[column] == expressionStringQuote
+                            && line[column + 1] == expressionStringQuote
+                            && line[column + 2] == expressionStringQuote)
+                        {
+                            column += 3;
+                            inExpressionString = false;
+                            continue;
+                        }
+
+                        column++;
+                        continue;
+                    }
+
+                    if (line[column] == expressionStringQuote)
+                    {
+                        column++;
+                        inExpressionString = false;
+                        continue;
+                    }
+
+                    column++;
+                    continue;
+                }
+
+                if (line[column] == '\'' || line[column] == '"')
+                {
+                    expressionStringQuote = line[column];
+                    expressionStringTripleQuoted = column + 2 < line.Length
+                        && line[column + 1] == expressionStringQuote
+                        && line[column + 2] == expressionStringQuote;
+                    column += expressionStringTripleQuoted ? 3 : 1;
+                    inExpressionString = true;
+                    continue;
+                }
+
+                if (line[column] == '{')
+                {
+                    expressionDepth++;
+                    column++;
+                    continue;
+                }
+
+                if (line[column] == '}')
+                {
+                    expressionDepth--;
+                    chars[column++] = ' ';
+                    if (expressionDepth == 0)
+                        inExpression = false;
+                    continue;
+                }
+
+                column++;
+            }
+
+            lines[lineIndex] = new string(chars);
+            lineIndex++;
+        }
+
+        endLineIndex = Math.Max(startLineIndex, lines.Length - 1);
+        endColumn = 0;
+    }
+
     private static void ReplaceWithSpaces(char[] buffer, int start, int length)
     {
         for (var i = start; i < start + length && i < buffer.Length; i++)
@@ -3235,6 +3438,63 @@ private static bool TryOpenPythonSingleLineString(
         return true;
     }
 
+    private static bool TryOpenPythonString(
+        string line,
+        int startIndex,
+        out int prefixLength,
+        out char quoteChar,
+        out bool isRaw,
+        out bool isFString,
+        out bool isTripleQuoted)
+    {
+        isTripleQuoted = false;
+        if (!TryOpenPythonSingleOrTripleString(line, startIndex, out prefixLength, out quoteChar, out isRaw, out isFString, out isTripleQuoted))
+            return false;
+        return true;
+    }
+
+    private static bool TryOpenPythonSingleOrTripleString(
+        string line,
+        int startIndex,
+        out int prefixLength,
+        out char quoteChar,
+        out bool isRaw,
+        out bool isFString,
+        out bool isTripleQuoted)
+    {
+        prefixLength = 0;
+        quoteChar = '\0';
+        isRaw = false;
+        isFString = false;
+        isTripleQuoted = false;
+
+        if (startIndex < 0 || startIndex >= line.Length)
+            return false;
+
+        if (startIndex > 0 && IsIdentifierChar(line[startIndex - 1]))
+            return false;
+
+        var p = startIndex;
+        var prefixChars = 0;
+        while (p < line.Length && prefixChars < 2 && IsPythonStringPrefixChar(line[p]))
+        {
+            if (line[p] is 'r' or 'R')
+                isRaw = true;
+            if (line[p] is 'f' or 'F')
+                isFString = true;
+            p++;
+            prefixChars++;
+        }
+
+        if (p >= line.Length || (line[p] != '\'' && line[p] != '"'))
+            return false;
+
+        prefixLength = p - startIndex;
+        quoteChar = line[p];
+        isTripleQuoted = p + 2 < line.Length && line[p + 1] == quoteChar && line[p + 2] == quoteChar;
+        return true;
+    }
+
     private static bool IsIgnoredCallName(string language, string name)
     {
         if (LanguageSpecificCallNameKeeps.TryGetValue(language, out var languageSpecificKeepNames)