[codex] Split symbol and reference extraction modules

changelog.d/unreleased/3381.internal.md

@@ -0,0 +1,16 @@
+---
+category: internal
+issues:
+  - 3381
+affected:
+  - src/CodeIndex/Indexer/Symbols/SymbolExtractor.cs
+  - src/CodeIndex/Indexer/Symbols/SymbolExtractor.CSharpPatterns.cs
+---
+
+## English
+
+- **Split C# symbol pattern definitions out of the central symbol extractor (#3381)** — C# declaration and type regex constants now live in a dedicated partial module, reducing the mixed-language pattern surface in `SymbolExtractor.cs` without changing extraction behavior.
+
+## 日本語
+
+- **C# シンボル pattern 定義を central symbol extractor から分離しました (#3381)** — C# の宣言・型 regex 定数を専用 partial module に移し、抽出挙動は変えずに `SymbolExtractor.cs` の多言語 pattern 混在を減らしました。

changelog.d/unreleased/3383.internal.md

@@ -0,0 +1,16 @@
+---
+category: internal
+issues:
+  - 3383
+affected:
+  - tests/CodeIndex.Tests/SymbolExtractorTests.cs
+  - tests/CodeIndex.Tests/SymbolExtractorLuaTests.cs
+---
+
+## English
+
+- **Split Lua symbol extractor coverage into a language-specific test file (#3383)** — Moved the Lua symbol extraction regression case out of the mega `SymbolExtractorTests.cs` file and into `SymbolExtractorLuaTests.cs` for more targeted validation and review.
+
+## 日本語
+
+- **Lua symbol extractor coverage を言語別 test file に分離しました (#3383)** — Lua の symbol extraction regression case を巨大な `SymbolExtractorTests.cs` から `SymbolExtractorLuaTests.cs` に移し、より対象を絞った検証と review をしやすくしました。

changelog.d/unreleased/3421.internal.md

@@ -0,0 +1,16 @@
+---
+category: internal
+issues:
+  - 3421
+affected:
+  - src/CodeIndex/Indexer/References/Support/LanguageReferenceExtractionSupport.cs
+  - src/CodeIndex/Indexer/References/Languages/LuaReferenceExtractor.cs
+---
+
+## English
+
+- **Moved Lua reference extraction helpers into the Lua reference extractor (#3421)** — Lua require/type, colon-call, table-field, and long-bracket masking logic now live in the language module instead of the shared support class, reducing the central helper surface without changing extraction behavior.
+
+## 日本語
+
+- **Lua 参照抽出 helper を Lua reference extractor へ移しました (#3421)** — Lua の require/type、colon-call、table-field、long-bracket masking の処理を共有 support class ではなく言語 module に置き、抽出挙動は変えずに central helper surface を減らしました。

src/CodeIndex/Indexer/References/Languages/LuaReferenceExtractor.cs

@@ -1,11 +1,82 @@
+using System.Text.RegularExpressions;
+using Regex = CodeIndex.Indexer.BoundedRegex;
 using CodeIndex.Models;
 
 namespace CodeIndex.Indexer;
 
 internal static class LuaReferenceExtractor
 {
+    private static readonly Regex LuaRequireRegex = new(
+        @"\brequire\s*\(?\s*[""'](?<name>[^""']+)[""']",
+        RegexOptions.Compiled | RegexOptions.CultureInvariant);
+    private static readonly Regex LuaCommandCallRegex = new(
+        @"^\s*(?<name>[A-Za-z_]\w*(?:\.[A-Za-z_]\w*)?)\s+(?=[""'{A-Za-z_])",
+        RegexOptions.Compiled | RegexOptions.CultureInvariant);
+    private static readonly Regex LuaColonCallRegex = new(
+        @"(?<![\w.])(?:[A-Za-z_]\w*(?:\.[A-Za-z_]\w*)*):(?<name>[A-Za-z_]\w*)\s*\(",
+        RegexOptions.Compiled | RegexOptions.CultureInvariant);
+    private static readonly Regex LuaTableFieldReferenceRegex = new(
+        @"(?<![\w.])(?:[A-Za-z_]\w*\.)+(?<name>[A-Za-z_]\w*)\b(?!\s*(?:=|function\b|\())",
+        RegexOptions.Compiled | RegexOptions.CultureInvariant);
+
     public static string[] MaskLongCommentAndStringLines(IReadOnlyList<string> originalLines)
-        => LanguageReferenceExtractionSupport.MaskLuaLongCommentAndStringLines(originalLines);
+    {
+        var result = new string[originalLines.Count];
+        var longTextEqualsCount = -1;
+
+        for (var lineIndex = 0; lineIndex < originalLines.Count; lineIndex++)
+        {
+            var line = originalLines[lineIndex];
+            var chars = line.ToCharArray();
+            for (var cursor = 0; cursor < chars.Length; cursor++)
+            {
+                if (longTextEqualsCount >= 0)
+                {
+                    if (TryGetLuaLongBracketClose(line, cursor, longTextEqualsCount, out var closeLength))
+                    {
+                        MaskRange(chars, cursor, cursor + closeLength);
+                        cursor += closeLength - 1;
+                        longTextEqualsCount = -1;
+                        continue;
+                    }
+
+                    chars[cursor] = ' ';
+                    continue;
+                }
+
+                if (chars[cursor] is '"' or '\'')
+                {
+                    cursor = SkipQuotedLiteral(line, cursor);
+                    continue;
+                }
+
+                if (chars[cursor] == '-'
+                    && cursor + 2 < chars.Length
+                    && chars[cursor + 1] == '-'
+                    && TryGetLuaLongBracketOpen(line, cursor + 2, out var commentEqualsCount, out var commentOpenLength))
+                {
+                    MaskRange(chars, cursor, cursor + 2 + commentOpenLength);
+                    cursor += 1 + commentOpenLength;
+                    longTextEqualsCount = commentEqualsCount;
+                    continue;
+                }
+
+                if (chars[cursor] == '-' && cursor + 1 < chars.Length && chars[cursor + 1] == '-')
+                    break;
+
+                if (TryGetLuaLongBracketOpen(line, cursor, out var stringEqualsCount, out var stringOpenLength))
+                {
+                    MaskRange(chars, cursor, cursor + stringOpenLength);
+                    cursor += stringOpenLength - 1;
+                    longTextEqualsCount = stringEqualsCount;
+                }
+            }
+
+            result[lineIndex] = new string(chars);
+        }
+
+        return result;
+    }
 
     public static void EmitTypePositionReferences(
         string originalLine,
@@ -16,17 +87,8 @@ public static void EmitTypePositionReferences(
         int lineNumber,
         SymbolRecord? container)
     {
-        LanguageReferenceExtractionSupport.EmitTypePositionReferences(
-            "lua",
-            originalLine,
-            originalLine,
-            references,
-            seen,
-            fileId,
-            context,
-            lineNumber,
-            _ => container,
-            container);
+        foreach (var (name, index) in EnumerateLuaRequireReferences(originalLine))
+            ReferenceExtractor.AddReference(references, seen, fileId, name, index, "type_reference", context, lineNumber, container);
     }
 
     public static void EmitAdditionalCallReferences(
@@ -40,17 +102,183 @@ public static void EmitAdditionalCallReferences(
         Func<int, SymbolRecord?> resolveContainerForColumn,
         IReadOnlySet<string>? definitionNames)
     {
-        LanguageReferenceExtractionSupport.EmitAdditionalCallReferences(
-            "lua",
-            preparedLine,
-            preparedLine,
-            addCallLikeReference,
-            references,
-            seen,
-            fileId,
-            context,
-            lineNumber,
-            resolveContainerForColumn,
-            definitionNames);
+        var match = LuaCommandCallRegex.Match(preparedLine);
+        if (match.Success)
+        {
+            var name = LastQualifiedSegment(match.Groups["name"].Value);
+            if (definitionNames?.Contains(name) != true)
+                addCallLikeReference(name, match.Groups["name"].Index + match.Groups["name"].Value.LastIndexOf(name, StringComparison.Ordinal));
+        }
+
+        foreach (Match colonMatch in LuaColonCallRegex.Matches(preparedLine))
+        {
+            var name = colonMatch.Groups["name"].Value;
+            if (definitionNames?.Contains(name) == true)
+                continue;
+            addCallLikeReference(name, colonMatch.Groups["name"].Index);
+        }
+
+        foreach (Match fieldMatch in LuaTableFieldReferenceRegex.Matches(preparedLine))
+        {
+            var trimmed = preparedLine.TrimStart();
+            if (trimmed.StartsWith("function ", StringComparison.Ordinal)
+                || trimmed.StartsWith("local function ", StringComparison.Ordinal))
+            {
+                break;
+            }
+
+            var name = fieldMatch.Groups["name"].Value;
+            if (definitionNames?.Contains(name) == true)
+                continue;
+            var index = fieldMatch.Groups["name"].Index;
+            ReferenceExtractor.AddReference(references, seen, fileId, name, index, "reference", context, lineNumber, resolveContainerForColumn(index));
+        }
+    }
+
+    private static bool TryGetLuaLongBracketOpen(string line, int start, out int equalsCount, out int length)
+    {
+        equalsCount = 0;
+        length = 0;
+        if (start < 0 || start >= line.Length || line[start] != '[')
+            return false;
+
+        var cursor = start + 1;
+        while (cursor < line.Length && line[cursor] == '=')
+        {
+            equalsCount++;
+            cursor++;
+        }
+
+        if (cursor >= line.Length || line[cursor] != '[')
+            return false;
+
+        length = cursor - start + 1;
+        return true;
+    }
+
+    private static bool TryGetLuaLongBracketClose(string line, int start, int equalsCount, out int length)
+    {
+        length = 0;
+        if (start < 0 || start >= line.Length || line[start] != ']')
+            return false;
+
+        var cursor = start + 1;
+        for (var i = 0; i < equalsCount; i++)
+        {
+            if (cursor >= line.Length || line[cursor] != '=')
+                return false;
+            cursor++;
+        }
+
+        if (cursor >= line.Length || line[cursor] != ']')
+            return false;
+
+        length = cursor - start + 1;
+        return true;
+    }
+
+    private static IEnumerable<(string Name, int Index)> EnumerateLuaRequireReferences(string line)
+    {
+        for (var cursor = 0; cursor < line.Length; cursor++)
+        {
+            if (line[cursor] == '-' && cursor + 1 < line.Length && line[cursor + 1] == '-')
+                yield break;
+
+            if (line[cursor] is '"' or '\'')
+            {
+                cursor = SkipQuotedLiteral(line, cursor);
+                continue;
+            }
+
+            if (line[cursor] == '[' && cursor + 1 < line.Length && line[cursor + 1] == '[')
+            {
+                var close = line.IndexOf("]]", cursor + 2, StringComparison.Ordinal);
+                cursor = close < 0 ? line.Length : close + 1;
+                continue;
+            }
+
+            if (!IsLuaIdentifierAt(line, cursor, "require"))
+                continue;
+
+            var argStart = cursor + "require".Length;
+            while (argStart < line.Length && char.IsWhiteSpace(line[argStart]))
+                argStart++;
+            if (argStart < line.Length && line[argStart] == '(')
+            {
+                argStart++;
+                while (argStart < line.Length && char.IsWhiteSpace(line[argStart]))
+                    argStart++;
+            }
+
+            if (argStart >= line.Length || line[argStart] is not ('"' or '\''))
+                continue;
+
+            var quote = line[argStart++];
+            var nameStart = argStart;
+            while (argStart < line.Length)
+            {
+                if (line[argStart] == '\\' && argStart + 1 < line.Length)
+                {
+                    argStart += 2;
+                    continue;
+                }
+
+                if (line[argStart] == quote)
+                    break;
+                argStart++;
+            }
+
+            if (argStart > nameStart)
+                yield return (line[nameStart..argStart], nameStart);
+            cursor = argStart;
+        }
+    }
+
+    private static int SkipQuotedLiteral(string line, int start)
+    {
+        var quote = line[start];
+        var cursor = start + 1;
+        while (cursor < line.Length)
+        {
+            if (line[cursor] == '\\' && cursor + 1 < line.Length)
+            {
+                cursor += 2;
+                continue;
+            }
+
+            if (line[cursor] == quote)
+                return cursor;
+            cursor++;
+        }
+
+        return line.Length;
+    }
+
+    private static bool IsLuaIdentifierAt(string line, int index, string identifier)
+    {
+        if (index < 0 || index + identifier.Length > line.Length)
+            return false;
+        if (string.CompareOrdinal(line, index, identifier, 0, identifier.Length) != 0)
+            return false;
+        if (index > 0 && IsLuaIdentifierPart(line[index - 1]))
+            return false;
+
+        var after = index + identifier.Length;
+        return after >= line.Length || !IsLuaIdentifierPart(line[after]);
+    }
+
+    private static bool IsLuaIdentifierPart(char ch) =>
+        ch == '_' || char.IsLetterOrDigit(ch);
+
+    private static string LastQualifiedSegment(string value)
+    {
+        var dot = value.LastIndexOf('.');
+        return dot >= 0 && dot + 1 < value.Length ? value[(dot + 1)..] : value;
+    }
+
+    private static void MaskRange(char[] chars, int start, int end)
+    {
+        for (var i = Math.Max(0, start); i < end && i < chars.Length; i++)
+            chars[i] = ' ';
     }
 }