Fix Lua and Ruby reference extraction

changelog.d/unreleased/1478.fixed.md

@@ -0,0 +1,20 @@
+---
+category: fixed
+issues:
+  - 1478
+affected:
+  - src/CodeIndex/Indexer/Symbols/SymbolExtractor.cs
+  - src/CodeIndex/Indexer/References/ReferenceExtractor.Core.cs
+  - src/CodeIndex/Indexer/References/Languages/LuaReferenceExtractor.cs
+  - src/CodeIndex/Indexer/References/Support/LanguageReferenceExtractionSupport.cs
+  - tests/CodeIndex.Tests/ReferenceExtractorLuaTests.cs
+  - tests/CodeIndex.Tests/SymbolExtractorTests.cs
+---
+
+## English
+
+- **Lua references now cover colon method calls and table field reads (#1478)** — Lua extraction now records `obj:method(...)` calls and table field read references, with tests covering method calls, table-key definitions, and vararg signatures.
+
+## 日本語
+
+- **Lua の参照抽出が colon メソッド呼び出しと table field 読み取りを扱うようになりました (#1478)** — Lua 抽出は `obj:method(...)` 呼び出しと table field 読み取り参照を記録し、method call、table-key 定義、vararg signature をテストで固定しました。

changelog.d/unreleased/1481.fixed.md

@@ -0,0 +1,16 @@
+---
+category: fixed
+issues:
+  - 1481
+affected:
+  - src/CodeIndex/Indexer/References/Languages/RubyReferenceExtractor.cs
+  - tests/CodeIndex.Tests/ReferenceExtractorRubyTests.cs
+---
+
+## English
+
+- **Ruby DSL and metaprogramming references are no longer limited to the static command allowlist (#1481)** — Ruby extraction now records symbol-literal targets from custom DSL commands plus `send`, `public_send`, and `define_method`.
+
+## 日本語
+
+- **Ruby DSL と metaprogramming の参照抽出が静的 command allowlist だけに制限されなくなりました (#1481)** — Ruby 抽出は custom DSL command と `send`、`public_send`、`define_method` の symbol literal target を記録するようになりました。

src/CodeIndex/Indexer/References/Languages/LuaReferenceExtractor.cs

@@ -32,19 +32,25 @@ public static void EmitTypePositionReferences(
     public static void EmitAdditionalCallReferences(
         string preparedLine,
         Action<string, int> addCallLikeReference,
+        List<ReferenceRecord> references,
+        HashSet<string> seen,
+        long fileId,
+        string context,
+        int lineNumber,
+        Func<int, SymbolRecord?> resolveContainerForColumn,
         IReadOnlySet<string>? definitionNames)
     {
         LanguageReferenceExtractionSupport.EmitAdditionalCallReferences(
             "lua",
             preparedLine,
             preparedLine,
             addCallLikeReference,
-            [],
-            [],
-            0,
-            string.Empty,
-            0,
-            _ => null,
+            references,
+            seen,
+            fileId,
+            context,
+            lineNumber,
+            resolveContainerForColumn,
             definitionNames);
     }
 }

src/CodeIndex/Indexer/References/Languages/RubyReferenceExtractor.cs

@@ -32,10 +32,19 @@ internal static class RubyReferenceExtractor
         "has_many", "has_one", "belongs_to", "composed_of",
     };
 
+    private static readonly HashSet<string> SymbolLiteralFirstArgumentCommandNames = new(StringComparer.Ordinal)
+    {
+        "send", "public_send", "define_method",
+    };
+
     private static readonly Regex CommandTargetTokenRegex = new(
         @"(?<![\w$@])(?<token>:(?:""(?:[^""\\]|\\.)*""|'(?:[^'\\]|\\.)*'|[A-Za-z_]\w*[?!]?)|[A-Za-z_]\w*(?:::[A-Za-z_]\w*)*|""(?:[^""\\]|\\.)*""|'(?:[^'\\]|\\.)*')",
         RegexOptions.Compiled);
 
+    private static readonly Regex SymbolLiteralFirstArgumentRegex = new(
+        @"(?<![\w$@])(?<name>send|public_send|define_method)\s*\(\s*(?<token>:(?:""(?:[^""\\]|\\.)*""|'(?:[^'\\]|\\.)*'|[A-Za-z_]\w*[?!]?))",
+        RegexOptions.Compiled);
+
     private static readonly Regex ClassNameOptionRegex = new(
         @"(?<![\w$@]):?class_name\s*(?::|=>)\s*(?<quote>['""])(?<name>[A-Za-z_]\w*(?:::[A-Za-z_]\w*)*)\k<quote>",
         RegexOptions.Compiled);
@@ -100,6 +109,15 @@ public static void EmitAdditionalCallReferences(
                 resolveContainerForCall);
         }
 
+        EmitSymbolLiteralFirstArgumentReferences(
+            preparedLine,
+            references,
+            seen,
+            fileId,
+            context,
+            lineNumber,
+            resolveContainerForCall);
+
         foreach (Match match in BlockCallRegex.Matches(preparedLine))
         {
             var name = match.Groups["name"].Value;
@@ -119,7 +137,9 @@ public static void EmitCommandTargetReferences(
         int lineNumber,
         Func<int, SymbolRecord?> resolveContainerForCall)
     {
-        if (!CommandTargetReferenceNames.Contains(name))
+        var isKnownCommand = CommandTargetReferenceNames.Contains(name);
+        var onlyFirstSymbolLiteral = SymbolLiteralFirstArgumentCommandNames.Contains(name) || !isKnownCommand;
+        if (!isKnownCommand && !onlyFirstSymbolLiteral)
             return;
 
         var argsStart = callIndex + name.Length;
@@ -168,6 +188,9 @@ public static void EmitCommandTargetReferences(
             if (IsHashOptionKey(tail, match, rawToken))
                 break;
 
+            if (onlyFirstSymbolLiteral && rawToken[0] != ':')
+                break;
+
             if (string.Equals(name, "raise", StringComparison.Ordinal))
             {
                 if (rawToken[0] == ':' || rawToken[0] == '\'' || rawToken[0] == '"')
@@ -212,6 +235,38 @@ public static void EmitCommandTargetReferences(
 
             if (CommandTargetSingleTokenNames.Contains(name))
                 break;
+            if (onlyFirstSymbolLiteral)
+                break;
+        }
+    }
+
+    private static void EmitSymbolLiteralFirstArgumentReferences(
+        string preparedLine,
+        List<ReferenceRecord> references,
+        HashSet<string> seen,
+        long fileId,
+        string context,
+        int lineNumber,
+        Func<int, SymbolRecord?> resolveContainerForCall)
+    {
+        foreach (Match match in SymbolLiteralFirstArgumentRegex.Matches(preparedLine))
+        {
+            var rawToken = match.Groups["token"].Value;
+            var token = NormalizeCommandTargetToken(rawToken);
+            if (string.IsNullOrWhiteSpace(token))
+                continue;
+
+            var tokenIndex = match.Groups["token"].Index;
+            ReferenceExtractor.AddReference(
+                references,
+                seen,
+                fileId,
+                token,
+                tokenIndex,
+                "reference",
+                context,
+                lineNumber,
+                resolveContainerForCall(tokenIndex));
         }
     }
 

src/CodeIndex/Indexer/References/ReferenceExtractor.Core.cs

@@ -1524,7 +1524,7 @@ void AddGradleDslReference(string name, int callIndex)
                 else if (language == "elixir")
                     ElixirReferenceExtractor.EmitAdditionalCallReferences(preparedLine, AddCallLikeReference, definitionNames);
                 else if (language == "lua")
-                    LuaReferenceExtractor.EmitAdditionalCallReferences(preparedLine, AddCallLikeReference, definitionNames);
+                    LuaReferenceExtractor.EmitAdditionalCallReferences(preparedLine, AddCallLikeReference, references, seen, fileId, context, lineNumber, ResolveContainerForCall, definitionNames);
                 else if (language == "smalltalk")
                     SmalltalkReferenceExtractor.EmitAdditionalCallReferences(preparedLine, AddCallLikeReference, definitionNames);
                 else if (language == "vb")

src/CodeIndex/Indexer/References/Support/LanguageReferenceExtractionSupport.cs

@@ -517,6 +517,12 @@ internal static class LanguageReferenceExtractionSupport
     private static readonly Regex LuaCommandCallRegex = new(
         @"^\s*(?<name>[A-Za-z_]\w*(?:\.[A-Za-z_]\w*)?)\s+(?=[""'{A-Za-z_])",
         RegexOptions.Compiled | RegexOptions.CultureInvariant);
+    private static readonly Regex LuaColonCallRegex = new(
+        @"(?<![\w.])(?:[A-Za-z_]\w*(?:\.[A-Za-z_]\w*)*):(?<name>[A-Za-z_]\w*)\s*\(",
+        RegexOptions.Compiled | RegexOptions.CultureInvariant);
+    private static readonly Regex LuaTableFieldReferenceRegex = new(
+        @"(?<![\w.])(?:[A-Za-z_]\w*\.)+(?<name>[A-Za-z_]\w*)\b(?!\s*(?:=|function\b|\())",
+        RegexOptions.Compiled | RegexOptions.CultureInvariant);
 
     private static readonly Regex SmalltalkClassDeclarationRegex = new(
         @"^\s*(?:(?:[A-Za-z_]\w*)\s+subclass:|Class\s+named:|Object\s+subclass:)\s*#",
@@ -624,7 +630,7 @@ public static void EmitAdditionalCallReferences(
                 EmitElixirParenlessCallReferences(preparedLine, addCallLikeReference, definitionNames);
                 break;
             case "lua":
-                EmitLuaCommandCallReferences(preparedLine, addCallLikeReference, definitionNames);
+                EmitLuaCallReferences(preparedLine, addCallLikeReference, references, seen, fileId, context, lineNumber, resolveContainerForColumn, definitionNames);
                 break;
             case "smalltalk":
                 EmitSmalltalkMessageReferences(preparedLine, addCallLikeReference, definitionNames);
@@ -4570,16 +4576,48 @@ private static void EmitElixirParenlessCallReferences(string preparedLine, Actio
         }
     }
 
-    private static void EmitLuaCommandCallReferences(string preparedLine, Action<string, int> addCallLikeReference, IReadOnlySet<string>? definitionNames)
+    private static void EmitLuaCallReferences(
+        string preparedLine,
+        Action<string, int> addCallLikeReference,
+        List<ReferenceRecord> references,
+        HashSet<string> seen,
+        long fileId,
+        string context,
+        int lineNumber,
+        Func<int, SymbolRecord?> resolveContainerForColumn,
+        IReadOnlySet<string>? definitionNames)
     {
         var match = LuaCommandCallRegex.Match(preparedLine);
-        if (!match.Success)
-            return;
+        if (match.Success)
+        {
+            var name = LastQualifiedSegment(match.Groups["name"].Value);
+            if (definitionNames?.Contains(name) != true)
+                addCallLikeReference(name, match.Groups["name"].Index + match.Groups["name"].Value.LastIndexOf(name, StringComparison.Ordinal));
+        }
 
-        var name = LastQualifiedSegment(match.Groups["name"].Value);
-        if (definitionNames?.Contains(name) == true)
-            return;
-        addCallLikeReference(name, match.Groups["name"].Index + match.Groups["name"].Value.LastIndexOf(name, StringComparison.Ordinal));
+        foreach (Match colonMatch in LuaColonCallRegex.Matches(preparedLine))
+        {
+            var name = colonMatch.Groups["name"].Value;
+            if (definitionNames?.Contains(name) == true)
+                continue;
+            addCallLikeReference(name, colonMatch.Groups["name"].Index);
+        }
+
+        foreach (Match fieldMatch in LuaTableFieldReferenceRegex.Matches(preparedLine))
+        {
+            var trimmed = preparedLine.TrimStart();
+            if (trimmed.StartsWith("function ", StringComparison.Ordinal)
+                || trimmed.StartsWith("local function ", StringComparison.Ordinal))
+            {
+                break;
+            }
+
+            var name = fieldMatch.Groups["name"].Value;
+            if (definitionNames?.Contains(name) == true)
+                continue;
+            var index = fieldMatch.Groups["name"].Index;
+            ReferenceExtractor.AddReference(references, seen, fileId, name, index, "reference", context, lineNumber, resolveContainerForColumn(index));
+        }
     }
 
     private static void EmitSmalltalkMessageReferences(string preparedLine, Action<string, int> addCallLikeReference, IReadOnlySet<string>? definitionNames)

src/CodeIndex/Indexer/Symbols/SymbolExtractor.cs

@@ -1757,9 +1757,9 @@ private enum JavaScriptTypeScriptFunctionHeaderConsumeResult
         ],
         ["lua"] =
         [
-            new("function", new Regex(@"^\s*(?:local\s+)?function\s+(?<name>[\w.:]+)\s*\(", RegexOptions.Compiled), BodyStyle.None),
-            new("function", new Regex(@"^\s*local\s+(?<name>[\w]+)\s*=\s*function\s*\(", RegexOptions.Compiled), BodyStyle.None),
-            new("function", new Regex(@"^\s*(?<name>[\w]+(?:[.:][\w]+)+)\s*=\s*function\s*\(", RegexOptions.Compiled), BodyStyle.None),
+            new("function", new Regex(@"^\s*(?:local\s+)?function\s+(?<name>[\w.:]+)\s*\(", RegexOptions.Compiled), BodyStyle.ElixirEnd),
+            new("function", new Regex(@"^\s*local\s+(?<name>[\w]+)\s*=\s*function\s*\(", RegexOptions.Compiled), BodyStyle.ElixirEnd),
+            new("function", new Regex(@"^\s*(?<name>[\w]+(?:[.:][\w]+)+)\s*=\s*function\s*\(", RegexOptions.Compiled), BodyStyle.ElixirEnd),
             new("import",   new Regex(@"^\s*(?:local\s+\w+\s*=\s*)?require\s*\(?['""](?<name>[^'""]+)['""]", RegexOptions.Compiled), BodyStyle.None),
         ],
         ["elixir"] =

tests/CodeIndex.Tests/ReferenceExtractorLuaTests.cs

@@ -0,0 +1,29 @@
+using CodeIndex.Indexer;
+
+namespace CodeIndex.Tests;
+
+public class ReferenceExtractorLuaTests
+{
+    [Fact]
+    public void Extract_Lua_EmitsColonCallsAndTableFieldReferences()
+    {
+        const string content = """
+            local M = {}
+
+            function M.work(self, x, ...)
+                local value = self.runner.status
+                return self.runner:run(x, ...)
+            end
+            """;
+
+        var symbols = SymbolExtractor.Extract(1, "lua", content);
+        var references = ReferenceExtractor.Extract(1, "lua", content, symbols);
+
+        Assert.Contains(references, reference =>
+            reference.SymbolName == "run"
+            && reference.ReferenceKind == "call");
+        Assert.Contains(references, reference =>
+            reference.SymbolName == "status"
+            && reference.ReferenceKind == "reference");
+    }
+}

tests/CodeIndex.Tests/ReferenceExtractorRubyTests.cs

@@ -0,0 +1,39 @@
+using CodeIndex.Indexer;
+
+namespace CodeIndex.Tests;
+
+public class ReferenceExtractorRubyTests
+{
+    [Fact]
+    public void Extract_Ruby_EmitsCustomDslAndMetaprogrammingSymbolReferences()
+    {
+        const string content = """
+            class Order
+              custom_callback :notify
+              define_method(:nice_name) { name }
+              send(:cleanup) if respond_to?(:cleanup)
+              public_send(:deliver)
+            end
+            """;
+
+        var symbols = SymbolExtractor.Extract(1, "ruby", content);
+        var references = ReferenceExtractor.Extract(1, "ruby", content, symbols);
+
+        Assert.Contains(references, reference =>
+            reference.SymbolName == "notify"
+            && reference.ReferenceKind == "reference"
+            && reference.ContainerName == "Order");
+        Assert.Contains(references, reference =>
+            reference.SymbolName == "nice_name"
+            && reference.ReferenceKind == "reference"
+            && reference.ContainerName == "Order");
+        Assert.Contains(references, reference =>
+            reference.SymbolName == "cleanup"
+            && reference.ReferenceKind == "reference"
+            && reference.ContainerName == "Order");
+        Assert.Contains(references, reference =>
+            reference.SymbolName == "deliver"
+            && reference.ReferenceKind == "reference"
+            && reference.ContainerName == "Order");
+    }
+}

tests/CodeIndex.Tests/SymbolExtractorTests.cs

@@ -20776,6 +20776,10 @@ function M.dot_form(arg)
               return arg
             end
 
+            function M.deep.table_key(arg, ...)
+              return arg
+            end
+
             local function top_local(x)
               return x
             end
@@ -20791,6 +20795,7 @@ function plain_function(a)
         Assert.Contains(symbols, s => s.Kind == "function" && s.Name == "M.named");
         Assert.Contains(symbols, s => s.Kind == "function" && s.Name == "M:method_form");
         Assert.Contains(symbols, s => s.Kind == "function" && s.Name == "M.dot_form");
+        Assert.Contains(symbols, s => s.Kind == "function" && s.Name == "M.deep.table_key");
         Assert.Contains(symbols, s => s.Kind == "function" && s.Name == "top_local");
         Assert.Contains(symbols, s => s.Kind == "function" && s.Name == "plain_function");
     }