diff --git a/changelog.d/unreleased/1478.fixed.md b/changelog.d/unreleased/1478.fixed.md new file mode 100644 index 0000000000..52f2546a80 --- /dev/null +++ b/changelog.d/unreleased/1478.fixed.md @@ -0,0 +1,20 @@ +--- +category: fixed +issues: + - 1478 +affected: + - src/CodeIndex/Indexer/Symbols/SymbolExtractor.cs + - src/CodeIndex/Indexer/References/ReferenceExtractor.Core.cs + - src/CodeIndex/Indexer/References/Languages/LuaReferenceExtractor.cs + - src/CodeIndex/Indexer/References/Support/LanguageReferenceExtractionSupport.cs + - tests/CodeIndex.Tests/ReferenceExtractorLuaTests.cs + - tests/CodeIndex.Tests/SymbolExtractorTests.cs +--- + +## English + +- **Lua references now cover colon method calls and table field reads (#1478)** — Lua extraction now records `obj:method(...)` calls and table field read references, with tests covering method calls, table-key definitions, and vararg signatures. + +## 日本語 + +- **Lua の参照抽出が colon メソッド呼び出しと table field 読み取りを扱うようになりました (#1478)** — Lua 抽出は `obj:method(...)` 呼び出しと table field 読み取り参照を記録し、method call、table-key 定義、vararg signature をテストで固定しました。 diff --git a/changelog.d/unreleased/1481.fixed.md b/changelog.d/unreleased/1481.fixed.md new file mode 100644 index 0000000000..33a4e96c75 --- /dev/null +++ b/changelog.d/unreleased/1481.fixed.md @@ -0,0 +1,16 @@ +--- +category: fixed +issues: + - 1481 +affected: + - src/CodeIndex/Indexer/References/Languages/RubyReferenceExtractor.cs + - tests/CodeIndex.Tests/ReferenceExtractorRubyTests.cs +--- + +## English + +- **Ruby DSL and metaprogramming references are no longer limited to the static command allowlist (#1481)** — Ruby extraction now records symbol-literal targets from custom DSL commands plus `send`, `public_send`, and `define_method`. + +## 日本語 + +- **Ruby DSL と metaprogramming の参照抽出が静的 command allowlist だけに制限されなくなりました (#1481)** — Ruby 抽出は custom DSL command と `send`、`public_send`、`define_method` の symbol literal target を記録するようになりました。 diff --git a/src/CodeIndex/Indexer/References/Languages/LuaReferenceExtractor.cs b/src/CodeIndex/Indexer/References/Languages/LuaReferenceExtractor.cs index a1b76195f4..822e1fd12e 100644 --- a/src/CodeIndex/Indexer/References/Languages/LuaReferenceExtractor.cs +++ b/src/CodeIndex/Indexer/References/Languages/LuaReferenceExtractor.cs @@ -32,6 +32,12 @@ public static void EmitTypePositionReferences( public static void EmitAdditionalCallReferences( string preparedLine, Action addCallLikeReference, + List references, + HashSet seen, + long fileId, + string context, + int lineNumber, + Func resolveContainerForColumn, IReadOnlySet? definitionNames) { LanguageReferenceExtractionSupport.EmitAdditionalCallReferences( @@ -39,12 +45,12 @@ public static void EmitAdditionalCallReferences( preparedLine, preparedLine, addCallLikeReference, - [], - [], - 0, - string.Empty, - 0, - _ => null, + references, + seen, + fileId, + context, + lineNumber, + resolveContainerForColumn, definitionNames); } } diff --git a/src/CodeIndex/Indexer/References/Languages/RubyReferenceExtractor.cs b/src/CodeIndex/Indexer/References/Languages/RubyReferenceExtractor.cs index ff86d11e52..9eee202330 100644 --- a/src/CodeIndex/Indexer/References/Languages/RubyReferenceExtractor.cs +++ b/src/CodeIndex/Indexer/References/Languages/RubyReferenceExtractor.cs @@ -32,10 +32,19 @@ internal static class RubyReferenceExtractor "has_many", "has_one", "belongs_to", "composed_of", }; + private static readonly HashSet SymbolLiteralFirstArgumentCommandNames = new(StringComparer.Ordinal) + { + "send", "public_send", "define_method", + }; + private static readonly Regex CommandTargetTokenRegex = new( @"(?:(?:""(?:[^""\\]|\\.)*""|'(?:[^'\\]|\\.)*'|[A-Za-z_]\w*[?!]?)|[A-Za-z_]\w*(?:::[A-Za-z_]\w*)*|""(?:[^""\\]|\\.)*""|'(?:[^'\\]|\\.)*')", RegexOptions.Compiled); + private static readonly Regex SymbolLiteralFirstArgumentRegex = new( + @"(?send|public_send|define_method)\s*\(\s*(?:(?:""(?:[^""\\]|\\.)*""|'(?:[^'\\]|\\.)*'|[A-Za-z_]\w*[?!]?))", + RegexOptions.Compiled); + private static readonly Regex ClassNameOptionRegex = new( @"(?)\s*(?['""])(?[A-Za-z_]\w*(?:::[A-Za-z_]\w*)*)\k", RegexOptions.Compiled); @@ -100,6 +109,15 @@ public static void EmitAdditionalCallReferences( resolveContainerForCall); } + EmitSymbolLiteralFirstArgumentReferences( + preparedLine, + references, + seen, + fileId, + context, + lineNumber, + resolveContainerForCall); + foreach (Match match in BlockCallRegex.Matches(preparedLine)) { var name = match.Groups["name"].Value; @@ -119,7 +137,9 @@ public static void EmitCommandTargetReferences( int lineNumber, Func resolveContainerForCall) { - if (!CommandTargetReferenceNames.Contains(name)) + var isKnownCommand = CommandTargetReferenceNames.Contains(name); + var onlyFirstSymbolLiteral = SymbolLiteralFirstArgumentCommandNames.Contains(name) || !isKnownCommand; + if (!isKnownCommand && !onlyFirstSymbolLiteral) return; var argsStart = callIndex + name.Length; @@ -168,6 +188,9 @@ public static void EmitCommandTargetReferences( if (IsHashOptionKey(tail, match, rawToken)) break; + if (onlyFirstSymbolLiteral && rawToken[0] != ':') + break; + if (string.Equals(name, "raise", StringComparison.Ordinal)) { if (rawToken[0] == ':' || rawToken[0] == '\'' || rawToken[0] == '"') @@ -212,6 +235,38 @@ public static void EmitCommandTargetReferences( if (CommandTargetSingleTokenNames.Contains(name)) break; + if (onlyFirstSymbolLiteral) + break; + } + } + + private static void EmitSymbolLiteralFirstArgumentReferences( + string preparedLine, + List references, + HashSet seen, + long fileId, + string context, + int lineNumber, + Func resolveContainerForCall) + { + foreach (Match match in SymbolLiteralFirstArgumentRegex.Matches(preparedLine)) + { + var rawToken = match.Groups["token"].Value; + var token = NormalizeCommandTargetToken(rawToken); + if (string.IsNullOrWhiteSpace(token)) + continue; + + var tokenIndex = match.Groups["token"].Index; + ReferenceExtractor.AddReference( + references, + seen, + fileId, + token, + tokenIndex, + "reference", + context, + lineNumber, + resolveContainerForCall(tokenIndex)); } } diff --git a/src/CodeIndex/Indexer/References/ReferenceExtractor.Core.cs b/src/CodeIndex/Indexer/References/ReferenceExtractor.Core.cs index 2880954945..62fc2289b5 100644 --- a/src/CodeIndex/Indexer/References/ReferenceExtractor.Core.cs +++ b/src/CodeIndex/Indexer/References/ReferenceExtractor.Core.cs @@ -1524,7 +1524,7 @@ void AddGradleDslReference(string name, int callIndex) else if (language == "elixir") ElixirReferenceExtractor.EmitAdditionalCallReferences(preparedLine, AddCallLikeReference, definitionNames); else if (language == "lua") - LuaReferenceExtractor.EmitAdditionalCallReferences(preparedLine, AddCallLikeReference, definitionNames); + LuaReferenceExtractor.EmitAdditionalCallReferences(preparedLine, AddCallLikeReference, references, seen, fileId, context, lineNumber, ResolveContainerForCall, definitionNames); else if (language == "smalltalk") SmalltalkReferenceExtractor.EmitAdditionalCallReferences(preparedLine, AddCallLikeReference, definitionNames); else if (language == "vb") diff --git a/src/CodeIndex/Indexer/References/Support/LanguageReferenceExtractionSupport.cs b/src/CodeIndex/Indexer/References/Support/LanguageReferenceExtractionSupport.cs index e8fca13f04..2e93b8928a 100644 --- a/src/CodeIndex/Indexer/References/Support/LanguageReferenceExtractionSupport.cs +++ b/src/CodeIndex/Indexer/References/Support/LanguageReferenceExtractionSupport.cs @@ -517,6 +517,12 @@ internal static class LanguageReferenceExtractionSupport private static readonly Regex LuaCommandCallRegex = new( @"^\s*(?[A-Za-z_]\w*(?:\.[A-Za-z_]\w*)?)\s+(?=[""'{A-Za-z_])", RegexOptions.Compiled | RegexOptions.CultureInvariant); + private static readonly Regex LuaColonCallRegex = new( + @"(?[A-Za-z_]\w*)\s*\(", + RegexOptions.Compiled | RegexOptions.CultureInvariant); + private static readonly Regex LuaTableFieldReferenceRegex = new( + @"(?[A-Za-z_]\w*)\b(?!\s*(?:=|function\b|\())", + RegexOptions.Compiled | RegexOptions.CultureInvariant); private static readonly Regex SmalltalkClassDeclarationRegex = new( @"^\s*(?:(?:[A-Za-z_]\w*)\s+subclass:|Class\s+named:|Object\s+subclass:)\s*#", @@ -624,7 +630,7 @@ public static void EmitAdditionalCallReferences( EmitElixirParenlessCallReferences(preparedLine, addCallLikeReference, definitionNames); break; case "lua": - EmitLuaCommandCallReferences(preparedLine, addCallLikeReference, definitionNames); + EmitLuaCallReferences(preparedLine, addCallLikeReference, references, seen, fileId, context, lineNumber, resolveContainerForColumn, definitionNames); break; case "smalltalk": EmitSmalltalkMessageReferences(preparedLine, addCallLikeReference, definitionNames); @@ -4570,16 +4576,48 @@ private static void EmitElixirParenlessCallReferences(string preparedLine, Actio } } - private static void EmitLuaCommandCallReferences(string preparedLine, Action addCallLikeReference, IReadOnlySet? definitionNames) + private static void EmitLuaCallReferences( + string preparedLine, + Action addCallLikeReference, + List references, + HashSet seen, + long fileId, + string context, + int lineNumber, + Func resolveContainerForColumn, + IReadOnlySet? definitionNames) { var match = LuaCommandCallRegex.Match(preparedLine); - if (!match.Success) - return; + if (match.Success) + { + var name = LastQualifiedSegment(match.Groups["name"].Value); + if (definitionNames?.Contains(name) != true) + addCallLikeReference(name, match.Groups["name"].Index + match.Groups["name"].Value.LastIndexOf(name, StringComparison.Ordinal)); + } - var name = LastQualifiedSegment(match.Groups["name"].Value); - if (definitionNames?.Contains(name) == true) - return; - addCallLikeReference(name, match.Groups["name"].Index + match.Groups["name"].Value.LastIndexOf(name, StringComparison.Ordinal)); + foreach (Match colonMatch in LuaColonCallRegex.Matches(preparedLine)) + { + var name = colonMatch.Groups["name"].Value; + if (definitionNames?.Contains(name) == true) + continue; + addCallLikeReference(name, colonMatch.Groups["name"].Index); + } + + foreach (Match fieldMatch in LuaTableFieldReferenceRegex.Matches(preparedLine)) + { + var trimmed = preparedLine.TrimStart(); + if (trimmed.StartsWith("function ", StringComparison.Ordinal) + || trimmed.StartsWith("local function ", StringComparison.Ordinal)) + { + break; + } + + var name = fieldMatch.Groups["name"].Value; + if (definitionNames?.Contains(name) == true) + continue; + var index = fieldMatch.Groups["name"].Index; + ReferenceExtractor.AddReference(references, seen, fileId, name, index, "reference", context, lineNumber, resolveContainerForColumn(index)); + } } private static void EmitSmalltalkMessageReferences(string preparedLine, Action addCallLikeReference, IReadOnlySet? definitionNames) diff --git a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.cs b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.cs index 1bf55a6cc4..1019f4cab9 100644 --- a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.cs +++ b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.cs @@ -1757,9 +1757,9 @@ private enum JavaScriptTypeScriptFunctionHeaderConsumeResult ], ["lua"] = [ - new("function", new Regex(@"^\s*(?:local\s+)?function\s+(?[\w.:]+)\s*\(", RegexOptions.Compiled), BodyStyle.None), - new("function", new Regex(@"^\s*local\s+(?[\w]+)\s*=\s*function\s*\(", RegexOptions.Compiled), BodyStyle.None), - new("function", new Regex(@"^\s*(?[\w]+(?:[.:][\w]+)+)\s*=\s*function\s*\(", RegexOptions.Compiled), BodyStyle.None), + new("function", new Regex(@"^\s*(?:local\s+)?function\s+(?[\w.:]+)\s*\(", RegexOptions.Compiled), BodyStyle.ElixirEnd), + new("function", new Regex(@"^\s*local\s+(?[\w]+)\s*=\s*function\s*\(", RegexOptions.Compiled), BodyStyle.ElixirEnd), + new("function", new Regex(@"^\s*(?[\w]+(?:[.:][\w]+)+)\s*=\s*function\s*\(", RegexOptions.Compiled), BodyStyle.ElixirEnd), new("import", new Regex(@"^\s*(?:local\s+\w+\s*=\s*)?require\s*\(?['""](?[^'""]+)['""]", RegexOptions.Compiled), BodyStyle.None), ], ["elixir"] = diff --git a/tests/CodeIndex.Tests/ReferenceExtractorLuaTests.cs b/tests/CodeIndex.Tests/ReferenceExtractorLuaTests.cs new file mode 100644 index 0000000000..054caac270 --- /dev/null +++ b/tests/CodeIndex.Tests/ReferenceExtractorLuaTests.cs @@ -0,0 +1,29 @@ +using CodeIndex.Indexer; + +namespace CodeIndex.Tests; + +public class ReferenceExtractorLuaTests +{ + [Fact] + public void Extract_Lua_EmitsColonCallsAndTableFieldReferences() + { + const string content = """ + local M = {} + + function M.work(self, x, ...) + local value = self.runner.status + return self.runner:run(x, ...) + end + """; + + var symbols = SymbolExtractor.Extract(1, "lua", content); + var references = ReferenceExtractor.Extract(1, "lua", content, symbols); + + Assert.Contains(references, reference => + reference.SymbolName == "run" + && reference.ReferenceKind == "call"); + Assert.Contains(references, reference => + reference.SymbolName == "status" + && reference.ReferenceKind == "reference"); + } +} diff --git a/tests/CodeIndex.Tests/ReferenceExtractorRubyTests.cs b/tests/CodeIndex.Tests/ReferenceExtractorRubyTests.cs new file mode 100644 index 0000000000..9ee91441b1 --- /dev/null +++ b/tests/CodeIndex.Tests/ReferenceExtractorRubyTests.cs @@ -0,0 +1,39 @@ +using CodeIndex.Indexer; + +namespace CodeIndex.Tests; + +public class ReferenceExtractorRubyTests +{ + [Fact] + public void Extract_Ruby_EmitsCustomDslAndMetaprogrammingSymbolReferences() + { + const string content = """ + class Order + custom_callback :notify + define_method(:nice_name) { name } + send(:cleanup) if respond_to?(:cleanup) + public_send(:deliver) + end + """; + + var symbols = SymbolExtractor.Extract(1, "ruby", content); + var references = ReferenceExtractor.Extract(1, "ruby", content, symbols); + + Assert.Contains(references, reference => + reference.SymbolName == "notify" + && reference.ReferenceKind == "reference" + && reference.ContainerName == "Order"); + Assert.Contains(references, reference => + reference.SymbolName == "nice_name" + && reference.ReferenceKind == "reference" + && reference.ContainerName == "Order"); + Assert.Contains(references, reference => + reference.SymbolName == "cleanup" + && reference.ReferenceKind == "reference" + && reference.ContainerName == "Order"); + Assert.Contains(references, reference => + reference.SymbolName == "deliver" + && reference.ReferenceKind == "reference" + && reference.ContainerName == "Order"); + } +} diff --git a/tests/CodeIndex.Tests/SymbolExtractorTests.cs b/tests/CodeIndex.Tests/SymbolExtractorTests.cs index 104a10ed47..2d8d779614 100644 --- a/tests/CodeIndex.Tests/SymbolExtractorTests.cs +++ b/tests/CodeIndex.Tests/SymbolExtractorTests.cs @@ -20776,6 +20776,10 @@ function M.dot_form(arg) return arg end + function M.deep.table_key(arg, ...) + return arg + end + local function top_local(x) return x end @@ -20791,6 +20795,7 @@ function plain_function(a) Assert.Contains(symbols, s => s.Kind == "function" && s.Name == "M.named"); Assert.Contains(symbols, s => s.Kind == "function" && s.Name == "M:method_form"); Assert.Contains(symbols, s => s.Kind == "function" && s.Name == "M.dot_form"); + Assert.Contains(symbols, s => s.Kind == "function" && s.Name == "M.deep.table_key"); Assert.Contains(symbols, s => s.Kind == "function" && s.Name == "top_local"); Assert.Contains(symbols, s => s.Kind == "function" && s.Name == "plain_function"); }