diff --git a/changelog.d/unreleased/3381.internal.md b/changelog.d/unreleased/3381.internal.md new file mode 100644 index 0000000000..a3916f5fc8 --- /dev/null +++ b/changelog.d/unreleased/3381.internal.md @@ -0,0 +1,16 @@ +--- +category: internal +issues: + - 3381 +affected: + - src/CodeIndex/Indexer/Symbols/SymbolExtractor.cs + - src/CodeIndex/Indexer/Symbols/SymbolExtractor.CSharpPatterns.cs +--- + +## English + +- **Split C# symbol pattern definitions out of the central symbol extractor (#3381)** — C# declaration and type regex constants now live in a dedicated partial module, reducing the mixed-language pattern surface in `SymbolExtractor.cs` without changing extraction behavior. + +## 日本語 + +- **C# シンボル pattern 定義を central symbol extractor から分離しました (#3381)** — C# の宣言・型 regex 定数を専用 partial module に移し、抽出挙動は変えずに `SymbolExtractor.cs` の多言語 pattern 混在を減らしました。 diff --git a/changelog.d/unreleased/3383.internal.md b/changelog.d/unreleased/3383.internal.md new file mode 100644 index 0000000000..ea83af1560 --- /dev/null +++ b/changelog.d/unreleased/3383.internal.md @@ -0,0 +1,16 @@ +--- +category: internal +issues: + - 3383 +affected: + - tests/CodeIndex.Tests/SymbolExtractorTests.cs + - tests/CodeIndex.Tests/SymbolExtractorLuaTests.cs +--- + +## English + +- **Split Lua symbol extractor coverage into a language-specific test file (#3383)** — Moved the Lua symbol extraction regression case out of the mega `SymbolExtractorTests.cs` file and into `SymbolExtractorLuaTests.cs` for more targeted validation and review. + +## 日本語 + +- **Lua symbol extractor coverage を言語別 test file に分離しました (#3383)** — Lua の symbol extraction regression case を巨大な `SymbolExtractorTests.cs` から `SymbolExtractorLuaTests.cs` に移し、より対象を絞った検証と review をしやすくしました。 diff --git a/changelog.d/unreleased/3421.internal.md b/changelog.d/unreleased/3421.internal.md new file mode 100644 index 0000000000..467f7e1dbd --- /dev/null +++ b/changelog.d/unreleased/3421.internal.md @@ -0,0 +1,16 @@ +--- +category: internal +issues: + - 3421 +affected: + - src/CodeIndex/Indexer/References/Support/LanguageReferenceExtractionSupport.cs + - src/CodeIndex/Indexer/References/Languages/LuaReferenceExtractor.cs +--- + +## English + +- **Moved Lua reference extraction helpers into the Lua reference extractor (#3421)** — Lua require/type, colon-call, table-field, and long-bracket masking logic now live in the language module instead of the shared support class, reducing the central helper surface without changing extraction behavior. + +## 日本語 + +- **Lua 参照抽出 helper を Lua reference extractor へ移しました (#3421)** — Lua の require/type、colon-call、table-field、long-bracket masking の処理を共有 support class ではなく言語 module に置き、抽出挙動は変えずに central helper surface を減らしました。 diff --git a/src/CodeIndex/Indexer/References/Languages/LuaReferenceExtractor.cs b/src/CodeIndex/Indexer/References/Languages/LuaReferenceExtractor.cs index 822e1fd12e..19a4fc098e 100644 --- a/src/CodeIndex/Indexer/References/Languages/LuaReferenceExtractor.cs +++ b/src/CodeIndex/Indexer/References/Languages/LuaReferenceExtractor.cs @@ -1,11 +1,82 @@ +using System.Text.RegularExpressions; +using Regex = CodeIndex.Indexer.BoundedRegex; using CodeIndex.Models; namespace CodeIndex.Indexer; internal static class LuaReferenceExtractor { + private static readonly Regex LuaRequireRegex = new( + @"\brequire\s*\(?\s*[""'](?[^""']+)[""']", + RegexOptions.Compiled | RegexOptions.CultureInvariant); + private static readonly Regex LuaCommandCallRegex = new( + @"^\s*(?[A-Za-z_]\w*(?:\.[A-Za-z_]\w*)?)\s+(?=[""'{A-Za-z_])", + RegexOptions.Compiled | RegexOptions.CultureInvariant); + private static readonly Regex LuaColonCallRegex = new( + @"(?[A-Za-z_]\w*)\s*\(", + RegexOptions.Compiled | RegexOptions.CultureInvariant); + private static readonly Regex LuaTableFieldReferenceRegex = new( + @"(?[A-Za-z_]\w*)\b(?!\s*(?:=|function\b|\())", + RegexOptions.Compiled | RegexOptions.CultureInvariant); + public static string[] MaskLongCommentAndStringLines(IReadOnlyList originalLines) - => LanguageReferenceExtractionSupport.MaskLuaLongCommentAndStringLines(originalLines); + { + var result = new string[originalLines.Count]; + var longTextEqualsCount = -1; + + for (var lineIndex = 0; lineIndex < originalLines.Count; lineIndex++) + { + var line = originalLines[lineIndex]; + var chars = line.ToCharArray(); + for (var cursor = 0; cursor < chars.Length; cursor++) + { + if (longTextEqualsCount >= 0) + { + if (TryGetLuaLongBracketClose(line, cursor, longTextEqualsCount, out var closeLength)) + { + MaskRange(chars, cursor, cursor + closeLength); + cursor += closeLength - 1; + longTextEqualsCount = -1; + continue; + } + + chars[cursor] = ' '; + continue; + } + + if (chars[cursor] is '"' or '\'') + { + cursor = SkipQuotedLiteral(line, cursor); + continue; + } + + if (chars[cursor] == '-' + && cursor + 2 < chars.Length + && chars[cursor + 1] == '-' + && TryGetLuaLongBracketOpen(line, cursor + 2, out var commentEqualsCount, out var commentOpenLength)) + { + MaskRange(chars, cursor, cursor + 2 + commentOpenLength); + cursor += 1 + commentOpenLength; + longTextEqualsCount = commentEqualsCount; + continue; + } + + if (chars[cursor] == '-' && cursor + 1 < chars.Length && chars[cursor + 1] == '-') + break; + + if (TryGetLuaLongBracketOpen(line, cursor, out var stringEqualsCount, out var stringOpenLength)) + { + MaskRange(chars, cursor, cursor + stringOpenLength); + cursor += stringOpenLength - 1; + longTextEqualsCount = stringEqualsCount; + } + } + + result[lineIndex] = new string(chars); + } + + return result; + } public static void EmitTypePositionReferences( string originalLine, @@ -16,17 +87,8 @@ public static void EmitTypePositionReferences( int lineNumber, SymbolRecord? container) { - LanguageReferenceExtractionSupport.EmitTypePositionReferences( - "lua", - originalLine, - originalLine, - references, - seen, - fileId, - context, - lineNumber, - _ => container, - container); + foreach (var (name, index) in EnumerateLuaRequireReferences(originalLine)) + ReferenceExtractor.AddReference(references, seen, fileId, name, index, "type_reference", context, lineNumber, container); } public static void EmitAdditionalCallReferences( @@ -40,17 +102,183 @@ public static void EmitAdditionalCallReferences( Func resolveContainerForColumn, IReadOnlySet? definitionNames) { - LanguageReferenceExtractionSupport.EmitAdditionalCallReferences( - "lua", - preparedLine, - preparedLine, - addCallLikeReference, - references, - seen, - fileId, - context, - lineNumber, - resolveContainerForColumn, - definitionNames); + var match = LuaCommandCallRegex.Match(preparedLine); + if (match.Success) + { + var name = LastQualifiedSegment(match.Groups["name"].Value); + if (definitionNames?.Contains(name) != true) + addCallLikeReference(name, match.Groups["name"].Index + match.Groups["name"].Value.LastIndexOf(name, StringComparison.Ordinal)); + } + + foreach (Match colonMatch in LuaColonCallRegex.Matches(preparedLine)) + { + var name = colonMatch.Groups["name"].Value; + if (definitionNames?.Contains(name) == true) + continue; + addCallLikeReference(name, colonMatch.Groups["name"].Index); + } + + foreach (Match fieldMatch in LuaTableFieldReferenceRegex.Matches(preparedLine)) + { + var trimmed = preparedLine.TrimStart(); + if (trimmed.StartsWith("function ", StringComparison.Ordinal) + || trimmed.StartsWith("local function ", StringComparison.Ordinal)) + { + break; + } + + var name = fieldMatch.Groups["name"].Value; + if (definitionNames?.Contains(name) == true) + continue; + var index = fieldMatch.Groups["name"].Index; + ReferenceExtractor.AddReference(references, seen, fileId, name, index, "reference", context, lineNumber, resolveContainerForColumn(index)); + } + } + + private static bool TryGetLuaLongBracketOpen(string line, int start, out int equalsCount, out int length) + { + equalsCount = 0; + length = 0; + if (start < 0 || start >= line.Length || line[start] != '[') + return false; + + var cursor = start + 1; + while (cursor < line.Length && line[cursor] == '=') + { + equalsCount++; + cursor++; + } + + if (cursor >= line.Length || line[cursor] != '[') + return false; + + length = cursor - start + 1; + return true; + } + + private static bool TryGetLuaLongBracketClose(string line, int start, int equalsCount, out int length) + { + length = 0; + if (start < 0 || start >= line.Length || line[start] != ']') + return false; + + var cursor = start + 1; + for (var i = 0; i < equalsCount; i++) + { + if (cursor >= line.Length || line[cursor] != '=') + return false; + cursor++; + } + + if (cursor >= line.Length || line[cursor] != ']') + return false; + + length = cursor - start + 1; + return true; + } + + private static IEnumerable<(string Name, int Index)> EnumerateLuaRequireReferences(string line) + { + for (var cursor = 0; cursor < line.Length; cursor++) + { + if (line[cursor] == '-' && cursor + 1 < line.Length && line[cursor + 1] == '-') + yield break; + + if (line[cursor] is '"' or '\'') + { + cursor = SkipQuotedLiteral(line, cursor); + continue; + } + + if (line[cursor] == '[' && cursor + 1 < line.Length && line[cursor + 1] == '[') + { + var close = line.IndexOf("]]", cursor + 2, StringComparison.Ordinal); + cursor = close < 0 ? line.Length : close + 1; + continue; + } + + if (!IsLuaIdentifierAt(line, cursor, "require")) + continue; + + var argStart = cursor + "require".Length; + while (argStart < line.Length && char.IsWhiteSpace(line[argStart])) + argStart++; + if (argStart < line.Length && line[argStart] == '(') + { + argStart++; + while (argStart < line.Length && char.IsWhiteSpace(line[argStart])) + argStart++; + } + + if (argStart >= line.Length || line[argStart] is not ('"' or '\'')) + continue; + + var quote = line[argStart++]; + var nameStart = argStart; + while (argStart < line.Length) + { + if (line[argStart] == '\\' && argStart + 1 < line.Length) + { + argStart += 2; + continue; + } + + if (line[argStart] == quote) + break; + argStart++; + } + + if (argStart > nameStart) + yield return (line[nameStart..argStart], nameStart); + cursor = argStart; + } + } + + private static int SkipQuotedLiteral(string line, int start) + { + var quote = line[start]; + var cursor = start + 1; + while (cursor < line.Length) + { + if (line[cursor] == '\\' && cursor + 1 < line.Length) + { + cursor += 2; + continue; + } + + if (line[cursor] == quote) + return cursor; + cursor++; + } + + return line.Length; + } + + private static bool IsLuaIdentifierAt(string line, int index, string identifier) + { + if (index < 0 || index + identifier.Length > line.Length) + return false; + if (string.CompareOrdinal(line, index, identifier, 0, identifier.Length) != 0) + return false; + if (index > 0 && IsLuaIdentifierPart(line[index - 1])) + return false; + + var after = index + identifier.Length; + return after >= line.Length || !IsLuaIdentifierPart(line[after]); + } + + private static bool IsLuaIdentifierPart(char ch) => + ch == '_' || char.IsLetterOrDigit(ch); + + private static string LastQualifiedSegment(string value) + { + var dot = value.LastIndexOf('.'); + return dot >= 0 && dot + 1 < value.Length ? value[(dot + 1)..] : value; + } + + private static void MaskRange(char[] chars, int start, int end) + { + for (var i = Math.Max(0, start); i < end && i < chars.Length; i++) + chars[i] = ' '; } } diff --git a/src/CodeIndex/Indexer/References/Support/LanguageReferenceExtractionSupport.cs b/src/CodeIndex/Indexer/References/Support/LanguageReferenceExtractionSupport.cs index 482183df71..c55fa1bab5 100644 --- a/src/CodeIndex/Indexer/References/Support/LanguageReferenceExtractionSupport.cs +++ b/src/CodeIndex/Indexer/References/Support/LanguageReferenceExtractionSupport.cs @@ -517,19 +517,6 @@ internal static class LanguageReferenceExtractionSupport @"(?[a-z_]\w*[?!]?)\s+(?=(?:[A-Za-z_:@\[""']))", RegexOptions.Compiled | RegexOptions.CultureInvariant); - private static readonly Regex LuaRequireRegex = new( - @"\brequire\s*\(?\s*[""'](?[^""']+)[""']", - RegexOptions.Compiled | RegexOptions.CultureInvariant); - private static readonly Regex LuaCommandCallRegex = new( - @"^\s*(?[A-Za-z_]\w*(?:\.[A-Za-z_]\w*)?)\s+(?=[""'{A-Za-z_])", - RegexOptions.Compiled | RegexOptions.CultureInvariant); - private static readonly Regex LuaColonCallRegex = new( - @"(?[A-Za-z_]\w*)\s*\(", - RegexOptions.Compiled | RegexOptions.CultureInvariant); - private static readonly Regex LuaTableFieldReferenceRegex = new( - @"(?[A-Za-z_]\w*)\b(?!\s*(?:=|function\b|\())", - RegexOptions.Compiled | RegexOptions.CultureInvariant); - private static readonly Regex SmalltalkClassDeclarationRegex = new( @"^\s*(?:(?:[A-Za-z_]\w*)\s+subclass:|Class\s+named:|Object\s+subclass:)\s*#", RegexOptions.Compiled | RegexOptions.CultureInvariant); @@ -600,7 +587,7 @@ public static void EmitTypePositionReferences( EmitElixirTypeReferences(preparedLine, references, seen, fileId, context, lineNumber, container); break; case "lua": - EmitLuaTypeReferences(originalLine, references, seen, fileId, context, lineNumber, container); + LuaReferenceExtractor.EmitTypePositionReferences(originalLine, references, seen, fileId, context, lineNumber, container); break; } } @@ -636,7 +623,7 @@ public static void EmitAdditionalCallReferences( EmitElixirParenlessCallReferences(preparedLine, addCallLikeReference, definitionNames); break; case "lua": - EmitLuaCallReferences(preparedLine, addCallLikeReference, references, seen, fileId, context, lineNumber, resolveContainerForColumn, definitionNames); + LuaReferenceExtractor.EmitAdditionalCallReferences(preparedLine, addCallLikeReference, references, seen, fileId, context, lineNumber, resolveContainerForColumn, definitionNames); break; case "smalltalk": EmitSmalltalkMessageReferences(preparedLine, addCallLikeReference, definitionNames); @@ -852,107 +839,6 @@ private static int SkipGoStringLiteral(string line, int start) return line.Length; } - public static string[] MaskLuaLongCommentAndStringLines(IReadOnlyList originalLines) - { - var result = new string[originalLines.Count]; - var longTextEqualsCount = -1; - - for (var lineIndex = 0; lineIndex < originalLines.Count; lineIndex++) - { - var line = originalLines[lineIndex]; - var chars = line.ToCharArray(); - for (var cursor = 0; cursor < chars.Length; cursor++) - { - if (longTextEqualsCount >= 0) - { - if (TryGetLuaLongBracketClose(line, cursor, longTextEqualsCount, out var closeLength)) - { - MaskRange(chars, cursor, cursor + closeLength); - cursor += closeLength - 1; - longTextEqualsCount = -1; - continue; - } - - chars[cursor] = ' '; - continue; - } - - if (chars[cursor] is '"' or '\'') - { - cursor = SkipQuotedLiteral(line, cursor); - continue; - } - - if (chars[cursor] == '-' - && cursor + 2 < chars.Length - && chars[cursor + 1] == '-' - && TryGetLuaLongBracketOpen(line, cursor + 2, out var commentEqualsCount, out var commentOpenLength)) - { - MaskRange(chars, cursor, cursor + 2 + commentOpenLength); - cursor += 1 + commentOpenLength; - longTextEqualsCount = commentEqualsCount; - continue; - } - - if (chars[cursor] == '-' && cursor + 1 < chars.Length && chars[cursor + 1] == '-') - break; - - if (TryGetLuaLongBracketOpen(line, cursor, out var stringEqualsCount, out var stringOpenLength)) - { - MaskRange(chars, cursor, cursor + stringOpenLength); - cursor += stringOpenLength - 1; - longTextEqualsCount = stringEqualsCount; - } - } - - result[lineIndex] = new string(chars); - } - - return result; - } - - private static bool TryGetLuaLongBracketOpen(string line, int start, out int equalsCount, out int length) - { - equalsCount = 0; - length = 0; - if (start < 0 || start >= line.Length || line[start] != '[') - return false; - - var cursor = start + 1; - while (cursor < line.Length && line[cursor] == '=') - { - equalsCount++; - cursor++; - } - - if (cursor >= line.Length || line[cursor] != '[') - return false; - - length = cursor - start + 1; - return true; - } - - private static bool TryGetLuaLongBracketClose(string line, int start, int equalsCount, out int length) - { - length = 0; - if (start < 0 || start >= line.Length || line[start] != ']') - return false; - - var cursor = start + 1; - for (var i = 0; i < equalsCount; i++) - { - if (cursor >= line.Length || line[cursor] != '=') - return false; - cursor++; - } - - if (cursor >= line.Length || line[cursor] != ']') - return false; - - length = cursor - start + 1; - return true; - } - public static string[] MaskRazorCommentLines(IReadOnlyList originalLines) { var result = new string[originalLines.Count]; @@ -4544,96 +4430,6 @@ private static void EmitElixirTypeReferences( ReferenceExtractor.AddReference(references, seen, fileId, match, "type_reference", context, lineNumber, container); } - private static void EmitLuaTypeReferences( - string originalLine, - List references, - HashSet seen, - long fileId, - string context, - int lineNumber, - SymbolRecord? container) - { - foreach (var (name, index) in EnumerateLuaRequireReferences(originalLine)) - ReferenceExtractor.AddReference(references, seen, fileId, name, index, "type_reference", context, lineNumber, container); - } - - private static IEnumerable<(string Name, int Index)> EnumerateLuaRequireReferences(string line) - { - for (var cursor = 0; cursor < line.Length; cursor++) - { - if (line[cursor] == '-' && cursor + 1 < line.Length && line[cursor + 1] == '-') - yield break; - - if (line[cursor] is '"' or '\'') - { - cursor = SkipQuotedLiteral(line, cursor); - continue; - } - - if (line[cursor] == '[' && cursor + 1 < line.Length && line[cursor + 1] == '[') - { - var close = line.IndexOf("]]", cursor + 2, StringComparison.Ordinal); - cursor = close < 0 ? line.Length : close + 1; - continue; - } - - if (!IsIdentifierAt(line, cursor, "require")) - continue; - - var argStart = cursor + "require".Length; - while (argStart < line.Length && char.IsWhiteSpace(line[argStart])) - argStart++; - if (argStart < line.Length && line[argStart] == '(') - { - argStart++; - while (argStart < line.Length && char.IsWhiteSpace(line[argStart])) - argStart++; - } - - if (argStart >= line.Length || line[argStart] is not ('"' or '\'')) - continue; - - var quote = line[argStart++]; - var nameStart = argStart; - while (argStart < line.Length) - { - if (line[argStart] == '\\' && argStart + 1 < line.Length) - { - argStart += 2; - continue; - } - - if (line[argStart] == quote) - break; - argStart++; - } - - if (argStart > nameStart) - yield return (line[nameStart..argStart], nameStart); - cursor = argStart; - } - } - - private static int SkipQuotedLiteral(string line, int start) - { - var quote = line[start]; - var cursor = start + 1; - while (cursor < line.Length) - { - if (line[cursor] == '\\' && cursor + 1 < line.Length) - { - cursor += 2; - continue; - } - - if (line[cursor] == quote) - return cursor; - cursor++; - } - - return line.Length; - } - private static bool IsIdentifierAt(string line, int index, string identifier) { if (index < 0 || index + identifier.Length > line.Length) @@ -4737,50 +4533,6 @@ private static void EmitElixirParenlessCallReferences(string preparedLine, Actio } } - private static void EmitLuaCallReferences( - string preparedLine, - Action addCallLikeReference, - List references, - HashSet seen, - long fileId, - string context, - int lineNumber, - Func resolveContainerForColumn, - IReadOnlySet? definitionNames) - { - var match = LuaCommandCallRegex.Match(preparedLine); - if (match.Success) - { - var name = LastQualifiedSegment(match.Groups["name"].Value); - if (definitionNames?.Contains(name) != true) - addCallLikeReference(name, match.Groups["name"].Index + match.Groups["name"].Value.LastIndexOf(name, StringComparison.Ordinal)); - } - - foreach (Match colonMatch in LuaColonCallRegex.Matches(preparedLine)) - { - var name = colonMatch.Groups["name"].Value; - if (definitionNames?.Contains(name) == true) - continue; - addCallLikeReference(name, colonMatch.Groups["name"].Index); - } - - foreach (Match fieldMatch in LuaTableFieldReferenceRegex.Matches(preparedLine)) - { - var trimmed = preparedLine.TrimStart(); - if (trimmed.StartsWith("function ", StringComparison.Ordinal) - || trimmed.StartsWith("local function ", StringComparison.Ordinal)) - { - break; - } - - var name = fieldMatch.Groups["name"].Value; - if (definitionNames?.Contains(name) == true) - continue; - var index = fieldMatch.Groups["name"].Index; - ReferenceExtractor.AddReference(references, seen, fileId, name, index, "reference", context, lineNumber, resolveContainerForColumn(index)); - } - } - private static void EmitSmalltalkMessageReferences(string preparedLine, Action addCallLikeReference, IReadOnlySet? definitionNames) { var definitionMatch = SmalltalkMethodDefinitionRegex.Match(preparedLine); diff --git a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.CSharpPatterns.cs b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.CSharpPatterns.cs new file mode 100644 index 0000000000..7005e70e76 --- /dev/null +++ b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.CSharpPatterns.cs @@ -0,0 +1,85 @@ +using System.Text.RegularExpressions; +using Regex = CodeIndex.Indexer.BoundedRegex; + +namespace CodeIndex.Indexer; + +public static partial class SymbolExtractor +{ + private const string CSharpVisibilityPattern = @"protected\s+internal|private\s+protected|public|protected|internal|private"; + // Return-type character class includes `*` so pointer and function-pointer returns + // (`int*`, `void**`, `delegate*`, `int*[]`) are not silently dropped. + // The trailing CSharpTupleSuffixPattern lets a tuple group carry suffixes + // (`(int, int)[]`, `(int, int)?`, `(int, int)[][]`, `(int, int)[,]`, and whitespaced + // variants like `(int, int) []` / `(int, int) ?`) so tuple-array and nullable-tuple + // return types are captured on methods, properties, indexers, and explicit interface + // implementations. The shared segment matcher also allows tuple groups inside generic + // arguments (`Task<(int, int)>`, `Dictionary`, + // `List<(int, int)> IFoo.GetList()`), so ordinary methods and explicit-interface + // implementations stay aligned. Delegate and event declarations with tuple-array returns + // remain blocked by the pre-existing pattern-order issue (#340); the identifier branch + // already absorbs non-tuple suffix characters via its char class, but keeping the suffix + // loop outside both branches is harmless and makes the tuple branch's responsibilities + // explicit. + // 戻り値型のクラスに `*` を含め、ポインタ / 関数ポインタ戻り値型（`int*` / `void**` / `delegate*` / `int*[]`）を取りこぼさない。 + // 末尾の CSharpTupleSuffixPattern で tuple 分岐にも `[]` / `?` / `[][]` / `[,]` と、 + // `(int, int) []` / `(int, int) ?` のような空白を挟んだ整形バリエーションまで許容し、 + // tuple-array / nullable-tuple 戻り値をメソッド・プロパティ・インデクサ・明示的 + // インターフェース実装で捕捉できるようにする。共有の segment matcher により + // `Task<(int, int)>` / `Dictionary` / + // `List<(int, int)> IFoo.GetList()` のような generic-over-tuple も通常メソッドと + // 明示的インターフェース実装の両方で同じ経路で扱える。delegate / event 宣言で + // tuple-array 戻り値を扱う件は既存のパターン評価順問題 (#340) が残っており、この + // ループの範囲外。識別子側の分岐は文字クラスに `[`/`]`/`?` を既に含むため無害な冗長だが、 + // tuple 分岐側の責務が明確になる。 + // Tuple / array / nullable suffix tokens that may trail a C# return type. Each iteration + // matches a single `?` or a bracketed `[]` / `[,]` / `[,,]` group and allows whitespace + // between the preceding `)` / identifier and the suffix token (the `\s*` sits inside the + // group so a type with no suffix still matches zero iterations and consumes no + // whitespace). Shared by CSharpTypePattern and the C# constructor regex negative + // lookahead so legal formatting variants like `public required (int, int) [] R4 { ... }` + // and `public readonly (int, int) ? M3() => default;` are both rejected as ctor shapes + // (via the lookahead) and accepted as property / method shapes (via the upstream rows). + // Closes #349 follow-up. + // C# の戻り値型末尾に付きうる tuple / 配列 / nullable サフィックストークン列。各繰り返しは + // `?` 1 個または `[]` / `[,]` / `[,,]` の bracket ブロック 1 個を受理し、先行する `)` や + // 識別子とサフィックストークンの間に空白を許容する（`\s*` を繰り返しの内側に入れているため、 + // サフィックスを持たない型は 0 回繰り返しで一致し、空白を消費しない）。CSharpTypePattern と + // C# コンストラクタ regex の否定先読みで共有し、`public required (int, int) [] R4 { ... }` + // や `public readonly (int, int) ? M3() => default;` のような合法な整形を、 + // 否定先読みで ctor 形状として弾きつつ、上流の property / method 行で本来のシンボルとして + // 拾えるようにする。#349 のフォローアップ。 + private const string CSharpTupleSuffixPattern = @"(?:\s*(?:\?|\[[\],\s]*\]))*"; + // Embedded tuple groups must contain a comma at the OUTER tuple level so ordinary + // call/ctor parens (`Make()`, `Parent(value)`) keep falling through, while real tuple + // segments inside generics can nest arbitrarily deep (`Task<((int A, int B), string Name)>`, + // `Task<(((int A, int B), int C), string Name)>`). The balancing-group variant tracks nested + // parens and only records commas seen at depth 0. + // 埋め込み tuple group は最外 tuple レベルの comma を必須にし、`Make()` / `Parent(value)` の + // ような通常の call/ctor 括弧列は従来どおり不一致に落としつつ、generic 内の実 tuple segment + // は `Task<((int A, int B), string Name)>` / `Task<(((int A, int B), int C), string Name)>` + // のような深い入れ子まで通せるようにする。balancing-group 版で入れ子括弧を追跡し、 + // 深さ 0 で見えた comma だけを tuple 判定に使う。 + private const string CSharpTupleGroupPattern = + @"\((?>(?:[^(),]+|\((?)|\)(?<-TupleDepth>)|(?(TupleDepth),|(?,))))*(?(TupleDepth)(?!))(?(TupleComma)|(?!))\)"; + private const string CSharpUnicodeEscapePattern = @"\\(?:u[0-9A-Fa-f]{4}|U[0-9A-Fa-f]{8})"; + private const string CSharpIdentifierPattern = + @"@?(?:[_\p{L}]|" + CSharpUnicodeEscapePattern + @")(?:\w|" + CSharpUnicodeEscapePattern + @")*"; + private const string CSharpNamespacePattern = CSharpIdentifierPattern + @"(?:\." + CSharpIdentifierPattern + @")*"; + private const string CSharpTypeTokenCharsPattern = @"[\w@?.<>\[\],:*]"; + private const string CSharpTypeTokenPattern = @"(?:" + CSharpUnicodeEscapePattern + @"|" + CSharpTypeTokenCharsPattern + @")"; + private const string CSharpTypeSegmentPattern = + @"(?:" + CSharpTypeTokenPattern + @"+(?:" + CSharpTupleGroupPattern + CSharpTypeTokenPattern + @"*)*|" + CSharpTupleGroupPattern + CSharpTypeTokenPattern + @"*)"; + private const string CSharpTypePattern = + @"(?:(?:global::)?(?:" + CSharpTypeSegmentPattern + @")(?:\s+(?:" + CSharpTypeSegmentPattern + @"))*" + CSharpTupleSuffixPattern + @")"; + private const string CSharpMethodTypeParameterListPattern = + @"(?:<(?:(?>[^<>]+)|<(?)|>(?<-CSharpMethodTypeParameterDepth>))*(?(CSharpMethodTypeParameterDepth)(?!))>\s*)?"; + private static readonly Regex CSharpPartialFunctionDeclarationSignatureRegex = new( + $@"^(?:(?:{CSharpVisibilityPattern}|abstract|async|extern|new|override|sealed|static|unsafe|virtual)\s+)*partial\s+", + RegexOptions.Compiled | RegexOptions.CultureInvariant); + private static readonly Regex CSharpTestMethodAttributeRegex = new( + @"(?:^|,)\s*(?:(?:\w+\.)*)?(?:Fact|Theory|Test|TestCase|TestCaseSource|TestMethod|DataTestMethod)(?:Attribute)?\b", + RegexOptions.Compiled | RegexOptions.CultureInvariant); + // `delegate` is a non-type keyword only when it is NOT followed by `*` — `delegate*<...>` is a valid return type. + // `delegate` は `*` を伴わないときだけ非型キーワード扱い。`delegate*<...>` は戻り値型として有効。 + private const string CSharpNonTypeKeywordPattern = @"(?:(?:public|private|protected|internal|static|sealed|partial|readonly|unsafe|extern|virtual|override|abstract|async|new|file|required|ref)\b|delegate\b(?!\s*\*))"; +} diff --git a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.cs b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.cs index 0bcf26ec0d..ad8388290b 100644 --- a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.cs +++ b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.cs @@ -64,68 +64,6 @@ public static int GetContractVersion(string? lang) // instances and lookup tables are initialized once by the CLR and must be treated as // immutable after type initialization; per-file extraction state belongs in local // variables or per-call collections, never in static mutable caches. - private const string CSharpVisibilityPattern = @"protected\s+internal|private\s+protected|public|protected|internal|private"; - // Return-type character class includes `*` so pointer and function-pointer returns - // (`int*`, `void**`, `delegate*`, `int*[]`) are not silently dropped. - // The trailing CSharpTupleSuffixPattern lets a tuple group carry suffixes - // (`(int, int)[]`, `(int, int)?`, `(int, int)[][]`, `(int, int)[,]`, and whitespaced - // variants like `(int, int) []` / `(int, int) ?`) so tuple-array and nullable-tuple - // return types are captured on methods, properties, indexers, and explicit interface - // implementations. The shared segment matcher also allows tuple groups inside generic - // arguments (`Task<(int, int)>`, `Dictionary`, - // `List<(int, int)> IFoo.GetList()`), so ordinary methods and explicit-interface - // implementations stay aligned. Delegate and event declarations with tuple-array returns - // remain blocked by the pre-existing pattern-order issue (#340); the identifier branch - // already absorbs non-tuple suffix characters via its char class, but keeping the suffix - // loop outside both branches is harmless and makes the tuple branch's responsibilities - // explicit. - // 戻り値型のクラスに `*` を含め、ポインタ / 関数ポインタ戻り値型（`int*` / `void**` / `delegate*` / `int*[]`）を取りこぼさない。 - // 末尾の CSharpTupleSuffixPattern で tuple 分岐にも `[]` / `?` / `[][]` / `[,]` と、 - // `(int, int) []` / `(int, int) ?` のような空白を挟んだ整形バリエーションまで許容し、 - // tuple-array / nullable-tuple 戻り値をメソッド・プロパティ・インデクサ・明示的 - // インターフェース実装で捕捉できるようにする。共有の segment matcher により - // `Task<(int, int)>` / `Dictionary` / - // `List<(int, int)> IFoo.GetList()` のような generic-over-tuple も通常メソッドと - // 明示的インターフェース実装の両方で同じ経路で扱える。delegate / event 宣言で - // tuple-array 戻り値を扱う件は既存のパターン評価順問題 (#340) が残っており、この - // ループの範囲外。識別子側の分岐は文字クラスに `[`/`]`/`?` を既に含むため無害な冗長だが、 - // tuple 分岐側の責務が明確になる。 - // Tuple / array / nullable suffix tokens that may trail a C# return type. Each iteration - // matches a single `?` or a bracketed `[]` / `[,]` / `[,,]` group and allows whitespace - // between the preceding `)` / identifier and the suffix token (the `\s*` sits inside the - // group so a type with no suffix still matches zero iterations and consumes no - // whitespace). Shared by CSharpTypePattern and the C# constructor regex negative - // lookahead so legal formatting variants like `public required (int, int) [] R4 { ... }` - // and `public readonly (int, int) ? M3() => default;` are both rejected as ctor shapes - // (via the lookahead) and accepted as property / method shapes (via the upstream rows). - // Closes #349 follow-up. - // C# の戻り値型末尾に付きうる tuple / 配列 / nullable サフィックストークン列。各繰り返しは - // `?` 1 個または `[]` / `[,]` / `[,,]` の bracket ブロック 1 個を受理し、先行する `)` や - // 識別子とサフィックストークンの間に空白を許容する（`\s*` を繰り返しの内側に入れているため、 - // サフィックスを持たない型は 0 回繰り返しで一致し、空白を消費しない）。CSharpTypePattern と - // C# コンストラクタ regex の否定先読みで共有し、`public required (int, int) [] R4 { ... }` - // や `public readonly (int, int) ? M3() => default;` のような合法な整形を、 - // 否定先読みで ctor 形状として弾きつつ、上流の property / method 行で本来のシンボルとして - // 拾えるようにする。#349 のフォローアップ。 - private const string CSharpTupleSuffixPattern = @"(?:\s*(?:\?|\[[\],\s]*\]))*"; - // Embedded tuple groups must contain a comma at the OUTER tuple level so ordinary - // call/ctor parens (`Make()`, `Parent(value)`) keep falling through, while real tuple - // segments inside generics can nest arbitrarily deep (`Task<((int A, int B), string Name)>`, - // `Task<(((int A, int B), int C), string Name)>`). The balancing-group variant tracks nested - // parens and only records commas seen at depth 0. - // 埋め込み tuple group は最外 tuple レベルの comma を必須にし、`Make()` / `Parent(value)` の - // ような通常の call/ctor 括弧列は従来どおり不一致に落としつつ、generic 内の実 tuple segment - // は `Task<((int A, int B), string Name)>` / `Task<(((int A, int B), int C), string Name)>` - // のような深い入れ子まで通せるようにする。balancing-group 版で入れ子括弧を追跡し、 - // 深さ 0 で見えた comma だけを tuple 判定に使う。 - private const string CSharpTupleGroupPattern = - @"\((?>(?:[^(),]+|\((?)|\)(?<-TupleDepth>)|(?(TupleDepth),|(?,))))*(?(TupleDepth)(?!))(?(TupleComma)|(?!))\)"; - private const string CSharpUnicodeEscapePattern = @"\\(?:u[0-9A-Fa-f]{4}|U[0-9A-Fa-f]{8})"; - private const string CSharpIdentifierPattern = - @"@?(?:[_\p{L}]|" + CSharpUnicodeEscapePattern + @")(?:\w|" + CSharpUnicodeEscapePattern + @")*"; - private const string CSharpNamespacePattern = CSharpIdentifierPattern + @"(?:\." + CSharpIdentifierPattern + @")*"; - private const string CSharpTypeTokenCharsPattern = @"[\w@?.<>\[\],:*]"; - private const string CSharpTypeTokenPattern = @"(?:" + CSharpUnicodeEscapePattern + @"|" + CSharpTypeTokenCharsPattern + @")"; private const string SqlQualifiedIdentifierSegmentPattern = @"(?:\[(?:[^\]\r\n]|\]\])+\]|""[^""]+""|[\w$#]+)"; private const string SqlQualifiedIdentifierPattern = @"(?:" + SqlQualifiedIdentifierSegmentPattern + @")(?:\s*\.\s*(?:" + SqlQualifiedIdentifierSegmentPattern + @"))*"; @@ -170,18 +108,6 @@ public static int GetContractVersion(string? lang) @"(?:\[\[[^\r\n]*?\]\]\s*|__attribute__\s*\(\((?:(?>[^()]+)|\((?)|\)(?<-CAttributeDepth>))*(?(CAttributeDepth)(?!))\)\)\s*|__declspec\s*\((?:(?>[^()]+)|\((?)|\)(?<-CAttributeDepth>))*(?(CAttributeDepth)(?!))\)\s*|_Noreturn\s+)"; private const string CFunctionReturnTypePattern = @"(?(?:(?:\w+[\s*]+)|" + CAttributeSpecifierTokenPattern + @")+)"; - private const string CSharpTypeSegmentPattern = - @"(?:" + CSharpTypeTokenPattern + @"+(?:" + CSharpTupleGroupPattern + CSharpTypeTokenPattern + @"*)*|" + CSharpTupleGroupPattern + CSharpTypeTokenPattern + @"*)"; - private const string CSharpTypePattern = - @"(?:(?:global::)?(?:" + CSharpTypeSegmentPattern + @")(?:\s+(?:" + CSharpTypeSegmentPattern + @"))*" + CSharpTupleSuffixPattern + @")"; - private const string CSharpMethodTypeParameterListPattern = - @"(?:<(?:(?>[^<>]+)|<(?)|>(?<-CSharpMethodTypeParameterDepth>))*(?(CSharpMethodTypeParameterDepth)(?!))>\s*)?"; - private static readonly Regex CSharpPartialFunctionDeclarationSignatureRegex = new( - $@"^(?:(?:{CSharpVisibilityPattern}|abstract|async|extern|new|override|sealed|static|unsafe|virtual)\s+)*partial\s+", - RegexOptions.Compiled | RegexOptions.CultureInvariant); - private static readonly Regex CSharpTestMethodAttributeRegex = new( - @"(?:^|,)\s*(?:(?:\w+\.)*)?(?:Fact|Theory|Test|TestCase|TestCaseSource|TestMethod|DataTestMethod)(?:Attribute)?\b", - RegexOptions.Compiled | RegexOptions.CultureInvariant); private const string JavaUnicodeEscapePattern = @"\\u+[0-9A-Fa-f]{4}"; private const string JavaIdentifierPattern = @"(?:[\p{L}_$]|" + JavaUnicodeEscapePattern + @")(?:[\p{L}\p{Nd}_$]|" + JavaUnicodeEscapePattern + @")*"; @@ -227,9 +153,6 @@ public static int GetContractVersion(string? lang) private static readonly Regex PhpPrefixedRequireIncludeRegex = new( @"^\s*(?:require|include)(?:_once)?\s*\(?\s*(?(?:(?:__DIR__|__FILE__|dirname\s*\(\s*__FILE__\s*\))\s*\.\s*)+)\s*(?:'(?[^']+)'|""(?[^""]+)"")\s*\)?\s*;", RegexOptions.Compiled | RegexOptions.CultureInvariant | RegexOptions.IgnoreCase | RegexOptions.CultureInvariant); - // `delegate` is a non-type keyword only when it is NOT followed by `*` — `delegate*<...>` is a valid return type. - // `delegate` は `*` を伴わないときだけ非型キーワード扱い。`delegate*<...>` は戻り値型として有効。 - private const string CSharpNonTypeKeywordPattern = @"(?:(?:public|private|protected|internal|static|sealed|partial|readonly|unsafe|extern|virtual|override|abstract|async|new|file|required|ref)\b|delegate\b(?!\s*\*))"; private const string CFunctionStartBlacklistPattern = @"^(?!\s*typedef\b)(?!\s*(?:if|else|for|while|switch|return|sizeof)\s*[\(\{;])"; private const string CFunctionNameBlacklistPattern = @"(?!(?:int|void|char|short|long|float|double|signed|unsigned|bool|_Bool|size_t|ssize_t|intptr_t|uintptr_t|int8_t|int16_t|int32_t|int64_t|uint8_t|uint16_t|uint32_t|uint64_t)\b)"; private const string CppFunctionStartBlacklistPattern = @"^(?!\s*typedef\b)(?!\s*(?:if|else|for|while|switch|return|sizeof|using|namespace)\s*[\(\{;<])"; diff --git a/tests/CodeIndex.Tests/SymbolExtractorLuaTests.cs b/tests/CodeIndex.Tests/SymbolExtractorLuaTests.cs new file mode 100644 index 0000000000..a6572de463 --- /dev/null +++ b/tests/CodeIndex.Tests/SymbolExtractorLuaTests.cs @@ -0,0 +1,53 @@ +using CodeIndex.Indexer; + +namespace CodeIndex.Tests; + +public partial class SymbolExtractorTests +{ + [Fact] + public void Extract_Lua_DetectsFunctionsAndRequire() + { + // Lua: function, local function, assignment forms, require / Lua: 関数、ローカル関数、代入形式、require + var content = """ + local http = require('socket.http') + + local helper = function(x) + return x + end + + M.named = function(name) + return "hello " .. name + end + + function M:method_form(arg) + return arg + end + + function M.dot_form(arg) + return arg + end + + function M.deep.table_key(arg, ...) + return arg + end + + local function top_local(x) + return x + end + + function plain_function(a) + return a + end + """; + var symbols = SymbolExtractor.Extract(1, "lua", content); + + Assert.Contains(symbols, s => s.Kind == "import" && s.Name == "socket.http"); + Assert.Contains(symbols, s => s.Kind == "function" && s.Name == "helper"); + Assert.Contains(symbols, s => s.Kind == "function" && s.Name == "M.named"); + Assert.Contains(symbols, s => s.Kind == "function" && s.Name == "M:method_form"); + Assert.Contains(symbols, s => s.Kind == "function" && s.Name == "M.dot_form"); + Assert.Contains(symbols, s => s.Kind == "function" && s.Name == "M.deep.table_key"); + Assert.Contains(symbols, s => s.Kind == "function" && s.Name == "top_local"); + Assert.Contains(symbols, s => s.Kind == "function" && s.Name == "plain_function"); + } +} diff --git a/tests/CodeIndex.Tests/SymbolExtractorTests.cs b/tests/CodeIndex.Tests/SymbolExtractorTests.cs index 1f116d74e8..6ff57db191 100644 --- a/tests/CodeIndex.Tests/SymbolExtractorTests.cs +++ b/tests/CodeIndex.Tests/SymbolExtractorTests.cs @@ -14050,53 +14050,6 @@ public void Extract_R_DetectsFunctionAssignmentAndLibrary() Assert.Contains(symbols, s => s.Kind == "function" && s.Name == "my_plot"); } - [Fact] - public void Extract_Lua_DetectsFunctionsAndRequire() - { - // Lua: function, local function, assignment forms, require / Lua: 関数、ローカル関数、代入形式、require - var content = """ - local http = require('socket.http') - - local helper = function(x) - return x - end - - M.named = function(name) - return "hello " .. name - end - - function M:method_form(arg) - return arg - end - - function M.dot_form(arg) - return arg - end - - function M.deep.table_key(arg, ...) - return arg - end - - local function top_local(x) - return x - end - - function plain_function(a) - return a - end - """; - var symbols = SymbolExtractor.Extract(1, "lua", content); - - Assert.Contains(symbols, s => s.Kind == "import" && s.Name == "socket.http"); - Assert.Contains(symbols, s => s.Kind == "function" && s.Name == "helper"); - Assert.Contains(symbols, s => s.Kind == "function" && s.Name == "M.named"); - Assert.Contains(symbols, s => s.Kind == "function" && s.Name == "M:method_form"); - Assert.Contains(symbols, s => s.Kind == "function" && s.Name == "M.dot_form"); - Assert.Contains(symbols, s => s.Kind == "function" && s.Name == "M.deep.table_key"); - Assert.Contains(symbols, s => s.Kind == "function" && s.Name == "top_local"); - Assert.Contains(symbols, s => s.Kind == "function" && s.Name == "plain_function"); - } - [Fact] public void Extract_Elixir_DetectsModuleAndFunctions() {