diff --git a/changelog.d/unreleased/2055.fixed.md b/changelog.d/unreleased/2055.fixed.md new file mode 100644 index 0000000000..a86fcb47fa --- /dev/null +++ b/changelog.d/unreleased/2055.fixed.md @@ -0,0 +1,16 @@ +--- +category: fixed +issues: + - 2055 +affected: + - src/CodeIndex/Indexer/References/Languages/PythonReferenceExtractor.cs + - tests/CodeIndex.Tests/ReferenceExtractorTests.cs +--- + +## English + +- **Python decorator references now include decorator argument and composition callables (#2055)** — Python reference extraction now records callable symbols used inside parameterized decorators and composed decorator chains. + +## 日本語 + +- **Python decorator references が decorator 引数と合成 chain 内の callable も含むようになりました (#2055)** — Python reference extraction は、parameterized decorator や composed decorator chain 内で使われる callable symbol も記録するようになりました。 diff --git a/src/CodeIndex/Indexer/References/Languages/PythonReferenceExtractor.cs b/src/CodeIndex/Indexer/References/Languages/PythonReferenceExtractor.cs index 2c52ffa6c4..462f8cb86b 100644 --- a/src/CodeIndex/Indexer/References/Languages/PythonReferenceExtractor.cs +++ b/src/CodeIndex/Indexer/References/Languages/PythonReferenceExtractor.cs @@ -15,6 +15,9 @@ internal static class PythonReferenceExtractor private static readonly Regex DecoratorCallRegex = new( @"^\s*@(?[_\p{L}]\w*(?:\.[_\p{L}]\w*)*)\s*\(", RegexOptions.Compiled); + private static readonly Regex PythonIdentifierRegex = new( + @"(?[_\p{L}]\w*(?:\.[_\p{L}]\w*)*)", + RegexOptions.Compiled); private static readonly Regex BareRaiseTypeRegex = new( @"^\s*raise\s+(?(?:[_\p{L}]\w*\.)*[_\p{Lu}]\w*)(?:\s+from\s+[_\p{L}]\w*)?\s*(?:#.*)?$", RegexOptions.Compiled); @@ -262,6 +265,16 @@ public static void EmitDecoratorReferences( continue; ReferenceExtractor.AddReference(references, seen, fileId, match, "decorator", context, lineNumber, container); + EmitDecoratorArgumentReferences( + preparedLine, + match, + references, + seen, + fileId, + context, + lineNumber, + container, + isIgnoredName); } foreach (Match match in DecoratorRegex.Matches(preparedLine)) @@ -276,6 +289,78 @@ public static void EmitDecoratorReferences( } } + private static void EmitDecoratorArgumentReferences( + string preparedLine, + Match decoratorMatch, + List references, + HashSet seen, + long fileId, + string context, + int lineNumber, + SymbolRecord? container, + Func isIgnoredName) + { + var decoratorName = decoratorMatch.Groups["name"].Value; + var argumentStart = preparedLine.IndexOf('(', decoratorMatch.Index + decoratorMatch.Length - 1); + if (argumentStart < 0) + return; + + foreach (Match identifierMatch in PythonIdentifierRegex.Matches(preparedLine, argumentStart + 1)) + { + var nameGroup = identifierMatch.Groups["name"]; + var name = nameGroup.Value; + if (name == decoratorName || isIgnoredName(name) || IsPythonLiteralName(name)) + continue; + if (IsKeywordArgumentName(preparedLine, nameGroup.Index + nameGroup.Length)) + continue; + var isCallTarget = IsCallTarget(preparedLine, nameGroup.Index + nameGroup.Length); + if (IsKeywordArgumentValue(preparedLine, nameGroup.Index) && !isCallTarget) + continue; + + ReferenceExtractor.AddReference( + references, + seen, + fileId, + name, + nameGroup.Index, + isCallTarget ? "call" : "reference", + context, + lineNumber, + container, + "python"); + } + } + + private static bool IsKeywordArgumentName(string value, int afterNameIndex) + { + while (afterNameIndex < value.Length && char.IsWhiteSpace(value[afterNameIndex])) + afterNameIndex++; + + return afterNameIndex < value.Length && value[afterNameIndex] == '='; + } + + private static bool IsKeywordArgumentValue(string value, int nameIndex) + { + var beforeNameIndex = nameIndex - 1; + while (beforeNameIndex >= 0 && char.IsWhiteSpace(value[beforeNameIndex])) + beforeNameIndex--; + + return beforeNameIndex >= 0 && value[beforeNameIndex] == '='; + } + + private static bool IsCallTarget(string value, int afterNameIndex) + { + while (afterNameIndex < value.Length && char.IsWhiteSpace(value[afterNameIndex])) + afterNameIndex++; + + return afterNameIndex < value.Length && value[afterNameIndex] == '('; + } + + private static bool IsPythonLiteralName(string name) + { + return name is "True" or "False" or "None" or "Ellipsis"; + } + public static void EmitRaiseReferences( string preparedLine, List references, diff --git a/tests/CodeIndex.Tests/ReferenceExtractorTests.cs b/tests/CodeIndex.Tests/ReferenceExtractorTests.cs index be0acd776d..960dbae9f0 100644 --- a/tests/CodeIndex.Tests/ReferenceExtractorTests.cs +++ b/tests/CodeIndex.Tests/ReferenceExtractorTests.cs @@ -882,11 +882,43 @@ def wrap(f): return f return wrap + def target_func(): + pass + + def memoize(fn): + return fn + + def cache_with(timeout): + def wrap(f): + return f + return wrap + + def make_factory(): + return target_func + + DEFAULT_TIMEOUT = 30 + @bare_decorator @parametrized("value") def wrapped(): pass + @functools.wraps(target_func) + def wrapped_target(): + pass + + @cache_with(timeout=30)(memoize(target_func)) + def composed_target(): + pass + + @cache_with(timeout=DEFAULT_TIMEOUT) + def configured_target(): + pass + + @cache_with(factory=make_factory()) + def keyword_factory_target(): + pass + @staticmethod def method(): pass @@ -903,7 +935,7 @@ def parametrized_fixture(value): var symbols = SymbolExtractor.Extract(1, "python", content); var references = ReferenceExtractor.Extract(1, "python", content, symbols); - Assert.Equal(5, references.Count(reference => reference.ReferenceKind == "decorator")); + Assert.Equal(9, references.Count(reference => reference.ReferenceKind == "decorator")); Assert.Contains(references, reference => reference.SymbolName == "bare_decorator" && reference.ReferenceKind == "decorator"); @@ -922,6 +954,25 @@ def parametrized_fixture(value): Assert.Contains(references, reference => reference.SymbolName == "parametrized" && reference.ReferenceKind == "call"); + Assert.Contains(references, reference => + reference.SymbolName == "target_func" + && reference.ReferenceKind == "reference" + && reference.Context == "@functools.wraps(target_func)"); + Assert.Contains(references, reference => + reference.SymbolName == "memoize" + && reference.ReferenceKind == "call" + && reference.Context == "@cache_with(timeout=30)(memoize(target_func))"); + Assert.Contains(references, reference => + reference.SymbolName == "target_func" + && reference.ReferenceKind == "reference" + && reference.Context == "@cache_with(timeout=30)(memoize(target_func))"); + Assert.Contains(references, reference => + reference.SymbolName == "make_factory" + && reference.ReferenceKind == "call" + && reference.Context == "@cache_with(factory=make_factory())"); + Assert.DoesNotContain(references, reference => + reference.SymbolName == "DEFAULT_TIMEOUT" + && reference.ReferenceKind == "call"); } [Fact]