From 4522044b274de73b10e20a31409b4073dc0f9f02 Mon Sep 17 00:00:00 2001 From: Widthdom Date: Mon, 25 May 2026 00:04:57 +0900 Subject: [PATCH 1/3] Fix Python decorator argument references (#2055) --- changelog.d/unreleased/2055.fixed.md | 16 +++++ .../Languages/PythonReferenceExtractor.cs | 65 +++++++++++++++++++ .../ReferenceExtractorTests.cs | 33 +++++++++- 3 files changed, 113 insertions(+), 1 deletion(-) create mode 100644 changelog.d/unreleased/2055.fixed.md diff --git a/changelog.d/unreleased/2055.fixed.md b/changelog.d/unreleased/2055.fixed.md new file mode 100644 index 0000000000..a86fcb47fa --- /dev/null +++ b/changelog.d/unreleased/2055.fixed.md @@ -0,0 +1,16 @@ +--- +category: fixed +issues: + - 2055 +affected: + - src/CodeIndex/Indexer/References/Languages/PythonReferenceExtractor.cs + - tests/CodeIndex.Tests/ReferenceExtractorTests.cs +--- + +## English + +- **Python decorator references now include decorator argument and composition callables (#2055)** — Python reference extraction now records callable symbols used inside parameterized decorators and composed decorator chains. + +## 日本語 + +- **Python decorator references が decorator 引数と合成 chain 内の callable も含むようになりました (#2055)** — Python reference extraction は、parameterized decorator や composed decorator chain 内で使われる callable symbol も記録するようになりました。 diff --git a/src/CodeIndex/Indexer/References/Languages/PythonReferenceExtractor.cs b/src/CodeIndex/Indexer/References/Languages/PythonReferenceExtractor.cs index 2c52ffa6c4..443a53175c 100644 --- a/src/CodeIndex/Indexer/References/Languages/PythonReferenceExtractor.cs +++ b/src/CodeIndex/Indexer/References/Languages/PythonReferenceExtractor.cs @@ -15,6 +15,9 @@ internal static class PythonReferenceExtractor private static readonly Regex DecoratorCallRegex = new( @"^\s*@(?[_\p{L}]\w*(?:\.[_\p{L}]\w*)*)\s*\(", RegexOptions.Compiled); + private static readonly Regex PythonIdentifierRegex = new( + @"(?[_\p{L}]\w*(?:\.[_\p{L}]\w*)*)", + RegexOptions.Compiled); private static readonly Regex BareRaiseTypeRegex = new( @"^\s*raise\s+(?(?:[_\p{L}]\w*\.)*[_\p{Lu}]\w*)(?:\s+from\s+[_\p{L}]\w*)?\s*(?:#.*)?$", RegexOptions.Compiled); @@ -262,6 +265,16 @@ public static void EmitDecoratorReferences( continue; ReferenceExtractor.AddReference(references, seen, fileId, match, "decorator", context, lineNumber, container); + EmitDecoratorArgumentReferences( + preparedLine, + match, + references, + seen, + fileId, + context, + lineNumber, + container, + isIgnoredName); } foreach (Match match in DecoratorRegex.Matches(preparedLine)) @@ -276,6 +289,58 @@ public static void EmitDecoratorReferences( } } + private static void EmitDecoratorArgumentReferences( + string preparedLine, + Match decoratorMatch, + List references, + HashSet seen, + long fileId, + string context, + int lineNumber, + SymbolRecord? container, + Func isIgnoredName) + { + var decoratorName = decoratorMatch.Groups["name"].Value; + var argumentStart = preparedLine.IndexOf('(', decoratorMatch.Index + decoratorMatch.Length - 1); + if (argumentStart < 0) + return; + + foreach (Match identifierMatch in PythonIdentifierRegex.Matches(preparedLine, argumentStart + 1)) + { + var nameGroup = identifierMatch.Groups["name"]; + var name = nameGroup.Value; + if (name == decoratorName || isIgnoredName(name) || IsPythonLiteralName(name)) + continue; + if (IsKeywordArgumentName(preparedLine, nameGroup.Index + nameGroup.Length)) + continue; + + ReferenceExtractor.AddReference( + references, + seen, + fileId, + name, + nameGroup.Index, + "call", + context, + lineNumber, + container, + "python"); + } + } + + private static bool IsKeywordArgumentName(string value, int afterNameIndex) + { + while (afterNameIndex < value.Length && char.IsWhiteSpace(value[afterNameIndex])) + afterNameIndex++; + + return afterNameIndex < value.Length && value[afterNameIndex] == '='; + } + + private static bool IsPythonLiteralName(string name) + { + return name is "True" or "False" or "None" or "Ellipsis"; + } + public static void EmitRaiseReferences( string preparedLine, List references, diff --git a/tests/CodeIndex.Tests/ReferenceExtractorTests.cs b/tests/CodeIndex.Tests/ReferenceExtractorTests.cs index be0acd776d..a869b09ad7 100644 --- a/tests/CodeIndex.Tests/ReferenceExtractorTests.cs +++ b/tests/CodeIndex.Tests/ReferenceExtractorTests.cs @@ -882,11 +882,30 @@ def wrap(f): return f return wrap + def target_func(): + pass + + def memoize(fn): + return fn + + def cache_with(timeout): + def wrap(f): + return f + return wrap + @bare_decorator @parametrized("value") def wrapped(): pass + @functools.wraps(target_func) + def wrapped_target(): + pass + + @cache_with(timeout=30)(memoize(target_func)) + def composed_target(): + pass + @staticmethod def method(): pass @@ -903,7 +922,7 @@ def parametrized_fixture(value): var symbols = SymbolExtractor.Extract(1, "python", content); var references = ReferenceExtractor.Extract(1, "python", content, symbols); - Assert.Equal(5, references.Count(reference => reference.ReferenceKind == "decorator")); + Assert.Equal(7, references.Count(reference => reference.ReferenceKind == "decorator")); Assert.Contains(references, reference => reference.SymbolName == "bare_decorator" && reference.ReferenceKind == "decorator"); @@ -922,6 +941,18 @@ def parametrized_fixture(value): Assert.Contains(references, reference => reference.SymbolName == "parametrized" && reference.ReferenceKind == "call"); + Assert.Contains(references, reference => + reference.SymbolName == "target_func" + && reference.ReferenceKind == "call" + && reference.Context == "@functools.wraps(target_func)"); + Assert.Contains(references, reference => + reference.SymbolName == "memoize" + && reference.ReferenceKind == "call" + && reference.Context == "@cache_with(timeout=30)(memoize(target_func))"); + Assert.Contains(references, reference => + reference.SymbolName == "target_func" + && reference.ReferenceKind == "call" + && reference.Context == "@cache_with(timeout=30)(memoize(target_func))"); } [Fact] From 4a21b9ef8986ccab628ec00c68ce02bd93e54ee2 Mon Sep 17 00:00:00 2001 From: Widthdom Date: Mon, 25 May 2026 00:19:15 +0900 Subject: [PATCH 2/3] Address Python decorator review findings (#2055) --- .../Languages/PythonReferenceExtractor.cs | 21 ++++++++++++++++++- .../ReferenceExtractorTests.cs | 15 ++++++++++--- 2 files changed, 32 insertions(+), 4 deletions(-) diff --git a/src/CodeIndex/Indexer/References/Languages/PythonReferenceExtractor.cs b/src/CodeIndex/Indexer/References/Languages/PythonReferenceExtractor.cs index 443a53175c..2f27b09425 100644 --- a/src/CodeIndex/Indexer/References/Languages/PythonReferenceExtractor.cs +++ b/src/CodeIndex/Indexer/References/Languages/PythonReferenceExtractor.cs @@ -313,6 +313,8 @@ private static void EmitDecoratorArgumentReferences( continue; if (IsKeywordArgumentName(preparedLine, nameGroup.Index + nameGroup.Length)) continue; + if (IsKeywordArgumentValue(preparedLine, nameGroup.Index)) + continue; ReferenceExtractor.AddReference( references, @@ -320,7 +322,7 @@ private static void EmitDecoratorArgumentReferences( fileId, name, nameGroup.Index, - "call", + IsCallTarget(preparedLine, nameGroup.Index + nameGroup.Length) ? "call" : "reference", context, lineNumber, container, @@ -336,6 +338,23 @@ private static bool IsKeywordArgumentName(string value, int afterNameIndex) return afterNameIndex < value.Length && value[afterNameIndex] == '='; } + private static bool IsKeywordArgumentValue(string value, int nameIndex) + { + var beforeNameIndex = nameIndex - 1; + while (beforeNameIndex >= 0 && char.IsWhiteSpace(value[beforeNameIndex])) + beforeNameIndex--; + + return beforeNameIndex >= 0 && value[beforeNameIndex] == '='; + } + + private static bool IsCallTarget(string value, int afterNameIndex) + { + while (afterNameIndex < value.Length && char.IsWhiteSpace(value[afterNameIndex])) + afterNameIndex++; + + return afterNameIndex < value.Length && value[afterNameIndex] == '('; + } + private static bool IsPythonLiteralName(string name) { return name is "True" or "False" or "None" or "Ellipsis"; diff --git a/tests/CodeIndex.Tests/ReferenceExtractorTests.cs b/tests/CodeIndex.Tests/ReferenceExtractorTests.cs index a869b09ad7..665c646d1b 100644 --- a/tests/CodeIndex.Tests/ReferenceExtractorTests.cs +++ b/tests/CodeIndex.Tests/ReferenceExtractorTests.cs @@ -893,6 +893,8 @@ def wrap(f): return f return wrap + DEFAULT_TIMEOUT = 30 + @bare_decorator @parametrized("value") def wrapped(): @@ -906,6 +908,10 @@ def wrapped_target(): def composed_target(): pass + @cache_with(timeout=DEFAULT_TIMEOUT) + def configured_target(): + pass + @staticmethod def method(): pass @@ -922,7 +928,7 @@ def parametrized_fixture(value): var symbols = SymbolExtractor.Extract(1, "python", content); var references = ReferenceExtractor.Extract(1, "python", content, symbols); - Assert.Equal(7, references.Count(reference => reference.ReferenceKind == "decorator")); + Assert.Equal(8, references.Count(reference => reference.ReferenceKind == "decorator")); Assert.Contains(references, reference => reference.SymbolName == "bare_decorator" && reference.ReferenceKind == "decorator"); @@ -943,7 +949,7 @@ def parametrized_fixture(value): && reference.ReferenceKind == "call"); Assert.Contains(references, reference => reference.SymbolName == "target_func" - && reference.ReferenceKind == "call" + && reference.ReferenceKind == "reference" && reference.Context == "@functools.wraps(target_func)"); Assert.Contains(references, reference => reference.SymbolName == "memoize" @@ -951,8 +957,11 @@ def parametrized_fixture(value): && reference.Context == "@cache_with(timeout=30)(memoize(target_func))"); Assert.Contains(references, reference => reference.SymbolName == "target_func" - && reference.ReferenceKind == "call" + && reference.ReferenceKind == "reference" && reference.Context == "@cache_with(timeout=30)(memoize(target_func))"); + Assert.DoesNotContain(references, reference => + reference.SymbolName == "DEFAULT_TIMEOUT" + && reference.ReferenceKind == "call"); } [Fact] From db69058e3cfdbd1672629594a01fbf4076851d82 Mon Sep 17 00:00:00 2001 From: Widthdom Date: Mon, 25 May 2026 00:23:24 +0900 Subject: [PATCH 3/3] Handle decorator keyword callables (#2055) --- .../Languages/PythonReferenceExtractor.cs | 5 +++-- tests/CodeIndex.Tests/ReferenceExtractorTests.cs | 13 ++++++++++++- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/src/CodeIndex/Indexer/References/Languages/PythonReferenceExtractor.cs b/src/CodeIndex/Indexer/References/Languages/PythonReferenceExtractor.cs index 2f27b09425..462f8cb86b 100644 --- a/src/CodeIndex/Indexer/References/Languages/PythonReferenceExtractor.cs +++ b/src/CodeIndex/Indexer/References/Languages/PythonReferenceExtractor.cs @@ -313,7 +313,8 @@ private static void EmitDecoratorArgumentReferences( continue; if (IsKeywordArgumentName(preparedLine, nameGroup.Index + nameGroup.Length)) continue; - if (IsKeywordArgumentValue(preparedLine, nameGroup.Index)) + var isCallTarget = IsCallTarget(preparedLine, nameGroup.Index + nameGroup.Length); + if (IsKeywordArgumentValue(preparedLine, nameGroup.Index) && !isCallTarget) continue; ReferenceExtractor.AddReference( @@ -322,7 +323,7 @@ private static void EmitDecoratorArgumentReferences( fileId, name, nameGroup.Index, - IsCallTarget(preparedLine, nameGroup.Index + nameGroup.Length) ? "call" : "reference", + isCallTarget ? "call" : "reference", context, lineNumber, container, diff --git a/tests/CodeIndex.Tests/ReferenceExtractorTests.cs b/tests/CodeIndex.Tests/ReferenceExtractorTests.cs index 665c646d1b..960dbae9f0 100644 --- a/tests/CodeIndex.Tests/ReferenceExtractorTests.cs +++ b/tests/CodeIndex.Tests/ReferenceExtractorTests.cs @@ -893,6 +893,9 @@ def wrap(f): return f return wrap + def make_factory(): + return target_func + DEFAULT_TIMEOUT = 30 @bare_decorator @@ -912,6 +915,10 @@ def composed_target(): def configured_target(): pass + @cache_with(factory=make_factory()) + def keyword_factory_target(): + pass + @staticmethod def method(): pass @@ -928,7 +935,7 @@ def parametrized_fixture(value): var symbols = SymbolExtractor.Extract(1, "python", content); var references = ReferenceExtractor.Extract(1, "python", content, symbols); - Assert.Equal(8, references.Count(reference => reference.ReferenceKind == "decorator")); + Assert.Equal(9, references.Count(reference => reference.ReferenceKind == "decorator")); Assert.Contains(references, reference => reference.SymbolName == "bare_decorator" && reference.ReferenceKind == "decorator"); @@ -959,6 +966,10 @@ def parametrized_fixture(value): reference.SymbolName == "target_func" && reference.ReferenceKind == "reference" && reference.Context == "@cache_with(timeout=30)(memoize(target_func))"); + Assert.Contains(references, reference => + reference.SymbolName == "make_factory" + && reference.ReferenceKind == "call" + && reference.Context == "@cache_with(factory=make_factory())"); Assert.DoesNotContain(references, reference => reference.SymbolName == "DEFAULT_TIMEOUT" && reference.ReferenceKind == "call");