diff --git a/changelog.d/unreleased/2057.fixed.md b/changelog.d/unreleased/2057.fixed.md new file mode 100644 index 0000000000..533ce769f5 --- /dev/null +++ b/changelog.d/unreleased/2057.fixed.md @@ -0,0 +1,17 @@ +--- +category: fixed +issues: + - 2057 +affected: + - src/CodeIndex/Indexer/Symbols/SymbolExtractor.Python.cs + - src/CodeIndex/Indexer/References/Languages/PythonReferenceExtractor.cs + - src/CodeIndex/Indexer/References/ReferenceExtractor.cs +--- + +## English + +- **Python dataclass field metadata is now indexed (#2057)** — `field(...)` class attributes are distinguished as dataclass fields, metadata keys are indexed, default factories are referenced, and imported `fields(MyClass)` introspection now links back to the dataclass. + +## 日本語 + +- **Python dataclass field metadata を index するようにしました (#2057)** — `field(...)` class attribute を dataclass field として区別し、metadata key、default factory 参照、import 済み `fields(MyClass)` introspection から dataclass への参照を取得します。 diff --git a/src/CodeIndex/Indexer/References/Languages/PythonReferenceExtractor.cs b/src/CodeIndex/Indexer/References/Languages/PythonReferenceExtractor.cs index 4057b21ae3..876343a10a 100644 --- a/src/CodeIndex/Indexer/References/Languages/PythonReferenceExtractor.cs +++ b/src/CodeIndex/Indexer/References/Languages/PythonReferenceExtractor.cs @@ -100,7 +100,16 @@ internal static class PythonReferenceExtractor @"\b(?:typing|typing_extensions)\.get_type_hints\s*\(\s*(?(?:[_\p{L}]\w*\.)*[_\p{Lu}]\w*)", RegexOptions.Compiled); private static readonly Regex DataclassesFieldsTargetRegex = new( - @"\bdataclasses\.fields\s*\(\s*(?(?:[_\p{L}]\w*\.)*[_\p{Lu}]\w*)", + @"(?(?:[_\p{L}]\w*\.)*[_\p{Lu}]\w*)|\bdataclasses\.fields\s*\(\s*(?(?:[_\p{L}]\w*\.)*[_\p{Lu}]\w*)", + RegexOptions.Compiled); + private static readonly Regex DataclassFieldCallRegex = new( + @"^\s*[_\p{L}]\w*\s*(?::\s*[^=]+)?=\s*(?:(?:dataclasses\.)?field)\s*\(", + RegexOptions.Compiled); + private static readonly Regex DataclassFieldDefaultFactoryRegex = new( + @"\bdefault_factory\s*=\s*(?(?:[_\p{L}]\w*\.)*[_\p{L}]\w*)", + RegexOptions.Compiled); + private static readonly Regex DataclassFieldMetadataRegex = new( + @"\bmetadata\s*=\s*(?\{)", RegexOptions.Compiled); private static readonly Regex AttrsFieldsTargetRegex = new( @"\b(?:attr|attrs)\.fields\s*\(\s*(?(?:[_\p{L}]\w*\.)*[_\p{Lu}]\w*)", @@ -909,6 +918,234 @@ public static void EmitDataclassesFieldsReferences( } } + public static void EmitDataclassFieldReferences( + string[] preparedLines, + string[] originalLines, + int lineIndex, + List references, + HashSet seen, + long fileId, + SymbolRecord? container, + Func isIgnoredName) + { + var preparedLine = preparedLines[lineIndex]; + if (!DataclassFieldCallRegex.IsMatch(preparedLine)) + return; + + var depth = 0; + var sawFieldCall = false; + var inString = false; + var quoteChar = '\0'; + + for (var currentLineIndex = lineIndex; currentLineIndex < preparedLines.Length; currentLineIndex++) + { + var currentPreparedLine = preparedLines[currentLineIndex]; + var currentOriginalLine = originalLines[currentLineIndex]; + var currentContext = currentOriginalLine.Trim(); + var currentLineNumber = currentLineIndex + 1; + + EmitDataclassFieldDefaultFactoryReferences( + currentPreparedLine, + references, + seen, + fileId, + currentContext, + currentLineNumber, + container, + isIgnoredName); + EmitDataclassFieldMetadataReferences( + originalLines, + currentLineIndex, + references, + seen, + fileId, + container, + isIgnoredName); + + for (var column = 0; column < currentPreparedLine.Length; column++) + { + var ch = currentPreparedLine[column]; + if (inString) + { + if (ch == '\\') + { + column++; + continue; + } + + if (ch == quoteChar) + inString = false; + continue; + } + + if (ch == '#') + break; + if (ch is '\'' or '"') + { + inString = true; + quoteChar = ch; + continue; + } + + if (ch == '(') + { + depth++; + sawFieldCall = true; + } + else if (ch == ')' && depth > 0) + { + depth--; + if (sawFieldCall && depth == 0) + return; + } + } + + if (sawFieldCall && depth <= 0) + return; + } + } + + private static void EmitDataclassFieldDefaultFactoryReferences( + string preparedLine, + List references, + HashSet seen, + long fileId, + string context, + int lineNumber, + SymbolRecord? container, + Func isIgnoredName) + { + foreach (Match match in DataclassFieldDefaultFactoryRegex.Matches(preparedLine)) + { + var name = match.Groups["name"].Value; + if (isIgnoredName(name)) + continue; + + ReferenceExtractor.AddReference( + references, + seen, + fileId, + name, + match.Groups["name"].Index, + "call", + context, + lineNumber, + container, + "python"); + } + } + + private static void EmitDataclassFieldMetadataReferences( + string[] originalLines, + int lineIndex, + List references, + HashSet seen, + long fileId, + SymbolRecord? container, + Func isIgnoredName) + { + var metadataMatch = DataclassFieldMetadataRegex.Match(originalLines[lineIndex]); + if (!metadataMatch.Success) + return; + + var currentLineIndex = lineIndex; + var currentColumn = metadataMatch.Groups["values"].Index; + var depth = 0; + var inString = false; + var quoteChar = '\0'; + var stringStartColumn = -1; + + while (currentLineIndex < originalLines.Length) + { + var currentLine = originalLines[currentLineIndex]; + if (currentColumn >= currentLine.Length) + { + if (depth <= 0 && !inString) + break; + + currentLineIndex++; + currentColumn = 0; + continue; + } + + var ch = currentLine[currentColumn]; + if (inString) + { + if (ch == '\\' && currentColumn + 1 < currentLine.Length) + { + currentColumn += 2; + continue; + } + + if (ch == quoteChar) + { + var afterStringColumn = currentColumn + 1; + while (afterStringColumn < currentLine.Length && char.IsWhiteSpace(currentLine[afterStringColumn])) + afterStringColumn++; + + if (afterStringColumn < currentLine.Length && currentLine[afterStringColumn] == ':') + { + var name = currentLine[stringStartColumn..currentColumn].Trim(); + if (name.Length > 0 && !isIgnoredName(name)) + { + ReferenceExtractor.AddReference( + references, + seen, + fileId, + name, + stringStartColumn, + "annotation", + currentLine.Trim(), + currentLineIndex + 1, + container, + "python"); + } + } + + inString = false; + quoteChar = '\0'; + stringStartColumn = -1; + currentColumn++; + continue; + } + + currentColumn++; + continue; + } + + if (ch == '#') + break; + + if (ch is '\'' or '"') + { + inString = true; + quoteChar = ch; + stringStartColumn = currentColumn + 1; + currentColumn++; + continue; + } + + if (ch is '{' or '[' or '(') + { + depth++; + currentColumn++; + continue; + } + + if (ch is '}' or ']' or ')') + { + if (depth > 0) + depth--; + currentColumn++; + if (depth <= 0) + break; + continue; + } + + currentColumn++; + } + } + public static void EmitAttrsFieldsReferences( string preparedLine, List references, diff --git a/src/CodeIndex/Indexer/References/ReferenceExtractor.cs b/src/CodeIndex/Indexer/References/ReferenceExtractor.cs index 5ffd082207..ef42f24ab5 100644 --- a/src/CodeIndex/Indexer/References/ReferenceExtractor.cs +++ b/src/CodeIndex/Indexer/References/ReferenceExtractor.cs @@ -3119,6 +3119,15 @@ void AddGradleDslReference(string name, int callIndex) lineNumber, container, name => IsIgnoredCallName(language, name)); + PythonReferenceExtractor.EmitDataclassFieldReferences( + preparedLines, + lines, + i, + references, + seen, + fileId, + container, + name => IsIgnoredCallName(language, name)); PythonReferenceExtractor.EmitAttrsFieldsReferences( preparedLine, references, diff --git a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.Python.cs b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.Python.cs index 291bf516b7..0b3b2f815b 100644 --- a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.Python.cs +++ b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.Python.cs @@ -16,6 +16,8 @@ public static partial class SymbolExtractor private static readonly Regex PythonAllExtendRegex = new(@"^\s*__all__\.extend\(\s*(?.*)$", RegexOptions.Compiled | RegexOptions.CultureInvariant); private static readonly Regex PythonClassAnnotatedAttributeRegex = new(@"^\s*(?[_\p{L}]\w*)\s*:\s*[^=].*$", RegexOptions.Compiled | RegexOptions.CultureInvariant); private static readonly Regex PythonClassAssignedAttributeRegex = new(@"^\s*(?[_\p{L}]\w*)\s*=(?!=).*$", RegexOptions.Compiled | RegexOptions.CultureInvariant); + private static readonly Regex PythonDataclassFieldAttributeRegex = new(@"^\s*(?[_\p{L}]\w*)\s*(?::\s*[^=]+)?=\s*(?:(?:dataclasses\.)?field)\s*\(", RegexOptions.Compiled | RegexOptions.CultureInvariant); + private static readonly Regex PythonDataclassFieldMetadataRegex = new(@"\bmetadata\s*=\s*(?\{)", RegexOptions.Compiled | RegexOptions.CultureInvariant); private static readonly Regex PythonClassSlotsAssignmentRegex = new(@"^\s*__slots__\s*(?:\+?=)\s*(?.+)$", RegexOptions.Compiled | RegexOptions.CultureInvariant); private static readonly Regex PythonClassMatchArgsAssignmentRegex = new(@"^\s*__match_args__\s*(?:\+?=)\s*(?.+)$", RegexOptions.Compiled | RegexOptions.CultureInvariant); private static readonly Regex PythonClassAnnotationsAssignmentRegex = new(@"^\s*__annotations__\s*(?:\+?=)\s*(?.+)$", RegexOptions.Compiled | RegexOptions.CultureInvariant); @@ -381,6 +383,36 @@ private static void ExtractPythonClassAttributeSymbols( continue; } + var fieldMatch = PythonDataclassFieldAttributeRegex.Match(line); + if (fieldMatch.Success) + { + AddPythonClassPropertySymbol( + fileId, + lines, + symbols, + fieldMatch.Groups["name"].Value, + i, + fieldMatch.Groups["name"].Index, + subKind: "dataclass_field"); + + var metadataKeys = TryExpandPythonDataclassFieldMetadataKeys(lines, i, fieldMatch.Groups["name"].Index); + if (metadataKeys != null) + { + foreach (var key in metadataKeys) + { + AddPythonDataclassFieldMetadataSymbol( + fileId, + lines, + symbols, + key.Name, + key.LineIndex, + key.StartColumn); + } + } + + continue; + } + var match = PythonClassAnnotatedAttributeRegex.Match(line); if (!match.Success) match = PythonClassAssignedAttributeRegex.Match(line); @@ -404,7 +436,8 @@ private static void AddPythonClassPropertySymbol( List symbols, string name, int lineIndex, - int startColumn) + int startColumn, + string? subKind = null) { AddSymbolRecord( symbols, @@ -414,6 +447,34 @@ private static void AddPythonClassPropertySymbol( { FileId = fileId, Kind = "property", + SubKind = subKind, + Name = name, + Line = lineIndex + 1, + StartLine = lineIndex + 1, + StartColumn = startColumn, + EndLine = lineIndex + 1, + Signature = lines[lineIndex].Trim(), + }, + lines[lineIndex]); + } + + private static void AddPythonDataclassFieldMetadataSymbol( + long fileId, + string[] lines, + List symbols, + string name, + int lineIndex, + int startColumn) + { + AddSymbolRecord( + symbols, + cssSeenSymbols: null, + lineIndex + 1, + new SymbolRecord + { + FileId = fileId, + Kind = "reference", + SubKind = "dataclass_field_metadata", Name = name, Line = lineIndex + 1, StartLine = lineIndex + 1, @@ -559,6 +620,73 @@ private static void ExtractPythonWalrusSymbols( return entries.Count > 0 ? entries : null; } + private static List? TryExpandPythonDataclassFieldMetadataKeys( + string[] lines, + int fieldLineIndex, + int fieldStartColumn) + { + var currentLineIndex = fieldLineIndex; + var currentColumn = fieldStartColumn; + var depth = 0; + var sawFieldCall = false; + var inString = false; + var quoteChar = '\0'; + + while (currentLineIndex < lines.Length) + { + var currentLine = lines[currentLineIndex]; + var metadataMatch = PythonDataclassFieldMetadataRegex.Match(currentLine); + if (metadataMatch.Success) + return TryExpandPythonStringDictionaryKeys(lines, currentLineIndex, metadataMatch.Groups["values"].Index); + + for (; currentColumn < currentLine.Length; currentColumn++) + { + var ch = currentLine[currentColumn]; + if (inString) + { + if (ch == '\\') + { + currentColumn++; + continue; + } + + if (ch == quoteChar) + inString = false; + continue; + } + + if (ch == '#') + break; + if (ch is '\'' or '"') + { + inString = true; + quoteChar = ch; + continue; + } + + if (ch == '(') + { + depth++; + sawFieldCall = true; + } + else if (ch == ')' && depth > 0) + { + depth--; + if (sawFieldCall && depth == 0) + return null; + } + } + + if (sawFieldCall && depth <= 0) + return null; + + currentLineIndex++; + currentColumn = 0; + } + + return null; + } + private static List? TryExpandPythonAllExportSymbols(string[] lines, int lineIndex) { var line = lines[lineIndex]; diff --git a/tests/CodeIndex.Tests/ReferenceExtractorTests.cs b/tests/CodeIndex.Tests/ReferenceExtractorTests.cs index 0ffc9c4f68..0ac459adde 100644 --- a/tests/CodeIndex.Tests/ReferenceExtractorTests.cs +++ b/tests/CodeIndex.Tests/ReferenceExtractorTests.cs @@ -57,6 +57,50 @@ def beta(): && !reference.IsSelfReference); } + [Fact] + public void Extract_PythonDataclassField_EmitsMetadataAndDefaultFactoryReferences() + { + const string content = """ + from dataclasses import dataclass, field, fields + + @dataclass + class Job: + callback: Callable[[Payload], Result] = field( + default_factory=list, + metadata={ + "wire_name": "callback", + }, + ) + + def inspect_job(): + return fields(Job) + """; + + var symbols = SymbolExtractor.Extract(1, "python", content); + var references = ReferenceExtractor.Extract(1, "python", content, symbols); + + Assert.Contains(references, reference => + reference.SymbolName == "Payload" + && reference.ReferenceKind == "type_reference" + && reference.ContainerName == "Job"); + Assert.Contains(references, reference => + reference.SymbolName == "Result" + && reference.ReferenceKind == "type_reference" + && reference.ContainerName == "Job"); + Assert.Contains(references, reference => + reference.SymbolName == "list" + && reference.ReferenceKind == "call" + && reference.ContainerName == "Job"); + Assert.Contains(references, reference => + reference.SymbolName == "wire_name" + && reference.ReferenceKind == "annotation" + && reference.ContainerName == "Job"); + Assert.Contains(references, reference => + reference.SymbolName == "Job" + && reference.ReferenceKind == "type_reference" + && reference.ContainerName == "inspect_job"); + } + [Fact] public void BuildReferenceDedupeKey_IncludesFileIdAndLanguage() { diff --git a/tests/CodeIndex.Tests/SymbolExtractorTests.cs b/tests/CodeIndex.Tests/SymbolExtractorTests.cs index de55449729..1edc77ecfa 100644 --- a/tests/CodeIndex.Tests/SymbolExtractorTests.cs +++ b/tests/CodeIndex.Tests/SymbolExtractorTests.cs @@ -53,6 +53,39 @@ internal static string[] MaskLines(string? lang, string[] originalLines) Assert.Contains(symbols, symbol => symbol.Kind == "class" && symbol.Name == "StructuralLineMasker"); } + [Fact] + public void Extract_PythonDataclassField_IndexesFieldAndMetadataKeys() + { + const string content = """ + from dataclasses import dataclass, field + + @dataclass + class Job: + callback: Callable[[Payload], Result] = field( + default_factory=list, + metadata={"wire_name": "callback", "role": "handler"}, + ) + """; + + var symbols = SymbolExtractor.Extract(1, "python", content); + + Assert.Contains(symbols, symbol => + symbol.Kind == "property" + && symbol.SubKind == "dataclass_field" + && symbol.Name == "callback" + && symbol.Line == 5); + Assert.Contains(symbols, symbol => + symbol.Kind == "reference" + && symbol.SubKind == "dataclass_field_metadata" + && symbol.Name == "wire_name" + && symbol.Line == 7); + Assert.Contains(symbols, symbol => + symbol.Kind == "reference" + && symbol.SubKind == "dataclass_field_metadata" + && symbol.Name == "role" + && symbol.Line == 7); + } + [Theory] [InlineData("csharp", "Pages/Product.razor")] [InlineData("csharp", "Views/Product.cshtml")]