diff --git a/changelog.d/unreleased/2059.fixed.md b/changelog.d/unreleased/2059.fixed.md new file mode 100644 index 0000000000..460e771f72 --- /dev/null +++ b/changelog.d/unreleased/2059.fixed.md @@ -0,0 +1,17 @@ +--- +category: fixed +issues: + - 2059 +affected: + - src/CodeIndex/Indexer/References/Languages/PythonReferenceExtractor.cs + - src/CodeIndex/Indexer/References/ReferenceExtractor.cs + - tests/CodeIndex.Tests/ReferenceExtractorTests.cs +--- + +## English + +- **Python type-hint references now cover ParamSpec and TypeVarTuple edge cases (#2059)** — Python reference extraction now traverses comma-containing Callable annotations, multiline TypeVar constraints, ParamSpec bounds, TypeVarTuple unpacking, and nested Literal union operands. + +## 日本語 + +- **Python の型ヒント参照が ParamSpec と TypeVarTuple の端ケースを扱うようになりました (#2059)** — Python reference extraction は、カンマを含む Callable annotation、複数行 TypeVar constraint、ParamSpec bound、TypeVarTuple unpacking、ネストした Literal union operand を走査するようになりました。 diff --git a/src/CodeIndex/Indexer/References/Languages/PythonReferenceExtractor.cs b/src/CodeIndex/Indexer/References/Languages/PythonReferenceExtractor.cs index 02572c3d9a..2c52ffa6c4 100644 --- a/src/CodeIndex/Indexer/References/Languages/PythonReferenceExtractor.cs +++ b/src/CodeIndex/Indexer/References/Languages/PythonReferenceExtractor.cs @@ -73,7 +73,7 @@ internal static class PythonReferenceExtractor @":\s*(?(?:[_\p{L}]\w*\.)*[_\p{Lu}]\w*)(?=\s*(?:=|,|$))", RegexOptions.Compiled); private static readonly Regex AnnotationExpressionTypeRegex = new( - @":\s*(?[^=,]+)(?=\s*(?:=|,|$))", + @":\s*(?[^=]+)(?=\s*(?:=|$))", RegexOptions.Compiled); private static readonly Regex VariableAnnotationTypeRegex = new( @"^\s*(?:self\.)?\w+\s*:\s*(?(?:[_\p{L}]\w*\.)*[_\p{Lu}]\w*)(?=\s*(?:=|#|$))", @@ -88,11 +88,11 @@ internal static class PythonReferenceExtractor @"\b(?:(?:typing|typing_extensions)\.)?NewType\s*\(\s*[^,\n]+,\s*(?(?:[_\p{L}]\w*\.)*[_\p{Lu}]\w*)", RegexOptions.Compiled); private static readonly Regex TypeVarBoundTypeRegex = new( - @"\b(?:(?:typing|typing_extensions)\.)?TypeVar\s*\([^)]*\bbound\s*=\s*(?(?:[_\p{L}]\w*\.)*[_\p{Lu}]\w*)", + @"\b(?:(?:typing|typing_extensions)\.)?(?:TypeVar|ParamSpec)\s*\([^)]*\bbound\s*=\s*(?[^)]*)\)", RegexOptions.Compiled); private static readonly Regex TypeVarConstraintTypesRegex = new( - @"\b(?:(?:typing|typing_extensions)\.)?TypeVar\s*\(\s*[^,\n]+,\s*(?[^)=]*,[^)=]*)\)", - RegexOptions.Compiled); + @"\b(?:(?:typing|typing_extensions)\.)?(?:TypeVar|ParamSpec|TypeVarTuple)\s*\(\s*[^,\n]+,\s*(?[^)]*)\)", + RegexOptions.Compiled | RegexOptions.Singleline); private static readonly Regex GetTypeHintsTargetRegex = new( @"(?(?:[_\p{L}]\w*\.)*[_\p{Lu}]\w*)", RegexOptions.Compiled); @@ -187,6 +187,61 @@ private static void EmitPythonTypeExpressionReferences( } } + private static IEnumerable<(string Text, int Offset)> EnumeratePythonTopLevelCommaSegments(string value) + { + var start = 0; + var parenDepth = 0; + var bracketDepth = 0; + var braceDepth = 0; + var inString = false; + var quote = '\0'; + + for (var index = 0; index < value.Length; index++) + { + var ch = value[index]; + if (inString) + { + if (ch == '\\') + { + index++; + continue; + } + + if (ch == quote) + inString = false; + continue; + } + + if (ch is '\'' or '"') + { + inString = true; + quote = ch; + continue; + } + + if (ch == '(') + parenDepth++; + else if (ch == ')' && parenDepth > 0) + parenDepth--; + else if (ch == '[') + bracketDepth++; + else if (ch == ']' && bracketDepth > 0) + bracketDepth--; + else if (ch == '{') + braceDepth++; + else if (ch == '}' && braceDepth > 0) + braceDepth--; + else if (ch == ',' && parenDepth == 0 && bracketDepth == 0 && braceDepth == 0) + { + yield return (value[start..index], start); + start = index + 1; + } + } + + if (start <= value.Length) + yield return (value[start..], start); + } + public static void EmitDecoratorReferences( string preparedLine, List references, @@ -656,39 +711,42 @@ public static void EmitFunctionParameterReferences( foreach (Match functionMatch in FunctionParameterListRegex.Matches(preparedLine)) { var paramsGroup = functionMatch.Groups["params"]; - foreach (Match annotationMatch in AnnotationExpressionTypeRegex.Matches(paramsGroup.Value)) + foreach (var (parameterSegment, parameterOffset) in EnumeratePythonTopLevelCommaSegments(paramsGroup.Value)) { - var typeGroup = annotationMatch.Groups["type"]; - EmitPythonTypeExpressionReferences( - typeGroup, - references, - seen, - fileId, - context, - lineNumber, - container, - index => resolveContainerForReference(paramsGroup.Index + index), - isIgnoredName, - paramsGroup.Index); - } + foreach (Match annotationMatch in AnnotationExpressionTypeRegex.Matches(parameterSegment)) + { + var typeGroup = annotationMatch.Groups["type"]; + EmitPythonTypeExpressionReferences( + typeGroup, + references, + seen, + fileId, + context, + lineNumber, + container, + index => resolveContainerForReference(paramsGroup.Index + parameterOffset + index), + isIgnoredName, + paramsGroup.Index + parameterOffset); + } - foreach (Match annotationMatch in DirectAnnotationTypeRegex.Matches(paramsGroup.Value)) - { - var name = annotationMatch.Groups["name"].Value; - if (isIgnoredName(name)) - continue; + foreach (Match annotationMatch in DirectAnnotationTypeRegex.Matches(parameterSegment)) + { + var name = annotationMatch.Groups["name"].Value; + if (isIgnoredName(name)) + continue; - var nameIndex = paramsGroup.Index + annotationMatch.Groups["name"].Index; - ReferenceExtractor.AddTypeReferenceSegments( - references, - seen, - fileId, - name, - nameIndex, - context, - lineNumber, - resolveContainerForReference(nameIndex) ?? container, - "python"); + var nameIndex = paramsGroup.Index + parameterOffset + annotationMatch.Groups["name"].Index; + ReferenceExtractor.AddTypeReferenceSegments( + references, + seen, + fileId, + name, + nameIndex, + context, + lineNumber, + resolveContainerForReference(nameIndex) ?? container, + "python"); + } } } } @@ -804,20 +862,16 @@ public static void EmitTypeVarBoundReferences( { foreach (Match match in TypeVarBoundTypeRegex.Matches(preparedLine)) { - var name = match.Groups["name"].Value; - if (isIgnoredName(name)) - continue; - - ReferenceExtractor.AddTypeReferenceSegments( + EmitPythonTypeExpressionReferences( + match.Groups["type"], references, seen, fileId, - name, - match.Groups["name"].Index, context, lineNumber, container, - "python"); + resolveContainerForReference: null, + isIgnoredName); } } @@ -834,23 +888,16 @@ public static void EmitTypeVarConstraintReferences( foreach (Match match in TypeVarConstraintTypesRegex.Matches(preparedLine)) { var typesGroup = match.Groups["types"]; - foreach (Match typeMatch in TypeNameRegex.Matches(typesGroup.Value)) - { - var name = typeMatch.Groups["name"].Value; - if (isIgnoredName(name)) - continue; - - ReferenceExtractor.AddTypeReferenceSegments( - references, - seen, - fileId, - name, - typesGroup.Index + typeMatch.Groups["name"].Index, - context, - lineNumber, - container, - "python"); - } + EmitPythonTypeExpressionReferences( + typesGroup, + references, + seen, + fileId, + context, + lineNumber, + container, + resolveContainerForReference: null, + isIgnoredName); } } diff --git a/src/CodeIndex/Indexer/References/ReferenceExtractor.cs b/src/CodeIndex/Indexer/References/ReferenceExtractor.cs index fa5aafcfd8..612ea1dc0a 100644 --- a/src/CodeIndex/Indexer/References/ReferenceExtractor.cs +++ b/src/CodeIndex/Indexer/References/ReferenceExtractor.cs @@ -2981,6 +2981,20 @@ void AddGradleDslReference(string name, int callIndex) pythonPreparedLine = builtPythonHeaderMap.Text; pythonHeaderMap = builtPythonHeaderMap; } + var pythonTypeFactoryLine = preparedLine; + var pythonTypeFactoryMap = default(PythonLogicalHeaderReferenceLine?); + if (preparedLine.Contains("TypeVar", StringComparison.Ordinal) + || preparedLine.Contains("ParamSpec", StringComparison.Ordinal)) + { + var typeFactoryStartColumn = originalLine.IndexOfAny(['T', 'P']); + if (typeFactoryStartColumn < 0) + typeFactoryStartColumn = 0; + if (TryBuildPythonLogicalStatementReferenceLine(lines, i, typeFactoryStartColumn, out var builtPythonTypeFactoryMap)) + { + pythonTypeFactoryLine = builtPythonTypeFactoryMap.Text; + pythonTypeFactoryMap = builtPythonTypeFactoryMap; + } + } var pythonHeaderContainer = pythonHeaderSymbol ?? container; var pythonReferenceStart = references.Count; @@ -3105,8 +3119,9 @@ void AddGradleDslReference(string name, int callIndex) lineNumber, container, name => IsIgnoredCallName(language, name)); + var pythonTypeFactoryReferenceStart = references.Count; PythonReferenceExtractor.EmitTypeVarBoundReferences( - preparedLine, + pythonTypeFactoryLine, references, seen, fileId, @@ -3115,7 +3130,7 @@ void AddGradleDslReference(string name, int callIndex) container, name => IsIgnoredCallName(language, name)); PythonReferenceExtractor.EmitTypeVarConstraintReferences( - preparedLine, + pythonTypeFactoryLine, references, seen, fileId, @@ -3186,6 +3201,9 @@ void AddGradleDslReference(string name, int callIndex) lineNumber, container, name => IsIgnoredCallName(language, name)); + + if (pythonTypeFactoryMap.HasValue) + RemapPythonLogicalHeaderReferences(references, pythonTypeFactoryReferenceStart, pythonTypeFactoryMap.Value, lines); PythonReferenceExtractor.EmitDynamicImportReferences( preparedLine, originalLine, @@ -3624,7 +3642,8 @@ private static bool TryBuildPythonLogicalHeaderReferenceLine( { var line = lines[lineIndex]; var column = lineIndex == startLineIndex ? startColumn : FindFirstNonWhitespaceColumn(line); - if (column < line.Length) + var fragmentEndColumn = FindPythonCommentColumn(line, column); + if (column < fragmentEndColumn) { if (builder.Length > 0) { @@ -3633,10 +3652,10 @@ private static bool TryBuildPythonLogicalHeaderReferenceLine( physicalColumns.Add(column); } - for (var fragmentColumn = column; fragmentColumn < line.Length; fragmentColumn++) + for (var fragmentColumn = column; fragmentColumn < fragmentEndColumn; fragmentColumn++) { var fragmentChar = line[fragmentColumn]; - if (fragmentChar == '\\' && fragmentColumn == line.Length - 1) + if (fragmentChar == '\\' && fragmentColumn == fragmentEndColumn - 1) break; builder.Append(fragmentChar); @@ -3693,6 +3712,123 @@ private static bool TryBuildPythonLogicalHeaderReferenceLine( return header.Text.Length > 0; } + private static bool TryBuildPythonLogicalStatementReferenceLine( + string[] lines, + int startLineIndex, + int startColumn, + out PythonLogicalHeaderReferenceLine header) + { + var builder = new StringBuilder(); + var physicalLines = new List(); + var physicalColumns = new List(); + var parenDepth = 0; + var bracketDepth = 0; + var inString = false; + var quote = '\0'; + + for (var lineIndex = startLineIndex; lineIndex < lines.Length; lineIndex++) + { + var line = lines[lineIndex]; + var column = lineIndex == startLineIndex ? startColumn : FindFirstNonWhitespaceColumn(line); + var fragmentEndColumn = FindPythonCommentColumn(line, column); + if (column < fragmentEndColumn) + { + if (builder.Length > 0) + { + builder.Append(' '); + physicalLines.Add(lineIndex); + physicalColumns.Add(column); + } + + for (var fragmentColumn = column; fragmentColumn < fragmentEndColumn; fragmentColumn++) + { + var fragmentChar = line[fragmentColumn]; + if (fragmentChar == '\\' && fragmentColumn == fragmentEndColumn - 1) + break; + + builder.Append(fragmentChar); + physicalLines.Add(lineIndex); + physicalColumns.Add(fragmentColumn); + } + } + + for (var scan = column; scan < line.Length; scan++) + { + var ch = line[scan]; + if (inString) + { + if (ch == '\\') + { + scan++; + continue; + } + + if (ch == quote) + inString = false; + continue; + } + + if (ch is '\'' or '"') + { + inString = true; + quote = ch; + continue; + } + + if (ch == '#') + break; + if (ch == '(') + parenDepth++; + else if (ch == ')' && parenDepth > 0) + parenDepth--; + else if (ch == '[') + bracketDepth++; + else if (ch == ']' && bracketDepth > 0) + bracketDepth--; + } + + if (parenDepth == 0 && bracketDepth == 0 && !line.TrimEnd().EndsWith('\\')) + break; + } + + header = new PythonLogicalHeaderReferenceLine(builder.ToString(), physicalLines.ToArray(), physicalColumns.ToArray()); + return header.Text.Length > 0; + } + + private static int FindPythonCommentColumn(string line, int startColumn) + { + var inString = false; + var quote = '\0'; + for (var index = startColumn; index < line.Length; index++) + { + var ch = line[index]; + if (inString) + { + if (ch == '\\') + { + index++; + continue; + } + + if (ch == quote) + inString = false; + continue; + } + + if (ch is '\'' or '"') + { + inString = true; + quote = ch; + continue; + } + + if (ch == '#') + return index; + } + + return line.Length; + } + private static int FindFirstNonWhitespaceColumn(string line) { var index = 0; diff --git a/tests/CodeIndex.Tests/ReferenceExtractorTests.cs b/tests/CodeIndex.Tests/ReferenceExtractorTests.cs index 3f6ab4a206..be0acd776d 100644 --- a/tests/CodeIndex.Tests/ReferenceExtractorTests.cs +++ b/tests/CodeIndex.Tests/ReferenceExtractorTests.cs @@ -1557,6 +1557,147 @@ public void Extract_PythonTypeVarConstraints_CapturesConstraintTypeReferences() && reference.ReferenceKind == "type_reference"); } + [Fact] + public void Extract_PythonTypeVarConstraints_MultilineCapturesConstraintTypeReferences() + { + const string content = """ + TAccount = TypeVar( + "TAccount", + models.User, + models.Admin, + ) + """; + + var symbols = SymbolExtractor.Extract(1, "python", content); + var references = ReferenceExtractor.Extract(1, "python", content, symbols); + + Assert.Contains(references, reference => + reference.SymbolName == "User" + && reference.ReferenceKind == "type_reference" + && reference.Line == 3); + Assert.Contains(references, reference => + reference.SymbolName == "Admin" + && reference.ReferenceKind == "type_reference" + && reference.Line == 4); + } + + [Fact] + public void Extract_PythonTypeVarConstraints_MultilineDoesNotCaptureCommentTypeNames() + { + const string content = """ + TAccount = TypeVar( + "TAccount", + models.Admin, # models.User should stay a comment + ) + """; + + var symbols = SymbolExtractor.Extract(1, "python", content); + var references = ReferenceExtractor.Extract(1, "python", content, symbols); + + Assert.Contains(references, reference => + reference.SymbolName == "Admin" + && reference.ReferenceKind == "type_reference" + && reference.Line == 3); + Assert.DoesNotContain(references, reference => + reference.SymbolName == "User" + && reference.ReferenceKind == "type_reference"); + } + + [Fact] + public void Extract_PythonParamSpecBound_CapturesNestedCallableTypeReferences() + { + const string content = """ + P = ParamSpec("P", bound=Callable[models.User, results.Result]) + """; + + var symbols = SymbolExtractor.Extract(1, "python", content); + var references = ReferenceExtractor.Extract(1, "python", content, symbols); + + Assert.Contains(references, reference => + reference.SymbolName == "User" + && reference.ReferenceKind == "type_reference"); + Assert.Contains(references, reference => + reference.SymbolName == "Result" + && reference.ReferenceKind == "type_reference"); + } + + [Fact] + public void Extract_PythonCallableParamSpecAnnotation_CapturesReturnTypeAfterComma() + { + const string content = """ + def bind(callback: Callable[P.args, results.Result]): + return callback + """; + + var symbols = SymbolExtractor.Extract(1, "python", content); + var references = ReferenceExtractor.Extract(1, "python", content, symbols); + + Assert.Contains(references, reference => + reference.SymbolName == "P" + && reference.ReferenceKind == "type_reference" + && reference.ContainerName == "bind"); + Assert.Contains(references, reference => + reference.SymbolName == "Result" + && reference.ReferenceKind == "type_reference" + && reference.ContainerName == "bind"); + } + + [Fact] + public void Extract_PythonCallableParameterAnnotation_DoesNotCaptureNextParameterName() + { + const string content = """ + def bind(callback: Callable[P.args, results.Result], Request=None): + return callback + """; + + var symbols = SymbolExtractor.Extract(1, "python", content); + var references = ReferenceExtractor.Extract(1, "python", content, symbols); + + Assert.Contains(references, reference => + reference.SymbolName == "Result" + && reference.ReferenceKind == "type_reference" + && reference.ContainerName == "bind"); + Assert.DoesNotContain(references, reference => + reference.SymbolName == "Request" + && reference.ReferenceKind == "type_reference"); + } + + [Fact] + public void Extract_PythonTypeVarTupleUnpack_CapturesTupleTypeReference() + { + const string content = """ + type Packed = tuple[*Ts, results.Result] + """; + + var symbols = SymbolExtractor.Extract(1, "python", content); + var references = ReferenceExtractor.Extract(1, "python", content, symbols); + + Assert.Contains(references, reference => + reference.SymbolName == "Ts" + && reference.ReferenceKind == "type_reference"); + Assert.Contains(references, reference => + reference.SymbolName == "Result" + && reference.ReferenceKind == "type_reference"); + } + + [Fact] + public void Extract_PythonLiteralUnion_CapturesNestedUnionTypeReferences() + { + const string content = """ + type Choice = Literal["a", "b"] | models.User | results.Result + """; + + var symbols = SymbolExtractor.Extract(1, "python", content); + var references = ReferenceExtractor.Extract(1, "python", content, symbols); + + Assert.Contains(references, reference => + reference.SymbolName == "User" + && reference.ReferenceKind == "type_reference"); + Assert.Contains(references, reference => + reference.SymbolName == "Result" + && reference.ReferenceKind == "type_reference"); + } + [Fact] public void Extract_PythonGetTypeHints_CapturesTargetTypeReference() {