diff --git a/changelog.d/unreleased/2056.fixed.md b/changelog.d/unreleased/2056.fixed.md new file mode 100644 index 0000000000..88080e4f57 --- /dev/null +++ b/changelog.d/unreleased/2056.fixed.md @@ -0,0 +1,17 @@ +--- +category: fixed +issues: + - 2056 +affected: + - src/CodeIndex/Indexer/Symbols/SymbolExtractor.Python.cs + - src/CodeIndex/Indexer/References/Languages/PythonReferenceExtractor.cs + - src/CodeIndex/Indexer/References/ReferenceExtractor.cs +--- + +## English + +- **Python dynamic import literals are now indexed (#2056)** — `importlib.import_module(...)`, `importlib.util.find_spec(...)`, and `__import__(...)` string-literal module names now produce import symbols and references, while `importlib` calls remain visible in the reference graph. + +## 日本語 + +- **Python の dynamic import literal を index するようになりました (#2056)** — `importlib.import_module(...)`、`importlib.util.find_spec(...)`、`__import__(...)` の文字列 literal モジュール名が import symbol / reference として記録され、`importlib` 呼び出しも reference graph に残るようになりました。 diff --git a/src/CodeIndex/Indexer/References/Languages/PythonReferenceExtractor.cs b/src/CodeIndex/Indexer/References/Languages/PythonReferenceExtractor.cs index 4057b21ae3..36c5a33e84 100644 --- a/src/CodeIndex/Indexer/References/Languages/PythonReferenceExtractor.cs +++ b/src/CodeIndex/Indexer/References/Languages/PythonReferenceExtractor.cs @@ -114,6 +114,18 @@ internal static class PythonReferenceExtractor private static readonly Regex ContextlibSuppressTypeRegex = new( @"\bcontextlib\.suppress\s*\(\s*(?(?:[_\p{L}]\w*\.)*[_\p{Lu}]\w*)", RegexOptions.Compiled); + private static readonly Regex ImportlibDynamicImportRegex = new( + @"\bimportlib(?:\.util)?\.(?:import_module|find_spec)\s*\(", + RegexOptions.Compiled); + private static readonly Regex ImportlibDynamicImportLiteralRegex = new( + @"\bimportlib(?:\.util)?\.(?:import_module|find_spec)\s*\(\s*(?['""])(?[^'""]+)\k", + RegexOptions.Compiled); + private static readonly Regex BuiltinDynamicImportRegex = new( + @"(?['""])(?[^'""]+)\k", + RegexOptions.Compiled); private static string NormalizePythonAnnotationExpression(string expression) { @@ -1024,4 +1036,70 @@ public static void EmitContextlibSuppressReferences( "python"); } } + + public static void EmitDynamicImportReferences( + string preparedLine, + string originalLine, + List references, + HashSet seen, + long fileId, + string context, + int lineNumber, + SymbolRecord? container) + { + foreach (Match match in ImportlibDynamicImportRegex.Matches(preparedLine)) + { + ReferenceExtractor.AddReference( + references, + seen, + fileId, + "importlib", + match.Index, + "call", + context, + lineNumber, + container, + "python"); + + var literalMatch = ImportlibDynamicImportLiteralRegex.Match(originalLine, match.Index); + if (!literalMatch.Success || literalMatch.Index != match.Index) + continue; + + var moduleGroup = literalMatch.Groups["module"]; + if (moduleGroup.Success && moduleGroup.Value.Length > 0) + { + ReferenceExtractor.AddReference( + references, + seen, + fileId, + moduleGroup.Value, + moduleGroup.Index, + "import", + context, + lineNumber, + container, + "python"); + } + } + + foreach (Match match in BuiltinDynamicImportRegex.Matches(preparedLine)) + { + var literalMatch = BuiltinDynamicImportLiteralRegex.Match(originalLine, match.Index); + if (!literalMatch.Success || literalMatch.Index != match.Index) + continue; + + var moduleGroup = literalMatch.Groups["module"]; + ReferenceExtractor.AddReference( + references, + seen, + fileId, + moduleGroup.Value, + moduleGroup.Index, + "import", + context, + lineNumber, + container, + "python"); + } + } } diff --git a/src/CodeIndex/Indexer/References/ReferenceExtractor.cs b/src/CodeIndex/Indexer/References/ReferenceExtractor.cs index 5ffd082207..13a12a8aee 100644 --- a/src/CodeIndex/Indexer/References/ReferenceExtractor.cs +++ b/src/CodeIndex/Indexer/References/ReferenceExtractor.cs @@ -3155,6 +3155,15 @@ void AddGradleDslReference(string name, int callIndex) lineNumber, container, name => IsIgnoredCallName(language, name)); + PythonReferenceExtractor.EmitDynamicImportReferences( + preparedLine, + originalLine, + references, + seen, + fileId, + context, + lineNumber, + container); if (pythonHeaderMap.HasValue) RemapPythonLogicalHeaderReferences(references, pythonReferenceStart, pythonHeaderMap.Value, lines); } diff --git a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.Python.cs b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.Python.cs index 291bf516b7..68eef851be 100644 --- a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.Python.cs +++ b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.Python.cs @@ -11,6 +11,7 @@ public static partial class SymbolExtractor private readonly record struct PythonExportSymbolEntry(string Name, int LineIndex, int StartColumn); private static readonly Regex PythonDirectImportRegex = new(@"^import\s+(?.+)$", RegexOptions.Compiled | RegexOptions.CultureInvariant); private static readonly Regex PythonFromImportRegex = new(@"^from\s+(?(?:\.+[\w.]*|[\w.]+))\s+import\s+(?.+)$", RegexOptions.Compiled | RegexOptions.CultureInvariant); + private static readonly Regex PythonDynamicImportLiteralRegex = new(@"\b(?:importlib\.import_module|importlib\.util\.find_spec|__import__)\s*\(\s*(?['""])(?[^'""]+)\k", RegexOptions.Compiled | RegexOptions.CultureInvariant); private static readonly Regex PythonAllAssignmentRegex = new(@"^\s*__all__\s*(?:\+?=)\s*(?.+)$", RegexOptions.Compiled | RegexOptions.CultureInvariant); private static readonly Regex PythonAllAppendRegex = new(@"^\s*__all__\.append\(\s*(?['""])(?[^'""]+)\k\s*\)", RegexOptions.Compiled | RegexOptions.CultureInvariant); private static readonly Regex PythonAllExtendRegex = new(@"^\s*__all__\.extend\(\s*(?.*)$", RegexOptions.Compiled | RegexOptions.CultureInvariant); @@ -194,6 +195,20 @@ private static string BuildPythonLogicalHeaderSignature(string[] lines, int star var entries = new List(); var seenNames = new HashSet(StringComparer.Ordinal); + foreach (Match match in PythonDynamicImportLiteralRegex.Matches(statement)) + { + AddPythonImportEntry( + line, + absoluteStartColumn, + match.Groups["module"].Value, + entries, + seenNames, + ref absoluteStartColumn); + } + + if (entries.Count > 0) + return entries; + var directImportMatch = PythonDirectImportRegex.Match(statement); if (directImportMatch.Success) { diff --git a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.cs b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.cs index f8d722d8e8..8f2ff12b88 100644 --- a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.cs +++ b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.cs @@ -813,6 +813,7 @@ private enum JavaScriptTypeScriptFunctionHeaderConsumeResult new("import", new Regex(@"^\s*(?\w+)\s*=\s*(?:(?:typing|typing_extensions)\.)?(?:TypeVar|ParamSpec|TypeVarTuple)\s*\(", RegexOptions.Compiled), BodyStyle.None), new("property", new Regex(@"^\s*(?\w+)\s*:\s*(?:(?:typing|typing_extensions)\.)?Final(?:\[[^\]]+\])?\s*=", RegexOptions.Compiled), BodyStyle.None), new("import", new Regex(@"^\s*(?:from\s+(?(?:\.+[\w.]*|[\w.]+))\s+import\b|import\s+(?[\w.]+))", RegexOptions.Compiled), BodyStyle.None), + new("import", new Regex(@"^\s*(?:[_\p{L}]\w*\s*=\s*)?(?:importlib\.import_module|importlib\.util\.find_spec|__import__)\s*\(\s*['""](?[^'""]+)['""]", RegexOptions.Compiled), BodyStyle.None), ], ["cobol"] = [ diff --git a/tests/CodeIndex.Tests/ReferenceExtractorTests.cs b/tests/CodeIndex.Tests/ReferenceExtractorTests.cs index 0ffc9c4f68..5efdd928ba 100644 --- a/tests/CodeIndex.Tests/ReferenceExtractorTests.cs +++ b/tests/CodeIndex.Tests/ReferenceExtractorTests.cs @@ -1145,6 +1145,47 @@ def __init_subclass__(cls) -> None: && reference.ReferenceKind == "call"); } + [Fact] + public void Extract_PythonDynamicImports_EmitImportAndImportlibReferences() + { + const string content = """ + import importlib + + def load(module_name): + importlib.import_module("plugins.alpha") + __import__('legacy.loader') + importlib.util.find_spec("optional.backend") + importlib.import_module(module_name) + note = "importlib.import_module('not.real')" + # importlib.import_module("commented.out") + """; + + var symbols = SymbolExtractor.Extract(1, "python", content); + var references = ReferenceExtractor.Extract(1, "python", content, symbols); + + Assert.Equal(3, references.Count(reference => + reference.SymbolName == "importlib" + && reference.ReferenceKind == "call" + && reference.ContainerName == "load")); + Assert.Contains(references, reference => + reference.SymbolName == "plugins.alpha" + && reference.ReferenceKind == "import" + && reference.ContainerName == "load"); + Assert.Contains(references, reference => + reference.SymbolName == "legacy.loader" + && reference.ReferenceKind == "import" + && reference.ContainerName == "load"); + Assert.Contains(references, reference => + reference.SymbolName == "optional.backend" + && reference.ReferenceKind == "import" + && reference.ContainerName == "load"); + Assert.DoesNotContain(references, reference => + reference.SymbolName == "module_name" + && reference.ReferenceKind == "import"); + Assert.DoesNotContain(references, reference => reference.SymbolName == "not.real"); + Assert.DoesNotContain(references, reference => reference.SymbolName == "commented.out"); + } + [Fact] public void Extract_PythonStringifiedAnnotations_CapturesNestedForwardReferences() { diff --git a/tests/CodeIndex.Tests/SymbolExtractorTests.cs b/tests/CodeIndex.Tests/SymbolExtractorTests.cs index de55449729..adc01029e4 100644 --- a/tests/CodeIndex.Tests/SymbolExtractorTests.cs +++ b/tests/CodeIndex.Tests/SymbolExtractorTests.cs @@ -855,6 +855,31 @@ from package.subpackage import helper Assert.Contains(imports, symbol => symbol.Name == "helper"); } + [Fact] + public void Extract_Python_IndexesDynamicImportLiteralModules() + { + var content = """ + importlib.import_module("plugins.alpha") + loaded = importlib.import_module("plugins.beta") + __import__('legacy.loader') + importlib.util.find_spec("optional.backend") + importlib.import_module(module_name) + note = "importlib.import_module('not.real')" + # importlib.import_module("commented.out") + """; + + var symbols = SymbolExtractor.Extract(1, "python", content); + var imports = symbols.Where(symbol => symbol.Kind == "import").Select(symbol => symbol.Name).ToList(); + + Assert.Contains("plugins.alpha", imports); + Assert.Contains("plugins.beta", imports); + Assert.Contains("legacy.loader", imports); + Assert.Contains("optional.backend", imports); + Assert.DoesNotContain("module_name", imports); + Assert.DoesNotContain("not.real", imports); + Assert.DoesNotContain("commented.out", imports); + } + [Fact] public void Extract_Python_IndexesAllExportsFromInitModules() {