Fix Python dynamic import indexing

changelog.d/unreleased/2056.fixed.md

@@ -0,0 +1,17 @@
+---
+category: fixed
+issues:
+  - 2056
+affected:
+  - src/CodeIndex/Indexer/Symbols/SymbolExtractor.Python.cs
+  - src/CodeIndex/Indexer/References/Languages/PythonReferenceExtractor.cs
+  - src/CodeIndex/Indexer/References/ReferenceExtractor.cs
+---
+
+## English
+
+- **Python dynamic import literals are now indexed (#2056)** — `importlib.import_module(...)`, `importlib.util.find_spec(...)`, and `__import__(...)` string-literal module names now produce import symbols and references, while `importlib` calls remain visible in the reference graph.
+
+## 日本語
+
+- **Python の dynamic import literal を index するようになりました (#2056)** — `importlib.import_module(...)`、`importlib.util.find_spec(...)`、`__import__(...)` の文字列 literal モジュール名が import symbol / reference として記録され、`importlib` 呼び出しも reference graph に残るようになりました。

src/CodeIndex/Indexer/References/Languages/PythonReferenceExtractor.cs

@@ -114,6 +114,18 @@ internal static class PythonReferenceExtractor
     private static readonly Regex ContextlibSuppressTypeRegex = new(
         @"\bcontextlib\.suppress\s*\(\s*(?<name>(?:[_\p{L}]\w*\.)*[_\p{Lu}]\w*)",
         RegexOptions.Compiled);
+    private static readonly Regex ImportlibDynamicImportRegex = new(
+        @"\bimportlib(?:\.util)?\.(?:import_module|find_spec)\s*\(",
+        RegexOptions.Compiled);
+    private static readonly Regex ImportlibDynamicImportLiteralRegex = new(
+        @"\bimportlib(?:\.util)?\.(?:import_module|find_spec)\s*\(\s*(?<quote>['""])(?<module>[^'""]+)\k<quote>",
+        RegexOptions.Compiled);
+    private static readonly Regex BuiltinDynamicImportRegex = new(
+        @"(?<!\.)\b__import__\s*\(",
+        RegexOptions.Compiled);
+    private static readonly Regex BuiltinDynamicImportLiteralRegex = new(
+        @"(?<!\.)\b__import__\s*\(\s*(?<quote>['""])(?<module>[^'""]+)\k<quote>",
+        RegexOptions.Compiled);
 
     private static string NormalizePythonAnnotationExpression(string expression)
     {
@@ -1024,4 +1036,70 @@ public static void EmitContextlibSuppressReferences(
                 "python");
         }
     }
+
+    public static void EmitDynamicImportReferences(
+        string preparedLine,
+        string originalLine,
+        List<ReferenceRecord> references,
+        HashSet<string> seen,
+        long fileId,
+        string context,
+        int lineNumber,
+        SymbolRecord? container)
+    {
+        foreach (Match match in ImportlibDynamicImportRegex.Matches(preparedLine))
+        {
+            ReferenceExtractor.AddReference(
+                references,
+                seen,
+                fileId,
+                "importlib",
+                match.Index,
+                "call",
+                context,
+                lineNumber,
+                container,
+                "python");
+
+            var literalMatch = ImportlibDynamicImportLiteralRegex.Match(originalLine, match.Index);
+            if (!literalMatch.Success || literalMatch.Index != match.Index)
+                continue;
+
+            var moduleGroup = literalMatch.Groups["module"];
+            if (moduleGroup.Success && moduleGroup.Value.Length > 0)
+            {
+                ReferenceExtractor.AddReference(
+                    references,
+                    seen,
+                    fileId,
+                    moduleGroup.Value,
+                    moduleGroup.Index,
+                    "import",
+                    context,
+                    lineNumber,
+                    container,
+                    "python");
+            }
+        }
+
+        foreach (Match match in BuiltinDynamicImportRegex.Matches(preparedLine))
+        {
+            var literalMatch = BuiltinDynamicImportLiteralRegex.Match(originalLine, match.Index);
+            if (!literalMatch.Success || literalMatch.Index != match.Index)
+                continue;
+
+            var moduleGroup = literalMatch.Groups["module"];
+            ReferenceExtractor.AddReference(
+                references,
+                seen,
+                fileId,
+                moduleGroup.Value,
+                moduleGroup.Index,
+                "import",
+                context,
+                lineNumber,
+                container,
+                "python");
+        }
+    }
 }

src/CodeIndex/Indexer/References/ReferenceExtractor.cs

@@ -3155,6 +3155,15 @@ void AddGradleDslReference(string name, int callIndex)
                     lineNumber,
                     container,
                     name => IsIgnoredCallName(language, name));
+                PythonReferenceExtractor.EmitDynamicImportReferences(
+                    preparedLine,
+                    originalLine,
+                    references,
+                    seen,
+                    fileId,
+                    context,
+                    lineNumber,
+                    container);
                 if (pythonHeaderMap.HasValue)
                     RemapPythonLogicalHeaderReferences(references, pythonReferenceStart, pythonHeaderMap.Value, lines);
             }

src/CodeIndex/Indexer/Symbols/SymbolExtractor.Python.cs

@@ -11,6 +11,7 @@ public static partial class SymbolExtractor
     private readonly record struct PythonExportSymbolEntry(string Name, int LineIndex, int StartColumn);
     private static readonly Regex PythonDirectImportRegex = new(@"^import\s+(?<imports>.+)$", RegexOptions.Compiled | RegexOptions.CultureInvariant);
     private static readonly Regex PythonFromImportRegex = new(@"^from\s+(?<module>(?:\.+[\w.]*|[\w.]+))\s+import\s+(?<imports>.+)$", RegexOptions.Compiled | RegexOptions.CultureInvariant);
+    private static readonly Regex PythonDynamicImportLiteralRegex = new(@"\b(?:importlib\.import_module|importlib\.util\.find_spec|__import__)\s*\(\s*(?<quote>['""])(?<module>[^'""]+)\k<quote>", RegexOptions.Compiled | RegexOptions.CultureInvariant);
     private static readonly Regex PythonAllAssignmentRegex = new(@"^\s*__all__\s*(?:\+?=)\s*(?<values>.+)$", RegexOptions.Compiled | RegexOptions.CultureInvariant);
     private static readonly Regex PythonAllAppendRegex = new(@"^\s*__all__\.append\(\s*(?<quote>['""])(?<name>[^'""]+)\k<quote>\s*\)", RegexOptions.Compiled | RegexOptions.CultureInvariant);
     private static readonly Regex PythonAllExtendRegex = new(@"^\s*__all__\.extend\(\s*(?<values>.*)$", RegexOptions.Compiled | RegexOptions.CultureInvariant);
@@ -194,6 +195,20 @@ private static string BuildPythonLogicalHeaderSignature(string[] lines, int star
         var entries = new List<PythonImportSymbolEntry>();
         var seenNames = new HashSet<string>(StringComparer.Ordinal);
 
+        foreach (Match match in PythonDynamicImportLiteralRegex.Matches(statement))
+        {
+            AddPythonImportEntry(
+                line,
+                absoluteStartColumn,
+                match.Groups["module"].Value,
+                entries,
+                seenNames,
+                ref absoluteStartColumn);
+        }
+
+        if (entries.Count > 0)
+            return entries;
+
         var directImportMatch = PythonDirectImportRegex.Match(statement);
         if (directImportMatch.Success)
         {

src/CodeIndex/Indexer/Symbols/SymbolExtractor.cs

@@ -813,6 +813,7 @@ private enum JavaScriptTypeScriptFunctionHeaderConsumeResult
             new("import",   new Regex(@"^\s*(?<name>\w+)\s*=\s*(?:(?:typing|typing_extensions)\.)?(?:TypeVar|ParamSpec|TypeVarTuple)\s*\(", RegexOptions.Compiled), BodyStyle.None),
             new("property", new Regex(@"^\s*(?<name>\w+)\s*:\s*(?:(?:typing|typing_extensions)\.)?Final(?:\[[^\]]+\])?\s*=", RegexOptions.Compiled), BodyStyle.None),
             new("import",   new Regex(@"^\s*(?:from\s+(?<name>(?:\.+[\w.]*|[\w.]+))\s+import\b|import\s+(?<name>[\w.]+))", RegexOptions.Compiled), BodyStyle.None),
+            new("import",   new Regex(@"^\s*(?:[_\p{L}]\w*\s*=\s*)?(?:importlib\.import_module|importlib\.util\.find_spec|__import__)\s*\(\s*['""](?<name>[^'""]+)['""]", RegexOptions.Compiled), BodyStyle.None),
         ],
         ["cobol"] =
         [

tests/CodeIndex.Tests/ReferenceExtractorTests.cs

@@ -1145,6 +1145,47 @@ def __init_subclass__(cls) -> None:
             && reference.ReferenceKind == "call");
     }
 
+    [Fact]
+    public void Extract_PythonDynamicImports_EmitImportAndImportlibReferences()
+    {
+        const string content = """
+            import importlib
+
+            def load(module_name):
+                importlib.import_module("plugins.alpha")
+                __import__('legacy.loader')
+                importlib.util.find_spec("optional.backend")
+                importlib.import_module(module_name)
+                note = "importlib.import_module('not.real')"
+                # importlib.import_module("commented.out")
+            """;
+
+        var symbols = SymbolExtractor.Extract(1, "python", content);
+        var references = ReferenceExtractor.Extract(1, "python", content, symbols);
+
+        Assert.Equal(3, references.Count(reference =>
+            reference.SymbolName == "importlib"
+            && reference.ReferenceKind == "call"
+            && reference.ContainerName == "load"));
+        Assert.Contains(references, reference =>
+            reference.SymbolName == "plugins.alpha"
+            && reference.ReferenceKind == "import"
+            && reference.ContainerName == "load");
+        Assert.Contains(references, reference =>
+            reference.SymbolName == "legacy.loader"
+            && reference.ReferenceKind == "import"
+            && reference.ContainerName == "load");
+        Assert.Contains(references, reference =>
+            reference.SymbolName == "optional.backend"
+            && reference.ReferenceKind == "import"
+            && reference.ContainerName == "load");
+        Assert.DoesNotContain(references, reference =>
+            reference.SymbolName == "module_name"
+            && reference.ReferenceKind == "import");
+        Assert.DoesNotContain(references, reference => reference.SymbolName == "not.real");
+        Assert.DoesNotContain(references, reference => reference.SymbolName == "commented.out");
+    }
+
     [Fact]
     public void Extract_PythonStringifiedAnnotations_CapturesNestedForwardReferences()
     {

tests/CodeIndex.Tests/SymbolExtractorTests.cs

@@ -855,6 +855,31 @@ from package.subpackage import helper
         Assert.Contains(imports, symbol => symbol.Name == "helper");
     }
 
+    [Fact]
+    public void Extract_Python_IndexesDynamicImportLiteralModules()
+    {
+        var content = """
+            importlib.import_module("plugins.alpha")
+            loaded = importlib.import_module("plugins.beta")
+            __import__('legacy.loader')
+            importlib.util.find_spec("optional.backend")
+            importlib.import_module(module_name)
+            note = "importlib.import_module('not.real')"
+            # importlib.import_module("commented.out")
+            """;
+
+        var symbols = SymbolExtractor.Extract(1, "python", content);
+        var imports = symbols.Where(symbol => symbol.Kind == "import").Select(symbol => symbol.Name).ToList();
+
+        Assert.Contains("plugins.alpha", imports);
+        Assert.Contains("plugins.beta", imports);
+        Assert.Contains("legacy.loader", imports);
+        Assert.Contains("optional.backend", imports);
+        Assert.DoesNotContain("module_name", imports);
+        Assert.DoesNotContain("not.real", imports);
+        Assert.DoesNotContain("commented.out", imports);
+    }
+
     [Fact]
     public void Extract_Python_IndexesAllExportsFromInitModules()
     {