From 8588be4e3ddddb942718fa53735506366a2a479b Mon Sep 17 00:00:00 2001 From: Widthdom Date: Sun, 24 May 2026 20:49:35 +0900 Subject: [PATCH 1/2] Fix Python dynamic import indexing (#2056) --- changelog.d/unreleased/2056.fixed.md | 17 +++++ .../Languages/PythonReferenceExtractor.cs | 63 +++++++++++++++++++ .../Indexer/References/ReferenceExtractor.cs | 8 +++ .../Indexer/Symbols/SymbolExtractor.Python.cs | 15 +++++ .../Indexer/Symbols/SymbolExtractor.cs | 1 + .../ReferenceExtractorTests.cs | 37 +++++++++++ tests/CodeIndex.Tests/SymbolExtractorTests.cs | 19 ++++++ 7 files changed, 160 insertions(+) create mode 100644 changelog.d/unreleased/2056.fixed.md diff --git a/changelog.d/unreleased/2056.fixed.md b/changelog.d/unreleased/2056.fixed.md new file mode 100644 index 0000000000..88080e4f57 --- /dev/null +++ b/changelog.d/unreleased/2056.fixed.md @@ -0,0 +1,17 @@ +--- +category: fixed +issues: + - 2056 +affected: + - src/CodeIndex/Indexer/Symbols/SymbolExtractor.Python.cs + - src/CodeIndex/Indexer/References/Languages/PythonReferenceExtractor.cs + - src/CodeIndex/Indexer/References/ReferenceExtractor.cs +--- + +## English + +- **Python dynamic import literals are now indexed (#2056)** — `importlib.import_module(...)`, `importlib.util.find_spec(...)`, and `__import__(...)` string-literal module names now produce import symbols and references, while `importlib` calls remain visible in the reference graph. + +## 日本語 + +- **Python の dynamic import literal を index するようになりました (#2056)** — `importlib.import_module(...)`、`importlib.util.find_spec(...)`、`__import__(...)` の文字列 literal モジュール名が import symbol / reference として記録され、`importlib` 呼び出しも reference graph に残るようになりました。 diff --git a/src/CodeIndex/Indexer/References/Languages/PythonReferenceExtractor.cs b/src/CodeIndex/Indexer/References/Languages/PythonReferenceExtractor.cs index 4057b21ae3..d1594b303c 100644 --- a/src/CodeIndex/Indexer/References/Languages/PythonReferenceExtractor.cs +++ b/src/CodeIndex/Indexer/References/Languages/PythonReferenceExtractor.cs @@ -114,6 +114,12 @@ internal static class PythonReferenceExtractor private static readonly Regex ContextlibSuppressTypeRegex = new( @"\bcontextlib\.suppress\s*\(\s*(?(?:[_\p{L}]\w*\.)*[_\p{Lu}]\w*)", RegexOptions.Compiled); + private static readonly Regex ImportlibDynamicImportRegex = new( + @"\bimportlib(?:\.util)?\.(?:import_module|find_spec)\s*\(\s*(?:(?['""])(?[^'""]+)\k)?", + RegexOptions.Compiled); + private static readonly Regex BuiltinDynamicImportRegex = new( + @"(?['""])(?[^'""]+)\k", + RegexOptions.Compiled); private static string NormalizePythonAnnotationExpression(string expression) { @@ -1024,4 +1030,61 @@ public static void EmitContextlibSuppressReferences( "python"); } } + + public static void EmitDynamicImportReferences( + string preparedLine, + List references, + HashSet seen, + long fileId, + string context, + int lineNumber, + SymbolRecord? container) + { + foreach (Match match in ImportlibDynamicImportRegex.Matches(preparedLine)) + { + ReferenceExtractor.AddReference( + references, + seen, + fileId, + "importlib", + match.Index, + "call", + context, + lineNumber, + container, + "python"); + + var moduleGroup = match.Groups["module"]; + if (moduleGroup.Success && moduleGroup.Value.Length > 0) + { + ReferenceExtractor.AddReference( + references, + seen, + fileId, + moduleGroup.Value, + moduleGroup.Index, + "import", + context, + lineNumber, + container, + "python"); + } + } + + foreach (Match match in BuiltinDynamicImportRegex.Matches(preparedLine)) + { + var moduleGroup = match.Groups["module"]; + ReferenceExtractor.AddReference( + references, + seen, + fileId, + moduleGroup.Value, + moduleGroup.Index, + "import", + context, + lineNumber, + container, + "python"); + } + } } diff --git a/src/CodeIndex/Indexer/References/ReferenceExtractor.cs b/src/CodeIndex/Indexer/References/ReferenceExtractor.cs index 5ffd082207..d07f12b0c3 100644 --- a/src/CodeIndex/Indexer/References/ReferenceExtractor.cs +++ b/src/CodeIndex/Indexer/References/ReferenceExtractor.cs @@ -3155,6 +3155,14 @@ void AddGradleDslReference(string name, int callIndex) lineNumber, container, name => IsIgnoredCallName(language, name)); + PythonReferenceExtractor.EmitDynamicImportReferences( + context, + references, + seen, + fileId, + context, + lineNumber, + container); if (pythonHeaderMap.HasValue) RemapPythonLogicalHeaderReferences(references, pythonReferenceStart, pythonHeaderMap.Value, lines); } diff --git a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.Python.cs b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.Python.cs index 291bf516b7..68eef851be 100644 --- a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.Python.cs +++ b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.Python.cs @@ -11,6 +11,7 @@ public static partial class SymbolExtractor private readonly record struct PythonExportSymbolEntry(string Name, int LineIndex, int StartColumn); private static readonly Regex PythonDirectImportRegex = new(@"^import\s+(?.+)$", RegexOptions.Compiled | RegexOptions.CultureInvariant); private static readonly Regex PythonFromImportRegex = new(@"^from\s+(?(?:\.+[\w.]*|[\w.]+))\s+import\s+(?.+)$", RegexOptions.Compiled | RegexOptions.CultureInvariant); + private static readonly Regex PythonDynamicImportLiteralRegex = new(@"\b(?:importlib\.import_module|importlib\.util\.find_spec|__import__)\s*\(\s*(?['""])(?[^'""]+)\k", RegexOptions.Compiled | RegexOptions.CultureInvariant); private static readonly Regex PythonAllAssignmentRegex = new(@"^\s*__all__\s*(?:\+?=)\s*(?.+)$", RegexOptions.Compiled | RegexOptions.CultureInvariant); private static readonly Regex PythonAllAppendRegex = new(@"^\s*__all__\.append\(\s*(?['""])(?[^'""]+)\k\s*\)", RegexOptions.Compiled | RegexOptions.CultureInvariant); private static readonly Regex PythonAllExtendRegex = new(@"^\s*__all__\.extend\(\s*(?.*)$", RegexOptions.Compiled | RegexOptions.CultureInvariant); @@ -194,6 +195,20 @@ private static string BuildPythonLogicalHeaderSignature(string[] lines, int star var entries = new List(); var seenNames = new HashSet(StringComparer.Ordinal); + foreach (Match match in PythonDynamicImportLiteralRegex.Matches(statement)) + { + AddPythonImportEntry( + line, + absoluteStartColumn, + match.Groups["module"].Value, + entries, + seenNames, + ref absoluteStartColumn); + } + + if (entries.Count > 0) + return entries; + var directImportMatch = PythonDirectImportRegex.Match(statement); if (directImportMatch.Success) { diff --git a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.cs b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.cs index f8d722d8e8..fa3bcf1f15 100644 --- a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.cs +++ b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.cs @@ -813,6 +813,7 @@ private enum JavaScriptTypeScriptFunctionHeaderConsumeResult new("import", new Regex(@"^\s*(?\w+)\s*=\s*(?:(?:typing|typing_extensions)\.)?(?:TypeVar|ParamSpec|TypeVarTuple)\s*\(", RegexOptions.Compiled), BodyStyle.None), new("property", new Regex(@"^\s*(?\w+)\s*:\s*(?:(?:typing|typing_extensions)\.)?Final(?:\[[^\]]+\])?\s*=", RegexOptions.Compiled), BodyStyle.None), new("import", new Regex(@"^\s*(?:from\s+(?(?:\.+[\w.]*|[\w.]+))\s+import\b|import\s+(?[\w.]+))", RegexOptions.Compiled), BodyStyle.None), + new("import", new Regex(@"\b(?:importlib\.import_module|importlib\.util\.find_spec|__import__)\s*\(\s*['""](?[^'""]+)['""]", RegexOptions.Compiled), BodyStyle.None), ], ["cobol"] = [ diff --git a/tests/CodeIndex.Tests/ReferenceExtractorTests.cs b/tests/CodeIndex.Tests/ReferenceExtractorTests.cs index 9614904ef6..a23eb97eba 100644 --- a/tests/CodeIndex.Tests/ReferenceExtractorTests.cs +++ b/tests/CodeIndex.Tests/ReferenceExtractorTests.cs @@ -1145,6 +1145,43 @@ def __init_subclass__(cls) -> None: && reference.ReferenceKind == "call"); } + [Fact] + public void Extract_PythonDynamicImports_EmitImportAndImportlibReferences() + { + const string content = """ + import importlib + + def load(module_name): + importlib.import_module("plugins.alpha") + __import__('legacy.loader') + importlib.util.find_spec("optional.backend") + importlib.import_module(module_name) + """; + + var symbols = SymbolExtractor.Extract(1, "python", content); + var references = ReferenceExtractor.Extract(1, "python", content, symbols); + + Assert.Equal(3, references.Count(reference => + reference.SymbolName == "importlib" + && reference.ReferenceKind == "call" + && reference.ContainerName == "load")); + Assert.Contains(references, reference => + reference.SymbolName == "plugins.alpha" + && reference.ReferenceKind == "import" + && reference.ContainerName == "load"); + Assert.Contains(references, reference => + reference.SymbolName == "legacy.loader" + && reference.ReferenceKind == "import" + && reference.ContainerName == "load"); + Assert.Contains(references, reference => + reference.SymbolName == "optional.backend" + && reference.ReferenceKind == "import" + && reference.ContainerName == "load"); + Assert.DoesNotContain(references, reference => + reference.SymbolName == "module_name" + && reference.ReferenceKind == "import"); + } + [Fact] public void Extract_PythonStringifiedAnnotations_CapturesNestedForwardReferences() { diff --git a/tests/CodeIndex.Tests/SymbolExtractorTests.cs b/tests/CodeIndex.Tests/SymbolExtractorTests.cs index de55449729..bb94fec557 100644 --- a/tests/CodeIndex.Tests/SymbolExtractorTests.cs +++ b/tests/CodeIndex.Tests/SymbolExtractorTests.cs @@ -855,6 +855,25 @@ from package.subpackage import helper Assert.Contains(imports, symbol => symbol.Name == "helper"); } + [Fact] + public void Extract_Python_IndexesDynamicImportLiteralModules() + { + var content = """ + importlib.import_module("plugins.alpha") + __import__('legacy.loader') + importlib.util.find_spec("optional.backend") + importlib.import_module(module_name) + """; + + var symbols = SymbolExtractor.Extract(1, "python", content); + var imports = symbols.Where(symbol => symbol.Kind == "import").Select(symbol => symbol.Name).ToList(); + + Assert.Contains("plugins.alpha", imports); + Assert.Contains("legacy.loader", imports); + Assert.Contains("optional.backend", imports); + Assert.DoesNotContain("module_name", imports); + } + [Fact] public void Extract_Python_IndexesAllExportsFromInitModules() { From 85ea5ded6c1a018909e79d95716af7e0e65b8f57 Mon Sep 17 00:00:00 2001 From: Widthdom Date: Sun, 24 May 2026 20:55:51 +0900 Subject: [PATCH 2/2] Avoid Python dynamic import false positives (#2056) --- .../Languages/PythonReferenceExtractor.cs | 21 ++++++++++++++++--- .../Indexer/References/ReferenceExtractor.cs | 3 ++- .../Indexer/Symbols/SymbolExtractor.cs | 2 +- .../ReferenceExtractorTests.cs | 4 ++++ tests/CodeIndex.Tests/SymbolExtractorTests.cs | 6 ++++++ 5 files changed, 31 insertions(+), 5 deletions(-) diff --git a/src/CodeIndex/Indexer/References/Languages/PythonReferenceExtractor.cs b/src/CodeIndex/Indexer/References/Languages/PythonReferenceExtractor.cs index d1594b303c..36c5a33e84 100644 --- a/src/CodeIndex/Indexer/References/Languages/PythonReferenceExtractor.cs +++ b/src/CodeIndex/Indexer/References/Languages/PythonReferenceExtractor.cs @@ -115,9 +115,15 @@ internal static class PythonReferenceExtractor @"\bcontextlib\.suppress\s*\(\s*(?(?:[_\p{L}]\w*\.)*[_\p{Lu}]\w*)", RegexOptions.Compiled); private static readonly Regex ImportlibDynamicImportRegex = new( - @"\bimportlib(?:\.util)?\.(?:import_module|find_spec)\s*\(\s*(?:(?['""])(?[^'""]+)\k)?", + @"\bimportlib(?:\.util)?\.(?:import_module|find_spec)\s*\(", + RegexOptions.Compiled); + private static readonly Regex ImportlibDynamicImportLiteralRegex = new( + @"\bimportlib(?:\.util)?\.(?:import_module|find_spec)\s*\(\s*(?['""])(?[^'""]+)\k", RegexOptions.Compiled); private static readonly Regex BuiltinDynamicImportRegex = new( + @"(?['""])(?[^'""]+)\k", RegexOptions.Compiled); @@ -1033,6 +1039,7 @@ public static void EmitContextlibSuppressReferences( public static void EmitDynamicImportReferences( string preparedLine, + string originalLine, List references, HashSet seen, long fileId, @@ -1054,7 +1061,11 @@ public static void EmitDynamicImportReferences( container, "python"); - var moduleGroup = match.Groups["module"]; + var literalMatch = ImportlibDynamicImportLiteralRegex.Match(originalLine, match.Index); + if (!literalMatch.Success || literalMatch.Index != match.Index) + continue; + + var moduleGroup = literalMatch.Groups["module"]; if (moduleGroup.Success && moduleGroup.Value.Length > 0) { ReferenceExtractor.AddReference( @@ -1073,7 +1084,11 @@ public static void EmitDynamicImportReferences( foreach (Match match in BuiltinDynamicImportRegex.Matches(preparedLine)) { - var moduleGroup = match.Groups["module"]; + var literalMatch = BuiltinDynamicImportLiteralRegex.Match(originalLine, match.Index); + if (!literalMatch.Success || literalMatch.Index != match.Index) + continue; + + var moduleGroup = literalMatch.Groups["module"]; ReferenceExtractor.AddReference( references, seen, diff --git a/src/CodeIndex/Indexer/References/ReferenceExtractor.cs b/src/CodeIndex/Indexer/References/ReferenceExtractor.cs index d07f12b0c3..13a12a8aee 100644 --- a/src/CodeIndex/Indexer/References/ReferenceExtractor.cs +++ b/src/CodeIndex/Indexer/References/ReferenceExtractor.cs @@ -3156,7 +3156,8 @@ void AddGradleDslReference(string name, int callIndex) container, name => IsIgnoredCallName(language, name)); PythonReferenceExtractor.EmitDynamicImportReferences( - context, + preparedLine, + originalLine, references, seen, fileId, diff --git a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.cs b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.cs index fa3bcf1f15..8f2ff12b88 100644 --- a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.cs +++ b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.cs @@ -813,7 +813,7 @@ private enum JavaScriptTypeScriptFunctionHeaderConsumeResult new("import", new Regex(@"^\s*(?\w+)\s*=\s*(?:(?:typing|typing_extensions)\.)?(?:TypeVar|ParamSpec|TypeVarTuple)\s*\(", RegexOptions.Compiled), BodyStyle.None), new("property", new Regex(@"^\s*(?\w+)\s*:\s*(?:(?:typing|typing_extensions)\.)?Final(?:\[[^\]]+\])?\s*=", RegexOptions.Compiled), BodyStyle.None), new("import", new Regex(@"^\s*(?:from\s+(?(?:\.+[\w.]*|[\w.]+))\s+import\b|import\s+(?[\w.]+))", RegexOptions.Compiled), BodyStyle.None), - new("import", new Regex(@"\b(?:importlib\.import_module|importlib\.util\.find_spec|__import__)\s*\(\s*['""](?[^'""]+)['""]", RegexOptions.Compiled), BodyStyle.None), + new("import", new Regex(@"^\s*(?:[_\p{L}]\w*\s*=\s*)?(?:importlib\.import_module|importlib\.util\.find_spec|__import__)\s*\(\s*['""](?[^'""]+)['""]", RegexOptions.Compiled), BodyStyle.None), ], ["cobol"] = [ diff --git a/tests/CodeIndex.Tests/ReferenceExtractorTests.cs b/tests/CodeIndex.Tests/ReferenceExtractorTests.cs index a23eb97eba..513a2e0654 100644 --- a/tests/CodeIndex.Tests/ReferenceExtractorTests.cs +++ b/tests/CodeIndex.Tests/ReferenceExtractorTests.cs @@ -1156,6 +1156,8 @@ def load(module_name): __import__('legacy.loader') importlib.util.find_spec("optional.backend") importlib.import_module(module_name) + note = "importlib.import_module('not.real')" + # importlib.import_module("commented.out") """; var symbols = SymbolExtractor.Extract(1, "python", content); @@ -1180,6 +1182,8 @@ def load(module_name): Assert.DoesNotContain(references, reference => reference.SymbolName == "module_name" && reference.ReferenceKind == "import"); + Assert.DoesNotContain(references, reference => reference.SymbolName == "not.real"); + Assert.DoesNotContain(references, reference => reference.SymbolName == "commented.out"); } [Fact] diff --git a/tests/CodeIndex.Tests/SymbolExtractorTests.cs b/tests/CodeIndex.Tests/SymbolExtractorTests.cs index bb94fec557..adc01029e4 100644 --- a/tests/CodeIndex.Tests/SymbolExtractorTests.cs +++ b/tests/CodeIndex.Tests/SymbolExtractorTests.cs @@ -860,18 +860,24 @@ public void Extract_Python_IndexesDynamicImportLiteralModules() { var content = """ importlib.import_module("plugins.alpha") + loaded = importlib.import_module("plugins.beta") __import__('legacy.loader') importlib.util.find_spec("optional.backend") importlib.import_module(module_name) + note = "importlib.import_module('not.real')" + # importlib.import_module("commented.out") """; var symbols = SymbolExtractor.Extract(1, "python", content); var imports = symbols.Where(symbol => symbol.Kind == "import").Select(symbol => symbol.Name).ToList(); Assert.Contains("plugins.alpha", imports); + Assert.Contains("plugins.beta", imports); Assert.Contains("legacy.loader", imports); Assert.Contains("optional.backend", imports); Assert.DoesNotContain("module_name", imports); + Assert.DoesNotContain("not.real", imports); + Assert.DoesNotContain("commented.out", imports); } [Fact]