diff --git a/src/lake/Lake/Build/Infos.lean b/src/lake/Lake/Build/Infos.lean index cc9e9d546b7e..c4a31d6cb7cb 100644 --- a/src/lake/Lake/Build/Infos.lean +++ b/src/lake/Lake/Build/Infos.lean @@ -111,8 +111,9 @@ builtin_facet precompileImports : Module => Array Module /-- Shared library for `--load-dynlib`. -/ builtin_facet dynlib : Module => Dynlib -/-- A Lean library's Lean modules. -/ -builtin_facet modules : LeanLib => Array Module +/-- A Lean library's Lean modules, topologically sorted +(`A` comes before `B` when `B` imports `A`). -/ +builtin_facet modules : LeanLib => OrdModuleSet /-- The package's array of dependencies. -/ builtin_facet deps : Package => Array Package diff --git a/src/lake/Lake/Build/Library.lean b/src/lake/Lake/Build/Library.lean index 4ad16e148ab3..7888020277d2 100644 --- a/src/lake/Lake/Build/Library.lean +++ b/src/lake/Lake/Build/Library.lean @@ -34,7 +34,7 @@ Collect the local modules of a library. That is, the modules from `getModuleArray` plus their local transitive imports. -/ partial def LeanLib.recCollectLocalModules - (self : LeanLib) : FetchM (Job (Array Module)) + (self : LeanLib) : FetchM (Job OrdModuleSet) := ensureJob do let mut col : ModuleCollection := {} for mod in (← self.getModuleArray) do @@ -43,7 +43,7 @@ partial def LeanLib.recCollectLocalModules -- This is not considered a fatal error because we want the modules -- built to provide better error categorization in the monitor. logError s!"{self.name}: some modules have bad imports" - return Job.pure col.mods + return Job.pure ⟨col.modSet, col.mods⟩ where go root col := do let mut col := col @@ -83,7 +83,7 @@ public def LeanLib.leanArtsFacetConfig : LibraryFacetConfig leanArtsFacet := "" withRegisterJob s!"{self.name}:static{suffix}" <| withCurrPackage self.pkg do let mods ← (← self.modules.fetch).await - let oJobs ← mods.flatMapM fun mod => + let oJobs ← mods.toArray.flatMapM fun mod => mod.nativeFacets shouldExport |>.mapM (·.fetch mod) let moreOJobs ← self.moreLinkObjs.mapM (·.fetchIn self.pkg) let libFile := if shouldExport then self.staticExportLibFile else self.staticLibFile @@ -126,7 +126,7 @@ public def LeanLib.staticExportFacetConfig : LibraryFacetConfig staticExportFace def LeanLib.recBuildShared (self : LeanLib) : FetchM (Job Dynlib) := do withRegisterJob s!"{self.name}:shared" <| withCurrPackage self.pkg do let mods ← (← self.modules.fetch).await - let objJobs ← mods.flatMapM fun mod => + let objJobs ← mods.toArray.flatMapM fun mod => mod.nativeFacets true |>.mapM (·.fetch mod) let objJobs ← self.moreLinkObjs.foldlM (init := objJobs) (·.push <$> ·.fetchIn self.pkg) diff --git a/src/lake/Lake/Build/Module.lean b/src/lake/Lake/Build/Module.lean index 1266fd0bf409..6b9ee05e7f7f 100644 --- a/src/lake/Lake/Build/Module.lean +++ b/src/lake/Lake/Build/Module.lean @@ -124,40 +124,58 @@ def Module.recComputePrecompileImports (mod : Module) : FetchM (Job (Array Modul public def Module.precompileImportsFacetConfig : ModuleFacetConfig precompileImportsFacet := mkFacetJobConfig recComputePrecompileImports (buildable := false) +/-- Whether this module is included in `lib.modules` for its parent library `lib`. +This is false of "orphaned" modules not imported by their parent library's root module. -/ +def Module.containedInLibModules (mod : Module) : FetchM Bool := do + return (← (← mod.lib.modules.fetch).await).contains mod + /-- -Computes the transitive dynamic libraries of a module's imports. -Modules from the same library are loaded individually, while modules -from other libraries are loaded as part of the whole library. --/ +Computes the dynamic libraries of `self`'s transitive imports `imps`. +Modules from the same library as `self` are loaded individually, +while modules from other libraries are +- loaded through their library's shared target if they are included in this target, +- otherwise also individually. -/ def Module.fetchImportLibs (self : Module) (imps : Array Module) (compileSelf : Bool) : FetchM (Array (Job Dynlib)) := do let (_, jobs) ← imps.foldlM (init := (({} : NameSet), #[])) fun (libs, jobs) imp => do - if libs.contains imp.lib.name then - return (libs, jobs) - else if compileSelf && self.lib.name = imp.lib.name then + if compileSelf && self.lib.name = imp.lib.name then let job ← imp.dynlib.fetch return (libs, jobs.push job) else if compileSelf || imp.shouldPrecompile then - let jobs ← jobs.push <$> imp.lib.shared.fetch - return (libs.insert imp.lib.name, jobs) + if ← imp.containedInLibModules then + if libs.contains imp.lib.name then + return (libs, jobs) + else + let job ← imp.lib.shared.fetch + return (libs.insert imp.lib.name, jobs.push job) + else + -- TODO: consider linting for orphaned modules + let job ← imp.dynlib.fetch + return (libs, jobs.push job) else return (libs, jobs) return jobs /-- -Fetches the library dynlibs of a list of non-local imports. -Modules are loaded as part of their whole library. +Fetches the dynamic libraries of a list `mods` of non-local imports. +Modules are loaded through their library's shared target if they are included in this target, +otherwise individually. -/ def fetchImportLibs (mods : Array Module) : FetchM (Job (Array Dynlib)) := do let (_, jobs) ← mods.foldlM (init := (({} : NameSet), #[])) fun (libs, jobs) imp => do - if libs.contains imp.lib.name then - return (libs, jobs) - else if imp.shouldPrecompile then - let jobs ← jobs.push <$> imp.lib.shared.fetch - return (libs.insert imp.lib.name, jobs) + if imp.shouldPrecompile then + if ← imp.containedInLibModules then + if libs.contains imp.lib.name then + return (libs, jobs) + else + let job ← imp.lib.shared.fetch + return (libs.insert imp.lib.name, jobs.push job) + else + let job ← imp.dynlib.fetch + return (libs, jobs.push job) else return (libs, jobs) return Job.collectArray jobs "import dynlibs" @@ -206,17 +224,19 @@ def computeModuleDeps -/ let impLibs ← mkLoadOrder impLibs let mut dynlibs := externLibs ++ dynlibs - let mut plugins := plugins for impLib in impLibs do - if impLib.plugin then - plugins := plugins.push impLib - else - dynlibs := dynlibs.push impLib + /- Load as dynlib (not plugin) because: + - imported modules will be initialized in `importModules` anyway; and + - since imports from a different `pkg` are loaded together through `pkg:shared`, + passing `pkg:shared` as a plugin would initialize too much, + namely all modules in `pkg` rather than just the ones we have imported. -/ + dynlibs := dynlibs.push impLib /- On MacOS, Lake must be loaded as a plugin for `import Lake` to work with precompiled modules. https://github.com/leanprover/lean4/issues/7388 -/ + let mut plugins := plugins if Platform.isOSX && !(plugins.isEmpty && dynlibs.isEmpty) then plugins := plugins.push (← getLakeInstall).sharedDynlib return {dynlibs, plugins} diff --git a/src/lake/Lake/Util/OrdHashSet.lean b/src/lake/Lake/Util/OrdHashSet.lean index 25b8e9a221c6..0ceae09c4d76 100644 --- a/src/lake/Lake/Util/OrdHashSet.lean +++ b/src/lake/Lake/Util/OrdHashSet.lean @@ -30,6 +30,9 @@ public instance : EmptyCollection (OrdHashSet α) := ⟨empty⟩ public def mkEmpty (size : Nat) : OrdHashSet α := ⟨∅, .mkEmpty size⟩ +public def contains (self : OrdHashSet α) (a : α) : Bool := + self.toHashSet.contains a + public def insert (self : OrdHashSet α) (a : α) : OrdHashSet α := if self.toHashSet.contains a then self diff --git a/tests/lake/tests/precompileLink/test.sh b/tests/lake/tests/precompileLink/test.sh index 5e52c36a1213..40f5ab84136f 100755 --- a/tests/lake/tests/precompileLink/test.sh +++ b/tests/lake/tests/precompileLink/test.sh @@ -3,6 +3,7 @@ source ../common.sh ./clean.sh +PKG=precompileArgs # Test that precompilation works with a Lake import # https://github.com/leanprover/lean4/issues/7388 @@ -13,8 +14,8 @@ test_run -v build LakeTest test_run -v exe orderTest # Test that transitively importing a precompiled module -# from a non-precompiled module works -test_not_out '"plugins":[]' -v setup-file ImportDownstream.lean +# from a non-precompiled module loads the correct shared object. +test_out "${PKG}_Foo.$SHARED_LIB_EXT" -v setup-file ImportDownstream.lean test_run -v build Downstream # Test that `moreLinkArgs` are included when linking precompiled modules @@ -23,7 +24,6 @@ test_maybe_err "-lBogus" build -KlinkArgs=-lBogus ./clean.sh # Test that dynlibs are part of the module trace unless `platformIndependent` is set -PKG=precompileArgs test_run build -R echo foo > .lake/build/lib/lean/${PKG}_Foo_Bar.$SHARED_LIB_EXT test_err "Building Foo" build --rehash diff --git a/tests/lake/tests/precompileModules-detached/Downstream/ImportDetached.lean b/tests/lake/tests/precompileModules-detached/Downstream/ImportDetached.lean new file mode 100644 index 000000000000..dbca95f8b630 --- /dev/null +++ b/tests/lake/tests/precompileModules-detached/Downstream/ImportDetached.lean @@ -0,0 +1 @@ +import Upstream.Detached diff --git a/tests/lake/tests/precompileModules-detached/Downstream/ImportImportDetached.lean b/tests/lake/tests/precompileModules-detached/Downstream/ImportImportDetached.lean new file mode 100644 index 000000000000..8a8ad5645558 --- /dev/null +++ b/tests/lake/tests/precompileModules-detached/Downstream/ImportImportDetached.lean @@ -0,0 +1,3 @@ +import Downstream.ImportDetached + +#eval detachedValue diff --git a/tests/lake/tests/precompileModules-detached/ImportDetached.lean b/tests/lake/tests/precompileModules-detached/ImportDetached.lean new file mode 100644 index 000000000000..42c3d052554f --- /dev/null +++ b/tests/lake/tests/precompileModules-detached/ImportDetached.lean @@ -0,0 +1,3 @@ +import Upstream.Detached + +#eval detachedValue diff --git a/tests/lake/tests/precompileModules-detached/Upstream.lean b/tests/lake/tests/precompileModules-detached/Upstream.lean new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/tests/lake/tests/precompileModules-detached/Upstream/Detached.lean b/tests/lake/tests/precompileModules-detached/Upstream/Detached.lean new file mode 100644 index 000000000000..7af489468e06 --- /dev/null +++ b/tests/lake/tests/precompileModules-detached/Upstream/Detached.lean @@ -0,0 +1,4 @@ +/-! A module in the library `Upstream` not imported by `Upstream.lean`. -/ + +initialize detachedValue : Nat ← + return 1234 diff --git a/tests/lake/tests/precompileModules-detached/clean.sh b/tests/lake/tests/precompileModules-detached/clean.sh new file mode 100755 index 000000000000..333b50aba242 --- /dev/null +++ b/tests/lake/tests/precompileModules-detached/clean.sh @@ -0,0 +1 @@ +rm -rf .lake lake-manifest.json produced.out diff --git a/tests/lake/tests/precompileModules-detached/lakefile.toml b/tests/lake/tests/precompileModules-detached/lakefile.toml new file mode 100644 index 000000000000..956e5ebdfb2b --- /dev/null +++ b/tests/lake/tests/precompileModules-detached/lakefile.toml @@ -0,0 +1,8 @@ +name = "precompileModules-detached" +precompileModules = true + +[[lean_lib]] +name = "Upstream" + +[[lean_lib]] +name = "Downstream" diff --git a/tests/lake/tests/precompileModules-detached/test.sh b/tests/lake/tests/precompileModules-detached/test.sh new file mode 100755 index 000000000000..f1bb6f3349e5 --- /dev/null +++ b/tests/lake/tests/precompileModules-detached/test.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash +source ../common.sh + +./clean.sh + +# Test that a precompiled module absent from its parent library's shared target +# is loaded as an individual dylib when elaborating a downstream module that imports it. +PKG="precompileModules_x2ddetached" +test_out "${PKG}_Upstream_Detached.$SHARED_LIB_EXT" -v setup-file ImportDetached.lean +test_out "${PKG}_Upstream_Detached.$SHARED_LIB_EXT" -v setup-file Downstream/ImportDetached.lean + +# This segfaults if the individual module dynlib isn't loaded. +test_run build -R Downstream.ImportImportDetached + +# cleanup +rm -f produced.out diff --git a/tests/lake/tests/precompileModules-pkgInit/Downstream.lean b/tests/lake/tests/precompileModules-pkgInit/Downstream.lean new file mode 100644 index 000000000000..9a064b7fb6a1 --- /dev/null +++ b/tests/lake/tests/precompileModules-pkgInit/Downstream.lean @@ -0,0 +1,2 @@ +import Downstream.A +import Downstream.B diff --git a/tests/lake/tests/precompileModules-pkgInit/Downstream/A.lean b/tests/lake/tests/precompileModules-pkgInit/Downstream/A.lean new file mode 100644 index 000000000000..dbca95f8b630 --- /dev/null +++ b/tests/lake/tests/precompileModules-pkgInit/Downstream/A.lean @@ -0,0 +1 @@ +import Upstream.Detached diff --git a/tests/lake/tests/precompileModules-pkgInit/Downstream/B.lean b/tests/lake/tests/precompileModules-pkgInit/Downstream/B.lean new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/tests/lake/tests/precompileModules-pkgInit/ExternalMod.lean b/tests/lake/tests/precompileModules-pkgInit/ExternalMod.lean new file mode 100644 index 000000000000..d40cc58b06e0 --- /dev/null +++ b/tests/lake/tests/precompileModules-pkgInit/ExternalMod.lean @@ -0,0 +1 @@ +import Downstream.B diff --git a/tests/lake/tests/precompileModules-pkgInit/Upstream.lean b/tests/lake/tests/precompileModules-pkgInit/Upstream.lean new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/tests/lake/tests/precompileModules-pkgInit/Upstream/Detached.lean b/tests/lake/tests/precompileModules-pkgInit/Upstream/Detached.lean new file mode 100644 index 000000000000..7af489468e06 --- /dev/null +++ b/tests/lake/tests/precompileModules-pkgInit/Upstream/Detached.lean @@ -0,0 +1,4 @@ +/-! A module in the library `Upstream` not imported by `Upstream.lean`. -/ + +initialize detachedValue : Nat ← + return 1234 diff --git a/tests/lake/tests/precompileModules-pkgInit/clean.sh b/tests/lake/tests/precompileModules-pkgInit/clean.sh new file mode 100755 index 000000000000..333b50aba242 --- /dev/null +++ b/tests/lake/tests/precompileModules-pkgInit/clean.sh @@ -0,0 +1 @@ +rm -rf .lake lake-manifest.json produced.out diff --git a/tests/lake/tests/precompileModules-pkgInit/lakefile.toml b/tests/lake/tests/precompileModules-pkgInit/lakefile.toml new file mode 100644 index 000000000000..3dfc10719719 --- /dev/null +++ b/tests/lake/tests/precompileModules-pkgInit/lakefile.toml @@ -0,0 +1,8 @@ +name = "precompileModules-pkgInit" +precompileModules = true + +[[lean_lib]] +name = "Upstream" + +[[lean_lib]] +name = "Downstream" diff --git a/tests/lake/tests/precompileModules-pkgInit/test.sh b/tests/lake/tests/precompileModules-pkgInit/test.sh new file mode 100755 index 000000000000..b856278161df --- /dev/null +++ b/tests/lake/tests/precompileModules-pkgInit/test.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash +source ../common.sh + +./clean.sh + +# Test that elaborating `ExternalMod` succeeds. +# Prior to https://github.com/leanprover/lean4/pull/14326, +# we used to load `Downstream` as a plugin rather than a dynlib. +# This would try to initialize `Downstream`, +# and then (transitively through `Downstream.A`) `Upstream.Detached`. +# But the native symbol to initialize `Upstream.Detached` is not in `Upstream:shared`, +# so we would get a crash. +test_run -v lean ExternalMod.lean + +# cleanup +rm -f produced.out