From 8bc1ea90432155e96b31f79481605108f53664fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patrick=20H=C3=A4cker?= Date: Tue, 7 Jul 2026 06:51:49 +0200 Subject: [PATCH 1/3] feat: index only the packages the workspace imports The environment indexer (`get_store`) loaded every top-level dependency of a project's environment on a cold cache, even packages the workspace never `using`/`import`s. On a large shared environment this wastes gigabytes of RAM and minutes of CPU, and runs unrelated packages' `__init__` side effects (conda installs, GPU driver init, ...). Extract, from the workspace's own source, the set of top-level packages each project actually imports, thread it through the environment process to the out-of-process indexer, and load only those. Correctness is preserved: - In-workspace `develop`ed packages are always indexed, so a package's own symbols are never dropped. - Transitive dependencies of an imported package are still cached: importing it loads them and the whole-process symbol walk records them, so re-exports keep resolving. - The imported-package set is part of the `WatchEnvironmentKey` identity, so changing which packages are imported re-triggers indexing through the normal reconcile diff. - The fast-lane missing-cache check is scoped to the dependency closure of the imported and developed packages, so a never-loaded dependency is not reported "missing" forever (which would respawn an indexer every session). Measured on a 145-dependency environment (worst case: 24 developed packages): peak RSS 4.4 GB -> 1.3 GB, wall time 22.5 min -> 4 min. --- .../src/JuliaDynamicAnalysisProcess.jl | 2 +- .../src/symbolserver.jl | 17 +++- ...julia_dynamic_analysis_process_protocol.jl | 1 + src/dynamic_feature/dynamic_feature.jl | 80 +++++++++++++--- src/dynamic_feature/dynamic_messages.jl | 11 ++- src/layer_environment.jl | 93 ++++++++++++++++++- src/types.jl | 6 +- test/test_symbolserver.jl | 76 +++++++++++++++ 8 files changed, 266 insertions(+), 20 deletions(-) diff --git a/juliadynamicanalysisprocess/JuliaDynamicAnalysisProcess/src/JuliaDynamicAnalysisProcess.jl b/juliadynamicanalysisprocess/JuliaDynamicAnalysisProcess/src/JuliaDynamicAnalysisProcess.jl index d0bd311d..1b92bc2d 100644 --- a/juliadynamicanalysisprocess/JuliaDynamicAnalysisProcess/src/JuliaDynamicAnalysisProcess.jl +++ b/juliadynamicanalysisprocess/JuliaDynamicAnalysisProcess/src/JuliaDynamicAnalysisProcess.jl @@ -16,7 +16,7 @@ function index_project_request(params::JuliaDynamicAnalysisProtocol.IndexProject TestEnv.activate(params.package); end - SymbolServer.get_store(params.storePath, nothing) + SymbolServer.get_store(params.storePath, nothing; used_packages=params.usedPackages) return dirname(Base.active_project()) end diff --git a/juliadynamicanalysisprocess/JuliaDynamicAnalysisProcess/src/symbolserver.jl b/juliadynamicanalysisprocess/JuliaDynamicAnalysisProcess/src/symbolserver.jl index 97ac849d..396cd3d3 100644 --- a/juliadynamicanalysisprocess/JuliaDynamicAnalysisProcess/src/symbolserver.jl +++ b/juliadynamicanalysisprocess/JuliaDynamicAnalysisProcess/src/symbolserver.jl @@ -25,7 +25,7 @@ else is_stdlib(uuid::UUID) = uuid in keys(ctx.stdlibs) end -function get_store(store_path::String, progress_callback) +function get_store(store_path::String, progress_callback; used_packages=nothing) loading_bay = Module(:LoadingBay) ctx = try @@ -34,7 +34,7 @@ function get_store(store_path::String, progress_callback) @info "Package environment can't be read." exit() end - + server = Server(store_path, ctx, Dict{UUID,Package}()) written_caches = String[] # List of caches that have already been written @@ -52,6 +52,19 @@ function get_store(store_path::String, progress_callback) @info "$pk_name not in manifest, skipping." continue end + + # Only load registry packages the workspace actually `using`/`import`s. + # `used_packages === nothing` means "index everything" (the default used + # by the cloud indexer and standalone-project indexing). In-workspace + # `develop`ed packages are always indexed, so a package's own symbols and + # those of workspace deps are never dropped. Dependencies of an imported + # package are still cached: importing it loads them, and the whole-process + # `getenvtree`/`symbols` walk below picks them up. + if !isnothing(used_packages) && pk_name ∉ used_packages && !is_package_deved(manifest(ctx), uuid) + @debug "$pk_name ($uuid) not imported by the workspace, skipping load." + continue + end + pe = frommanifest(manifest(ctx), uuid) cache_path = joinpath(server.storedir, SymbolServer.get_cache_path(manifest(ctx), uuid)...) diff --git a/shared/julia_dynamic_analysis_process_protocol.jl b/shared/julia_dynamic_analysis_process_protocol.jl index f0c778b9..fd388df7 100644 --- a/shared/julia_dynamic_analysis_process_protocol.jl +++ b/shared/julia_dynamic_analysis_process_protocol.jl @@ -9,6 +9,7 @@ using ..JSONRPC: @dict_readable, RequestType, NotificationType, Outbound projectPath::String package::Union{Nothing,String} storePath::String + usedPackages::Union{Nothing,Vector{String}} end @dict_readable struct CreateStandaloneProjectParams <: JSONRPC.Outbound diff --git a/src/dynamic_feature/dynamic_feature.jl b/src/dynamic_feature/dynamic_feature.jl index 682a8b05..d2dfbfa5 100644 --- a/src/dynamic_feature/dynamic_feature.jl +++ b/src/dynamic_feature/dynamic_feature.jl @@ -51,13 +51,18 @@ mutable struct DynamicJuliaProcess end function index_project(djp::DynamicJuliaProcess, store_path::String) + # For a watch-environment process the key carries the workspace's imported + # package set, so the child only loads those (plus deved packages). For a + # test-environment process there is no such set, so `nothing` = index all. + used_packages = djp.key isa WatchEnvironmentKey ? djp.key.imported_packages : nothing JSONRPC.send( djp.endpoint, JuliaDynamicAnalysisProtocol.index_project_request_type, JuliaDynamicAnalysisProtocol.IndexProjectParams( djp.project_path, djp.package, - store_path + store_path, + used_packages ) ) end @@ -357,14 +362,49 @@ end const MissingPackage = @NamedTuple{name::String, uuid::UUID, version::String, git_tree_sha1::Union{String,Nothing}} +# Names of every manifest package reachable from `roots` by following `deps` +# edges (roots included). `manifest_deps` is keyed by package name, and each +# entry's dependencies are listed either as an array of names or a name⇒uuid +# table. Used to restrict the missing-cache check to the packages the child will +# actually load (the closure of the workspace's imported and deved packages). +function _packages_reachable_from(manifest_deps, roots) + reachable = Set{String}() + stack = collect(String, roots) + while !isempty(stack) + name = pop!(stack) + name in reachable && continue + push!(reachable, name) + v_entry = get(manifest_deps, name, nothing) + (v_entry isa Vector && length(v_entry) == 1 && v_entry[1] isa Dict) || continue + d = get(v_entry[1], "deps", nothing) + depnames = if d isa Vector + String[x for x in d if x isa AbstractString] + elseif d isa Dict + collect(keys(d)) + else + String[] + end + for dn in depnames + dn in reachable || push!(stack, dn) + end + end + return reachable +end + """ - _get_missing_packages(project_path, store_path) -> Vector{MissingPackage} + _get_missing_packages(project_path, store_path, imported_packages=nothing) -> Vector{MissingPackage} Parse the Manifest.toml at `project_path` and return a list of regular and stdlib packages whose .jstore cache files do not yet exist on disk. Deved packages are skipped entirely (they have no git_tree_sha1 and are handled by StaticLint). + +When `imported_packages` is given, only packages in the dependency closure of the +workspace's imported and deved packages are considered — i.e. exactly the set the +child indexer will load. This prevents a never-loaded dependency from being +reported "missing" forever, which would otherwise spawn an indexing child every +session that could never satisfy it. `nothing` considers the whole manifest. """ -function _get_missing_packages(project_path::String, store_path::String) +function _get_missing_packages(project_path::String, store_path::String, imported_packages=nothing) manifest_path = joinpath(project_path, "Manifest.toml") isfile(manifest_path) || return MissingPackage[] @@ -386,6 +426,20 @@ function _get_missing_packages(project_path::String, store_path::String) return MissingPackage[] end + # Restrict to the packages the child will actually load: the dependency + # closure of the workspace's imported packages plus all deved packages + # (which the child always loads). `nothing` means "consider everything". + allowed = if isnothing(imported_packages) + nothing + else + deved_names = Set{String}( + k for (k, v) in pairs(manifest_deps) + if v isa Vector && length(v) == 1 && v[1] isa Dict && haskey(v[1], "path") + ) + roots = union(Set{String}(imported_packages), deved_names) + _packages_reachable_from(manifest_deps, roots) + end + missing = MissingPackage[] for (k_entry, v_entry) in pairs(manifest_deps) @@ -394,6 +448,9 @@ function _get_missing_packages(project_path::String, store_path::String) v_entry[1] isa Dict || continue entry = v_entry[1] + # Skip packages the workspace never loads. + allowed === nothing || k_entry in allowed || continue + # Skip deved packages (have "path" key) haskey(entry, "path") && continue @@ -623,7 +680,7 @@ end # ─── Work messages ────────────────────────────────────────────────────────── function handle!(df::DynamicFeature, msg::WatchEnvironmentMsg) - key = WatchEnvironmentKey(msg.project_path, msg.content_hash) + key = WatchEnvironmentKey(msg.project_path, msg.content_hash, msg.imported_packages) push!(df.inflight, key) if key in df.failed_projects @@ -638,8 +695,9 @@ function handle!(df::DynamicFeature, msg::WatchEnvironmentMsg) # `EnvironmentPrepDoneMsg`, so all state mutation stays on the reactor. project_path = msg.project_path content_hash = msg.content_hash + imported_packages = msg.imported_packages Threads.@async try - missing_pkgs = _get_missing_packages(project_path, df.store_path) + missing_pkgs = _get_missing_packages(project_path, df.store_path, imported_packages) if !isempty(missing_pkgs) && df.download_enabled @info "Downloading missing package caches" count=length(missing_pkgs) @@ -647,7 +705,7 @@ function handle!(df::DynamicFeature, msg::WatchEnvironmentMsg) missing_pkgs = _download_missing_caches(missing_pkgs, df.store_path, df.upstream_url; df=df) end - put!(df.in_channel, EnvironmentPrepDoneMsg(project_path, content_hash, !isempty(missing_pkgs))) + put!(df.in_channel, EnvironmentPrepDoneMsg(project_path, content_hash, imported_packages, !isempty(missing_pkgs))) catch err @error "Environment prep failed" project_path=project_path exception=(err, catch_backtrace()) put!(df.in_channel, ProcessIndexFailedMsg(key, err)) @@ -657,13 +715,13 @@ function handle!(df::DynamicFeature, msg::WatchEnvironmentMsg) end function handle!(df::DynamicFeature, msg::EnvironmentPrepDoneMsg) - key = WatchEnvironmentKey(msg.project_path, msg.content_hash) + key = WatchEnvironmentKey(msg.project_path, msg.content_hash, msg.imported_packages) df.progress_state.current_sub_progress = 0.5 if !msg.still_missing @info "All package caches available, skipping DJP" project_path=msg.project_path _report_progress(df, "All caches available for $(basename(msg.project_path))") - put!(df.out_channel, EnvironmentReadyResult(msg.project_path, msg.content_hash)) + put!(df.out_channel, EnvironmentReadyResult(key)) push!(df.done, key) _complete_work_item!(df, key) elseif df.djp_mode != DynamicOff @@ -674,7 +732,7 @@ function handle!(df::DynamicFeature, msg::EnvironmentPrepDoneMsg) _launch_process!(df, djp) else @info "Some packages missing but DJP disabled, proceeding with best-effort" project_path=msg.project_path - put!(df.out_channel, EnvironmentReadyResult(msg.project_path, msg.content_hash)) + put!(df.out_channel, EnvironmentReadyResult(key)) push!(df.done, key) _complete_work_item!(df, key) end @@ -766,7 +824,7 @@ function handle!(df::DynamicFeature, msg::ProcessIndexedMsg) end if key isa WatchEnvironmentKey - put!(df.out_channel, EnvironmentReadyResult(key.project_path, key.content_hash)) + put!(df.out_channel, EnvironmentReadyResult(key)) elseif key isa WatchTestEnvironmentKey test_project_uri = filepath2uri(msg.result_dir) put!(df.out_channel, TestEnvironmentReadyResult(filepath2uri(key.project_path), key.package_name, test_project_uri, key.content_hash)) @@ -899,7 +957,7 @@ function handle!(df::DynamicFeature, msg::ReconcileMsg) _report_progress(df, "Preparing to index...") if key isa WatchEnvironmentKey - handle!(df, WatchEnvironmentMsg(key.project_path, key.content_hash)) + handle!(df, WatchEnvironmentMsg(key.project_path, key.content_hash, key.imported_packages)) elseif key isa WatchTestEnvironmentKey handle!(df, WatchTestEnvironmentMsg(key.project_path, key.package_name, key.content_hash)) elseif key isa CreateStandaloneProjectKey diff --git a/src/dynamic_feature/dynamic_messages.jl b/src/dynamic_feature/dynamic_messages.jl index 73cade79..06e3e1b9 100644 --- a/src/dynamic_feature/dynamic_messages.jl +++ b/src/dynamic_feature/dynamic_messages.jl @@ -14,9 +14,15 @@ # `@auto_hash_equals` gives them value-based `==`/`hash` (the default for plain # structs compares `String` fields by identity, which is unsafe for use as # `Set`/`Dict` keys). +# `imported_packages` is the sorted set of top-level package names the +# workspace's own source `using`/`import`s within this project. It is part of +# the key's identity so that a change to which packages are used (re)triggers +# indexing via the normal reconcile diff, and it is forwarded to the child so it +# only loads packages the workspace actually references. @auto_hash_equals struct WatchEnvironmentKey project_path::String content_hash::UInt64 + imported_packages::Vector{String} end @auto_hash_equals struct WatchTestEnvironmentKey @@ -51,6 +57,7 @@ abstract type DynamicReactorMessage end struct WatchEnvironmentMsg <: DynamicReactorMessage project_path::String content_hash::UInt64 + imported_packages::Vector{String} end """Request to index/watch the test environment of a project + package.""" @@ -92,6 +99,7 @@ once the (potentially slow) missing-package check + cloud download finished. struct EnvironmentPrepDoneMsg <: DynamicReactorMessage project_path::String content_hash::UInt64 + imported_packages::Vector{String} still_missing::Bool end @@ -140,8 +148,7 @@ end """The environment for a project has been fully processed.""" struct EnvironmentReadyResult <: DynamicResultMessage - project_path::String - content_hash::UInt64 + key::WatchEnvironmentKey end """The test environment for a project + package is ready.""" diff --git a/src/layer_environment.jl b/src/layer_environment.jl index 856db90d..ccbc35e2 100644 --- a/src/layer_environment.jl +++ b/src/layer_environment.jl @@ -19,7 +19,7 @@ end Whether the environment for `project_uri` (at `content_hash`) has been indexed. """ Salsa.@derived function derived_project_environment_ready(rt, project_uri, content_hash::UInt64) - key = WatchEnvironmentKey(uri2filepath(project_uri), content_hash) + key = WatchEnvironmentKey(uri2filepath(project_uri), content_hash, derived_project_imported_packages(rt, project_uri)) return key in input_ready_project_environments(rt) end @@ -294,6 +294,96 @@ function _file_has_testitems(rt, uri) end end +# ─── Workspace-imported package extraction ─────────────────────────────────── +# +# The dynamic indexer only needs to load the packages the workspace's own source +# actually `using`/`import`s (plus in-workspace deved packages, which the child +# always indexes). These helpers pull the top-level package names out of the +# parsed source so the set can be attached to the environment DJP key and +# forwarded to the child. + +# The package/module named by a single import path (`Foo` in `import Foo`, +# `Foo.Bar`, or `Foo as F`). Relative imports (`using .Local`) reference an +# in-package module rather than an environment package, so they yield `nothing`. +function _import_path_package(path) + path isa CSTParser.EXPR || return nothing + if CSTParser.headof(path) === :as + return isempty(path.args) ? nothing : _import_path_package(path.args[1]) + end + CSTParser.isidentifier(path) && return CSTParser.valof(path) + (isnothing(path.args) || isempty(path.args)) && return nothing + first_arg = path.args[1] + # Leading dot ⇒ relative import. + if CSTParser.isoperator(first_arg) && CSTParser.valof(first_arg) == "." + return nothing + end + CSTParser.isidentifier(first_arg) ? CSTParser.valof(first_arg) : nothing +end + +# Walk an EXPR tree collecting the top-level package names from every `using` +# and `import` statement (including those nested inside modules or blocks). +function _collect_imported_packages!(names::Set{String}, x) + x isa CSTParser.EXPR || return names + h = CSTParser.headof(x) + if h in (:using, :import) + if !isempty(x.args) + a1 = x.args[1] + if a1 isa CSTParser.EXPR && CSTParser.isoperator(CSTParser.headof(a1)) && CSTParser.valof(CSTParser.headof(a1)) == ":" + # `using A: x, y` — only `A` names a package. + pkg = isempty(a1.args) ? nothing : _import_path_package(a1.args[1]) + !isnothing(pkg) && push!(names, pkg) + else + for path in x.args + pkg = _import_path_package(path) + !isnothing(pkg) && push!(names, pkg) + end + end + end + elseif !isnothing(x.args) + for a in x.args + _collect_imported_packages!(names, a) + end + end + return names +end + +""" + derived_file_imported_packages(rt, uri) -> Vector{String} + +The sorted set of top-level package names `using`/`import`ed by `uri`'s source. +""" +Salsa.@derived function derived_file_imported_packages(rt, uri) + cst = derived_julia_legacy_syntax_tree(rt, uri) + names = _collect_imported_packages!(Set{String}(), cst) + return sort!(collect(names)) +end + +# Group every workspace Julia file's imported packages by the project that +# resolves that file, so each project's environment DJP can be given exactly the +# packages its files reference. +Salsa.@derived function derived_all_project_imported_packages(rt) + @debug "derived_all_project_imported_packages" + + by_project = Dict{URI,Set{String}}() + for uri in derived_all_julia_files(rt) + project_uri = derived_project_uri_for_root(rt, uri) + isnothing(project_uri) && continue + union!(get!(by_project, project_uri, Set{String}()), derived_file_imported_packages(rt, uri)) + end + + return Dict{URI,Vector{String}}(k => sort!(collect(v)) for (k, v) in by_project) +end + +""" + derived_project_imported_packages(rt, project_uri) -> Vector{String} + +The sorted set of top-level package names the workspace's own source +`using`/`import`s within `project_uri`'s environment. +""" +Salsa.@derived function derived_project_imported_packages(rt, project_uri) + return get(derived_all_project_imported_packages(rt), project_uri, String[]) +end + Salsa.@derived function derived_required_dynamic_projects(rt) @debug "derived_required_dynamic_projects" @@ -306,6 +396,7 @@ Salsa.@derived function derived_required_dynamic_projects(rt) push!(required, WatchEnvironmentKey( uri2filepath(project_uri), project.content_hash, + derived_project_imported_packages(rt, project_uri), )) end diff --git a/src/types.jl b/src/types.jl index 1dff01f3..25568db6 100644 --- a/src/types.jl +++ b/src/types.jl @@ -458,7 +458,7 @@ function process_from_dynamic(jw::JuliaWorkspace) # gating (in derived_file_env_ready) prevents env-dependent # diagnostics for other projects from being flushed prematurely # while their own DJPs are still pending. - push!(ready_envs, WatchEnvironmentKey(msg.project_path, msg.content_hash)) + push!(ready_envs, msg.key) envs_dirty = true any_env_ready = true @@ -473,7 +473,7 @@ function process_from_dynamic(jw::JuliaWorkspace) _load_package_caches_for_project!(jw, msg.test_project_uri) test_proj = derived_project(jw.runtime, msg.test_project_uri) test_proj_hash = test_proj === nothing ? UInt64(0) : test_proj.content_hash - push!(ready_envs, WatchEnvironmentKey(uri2filepath(msg.test_project_uri), test_proj_hash)) + push!(ready_envs, WatchEnvironmentKey(uri2filepath(msg.test_project_uri), test_proj_hash, derived_project_imported_packages(jw.runtime, msg.test_project_uri))) envs_dirty = true any_env_ready = true @@ -486,7 +486,7 @@ function process_from_dynamic(jw::JuliaWorkspace) _load_package_caches_for_project!(jw, msg.project_uri) standalone_proj = derived_project(jw.runtime, msg.project_uri) standalone_proj_hash = standalone_proj === nothing ? UInt64(0) : standalone_proj.content_hash - push!(ready_envs, WatchEnvironmentKey(uri2filepath(msg.project_uri), standalone_proj_hash)) + push!(ready_envs, WatchEnvironmentKey(uri2filepath(msg.project_uri), standalone_proj_hash, derived_project_imported_packages(jw.runtime, msg.project_uri))) envs_dirty = true any_env_ready = true else diff --git a/test/test_symbolserver.jl b/test/test_symbolserver.jl index a755a0bc..4b32590d 100644 --- a/test/test_symbolserver.jl +++ b/test/test_symbolserver.jl @@ -337,6 +337,38 @@ end end end +@testitem "SymbolServer: workspace import extraction" begin + using JuliaWorkspaces: JuliaWorkspace, derived_file_imported_packages + using JuliaWorkspaces.URIs2: @uri_str + + uri = uri"file:///Foo.jl" + content = """ + module Foo + using Pkg + using LinearAlgebra, Statistics + import JSON + import CSV as C + using DataFrames: DataFrame, groupby + using .Internal + import ..Sibling + using Base.Threads + end + """ + + jw = JuliaWorkspace() + add_file!(jw, TextFile(uri, SourceText(content, "julia"))) + + # Every top-level package is captured (with alias and `A: x` forms), while + # relative imports (`.Internal`, `..Sibling`) are excluded. + @test derived_file_imported_packages(jw.runtime, uri) == + ["Base", "CSV", "DataFrames", "JSON", "LinearAlgebra", "Pkg", "Statistics"] + + # A file with no imports yields an empty set. + empty_uri = uri"file:///Bar.jl" + add_file!(jw, TextFile(empty_uri, SourceText("f(x) = x + 1\n", "julia"))) + @test isempty(derived_file_imported_packages(jw.runtime, empty_uri)) +end + @testitem "SymbolServer: #1395 DataTypeStore field types are serializable and aligned" begin using JuliaWorkspaces.SymbolServer: DataTypeStore, FakeTypeName, MethodStore using JuliaWorkspaces.SymbolServer.CacheStore: write, read @@ -662,6 +694,50 @@ end end end +@testitem "SymbolServer: missing-package check is scoped to imported closure" begin + using JuliaWorkspaces: JuliaWorkspaces + + mktempdir() do root + proj = joinpath(root, "proj") + store = joinpath(root, "store") + mkpath(proj) + write(joinpath(proj, "Manifest.toml"), """ + julia_version = "1.11.0" + manifest_format = "2.0" + + [[deps.Example]] + uuid = "7876af07-990d-54b4-ab0e-23690620f79a" + version = "0.5.3" + git-tree-sha1 = "46e44e869b4d90b96bd8ed1fdcf32244fddfb6cc" + deps = ["Dep"] + + [[deps.Dep]] + uuid = "11111111-1111-1111-1111-111111111111" + version = "1.0.0" + git-tree-sha1 = "1111111111111111111111111111111111111111" + + [[deps.Unused]] + uuid = "22222222-2222-2222-2222-222222222222" + version = "2.0.0" + git-tree-sha1 = "2222222222222222222222222222222222222222" + """) + + names(pkgs) = Set(m.name for m in pkgs) + + # No filter ⇒ every uncached manifest package is missing. + @test names(JuliaWorkspaces._get_missing_packages(proj, store)) == + Set(["Example", "Dep", "Unused"]) + + # Importing Example pulls in its transitive dep, but never the unrelated + # Unused package. + @test names(JuliaWorkspaces._get_missing_packages(proj, store, ["Example"])) == + Set(["Example", "Dep"]) + + # Importing nothing (and no deved packages) ⇒ nothing to index. + @test isempty(JuliaWorkspaces._get_missing_packages(proj, store, String[])) + end +end + @testitem "SymbolServer: FakeTypeName caches nested type parameters" begin using JuliaWorkspaces.SymbolServer: FakeTypeName using JuliaWorkspaces.SymbolServer.CacheStore: storeunstore From de83f6a882a2eaa484ce6e6209b988d050f2a374 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patrick=20H=C3=A4cker?= Date: Tue, 7 Jul 2026 06:58:37 +0200 Subject: [PATCH 2/3] refactor: carry the DJP key in reactor work messages Each reactor work message (`WatchEnvironmentMsg`, `WatchTestEnvironmentMsg`, `CreateStandaloneProjectMsg`, `EnvironmentPrepDoneMsg`) now holds its DJP key directly instead of duplicating the key's fields. This removes the field duplication and the repeated key reconstruction in the `handle!` methods, matching the lifecycle messages (`ProcessLaunchedMsg`, ...) and `EnvironmentReadyResult`, which already carry their key. No behavior change. --- src/dynamic_feature/dynamic_feature.jl | 41 ++++++++++++------------- src/dynamic_feature/dynamic_messages.jl | 15 +++------ 2 files changed, 24 insertions(+), 32 deletions(-) diff --git a/src/dynamic_feature/dynamic_feature.jl b/src/dynamic_feature/dynamic_feature.jl index d2dfbfa5..92dee0d5 100644 --- a/src/dynamic_feature/dynamic_feature.jl +++ b/src/dynamic_feature/dynamic_feature.jl @@ -680,7 +680,7 @@ end # ─── Work messages ────────────────────────────────────────────────────────── function handle!(df::DynamicFeature, msg::WatchEnvironmentMsg) - key = WatchEnvironmentKey(msg.project_path, msg.content_hash, msg.imported_packages) + key = msg.key push!(df.inflight, key) if key in df.failed_projects @@ -693,9 +693,8 @@ function handle!(df::DynamicFeature, msg::WatchEnvironmentMsg) # Offload the (potentially slow) missing-package check + cloud download to a # task so the reactor stays responsive. The result is fed back as an # `EnvironmentPrepDoneMsg`, so all state mutation stays on the reactor. - project_path = msg.project_path - content_hash = msg.content_hash - imported_packages = msg.imported_packages + project_path = key.project_path + imported_packages = key.imported_packages Threads.@async try missing_pkgs = _get_missing_packages(project_path, df.store_path, imported_packages) @@ -705,7 +704,7 @@ function handle!(df::DynamicFeature, msg::WatchEnvironmentMsg) missing_pkgs = _download_missing_caches(missing_pkgs, df.store_path, df.upstream_url; df=df) end - put!(df.in_channel, EnvironmentPrepDoneMsg(project_path, content_hash, imported_packages, !isempty(missing_pkgs))) + put!(df.in_channel, EnvironmentPrepDoneMsg(key, !isempty(missing_pkgs))) catch err @error "Environment prep failed" project_path=project_path exception=(err, catch_backtrace()) put!(df.in_channel, ProcessIndexFailedMsg(key, err)) @@ -715,23 +714,23 @@ function handle!(df::DynamicFeature, msg::WatchEnvironmentMsg) end function handle!(df::DynamicFeature, msg::EnvironmentPrepDoneMsg) - key = WatchEnvironmentKey(msg.project_path, msg.content_hash, msg.imported_packages) + key = msg.key df.progress_state.current_sub_progress = 0.5 if !msg.still_missing - @info "All package caches available, skipping DJP" project_path=msg.project_path - _report_progress(df, "All caches available for $(basename(msg.project_path))") + @info "All package caches available, skipping DJP" project_path=key.project_path + _report_progress(df, "All caches available for $(basename(key.project_path))") put!(df.out_channel, EnvironmentReadyResult(key)) push!(df.done, key) _complete_work_item!(df, key) elseif df.djp_mode != DynamicOff - @info "Launching DJP for remaining missing packages" project_path=msg.project_path - _report_progress(df, "Indexing $(basename(msg.project_path))...") - djp = DynamicJuliaProcess(key, msg.project_path, nothing, :watch_environment) + @info "Launching DJP for remaining missing packages" project_path=key.project_path + _report_progress(df, "Indexing $(basename(key.project_path))...") + djp = DynamicJuliaProcess(key, key.project_path, nothing, :watch_environment) df.procs[key] = djp _launch_process!(df, djp) else - @info "Some packages missing but DJP disabled, proceeding with best-effort" project_path=msg.project_path + @info "Some packages missing but DJP disabled, proceeding with best-effort" project_path=key.project_path put!(df.out_channel, EnvironmentReadyResult(key)) push!(df.done, key) _complete_work_item!(df, key) @@ -741,7 +740,7 @@ function handle!(df::DynamicFeature, msg::EnvironmentPrepDoneMsg) end function handle!(df::DynamicFeature, msg::WatchTestEnvironmentMsg) - key = WatchTestEnvironmentKey(msg.project_path, msg.package, msg.content_hash) + key = msg.key push!(df.inflight, key) if key in df.failed_projects @@ -751,8 +750,8 @@ function handle!(df::DynamicFeature, msg::WatchTestEnvironmentMsg) return false end - _report_progress(df, "Indexing test environment for $(msg.package)...") - djp = DynamicJuliaProcess(key, msg.project_path, msg.package, :watch_test_environment) + _report_progress(df, "Indexing test environment for $(key.package_name)...") + djp = DynamicJuliaProcess(key, key.project_path, key.package_name, :watch_test_environment) df.procs[key] = djp _launch_process!(df, djp) @@ -760,7 +759,7 @@ function handle!(df::DynamicFeature, msg::WatchTestEnvironmentMsg) end function handle!(df::DynamicFeature, msg::CreateStandaloneProjectMsg) - key = CreateStandaloneProjectKey(msg.package_path, msg.content_hash) + key = msg.key push!(df.inflight, key) if key in df.failed_projects @@ -770,8 +769,8 @@ function handle!(df::DynamicFeature, msg::CreateStandaloneProjectMsg) return false end - _report_progress(df, "Creating standalone project for $(basename(msg.package_path))...") - djp = DynamicJuliaProcess(key, msg.package_path, nothing, :create_standalone_project) + _report_progress(df, "Creating standalone project for $(basename(key.package_path))...") + djp = DynamicJuliaProcess(key, key.package_path, nothing, :create_standalone_project) df.procs[key] = djp _launch_process!(df, djp) @@ -957,11 +956,11 @@ function handle!(df::DynamicFeature, msg::ReconcileMsg) _report_progress(df, "Preparing to index...") if key isa WatchEnvironmentKey - handle!(df, WatchEnvironmentMsg(key.project_path, key.content_hash, key.imported_packages)) + handle!(df, WatchEnvironmentMsg(key)) elseif key isa WatchTestEnvironmentKey - handle!(df, WatchTestEnvironmentMsg(key.project_path, key.package_name, key.content_hash)) + handle!(df, WatchTestEnvironmentMsg(key)) elseif key isa CreateStandaloneProjectKey - handle!(df, CreateStandaloneProjectMsg(key.package_path, key.content_hash)) + handle!(df, CreateStandaloneProjectMsg(key)) end end diff --git a/src/dynamic_feature/dynamic_messages.jl b/src/dynamic_feature/dynamic_messages.jl index 06e3e1b9..cddd10d8 100644 --- a/src/dynamic_feature/dynamic_messages.jl +++ b/src/dynamic_feature/dynamic_messages.jl @@ -55,22 +55,17 @@ abstract type DynamicReactorMessage end """Request to index/watch the environment of a project.""" struct WatchEnvironmentMsg <: DynamicReactorMessage - project_path::String - content_hash::UInt64 - imported_packages::Vector{String} + key::WatchEnvironmentKey end """Request to index/watch the test environment of a project + package.""" struct WatchTestEnvironmentMsg <: DynamicReactorMessage - project_path::String - package::String - content_hash::UInt64 + key::WatchTestEnvironmentKey end """Request to create a standalone project for a package folder.""" struct CreateStandaloneProjectMsg <: DynamicReactorMessage - package_path::String - content_hash::UInt64 + key::CreateStandaloneProjectKey end """Request an orderly shutdown of the reactor.""" @@ -97,9 +92,7 @@ once the (potentially slow) missing-package check + cloud download finished. `still_missing` indicates whether a DJP is still required afterwards. """ struct EnvironmentPrepDoneMsg <: DynamicReactorMessage - project_path::String - content_hash::UInt64 - imported_packages::Vector{String} + key::WatchEnvironmentKey still_missing::Bool end From 8931f1b592e83ba4a3b5a890ca284999aaf76df4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patrick=20H=C3=A4cker?= Date: Tue, 7 Jul 2026 11:07:32 +0200 Subject: [PATCH 3/3] Add test coverage for loading relevant packages only --- test/test_symbolserver.jl | 120 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 120 insertions(+) diff --git a/test/test_symbolserver.jl b/test/test_symbolserver.jl index 4b32590d..b0bc9e90 100644 --- a/test/test_symbolserver.jl +++ b/test/test_symbolserver.jl @@ -369,6 +369,43 @@ end @test isempty(derived_file_imported_packages(jw.runtime, empty_uri)) end +@testitem "SymbolServer: project imported packages aggregate into the env key" begin + using JuliaWorkspaces: JuliaWorkspaces, JuliaWorkspace, derived_project_imported_packages + using JuliaWorkspaces.URIs2: URI + + project_toml = """ + name = "AggTest" + uuid = "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee" + version = "0.1.0" + """ + manifest_toml = """ + julia_version = "1.11.0" + manifest_format = "2.0" + project_hash = "abc123" + + [deps] + """ + + jw = JuliaWorkspace() + add_file!(jw, TextFile(URI("file:///aggtest/Project.toml"), SourceText(project_toml, "toml"))) + add_file!(jw, TextFile(URI("file:///aggtest/Manifest.toml"), SourceText(manifest_toml, "toml"))) + add_file!(jw, TextFile(URI("file:///aggtest/src/AggTest.jl"), + SourceText("module AggTest\nusing Foo\nend\n", "julia"))) + add_file!(jw, TextFile(URI("file:///aggtest/src/extra.jl"), + SourceText("import Bar\nusing .Local\n", "julia"))) + + project_uri = JuliaWorkspaces.derived_project_uri_for_root( + jw.runtime, URI("file:///aggtest/src/AggTest.jl")) + + # Imports are unioned across every file the project resolves; relative imports drop out. + @test derived_project_imported_packages(jw.runtime, project_uri) == ["Bar", "Foo"] + + # ...and the aggregate is carried on the project's environment DJP key. + reqs = JuliaWorkspaces.derived_required_dynamic_projects(jw.runtime) + wek = only(k for k in reqs if k isa JuliaWorkspaces.WatchEnvironmentKey) + @test wek.imported_packages == ["Bar", "Foo"] +end + @testitem "SymbolServer: #1395 DataTypeStore field types are serializable and aligned" begin using JuliaWorkspaces.SymbolServer: DataTypeStore, FakeTypeName, MethodStore using JuliaWorkspaces.SymbolServer.CacheStore: write, read @@ -738,6 +775,89 @@ end end end +@testitem "SymbolServer: get_store loads only imported plus deved packages" begin + a_uuid = "5a61a11e-3d95-423b-8231-1a5bca90429f" + b_uuid = "09fe8bcb-ac26-410c-9c93-4f8b45323ba9" + la_uuid = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" # LinearAlgebra (stdlib, non-deved) + symbolserver_jl = abspath(joinpath(@__DIR__, "..", "juliadynamicanalysisprocess", + "JuliaDynamicAnalysisProcess", "src", "symbolserver.jl")) + + mktempdir() do root + proj = joinpath(root, "proj") + adir = joinpath(root, "A") + bdir = joinpath(root, "B") + store = joinpath(root, "store") + mkpath(joinpath(adir, "src")) + mkpath(joinpath(bdir, "src")) + mkpath(proj) + mkpath(store) + + write(joinpath(adir, "Project.toml"), """ + name = "A" + uuid = "$a_uuid" + version = "0.1.0" + """) + write(joinpath(adir, "src", "A.jl"), "module A\nfoo(x) = 1\nend # module A\n") + + write(joinpath(bdir, "Project.toml"), """ + name = "B" + uuid = "$b_uuid" + version = "0.1.0" + """) + write(joinpath(bdir, "src", "B.jl"), "module B\nbar(x) = 2\nend # module B\n") + + write(joinpath(proj, "Project.toml"), """ + [deps] + A = "$a_uuid" + B = "$b_uuid" + LinearAlgebra = "$la_uuid" + """) + write(joinpath(proj, "Manifest.toml"), """ + julia_version = "1.11.0" + manifest_format = "2.0" + project_hash = "0000000000000000000000000000000000000000" + + [[deps.A]] + path = "../A" + uuid = "$a_uuid" + version = "0.1.0" + + [[deps.B]] + path = "../B" + uuid = "$b_uuid" + version = "0.1.0" + + [[deps.LinearAlgebra]] + uuid = "$la_uuid" + """) + + runner = joinpath(root, "run_indexer.jl") + write(runner, """ + include(raw"$symbolserver_jl") + using Pkg + Pkg.activate(raw"$proj") + SymbolServer.get_store(raw"$store", nothing; used_packages=["A"]) + """) + + jl = joinpath(Sys.BINDIR, Base.julia_exename()) + logfile = joinpath(root, "indexer.log") + cmd = `$jl --startup-file=no --project=$proj $runner` + proc = withenv("JULIA_PKG_PRECOMPILE_AUTO" => "0", "JULIA_DEBUG" => "SymbolServer") do + run(pipeline(ignorestatus(cmd), stderr=logfile)) + end + log = read(logfile, String) + @test proc.exitcode == 0 + + # A is imported by the workspace → indexed. + @test isfile(joinpath(store, "A", "A", a_uuid, "0.1.0.jstore")) + # B isn't imported but is a workspace (deved) package → still indexed. + @test isfile(joinpath(store, "B", "B", b_uuid, "0.1.0.jstore")) + # LinearAlgebra is neither imported nor deved, so the loader skips loading + # it on its own account (it may still be captured if already resident). + @test occursin(r"LinearAlgebra.*not imported by the workspace, skipping load", log) + end +end + @testitem "SymbolServer: FakeTypeName caches nested type parameters" begin using JuliaWorkspaces.SymbolServer: FakeTypeName using JuliaWorkspaces.SymbolServer.CacheStore: storeunstore