feat: batch initial file load and improve progress messaging

Project.toml

@@ -18,7 +18,7 @@ UUIDs = "cf7118a7-6976-5b1a-9a39-7adc72f591a4"
 [compat]
 JSON = "0.20, 0.21"
 JSONRPC = "3"
-JuliaWorkspaces = "8"
+JuliaWorkspaces = "8.1"
 LoggingExtras = "1.2"
 PrecompileTools = "1"
 URIs = "1.3"

src/progress.jl

@@ -1,45 +1,71 @@
 """
     create_progress_callback(server::LanguageServerInstance) -> Function
 
-Return a closure `(message::String, percentage::Int) -> Nothing` that translates
-JuliaWorkspaces progress updates into LSP `\$/progress` notifications.
+Return a closure `(key::String, message::String, percentage::Int) -> Nothing`
+that translates JuliaWorkspaces progress updates into LSP `\$/progress`
+notifications.
 
-The closure manages a single progress token lifecycle:
-- On the first call it creates a token via `window/workDoneProgress/create` and
-  sends `WorkDoneProgressBegin`.
-- Subsequent calls send `WorkDoneProgressReport` with message and percentage.
-- When percentage reaches 100 it sends `WorkDoneProgressEnd` and resets, ready
-  for a new round (e.g. after a manifest change triggers re-indexing).
+Each distinct `key` is its own work-done-progress token (its own progress bar
+on the client) with the full 0–100 percentage range, so concurrently running
+operations — downloading caches, indexing different projects, loading caches —
+are presented independently instead of being squeezed into slices of one bar.
+
+The closure only enqueues the report and never blocks: a worker task owns the
+token lifecycles and performs the client round-trips. This matters because the
+callback is invoked from the dynamic-feature reactor, which must not stall on
+`window/workDoneProgress/create` (a blocking client request) — reports arriving
+while a token is being created queue up and are delivered in order.
+
+Per-key lifecycle managed by the worker:
+- The first report for a key creates a token via `window/workDoneProgress/create`
+  and sends `WorkDoneProgressBegin`.
+- Subsequent reports are sent as `WorkDoneProgressReport`.
+- A report with `percentage >= 100` sends `WorkDoneProgressEnd`, retires the
+  key (a later report for it starts a fresh bar), and queues
+  `:jw_indexing_complete` so diagnostics are refreshed. Ending a key that has
+  no open bar is a no-op.
 """
 function create_progress_callback(server::LanguageServerInstance)
-    # Mutable state captured by the closure.
-    active = Ref(false)
-    token = Ref("")
+    reports = Channel{Tuple{String,String,Int}}(Inf)
+
+    @async try
+        tokens = Dict{String,String}()
 
-    return function (message::String, percentage::Int)
-        # Guard: do nothing when the client doesn't support work-done progress
-        # or the endpoint isn't ready yet.
-        server.clientcapability_window_workdoneprogress || return
-        ep = server.jr_endpoint
+        for (key, message, percentage) in reports
+            # Guard: do nothing when the client doesn't support work-done progress
+            # or the endpoint isn't ready yet.
+            server.clientcapability_window_workdoneprogress || continue
+            ep = server.jr_endpoint
 
-        if !active[]
-            # Start a new progress session
-            token[] = "jw-indexing-$(UUIDs.uuid4())"
-            try
-                JSONRPC.send(ep, window_workDoneProgress_create_request_type, WorkDoneProgressCreateParams(token[]))
-            catch err
-                @warn "Failed to create progress token" exception=(err, catch_backtrace())
-                return
+            token = get(tokens, key, nothing)
+            if token === nothing
+                # Ending an operation that never opened a bar is a no-op.
+                percentage >= 100 && continue
+
+                # Start a new progress bar for this operation
+                token = "jw-$(UUIDs.uuid4())"
+                try
+                    JSONRPC.send(ep, window_workDoneProgress_create_request_type, WorkDoneProgressCreateParams(token))
+                catch err
+                    @warn "Failed to create progress token" exception=(err, catch_backtrace())
+                    continue
+                end
+                JSONRPC.send(ep, progress_notification_type, ProgressParams(token, WorkDoneProgressBegin("Julia", false, message, percentage)))
+                tokens[key] = token
+            elseif percentage >= 100
+                JSONRPC.send(ep, progress_notification_type, ProgressParams(token, WorkDoneProgressEnd(message)))
+                delete!(tokens, key)
+                put!(server.combined_msg_queue, (type=:jw_indexing_complete,))
+            else
+                JSONRPC.send(ep, progress_notification_type, ProgressParams(token, WorkDoneProgressReport(false, message, percentage)))
             end
-            JSONRPC.send(ep, progress_notification_type, ProgressParams(token[], WorkDoneProgressBegin("Julia", false, message, percentage)))
-            active[] = true
-        elseif percentage >= 100
-            JSONRPC.send(ep, progress_notification_type, ProgressParams(token[], WorkDoneProgressEnd(message)))
-            active[] = false
-            put!(server.combined_msg_queue, (type=:jw_indexing_complete,))
-        else
-            JSONRPC.send(ep, progress_notification_type, ProgressParams(token[], WorkDoneProgressReport(false, message, percentage)))
         end
+    catch err
+        @error "Progress reporting task failed" exception=(err, catch_backtrace())
+    end
+
+    return function (key::String, message::String, percentage::Int)
+        put!(reports, (key, message, percentage))
         return
     end
 end
@@ -55,12 +81,12 @@ while the LanguageServerInstance is still being built.
 function _create_deferred_progress_callback(server_ref::Ref)
     inner_cb = Ref{Union{Nothing,Function}}(nothing)
 
-    return function (message::String, percentage::Int)
+    return function (key::String, message::String, percentage::Int)
         isassigned(server_ref) || return
         if inner_cb[] === nothing
             inner_cb[] = create_progress_callback(server_ref[])
         end
-        inner_cb[](message, percentage)
+        inner_cb[](key, message, percentage)
         return
     end
 end

src/requests/init.jl

@@ -105,6 +105,9 @@ function load_folder(path::String, server, added_uris)
     if load_rootpath(path)
         try
             for (root, _, files) in walkdir(path, onerror=x -> x)
+                # Walking a big workspace shouldn't starve other tasks (the
+                # dynamic-feature reactor, the client connection).
+                yield()
                 for file in files
                     filepath = joinpath(root, file)
                     if isvalidjlfile(filepath)
@@ -285,6 +288,7 @@ function initialized_notification(params::InitializedParams, server::LanguageSer
 
     TraceLogging.@trace "initial_workspace_load" begin
         if server.workspaceFolders !== nothing
+            files_to_add = JuliaWorkspaces.TextFile[]
             TraceLogging.@trace "first workspace folder loop" for i in server.workspaceFolders
                 files = JuliaWorkspaces.read_path_into_textdocuments(filepath2uri(i), ignore_io_errors=true)
 
@@ -295,12 +299,18 @@ function initialized_notification(params::InitializedParams, server::LanguageSer
                         server._files_from_disc[i.uri] = i
 
                         if !haskey(server._open_file_versions, i.uri)
-                            JuliaWorkspaces.add_file!(server.workspace, i)
+                            push!(files_to_add, i)
                         end
                     end
                 end
             end
 
+            # Add the whole batch at once: this reconciles the required dynamic
+            # processes a single time instead of once per file, so downloading/
+            # indexing can start right after this call rather than after the
+            # whole initial load.
+            TraceLogging.@trace JuliaWorkspaces.add_files!(server.workspace, files_to_add)
+
             TraceLogging.@trace JuliaWorkspaces.set_active_project!(server.workspace, isempty(server.env_path) ? nothing : filepath2uri(server.env_path))
 
             TraceLogging.@trace "second workspace folder loop" for wkspc in server.workspaceFolders

test/test_progress.jl

@@ -16,42 +16,73 @@
 
     cb = create_progress_callback(server)
 
-    # First call → Begin
-    cb("Downloading caches...", 10)
-    @test length(sent) == 2  # create + begin
+    # Reports are delivered asynchronously by the worker task that owns the
+    # progress tokens, so wait for the expected number of sends.
+    wait_for_sends(n) = timedwait(() -> length(sent) >= n, 5.0) === :ok
+
+    # First report for a key → create + Begin
+    cb("download:/p", "Downloading caches...", 10)
+    @test wait_for_sends(2)
     @test sent[1][1] === window_workDoneProgress_create_request_type
     @test sent[1][2] isa WorkDoneProgressCreateParams
-    token = sent[1][2].token
-    @test startswith(token, "jw-indexing-")
+    dl_token = sent[1][2].token
+    @test startswith(dl_token, "jw-")
 
     @test sent[2][1] === progress_notification_type
     @test sent[2][2] isa ProgressParams{WorkDoneProgressBegin}
-    @test sent[2][2].token == token
+    @test sent[2][2].token == dl_token
     @test sent[2][2].value.title == "Julia"
     @test sent[2][2].value.message == "Downloading caches..."
     @test sent[2][2].value.percentage == 10
 
-    # Subsequent call → Report
-    cb("Indexing project...", 50)
-    @test length(sent) == 3
-    @test sent[3][1] === progress_notification_type
-    @test sent[3][2] isa ProgressParams{WorkDoneProgressReport}
-    @test sent[3][2].value.message == "Indexing project..."
-    @test sent[3][2].value.percentage == 50
-
-    # Final call → End
-    cb("Indexing complete", 100)
-    @test length(sent) == 4
-    @test sent[4][1] === progress_notification_type
-    @test sent[4][2] isa ProgressParams{WorkDoneProgressEnd}
-    @test sent[4][2].value.message == "Indexing complete"
-
-    # After End, a new call starts a fresh session
+    # A different key gets its own token — bars run concurrently.
+    cb("index:/p", "Indexing project...", 5)
+    @test wait_for_sends(4)
+    @test sent[3][2] isa WorkDoneProgressCreateParams
+    idx_token = sent[3][2].token
+    @test idx_token != dl_token
+    @test sent[4][2] isa ProgressParams{WorkDoneProgressBegin}
+    @test sent[4][2].token == idx_token
+
+    # Subsequent reports go to their own bars.
+    cb("download:/p", "Downloading caches (50/100)...", 50)
+    @test wait_for_sends(5)
+    @test sent[5][2] isa ProgressParams{WorkDoneProgressReport}
+    @test sent[5][2].token == dl_token
+    @test sent[5][2].value.percentage == 50
+
+    # percentage >= 100 ends only that key's bar.
+    cb("download:/p", "Downloads done", 100)
+    @test wait_for_sends(6)
+    @test sent[6][2] isa ProgressParams{WorkDoneProgressEnd}
+    @test sent[6][2].token == dl_token
+
+    cb("index:/p", "Indexing Foo...", 40)
+    @test wait_for_sends(7)
+    @test sent[7][2] isa ProgressParams{WorkDoneProgressReport}
+    @test sent[7][2].token == idx_token
+
+    # Ending a key without an open bar is a no-op; ending the index bar works.
+    cb("unknown-op", "Done", 100)
+    cb("index:/p", "Done", 100)
+    @test wait_for_sends(8)
+    @test sent[8][2] isa ProgressParams{WorkDoneProgressEnd}
+    @test sent[8][2].token == idx_token
+
+    # After End, a new report for the same key starts a fresh bar.
+    empty!(sent)
+    cb("index:/p", "Re-indexing...", 5)
+    @test wait_for_sends(2)  # new create + begin
+    @test sent[1][2].token != idx_token
+
+    # The callback itself must never block: enqueueing a burst returns
+    # immediately and everything is delivered in order.
     empty!(sent)
-    cb("Re-indexing...", 5)
-    @test length(sent) == 2  # new create + begin
-    new_token = sent[1][2].token
-    @test new_token != token  # different token
+    for i in 1:10
+        cb("index:/p", "Report $i", 10 + i)
+    end
+    @test wait_for_sends(10)
+    @test [p.value.message for (_, p) in sent] == ["Report $i" for i in 1:10]
 end
 
 @testitem "Progress callback without workDoneProgress support" begin
@@ -69,9 +100,10 @@ end
     cb = create_progress_callback(server)
 
     # Should be a no-op — no sends at all
-    cb("Downloading...", 10)
-    cb("Indexing...", 50)
-    cb("Done", 100)
+    cb("op", "Downloading...", 10)
+    cb("op", "Indexing...", 50)
+    cb("op", "Done", 100)
+    sleep(0.5)
     @test isempty(sent)
 end
 
@@ -86,8 +118,10 @@ end
 
     # With JSONRPC.send(::Nothing,...) defined, callback should work without error
     JSONRPC.send(::Nothing, typ, params) = nothing
+
     cb = create_progress_callback(server)
 
-    cb("test", 10)
+    cb("op", "test", 10)
+    sleep(0.5)
     @test true  # if we got here without error, the test passes
 end