fix: memory-leak due to unbound type-parameter recursion

shared/symbolserver/faketypes.jl

@@ -14,24 +14,49 @@ struct FakeTypeName
     parameters::Vector{Any}
 end
 
-function FakeTypeName(@nospecialize(x))
+# Type parameters are expanded recursively so cached signatures keep their
+# structure, but a few types expand into huge trees — type-domain packages
+# whose types recurse into themselves (naturals as nested `NonnegativeInteger`,
+# rationals as continued fractions), and even some Base/LinearAlgebra
+# signatures whose `where`-bounded `Union`s reach tens of thousands of nodes —
+# which unbounded would produce multi-gigabyte caches that exhaust memory on
+# read. To prevent that, `budget` caps how many `DataType`s are expanded while
+# building one type: once it is spent, a type keeps only its name and a single
+# `Unserializable` parameter marking the aborted expansion (displayed as `…`,
+# same sentinel the cache writer uses). The limit sits far above any ordinary
+# type, so only these outliers are truncated; the serializer's own MAX_DEPTH
+# guard is a separate cycle check, not a size bound.
+const MAX_EXPANDED_TYPES = 256
+
+mutable struct ExpandBudget
+    remaining::Int
+end
+ExpandBudget() = ExpandBudget(MAX_EXPANDED_TYPES)
+
+function FakeTypeName(@nospecialize(x), budget::ExpandBudget=ExpandBudget())
     @static if !(Vararg isa Type)
         x isa typeof(Vararg) && return FakeTypeofVararg(x)
     end
     if x isa DataType
         xname = x.name
-        xnamename = xname.name
-        ft = FakeTypeName(VarRef(VarRef(x.name.module), xnamename), [])
-        for p in x.parameters
-            push!(ft.parameters, _parameter(p))
+        ft = FakeTypeName(VarRef(VarRef(xname.module), xname.name), [])
+        if isempty(x.parameters)
+            # leaf type, nothing to expand
+        elseif budget.remaining > 0
+            budget.remaining -= 1
+            for p in x.parameters
+                push!(ft.parameters, _parameter(p, budget))
+            end
+        else
+            push!(ft.parameters, Unserializable())
         end
         ft
     elseif x isa Union
-        FakeUnion(x)
+        FakeUnion(x, budget)
     elseif x isa UnionAll
-        FakeUnionAll(x)
+        FakeUnionAll(x, budget)
     elseif x isa TypeVar
-        FakeTypeVar(x)
+        FakeTypeVar(x, budget)
     elseif x isa Core.TypeofBottom
         FakeTypeofBottom()
     elseif x isa Module
@@ -50,28 +75,28 @@ struct FakeUnion
     a
     b
 end
-FakeUnion(u::Union) = FakeUnion(FakeTypeName(u.a), FakeTypeName(u.b))
+FakeUnion(u::Union, budget::ExpandBudget=ExpandBudget()) = FakeUnion(FakeTypeName(u.a, budget), FakeTypeName(u.b, budget))
 struct FakeTypeVar
     name::Symbol
     lb
     ub
 end
-FakeTypeVar(tv::TypeVar) = FakeTypeVar(tv.name, FakeTypeName(tv.lb), FakeTypeName(tv.ub))
+FakeTypeVar(tv::TypeVar, budget::ExpandBudget=ExpandBudget()) = FakeTypeVar(tv.name, FakeTypeName(tv.lb, budget), FakeTypeName(tv.ub, budget))
 struct FakeUnionAll
     var::FakeTypeVar
     body::Any
 end
-FakeUnionAll(ua::UnionAll) = FakeUnionAll(FakeTypeVar(ua.var), FakeTypeName(ua.body))
+FakeUnionAll(ua::UnionAll, budget::ExpandBudget=ExpandBudget()) = FakeUnionAll(FakeTypeVar(ua.var, budget), FakeTypeName(ua.body, budget))
 
-function _parameter(@nospecialize(p))
+function _parameter(@nospecialize(p), budget::ExpandBudget=ExpandBudget())
     if p isa Union{Int,Symbol,Bool,Char}
         p
     elseif !(p isa Type) && isbitstype(typeof(p))
         0
     elseif p isa Tuple
-        _parameter.(p)
+        map(pp -> _parameter(pp, budget), p)
     else
-        FakeTypeName(p)
+        FakeTypeName(p, budget)
     end
 end
 

shared/symbolserver/serialize.jl

@@ -87,6 +87,9 @@ end
 function _write(io, x::Nothing, depth::Int)
     Base.write(io, NothingHeader)
 end
+function _write(io, x::Unserializable, depth::Int)
+    Base.write(io, UnserializableHeader)
+end
 function _write(io, x::Char, depth::Int)
     Base.write(io, CharHeader)
     Base.write(io, UInt32(x))

test/test_symbolserver.jl

@@ -779,3 +779,36 @@ end
         @test VarRef(nothing, :B) in exts[b_foo.extends]
     end
 end
+
+@testitem "SymbolServer: recursive type parameters are bounded" begin
+    using JuliaWorkspaces.SymbolServer: FakeTypeName, ExpandBudget, Unserializable, CacheStore
+
+    # Number of DataTypes that were expanded (i.e. kept their parameters).
+    expanded(x) = x isa FakeTypeName && !isempty(x.parameters) && !(x.parameters[1] isa Unserializable) ? 1 + sum(expanded, x.parameters) : 0
+    # Number of expansions aborted by the budget.
+    truncated(x) = x isa FakeTypeName ? count(p -> p isa Unserializable, x.parameters) + sum(truncated, x.parameters; init = 0) : 0
+
+    limit = 8
+    deep = foldl((T, _) -> Tuple{T}, 1:100; init = Int)   # Tuple{Tuple{…{Int}}}
+    wide = Tuple{ntuple(i -> Val{i}, 100)...}              # Tuple{Val{1},…,Val{100}}
+
+    # However a type explodes — by depth or by width — expansion stops at the
+    # budget, and each cut is marked rather than silently dropped.
+    for T in (deep, wide)
+        ft = FakeTypeName(T, ExpandBudget(limit))
+        @test expanded(ft) <= limit
+        @test truncated(ft) > 0
+    end
+
+    # An ordinary type has far fewer nodes than the budget, so nothing is dropped.
+    ordinary = Dict{String,Vector{Tuple{Int,Float64}}}
+    @test FakeTypeName(ordinary, ExpandBudget(limit)) == FakeTypeName(ordinary)
+    @test truncated(FakeTypeName(ordinary, ExpandBudget(limit))) == 0
+
+    # The marker renders as `…` and roundtrips through the cache format.
+    ft = FakeTypeName(deep, ExpandBudget(limit))
+    @test contains(string(ft), "…")
+    io = IOBuffer()
+    CacheStore.write(io, ft)
+    @test CacheStore.read(IOBuffer(take!(io))) == ft
+end