From 5945e4d594d27bf4d01518e97e5ad52666a67fdd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patrick=20H=C3=A4cker?= Date: Sun, 5 Jul 2026 09:01:45 +0200 Subject: [PATCH 1/2] fix: bound type-parameter expansion to prevent gigabyte caches MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit FakeTypeName expanded a type's parameters recursively with no size bound. A few types explode into enormous trees: type-domain packages whose types recurse into themselves (e.g. TypeDomainNaturalNumbers — naturals as nested types, rationals as continued fractions), and even some Base/LinearAlgebra signatures whose `where`-bounded `Union`s reach tens of thousands of nodes. This produced multi-gigabyte cache files (2.2 GB for TypeDomainNaturalNumbers) that exhausted memory — OOMing the machine — when read back into the store. Cap expansion with a per-type budget of expanded DataTypes (MAX_EXPANDED_TYPES = 128): once it is spent, a type's remaining parameters are dropped and only its name is kept. The budget bounds any shape of explosion (depth, width, or indirect recursion through variable bounds), while ordinary types stay far below it and are unaffected. Add a regression test. Ported from julia-vscode/SymbolServer.jl#320 (SymbolServer.jl is deprecated; the code now lives here under shared/symbolserver/). --- shared/symbolserver/faketypes.jl | 48 ++++++++++++++++++++++---------- test/test_symbolserver.jl | 20 +++++++++++++ 2 files changed, 54 insertions(+), 14 deletions(-) diff --git a/shared/symbolserver/faketypes.jl b/shared/symbolserver/faketypes.jl index 8b59ef94..be6f6e02 100644 --- a/shared/symbolserver/faketypes.jl +++ b/shared/symbolserver/faketypes.jl @@ -14,24 +14,44 @@ struct FakeTypeName parameters::Vector{Any} end -function FakeTypeName(@nospecialize(x)) +# Type parameters are expanded recursively so cached signatures keep their +# structure, but a few types expand into huge trees — type-domain packages +# whose types recurse into themselves (naturals as nested `NonnegativeInteger`, +# rationals as continued fractions), and even some Base/LinearAlgebra +# signatures whose `where`-bounded `Union`s reach tens of thousands of nodes — +# which unbounded would produce multi-gigabyte caches that exhaust memory on +# read. To prevent that, `budget` caps how many `DataType`s are expanded while +# building one type: once it is spent the remaining parameters are dropped, +# keeping only their names. The limit sits far above any ordinary type, so only +# these outliers are truncated; the serializer's own MAX_DEPTH guard is a +# separate cycle check, not a size bound. +const MAX_EXPANDED_TYPES = 128 + +mutable struct ExpandBudget + remaining::Int +end +ExpandBudget() = ExpandBudget(MAX_EXPANDED_TYPES) + +function FakeTypeName(@nospecialize(x), budget::ExpandBudget=ExpandBudget()) @static if !(Vararg isa Type) x isa typeof(Vararg) && return FakeTypeofVararg(x) end if x isa DataType xname = x.name - xnamename = xname.name - ft = FakeTypeName(VarRef(VarRef(x.name.module), xnamename), []) - for p in x.parameters - push!(ft.parameters, _parameter(p)) + ft = FakeTypeName(VarRef(VarRef(xname.module), xname.name), []) + if budget.remaining > 0 + budget.remaining -= 1 + for p in x.parameters + push!(ft.parameters, _parameter(p, budget)) + end end ft elseif x isa Union - FakeUnion(x) + FakeUnion(x, budget) elseif x isa UnionAll - FakeUnionAll(x) + FakeUnionAll(x, budget) elseif x isa TypeVar - FakeTypeVar(x) + FakeTypeVar(x, budget) elseif x isa Core.TypeofBottom FakeTypeofBottom() elseif x isa Module @@ -50,28 +70,28 @@ struct FakeUnion a b end -FakeUnion(u::Union) = FakeUnion(FakeTypeName(u.a), FakeTypeName(u.b)) +FakeUnion(u::Union, budget::ExpandBudget=ExpandBudget()) = FakeUnion(FakeTypeName(u.a, budget), FakeTypeName(u.b, budget)) struct FakeTypeVar name::Symbol lb ub end -FakeTypeVar(tv::TypeVar) = FakeTypeVar(tv.name, FakeTypeName(tv.lb), FakeTypeName(tv.ub)) +FakeTypeVar(tv::TypeVar, budget::ExpandBudget=ExpandBudget()) = FakeTypeVar(tv.name, FakeTypeName(tv.lb, budget), FakeTypeName(tv.ub, budget)) struct FakeUnionAll var::FakeTypeVar body::Any end -FakeUnionAll(ua::UnionAll) = FakeUnionAll(FakeTypeVar(ua.var), FakeTypeName(ua.body)) +FakeUnionAll(ua::UnionAll, budget::ExpandBudget=ExpandBudget()) = FakeUnionAll(FakeTypeVar(ua.var, budget), FakeTypeName(ua.body, budget)) -function _parameter(@nospecialize(p)) +function _parameter(@nospecialize(p), budget::ExpandBudget=ExpandBudget()) if p isa Union{Int,Symbol,Bool,Char} p elseif !(p isa Type) && isbitstype(typeof(p)) 0 elseif p isa Tuple - _parameter.(p) + map(pp -> _parameter(pp, budget), p) else - FakeTypeName(p) + FakeTypeName(p, budget) end end diff --git a/test/test_symbolserver.jl b/test/test_symbolserver.jl index c9d59bb6..812731ef 100644 --- a/test/test_symbolserver.jl +++ b/test/test_symbolserver.jl @@ -779,3 +779,23 @@ end @test VarRef(nothing, :B) in exts[b_foo.extends] end end + +@testitem "SymbolServer: recursive type parameters are bounded" begin + using JuliaWorkspaces.SymbolServer: FakeTypeName, ExpandBudget + + # Number of DataTypes that were expanded (i.e. kept their parameters). + expanded(x) = x isa FakeTypeName && !isempty(x.parameters) ? 1 + sum(expanded, x.parameters) : 0 + + limit = 8 + deep = foldl((T, _) -> Tuple{T}, 1:100; init = Int) # Tuple{Tuple{…{Int}}} + wide = Tuple{ntuple(i -> Val{i}, 100)...} # Tuple{Val{1},…,Val{100}} + + # However a type explodes — by depth or by width — expansion stops at the budget. + for T in (deep, wide) + @test expanded(FakeTypeName(T, ExpandBudget(limit))) <= limit + end + + # An ordinary type has far fewer nodes than the budget, so nothing is dropped. + ordinary = Dict{String,Vector{Tuple{Int,Float64}}} + @test FakeTypeName(ordinary, ExpandBudget(limit)) == FakeTypeName(ordinary) +end From b7a7141fab6334fc6a8ba663e9342331f4928f8d Mon Sep 17 00:00:00 2001 From: Sebastian Pfitzner Date: Mon, 6 Jul 2026 11:28:42 +0000 Subject: [PATCH 2/2] fix: mark aborted type-parameter expansion with the Unserializable sentinel MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Instead of silently dropping all parameters when the ExpandBudget is spent, keep the type's name and a single Unserializable() parameter, so truncated types render as Foo{…} instead of a misleadingly bare Foo. Budget is now only spent on types that actually have parameters, and bumped to 256 to clear the p99 of Base/stdlib signature slots (~160 expanded DataTypes) with some margin. The sentinel needs a writer method to roundtrip: without one, a parameters vector containing it would MethodError during the cache write and degrade the whole module binding to Unserializable. Co-Authored-By: Claude Fable 5 --- shared/symbolserver/faketypes.jl | 17 +++++++++++------ shared/symbolserver/serialize.jl | 3 +++ test/test_symbolserver.jl | 21 +++++++++++++++++---- 3 files changed, 31 insertions(+), 10 deletions(-) diff --git a/shared/symbolserver/faketypes.jl b/shared/symbolserver/faketypes.jl index be6f6e02..7cd04d37 100644 --- a/shared/symbolserver/faketypes.jl +++ b/shared/symbolserver/faketypes.jl @@ -21,11 +21,12 @@ end # signatures whose `where`-bounded `Union`s reach tens of thousands of nodes — # which unbounded would produce multi-gigabyte caches that exhaust memory on # read. To prevent that, `budget` caps how many `DataType`s are expanded while -# building one type: once it is spent the remaining parameters are dropped, -# keeping only their names. The limit sits far above any ordinary type, so only -# these outliers are truncated; the serializer's own MAX_DEPTH guard is a -# separate cycle check, not a size bound. -const MAX_EXPANDED_TYPES = 128 +# building one type: once it is spent, a type keeps only its name and a single +# `Unserializable` parameter marking the aborted expansion (displayed as `…`, +# same sentinel the cache writer uses). The limit sits far above any ordinary +# type, so only these outliers are truncated; the serializer's own MAX_DEPTH +# guard is a separate cycle check, not a size bound. +const MAX_EXPANDED_TYPES = 256 mutable struct ExpandBudget remaining::Int @@ -39,11 +40,15 @@ function FakeTypeName(@nospecialize(x), budget::ExpandBudget=ExpandBudget()) if x isa DataType xname = x.name ft = FakeTypeName(VarRef(VarRef(xname.module), xname.name), []) - if budget.remaining > 0 + if isempty(x.parameters) + # leaf type, nothing to expand + elseif budget.remaining > 0 budget.remaining -= 1 for p in x.parameters push!(ft.parameters, _parameter(p, budget)) end + else + push!(ft.parameters, Unserializable()) end ft elseif x isa Union diff --git a/shared/symbolserver/serialize.jl b/shared/symbolserver/serialize.jl index b86a254a..51389084 100644 --- a/shared/symbolserver/serialize.jl +++ b/shared/symbolserver/serialize.jl @@ -87,6 +87,9 @@ end function _write(io, x::Nothing, depth::Int) Base.write(io, NothingHeader) end +function _write(io, x::Unserializable, depth::Int) + Base.write(io, UnserializableHeader) +end function _write(io, x::Char, depth::Int) Base.write(io, CharHeader) Base.write(io, UInt32(x)) diff --git a/test/test_symbolserver.jl b/test/test_symbolserver.jl index 812731ef..a755a0bc 100644 --- a/test/test_symbolserver.jl +++ b/test/test_symbolserver.jl @@ -781,21 +781,34 @@ end end @testitem "SymbolServer: recursive type parameters are bounded" begin - using JuliaWorkspaces.SymbolServer: FakeTypeName, ExpandBudget + using JuliaWorkspaces.SymbolServer: FakeTypeName, ExpandBudget, Unserializable, CacheStore # Number of DataTypes that were expanded (i.e. kept their parameters). - expanded(x) = x isa FakeTypeName && !isempty(x.parameters) ? 1 + sum(expanded, x.parameters) : 0 + expanded(x) = x isa FakeTypeName && !isempty(x.parameters) && !(x.parameters[1] isa Unserializable) ? 1 + sum(expanded, x.parameters) : 0 + # Number of expansions aborted by the budget. + truncated(x) = x isa FakeTypeName ? count(p -> p isa Unserializable, x.parameters) + sum(truncated, x.parameters; init = 0) : 0 limit = 8 deep = foldl((T, _) -> Tuple{T}, 1:100; init = Int) # Tuple{Tuple{…{Int}}} wide = Tuple{ntuple(i -> Val{i}, 100)...} # Tuple{Val{1},…,Val{100}} - # However a type explodes — by depth or by width — expansion stops at the budget. + # However a type explodes — by depth or by width — expansion stops at the + # budget, and each cut is marked rather than silently dropped. for T in (deep, wide) - @test expanded(FakeTypeName(T, ExpandBudget(limit))) <= limit + ft = FakeTypeName(T, ExpandBudget(limit)) + @test expanded(ft) <= limit + @test truncated(ft) > 0 end # An ordinary type has far fewer nodes than the budget, so nothing is dropped. ordinary = Dict{String,Vector{Tuple{Int,Float64}}} @test FakeTypeName(ordinary, ExpandBudget(limit)) == FakeTypeName(ordinary) + @test truncated(FakeTypeName(ordinary, ExpandBudget(limit))) == 0 + + # The marker renders as `…` and roundtrips through the cache format. + ft = FakeTypeName(deep, ExpandBudget(limit)) + @test contains(string(ft), "…") + io = IOBuffer() + CacheStore.write(io, ft) + @test CacheStore.read(IOBuffer(take!(io))) == ft end