diff --git a/crates/cranelift/src/func_environ/gc/drc.rs b/crates/cranelift/src/func_environ/gc/drc.rs index fe75676554ab..5c9bc3ab85da 100644 --- a/crates/cranelift/src/func_environ/gc/drc.rs +++ b/crates/cranelift/src/func_environ/gc/drc.rs @@ -15,10 +15,6 @@ use wasmtime_environ::{ WasmValType, drc::DrcTypeLayouts, }; -// The minimum over-approximated stack roots list size for which we will trigger -// a GC. -const MIN_OVER_APPROX_STACK_ROOTS_GC_THRESHOLD: i64 = 1024; - #[derive(Default)] pub struct DrcCompiler { layouts: DrcTypeLayouts, @@ -190,9 +186,10 @@ impl DrcCompiler { ); let doubled_last_len = builder.ins().iadd(last_len, last_len); - let min_threshold = builder - .ins() - .iconst(ir::types::I32, MIN_OVER_APPROX_STACK_ROOTS_GC_THRESHOLD); + let min_threshold = builder.ins().iconst( + ir::types::I32, + i64::from(wasmtime_environ::drc::MIN_OVER_APPROX_STACK_ROOTS_GC_THRESHOLD), + ); let threshold = builder.ins().umax(doubled_last_len, min_threshold); let should_gc = diff --git a/crates/cranelift/src/lib.rs b/crates/cranelift/src/lib.rs index 50c0b26cbe58..150726f70c8b 100644 --- a/crates/cranelift/src/lib.rs +++ b/crates/cranelift/src/lib.rs @@ -49,7 +49,7 @@ mod trap; use self::compiler::Compiler; const TRAP_INTERNAL_ASSERT: TrapCode = TrapCode::unwrap_user(1); -const TRAP_GC_HEAP_CORRUPT: TrapCode = TrapCode::unwrap_user(2); +pub const TRAP_GC_HEAP_CORRUPT: TrapCode = TrapCode::unwrap_user(2); const TRAP_OFFSET: u8 = 3; pub const TRAP_CANNOT_LEAVE_COMPONENT: TrapCode = TrapCode::unwrap_user(Trap::CannotLeaveComponent as u8 + TRAP_OFFSET); diff --git a/crates/environ/src/gc.rs b/crates/environ/src/gc.rs index 5a5c9d64bf90..9e588f774f78 100644 --- a/crates/environ/src/gc.rs +++ b/crates/environ/src/gc.rs @@ -29,6 +29,19 @@ use core::alloc::Layout; /// enabled. pub const POISON: u8 = 0b00001111; +/// The bit within a `VMDrcHeader`'s reserved bits that is the mark +/// bit. Collectively, this bit in all the heap's objects' headers implements +/// the precise-stack-roots set. +pub const DRC_HEADER_MARK_BIT: u32 = 1 << 0; + +/// The bit within a `VMDrcHeader`'s reserved bits that is the +/// in-the-over-approximated-stack-roots list bit. +pub const DRC_HEADER_IN_OVER_APPROX_LIST_BIT: u32 = 1 << 1; + +/// The minimum length the over-approximated-stack-roots list must reach +/// before a read barrier considers forcing a GC. +pub const DRC_MIN_OVER_APPROX_STACK_ROOTS_GC_THRESHOLD: u32 = 1024; + /// Assert a condition, but only when `gc_zeal` is enabled. #[macro_export] macro_rules! gc_assert { diff --git a/crates/environ/src/gc/drc.rs b/crates/environ/src/gc/drc.rs index 9f557506304b..a24a21f66862 100644 --- a/crates/environ/src/gc/drc.rs +++ b/crates/environ/src/gc/drc.rs @@ -17,14 +17,9 @@ pub const EXCEPTION_TAG_INSTANCE_OFFSET: u32 = HEADER_SIZE; /// The offset of the tag-defined-index field in an exception header. pub const EXCEPTION_TAG_DEFINED_OFFSET: u32 = HEADER_SIZE + 4; -/// The bit within a `VMDrcHeader`'s reserved bits that is the mark -/// bit. Collectively, this bit in all the heap's objects' headers implements -/// the precise-stack-roots set. -pub const HEADER_MARK_BIT: u32 = 1 << 0; - -/// The bit within a `VMDrcHeader`'s reserved bits that is the -/// in-the-over-approximated-stack-roots list bit. -pub const HEADER_IN_OVER_APPROX_LIST_BIT: u32 = 1 << 1; +pub use super::DRC_HEADER_IN_OVER_APPROX_LIST_BIT as HEADER_IN_OVER_APPROX_LIST_BIT; +pub use super::DRC_HEADER_MARK_BIT as HEADER_MARK_BIT; +pub use super::DRC_MIN_OVER_APPROX_STACK_ROOTS_GC_THRESHOLD as MIN_OVER_APPROX_STACK_ROOTS_GC_THRESHOLD; /// The layout of Wasm GC objects in the deferred reference-counting collector. #[derive(Default)] diff --git a/crates/test-util/src/wast.rs b/crates/test-util/src/wast.rs index 835c41e4f2d3..a3146a77457a 100644 --- a/crates/test-util/src/wast.rs +++ b/crates/test-util/src/wast.rs @@ -562,16 +562,6 @@ impl WastTest { return true; } - // Winch supports GC types only under the barrier-free collectors; - // the deferred reference-counting collector is refused until Winch - // emits GC barriers. - if config.compiler == Compiler::Winch - && config.collector == Collector::DeferredReferenceCounting - && self.config.gc_types() - { - return true; - } - // Disable spec tests per target for proposals that Winch does not implement yet. if config.compiler == Compiler::Winch { // Common list for tests that fail in all targets supported by Winch. diff --git a/crates/wasmtime/src/config.rs b/crates/wasmtime/src/config.rs index 98090ecfc4e5..09e394af0e84 100644 --- a/crates/wasmtime/src/config.rs +++ b/crates/wasmtime/src/config.rs @@ -2458,13 +2458,6 @@ impl Config { | WasmFeatures::LEGACY_EXCEPTIONS | WasmFeatures::STACK_SWITCHING; - // Winch supports GC types only under the barrier-free - // collectors; the deferred reference-counting collector - // requires GC barriers that Winch does not emit yet. - #[cfg(feature = "gc")] - if self.collector.not_auto() == Some(Collector::DeferredReferenceCounting) { - unsupported |= WasmFeatures::GC_TYPES; - } match self.compiler_target().architecture { target_lexicon::Architecture::Aarch64(_) => { unsupported |= WasmFeatures::THREADS; diff --git a/docs/stability-tiers.md b/docs/stability-tiers.md index bf14e93398c9..b214fb741776 100644 --- a/docs/stability-tiers.md +++ b/docs/stability-tiers.md @@ -340,10 +340,11 @@ emitting Pulley bytecode. | [`exception-handling`] | ✅ | ❌ | | [`stack-switching`] | ❌ | ❌ | -[^a]: Winch supports some features of the [`reference-types`] proposal such as - the change to support multiple tables and LEB-encoding table indices in - instructions, but it does not support GC types such as `externref` or the - new table opcodes in the [`reference-types`] proposal. +[^a]: Winch supports GC reference values such as `externref` through parameters, + results, locals, globals, and calls, including the required stack maps and + collector barriers. It also supports multiple tables and LEB-encoded table + indices, but does not yet support every table and element-segment case in the + [`reference-types`] proposal. [^b]: Pulley does not support the [`threads`] proposal because there is no known safe way to implement this with Rust's memory model. [^c]: Winch's support for aarch64 is complete for Core Wasm. diff --git a/tests/all/gc.rs b/tests/all/gc.rs index ac381877f7d4..5e3154002832 100644 --- a/tests/all/gc.rs +++ b/tests/all/gc.rs @@ -3839,6 +3839,217 @@ fn winch_externref_survives_gc_in_frame() -> Result<()> { Ok(()) } +/// The write barrier's decrement chain releases an object once a global stops +/// holding the last reference to it. +#[test] +#[cfg_attr(miri, ignore)] +fn winch_drc_write_barrier_drops_old_global_value() -> Result<()> { + let mut config = Config::new(); + config.strategy(Strategy::Winch); + config.collector(Collector::DeferredReferenceCounting); + let Ok(engine) = Engine::new(&config) else { + return Ok(()); + }; + let module = Module::new( + &engine, + r#" + (module + (global $g (mut externref) (ref.null extern)) + (func (export "set") (param externref) + (global.set $g (local.get 0)))) + "#, + )?; + let mut store = Store::new(&engine, ()); + let instance = Instance::new(&mut store, &module, &[])?; + let set = instance.get_func(&mut store, "set").unwrap(); + + let dropped = Arc::new(AtomicBool::new(false)); + { + let mut scope = RootScope::new(&mut store); + let r = ExternRef::new(&mut scope, SetFlagOnDrop(dropped.clone()))?; + set.call(&mut scope, &[Val::ExternRef(Some(r))], &mut [])?; + } + + // The global holds the only reference; nothing may be dropped yet. + store.gc(None)?; + assert!(!dropped.load(SeqCst)); + + // Overwriting the global decrements the count to zero and releases the + // old value. + set.call(&mut store, &[Val::ExternRef(None)], &mut [])?; + store.gc(None)?; + assert!(dropped.load(SeqCst)); + + Ok(()) +} + +/// The read barrier holds a count for references entering the stack, so +/// overwriting their last long-lived home cannot free them out from under +/// the frame that loaded them. +#[test] +#[cfg_attr(miri, ignore)] +fn winch_drc_read_barrier_keeps_loaded_ref_alive() -> Result<()> { + let mut config = Config::new(); + config.strategy(Strategy::Winch); + config.collector(Collector::DeferredReferenceCounting); + let Ok(engine) = Engine::new(&config) else { + return Ok(()); + }; + let module = Module::new( + &engine, + r#" + (module + (import "" "gc" (func $gc)) + (global $g (mut externref) (ref.null extern)) + (func (export "set") (param externref) + (global.set $g (local.get 0))) + (func (export "swap") (result externref) + (local $tmp externref) + (local.set $tmp (global.get $g)) + (global.set $g (ref.null extern)) + (call $gc) + (local.get $tmp))) + "#, + )?; + let mut store = Store::new(&engine, ()); + let gc = Func::wrap(&mut store, |mut cx: Caller<'_, ()>| { + let _ = cx.gc(None); + }); + let instance = Instance::new(&mut store, &module, &[gc.into()])?; + let set = instance.get_func(&mut store, "set").unwrap(); + let swap = instance.get_typed_func::<(), Option>>(&mut store, "swap")?; + + { + let mut scope = RootScope::new(&mut store); + let r = ExternRef::new(&mut scope, 0xDECAFu32)?; + set.call(&mut scope, &[Val::ExternRef(Some(r))], &mut [])?; + } + + // Settle the deferred unroot so the global truly holds the last count. + store.gc(None)?; + + // `swap` loads the reference onto the stack, overwrites the global, and + // collects while the stack copy is live. + let out = swap.call(&mut store, ())?.expect("must not be null"); + let got = out + .data(&store)? + .and_then(|d| d.downcast_ref::().copied()); + assert_eq!(got, Some(0xDECAF)); + + Ok(()) +} + +/// An `externref` may wrap an unboxed i31 created through the host API. Such +/// values do not have reference counts and must bypass both DRC barriers. +#[test] +#[cfg_attr(miri, ignore)] +fn winch_drc_i31_wrapped_as_externref_skips_global_barriers() -> Result<()> { + let mut config = Config::new(); + config.strategy(Strategy::Winch); + config.collector(Collector::DeferredReferenceCounting); + let Ok(engine) = Engine::new(&config) else { + return Ok(()); + }; + let module = Module::new( + &engine, + r#" + (module + (global $g (mut externref) (ref.null extern)) + (func (export "set") (param externref) + local.get 0 + global.set $g) + (func (export "get") (result externref) + global.get $g)) + "#, + )?; + let mut store = Store::new(&engine, ()); + let instance = Instance::new(&mut store, &module, &[])?; + let set = instance.get_func(&mut store, "set").unwrap(); + let get = instance.get_func(&mut store, "get").unwrap(); + + let anyref = AnyRef::from_i31(&mut store, I31::wrapping_u32(0x1234)); + let externref = ExternRef::convert_any(&mut store, anyref)?; + set.call(&mut store, &[Val::ExternRef(Some(externref))], &mut [])?; + + let mut results = [Val::null_extern_ref()]; + get.call(&mut store, &[], &mut results)?; + let externref = results[0] + .unwrap_externref() + .expect("global.get returned null"); + let anyref = AnyRef::convert_extern(&mut store, *externref)?; + assert_eq!(anyref.unwrap_i31(&store)?.get_u32(), 0x1234); + + // Replacing the i31-backed externref exercises the old-value side of the + // write barrier as well. + set.call(&mut store, &[Val::null_extern_ref()], &mut [])?; + Ok(()) +} + +/// Growing the over-approximated-stack-roots list to its threshold forces a +/// collection from the read barrier and preserves the reference whose load +/// triggered that collection. +#[test] +#[cfg_attr(miri, ignore)] +fn winch_drc_read_barrier_forces_gc_at_threshold() -> Result<()> { + let mut config = Config::new(); + config.strategy(Strategy::Winch); + config.collector(Collector::DeferredReferenceCounting); + let Ok(engine) = Engine::new(&config) else { + return Ok(()); + }; + + let num_refs = wasmtime_environ::DRC_MIN_OVER_APPROX_STACK_ROOTS_GC_THRESHOLD as usize; + let mut wat = "(module\n".to_string(); + for i in 0..num_refs { + wat.push_str(&format!( + r#"(global $g{i} (export "g{i}") (mut externref) (ref.null extern)) +"#, + )); + } + wat.push_str("(func (export \"drain\")\n"); + for i in 0..num_refs { + wat.push_str(&format!( + "(drop (global.get $g{i}))\n(global.set $g{i} (ref.null extern))\n" + )); + } + wat.push_str("))"); + + let module = Module::new(&engine, &wat)?; + let mut store = Store::new(&engine, ()); + let instance = Instance::new(&mut store, &module, &[])?; + let dropped = Arc::new(AtomicUsize::new(0)); + + { + let mut scope = RootScope::new(&mut store); + for i in 0..num_refs { + let gc_ref = ExternRef::new(&mut scope, CountDrops(dropped.clone()))?; + let global = instance + .get_global(&mut scope, &format!("g{i}")) + .expect("global must be exported"); + global.set(&mut scope, Val::ExternRef(Some(gc_ref)))?; + } + } + + // Settle the host roots. Each global is now the only long-lived home for + // its reference. + store.gc(None)?; + assert_eq!(dropped.load(SeqCst), 0); + + let drain = instance.get_typed_func::<(), ()>(&mut store, "drain")?; + drain.call(&mut store, ())?; + + // The final global.get reaches the threshold and forces a collection. + // The preceding references are no longer on the stack or in globals, but + // the triggering reference is still live in the global.get result slot. + assert_eq!(dropped.load(SeqCst), num_refs - 1); + + // The triggering reference becomes collectible after `drain` returns. + store.gc(None)?; + assert_eq!(dropped.load(SeqCst), num_refs); + + Ok(()) +} + /// Reference values crossing the ABI boundary in every position: stack-passed /// externref params and multi-value externref results (more than fit in registers) #[test] diff --git a/tests/all/winch_engine_features.rs b/tests/all/winch_engine_features.rs index 966588eb07ac..fe252842a14a 100644 --- a/tests/all/winch_engine_features.rs +++ b/tests/all/winch_engine_features.rs @@ -64,41 +64,3 @@ fn ensure_compatibility_between_winch_and_debug_native(config: &mut Config) -> R Ok(()) } - -#[wasmtime_test(strategies(only(Winch)))] -#[cfg_attr(miri, ignore)] -fn ensure_compatibility_between_winch_and_drc_collector(config: &mut Config) -> Result<()> { - config.collector(Collector::DeferredReferenceCounting); - config.gc_support(true); - let result = Engine::new(&config); - match result { - Ok(_) => { - wasmtime::bail!( - "Expected incompatibility between the deferred reference-counting \ - collector and Winch" - ) - } - Err(e) => { - assert_eq!( - e.to_string(), - "the wasm_gc_types feature is not supported on this compiler configuration" - ); - } - } - - Ok(()) -} - -#[wasmtime_test(strategies(only(Winch)))] -#[cfg_attr(miri, ignore)] -fn winch_with_drc_collector_disables_gc_types_by_default(config: &mut Config) -> Result<()> { - config.collector(Collector::DeferredReferenceCounting); - let engine = Engine::new(&config)?; - let result = Module::new( - &engine, - r#"(module (global (mut externref) (ref.null extern)))"#, - ); - assert!(result.is_err()); - - Ok(()) -} diff --git a/tests/disas/winch/aarch64/global-get-externref-drc.wat b/tests/disas/winch/aarch64/global-get-externref-drc.wat new file mode 100644 index 000000000000..39e2d11cc041 --- /dev/null +++ b/tests/disas/winch/aarch64/global-get-externref-drc.wat @@ -0,0 +1,94 @@ +;;! target = "aarch64" +;;! test = "winch" +;;! flags = "-Ccollector=drc" + +(module + (global $g (mut externref) (ref.null extern)) + (func (export "get") (result externref) + (global.get $g))) +;; wasm[0]::function[0]: +;; stp x29, x30, [sp, #-0x10]! +;; mov x29, sp +;; str x28, [sp, #-0x10]! +;; mov x28, sp +;; ldur x16, [x0, #8] +;; ldur x16, [x16, #0x18] +;; mov x17, #0 +;; movk x17, #0x20 +;; add x16, x16, x17 +;; cmp sp, x16 +;; b.lo #0x148 +;; 2c: mov x9, x0 +;; sub x28, x28, #0x10 +;; mov sp, x28 +;; stur x0, [x28, #8] +;; stur x1, [x28] +;; ldur w0, [x9, #0x30] +;; sub x28, x28, #4 +;; mov sp, x28 +;; stur w0, [x28] +;; ldur w0, [x28] +;; tst w0, w0 +;; b.eq #0x124 +;; b #0x60 +;; 60: mov w1, w0 +;; and w1, w1, #1 +;; tst w1, w1 +;; b.ne #0x124 +;; b #0x74 +;; 74: ldur x1, [x9, #8] +;; ldur x2, [x1, #0x28] +;; ldur x1, [x1, #0x20] +;; mov x3, x0 +;; add x3, x3, #0x14 +;; cmp x3, x2, uxtx +;; sub sp, x28, #0xc +;; b.hi #0x14c +;; 94: mov sp, x28 +;; mov x2, x1 +;; add x2, x2, x0, uxtx +;; ldur w3, [x2] +;; and w3, w3, #2 +;; tst w3, w3 +;; b.ne #0x124 +;; b #0xb4 +;; b4: ldur x4, [x9, #0x20] +;; ldur w3, [x4] +;; stur w3, [x2, #0x10] +;; ldur w3, [x2] +;; orr w3, w3, #2 +;; stur w3, [x2] +;; ldur x3, [x2, #8] +;; add x3, x3, #1 +;; stur x3, [x2, #8] +;; stur w0, [x4] +;; ldur w3, [x4, #4] +;; add w3, w3, #1 +;; stur w3, [x4, #4] +;; ldur w2, [x4, #8] +;; add w2, w2, w2, uxtx +;; cmp w3, w2, uxtx +;; b.lo #0x124 +;; b #0xfc +;; fc: cmp w3, #0x400 +;; b.lo #0x124 +;; b #0x108 +;; 108: sub x28, x28, #0xc +;; mov sp, x28 +;; mov x0, x9 +;; bl #0x29c +;; 118: add x28, x28, #0xc +;; ╰─╼ stack_map: frame_size=48, frame_offsets=[12] +;; mov sp, x28 +;; ldur x9, [x28, #0xc] +;; ldur w0, [x28] +;; add x28, x28, #4 +;; mov sp, x28 +;; add x28, x28, #0x10 +;; mov sp, x28 +;; mov sp, x28 +;; ldr x28, [sp], #0x10 +;; ldp x29, x30, [sp], #0x10 +;; ret +;; 148: udf #0xc11f +;; 14c: udf #0xc11f diff --git a/tests/disas/winch/aarch64/global-set-externref-drc.wat b/tests/disas/winch/aarch64/global-set-externref-drc.wat new file mode 100644 index 000000000000..e69f8fa4cf91 --- /dev/null +++ b/tests/disas/winch/aarch64/global-set-externref-drc.wat @@ -0,0 +1,103 @@ +;;! target = "aarch64" +;;! test = "winch" +;;! flags = "-Ccollector=drc" + +(module + (global $g (mut externref) (ref.null extern)) + (func (param externref) + (global.set $g (local.get 0)))) +;; wasm[0]::function[0]: +;; stp x29, x30, [sp, #-0x10]! +;; mov x29, sp +;; str x28, [sp, #-0x10]! +;; mov x28, sp +;; ldur x16, [x0, #8] +;; ldur x16, [x16, #0x18] +;; mov x17, #0 +;; movk x17, #0x20 +;; add x16, x16, x17 +;; cmp sp, x16 +;; b.lo #0x168 +;; 2c: mov x9, x0 +;; sub x28, x28, #0x18 +;; mov sp, x28 +;; stur x0, [x28, #0x10] +;; stur x1, [x28, #8] +;; stur w2, [x28, #4] +;; ldur w16, [x28, #4] +;; sub x28, x28, #4 +;; mov sp, x28 +;; stur w16, [x28] +;; ldur w0, [x28] +;; add x28, x28, #4 +;; mov sp, x28 +;; ldur x1, [x9, #8] +;; ldur x2, [x1, #0x28] +;; ldur x1, [x1, #0x20] +;; ldur w3, [x9, #0x30] +;; tst w0, w0 +;; b.eq #0xbc +;; b #0x7c +;; 7c: mov w4, w0 +;; and w4, w4, #1 +;; tst w4, w4 +;; b.ne #0xbc +;; b #0x90 +;; 90: mov x4, x0 +;; add x4, x4, #0x10 +;; cmp x4, x2, uxtx +;; sub sp, x28, #8 +;; b.hi #0x16c +;; a4: mov sp, x28 +;; mov x4, x1 +;; add x4, x4, x0, uxtx +;; ldur x5, [x4, #8] +;; add x5, x5, #1 +;; stur x5, [x4, #8] +;; stur w0, [x9, #0x30] +;; tst w3, w3 +;; b.eq #0x150 +;; b #0xcc +;; cc: mov w4, w3 +;; and w4, w4, #1 +;; tst w4, w4 +;; b.ne #0x150 +;; b #0xe0 +;; e0: mov x4, x3 +;; add x4, x4, #0x10 +;; cmp x4, x2, uxtx +;; sub sp, x28, #8 +;; b.hi #0x170 +;; f4: mov sp, x28 +;; mov x4, x1 +;; add x4, x4, x3, uxtx +;; ldur x5, [x4, #8] +;; sub x5, x5, #1 +;; cmp x5, #0 +;; b.eq #0x11c +;; b #0x114 +;; 114: stur x5, [x4, #8] +;; b #0x150 +;; 11c: sub x28, x28, #4 +;; mov sp, x28 +;; stur w3, [x28] +;; sub x28, x28, #4 +;; mov sp, x28 +;; mov x0, x9 +;; ldur w1, [x28, #4] +;; bl #0x1ec +;; 13c: add x28, x28, #4 +;; ╰─╼ stack_map: frame_size=48, frame_offsets=[12] +;; mov sp, x28 +;; add x28, x28, #4 +;; mov sp, x28 +;; ldur x9, [x28, #0x10] +;; add x28, x28, #0x18 +;; mov sp, x28 +;; mov sp, x28 +;; ldr x28, [sp], #0x10 +;; ldp x29, x30, [sp], #0x10 +;; ret +;; 168: udf #0xc11f +;; 16c: udf #0xc11f +;; 170: udf #0xc11f diff --git a/tests/disas/winch/x64/global-get-externref-drc.wat b/tests/disas/winch/x64/global-get-externref-drc.wat new file mode 100644 index 000000000000..63f6f4093420 --- /dev/null +++ b/tests/disas/winch/x64/global-get-externref-drc.wat @@ -0,0 +1,75 @@ +;;! target = "x86_64" +;;! test = "winch" +;;! flags = "-Ccollector=drc" + +(module + (global $g (mut externref) (ref.null extern)) + (func (export "get") (result externref) + (global.get $g))) +;; wasm[0]::function[0]: +;; pushq %rbp +;; movq %rsp, %rbp +;; movq 8(%rdi), %r11 +;; movq 0x18(%r11), %r11 +;; addq $0x20, %r11 +;; cmpq %rsp, %r11 +;; ja 0x104 +;; 1c: movq %rdi, %r14 +;; subq $0x10, %rsp +;; movq %rdi, 8(%rsp) +;; movq %rsi, (%rsp) +;; movl 0x30(%r14), %eax +;; subq $4, %rsp +;; movl %eax, (%rsp) +;; movl (%rsp), %eax +;; testl %eax, %eax +;; je 0xf1 +;; 48: movl %eax, %ecx +;; andl $1, %ecx +;; testl %ecx, %ecx +;; jne 0xf1 +;; 58: movq 8(%r14), %rcx +;; movq 0x28(%rcx), %rdx +;; movq 0x20(%rcx), %rcx +;; movq %rax, %rbx +;; addq $0x14, %rbx +;; cmpq %rdx, %rbx +;; ja 0x106 +;; 77: movq %rcx, %rdx +;; addq %rax, %rdx +;; movl (%rdx), %ebx +;; andl $2, %ebx +;; testl %ebx, %ebx +;; jne 0xf1 +;; 8d: movq 0x20(%r14), %rsi +;; movl (%rsi), %ebx +;; movl %ebx, 0x10(%rdx) +;; movl (%rdx), %ebx +;; orl $2, %ebx +;; movl %ebx, (%rdx) +;; movq 8(%rdx), %rbx +;; addq $1, %rbx +;; movq %rbx, 8(%rdx) +;; movl %eax, (%rsi) +;; movl 4(%rsi), %ebx +;; addl $1, %ebx +;; movl %ebx, 4(%rsi) +;; movl 8(%rsi), %edx +;; addl %edx, %edx +;; cmpl %edx, %ebx +;; jb 0xf1 +;; ca: cmpl $0x400, %ebx +;; jb 0xf1 +;; d6: subq $0xc, %rsp +;; movq %r14, %rdi +;; callq 0x20c +;; addq $0xc, %rsp +;; ╰─╼ stack_map: frame_size=32, frame_offsets=[12] +;; movq 0xc(%rsp), %r14 +;; movl (%rsp), %eax +;; addq $4, %rsp +;; addq $0x10, %rsp +;; popq %rbp +;; retq +;; 104: ud2 +;; 106: ud2 diff --git a/tests/disas/winch/x64/global-set-externref-drc.wat b/tests/disas/winch/x64/global-set-externref-drc.wat new file mode 100644 index 000000000000..1426b1270a8d --- /dev/null +++ b/tests/disas/winch/x64/global-set-externref-drc.wat @@ -0,0 +1,80 @@ +;;! target = "x86_64" +;;! test = "winch" +;;! flags = "-Ccollector=drc" + +(module + (global $g (mut externref) (ref.null extern)) + (func (param externref) + (global.set $g (local.get 0)))) +;; wasm[0]::function[0]: +;; pushq %rbp +;; movq %rsp, %rbp +;; movq 8(%rdi), %r11 +;; movq 0x18(%r11), %r11 +;; addq $0x30, %r11 +;; cmpq %rsp, %r11 +;; ja 0x12a +;; 1c: movq %rdi, %r14 +;; subq $0x20, %rsp +;; movq %rdi, 0x18(%rsp) +;; movq %rsi, 0x10(%rsp) +;; movl %edx, 0xc(%rsp) +;; movl 0xc(%rsp), %r11d +;; subq $4, %rsp +;; movl %r11d, (%rsp) +;; movl (%rsp), %eax +;; addq $4, %rsp +;; movq 8(%r14), %rcx +;; movq 0x28(%rcx), %rdx +;; movq 0x20(%rcx), %rcx +;; movl 0x30(%r14), %ebx +;; testl %eax, %eax +;; je 0x9e +;; 66: movl %eax, %esi +;; andl $1, %esi +;; testl %esi, %esi +;; jne 0x9e +;; 76: movq %rax, %rsi +;; addq $0x10, %rsi +;; cmpq %rdx, %rsi +;; ja 0x12c +;; 89: movq %rcx, %rsi +;; addq %rax, %rsi +;; movq 8(%rsi), %rdi +;; addq $1, %rdi +;; movq %rdi, 8(%rsi) +;; movl %eax, 0x30(%r14) +;; testl %ebx, %ebx +;; je 0x121 +;; aa: movl %ebx, %esi +;; andl $1, %esi +;; testl %esi, %esi +;; jne 0x121 +;; ba: movq %rbx, %rsi +;; addq $0x10, %rsi +;; cmpq %rdx, %rsi +;; ja 0x12e +;; cd: movq %rcx, %rsi +;; addq %rbx, %rsi +;; movq 8(%rsi), %rdi +;; subq $1, %rdi +;; cmpq $0, %rdi +;; je 0xf1 +;; e8: movq %rdi, 8(%rsi) +;; jmp 0x121 +;; f1: subq $4, %rsp +;; movl %ebx, (%rsp) +;; subq $0xc, %rsp +;; movq %r14, %rdi +;; movl 0xc(%rsp), %esi +;; callq 0x18d +;; addq $0xc, %rsp +;; ╰─╼ stack_map: frame_size=48, frame_offsets=[28] +;; addq $4, %rsp +;; movq 0x18(%rsp), %r14 +;; addq $0x20, %rsp +;; popq %rbp +;; retq +;; 12a: ud2 +;; 12c: ud2 +;; 12e: ud2 diff --git a/tests/misc_testsuite/winch/gc-refs.wast b/tests/misc_testsuite/winch/gc-refs.wast index 4f242b02239e..d9ba79a13a97 100644 --- a/tests/misc_testsuite/winch/gc-refs.wast +++ b/tests/misc_testsuite/winch/gc-refs.wast @@ -2,8 +2,8 @@ ;;! gc_types = true ;; Winch reference-value support: externref flows through params, results, -;; locals, and globals, and survives calls (stack maps). Runs under the -;; barrier-free collectors; the DRC collector is refused at config time until Winch emits barriers. +;; locals, and globals, and survives calls (stack maps). Global reads and +;; writes use barriers when running under the DRC collector. (module (global $g (mut externref) (ref.null extern)) diff --git a/winch/codegen/src/codegen/mod.rs b/winch/codegen/src/codegen/mod.rs index 0df13d76d950..ecfd2b5c5574 100644 --- a/winch/codegen/src/codegen/mod.rs +++ b/winch/codegen/src/codegen/mod.rs @@ -490,6 +490,14 @@ where } } + /// Whether a GC barrier must be emitted when writing or reading a + /// reference of the given type through a collector-visible location. + pub fn gc_barrier_needed(&self, ty: &WasmValType) -> bool { + ty.is_vmgcref_type_and_not_i31() + && self.tunables.collector + == Some(wasmtime_environ::Collector::DeferredReferenceCounting) + } + /// Emits a a series of instructions that will type check a function reference call. pub fn emit_typecheck_funcref( &mut self, diff --git a/winch/codegen/src/visitor.rs b/winch/codegen/src/visitor.rs index 791f7354eea0..02f183c65190 100644 --- a/winch/codegen/src/visitor.rs +++ b/winch/codegen/src/visitor.rs @@ -26,10 +26,11 @@ use wasmparser::{ BlockType, BrTable, HeapType, Ieee32, Ieee64, MemArg, V128, ValType, VisitOperator, VisitSimdOperator, }; -use wasmtime_cranelift::TRAP_INDIRECT_CALL_TO_NULL; +use wasmtime_cranelift::{TRAP_GC_HEAP_CORRUPT, TRAP_INDIRECT_CALL_TO_NULL}; use wasmtime_environ::{ - DataIndex, ElemIndex, FuncIndex, GlobalIndex, MemoryIndex, TableIndex, TypeIndex, WasmHeapType, - WasmValType, + DRC_HEADER_IN_OVER_APPROX_LIST_BIT, DRC_MIN_OVER_APPROX_STACK_ROOTS_GC_THRESHOLD, DataIndex, + ElemIndex, FuncIndex, GlobalIndex, I31_DISCRIMINANT, MemoryIndex, PtrSize, TableIndex, + TypeIndex, WasmHeapType, WasmValType, }; /// A macro to define unsupported WebAssembly operators. @@ -2066,11 +2067,278 @@ where let index = GlobalIndex::from_u32(global_index); let (ty, base, offset) = self.emit_get_global_addr(index)?; let addr = self.masm.address_at_reg(base, offset)?; - let dst = self.context.reg_for_type(ty, self.masm)?; - self.masm.load(addr, writable!(dst), ty.try_into()?)?; - self.context.stack.push(Val::reg(dst, ty)); + let gc_ref = self.context.reg_for_type(ty, self.masm)?; + if self.gc_barrier_needed(&ty) { + self.masm.load(addr, writable!(gc_ref), ty.try_into()?)?; + self.context.stack.push(Val::reg(gc_ref, ty)); + self.context.spill(self.masm)?; + let slot = self + .context + .stack + .peek() + .ok_or_else(|| CodeGenError::missing_values_in_stack())? + .unwrap_mem() + .slot; + + let ref_reg = self.context.any_gpr(self.masm)?; + self.masm.load( + self.masm.address_from_sp(slot.offset)?, + writable!(ref_reg), + OperandSize::S32, + )?; + + let store_context_offset = self.env.vmoffsets.ptr.vmctx_store_context(); + let gc_heap_base_offset = self.env.vmoffsets.ptr.vm_store_context().gc_heap_base(); + let gc_heap_len_offset = self + .env + .vmoffsets + .ptr + .vm_store_context() + .gc_heap_current_length(); + let heap_reg = self.context.any_gpr(self.masm)?; + let skip_barrier = self.masm.get_label()?; + self.masm.branch( + IntCmpKind::Eq, + ref_reg, + ref_reg.into(), + skip_barrier, + OperandSize::S32, + )?; + self.masm + .mov(writable!(heap_reg), ref_reg.into(), OperandSize::S32)?; + self.masm.and( + writable!(heap_reg), + heap_reg, + RegImm::i32(I31_DISCRIMINANT as i32), + OperandSize::S32, + )?; + self.masm.branch( + IntCmpKind::Ne, + heap_reg, + heap_reg.into(), + skip_barrier, + OperandSize::S32, + )?; + + self.masm.load_ptr( + self.masm + .address_at_vmctx(u32::from(store_context_offset))?, + writable!(heap_reg), + )?; + let fold_reg = self.context.any_gpr(self.masm)?; + self.masm.load( + self.masm + .address_at_reg(heap_reg, u32::from(gc_heap_len_offset))?, + writable!(fold_reg), + OperandSize::S64, + )?; + self.masm.load_ptr( + self.masm + .address_at_reg(heap_reg, u32::from(gc_heap_base_offset))?, + writable!(heap_reg), + )?; + + let header_extent = i64::from( + self.env + .vmoffsets + .vm_drc_header_next_over_approximated_stack_root(), + ) + 4; + let extent_reg = self.context.any_gpr(self.masm)?; + self.masm + .mov(writable!(extent_reg), ref_reg.into(), OperandSize::S64)?; + self.masm.add( + writable!(extent_reg), + extent_reg, + RegImm::i64(header_extent), + OperandSize::S64, + )?; + self.masm + .cmp(extent_reg, fold_reg.into(), OperandSize::S64)?; + self.masm.trapif(IntCmpKind::GtU, TRAP_GC_HEAP_CORRUPT)?; + self.context.free_reg(extent_reg); + + self.masm + .mov(writable!(fold_reg), heap_reg.into(), OperandSize::S64)?; + self.masm.add( + writable!(fold_reg), + fold_reg, + ref_reg.into(), + OperandSize::S64, + )?; + + let reserved_offset = self.env.vmoffsets.vm_gc_header_reserved_bits(); + let bits_reg = self.context.any_gpr(self.masm)?; + self.masm.load( + self.masm.address_at_reg(fold_reg, reserved_offset)?, + writable!(bits_reg), + OperandSize::S32, + )?; + self.masm.and( + writable!(bits_reg), + bits_reg, + RegImm::i32(DRC_HEADER_IN_OVER_APPROX_LIST_BIT as i32), + OperandSize::S32, + )?; + self.masm.branch( + IntCmpKind::Ne, + bits_reg, + bits_reg.into(), + skip_barrier, + OperandSize::S32, + )?; + + let heap_data_offset = self.env.vmoffsets.ptr.vmctx_gc_heap_data(); + let roots_head_offset = u32::from( + self.env + .vmoffsets + .ptr + .vmdrc_heap_data_over_approximated_stack_roots(), + ); + let roots_len_offset = u32::from( + self.env + .vmoffsets + .ptr + .vmdrc_heap_data_current_over_approximated_stack_roots_len(), + ); + let next_offset = self + .env + .vmoffsets + .vm_drc_header_next_over_approximated_stack_root(); + let ref_count_offset = self.env.vmoffsets.vm_drc_header_ref_count(); + + let heap_data_reg = self.context.any_gpr(self.masm)?; + self.masm.load_ptr( + self.masm.address_at_vmctx(u32::from(heap_data_offset))?, + writable!(heap_data_reg), + )?; + + self.masm.load( + self.masm.address_at_reg(heap_data_reg, roots_head_offset)?, + writable!(bits_reg), + OperandSize::S32, + )?; + self.masm.store( + bits_reg.into(), + self.masm.address_at_reg(fold_reg, next_offset)?, + OperandSize::S32, + )?; + + self.masm.load( + self.masm.address_at_reg(fold_reg, reserved_offset)?, + writable!(bits_reg), + OperandSize::S32, + )?; + self.masm.or( + writable!(bits_reg), + bits_reg, + RegImm::i32(DRC_HEADER_IN_OVER_APPROX_LIST_BIT as i32), + OperandSize::S32, + )?; + self.masm.store( + bits_reg.into(), + self.masm.address_at_reg(fold_reg, reserved_offset)?, + OperandSize::S32, + )?; + + self.masm.load( + self.masm.address_at_reg(fold_reg, ref_count_offset)?, + writable!(bits_reg), + OperandSize::S64, + )?; + self.masm.add( + writable!(bits_reg), + bits_reg, + RegImm::i64(1), + OperandSize::S64, + )?; + self.masm.store( + bits_reg.into(), + self.masm.address_at_reg(fold_reg, ref_count_offset)?, + OperandSize::S64, + )?; - self.context.free_reg(base); + self.masm.store( + ref_reg.into(), + self.masm.address_at_reg(heap_data_reg, roots_head_offset)?, + OperandSize::S32, + )?; + + self.masm.load( + self.masm.address_at_reg(heap_data_reg, roots_len_offset)?, + writable!(bits_reg), + OperandSize::S32, + )?; + self.masm.add( + writable!(bits_reg), + bits_reg, + RegImm::i32(1), + OperandSize::S32, + )?; + self.masm.store( + bits_reg.into(), + self.masm.address_at_reg(heap_data_reg, roots_len_offset)?, + OperandSize::S32, + )?; + + let last_len_offset = u32::from( + self.env + .vmoffsets + .ptr + .vmdrc_heap_data_over_approximated_stack_roots_len_after_last_gc(), + ); + self.masm.load( + self.masm.address_at_reg(heap_data_reg, last_len_offset)?, + writable!(fold_reg), + OperandSize::S32, + )?; + self.masm.add( + writable!(fold_reg), + fold_reg, + fold_reg.into(), + OperandSize::S32, + )?; + + let skip_gc = self.masm.get_label()?; + self.masm.branch( + IntCmpKind::LtU, + bits_reg, + fold_reg.into(), + skip_gc, + OperandSize::S32, + )?; + self.masm.branch( + IntCmpKind::LtU, + bits_reg, + RegImm::i32(DRC_MIN_OVER_APPROX_STACK_ROOTS_GC_THRESHOLD as i32), + skip_gc, + OperandSize::S32, + )?; + + self.context.free_reg(heap_data_reg); + self.context.free_reg(ref_reg); + self.context.free_reg(bits_reg); + self.context.free_reg(fold_reg); + self.context.free_reg(heap_reg); + + let force_gc = self.env.builtins.force_gc::()?; + FnCall::emit::( + &mut self.env, + self.masm, + &mut self.context, + Callee::Builtin(force_gc), + )?; + self.context.pop_and_free(self.masm)?; + + self.masm.bind(skip_gc)?; + self.masm.bind(skip_barrier)?; + + self.context.free_reg(base); + } else { + self.masm.load(addr, writable!(gc_ref), ty.try_into()?)?; + self.context.stack.push(Val::reg(gc_ref, ty)); + + self.context.free_reg(base); + } Ok(()) } @@ -2080,11 +2348,212 @@ where let (ty, base, offset) = self.emit_get_global_addr(index)?; let addr = self.masm.address_at_reg(base, offset)?; - let typed_reg = self.context.pop_to_reg(self.masm, None)?; - self.masm - .store(typed_reg.reg.into(), addr, ty.try_into()?)?; - self.context.free_reg(typed_reg.reg); - self.context.free_reg(base); + if self.gc_barrier_needed(&ty) { + self.context.spill(self.masm)?; + let typed_reg = self.context.pop_to_reg(self.masm, None)?; + let store_context_offset = self.env.vmoffsets.ptr.vmctx_store_context(); + let gc_heap_base_offset = self.env.vmoffsets.ptr.vm_store_context().gc_heap_base(); + let gc_heap_len_offset = self + .env + .vmoffsets + .ptr + .vm_store_context() + .gc_heap_current_length(); + let heap_reg = self.context.any_gpr(self.masm)?; + let bound_reg = self.context.any_gpr(self.masm)?; + self.masm.load_ptr( + self.masm + .address_at_vmctx(u32::from(store_context_offset))?, + writable!(heap_reg), + )?; + self.masm.load( + self.masm + .address_at_reg(heap_reg, u32::from(gc_heap_len_offset))?, + writable!(bound_reg), + OperandSize::S64, + )?; + self.masm.load_ptr( + self.masm + .address_at_reg(heap_reg, u32::from(gc_heap_base_offset))?, + writable!(heap_reg), + )?; + + let old_reg = self.context.any_gpr(self.masm)?; + self.masm.load(addr, writable!(old_reg), OperandSize::S32)?; + + let ref_count_offset = self.env.vmoffsets.vm_drc_header_ref_count(); + let count_addr_reg = self.context.any_gpr(self.masm)?; + let count_reg = self.context.any_gpr(self.masm)?; + let header_extent = i64::from(ref_count_offset) + 8; + + let skip_inc = self.masm.get_label()?; + self.masm.branch( + IntCmpKind::Eq, + typed_reg.reg, + typed_reg.reg.into(), + skip_inc, + OperandSize::S32, + )?; + self.masm.mov( + writable!(count_addr_reg), + typed_reg.reg.into(), + OperandSize::S32, + )?; + self.masm.and( + writable!(count_addr_reg), + count_addr_reg, + RegImm::i32(I31_DISCRIMINANT as i32), + OperandSize::S32, + )?; + self.masm.branch( + IntCmpKind::Ne, + count_addr_reg, + count_addr_reg.into(), + skip_inc, + OperandSize::S32, + )?; + + self.masm.mov( + writable!(count_addr_reg), + typed_reg.reg.into(), + OperandSize::S64, + )?; + self.masm.add( + writable!(count_addr_reg), + count_addr_reg, + RegImm::i64(header_extent), + OperandSize::S64, + )?; + self.masm + .cmp(count_addr_reg, bound_reg.into(), OperandSize::S64)?; + self.masm.trapif(IntCmpKind::GtU, TRAP_GC_HEAP_CORRUPT)?; + + self.masm + .mov(writable!(count_addr_reg), heap_reg.into(), OperandSize::S64)?; + self.masm.add( + writable!(count_addr_reg), + count_addr_reg, + typed_reg.reg.into(), + OperandSize::S64, + )?; + self.masm.load( + self.masm.address_at_reg(count_addr_reg, ref_count_offset)?, + writable!(count_reg), + OperandSize::S64, + )?; + self.masm.add( + writable!(count_reg), + count_reg, + RegImm::i64(1), + OperandSize::S64, + )?; + self.masm.store( + count_reg.into(), + self.masm.address_at_reg(count_addr_reg, ref_count_offset)?, + OperandSize::S64, + )?; + + self.masm.bind(skip_inc)?; + + self.masm + .store(typed_reg.reg.into(), addr, ty.try_into()?)?; + + let skip_dec = self.masm.get_label()?; + self.masm.branch( + IntCmpKind::Eq, + old_reg, + old_reg.into(), + skip_dec, + OperandSize::S32, + )?; + self.masm + .mov(writable!(count_addr_reg), old_reg.into(), OperandSize::S32)?; + self.masm.and( + writable!(count_addr_reg), + count_addr_reg, + RegImm::i32(I31_DISCRIMINANT as i32), + OperandSize::S32, + )?; + self.masm.branch( + IntCmpKind::Ne, + count_addr_reg, + count_addr_reg.into(), + skip_dec, + OperandSize::S32, + )?; + + self.masm + .mov(writable!(count_addr_reg), old_reg.into(), OperandSize::S64)?; + self.masm.add( + writable!(count_addr_reg), + count_addr_reg, + RegImm::i64(header_extent), + OperandSize::S64, + )?; + self.masm + .cmp(count_addr_reg, bound_reg.into(), OperandSize::S64)?; + self.masm.trapif(IntCmpKind::GtU, TRAP_GC_HEAP_CORRUPT)?; + + self.masm + .mov(writable!(count_addr_reg), heap_reg.into(), OperandSize::S64)?; + self.masm.add( + writable!(count_addr_reg), + count_addr_reg, + old_reg.into(), + OperandSize::S64, + )?; + self.masm.load( + self.masm.address_at_reg(count_addr_reg, ref_count_offset)?, + writable!(count_reg), + OperandSize::S64, + )?; + self.masm.sub( + writable!(count_reg), + count_reg, + RegImm::i64(1), + OperandSize::S64, + )?; + + let drop_old = self.masm.get_label()?; + self.masm.branch( + IntCmpKind::Eq, + count_reg, + RegImm::i64(0), + drop_old, + OperandSize::S64, + )?; + self.masm.store( + count_reg.into(), + self.masm.address_at_reg(count_addr_reg, ref_count_offset)?, + OperandSize::S64, + )?; + self.masm.jmp(skip_dec)?; + + self.context.free_reg(count_reg); + self.context.free_reg(count_addr_reg); + self.context.free_reg(bound_reg); + self.context.free_reg(heap_reg); + self.context.free_reg(typed_reg.reg); + self.context.free_reg(base); + + self.masm.bind(drop_old)?; + let drop_gc_ref = self.env.builtins.drop_gc_ref::()?; + self.context.stack.push(TypedReg::i32(old_reg).into()); + FnCall::emit::( + &mut self.env, + self.masm, + &mut self.context, + Callee::Builtin(drop_gc_ref), + )?; + + self.masm.bind(skip_dec)?; + } else { + let typed_reg = self.context.pop_to_reg(self.masm, None)?; + self.masm + .store(typed_reg.reg.into(), addr, ty.try_into()?)?; + self.context.free_reg(typed_reg.reg); + self.context.free_reg(base); + } Ok(()) }