From 10e2dbee671f73e4e258995538f3c2a96a13769a Mon Sep 17 00:00:00 2001 From: Alex Crichton Date: Mon, 27 Jul 2026 12:17:30 -0700 Subject: [PATCH] Remove preemption points in bulk operations This commit updates the translation of bulk operations such as `memory.grow` which were recently refactored to not have preemption points within the operation itself. Preemption points within the operation, while useful for very large operations, expose internal and intermediate state to embedders and the rest of the runtime. For example tables that are grown are initially filled with null, which may not be valid for the table's type. These bulk operations didn't recompute pointers/indices after a possible preemption meaning if memories were grown/moved then it would cause faults. In general this is seen as too risky of an operation to perform. The fix in this commit is to move all preemption checks to the start of the operation itself. This means that bulk operations continue to be metered with a cost proportional to the size of the operation for fuel, and they all contain an initial epoch check for epochs. Once the operation is committed to, however, there's no cancelling it and it'll continue to run. In practice this means that extremely large copies, for example, can blow the epoch budget. To re-add preemption checks within the operation, however, will require very careful reintroduction to avoid these sorts of problems/faults. --- crates/cranelift/src/func_environ.rs | 575 +++++---------------- crates/cranelift/src/func_environ/gc.rs | 22 +- crates/wasmtime/src/config.rs | 25 +- crates/wasmtime/src/runtime/store.rs | 4 + tests/all/epoch_interruption.rs | 127 +++++ tests/all/fuel.rs | 82 +++ tests/all/gc.rs | 104 ++++ tests/disas/gc/array-copy-with-fuel.wat | 211 ++++---- tests/disas/memory-copy-epochs.wat | 124 ++--- tests/disas/memory-copy-fuel-const-len.wat | 184 +++++++ tests/disas/memory-copy-fuel.wat | 130 ++--- 11 files changed, 824 insertions(+), 764 deletions(-) create mode 100644 tests/disas/memory-copy-fuel-const-len.wat diff --git a/crates/cranelift/src/func_environ.rs b/crates/cranelift/src/func_environ.rs index 10d20e63b170..64e9642bf584 100644 --- a/crates/cranelift/src/func_environ.rs +++ b/crates/cranelift/src/func_environ.rs @@ -663,85 +663,6 @@ impl<'module_environment> FuncEnvironment<'module_environment> { builder.switch_to_block(continuation_block); } - /// Manually insert a fuel check, as opposed to what already happens around - /// normal loops headers and function entries. - /// - /// This can be used for expensive opcodes, such as `array.copy`, where the - /// operation's runtime is a function of the runtime state. - fn manual_fuel_check(&mut self, builder: &mut FunctionBuilder<'_>, fuel_to_consume: ir::Value) { - self.fuel_increment_var(builder); - - let fuel = builder.use_var(self.fuel_var); - let fuel = builder.ins().iadd(fuel, fuel_to_consume); - builder.def_var(self.fuel_var, fuel); - - self.fuel_check(builder); - } - - /// Consumes `units * cost_per_unit` fuel, saturating the charge at - /// `i64::MAX` so that an oversized unsigned operand cannot wrap around and - /// add fuel instead. - fn consume_variable_fuel( - &mut self, - builder: &mut FunctionBuilder<'_>, - units: ir::Value, - cost_per_unit: u8, - ) { - let may_exceed_i64_max = match builder.func.dfg.value_type(units) { - I32 => false, - I64 => true, - ty => unreachable!("unsupported variable fuel unit type: {ty}"), - }; - self.consume_variable_fuel_impl(builder, units, cost_per_unit, may_exceed_i64_max); - } - - /// Like [`Self::consume_variable_fuel`], but for a unit count whose product - /// with `cost_per_unit` is statically known to fit in an `i64`. - fn consume_bounded_variable_fuel( - &mut self, - builder: &mut FunctionBuilder<'_>, - units: ir::Value, - cost_per_unit: u8, - ) { - self.consume_variable_fuel_impl(builder, units, cost_per_unit, false); - } - - fn consume_variable_fuel_impl( - &mut self, - builder: &mut FunctionBuilder<'_>, - units: ir::Value, - cost_per_unit: u8, - may_exceed_i64_max: bool, - ) { - if !self.tunables.consume_fuel || cost_per_unit == 0 { - return; - } - - let units = match builder.func.dfg.value_type(units) { - I32 => builder.ins().uextend(I64, units), - I64 => units, - ty => unreachable!("unsupported variable fuel unit type: {ty}"), - }; - let fuel = if cost_per_unit == 1 { - units - } else { - builder.ins().imul_imm_s(units, i64::from(cost_per_unit)) - }; - let fuel = if may_exceed_i64_max { - let max = builder.ins().iconst(I64, i64::MAX); - let max_units = builder - .ins() - .iconst(I64, i64::MAX / i64::from(cost_per_unit)); - let saturate = builder - .ins() - .icmp(IntCC::UnsignedGreaterThan, units, max_units); - builder.ins().select(saturate, max, fuel) - } else { - fuel - }; - self.manual_fuel_check(builder, fuel); - } - fn epoch_function_entry(&mut self, builder: &mut FunctionBuilder<'_>) { debug_assert!(self.epoch_deadline_var.is_reserved_value()); self.epoch_deadline_var = builder.declare_var(ir::types::I64); @@ -2684,17 +2605,19 @@ impl FuncEnvironment<'_> { delta: ir::Value, init_value: ir::Value, ) -> WasmResult { + let cost = self + .tunables + .operator_cost + .variable() + .table_grow_per_element; + self.pre_translate_bulk_op(builder, delta, cost)?; + let mut pos = builder.cursor(); let table = self.table(table_index); let (table_vmctx, defined_table_index) = self.table_vmctx_and_defined_index(&mut pos, table_index); let index_type = table.idx_type; let delta64 = self.cast_index_to_i64(&mut pos, delta, index_type); - let cost = self - .tunables - .operator_cost - .variable() - .table_grow_per_element; // Call out to the host to perform the actual growth of the underlying // table. This will initialize table slots as all null. Afterwards the @@ -2740,7 +2663,6 @@ impl FuncEnvironment<'_> { // A failed attempt performs no initialization loop, but still charge // for the requested growth so repeated failures are not free. builder.switch_to_block(failed_block); - self.consume_variable_fuel(builder, delta, cost); builder.ins().jump(done_block, &[]); builder.switch_to_block(fill_block); @@ -2748,12 +2670,11 @@ impl FuncEnvironment<'_> { builder, CheckedEntity::Table { table: table_index, - initialized: true, + initialized: false, }, result_idx, init_value, delta, - cost, )?; builder.ins().jump(done_block, &[]); @@ -2907,6 +2828,7 @@ impl FuncEnvironment<'_> { .operator_cost .variable() .table_fill_per_element; + self.pre_translate_bulk_op(builder, len, cost)?; self.translate_entity_fill( builder, CheckedEntity::Table { @@ -2916,7 +2838,6 @@ impl FuncEnvironment<'_> { dst, val, len, - cost, ) } @@ -3053,8 +2974,7 @@ impl FuncEnvironment<'_> { elem: ir::Value, len: ir::Value, ) -> WasmResult { - let cost = self.tunables.operator_cost.variable().array_new_per_element; - gc::translate_array_new(self, builder, array_type_index, elem, len, cost) + gc::translate_array_new(self, builder, array_type_index, elem, len) } pub fn translate_array_new_default( @@ -3063,12 +2983,7 @@ impl FuncEnvironment<'_> { array_type_index: TypeIndex, len: ir::Value, ) -> WasmResult { - let cost = self - .tunables - .operator_cost - .variable() - .array_new_default_per_element; - gc::translate_array_new_default(self, builder, array_type_index, len, cost) + gc::translate_array_new_default(self, builder, array_type_index, len) } pub fn translate_array_new_fixed( @@ -3151,6 +3066,7 @@ impl FuncEnvironment<'_> { .operator_cost .variable() .array_copy_per_element; + self.pre_translate_bulk_op(builder, len, cost)?; self.translate_entity_copy( builder, CheckedEntity::Array { @@ -3166,7 +3082,6 @@ impl FuncEnvironment<'_> { dst_index, src_index, len, - cost, ) } @@ -3185,6 +3100,7 @@ impl FuncEnvironment<'_> { .operator_cost .variable() .array_fill_per_element; + self.pre_translate_bulk_op(builder, len, cost)?; self.translate_entity_fill( builder, CheckedEntity::Array { @@ -3195,7 +3111,6 @@ impl FuncEnvironment<'_> { index, value, len, - cost, ) } @@ -3216,6 +3131,7 @@ impl FuncEnvironment<'_> { .operator_cost .variable() .array_init_data_per_element; + self.pre_translate_bulk_op(builder, len, cost)?; self.translate_entity_copy( builder, CheckedEntity::Array { @@ -3230,7 +3146,6 @@ impl FuncEnvironment<'_> { dst_index, data_offset, len, - cost, ) } @@ -3250,6 +3165,7 @@ impl FuncEnvironment<'_> { .operator_cost .variable() .array_init_elem_per_element; + self.pre_translate_bulk_op(builder, len, cost)?; self.translate_entity_copy( builder, CheckedEntity::Array { @@ -3261,7 +3177,6 @@ impl FuncEnvironment<'_> { dst_index, elem_offset, len, - cost, ) } @@ -3634,7 +3549,7 @@ impl FuncEnvironment<'_> { let index_type = self.memory(index).idx_type; let cost = self.tunables.operator_cost.variable().memory_grow_per_page; - self.consume_variable_fuel(builder, val, cost); + self.pre_translate_bulk_op(builder, val, cost)?; let mut pos = builder.cursor(); let val = self.cast_index_to_i64(&mut pos, val, index_type); let call_inst = pos @@ -3733,7 +3648,8 @@ impl FuncEnvironment<'_> { len: ir::Value, ) -> WasmResult<()> { let cost = self.tunables.operator_cost.variable().memory_copy_per_byte; - self.translate_entity_copy(builder, dst_index, src_index, dst, src, len, cost) + self.pre_translate_bulk_op(builder, len, cost)?; + self.translate_entity_copy(builder, dst_index, src_index, dst, src, len) } /// Perform a raw bulk-memory-like libcall. @@ -3741,7 +3657,11 @@ impl FuncEnvironment<'_> { /// The main purpose of this helper is to handle situations when fuel and /// epochs are enabled to break up the copy into a loop of chunks with /// preemption checks between them. - fn raw_bulk_memory_operation(&mut self, builder: &mut FunctionBuilder<'_>, mut op: BulkOp) { + fn raw_bulk_memory_operation( + &mut self, + builder: &mut FunctionBuilder<'_>, + op: BulkOp, + ) -> WasmResult<()> { // Fast path: a copy whose byte length is a small compile-time constant is // expanded inline (see `emit_inline_memcpy`), skipping the libcall's fixed // per-call cost (a wasm/host transition and an indirect call) that @@ -3758,259 +3678,33 @@ impl FuncEnvironment<'_> { const_len: Some(bytes), src_entity, dst_entity, - fuel, .. } = op { if bytes <= INLINE_COPY_MAX_BYTES { - if self.tunables.consume_fuel { - let units = bytes / u64::from(fuel.bytes_per_unit); - self.fuel_consumed += - i64::try_from(units * u64::from(fuel.cost_per_unit)).unwrap(); - } let src_region = self.bulk_copy_alias_region(builder.func, src_entity); let dst_region = self.bulk_copy_alias_region(builder.func, dst_entity); self.emit_inline_memcpy(builder, dst, src, bytes, src_region, dst_region); - return; + return Ok(()); } } - // Very scientifically chosen. Or, more seriously, this is just an - // arbitrary number for now. 100k copies of this size locally takes half - // a second, so seems like a reasonably large chunk size to not hit perf - // too much by chunking but also enable time slicing. - const UNINTERRUPTABLE_CHUNK_SIZE: i64 = 128 << 20; - let mut pos = builder.cursor(); let vmctx = self.vmctx_val(&mut pos); - let pointer_type = self.pointer_type(); // Performs a raw call to the actual libcall, as dictated by the - // provided `op`. This inserts configured epoch/fuel checks before the - // call. - let raw_call = - |env: &mut FuncEnvironment<'_>, builder: &mut FunctionBuilder<'_>, op: &BulkOp| { - if env.tunables.epoch_interruption { - env.epoch_check(builder); - } - let fuel = op.fuel(); - if env.tunables.consume_fuel && fuel.cost_per_unit != 0 { - let byte_len = op.len(); - debug_assert!(fuel.bytes_per_unit.is_power_of_two()); - let units = if fuel.bytes_per_unit == 1 { - byte_len - } else { - builder - .ins() - .ushr_imm_u(byte_len, i64::from(fuel.bytes_per_unit.trailing_zeros())) - }; - // With fuel enabled all calls emitted below are limited to - // `UNINTERRUPTABLE_CHUNK_SIZE`, so this multiplication - // cannot exceed `i64::MAX` even at the maximum `u8` rate. - env.consume_bounded_variable_fuel(builder, units, fuel.cost_per_unit); - } - match *op { - BulkOp::MemoryCopy { dst, src, len, .. } => { - let memory_copy = env.builtin_functions.memory_copy(&mut builder.func); - builder.ins().call(memory_copy, &[vmctx, dst, src, len]); - } - BulkOp::MemoryFill { dst, val, len, .. } => { - let memory_fill = env.builtin_functions.memory_fill(&mut builder.func); - builder.ins().call(memory_fill, &[vmctx, dst, val, len]); - } - } - }; - - // If epochs and fuel are disabled, then just call the libcall and - // return. No need for the loops below. - if !self.tunables.epoch_interruption && !self.tunables.consume_fuel { - raw_call(self, builder, &op); - return; - } - - // If fuel is enabled, first take all the pending fuel and flush it to - // our internal variable. This is necessary to avoid picking up all - // pending fuel on each turn of the loop below. - if self.tunables.consume_fuel { - self.fuel_increment_var(builder); - } - - let current_block = builder.current_block().unwrap(); - let chunk_block = builder.create_block(); - let last_chunk_block = builder.create_block(); - - builder.ensure_inserted_block(); - builder.insert_block_after(chunk_block, current_block); - builder.insert_block_after(last_chunk_block, chunk_block); - - let chunk = builder - .ins() - .iconst(pointer_type, UNINTERRUPTABLE_CHUNK_SIZE); - - // For `memcpy` when chunking this up we might need to do a backwards - // copy or a forwards copy. Determine that here and jump to the - // backwards copy if needed. - let backwards_block = if let BulkOp::MemoryCopy { dst, src, .. } = op { - let forwards = builder.ins().icmp(IntCC::UnsignedGreaterThan, src, dst); - let forwards_block = builder.create_block(); - let backwards_block = builder.create_block(); - builder - .ins() - .brif(forwards, forwards_block, &[], backwards_block, &[]); - builder.switch_to_block(forwards_block); - builder.seal_block(forwards_block); - Some((backwards_block, op.clone())) - } else { - None - }; - - // Helper closure to test if the length in `op` is larger than `chunk`, - // and if so do a single chunk. Else this goes to the final block with - // the final operation. - let has_chunk_branch = - |builder: &mut FunctionBuilder<'_>, op: &_, chunk_block, last_chunk_block| { - let len = match *op { - BulkOp::MemoryCopy { len, .. } | BulkOp::MemoryFill { len, .. } => len, - }; - let has_chunk = builder.ins().icmp(IntCC::UnsignedGreaterThan, len, chunk); - match *op { - BulkOp::MemoryCopy { dst, src, len, .. } => { - builder.ins().brif( - has_chunk, - chunk_block, - &[dst.into(), src.into(), len.into()], - last_chunk_block, - &[dst.into(), src.into(), len.into()], - ); - } - BulkOp::MemoryFill { dst, len, .. } => { - builder.ins().brif( - has_chunk, - chunk_block, - &[dst.into(), len.into()], - last_chunk_block, - &[dst.into(), len.into()], - ); - } - } - }; - - let append_block_params = |builder: &mut FunctionBuilder<'_>, block, op: &mut _| match op { - BulkOp::MemoryCopy { dst, src, len, .. } => { - *dst = builder.append_block_param(block, pointer_type); - *src = builder.append_block_param(block, pointer_type); - *len = builder.append_block_param(block, pointer_type); - } - BulkOp::MemoryFill { dst, len, .. } => { - *dst = builder.append_block_param(block, pointer_type); - *len = builder.append_block_param(block, pointer_type); - } - }; - - // Forwards copy: dispatch to the per-chunk loop or the final iteration - // if there's no chunks. - has_chunk_branch(builder, &op, chunk_block, last_chunk_block); - - // Forwards copy: In the block with per-chunk copies, each operation - // performs `chunk` length of bytes and then decrements the current - // length by `chunk`. Afterwards a condition tests if we do another - // chunk or break out for the final chunk. - builder.switch_to_block(chunk_block); - append_block_params(builder, chunk_block, &mut op); - let op_len = match &mut op { - BulkOp::MemoryCopy { len, .. } | BulkOp::MemoryFill { len, .. } => len, - }; - let remaining_len = *op_len; - *op_len = chunk; - raw_call(self, builder, &op); - match &mut op { + // provided `op`. + match op { BulkOp::MemoryCopy { dst, src, len, .. } => { - *dst = builder.ins().iadd(*dst, chunk); - *src = builder.ins().iadd(*src, chunk); - *len = builder.ins().isub(remaining_len, chunk); + let memory_copy = self.builtin_functions.memory_copy(&mut builder.func); + builder.ins().call(memory_copy, &[vmctx, dst, src, len]); } - BulkOp::MemoryFill { len, dst, .. } => { - *dst = builder.ins().iadd(*dst, chunk); - *len = builder.ins().isub(remaining_len, chunk); + BulkOp::MemoryFill { dst, val, len } => { + let memory_fill = self.builtin_functions.memory_fill(&mut builder.func); + builder.ins().call(memory_fill, &[vmctx, dst, val, len]); } - }; - has_chunk_branch(builder, &op, chunk_block, last_chunk_block); - builder.seal_block(chunk_block); - - // Backwards copy: similar to the above but with adjustments on where - // increments/decrements happen. Notably: - // - // * Initial src/end are the final byte address - // * Each chunk starts out by decrementing src/end as opposed to above - // where the increment happens at the end. - // * The final block performs the final decrement before jumping to the - // shared `last_chunk_block` between the forwards/backwards paths. - if let Some((backwards_block, mut op)) = backwards_block { - // Setup `dst=dst+len` and `src=src+len`, then see if we have a - // chunk. - builder.switch_to_block(backwards_block); - builder.seal_block(backwards_block); - let backwards_chunk_block = builder.create_block(); - let backwards_last_chunk_block = builder.create_block(); - let BulkOp::MemoryCopy { dst, src, len, .. } = &mut op else { - unreachable!() - }; - *dst = builder.ins().iadd(*dst, *len); - *src = builder.ins().iadd(*src, *len); - has_chunk_branch( - builder, - &op, - backwards_chunk_block, - backwards_last_chunk_block, - ); - - // Execute the per-chunk backwards copy, adjusting pointers before - // the copy itself. - builder.switch_to_block(backwards_chunk_block); - append_block_params(builder, backwards_chunk_block, &mut op); - let BulkOp::MemoryCopy { dst, src, len, .. } = &mut op else { - unreachable!() - }; - let remaining_len = *len; - *len = chunk; - *dst = builder.ins().isub(*dst, chunk); - *src = builder.ins().isub(*src, chunk); - raw_call(self, builder, &op); - let BulkOp::MemoryCopy { len, .. } = &mut op else { - unreachable!() - }; - *len = builder.ins().isub(remaining_len, chunk); - has_chunk_branch( - builder, - &op, - backwards_chunk_block, - backwards_last_chunk_block, - ); - builder.seal_block(backwards_chunk_block); - - // Final backwards chunk: adjust the dst/src to be their true base - // pointers and then delegate to `last_chunk_block` for the actual - // memcpy. - builder.switch_to_block(backwards_last_chunk_block); - builder.seal_block(backwards_last_chunk_block); - append_block_params(builder, backwards_last_chunk_block, &mut op); - let BulkOp::MemoryCopy { dst, src, len, .. } = &mut op else { - unreachable!() - }; - *dst = builder.ins().isub(*dst, *len); - *src = builder.ins().isub(*src, *len); - builder.ins().jump( - last_chunk_block, - &[(*dst).into(), (*src).into(), (*len).into()], - ); } - - // In the final block we know that the length of the operation is less - // than `chunk`. - builder.switch_to_block(last_chunk_block); - builder.seal_block(last_chunk_block); - append_block_params(builder, last_chunk_block, &mut op); - raw_call(self, builder, &op); + Ok(()) } /// Emits a generic "fill" of `entity` from `dst` for `len` elements, @@ -4021,6 +3715,9 @@ impl FuncEnvironment<'_> { /// `dst` and `len` values must be typed appropriately for `entity`. This /// will perform a bounds-check before actually executing the operation and /// then afterwards will perform the operation. + /// + /// Callers must invoke `pre_translate_bulk_op` before calling this method + /// to properly account for fuel/epoch checks for this bulk operation. fn translate_entity_fill( &mut self, builder: &mut FunctionBuilder<'_>, @@ -4028,7 +3725,6 @@ impl FuncEnvironment<'_> { dst: ir::Value, val: ir::Value, len: ir::Value, - cost_per_unit: u8, ) -> WasmResult<()> { let entity = entity.into(); let idx_type = entity.index_type(self); @@ -4042,37 +3738,22 @@ impl FuncEnvironment<'_> { self.unchecked_cast_wasm_addr_to_native_addr(&mut builder.cursor(), len, idx_type); match entity { - CheckedEntity::Memory(_) => { - self.raw_bulk_memory_operation( - builder, - BulkOp::MemoryFill { - dst: raw_dst_addr, - val, - len: len_ptr, - fuel: BulkFuel { - cost_per_unit, - bytes_per_unit: 1, - }, - }, - ); - } - CheckedEntity::Table { .. } | CheckedEntity::Array { .. } => { - self.emit_raw_array_or_table_fill( - builder, - entity, - raw_dst_addr, + CheckedEntity::Memory(_) => self.raw_bulk_memory_operation( + builder, + BulkOp::MemoryFill { + dst: raw_dst_addr, val, - len_ptr, - cost_per_unit, - )?; + len: len_ptr, + }, + ), + CheckedEntity::Table { .. } | CheckedEntity::Array { .. } => { + self.emit_raw_array_or_table_fill(builder, entity, raw_dst_addr, val, len_ptr) } // Not allowed to be written to in wasm. CheckedEntity::Data { .. } | CheckedEntity::Elem(_) | CheckedEntity::RuntimeData(_) => { unreachable!() } } - - Ok(()) } /// Performs a manual element-by-element fill of `entity`, starting at @@ -4096,7 +3777,6 @@ impl FuncEnvironment<'_> { dst_elem_addr: ir::Value, value: ir::Value, copy_len: ir::Value, - cost_per_element: u8, ) -> WasmResult<()> { let pointer_ty = self.pointer_type(); @@ -4116,19 +3796,14 @@ impl FuncEnvironment<'_> { if entity.allows_memset(self) && let Some(value) = self.fill_value_as_memset(builder, elem_ty, value) { - self.raw_bulk_memory_operation( + return self.raw_bulk_memory_operation( builder, BulkOp::MemoryFill { dst: dst_elem_addr, val: value, len: copy_byte_len, - fuel: BulkFuel { - cost_per_unit: cost_per_element, - bytes_per_unit: u8::try_from(elem_size).unwrap(), - }, }, ); - return Ok(()); } // Funcref values are intern'd when stored on the GC heap, and the @@ -4169,12 +3844,6 @@ impl FuncEnvironment<'_> { builder.insert_block_after(loop_block, current_block); builder.insert_block_after(continue_block, loop_block); - // Before entering the loop below flush our fuel counters to ensure - // that previous instructions' fuel isn't counted once-per-iteration. - if self.tunables.consume_fuel { - self.fuel_increment_var(builder); - } - // Current block: test to see if this is actually an empty copy. If it // is then skip over the entire loop, otherwise enter the loop and // perform the first ieration. @@ -4192,11 +3861,6 @@ impl FuncEnvironment<'_> { // by the element size, then see if we turn again or exit. builder.switch_to_block(loop_block); let elem_addr = builder.append_block_param(loop_block, pointer_ty); - // Consume the configured cost for this element before writing it. - if self.tunables.consume_fuel { - self.fuel_consumed += i64::from(cost_per_element); - } - self.translate_loop_header(builder)?; match entity { CheckedEntity::Table { table, initialized } => { assert!(!is_pre_interned_funcref); @@ -4312,7 +3976,8 @@ impl FuncEnvironment<'_> { len: ir::Value, ) -> WasmResult<()> { let cost = self.tunables.operator_cost.variable().memory_fill_per_byte; - self.translate_entity_fill(builder, memory_index, dst, val, len, cost) + self.pre_translate_bulk_op(builder, len, cost)?; + self.translate_entity_fill(builder, memory_index, dst, val, len) } pub fn translate_memory_init( @@ -4326,6 +3991,7 @@ impl FuncEnvironment<'_> { ) -> WasmResult<()> { let seg_index = DataIndex::from_u32(seg_index); let cost = self.tunables.operator_cost.variable().memory_init_per_byte; + self.pre_translate_bulk_op(builder, len, cost)?; self.translate_entity_copy( builder, memory_index, @@ -4336,7 +4002,6 @@ impl FuncEnvironment<'_> { dst, src, len, - cost, ) } @@ -4379,6 +4044,9 @@ impl FuncEnvironment<'_> { /// of the copy. Both `dst` and `src` have types appropriate to index their /// respective entities, and `len` has a type that's the smaller of the two /// index types. + /// + /// Callers must invoke `pre_translate_bulk_op` before calling this method + /// to properly account for fuel/epoch checks for this bulk operation. fn translate_entity_copy( &mut self, builder: &mut FunctionBuilder<'_>, @@ -4387,7 +4055,6 @@ impl FuncEnvironment<'_> { dst: ir::Value, src: ir::Value, len: ir::Value, - cost_per_unit: u8, ) -> WasmResult<()> { let dst_entity = dst_entity.into(); let src_entity = src_entity.into(); @@ -4449,13 +4116,8 @@ impl FuncEnvironment<'_> { const_len: const_count, src_entity, dst_entity, - fuel: BulkFuel { - cost_per_unit, - bytes_per_unit: 1, - }, }, - ); - Ok(()) + ) } // Tables/arrays are sometimes a memcpy, sometimes a per-element @@ -4470,7 +4132,6 @@ impl FuncEnvironment<'_> { len_ptr, src, const_count, - cost_per_unit, ), // Cannot copy into a data or element segment in wasm. @@ -4686,6 +4347,7 @@ impl FuncEnvironment<'_> { .operator_cost .variable() .table_copy_per_element; + self.pre_translate_bulk_op(builder, len, cost)?; self.translate_entity_copy( builder, CheckedEntity::Table { @@ -4699,7 +4361,6 @@ impl FuncEnvironment<'_> { dst, src, len, - cost, ) } @@ -4729,7 +4390,6 @@ impl FuncEnvironment<'_> { copy_len: ir::Value, src_index: ir::Value, const_count: Option, - cost_per_element: u8, ) -> WasmResult<()> { let pointer_type = self.pointer_type(); assert_eq!(builder.func.dfg.value_type(dst_elem_addr), pointer_type); @@ -4808,7 +4468,7 @@ impl FuncEnvironment<'_> { // parameters (or expand it inline; see `raw_bulk_memory_operation`). if !type_forbids_memcpy && dst_element_size == src_element_size { let const_len = const_count.and_then(|c| c.checked_mul(u64::from(dst_element_size))); - self.raw_bulk_memory_operation( + return self.raw_bulk_memory_operation( builder, BulkOp::MemoryCopy { dst: dst_elem_addr, @@ -4817,13 +4477,8 @@ impl FuncEnvironment<'_> { const_len, src_entity, dst_entity, - fuel: BulkFuel { - cost_per_unit: cost_per_element, - bytes_per_unit: u8::try_from(dst_element_size).unwrap(), - }, }, ); - return Ok(()); } // For other copies, this is a per-element loop. Use the helper to @@ -4837,7 +4492,6 @@ impl FuncEnvironment<'_> { src_elem_addr, copy_len, src_index, - cost_per_element, &|this, builder, dst, src, src_index| { let write_ty = dst_entity.storage_type(this); let val = match src_entity { @@ -5044,7 +4698,6 @@ impl FuncEnvironment<'_> { src_elem_addr: ir::Value, copy_len: ir::Value, src_index: ir::Value, - cost_per_element: u8, copy_one: &dyn Fn( &mut Self, &mut FunctionBuilder<'_>, @@ -5096,13 +4749,6 @@ impl FuncEnvironment<'_> { builder.insert_block_after(backwards_block, forward_block); builder.insert_block_after(done_block, backwards_block); - // Update our local fuel counter, if enabled, before entering the loops - // below. This zeros out `self.fuel_consumed` so we don't consume - // previous fuel on each iteration of the loop. - if self.tunables.consume_fuel { - self.fuel_increment_var(builder); - } - // Terminate `current_block` by testing to see if we're copying any // elements at all. builder @@ -5192,11 +4838,6 @@ impl FuncEnvironment<'_> { let src_cur = builder.append_block_param(forward_block, self.pointer_type()); let src_index = builder.append_block_param(forward_block, src_index_ty); let forward_keepalives = append_keepalive_params(builder, forward_block); - // Consume the configured cost for this element before copying it. - if self.tunables.consume_fuel { - self.fuel_consumed += i64::from(cost_per_element); - } - self.translate_loop_header(builder)?; copy_one(self, builder, dst_cur, src_cur, src_index)?; let dst_next = builder .ins() @@ -5222,10 +4863,6 @@ impl FuncEnvironment<'_> { let src_cur = builder.append_block_param(backwards_block, self.pointer_type()); let src_index = builder.append_block_param(backwards_block, src_index_ty); let backward_keepalives = append_keepalive_params(builder, backwards_block); - if self.tunables.consume_fuel { - self.fuel_consumed += i64::from(cost_per_element); - } - self.translate_loop_header(builder)?; let dst_cur = { let size = builder .ins() @@ -5281,6 +4918,7 @@ impl FuncEnvironment<'_> { .operator_cost .variable() .table_init_per_element; + self.pre_translate_bulk_op(builder, len, cost)?; self.translate_entity_copy( builder, CheckedEntity::Table { @@ -5291,7 +4929,6 @@ impl FuncEnvironment<'_> { dst, src, len, - cost, ) } @@ -5355,6 +4992,78 @@ impl FuncEnvironment<'_> { Ok(builder.ins().ireduce(ir::types::I32, ret)) } + /// Translation prefix before bulk operations such as `memory.copy`. + /// + /// Takes a dynamic value `units` for the size of the operation as well as + /// a `cost_per_unit` configured for this operation. If fuel is enabled + /// this fuel will be consumed, and if epochs are enabled then an epoch + /// check happens. If neither epochs nor fuel are enabled this is a noop. + fn pre_translate_bulk_op( + &mut self, + builder: &mut FunctionBuilder, + units: ir::Value, + cost_per_unit: u8, + ) -> WasmResult<()> { + let const_units = + Self::value_as_const_int(builder, units).map(|c| i64::try_from(c).unwrap_or(i64::MAX)); + + if self.tunables.consume_fuel && cost_per_unit > 0 { + match const_units { + // Fold constant costs directly into internal state. + Some(units) => { + self.fuel_consumed = self + .fuel_consumed + .saturating_add(units.saturating_mul(i64::from(cost_per_unit))) + } + + None => { + // Note that fuel is always a 64-bit counter. + // + // Also note that the cost is clamped to `i64::MAX` to + // prevent fuel counter overflows since `cost` is otherwise + // an untrusted value. + let units_clamped64 = match builder.func.dfg.value_type(units) { + ir::types::I32 => { + let units64 = builder.ins().uextend(ir::types::I64, units); + builder.ins().imul_imm_u(units64, i64::from(cost_per_unit)) + } + ir::types::I64 => { + let fuel = builder.ins().imul_imm_u(units, i64::from(cost_per_unit)); + let max = builder.ins().iconst(ir::types::I64, i64::MAX); + let max_units = builder + .ins() + .iconst(I64, i64::MAX / i64::from(cost_per_unit)); + let saturate = + builder + .ins() + .icmp(IntCC::UnsignedGreaterThan, units, max_units); + builder.ins().select(saturate, max, fuel) + } + _ => unreachable!(), + }; + self.fuel_increment_var(builder); + let fuel = builder.use_var(self.fuel_var); + let fuel = builder.ins().iadd(fuel, units_clamped64); + builder.def_var(self.fuel_var, fuel); + } + } + } + + // Skip explicit fuel/epoch checks for operations which are + // subjectively, and statically, considered cheap. + const SMALL_BULK_OP_COST: i64 = 128; + if let Some(units) = const_units + && let Some(cost) = units.checked_mul(i64::from(cost_per_unit)) + && cost <= SMALL_BULK_OP_COST + { + return Ok(()); + } + + // This isn't a loop header but for fuel/epoch purposes it's the same + // thing. + self.translate_loop_header(builder) + } + pub fn translate_loop_header(&mut self, builder: &mut FunctionBuilder) -> WasmResult<()> { // Additionally if enabled check how much fuel we have remaining to see // if we've run out by this point. @@ -6233,6 +5942,7 @@ impl FuncEnvironment<'_> { .operator_cost .variable() .table_fill_per_element; + self.pre_translate_bulk_op(builder, len, cost)?; self.translate_entity_fill( builder, CheckedEntity::Table { @@ -6242,7 +5952,6 @@ impl FuncEnvironment<'_> { dst, val, len, - cost, ) } @@ -6278,7 +5987,7 @@ impl FuncEnvironment<'_> { .operator_cost .variable() .table_init_per_element; - self.consume_variable_fuel(builder, segment_len, cost); + self.pre_translate_bulk_op(builder, segment_len, cost)?; // Re-use the `table.set` translation for making this a simple function // to define. That re-executes the bounds check which is a bit @@ -6351,7 +6060,8 @@ impl FuncEnvironment<'_> { let len = self.load_runtime_data_length(builder, data); let start = builder.ins().iconst(I32, 0); let cost = self.tunables.operator_cost.variable().memory_init_per_byte; - self.translate_entity_copy(builder, memory, data, offset, start, len, cost)?; + self.pre_translate_bulk_op(builder, len, cost)?; + self.translate_entity_copy(builder, memory, data, offset, start, len)?; // Finalize control-flow for the `MemorySegmentOffset::Static` case // above. @@ -6521,7 +6231,6 @@ enum BulkOp { const_len: Option, src_entity: CheckedEntity, dst_entity: CheckedEntity, - fuel: BulkFuel, }, /// A `memory.fill` operation, setting all bytes of `dst` to `val`. @@ -6534,31 +6243,9 @@ enum BulkOp { dst: ir::Value, val: ir::Value, len: ir::Value, - fuel: BulkFuel, }, } -impl BulkOp { - fn len(&self) -> ir::Value { - match self { - BulkOp::MemoryCopy { len, .. } | BulkOp::MemoryFill { len, .. } => *len, - } - } - - fn fuel(&self) -> BulkFuel { - match self { - BulkOp::MemoryCopy { fuel, .. } | BulkOp::MemoryFill { fuel, .. } => *fuel, - } - } -} - -#[derive(Copy, Clone)] -struct BulkFuel { - cost_per_unit: u8, - /// Number of copied bytes represented by one billable unit. - bytes_per_unit: u8, -} - /// A list of entities which can participate in various kinds of bulk operations /// in wasm. /// diff --git a/crates/cranelift/src/func_environ/gc.rs b/crates/cranelift/src/func_environ/gc.rs index 183b0d1d1cd4..ac8275f027a9 100644 --- a/crates/cranelift/src/func_environ/gc.rs +++ b/crates/cranelift/src/func_environ/gc.rs @@ -801,9 +801,15 @@ pub fn translate_array_new( array_type_index: TypeIndex, elem: ir::Value, len: ir::Value, - cost_per_element: u8, ) -> WasmResult { log::trace!("translate_array_new({array_type_index:?}, {elem:?}, {len:?})"); + let cost = func_env + .tunables + .operator_cost + .variable() + .array_new_per_element; + func_env.pre_translate_bulk_op(builder, len, cost)?; + let result = gc_compiler(func_env)?.alloc_uninit_array(func_env, builder, array_type_index, len)?; let zero = builder.ins().iconst(ir::types::I32, 0); @@ -818,7 +824,6 @@ pub fn translate_array_new( zero, elem, len, - cost_per_element, )?; log::trace!("translate_array_new(..) -> {result:?}"); Ok(result) @@ -829,9 +834,14 @@ pub fn translate_array_new_default( builder: &mut FunctionBuilder, array_type_index: TypeIndex, len: ir::Value, - cost_per_element: u8, ) -> WasmResult { log::trace!("translate_array_new_default({array_type_index:?}, {len:?})"); + let cost = func_env + .tunables + .operator_cost + .variable() + .array_new_default_per_element; + func_env.pre_translate_bulk_op(builder, len, cost)?; let interned_ty = func_env.module.types[array_type_index].unwrap_module_type_index(); let array_ty = func_env.types.unwrap_array(interned_ty)?; @@ -850,7 +860,6 @@ pub fn translate_array_new_default( zero, elem, len, - cost_per_element, )?; Ok(result) } @@ -1754,8 +1763,10 @@ pub fn translate_array_new_entity( entity: CheckedEntity, entity_offset: ir::Value, len: ir::Value, - cost_per_element: u8, + cost_per_unit: u8, ) -> WasmResult { + env.pre_translate_bulk_op(builder, len, cost_per_unit)?; + // Before actually allocating this array first do a bounds-check on the // passive entity itself. let interned_type_index = env.module.types[array_type_index].unwrap_module_type_index(); @@ -1774,7 +1785,6 @@ pub fn translate_array_new_entity( dst, entity_offset, len, - cost_per_element, )?; Ok(array) diff --git a/crates/wasmtime/src/config.rs b/crates/wasmtime/src/config.rs index 98090ecfc4e5..fd411683efe5 100644 --- a/crates/wasmtime/src/config.rs +++ b/crates/wasmtime/src/config.rs @@ -690,10 +690,6 @@ impl Config { /// signal handler), then we can ensure that all async code will /// yield to the executor within a bounded time. /// - /// The deadline check cannot be avoided by malicious wasm code. It is safe - /// to use epoch deadlines to limit the execution time of untrusted - /// code. - /// /// The [`Store`](crate::Store) tracks the deadline, and controls /// what happens when the deadline is reached during /// execution. Several behaviors are possible: @@ -739,6 +735,27 @@ impl Config { /// computation and have the desired effect of cancelling a blocking /// operation when a timeout expires. /// + /// ## Limitations with malicious guests + /// + /// Epochs are designed to handle malicious WebAssembly guests -- the + /// deadline check cannot be avoided by WebAssembly code. It is safe to use + /// epoch deadlines to limit the execution time of untrusted code. + /// + /// Note, though, that a current limitation to this is that + /// bulk-data-transfer instructions, such as `memory.copy`, only check the + /// epoch once at the start of the operation. These operations can take a + /// variable amount of time to complete based on how many bytes are being + /// copied. This means that the maximal time slice a guest might take is + /// the maximum of the epoch interval and the largest + /// memory-copy-style-instruction executed. The size of a copy is bounded + /// on the size of linear memory or GC heap size. In the limit, however, a + /// guest using a 64-bit linear memory with a 128GiB size could issue a + /// 128GiB `memory.copy` which would have no preemption within the + /// instruction itself. Hosts which need strict time limits for guests right + /// now are recommended to ensure that the store's allocated heap size + /// (linear memory + GC heap) are bounded with a + /// [`ResourceLimiter`](crate::ResourceLimiter). + /// /// ## When to use fuel vs. epochs /// /// In general, epoch-based interruption results in faster diff --git a/crates/wasmtime/src/runtime/store.rs b/crates/wasmtime/src/runtime/store.rs index 9cc49ea8361c..a0816548b844 100644 --- a/crates/wasmtime/src/runtime/store.rs +++ b/crates/wasmtime/src/runtime/store.rs @@ -1041,6 +1041,10 @@ impl Store { /// The `interval` parameter indicates how much fuel should be /// consumed between yields of an async future. When fuel runs out wasm will trap. /// + /// For limitations related to consumption of fuel and when yield points are + /// injected, see the discussion in + /// [`Config::epoch_interruption`](crate::Config::epoch_interruption). + /// /// # Error /// /// This method will error if fuel is not enabled or `interval` is diff --git a/tests/all/epoch_interruption.rs b/tests/all/epoch_interruption.rs index 6a1c1ff7bb92..1d2b4664f3dc 100644 --- a/tests/all/epoch_interruption.rs +++ b/tests/all/epoch_interruption.rs @@ -496,3 +496,130 @@ async fn drop_future_on_epoch_yield(config: &mut Config) -> Result<()> { assert_eq!(true, alive_flag.load(Ordering::Acquire)); Ok(()) } + +#[test] +fn memory_grow_in_epoch_callback() -> Result<()> { + let mut config = Config::new(); + config.epoch_interruption(true); + config.memory_reservation(0); + config.memory_reservation_for_growth(0); + config.memory_guard_size(0); + config.memory_may_move(true); + config.memory_init_cow(false); + let engine = Engine::new(&config)?; + let module = Module::new( + &engine, + r#" + (module + (memory (export "mem") 1) + (func (export "go") + (memory.fill (i32.const 0) (i32.const 0x41) (i32.const 65536)))) + "#, + )?; + + let mut store: Store> = Store::new(&engine, None); + store.set_epoch_deadline(1); + store.epoch_deadline_callback(move |mut cx| { + if let Some(mem) = *cx.data() { + mem.grow(&mut cx, 5)?; + } + Ok(UpdateDeadline::Continue(0)) + }); + + let instance = Instance::new(&mut store, &module, &[])?; + let mem = instance.get_memory(&mut store, "mem").unwrap(); + *store.data_mut() = Some(mem); + engine.increment_epoch(); + + instance + .get_typed_func::<(), ()>(&mut store, "go")? + .call(&mut store, ())?; + + let data = mem.data(&store); + assert_eq!(data[0], 0x41); + assert_eq!(data[65535], 0x41); + Ok(()) +} + +#[test] +fn table_grow_in_epoch_callback() -> Result<()> { + let mut config = Config::new(); + config.epoch_interruption(true); + let engine = Engine::new(&config)?; + let module = Module::new( + &engine, + r#" + (module + (table $t (export "t") 1 funcref) + (func (export "go") + (table.fill $t (i32.const 0) (ref.null func) (i32.const 1)))) + "#, + )?; + + let mut store: Store> = Store::new(&engine, None); + store.set_epoch_deadline(1); + store.epoch_deadline_callback(move |mut cx| { + if let Some(t) = *cx.data() { + t.grow(&mut cx, 5, Ref::Func(None))?; + } + Ok(UpdateDeadline::Continue(0)) + }); + + let instance = Instance::new(&mut store, &module, &[])?; + let t = instance.get_table(&mut store, "t").unwrap(); + *store.data_mut() = Some(t); + engine.increment_epoch(); + + instance + .get_typed_func::<(), ()>(&mut store, "go")? + .call(&mut store, ())?; + Ok(()) +} + +#[test] +fn gc_during_epoch_callback() -> Result<()> { + let mut config = Config::new(); + config.epoch_interruption(true); + let engine = Engine::new(&config)?; + let module = Module::new( + &engine, + r#" + (module + (type $box (struct (field i32))) + (type $arr (array (mut (ref null $box)))) + (global $sink (mut (ref null $arr)) (ref.null $arr)) + (func $mk (param $n i32) (result (ref $arr)) + (array.new_default $arr (local.get $n))) + (func (export "run") (param $n i32) (result i32) + (local $i i32) + (block $done + (loop $l + (br_if $done (i32.ge_u (local.get $i) (i32.const 40))) + (array.fill $arr (call $mk (local.get $n)) (i32.const 0) + (struct.new $box (i32.const 7)) (local.get $n)) + (global.set $sink (call $mk (i32.const 8))) + (local.set $i (i32.add (local.get $i) (i32.const 1))) + (br $l) + ) + ) + (i32.mul (local.get $n) (i32.const 7)))) + "#, + )?; + + let mut store = Store::new(&engine, ()); + store.set_epoch_deadline(1); + store.epoch_deadline_callback(|mut caller| { + caller.gc(None)?; + Ok(UpdateDeadline::Continue(0)) + }); + engine.increment_epoch(); + + let instance = Instance::new(&mut store, &module, &[])?; + let run = instance.get_typed_func::(&mut store, "run")?; + let n = 200; + for i in 0..5 { + let got = run.call(&mut store, n)?; + assert_eq!(got, 7 * n, "iteration {i} read back {got}"); + } + Ok(()) +} diff --git a/tests/all/fuel.rs b/tests/all/fuel.rs index 445a6c1cbd4b..d90ee5c97209 100644 --- a/tests/all/fuel.rs +++ b/tests/all/fuel.rs @@ -966,6 +966,88 @@ fn table64_variable_operator_cost_saturates(config: &mut Config) -> Result<()> { // i64::MAX * 2 must saturate at i64::MAX rather than wrap to -2. let error = grow.call(&mut store, i64::MAX).unwrap_err(); assert_eq!(error.downcast::().unwrap(), Trap::OutOfFuel); + Ok(()) +} + +#[test] +#[cfg_attr(miri, ignore)] +fn huge_table64_grow_cannot_mint_fuel() -> Result<()> { + huge_table64_grow_cannot_mint_fuel_impl( + r#" + (module + (table $t i64 0 0x10000 (ref null func)) + (func (export "run") (param $delta i64) + (loop $l + (drop (table.grow $t (ref.null func) (local.get $delta))) + (br $l)))) + "#, + ) +} +#[test] +#[cfg_attr(miri, ignore)] +fn huge_table64_grow_cannot_mint_fuel_const() -> Result<()> { + huge_table64_grow_cannot_mint_fuel_impl( + r#" + (module + (table $t i64 0 0x10000 (ref null func)) + (func (export "run") (param $delta i64) + (loop $l + (drop (table.grow $t (ref.null func) (i64.const -500))) + (br $l)))) + "#, + ) +} + +fn huge_table64_grow_cannot_mint_fuel_impl(wat: &str) -> Result<()> { + let mut config = Config::new(); + config.consume_fuel(true); + let engine = Engine::new(&config)?; + let module = Module::new(&engine, wat)?; + + let mut store = Store::new(&engine, ()); + store.set_fuel(100_000)?; + let instance = Instance::new(&mut store, &module, &[])?; + let run = instance.get_typed_func::(&mut store, "run")?; + + let trap = run.call(&mut store, -500).unwrap_err().downcast::()?; + assert_eq!(trap, Trap::OutOfFuel); + assert_eq!(store.get_fuel()?, 0); + Ok(()) +} + +#[test] +fn fuel_around_table_grow() -> Result<()> { + let mut config = Config::new(); + config.consume_fuel(true); + let engine = Engine::new(&config)?; + let module = Module::new( + &engine, + r#" + (module + (type $ft (func)) + (func $f (type $ft)) + (table $t 1 10000000 (ref $ft) (ref.func $f)) + (func (export "grow") (result i32) + (table.grow $t (ref.func $f) (i32.const 9999999))) + (func (export "call") (param i32) + (call_indirect $t (type $ft) (local.get 0)))) + "#, + )?; + + let mut store = Store::new(&engine, ()); + store.set_fuel(2)?; + let instance = Instance::new(&mut store, &module, &[])?; + let grow = instance.get_typed_func::<(), i32>(&mut store, "grow")?; + let trap = grow.call(&mut store, ()).unwrap_err().downcast::()?; + assert_eq!(trap, Trap::OutOfFuel); + + store.set_fuel(u64::MAX)?; + let call = instance.get_typed_func::(&mut store, "call")?; + let trap = call + .call(&mut store, 9999999) + .unwrap_err() + .downcast::()?; + assert_eq!(trap, Trap::TableOutOfBounds); Ok(()) } diff --git a/tests/all/gc.rs b/tests/all/gc.rs index ac381877f7d4..ff32f09fc463 100644 --- a/tests/all/gc.rs +++ b/tests/all/gc.rs @@ -3892,3 +3892,107 @@ fn winch_ref_params_and_results_across_gc() -> Result<()> { } Ok(()) } + +#[test] +#[cfg_attr(miri, ignore)] +fn array_fill_i64_gc_during_epoch() -> Result<()> { + gc_during_epoch( + r#" + (module + (type $arr (array (mut i64))) + (type $box (struct (field i32))) + (func (export "run") (param $n i32) (result i32) + (local $a (ref null $arr)) (local $i i32) (local $s i32) + (local.set $a (array.new_default $arr (local.get $n))) + ;; Keep the collector busy so it has something to move. + (drop (struct.new $box (i32.const 1))) + (array.fill $arr (local.get $a) (i32.const 0) (i64.const 7) (local.get $n)) + (block $done (loop $l + (br_if $done (i32.ge_u (local.get $i) (local.get $n))) + (local.set $s (i32.add (local.get $s) + (i32.wrap_i64 (array.get $arr (local.get $a) (local.get $i))))) + (local.set $i (i32.add (local.get $i) (i32.const 1))) + (br $l))) + (local.get $s))) + "#, + ) +} + +#[test] +#[cfg_attr(miri, ignore)] +fn array_new_gc_during_epoch() -> Result<()> { + gc_during_epoch( + r#" + (module + (type $box (struct (field i32))) + (type $arr (array (mut (ref null $box)))) + (func (export "run") (param $n i32) (result i32) + (local $a (ref null $arr)) (local $i i32) (local $s i32) + (local.set $a (array.new $arr (struct.new $box (i32.const 7)) (local.get $n))) + (block $done (loop $l + (br_if $done (i32.ge_u (local.get $i) (local.get $n))) + (local.set $s (i32.add (local.get $s) + (struct.get $box 0 (ref.as_non_null + (array.get $arr (local.get $a) (local.get $i)))))) + (local.set $i (i32.add (local.get $i) (i32.const 1))) + (br $l))) + (local.get $s))) + "#, + ) +} + +#[test] +#[cfg_attr(miri, ignore)] +fn array_copy_gc_during_epoch() -> Result<()> { + gc_during_epoch( + r#" + (module + (type $box (struct (field i32))) + (type $arr (array (mut (ref null $box)))) + (func (export "run") (param $n i32) (result i32) + (local $a (ref null $arr)) (local $b (ref null $arr)) + (local $i i32) (local $s i32) + (local.set $a (array.new_default $arr (local.get $n))) + (local.set $b (array.new_default $arr (local.get $n))) + (array.fill $arr (local.get $b) (i32.const 0) + (struct.new $box (i32.const 7)) (local.get $n)) + (array.copy $arr $arr (local.get $a) (i32.const 0) + (local.get $b) (i32.const 0) (local.get $n)) + (block $done (loop $l + (br_if $done (i32.ge_u (local.get $i) (local.get $n))) + (local.set $s (i32.add (local.get $s) + (struct.get $box 0 (ref.as_non_null + (array.get $arr (local.get $a) (local.get $i)))))) + (local.set $i (i32.add (local.get $i) (i32.const 1))) + (br $l))) + (local.get $s))) + "#, + ) +} + +fn gc_during_epoch(wat: &str) -> Result<()> { + let mut config = Config::new(); + config.epoch_interruption(true); + let engine = Engine::new(&config)?; + let module = Module::new(&engine, wat)?; + + let mut store = Store::new(&engine, ()); + store.set_epoch_deadline(1); + store.epoch_deadline_callback(|mut caller| { + caller.gc(None)?; + Ok(UpdateDeadline::Continue(0)) + }); + engine.increment_epoch(); + + let instance = Instance::new(&mut store, &module, &[])?; + let f = instance.get_typed_func::(&mut store, "run")?; + + let n = 100; + for i in 0..5 { + match f.call(&mut store, n) { + Ok(got) => assert_eq!(got, 7 * n, "iteration {i} read back {got}"), + Err(e) => panic!("iteration {i} failed: {e:?}"), + } + } + Ok(()) +} diff --git a/tests/disas/gc/array-copy-with-fuel.wat b/tests/disas/gc/array-copy-with-fuel.wat index aeab1b17ebd3..8371c0eed235 100644 --- a/tests/disas/gc/array-copy-with-fuel.wat +++ b/tests/disas/gc/array-copy-with-fuel.wat @@ -28,10 +28,10 @@ ;; stack_limit = gv2 ;; ;; block0(v0: i64, v1: i64, v2: i32, v3: i32, v4: i32, v5: i32, v6: i32): -;; v181 = stack_addr.i64 ss0 -;; store notrap aligned region6 v2, v181 -;; v182 = stack_addr.i64 ss1 -;; store notrap aligned region7 v4, v182 +;; v162 = stack_addr.i64 ss0 +;; store notrap aligned region6 v2, v162 +;; v163 = stack_addr.i64 ss1 +;; store notrap aligned region7 v4, v163 ;; @0020 v7 = load.i64 notrap aligned readonly can_move region0 v0+8 ;; @0020 v8 = load.i64 notrap aligned region2 v7 ;; @0020 v9 = iconst.i64 1 @@ -41,131 +41,108 @@ ;; @0020 brif v12, block2, block3(v10) ;; ;; block2: -;; v191 = iadd.i64 v8, v9 ; v9 = 1 -;; @0020 store notrap aligned region2 v191, v7 +;; v172 = iadd.i64 v8, v9 ; v9 = 1 +;; @0020 store notrap aligned region2 v172, v7 ;; @0020 v14 = call fn0(v0), stack_map=[i32 @ ss0+0, i32 @ ss1+0] ;; @0020 v16 = load.i64 notrap aligned region2 v7 ;; @0020 jump block3(v16) ;; -;; block3(v89: i64): -;; v180 = load.i32 notrap aligned region6 v181 -;; @002b trapz v180, user16 -;; @002b v24 = load.i64 notrap aligned readonly can_move region3 v7+32 -;; @002b v22 = uextend.i64 v180 -;; @002b v25 = iadd v24, v22 -;; @002b v26 = iconst.i64 16 -;; @002b v27 = iadd v25, v26 ; v26 = 16 -;; @002b v28 = load.i32 user2 readonly region5 v27 -;; @002b v30 = uextend.i64 v3 -;; @002b v31 = uextend.i64 v6 -;; @002b v34 = iadd v30, v31 -;; @002b v29 = uextend.i64 v28 -;; @002b v35 = icmp ugt v34, v29 -;; @002b trapnz v35, user17 -;; v174 = load.i32 notrap aligned region7 v182 -;; @002b trapz v174, user16 -;; @002b v46 = uextend.i64 v174 -;; @002b v49 = iadd v24, v46 -;; @002b v51 = iadd v49, v26 ; v26 = 16 -;; @002b v52 = load.i32 user2 readonly region5 v51 -;; @002b v54 = uextend.i64 v5 -;; @002b v58 = iadd v54, v31 -;; @002b v53 = uextend.i64 v52 -;; @002b v59 = icmp ugt v58, v53 -;; @002b trapnz v59, user17 -;; @002b v78 = load.i64 notrap aligned region4 v7+40 -;; @002b v40 = iconst.i64 20 -;; @002b v41 = iadd v25, v40 ; v40 = 20 -;; v184 = iconst.i64 2 -;; v185 = ishl v30, v184 ; v184 = 2 -;; @002b v45 = iadd v41, v185 -;; v189 = ishl v31, v184 ; v184 = 2 -;; @002b v80 = uadd_overflow_trap v45, v189, user2 -;; @002b v79 = iadd v24, v78 -;; @002b v81 = icmp ugt v80, v79 -;; @002b trapnz v81, user2 -;; @002b v65 = iadd v49, v40 ; v40 = 20 -;; v187 = ishl v54, v184 ; v184 = 2 -;; @002b v69 = iadd v65, v187 -;; @002b v87 = uadd_overflow_trap v69, v189, user2 -;; @002b v88 = icmp ugt v87, v79 -;; @002b trapnz v88, user2 -;; @002b v90 = iconst.i64 6 -;; @002b v91 = iadd v89, v90 ; v90 = 6 -;; @002b brif.i32 v6, block4, block7(v91) +;; block3(v25: i64): +;; @002b v26 = iconst.i64 6 +;; @002b v27 = iadd v25, v26 ; v26 = 6 +;; @002b v22 = uextend.i64 v6 +;; @002b v28 = iadd v27, v22 +;; v173 = iconst.i64 0 +;; v174 = icmp sge v28, v173 ; v173 = 0 +;; @002b brif v174, block4, block5(v28) ;; ;; block4: -;; v162 = load.i32 notrap aligned region6 v181 -;; v164 = load.i32 notrap aligned region7 v182 -;; @002b v92 = icmp.i64 ult v45, v69 -;; v192 = iadd.i64 v89, v90 ; v90 = 6 -;; @002b v97 = iadd.i64 v45, v189 -;; @002b v98 = iadd.i64 v69, v189 -;; @002b v100 = iadd.i32 v5, v6 -;; @002b v43 = iconst.i64 4 -;; @002b v141 = iconst.i32 1 -;; @002b brif v92, block5(v45, v69, v5, v162, v164, v192), block6(v97, v98, v100, v162, v164, v192) +;; @002b store.i64 notrap aligned region2 v28, v7 +;; @002b v32 = call fn0(v0), stack_map=[i32 @ ss0+0, i32 @ ss1+0] +;; @002b v34 = load.i64 notrap aligned region2 v7 +;; @002b jump block5(v34) ;; -;; block5(v101: i64, v102: i64, v103: i32, v104: i32, v105: i32, v106: i64): -;; store notrap aligned region6 v104, v181 -;; store notrap aligned region7 v105, v182 -;; v202 = iconst.i64 1 -;; v203 = iadd v106, v202 ; v202 = 1 -;; v204 = iconst.i64 0 -;; v205 = icmp sge v203, v204 ; v204 = 0 -;; @002b brif v205, block8, block9(v203) +;; block5(v139: i64): +;; v161 = load.i32 notrap aligned region6 v162 +;; @002b trapz v161, user16 +;; @002b v37 = load.i64 notrap aligned readonly can_move region3 v7+32 +;; @002b v35 = uextend.i64 v161 +;; @002b v38 = iadd v37, v35 +;; @002b v39 = iconst.i64 16 +;; @002b v40 = iadd v38, v39 ; v39 = 16 +;; @002b v41 = load.i32 user2 readonly region5 v40 +;; @002b v43 = uextend.i64 v3 +;; @002b v47 = iadd v43, v22 +;; @002b v42 = uextend.i64 v41 +;; @002b v48 = icmp ugt v47, v42 +;; @002b trapnz v48, user17 +;; v155 = load.i32 notrap aligned region7 v163 +;; @002b trapz v155, user16 +;; @002b v59 = uextend.i64 v155 +;; @002b v62 = iadd v37, v59 +;; @002b v64 = iadd v62, v39 ; v39 = 16 +;; @002b v65 = load.i32 user2 readonly region5 v64 +;; @002b v67 = uextend.i64 v5 +;; @002b v71 = iadd v67, v22 +;; @002b v66 = uextend.i64 v65 +;; @002b v72 = icmp ugt v71, v66 +;; @002b trapnz v72, user17 +;; @002b v91 = load.i64 notrap aligned region4 v7+40 +;; @002b v53 = iconst.i64 20 +;; @002b v54 = iadd v38, v53 ; v53 = 20 +;; v165 = iconst.i64 2 +;; v166 = ishl v43, v165 ; v165 = 2 +;; @002b v58 = iadd v54, v166 +;; v170 = ishl.i64 v22, v165 ; v165 = 2 +;; @002b v93 = uadd_overflow_trap v58, v170, user2 +;; @002b v92 = iadd v37, v91 +;; @002b v94 = icmp ugt v93, v92 +;; @002b trapnz v94, user2 +;; @002b v78 = iadd v62, v53 ; v53 = 20 +;; v168 = ishl v67, v165 ; v165 = 2 +;; @002b v82 = iadd v78, v168 +;; @002b v100 = uadd_overflow_trap v82, v170, user2 +;; @002b v101 = icmp ugt v100, v92 +;; @002b trapnz v101, user2 +;; @002b brif.i32 v6, block6, block9 ;; -;; block6(v123: i64, v124: i64, v125: i32, v126: i32, v127: i32, v128: i64): -;; store notrap aligned region7 v126, v182 -;; store notrap aligned region6 v127, v181 -;; v193 = iconst.i64 1 -;; v194 = iadd v128, v193 ; v193 = 1 -;; v195 = iconst.i64 0 -;; v196 = icmp sge v194, v195 ; v195 = 0 -;; @002b brif v196, block10, block11(v194) +;; block6: +;; v143 = load.i32 notrap aligned region6 v162 +;; v145 = load.i32 notrap aligned region7 v163 +;; @002b v102 = icmp.i64 ult v58, v82 +;; @002b v107 = iadd.i64 v58, v170 +;; @002b v108 = iadd.i64 v82, v170 +;; @002b v110 = iadd.i32 v5, v6 +;; @002b v56 = iconst.i64 4 +;; @002b v133 = iconst.i32 1 +;; @002b brif v102, block7(v58, v82, v5), block8(v107, v108, v110) ;; -;; block7(v148: i64): -;; @002f jump block1 -;; -;; block8: -;; @002b store.i64 notrap aligned region2 v203, v7 -;; @002b v112 = call fn0(v0), stack_map=[i32 @ ss0+0, i32 @ ss1+0] -;; @002b v114 = load.i64 notrap aligned region2 v7 -;; @002b jump block9(v114) +;; block7(v111: i64, v112: i64, v113: i32): +;; @002b v116 = load.i32 user2 little region5 v112 +;; @002b store user2 little region5 v116, v111 +;; v180 = iconst.i64 4 +;; v181 = iadd v112, v180 ; v180 = 4 +;; @002b v123 = icmp eq v181, v108 +;; v182 = iadd v111, v180 ; v180 = 4 +;; v183 = iconst.i32 1 +;; v184 = iadd v113, v183 ; v183 = 1 +;; @002b brif v123, block9, block7(v182, v181, v184) ;; -;; block9(v145: i64): -;; @002b v115 = load.i32 user2 little region5 v102 -;; @002b store user2 little region5 v115, v101 -;; v150 = load.i32 notrap aligned region6 v181 -;; v152 = load.i32 notrap aligned region7 v182 -;; v206 = iconst.i64 4 -;; v207 = iadd.i64 v102, v206 ; v206 = 4 -;; @002b v122 = icmp eq v207, v98 -;; v208 = iadd.i64 v101, v206 ; v206 = 4 -;; v209 = iconst.i32 1 -;; v210 = iadd.i32 v103, v209 ; v209 = 1 -;; @002b brif v122, block7(v145), block5(v208, v207, v210, v150, v152, v145) +;; block8(v124: i64, v125: i64, v126: i32): +;; v175 = iconst.i64 4 +;; v176 = isub v125, v175 ; v175 = 4 +;; @002b v135 = load.i32 user2 little region5 v176 +;; v177 = isub v124, v175 ; v175 = 4 +;; @002b store user2 little region5 v135, v177 +;; @002b v136 = icmp eq v176, v82 +;; v178 = iconst.i32 1 +;; v179 = isub v126, v178 ; v178 = 1 +;; @002b brif v136, block9, block8(v177, v176, v179) ;; -;; block10: -;; @002b store.i64 notrap aligned region2 v194, v7 -;; @002b v134 = call fn0(v0), stack_map=[i32 @ ss1+0, i32 @ ss0+0] -;; @002b v136 = load.i64 notrap aligned region2 v7 -;; @002b jump block11(v136) -;; -;; block11(v146: i64): -;; v197 = iconst.i64 4 -;; v198 = isub.i64 v124, v197 ; v197 = 4 -;; @002b v143 = load.i32 user2 little region5 v198 -;; v199 = isub.i64 v123, v197 ; v197 = 4 -;; @002b store user2 little region5 v143, v199 -;; v156 = load.i32 notrap aligned region7 v182 -;; v158 = load.i32 notrap aligned region6 v181 -;; @002b v144 = icmp eq v198, v69 -;; v200 = iconst.i32 1 -;; v201 = isub.i32 v125, v200 ; v200 = 1 -;; @002b brif v144, block7(v146), block6(v199, v198, v201, v156, v158, v146) +;; block9: +;; @002f jump block1 ;; ;; block1: -;; @002f store.i64 notrap aligned region2 v148, v7 +;; @002f store.i64 notrap aligned region2 v139, v7 ;; @002f return ;; } diff --git a/tests/disas/memory-copy-epochs.wat b/tests/disas/memory-copy-epochs.wat index f712972b8774..d6113471a161 100644 --- a/tests/disas/memory-copy-epochs.wat +++ b/tests/disas/memory-copy-epochs.wat @@ -37,101 +37,35 @@ ;; @001e v10 = call fn0(v0) ;; @001e jump block2(v10) ;; -;; block2(v59: i64): -;; @0025 v14 = load.i64 notrap aligned region6 v0+64 -;; @0025 v15 = uextend.i64 v2 -;; @0025 v16 = uextend.i64 v4 -;; @0025 v19 = iadd v15, v16 -;; @0025 v20 = icmp ugt v19, v14 -;; @0025 trapnz v20, heap_oob -;; @0025 v27 = uextend.i64 v3 -;; @0025 v31 = iadd v27, v16 -;; @0025 v32 = icmp ugt v31, v14 -;; @0025 trapnz v32, heap_oob -;; @0025 v21 = load.i64 notrap aligned readonly can_move region5 v0+56 -;; @0025 v37 = iadd v21, v27 -;; @0025 v25 = iadd v21, v15 -;; @0025 v40 = icmp ugt v37, v25 -;; @0025 brif v40, block6, block7 -;; -;; block4(v42: i64, v43: i64, v44: i64, v47: i64): -;; @0025 v46 = load.i64 notrap aligned region3 v5 -;; @0025 v48 = icmp uge v46, v47 -;; @0025 brif v48, block9, block8(v47) -;; -;; block5(v86: i64, v87: i64, v88: i64, v92: i64): -;; @0025 v91 = load.i64 notrap aligned region3 v5 -;; @0025 v94 = icmp uge v91, v92 -;; @0025 brif v94, block17, block16 -;; -;; block6: -;; v108 = iconst.i64 0x0800_0000 -;; v109 = icmp.i64 ugt v16, v108 ; v108 = 0x0800_0000 -;; @0025 brif v109, block4(v25, v37, v16, v59), block5(v25, v37, v16, v59) -;; -;; block9 cold: -;; @0025 v50 = load.i64 notrap aligned region4 v7+8 -;; @0025 v51 = icmp.i64 uge v46, v50 -;; @0025 brif v51, block10, block8(v50) -;; -;; block10 cold: -;; @0025 v52 = call fn0(v0) -;; @0025 jump block8(v52) -;; -;; block8(v60: i64): -;; @0025 call fn1(v0, v42, v43, v108) ; v108 = 0x0800_0000 -;; @0025 v55 = isub.i64 v44, v108 ; v108 = 0x0800_0000 -;; @0025 v56 = icmp ugt v55, v108 ; v108 = 0x0800_0000 -;; @0025 v53 = iadd.i64 v42, v108 ; v108 = 0x0800_0000 -;; @0025 v54 = iadd.i64 v43, v108 ; v108 = 0x0800_0000 -;; @0025 brif v56, block4(v53, v54, v55, v60), block5(v53, v54, v55, v60) -;; -;; block7: -;; @0025 v39 = iconst.i64 0x0800_0000 -;; @0025 v63 = icmp.i64 ugt v16, v39 ; v39 = 0x0800_0000 -;; @0025 v61 = iadd.i64 v25, v16 -;; @0025 v62 = iadd.i64 v37, v16 -;; @0025 brif v63, block11(v61, v62, v16, v59), block12(v61, v62, v16, v59) -;; -;; block11(v64: i64, v65: i64, v66: i64, v71: i64): -;; @0025 v70 = load.i64 notrap aligned region3 v5 -;; @0025 v72 = icmp uge v70, v71 -;; @0025 brif v72, block14, block13(v71) -;; -;; block14 cold: -;; @0025 v74 = load.i64 notrap aligned region4 v7+8 -;; @0025 v75 = icmp.i64 uge v70, v74 -;; @0025 brif v75, block15, block13(v74) -;; -;; block15 cold: -;; @0025 v76 = call fn0(v0) -;; @0025 jump block13(v76) -;; -;; block13(v80: i64): -;; v103 = iconst.i64 0x0800_0000 -;; v104 = isub.i64 v64, v103 ; v103 = 0x0800_0000 -;; v105 = isub.i64 v65, v103 ; v103 = 0x0800_0000 -;; @0025 call fn1(v0, v104, v105, v103) ; v103 = 0x0800_0000 -;; v106 = isub.i64 v66, v103 ; v103 = 0x0800_0000 -;; v107 = icmp ugt v106, v103 ; v103 = 0x0800_0000 -;; @0025 brif v107, block11(v104, v105, v106, v80), block12(v104, v105, v106, v80) -;; -;; block12(v81: i64, v82: i64, v83: i64, v93: i64): -;; @0025 v84 = isub v81, v83 -;; @0025 v85 = isub v82, v83 -;; @0025 jump block5(v84, v85, v83, v93) -;; -;; block17 cold: -;; @0025 v96 = load.i64 notrap aligned region4 v7+8 -;; @0025 v97 = icmp.i64 uge v91, v96 -;; @0025 brif v97, block18, block16 -;; -;; block18 cold: -;; @0025 v98 = call fn0(v0) -;; @0025 jump block16 -;; -;; block16: -;; @0025 call fn1(v0, v86, v87, v88) +;; block2(v16: i64): +;; @0025 v15 = load.i64 notrap aligned region3 v5 +;; @0025 v17 = icmp uge v15, v16 +;; @0025 brif v17, block5, block4 +;; +;; block5 cold: +;; @0025 v19 = load.i64 notrap aligned region4 v7+8 +;; @0025 v20 = icmp.i64 uge v15, v19 +;; @0025 brif v20, block6, block4 +;; +;; block6 cold: +;; @0025 v21 = call fn0(v0) +;; @0025 jump block4 +;; +;; block4: +;; @0025 v22 = load.i64 notrap aligned region6 v0+64 +;; @0025 v23 = uextend.i64 v2 +;; @0025 v24 = uextend.i64 v4 +;; @0025 v27 = iadd v23, v24 +;; @0025 v28 = icmp ugt v27, v22 +;; @0025 trapnz v28, heap_oob +;; @0025 v35 = uextend.i64 v3 +;; @0025 v39 = iadd v35, v24 +;; @0025 v40 = icmp ugt v39, v22 +;; @0025 trapnz v40, heap_oob +;; @0025 v29 = load.i64 notrap aligned readonly can_move region5 v0+56 +;; @0025 v33 = iadd v29, v23 +;; @0025 v45 = iadd v29, v35 +;; @0025 call fn1(v0, v33, v45, v24) ;; @0029 jump block1 ;; ;; block1: diff --git a/tests/disas/memory-copy-fuel-const-len.wat b/tests/disas/memory-copy-fuel-const-len.wat new file mode 100644 index 000000000000..aee7af900c1b --- /dev/null +++ b/tests/disas/memory-copy-fuel-const-len.wat @@ -0,0 +1,184 @@ +;;! target = 'x86_64' +;;! test = 'optimize' +;;! flags = '-Wfuel=100' + +(module + (memory 1) + (func $copy_16 (param i32 i32) + (memory.copy (local.get 0) (local.get 1) (i32.const 16)) + ) + (func $fill_128 (param i32) + (memory.fill (local.get 0) (i32.const 0) (i32.const 128)) + ) + (func $fill_4096 (param i32) + (memory.fill (local.get 0) (i32.const 0) (i32.const 4096)) + ) +) +;; function u0:0(i64 vmctx, i64, i32, i32) tail { +;; region0 = 8 "VMContext+0x8" +;; region1 = 67108888 "VMStoreContext+0x18" +;; region2 = 67108864 "VMStoreContext+0x0" +;; region3 = 603979776 "VMMemoryDefinition+0x0" +;; region4 = 603979784 "VMMemoryDefinition+0x8" +;; region5 = 201326592 "DefinedMemory(StaticModuleIndex(0), DefinedMemoryIndex(0))" +;; gv0 = vmctx +;; gv1 = load.i64 notrap aligned readonly can_move region0 gv0+8 +;; gv2 = load.i64 notrap aligned region1 gv1+24 +;; sig0 = (i64 vmctx) -> i8 tail +;; fn0 = colocated u805306368:12 sig0 +;; stack_limit = gv2 +;; +;; block0(v0: i64, v1: i64, v2: i32, v3: i32): +;; @0023 v4 = load.i64 notrap aligned readonly can_move region0 v0+8 +;; @0023 v5 = load.i64 notrap aligned region2 v4 +;; @0023 v6 = iconst.i64 1 +;; @0023 v7 = iadd v5, v6 ; v6 = 1 +;; @0023 v8 = iconst.i64 0 +;; @0023 v9 = icmp sge v7, v8 ; v8 = 0 +;; @0023 brif v9, block2, block3(v7) +;; +;; block2: +;; v58 = iadd.i64 v5, v6 ; v6 = 1 +;; @0023 store notrap aligned region2 v58, v4 +;; @0023 v11 = call fn0(v0) +;; @0023 v13 = load.i64 notrap aligned region2 v4 +;; @0023 jump block3(v13) +;; +;; block3(v43: i64): +;; @002a v17 = load.i64 notrap aligned region4 v0+64 +;; @002a v18 = uextend.i64 v2 +;; v47 = iconst.i64 16 +;; @002a v22 = iadd v18, v47 ; v47 = 16 +;; @002a v23 = icmp ugt v22, v17 +;; @002a trapnz v23, heap_oob +;; @002a v30 = uextend.i64 v3 +;; @002a v34 = iadd v30, v47 ; v47 = 16 +;; @002a v35 = icmp ugt v34, v17 +;; @002a trapnz v35, heap_oob +;; @002a v24 = load.i64 notrap aligned readonly can_move region3 v0+56 +;; @002a v40 = iadd v24, v30 +;; @002a v42 = load.i8x16 notrap aligned little region5 v40 +;; @002a v28 = iadd v24, v18 +;; @002a store notrap aligned little region5 v42, v28 +;; @002e jump block1 +;; +;; block1: +;; @002e v44 = iconst.i64 20 +;; @002e v45 = iadd.i64 v43, v44 ; v44 = 20 +;; @002e store notrap aligned region2 v45, v4 +;; @002e return +;; } +;; +;; function u0:1(i64 vmctx, i64, i32) tail { +;; region0 = 8 "VMContext+0x8" +;; region1 = 67108888 "VMStoreContext+0x18" +;; region2 = 67108864 "VMStoreContext+0x0" +;; region3 = 603979776 "VMMemoryDefinition+0x0" +;; region4 = 603979784 "VMMemoryDefinition+0x8" +;; gv0 = vmctx +;; gv1 = load.i64 notrap aligned readonly can_move region0 gv0+8 +;; gv2 = load.i64 notrap aligned region1 gv1+24 +;; sig0 = (i64 vmctx) -> i8 tail +;; sig1 = (i64 vmctx, i64, i32, i64) tail +;; fn0 = colocated u805306368:12 sig0 +;; fn1 = colocated u805306368:2 sig1 +;; stack_limit = gv2 +;; +;; block0(v0: i64, v1: i64, v2: i32): +;; @0030 v3 = load.i64 notrap aligned readonly can_move region0 v0+8 +;; @0030 v4 = load.i64 notrap aligned region2 v3 +;; @0030 v5 = iconst.i64 1 +;; @0030 v6 = iadd v4, v5 ; v5 = 1 +;; @0030 v7 = iconst.i64 0 +;; @0030 v8 = icmp sge v6, v7 ; v7 = 0 +;; @0030 brif v8, block2, block3(v6) +;; +;; block2: +;; v43 = iadd.i64 v4, v5 ; v5 = 1 +;; @0030 store notrap aligned region2 v43, v3 +;; @0030 v10 = call fn0(v0) +;; @0030 v12 = load.i64 notrap aligned region2 v3 +;; @0030 jump block3(v12) +;; +;; block3(v29: i64): +;; @0038 v16 = load.i64 notrap aligned region4 v0+64 +;; @0038 v17 = uextend.i64 v2 +;; v33 = iconst.i64 128 +;; @0038 v21 = iadd v17, v33 ; v33 = 128 +;; @0038 v22 = icmp ugt v21, v16 +;; @0038 trapnz v22, heap_oob +;; @0038 v23 = load.i64 notrap aligned readonly can_move region3 v0+56 +;; @0038 v27 = iadd v23, v17 +;; @0033 v14 = iconst.i32 0 +;; @0038 call fn1(v0, v27, v14, v33) ; v14 = 0, v33 = 128 +;; @003b jump block1 +;; +;; block1: +;; @003b v30 = iconst.i64 132 +;; @003b v31 = iadd.i64 v29, v30 ; v30 = 132 +;; @003b store notrap aligned region2 v31, v3 +;; @003b return +;; } +;; +;; function u0:2(i64 vmctx, i64, i32) tail { +;; region0 = 8 "VMContext+0x8" +;; region1 = 67108888 "VMStoreContext+0x18" +;; region2 = 67108864 "VMStoreContext+0x0" +;; region3 = 603979776 "VMMemoryDefinition+0x0" +;; region4 = 603979784 "VMMemoryDefinition+0x8" +;; gv0 = vmctx +;; gv1 = load.i64 notrap aligned readonly can_move region0 gv0+8 +;; gv2 = load.i64 notrap aligned region1 gv1+24 +;; sig0 = (i64 vmctx) -> i8 tail +;; sig1 = (i64 vmctx, i64, i32, i64) tail +;; fn0 = colocated u805306368:12 sig0 +;; fn1 = colocated u805306368:2 sig1 +;; stack_limit = gv2 +;; +;; block0(v0: i64, v1: i64, v2: i32): +;; @003d v3 = load.i64 notrap aligned readonly can_move region0 v0+8 +;; @003d v4 = load.i64 notrap aligned region2 v3 +;; @003d v5 = iconst.i64 1 +;; @003d v6 = iadd v4, v5 ; v5 = 1 +;; @003d v7 = iconst.i64 0 +;; @003d v8 = icmp sge v6, v7 ; v7 = 0 +;; @003d brif v8, block2, block3(v6) +;; +;; block2: +;; v50 = iadd.i64 v4, v5 ; v5 = 1 +;; @003d store notrap aligned region2 v50, v3 +;; @003d v10 = call fn0(v0) +;; @003d v12 = load.i64 notrap aligned region2 v3 +;; @003d jump block3(v12) +;; +;; block3(v16: i64): +;; @0045 v17 = iconst.i64 4100 +;; @0045 v18 = iadd v16, v17 ; v17 = 4100 +;; v51 = iconst.i64 0 +;; v52 = icmp sge v18, v51 ; v51 = 0 +;; @0045 brif v52, block4, block5(v18) +;; +;; block4: +;; v53 = iadd.i64 v16, v17 ; v17 = 4100 +;; @0045 store notrap aligned region2 v53, v3 +;; @0045 v22 = call fn0(v0) +;; @0045 v24 = load.i64 notrap aligned region2 v3 +;; @0045 jump block5(v24) +;; +;; block5(v39: i64): +;; @0045 v25 = load.i64 notrap aligned region4 v0+64 +;; @0045 v26 = uextend.i64 v2 +;; v40 = iconst.i64 4096 +;; @0045 v30 = iadd v26, v40 ; v40 = 4096 +;; @0045 v31 = icmp ugt v30, v25 +;; @0045 trapnz v31, heap_oob +;; @0045 v32 = load.i64 notrap aligned readonly can_move region3 v0+56 +;; @0045 v36 = iadd v32, v26 +;; @0040 v14 = iconst.i32 0 +;; @0045 call fn1(v0, v36, v14, v40) ; v14 = 0, v40 = 4096 +;; @0048 jump block1 +;; +;; block1: +;; @0048 store.i64 notrap aligned region2 v39, v3 +;; @0048 return +;; } diff --git a/tests/disas/memory-copy-fuel.wat b/tests/disas/memory-copy-fuel.wat index 45e72b7043d4..29104ad29719 100644 --- a/tests/disas/memory-copy-fuel.wat +++ b/tests/disas/memory-copy-fuel.wat @@ -33,110 +33,44 @@ ;; @001e brif v10, block2, block3(v8) ;; ;; block2: -;; v106 = iadd.i64 v6, v7 ; v7 = 1 -;; @001e store notrap aligned region2 v106, v5 +;; v61 = iadd.i64 v6, v7 ; v7 = 1 +;; @001e store notrap aligned region2 v61, v5 ;; @001e v12 = call fn0(v0) ;; @001e v14 = load.i64 notrap aligned region2 v5 ;; @001e jump block3(v14) ;; -;; block3(v43: i64): -;; @0025 v18 = load.i64 notrap aligned region4 v0+64 -;; @0025 v19 = uextend.i64 v2 -;; @0025 v20 = uextend.i64 v4 -;; @0025 v23 = iadd v19, v20 -;; @0025 v24 = icmp ugt v23, v18 -;; @0025 trapnz v24, heap_oob -;; @0025 v31 = uextend.i64 v3 -;; @0025 v35 = iadd v31, v20 -;; @0025 v36 = icmp ugt v35, v18 -;; @0025 trapnz v36, heap_oob -;; @0025 v25 = load.i64 notrap aligned readonly can_move region3 v0+56 -;; @0025 v41 = iadd v25, v31 -;; @0025 v29 = iadd v25, v19 -;; @0025 v47 = icmp ugt v41, v29 -;; @0025 brif v47, block6, block7 -;; -;; block4(v49: i64, v50: i64, v51: i64, v52: i64): -;; @0025 v53 = iadd v52, v116 ; v116 = 0x0800_0000 -;; v120 = iconst.i64 0 -;; v121 = icmp sge v53, v120 ; v120 = 0 -;; @0025 brif v121, block8, block9(v53) -;; -;; block5(v89: i64, v90: i64, v91: i64, v92: i64): -;; @0025 v94 = iadd v92, v91 -;; v123 = iconst.i64 0 -;; v124 = icmp sge v94, v123 ; v123 = 0 -;; @0025 brif v124, block14, block15(v94) -;; -;; block6: -;; v116 = iconst.i64 0x0800_0000 -;; v117 = icmp.i64 ugt v20, v116 ; v116 = 0x0800_0000 -;; v118 = iconst.i64 4 -;; v119 = iadd.i64 v43, v118 ; v118 = 4 -;; @0025 brif v117, block4(v29, v41, v20, v119), block5(v29, v41, v20, v119) -;; -;; block8: -;; v122 = iadd.i64 v52, v116 ; v116 = 0x0800_0000 -;; @0025 store notrap aligned region2 v122, v5 -;; @0025 v57 = call fn0(v0) -;; @0025 v59 = load.i64 notrap aligned region2 v5 -;; @0025 jump block9(v59) -;; -;; block9(v64: i64): -;; @0025 call fn1(v0, v49, v50, v116) ; v116 = 0x0800_0000 -;; @0025 v62 = isub.i64 v51, v116 ; v116 = 0x0800_0000 -;; @0025 v63 = icmp ugt v62, v116 ; v116 = 0x0800_0000 -;; @0025 v60 = iadd.i64 v49, v116 ; v116 = 0x0800_0000 -;; @0025 v61 = iadd.i64 v50, v116 ; v116 = 0x0800_0000 -;; @0025 brif v63, block4(v60, v61, v62, v64), block5(v60, v61, v62, v64) -;; -;; block7: -;; @0025 v46 = iconst.i64 0x0800_0000 -;; @0025 v67 = icmp.i64 ugt v20, v46 ; v46 = 0x0800_0000 -;; @0025 v65 = iadd.i64 v29, v20 -;; @0025 v66 = iadd.i64 v41, v20 -;; @0025 v44 = iconst.i64 4 -;; @0025 v45 = iadd.i64 v43, v44 ; v44 = 4 -;; @0025 brif v67, block10(v65, v66, v20, v45), block11(v65, v66, v20, v45) -;; -;; block10(v68: i64, v69: i64, v70: i64, v73: i64): -;; v107 = iconst.i64 0x0800_0000 -;; v108 = iadd v73, v107 ; v107 = 0x0800_0000 -;; v109 = iconst.i64 0 -;; v110 = icmp sge v108, v109 ; v109 = 0 -;; @0025 brif v110, block12, block13(v108) -;; -;; block12: -;; @0025 store.i64 notrap aligned region2 v108, v5 -;; @0025 v78 = call fn0(v0) -;; @0025 v80 = load.i64 notrap aligned region2 v5 -;; @0025 jump block13(v80) -;; -;; block13(v83: i64): -;; v111 = iconst.i64 0x0800_0000 -;; v112 = isub.i64 v68, v111 ; v111 = 0x0800_0000 -;; v113 = isub.i64 v69, v111 ; v111 = 0x0800_0000 -;; @0025 call fn1(v0, v112, v113, v111) ; v111 = 0x0800_0000 -;; v114 = isub.i64 v70, v111 ; v111 = 0x0800_0000 -;; v115 = icmp ugt v114, v111 ; v111 = 0x0800_0000 -;; @0025 brif v115, block10(v112, v113, v114, v83), block11(v112, v113, v114, v83) -;; -;; block11(v84: i64, v85: i64, v86: i64, v93: i64): -;; @0025 v87 = isub v84, v86 -;; @0025 v88 = isub v85, v86 -;; @0025 jump block5(v87, v88, v86, v93) -;; -;; block14: -;; @0025 store.i64 notrap aligned region2 v94, v5 -;; @0025 v98 = call fn0(v0) -;; @0025 v100 = load.i64 notrap aligned region2 v5 -;; @0025 jump block15(v100) -;; -;; block15(v102: i64): -;; @0025 call fn1(v0, v89, v90, v91) +;; block3(v21: i64): +;; @0025 v22 = iconst.i64 4 +;; @0025 v23 = iadd v21, v22 ; v22 = 4 +;; @0025 v18 = uextend.i64 v4 +;; @0025 v24 = iadd v23, v18 +;; v62 = iconst.i64 0 +;; v63 = icmp sge v24, v62 ; v62 = 0 +;; @0025 brif v63, block4, block5(v24) +;; +;; block4: +;; @0025 store.i64 notrap aligned region2 v24, v5 +;; @0025 v28 = call fn0(v0) +;; @0025 v30 = load.i64 notrap aligned region2 v5 +;; @0025 jump block5(v30) +;; +;; block5(v57: i64): +;; @0025 v31 = load.i64 notrap aligned region4 v0+64 +;; @0025 v32 = uextend.i64 v2 +;; @0025 v36 = iadd v32, v18 +;; @0025 v37 = icmp ugt v36, v31 +;; @0025 trapnz v37, heap_oob +;; @0025 v44 = uextend.i64 v3 +;; @0025 v48 = iadd v44, v18 +;; @0025 v49 = icmp ugt v48, v31 +;; @0025 trapnz v49, heap_oob +;; @0025 v38 = load.i64 notrap aligned readonly can_move region3 v0+56 +;; @0025 v42 = iadd v38, v32 +;; @0025 v54 = iadd v38, v44 +;; @0025 call fn1(v0, v42, v54, v18) ;; @0029 jump block1 ;; ;; block1: -;; @0029 store.i64 notrap aligned region2 v102, v5 +;; @0029 store.i64 notrap aligned region2 v57, v5 ;; @0029 return ;; }