diff --git a/crates/cranelift/src/alias_region.rs b/crates/cranelift/src/alias_region.rs index 873f34b7a21b..9963f5ef22d9 100644 --- a/crates/cranelift/src/alias_region.rs +++ b/crates/cranelift/src/alias_region.rs @@ -5,13 +5,12 @@ use cranelift_codegen::{ ir::{self, InstBuilder as _}, }; use wasmtime_environ::{ - BuiltinFunctionIndex, DefinedGlobalIndex, DefinedMemoryIndex, DefinedTableIndex, FuncIndex, - GetPtrSize, GlobalIndex, MemoryIndex, ModuleInternedTypeIndex, PtrSize as _, RuntimeDataIndex, - StaticModuleIndex, TableIndex, TagIndex, VMOffsets, + BuiltinFunctionIndex, DefinedGlobalIndex, DefinedMemoryIndex, DefinedTableIndex, GetPtrSize, + ModuleInternedTypeIndex, PtrSize as _, RuntimeDataIndex, StaticModuleIndex, VMOffsets, component::{ ComponentBuiltinFunctionIndex, LoweredIndex, ResourceIndex, RuntimeCallbackIndex, RuntimeComponentInstanceIndex, RuntimeMemoryIndex, RuntimePostReturnIndex, - VMComponentOffsets, + RuntimeReallocIndex, VMComponentOffsets, }, }; @@ -501,10 +500,6 @@ impl<'a, Offsets> Field<'a, Offsets> { /// Emit a store of `value` to this field relative to `ptr`, a pointer to /// the containing `VM*` structure. - #[allow( - dead_code, - reason = "part of the general `Field` API; not all fields are stored to yet" - )] pub fn store(&mut self, cursor: &mut FuncCursor<'_>, ptr: ir::Value, value: ir::Value) { let flags = self.flags_with_region(cursor.func); cursor @@ -703,594 +698,294 @@ macro_rules! define_vm_type_alias_region_helpers { } wasmtime_environ::for_each_vm_type!(define_vm_type_alias_region_helpers); -impl AliasRegions -where - Offsets: GetPtrSize, -{ - /// Create a new `AliasRegions`. - pub fn new(offsets: Offsets) -> Self { - Self { - pointer_type: ir::Type::int_with_byte_size(offsets.get_ptr_size().size().into()) - .unwrap(), - offsets, - cache: std::collections::HashMap::default(), - } - } - - /// Get the alias region for accesses into the GC heap. - pub fn gc_heap_region(&mut self, func: &mut ir::Function) -> ir::AliasRegion { - self.region(func, AliasRegionKey::GcHeap) - } - - /// Get the alias region for an imported or exported memory access (shared - /// across all imported/exported memories). - pub fn public_memory_region(&mut self, func: &mut ir::Function) -> ir::AliasRegion { - self.region(func, AliasRegionKey::PublicMemory) - } - - /// Get the alias region for accessing a defined memory that is not - /// exported. - pub fn defined_memory_region( - &mut self, - func: &mut ir::Function, - module: StaticModuleIndex, - index: DefinedMemoryIndex, - ) -> ir::AliasRegion { - self.region(func, AliasRegionKey::DefinedMemory { module, index }) - } - - /// Get the alias region for an imported or exported table access (shared - /// across all imported/exported memories). - pub fn public_table_region(&mut self, func: &mut ir::Function) -> ir::AliasRegion { - self.region(func, AliasRegionKey::PublicTable) - } - - /// Get the alias region for accessing a defined table that is not - /// exported. - pub fn defined_table_region( - &mut self, - func: &mut ir::Function, - module: StaticModuleIndex, - index: DefinedTableIndex, - ) -> ir::AliasRegion { - self.region(func, AliasRegionKey::DefinedTable { module, index }) - } - - /// Get the alias region for an imported or exported global access (shared - /// across all imported/exported memories). - pub fn public_global_region(&mut self, func: &mut ir::Function) -> ir::AliasRegion { - self.region(func, AliasRegionKey::PublicGlobal) - } - - /// Get the alias region for accessing a defined global that is not - /// exported. - pub fn defined_global_region( - &mut self, - func: &mut ir::Function, - module: StaticModuleIndex, - index: DefinedGlobalIndex, - ) -> ir::AliasRegion { - self.region(func, AliasRegionKey::DefinedGlobal { module, index }) - } -} - -/// `VMContext`-related methods that are valid for any `VMContext`, regardless -/// of its particular `VMOffsets`. -impl AliasRegions -where - Offsets: GetPtrSize, -{ - /// Get the alias region for the given offset into the `VMContext`. - fn vmctx_region(&mut self, func: &mut ir::Function, offset: u32) -> ir::AliasRegion { - self.region( - func, - AliasRegionKey::Vm { - ty: VmType::VMContext, - offset, - }, - ) - } +/// Define, for each of Wasmtime's vmctx types, an [`AliasRegions`] accessor +/// that returns a wrapper exposing a [`Field`]-returning method per field of +/// that vmctx. +/// +/// For example, `alias_regions.vmctx().epoch_ptr()` is the `VMContext::epoch_ptr` +/// field, and `alias_regions.vmcomponent().callbacks(i)` is the `i`th element of +/// the `VMComponentContext`'s runtime-callbacks array. +/// +/// A vmctx's `static` fields sit at offsets that depend only on the target +/// pointer size, so their accessors are available for any `Offsets: GetPtrSize`. +/// Its `dynamic` fields sit at offsets that additionally depend on the module or +/// component being compiled, so their accessors are only available when the +/// `AliasRegions` carries that vmctx's own fully-computed offsets. +/// +/// A field marked `#[aggregate]` gets no accessor, for the same reason it gets +/// none in [`define_vm_type_alias_region_helpers!`]: it has no single Cranelift +/// type. Instead, its interior is reached by rebasing a [`Field`] of the nested +/// `VM*` type onto the aggregate's own offset within the vmctx (see +/// [`Field::relative_to`]), which produces exactly the same alias region as +/// accessing that `VM*` type through a pointer would. For the few aggregates that +/// have no `VM*` type of their own, and hence no alias region of their own, there +/// are hand-written helpers below. +#[allow( + unused_macro_rules, + reason = "entry shapes and marker attributes are handled uniformly for both \ + sections of both vmctx types, but not every combination occurs" +)] +macro_rules! define_vmctx_alias_region_helpers { + // Classify a field type to its Cranelift `ir::Type`, given `$pt` (the target + // pointer type as an `ir::Type`). + (@field_ty $pt:expr, u32) => { ir::types::I32 }; + (@field_ty $pt:expr, VmPtr < $g:ident >) => { $pt }; - fn vmctx_load( - &mut self, - cursor: &mut FuncCursor<'_>, - ty: ir::Type, - base_flags: ir::MemFlagsData, - vmctx: ir::Value, - offset: u32, - ) -> ir::Value { - let region = self.vmctx_region(cursor.func, offset); - cursor.ins().load( - ty, - base_flags.with_alias_region(Some(region)), - vmctx, - i32::try_from(offset).unwrap(), - ) - } + // Determine the Cranelift type a field is *accessed* as: the classification + // of its `#[access_as = T]` type if it has one, and of its declared type + // otherwise. + (@access_ty $pt:expr, [ $($fty:tt)* ] []) => { + define_vmctx_alias_region_helpers!(@field_ty $pt, $($fty)*) + }; + (@access_ty $pt:expr, $fty:tt [ #[access_as = $($t:tt)*] $($rest:tt)* ]) => { + define_vmctx_alias_region_helpers!(@field_ty $pt, $($t)*) + }; + (@access_ty $pt:expr, $fty:tt [ # $skip:tt $($rest:tt)* ]) => { + define_vmctx_alias_region_helpers!(@access_ty $pt, $fty [ $($rest)* ]) + }; - fn vmctx_store( - &mut self, - cursor: &mut FuncCursor<'_>, - base_flags: ir::MemFlagsData, - vmctx: ir::Value, - offset: u32, - val: ir::Value, - ) { - let region = self.vmctx_region(cursor.func, offset); - cursor.ins().store( - base_flags.with_alias_region(Some(region)), - val, - vmctx, - i32::try_from(offset).unwrap(), + // Apply a field attribute to its access flags. + (@apply_attr $flags:expr, [readonly]) => { $flags.with_readonly() }; + (@apply_attr $flags:expr, [can_move]) => { $flags.with_can_move() }; + (@apply_attr $flags:expr, [access_as = $($t:tt)*]) => { $flags }; + + // Compute a field's access flags and Cranelift type from its declared type + // and marker attributes, and build the `Field` for it at `$offset`. + (@field $Name:ident ($self:expr, $offset:expr) [ $($fty:tt)* ] [ $(# $fattr:tt)* ]) => {{ + let this = $self; + let flags = ir::MemFlagsData::trusted(); + $( let flags = define_vmctx_alias_region_helpers!(@apply_attr flags, $fattr); )* + let ty = define_vmctx_alias_region_helpers!( + @access_ty this.regions.pointer_type, [ $($fty)* ] [ $(# $fattr)* ] ); - } + let offset = $offset; + Field::new(this.regions, VmType::$Name, offset, flags, ty) + }}; - /// Load the `VMContext::magic` field. - pub fn vmctx_magic(&mut self, cursor: &mut FuncCursor<'_>, vmctx: ir::Value) -> ir::Value { - self.vmctx_load( - cursor, - ir::types::I32, - ir::MemFlagsData::trusted().with_readonly().with_can_move(), - vmctx, - self.offsets.get_ptr_size().vmctx_magic().into(), - ) - } + // ### `static` Section Entries - /// Load the `*mut VMStoreContext` value out of the given `*mut VMContext`. - pub fn vmctx_store_context( - &mut self, - cursor: &mut FuncCursor<'_>, - vmctx: ir::Value, - ) -> ir::Value { - self.vmctx_store_context_load(cursor.func) - .emit(cursor, vmctx) - } + (@static_entry $Name:ident $snake:ident align { $al:tt }) => {}; - /// Get a `Load` for the `*mut VMStoreContext` value out of a `*mut VMContext`. - pub fn vmctx_store_context_load(&mut self, func: &mut ir::Function) -> Load { - let offset = u32::from(self.offsets.get_ptr_size().vmctx_store_context()); - let region = self.vmctx_region(func, offset); - Load { - offset, - flags: ir::MemFlagsData::trusted() - .with_readonly() - .with_can_move() - .with_alias_region(Some(region)), - ty: self.pointer_type, - } - } - - /// Load the `*mut i64` epoch pointer out of the given `*mut VMContext`. - pub fn vmctx_epoch_ptr(&mut self, cursor: &mut FuncCursor<'_>, vmctx: ir::Value) -> ir::Value { - self.vmctx_load( - cursor, - self.pointer_type, - ir::MemFlagsData::trusted(), - vmctx, - self.offsets.get_ptr_size().vmctx_epoch_ptr().into(), - ) - } + // Aggregates get no accessor. + (@static_entry $Name:ident $snake:ident $kind:ident { #[aggregate] $($rest:tt)* }) => {}; - /// Load the base pointer of the `[VMSharedTypeIndex]` array out of the - /// given `*mut VMContext`. - pub fn vmctx_shared_type_ids_array( - &mut self, - cursor: &mut FuncCursor<'_>, - vmctx: ir::Value, - ) -> ir::Value { - self.vmctx_load( - cursor, - self.pointer_type, - ir::MemFlagsData::trusted().with_readonly().with_can_move(), - vmctx, - self.offsets.get_ptr_size().vmctx_type_ids_array().into(), - ) - } - - /// Load the collector's heap data pointer out of the `*mut VMContext`. - pub fn vmctx_gc_heap_data( - &mut self, - cursor: &mut FuncCursor<'_>, - vmctx: ir::Value, - ) -> ir::Value { - self.vmctx_load( - cursor, - self.pointer_type, - ir::MemFlagsData::trusted().with_readonly().with_can_move(), - vmctx, - self.offsets.get_ptr_size().vmctx_gc_heap_data().into(), - ) - } - - /// Load the base pointer to the builtin-functions array from a `*mut - /// VMContext`. - pub fn vmctx_builtin_functions( - &mut self, - cursor: &mut FuncCursor<'_>, - vmctx: ir::Value, - ) -> ir::Value { - self.vmctx_load( - cursor, - self.pointer_type, - ir::MemFlagsData::trusted().with_readonly().with_can_move(), - vmctx, - self.offsets - .get_ptr_size() - .vmcontext_builtin_functions() - .into(), - ) - } -} - -/// `VMContext`-related methods that are specific to a particular Wasm module's -/// `VMOffsets`. -impl AliasRegions> { - /// Like `vmctx_load`, but tags the load with a per-import alias region - /// rather than the coarse `VMContext` region. Used for fields of the - /// `VM*Import` structs inlined into the `VMContext`. - fn vmimport_load( - &mut self, - cursor: &mut FuncCursor<'_>, - ty: ir::Type, - base_flags: ir::MemFlagsData, - vmctx: ir::Value, - vmctx_offset: u32, - field_offset: u32, - vm_type: VmType, - ) -> ir::Value { - let region = self.region( - cursor.func, - AliasRegionKey::Vm { - ty: vm_type, - offset: field_offset, - }, - ); - cursor.ins().load( - ty, - base_flags.with_alias_region(Some(region)), - vmctx, - i32::try_from(vmctx_offset).unwrap(), - ) - } - - /// Load the imported tag's `VMTagImport::vmctx` field from the `*mut - /// VMContext`. - pub fn vmctx_vmtag_import_vmctx( - &mut self, - cursor: &mut FuncCursor<'_>, - vmctx: ir::Value, - tag: TagIndex, - ) -> ir::Value { - self.vmimport_load( - cursor, - self.pointer_type, - ir::MemFlagsData::trusted().with_readonly().with_can_move(), - vmctx, - self.offsets.vmctx_vmtag_import_vmctx(tag), - self.offsets.ptr.vm_tag_import().vmctx().into(), - VmType::VMTagImport, - ) - } - - /// Load the imported tag's `VMTagImport::index` field from the `*mut - /// VMContext`. - pub fn vmctx_vmtag_import_index( - &mut self, - cursor: &mut FuncCursor<'_>, - vmctx: ir::Value, - tag: TagIndex, - ) -> ir::Value { - self.vmimport_load( - cursor, - ir::types::I32, - ir::MemFlagsData::trusted().with_readonly().with_can_move(), - vmctx, - self.offsets.vmctx_vmtag_import_index(tag), - self.offsets.ptr.vm_tag_import().index().into(), - VmType::VMTagImport, - ) - } + (@static_entry $Name:ident $snake:ident field { + $(# $fattr:tt)* $fname:ident : $($fty:tt)* + }) => { + #[doc = concat!( + "Get the [`Field`] for the `", stringify!($fname), "` field of `", + stringify!($Name), "`." + )] + pub fn $fname(self) -> Field<'a, Offsets> { + let offset = u32::from(self.regions.offsets.get_ptr_size().$snake().$fname()); + define_vmctx_alias_region_helpers!( + @field $Name (self, offset) [ $($fty)* ] [ $(# $fattr)* ] + ) + } + }; - /// Load the imported tag's `VMTagImport::from` field from the `*mut - /// VMContext`. - pub fn vmctx_vmtag_import_from( - &mut self, - cursor: &mut FuncCursor<'_>, - vmctx: ir::Value, - tag: TagIndex, - ) -> ir::Value { - self.vmimport_load( - cursor, - self.pointer_type, - ir::MemFlagsData::trusted().with_readonly().with_can_move(), - vmctx, - self.offsets.vmctx_vmtag_import_from(tag), - self.offsets.ptr.vm_tag_import().from().into(), - VmType::VMTagImport, - ) - } + // ### `dynamic` Section Entries - /// Load the import function's `VMFunctionImport::vmctx` field from the - /// `*mut VMContext`. - pub fn vmctx_vmfunction_import_vmctx( - &mut self, - cursor: &mut FuncCursor<'_>, - vmctx: ir::Value, - func: FuncIndex, - ) -> ir::Value { - self.vmimport_load( - cursor, - self.pointer_type, - ir::MemFlagsData::trusted().with_readonly().with_can_move(), - vmctx, - self.offsets.vmctx_vmfunction_import_vmctx(func), - self.offsets.ptr.vm_function_import().vmctx().into(), - VmType::VMFunctionImport, - ) - } + (@dynamic_entry $Name:ident $Offsets:tt align { $al:tt }) => {}; - /// Load the import function's `VMFunctionImport::wasm_call` field from the - /// `*mut VMContext`. - pub fn vmctx_vmfunction_import_wasm_call( - &mut self, - cursor: &mut FuncCursor<'_>, - vmctx: ir::Value, - func: FuncIndex, - ) -> ir::Value { - self.vmimport_load( - cursor, - self.pointer_type, - ir::MemFlagsData::trusted().with_readonly().with_can_move(), - vmctx, - self.offsets.vmctx_vmfunction_import_wasm_call(func), - self.offsets.ptr.vm_function_import().wasm_call().into(), - VmType::VMFunctionImport, - ) - } + // Aggregates get no accessor. + (@dynamic_entry $Name:ident $Offsets:tt $kind:ident { #[aggregate] $($rest:tt)* }) => {}; - /// Load the imported memory's `VMMemoryImport::vmctx` field from the `*mut - /// VMContext`. - pub fn vmctx_vmmemory_import_vmctx( - &mut self, - cursor: &mut FuncCursor<'_>, - vmctx: ir::Value, - memory: MemoryIndex, - ) -> ir::Value { - let mem_offset = self.offsets.vmctx_vmmemory_import(memory); - let mem_vmctx_offset = mem_offset + u32::from(self.offsets.ptr.vm_memory_import().vmctx()); - self.vmimport_load( - cursor, - self.pointer_type, - ir::MemFlagsData::trusted().with_readonly().with_can_move(), - vmctx, - mem_vmctx_offset, - self.offsets.ptr.vm_memory_import().vmctx().into(), - VmType::VMMemoryImport, - ) - } + (@dynamic_entry $Name:ident [ $($Offsets:tt)* ] array { + $(# $fattr:tt)* $fname:ident [ $count:ident ; $Index:ident ] : $($fty:tt)* + }) => { + #[doc = concat!( + "Get the [`Field`] for the `index`th element of `", stringify!($Name), + "`'s `", stringify!($fname), "` array.\n\nPanics if `index` is out of \ + bounds for this vmctx." + )] + pub fn $fname(self, index: $Index) -> Field<'a, $($Offsets)*> { + let offset = self.regions.offsets.$fname().at(index); + define_vmctx_alias_region_helpers!( + @field $Name (self, offset) [ $($fty)* ] [ $(# $fattr)* ] + ) + } + }; - /// Load the imported memory's `VMMemoryImport::index` field from the `*mut - /// VMContext`. - pub fn vmctx_vmmemory_import_index( - &mut self, - cursor: &mut FuncCursor<'_>, - vmctx: ir::Value, - memory: MemoryIndex, - ) -> ir::Value { - let mem_offset = self.offsets.vmctx_vmmemory_import(memory); - let mem_index_offset = mem_offset + u32::from(self.offsets.ptr.vm_memory_import().index()); - self.vmimport_load( - cursor, - ir::types::I32, - ir::MemFlagsData::trusted().with_readonly().with_can_move(), - vmctx, - mem_index_offset, - self.offsets.ptr.vm_memory_import().index().into(), - VmType::VMMemoryImport, - ) - } + // A single field, whether unconditionally present (`field`) or only + // conditionally (`optional`). + (@dynamic_entry $Name:ident [ $($Offsets:tt)* ] $kind:ident { + $(# $fattr:tt)* $fname:ident $([ if $flag:ident ])? : $($fty:tt)* + }) => { + #[doc = concat!( + "Get the [`Field`] for the `", stringify!($fname), "` field of `", + stringify!($Name), "`." + $(, "\n\nPanics if `", stringify!($flag), "` is false, in which case \ + this field is not present at all.")? + )] + pub fn $fname(self) -> Field<'a, $($Offsets)*> { + let offset = self.regions.offsets.$fname(); + define_vmctx_alias_region_helpers!( + @field $Name (self, offset) [ $($fty)* ] [ $(# $fattr)* ] + ) + } + }; - /// Load the imported memory's `VMMemoryImport::from` field from the `*mut - /// VMContext`. - pub fn vmctx_vmmemory_import_from( - &mut self, - cursor: &mut FuncCursor<'_>, - vmctx: ir::Value, - memory: MemoryIndex, - ) -> ir::Value { - self.vmctx_vmmemory_import_from_load(cursor.func, memory) - .emit(cursor, vmctx) - } + // Emit the `impl` block holding the accessors for the dynamically-positioned + // fields. + (@dynamic_impl $Name:ident [ $($Offsets:tt)* ] $OffsetsTt:tt { + $($kind:ident $entry:tt)* + }) => { + #[allow( + dead_code, + reason = "generated uniformly for every field; not all fields are \ + accessed by compiled code" + )] + impl<'a> $Name<'a, $($Offsets)*> { + $( + define_vmctx_alias_region_helpers!( + @dynamic_entry $Name $OffsetsTt $kind $entry + ); + )* + } + }; - /// Get a `Load` for the imported memory's `VMMemoryImport::from` field from - /// a `*mut VMContext`. - pub fn vmctx_vmmemory_import_from_load( - &mut self, - func: &mut ir::Function, - memory: MemoryIndex, - ) -> Load { - let mem_offset = self.offsets.vmctx_vmmemory_import(memory); - let offset = mem_offset + u32::from(self.offsets.ptr.vm_memory_import().from()); - let region = self.region( - func, - AliasRegionKey::Vm { - ty: VmType::VMMemoryImport, - offset: self.offsets.ptr.vm_memory_import().from().into(), - }, - ); - Load { - offset, - flags: ir::MemFlagsData::trusted() - .with_readonly() - .with_can_move() - .with_alias_region(Some(region)), - ty: self.pointer_type, + // Emit the accessor `struct` and both `impl` blocks for one vmctx type. + (@emit $Name:ident $snake:ident $Offsets:tt + static { $($skind:ident $sentry:tt)* } + dynamic { $($dyn:tt)* } + ) => { + #[doc = concat!( + "An [`AliasRegions`] accessor for the fields of a `", stringify!($Name), + "`." + )] + pub struct $Name<'a, Offsets> { + regions: &'a mut AliasRegions, } - } - /// Load the imported table's `VMTableImport::vmctx` field from the `*mut - /// VMContext`. - pub fn vmctx_vmtable_import_vmctx( - &mut self, - cursor: &mut FuncCursor<'_>, - vmctx: ir::Value, - table: TableIndex, - ) -> ir::Value { - let table_offset = self.offsets.vmctx_vmtable_import(table); - let table_vmctx_offset = - table_offset + u32::from(self.offsets.ptr.vm_table_import().vmctx()); - self.vmimport_load( - cursor, - self.pointer_type, - ir::MemFlagsData::trusted().with_readonly().with_can_move(), - vmctx, - table_vmctx_offset, - self.offsets.ptr.vm_table_import().vmctx().into(), - VmType::VMTableImport, - ) - } + impl AliasRegions { + #[doc = concat!( + "Get an accessor for the fields of a `", stringify!($Name), "`." + )] + pub fn $snake(&mut self) -> $Name<'_, Offsets> { + $Name { regions: self } + } + } - /// Load the imported table's `VMTableImport::index` field from the `*mut - /// VMContext`. - pub fn vmctx_vmtable_import_index( - &mut self, - cursor: &mut FuncCursor<'_>, - vmctx: ir::Value, - table: TableIndex, - ) -> ir::Value { - let table_offset = self.offsets.vmctx_vmtable_import(table); - let table_index_offset = - table_offset + u32::from(self.offsets.ptr.vm_table_import().index()); - self.vmimport_load( - cursor, - ir::types::I32, - ir::MemFlagsData::trusted().with_readonly().with_can_move(), - vmctx, - table_index_offset, - self.offsets.ptr.vm_table_import().index().into(), - VmType::VMTableImport, - ) - } + // A statically-positioned field's offset depends only on the target + // pointer size, so these accessors work with any `Offsets`. + #[allow( + dead_code, + reason = "generated uniformly for every field; not all fields are \ + accessed by compiled code" + )] + impl<'a, Offsets> $Name<'a, Offsets> + where + Offsets: GetPtrSize, + { + $( define_vmctx_alias_region_helpers!(@static_entry $Name $snake $skind $sentry); )* + } - /// Get a `Load` for the imported table's `VMTableImport::from` field (a - /// `*mut VMTableDefinition`) out of a `*mut VMContext`. - pub fn vmctx_vmtable_from_load(&mut self, func: &mut ir::Function, table: TableIndex) -> Load { - let offset = self.offsets.vmctx_vmtable_from(table); - let region = self.region( - func, - AliasRegionKey::Vm { - ty: VmType::VMTableImport, - offset: self.offsets.ptr.vm_table_import().from().into(), - }, + // A dynamically-positioned field's offset depends on the module or + // component being compiled, so these accessors require this vmctx's own + // fully-computed offsets. + define_vmctx_alias_region_helpers!( + @dynamic_impl $Name $Offsets $Offsets { $($dyn)* } ); - Load { - offset, - flags: ir::MemFlagsData::trusted() - .with_readonly() - .with_can_move() - .with_alias_region(Some(region)), - ty: self.pointer_type, + }; + + // Map each vmctx type to the offsets type that computes its + // dynamically-positioned fields' offsets. + (@one VMContext $snake:ident static { $($stat:tt)* } dynamic { $($dyn:tt)* }) => { + define_vmctx_alias_region_helpers!(@emit VMContext $snake [ VMOffsets ] + static { $($stat)* } dynamic { $($dyn)* }); + }; + (@one VMComponentContext $snake:ident static { $($stat:tt)* } dynamic { $($dyn:tt)* }) => { + define_vmctx_alias_region_helpers!( + @emit VMComponentContext $snake [ VMComponentOffsets ] + static { $($stat)* } dynamic { $($dyn)* } + ); + }; + + // Top-level entry. + ( $( + { + $Name:ident $snake:ident + static { $($stat:tt)* } + dynamic { $($dyn:tt)* } + } + )* ) => { + $( + define_vmctx_alias_region_helpers!(@one $Name $snake + static { $($stat)* } dynamic { $($dyn)* }); + )* + }; +} +wasmtime_environ::for_each_vmctx_type!(define_vmctx_alias_region_helpers); + +impl AliasRegions +where + Offsets: GetPtrSize, +{ + /// Create a new `AliasRegions`. + pub fn new(offsets: Offsets) -> Self { + Self { + pointer_type: ir::Type::int_with_byte_size(offsets.get_ptr_size().size().into()) + .unwrap(), + offsets, + cache: std::collections::HashMap::default(), } } - /// Load the imported global's address (`VMGlobalImport::from`) out of the - /// `*mut VMContext`. - pub fn vmctx_vmglobal_import_from( - &mut self, - cursor: &mut FuncCursor<'_>, - vmctx: ir::Value, - global: GlobalIndex, - ) -> ir::Value { - let from_offset = self.offsets.vmctx_vmglobal_import_from(global); - self.vmimport_load( - cursor, - self.pointer_type, - ir::MemFlagsData::trusted().with_readonly().with_can_move(), - vmctx, - from_offset, - self.offsets.ptr.vm_global_import().from().into(), - VmType::VMGlobalImport, - ) + /// Get the alias region for accesses into the GC heap. + pub fn gc_heap_region(&mut self, func: &mut ir::Function) -> ir::AliasRegion { + self.region(func, AliasRegionKey::GcHeap) } - /// Load the defined memory's `*mut VMMemoryDefinition` out of the `*mut - /// VMContext`. - pub fn vmctx_vmmemory_pointer( - &mut self, - cursor: &mut FuncCursor<'_>, - vmctx: ir::Value, - memory: DefinedMemoryIndex, - ) -> ir::Value { - self.vmctx_vmmemory_pointer_load(cursor.func, memory) - .emit(cursor, vmctx) + /// Get the alias region for an imported or exported memory access (shared + /// across all imported/exported memories). + pub fn public_memory_region(&mut self, func: &mut ir::Function) -> ir::AliasRegion { + self.region(func, AliasRegionKey::PublicMemory) } - /// Get a `Load` for the defined memory's `*mut VMMemoryDefinition` out of a - /// `*mut VMContext`. - pub fn vmctx_vmmemory_pointer_load( + /// Get the alias region for accessing a defined memory that is not + /// exported. + pub fn defined_memory_region( &mut self, func: &mut ir::Function, - memory: DefinedMemoryIndex, - ) -> Load { - let offset = self.offsets.vmctx_vmmemory_pointer(memory); - let region = self.vmctx_region(func, offset); - Load { - offset, - flags: ir::MemFlagsData::trusted() - .with_readonly() - .with_can_move() - .with_alias_region(Some(region)), - ty: self.pointer_type, - } + module: StaticModuleIndex, + index: DefinedMemoryIndex, + ) -> ir::AliasRegion { + self.region(func, AliasRegionKey::DefinedMemory { module, index }) } - /// Load the base of the given runtime data out of the `*mut VMContext`. - pub fn vmctx_runtime_data_base( - &mut self, - cursor: &mut FuncCursor<'_>, - vmctx: ir::Value, - runtime_data: RuntimeDataIndex, - ) -> ir::Value { - self.vmctx_load( - cursor, - self.pointer_type, - ir::MemFlagsData::trusted(), - vmctx, - self.offsets.vmctx_runtime_data_base(runtime_data), - ) + /// Get the alias region for an imported or exported table access (shared + /// across all imported/exported memories). + pub fn public_table_region(&mut self, func: &mut ir::Function) -> ir::AliasRegion { + self.region(func, AliasRegionKey::PublicTable) } - /// Load the length of the given runtime data out of the `*mut VMContext`. - pub fn vmctx_runtime_data_length( + /// Get the alias region for accessing a defined table that is not + /// exported. + pub fn defined_table_region( &mut self, - cursor: &mut FuncCursor<'_>, - vmctx: ir::Value, - runtime_data: RuntimeDataIndex, - ) -> ir::Value { - self.vmctx_load( - cursor, - ir::types::I32, - ir::MemFlagsData::trusted(), - vmctx, - self.offsets.vmctx_runtime_data_length(runtime_data), - ) + func: &mut ir::Function, + module: StaticModuleIndex, + index: DefinedTableIndex, + ) -> ir::AliasRegion { + self.region(func, AliasRegionKey::DefinedTable { module, index }) + } + + /// Get the alias region for an imported or exported global access (shared + /// across all imported/exported memories). + pub fn public_global_region(&mut self, func: &mut ir::Function) -> ir::AliasRegion { + self.region(func, AliasRegionKey::PublicGlobal) } - /// Load the length of the given runtime data out of the `*mut VMContext`. - pub fn store_vmctx_runtime_data_length( + /// Get the alias region for accessing a defined global that is not + /// exported. + pub fn defined_global_region( &mut self, - cursor: &mut FuncCursor<'_>, - vmctx: ir::Value, - runtime_data: RuntimeDataIndex, - new_length: ir::Value, - ) { - self.vmctx_store( - cursor, - ir::MemFlagsData::trusted(), - vmctx, - self.offsets.vmctx_runtime_data_length(runtime_data), - new_length, - ) + func: &mut ir::Function, + module: StaticModuleIndex, + index: DefinedGlobalIndex, + ) -> ir::AliasRegion { + self.region(func, AliasRegionKey::DefinedGlobal { module, index }) } } @@ -2332,243 +2027,45 @@ where } } -/// `VMComponentContext`-related methods, used when compiling component -/// trampolines. +/// `VMComponentContext` fields that are not simply one of the layout's own +/// fields, and so are not generated by [`define_vmctx_alias_region_helpers!`]. impl AliasRegions> { - fn vmcomponent_region(&mut self, func: &mut ir::Function, offset: u32) -> ir::AliasRegion { - self.region( - func, - AliasRegionKey::Vm { - ty: VmType::VMComponentContext, - offset, - }, - ) - } - - fn vmcomponent_load( - &mut self, - cursor: &mut FuncCursor<'_>, - ty: ir::Type, - base_flags: ir::MemFlagsData, - vmctx: ir::Value, - offset: u32, - ) -> ir::Value { - let region = self.vmcomponent_region(cursor.func, offset); - cursor.ins().load( - ty, - base_flags.with_alias_region(Some(region)), - vmctx, - i32::try_from(offset).unwrap(), - ) - } - - fn vmcomponent_store( - &mut self, - cursor: &mut FuncCursor<'_>, - base_flags: ir::MemFlagsData, - vmctx: ir::Value, - offset: u32, - val: ir::Value, - ) { - let region = self.vmcomponent_region(cursor.func, offset); - cursor.ins().store( - base_flags.with_alias_region(Some(region)), - val, - vmctx, - i32::try_from(offset).unwrap(), - ); - } - - /// Load a lowering's host-data pointer from the `VMComponentContext`. - pub fn vmcomponent_lowering_data( - &mut self, - cursor: &mut FuncCursor<'_>, - vmctx: ir::Value, - index: LoweredIndex, - ) -> ir::Value { - self.vmcomponent_load( - cursor, - self.pointer_type, - ir::MemFlagsData::trusted(), - vmctx, - self.offsets.lowering_data(index), - ) - } - - /// Load a lowering's host callee pointer from the `VMComponentContext`. + /// Get the [`Field`] for the `callee` of the `index`th lowering in the + /// `VMComponentContext`'s `lowerings` array. pub fn vmcomponent_lowering_callee( &mut self, - cursor: &mut FuncCursor<'_>, - vmctx: ir::Value, index: LoweredIndex, - ) -> ir::Value { - self.vmcomponent_load( - cursor, - self.pointer_type, - ir::MemFlagsData::trusted(), - vmctx, - self.offsets.lowering_callee(index), - ) - } - - /// Load the current task's `may_block` flag from the `VMComponentContext`. - pub fn vmcomponent_task_may_block( - &mut self, - cursor: &mut FuncCursor<'_>, - vmctx: ir::Value, - ) -> ir::Value { - self.vmcomponent_load( - cursor, - ir::types::I32, - ir::MemFlagsData::trusted().with_readonly(), - vmctx, - self.offsets.task_may_block(), - ) - } - - /// Store the current task's `may_block` flag into the `VMComponentContext`. - pub fn store_vmcomponent_task_may_block( - &mut self, - cursor: &mut FuncCursor<'_>, - vmctx: ir::Value, - val: ir::Value, - ) { - self.vmcomponent_store( - cursor, - ir::MemFlagsData::trusted(), - vmctx, - self.offsets.task_may_block(), - val, - ) - } - - /// Load a resource's destructor function pointer from the - /// `VMComponentContext`. - pub fn vmcomponent_resource_destructor( - &mut self, - cursor: &mut FuncCursor<'_>, - vmctx: ir::Value, - index: ResourceIndex, - ) -> ir::Value { - self.vmcomponent_load( - cursor, - self.pointer_type, - ir::MemFlagsData::trusted().with_readonly(), - vmctx, - self.offsets.resource_destructor(index), - ) - } - - /// Load a runtime memory's `*mut VMMemoryDefinition` from the - /// `VMComponentContext`. - pub fn vmcomponent_runtime_memory( - &mut self, - cursor: &mut FuncCursor<'_>, - vmctx: ir::Value, - index: RuntimeMemoryIndex, - ) -> ir::Value { - self.vmcomponent_load( - cursor, - self.pointer_type, - ir::MemFlagsData::trusted(), - vmctx, - self.offsets.runtime_memory(index), - ) - } - - /// Load a runtime callback function pointer from the `VMComponentContext`. - pub fn vmcomponent_runtime_callback( - &mut self, - cursor: &mut FuncCursor<'_>, - vmctx: ir::Value, - index: RuntimeCallbackIndex, - ) -> ir::Value { - self.vmcomponent_load( - cursor, - self.pointer_type, - ir::MemFlagsData::trusted(), - vmctx, - self.offsets.runtime_callback(index), - ) + ) -> Field<'_, VMComponentOffsets> { + let offset = self.offsets.lowering_callee(index); + self.vmlowering_field(offset) } - /// Load a runtime post-return function pointer from the - /// `VMComponentContext`. - pub fn vmcomponent_runtime_post_return( + /// Get the [`Field`] for the host data of the `index`th lowering in the + /// `VMComponentContext`'s `lowerings` array. + pub fn vmcomponent_lowering_data( &mut self, - cursor: &mut FuncCursor<'_>, - vmctx: ir::Value, - index: RuntimePostReturnIndex, - ) -> ir::Value { - self.vmcomponent_load( - cursor, - self.pointer_type, - ir::MemFlagsData::trusted(), - vmctx, - self.offsets.runtime_post_return(index), - ) + index: LoweredIndex, + ) -> Field<'_, VMComponentOffsets> { + let offset = self.offsets.lowering_data(index); + self.vmlowering_field(offset) } - /// Load the base pointer of the component builtins array from the + /// Get the [`Field`] at `offset` within a `VMLowering` inlined into the /// `VMComponentContext`. - pub fn vmcomponent_builtins( - &mut self, - cursor: &mut FuncCursor<'_>, - vmctx: ir::Value, - ) -> ir::Value { - self.vmcomponent_load( - cursor, - self.pointer_type, - ir::MemFlagsData::trusted().with_readonly(), - vmctx, - self.offsets.builtins(), - ) - } - - /// Load a component instance's `may_leave` flag from the `VMComponentContext`. - pub fn vmcomponent_instance_may_leave( - &mut self, - cursor: &mut FuncCursor<'_>, - vmctx: ir::Value, - instance: RuntimeComponentInstanceIndex, - ) -> ir::Value { - self.vmcomponent_load( - cursor, - ir::types::I32, + /// + /// Unlike the other aggregates inlined into a vmctx, `VMLowering` is not one + /// of the types defined by `for_each_vm_type!`, and so has no alias region of + /// its own to rebase onto the aggregate's offset. Its fields are therefore + /// part of the containing `VMComponentContext`'s region, keyed by their + /// offset within it. + fn vmlowering_field(&mut self, offset: u32) -> Field<'_, VMComponentOffsets> { + let ty = self.pointer_type; + Field::new( + self, + VmType::VMComponentContext, + offset, ir::MemFlagsData::trusted(), - vmctx, - self.offsets.may_leave(instance), - ) - } -} - -/// `VMComponentContext`-related methods that need to be generic over `Offsets` -/// due to the call context. -impl AliasRegions -where - Offsets: GetPtrSize, -{ - /// Load a field at `offset` within a `VMComponentContext`. - pub fn vmcomponent_context_generic_load( - &mut self, - cursor: &mut FuncCursor<'_>, - ty: ir::Type, - base_flags: ir::MemFlagsData, - vmctx: ir::Value, - offset: u32, - ) -> ir::Value { - let region = self.region( - cursor.func, - AliasRegionKey::Vm { - ty: VmType::VMComponentContext, - offset, - }, - ); - cursor.ins().load( ty, - base_flags.with_alias_region(Some(region)), - vmctx, - i32::try_from(offset).unwrap(), ) } } diff --git a/crates/cranelift/src/compiler.rs b/crates/cranelift/src/compiler.rs index 5f3fdee1c6e0..7aa781c40fd6 100644 --- a/crates/cranelift/src/compiler.rs +++ b/crates/cranelift/src/compiler.rs @@ -230,8 +230,10 @@ impl Compiler { caller_vmctx, wasmtime_environ::VMCONTEXT_MAGIC, ); - let vm_store_context = - alias_regions.vmctx_store_context(&mut builder.cursor(), caller_vmctx); + let vm_store_context = alias_regions + .vmctx() + .store_context() + .load(&mut builder.cursor(), caller_vmctx); save_last_wasm_exit_fp_and_pc( &mut builder, pointer_type, @@ -266,8 +268,10 @@ impl Compiler { // Increment the "execution version" on the VMStoreContext if // guest debugging is enabled. if self.tunables.debug_guest { - let vmstore_ctx_ptr = - alias_regions.vmctx_store_context(&mut builder.cursor(), caller_vmctx); + let vmstore_ctx_ptr = alias_regions + .vmctx() + .store_context() + .load(&mut builder.cursor(), caller_vmctx); let old_version = alias_regions .vmstore_context_execution_version(&mut builder.cursor(), vmstore_ctx_ptr); let new_version = builder.ins().iadd_imm_s(old_version, 1); @@ -343,7 +347,10 @@ impl Compiler { vmctx, wasmtime_environ::VMCONTEXT_MAGIC, ); - let vm_store_context = alias_regions.vmctx_store_context(&mut builder.cursor(), vmctx); + let vm_store_context = alias_regions + .vmctx() + .store_context() + .load(&mut builder.cursor(), vmctx); save_last_wasm_exit_fp_and_pc( &mut builder, pointer_type, @@ -550,7 +557,9 @@ impl wasmtime_environ::Compiler for Compiler { if !isa.triple().is_pulley() { let store_ctx = func_env .alias_regions - .vmctx_store_context_load(&mut context.func); + .vmctx() + .store_context() + .to_deferred_load(&mut context.func); let stack_limit = func_env .alias_regions .vmstore_context_stack_limit_load(&mut context.func); @@ -608,7 +617,7 @@ impl wasmtime_environ::Compiler for Compiler { symbol, wasmtime_environ::VMCONTEXT_MAGIC, |alias_regions, _pointer_type, cursor, vmctx| { - alias_regions.vmctx_store_context(cursor, vmctx) + alias_regions.vmctx().store_context().load(cursor, vmctx) }, ) } @@ -656,7 +665,7 @@ impl wasmtime_environ::Compiler for Compiler { symbol, wasmtime_environ::VMCONTEXT_MAGIC, |alias_regions, _pointer_type, cursor, vmctx| { - alias_regions.vmctx_store_context(cursor, vmctx) + alias_regions.vmctx().store_context().load(cursor, vmctx) }, ), // Delegate to a helper to finish compiling this. @@ -1358,7 +1367,10 @@ impl Compiler { { // Builtins are stored in an array in all `VMContext`s. First load the // base pointer of the array... - let array_addr = alias_regions.vmctx_builtin_functions(&mut builder.cursor(), vmctx); + let array_addr = alias_regions + .vmctx() + .builtin_functions() + .load(&mut builder.cursor(), vmctx); // ... and then load the entry in the array that corresponds to this // builtin. let func_addr = alias_regions.builtin_functions_array_element( @@ -1408,7 +1420,14 @@ impl Compiler { if !self.emit_debug_checks { return; } - let magic = alias_regions.vmctx_magic(&mut builder.cursor(), vmctx); + // NB: a `VMComponentContext`'s `magic` field is deliberately read + // through the `VMContext` accessor: both types keep `magic` at offset + // zero, and this check is the one place that reads a vmctx's magic + // without knowing which of the two it has. + let magic = alias_regions + .vmctx() + .magic() + .load(&mut builder.cursor(), vmctx); let is_expected_vmctx = builder.ins().icmp_imm_s( ir::condcodes::IntCC::Equal, magic, diff --git a/crates/cranelift/src/compiler/component.rs b/crates/cranelift/src/compiler/component.rs index 3d49e8daac86..05281ca85cb3 100644 --- a/crates/cranelift/src/compiler/component.rs +++ b/crates/cranelift/src/compiler/component.rs @@ -155,11 +155,10 @@ impl<'a> TrampolineCompiler<'a> { WasmArgs::ValRawList, |me, params| { let vmctx = params[0]; - let lowering_data = me.alias_regions.vmcomponent_lowering_data( - &mut me.builder.cursor(), - vmctx, - *index, - ); + let lowering_data = me + .alias_regions + .vmcomponent_lowering_data(*index) + .load(&mut me.builder.cursor(), vmctx); params.extend([ lowering_data, me.index_value(*lower_ty), @@ -993,11 +992,10 @@ impl<'a> TrampolineCompiler<'a> { HostCallee::Lowering(index) => { // Load host function pointer from the vmcontext and then call that // indirect function pointer with the list of arguments. - let host_fn = self.alias_regions.vmcomponent_lowering_callee( - &mut self.builder.cursor(), - vmctx, - index, - ); + let host_fn = self + .alias_regions + .vmcomponent_lowering_callee(index) + .load(&mut self.builder.cursor(), vmctx); let host_sig = { let mut sig = ir::Signature::new(CallConv::triple_default(self.isa.triple())); for param in host_args.iter() { @@ -1209,9 +1207,12 @@ impl<'a> TrampolineCompiler<'a> { // Stash the old value of `may_block` and then set it to false. let old_may_block = self .alias_regions - .vmcomponent_task_may_block(&mut self.builder.cursor(), vmctx); + .vmcomponent() + .task_may_block() + .readonly() + .load(&mut self.builder.cursor(), vmctx); let zero = self.builder.ins().iconst(ir::types::I32, i64::from(0)); - self.alias_regions.store_vmcomponent_task_may_block( + self.alias_regions.vmcomponent().task_may_block().store( &mut self.builder.cursor(), vmctx, zero, @@ -1254,11 +1255,11 @@ impl<'a> TrampolineCompiler<'a> { // NB: despite the vmcontext storing nullable funcrefs for function // pointers we know this is statically never null due to the // `has_destructor` check above. - let dtor_func_ref = self.alias_regions.vmcomponent_resource_destructor( - &mut self.builder.cursor(), - vmctx, - index, - ); + let dtor_func_ref = self + .alias_regions + .vmcomponent() + .resource_destructors(index) + .load(&mut self.builder.cursor(), vmctx); if self.compiler.emit_debug_checks { self.builder .ins() @@ -1326,7 +1327,7 @@ impl<'a> TrampolineCompiler<'a> { self.raise_if_host_trapped(result.unwrap()); // Restore the old value of `may_block` - self.alias_regions.store_vmcomponent_task_may_block( + self.alias_regions.vmcomponent().task_may_block().store( &mut self.builder.cursor(), vmctx, old_may_block, @@ -1354,7 +1355,9 @@ impl<'a> TrampolineCompiler<'a> { fn load_memory(&mut self, vmctx: ir::Value, memory: RuntimeMemoryIndex) -> ir::Value { self.alias_regions - .vmcomponent_runtime_memory(&mut self.builder.cursor(), vmctx, memory) + .vmcomponent() + .memories(memory) + .load(&mut self.builder.cursor(), vmctx) } fn load_callback( @@ -1364,11 +1367,11 @@ impl<'a> TrampolineCompiler<'a> { ) -> ir::Value { let pointer_type = self.isa.pointer_type(); match callback { - Some(idx) => self.alias_regions.vmcomponent_runtime_callback( - &mut self.builder.cursor(), - vmctx, - idx, - ), + Some(idx) => self + .alias_regions + .vmcomponent() + .callbacks(idx) + .load(&mut self.builder.cursor(), vmctx), None => self.builder.ins().iconst(pointer_type, 0), } } @@ -1380,11 +1383,11 @@ impl<'a> TrampolineCompiler<'a> { ) -> ir::Value { let pointer_type = self.isa.pointer_type(); match post_return { - Some(idx) => self.alias_regions.vmcomponent_runtime_post_return( - &mut self.builder.cursor(), - vmctx, - idx, - ), + Some(idx) => self + .alias_regions + .vmcomponent() + .post_returns(idx) + .load(&mut self.builder.cursor(), vmctx), None => self.builder.ins().iconst(pointer_type, 0), } } @@ -1402,7 +1405,9 @@ impl<'a> TrampolineCompiler<'a> { // per-process. let builtins_array = self .alias_regions - .vmcomponent_builtins(&mut self.builder.cursor(), vmctx); + .vmcomponent() + .builtins() + .load(&mut self.builder.cursor(), vmctx); // Next load the function pointer at `offset` and return that. self.alias_regions .component_builtin_functions_array_element( @@ -1543,11 +1548,11 @@ impl<'a> TrampolineCompiler<'a> { fn check_may_leave_instance(&mut self, instance: RuntimeComponentInstanceIndex) { let vmctx = self.builder.func.dfg.block_params(self.block0)[0]; - let may_leave = self.alias_regions.vmcomponent_instance_may_leave( - &mut self.builder.cursor(), - vmctx, - instance, - ); + let may_leave = self + .alias_regions + .vmcomponent() + .may_leave(instance) + .load(&mut self.builder.cursor(), vmctx); let (mut traps, builder) = self.traps(); traps.trapz(builder, may_leave, TRAP_CANNOT_LEAVE_COMPONENT); } @@ -1581,7 +1586,9 @@ impl<'a> TrampolineCompiler<'a> { fn load_vm_store_context(&mut self) -> ir::Value { let caller_vmctx = self.abi_load_params()[1]; self.alias_regions - .vmctx_store_context(&mut self.builder.cursor(), caller_vmctx) + .vmctx() + .store_context() + .load(&mut self.builder.cursor(), caller_vmctx) } } @@ -1690,22 +1697,17 @@ impl ComponentCompiler for Compiler { // Implement the array-abi trampoline in terms of calling the // wasm-abi trampoline. Abi::Array => { - let offsets = - VMComponentOffsets::new(self.isa.pointer_bytes(), &component.component); return Ok(self.array_to_wasm_trampoline( key, FuncKey::ComponentTrampoline(Abi::Wasm, trampoline_index), sig, symbol, wasmtime_environ::component::VMCOMPONENT_MAGIC, - |alias_regions, pointer_type, cursor, vmctx| { - alias_regions.vmcomponent_context_generic_load( - cursor, - pointer_type, - ir::MemFlagsData::trusted().with_readonly().with_can_move(), - vmctx, - offsets.vm_store_context(), - ) + |alias_regions, _pointer_type, cursor, vmctx| { + alias_regions + .vmcomponent() + .store_context() + .load(cursor, vmctx) }, )?); } @@ -1774,22 +1776,17 @@ impl ComponentCompiler for Compiler { // Implement the array-abi trampoline in terms of calling the // wasm-abi trampoline. Abi::Array => { - let offsets = - VMComponentOffsets::new(self.isa.pointer_bytes(), &component.component); return Ok(self.array_to_wasm_trampoline( FuncKey::UnsafeIntrinsic(abi, intrinsic), FuncKey::UnsafeIntrinsic(Abi::Wasm, intrinsic), &wasm_func_ty, symbol, wasmtime_environ::component::VMCOMPONENT_MAGIC, - |alias_regions, pointer_type, cursor, vmctx| { - alias_regions.vmcomponent_context_generic_load( - cursor, - pointer_type, - ir::MemFlagsData::trusted().with_readonly().with_can_move(), - vmctx, - offsets.vm_store_context(), - ) + |alias_regions, _pointer_type, cursor, vmctx| { + alias_regions + .vmcomponent() + .store_context() + .load(cursor, vmctx) }, )?); } @@ -2427,7 +2424,9 @@ where let caller_vmctx = params[1]; self.traps .alias_regions() - .vmctx_store_context(&mut self.builder.cursor(), caller_vmctx) + .vmctx() + .store_context() + .load(&mut self.builder.cursor(), caller_vmctx) } } diff --git a/crates/cranelift/src/debug.rs b/crates/cranelift/src/debug.rs index 55a0a7a2fb09..dbfedf8631db 100644 --- a/crates/cranelift/src/debug.rs +++ b/crates/cranelift/src/debug.rs @@ -102,17 +102,20 @@ impl<'a> Compilation<'a> { let memory_offset = if ofs.num_imported_memories > 0 { let index = MemoryIndex::new(0); ModuleMemoryOffset::Imported { - offset_to_vm_memory_definition: ofs.vmctx_vmmemory_import(index) + offset_to_vm_memory_definition: ofs.imported_memories().at(index) + u32::from(ofs.ptr.vm_memory_import().from()), offset_to_memory_base: ofs.ptr.vm_memory_definition().base().into(), } } else if ofs.num_owned_memories > 0 { let index = OwnedMemoryIndex::new(0); - ModuleMemoryOffset::Defined(ofs.vmctx_vmmemory_definition_base(index)) + ModuleMemoryOffset::Defined( + ofs.owned_memories().at(index) + + u32::from(ofs.ptr.vm_memory_definition().base()), + ) } else if ofs.num_defined_memories > 0 { let index = DefinedMemoryIndex::new(0); ModuleMemoryOffset::Imported { - offset_to_vm_memory_definition: ofs.vmctx_vmmemory_pointer(index), + offset_to_vm_memory_definition: ofs.memories().at(index), offset_to_memory_base: ofs.ptr.vm_memory_definition().base().into(), } } else { diff --git a/crates/cranelift/src/func_environ.rs b/crates/cranelift/src/func_environ.rs index 10d20e63b170..d358974a2d67 100644 --- a/crates/cranelift/src/func_environ.rs +++ b/crates/cranelift/src/func_environ.rs @@ -450,12 +450,16 @@ impl<'module_environment> FuncEnvironment<'module_environment> { ) -> (ir::Value, i32) { let vmctx = self.vmctx_val(pos); if let Some(def_index) = self.module.defined_global_index(index) { - let offset = i32::try_from(self.offsets.vmctx_vmglobal_definition(def_index)).unwrap(); + let offset = i32::try_from(self.offsets.globals().at(def_index)).unwrap(); (vmctx, offset) } else { + let import_off = self.offsets.imported_globals().at(index); let addr = self .alias_regions - .vmctx_vmglobal_import_from(pos, vmctx, index); + .vm_global_import() + .from() + .relative_to(import_off) + .load(pos, vmctx); (addr, 0) } } @@ -464,7 +468,9 @@ impl<'module_environment> FuncEnvironment<'module_environment> { fn get_vmstore_context_ptr(&mut self, builder: &mut FunctionBuilder) -> ir::Value { let vmctx = self.vmctx_val(&mut builder.cursor()); self.alias_regions - .vmctx_store_context(&mut builder.cursor(), vmctx) + .vmctx() + .store_context() + .load(&mut builder.cursor(), vmctx) } fn fuel_function_entry(&mut self, builder: &mut FunctionBuilder<'_>) { @@ -816,7 +822,9 @@ impl<'module_environment> FuncEnvironment<'module_environment> { fn epoch_ptr(&mut self, builder: &mut FunctionBuilder<'_>) -> ir::Value { let vmctx = self.vmctx_val(&mut builder.cursor()); self.alias_regions - .vmctx_epoch_ptr(&mut builder.cursor(), vmctx) + .vmctx() + .epoch_ptr() + .load(&mut builder.cursor(), vmctx) } fn epoch_load_current(&mut self, builder: &mut FunctionBuilder<'_>) -> ir::Value { @@ -1176,7 +1184,7 @@ impl<'module_environment> FuncEnvironment<'module_environment> { ) -> ir::Value { let vmctx = self.vmctx_val(pos); // Load the base pointer of the array of `VMSharedTypeIndex`es. - let shared_indices = self.alias_regions.vmctx_shared_type_ids_array(pos, vmctx); + let shared_indices = self.alias_regions.vmctx().type_ids().load(pos, vmctx); // Calculate the offset in that array for this type's entry. @@ -1637,7 +1645,9 @@ impl FuncEnvironment<'_> { // atomically growing it. let mem = self .alias_regions - .vmctx_vmmemory_pointer_load(func, def_index); + .vmctx() + .memories(def_index) + .to_deferred_load(func); let mut base = self.alias_regions.vm_memory_definition().base(); base.can_move(); if base_readonly { @@ -1659,7 +1669,7 @@ impl FuncEnvironment<'_> { // relative to the `vmctx` itself: fold that offset into each // field's offset. let owned_index = self.module.owned_memory_index(def_index); - let vmctx_off = self.offsets.vmctx_vmmemory_definition(owned_index); + let vmctx_off = self.offsets.owned_memories().at(owned_index); let mut base = self.alias_regions.vm_memory_definition().base(); base.can_move(); if base_readonly { @@ -1678,9 +1688,13 @@ impl FuncEnvironment<'_> { ) } } else { + let import_off = self.offsets.imported_memories().at(index); let mem = self .alias_regions - .vmctx_vmmemory_import_from_load(func, index); + .vm_memory_import() + .from() + .relative_to(import_off) + .to_deferred_load(func); let mut base = self.alias_regions.vm_memory_definition().base(); base.can_move(); if base_readonly { @@ -1733,7 +1747,7 @@ impl FuncEnvironment<'_> { if let Some(def_index) = self.module.defined_table_index(index) { // A defined table's `VMTableDefinition` is inlined into the vmctx, // reached at an absolute `vmctx` offset. - let vmctx_off = self.offsets.vmctx_vmtable_definition(def_index); + let vmctx_off = self.offsets.tables().at(def_index); let mut base = self.alias_regions.vm_table_definition().base(); if is_static { base.readonly().can_move(); @@ -1761,7 +1775,13 @@ impl FuncEnvironment<'_> { } else { // An imported table is reached through a `*mut VMTableDefinition` // loaded from the `vmctx`. - let from = self.alias_regions.vmctx_vmtable_from_load(func, index); + let import_off = self.offsets.imported_tables().at(index); + let from = self + .alias_regions + .vm_table_import() + .from() + .relative_to(import_off) + .to_deferred_load(func); let mut base = self.alias_regions.vm_table_definition().base(); if is_static { base.readonly().can_move(); @@ -1822,16 +1842,19 @@ impl FuncEnvironment<'_> { } else { // An imported tag -- we need to load the VMTagImport struct. let vmctx = self.vmctx_val(&mut builder.cursor()); - let from_vmctx = self.alias_regions.vmctx_vmtag_import_vmctx( - &mut builder.cursor(), - vmctx, - tag_index, - ); - let index = self.alias_regions.vmctx_vmtag_import_index( - &mut builder.cursor(), - vmctx, - tag_index, - ); + let import_off = self.offsets.imported_tags().at(tag_index); + let from_vmctx = self + .alias_regions + .vm_tag_import() + .vmctx() + .relative_to(import_off) + .load(&mut builder.cursor(), vmctx); + let index = self + .alias_regions + .vm_tag_import() + .index() + .relative_to(import_off) + .load(&mut builder.cursor(), vmctx); let builtin = self.builtin_functions.get_instance_id(builder.func); let call = builder.ins().call(builtin, &[from_vmctx]); let from_instance_id = builder.func.dfg.inst_results(call)[0]; @@ -1926,11 +1949,14 @@ impl<'a, 'func, 'module_env> Call<'a, 'func, 'module_env> { // First append the callee vmctx address. let vmctx = self.env.vmctx_val(&mut self.builder.cursor()); - let callee_vmctx = self.env.alias_regions.vmctx_vmfunction_import_vmctx( - &mut self.builder.cursor(), - vmctx, - callee_index, - ); + let import_off = self.env.offsets.imported_functions().at(callee_index); + let callee_vmctx = self + .env + .alias_regions + .vm_function_import() + .vmctx() + .relative_to(import_off) + .load(&mut self.builder.cursor(), vmctx); real_call_args.push(callee_vmctx); real_call_args.push(caller_vmctx); @@ -1999,11 +2025,14 @@ impl<'a, 'func, 'module_env> Call<'a, 'func, 'module_env> { } } } - let func_addr = self.env.alias_regions.vmctx_vmfunction_import_wasm_call( - &mut self.builder.cursor(), - vmctx, - callee_index, - ); + let import_off = self.env.offsets.imported_functions().at(callee_index); + let func_addr = self + .env + .alias_regions + .vm_function_import() + .wasm_call() + .relative_to(import_off) + .load(&mut self.builder.cursor(), vmctx); Ok(self.indirect_call_inst(sig_ref, func_addr, &real_call_args)) } @@ -2013,11 +2042,14 @@ impl<'a, 'func, 'module_env> Call<'a, 'func, 'module_env> { // and with different functions. Either way, we have to do the // indirect call. None => { - let func_addr = self.env.alias_regions.vmctx_vmfunction_import_wasm_call( - &mut self.builder.cursor(), - vmctx, - callee_index, - ); + let import_off = self.env.offsets.imported_functions().at(callee_index); + let func_addr = self + .env + .alias_regions + .vm_function_import() + .wasm_call() + .relative_to(import_off) + .load(&mut self.builder.cursor(), vmctx); Ok(self.indirect_call_inst(sig_ref, func_addr, &real_call_args)) } } @@ -2205,11 +2237,14 @@ impl<'a, 'func, 'module_env> Call<'a, 'func, 'module_env> { // equivalent out-of-line teardown via the `exit_sync_call` libcall. self.builder.switch_to_block(slow_block); let vmctx = self.env.vmctx_val(&mut self.builder.cursor()); - let func_addr = self.env.alias_regions.vmctx_vmfunction_import_wasm_call( - &mut self.builder.cursor(), - vmctx, - callee_index, - ); + let import_off = self.env.offsets.imported_functions().at(callee_index); + let func_addr = self + .env + .alias_regions + .vm_function_import() + .wasm_call() + .relative_to(import_off) + .load(&mut self.builder.cursor(), vmctx); self.indirect_call_inst(sig_ref, func_addr, real_call_args); self.builder.ins().jump(cont_block, &[]); @@ -3583,12 +3618,19 @@ impl FuncEnvironment<'_> { // This is an imported memory, so load the vmctx/defined index from // the import definition itself. None => { + let import_off = self.offsets.imported_memories().at(index); let vmctx = self .alias_regions - .vmctx_vmmemory_import_vmctx(pos, cur_vmctx, index); + .vm_memory_import() + .vmctx() + .relative_to(import_off) + .load(pos, cur_vmctx); let index = self .alias_regions - .vmctx_vmmemory_import_index(pos, cur_vmctx, index); + .vm_memory_import() + .index() + .relative_to(import_off) + .load(pos, cur_vmctx); (vmctx, index) } } @@ -3609,12 +3651,19 @@ impl FuncEnvironment<'_> { match self.module.defined_table_index(index) { Some(index) => (cur_vmctx, pos.ins().iconst(I32, i64::from(index.as_u32()))), None => { + let import_off = self.offsets.imported_tables().at(index); let vmctx = self .alias_regions - .vmctx_vmtable_import_vmctx(pos, cur_vmctx, index); + .vm_table_import() + .vmctx() + .relative_to(import_off) + .load(pos, cur_vmctx); let index = self .alias_regions - .vmctx_vmtable_import_index(pos, cur_vmctx, index); + .vm_table_import() + .index() + .relative_to(import_off) + .load(pos, cur_vmctx); (vmctx, index) } } @@ -3665,7 +3714,9 @@ impl FuncEnvironment<'_> { if is_shared { let mem_ptr = self .alias_regions - .vmctx_vmmemory_pointer(pos, vmctx, def_index); + .vmctx() + .memories(def_index) + .load(pos, vmctx); self.alias_regions .vm_memory_definition() .current_length() @@ -3674,7 +3725,7 @@ impl FuncEnvironment<'_> { // A defined, owned memory's `VMMemoryDefinition` is inlined into // the `vmctx` at an absolute offset. let owned_index = self.module.owned_memory_index(def_index); - let vmctx_off = self.offsets.vmctx_vmmemory_definition(owned_index); + let vmctx_off = self.offsets.owned_memories().at(owned_index); self.alias_regions .vm_memory_definition() .current_length() @@ -3682,9 +3733,13 @@ impl FuncEnvironment<'_> { .load(pos, vmctx) } } else { + let import_off = self.offsets.imported_memories().at(index); let mem_ptr = self .alias_regions - .vmctx_vmmemory_import_from(pos, vmctx, index); + .vm_memory_import() + .from() + .relative_to(import_off) + .load(pos, vmctx); if is_shared { self.alias_regions .vm_memory_definition() @@ -4355,12 +4410,10 @@ impl FuncEnvironment<'_> { // the value 0 to the `VMContext`'s slot for this passive data segment. let vmctx = self.vmctx_val(&mut pos); let new_length = pos.ins().iconst(I32, 0); - self.alias_regions.store_vmctx_runtime_data_length( - &mut pos, - vmctx, - runtime_index, - new_length, - ); + self.alias_regions + .vmctx() + .runtime_data_lengths(runtime_index) + .store(&mut pos, vmctx, new_length); Ok(()) } @@ -4650,7 +4703,9 @@ impl FuncEnvironment<'_> { ) -> ir::Value { let vmctx = self.vmctx_val(&mut builder.cursor()); self.alias_regions - .vmctx_runtime_data_length(&mut builder.cursor(), vmctx, runtime_index) + .vmctx() + .runtime_data_lengths(runtime_index) + .load(&mut builder.cursor(), vmctx) } fn load_runtime_data_length_as_pointer( @@ -4669,7 +4724,9 @@ impl FuncEnvironment<'_> { ) -> ir::Value { let vmctx = self.vmctx_val(&mut builder.cursor()); self.alias_regions - .vmctx_runtime_data_base(&mut builder.cursor(), vmctx, runtime_index) + .vmctx() + .runtime_data_bases(runtime_index) + .load(&mut builder.cursor(), vmctx) } pub fn translate_table_copy( diff --git a/crates/cranelift/src/func_environ/gc.rs b/crates/cranelift/src/func_environ/gc.rs index 183b0d1d1cd4..11379b169669 100644 --- a/crates/cranelift/src/func_environ/gc.rs +++ b/crates/cranelift/src/func_environ/gc.rs @@ -1515,7 +1515,11 @@ impl FuncEnvironment<'_> { return heap; } - let store_ctx = self.alias_regions.vmctx_store_context_load(func); + let store_ctx = self + .alias_regions + .vmctx() + .store_context() + .to_deferred_load(func); // The base pointer's load is `can_move` and `readonly` when the GC // heap's base can never move. diff --git a/crates/cranelift/src/func_environ/gc/copying.rs b/crates/cranelift/src/func_environ/gc/copying.rs index e4eac7788512..7f3a56541e67 100644 --- a/crates/cranelift/src/func_environ/gc/copying.rs +++ b/crates/cranelift/src/func_environ/gc/copying.rs @@ -34,7 +34,9 @@ impl CopyingCompiler { let vmctx = func_env.vmctx_val(&mut builder.cursor()); func_env .alias_regions - .vmctx_gc_heap_data(&mut builder.cursor(), vmctx) + .vmctx() + .gc_heap_data() + .load(&mut builder.cursor(), vmctx) } /// Load the current bump pointer and active-space end from a `*mut diff --git a/crates/cranelift/src/func_environ/gc/drc.rs b/crates/cranelift/src/func_environ/gc/drc.rs index fe75676554ab..1bed3771796a 100644 --- a/crates/cranelift/src/func_environ/gc/drc.rs +++ b/crates/cranelift/src/func_environ/gc/drc.rs @@ -109,7 +109,9 @@ impl DrcCompiler { let vmctx = func_env.vmctx_val(&mut builder.cursor()); let heap_data = func_env .alias_regions - .vmctx_gc_heap_data(&mut builder.cursor(), vmctx); + .vmctx() + .gc_heap_data() + .load(&mut builder.cursor(), vmctx); // Load the current first list element, which will be our new next list // element. @@ -175,7 +177,9 @@ impl DrcCompiler { let vmctx = func_env.vmctx_val(&mut builder.cursor()); let heap_data = func_env .alias_regions - .vmctx_gc_heap_data(&mut builder.cursor(), vmctx); + .vmctx() + .gc_heap_data() + .load(&mut builder.cursor(), vmctx); let current_len = func_env .alias_regions .vmdrc_heap_data_current_over_approximated_stack_roots_len( diff --git a/crates/cranelift/src/func_environ/gc/null.rs b/crates/cranelift/src/func_environ/gc/null.rs index b951d9b30ba4..454166c2782a 100644 --- a/crates/cranelift/src/func_environ/gc/null.rs +++ b/crates/cranelift/src/func_environ/gc/null.rs @@ -72,7 +72,9 @@ impl NullCompiler { let vmctx = func_env.vmctx_val(&mut builder.cursor()); let ptr_to_next = func_env .alias_regions - .vmctx_gc_heap_data(&mut builder.cursor(), vmctx); + .vmctx() + .gc_heap_data() + .load(&mut builder.cursor(), vmctx); let next = func_env .alias_regions .vmnull_heap_data_bump_finger(&mut builder.cursor(), ptr_to_next); diff --git a/crates/cranelift/src/func_environ/stack_switching/instructions.rs b/crates/cranelift/src/func_environ/stack_switching/instructions.rs index 690c5d5d5479..8f8b342f9c6c 100644 --- a/crates/cranelift/src/func_environ/stack_switching/instructions.rs +++ b/crates/cranelift/src/func_environ/stack_switching/instructions.rs @@ -1012,11 +1012,15 @@ pub(crate) fn tag_address<'a>( let vmctx = env.vmctx_val(&mut builder.cursor()); let tag_index = wasmtime_environ::TagIndex::from_u32(index); if let Some(def_index) = env.module.defined_tag_index(tag_index) { - let offset = i32::try_from(env.offsets.vmctx_vmtag_definition(def_index)).unwrap(); + let offset = i32::try_from(env.offsets.tags().at(def_index)).unwrap(); builder.ins().iadd_imm_s(vmctx, i64::from(offset)) } else { + let import_off = env.offsets.imported_tags().at(tag_index); env.alias_regions - .vmctx_vmtag_import_from(&mut builder.cursor(), vmctx, tag_index) + .vm_tag_import() + .from() + .relative_to(import_off) + .load(&mut builder.cursor(), vmctx) } } @@ -1033,7 +1037,9 @@ pub fn vmctx_load_stack_chain<'a>( // First we need to get the `VMStoreContext`. let vm_store_context = env .alias_regions - .vmctx_store_context(&mut builder.cursor(), vmctx); + .vmctx() + .store_context() + .load(&mut builder.cursor(), vmctx); let stack_chain_region = env .alias_regions @@ -1061,7 +1067,9 @@ pub fn vmctx_store_stack_chain<'a>( // First we need to get the `VMStoreContext`. let vm_store_context = env .alias_regions - .vmctx_store_context(&mut builder.cursor(), vmctx); + .vmctx() + .store_context() + .load(&mut builder.cursor(), vmctx); let stack_chain_region = env .alias_regions @@ -1571,7 +1579,9 @@ fn translate_resume_impl<'a>( // of the invariants that we maintain for the various stack limits. let vm_runtime_limits_ptr = env .alias_regions - .vmctx_store_context(&mut builder.cursor(), vmctx); + .vmctx() + .store_context() + .load(&mut builder.cursor(), vmctx); parent_csi.load_limits_from_vmcontext(env, builder, vm_runtime_limits_ptr, true); resume_csi.write_limits_to_vmcontext(env, builder, vm_runtime_limits_ptr); @@ -2042,7 +2052,9 @@ pub(crate) fn translate_switch<'a>( // switcher continuation. let vm_runtime_limits_ptr = env .alias_regions - .vmctx_store_context(&mut builder.cursor(), vmctx); + .vmctx() + .store_context() + .load(&mut builder.cursor(), vmctx); switcher_contref_csi.load_limits_from_vmcontext(env, builder, vm_runtime_limits_ptr, false); let revision = switcher_contref.get_revision(env, builder); diff --git a/crates/environ/src/component/vmcomponent_offsets.rs b/crates/environ/src/component/vmcomponent_offsets.rs index 1b1c14834666..937b7445278c 100644 --- a/crates/environ/src/component/vmcomponent_offsets.rs +++ b/crates/environ/src/component/vmcomponent_offsets.rs @@ -1,20 +1,9 @@ -// Currently the `VMComponentContext` allocation by field looks like this: -// -// struct VMComponentContext { -// magic: u32, -// builtins: &'static VMComponentBuiltins, -// limits: *const VMStoreContext, -// may_leave: [VMGlobalDefinition; component.num_runtime_component_instances], -// task_may_block: u32, -// trampoline_func_refs: [VMFuncRef; component.num_trampolines], -// unsafe_intrinsics: [VMFuncRef; component.num_unsafe_intrinsics], -// lowerings: [VMLowering; component.num_lowerings], -// memories: [*mut VMMemoryDefinition; component.num_runtime_memories], -// tables: [VMTable; component.num_runtime_tables], -// reallocs: [*mut VMFuncRef; component.num_runtime_reallocs], -// post_returns: [*mut VMFuncRef; component.num_runtime_post_returns], -// resource_destructors: [*mut VMFuncRef; component.num_resources], -// } +//! Offsets of the fields within a `VMComponentContext`. +//! +//! The layout itself is not defined here: it is defined once, alongside +//! `VMContext`'s, in `for_each_vmctx_type!`. Everything in this module is either +//! generated from that definition or is an offset that is not simply the offset +//! of one of the layout's fields. use crate::GetPtrSize; use crate::PtrSize; @@ -55,10 +44,11 @@ pub struct VMComponentOffsets

{ /// Number of resources within a component which need destructors stored. pub num_resources: u32, - // precalculated offsets of various member fields - magic: u32, - builtins: u32, - vm_store_context: u32, + // Precalculated offsets of the dynamically-positioned fields, one per entry + // in `VMComponentContext`'s `dynamic` section in `for_each_vmctx_type!`, + // plus this `VMComponentContext`'s total size. These are all computed by the + // generated `compute_field_offsets` and read by the generated accessors of + // the same names. may_leave: u32, task_may_block: u32, trampoline_func_refs: u32, @@ -82,16 +72,66 @@ impl GetPtrSize for VMComponentOffsets

{ } } -#[inline] -fn align(offset: u32, align: u32) -> u32 { - assert!(align.is_power_of_two()); - (offset + (align - 1)) & !(align - 1) +impl_vmctx_array_index! { + RuntimeComponentInstanceIndex, + TrampolineIndex, + LoweredIndex, + RuntimeMemoryIndex, + RuntimeTableIndex, + RuntimeReallocIndex, + RuntimeCallbackIndex, + RuntimePostReturnIndex, + ResourceIndex, } +impl crate::VmctxArrayIndex for UnsafeIntrinsic { + #[inline] + fn vmctx_array_index(self) -> u32 { + self.index() + } +} + +/// Generate the offsets of `VMComponentContext`'s dynamically-positioned fields, +/// ignoring every other vmctx type. +macro_rules! define_vmcomponent_offsets_dynamic_offsets { + (@one VMComponentContext $snake:ident { $($dyn:tt)* }) => { + /// Offsets of the dynamically-positioned fields of `VMComponentContext`. + impl VMComponentOffsets

{ + define_vmctx_dynamic_offsets!(@accessors (self) [ $($dyn)* ]); + + define_vmctx_dynamic_offsets!(@compute_fn (self, next) $snake [ $($dyn)* ]); + + /// Return the size of the `VMComponentContext` allocation. + #[inline] + pub fn size_of_vmctx(&self) -> u32 { + self.size + } + } + }; + (@one $other:ident $snake:ident { $($dyn:tt)* }) => {}; + + ( $( + { + $Name:ident $snake:ident + static { $($stat:tt)* } + dynamic { $($dyn:tt)* } + } + )* ) => { + $( define_vmcomponent_offsets_dynamic_offsets!(@one $Name $snake { $($dyn)* }); )* + }; +} + +for_each_vmctx_type!(define_vmcomponent_offsets_dynamic_offsets); + impl VMComponentOffsets

{ /// Creates a new set of offsets for the `component` specified configured /// additionally for the `ptr` size specified. pub fn new(ptr: P, component: &Component) -> Self { + // This is required by the implementation of + // `VMComponentContext::from_opaque`. If this value changes then this + // location needs to be updated. + assert_eq!(ptr.vmcomponent().magic(), 0); + let mut ret = Self { ptr, num_lowerings: component.num_lowerings, @@ -125,9 +165,6 @@ impl VMComponentOffsets

{ 0 }, num_resources: component.num_resources, - magic: 0, - builtins: 0, - vm_store_context: 0, may_leave: 0, task_may_block: 0, trampoline_func_refs: 0, @@ -142,58 +179,9 @@ impl VMComponentOffsets

{ size: 0, }; - // Convenience functions for checked addition and multiplication. - // As side effect this reduces binary size by using only a single - // `#[track_caller]` location for each function instead of one for - // each individual invocation. - #[inline] - fn cmul(count: u32, size: u8) -> u32 { - count.checked_mul(u32::from(size)).unwrap() - } - - let mut next_field_offset = 0; - - macro_rules! fields { - (size($field:ident) = $size:expr, $($rest:tt)*) => { - ret.$field = next_field_offset; - next_field_offset = next_field_offset.checked_add(u32::from($size)).unwrap(); - fields!($($rest)*); - }; - (align($align:expr), $($rest:tt)*) => { - next_field_offset = align(next_field_offset, $align); - fields!($($rest)*); - }; - () => {}; - } - - fields! { - size(magic) = 4u32, - align(u32::from(ret.ptr.size())), - size(builtins) = ret.ptr.size(), - size(vm_store_context) = ret.ptr.size(), - align(16), - size(may_leave) = cmul(ret.num_runtime_component_instances, ret.ptr.vm_global_definition().size()), - size(task_may_block) = ret.ptr.vm_global_definition().size(), - align(u32::from(ret.ptr.size())), - size(trampoline_func_refs) = cmul(ret.num_trampolines, ret.ptr.vm_func_ref().size()), - size(intrinsic_func_refs) = cmul(ret.num_unsafe_intrinsics, ret.ptr.vm_func_ref().size()), - size(lowerings) = cmul(ret.num_lowerings, ret.ptr.size() * 2), - size(memories) = cmul(ret.num_runtime_memories, ret.ptr.size()), - size(tables) = cmul(ret.num_runtime_tables, ret.ptr.vm_table_import().size()), - size(reallocs) = cmul(ret.num_runtime_reallocs, ret.ptr.size()), - size(callbacks) = cmul(ret.num_runtime_callbacks, ret.ptr.size()), - size(post_returns) = cmul(ret.num_runtime_post_returns, ret.ptr.size()), - size(resource_destructors) = cmul(ret.num_resources, ret.ptr.size()), - } - - ret.size = next_field_offset; - - // This is required by the implementation of - // `VMComponentContext::from_opaque`. If this value changes then this - // location needs to be updated. - assert_eq!(ret.magic, 0); + ret.compute_field_offsets(); - return ret; + ret } /// The size, in bytes, of the host pointer. @@ -202,86 +190,16 @@ impl VMComponentOffsets

{ self.ptr.size() } - /// The offset of the `magic` field. - #[inline] - pub fn magic(&self) -> u32 { - self.magic - } - - /// The offset of the `builtins` field. - #[inline] - pub fn builtins(&self) -> u32 { - self.builtins - } - - /// The offset of the `may_leave` flag for the given component instance. - #[inline] - pub fn may_leave(&self, index: RuntimeComponentInstanceIndex) -> u32 { - assert!(index.as_u32() < self.num_runtime_component_instances); - self.may_leave + index.as_u32() * u32::from(self.ptr.vm_global_definition().size()) - } - - /// The offset of the `task_may_block` field. - pub fn task_may_block(&self) -> u32 { - self.task_may_block - } - - /// The offset of the `vm_store_context` field. - #[inline] - pub fn vm_store_context(&self) -> u32 { - self.vm_store_context - } - - /// The offset of the `trampoline_func_refs` field. - #[inline] - pub fn trampoline_func_refs(&self) -> u32 { - self.trampoline_func_refs - } - - /// The offset of `VMFuncRef` for the `index` specified. - #[inline] - pub fn trampoline_func_ref(&self, index: TrampolineIndex) -> u32 { - assert!(index.as_u32() < self.num_trampolines); - self.trampoline_func_refs() + index.as_u32() * u32::from(self.ptr.vm_func_ref().size()) - } - - /// The offset of the `unsafe_intrinsic_func_refs` field. - #[inline] - pub fn unsafe_intrinsic_func_refs(&self) -> u32 { - self.intrinsic_func_refs - } - - /// The offset of the `VMFuncRef` for the `intrinsic` specified. - #[inline] - pub fn unsafe_intrinsic_func_ref(&self, intrinsic: UnsafeIntrinsic) -> u32 { - assert!(intrinsic.index() < self.num_unsafe_intrinsics); - self.unsafe_intrinsic_func_refs() - + intrinsic.index() * u32::from(self.ptr.vm_func_ref().size()) - } - - /// The offset of the `lowerings` field. - #[inline] - pub fn lowerings(&self) -> u32 { - self.lowerings - } - - /// The offset of the `VMLowering` for the `index` specified. - #[inline] - pub fn lowering(&self, index: LoweredIndex) -> u32 { - assert!(index.as_u32() < self.num_lowerings); - self.lowerings() + index.as_u32() * u32::from(2 * self.ptr.size()) - } - /// The offset of the `callee` for the `index` specified. #[inline] pub fn lowering_callee(&self, index: LoweredIndex) -> u32 { - self.lowering(index) + self.lowering_callee_offset() + self.lowerings().at(index) + self.lowering_callee_offset() } /// The offset of the `data` for the `index` specified. #[inline] pub fn lowering_data(&self, index: LoweredIndex) -> u32 { - self.lowering(index) + self.lowering_data_offset() + self.lowerings().at(index) + self.lowering_data_offset() } /// The size of the `VMLowering` type @@ -301,93 +219,4 @@ impl VMComponentOffsets

{ pub fn lowering_data_offset(&self) -> u32 { u32::from(self.ptr.size()) } - - /// The offset of the base of the `runtime_memories` field - #[inline] - pub fn runtime_memories(&self) -> u32 { - self.memories - } - - /// The offset of the `*mut VMMemoryDefinition` for the runtime index - /// provided. - #[inline] - pub fn runtime_memory(&self, index: RuntimeMemoryIndex) -> u32 { - assert!(index.as_u32() < self.num_runtime_memories); - self.runtime_memories() + index.as_u32() * u32::from(self.ptr.size()) - } - - /// The offset of the base of the `runtime_tables` field - #[inline] - pub fn runtime_tables(&self) -> u32 { - self.tables - } - - /// The offset of the table for the runtime index provided. - #[inline] - pub fn runtime_table(&self, index: RuntimeTableIndex) -> u32 { - assert!(index.as_u32() < self.num_runtime_tables); - self.runtime_tables() + index.as_u32() * u32::from(self.ptr.vm_table_import().size()) - } - - /// The offset of the base of the `runtime_reallocs` field - #[inline] - pub fn runtime_reallocs(&self) -> u32 { - self.reallocs - } - - /// The offset of the `*mut VMFuncRef` for the runtime index - /// provided. - #[inline] - pub fn runtime_realloc(&self, index: RuntimeReallocIndex) -> u32 { - assert!(index.as_u32() < self.num_runtime_reallocs); - self.runtime_reallocs() + index.as_u32() * u32::from(self.ptr.size()) - } - - /// The offset of the base of the `runtime_callbacks` field - #[inline] - pub fn runtime_callbacks(&self) -> u32 { - self.callbacks - } - - /// The offset of the `*mut VMFuncRef` for the runtime index - /// provided. - #[inline] - pub fn runtime_callback(&self, index: RuntimeCallbackIndex) -> u32 { - assert!(index.as_u32() < self.num_runtime_callbacks); - self.runtime_callbacks() + index.as_u32() * u32::from(self.ptr.size()) - } - - /// The offset of the base of the `runtime_post_returns` field - #[inline] - pub fn runtime_post_returns(&self) -> u32 { - self.post_returns - } - - /// The offset of the `*mut VMFuncRef` for the runtime index - /// provided. - #[inline] - pub fn runtime_post_return(&self, index: RuntimePostReturnIndex) -> u32 { - assert!(index.as_u32() < self.num_runtime_post_returns); - self.runtime_post_returns() + index.as_u32() * u32::from(self.ptr.size()) - } - - /// The offset of the base of the `resource_destructors` field - #[inline] - pub fn resource_destructors(&self) -> u32 { - self.resource_destructors - } - - /// The offset of the `*mut VMFuncRef` for the runtime index - /// provided. - #[inline] - pub fn resource_destructor(&self, index: ResourceIndex) -> u32 { - assert!(index.as_u32() < self.num_resources); - self.resource_destructors() + index.as_u32() * u32::from(self.ptr.size()) - } - - /// Return the size of the `VMComponentContext` allocation. - #[inline] - pub fn size_of_vmctx(&self) -> u32 { - self.size - } } diff --git a/crates/environ/src/lib.rs b/crates/environ/src/lib.rs index 2be10ed159e7..774f140423a6 100644 --- a/crates/environ/src/lib.rs +++ b/crates/environ/src/lib.rs @@ -43,6 +43,8 @@ mod trap_encoding; mod tunables; mod types; #[macro_use] +mod vmctxtypes; +#[macro_use] mod vmtypes; mod vmoffsets; mod wasm_error; @@ -66,6 +68,7 @@ pub use crate::string_pool::{Atom, StringPool}; pub use crate::trap_encoding::*; pub use crate::tunables::*; pub use crate::types::*; +pub use crate::vmctxtypes::{ArrayOffsets, VmctxArrayIndex}; pub use crate::vmoffsets::*; pub use crate::wasm_error::*; pub use object; diff --git a/crates/environ/src/vmctxtypes.rs b/crates/environ/src/vmctxtypes.rs new file mode 100644 index 000000000000..ddac95c64e0b --- /dev/null +++ b/crates/environ/src/vmctxtypes.rs @@ -0,0 +1,514 @@ +//! Centralized definitions of the layouts of Wasmtime's two "vmctx" types: +//! `VMContext`, the runtime context for a core Wasm instance, and +//! `VMComponentContext`, the runtime context for a component. +//! +//! Unlike the `VM*` types defined by `for_each_vm_type!`, neither of these has +//! a corresponding `#[repr(C)]` Rust `struct`: their sizes depend on the module +//! or component being instantiated, so they are dynamically laid out and +//! accessed exclusively through the computed offsets in `VMOffsets` and +//! `VMComponentOffsets`. That layout is defined exactly once here, via the +//! higher-order `for_each_vmctx_type!` macro, and each consumer generates its +//! view of it from that single source of truth. + +/// Invoke the given macro `$mac` once, passing it the layout of each of +/// Wasmtime's "vmctx" types. +/// +/// This is a higher-order macro: callers define a `macro_rules!` macro that +/// matches the grammar described below and pass its name as an argument to this +/// macro's invocation, e.g. `for_each_vmctx_type!(define_vmctx_offsets)`. +/// +/// # Grammar +/// +/// The layout below is written in exactly the same grammar that it is handed to +/// `$mac` in: this macro has a single rule, and that rule does nothing but +/// forward those tokens along. Writing the layout in the grammar that consumers +/// match makes it more verbose than a bespoke input syntax would be, but in +/// exchange there is no normalization pass in between, so what a consumer matches +/// is exactly what a reader of the layout sees. +/// +/// `$mac` receives one brace-delimited group per vmctx type: +/// +/// ```ignore +/// { +/// VMContext vmctx +/// static { ...entries... } +/// dynamic { ...entries... } +/// } +/// ``` +/// +/// where `vmctx` is the name used for accessor methods of that type, `static` +/// holds the fixed-width prefix whose offsets depend only on the target pointer +/// size, and `dynamic` holds the rest, whose offsets additionally depend on the +/// module or component being compiled. +/// +/// Each entry within a section is a keyword naming the entry's shape followed by +/// a single delimited group, so that a consumer can iterate over a section with +/// `$( ...$kind:ident $entry:tt... )*` and dispatch on one whole entry at a time +/// instead of munching the section token by token. An entry is one of: +/// +/// * `align { ptr }` or `align { N }`: round the running offset up to the target +/// pointer size, or to `N` bytes. Alignment is *always* explicit: it is never +/// derived from a field's type, because the layout being described here does +/// not necessarily align every field to its natural alignment, and inserting +/// padding that the layout does not actually have would silently corrupt every +/// subsequent offset. +/// +/// * `field { : }`: a single field. +/// +/// * `array { [; ]: }` (`dynamic` only): an +/// array of `self.` elements, indexed by ``. +/// +/// * `optional { [if ]: }` (`dynamic` only): a field that +/// is present when `self.` is true and absent (zero-sized) otherwise. +/// +/// A field's type is always the last thing in its entry. Consumers capture it +/// as a trailing `$($fty:tt)*` and re-dispatch on its tokens only where they +/// actually need to classify it into a size or a Cranelift type; a `:ty` +/// capture would be opaque forever, and so could never be classified at +/// all. +/// +/// `` is a possibly-empty sequence of marker attributes, which consumers +/// match with `$(# $fattr:tt)*`. They do not affect the layout, but they do affect +/// the accesses generated for the field: +/// +/// * `#[aggregate]`: this field is a composite (a nested struct, or an array of +/// them) rather than a single scalar. Compiled code accesses such a field's +/// interior piecewise, so there is no one Cranelift type for the field as a +/// whole and no alias-region accessor is generated for it. Its offset is still +/// generated, since that is what interior accesses are computed relative to. +/// +/// * `#[readonly]` and/or `#[can_move]`: describe how Cranelift may treat loads +/// and stores of this field. +/// +/// * `#[access_as = Type]`: this field is *declared* with one type (because that +/// is what determines its size and stride) but *accessed* as another. For +/// example, the component context's `may_leave` flags are each stored in a +/// whole `VMGlobalDefinition` but only ever accessed as a `u32`. +/// +/// Doc comments are deliberately *not* accepted on fields; use `//` comments for +/// prose about the layout. Accessor documentation is synthesized from the field +/// names instead, so that there is only one place a field can be described. +/// +/// A consumer that only cares about one of the two types can filter with a +/// literal-name arm followed by a catch-all, e.g. +/// +/// ```ignore +/// (@one VMContext $snake:ident dynamic { $($dyn:tt)* }) => { ...generate... }; +/// (@one $other:ident $snake:ident dynamic { $($dyn:tt)* }) => {}; +/// ``` +#[macro_export] +macro_rules! for_each_vmctx_type { + ($mac:ident) => { + $mac! { + { + VMContext vmctx + + // Fixed-width data comes first so that the calculation of these + // fields' offsets is a compile-time constant when using + // `HostPtr`. + static { + field { #[readonly] #[can_move] magic: u32 } + + // NB: this is where the four bytes of padding after `magic` + // live on targets with eight-byte pointers. + align { ptr } + + field { #[readonly] #[can_move] store_context: VmPtr } + + field { #[readonly] #[can_move] builtin_functions: VmPtr } + + field { epoch_ptr: VmPtr } + + // A pointer that different collectors use however they see + // fit. + field { #[readonly] #[can_move] gc_heap_data: VmPtr } + + field { #[readonly] #[can_move] type_ids: VmPtr } + } + + // Variable-width fields come after the fixed-width fields + // above. Memory-related items are placed first as they are some + // of the most frequently accessed items, and minimizing their + // offset can shrink the size of load/store instruction offset + // immediates on platforms like x64 and Pulley (e.g. fit in an + // 8-bit offset instead of needing a 32-bit offset). + dynamic { + array { + #[aggregate] + imported_memories[num_imported_memories; MemoryIndex]: VMMemoryImport + } + + array { + #[readonly] + #[can_move] + memories[num_defined_memories; DefinedMemoryIndex]: VmPtr + } + + array { + #[aggregate] + owned_memories[num_owned_memories; OwnedMemoryIndex]: VMMemoryDefinition + } + + array { + #[aggregate] + imported_functions[num_imported_functions; FuncIndex]: VMFunctionImport + } + + array { + #[aggregate] + imported_tables[num_imported_tables; TableIndex]: VMTableImport + } + + array { + #[aggregate] + imported_globals[num_imported_globals; GlobalIndex]: VMGlobalImport + } + + array { + #[aggregate] + imported_tags[num_imported_tags; TagIndex]: VMTagImport + } + + array { + #[aggregate] + tables[num_defined_tables; DefinedTableIndex]: VMTableDefinition + } + + align { 16 } + + array { + #[aggregate] + globals[num_defined_globals; DefinedGlobalIndex]: VMGlobalDefinition + } + + array { + #[aggregate] + tags[num_defined_tags; DefinedTagIndex]: VMTagDefinition + } + + array { + #[aggregate] + func_refs[num_escaped_funcs; FuncRefIndex]: VMFuncRef + } + + optional { + #[aggregate] + startup_func_ref[if has_startup_func]: VMFuncRef + } + + array { + runtime_data_bases[num_runtime_data; RuntimeDataIndex]: VmPtr + } + + array { + runtime_data_lengths[num_runtime_data; RuntimeDataIndex]: u32 + } + } + } + + { + VMComponentContext vmcomponent + + static { + // NB: `magic` must be at offset zero; this is relied upon by + // `VMComponentContext::from_opaque`. + field { #[readonly] #[can_move] magic: u32 } + + align { ptr } + + field { #[readonly] builtins: VmPtr } + + field { #[readonly] #[can_move] store_context: VmPtr } + } + + dynamic { + align { 16 } + + // Each of these flags gets a whole `VMGlobalDefinition`'s + // worth of space, but only its first four bytes are ever + // accessed. + array { + #[access_as = u32] + may_leave[num_runtime_component_instances; RuntimeComponentInstanceIndex]: VMGlobalDefinition + } + + field { #[access_as = u32] task_may_block: VMGlobalDefinition } + + align { ptr } + + array { + #[aggregate] + trampoline_func_refs[num_trampolines; TrampolineIndex]: VMFuncRef + } + + array { + #[aggregate] + intrinsic_func_refs[num_unsafe_intrinsics; UnsafeIntrinsic]: VMFuncRef + } + + array { + #[aggregate] + lowerings[num_lowerings; LoweredIndex]: VMLowering + } + + array { + memories + [num_runtime_memories; RuntimeMemoryIndex]: VmPtr + } + + array { + #[aggregate] + tables[num_runtime_tables; RuntimeTableIndex]: VMTableImport + } + + array { + reallocs[num_runtime_reallocs; RuntimeReallocIndex]: VmPtr + } + + array { + callbacks[num_runtime_callbacks; RuntimeCallbackIndex]: VmPtr + } + + array { + post_returns[num_runtime_post_returns; RuntimePostReturnIndex]: VmPtr + } + + array { + #[readonly] + resource_destructors[num_resources; ResourceIndex]: VmPtr + } + } + } + } + }; +} + +/// Round `offset` up to a multiple of `align`. +#[inline] +pub(crate) fn align_up(offset: u32, align: u32) -> u32 { + debug_assert!(align.is_power_of_two()); + (offset + (align - 1)) & !(align - 1) +} + +/// Add two offsets, panicking on overflow. +#[inline] +pub(crate) fn cadd(a: u32, b: u32) -> u32 { + a.checked_add(b).unwrap() +} + +/// Multiply an element count by an element size, panicking on overflow. +#[inline] +pub(crate) fn cmul(count: u32, size: u32) -> u32 { + count.checked_mul(size).unwrap() +} + +/// Classify one of the field types accepted by `for_each_vmctx_type!` to its +/// size in bytes as a `u32`, given a `PtrSize`. +#[allow( + unused_macro_rules, + reason = "some element types only appear in `VMComponentContext`, and so are \ + unused when the `component-model` feature is disabled" +)] +macro_rules! vmctx_field_size { + (($p:expr) u32) => { + 4u32 + }; + (($p:expr) VmPtr < $g:ident >) => { + u32::from($p) + }; + // `VMLowering` is a pair of pointers, and is not itself defined by + // `for_each_vm_type!`. + (($p:expr) VMLowering) => { + 2u32 * u32::from($p) + }; + // Anything else names one of the `VM*` types, and so has an + // `offsets::VMFoo` generated for it by `for_each_vm_type!`. Deferring to + // that keeps this macro from having to mirror the list of `VM*` types, and + // a type that has no such entry fails to resolve here. + (($p:expr) $Name:ident) => { + u32::from(crate::vmoffsets::offsets::$Name($p).size()) + }; +} + +/// Classify a `for_each_vmctx_type!` alignment step to its alignment in bytes +/// as a `u32`, given a `PtrSize`. +macro_rules! vmctx_align_value { + (($p:expr) ptr) => { + u32::from($p) + }; + (($p:expr) $n:literal) => { + $n + }; +} + +/// Generate the accessors for, and the layout computation of, the +/// dynamically-positioned fields of one of the vmctx types. +#[allow( + unused_macro_rules, + reason = "single dynamically-positioned fields only appear in \ + `VMComponentContext`, and so are unused when the `component-model` \ + feature is disabled" +)] +macro_rules! define_vmctx_dynamic_offsets { + (@accessors ($s:ident) [ $($kind:ident $entry:tt)* ]) => { + $( define_vmctx_dynamic_offsets!(@accessor ($s) $kind $entry); )* + }; + + (@accessor ($s:ident) align { $al:tt }) => {}; + + (@accessor ($s:ident) field { $(# $fattr:tt)* $fname:ident : $($fty:tt)* }) => { + #[doc = concat!("The offset of the `", stringify!($fname), "` field.")] + #[inline] + pub fn $fname(&$s) -> u32 { + $s.$fname + } + }; + + (@accessor ($s:ident) array { + $(# $fattr:tt)* $fname:ident [ $count:ident ; $Index:ident ] : $($fty:tt)* + }) => { + #[doc = concat!("The offsets of the `", stringify!($fname), "` array.")] + #[inline] + pub fn $fname(&$s) -> $crate::ArrayOffsets<$Index> { + $crate::ArrayOffsets::new( + $s.$fname, + vmctx_field_size!(($s.ptr.size()) $($fty)*), + $s.$count, + ) + } + }; + + (@accessor ($s:ident) optional { + $(# $fattr:tt)* $fname:ident [ if $flag:ident ] : $($fty:tt)* + }) => { + #[doc = concat!( + "The offset of the `", stringify!($fname), "` field.\n\n", + "Panics if `", stringify!($flag), "` is false, in which case this \ + field is not present at all." + )] + #[inline] + pub fn $fname(&$s) -> u32 { + assert!($s.$flag); + $s.$fname + } + }; + + (@compute_fn ($s:ident, $next:ident) $snake:ident [ $($kind:ident $entry:tt)* ]) => { + /// Compute the offset of each dynamically-positioned field, and this + /// vmctx's total size. + fn compute_field_offsets(&mut $s) { + let mut $next = u32::from($s.ptr.$snake().end_of_static_fields()); + $( define_vmctx_dynamic_offsets!(@compute ($s, $next) $kind $entry); )* + $s.size = $next; + } + }; + + (@compute ($s:ident, $next:ident) align { $al:tt }) => { + $next = crate::vmctxtypes::align_up($next, vmctx_align_value!(($s.ptr.size()) $al)); + }; + + (@compute ($s:ident, $next:ident) field { + $(# $fattr:tt)* $fname:ident : $($fty:tt)* + }) => { + $s.$fname = $next; + $next = crate::vmctxtypes::cadd($next, vmctx_field_size!(($s.ptr.size()) $($fty)*)); + }; + + (@compute ($s:ident, $next:ident) array { + $(# $fattr:tt)* $fname:ident [ $count:ident ; $Index:ident ] : $($fty:tt)* + }) => { + $s.$fname = $next; + $next = crate::vmctxtypes::cadd( + $next, + crate::vmctxtypes::cmul($s.$count, vmctx_field_size!(($s.ptr.size()) $($fty)*)), + ); + }; + + (@compute ($s:ident, $next:ident) optional { + $(# $fattr:tt)* $fname:ident [ if $flag:ident ] : $($fty:tt)* + }) => { + $s.$fname = $next; + $next = crate::vmctxtypes::cadd( + $next, + if $s.$flag { + vmctx_field_size!(($s.ptr.size()) $($fty)*) + } else { + 0 + }, + ); + }; +} + +/// The offsets of one array field within a vmctx. +#[derive(Debug, Clone, Copy)] +pub struct ArrayOffsets { + begin: u32, + stride: u32, + count: u32, + _index: core::marker::PhantomData, +} + +impl ArrayOffsets { + /// Create the offsets for an array of `count` elements which begins at + /// `begin` within its vmctx and whose elements are `stride` bytes apart. + #[inline] + pub fn new(begin: u32, stride: u32, count: u32) -> Self { + ArrayOffsets { + begin, + stride, + count, + _index: core::marker::PhantomData, + } + } + + /// The offset of the start of this array within its vmctx. + #[inline] + pub fn begin(&self) -> u32 { + self.begin + } + + /// The number of bytes between the start of consecutive elements of this + /// array. + #[inline] + pub fn stride(&self) -> u32 { + self.stride + } + + /// The number of elements in this array. + #[inline] + pub fn count(&self) -> u32 { + self.count + } +} + +impl ArrayOffsets { + /// The offset of the given element within this array's vmctx. + /// + /// Panics if `index` is out of bounds for this array. + #[inline] + pub fn at(&self, index: I) -> u32 { + let index = index.vmctx_array_index(); + assert!(index < self.count); + self.begin + index * self.stride + } +} + +/// An index type that can be used to index one of a vmctx's arrays. +pub trait VmctxArrayIndex: Copy { + /// This index's position within its array. + fn vmctx_array_index(self) -> u32; +} + +/// Implement `VmctxArrayIndex` for entity references. +macro_rules! impl_vmctx_array_index { + ($($ty:ty),* $(,)?) => { + $( + impl $crate::VmctxArrayIndex for $ty { + #[inline] + fn vmctx_array_index(self) -> u32 { + self.as_u32() + } + } + )* + }; +} diff --git a/crates/environ/src/vmoffsets.rs b/crates/environ/src/vmoffsets.rs index 79f7bda603a6..f4f6b33b14cc 100644 --- a/crates/environ/src/vmoffsets.rs +++ b/crates/environ/src/vmoffsets.rs @@ -1,47 +1,16 @@ //! Offsets and sizes of various structs in `wasmtime::runtime::vm::*` that are //! accessed directly by compiled Wasm code. -// Currently the `VMContext` allocation by field looks like this: -// -// struct VMContext { -// // Fixed-width data comes first so the calculation of the offset of -// // these fields is a compile-time constant when using `HostPtr`. -// magic: u32, -// _padding: u32, // (On 64-bit systems) -// vm_store_context: *const VMStoreContext, -// builtin_functions: *mut VMBuiltinFunctionsArray, -// epoch_ptr: *mut AtomicU64, -// gc_heap_data: *mut T, // Collector-specific pointer -// type_ids: *const VMSharedTypeIndex, -// -// // Variable-width fields come after the fixed-width fields above. Place -// // memory-related items first as they're some of the most frequently -// // accessed items and minimizing their offset in this structure can -// // shrink the size of load/store instruction offset immediates on -// // platforms like x64 and Pulley (e.g. fit in an 8-bit offset instead -// // of needing a 32-bit offset) -// imported_memories: [VMMemoryImport; module.num_imported_memories], -// memories: [*mut VMMemoryDefinition; module.num_defined_memories], -// owned_memories: [VMMemoryDefinition; module.num_owned_memories], -// imported_functions: [VMFunctionImport; module.num_imported_functions], -// imported_tables: [VMTableImport; module.num_imported_tables], -// imported_globals: [VMGlobalImport; module.num_imported_globals], -// imported_tags: [VMTagImport; module.num_imported_tags], -// tables: [VMTableDefinition; module.num_defined_tables], -// globals: [VMGlobalDefinition; module.num_defined_globals], -// tags: [VMTagDefinition; module.num_defined_tags], -// func_refs: [VMFuncRef; module.num_escaped_funcs], -// startup_func_ref: [VMFuncRef; module.has_startup_func ? 1 : 0], -// runtime_data_bases: [*const u8; module.num_runtime_data], -// runtime_data_lengths: [u32; module.num_runtime_data], -// } +// The `VMContext` layout is not defined here: it is defined once, alongside +// `VMComponentContext`'s, in `for_each_vmctx_type!`. Everything in this module +// is either generated from that definition or is an offset that is not simply +// the offset of one of the layout's fields. use crate::{ DefinedGlobalIndex, DefinedMemoryIndex, DefinedTableIndex, DefinedTagIndex, FuncIndex, FuncRefIndex, GlobalIndex, MemoryIndex, Module, OwnedMemoryIndex, RuntimeDataIndex, TableIndex, TagIndex, }; -use cranelift_entity::packed_option::ReservedValue; /// Number of slots in for `component_context` in the `VMStoreContext`. This is /// defined by the component model's `context.{get,set}` intrinsics. @@ -243,25 +212,87 @@ macro_rules! define_vm_type_offsets { $($body:tt)* } )* ) => { - /// Offsets of fields within the various `VM*` types, parameterized over - /// a target [`PtrSize`] so that they can be computed during cross - /// compilation. - /// - /// These types are namespaced within their own module so that they never - /// collide with the real definitions of the `VM*` types themselves. - pub mod offsets { - use super::{align, PtrSize, NUM_COMPONENT_CONTEXT_SLOTS}; + $( + #[doc = concat!("Offsets of fields within the `", stringify!($Name), "` type.")] + pub struct $Name(pub P); - $( - #[doc = concat!("Offsets of fields within the `", stringify!($Name), "` type.")] - pub struct $Name(pub P); + define_vm_type_offsets!(@impl $Name [$($repr)*] {} $($body)*); + )* + }; +} + +/// Generate a `struct VMContext(P)`-style wrapper for each vmctx +/// type, with a method per statically-positioned field returning that field's +/// offset. +macro_rules! define_vmctx_static_offsets { + // Munch the `static` section, threading a closed-form expression for the + // running offset. + (@chain $p:ident [ $($prev:tt)* ] []) => { + /// The offset just past this type's last statically-positioned field. + /// + /// Everything after this point is dynamically sized. + #[inline] + pub fn end_of_static_fields(&self) -> u8 { + let $p = self.0.size(); + let _ = $p; + u8::try_from($($prev)*).unwrap() + } + }; + (@chain $p:ident [ $($prev:tt)* ] [ align { $al:tt } $($rest:tt)* ]) => { + define_vmctx_static_offsets!( + @chain $p + [ crate::vmctxtypes::align_up($($prev)*, vmctx_align_value!(($p) $al)) ] + [ $($rest)* ] + ); + }; + (@chain $p:ident [ $($prev:tt)* ] [ + field { $(# $fattr:tt)* $fname:ident : $($fty:tt)* } $($rest:tt)* + ]) => { + #[doc = concat!("The offset of the `", stringify!($fname), "` field.")] + #[inline] + pub fn $fname(&self) -> u8 { + let $p = self.0.size(); + let _ = $p; + u8::try_from($($prev)*).unwrap() + } + define_vmctx_static_offsets!( + @chain $p + [ $($prev)* + vmctx_field_size!(($p) $($fty)*) ] + [ $($rest)* ] + ); + }; - define_vm_type_offsets!(@impl $Name [$($repr)*] {} $($body)*); - )* + ( $( + { + $Name:ident $snake:ident + static { $($stat:tt)* } + dynamic { $($dyn:tt)* } } + )* ) => { + $( + #[doc = concat!("Offsets of the statically-positioned fields within the `", + stringify!($Name), "` type.")] + pub struct $Name(pub P); + + impl $Name

{ + define_vmctx_static_offsets!(@chain ptr [ 0u32 ] [ $($stat)* ]); + } + )* }; } -for_each_vm_type!(define_vm_type_offsets); + +/// Offsets of fields within the various `VM*` types and within the two vmctx +/// types, parameterized over a target `PtrSize` so that they can be computed +/// during cross compilation. +/// +/// These types are namespaced within their own module so that they never collide +/// with the real definitions of the `VM*` types themselves. +pub mod offsets { + use super::{NUM_COMPONENT_CONTEXT_SLOTS, PtrSize, align}; + + for_each_vm_type!(define_vm_type_offsets); + for_each_vmctx_type!(define_vmctx_static_offsets); +} /// The size, in bytes, of one `context.{get,set}` slot. These slots are `u32`s, /// both in `VMStoreContext::component_context` and in @@ -303,9 +334,8 @@ impl offsets::VMDeferredThread

{ } } -/// Add a `fn vm_foo(&self) -> offsets::VMFoo<&Self>` accessor to [`PtrSize`] for -/// each `VM*` type, using the `#[snake_name = ...]` attribute for the method -/// name. +/// Add a `fn vm_foo(&self) -> offsets::VMFoo<&Self>` accessor to `PtrSize` for +/// each `VM*` type. macro_rules! define_ptr_size_vm_type_accessors { ( $( $(#[doc = $sdoc:literal])* @@ -313,8 +343,6 @@ macro_rules! define_ptr_size_vm_type_accessors { #[repr($($repr:tt)*)] #[snake_name = $snake:ident] $svis:vis struct $Name:ident { - // This macro only needs each type's name and snake name, so the - // body is captured raw rather than parsed into fields. $($body:tt)* } )* ) => { @@ -328,6 +356,27 @@ macro_rules! define_ptr_size_vm_type_accessors { }; } +/// Add a `fn vmctx(&self) -> offsets::VMContext<&Self>` accessor to `PtrSize` +/// for each vmctx type. +macro_rules! define_ptr_size_vmctx_type_accessors { + ( $( + { + $Name:ident $snake:ident + static { $($stat:tt)* } + dynamic { $($dyn:tt)* } + } + )* ) => { + $( + #[doc = concat!("Get the [`offsets::", stringify!($Name), + "`] offsets for this pointer size.")] + #[inline] + fn $snake(&self) -> offsets::$Name<&Self> { + offsets::$Name(self) + } + )* + }; +} + /// This class computes offsets to fields within `VMContext` and other /// related structs that JIT code accesses directly. #[derive(Debug, Clone, Copy)] @@ -362,18 +411,18 @@ pub struct VMOffsets

{ /// Whether or not the module has a start function. pub has_startup_func: bool, - // precalculated offsets of various member fields + // Precalculated offsets of the dynamically-positioned fields. + imported_memories: u32, + memories: u32, + owned_memories: u32, imported_functions: u32, imported_tables: u32, - imported_memories: u32, imported_globals: u32, imported_tags: u32, - defined_tables: u32, - defined_memories: u32, - owned_memories: u32, - defined_globals: u32, - defined_tags: u32, - defined_func_refs: u32, + tables: u32, + globals: u32, + tags: u32, + func_refs: u32, startup_func_ref: u32, runtime_data_bases: u32, runtime_data_lengths: u32, @@ -386,23 +435,12 @@ pub trait PtrSize { fn size(&self) -> u8; // Generate a `fn vm_foo(&self) -> offsets::VMFoo<&Self>` accessor for each - // `VM*` type, giving access to that type's field offsets, size, and - // alignment for this pointer size. + // `VM*` type. for_each_vm_type!(define_ptr_size_vm_type_accessors); - /// The offset of the `VMContext::store_context` field - fn vmcontext_store_context(&self) -> u8 { - u8::try_from(align( - u32::try_from(core::mem::size_of::()).unwrap(), - u32::from(self.size()), - )) - .unwrap() - } - - /// The offset of the `VMContext::builtin_functions` field - fn vmcontext_builtin_functions(&self) -> u8 { - self.vmcontext_store_context() + self.size() - } + // Generate a `fn vmctx(&self) -> offsets::VMContext<&Self>` accessor for + // each vmctx type. + for_each_vmctx_type!(define_ptr_size_vmctx_type_accessors); /// Return the size of `VMSharedTypeIndex`. #[inline] @@ -587,43 +625,6 @@ pub trait PtrSize { self.vmcontref_args() + self.size_of_vmhostarray() } - /// Return the offset to the `magic` value in this `VMContext`. - #[inline] - fn vmctx_magic(&self) -> u8 { - // This is required by the implementation of `VMContext::instance` and - // `VMContext::instance_mut`. If this value changes then those locations - // need to be updated. - 0 - } - - /// Return the offset to the `VMStoreContext` structure - #[inline] - fn vmctx_store_context(&self) -> u8 { - self.vmctx_magic() + self.size() - } - - /// Return the offset to the `VMBuiltinFunctionsArray` structure - #[inline] - fn vmctx_builtin_functions(&self) -> u8 { - self.vmctx_store_context() + self.size() - } - - /// Return the offset to the `*const AtomicU64` epoch-counter - /// pointer. - #[inline] - fn vmctx_epoch_ptr(&self) -> u8 { - self.vmctx_builtin_functions() + self.size() - } - - /// Return the offset to the `*mut T` collector-specific data. - /// - /// This is a pointer that different collectors can use however they see - /// fit. - #[inline] - fn vmctx_gc_heap_data(&self) -> u8 { - self.vmctx_epoch_ptr() + self.size() - } - /// Return the offset of the `over_approximated_stack_roots` field within /// `VMDrcHeapData`. #[inline] @@ -682,20 +683,6 @@ pub trait PtrSize { fn align_of_vmcopying_heap_data(&self) -> u8 { 4 } - - /// The offset of the `type_ids` array pointer. - #[inline] - fn vmctx_type_ids_array(&self) -> u8 { - self.vmctx_gc_heap_data() + self.size() - } - - /// The end of statically known offsets in `VMContext`. - /// - /// Data after this is dynamically sized. - #[inline] - fn vmctx_dynamic_data_start(&self) -> u8 { - self.vmctx_type_ids_array() + self.size() - } } /// A trait to abstract over various types that contain a `P: PtrSize`. @@ -871,16 +858,16 @@ impl VMOffsets

{ runtime_data_lengths: "runtime data lengths", runtime_data_bases: "runtime data base pointers", startup_func_ref: "startup funcref", - defined_func_refs: "module functions", - defined_tags: "defined tags", - defined_globals: "defined globals", - defined_tables: "defined tables", + func_refs: "module functions", + tags: "defined tags", + globals: "defined globals", + tables: "defined tables", imported_tags: "imported tags", imported_globals: "imported globals", imported_tables: "imported tables", imported_functions: "imported functions", owned_memories: "owned memories", - defined_memories: "defined memories", + memories: "defined memories", imported_memories: "imported memories", } } @@ -912,90 +899,24 @@ impl From> for VMOffsets

{ num_escaped_funcs: fields.num_escaped_funcs, num_runtime_data: fields.num_runtime_data, has_startup_func: fields.has_startup_func, + imported_memories: 0, + memories: 0, + owned_memories: 0, imported_functions: 0, imported_tables: 0, - imported_memories: 0, imported_globals: 0, imported_tags: 0, - defined_tables: 0, - defined_memories: 0, - owned_memories: 0, - defined_globals: 0, - defined_tags: 0, - defined_func_refs: 0, + tables: 0, + globals: 0, + tags: 0, + func_refs: 0, startup_func_ref: 0, runtime_data_bases: 0, runtime_data_lengths: 0, size: 0, }; - - // Convenience functions for checked addition and multiplication. - // As side effect this reduces binary size by using only a single - // `#[track_caller]` location for each function instead of one for - // each individual invocation. - #[inline] - fn cadd(count: u32, size: u32) -> u32 { - count.checked_add(size).unwrap() - } - - #[inline] - fn cmul(count: u32, size: u8) -> u32 { - count.checked_mul(u32::from(size)).unwrap() - } - - let mut next_field_offset = u32::from(ret.ptr.vmctx_dynamic_data_start()); - - macro_rules! fields { - (size($field:ident) = $size:expr, $($rest:tt)*) => { - ret.$field = next_field_offset; - next_field_offset = cadd(next_field_offset, u32::from($size)); - fields!($($rest)*); - }; - (align($align:expr), $($rest:tt)*) => { - next_field_offset = align(next_field_offset, $align); - fields!($($rest)*); - }; - () => {}; - } - - fields! { - size(imported_memories) - = cmul(ret.num_imported_memories, ret.ptr.vm_memory_import().size()), - size(defined_memories) - = cmul(ret.num_defined_memories, ret.ptr.size_of_vmmemory_pointer()), - size(owned_memories) - = cmul(ret.num_owned_memories, ret.ptr.vm_memory_definition().size()), - size(imported_functions) - = cmul(ret.num_imported_functions, ret.ptr.vm_function_import().size()), - size(imported_tables) - = cmul(ret.num_imported_tables, ret.ptr.vm_table_import().size()), - size(imported_globals) - = cmul(ret.num_imported_globals, ret.ptr.vm_global_import().size()), - size(imported_tags) - = cmul(ret.num_imported_tags, ret.ptr.vm_tag_import().size()), - size(defined_tables) - = cmul(ret.num_defined_tables, ret.ptr.vm_table_definition().size()), - align(16), - size(defined_globals) - = cmul(ret.num_defined_globals, ret.ptr.vm_global_definition().size()), - size(defined_tags) - = cmul(ret.num_defined_tags, ret.ptr.vm_tag_definition().size()), - size(defined_func_refs) = cmul( - ret.num_escaped_funcs, - ret.ptr.vm_func_ref().size(), - ), - size(startup_func_ref) = if ret.has_startup_func { - ret.ptr.vm_func_ref().size() - } else { - 0 - }, - size(runtime_data_bases) = cmul(ret.num_runtime_data, ret.ptr.size()), - size(runtime_data_lengths) = cmul(ret.num_runtime_data, 4), - } - - ret.size = next_field_offset; - - return ret; + ret.compute_field_offsets(); + ret } } @@ -1025,286 +946,50 @@ impl VMOffsets

{ } } -/// Offsets for `VMContext`. -impl VMOffsets

{ - /// The offset of the `tables` array. - #[inline] - pub fn vmctx_imported_functions_begin(&self) -> u32 { - self.imported_functions - } - - /// The offset of the `tables` array. - #[inline] - pub fn vmctx_imported_tables_begin(&self) -> u32 { - self.imported_tables - } - - /// The offset of the `memories` array. - #[inline] - pub fn vmctx_imported_memories_begin(&self) -> u32 { - self.imported_memories - } - - /// The offset of the `globals` array. - #[inline] - pub fn vmctx_imported_globals_begin(&self) -> u32 { - self.imported_globals - } - - /// The offset of the `tags` array. - #[inline] - pub fn vmctx_imported_tags_begin(&self) -> u32 { - self.imported_tags - } - - /// The offset of the `tables` array. - #[inline] - pub fn vmctx_tables_begin(&self) -> u32 { - self.defined_tables - } - - /// The offset of the `memories` array. - #[inline] - pub fn vmctx_memories_begin(&self) -> u32 { - self.defined_memories - } - - /// The offset of the `owned_memories` array. - #[inline] - pub fn vmctx_owned_memories_begin(&self) -> u32 { - self.owned_memories - } - - /// The offset of the `globals` array. - #[inline] - pub fn vmctx_globals_begin(&self) -> u32 { - self.defined_globals - } - - /// The offset of the `tags` array. - #[inline] - pub fn vmctx_tags_begin(&self) -> u32 { - self.defined_tags - } - - /// The offset of the `func_refs` array. - #[inline] - pub fn vmctx_func_refs_begin(&self) -> u32 { - self.defined_func_refs - } - - /// The offset of the `runtime_data_bases` array. - #[inline] - pub fn vmctx_runtime_data_bases_begin(&self) -> u32 { - self.runtime_data_bases - } - - /// The offset of the `runtime_data_lengths` array. - #[inline] - pub fn vmctx_runtime_data_lengths_begin(&self) -> u32 { - self.runtime_data_lengths - } - - /// Return the size of the `VMContext` allocation. - #[inline] - pub fn size_of_vmctx(&self) -> u32 { - self.size - } - - /// Return the offset to `VMFunctionImport` index `index`. - #[inline] - pub fn vmctx_vmfunction_import(&self, index: FuncIndex) -> u32 { - assert!(index.as_u32() < self.num_imported_functions); - self.vmctx_imported_functions_begin() - + index.as_u32() * u32::from(self.ptr.vm_function_import().size()) - } - - /// Return the offset to `VMTable` index `index`. - #[inline] - pub fn vmctx_vmtable_import(&self, index: TableIndex) -> u32 { - assert!(index.as_u32() < self.num_imported_tables); - self.vmctx_imported_tables_begin() - + index.as_u32() * u32::from(self.ptr.vm_table_import().size()) - } - - /// Return the offset to `VMMemoryImport` index `index`. - #[inline] - pub fn vmctx_vmmemory_import(&self, index: MemoryIndex) -> u32 { - assert!(index.as_u32() < self.num_imported_memories); - self.vmctx_imported_memories_begin() - + index.as_u32() * u32::from(self.ptr.vm_memory_import().size()) - } - - /// Return the offset to `VMGlobalImport` index `index`. - #[inline] - pub fn vmctx_vmglobal_import(&self, index: GlobalIndex) -> u32 { - assert!(index.as_u32() < self.num_imported_globals); - self.vmctx_imported_globals_begin() - + index.as_u32() * u32::from(self.ptr.vm_global_import().size()) - } - - /// Return the offset to `VMTagImport` index `index`. - #[inline] - pub fn vmctx_vmtag_import(&self, index: TagIndex) -> u32 { - assert!(index.as_u32() < self.num_imported_tags); - self.vmctx_imported_tags_begin() - + index.as_u32() * u32::from(self.ptr.vm_tag_import().size()) - } - - /// Return the offset to `VMTableDefinition` index `index`. - #[inline] - pub fn vmctx_vmtable_definition(&self, index: DefinedTableIndex) -> u32 { - assert!(index.as_u32() < self.num_defined_tables); - self.vmctx_tables_begin() - + index.as_u32() * u32::from(self.ptr.vm_table_definition().size()) - } - - /// Return the offset to the `*mut VMMemoryDefinition` at index `index`. - #[inline] - pub fn vmctx_vmmemory_pointer(&self, index: DefinedMemoryIndex) -> u32 { - assert!(index.as_u32() < self.num_defined_memories); - self.vmctx_memories_begin() - + index.as_u32() * u32::from(self.ptr.size_of_vmmemory_pointer()) - } - - /// Return the offset to the owned `VMMemoryDefinition` at index `index`. - #[inline] - pub fn vmctx_vmmemory_definition(&self, index: OwnedMemoryIndex) -> u32 { - assert!(index.as_u32() < self.num_owned_memories); - self.vmctx_owned_memories_begin() - + index.as_u32() * u32::from(self.ptr.vm_memory_definition().size()) - } - - /// Return the offset to the `VMGlobalDefinition` index `index`. - #[inline] - pub fn vmctx_vmglobal_definition(&self, index: DefinedGlobalIndex) -> u32 { - assert!(index.as_u32() < self.num_defined_globals); - self.vmctx_globals_begin() - + index.as_u32() * u32::from(self.ptr.vm_global_definition().size()) - } - - /// Return the offset to the `VMTagDefinition` index `index`. - #[inline] - pub fn vmctx_vmtag_definition(&self, index: DefinedTagIndex) -> u32 { - assert!(index.as_u32() < self.num_defined_tags); - self.vmctx_tags_begin() + index.as_u32() * u32::from(self.ptr.vm_tag_definition().size()) - } - - /// Return the offset to the `VMFuncRef` for the given function - /// index (either imported or defined). - #[inline] - pub fn vmctx_func_ref(&self, index: FuncRefIndex) -> u32 { - assert!(!index.is_reserved_value()); - assert!(index.as_u32() < self.num_escaped_funcs); - self.vmctx_func_refs_begin() + index.as_u32() * u32::from(self.ptr.vm_func_ref().size()) - } - - /// Returns the offset to the `VMFuncRef` for the module startup function. - /// - /// Panics if this module does not have a startup function. - #[inline] - pub fn vmctx_startup_func_ref(&self) -> u32 { - assert!(self.has_startup_func); - self.startup_func_ref - } - - /// Return the offset to the base of the runtime data segment at `index`. - #[inline] - pub fn vmctx_runtime_data_base(&self, index: RuntimeDataIndex) -> u32 { - assert!(!index.is_reserved_value()); - assert!(index.as_u32() < self.num_runtime_data); - self.vmctx_runtime_data_bases_begin() + index.as_u32() * u32::from(self.ptr.size()) - } - - /// Return the offset to the length of the runtime data segment at `index`. - #[inline] - pub fn vmctx_runtime_data_length(&self, index: RuntimeDataIndex) -> u32 { - assert!(!index.is_reserved_value()); - assert!(index.as_u32() < self.num_runtime_data); - self.vmctx_runtime_data_lengths_begin() + index.as_u32() * 4 - } - - /// Return the offset to the `wasm_call` field in `*const VMFunctionBody` index `index`. - #[inline] - pub fn vmctx_vmfunction_import_wasm_call(&self, index: FuncIndex) -> u32 { - self.vmctx_vmfunction_import(index) + u32::from(self.ptr.vm_function_import().wasm_call()) - } - - /// Return the offset to the `array_call` field in `*const VMFunctionBody` index `index`. - #[inline] - pub fn vmctx_vmfunction_import_array_call(&self, index: FuncIndex) -> u32 { - self.vmctx_vmfunction_import(index) + u32::from(self.ptr.vm_function_import().array_call()) - } - - /// Return the offset to the `vmctx` field in `*const VMFunctionBody` index `index`. - #[inline] - pub fn vmctx_vmfunction_import_vmctx(&self, index: FuncIndex) -> u32 { - self.vmctx_vmfunction_import(index) + u32::from(self.ptr.vm_function_import().vmctx()) - } - - /// Return the offset to the `from` field in the imported `VMTable` at index - /// `index`. - #[inline] - pub fn vmctx_vmtable_from(&self, index: TableIndex) -> u32 { - self.vmctx_vmtable_import(index) + u32::from(self.ptr.vm_table_import().from()) - } - - /// Return the offset to the `base` field in `VMTableDefinition` index `index`. - #[inline] - pub fn vmctx_vmtable_definition_base(&self, index: DefinedTableIndex) -> u32 { - self.vmctx_vmtable_definition(index) + u32::from(self.ptr.vm_table_definition().base()) - } - - /// Return the offset to the `current_elements` field in `VMTableDefinition` index `index`. - #[inline] - pub fn vmctx_vmtable_definition_current_elements(&self, index: DefinedTableIndex) -> u32 { - self.vmctx_vmtable_definition(index) - + u32::from(self.ptr.vm_table_definition().current_elements()) - } - - /// Return the offset to the `from` field in `VMMemoryImport` index `index`. - #[inline] - pub fn vmctx_vmmemory_import_from(&self, index: MemoryIndex) -> u32 { - self.vmctx_vmmemory_import(index) + u32::from(self.ptr.vm_memory_import().from()) - } - - /// Return the offset to the `base` field in `VMMemoryDefinition` index `index`. - #[inline] - pub fn vmctx_vmmemory_definition_base(&self, index: OwnedMemoryIndex) -> u32 { - self.vmctx_vmmemory_definition(index) + u32::from(self.ptr.vm_memory_definition().base()) - } - - /// Return the offset to the `current_length` field in `VMMemoryDefinition` index `index`. - #[inline] - pub fn vmctx_vmmemory_definition_current_length(&self, index: OwnedMemoryIndex) -> u32 { - self.vmctx_vmmemory_definition(index) - + u32::from(self.ptr.vm_memory_definition().current_length()) - } +impl_vmctx_array_index! { + MemoryIndex, + DefinedMemoryIndex, + OwnedMemoryIndex, + FuncIndex, + TableIndex, + DefinedTableIndex, + GlobalIndex, + DefinedGlobalIndex, + TagIndex, + DefinedTagIndex, + FuncRefIndex, + RuntimeDataIndex, +} - /// Return the offset to the `from` field in `VMGlobalImport` index `index`. - #[inline] - pub fn vmctx_vmglobal_import_from(&self, index: GlobalIndex) -> u32 { - self.vmctx_vmglobal_import(index) + u32::from(self.ptr.vm_global_import().from()) - } +/// Generate the accessors for the offsets of `VMContext`'s +/// dynamically-positioned fields. +macro_rules! define_vmoffsets_dynamic_offsets { + (@one VMContext $snake:ident { $($dyn:tt)* }) => { + /// Offsets of the dynamically-positioned fields of `VMContext`. + impl VMOffsets

{ + define_vmctx_dynamic_offsets!(@accessors (self) [ $($dyn)* ]); + define_vmctx_dynamic_offsets!(@compute_fn (self, next) $snake [ $($dyn)* ]); - /// Return the offset to the `from` field in `VMTagImport` index `index`. - #[inline] - pub fn vmctx_vmtag_import_from(&self, index: TagIndex) -> u32 { - self.vmctx_vmtag_import(index) + u32::from(self.ptr.vm_tag_import().from()) - } - - /// Return the offset to the `vmctx` field in `VMTagImport` index `index`. - #[inline] - pub fn vmctx_vmtag_import_vmctx(&self, index: TagIndex) -> u32 { - self.vmctx_vmtag_import(index) + u32::from(self.ptr.vm_tag_import().vmctx()) - } + /// Return the size of the `VMContext` allocation. + #[inline] + pub fn size_of_vmctx(&self) -> u32 { + self.size + } + } + }; + (@one $other:ident $snake:ident { $($dyn:tt)* }) => {}; - /// Return the offset to the `index` field in `VMTagImport` index `index`. - #[inline] - pub fn vmctx_vmtag_import_index(&self, index: TagIndex) -> u32 { - self.vmctx_vmtag_import(index) + u32::from(self.ptr.vm_tag_import().index()) - } + ( $( + { + $Name:ident $snake:ident + static { $($stat:tt)* } + dynamic { $($dyn:tt)* } + } + )* ) => { + $( define_vmoffsets_dynamic_offsets!(@one $Name $snake { $($dyn)* }); )* + }; } +for_each_vmctx_type!(define_vmoffsets_dynamic_offsets); /// Offsets for `VMGcHeader`. impl VMOffsets

{ diff --git a/crates/environ/src/vmtypes.rs b/crates/environ/src/vmtypes.rs index 867c1b80609a..8011ddd8c096 100644 --- a/crates/environ/src/vmtypes.rs +++ b/crates/environ/src/vmtypes.rs @@ -147,10 +147,14 @@ macro_rules! for_each_vm_type { #[snake_name = vm_function_import] pub struct VMFunctionImport { /// Same as `VMFuncRef::array_call`. + #[readonly] + #[can_move] pub array_call: VmPtr, /// Same as `VMFuncRef::wasm_call`, except always non-null. Must be filled /// in by the time Wasm is importing this function! + #[readonly] + #[can_move] pub wasm_call: VmPtr, /// Function signature's _actual_ type id. @@ -158,9 +162,13 @@ macro_rules! for_each_vm_type { /// This is the type that the function was defined with, not the type that /// it was imported as. These two can be different in the face of subtyping /// and we need the former for to correctly implement dynamic downcasts. + #[readonly] + #[can_move] pub type_index: VMSharedTypeIndex, /// Same as `VMFuncRef::vmctx`. + #[readonly] + #[can_move] pub vmctx: VmPtr, } @@ -171,12 +179,18 @@ macro_rules! for_each_vm_type { #[snake_name = vm_table_import] pub struct VMTableImport { /// A pointer to the imported table description. + #[readonly] + #[can_move] pub from: VmPtr, /// A pointer to the `VMContext` that owns the table description. + #[readonly] + #[can_move] pub vmctx: VmPtr, /// The table index, within `vmctx`, this definition resides at. + #[readonly] + #[can_move] pub index: DefinedTableIndex, } @@ -187,12 +201,18 @@ macro_rules! for_each_vm_type { #[snake_name = vm_memory_import] pub struct VMMemoryImport { /// A pointer to the imported memory description. + #[readonly] + #[can_move] pub from: VmPtr, /// A pointer to the `VMContext` that owns the memory description. + #[readonly] + #[can_move] pub vmctx: VmPtr, /// The index of the memory in the containing `vmctx`. + #[readonly] + #[can_move] pub index: DefinedMemoryIndex, } @@ -207,6 +227,8 @@ macro_rules! for_each_vm_type { #[snake_name = vm_global_import] pub struct VMGlobalImport { /// A pointer to the imported global variable description. + #[readonly] + #[can_move] pub from: VmPtr, /// A pointer to the context that owns the global. @@ -215,10 +237,14 @@ macro_rules! for_each_vm_type { /// for `VMGlobalKind::Host`, it's a `VMContext` for /// `VMGlobalKind::Instance`, and it's `VMComponentContext` for /// `VMGlobalKind::ComponentFlags`. + #[readonly] + #[can_move] pub vmctx: Option>, /// The kind of global, and extra location information in addition to /// `vmctx` above. + #[readonly] + #[can_move] pub kind: VMGlobalKind, } @@ -229,12 +255,18 @@ macro_rules! for_each_vm_type { #[snake_name = vm_tag_import] pub struct VMTagImport { /// A pointer to the imported tag description. + #[readonly] + #[can_move] pub from: VmPtr, /// The instance that owns this tag. + #[readonly] + #[can_move] pub vmctx: VmPtr, /// The index of the tag in the containing `vmctx`. + #[readonly] + #[can_move] pub index: DefinedTagIndex, } diff --git a/crates/wasmtime/src/runtime/vm/component.rs b/crates/wasmtime/src/runtime/vm/component.rs index de21c6738bfd..bafaada3d714 100644 --- a/crates/wasmtime/src/runtime/vm/component.rs +++ b/crates/wasmtime/src/runtime/vm/component.rs @@ -30,7 +30,7 @@ use core::ptr::NonNull; use wasmtime_environ::component::*; use wasmtime_environ::error::OutOfMemory; use wasmtime_environ::prelude::TryPrimaryMap; -use wasmtime_environ::{HostPtr, PrimaryMap, VMSharedTypeIndex}; +use wasmtime_environ::{HostPtr, PrimaryMap, PtrSize as _, VMSharedTypeIndex}; #[allow( clippy::cast_possible_truncation, @@ -365,8 +365,8 @@ impl ComponentInstance { #[inline] pub fn instance_flags(&self, instance: RuntimeComponentInstanceIndex) -> InstanceFlags { unsafe { - let ptr = - self.vmctx_plus_offset_raw::(self.offsets.may_leave(instance)); + let ptr = self + .vmctx_plus_offset_raw::(self.offsets.may_leave().at(instance)); InstanceFlags(SendSyncPtr::new(ptr)) } } @@ -378,7 +378,7 @@ impl ComponentInstance { /// during the instantiation process of a component. pub fn runtime_memory(&self, idx: RuntimeMemoryIndex) -> NonNull { unsafe { - let ret = *self.vmctx_plus_offset::>(self.offsets.runtime_memory(idx)); + let ret = *self.vmctx_plus_offset::>(self.offsets.memories().at(idx)); debug_assert!(ret.as_ptr() as usize != INVALID_PTR); ret.as_non_null() } @@ -391,7 +391,7 @@ impl ComponentInstance { /// during the instantiation process of a component. pub fn runtime_table(&self, idx: RuntimeTableIndex) -> VMTableImport { unsafe { - let ret = *self.vmctx_plus_offset::(self.offsets.runtime_table(idx)); + let ret = *self.vmctx_plus_offset::(self.offsets.tables().at(idx)); debug_assert!(ret.from.as_ptr() as usize != INVALID_PTR); debug_assert!(ret.vmctx.as_ptr() as usize != INVALID_PTR); ret @@ -430,7 +430,7 @@ impl ComponentInstance { /// during the instantiation process of a component. pub fn runtime_realloc(&self, idx: RuntimeReallocIndex) -> NonNull { unsafe { - let ret = *self.vmctx_plus_offset::>(self.offsets.runtime_realloc(idx)); + let ret = *self.vmctx_plus_offset::>(self.offsets.reallocs().at(idx)); debug_assert!(ret.as_ptr() as usize != INVALID_PTR); ret.as_non_null() } @@ -442,7 +442,7 @@ impl ComponentInstance { /// during the instantiation process of a component. pub fn runtime_callback(&self, idx: RuntimeCallbackIndex) -> NonNull { unsafe { - let ret = *self.vmctx_plus_offset::>(self.offsets.runtime_callback(idx)); + let ret = *self.vmctx_plus_offset::>(self.offsets.callbacks().at(idx)); debug_assert!(ret.as_ptr() as usize != INVALID_PTR); ret.as_non_null() } @@ -454,7 +454,7 @@ impl ComponentInstance { /// during the instantiation process of a component. pub fn runtime_post_return(&self, idx: RuntimePostReturnIndex) -> NonNull { unsafe { - let ret = *self.vmctx_plus_offset::>(self.offsets.runtime_post_return(idx)); + let ret = *self.vmctx_plus_offset::>(self.offsets.post_returns().at(idx)); debug_assert!(ret.as_ptr() as usize != INVALID_PTR); ret.as_non_null() } @@ -467,7 +467,7 @@ impl ComponentInstance { /// during the instantiation process of a component. pub fn lowering(&self, idx: LoweredIndex) -> VMLowering { unsafe { - let ret = *self.vmctx_plus_offset::(self.offsets.lowering(idx)); + let ret = *self.vmctx_plus_offset::(self.offsets.lowerings().at(idx)); debug_assert!(ret.callee.as_ptr() as usize != INVALID_PTR); debug_assert!(ret.data.as_ptr() as usize != INVALID_PTR); ret @@ -484,7 +484,7 @@ impl ComponentInstance { /// during the instantiation process of a component. pub fn trampoline_func_ref(&self, idx: TrampolineIndex) -> NonNull { unsafe { - let offset = self.offsets.trampoline_func_ref(idx); + let offset = self.offsets.trampoline_func_refs().at(idx); let ret = self.vmctx_plus_offset_raw::(offset); debug_assert!( mem::transmute::>, usize>(ret.as_ref().wasm_call) @@ -498,7 +498,7 @@ impl ComponentInstance { /// Get the core Wasm function reference for the given unsafe intrinsic. pub fn unsafe_intrinsic_func_ref(&self, idx: UnsafeIntrinsic) -> NonNull { unsafe { - let offset = self.offsets.unsafe_intrinsic_func_ref(idx); + let offset = self.offsets.intrinsic_func_refs().at(idx); let ret = self.vmctx_plus_offset_raw::(offset); debug_assert!( mem::transmute::>, usize>(ret.as_ref().wasm_call) @@ -523,7 +523,7 @@ impl ComponentInstance { ptr: NonNull, ) { unsafe { - let offset = self.offsets.runtime_memory(idx); + let offset = self.offsets.memories().at(idx); let storage = self.vmctx_plus_offset_mut::>(offset); debug_assert!((*storage).as_ptr() as usize == INVALID_PTR); *storage = ptr.into(); @@ -537,7 +537,7 @@ impl ComponentInstance { ptr: NonNull, ) { unsafe { - let offset = self.offsets.runtime_realloc(idx); + let offset = self.offsets.reallocs().at(idx); let storage = self.vmctx_plus_offset_mut::>(offset); debug_assert!((*storage).as_ptr() as usize == INVALID_PTR); *storage = ptr.into(); @@ -551,7 +551,7 @@ impl ComponentInstance { ptr: NonNull, ) { unsafe { - let offset = self.offsets.runtime_callback(idx); + let offset = self.offsets.callbacks().at(idx); let storage = self.vmctx_plus_offset_mut::>(offset); debug_assert!((*storage).as_ptr() as usize == INVALID_PTR); *storage = ptr.into(); @@ -565,7 +565,7 @@ impl ComponentInstance { ptr: NonNull, ) { unsafe { - let offset = self.offsets.runtime_post_return(idx); + let offset = self.offsets.post_returns().at(idx); let storage = self.vmctx_plus_offset_mut::>(offset); debug_assert!((*storage).as_ptr() as usize == INVALID_PTR); *storage = ptr.into(); @@ -582,7 +582,7 @@ impl ComponentInstance { /// here is never needed prior to it being configured here in the instance. pub fn set_runtime_table(self: Pin<&mut Self>, idx: RuntimeTableIndex, import: VMTableImport) { unsafe { - let offset = self.offsets.runtime_table(idx); + let offset = self.offsets.tables().at(idx); let storage = self.vmctx_plus_offset_mut::(offset); debug_assert!((*storage).vmctx.as_ptr() as usize == INVALID_PTR); debug_assert!((*storage).from.as_ptr() as usize == INVALID_PTR); @@ -598,7 +598,7 @@ impl ComponentInstance { debug_assert!(*self.vmctx_plus_offset::(callee) == INVALID_PTR); let data = self.offsets.lowering_data(idx); debug_assert!(*self.vmctx_plus_offset::(data) == INVALID_PTR); - let offset = self.offsets.lowering(idx); + let offset = self.offsets.lowerings().at(idx); *self.vmctx_plus_offset_mut(offset) = lowering; } } @@ -612,7 +612,7 @@ impl ComponentInstance { type_index: VMSharedTypeIndex, ) { unsafe { - let offset = self.offsets.trampoline_func_ref(idx); + let offset = self.offsets.trampoline_func_refs().at(idx); debug_assert!(*self.vmctx_plus_offset::(offset) == INVALID_PTR); let vmctx = VMOpaqueContext::from_vmcomponent(self.vmctx()); *self.vmctx_plus_offset_mut(offset) = VMFuncRef { @@ -633,7 +633,7 @@ impl ComponentInstance { type_index: VMSharedTypeIndex, ) { unsafe { - let offset = self.offsets.unsafe_intrinsic_func_ref(intrinsic); + let offset = self.offsets.intrinsic_func_refs().at(intrinsic); debug_assert!(*self.vmctx_plus_offset::(offset) == INVALID_PTR); let vmctx = VMOpaqueContext::from_vmcomponent(self.vmctx()); *self.vmctx_plus_offset_mut(offset) = VMFuncRef { @@ -655,7 +655,7 @@ impl ComponentInstance { dtor: Option>, ) { unsafe { - let offset = self.offsets.resource_destructor(idx); + let offset = self.offsets.resource_destructors().at(idx); debug_assert!(*self.vmctx_plus_offset::(offset) == INVALID_PTR); *self.vmctx_plus_offset_mut(offset) = dtor.map(VmPtr::from); } @@ -667,14 +667,14 @@ impl ComponentInstance { /// after instantiation. pub fn resource_destructor(&self, idx: ResourceIndex) -> Option> { unsafe { - let offset = self.offsets.resource_destructor(idx); + let offset = self.offsets.resource_destructors().at(idx); debug_assert!(*self.vmctx_plus_offset::(offset) != INVALID_PTR); (*self.vmctx_plus_offset::>>(offset)).map(|p| p.as_non_null()) } } unsafe fn initialize_vmctx(mut self: Pin<&mut Self>) { - let offset = self.offsets.magic(); + let offset = u32::from(self.offsets.ptr.vmcomponent().magic()); // SAFETY: it's safe to write the magic value during initialization and // this is also the right type of value to write. unsafe { @@ -687,14 +687,14 @@ impl ComponentInstance { // is also the right type of value to store in the vmctx. static BUILTINS: libcalls::VMComponentBuiltins = libcalls::VMComponentBuiltins::INIT; let ptr = BUILTINS.expose_provenance(); - let offset = self.offsets.builtins(); + let offset = u32::from(self.offsets.ptr.vmcomponent().builtins()); unsafe { *self.as_mut().vmctx_plus_offset_mut(offset) = VmPtr::from(ptr); } // SAFETY: it's safe to initialize the vmctx in this function and this // is also the right type of value to store in the vmctx. - let offset = self.offsets.vm_store_context(); + let offset = u32::from(self.offsets.ptr.vmcomponent().store_context()); unsafe { *self.as_mut().vmctx_plus_offset_mut(offset) = VmPtr::from(self.store.0.as_ref().vm_store_context_ptr()); @@ -734,7 +734,7 @@ impl ComponentInstance { } for i in 0..self.offsets.num_trampolines { let i = TrampolineIndex::from_u32(i); - let offset = self.offsets.trampoline_func_ref(i); + let offset = self.offsets.trampoline_func_refs().at(i); // SAFETY: see above unsafe { *self.as_mut().vmctx_plus_offset_mut(offset) = INVALID_PTR; @@ -742,7 +742,7 @@ impl ComponentInstance { } for i in 0..self.offsets.num_unsafe_intrinsics { let i = UnsafeIntrinsic::from_u32(i); - let offset = self.offsets.unsafe_intrinsic_func_ref(i); + let offset = self.offsets.intrinsic_func_refs().at(i); // SAFETY: see above unsafe { *self.as_mut().vmctx_plus_offset_mut(offset) = INVALID_PTR; @@ -750,7 +750,7 @@ impl ComponentInstance { } for i in 0..self.offsets.num_runtime_memories { let i = RuntimeMemoryIndex::from_u32(i); - let offset = self.offsets.runtime_memory(i); + let offset = self.offsets.memories().at(i); // SAFETY: see above unsafe { *self.as_mut().vmctx_plus_offset_mut(offset) = INVALID_PTR; @@ -758,7 +758,7 @@ impl ComponentInstance { } for i in 0..self.offsets.num_runtime_reallocs { let i = RuntimeReallocIndex::from_u32(i); - let offset = self.offsets.runtime_realloc(i); + let offset = self.offsets.reallocs().at(i); // SAFETY: see above unsafe { *self.as_mut().vmctx_plus_offset_mut(offset) = INVALID_PTR; @@ -766,7 +766,7 @@ impl ComponentInstance { } for i in 0..self.offsets.num_runtime_callbacks { let i = RuntimeCallbackIndex::from_u32(i); - let offset = self.offsets.runtime_callback(i); + let offset = self.offsets.callbacks().at(i); // SAFETY: see above unsafe { *self.as_mut().vmctx_plus_offset_mut(offset) = INVALID_PTR; @@ -774,7 +774,7 @@ impl ComponentInstance { } for i in 0..self.offsets.num_runtime_post_returns { let i = RuntimePostReturnIndex::from_u32(i); - let offset = self.offsets.runtime_post_return(i); + let offset = self.offsets.post_returns().at(i); // SAFETY: see above unsafe { *self.as_mut().vmctx_plus_offset_mut(offset) = INVALID_PTR; @@ -782,7 +782,7 @@ impl ComponentInstance { } for i in 0..self.offsets.num_resources { let i = ResourceIndex::from_u32(i); - let offset = self.offsets.resource_destructor(i); + let offset = self.offsets.resource_destructors().at(i); // SAFETY: see above unsafe { *self.as_mut().vmctx_plus_offset_mut(offset) = INVALID_PTR; @@ -790,7 +790,7 @@ impl ComponentInstance { } for i in 0..self.offsets.num_runtime_tables { let i = RuntimeTableIndex::from_u32(i); - let offset = self.offsets.runtime_table(i); + let offset = self.offsets.tables().at(i); // SAFETY: see above #[allow(clippy::cast_possible_truncation, reason = "known to not overflow")] unsafe { diff --git a/crates/wasmtime/src/runtime/vm/instance.rs b/crates/wasmtime/src/runtime/vm/instance.rs index 06ba7e44551c..321f140def87 100644 --- a/crates/wasmtime/src/runtime/vm/instance.rs +++ b/crates/wasmtime/src/runtime/vm/instance.rs @@ -418,32 +418,32 @@ impl Instance { /// Return the indexed `VMFunctionImport`. fn imported_function(&self, index: FuncIndex) -> &VMFunctionImport { - unsafe { self.vmctx_plus_offset(self.offsets().vmctx_vmfunction_import(index)) } + unsafe { self.vmctx_plus_offset(self.offsets().imported_functions().at(index)) } } /// Return the index `VMTableImport`. fn imported_table(&self, index: TableIndex) -> &VMTableImport { - unsafe { self.vmctx_plus_offset(self.offsets().vmctx_vmtable_import(index)) } + unsafe { self.vmctx_plus_offset(self.offsets().imported_tables().at(index)) } } /// Return the indexed `VMMemoryImport`. fn imported_memory(&self, index: MemoryIndex) -> &VMMemoryImport { - unsafe { self.vmctx_plus_offset(self.offsets().vmctx_vmmemory_import(index)) } + unsafe { self.vmctx_plus_offset(self.offsets().imported_memories().at(index)) } } /// Return the indexed `VMGlobalImport`. fn imported_global(&self, index: GlobalIndex) -> &VMGlobalImport { - unsafe { self.vmctx_plus_offset(self.offsets().vmctx_vmglobal_import(index)) } + unsafe { self.vmctx_plus_offset(self.offsets().imported_globals().at(index)) } } /// Return the indexed `VMTagImport`. fn imported_tag(&self, index: TagIndex) -> &VMTagImport { - unsafe { self.vmctx_plus_offset(self.offsets().vmctx_vmtag_import(index)) } + unsafe { self.vmctx_plus_offset(self.offsets().imported_tags().at(index)) } } /// Return the indexed `VMTagDefinition`. pub fn tag_ptr(&self, index: DefinedTagIndex) -> NonNull { - unsafe { self.vmctx_plus_offset_raw(self.offsets().vmctx_vmtag_definition(index)) } + unsafe { self.vmctx_plus_offset_raw(self.offsets().tags().at(index)) } } /// Return the indexed `VMTableDefinition`. @@ -461,7 +461,7 @@ impl Instance { /// Return a pointer to the `index`'th table within this instance, stored /// in vmctx memory. pub fn table_ptr(&self, index: DefinedTableIndex) -> NonNull { - unsafe { self.vmctx_plus_offset_raw(self.offsets().vmctx_vmtable_definition(index)) } + unsafe { self.vmctx_plus_offset_raw(self.offsets().tables().at(index)) } } /// Get a locally defined or imported memory. @@ -496,14 +496,14 @@ impl Instance { #[inline] pub fn memory_ptr(&self, index: DefinedMemoryIndex) -> NonNull { unsafe { - self.vmctx_plus_offset::>(self.offsets().vmctx_vmmemory_pointer(index)) + self.vmctx_plus_offset::>(self.offsets().memories().at(index)) .as_non_null() } } /// Return the indexed `VMGlobalDefinition`. pub fn global_ptr(&self, index: DefinedGlobalIndex) -> NonNull { - unsafe { self.vmctx_plus_offset_raw(self.offsets().vmctx_vmglobal_definition(index)) } + unsafe { self.vmctx_plus_offset_raw(self.offsets().globals().at(index)) } } /// Get all globals within this instance. @@ -538,19 +538,19 @@ impl Instance { /// Return a pointer to the interrupts structure #[inline] pub fn vm_store_context(&self) -> NonNull>> { - unsafe { self.vmctx_plus_offset_raw(self.offsets().ptr.vmctx_store_context()) } + unsafe { self.vmctx_plus_offset_raw(self.offsets().ptr.vmctx().store_context()) } } /// Return a pointer to the global epoch counter used by this instance. #[cfg(target_has_atomic = "64")] pub fn epoch_ptr(self: Pin<&mut Self>) -> &mut Option> { - let offset = self.offsets().ptr.vmctx_epoch_ptr(); + let offset = self.offsets().ptr.vmctx().epoch_ptr(); unsafe { self.vmctx_plus_offset_mut(offset) } } /// Return a pointer to the collector-specific heap data. pub fn gc_heap_data(self: Pin<&mut Self>) -> &mut Option> { - let offset = self.offsets().ptr.vmctx_gc_heap_data(); + let offset = self.offsets().ptr.vmctx().gc_heap_data(); unsafe { self.vmctx_plus_offset_mut(offset) } } @@ -841,7 +841,7 @@ impl Instance { } fn type_ids_array(&self) -> NonNull> { - unsafe { self.vmctx_plus_offset_raw(self.offsets().ptr.vmctx_type_ids_array()) } + unsafe { self.vmctx_plus_offset_raw(self.offsets().ptr.vmctx().type_ids()) } } /// Get a `&VMFuncRef` for the given `FuncIndex`. @@ -879,7 +879,7 @@ impl Instance { let index = module.func_index(def_index); let func = &module.functions[index]; let type_index = func.signature.unwrap_engine_type_index(); - let vmctx_offset = self.offsets().vmctx_func_ref(func.func_ref); + let vmctx_offset = self.offsets().func_refs().at(func.func_ref); let array_to_wasm_key = FuncKey::ArrayToWasmTrampoline(module.module_index, def_index); let wasm_key = FuncKey::DefinedWasmFunction(module.module_index, def_index); // SAFETY: the type/offset/keys here are all valid for the defined @@ -906,7 +906,7 @@ impl Instance { t.unwrap_engine_type_index() } }; - let vmctx_offset = self.offsets().vmctx_startup_func_ref(); + let vmctx_offset = self.offsets().startup_func_ref(); let array_to_wasm_key = FuncKey::ModuleStartup(Abi::Array, module.module_index); let wasm_key = FuncKey::ModuleStartup(Abi::Wasm, module.module_index); // SAFETY: the type/offset/keys here are all valid for the module @@ -1185,7 +1185,7 @@ impl Instance { unsafe { let offsets = instance.runtime_info.offsets(); instance - .vmctx_plus_offset_raw::(offsets.ptr.vmctx_magic()) + .vmctx_plus_offset_raw::(offsets.ptr.vmctx().magic()) .write(VMCONTEXT_MAGIC); } @@ -1213,7 +1213,7 @@ impl Instance { let ptr = BUILTINS.expose_provenance(); let offsets = instance.runtime_info.offsets(); instance - .vmctx_plus_offset_raw(offsets.ptr.vmctx_builtin_functions()) + .vmctx_plus_offset_raw(offsets.ptr.vmctx().builtin_functions()) .write(VmPtr::from(ptr)); } @@ -1227,7 +1227,7 @@ impl Instance { ptr::copy_nonoverlapping( imports.functions.as_ptr(), instance - .vmctx_plus_offset_raw(offsets.vmctx_imported_functions_begin()) + .vmctx_plus_offset_raw(offsets.imported_functions().begin()) .as_ptr(), imports.functions.len(), ); @@ -1235,7 +1235,7 @@ impl Instance { ptr::copy_nonoverlapping( imports.tables.as_ptr(), instance - .vmctx_plus_offset_raw(offsets.vmctx_imported_tables_begin()) + .vmctx_plus_offset_raw(offsets.imported_tables().begin()) .as_ptr(), imports.tables.len(), ); @@ -1243,7 +1243,7 @@ impl Instance { ptr::copy_nonoverlapping( imports.memories.as_ptr(), instance - .vmctx_plus_offset_raw(offsets.vmctx_imported_memories_begin()) + .vmctx_plus_offset_raw(offsets.imported_memories().begin()) .as_ptr(), imports.memories.len(), ); @@ -1251,7 +1251,7 @@ impl Instance { ptr::copy_nonoverlapping( imports.globals.as_ptr(), instance - .vmctx_plus_offset_raw(offsets.vmctx_imported_globals_begin()) + .vmctx_plus_offset_raw(offsets.imported_globals().begin()) .as_ptr(), imports.globals.len(), ); @@ -1259,7 +1259,7 @@ impl Instance { ptr::copy_nonoverlapping( imports.tags.as_ptr(), instance - .vmctx_plus_offset_raw(offsets.vmctx_imported_tags_begin()) + .vmctx_plus_offset_raw(offsets.imported_tags().begin()) .as_ptr(), imports.tags.len(), ); @@ -1277,7 +1277,7 @@ impl Instance { // valid. unsafe { let offsets = instance.runtime_info.offsets(); - let mut ptr = instance.vmctx_plus_offset_raw(offsets.vmctx_tables_begin()); + let mut ptr = instance.vmctx_plus_offset_raw(offsets.tables().begin()); let tables = instance.as_mut().tables_mut(); for i in 0..module.num_defined_tables() { ptr.write(tables[DefinedTableIndex::new(i)].1.vmtable()); @@ -1296,9 +1296,8 @@ impl Instance { // valid. unsafe { let offsets = instance.runtime_info.offsets(); - let mut ptr = instance.vmctx_plus_offset_raw(offsets.vmctx_memories_begin()); - let mut owned_ptr = - instance.vmctx_plus_offset_raw(offsets.vmctx_owned_memories_begin()); + let mut ptr = instance.vmctx_plus_offset_raw(offsets.memories().begin()); + let mut owned_ptr = instance.vmctx_plus_offset_raw(offsets.owned_memories().begin()); let memories = instance.as_mut().memories_mut(); for i in 0..module.num_defined_memories() { let defined_memory_index = DefinedMemoryIndex::new(i); @@ -1355,7 +1354,7 @@ impl Instance { // valid. unsafe { let offsets = instance.runtime_info.offsets(); - let mut ptr = instance.vmctx_plus_offset_raw(offsets.vmctx_tags_begin()); + let mut ptr = instance.vmctx_plus_offset_raw(offsets.tags().begin()); for i in 0..module.num_defined_tags() { let defined_index = DefinedTagIndex::new(i); let tag_index = module.tag_index(defined_index); @@ -1375,9 +1374,8 @@ impl Instance { unsafe { let offsets = instance.runtime_info.offsets(); let mut lengths = - instance.vmctx_plus_offset_raw(offsets.vmctx_runtime_data_lengths_begin()); - let mut bases = - instance.vmctx_plus_offset_raw(offsets.vmctx_runtime_data_bases_begin()); + instance.vmctx_plus_offset_raw(offsets.runtime_data_lengths().begin()); + let mut bases = instance.vmctx_plus_offset_raw(offsets.runtime_data_bases().begin()); for i in module.runtime_data.keys() { let data = instance.runtime_data(i); lengths.write(u32::try_from(data.len()).unwrap()); @@ -1408,10 +1406,10 @@ impl Instance { let offsets = instance.runtime_info.offsets(); unsafe { instance - .vmctx_plus_offset_raw(offsets.vmctx_runtime_data_length(*data)) + .vmctx_plus_offset_raw(offsets.runtime_data_lengths().at(*data)) .write(0u32); instance - .vmctx_plus_offset_raw(offsets.vmctx_runtime_data_base(*data)) + .vmctx_plus_offset_raw(offsets.runtime_data_bases().at(*data)) .write(0usize); } } diff --git a/crates/wasmtime/src/runtime/vm/vmcontext.rs b/crates/wasmtime/src/runtime/vm/vmcontext.rs index 9324674f1e90..ba708d03adb3 100644 --- a/crates/wasmtime/src/runtime/vm/vmcontext.rs +++ b/crates/wasmtime/src/runtime/vm/vmcontext.rs @@ -331,7 +331,7 @@ mod test_vmglobal_definition { fn check_vmglobal_begins_aligned() { let module = Module::new(StaticModuleIndex::from_u32(0)); let offsets = VMOffsets::new(HostPtr, &module); - assert_eq!(offsets.vmctx_globals_begin() % 16, 0); + assert_eq!(offsets.globals().begin() % 16, 0); } #[test] @@ -544,7 +544,7 @@ mod test_vmtag_definition { fn check_vmtag_begins_aligned() { let module = Module::new(StaticModuleIndex::from_u32(0)); let offsets = VMOffsets::new(HostPtr, &module); - assert_eq!(offsets.vmctx_tags_begin() % 16, 0); + assert_eq!(offsets.tags().begin() % 16, 0); } } @@ -969,7 +969,7 @@ impl VMContext { #[inline] pub unsafe fn from_opaque(opaque: NonNull) -> NonNull { // Note that in general the offset of the "magic" field is stored in - // `VMOffsets::vmctx_magic`. Given though that this is a sanity check + // `VMContext::magic`. Given though that this is a sanity check // about converting this pointer to another type we ideally don't want // to read the offset from potentially corrupt memory. Instead it would // be better to catch errors here as soon as possible. diff --git a/winch/codegen/src/codegen/call.rs b/winch/codegen/src/codegen/call.rs index 0f54c9254cd5..e8a8f7914e52 100644 --- a/winch/codegen/src/codegen/call.rs +++ b/winch/codegen/src/codegen/call.rs @@ -201,11 +201,13 @@ impl FnCall { context.without::, M, _>(&sig.regs, masm, |context, masm| { Ok((context.any_gpr(masm)?, context.any_gpr(masm)?)) })??; - let callee_vmctx_offset = vmoffsets.vmctx_vmfunction_import_vmctx(index); + let vmimport = vmoffsets.imported_functions().at(index); + let callee_vmctx_offset = vmimport + u32::from(vmoffsets.ptr.vm_function_import().vmctx()); let callee_vmctx_addr = masm.address_at_vmctx(callee_vmctx_offset)?; masm.load_ptr(callee_vmctx_addr, writable!(callee_vmctx))?; - let callee_body_offset = vmoffsets.vmctx_vmfunction_import_wasm_call(index); + let callee_body_offset = + vmimport + u32::from(vmoffsets.ptr.vm_function_import().wasm_call()); let callee_addr = masm.address_at_vmctx(callee_body_offset)?; masm.load_ptr(callee_addr, writable!(callee))?; diff --git a/winch/codegen/src/codegen/env.rs b/winch/codegen/src/codegen/env.rs index f53fdba350d0..07f88c120bb5 100644 --- a/winch/codegen/src/codegen/env.rs +++ b/winch/codegen/src/codegen/env.rs @@ -215,12 +215,13 @@ impl<'a, 'translation, 'data, P: PtrSize> FuncEnv<'a, 'translation, 'data, P> { let ty = self.translation.module.globals[index].wasm_ty; let val = || match self.translation.module.defined_global_index(index) { Some(defined_index) => GlobalData { - offset: self.vmoffsets.vmctx_vmglobal_definition(defined_index), + offset: self.vmoffsets.globals().at(defined_index), imported: false, ty, }, None => GlobalData { - offset: self.vmoffsets.vmctx_vmglobal_import_from(index), + offset: self.vmoffsets.imported_globals().at(index) + + u32::from(self.vmoffsets.ptr.vm_global_import().from()), imported: true, ty, }, @@ -238,12 +239,18 @@ impl<'a, 'translation, 'data, P: PtrSize> FuncEnv<'a, 'translation, 'data, P> { match self.translation.module.defined_table_index(index) { Some(defined) => ( None, - self.vmoffsets.vmctx_vmtable_definition_base(defined), - self.vmoffsets - .vmctx_vmtable_definition_current_elements(defined), + self.vmoffsets.tables().at(defined) + + u32::from(self.vmoffsets.ptr.vm_table_definition().base()), + self.vmoffsets.tables().at(defined) + + u32::from( + self.vmoffsets.ptr.vm_table_definition().current_elements(), + ), ), None => ( - Some(self.vmoffsets.vmctx_vmtable_from(index)), + Some( + self.vmoffsets.imported_tables().at(index) + + u32::from(self.vmoffsets.ptr.vm_table_import().from()), + ), self.vmoffsets.ptr.vm_table_definition().base().into(), self.vmoffsets .ptr @@ -274,39 +281,48 @@ impl<'a, 'translation, 'data, P: PtrSize> FuncEnv<'a, 'translation, 'data, P> { match self.resolved_heaps.entry(index) { Occupied(entry) => *entry.get(), Vacant(entry) => { - let (import_from, base_offset, current_length_offset) = - match self.translation.module.defined_memory_index(index) { - Some(defined) => { - if is_shared { - ( - Some(self.vmoffsets.vmctx_vmmemory_pointer(defined)), - self.vmoffsets.ptr.vm_memory_definition().base().into(), - self.vmoffsets - .ptr - .vm_memory_definition() - .current_length() - .into(), - ) - } else { - let owned = self.translation.module.owned_memory_index(defined); - ( - None, - self.vmoffsets.vmctx_vmmemory_definition_base(owned), - self.vmoffsets - .vmctx_vmmemory_definition_current_length(owned), - ) - } + let (import_from, base_offset, current_length_offset) = match self + .translation + .module + .defined_memory_index(index) + { + Some(defined) => { + if is_shared { + ( + Some(self.vmoffsets.memories().at(defined)), + self.vmoffsets.ptr.vm_memory_definition().base().into(), + self.vmoffsets + .ptr + .vm_memory_definition() + .current_length() + .into(), + ) + } else { + let owned = self.translation.module.owned_memory_index(defined); + ( + None, + self.vmoffsets.owned_memories().at(owned) + + u32::from(self.vmoffsets.ptr.vm_memory_definition().base()), + self.vmoffsets.owned_memories().at(owned) + + u32::from( + self.vmoffsets.ptr.vm_memory_definition().current_length(), + ), + ) } - None => ( - Some(self.vmoffsets.vmctx_vmmemory_import_from(index)), - self.vmoffsets.ptr.vm_memory_definition().base().into(), - self.vmoffsets - .ptr - .vm_memory_definition() - .current_length() - .into(), + } + None => ( + Some( + self.vmoffsets.imported_memories().at(index) + + u32::from(self.vmoffsets.ptr.vm_memory_import().from()), ), - }; + self.vmoffsets.ptr.vm_memory_definition().base().into(), + self.vmoffsets + .ptr + .vm_memory_definition() + .current_length() + .into(), + ), + }; let memory = &self.translation.module.memories[index]; diff --git a/winch/codegen/src/codegen/mod.rs b/winch/codegen/src/codegen/mod.rs index ac0fd0b1c145..9cea95379528 100644 --- a/winch/codegen/src/codegen/mod.rs +++ b/winch/codegen/src/codegen/mod.rs @@ -466,7 +466,7 @@ where .as_u32() .checked_mul(sig_index_bytes.into()) .unwrap(); - let signatures_base_offset = self.env.vmoffsets.ptr.vmctx_type_ids_array(); + let signatures_base_offset = self.env.vmoffsets.ptr.vmctx().type_ids(); let funcref_sig_offset = self.env.vmoffsets.ptr.vm_func_ref().type_index(); // Get the caller id. let caller_id = self.context.any_gpr(self.masm)?; @@ -1747,7 +1747,8 @@ where let data_segment_length_offset = self .env .vmoffsets - .vmctx_runtime_data_length(runtime_data_index); + .runtime_data_lengths() + .at(runtime_data_index); let tmp1 = self.context.any_gpr(self.masm)?; let tmp2 = self.context.any_gpr(self.masm)?; self.masm.load( @@ -1774,7 +1775,8 @@ where let data_segment_base_offset = self .env .vmoffsets - .vmctx_runtime_data_base(runtime_data_index); + .runtime_data_bases() + .at(runtime_data_index); self.masm.load( self.masm.address_at_vmctx(data_segment_base_offset)?, writable!(tmp1), @@ -1812,7 +1814,8 @@ where let data_segment_offset = self .env .vmoffsets - .vmctx_runtime_data_length(runtime_data_index); + .runtime_data_lengths() + .at(runtime_data_index); let len_addr = self.masm.address_at_vmctx(data_segment_offset)?; self.masm.store(RegImm::i32(0), len_addr, OperandSize::S32) } @@ -2098,7 +2101,7 @@ where /// Emits a series of instructions that load the `fuel_consumed` field from /// `VMStoreContext`. fn emit_load_fuel_consumed(&mut self, fuel_reg: Reg) -> Result<()> { - let store_context_offset = self.env.vmoffsets.ptr.vmctx_store_context(); + let store_context_offset = self.env.vmoffsets.ptr.vmctx().store_context(); let fuel_offset = self.env.vmoffsets.ptr.vm_store_context().fuel_consumed(); self.masm.load_ptr( self.masm @@ -2176,8 +2179,8 @@ where epoch_deadline_reg: Reg, epoch_counter_reg: Reg, ) -> Result<()> { - let epoch_ptr_offset = self.env.vmoffsets.ptr.vmctx_epoch_ptr(); - let store_context_offset = self.env.vmoffsets.ptr.vmctx_store_context(); + let epoch_ptr_offset = self.env.vmoffsets.ptr.vmctx().epoch_ptr(); + let store_context_offset = self.env.vmoffsets.ptr.vmctx().store_context(); let epoch_deadline_offset = self.env.vmoffsets.ptr.vm_store_context().epoch_deadline(); // Load the current epoch value into `epoch_counter_var`. @@ -2218,7 +2221,7 @@ where return Ok(()); } - let store_context_offset = self.env.vmoffsets.ptr.vmctx_store_context(); + let store_context_offset = self.env.vmoffsets.ptr.vmctx().store_context(); let fuel_offset = self.env.vmoffsets.ptr.vm_store_context().fuel_consumed(); let limits_reg = self.context.any_gpr(self.masm)?; @@ -2527,7 +2530,7 @@ where // is loaded from the `VMMemoryImport` and the vmctx is loaded from // the vmctx itself. None => { - let vmimport = self.env.vmoffsets.vmctx_vmmemory_import(mem); + let vmimport = self.env.vmoffsets.imported_memories().at(mem); let vmctx_offset = vmimport + u32::from(self.env.vmoffsets.ptr.vm_memory_import().vmctx()); let index_offset = @@ -2559,7 +2562,7 @@ where Ok(Callee::Builtin(builtin)) } None => { - let vmimport = self.env.vmoffsets.vmctx_vmtable_import(table); + let vmimport = self.env.vmoffsets.imported_tables().at(table); let vmctx_offset = vmimport + u32::from(self.env.vmoffsets.ptr.vm_table_import().vmctx()); let index_offset = diff --git a/winch/codegen/src/isa/aarch64/masm.rs b/winch/codegen/src/isa/aarch64/masm.rs index 9ac7842106a7..5a376aa99b47 100644 --- a/winch/codegen/src/isa/aarch64/masm.rs +++ b/winch/codegen/src/isa/aarch64/masm.rs @@ -191,7 +191,7 @@ impl Masm for MacroAssembler { masm.with_scratch::(|masm, scratch_stk_limit| { masm.with_scratch::(|masm, scratch_tmp| { masm.load_ptr( - masm.address_at_reg(vmctx, ptr_size_u8.vmcontext_store_context().into())?, + masm.address_at_reg(vmctx, ptr_size_u8.vmctx().store_context().into())?, scratch_stk_limit.writable(), )?; diff --git a/winch/codegen/src/isa/x64/masm.rs b/winch/codegen/src/isa/x64/masm.rs index 0044af022298..9019c36a20dc 100644 --- a/winch/codegen/src/isa/x64/masm.rs +++ b/winch/codegen/src/isa/x64/masm.rs @@ -129,7 +129,7 @@ impl Masm for MacroAssembler { self.with_scratch::(|masm, scratch| { masm.load_ptr( - masm.address_at_reg(vmctx, ptr_size.vmcontext_store_context().into())?, + masm.address_at_reg(vmctx, ptr_size.vmctx().store_context().into())?, scratch.writable(), )?;