diff --git a/Cargo.lock b/Cargo.lock index e890d367f22c..eb4c2e5fc929 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5324,6 +5324,27 @@ dependencies = [ "wasm-encoder 0.254.0", ] +[[package]] +name = "wast-arm-runtest" +version = "48.0.0" +dependencies = [ + "anyhow", + "cranelift-codegen", + "cranelift-frontend", + "cranelift-module", + "cranelift-object", + "json-from-wast", + "serde", + "serde_json", + "target-lexicon", + "tempfile", + "wasmparser 0.253.0", + "wasmtime-environ", + "wasmtime-internal-cranelift", + "wast 253.0.0", + "wat", +] + [[package]] name = "wat" version = "1.254.0" diff --git a/Cargo.toml b/Cargo.toml index bdfba85d5c2a..7c90e5b75172 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -181,6 +181,7 @@ opt-level = 0 resolver = '2' members = [ "cranelift", + "cranelift/filetests/wast-arm-runtest", "cranelift/assembler-x64/fuzz", "cranelift/isle/fuzz", "cranelift/isle/islec", @@ -409,7 +410,7 @@ log = { version = "0.4.28", default-features = false } clap = { version = "4.5.48", default-features = false, features = ["std", "derive"] } clap_complete = "4.5.58" hashbrown = { version = "0.17", default-features = false } -capstone = { version = "0.14.0", default-features = false, features = ['full', 'arch_x86', 'arch_riscv', 'arch_arm64', 'arch_sysz'] } +capstone = { version = "0.14.0", default-features = false, features = ['full', 'arch_x86', 'arch_riscv', 'arch_arm', 'arch_arm64', 'arch_sysz'] } xed-sys = { version = "0.6" } smallvec = { version = "1.15.1", features = ["union"] } tracing = { version = "0.1.41", default-features = false } diff --git a/cranelift/codegen/Cargo.toml b/cranelift/codegen/Cargo.toml index 9235281ec986..4aa87d890e60 100644 --- a/cranelift/codegen/Cargo.toml +++ b/cranelift/codegen/Cargo.toml @@ -92,6 +92,7 @@ unwind = ["gimli"] # ISA targets for which we should build. # If no ISA targets are explicitly enabled, the ISA target for the host machine is enabled. x86 = [] +arm32 = [] arm64 = [] s390x = [] riscv64 = [] @@ -109,7 +110,7 @@ all-arch = ["all-native-arch", "pulley"] # Option to enable all architectures that correspond to an actual native target # (that is, exclude Pulley). -all-native-arch = ["x86", "arm64", "s390x", "riscv64"] +all-native-arch = ["x86", "arm32", "arm64", "s390x", "riscv64"] # For dependent crates that want to serialize some parts of cranelift enable-serde = [ diff --git a/cranelift/codegen/meta/src/isa/arm32.rs b/cranelift/codegen/meta/src/isa/arm32.rs new file mode 100644 index 000000000000..25aff482bb9b --- /dev/null +++ b/cranelift/codegen/meta/src/isa/arm32.rs @@ -0,0 +1,25 @@ +use crate::cdsl::isa::TargetIsa; +use crate::cdsl::settings::SettingGroupBuilder; + +pub(crate) fn define() -> TargetIsa { + let mut settings = SettingGroupBuilder::new("arm32"); + + settings.add_bool( + "has_neon", + "Has Advanced SIMD (NEON) support; does not have an effect on code \ + generation by itself yet, reserved for future use.", + "", + false, + ); + + settings.add_bool( + "has_idiv", + "Has hardware integer divide (the `sdiv`/`udiv` instructions, present \ + on ARMv7-R/M and ARMv7VE); when disabled, division must go through a \ + runtime library call.", + "", + false, + ); + + TargetIsa::new("arm32", settings.build()) +} diff --git a/cranelift/codegen/meta/src/isa/mod.rs b/cranelift/codegen/meta/src/isa/mod.rs index 655b14a9c5a1..08974c98966d 100644 --- a/cranelift/codegen/meta/src/isa/mod.rs +++ b/cranelift/codegen/meta/src/isa/mod.rs @@ -2,6 +2,7 @@ use crate::cdsl::isa::TargetIsa; use std::fmt; +mod arm32; mod arm64; mod pulley; mod riscv64; @@ -12,6 +13,7 @@ pub(crate) mod x86; #[derive(PartialEq, Copy, Clone)] pub enum Isa { X86, + Arm32, Arm64, S390x, Riscv64, @@ -32,6 +34,11 @@ impl Isa { pub fn from_arch(arch: &str) -> Option { match arch { "aarch64" => Some(Isa::Arm64), + // 32-bit ARM / Thumb, but not the 64-bit `arm64*` spellings + // (e.g. `arm64ec`), which are AArch64. + x if (x.starts_with("arm") || x.starts_with("thumb")) && !x.starts_with("arm64") => { + Some(Isa::Arm32) + } "s390x" => Some(Isa::S390x), x if ["x86_64", "i386", "i586", "i686"].contains(&x) => Some(Isa::X86), "riscv64" | "riscv64gc" | "riscv64imac" => Some(Isa::Riscv64), @@ -45,6 +52,7 @@ impl Isa { pub fn all() -> &'static [Isa] { &[ Isa::X86, + Isa::Arm32, Isa::Arm64, Isa::S390x, Isa::Riscv64, @@ -59,6 +67,7 @@ impl fmt::Display for Isa { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { match *self { Isa::X86 => write!(f, "x86"), + Isa::Arm32 => write!(f, "arm32"), Isa::Arm64 => write!(f, "arm64"), Isa::S390x => write!(f, "s390x"), Isa::Riscv64 => write!(f, "riscv64"), @@ -72,6 +81,7 @@ pub(crate) fn define(isas: &[Isa]) -> Vec { isas.iter() .map(|isa| match isa { Isa::X86 => x86::define(), + Isa::Arm32 => arm32::define(), Isa::Arm64 => arm64::define(), Isa::S390x => s390x::define(), Isa::Riscv64 => riscv64::define(), diff --git a/cranelift/codegen/meta/src/isle.rs b/cranelift/codegen/meta/src/isle.rs index 6f39a34e1390..5a824497069f 100644 --- a/cranelift/codegen/meta/src/isle.rs +++ b/cranelift/codegen/meta/src/isle.rs @@ -144,6 +144,7 @@ pub fn get_isle_compilations( // Directories for lowering backends. let src_isa_x64 = codegen_crate_dir.join("src").join("isa").join("x64"); + let src_isa_arm32 = codegen_crate_dir.join("src").join("isa").join("arm32"); let src_isa_aarch64 = codegen_crate_dir.join("src").join("isa").join("aarch64"); let src_isa_s390x = codegen_crate_dir.join("src").join("isa").join("s390x"); let src_isa_risc_v = codegen_crate_dir.join("src").join("isa").join("riscv64"); @@ -252,6 +253,21 @@ pub fn get_isle_compilations( .concat(), untracked_inputs: vec![numerics_isle.clone(), clif_lower_isle.clone()], }, + // The arm32 (AArch32 / A32) instruction selector. + IsleCompilation { + name: "arm32".to_string(), + output: gen_dir.join("isle_arm32.rs"), + tracked_inputs: [ + vec![prelude_isle.clone(), prelude_lower_isle.clone()], + spec_inputs(&[]), + vec![ + src_isa_arm32.join("inst.isle"), + src_isa_arm32.join("lower.isle"), + ], + ] + .concat(), + untracked_inputs: vec![numerics_isle.clone(), clif_lower_isle.clone()], + }, // The risc-v instruction selector. IsleCompilation { name: "riscv64".to_string(), diff --git a/cranelift/codegen/src/isa/arm32/abi.rs b/cranelift/codegen/src/isa/arm32/abi.rs new file mode 100644 index 000000000000..c5e72d8cb38a --- /dev/null +++ b/cranelift/codegen/src/isa/arm32/abi.rs @@ -0,0 +1,565 @@ +//! Implementation of a standard Arm32 ABI. + +use crate::CodegenResult; +use crate::ir; +use crate::ir::types::*; +use crate::ir::{Signature, Type}; +use crate::isa; +use crate::isa::arm32::inst::*; +use crate::isa::arm32::settings::Flags as Arm32Flags; +use crate::machinst::*; +use crate::settings; +use alloc::vec::Vec; +use regalloc2::{MachineEnv, PRegSet}; +use smallvec::{SmallVec, smallvec}; + +/// Support for the arm32 ABI from the callee side (within a function body). +pub(crate) type Arm32Callee = Callee; + +/// arm32-specific ABI behavior. This struct just serves as an implementation +/// point for the trait; it is never actually instantiated. +pub struct Arm32MachineDeps; + +impl IsaFlags for Arm32Flags {} + +impl ABIMachineSpec for Arm32MachineDeps { + type I = Inst; + type F = Arm32Flags; + + /// 128 MB, as with the other backends, to avoid overflow with 32-bit + /// arithmetic on stack offsets. + const STACK_ARG_RET_SIZE_LIMIT: u32 = 128 * 1024 * 1024; + + fn word_bits() -> u32 { + 32 + } + + fn stack_align(_call_conv: isa::CallConv) -> u32 { + // AAPCS requires 8-byte stack alignment at public interfaces. + 8 + } + + fn compute_arg_locs( + call_conv: isa::CallConv, + flags: &settings::Flags, + params: &[ir::AbiParam], + args_or_rets: ArgsOrRets, + add_ret_area_ptr: bool, + mut args: ArgsAccumulator, + ) -> CodegenResult<(u32, Option)> { + // Integer arguments/returns go in r0-r3 (r0-r1 for returns), the rest on + // the stack. This is a simplified AAPCS. + let (x_start, x_end, d_end) = match args_or_rets { + ArgsOrRets::Args => (0u8, 3u8, 7u8), + ArgsOrRets::Rets => (0u8, 1u8, 1u8), + }; + let mut next_x_reg = x_start; + // VFP (hard-float) register counter: each float value takes one D reg + // (an f32 uses the low S-half). This is a simplified AAPCS-VFP that does + // not back-fill S registers. + let mut next_d_reg = 0u8; + let mut next_stack: u32 = 0; + + let ret_area_ptr = if add_ret_area_ptr { + assert!(ArgsOrRets::Args == args_or_rets); + let reg = xreg(next_x_reg); + next_x_reg += 1; + Some(ABIArg::reg( + reg.to_real_reg().unwrap(), + I32, + ir::ArgumentExtension::None, + ir::ArgumentPurpose::Normal, + )) + } else { + None + }; + + for param in params { + if let ir::ArgumentPurpose::StructArgument(_) = param.purpose { + panic!("StructArgument parameters are not supported on arm32."); + } + + let (rcs, reg_tys) = Inst::rc_for_type(param.value_type)?; + + // AAPCS: a 64-bit value passed in registers must start in an + // even-numbered register (an r0:r1 / r2:r3 pair). If the aligned + // pair would not fully fit, it goes entirely on the (8-aligned) + // stack, so bump `next_x_reg` past the argument registers. + if rcs.len() == 2 { + next_x_reg = align_to(u32::from(next_x_reg), 2) as u8; + if next_x_reg + 1 > x_end { + next_x_reg = x_end + 1; + next_stack = align_to(next_stack, 8); + } + } + + let mut slots = ABIArgSlotVec::new(); + for (rc, reg_ty) in rcs.iter().zip(reg_tys.iter()) { + let reg = match rc { + RegClass::Float if next_d_reg <= d_end => { + let r = dreg(next_d_reg); + next_d_reg += 1; + Some(r) + } + RegClass::Int if next_x_reg <= x_end => { + let r = xreg(next_x_reg); + next_x_reg += 1; + Some(r) + } + RegClass::Int | RegClass::Float => None, + RegClass::Vector => unreachable!("arm32 has no vector registers"), + }; + if let Some(reg) = reg { + slots.push(ABIArgSlot::Reg { + reg: reg.to_real_reg().unwrap(), + ty: *reg_ty, + extension: param.extension, + }); + } else { + if args_or_rets == ArgsOrRets::Rets && !flags.enable_multi_ret_implicit_sret() { + return Err(crate::CodegenError::Unsupported( + "Too many return values to fit in registers.".into(), + )); + } + let size = core::cmp::max(reg_ty.bytes(), 4); + next_stack = align_to(next_stack, size); + slots.push(ABIArgSlot::Stack { + offset: next_stack as i64, + ty: *reg_ty, + extension: param.extension, + }); + next_stack += size; + } + } + args.push(ABIArg::Slots { + slots, + purpose: param.purpose, + }); + } + + let pos = if let Some(ret_area_ptr) = ret_area_ptr { + args.push_non_formal(ret_area_ptr); + Some(args.args().len() - 1) + } else { + None + }; + + next_stack = align_to(next_stack, Self::stack_align(call_conv)); + Ok((next_stack, pos)) + } + + fn gen_load_stack(mem: StackAMode, into_reg: Writable, _ty: Type) -> Inst { + if into_reg.to_reg().class() == RegClass::Float { + Inst::FpuLoad { + size: FpuSize::F64, + rd: into_reg, + mem: amode_from_stack(mem), + } + } else { + Inst::Load { + rt: into_reg, + mem: amode_from_stack(mem), + kind: LoadKind::Word, + } + } + } + + fn gen_store_stack(mem: StackAMode, from_reg: Reg, _ty: Type) -> Inst { + if from_reg.class() == RegClass::Float { + Inst::FpuStore { + size: FpuSize::F64, + rt: from_reg, + mem: amode_from_stack(mem), + } + } else { + Inst::Store { + rt: from_reg, + mem: amode_from_stack(mem), + kind: StoreKind::Word, + } + } + } + + fn gen_move(to_reg: Writable, from_reg: Reg, ty: Type) -> Inst { + Inst::gen_move(to_reg, from_reg, ty) + } + + fn gen_extend( + _to_reg: Writable, + _from_reg: Reg, + _signed: bool, + _from_bits: u8, + _to_bits: u8, + ) -> Inst { + unimplemented!("arm32: integer extension not yet implemented") + } + + fn get_ext_mode( + _call_conv: isa::CallConv, + specified: ir::ArgumentExtension, + ) -> ir::ArgumentExtension { + specified + } + + fn gen_args(args: Vec) -> Inst { + Inst::Args { args } + } + + fn gen_rets(rets: Vec) -> Inst { + Inst::Rets { rets } + } + + fn get_stacklimit_reg(_call_conv: isa::CallConv) -> Reg { + spilltmp_reg() + } + + fn gen_add_imm( + _call_conv: isa::CallConv, + _into_reg: Writable, + _from_reg: Reg, + _imm: u32, + ) -> SmallInstVec { + unimplemented!("arm32: gen_add_imm not yet implemented") + } + + fn gen_stack_lower_bound_trap(_limit_reg: Reg) -> SmallInstVec { + unimplemented!("arm32: stack-limit checks not yet implemented") + } + + fn gen_get_stack_addr(_mem: StackAMode, _into_reg: Writable) -> Inst { + unimplemented!("arm32: stack-address computation not yet implemented") + } + + fn gen_load_base_offset(into_reg: Writable, base: Reg, offset: i32, _ty: Type) -> Inst { + let mem = AMode::RegOffset { rn: base, offset }; + if into_reg.to_reg().class() == RegClass::Float { + Inst::FpuLoad { + size: FpuSize::F64, + rd: into_reg, + mem, + } + } else { + Inst::Load { + rt: into_reg, + mem, + kind: LoadKind::Word, + } + } + } + + fn gen_store_base_offset(base: Reg, offset: i32, from_reg: Reg, _ty: Type) -> Inst { + let mem = AMode::RegOffset { rn: base, offset }; + if from_reg.class() == RegClass::Float { + Inst::FpuStore { + size: FpuSize::F64, + rt: from_reg, + mem, + } + } else { + Inst::Store { + rt: from_reg, + mem, + kind: StoreKind::Word, + } + } + } + + fn gen_sp_reg_adjust(amount: i32) -> SmallInstVec { + if amount == 0 { + smallvec![] + } else { + smallvec![Inst::AdjustSp { amount }] + } + } + + fn gen_prologue_frame_setup( + _call_conv: isa::CallConv, + _flags: &settings::Flags, + _isa_flags: &Arm32Flags, + frame_layout: &FrameLayout, + ) -> SmallInstVec { + let mut insts = smallvec![]; + if frame_layout.setup_area_size > 0 { + // push {fp, lr}; mov fp, sp + insts.push(Inst::Push { + reg_list: FP_LR_REG_LIST, + }); + insts.push(Inst::MovReg { + rd: writable_fp_reg(), + rm: stack_reg(), + }); + } + insts + } + + fn gen_epilogue_frame_restore( + _call_conv: isa::CallConv, + _flags: &settings::Flags, + _isa_flags: &Arm32Flags, + frame_layout: &FrameLayout, + ) -> SmallInstVec { + let mut insts = smallvec![]; + if frame_layout.setup_area_size > 0 { + // pop {fp, lr} + insts.push(Inst::Pop { + reg_list: FP_LR_REG_LIST, + }); + } + insts + } + + fn gen_return( + _call_conv: isa::CallConv, + _isa_flags: &Arm32Flags, + _frame_layout: &FrameLayout, + ) -> SmallInstVec { + smallvec![Inst::Ret] + } + + fn gen_probestack(_insts: &mut SmallInstVec, _frame_size: u32) { + unimplemented!("arm32: probestack not yet implemented") + } + + fn gen_inline_probestack( + _insts: &mut SmallInstVec, + _call_conv: isa::CallConv, + _frame_size: u32, + _guard_size: u32, + ) { + unimplemented!("arm32: inline probestack not yet implemented") + } + + fn gen_clobber_save( + _call_conv: isa::CallConv, + _flags: &settings::Flags, + frame_layout: &FrameLayout, + ) -> SmallVec<[Inst; 16]> { + let mut insts = smallvec![]; + let stack_size = frame_layout.clobber_size + + frame_layout.fixed_frame_storage_size + + frame_layout.outgoing_args_size; + if stack_size > 0 { + insts.extend(Self::gen_sp_reg_adjust(-(stack_size as i32))); + // Callee-saved registers are saved at the *top* of the frame, above + // the fixed stack-slot and outgoing-argument regions (which resolve + // from sp upward). Placing them from sp=0 would alias those regions + // and corrupt saved registers. + let mut cur_offset = 0u32; + for reg in &frame_layout.clobbered_callee_saves { + let r = Reg::from(reg.to_reg()); + let bytes = if r.class() == RegClass::Float { 8 } else { 4 }; + cur_offset = align_to(cur_offset, bytes); + let mem = AMode::SPOffset { + offset: i64::from(stack_size - cur_offset - bytes), + }; + if r.class() == RegClass::Float { + insts.push(Inst::FpuStore { + size: FpuSize::F64, + rt: r, + mem, + }); + } else { + insts.push(Inst::Store { + rt: r, + mem, + kind: StoreKind::Word, + }); + } + cur_offset += bytes; + } + } + insts + } + + fn gen_clobber_restore( + _call_conv: isa::CallConv, + _flags: &settings::Flags, + frame_layout: &FrameLayout, + ) -> SmallVec<[Inst; 16]> { + let mut insts = smallvec![]; + let stack_size = frame_layout.clobber_size + + frame_layout.fixed_frame_storage_size + + frame_layout.outgoing_args_size; + // Mirror the top-of-frame placement used by `gen_clobber_save`. + let mut cur_offset = 0u32; + for reg in &frame_layout.clobbered_callee_saves { + let is_float = reg.to_reg().class() == RegClass::Float; + let bytes = if is_float { 8 } else { 4 }; + cur_offset = align_to(cur_offset, bytes); + let mem = AMode::SPOffset { + offset: i64::from(stack_size - cur_offset - bytes), + }; + if is_float { + insts.push(Inst::FpuLoad { + size: FpuSize::F64, + rd: reg.map(Reg::from), + mem, + }); + } else { + insts.push(Inst::Load { + rt: reg.map(Reg::from), + mem, + kind: LoadKind::Word, + }); + } + cur_offset += bytes; + } + if stack_size > 0 { + insts.extend(Self::gen_sp_reg_adjust(stack_size as i32)); + } + insts + } + + fn gen_memcpy Writable>( + _call_conv: isa::CallConv, + _dst: Reg, + _src: Reg, + _size: usize, + _alloc_tmp: F, + ) -> SmallVec<[Self::I; 8]> { + unimplemented!("arm32: memcpy not yet implemented") + } + + fn get_number_of_spillslots_for_value( + rc: RegClass, + _target_vector_bytes: u32, + _isa_flags: &Arm32Flags, + ) -> u32 { + match rc { + RegClass::Int => 1, + // A D register is 8 bytes = two 4-byte spill slots. + RegClass::Float => 2, + RegClass::Vector => unimplemented!("arm32 has no vector registers"), + } + } + + fn get_machine_env(_flags: &settings::Flags, _call_conv: isa::CallConv) -> &MachineEnv { + static MACHINE_ENV: MachineEnv = create_reg_environment(); + &MACHINE_ENV + } + + fn get_regs_clobbered_by_call( + _call_conv_of_callee: isa::CallConv, + _is_exception: bool, + ) -> PRegSet { + DEFAULT_CLOBBERS + } + + fn compute_frame_layout( + call_conv: isa::CallConv, + flags: &settings::Flags, + _sig: &Signature, + regs: &[Writable], + function_calls: FunctionCalls, + incoming_args_size: u32, + tail_args_size: u32, + stackslots_size: u32, + fixed_frame_storage_size: u32, + outgoing_args_size: u32, + ) -> FrameLayout { + let is_callee_saved = |reg: &Writable| match call_conv { + isa::CallConv::PreserveAll => true, + _ => DEFAULT_CALLEE_SAVES.contains(reg.to_reg().into()), + }; + let mut regs: Vec> = + regs.iter().cloned().filter(is_callee_saved).collect(); + regs.sort_unstable(); + + let clobber_size = compute_clobber_size(®s); + + let setup_area_size = if flags.preserve_frame_pointers() + || function_calls != FunctionCalls::None + || incoming_args_size > 0 + || clobber_size > 0 + || fixed_frame_storage_size > 0 + { + 8 // FP, LR + } else { + 0 + }; + + FrameLayout { + word_bytes: 4, + incoming_args_size, + tail_args_size, + setup_area_size, + clobber_size, + fixed_frame_storage_size, + stackslots_size, + outgoing_args_size, + clobbered_callee_saves: regs, + function_calls, + } + } + + fn retval_temp_reg(_call_conv_of_callee: isa::CallConv) -> Writable { + // r12 (ip) is caller-saved and never used to return a value. + writable_xreg(12) + } + + fn exception_payload_regs(_call_conv: isa::CallConv) -> &'static [Reg] { + &[] + } +} + +/// Register-list mask for `{fp, lr}` (r11 and r14), as used by push/pop. +const FP_LR_REG_LIST: u32 = (1 << 11) | (1 << 14); + +/// Convert a `StackAMode` (a nominal stack reference) into an `AMode` that is +/// resolved against the frame layout at emit time. +fn amode_from_stack(mem: StackAMode) -> AMode { + match mem { + StackAMode::IncomingArg(off, stack_args_size) => AMode::IncomingArg { + offset: i64::from(stack_args_size) - off, + }, + StackAMode::Slot(off) => AMode::SlotOffset { offset: off }, + StackAMode::OutgoingArg(off) => AMode::SPOffset { offset: off }, + } +} + +/// Callee-saved registers under AAPCS: GPRs r4-r11 and VFP D8-D15. +const DEFAULT_CALLEE_SAVES: PRegSet = PRegSet::empty() + .with(preg(4)) + .with(preg(5)) + .with(preg(6)) + .with(preg(7)) + .with(preg(8)) + .with(preg(9)) + .with(preg(10)) + .with(preg(11)) + .with(pdreg(8)) + .with(pdreg(9)) + .with(pdreg(10)) + .with(pdreg(11)) + .with(pdreg(12)) + .with(pdreg(13)) + .with(pdreg(14)) + .with(pdreg(15)); + +/// Caller-saved (clobbered) registers: GPRs r0-r3, r12 and VFP D0-D7. +const DEFAULT_CLOBBERS: PRegSet = PRegSet::empty() + .with(preg(0)) + .with(preg(1)) + .with(preg(2)) + .with(preg(3)) + .with(preg(12)) + .with(pdreg(0)) + .with(pdreg(1)) + .with(pdreg(2)) + .with(pdreg(3)) + .with(pdreg(4)) + .with(pdreg(5)) + .with(pdreg(6)) + .with(pdreg(7)); + +fn compute_clobber_size(clobbers: &[Writable]) -> u32 { + let mut size = 0; + for reg in clobbers { + match reg.to_reg().class() { + RegClass::Int => size += 4, + RegClass::Float => size += 8, + RegClass::Vector => unreachable!("arm32 has no vector registers"), + } + } + align_to(size, 8) +} diff --git a/cranelift/codegen/src/isa/arm32/inst.isle b/cranelift/codegen/src/isa/arm32/inst.isle new file mode 100644 index 000000000000..6fa1c7cdbe5b --- /dev/null +++ b/cranelift/codegen/src/isa/arm32/inst.isle @@ -0,0 +1,972 @@ +;; arm32 (AArch32 / A32) instruction definitions. +;; +;; The `MInst` type below is the machine-instruction enum for this backend; it +;; is generated into Rust and re-exported as `Inst` by `inst/mod.rs`. + +;;;; Supporting types ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +;; A data-processing ALU operation. +(type ALUOp + (enum + (Add) + (Sub) + (Rsb) + (Adc) + (Sbc) + (And) + (Orr) + (Eor) + (Bic))) + +;; A flag-setting compare/test operation (destination is discarded). +(type CmpOp + (enum + (Cmp) + (Cmn) + (Tst) + (Teq))) + +;; A shift/rotate operation. +(type ShiftOp + (enum + (Lsl) + (Lsr) + (Asr) + (Ror))) + +;; A single-operand bit-manipulation operation. +(type BitOp + (enum + (Clz) + (Rev) + (Rev16) + (Rbit) + (Revsh))) + +;; A saturating add/subtract (`qadd`/`qsub`/`qdadd`/`qdsub`). +(type QAluOp + (enum + (Qadd) + (Qsub) + (Qdadd) + (Qdsub))) + +;; A saturate operation (`ssat`/`usat`/`ssat16`/`usat16`). +(type SatOp + (enum + (Ssat) + (Usat) + (Ssat16) + (Usat16))) + +;; A bitfield extract (`sbfx`/`ubfx`). +(type BfxOp + (enum + (Sbfx) + (Ubfx))) + +;; A halfword-pack operation (`pkhbt`/`pkhtb`). +(type PkhOp + (enum + (Pkhbt) + (Pkhtb))) + +;; A parallel (SIMD-in-GPR) 8/16-bit add/subtract operation. Names follow the +;; ARM mnemonics: prefix S/Q/SH/U/UQ/UH, op ADD8/ADD16/SUB8/SUB16/ASX/SAX. +(type ParAluOp + (enum + (Sadd8) (Sadd16) (Ssub8) (Ssub16) (Sasx) (Ssax) + (Qadd8) (Qadd16) (Qsub8) (Qsub16) (Qasx) (Qsax) + (Shadd8) (Shadd16) (Shsub8) (Shsub16) (Shasx) (Shsax) + (Uadd8) (Uadd16) (Usub8) (Usub16) (Uasx) (Usax) + (Uqadd8) (Uqadd16) (Uqsub8) (Uqsub16) (Uqasx) (Uqsax) + (Uhadd8) (Uhadd16) (Uhsub8) (Uhsub16) (Uhasx) (Uhsax))) + +;; A sign/zero extension operation. +(type ExtOp + (enum + (Sxtb) + (Sxth) + (Uxtb) + (Uxth) + (Sxtb16) + (Uxtb16))) + +;; A sign/zero extend-and-add operation (`sxtab`/`uxtah`/...). +(type ExtAddOp + (enum + (Sxtab) + (Sxtah) + (Sxtab16) + (Uxtab) + (Uxtah) + (Uxtab16))) + +;; DSP multiplies with a single 32-bit result (`rd = f(rn, rm)`). +(type DspMul3Op + (enum + (Smulbb) (Smulbt) (Smultb) (Smultt) + (Smulwb) (Smulwt) + (Smmul) (Smuad) (Smusd))) + +;; DSP multiply-accumulates with a single 32-bit result (`rd = f(rn, rm) + ra`). +(type DspMul4Op + (enum + (Smlabb) (Smlabt) (Smlatb) (Smlatt) + (Smlawb) (Smlawt) + (Smmla) (Smmls) + (Smlad) (Smlsd))) + +;; DSP multiplies with a 64-bit accumulator (`{rd_hi:rd_lo} = ...`). +(type DspMulLOp + (enum + (Smlalbb) (Smlalbt) (Smlaltb) (Smlaltt) + (Smlald) (Smlsld) + (Umaal))) + +;; A memory-barrier / exclusive-monitor-clear instruction (all use the full +;; system option). +(type BarrierOp + (enum + (Dmb) + (Dsb) + (Isb) + (Clrex))) + +;; Access width for exclusive / acquire-release memory operations. `DWord` +;; (64-bit) uses an even/odd register pair and is only valid on the paired +;; exclusive instructions. +(type AtomicSize + (enum + (Byte) + (Half) + (Word) + (DWord))) + +;; The precision of a VFP floating-point operation. +(type FpuSize + (enum + (F32) + (F64))) + +;; A three-register VFP data-processing operation. +(type FpuOp3 + (enum + (Vadd) + (Vsub) + (Vmul) + (Vdiv))) + +;; A two-register VFP data-processing operation. +(type FpuOp2 + (enum + (Vmov) + (Vneg) + (Vabs) + (Vsqrt))) + +;; An ARM condition code (the "always" and "never" codes are omitted). +(type Cond + (enum + (Eq) (Ne) (Hs) (Lo) (Mi) (Pl) (Vs) (Vc) + (Hi) (Ls) (Ge) (Lt) (Gt) (Le))) + +;; The width/signedness of a load. +(type LoadKind + (enum + (Word) + (UByte) + (SByte) + (UHalf) + (SHalf))) + +;; The width of a store. +(type StoreKind + (enum + (Word) + (Byte) + (Half))) + +;; A memory addressing mode. +(type AMode + (enum + ;; `[rn, #offset]` + (RegOffset (rn Reg) (offset i32)) + ;; `[rn, rm]` + (RegReg (rn Reg) (rm Reg)) + ;; `[sp, #offset]` + (SPOffset (offset i64)) + ;; A reference to a stack slot, resolved to an SP offset at emit time. + (SlotOffset (offset i64)) + ;; A reference into the incoming-argument area, resolved at emit time. + (IncomingArg (offset i64)))) + +(type BoxCallInfo (primitive BoxCallInfo)) +(type BoxCallIndInfo (primitive BoxCallIndInfo)) +(type VecMachLabel extern (enum)) + +;;;; Instruction formats ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(type MInst + (enum + ;; A zero-length no-op. + (Nop0) + ;; A four-byte no-op. + (Nop4) + + ;; Load an arbitrary 32-bit constant into `rd` via `movw` + `movt`. + (MovImm (rd WritableReg) (imm u64)) + ;; `mov rd, #imm` with a rotated 8-bit immediate (pre-encoded imm12). + (MovRotImm (rd WritableReg) (imm12 u32)) + ;; `mvn rd, #imm` with a rotated 8-bit immediate (pre-encoded imm12). + (MvnRotImm (rd WritableReg) (imm12 u32)) + ;; `movw rd, #imm16`. + (Movw (rd WritableReg) (imm16 u32)) + ;; `movt rd, #imm16` (sets the top half, preserving the low half). + (Movt (rd WritableReg) (imm16 u32)) + ;; `mov rd, rm`. + (MovReg (rd WritableReg) (rm Reg)) + ;; `mvn rd, rm` — bitwise-NOT of a register. + (MvnReg (rd WritableReg) (rm Reg)) + + ;; `op rd, rn, rm`. + (AluRRR (op ALUOp) (rd WritableReg) (rn Reg) (rm Reg)) + ;; `op rd, rn, #imm` (pre-encoded rotated imm12). + (AluRRImm (op ALUOp) (rd WritableReg) (rn Reg) (imm12 u32)) + ;; `op{s} rd, rn, rm` — flag-setting form (adds/subs/adcs/sbcs). + (AluRRRFlags (op ALUOp) (rd WritableReg) (rn Reg) (rm Reg)) + + ;; A shift/rotate by a constant amount (`op rd, rm, #amount`). + (ShiftImm (op ShiftOp) (rd WritableReg) (rm Reg) (amount u8)) + ;; A shift/rotate by a register amount (`op rd, rm, rs`). + (ShiftReg (op ShiftOp) (rd WritableReg) (rm Reg) (rs Reg)) + + ;; `mul rd, rn, rm` — `rd = rn * rm` (low 32 bits). + (Mul (rd WritableReg) (rn Reg) (rm Reg)) + ;; `mla rd, rn, rm, ra` — `rd = rn * rm + ra` (low 32 bits). + (Mla (rd WritableReg) (rn Reg) (rm Reg) (ra Reg)) + ;; `mls rd, rn, rm, ra` — `rd = ra - rn * rm` (low 32 bits). + (Mls (rd WritableReg) (rn Reg) (rm Reg) (ra Reg)) + ;; `umull rd_lo, rd_hi, rn, rm` — unsigned 32x32->64 multiply. + (Umull (rd_lo WritableReg) (rd_hi WritableReg) (rn Reg) (rm Reg)) + ;; `smull rd_lo, rd_hi, rn, rm` — signed 32x32->64 multiply. + (Smull (rd_lo WritableReg) (rd_hi WritableReg) (rn Reg) (rm Reg)) + ;; `umlal rd_lo, rd_hi, rn, rm` — unsigned multiply-accumulate into 64 bits. + (Umlal (rd_lo WritableReg) (rd_hi WritableReg) (rn Reg) (rm Reg)) + ;; `smlal rd_lo, rd_hi, rn, rm` — signed multiply-accumulate into 64 bits. + (Smlal (rd_lo WritableReg) (rd_hi WritableReg) (rn Reg) (rm Reg)) + + ;; `sdiv rd, rn, rm` — signed divide `rd = rn / rm`. + (SDiv (rd WritableReg) (rn Reg) (rm Reg)) + ;; `udiv rd, rn, rm` — unsigned divide. + (UDiv (rd WritableReg) (rn Reg) (rm Reg)) + + ;; A single-operand bit operation (`clz`/`rev`/`rev16`/`rbit`). + (BitRR (op BitOp) (rd WritableReg) (rm Reg)) + ;; A sign/zero extension (`sxtb`/`sxth`/`uxtb`/`uxth`). + (ExtRR (op ExtOp) (rd WritableReg) (rm Reg)) + + ;; A conditional select, emitted as `mov rd, rm; mov rd, rn`. + (CSel (cond Cond) (rd WritableReg) (rn Reg) (rm Reg)) + + ;; Bitfield clear: `bfc rd, #lsb, #width`. + (Bfc (rd WritableReg) (lsb u8) (width u8)) + ;; Bitfield insert: `bfi rd, rn, #lsb, #width`. + (Bfi (rd WritableReg) (rn Reg) (lsb u8) (width u8)) + ;; Bitfield extract: `sbfx`/`ubfx rd, rn, #lsb, #width`. + (Bfx (op BfxOp) (rd WritableReg) (rn Reg) (lsb u8) (width u8)) + + ;; A saturating add/subtract (`op rd, rm, rn`). + (QAlu (op QAluOp) (rd WritableReg) (rm Reg) (rn Reg)) + ;; A saturate: `ssat/usat rd, #sat_bits, rm`. + (Sat (op SatOp) (rd WritableReg) (sat_bits u8) (rm Reg)) + + ;; A parallel 8/16-bit add/subtract (`op rd, rn, rm`). + (ParAlu (op ParAluOp) (rd WritableReg) (rn Reg) (rm Reg)) + + ;; A sign/zero extend-and-add (`op rd, rn, rm`). + (ExtAdd (op ExtAddOp) (rd WritableReg) (rn Reg) (rm Reg)) + + ;; A DSP multiply producing a single 32-bit result. + (DspMul3 (op DspMul3Op) (rd WritableReg) (rn Reg) (rm Reg)) + ;; A DSP multiply-accumulate producing a single 32-bit result. + (DspMul4 (op DspMul4Op) (rd WritableReg) (rn Reg) (rm Reg) (ra Reg)) + ;; A DSP multiply with a 64-bit accumulator pair. + (DspMulL (op DspMulLOp) (rd_lo WritableReg) (rd_hi WritableReg) (rn Reg) (rm Reg)) + + ;; Byte-wise select using the GE flags (`sel rd, rn, rm`). + (Sel (rd WritableReg) (rn Reg) (rm Reg)) + ;; Halfword pack (`pkhbt`/`pkhtb rd, rn, rm`). + (Pkh (op PkhOp) (rd WritableReg) (rn Reg) (rm Reg)) + ;; Rotate right with extend (`rrx rd, rm`). + (Rrx (rd WritableReg) (rm Reg)) + + ;; A permanently-undefined instruction used to encode a trap. + (Udf (code TrapCode)) + + ;; Conditionally trap on the current flags: emitted as + ;; `b 1f; udf ; 1:`. + (TrapIf (cond Cond) (code TrapCode)) + + ;; Load/store multiple, increment-after (`ldmia`/`stmia rn{!}, {list}`). + (LdmStm (load bool) (rn Reg) (writeback bool) (reg_list u32)) + + ;; Load-exclusive (`ldrex{b,h}`) or load-acquire-exclusive (`ldaex{b,h}`). + (LoadEx (acquire bool) (size AtomicSize) (rt WritableReg) (rn Reg)) + ;; Store-exclusive (`strex{b,h}`) or store-release-exclusive + ;; (`stlex{b,h}`); `rd` receives the success flag. + (StoreEx (release bool) (size AtomicSize) (rd WritableReg) (rt Reg) (rn Reg)) + ;; Load-acquire (`lda{b,h} rt, [rn]`). + (LoadAcq (size AtomicSize) (rt WritableReg) (rn Reg)) + ;; Store-release (`stl{b,h} rt, [rn]`). + (StoreRel (size AtomicSize) (rt Reg) (rn Reg)) + + ;; A memory barrier / clear-exclusive. + (Barrier (op BarrierOp)) + + ;; An atomic read-modify-write loop over a `size`-wide location: + ;; loop: ldaex{,b,h} rd, [addr]; tmp1, rd, operand; + ;; stlex{,b,h} tmp2, tmp1, [addr]; cmp tmp2, #0; bne loop + ;; `rd` receives the (zero-extended) old value. + (AtomicRmw (op AtomicRmwOp) (size AtomicSize) (rd WritableReg) (addr Reg) + (operand Reg) (tmp1 WritableReg) (tmp2 WritableReg)) + ;; An atomic compare-and-swap loop over a `size`-wide location; `rd` + ;; receives the (zero-extended) old value. + (AtomicCas (size AtomicSize) (rd WritableReg) (addr Reg) (expected Reg) + (new Reg) (tmp WritableReg)) + + ;; A 64-bit atomic read-modify-write loop using the paired exclusive + ;; instructions (`ldaexd`/`stlexd`). The old value is returned in + ;; (rd_lo, rd_hi). All registers are pinned to a fixed even/odd layout in + ;; the operand collector because A32 exclusive-pair instructions require + ;; consecutive even/odd register pairs, which regalloc2 cannot model. + (AtomicRmw64 (op AtomicRmwOp) (rd_lo WritableReg) (rd_hi WritableReg) + (addr Reg) (operand_lo Reg) (operand_hi Reg) + (tmp_new_lo WritableReg) (tmp_new_hi WritableReg) + (tmp_res WritableReg)) + ;; A 64-bit atomic compare-and-swap loop; the old value is in (rd_lo, rd_hi). + (AtomicCas64 (rd_lo WritableReg) (rd_hi WritableReg) (addr Reg) + (expected_lo Reg) (expected_hi Reg) (new_lo Reg) (new_hi Reg) + (tmp_res WritableReg)) + ;; A 64-bit sequentially-consistent atomic load (`ldaexd` + `clrex`). + (AtomicLoad64 (rd_lo WritableReg) (rd_hi WritableReg) (addr Reg)) + ;; A 64-bit sequentially-consistent atomic store via an `ldaexd`/`stlexd` + ;; retry loop; the loaded value is discarded into (scratch_lo, scratch_hi). + (AtomicStore64 (addr Reg) (src_lo Reg) (src_hi Reg) + (scratch_lo WritableReg) (scratch_hi WritableReg) + (tmp_res WritableReg)) + + ;; A three-register VFP op (`vadd`/`vsub`/`vmul`/`vdiv`). + (FpuRRR (op FpuOp3) (size FpuSize) (rd WritableReg) (rn Reg) (rm Reg)) + ;; A two-register VFP op (`vmov`/`vneg`/`vabs`/`vsqrt`). + (FpuRR (op FpuOp2) (size FpuSize) (rd WritableReg) (rm Reg)) + ;; A VFP load (`vldr`). + (FpuLoad (size FpuSize) (rd WritableReg) (mem AMode)) + ;; A VFP store (`vstr`). + (FpuStore (size FpuSize) (rt Reg) (mem AMode)) + ;; Move a GPR into the low half of an S register (`vmov s, r`) for f32 bits. + (MovToFpu32 (rd WritableReg) (rt Reg)) + ;; Move two GPRs into a D register (`vmov d, rlo, rhi`) for f64 bits. + (MovToFpu64 (rd WritableReg) (rt_lo Reg) (rt_hi Reg)) + ;; Move an S register into a GPR (`vmov r, s`). + (MovFromFpu32 (rt WritableReg) (rm Reg)) + ;; Move a D register into two GPRs (`vmov rlo, rhi, d`). + (MovFromFpu64 (rt_lo WritableReg) (rt_hi WritableReg) (rm Reg)) + + ;; Compare two float registers and move the VFP flags into the CPSR + ;; (`vcmp` + `vmrs APSR_nzcv, FPSCR`). + (VcmpMrs (size FpuSize) (rn Reg) (rm Reg)) + ;; Convert between f32 and f64 (`to_f64` selects fpromote vs fdemote). + (VcvtFF (to_f64 bool) (rd WritableReg) (rm Reg)) + ;; Convert a float to a 32-bit integer (round toward zero, saturating). + ;; `rd` receives the integer in an S register. + (VcvtToInt (signed bool) (size FpuSize) (rd WritableReg) (rm Reg)) + ;; Convert a 32-bit integer (held in the S register `rm`) to a float. + (VcvtFromInt (signed bool) (size FpuSize) (rd WritableReg) (rm Reg)) + ;; A conditional float select, emitted as `vmov rd, rm; vmov rd, rn`. + (FpuCSel (cond Cond) (rd WritableReg) (rn Reg) (rm Reg)) + ;; IEEE `vminnm`/`vmaxnm` (number min/max; ARMv8). `max` selects vmaxnm. + (FpuMinMax (max bool) (size FpuSize) (rd WritableReg) (rn Reg) (rm Reg)) + + ;; `cmp/cmn/tst/teq rn, rm`. + (CmpRR (op CmpOp) (rn Reg) (rm Reg)) + ;; `cmp/cmn/tst/teq rn, #imm` (pre-encoded rotated imm12). + (CmpRImm (op CmpOp) (rn Reg) (imm12 u32)) + + ;; A load of the given kind. + (Load (rt WritableReg) (mem AMode) (kind LoadKind)) + ;; A store of the given kind. + (Store (rt Reg) (mem AMode) (kind StoreKind)) + + ;; `push {reglist}` (STMDB sp!). The register list is a 16-bit mask. + (Push (reg_list u32)) + ;; `pop {reglist}` (LDMIA sp!). + (Pop (reg_list u32)) + + ;; Adjust the stack pointer by a signed constant (add/sub sp). + (AdjustSp (amount i32)) + + ;; A direct call (`bl`). + (Call (info BoxCallInfo)) + ;; An indirect call (`blx rm`). + (CallInd (info BoxCallIndInfo)) + + ;; Load the absolute address of an external symbol into `rd` via an + ;; inline literal: `ldr rd, [pc]; b .+8; .word ` where the + ;; `.word` carries an `Abs4` relocation resolved by the linker. + (LoadExtName (rd WritableReg) + (name BoxExternalName) + (offset i64)) + + ;; An unconditional branch to a label (`b`). + (Jump (dest MachLabel)) + + ;; A conditional branch: `b taken; b not_taken`. + (CondBr (cond Cond) (taken MachLabel) (not_taken MachLabel)) + + ;; A jump table: bounds-check `index` against the entry count and branch to + ;; `targets[index]`, else to the default (`targets[0]`). + (BrTable (index Reg) (tmp WritableReg) (targets VecMachLabel)) + + ;; Pseudo-instruction capturing incoming register arguments in vregs. + (Args (args VecArgPair)) + ;; Pseudo-instruction moving vregs into the return registers. + (Rets (rets VecRetPair)) + ;; Return from the current function (`bx lr`). + (Ret) + ) +) + +;;;; Constant materialization ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +;; Materialize an integer constant into a register, choosing the shortest of +;; `mov`/`mvn`/`movw`/`movw+movt`. +(decl gen_constant (u64) Reg) +(extern constructor gen_constant gen_constant) + +;; Materialize an integer constant of the given type into a register. +(decl imm (Type u64) Reg) +(rule (imm _ty n) (gen_constant n)) + +;;;; ALU helpers ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +;; `op rd, rn, rm`. +(decl alu_rrr (ALUOp Reg Reg) Reg) +(rule (alu_rrr op rn rm) + (let ((rd WritableReg (temp_writable_reg $I32)) + (_ Unit (emit (MInst.AluRRR op rd rn rm)))) + rd)) + +;; `op rd, rn, #imm` (immediate must be a rotated imm12). +(decl alu_rr_imm12 (ALUOp Reg u32) Reg) +(rule (alu_rr_imm12 op rn imm12) + (let ((rd WritableReg (temp_writable_reg $I32)) + (_ Unit (emit (MInst.AluRRImm op rd rn imm12)))) + rd)) + +;; `op{s} rd, rn, rm` — flag-setting form (adds/subs/adcs/sbcs). +(decl alu_rrr_flags (ALUOp Reg Reg) Reg) +(rule (alu_rrr_flags op rn rm) + (let ((rd WritableReg (temp_writable_reg $I32)) + (_ Unit (emit (MInst.AluRRRFlags op rd rn rm)))) + rd)) + +;; A constant is encodable as an A32 data-processing immediate; the result is +;; its pre-encoded 12-bit form. +(decl pure partial u64_from_rotated_imm12 (u64) u32) +(extern constructor u64_from_rotated_imm12 u64_from_rotated_imm12) + +;; Add: prefer the immediate form when the RHS is an encodable constant. +(decl add_reg (Reg Reg) Reg) +(rule (add_reg a b) (alu_rrr (ALUOp.Add) a b)) + +(decl add_imm (Reg u64) Reg) +(rule 1 (add_imm a b) + (if-let imm12 (u64_from_rotated_imm12 b)) + (alu_rr_imm12 (ALUOp.Add) a imm12)) +(rule (add_imm a b) (alu_rrr (ALUOp.Add) a (gen_constant b))) + +(decl sub_reg (Reg Reg) Reg) +(rule (sub_reg a b) (alu_rrr (ALUOp.Sub) a b)) + +(decl sub_imm (Reg u64) Reg) +(rule 1 (sub_imm a b) + (if-let imm12 (u64_from_rotated_imm12 b)) + (alu_rr_imm12 (ALUOp.Sub) a imm12)) +(rule (sub_imm a b) (alu_rrr (ALUOp.Sub) a (gen_constant b))) + +;; Reverse subtract, used for negation: `rsb rd, rn, #0`. +(decl rsb_imm (Reg u32) Reg) +(rule (rsb_imm a imm12) + (let ((rd WritableReg (temp_writable_reg $I32)) + (_ Unit (emit (MInst.AluRRImm (ALUOp.Rsb) rd a imm12)))) + rd)) + +;; A generic ALU op that prefers the immediate form when the RHS is an +;; encodable constant and otherwise materializes it into a register. +(decl alu_op_imm (ALUOp Reg u64) Reg) +(rule 1 (alu_op_imm op a b) + (if-let imm12 (u64_from_rotated_imm12 b)) + (alu_rr_imm12 op a imm12)) +(rule (alu_op_imm op a b) (alu_rrr op a (gen_constant b))) + +;;;; Multiplies ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(decl mul (Reg Reg) Reg) +(rule (mul a b) + (let ((rd WritableReg (temp_writable_reg $I32)) + (_ Unit (emit (MInst.Mul rd a b)))) + rd)) + +;; Bitwise-NOT of a register (`mvn rd, rm`). +(decl mvn_reg (Reg) Reg) +(rule (mvn_reg rm) + (let ((rd WritableReg (temp_writable_reg $I32)) + (_ Unit (emit (MInst.MvnReg rd rm)))) + rd)) + +;; `mls rd, rn, rm, ra` — `rd = ra - rn * rm`. +(decl mls (Reg Reg Reg) Reg) +(rule (mls rn rm ra) + (let ((rd WritableReg (temp_writable_reg $I32)) + (_ Unit (emit (MInst.Mls rd rn rm ra)))) + rd)) + +;;;; Divides ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +;; True when the hardware integer-divide instructions are available. +(decl pure use_idiv () bool) +(extern constructor use_idiv use_idiv) + +;; Signed/unsigned divide with the Cranelift-mandated trap checks (divide by +;; zero, and `INT_MIN / -1` overflow for `sdiv`) emitted before the divide. +(decl gen_sdiv (Reg Reg) Reg) +(extern constructor gen_sdiv gen_sdiv) +(decl gen_udiv (Reg Reg) Reg) +(extern constructor gen_udiv gen_udiv) + +;;;; Bit operations and extends ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(decl bit_rr (BitOp Reg) Reg) +(rule (bit_rr op rm) + (let ((rd WritableReg (temp_writable_reg $I32)) + (_ Unit (emit (MInst.BitRR op rd rm)))) + rd)) + +(decl ext_rr (ExtOp Reg) Reg) +(rule (ext_rr op rm) + (let ((rd WritableReg (temp_writable_reg $I32)) + (_ Unit (emit (MInst.ExtRR op rd rm)))) + rd)) + +;;;; Conditional select ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +;; Select `if_true` when `cond` holds (given the current flags), else +;; `if_false`. Emitted as `mov rd, if_false; mov rd, if_true`. +(decl csel (Cond Reg Reg) Reg) +(rule (csel cond if_true if_false) + (let ((rd WritableReg (temp_writable_reg $I32)) + (_ Unit (emit (MInst.CSel cond rd if_true if_false)))) + rd)) + +;;;; Traps ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(decl udf (TrapCode) SideEffectNoResult) +(rule (udf code) + (SideEffectNoResult.Inst (MInst.Udf code))) + +;;;; Barriers ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(decl dmb () SideEffectNoResult) +(rule (dmb) + (SideEffectNoResult.Inst (MInst.Barrier (BarrierOp.Dmb)))) + +;;;; Atomics (8/16/32-bit) ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +;; Map a scalar integer type to the exclusive-access width. +(decl atomic_size (Type) AtomicSize) +(rule (atomic_size $I8) (AtomicSize.Byte)) +(rule (atomic_size $I16) (AtomicSize.Half)) +(rule (atomic_size $I32) (AtomicSize.Word)) + +(decl atomic_rmw_size (Type AtomicRmwOp Reg Reg) Reg) +(rule (atomic_rmw_size ty op addr operand) + (let ((rd WritableReg (temp_writable_reg $I32)) + (tmp1 WritableReg (temp_writable_reg $I32)) + (tmp2 WritableReg (temp_writable_reg $I32)) + (_ Unit (emit (MInst.AtomicRmw op (atomic_size ty) rd addr operand tmp1 tmp2)))) + rd)) + +(decl atomic_cas_size (Type Reg Reg Reg) Reg) +(rule (atomic_cas_size ty addr expected new) + (let ((rd WritableReg (temp_writable_reg $I32)) + (tmp WritableReg (temp_writable_reg $I32)) + (_ Unit (emit (MInst.AtomicCas (atomic_size ty) rd addr expected new tmp)))) + rd)) + +;; Sequentially-consistent load/store via load-acquire / store-release. +(decl atomic_load_size (Type Reg) Reg) +(rule (atomic_load_size ty addr) + (let ((rd WritableReg (temp_writable_reg $I32)) + (_ Unit (emit (MInst.LoadAcq (atomic_size ty) rd addr)))) + rd)) + +(decl atomic_store_size (Type Reg Reg) InstOutput) +(rule (atomic_store_size ty src addr) + (side_effect (SideEffectNoResult.Inst (MInst.StoreRel (atomic_size ty) src addr)))) + +;;;; Atomics (64-bit) ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(decl atomic_rmw_i64 (AtomicRmwOp Reg ValueRegs) ValueRegs) +(rule (atomic_rmw_i64 op addr operand) + (let ((rd_lo WritableReg (temp_writable_reg $I32)) + (rd_hi WritableReg (temp_writable_reg $I32)) + (nl WritableReg (temp_writable_reg $I32)) + (nh WritableReg (temp_writable_reg $I32)) + (res WritableReg (temp_writable_reg $I32)) + (_ Unit (emit (MInst.AtomicRmw64 op rd_lo rd_hi addr + (vr_lo operand) (vr_hi operand) nl nh res)))) + (value_regs (writable_reg_to_reg rd_lo) (writable_reg_to_reg rd_hi)))) + +(decl atomic_cas_i64 (Reg ValueRegs ValueRegs) ValueRegs) +(rule (atomic_cas_i64 addr expected new) + (let ((rd_lo WritableReg (temp_writable_reg $I32)) + (rd_hi WritableReg (temp_writable_reg $I32)) + (res WritableReg (temp_writable_reg $I32)) + (_ Unit (emit (MInst.AtomicCas64 rd_lo rd_hi addr + (vr_lo expected) (vr_hi expected) + (vr_lo new) (vr_hi new) res)))) + (value_regs (writable_reg_to_reg rd_lo) (writable_reg_to_reg rd_hi)))) + +(decl atomic_load_i64 (Reg) ValueRegs) +(rule (atomic_load_i64 addr) + (let ((rd_lo WritableReg (temp_writable_reg $I32)) + (rd_hi WritableReg (temp_writable_reg $I32)) + (_ Unit (emit (MInst.AtomicLoad64 rd_lo rd_hi addr)))) + (value_regs (writable_reg_to_reg rd_lo) (writable_reg_to_reg rd_hi)))) + +(decl atomic_store_i64 (ValueRegs Reg) InstOutput) +(rule (atomic_store_i64 src addr) + (let ((sl WritableReg (temp_writable_reg $I32)) + (sh WritableReg (temp_writable_reg $I32)) + (res WritableReg (temp_writable_reg $I32))) + (side_effect (SideEffectNoResult.Inst + (MInst.AtomicStore64 addr (vr_lo src) (vr_hi src) sl sh res))))) + +;;;; Floating point ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(decl fpu_rrr (FpuOp3 FpuSize Reg Reg) Reg) +(rule (fpu_rrr op size rn rm) + (let ((rd WritableReg (temp_writable_reg $F64)) + (_ Unit (emit (MInst.FpuRRR op size rd rn rm)))) + rd)) + +(decl fpu_rr (FpuOp2 FpuSize Reg) Reg) +(rule (fpu_rr op size rm) + (let ((rd WritableReg (temp_writable_reg $F64)) + (_ Unit (emit (MInst.FpuRR op size rd rm)))) + rd)) + +;; An f32 constant: build the bit pattern in a GPR, then move it into an S reg. +(decl f32_const (u64) Reg) +(rule (f32_const bits) + (let ((r Reg (gen_constant bits)) + (rd WritableReg (temp_writable_reg $F32)) + (_ Unit (emit (MInst.MovToFpu32 rd r)))) + rd)) + +;; An f64 constant: build the two halves in GPRs, then move them into a D reg. +(decl f64_const (u64) Reg) +(rule (f64_const bits) + (let ((lo Reg (gen_constant bits)) + (hi Reg (gen_constant (u64_high32 bits))) + (rd WritableReg (temp_writable_reg $F64)) + (_ Unit (emit (MInst.MovToFpu64 rd lo hi)))) + rd)) + +(decl fpu_load (FpuSize AMode) Reg) +(rule (fpu_load size mem) + (let ((rd WritableReg (temp_writable_reg $F64)) + (_ Unit (emit (MInst.FpuLoad size rd mem)))) + rd)) + +(decl fpu_store (FpuSize Reg AMode) InstOutput) +(rule (fpu_store size val mem) + (side_effect (SideEffectNoResult.Inst (MInst.FpuStore size val mem)))) + +;; Map a Cranelift `FloatCC` to the ARM condition testing the VFP flags. +(decl cond_from_floatcc (FloatCC) Cond) +(extern constructor cond_from_floatcc cond_from_floatcc) + +;; The two float conditions that a single ARM condition can't express, built as +;; a 0/1 boolean from the flags left by a `vcmp`. `one` (ordered and not-equal) +;; keeps the not-equal result only when ordered (`vc`); `ueq` (unordered or +;; equal) forces the equal result to 1 when unordered (`vs`). +(decl gen_fcmp_one (FpuSize Reg Reg) Reg) +(rule (gen_fcmp_one size a b) + (let ((_ Unit (emit_side_effect (vcmp size a b)))) + (csel (Cond.Vc) (csetv (Cond.Ne)) (gen_constant 0)))) +(decl gen_fcmp_ueq (FpuSize Reg Reg) Reg) +(rule (gen_fcmp_ueq size a b) + (let ((_ Unit (emit_side_effect (vcmp size a b)))) + (csel (Cond.Vs) (gen_constant 1) (csetv (Cond.Eq))))) + +;; `fcopysign` via GPR bit manipulation (clear the sign of `a`, OR in `b`'s). +(decl gen_copysign_f32 (Reg Reg) Reg) +(extern constructor gen_copysign_f32 gen_copysign_f32) +(decl gen_copysign_f64 (Reg Reg) Reg) +(extern constructor gen_copysign_f64 gen_copysign_f64) + +;; `vcmp` + `vmrs` as a side effect (leaves the comparison in the CPSR). +(decl vcmp (FpuSize Reg Reg) SideEffectNoResult) +(rule (vcmp size rn rm) + (SideEffectNoResult.Inst (MInst.VcmpMrs size rn rm))) + +;; f32<->f64 conversion. +(decl vcvt_ff (bool Reg) Reg) +(rule (vcvt_ff to_f64 rm) + (let ((rd WritableReg (temp_writable_reg $F64)) + (_ Unit (emit (MInst.VcvtFF to_f64 rd rm)))) + rd)) + +;; int (in a GPR) -> float. +(decl vcvt_from_int (bool FpuSize Reg) Reg) +(rule (vcvt_from_int signed size gpr) + (let ((s WritableReg (temp_writable_reg $F32)) + (_ Unit (emit (MInst.MovToFpu32 s gpr))) + (rd WritableReg (temp_writable_reg $F64)) + (_2 Unit (emit (MInst.VcvtFromInt signed size rd (writable_reg_to_reg s))))) + rd)) + +;; float -> i32 (in a GPR), round toward zero, saturating. +(decl vcvt_to_int (bool FpuSize Reg) Reg) +(rule (vcvt_to_int signed size f) + (let ((s WritableReg (temp_writable_reg $F32)) + (_ Unit (emit (MInst.VcvtToInt signed size s f))) + (rd WritableReg (temp_writable_reg $I32)) + (_2 Unit (emit (MInst.MovFromFpu32 rd (writable_reg_to_reg s))))) + rd)) + +;; bitcast helpers (`vmov` between the GPR and VFP files). +(decl mov_from_fpu32 (Reg) Reg) +(rule (mov_from_fpu32 rm) + (let ((rd WritableReg (temp_writable_reg $I32)) + (_ Unit (emit (MInst.MovFromFpu32 rd rm)))) + rd)) + +(decl mov_to_fpu32 (Reg) Reg) +(rule (mov_to_fpu32 rt) + (let ((rd WritableReg (temp_writable_reg $F32)) + (_ Unit (emit (MInst.MovToFpu32 rd rt)))) + rd)) + +(decl mov_from_fpu64 (Reg) ValueRegs) +(rule (mov_from_fpu64 rm) + (let ((lo WritableReg (temp_writable_reg $I32)) + (hi WritableReg (temp_writable_reg $I32)) + (_ Unit (emit (MInst.MovFromFpu64 lo hi rm)))) + (value_regs (writable_reg_to_reg lo) (writable_reg_to_reg hi)))) + +(decl mov_to_fpu64 (ValueRegs) Reg) +(rule (mov_to_fpu64 regs) + (let ((rd WritableReg (temp_writable_reg $F64)) + (_ Unit (emit (MInst.MovToFpu64 rd (vr_lo regs) (vr_hi regs))))) + rd)) + +;; Conditional float select: `if_true` when `cond` holds, else `if_false`. +(decl fpu_csel (Cond Reg Reg) Reg) +(rule (fpu_csel cond if_true if_false) + (let ((rd WritableReg (temp_writable_reg $F64)) + (_ Unit (emit (MInst.FpuCSel cond rd if_true if_false)))) + rd)) + +;; `vminnm`/`vmaxnm` (IEEE number min/max, ignoring NaN). +(decl fpu_minmax (bool FpuSize Reg Reg) Reg) +(rule (fpu_minmax is_max size rn rm) + (let ((rd WritableReg (temp_writable_reg $F64)) + (_ Unit (emit (MInst.FpuMinMax is_max size rd rn rm)))) + rd)) + +;; fmin/fmax with WebAssembly NaN propagation: use the IEEE number min/max +;; (which handles signed zeros) when ordered, and `x+y` (a NaN) when unordered. +(decl gen_fminmax (bool FpuSize Reg Reg) Reg) +(rule (gen_fminmax is_max size x y) + (let ((nm Reg (fpu_minmax is_max size x y)) + (nan Reg (fpu_rrr (FpuOp3.Vadd) size x y)) + (_ Unit (emit_side_effect (vcmp size x y)))) + (fpu_csel (Cond.Vs) nan nm))) + +;;;; Shifts ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +;; Shift/rotate by a constant amount. Handles the Cranelift "modulo width" +;; semantics (and the shift-by-zero special case) in Rust. +(decl gen_shift_imm (ShiftOp Reg u64) Reg) +(extern constructor gen_shift_imm gen_shift_imm) + +;; Shift/rotate by a register amount. The amount is masked to 0..31 to match +;; Cranelift's modulo-width semantics. +(decl shift_reg (ShiftOp Reg Value) Reg) +(rule (shift_reg op rm amount) + (let ((masked Reg (alu_rr_imm12 (ALUOp.And) (put_in_reg amount) 31)) + (rd WritableReg (temp_writable_reg $I32)) + (_ Unit (emit (MInst.ShiftReg op rd rm masked)))) + rd)) + +;;;; 64-bit helpers (values held as a (lo, hi) register pair) ;;;;;;;;;;;;;;;;;; + +(decl vr_lo (ValueRegs) Reg) +(rule (vr_lo regs) (value_regs_get regs 0)) +(decl vr_hi (ValueRegs) Reg) +(rule (vr_hi regs) (value_regs_get regs 1)) + +;; The high 32 bits of a 64-bit constant. +(decl pure u64_high32 (u64) u64) +(extern constructor u64_high32 u64_high32) + +;; Widen a u32 to u64 (zero-extended). +(decl pure u32_to_u64 (u32) u64) +(extern constructor u32_to_u64 u32_to_u64) + +;; `adds`/`adc` and `subs`/`sbc` carry chains over the halves. +(decl add_i64 (ValueRegs ValueRegs) ValueRegs) +(rule (add_i64 x y) + (let ((lo Reg (alu_rrr_flags (ALUOp.Add) (vr_lo x) (vr_lo y))) + (hi Reg (alu_rrr (ALUOp.Adc) (vr_hi x) (vr_hi y)))) + (value_regs lo hi))) + +(decl sub_i64 (ValueRegs ValueRegs) ValueRegs) +(rule (sub_i64 x y) + (let ((lo Reg (alu_rrr_flags (ALUOp.Sub) (vr_lo x) (vr_lo y))) + (hi Reg (alu_rrr (ALUOp.Sbc) (vr_hi x) (vr_hi y)))) + (value_regs lo hi))) + +;; A bitwise op applied independently to each half. +(decl logical_i64 (ALUOp ValueRegs ValueRegs) ValueRegs) +(rule (logical_i64 op x y) + (value_regs (alu_rrr op (vr_lo x) (vr_lo y)) + (alu_rrr op (vr_hi x) (vr_hi y)))) + +(decl not_i64 (ValueRegs) ValueRegs) +(rule (not_i64 x) + (value_regs (mvn_reg (vr_lo x)) (mvn_reg (vr_hi x)))) + +;; `mla rd, rn, rm, ra` — `rd = rn * rm + ra`. +(decl mla (Reg Reg Reg) Reg) +(rule (mla rn rm ra) + (let ((rd WritableReg (temp_writable_reg $I32)) + (_ Unit (emit (MInst.Mla rd rn rm ra)))) + rd)) + +;; 64-bit multiply (low 64 bits): umull for the lo*lo product, then fold in the +;; two cross terms with mla. +(decl mul_i64 (ValueRegs ValueRegs) ValueRegs) +(rule (mul_i64 x y) + (let ((xlo Reg (vr_lo x)) + (xhi Reg (vr_hi x)) + (ylo Reg (vr_lo y)) + (yhi Reg (vr_hi y)) + (lo WritableReg (temp_writable_reg $I32)) + (hi0 WritableReg (temp_writable_reg $I32)) + (_ Unit (emit (MInst.Umull lo hi0 xlo ylo))) + (hi1 Reg (mla xlo yhi (writable_reg_to_reg hi0))) + (hi2 Reg (mla xhi ylo hi1))) + (value_regs (writable_reg_to_reg lo) hi2))) + +;; Constant shift/rotate of a 64-bit value; the funnel logic is in Rust. +(decl gen_i64_shift_imm (ShiftOp ValueRegs u64) ValueRegs) +(extern constructor gen_i64_shift_imm gen_i64_shift_imm) + +;;;; Loads and stores ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(decl arm_load (AMode LoadKind) Reg) +(rule (arm_load mem kind) + (let ((rd WritableReg (temp_writable_reg $I32)) + (_ Unit (emit (MInst.Load rd mem kind)))) + rd)) + +(decl arm_store (Reg AMode StoreKind) InstOutput) +(rule (arm_store val mem kind) + (side_effect (SideEffectNoResult.Inst (MInst.Store val mem kind)))) + +;; Build an addressing mode from an address value plus a constant offset. +(decl amode (Value Offset32) AMode) +(rule (amode base offset) (AMode.RegOffset base (offset32_to_i32 offset))) + +;; Addressing mode for the high word of a 64-bit access (offset + 4). +(decl amode_hi (Value Offset32) AMode) +(rule (amode_hi base offset) (AMode.RegOffset base (offset32_plus4 offset))) + +(decl pure offset32_plus4 (Offset32) i32) +(extern constructor offset32_plus4 offset32_plus4) + +;;;; Comparisons and branches ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +;; Emit a compare (`cmp rn, rm`) as a side effect. +(decl cmp_rr (Reg Reg) SideEffectNoResult) +(rule (cmp_rr rn rm) + (SideEffectNoResult.Inst (MInst.CmpRR (CmpOp.Cmp) rn rm))) + +;; Emit a compare against a rotated imm12 (`cmp rn, #imm`) as a side effect. +(decl cmp_imm (Reg u32) SideEffectNoResult) +(rule (cmp_imm rn imm12) + (SideEffectNoResult.Inst (MInst.CmpRImm (CmpOp.Cmp) rn imm12))) + +;; A conditional branch. +(decl cond_br (Cond MachLabel MachLabel) SideEffectNoResult) +(rule (cond_br cond taken not_taken) + (SideEffectNoResult.Inst (MInst.CondBr cond taken not_taken))) + +;; An unconditional branch. +(decl jump_impl (MachLabel) SideEffectNoResult) +(rule (jump_impl label) + (SideEffectNoResult.Inst (MInst.Jump label))) + +;; A jump table (the first target is the default block). +(decl lower_br_table (Reg MachLabelSlice) Unit) +(extern constructor lower_br_table lower_br_table) + +;;;; Calls ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(decl gen_call_info (Sig ExternalName CallArgList CallRetList OptionTryCallInfo bool) BoxCallInfo) +(extern constructor gen_call_info gen_call_info) + +(decl gen_call_ind_info (Sig Reg CallArgList CallRetList OptionTryCallInfo) BoxCallIndInfo) +(extern constructor gen_call_ind_info gen_call_ind_info) + +(decl call_impl (BoxCallInfo) SideEffectNoResult) +(rule (call_impl info) + (SideEffectNoResult.Inst (MInst.Call info))) + +;; Load the absolute address of an external symbol (plus a byte offset) into a +;; fresh register. Used for far/non-colocated calls and symbol addresses. +;; (`box_external_name` is applied implicitly by the `ExternalName -> +;; BoxExternalName` conversion in the prelude.) +(decl load_ext_name (ExternalName i64) Reg) +(rule (load_ext_name name offset) + (let ((rd WritableReg (temp_writable_reg $I32)) + (_ Unit (emit (MInst.LoadExtName rd name offset)))) + rd)) + +(decl call_ind_impl (BoxCallIndInfo) SideEffectNoResult) +(rule (call_ind_impl info) + (SideEffectNoResult.Inst (MInst.CallInd info))) + +;; Map an `IntCC` to the corresponding ARM condition code. +(decl cond_from_intcc (IntCC) Cond) +(extern constructor cond_from_intcc cond_from_intcc) + +;; Emit a `cmp` of two (possibly narrow) integer values as a side effect, +;; widening i8/i16 operands to 32 bits per the comparison's signedness, and +;; return the ARM condition that then tests the result. +(decl emit_icmp_cmp (IntCC Value Value) Cond) +(extern constructor emit_icmp_cmp emit_icmp_cmp) + +;; Emit the compare sequence for a 64-bit `icmp` and return the ARM condition +;; that then tests it (the compare itself is emitted as a side effect). +(decl lower_icmp_i64 (IntCC ValueRegs ValueRegs) Cond) +(extern constructor lower_icmp_i64 lower_icmp_i64) + +;; Materialize a boolean 0/1 from the current flags and a condition +;; (`mov rd, #0; mov rd, #1`). +(decl csetv (Cond) Reg) +(rule (csetv cond) (csel cond (gen_constant 1) (gen_constant 0))) diff --git a/cranelift/codegen/src/isa/arm32/inst/args.rs b/cranelift/codegen/src/isa/arm32/inst/args.rs new file mode 100644 index 000000000000..eba41a211350 --- /dev/null +++ b/cranelift/codegen/src/isa/arm32/inst/args.rs @@ -0,0 +1,736 @@ +//! arm32 addressing modes and immediate/operand helpers. + +use crate::isa::arm32::inst::*; +use crate::machinst::{OperandVisitor, Reg}; + +pub use crate::isa::arm32::lower::isle::generated_code::{ + ALUOp, AMode, AtomicSize, BarrierOp, BfxOp, BitOp, CmpOp, Cond, DspMul3Op, DspMul4Op, + DspMulLOp, ExtAddOp, ExtOp, FpuOp2, FpuOp3, FpuSize, LoadKind, ParAluOp, PkhOp, QAluOp, SatOp, + ShiftOp, StoreKind, +}; + +impl AtomicSize { + /// The instruction-mnemonic suffix for this access width (`ldrex`, + /// `ldrexb`, `ldrexh`, `ldrexd`). + pub(crate) fn suffix(self) -> &'static str { + match self { + AtomicSize::Byte => "b", + AtomicSize::Half => "h", + AtomicSize::Word => "", + AtomicSize::DWord => "d", + } + } + + /// The size field (bits 22:21) of an A32 exclusive/acquire-release + /// instruction encoding. + pub(crate) fn enc_bits(self) -> u32 { + let sz = match self { + AtomicSize::Word => 0b00, + AtomicSize::DWord => 0b01, + AtomicSize::Byte => 0b10, + AtomicSize::Half => 0b11, + }; + sz << 21 + } + + /// True for a sub-word (byte or halfword) access. + pub(crate) fn is_subword(self) -> bool { + matches!(self, AtomicSize::Byte | AtomicSize::Half) + } + + /// The sign/zero-extend op that widens a sub-word value of this size to a + /// full 32-bit register. + pub(crate) fn extend_op(self, signed: bool) -> ExtOp { + match (self, signed) { + (AtomicSize::Byte, false) => ExtOp::Uxtb, + (AtomicSize::Byte, true) => ExtOp::Sxtb, + (AtomicSize::Half, false) => ExtOp::Uxth, + (AtomicSize::Half, true) => ExtOp::Sxth, + _ => unreachable!("extend_op is only valid for sub-word sizes"), + } + } +} + +/// A memory address resolved to a concrete base register and either an +/// immediate or register offset, ready for encoding. +pub(crate) enum ResolvedAMode { + /// `[base, #offset]` + Imm { base: Reg, offset: i32 }, + /// `[base, index]` + Reg { base: Reg, index: Reg }, +} + +impl AMode { + /// Collect the register operands referenced by this addressing mode. + pub(crate) fn get_operands(&mut self, collector: &mut impl OperandVisitor) { + // Only virtual registers are tracked; sp is implicit for the + // stack-relative modes. + fn use_if_virtual(collector: &mut impl OperandVisitor, reg: &mut Reg) { + if reg.to_real_reg().is_none() { + collector.reg_use(reg); + } + } + match self { + AMode::RegOffset { rn, .. } => use_if_virtual(collector, rn), + AMode::RegReg { rn, rm } => { + use_if_virtual(collector, rn); + use_if_virtual(collector, rm); + } + AMode::SPOffset { .. } | AMode::SlotOffset { .. } | AMode::IncomingArg { .. } => {} + } + } + + /// Resolve this addressing mode against the current frame layout. + pub(crate) fn resolve(&self, state: &EmitState) -> ResolvedAMode { + match self { + &AMode::RegOffset { rn, offset } => ResolvedAMode::Imm { base: rn, offset }, + &AMode::RegReg { rn, rm } => ResolvedAMode::Reg { + base: rn, + index: rm, + }, + &AMode::SPOffset { offset } => ResolvedAMode::Imm { + base: stack_reg(), + offset: offset as i32, + }, + &AMode::SlotOffset { offset } => { + let fl = state.frame_layout(); + ResolvedAMode::Imm { + base: stack_reg(), + offset: (offset + i64::from(fl.outgoing_args_size)) as i32, + } + } + &AMode::IncomingArg { offset } => { + let fl = state.frame_layout(); + let sp_off = i64::from(fl.tail_args_size) + + i64::from(fl.setup_area_size) + + i64::from(fl.clobber_size) + + i64::from(fl.fixed_frame_storage_size) + + i64::from(fl.outgoing_args_size) + - offset; + ResolvedAMode::Imm { + base: stack_reg(), + offset: sp_off as i32, + } + } + } + } + + /// Pretty-print this addressing mode. + pub(crate) fn pretty_print(&self) -> String { + match self { + AMode::RegOffset { rn, offset } => { + alloc::format!("[{}, #{}]", reg_name(*rn), offset) + } + AMode::RegReg { rn, rm } => { + alloc::format!("[{}, {}]", reg_name(*rn), reg_name(*rm)) + } + AMode::SPOffset { offset } => alloc::format!("[sp, #{offset}]"), + AMode::SlotOffset { offset } => alloc::format!("[slot, #{offset}]"), + AMode::IncomingArg { offset } => alloc::format!("[incoming_arg, #{offset}]"), + } + } +} + +impl ALUOp { + /// The 4-bit data-processing opcode for this operation. + pub(crate) fn opcode(self) -> u32 { + match self { + ALUOp::And => 0b0000, + ALUOp::Eor => 0b0001, + ALUOp::Sub => 0b0010, + ALUOp::Rsb => 0b0011, + ALUOp::Add => 0b0100, + ALUOp::Adc => 0b0101, + ALUOp::Sbc => 0b0110, + ALUOp::Orr => 0b1100, + ALUOp::Bic => 0b1110, + } + } + + pub(crate) fn name(self) -> &'static str { + match self { + ALUOp::And => "and", + ALUOp::Eor => "eor", + ALUOp::Sub => "sub", + ALUOp::Rsb => "rsb", + ALUOp::Add => "add", + ALUOp::Adc => "adc", + ALUOp::Sbc => "sbc", + ALUOp::Orr => "orr", + ALUOp::Bic => "bic", + } + } +} + +impl CmpOp { + /// The 4-bit data-processing opcode (with the S bit set separately). + pub(crate) fn opcode(self) -> u32 { + match self { + CmpOp::Tst => 0b1000, + CmpOp::Teq => 0b1001, + CmpOp::Cmp => 0b1010, + CmpOp::Cmn => 0b1011, + } + } + + pub(crate) fn name(self) -> &'static str { + match self { + CmpOp::Tst => "tst", + CmpOp::Teq => "teq", + CmpOp::Cmp => "cmp", + CmpOp::Cmn => "cmn", + } + } +} + +impl BitOp { + /// The full instruction word for `op rd, rm`, with `rd`/`rm` zeroed. + pub(crate) fn template(self) -> u32 { + match self { + BitOp::Clz => 0x016f_0f10, + BitOp::Rev => 0x06bf_0f30, + BitOp::Rev16 => 0x06bf_0fb0, + BitOp::Rbit => 0x06ff_0f30, + BitOp::Revsh => 0x06ff_0fb0, + } + } + + pub(crate) fn name(self) -> &'static str { + match self { + BitOp::Clz => "clz", + BitOp::Rev => "rev", + BitOp::Rev16 => "rev16", + BitOp::Rbit => "rbit", + BitOp::Revsh => "revsh", + } + } +} + +impl QAluOp { + /// The instruction word for `op rd, rm, rn`, with the registers zeroed. + pub(crate) fn template(self) -> u32 { + match self { + QAluOp::Qadd => 0x0100_0050, + QAluOp::Qsub => 0x0120_0050, + QAluOp::Qdadd => 0x0140_0050, + QAluOp::Qdsub => 0x0160_0050, + } + } + + pub(crate) fn name(self) -> &'static str { + match self { + QAluOp::Qadd => "qadd", + QAluOp::Qsub => "qsub", + QAluOp::Qdadd => "qdadd", + QAluOp::Qdsub => "qdsub", + } + } +} + +impl SatOp { + /// The instruction word for `op rd, #sat, rm` (no shift), registers zeroed. + pub(crate) fn template(self) -> u32 { + match self { + SatOp::Ssat => 0x06a0_0010, + SatOp::Usat => 0x06e0_0010, + SatOp::Ssat16 => 0x06a0_0f30, + SatOp::Usat16 => 0x06e0_0f30, + } + } + + /// Signed operations encode `sat_bits - 1`; unsigned encode `sat_bits`. + pub(crate) fn is_signed(self) -> bool { + matches!(self, SatOp::Ssat | SatOp::Ssat16) + } + + pub(crate) fn name(self) -> &'static str { + match self { + SatOp::Ssat => "ssat", + SatOp::Usat => "usat", + SatOp::Ssat16 => "ssat16", + SatOp::Usat16 => "usat16", + } + } +} + +impl BfxOp { + pub(crate) fn template(self) -> u32 { + match self { + BfxOp::Sbfx => 0x07a0_0050, + BfxOp::Ubfx => 0x07e0_0050, + } + } + + pub(crate) fn name(self) -> &'static str { + match self { + BfxOp::Sbfx => "sbfx", + BfxOp::Ubfx => "ubfx", + } + } +} + +impl PkhOp { + pub(crate) fn template(self) -> u32 { + match self { + PkhOp::Pkhbt => 0x0680_0010, + PkhOp::Pkhtb => 0x0680_0050, + } + } + + pub(crate) fn name(self) -> &'static str { + match self { + PkhOp::Pkhbt => "pkhbt", + PkhOp::Pkhtb => "pkhtb", + } + } +} + +impl ParAluOp { + /// The `(op1, op2)` fields of the parallel add/subtract encoding: `op1` + /// (bits 22:20) selects the prefix (signedness/saturation) and `op2` (bits + /// 7:5) selects the operation. + fn parts(self) -> (u32, u32) { + // Prefixes: S=1, Q=2, SH=3, U=5, UQ=6, UH=7. + // Ops: ADD16=0, ASX=1, SAX=2, SUB16=3, ADD8=4, SUB8=7. + match self { + ParAluOp::Sadd16 => (1, 0), + ParAluOp::Sasx => (1, 1), + ParAluOp::Ssax => (1, 2), + ParAluOp::Ssub16 => (1, 3), + ParAluOp::Sadd8 => (1, 4), + ParAluOp::Ssub8 => (1, 7), + ParAluOp::Qadd16 => (2, 0), + ParAluOp::Qasx => (2, 1), + ParAluOp::Qsax => (2, 2), + ParAluOp::Qsub16 => (2, 3), + ParAluOp::Qadd8 => (2, 4), + ParAluOp::Qsub8 => (2, 7), + ParAluOp::Shadd16 => (3, 0), + ParAluOp::Shasx => (3, 1), + ParAluOp::Shsax => (3, 2), + ParAluOp::Shsub16 => (3, 3), + ParAluOp::Shadd8 => (3, 4), + ParAluOp::Shsub8 => (3, 7), + ParAluOp::Uadd16 => (5, 0), + ParAluOp::Uasx => (5, 1), + ParAluOp::Usax => (5, 2), + ParAluOp::Usub16 => (5, 3), + ParAluOp::Uadd8 => (5, 4), + ParAluOp::Usub8 => (5, 7), + ParAluOp::Uqadd16 => (6, 0), + ParAluOp::Uqasx => (6, 1), + ParAluOp::Uqsax => (6, 2), + ParAluOp::Uqsub16 => (6, 3), + ParAluOp::Uqadd8 => (6, 4), + ParAluOp::Uqsub8 => (6, 7), + ParAluOp::Uhadd16 => (7, 0), + ParAluOp::Uhasx => (7, 1), + ParAluOp::Uhsax => (7, 2), + ParAluOp::Uhsub16 => (7, 3), + ParAluOp::Uhadd8 => (7, 4), + ParAluOp::Uhsub8 => (7, 7), + } + } + + /// The instruction word for `op rd, rn, rm`, with the registers zeroed. + pub(crate) fn template(self) -> u32 { + let (op1, op2) = self.parts(); + 0x0600_0000 | (op1 << 20) | 0x0000_0f00 | (op2 << 5) | 0x10 + } + + pub(crate) fn name(self) -> &'static str { + match self { + ParAluOp::Sadd8 => "sadd8", + ParAluOp::Sadd16 => "sadd16", + ParAluOp::Ssub8 => "ssub8", + ParAluOp::Ssub16 => "ssub16", + ParAluOp::Sasx => "sasx", + ParAluOp::Ssax => "ssax", + ParAluOp::Qadd8 => "qadd8", + ParAluOp::Qadd16 => "qadd16", + ParAluOp::Qsub8 => "qsub8", + ParAluOp::Qsub16 => "qsub16", + ParAluOp::Qasx => "qasx", + ParAluOp::Qsax => "qsax", + ParAluOp::Shadd8 => "shadd8", + ParAluOp::Shadd16 => "shadd16", + ParAluOp::Shsub8 => "shsub8", + ParAluOp::Shsub16 => "shsub16", + ParAluOp::Shasx => "shasx", + ParAluOp::Shsax => "shsax", + ParAluOp::Uadd8 => "uadd8", + ParAluOp::Uadd16 => "uadd16", + ParAluOp::Usub8 => "usub8", + ParAluOp::Usub16 => "usub16", + ParAluOp::Uasx => "uasx", + ParAluOp::Usax => "usax", + ParAluOp::Uqadd8 => "uqadd8", + ParAluOp::Uqadd16 => "uqadd16", + ParAluOp::Uqsub8 => "uqsub8", + ParAluOp::Uqsub16 => "uqsub16", + ParAluOp::Uqasx => "uqasx", + ParAluOp::Uqsax => "uqsax", + ParAluOp::Uhadd8 => "uhadd8", + ParAluOp::Uhadd16 => "uhadd16", + ParAluOp::Uhsub8 => "uhsub8", + ParAluOp::Uhsub16 => "uhsub16", + ParAluOp::Uhasx => "uhasx", + ParAluOp::Uhsax => "uhsax", + } + } +} + +impl ExtOp { + /// The full instruction word for `op rd, rm` (rotation 0), with `rd`/`rm` + /// zeroed. + pub(crate) fn template(self) -> u32 { + match self { + ExtOp::Sxtb => 0x06af_0070, + ExtOp::Sxth => 0x06bf_0070, + ExtOp::Uxtb => 0x06ef_0070, + ExtOp::Uxth => 0x06ff_0070, + ExtOp::Sxtb16 => 0x068f_0070, + ExtOp::Uxtb16 => 0x06cf_0070, + } + } + + pub(crate) fn name(self) -> &'static str { + match self { + ExtOp::Sxtb => "sxtb", + ExtOp::Sxth => "sxth", + ExtOp::Uxtb => "uxtb", + ExtOp::Uxth => "uxth", + ExtOp::Sxtb16 => "sxtb16", + ExtOp::Uxtb16 => "uxtb16", + } + } +} + +impl ExtAddOp { + /// The instruction word for `op rd, rn, rm` (rotation 0), registers zeroed. + pub(crate) fn template(self) -> u32 { + match self { + ExtAddOp::Sxtab => 0x06a0_0070, + ExtAddOp::Sxtah => 0x06b0_0070, + ExtAddOp::Sxtab16 => 0x0680_0070, + ExtAddOp::Uxtab => 0x06e0_0070, + ExtAddOp::Uxtah => 0x06f0_0070, + ExtAddOp::Uxtab16 => 0x06c0_0070, + } + } + + pub(crate) fn name(self) -> &'static str { + match self { + ExtAddOp::Sxtab => "sxtab", + ExtAddOp::Sxtah => "sxtah", + ExtAddOp::Sxtab16 => "sxtab16", + ExtAddOp::Uxtab => "uxtab", + ExtAddOp::Uxtah => "uxtah", + ExtAddOp::Uxtab16 => "uxtab16", + } + } +} + +impl DspMul3Op { + /// The instruction word for `op rd, rn, rm`, with `rd`(19:16), `rm`(11:8), + /// `rn`(3:0) zeroed. The half-select (M/N) and pre-select bits are baked in. + pub(crate) fn template(self) -> u32 { + match self { + DspMul3Op::Smulbb => 0x0160_0080, + DspMul3Op::Smulbt => 0x0160_00c0, + DspMul3Op::Smultb => 0x0160_00a0, + DspMul3Op::Smultt => 0x0160_00e0, + DspMul3Op::Smulwb => 0x0120_00a0, + DspMul3Op::Smulwt => 0x0120_00e0, + DspMul3Op::Smmul => 0x0750_f010, + DspMul3Op::Smuad => 0x0700_f010, + DspMul3Op::Smusd => 0x0700_f050, + } + } + + pub(crate) fn name(self) -> &'static str { + match self { + DspMul3Op::Smulbb => "smulbb", + DspMul3Op::Smulbt => "smulbt", + DspMul3Op::Smultb => "smultb", + DspMul3Op::Smultt => "smultt", + DspMul3Op::Smulwb => "smulwb", + DspMul3Op::Smulwt => "smulwt", + DspMul3Op::Smmul => "smmul", + DspMul3Op::Smuad => "smuad", + DspMul3Op::Smusd => "smusd", + } + } +} + +impl DspMul4Op { + /// The instruction word for `op rd, rn, rm, ra`, with `rd`(19:16), + /// `ra`(15:12), `rm`(11:8), `rn`(3:0) zeroed. + pub(crate) fn template(self) -> u32 { + match self { + DspMul4Op::Smlabb => 0x0100_0080, + DspMul4Op::Smlabt => 0x0100_00c0, + DspMul4Op::Smlatb => 0x0100_00a0, + DspMul4Op::Smlatt => 0x0100_00e0, + DspMul4Op::Smlawb => 0x0120_0080, + DspMul4Op::Smlawt => 0x0120_00c0, + DspMul4Op::Smmla => 0x0750_0010, + DspMul4Op::Smmls => 0x0750_00d0, + DspMul4Op::Smlad => 0x0700_0010, + DspMul4Op::Smlsd => 0x0700_0050, + } + } + + pub(crate) fn name(self) -> &'static str { + match self { + DspMul4Op::Smlabb => "smlabb", + DspMul4Op::Smlabt => "smlabt", + DspMul4Op::Smlatb => "smlatb", + DspMul4Op::Smlatt => "smlatt", + DspMul4Op::Smlawb => "smlawb", + DspMul4Op::Smlawt => "smlawt", + DspMul4Op::Smmla => "smmla", + DspMul4Op::Smmls => "smmls", + DspMul4Op::Smlad => "smlad", + DspMul4Op::Smlsd => "smlsd", + } + } +} + +impl DspMulLOp { + /// The instruction word for `op rd_lo, rd_hi, rn, rm`, with `rd_hi`(19:16), + /// `rd_lo`(15:12), `rm`(11:8), `rn`(3:0) zeroed. + pub(crate) fn template(self) -> u32 { + match self { + DspMulLOp::Smlalbb => 0x0140_0080, + DspMulLOp::Smlalbt => 0x0140_00c0, + DspMulLOp::Smlaltb => 0x0140_00a0, + DspMulLOp::Smlaltt => 0x0140_00e0, + DspMulLOp::Smlald => 0x0740_0010, + DspMulLOp::Smlsld => 0x0740_0050, + DspMulLOp::Umaal => 0x0040_0090, + } + } + + pub(crate) fn name(self) -> &'static str { + match self { + DspMulLOp::Smlalbb => "smlalbb", + DspMulLOp::Smlalbt => "smlalbt", + DspMulLOp::Smlaltb => "smlaltb", + DspMulLOp::Smlaltt => "smlaltt", + DspMulLOp::Smlald => "smlald", + DspMulLOp::Smlsld => "smlsld", + DspMulLOp::Umaal => "umaal", + } + } +} + +impl BarrierOp { + /// The full (unconditional) instruction word, with the system option (SY). + pub(crate) fn encoding(self) -> u32 { + match self { + BarrierOp::Dmb => 0xf57f_f05f, + BarrierOp::Dsb => 0xf57f_f04f, + BarrierOp::Isb => 0xf57f_f06f, + BarrierOp::Clrex => 0xf57f_f01f, + } + } + + pub(crate) fn name(self) -> &'static str { + match self { + BarrierOp::Dmb => "dmb sy", + BarrierOp::Dsb => "dsb sy", + BarrierOp::Isb => "isb sy", + BarrierOp::Clrex => "clrex", + } + } +} + +impl ShiftOp { + /// The 2-bit shift-type field used in a shifted operand2. + pub(crate) fn bits(self) -> u32 { + match self { + ShiftOp::Lsl => 0b00, + ShiftOp::Lsr => 0b01, + ShiftOp::Asr => 0b10, + ShiftOp::Ror => 0b11, + } + } + + pub(crate) fn name(self) -> &'static str { + match self { + ShiftOp::Lsl => "lsl", + ShiftOp::Lsr => "lsr", + ShiftOp::Asr => "asr", + ShiftOp::Ror => "ror", + } + } +} + +impl Cond { + /// The 4-bit condition-code field. + pub(crate) fn bits(self) -> u32 { + match self { + Cond::Eq => 0b0000, + Cond::Ne => 0b0001, + Cond::Hs => 0b0010, + Cond::Lo => 0b0011, + Cond::Mi => 0b0100, + Cond::Pl => 0b0101, + Cond::Vs => 0b0110, + Cond::Vc => 0b0111, + Cond::Hi => 0b1000, + Cond::Ls => 0b1001, + Cond::Ge => 0b1010, + Cond::Lt => 0b1011, + Cond::Gt => 0b1100, + Cond::Le => 0b1101, + } + } + + /// The condition that is true exactly when `self` is false. + pub(crate) fn invert(self) -> Cond { + match self { + Cond::Eq => Cond::Ne, + Cond::Ne => Cond::Eq, + Cond::Hs => Cond::Lo, + Cond::Lo => Cond::Hs, + Cond::Mi => Cond::Pl, + Cond::Pl => Cond::Mi, + Cond::Vs => Cond::Vc, + Cond::Vc => Cond::Vs, + Cond::Hi => Cond::Ls, + Cond::Ls => Cond::Hi, + Cond::Ge => Cond::Lt, + Cond::Lt => Cond::Ge, + Cond::Gt => Cond::Le, + Cond::Le => Cond::Gt, + } + } + + pub(crate) fn name(self) -> &'static str { + match self { + Cond::Eq => "eq", + Cond::Ne => "ne", + Cond::Hs => "hs", + Cond::Lo => "lo", + Cond::Mi => "mi", + Cond::Pl => "pl", + Cond::Vs => "vs", + Cond::Vc => "vc", + Cond::Hi => "hi", + Cond::Ls => "ls", + Cond::Ge => "ge", + Cond::Lt => "lt", + Cond::Gt => "gt", + Cond::Le => "le", + } + } +} + +impl LoadKind { + pub(crate) fn mnemonic(self) -> &'static str { + match self { + LoadKind::Word => "ldr", + LoadKind::UByte => "ldrb", + LoadKind::SByte => "ldrsb", + LoadKind::UHalf => "ldrh", + LoadKind::SHalf => "ldrsh", + } + } +} + +impl StoreKind { + pub(crate) fn mnemonic(self) -> &'static str { + match self { + StoreKind::Word => "str", + StoreKind::Byte => "strb", + StoreKind::Half => "strh", + } + } +} + +impl FpuSize { + /// The size bit (bit 8) that distinguishes F64 (set) from F32 (clear). + pub(crate) fn size_bit(self) -> u32 { + match self { + FpuSize::F32 => 0, + FpuSize::F64 => 0x100, + } + } + + pub(crate) fn suffix(self) -> &'static str { + match self { + FpuSize::F32 => "f32", + FpuSize::F64 => "f64", + } + } +} + +impl FpuOp3 { + /// The F32 base word (`vadd.f32 s0, s0, s0`); F64 adds the size bit. + pub(crate) fn base(self) -> u32 { + match self { + FpuOp3::Vadd => 0xee30_0a00, + FpuOp3::Vsub => 0xee30_0a40, + FpuOp3::Vmul => 0xee20_0a00, + FpuOp3::Vdiv => 0xee80_0a00, + } + } + + pub(crate) fn name(self) -> &'static str { + match self { + FpuOp3::Vadd => "vadd", + FpuOp3::Vsub => "vsub", + FpuOp3::Vmul => "vmul", + FpuOp3::Vdiv => "vdiv", + } + } +} + +impl FpuOp2 { + /// The F32 base word; F64 adds the size bit. + pub(crate) fn base(self) -> u32 { + match self { + FpuOp2::Vmov => 0xeeb0_0a40, + FpuOp2::Vneg => 0xeeb1_0a40, + FpuOp2::Vabs => 0xeeb0_0ac0, + FpuOp2::Vsqrt => 0xeeb1_0ac0, + } + } + + pub(crate) fn name(self) -> &'static str { + match self { + FpuOp2::Vmov => "vmov", + FpuOp2::Vneg => "vneg", + FpuOp2::Vabs => "vabs", + FpuOp2::Vsqrt => "vsqrt", + } + } +} + +/// Decode a 12-bit rotated data-processing immediate back to its value, for +/// pretty-printing. +pub fn decode_rotated_imm(enc: u32) -> u32 { + let rot = (enc >> 8) & 0xf; + let imm8 = enc & 0xff; + imm8.rotate_right(2 * rot) +} + +/// Encode `val` as an A32 data-processing immediate: an 8-bit value rotated +/// right by an even amount. Returns the 12-bit encoded operand (`rot << 8 | +/// imm8`) if representable. +pub fn encode_rotated_imm(val: u32) -> Option { + if val <= 0xff { + return Some(val); + } + for rot in 1..16u32 { + // The immediate is `imm8` rotated right by `2 * rot`; to recover imm8 we + // rotate `val` left by the same amount. + let rotated = val.rotate_left(2 * rot); + if rotated <= 0xff { + return Some((rot << 8) | rotated); + } + } + None +} diff --git a/cranelift/codegen/src/isa/arm32/inst/emit.rs b/cranelift/codegen/src/isa/arm32/inst/emit.rs new file mode 100644 index 000000000000..5cd8466c9501 --- /dev/null +++ b/cranelift/codegen/src/isa/arm32/inst/emit.rs @@ -0,0 +1,1448 @@ +//! arm32 (A32) ISA: binary code emission. + +use crate::binemit::Reloc; +use crate::ir; +use crate::ir::AtomicRmwOp; +use crate::isa::arm32::inst::*; +use crate::settings; +use cranelift_control::ControlPlane; + +/// Information carried along during emission, derived from the shared and ISA +/// flags. +pub struct EmitInfo { + #[expect(dead_code, reason = "may be used once more instructions are added")] + shared_flags: settings::Flags, + #[expect(dead_code, reason = "may be used once more instructions are added")] + isa_flags: super::super::settings::Flags, +} + +impl EmitInfo { + pub(crate) fn new( + shared_flags: settings::Flags, + isa_flags: super::super::settings::Flags, + ) -> Self { + Self { + shared_flags, + isa_flags, + } + } +} + +/// State carried between emissions of a sequence of instructions. +#[derive(Default, Clone, Debug)] +pub struct EmitState { + /// The user stack map for the upcoming instruction. + user_stack_map: Option, + /// Only used during fuzz-testing. + ctrl_plane: ControlPlane, + frame_layout: FrameLayout, +} + +impl EmitState { + fn take_stack_map(&mut self) -> Option { + self.user_stack_map.take() + } +} + +impl MachInstEmitState for EmitState { + fn new( + abi: &Callee, + ctrl_plane: ControlPlane, + ) -> Self { + EmitState { + user_stack_map: None, + ctrl_plane, + frame_layout: abi.frame_layout().clone(), + } + } + + fn pre_safepoint(&mut self, user_stack_map: Option) { + self.user_stack_map = user_stack_map; + } + + fn ctrl_plane_mut(&mut self) -> &mut ControlPlane { + &mut self.ctrl_plane + } + + fn take_ctrl_plane(self) -> ControlPlane { + self.ctrl_plane + } + + fn frame_layout(&self) -> &FrameLayout { + &self.frame_layout + } +} + +/// Encoding of the "always" (AL) condition code, in the top nibble of an A32 +/// instruction word. +const COND_AL: u32 = 0xe000_0000; + +/// The hardware encoding number (0-15) of a real register. +fn machreg_to_gpr(reg: Reg) -> u32 { + u32::from(reg.to_real_reg().unwrap().hw_enc() & 0xf) +} + +//============================================================================= +// Instruction encoders. + +/// A data-processing instruction with an immediate operand2 (`op{s} rd, rn, +/// #imm12`). `imm12` is the already-encoded rotated immediate. +fn enc_dp_imm(opcode: u32, s: u32, rd: u32, rn: u32, imm12: u32) -> u32 { + COND_AL | (1 << 25) | (opcode << 21) | (s << 20) | (rn << 16) | (rd << 12) | (imm12 & 0xfff) +} + +/// A data-processing instruction with a register operand2 (`op{s} rd, rn, rm`). +fn enc_dp_reg(opcode: u32, s: u32, rd: u32, rn: u32, rm: u32) -> u32 { + COND_AL | (opcode << 21) | (s << 20) | (rn << 16) | (rd << 12) | rm +} + +/// `movw rd, #imm16` — load a 16-bit immediate into the low half of `rd`. +fn enc_movw(rd: u32, imm16: u32) -> u32 { + let imm4 = (imm16 >> 12) & 0xf; + let imm12 = imm16 & 0xfff; + COND_AL | 0x0300_0000 | (imm4 << 16) | (rd << 12) | imm12 +} + +/// `movt rd, #imm16` — load a 16-bit immediate into the high half of `rd`. +fn enc_movt(rd: u32, imm16: u32) -> u32 { + let imm4 = (imm16 >> 12) & 0xf; + let imm12 = imm16 & 0xfff; + COND_AL | 0x0340_0000 | (imm4 << 16) | (rd << 12) | imm12 +} + +/// `bx rm` — branch and exchange. +fn enc_bx(rm: u32) -> u32 { + COND_AL | 0x012f_ff10 | rm +} + +/// `b ` — unconditional branch (offset patched in later). +fn enc_b(imm24: u32) -> u32 { + COND_AL | 0x0a00_0000 | (imm24 & 0x00ff_ffff) +} + +/// `b ` — conditional branch. +fn enc_bcond(cond: Cond, imm24: u32) -> u32 { + (cond.bits() << 28) | 0x0a00_0000 | (imm24 & 0x00ff_ffff) +} + +/// `bl ` — branch with link (direct call). +fn enc_bl(imm24: u32) -> u32 { + COND_AL | 0x0b00_0000 | (imm24 & 0x00ff_ffff) +} + +/// `blx rm` — branch with link and exchange to a register (indirect call). +fn enc_blx(rm: u32) -> u32 { + COND_AL | 0x012f_ff30 | rm +} + +/// Common post-call sequence: record the safepoint/call site, pop callee stack +/// args, and load any stack-carried return values. +fn emit_call_epilogue( + sink: &mut MachBuffer, + emit_info: &EmitInfo, + state: &mut EmitState, + info: &crate::machinst::CallInfo, +) { + use crate::isa::arm32::abi::Arm32MachineDeps; + use crate::machinst::ABIMachineSpec; + + if let Some(s) = state.take_stack_map() { + let offset = sink.cur_offset(); + sink.push_user_stack_map(state, offset, s); + } + sink.add_call_site(); + + let callee_pop_size = i32::try_from(info.callee_pop_size).unwrap(); + if callee_pop_size > 0 { + for inst in Arm32MachineDeps::gen_sp_reg_adjust(-callee_pop_size) { + inst.emit(sink, emit_info, state); + } + } + + info.emit_retval_loads::( + state.frame_layout().stackslots_size, + |inst| inst.emit(sink, emit_info, state), + |_needed_space| None, + ); +} + +/// `add`/`sub sp, sp, #imm`. +fn enc_sp_adjust(amount: i32) -> u32 { + let (op, mag) = if amount < 0 { + (ALUOp::Sub, (-amount) as u32) + } else { + (ALUOp::Add, amount as u32) + }; + let imm12 = + encode_rotated_imm(mag).expect("arm32 sp adjust not encodable as a single immediate"); + enc_dp_imm(op.opcode(), 0, 13, 13, imm12) +} + +/// `ldr`/`str{b} rt, [base, #±offset]` (word or byte). +fn enc_ldr_str_imm(load: bool, byte: bool, rt: u32, base: u32, offset: i32) -> u32 { + let (u_bit, mag) = if offset < 0 { + (0u32, (-offset) as u32) + } else { + (1u32, offset as u32) + }; + assert!(mag < 4096, "arm32 ldr/str offset out of range: {offset}"); + COND_AL + | 0x0400_0000 + | (1 << 24) // P = 1 (pre-indexed) + | (u_bit << 23) + | (if byte { 1 } else { 0 } << 22) + | (if load { 1 } else { 0 } << 20) + | (base << 16) + | (rt << 12) + | mag +} + +/// `ldr`/`str{b} rt, [base, index]` (word or byte, register offset). +fn enc_ldr_str_reg(load: bool, byte: bool, rt: u32, base: u32, index: u32) -> u32 { + COND_AL + | 0x0600_0000 + | (1 << 24) // P = 1 + | (1 << 23) // U = 1 (add) + | (if byte { 1 } else { 0 } << 22) + | (if load { 1 } else { 0 } << 20) + | (base << 16) + | (rt << 12) + | index +} + +/// Extra load/store (halfword and signed byte) with an 8-bit immediate offset. +fn enc_ldrh_strh_imm(load: bool, s: u32, h: u32, rt: u32, base: u32, offset: i32) -> u32 { + let (u_bit, mag) = if offset < 0 { + (0u32, (-offset) as u32) + } else { + (1u32, offset as u32) + }; + assert!(mag < 256, "arm32 ldrh/strh offset out of range: {offset}"); + let imm_h = (mag >> 4) & 0xf; + let imm_l = mag & 0xf; + COND_AL + | (1 << 24) // P = 1 + | (u_bit << 23) + | (1 << 22) // immediate form + | (if load { 1 } else { 0 } << 20) + | (base << 16) + | (rt << 12) + | (imm_h << 8) + | (1 << 7) + | (s << 6) + | (h << 5) + | (1 << 4) + | imm_l +} + +/// `push {reglist}` (STMDB sp!). +fn enc_push(reg_list: u32) -> u32 { + COND_AL | 0x092d_0000 | (reg_list & 0xffff) +} + +/// `pop {reglist}` (LDMIA sp!). +fn enc_pop(reg_list: u32) -> u32 { + COND_AL | 0x08bd_0000 | (reg_list & 0xffff) +} + +/// A shift/rotate by a constant amount: `mov rd, rm, #amount`. +fn enc_shift_imm(op: ShiftOp, rd: u32, rm: u32, amount: u32) -> u32 { + COND_AL | 0x01a0_0000 | (rd << 12) | ((amount & 0x1f) << 7) | (op.bits() << 5) | rm +} + +/// A shift/rotate by a register amount: `mov rd, rm, rs`. +fn enc_shift_reg(op: ShiftOp, rd: u32, rm: u32, rs: u32) -> u32 { + COND_AL | 0x01a0_0000 | (rd << 12) | (rs << 8) | (op.bits() << 5) | (1 << 4) | rm +} + +/// `mul rd, rn, rm` — `rd = rn * rm`. +fn enc_mul(rd: u32, rn: u32, rm: u32) -> u32 { + COND_AL | (rd << 16) | (rm << 8) | 0x90 | rn +} + +/// `mla rd, rn, rm, ra` — `rd = rn * rm + ra`. +fn enc_mla(rd: u32, rn: u32, rm: u32, ra: u32) -> u32 { + COND_AL | 0x0020_0000 | (rd << 16) | (ra << 12) | (rm << 8) | 0x90 | rn +} + +/// A 32x32->64 long multiply (`umull`/`smull`). `signed` selects `smull`. +fn enc_mull(signed: bool, rd_lo: u32, rd_hi: u32, rn: u32, rm: u32) -> u32 { + let base = if signed { 0x00c0_0000 } else { 0x0080_0000 }; + COND_AL | base | (rd_hi << 16) | (rd_lo << 12) | (rm << 8) | 0x90 | rn +} + +/// A 32x32->64 multiply-accumulate (`umlal`/`smlal`). `signed` selects `smlal`. +fn enc_mlal(signed: bool, rd_lo: u32, rd_hi: u32, rn: u32, rm: u32) -> u32 { + let base = if signed { 0x00e0_0000 } else { 0x00a0_0000 }; + COND_AL | base | (rd_hi << 16) | (rd_lo << 12) | (rm << 8) | 0x90 | rn +} + +/// `mls rd, rn, rm, ra` — `rd = ra - rn * rm`. +fn enc_mls(rd: u32, rn: u32, rm: u32, ra: u32) -> u32 { + COND_AL | 0x0060_0090 | (rd << 16) | (ra << 12) | (rm << 8) | rn +} + +/// `sdiv`/`udiv rd, rn, rm`. `signed` selects `sdiv`. +fn enc_div(signed: bool, rd: u32, rn: u32, rm: u32) -> u32 { + let base = if signed { 0x0710_f010 } else { 0x0730_f010 }; + COND_AL | base | (rd << 16) | (rm << 8) | rn +} + +/// A single-operand `op rd, rm` built from an opcode template. +fn enc_op_rd_rm(template: u32, rd: u32, rm: u32) -> u32 { + COND_AL | template | (rd << 12) | rm +} + +/// A conditional register move: `mov rd, rm`. +fn enc_mov_cond(cond: Cond, rd: u32, rm: u32) -> u32 { + (cond.bits() << 28) | 0x01a0_0000 | (rd << 12) | rm +} + +/// `bfc rd, #lsb, #width`. +fn enc_bfc(rd: u32, lsb: u32, width: u32) -> u32 { + let msb = lsb + width - 1; + COND_AL | 0x07c0_001f | (msb << 16) | (rd << 12) | (lsb << 7) +} + +/// `bfi rd, rn, #lsb, #width`. +fn enc_bfi(rd: u32, rn: u32, lsb: u32, width: u32) -> u32 { + let msb = lsb + width - 1; + COND_AL | 0x07c0_0010 | (msb << 16) | (rd << 12) | (lsb << 7) | rn +} + +/// `sbfx`/`ubfx rd, rn, #lsb, #width` from the op template. +fn enc_bfx(template: u32, rd: u32, rn: u32, lsb: u32, width: u32) -> u32 { + COND_AL | template | ((width - 1) << 16) | (rd << 12) | (lsb << 7) | rn +} + +/// A three-register op `template rd, rn, rm` with `rn`/`rd`/`rm` zeroed +/// (used by the saturating, parallel, sel, pkh and extend-add families). +fn enc_rd_rn_rm(template: u32, rd: u32, rn: u32, rm: u32) -> u32 { + COND_AL | template | (rn << 16) | (rd << 12) | rm +} + +/// `ssat`/`usat rd, #sat_bits, rm`. +fn enc_sat(op: SatOp, rd: u32, sat_bits: u32, rm: u32) -> u32 { + let field = if op.is_signed() { + sat_bits - 1 + } else { + sat_bits + }; + COND_AL | op.template() | (field << 16) | (rd << 12) | rm +} + +/// `rrx rd, rm` (a `mov` with a rotate-right-extend shift). +fn enc_rrx(rd: u32, rm: u32) -> u32 { + COND_AL | 0x01a0_0060 | (rd << 12) | rm +} + +/// The permanently-undefined `udf` encoding (`udf #0`). +fn enc_udf() -> u32 { + 0xe7f0_00f0 +} + +/// A DSP multiply with a single result: `template` plus `rd`(19:16), +/// `rm`(11:8), `rn`(3:0). +fn enc_dsp3(template: u32, rd: u32, rn: u32, rm: u32) -> u32 { + COND_AL | template | (rd << 16) | (rm << 8) | rn +} + +/// A DSP multiply-accumulate: `template` plus `rd`(19:16), `ra`(15:12), +/// `rm`(11:8), `rn`(3:0). +fn enc_dsp4(template: u32, rd: u32, rn: u32, rm: u32, ra: u32) -> u32 { + COND_AL | template | (rd << 16) | (ra << 12) | (rm << 8) | rn +} + +/// A DSP long multiply: `template` plus `rd_hi`(19:16), `rd_lo`(15:12), +/// `rm`(11:8), `rn`(3:0). +fn enc_dsp_long(template: u32, rd_lo: u32, rd_hi: u32, rn: u32, rm: u32) -> u32 { + COND_AL | template | (rd_hi << 16) | (rd_lo << 12) | (rm << 8) | rn +} + +/// `ldmia`/`stmia rn{!}, {reglist}` (increment-after). +fn enc_ldm_stm(load: bool, rn: u32, writeback: bool, reg_list: u32) -> u32 { + let base = if load { 0x0890_0000 } else { 0x0880_0000 }; + let wb = if writeback { 0x0020_0000 } else { 0 }; + COND_AL | base | wb | (rn << 16) | (reg_list & 0xffff) +} + +/// `ldrex{b,h,d}`/`ldaex{b,h,d} rt, [rn]` (the size field selects the width). +fn enc_load_ex(acquire: bool, size: AtomicSize, rt: u32, rn: u32) -> u32 { + let base = if acquire { 0x0190_0e9f } else { 0x0190_0f9f }; + COND_AL | base | size.enc_bits() | (rn << 16) | (rt << 12) +} + +/// `strex{b,h,d}`/`stlex{b,h,d} rd, rt, [rn]`. +fn enc_store_ex(release: bool, size: AtomicSize, rd: u32, rt: u32, rn: u32) -> u32 { + let base = if release { 0x0180_0e90 } else { 0x0180_0f90 }; + COND_AL | base | size.enc_bits() | (rn << 16) | (rd << 12) | rt +} + +/// `lda{b,h} rt, [rn]`. +fn enc_lda(size: AtomicSize, rt: u32, rn: u32) -> u32 { + COND_AL | 0x0190_0c9f | size.enc_bits() | (rn << 16) | (rt << 12) +} + +/// `stl{b,h} rt, [rn]`. +fn enc_stl(size: AtomicSize, rt: u32, rn: u32) -> u32 { + COND_AL | 0x0180_fc90 | size.enc_bits() | (rn << 16) | rt +} + +/// The hardware encoding number (0-15) of a VFP register. An f32 uses the low +/// S-half of its D register, whose Vd/D fields encode identically, so this is +/// just the D-register number. +fn machreg_to_vfp(reg: Reg) -> u32 { + u32::from(reg.to_real_reg().unwrap().hw_enc() & 0xf) +} + +/// A three-register VFP data-processing instruction. +fn enc_fpu_rrr(op: FpuOp3, size: FpuSize, d: u32, n: u32, m: u32) -> u32 { + op.base() | size.size_bit() | (n << 16) | (d << 12) | m +} + +/// A two-register VFP data-processing instruction. +fn enc_fpu_rr(op: FpuOp2, size: FpuSize, d: u32, m: u32) -> u32 { + op.base() | size.size_bit() | (d << 12) | m +} + +/// `vldr`/`vstr , [rn, #±offset]` (offset must be a multiple of 4). +fn enc_vldr_vstr(load: bool, size: FpuSize, vd: u32, rn: u32, offset: i32) -> u32 { + let base = (if load { 0xed90_0a00 } else { 0xed80_0a00 }) | size.size_bit(); + assert!(offset % 4 == 0, "arm32 vldr/vstr offset must be 4-aligned"); + let (base, mag) = if offset < 0 { + (base & !0x0080_0000, ((-offset) as u32) / 4) + } else { + (base, (offset as u32) / 4) + }; + assert!(mag < 256, "arm32 vldr/vstr offset out of range"); + base | (rn << 16) | (vd << 12) | mag +} + +/// `vmov sn, rt` — move a GPR into a single-precision register. +fn enc_vmov_gpr_s(vn: u32, rt: u32) -> u32 { + COND_AL | 0x0e00_0a10 | (vn << 16) | (rt << 12) +} + +/// `vmov dm, rt_lo, rt_hi` — move two GPRs into a double register. +fn enc_vmov_gpr_d(vm: u32, rt_lo: u32, rt_hi: u32) -> u32 { + COND_AL | 0x0c40_0b10 | (rt_hi << 16) | (rt_lo << 12) | vm +} + +/// `vmov rt, sn` — move a single register into a GPR. +fn enc_vmov_s_to_gpr(rt: u32, vn: u32) -> u32 { + COND_AL | 0x0e10_0a10 | (vn << 16) | (rt << 12) +} + +/// `vmov rt_lo, rt_hi, dm` — move a double register into two GPRs. +fn enc_vmov_d_to_gpr(rt_lo: u32, rt_hi: u32, vm: u32) -> u32 { + COND_AL | 0x0c50_0b10 | (rt_hi << 16) | (rt_lo << 12) | vm +} + +/// The `vmrs APSR_nzcv, FPSCR` instruction (moves the VFP flags to the CPSR). +const VMRS_APSR_NZCV: u32 = 0xeef1_fa10; + +/// `vcmp.f32/f64 sn/dn, sm/dm`. +fn enc_vcmp(size: FpuSize, n: u32, m: u32) -> u32 { + (0xeeb4_0a40 | size.size_bit()) | (n << 12) | m +} + +/// `vcvt.f64.f32`/`vcvt.f32.f64` (`to_f64` selects promote vs demote). +fn enc_vcvt_ff(to_f64: bool, d: u32, m: u32) -> u32 { + let base = if to_f64 { 0xeeb7_0ac0 } else { 0xeeb7_0bc0 }; + base | (d << 12) | m +} + +/// `vcvt.s32/u32.f32/f64` — float to integer, round toward zero. +fn enc_vcvt_to_int(signed: bool, size: FpuSize, d: u32, m: u32) -> u32 { + let sign = if signed { 0x1_0000 } else { 0 }; + (0xeebc_0ac0 | sign | size.size_bit()) | (d << 12) | m +} + +/// `vcvt.f32/f64.s32/u32` — integer (in the S reg `m`) to float. +fn enc_vcvt_from_int(signed: bool, size: FpuSize, d: u32, m: u32) -> u32 { + let sign = if signed { 0x80 } else { 0 }; + (0xeeb8_0a40 | sign | size.size_bit()) | (d << 12) | m +} + +/// `vmov.f64 dd, dm` — a conditional double-register move. +fn enc_vmov_f64_cond(cond: Cond, d: u32, m: u32) -> u32 { + (cond.bits() << 28) | 0x0eb0_0b40 | (d << 12) | m +} + +/// `vminnm`/`vmaxnm` (IEEE number min/max; ARMv8). These are unconditional. +fn enc_fpu_minmax(max: bool, size: FpuSize, d: u32, n: u32, m: u32) -> u32 { + let op = if max { 0 } else { 0x40 }; + (0xfe80_0a00 | op | size.size_bit()) | (n << 16) | (d << 12) | m +} + +fn put_u32(sink: &mut MachBuffer, word: u32) { + for b in word.to_le_bytes() { + sink.put1(b); + } +} + +impl MachInstEmit for Inst { + type State = EmitState; + type Info = EmitInfo; + + fn emit(&self, sink: &mut MachBuffer, emit_info: &Self::Info, state: &mut EmitState) { + let start = sink.cur_offset(); + match self { + Inst::Nop0 => {} + Inst::Nop4 => put_u32(sink, COND_AL | 0x0320_f000), + Inst::Ret => put_u32(sink, enc_bx(14)), + + Inst::MovImm { rd, imm } => { + let rd = machreg_to_gpr(rd.to_reg()); + put_u32(sink, enc_movw(rd, (imm & 0xffff) as u32)); + if imm >> 16 != 0 { + put_u32(sink, enc_movt(rd, (imm >> 16) as u32)); + } + } + Inst::MovRotImm { rd, imm12 } => { + let rd = machreg_to_gpr(rd.to_reg()); + put_u32(sink, enc_dp_imm(0b1101, 0, rd, 0, *imm12)); + } + Inst::MvnRotImm { rd, imm12 } => { + let rd = machreg_to_gpr(rd.to_reg()); + put_u32(sink, enc_dp_imm(0b1111, 0, rd, 0, *imm12)); + } + Inst::Movw { rd, imm16 } => { + put_u32(sink, enc_movw(machreg_to_gpr(rd.to_reg()), *imm16)); + } + Inst::Movt { rd, imm16 } => { + put_u32(sink, enc_movt(machreg_to_gpr(rd.to_reg()), *imm16)); + } + Inst::MovReg { rd, rm } => { + let rd = machreg_to_gpr(rd.to_reg()); + let rm = machreg_to_gpr(*rm); + put_u32(sink, enc_dp_reg(0b1101, 0, rd, 0, rm)); + } + Inst::MvnReg { rd, rm } => { + let rd = machreg_to_gpr(rd.to_reg()); + let rm = machreg_to_gpr(*rm); + put_u32(sink, enc_dp_reg(0b1111, 0, rd, 0, rm)); + } + + Inst::AluRRR { op, rd, rn, rm } => { + let rd = machreg_to_gpr(rd.to_reg()); + let rn = machreg_to_gpr(*rn); + let rm = machreg_to_gpr(*rm); + put_u32(sink, enc_dp_reg(op.opcode(), 0, rd, rn, rm)); + } + Inst::AluRRImm { op, rd, rn, imm12 } => { + let rd = machreg_to_gpr(rd.to_reg()); + let rn = machreg_to_gpr(*rn); + put_u32(sink, enc_dp_imm(op.opcode(), 0, rd, rn, *imm12)); + } + Inst::AluRRRFlags { op, rd, rn, rm } => { + let rd = machreg_to_gpr(rd.to_reg()); + let rn = machreg_to_gpr(*rn); + let rm = machreg_to_gpr(*rm); + put_u32(sink, enc_dp_reg(op.opcode(), 1, rd, rn, rm)); + } + Inst::ShiftImm { op, rd, rm, amount } => { + let rd = machreg_to_gpr(rd.to_reg()); + let rm = machreg_to_gpr(*rm); + put_u32(sink, enc_shift_imm(*op, rd, rm, u32::from(*amount))); + } + Inst::ShiftReg { op, rd, rm, rs } => { + let rd = machreg_to_gpr(rd.to_reg()); + let rm = machreg_to_gpr(*rm); + let rs = machreg_to_gpr(*rs); + put_u32(sink, enc_shift_reg(*op, rd, rm, rs)); + } + Inst::Mul { rd, rn, rm } => { + let rd = machreg_to_gpr(rd.to_reg()); + let rn = machreg_to_gpr(*rn); + let rm = machreg_to_gpr(*rm); + put_u32(sink, enc_mul(rd, rn, rm)); + } + Inst::Mla { rd, rn, rm, ra } => { + let rd = machreg_to_gpr(rd.to_reg()); + let rn = machreg_to_gpr(*rn); + let rm = machreg_to_gpr(*rm); + let ra = machreg_to_gpr(*ra); + put_u32(sink, enc_mla(rd, rn, rm, ra)); + } + Inst::Umull { + rd_lo, + rd_hi, + rn, + rm, + } => { + let rd_lo = machreg_to_gpr(rd_lo.to_reg()); + let rd_hi = machreg_to_gpr(rd_hi.to_reg()); + let rn = machreg_to_gpr(*rn); + let rm = machreg_to_gpr(*rm); + put_u32(sink, enc_mull(false, rd_lo, rd_hi, rn, rm)); + } + Inst::Smull { + rd_lo, + rd_hi, + rn, + rm, + } => { + let rd_lo = machreg_to_gpr(rd_lo.to_reg()); + let rd_hi = machreg_to_gpr(rd_hi.to_reg()); + let rn = machreg_to_gpr(*rn); + let rm = machreg_to_gpr(*rm); + put_u32(sink, enc_mull(true, rd_lo, rd_hi, rn, rm)); + } + Inst::Umlal { + rd_lo, + rd_hi, + rn, + rm, + } => { + let rd_lo = machreg_to_gpr(rd_lo.to_reg()); + let rd_hi = machreg_to_gpr(rd_hi.to_reg()); + let rn = machreg_to_gpr(*rn); + let rm = machreg_to_gpr(*rm); + put_u32(sink, enc_mlal(false, rd_lo, rd_hi, rn, rm)); + } + Inst::Smlal { + rd_lo, + rd_hi, + rn, + rm, + } => { + let rd_lo = machreg_to_gpr(rd_lo.to_reg()); + let rd_hi = machreg_to_gpr(rd_hi.to_reg()); + let rn = machreg_to_gpr(*rn); + let rm = machreg_to_gpr(*rm); + put_u32(sink, enc_mlal(true, rd_lo, rd_hi, rn, rm)); + } + Inst::Mls { rd, rn, rm, ra } => { + let rd = machreg_to_gpr(rd.to_reg()); + let rn = machreg_to_gpr(*rn); + let rm = machreg_to_gpr(*rm); + let ra = machreg_to_gpr(*ra); + put_u32(sink, enc_mls(rd, rn, rm, ra)); + } + Inst::SDiv { rd, rn, rm } => { + let rd = machreg_to_gpr(rd.to_reg()); + let rn = machreg_to_gpr(*rn); + let rm = machreg_to_gpr(*rm); + put_u32(sink, enc_div(true, rd, rn, rm)); + } + Inst::UDiv { rd, rn, rm } => { + let rd = machreg_to_gpr(rd.to_reg()); + let rn = machreg_to_gpr(*rn); + let rm = machreg_to_gpr(*rm); + put_u32(sink, enc_div(false, rd, rn, rm)); + } + Inst::BitRR { op, rd, rm } => { + let rd = machreg_to_gpr(rd.to_reg()); + let rm = machreg_to_gpr(*rm); + put_u32(sink, enc_op_rd_rm(op.template(), rd, rm)); + } + Inst::ExtRR { op, rd, rm } => { + let rd = machreg_to_gpr(rd.to_reg()); + let rm = machreg_to_gpr(*rm); + put_u32(sink, enc_op_rd_rm(op.template(), rd, rm)); + } + Inst::CSel { cond, rd, rn, rm } => { + let rd = machreg_to_gpr(rd.to_reg()); + let rn = machreg_to_gpr(*rn); + let rm = machreg_to_gpr(*rm); + // Default to the "false" value, then conditionally overwrite. + put_u32(sink, enc_dp_reg(0b1101, 0, rd, 0, rm)); + put_u32(sink, enc_mov_cond(*cond, rd, rn)); + } + Inst::Bfc { rd, lsb, width } => { + let rd = machreg_to_gpr(rd.to_reg()); + put_u32(sink, enc_bfc(rd, u32::from(*lsb), u32::from(*width))); + } + Inst::Bfi { rd, rn, lsb, width } => { + let rd = machreg_to_gpr(rd.to_reg()); + let rn = machreg_to_gpr(*rn); + put_u32(sink, enc_bfi(rd, rn, u32::from(*lsb), u32::from(*width))); + } + Inst::Bfx { + op, + rd, + rn, + lsb, + width, + } => { + let rd = machreg_to_gpr(rd.to_reg()); + let rn = machreg_to_gpr(*rn); + put_u32( + sink, + enc_bfx(op.template(), rd, rn, u32::from(*lsb), u32::from(*width)), + ); + } + Inst::QAlu { op, rd, rm, rn } => { + let rd = machreg_to_gpr(rd.to_reg()); + let rm = machreg_to_gpr(*rm); + let rn = machreg_to_gpr(*rn); + put_u32(sink, enc_rd_rn_rm(op.template(), rd, rn, rm)); + } + Inst::Sat { + op, + rd, + sat_bits, + rm, + } => { + let rd = machreg_to_gpr(rd.to_reg()); + let rm = machreg_to_gpr(*rm); + put_u32(sink, enc_sat(*op, rd, u32::from(*sat_bits), rm)); + } + Inst::ParAlu { op, rd, rn, rm } => { + let rd = machreg_to_gpr(rd.to_reg()); + let rn = machreg_to_gpr(*rn); + let rm = machreg_to_gpr(*rm); + put_u32(sink, enc_rd_rn_rm(op.template(), rd, rn, rm)); + } + Inst::ExtAdd { op, rd, rn, rm } => { + let rd = machreg_to_gpr(rd.to_reg()); + let rn = machreg_to_gpr(*rn); + let rm = machreg_to_gpr(*rm); + put_u32(sink, enc_rd_rn_rm(op.template(), rd, rn, rm)); + } + Inst::DspMul3 { op, rd, rn, rm } => { + let rd = machreg_to_gpr(rd.to_reg()); + let rn = machreg_to_gpr(*rn); + let rm = machreg_to_gpr(*rm); + put_u32(sink, enc_dsp3(op.template(), rd, rn, rm)); + } + Inst::DspMul4 { op, rd, rn, rm, ra } => { + let rd = machreg_to_gpr(rd.to_reg()); + let rn = machreg_to_gpr(*rn); + let rm = machreg_to_gpr(*rm); + let ra = machreg_to_gpr(*ra); + put_u32(sink, enc_dsp4(op.template(), rd, rn, rm, ra)); + } + Inst::DspMulL { + op, + rd_lo, + rd_hi, + rn, + rm, + } => { + let rd_lo = machreg_to_gpr(rd_lo.to_reg()); + let rd_hi = machreg_to_gpr(rd_hi.to_reg()); + let rn = machreg_to_gpr(*rn); + let rm = machreg_to_gpr(*rm); + put_u32(sink, enc_dsp_long(op.template(), rd_lo, rd_hi, rn, rm)); + } + Inst::Sel { rd, rn, rm } => { + let rd = machreg_to_gpr(rd.to_reg()); + let rn = machreg_to_gpr(*rn); + let rm = machreg_to_gpr(*rm); + put_u32(sink, enc_rd_rn_rm(0x0680_0fb0, rd, rn, rm)); + } + Inst::Pkh { op, rd, rn, rm } => { + let rd = machreg_to_gpr(rd.to_reg()); + let rn = machreg_to_gpr(*rn); + let rm = machreg_to_gpr(*rm); + put_u32(sink, enc_rd_rn_rm(op.template(), rd, rn, rm)); + } + Inst::Rrx { rd, rm } => { + let rd = machreg_to_gpr(rd.to_reg()); + let rm = machreg_to_gpr(*rm); + put_u32(sink, enc_rrx(rd, rm)); + } + Inst::Udf { code } => { + sink.add_trap(*code); + put_u32(sink, enc_udf()); + } + Inst::TrapIf { cond, code } => { + // Branch over the trap when the condition does not hold. + let skip = sink.get_label(); + sink.use_label_at_offset(sink.cur_offset(), skip, LabelUse::Branch26); + put_u32(sink, enc_bcond(cond.invert(), 0)); + sink.add_trap(*code); + put_u32(sink, enc_udf()); + sink.bind_label(skip, state.ctrl_plane_mut()); + } + Inst::LdmStm { + load, + rn, + writeback, + reg_list, + } => { + let rn = machreg_to_gpr(*rn); + put_u32(sink, enc_ldm_stm(*load, rn, *writeback, *reg_list)); + } + Inst::LoadEx { + acquire, + size, + rt, + rn, + } => { + let rt = machreg_to_gpr(rt.to_reg()); + let rn = machreg_to_gpr(*rn); + put_u32(sink, enc_load_ex(*acquire, *size, rt, rn)); + } + Inst::StoreEx { + release, + size, + rd, + rt, + rn, + } => { + let rd = machreg_to_gpr(rd.to_reg()); + let rt = machreg_to_gpr(*rt); + let rn = machreg_to_gpr(*rn); + put_u32(sink, enc_store_ex(*release, *size, rd, rt, rn)); + } + Inst::LoadAcq { size, rt, rn } => { + let rt = machreg_to_gpr(rt.to_reg()); + let rn = machreg_to_gpr(*rn); + put_u32(sink, enc_lda(*size, rt, rn)); + } + Inst::StoreRel { size, rt, rn } => { + let rt = machreg_to_gpr(*rt); + let rn = machreg_to_gpr(*rn); + put_u32(sink, enc_stl(*size, rt, rn)); + } + Inst::Barrier { op } => put_u32(sink, op.encoding()), + Inst::AtomicRmw { + op, + size, + rd, + addr, + operand, + tmp1, + tmp2, + } => emit_atomic_rmw( + sink, + state, + *op, + *size, + machreg_to_gpr(rd.to_reg()), + machreg_to_gpr(*addr), + machreg_to_gpr(*operand), + machreg_to_gpr(tmp1.to_reg()), + machreg_to_gpr(tmp2.to_reg()), + ), + Inst::AtomicCas { + size, + rd, + addr, + expected, + new, + tmp, + } => emit_atomic_cas( + sink, + state, + *size, + machreg_to_gpr(rd.to_reg()), + machreg_to_gpr(*addr), + machreg_to_gpr(*expected), + machreg_to_gpr(*new), + machreg_to_gpr(tmp.to_reg()), + ), + Inst::AtomicRmw64 { + op, + rd_lo, + rd_hi, + addr, + operand_lo, + operand_hi, + tmp_new_lo, + tmp_new_hi, + tmp_res, + } => emit_atomic_rmw64( + sink, + state, + *op, + machreg_to_gpr(rd_lo.to_reg()), + machreg_to_gpr(rd_hi.to_reg()), + machreg_to_gpr(*addr), + machreg_to_gpr(*operand_lo), + machreg_to_gpr(*operand_hi), + machreg_to_gpr(tmp_new_lo.to_reg()), + machreg_to_gpr(tmp_new_hi.to_reg()), + machreg_to_gpr(tmp_res.to_reg()), + ), + Inst::AtomicCas64 { + rd_lo, + rd_hi, + addr, + expected_lo, + expected_hi, + new_lo, + new_hi, + tmp_res, + } => emit_atomic_cas64( + sink, + state, + machreg_to_gpr(rd_lo.to_reg()), + machreg_to_gpr(rd_hi.to_reg()), + machreg_to_gpr(*addr), + machreg_to_gpr(*expected_lo), + machreg_to_gpr(*expected_hi), + machreg_to_gpr(*new_lo), + machreg_to_gpr(*new_hi), + machreg_to_gpr(tmp_res.to_reg()), + ), + Inst::AtomicLoad64 { rd_lo, rd_hi, addr } => { + let rd_lo = machreg_to_gpr(rd_lo.to_reg()); + debug_assert_eq!(machreg_to_gpr(rd_hi.to_reg()), rd_lo + 1); + let addr = machreg_to_gpr(*addr); + // ldaexd rd_lo, rd_hi, [addr]; clrex + put_u32(sink, enc_load_ex(true, AtomicSize::DWord, rd_lo, addr)); + put_u32(sink, BarrierOp::Clrex.encoding()); + } + Inst::AtomicStore64 { + addr, + src_lo, + src_hi, + scratch_lo, + scratch_hi, + tmp_res, + } => emit_atomic_store64( + sink, + state, + machreg_to_gpr(*addr), + machreg_to_gpr(*src_lo), + machreg_to_gpr(*src_hi), + machreg_to_gpr(scratch_lo.to_reg()), + machreg_to_gpr(scratch_hi.to_reg()), + machreg_to_gpr(tmp_res.to_reg()), + ), + + Inst::FpuRRR { + op, + size, + rd, + rn, + rm, + } => { + let rd = machreg_to_vfp(rd.to_reg()); + let rn = machreg_to_vfp(*rn); + let rm = machreg_to_vfp(*rm); + put_u32(sink, enc_fpu_rrr(*op, *size, rd, rn, rm)); + } + Inst::FpuRR { op, size, rd, rm } => { + let rd = machreg_to_vfp(rd.to_reg()); + let rm = machreg_to_vfp(*rm); + put_u32(sink, enc_fpu_rr(*op, *size, rd, rm)); + } + Inst::FpuLoad { size, rd, mem } => { + let rd = machreg_to_vfp(rd.to_reg()); + let (base, offset) = resolve_fpu_amode(mem, state); + put_u32(sink, enc_vldr_vstr(true, *size, rd, base, offset)); + } + Inst::FpuStore { size, rt, mem } => { + let rt = machreg_to_vfp(*rt); + let (base, offset) = resolve_fpu_amode(mem, state); + put_u32(sink, enc_vldr_vstr(false, *size, rt, base, offset)); + } + Inst::MovToFpu32 { rd, rt } => { + let rd = machreg_to_vfp(rd.to_reg()); + let rt = machreg_to_gpr(*rt); + put_u32(sink, enc_vmov_gpr_s(rd, rt)); + } + Inst::MovToFpu64 { rd, rt_lo, rt_hi } => { + let rd = machreg_to_vfp(rd.to_reg()); + let rt_lo = machreg_to_gpr(*rt_lo); + let rt_hi = machreg_to_gpr(*rt_hi); + put_u32(sink, enc_vmov_gpr_d(rd, rt_lo, rt_hi)); + } + Inst::MovFromFpu32 { rt, rm } => { + let rt = machreg_to_gpr(rt.to_reg()); + let rm = machreg_to_vfp(*rm); + put_u32(sink, enc_vmov_s_to_gpr(rt, rm)); + } + Inst::MovFromFpu64 { rt_lo, rt_hi, rm } => { + let rt_lo = machreg_to_gpr(rt_lo.to_reg()); + let rt_hi = machreg_to_gpr(rt_hi.to_reg()); + let rm = machreg_to_vfp(*rm); + put_u32(sink, enc_vmov_d_to_gpr(rt_lo, rt_hi, rm)); + } + Inst::VcmpMrs { size, rn, rm } => { + let rn = machreg_to_vfp(*rn); + let rm = machreg_to_vfp(*rm); + put_u32(sink, enc_vcmp(*size, rn, rm)); + put_u32(sink, VMRS_APSR_NZCV); + } + Inst::VcvtFF { to_f64, rd, rm } => { + let rd = machreg_to_vfp(rd.to_reg()); + let rm = machreg_to_vfp(*rm); + put_u32(sink, enc_vcvt_ff(*to_f64, rd, rm)); + } + Inst::VcvtToInt { + signed, + size, + rd, + rm, + } => { + let rd = machreg_to_vfp(rd.to_reg()); + let rm = machreg_to_vfp(*rm); + put_u32(sink, enc_vcvt_to_int(*signed, *size, rd, rm)); + } + Inst::VcvtFromInt { + signed, + size, + rd, + rm, + } => { + let rd = machreg_to_vfp(rd.to_reg()); + let rm = machreg_to_vfp(*rm); + put_u32(sink, enc_vcvt_from_int(*signed, *size, rd, rm)); + } + Inst::FpuCSel { cond, rd, rn, rm } => { + let rd = machreg_to_vfp(rd.to_reg()); + let rn = machreg_to_vfp(*rn); + let rm = machreg_to_vfp(*rm); + // Default to `if_false`, then conditionally overwrite. + put_u32(sink, enc_fpu_rr(FpuOp2::Vmov, FpuSize::F64, rd, rm)); + put_u32(sink, enc_vmov_f64_cond(*cond, rd, rn)); + } + Inst::FpuMinMax { + max, + size, + rd, + rn, + rm, + } => { + let rd = machreg_to_vfp(rd.to_reg()); + let rn = machreg_to_vfp(*rn); + let rm = machreg_to_vfp(*rm); + put_u32(sink, enc_fpu_minmax(*max, *size, rd, rn, rm)); + } + Inst::CmpRR { op, rn, rm } => { + let rn = machreg_to_gpr(*rn); + let rm = machreg_to_gpr(*rm); + put_u32(sink, enc_dp_reg(op.opcode(), 1, 0, rn, rm)); + } + Inst::CmpRImm { op, rn, imm12 } => { + let rn = machreg_to_gpr(*rn); + put_u32(sink, enc_dp_imm(op.opcode(), 1, 0, rn, *imm12)); + } + + Inst::AdjustSp { amount } => put_u32(sink, enc_sp_adjust(*amount)), + + Inst::Load { rt, mem, kind } => { + let rt = machreg_to_gpr(rt.to_reg()); + emit_load_store(sink, rt, mem, state, LoadStore::Load(*kind)); + } + Inst::Store { rt, mem, kind } => { + let rt = machreg_to_gpr(*rt); + emit_load_store(sink, rt, mem, state, LoadStore::Store(*kind)); + } + + Inst::Push { reg_list } => put_u32(sink, enc_push(*reg_list)), + Inst::Pop { reg_list } => put_u32(sink, enc_pop(*reg_list)), + + Inst::Call { info } => { + sink.add_reloc(Reloc::Arm32Call, &info.dest, 0); + put_u32(sink, enc_bl(0)); + emit_call_epilogue(sink, emit_info, state, info); + } + Inst::CallInd { info } => { + let rm = machreg_to_gpr(info.dest); + put_u32(sink, enc_blx(rm)); + emit_call_epilogue(sink, emit_info, state, info); + } + Inst::LoadExtName { rd, name, offset } => { + let rd = machreg_to_gpr(rd.to_reg()); + // `ldr rd, [pc]` reads `pc` as this instruction's address + 8, + // i.e. the inline literal placed two slots ahead. + put_u32(sink, enc_ldr_str_imm(true, false, rd, 15, 0)); + // `b .+8` branches over the literal (its `pc` is already the + // instruction just past the `.word`, so the offset is 0). + put_u32(sink, enc_b(0)); + // The literal: the symbol's absolute address plus `offset`, + // supplied by the linker via an `Abs4` (32-bit absolute) reloc. + sink.add_reloc(Reloc::Abs4, name.as_ref(), *offset); + put_u32(sink, 0); + } + + Inst::Jump { dest } => { + sink.use_label_at_offset(sink.cur_offset(), *dest, LabelUse::Branch26); + sink.add_uncond_branch(sink.cur_offset(), sink.cur_offset() + 4, *dest); + put_u32(sink, enc_b(0)); + } + Inst::BrTable { + index, + tmp, + targets, + } => { + let (default, entries) = targets.split_first().unwrap(); + let index = machreg_to_gpr(*index); + let n = entries.len() as u32; + + // Bounds check: branch to the default block if `index >=u n`. + if let Some(imm12) = encode_rotated_imm(n) { + put_u32(sink, enc_dp_imm(CmpOp::Cmp.opcode(), 1, 0, index, imm12)); + } else { + let tmp = machreg_to_gpr(tmp.to_reg()); + put_u32(sink, enc_movw(tmp, n & 0xffff)); + if n >> 16 != 0 { + put_u32(sink, enc_movt(tmp, n >> 16)); + } + put_u32(sink, enc_dp_reg(CmpOp::Cmp.opcode(), 1, 0, index, tmp)); + } + sink.use_label_at_offset(sink.cur_offset(), *default, LabelUse::Branch26); + put_u32(sink, enc_bcond(Cond::Hs, 0)); + + // Jump into the inline table: `pc` reads as this instruction's + // address + 8, so the table begins one slot after the filler. + put_u32(sink, 0xe08f_f100 | index); // add pc, pc, index, lsl #2 + put_u32(sink, COND_AL | 0x0320_f000); // nop (filler) + for target in entries { + sink.use_label_at_offset(sink.cur_offset(), *target, LabelUse::Branch26); + put_u32(sink, enc_b(0)); + } + } + Inst::CondBr { + cond, + taken, + not_taken, + } => { + // Conditional branch to `taken`. + let inverted = enc_bcond(cond.invert(), 0).to_le_bytes(); + sink.use_label_at_offset(sink.cur_offset(), *taken, LabelUse::Branch26); + sink.add_cond_branch(sink.cur_offset(), sink.cur_offset() + 4, *taken, &inverted); + put_u32(sink, enc_bcond(*cond, 0)); + // Unconditional branch to `not_taken`. + sink.use_label_at_offset(sink.cur_offset(), *not_taken, LabelUse::Branch26); + sink.add_uncond_branch(sink.cur_offset(), sink.cur_offset() + 4, *not_taken); + put_u32(sink, enc_b(0)); + } + + Inst::Args { .. } | Inst::Rets { .. } => { + // Pseudo-instructions: no machine code. + } + } + + // `BrTable` and calls emit variable-length sequences, so they are + // exempt from the per-instruction worst-case-size check. + let variable_length = matches!( + self, + Inst::BrTable { .. } + | Inst::Call { .. } + | Inst::CallInd { .. } + | Inst::AtomicRmw { .. } + | Inst::AtomicCas { .. } + | Inst::AtomicRmw64 { .. } + | Inst::AtomicCas64 { .. } + | Inst::AtomicStore64 { .. } + | Inst::LoadExtName { .. } + ); + let end = sink.cur_offset(); + debug_assert!( + variable_length || (end - start) <= Inst::worst_case_size(), + "instruction {self:?} exceeded worst-case size" + ); + } + + fn pretty_print_inst(&self, state: &mut Self::State) -> String { + self.print_with_state(state) + } +} + +/// Resolve an `AMode` for a VFP access, which only supports an immediate +/// offset. Returns the base GPR encoding and the byte offset. +fn resolve_fpu_amode(mem: &AMode, state: &EmitState) -> (u32, i32) { + match mem.resolve(state) { + ResolvedAMode::Imm { base, offset } => (machreg_to_gpr(base), offset), + ResolvedAMode::Reg { .. } => { + unimplemented!("arm32: register-offset VFP addressing is not supported") + } + } +} + +/// Emit the LDAEX/op/STLEX retry loop for an atomic read-modify-write. +/// +/// The load zero-extends a sub-word value into `rd`; the store truncates. For +/// add/sub/and/or/xor/nand/xchg the low byte/halfword of the arithmetic is +/// correct regardless of the operand's upper bits, so no extension is needed. +/// The min/max comparisons do extend both inputs to the access width. +fn emit_atomic_rmw( + sink: &mut MachBuffer, + state: &mut EmitState, + op: AtomicRmwOp, + size: AtomicSize, + rd: u32, + addr: u32, + operand: u32, + tmp1: u32, + tmp2: u32, +) { + let loop_lbl = sink.get_label(); + sink.bind_label(loop_lbl, state.ctrl_plane_mut()); + put_u32(sink, enc_load_ex(true, size, rd, addr)); // ldaex{,b,h} rd, [addr] + + // Compute the new value into tmp1. + let alu = |sink: &mut MachBuffer, op: ALUOp| { + put_u32(sink, enc_dp_reg(op.opcode(), 0, tmp1, rd, operand)); + }; + match op { + AtomicRmwOp::Add => alu(sink, ALUOp::Add), + AtomicRmwOp::Sub => alu(sink, ALUOp::Sub), + AtomicRmwOp::And => alu(sink, ALUOp::And), + AtomicRmwOp::Or => alu(sink, ALUOp::Orr), + AtomicRmwOp::Xor => alu(sink, ALUOp::Eor), + AtomicRmwOp::Nand => { + alu(sink, ALUOp::And); + put_u32(sink, enc_dp_reg(0b1111, 0, tmp1, 0, tmp1)); // mvn tmp1, tmp1 + } + AtomicRmwOp::Xchg => { + put_u32(sink, enc_dp_reg(0b1101, 0, tmp1, 0, operand)); // mov tmp1, operand + } + AtomicRmwOp::Umin | AtomicRmwOp::Umax | AtomicRmwOp::Smin | AtomicRmwOp::Smax => { + let signed = matches!(op, AtomicRmwOp::Smin | AtomicRmwOp::Smax); + let cond = match op { + AtomicRmwOp::Umin => Cond::Lo, + AtomicRmwOp::Umax => Cond::Hi, + AtomicRmwOp::Smin => Cond::Lt, + _ => Cond::Gt, + }; + if size.is_subword() { + // Extend both inputs to 32 bits so the comparison is correct, + // using tmp1/tmp2 as scratch before selecting the result. + let ext = size.extend_op(signed).template(); + put_u32(sink, enc_op_rd_rm(ext, tmp1, rd)); // ext tmp1, rd + put_u32(sink, enc_op_rd_rm(ext, tmp2, operand)); // ext tmp2, operand + put_u32(sink, enc_dp_reg(CmpOp::Cmp.opcode(), 1, 0, tmp1, tmp2)); + put_u32(sink, enc_dp_reg(0b1101, 0, tmp1, 0, operand)); // mov tmp1, operand + put_u32(sink, enc_mov_cond(cond, tmp1, rd)); // tmp1 = rd if cond + } else { + // tmp1 = operand; cmp rd, operand; tmp1 = rd if `cond`. + put_u32(sink, enc_dp_reg(0b1101, 0, tmp1, 0, operand)); + put_u32(sink, enc_dp_reg(CmpOp::Cmp.opcode(), 1, 0, rd, operand)); + put_u32(sink, enc_mov_cond(cond, tmp1, rd)); + } + } + } + + put_u32(sink, enc_store_ex(true, size, tmp2, tmp1, addr)); // stlex{,b,h} tmp2, tmp1, [addr] + put_u32(sink, enc_dp_imm(CmpOp::Cmp.opcode(), 1, 0, tmp2, 0)); // cmp tmp2, #0 + sink.use_label_at_offset(sink.cur_offset(), loop_lbl, LabelUse::Branch26); + put_u32(sink, enc_bcond(Cond::Ne, 0)); // bne loop +} + +/// Emit the LDAEX/compare/STLEX retry loop for an atomic compare-and-swap. +fn emit_atomic_cas( + sink: &mut MachBuffer, + state: &mut EmitState, + size: AtomicSize, + rd: u32, + addr: u32, + expected: u32, + new: u32, + tmp: u32, +) { + let loop_lbl = sink.get_label(); + let done_lbl = sink.get_label(); + sink.bind_label(loop_lbl, state.ctrl_plane_mut()); + put_u32(sink, enc_load_ex(true, size, rd, addr)); // ldaex{,b,h} rd, [addr] + // `rd` is zero-extended; compare it against a zero-extended `expected` so + // only the accessed byte/halfword participates. + if size.is_subword() { + let ext = size.extend_op(false).template(); + put_u32(sink, enc_op_rd_rm(ext, tmp, expected)); // uxt tmp, expected + put_u32(sink, enc_dp_reg(CmpOp::Cmp.opcode(), 1, 0, rd, tmp)); // cmp rd, tmp + } else { + put_u32(sink, enc_dp_reg(CmpOp::Cmp.opcode(), 1, 0, rd, expected)); // cmp rd, expected + } + sink.use_label_at_offset(sink.cur_offset(), done_lbl, LabelUse::Branch26); + put_u32(sink, enc_bcond(Cond::Ne, 0)); // bne done + put_u32(sink, enc_store_ex(true, size, tmp, new, addr)); // stlex{,b,h} tmp, new, [addr] + put_u32(sink, enc_dp_imm(CmpOp::Cmp.opcode(), 1, 0, tmp, 0)); // cmp tmp, #0 + sink.use_label_at_offset(sink.cur_offset(), loop_lbl, LabelUse::Branch26); + put_u32(sink, enc_bcond(Cond::Ne, 0)); // bne loop + sink.bind_label(done_lbl, state.ctrl_plane_mut()); +} + +/// Emit the LDAEXD/op/STLEXD retry loop for a 64-bit atomic read-modify-write. +/// The old value is loaded into (rd_lo, rd_hi) and the new value computed into +/// (nl, nh) before the paired store-exclusive. +fn emit_atomic_rmw64( + sink: &mut MachBuffer, + state: &mut EmitState, + op: AtomicRmwOp, + rd_lo: u32, + rd_hi: u32, + addr: u32, + op_lo: u32, + op_hi: u32, + nl: u32, + nh: u32, + res: u32, +) { + debug_assert_eq!(rd_hi, rd_lo + 1, "ldaexd needs a consecutive pair"); + debug_assert_eq!(nh, nl + 1, "stlexd needs a consecutive pair"); + const MOV: u32 = 0b1101; + const MVN: u32 = 0b1111; + let loop_lbl = sink.get_label(); + sink.bind_label(loop_lbl, state.ctrl_plane_mut()); + put_u32(sink, enc_load_ex(true, AtomicSize::DWord, rd_lo, addr)); // ldaexd rd_lo,rd_hi,[addr] + + // Compute the new 64-bit value into (nl, nh). + let dp = |sink: &mut MachBuffer, opcode: u32, s: u32, rd: u32, rn: u32, rm: u32| { + put_u32(sink, enc_dp_reg(opcode, s, rd, rn, rm)); + }; + match op { + AtomicRmwOp::Add => { + dp(sink, ALUOp::Add.opcode(), 1, nl, rd_lo, op_lo); // adds nl, rd_lo, op_lo + dp(sink, ALUOp::Adc.opcode(), 0, nh, rd_hi, op_hi); // adc nh, rd_hi, op_hi + } + AtomicRmwOp::Sub => { + dp(sink, ALUOp::Sub.opcode(), 1, nl, rd_lo, op_lo); // subs nl, rd_lo, op_lo + dp(sink, ALUOp::Sbc.opcode(), 0, nh, rd_hi, op_hi); // sbc nh, rd_hi, op_hi + } + AtomicRmwOp::And | AtomicRmwOp::Nand => { + dp(sink, ALUOp::And.opcode(), 0, nl, rd_lo, op_lo); + dp(sink, ALUOp::And.opcode(), 0, nh, rd_hi, op_hi); + if let AtomicRmwOp::Nand = op { + dp(sink, MVN, 0, nl, 0, nl); // mvn nl, nl + dp(sink, MVN, 0, nh, 0, nh); // mvn nh, nh + } + } + AtomicRmwOp::Or => { + dp(sink, ALUOp::Orr.opcode(), 0, nl, rd_lo, op_lo); + dp(sink, ALUOp::Orr.opcode(), 0, nh, rd_hi, op_hi); + } + AtomicRmwOp::Xor => { + dp(sink, ALUOp::Eor.opcode(), 0, nl, rd_lo, op_lo); + dp(sink, ALUOp::Eor.opcode(), 0, nh, rd_hi, op_hi); + } + AtomicRmwOp::Xchg => { + dp(sink, MOV, 0, nl, 0, op_lo); // mov nl, op_lo + dp(sink, MOV, 0, nh, 0, op_hi); // mov nh, op_hi + } + AtomicRmwOp::Umin | AtomicRmwOp::Umax | AtomicRmwOp::Smin | AtomicRmwOp::Smax => { + // Set flags for `old - operand` (64-bit) via subs/sbcs, using + // (nl, nh) as scratch. The chosen condition is Z-independent (the + // sbcs Z flag reflects only the high word), so equality never + // affects the result — which is correct, since min/max of equal + // values is that value either way. + let cond = match op { + AtomicRmwOp::Umin => Cond::Lo, // old Cond::Hs, // old >=u operand + AtomicRmwOp::Smin => Cond::Lt, // old Cond::Ge, // old >=s operand + }; + dp(sink, ALUOp::Sub.opcode(), 1, nl, rd_lo, op_lo); // subs nl, rd_lo, op_lo + dp(sink, ALUOp::Sbc.opcode(), 1, nh, rd_hi, op_hi); // sbcs nh, rd_hi, op_hi + dp(sink, MOV, 0, nl, 0, op_lo); // nl = operand (default) + dp(sink, MOV, 0, nh, 0, op_hi); + put_u32(sink, enc_mov_cond(cond, nl, rd_lo)); // nl = rd_lo if cond + put_u32(sink, enc_mov_cond(cond, nh, rd_hi)); // nh = rd_hi if cond + } + } + + put_u32(sink, enc_store_ex(true, AtomicSize::DWord, res, nl, addr)); // stlexd res, nl,nh,[addr] + put_u32(sink, enc_dp_imm(CmpOp::Cmp.opcode(), 1, 0, res, 0)); // cmp res, #0 + sink.use_label_at_offset(sink.cur_offset(), loop_lbl, LabelUse::Branch26); + put_u32(sink, enc_bcond(Cond::Ne, 0)); // bne loop +} + +/// Emit the LDAEXD/compare/STLEXD retry loop for a 64-bit compare-and-swap. +fn emit_atomic_cas64( + sink: &mut MachBuffer, + state: &mut EmitState, + rd_lo: u32, + rd_hi: u32, + addr: u32, + exp_lo: u32, + exp_hi: u32, + new_lo: u32, + new_hi: u32, + res: u32, +) { + debug_assert_eq!(rd_hi, rd_lo + 1, "ldaexd needs a consecutive pair"); + debug_assert_eq!(new_hi, new_lo + 1, "stlexd needs a consecutive pair"); + let loop_lbl = sink.get_label(); + let done_lbl = sink.get_label(); + sink.bind_label(loop_lbl, state.ctrl_plane_mut()); + put_u32(sink, enc_load_ex(true, AtomicSize::DWord, rd_lo, addr)); // ldaexd rd_lo,rd_hi,[addr] + // Both halves must match `expected`. + put_u32(sink, enc_dp_reg(CmpOp::Cmp.opcode(), 1, 0, rd_lo, exp_lo)); // cmp rd_lo, exp_lo + sink.use_label_at_offset(sink.cur_offset(), done_lbl, LabelUse::Branch26); + put_u32(sink, enc_bcond(Cond::Ne, 0)); // bne done + put_u32(sink, enc_dp_reg(CmpOp::Cmp.opcode(), 1, 0, rd_hi, exp_hi)); // cmp rd_hi, exp_hi + sink.use_label_at_offset(sink.cur_offset(), done_lbl, LabelUse::Branch26); + put_u32(sink, enc_bcond(Cond::Ne, 0)); // bne done + put_u32( + sink, + enc_store_ex(true, AtomicSize::DWord, res, new_lo, addr), + ); // stlexd res,new_lo,new_hi + put_u32(sink, enc_dp_imm(CmpOp::Cmp.opcode(), 1, 0, res, 0)); // cmp res, #0 + sink.use_label_at_offset(sink.cur_offset(), loop_lbl, LabelUse::Branch26); + put_u32(sink, enc_bcond(Cond::Ne, 0)); // bne loop + sink.bind_label(done_lbl, state.ctrl_plane_mut()); +} + +/// Emit the LDAEXD/STLEXD retry loop for a 64-bit atomic store. The load is +/// required to arm the exclusive monitor; its result is discarded. +fn emit_atomic_store64( + sink: &mut MachBuffer, + state: &mut EmitState, + addr: u32, + src_lo: u32, + src_hi: u32, + scratch_lo: u32, + scratch_hi: u32, + res: u32, +) { + debug_assert_eq!( + scratch_hi, + scratch_lo + 1, + "ldaexd needs a consecutive pair" + ); + debug_assert_eq!(src_hi, src_lo + 1, "stlexd needs a consecutive pair"); + let loop_lbl = sink.get_label(); + sink.bind_label(loop_lbl, state.ctrl_plane_mut()); + put_u32(sink, enc_load_ex(true, AtomicSize::DWord, scratch_lo, addr)); // ldaexd scratch,[addr] + put_u32( + sink, + enc_store_ex(true, AtomicSize::DWord, res, src_lo, addr), + ); // stlexd res, src,[addr] + put_u32(sink, enc_dp_imm(CmpOp::Cmp.opcode(), 1, 0, res, 0)); // cmp res, #0 + sink.use_label_at_offset(sink.cur_offset(), loop_lbl, LabelUse::Branch26); + put_u32(sink, enc_bcond(Cond::Ne, 0)); // bne loop +} + +enum LoadStore { + Load(LoadKind), + Store(StoreKind), +} + +fn emit_load_store( + sink: &mut MachBuffer, + rt: u32, + mem: &AMode, + state: &EmitState, + op: LoadStore, +) { + match mem.resolve(state) { + ResolvedAMode::Imm { base, offset } => { + let base = machreg_to_gpr(base); + let word = match op { + LoadStore::Load(LoadKind::Word) => enc_ldr_str_imm(true, false, rt, base, offset), + LoadStore::Load(LoadKind::UByte) => enc_ldr_str_imm(true, true, rt, base, offset), + LoadStore::Load(LoadKind::SByte) => enc_ldrh_strh_imm(true, 1, 0, rt, base, offset), + LoadStore::Load(LoadKind::UHalf) => enc_ldrh_strh_imm(true, 0, 1, rt, base, offset), + LoadStore::Load(LoadKind::SHalf) => enc_ldrh_strh_imm(true, 1, 1, rt, base, offset), + LoadStore::Store(StoreKind::Word) => { + enc_ldr_str_imm(false, false, rt, base, offset) + } + LoadStore::Store(StoreKind::Byte) => enc_ldr_str_imm(false, true, rt, base, offset), + LoadStore::Store(StoreKind::Half) => { + enc_ldrh_strh_imm(false, 0, 1, rt, base, offset) + } + }; + put_u32(sink, word); + } + ResolvedAMode::Reg { base, index } => { + let base = machreg_to_gpr(base); + let index = machreg_to_gpr(index); + let word = match op { + LoadStore::Load(LoadKind::Word) => enc_ldr_str_reg(true, false, rt, base, index), + LoadStore::Load(LoadKind::UByte) => enc_ldr_str_reg(true, true, rt, base, index), + LoadStore::Store(StoreKind::Word) => enc_ldr_str_reg(false, false, rt, base, index), + LoadStore::Store(StoreKind::Byte) => enc_ldr_str_reg(false, true, rt, base, index), + _ => unimplemented!("arm32: register-offset sub-word access not yet implemented"), + }; + put_u32(sink, word); + } + } +} diff --git a/cranelift/codegen/src/isa/arm32/inst/emit_tests.rs b/cranelift/codegen/src/isa/arm32/inst/emit_tests.rs new file mode 100644 index 000000000000..bec102adaac2 --- /dev/null +++ b/cranelift/codegen/src/isa/arm32/inst/emit_tests.rs @@ -0,0 +1,963 @@ +//! Encoding tests for arm32 instructions. + +use super::*; + +/// Emit a single instruction and return the encoded bytes. +fn encode(inst: Inst) -> Vec { + use crate::isa::arm32::settings as arm32_settings; + use crate::settings::{self, Configurable}; + + let mut b = settings::builder(); + b.set("enable_verifier", "false").unwrap(); + let flags = settings::Flags::new(b); + let isa_flags = arm32_settings::Flags::new(&flags, &arm32_settings::builder()); + let emit_info = EmitInfo::new(flags, isa_flags); + + let mut buffer = MachBuffer::new(); + let mut state = EmitState::default(); + inst.emit(&mut buffer, &emit_info, &mut state); + let mut ctrl_plane = Default::default(); + let buffer = buffer.finish(&Default::default(), &mut ctrl_plane); + buffer.data().to_vec() +} + +fn u32_le(inst: Inst) -> u32 { + let bytes = encode(inst); + assert_eq!(bytes.len(), 4, "expected a single 4-byte instruction"); + u32::from_le_bytes([bytes[0], bytes[1], bytes[2], bytes[3]]) +} + +fn rot(v: u32) -> u32 { + encode_rotated_imm(v).unwrap() +} + +#[test] +fn data_movement() { + assert_eq!(u32_le(Inst::Ret), 0xe12f_ff1e); // bx lr + assert_eq!(u32_le(Inst::Nop4), 0xe320_f000); + assert_eq!( + u32_le(Inst::MovReg { + rd: writable_xreg(0), + rm: xreg(1), + }), + 0xe1a0_0001 + ); + assert_eq!( + u32_le(Inst::Movw { + rd: writable_xreg(0), + imm16: 42, + }), + 0xe300_002a + ); + assert_eq!( + u32_le(Inst::MovImm { + rd: writable_xreg(0), + imm: 42, + }), + 0xe300_002a + ); + assert_eq!( + u32_le(Inst::MovRotImm { + rd: writable_xreg(0), + imm12: rot(255), + }), + 0xe3a0_00ff // mov r0, #255 + ); + assert_eq!( + u32_le(Inst::MvnRotImm { + rd: writable_xreg(0), + imm12: rot(0), + }), + 0xe3e0_0000 // mvn r0, #0 + ); +} + +#[test] +fn arithmetic() { + assert_eq!( + u32_le(Inst::AluRRR { + op: ALUOp::Add, + rd: writable_xreg(0), + rn: xreg(1), + rm: xreg(2), + }), + 0xe081_0002 // add r0, r1, r2 + ); + assert_eq!( + u32_le(Inst::AluRRImm { + op: ALUOp::Add, + rd: writable_xreg(0), + rn: xreg(1), + imm12: rot(1), + }), + 0xe281_0001 // add r0, r1, #1 + ); + assert_eq!( + u32_le(Inst::AluRRR { + op: ALUOp::Sub, + rd: writable_xreg(0), + rn: xreg(1), + rm: xreg(2), + }), + 0xe041_0002 // sub r0, r1, r2 + ); + assert_eq!( + u32_le(Inst::AluRRImm { + op: ALUOp::Rsb, + rd: writable_xreg(0), + rn: xreg(1), + imm12: rot(0), + }), + 0xe261_0000 // rsb r0, r1, #0 + ); +} + +#[test] +fn logical() { + let cases = [ + (ALUOp::And, 0xe001_0002u32), // and r0, r1, r2 + (ALUOp::Orr, 0xe181_0002), // orr r0, r1, r2 + (ALUOp::Eor, 0xe021_0002), // eor r0, r1, r2 + (ALUOp::Bic, 0xe1c1_0002), // bic r0, r1, r2 + ]; + for (op, want) in cases { + assert_eq!( + u32_le(Inst::AluRRR { + op, + rd: writable_xreg(0), + rn: xreg(1), + rm: xreg(2), + }), + want, + "{op:?}" + ); + } + assert_eq!( + u32_le(Inst::MvnReg { + rd: writable_xreg(0), + rm: xreg(1), + }), + 0xe1e0_0001 // mvn r0, r1 + ); +} + +#[test] +fn flag_setting() { + assert_eq!( + u32_le(Inst::AluRRRFlags { + op: ALUOp::Add, + rd: writable_xreg(0), + rn: xreg(1), + rm: xreg(2), + }), + 0xe091_0002 // adds r0, r1, r2 + ); + assert_eq!( + u32_le(Inst::AluRRRFlags { + op: ALUOp::Sbc, + rd: writable_xreg(0), + rn: xreg(1), + rm: xreg(2), + }), + 0xe0d1_0002 // sbcs r0, r1, r2 + ); + assert_eq!( + u32_le(Inst::CmpRR { + op: CmpOp::Tst, + rn: xreg(0), + rm: xreg(1), + }), + 0xe110_0001 // tst r0, r1 + ); + assert_eq!( + u32_le(Inst::CmpRR { + op: CmpOp::Teq, + rn: xreg(0), + rm: xreg(1), + }), + 0xe130_0001 // teq r0, r1 + ); +} + +#[test] +fn shifts() { + assert_eq!( + u32_le(Inst::ShiftImm { + op: ShiftOp::Lsl, + rd: writable_xreg(0), + rm: xreg(1), + amount: 2, + }), + 0xe1a0_0101 // lsl r0, r1, #2 + ); + assert_eq!( + u32_le(Inst::ShiftImm { + op: ShiftOp::Asr, + rd: writable_xreg(0), + rm: xreg(1), + amount: 3, + }), + 0xe1a0_01c1 // asr r0, r1, #3 + ); + assert_eq!( + u32_le(Inst::ShiftReg { + op: ShiftOp::Lsl, + rd: writable_xreg(0), + rm: xreg(1), + rs: xreg(2), + }), + 0xe1a0_0211 // lsl r0, r1, r2 + ); +} + +#[test] +fn multiplies() { + assert_eq!( + u32_le(Inst::Mul { + rd: writable_xreg(0), + rn: xreg(1), + rm: xreg(2), + }), + 0xe000_0291 // mul r0, r1, r2 + ); + assert_eq!( + u32_le(Inst::Mla { + rd: writable_xreg(0), + rn: xreg(1), + rm: xreg(2), + ra: xreg(3), + }), + 0xe020_3291 // mla r0, r1, r2, r3 + ); + assert_eq!( + u32_le(Inst::Umull { + rd_lo: writable_xreg(0), + rd_hi: writable_xreg(1), + rn: xreg(2), + rm: xreg(3), + }), + 0xe081_0392 // umull r0, r1, r2, r3 + ); + assert_eq!( + u32_le(Inst::Smull { + rd_lo: writable_xreg(0), + rd_hi: writable_xreg(1), + rn: xreg(2), + rm: xreg(3), + }), + 0xe0c1_0392 // smull r0, r1, r2, r3 + ); +} + +#[test] +fn bit_ops() { + let bit_cases = [ + (BitOp::Clz, 0xe16f_0f11u32), + (BitOp::Rev, 0xe6bf_0f31), + (BitOp::Rev16, 0xe6bf_0fb1), + (BitOp::Rbit, 0xe6ff_0f31), + ]; + for (op, want) in bit_cases { + assert_eq!( + u32_le(Inst::BitRR { + op, + rd: writable_xreg(0), + rm: xreg(1), + }), + want, + "{op:?}" + ); + } + let ext_cases = [ + (ExtOp::Sxtb, 0xe6af_0071u32), + (ExtOp::Sxth, 0xe6bf_0071), + (ExtOp::Uxtb, 0xe6ef_0071), + (ExtOp::Uxth, 0xe6ff_0071), + ]; + for (op, want) in ext_cases { + assert_eq!( + u32_le(Inst::ExtRR { + op, + rd: writable_xreg(0), + rm: xreg(1), + }), + want, + "{op:?}" + ); + } +} + +#[test] +fn divides() { + assert_eq!( + u32_le(Inst::SDiv { + rd: writable_xreg(0), + rn: xreg(1), + rm: xreg(2), + }), + 0xe710_f211 // sdiv r0, r1, r2 + ); + assert_eq!( + u32_le(Inst::UDiv { + rd: writable_xreg(0), + rn: xreg(1), + rm: xreg(2), + }), + 0xe730_f211 // udiv r0, r1, r2 + ); +} + +#[test] +fn fused_multiply() { + assert_eq!( + u32_le(Inst::Mls { + rd: writable_xreg(0), + rn: xreg(1), + rm: xreg(2), + ra: xreg(3), + }), + 0xe060_3291 // mls r0, r1, r2, r3 + ); + assert_eq!( + u32_le(Inst::Umlal { + rd_lo: writable_xreg(0), + rd_hi: writable_xreg(1), + rn: xreg(2), + rm: xreg(3), + }), + 0xe0a1_0392 // umlal r0, r1, r2, r3 + ); + assert_eq!( + u32_le(Inst::Smlal { + rd_lo: writable_xreg(0), + rd_hi: writable_xreg(1), + rn: xreg(2), + rm: xreg(3), + }), + 0xe0e1_0392 // smlal r0, r1, r2, r3 + ); +} + +#[test] +fn conditional_select() { + // csel ne r0, r1, r2 => mov r0, r2 ; movne r0, r1 + let bytes = encode(Inst::CSel { + cond: Cond::Ne, + rd: writable_xreg(0), + rn: xreg(1), + rm: xreg(2), + }); + assert_eq!(bytes.len(), 8); + let w0 = u32::from_le_bytes([bytes[0], bytes[1], bytes[2], bytes[3]]); + let w1 = u32::from_le_bytes([bytes[4], bytes[5], bytes[6], bytes[7]]); + assert_eq!(w0, 0xe1a0_0002); // mov r0, r2 + assert_eq!(w1, 0x11a0_0001); // movne r0, r1 +} + +#[test] +fn bitfield() { + assert_eq!( + u32_le(Inst::Bfc { + rd: writable_xreg(0), + lsb: 4, + width: 8, + }), + 0xe7cb_021f // bfc r0, #4, #8 + ); + assert_eq!( + u32_le(Inst::Bfi { + rd: writable_xreg(0), + rn: xreg(1), + lsb: 4, + width: 8, + }), + 0xe7cb_0211 // bfi r0, r1, #4, #8 + ); + assert_eq!( + u32_le(Inst::Bfx { + op: BfxOp::Sbfx, + rd: writable_xreg(0), + rn: xreg(1), + lsb: 4, + width: 8, + }), + 0xe7a7_0251 // sbfx r0, r1, #4, #8 + ); + assert_eq!( + u32_le(Inst::Bfx { + op: BfxOp::Ubfx, + rd: writable_xreg(0), + rn: xreg(1), + lsb: 4, + width: 8, + }), + 0xe7e7_0251 // ubfx r0, r1, #4, #8 + ); +} + +#[test] +fn saturating() { + let q_cases = [ + (QAluOp::Qadd, 0xe102_0051u32), + (QAluOp::Qsub, 0xe122_0051), + (QAluOp::Qdadd, 0xe142_0051), + (QAluOp::Qdsub, 0xe162_0051), + ]; + for (op, want) in q_cases { + // qadd r0, r1, r2 => Rd=0, Rm=1, Rn=2 + assert_eq!( + u32_le(Inst::QAlu { + op, + rd: writable_xreg(0), + rm: xreg(1), + rn: xreg(2), + }), + want, + "{op:?}" + ); + } + assert_eq!( + u32_le(Inst::Sat { + op: SatOp::Ssat, + rd: writable_xreg(0), + sat_bits: 8, + rm: xreg(1), + }), + 0xe6a7_0011 // ssat r0, #8, r1 + ); + assert_eq!( + u32_le(Inst::Sat { + op: SatOp::Usat, + rd: writable_xreg(0), + sat_bits: 8, + rm: xreg(1), + }), + 0xe6e8_0011 // usat r0, #8, r1 + ); +} + +#[test] +fn misc_alu() { + assert_eq!( + u32_le(Inst::Sel { + rd: writable_xreg(0), + rn: xreg(1), + rm: xreg(2), + }), + 0xe681_0fb2 // sel r0, r1, r2 + ); + assert_eq!( + u32_le(Inst::Pkh { + op: PkhOp::Pkhbt, + rd: writable_xreg(0), + rn: xreg(1), + rm: xreg(2), + }), + 0xe681_0012 // pkhbt r0, r1, r2 + ); + assert_eq!( + u32_le(Inst::Pkh { + op: PkhOp::Pkhtb, + rd: writable_xreg(0), + rn: xreg(1), + rm: xreg(2), + }), + 0xe681_0052 // pkhtb r0, r1, r2 + ); + assert_eq!( + u32_le(Inst::Rrx { + rd: writable_xreg(0), + rm: xreg(1), + }), + 0xe1a0_0061 // rrx r0, r1 + ); + assert_eq!( + u32_le(Inst::BitRR { + op: BitOp::Revsh, + rd: writable_xreg(0), + rm: xreg(1), + }), + 0xe6ff_0fb1 // revsh r0, r1 + ); + assert_eq!( + u32_le(Inst::Udf { + code: crate::ir::TrapCode::STACK_OVERFLOW, + }), + 0xe7f0_00f0 // udf #0 + ); +} + +#[test] +fn extend_variants() { + assert_eq!( + u32_le(Inst::ExtRR { + op: ExtOp::Sxtb16, + rd: writable_xreg(0), + rm: xreg(1), + }), + 0xe68f_0071 // sxtb16 r0, r1 + ); + assert_eq!( + u32_le(Inst::ExtRR { + op: ExtOp::Uxtb16, + rd: writable_xreg(0), + rm: xreg(1), + }), + 0xe6cf_0071 // uxtb16 r0, r1 + ); + let add_cases = [ + (ExtAddOp::Sxtab, 0xe6a1_0072u32), + (ExtAddOp::Sxtah, 0xe6b1_0072), + (ExtAddOp::Sxtab16, 0xe681_0072), + (ExtAddOp::Uxtab, 0xe6e1_0072), + (ExtAddOp::Uxtah, 0xe6f1_0072), + (ExtAddOp::Uxtab16, 0xe6c1_0072), + ]; + for (op, want) in add_cases { + assert_eq!( + u32_le(Inst::ExtAdd { + op, + rd: writable_xreg(0), + rn: xreg(1), + rm: xreg(2), + }), + want, + "{op:?}" + ); + } +} + +#[test] +fn parallel_add_sub() { + let par = |op| { + u32_le(Inst::ParAlu { + op, + rd: writable_xreg(0), + rn: xreg(1), + rm: xreg(2), + }) + }; + // Hand-verified reference encodings (`op r0, r1, r2`). + assert_eq!(par(ParAluOp::Sadd8), 0xe611_0f92); + assert_eq!(par(ParAluOp::Ssub8), 0xe611_0ff2); + assert_eq!(par(ParAluOp::Uadd8), 0xe651_0f92); + assert_eq!(par(ParAluOp::Uadd16), 0xe651_0f12); + assert_eq!(par(ParAluOp::Uhadd16), 0xe671_0f12); + + // All 36 members must encode to distinct, well-formed words that share the + // parallel add/sub encoding skeleton. + let all = [ + ParAluOp::Sadd8, + ParAluOp::Sadd16, + ParAluOp::Ssub8, + ParAluOp::Ssub16, + ParAluOp::Sasx, + ParAluOp::Ssax, + ParAluOp::Qadd8, + ParAluOp::Qadd16, + ParAluOp::Qsub8, + ParAluOp::Qsub16, + ParAluOp::Qasx, + ParAluOp::Qsax, + ParAluOp::Shadd8, + ParAluOp::Shadd16, + ParAluOp::Shsub8, + ParAluOp::Shsub16, + ParAluOp::Shasx, + ParAluOp::Shsax, + ParAluOp::Uadd8, + ParAluOp::Uadd16, + ParAluOp::Usub8, + ParAluOp::Usub16, + ParAluOp::Uasx, + ParAluOp::Usax, + ParAluOp::Uqadd8, + ParAluOp::Uqadd16, + ParAluOp::Uqsub8, + ParAluOp::Uqsub16, + ParAluOp::Uqasx, + ParAluOp::Uqsax, + ParAluOp::Uhadd8, + ParAluOp::Uhadd16, + ParAluOp::Uhsub8, + ParAluOp::Uhsub16, + ParAluOp::Uhasx, + ParAluOp::Uhsax, + ]; + let mut seen = alloc::collections::BTreeSet::new(); + for op in all { + let w = par(op); + // Fixed skeleton: cond=AL + [27:24]=0110, bit23=0, [11:8]=1111 & bit4=1, + // and the register fields Rn=1, Rd=0, Rm=2. + assert_eq!(w & 0xff80_0000, 0xe600_0000, "{op:?}"); + assert_eq!(w & 0x0000_0f10, 0x0000_0f10, "{op:?}"); + assert_eq!(w & 0x000f_f00f, 0x0001_0002, "{op:?}"); + assert!(seen.insert(w), "duplicate encoding for {op:?}"); + } + assert_eq!(seen.len(), 36); +} + +#[test] +fn dsp_multiplies() { + let m3 = |op| { + u32_le(Inst::DspMul3 { + op, + rd: writable_xreg(0), + rn: xreg(1), + rm: xreg(2), + }) + }; + // `op r0, r1, r2` + assert_eq!(m3(DspMul3Op::Smulbb), 0xe160_0281); + assert_eq!(m3(DspMul3Op::Smulbt), 0xe160_02c1); + assert_eq!(m3(DspMul3Op::Smultb), 0xe160_02a1); + assert_eq!(m3(DspMul3Op::Smultt), 0xe160_02e1); + assert_eq!(m3(DspMul3Op::Smulwb), 0xe120_02a1); + assert_eq!(m3(DspMul3Op::Smulwt), 0xe120_02e1); + assert_eq!(m3(DspMul3Op::Smmul), 0xe750_f211); + assert_eq!(m3(DspMul3Op::Smuad), 0xe700_f211); + assert_eq!(m3(DspMul3Op::Smusd), 0xe700_f251); + + let m4 = |op| { + u32_le(Inst::DspMul4 { + op, + rd: writable_xreg(0), + rn: xreg(1), + rm: xreg(2), + ra: xreg(3), + }) + }; + // `op r0, r1, r2, r3` + assert_eq!(m4(DspMul4Op::Smlabb), 0xe100_3281); + assert_eq!(m4(DspMul4Op::Smlabt), 0xe100_32c1); + assert_eq!(m4(DspMul4Op::Smlatb), 0xe100_32a1); + assert_eq!(m4(DspMul4Op::Smlatt), 0xe100_32e1); + assert_eq!(m4(DspMul4Op::Smlawb), 0xe120_3281); + assert_eq!(m4(DspMul4Op::Smlawt), 0xe120_32c1); + assert_eq!(m4(DspMul4Op::Smmla), 0xe750_3211); + assert_eq!(m4(DspMul4Op::Smmls), 0xe750_32d1); + assert_eq!(m4(DspMul4Op::Smlad), 0xe700_3211); + assert_eq!(m4(DspMul4Op::Smlsd), 0xe700_3251); + + let ml = |op| { + u32_le(Inst::DspMulL { + op, + rd_lo: writable_xreg(0), + rd_hi: writable_xreg(1), + rn: xreg(2), + rm: xreg(3), + }) + }; + // `op r0, r1, r2, r3` + assert_eq!(ml(DspMulLOp::Smlalbb), 0xe141_0382); + assert_eq!(ml(DspMulLOp::Smlalbt), 0xe141_03c2); + assert_eq!(ml(DspMulLOp::Smlaltb), 0xe141_03a2); + assert_eq!(ml(DspMulLOp::Smlaltt), 0xe141_03e2); + assert_eq!(ml(DspMulLOp::Smlald), 0xe741_0312); + assert_eq!(ml(DspMulLOp::Smlsld), 0xe741_0352); + assert_eq!(ml(DspMulLOp::Umaal), 0xe041_0392); +} + +#[test] +fn memory_multiple_and_exclusive() { + assert_eq!( + u32_le(Inst::LdmStm { + load: true, + rn: xreg(0), + writeback: true, + reg_list: (1 << 1) | (1 << 2), + }), + 0xe8b0_0006 // ldmia r0!, {r1, r2} + ); + assert_eq!( + u32_le(Inst::LdmStm { + load: false, + rn: xreg(0), + writeback: false, + reg_list: (1 << 1) | (1 << 2), + }), + 0xe880_0006 // stmia r0, {r1, r2} + ); + assert_eq!( + u32_le(Inst::LoadEx { + acquire: false, + size: AtomicSize::Word, + rt: writable_xreg(1), + rn: xreg(0), + }), + 0xe190_1f9f // ldrex r1, [r0] + ); + assert_eq!( + u32_le(Inst::LoadEx { + acquire: true, + size: AtomicSize::Word, + rt: writable_xreg(1), + rn: xreg(0), + }), + 0xe190_1e9f // ldaex r1, [r0] + ); + assert_eq!( + u32_le(Inst::LoadEx { + acquire: true, + size: AtomicSize::Byte, + rt: writable_xreg(1), + rn: xreg(0), + }), + 0xe1d0_1e9f // ldaexb r1, [r0] + ); + assert_eq!( + u32_le(Inst::LoadEx { + acquire: true, + size: AtomicSize::Half, + rt: writable_xreg(1), + rn: xreg(0), + }), + 0xe1f0_1e9f // ldaexh r1, [r0] + ); + assert_eq!( + u32_le(Inst::StoreEx { + release: false, + size: AtomicSize::Word, + rd: writable_xreg(0), + rt: xreg(1), + rn: xreg(2), + }), + 0xe182_0f91 // strex r0, r1, [r2] + ); + assert_eq!( + u32_le(Inst::StoreEx { + release: true, + size: AtomicSize::Word, + rd: writable_xreg(0), + rt: xreg(1), + rn: xreg(2), + }), + 0xe182_0e91 // stlex r0, r1, [r2] + ); + assert_eq!( + u32_le(Inst::StoreEx { + release: true, + size: AtomicSize::Byte, + rd: writable_xreg(0), + rt: xreg(1), + rn: xreg(2), + }), + 0xe1c2_0e91 // stlexb r0, r1, [r2] + ); + assert_eq!( + u32_le(Inst::StoreEx { + release: true, + size: AtomicSize::Half, + rd: writable_xreg(0), + rt: xreg(1), + rn: xreg(2), + }), + 0xe1e2_0e91 // stlexh r0, r1, [r2] + ); + assert_eq!( + u32_le(Inst::LoadAcq { + size: AtomicSize::Word, + rt: writable_xreg(1), + rn: xreg(0), + }), + 0xe190_1c9f // lda r1, [r0] + ); + assert_eq!( + u32_le(Inst::LoadAcq { + size: AtomicSize::Byte, + rt: writable_xreg(1), + rn: xreg(0), + }), + 0xe1d0_1c9f // ldab r1, [r0] + ); + assert_eq!( + u32_le(Inst::LoadAcq { + size: AtomicSize::Half, + rt: writable_xreg(1), + rn: xreg(0), + }), + 0xe1f0_1c9f // ldah r1, [r0] + ); + assert_eq!( + u32_le(Inst::StoreRel { + size: AtomicSize::Word, + rt: xreg(1), + rn: xreg(0), + }), + 0xe180_fc91 // stl r1, [r0] + ); + assert_eq!( + u32_le(Inst::StoreRel { + size: AtomicSize::Byte, + rt: xreg(1), + rn: xreg(0), + }), + 0xe1c0_fc91 // stlb r1, [r0] + ); + assert_eq!( + u32_le(Inst::StoreRel { + size: AtomicSize::Half, + rt: xreg(1), + rn: xreg(0), + }), + 0xe1e0_fc91 // stlh r1, [r0] + ); +} + +#[test] +fn barriers() { + assert_eq!(u32_le(Inst::Barrier { op: BarrierOp::Dmb }), 0xf57f_f05f); + assert_eq!(u32_le(Inst::Barrier { op: BarrierOp::Dsb }), 0xf57f_f04f); + assert_eq!(u32_le(Inst::Barrier { op: BarrierOp::Isb }), 0xf57f_f06f); + assert_eq!( + u32_le(Inst::Barrier { + op: BarrierOp::Clrex + }), + 0xf57f_f01f + ); +} + +#[test] +fn compares() { + assert_eq!( + u32_le(Inst::CmpRR { + op: CmpOp::Cmp, + rn: xreg(0), + rm: xreg(1), + }), + 0xe150_0001 // cmp r0, r1 + ); + assert_eq!( + u32_le(Inst::CmpRImm { + op: CmpOp::Cmp, + rn: xreg(0), + imm12: rot(0), + }), + 0xe350_0000 // cmp r0, #0 + ); +} + +#[test] +fn memory() { + assert_eq!( + u32_le(Inst::Load { + rt: writable_xreg(0), + mem: AMode::RegOffset { + rn: xreg(1), + offset: 4 + }, + kind: LoadKind::Word, + }), + 0xe591_0004 // ldr r0, [r1, #4] + ); + assert_eq!( + u32_le(Inst::Store { + rt: xreg(0), + mem: AMode::RegOffset { + rn: xreg(1), + offset: 4 + }, + kind: StoreKind::Word, + }), + 0xe581_0004 // str r0, [r1, #4] + ); + assert_eq!( + u32_le(Inst::Load { + rt: writable_xreg(0), + mem: AMode::RegOffset { + rn: xreg(1), + offset: 4 + }, + kind: LoadKind::UByte, + }), + 0xe5d1_0004 // ldrb r0, [r1, #4] + ); + assert_eq!( + u32_le(Inst::Store { + rt: xreg(0), + mem: AMode::RegOffset { + rn: xreg(1), + offset: 4 + }, + kind: StoreKind::Byte, + }), + 0xe5c1_0004 // strb r0, [r1, #4] + ); + assert_eq!( + u32_le(Inst::Load { + rt: writable_xreg(0), + mem: AMode::RegOffset { + rn: xreg(1), + offset: 4 + }, + kind: LoadKind::UHalf, + }), + 0xe1d1_00b4 // ldrh r0, [r1, #4] + ); + assert_eq!( + u32_le(Inst::Store { + rt: xreg(0), + mem: AMode::RegOffset { + rn: xreg(1), + offset: 4 + }, + kind: StoreKind::Half, + }), + 0xe1c1_00b4 // strh r0, [r1, #4] + ); + assert_eq!( + u32_le(Inst::Load { + rt: writable_xreg(0), + mem: AMode::RegOffset { + rn: xreg(1), + offset: 4 + }, + kind: LoadKind::SByte, + }), + 0xe1d1_00d4 // ldrsb r0, [r1, #4] + ); + assert_eq!( + u32_le(Inst::Load { + rt: writable_xreg(0), + mem: AMode::RegOffset { + rn: xreg(1), + offset: 4 + }, + kind: LoadKind::SHalf, + }), + 0xe1d1_00f4 // ldrsh r0, [r1, #4] + ); + // Register-offset word forms. + assert_eq!( + u32_le(Inst::Load { + rt: writable_xreg(0), + mem: AMode::RegReg { + rn: xreg(1), + rm: xreg(2) + }, + kind: LoadKind::Word, + }), + 0xe791_0002 // ldr r0, [r1, r2] + ); + assert_eq!( + u32_le(Inst::Store { + rt: xreg(0), + mem: AMode::RegReg { + rn: xreg(1), + rm: xreg(2) + }, + kind: StoreKind::Word, + }), + 0xe781_0002 // str r0, [r1, r2] + ); +} + +#[test] +fn push_pop_and_sp() { + // push/pop {fp, lr} + let list = (1 << 11) | (1 << 14); + assert_eq!(u32_le(Inst::Push { reg_list: list }), 0xe92d_4800); + assert_eq!(u32_le(Inst::Pop { reg_list: list }), 0xe8bd_4800); + assert_eq!(u32_le(Inst::AdjustSp { amount: -8 }), 0xe24d_d008); + assert_eq!(u32_le(Inst::AdjustSp { amount: 8 }), 0xe28d_d008); +} diff --git a/cranelift/codegen/src/isa/arm32/inst/mod.rs b/cranelift/codegen/src/isa/arm32/inst/mod.rs new file mode 100644 index 000000000000..a0b6d82dc72a --- /dev/null +++ b/cranelift/codegen/src/isa/arm32/inst/mod.rs @@ -0,0 +1,1162 @@ +//! This module defines arm32 (AArch32 / A32) machine instruction types. + +use crate::binemit::{Addend, CodeOffset, Reloc}; +use crate::ir::types::{F32, F64, I8, I16, I32, I64}; +use crate::ir::{Type, types}; +use crate::isa::FunctionAlignment; +use crate::isa::arm32::abi::Arm32MachineDeps; +use crate::machinst::*; +use crate::{CodegenError, CodegenResult, settings}; + +use alloc::string::{String, ToString}; +use alloc::vec::Vec; +use core::fmt::Write; +use regalloc2::RegClass; + +pub mod args; +pub use self::args::*; +pub mod regs; +pub use self::regs::*; +pub mod emit; +pub use self::emit::*; + +#[cfg(test)] +mod emit_tests; + +// The `Inst` type itself is generated from `inst.isle` and re-exported here so +// that the rest of the backend can refer to `Inst` variants directly. +pub use crate::isa::arm32::lower::isle::generated_code::MInst as Inst; + +//============================================================================= +// Operand collection. + +fn arm32_get_operands(inst: &mut Inst, collector: &mut impl OperandVisitor) { + // Only collect virtual (allocatable) registers as operands. Physical + // registers such as sp/fp/lr are fixed in the encoding and are not tracked + // by the register allocator. + fn use_if_virtual(collector: &mut impl OperandVisitor, reg: &mut Reg) { + if reg.to_real_reg().is_none() { + collector.reg_use(reg); + } + } + fn def_if_virtual(collector: &mut impl OperandVisitor, reg: &mut Writable) { + if reg.to_reg().to_real_reg().is_none() { + collector.reg_def(reg); + } + } + + match inst { + Inst::Nop0 + | Inst::Nop4 + | Inst::Ret + | Inst::AdjustSp { .. } + | Inst::Jump { .. } + | Inst::CondBr { .. } + | Inst::Push { .. } + | Inst::Pop { .. } => {} + + Inst::BrTable { index, tmp, .. } => { + use_if_virtual(collector, index); + if tmp.to_reg().to_real_reg().is_none() { + collector.reg_early_def(tmp); + } + } + + Inst::MovImm { rd, .. } + | Inst::MovRotImm { rd, .. } + | Inst::MvnRotImm { rd, .. } + | Inst::Movw { rd, .. } + | Inst::Movt { rd, .. } => def_if_virtual(collector, rd), + + Inst::MovReg { rd, rm } | Inst::MvnReg { rd, rm } => { + use_if_virtual(collector, rm); + def_if_virtual(collector, rd); + } + Inst::AluRRR { rd, rn, rm, .. } => { + use_if_virtual(collector, rn); + use_if_virtual(collector, rm); + def_if_virtual(collector, rd); + } + Inst::AluRRImm { rd, rn, .. } => { + use_if_virtual(collector, rn); + def_if_virtual(collector, rd); + } + Inst::AluRRRFlags { rd, rn, rm, .. } => { + use_if_virtual(collector, rn); + use_if_virtual(collector, rm); + def_if_virtual(collector, rd); + } + Inst::ShiftImm { rd, rm, .. } => { + use_if_virtual(collector, rm); + def_if_virtual(collector, rd); + } + Inst::ShiftReg { rd, rm, rs, .. } => { + use_if_virtual(collector, rm); + use_if_virtual(collector, rs); + def_if_virtual(collector, rd); + } + Inst::Mul { rd, rn, rm } => { + use_if_virtual(collector, rn); + use_if_virtual(collector, rm); + def_if_virtual(collector, rd); + } + Inst::DspMul3 { rd, rn, rm, .. } => { + use_if_virtual(collector, rn); + use_if_virtual(collector, rm); + def_if_virtual(collector, rd); + } + Inst::Mla { rd, rn, rm, ra } + | Inst::Mls { rd, rn, rm, ra } + | Inst::DspMul4 { rd, rn, rm, ra, .. } => { + use_if_virtual(collector, rn); + use_if_virtual(collector, rm); + use_if_virtual(collector, ra); + def_if_virtual(collector, rd); + } + Inst::SDiv { rd, rn, rm } | Inst::UDiv { rd, rn, rm } => { + use_if_virtual(collector, rn); + use_if_virtual(collector, rm); + def_if_virtual(collector, rd); + } + Inst::BitRR { rd, rm, .. } | Inst::ExtRR { rd, rm, .. } => { + use_if_virtual(collector, rm); + def_if_virtual(collector, rd); + } + Inst::CSel { rd, rn, rm, .. } => { + use_if_virtual(collector, rn); + use_if_virtual(collector, rm); + // `rd` is written before the inputs are all consumed (the default + // move), so keep it distinct from the operands. + if rd.to_reg().to_real_reg().is_none() { + collector.reg_early_def(rd); + } + } + Inst::Bfc { rd, .. } => def_if_virtual(collector, rd), + Inst::Sat { rd, rm, .. } | Inst::Rrx { rd, rm } => { + use_if_virtual(collector, rm); + def_if_virtual(collector, rd); + } + Inst::Bfi { rd, rn, .. } | Inst::Bfx { rd, rn, .. } => { + use_if_virtual(collector, rn); + def_if_virtual(collector, rd); + } + Inst::QAlu { rd, rm, rn, .. } => { + use_if_virtual(collector, rm); + use_if_virtual(collector, rn); + def_if_virtual(collector, rd); + } + Inst::Sel { rd, rn, rm } + | Inst::Pkh { rd, rn, rm, .. } + | Inst::ParAlu { rd, rn, rm, .. } + | Inst::ExtAdd { rd, rn, rm, .. } => { + use_if_virtual(collector, rn); + use_if_virtual(collector, rm); + def_if_virtual(collector, rd); + } + Inst::Udf { .. } | Inst::TrapIf { .. } | Inst::Barrier { .. } => {} + + Inst::AtomicRmw { + rd, + addr, + operand, + tmp1, + tmp2, + .. + } => { + use_if_virtual(collector, addr); + use_if_virtual(collector, operand); + for d in [rd, tmp1, tmp2] { + if d.to_reg().to_real_reg().is_none() { + collector.reg_early_def(d); + } + } + } + Inst::AtomicCas { + rd, + addr, + expected, + new, + tmp, + .. + } => { + use_if_virtual(collector, addr); + use_if_virtual(collector, expected); + use_if_virtual(collector, new); + for d in [rd, tmp] { + if d.to_reg().to_real_reg().is_none() { + collector.reg_early_def(d); + } + } + } + + // The 64-bit atomics pin every operand to a fixed even/odd register + // layout: r0/r1 (old value), r2/r3 (new/store value), r4 (addr), + // r5/r6 (operand/expected), r7 (store-exclusive result). A32's paired + // exclusive instructions require consecutive even/odd pairs, which + // regalloc2 cannot express, so the pairs must be fixed physical + // registers. + Inst::AtomicRmw64 { + rd_lo, + rd_hi, + addr, + operand_lo, + operand_hi, + tmp_new_lo, + tmp_new_hi, + tmp_res, + .. + } => { + collector.reg_fixed_use(addr, xreg(4)); + collector.reg_fixed_use(operand_lo, xreg(5)); + collector.reg_fixed_use(operand_hi, xreg(6)); + collector.reg_fixed_def(rd_lo, xreg(0)); + collector.reg_fixed_def(rd_hi, xreg(1)); + collector.reg_fixed_def(tmp_new_lo, xreg(2)); + collector.reg_fixed_def(tmp_new_hi, xreg(3)); + collector.reg_fixed_def(tmp_res, xreg(7)); + } + Inst::AtomicCas64 { + rd_lo, + rd_hi, + addr, + expected_lo, + expected_hi, + new_lo, + new_hi, + tmp_res, + } => { + collector.reg_fixed_use(addr, xreg(4)); + collector.reg_fixed_use(expected_lo, xreg(5)); + collector.reg_fixed_use(expected_hi, xreg(6)); + collector.reg_fixed_use(new_lo, xreg(2)); + collector.reg_fixed_use(new_hi, xreg(3)); + collector.reg_fixed_def(rd_lo, xreg(0)); + collector.reg_fixed_def(rd_hi, xreg(1)); + collector.reg_fixed_def(tmp_res, xreg(7)); + } + Inst::AtomicLoad64 { rd_lo, rd_hi, addr } => { + collector.reg_fixed_use(addr, xreg(4)); + collector.reg_fixed_def(rd_lo, xreg(0)); + collector.reg_fixed_def(rd_hi, xreg(1)); + } + Inst::AtomicStore64 { + addr, + src_lo, + src_hi, + scratch_lo, + scratch_hi, + tmp_res, + } => { + collector.reg_fixed_use(addr, xreg(4)); + collector.reg_fixed_use(src_lo, xreg(2)); + collector.reg_fixed_use(src_hi, xreg(3)); + collector.reg_fixed_def(scratch_lo, xreg(0)); + collector.reg_fixed_def(scratch_hi, xreg(1)); + collector.reg_fixed_def(tmp_res, xreg(7)); + } + + Inst::FpuRRR { rd, rn, rm, .. } => { + use_if_virtual(collector, rn); + use_if_virtual(collector, rm); + def_if_virtual(collector, rd); + } + Inst::FpuRR { rd, rm, .. } => { + use_if_virtual(collector, rm); + def_if_virtual(collector, rd); + } + Inst::FpuLoad { rd, mem, .. } => { + mem.get_operands(collector); + def_if_virtual(collector, rd); + } + Inst::FpuStore { rt, mem, .. } => { + use_if_virtual(collector, rt); + mem.get_operands(collector); + } + Inst::MovToFpu32 { rd, rt } => { + use_if_virtual(collector, rt); + def_if_virtual(collector, rd); + } + Inst::MovToFpu64 { rd, rt_lo, rt_hi } => { + use_if_virtual(collector, rt_lo); + use_if_virtual(collector, rt_hi); + def_if_virtual(collector, rd); + } + Inst::MovFromFpu32 { rt, rm } => { + use_if_virtual(collector, rm); + def_if_virtual(collector, rt); + } + Inst::MovFromFpu64 { rt_lo, rt_hi, rm } => { + use_if_virtual(collector, rm); + def_if_virtual(collector, rt_lo); + def_if_virtual(collector, rt_hi); + } + Inst::VcmpMrs { rn, rm, .. } => { + use_if_virtual(collector, rn); + use_if_virtual(collector, rm); + } + Inst::VcvtFF { rd, rm, .. } + | Inst::VcvtToInt { rd, rm, .. } + | Inst::VcvtFromInt { rd, rm, .. } => { + use_if_virtual(collector, rm); + def_if_virtual(collector, rd); + } + Inst::FpuMinMax { rd, rn, rm, .. } => { + use_if_virtual(collector, rn); + use_if_virtual(collector, rm); + def_if_virtual(collector, rd); + } + Inst::FpuCSel { rd, rn, rm, .. } => { + use_if_virtual(collector, rn); + use_if_virtual(collector, rm); + if rd.to_reg().to_real_reg().is_none() { + collector.reg_early_def(rd); + } + } + Inst::LdmStm { rn, .. } => use_if_virtual(collector, rn), + Inst::LoadEx { rt, rn, .. } | Inst::LoadAcq { rt, rn, .. } => { + use_if_virtual(collector, rn); + def_if_virtual(collector, rt); + } + Inst::StoreEx { rd, rt, rn, .. } => { + use_if_virtual(collector, rt); + use_if_virtual(collector, rn); + def_if_virtual(collector, rd); + } + Inst::StoreRel { rt, rn, .. } => { + use_if_virtual(collector, rt); + use_if_virtual(collector, rn); + } + Inst::Umull { + rd_lo, + rd_hi, + rn, + rm, + } + | Inst::Smull { + rd_lo, + rd_hi, + rn, + rm, + } + | Inst::Umlal { + rd_lo, + rd_hi, + rn, + rm, + } + | Inst::Smlal { + rd_lo, + rd_hi, + rn, + rm, + } + | Inst::DspMulL { + rd_lo, + rd_hi, + rn, + rm, + .. + } => { + use_if_virtual(collector, rn); + use_if_virtual(collector, rm); + // The two destination registers must be distinct from each other + // and from the inputs, so use early defs. (For the accumulating + // `umlal`/`smlal` the destinations are also inputs; those are only + // constructed once i64 support wires them up.) + if rd_lo.to_reg().to_real_reg().is_none() { + collector.reg_early_def(rd_lo); + } + if rd_hi.to_reg().to_real_reg().is_none() { + collector.reg_early_def(rd_hi); + } + } + Inst::CmpRR { rn, rm, .. } => { + use_if_virtual(collector, rn); + use_if_virtual(collector, rm); + } + Inst::CmpRImm { rn, .. } => use_if_virtual(collector, rn), + + Inst::Store { rt, mem, .. } => { + use_if_virtual(collector, rt); + mem.get_operands(collector); + } + Inst::Load { rt, mem, .. } => { + mem.get_operands(collector); + def_if_virtual(collector, rt); + } + + Inst::Call { info } => { + let CallInfo { uses, defs, .. } = &mut **info; + for CallArgPair { vreg, preg } in uses { + collector.reg_fixed_use(vreg, *preg); + } + for CallRetPair { vreg, location } in defs { + match location { + RetLocation::Reg(preg, ..) => collector.reg_fixed_def(vreg, *preg), + RetLocation::Stack(..) => collector.any_def(vreg), + } + } + collector.reg_clobbers(info.clobbers); + } + Inst::LoadExtName { rd, .. } => { + collector.reg_def(rd); + } + Inst::CallInd { info } => { + let CallInfo { + dest, uses, defs, .. + } = &mut **info; + collector.reg_use(dest); + for CallArgPair { vreg, preg } in uses { + collector.reg_fixed_use(vreg, *preg); + } + for CallRetPair { vreg, location } in defs { + match location { + RetLocation::Reg(preg, ..) => collector.reg_fixed_def(vreg, *preg), + RetLocation::Stack(..) => collector.any_def(vreg), + } + } + collector.reg_clobbers(info.clobbers); + } + + Inst::Args { args } => { + for ArgPair { vreg, preg } in args { + collector.reg_fixed_def(vreg, *preg); + } + } + Inst::Rets { rets } => { + for RetPair { vreg, preg } in rets { + collector.reg_fixed_use(vreg, *preg); + } + } + } +} + +impl MachInst for Inst { + type LabelUse = LabelUse; + type ABIMachineSpec = Arm32MachineDeps; + + // The undefined instruction `udf`. Used to fill trap-reachable padding. + const TRAP_OPCODE: &'static [u8] = &0xe7f000f0u32.to_le_bytes(); + + fn gen_dummy_use(_reg: Reg) -> Self { + // Represented as a zero-length nop that "uses" nothing; a dummy use is + // only needed by targets that must keep a value live artificially. + Inst::Nop0 + } + + fn canonical_type_for_rc(rc: RegClass) -> Type { + match rc { + RegClass::Int => I32, + RegClass::Float => F64, + RegClass::Vector => types::I8X16, + } + } + + fn is_safepoint(&self) -> bool { + matches!(self, Inst::Call { .. } | Inst::CallInd { .. }) + } + + fn get_operands(&mut self, collector: &mut impl OperandVisitor) { + arm32_get_operands(self, collector); + } + + fn is_move(&self) -> Option<(Writable, Reg)> { + match self { + Inst::MovReg { rd, rm } => Some((*rd, *rm)), + _ => None, + } + } + + fn is_included_in_clobbers(&self) -> bool { + !matches!(self, Inst::Args { .. }) + } + + fn is_trap(&self) -> bool { + matches!(self, Inst::Udf { .. }) + } + + fn is_args(&self) -> bool { + matches!(self, Inst::Args { .. }) + } + + fn call_type(&self) -> CallType { + match self { + Inst::Call { .. } | Inst::CallInd { .. } => CallType::Regular, + _ => CallType::None, + } + } + + fn is_term(&self) -> MachTerminator { + match self { + Inst::Rets { .. } => MachTerminator::Ret, + Inst::Jump { .. } | Inst::CondBr { .. } | Inst::BrTable { .. } => { + MachTerminator::Branch + } + _ => MachTerminator::None, + } + } + + fn is_mem_access(&self) -> bool { + matches!(self, Inst::Load { .. } | Inst::Store { .. }) + } + + fn gen_move(to_reg: Writable, from_reg: Reg, _ty: Type) -> Inst { + if to_reg.to_reg().class() == RegClass::Float { + // A `vmov.f64` copies the whole D register, which is correct for an + // f32 held in the low half too. + Inst::FpuRR { + op: FpuOp2::Vmov, + size: FpuSize::F64, + rd: to_reg, + rm: from_reg, + } + } else { + Inst::MovReg { + rd: to_reg, + rm: from_reg, + } + } + } + + fn gen_nop(preferred_size: usize) -> Inst { + if preferred_size == 0 { + return Inst::Nop0; + } + assert!(preferred_size >= 4); + Inst::Nop4 + } + + fn gen_nop_units() -> Vec> { + vec![0xe320f000u32.to_le_bytes().to_vec()] + } + + fn rc_for_type(ty: Type) -> CodegenResult<(&'static [RegClass], &'static [Type])> { + match ty { + I8 => Ok((&[RegClass::Int], &[I8])), + I16 => Ok((&[RegClass::Int], &[I16])), + I32 => Ok((&[RegClass::Int], &[I32])), + // 64-bit values are held in a pair of 32-bit integer registers. + I64 => Ok((&[RegClass::Int, RegClass::Int], &[I32, I32])), + // Floats live in VFP registers (an f32 in the low half of a D reg). + F32 => Ok((&[RegClass::Float], &[F32])), + F64 => Ok((&[RegClass::Float], &[F64])), + _ => Err(CodegenError::Unsupported(alloc::format!( + "Unsupported type on arm32 (only i8/i16/i32/i64/f32/f64 are implemented so far): {ty}" + ))), + } + } + + fn gen_jump(target: MachLabel) -> Inst { + Inst::Jump { dest: target } + } + + fn worst_case_size() -> CodeOffset { + // Must be an upper bound on the number of bytes any single (non + // jump-table) instruction emits, since the buffer uses it as the + // per-instruction island lookahead. The largest such sequence is the + // 64-bit atomic min/max RMW loop, which expands to 10 words (40 bytes). + // Inline jump tables manage their own islands and are exempt. + 44 + } + + fn worst_case_island_growth() -> CodeOffset { + 8 + } + + fn ref_type_regclass(_settings: &settings::Flags) -> RegClass { + RegClass::Int + } + + fn function_alignment() -> FunctionAlignment { + FunctionAlignment { + minimum: 4, + preferred: 4, + } + } +} + +//============================================================================= +// Pretty-printing. + +impl Inst { + pub(crate) fn print_with_state(&self, _state: &mut EmitState) -> String { + let r = |reg: Reg| reg_name(reg); + let reglist = |mask: u32| -> String { + let names: Vec = (0..16) + .filter(|i| mask & (1 << i) != 0) + .map(|i| reg_name(xreg(i))) + .collect(); + alloc::format!("{{{}}}", names.join(", ")) + }; + match self { + Inst::Nop0 => "nop-zero-len".to_string(), + Inst::Nop4 => "nop".to_string(), + Inst::Ret => "bx lr".to_string(), + Inst::MovImm { rd, imm } => { + let rd = r(rd.to_reg()); + if *imm >> 16 == 0 { + alloc::format!("movw {rd}, #{imm}") + } else { + alloc::format!("movw {rd}, #{}; movt {rd}, #{}", imm & 0xffff, imm >> 16) + } + } + Inst::MovRotImm { rd, imm12 } => { + alloc::format!("mov {}, #{}", r(rd.to_reg()), decode_rotated_imm(*imm12)) + } + Inst::MvnRotImm { rd, imm12 } => { + alloc::format!("mvn {}, #{}", r(rd.to_reg()), decode_rotated_imm(*imm12)) + } + Inst::Movw { rd, imm16 } => alloc::format!("movw {}, #{}", r(rd.to_reg()), imm16), + Inst::Movt { rd, imm16 } => alloc::format!("movt {}, #{}", r(rd.to_reg()), imm16), + Inst::MovReg { rd, rm } => { + alloc::format!("mov {}, {}", r(rd.to_reg()), r(*rm)) + } + Inst::MvnReg { rd, rm } => { + alloc::format!("mvn {}, {}", r(rd.to_reg()), r(*rm)) + } + Inst::AluRRR { op, rd, rn, rm } => { + alloc::format!("{} {}, {}, {}", op.name(), r(rd.to_reg()), r(*rn), r(*rm)) + } + Inst::AluRRImm { op, rd, rn, imm12 } => { + alloc::format!( + "{} {}, {}, #{}", + op.name(), + r(rd.to_reg()), + r(*rn), + decode_rotated_imm(*imm12) + ) + } + Inst::AluRRRFlags { op, rd, rn, rm } => { + alloc::format!("{}s {}, {}, {}", op.name(), r(rd.to_reg()), r(*rn), r(*rm)) + } + Inst::ShiftImm { op, rd, rm, amount } => { + alloc::format!("{} {}, {}, #{}", op.name(), r(rd.to_reg()), r(*rm), amount) + } + Inst::ShiftReg { op, rd, rm, rs } => { + alloc::format!("{} {}, {}, {}", op.name(), r(rd.to_reg()), r(*rm), r(*rs)) + } + Inst::Mul { rd, rn, rm } => { + alloc::format!("mul {}, {}, {}", r(rd.to_reg()), r(*rn), r(*rm)) + } + Inst::Mla { rd, rn, rm, ra } => { + alloc::format!("mla {}, {}, {}, {}", r(rd.to_reg()), r(*rn), r(*rm), r(*ra)) + } + Inst::Mls { rd, rn, rm, ra } => { + alloc::format!("mls {}, {}, {}, {}", r(rd.to_reg()), r(*rn), r(*rm), r(*ra)) + } + Inst::Umull { + rd_lo, + rd_hi, + rn, + rm, + } + | Inst::Smull { + rd_lo, + rd_hi, + rn, + rm, + } + | Inst::Umlal { + rd_lo, + rd_hi, + rn, + rm, + } + | Inst::Smlal { + rd_lo, + rd_hi, + rn, + rm, + } => { + let mnem = match self { + Inst::Umull { .. } => "umull", + Inst::Smull { .. } => "smull", + Inst::Umlal { .. } => "umlal", + _ => "smlal", + }; + alloc::format!( + "{mnem} {}, {}, {}, {}", + r(rd_lo.to_reg()), + r(rd_hi.to_reg()), + r(*rn), + r(*rm) + ) + } + Inst::SDiv { rd, rn, rm } => { + alloc::format!("sdiv {}, {}, {}", r(rd.to_reg()), r(*rn), r(*rm)) + } + Inst::UDiv { rd, rn, rm } => { + alloc::format!("udiv {}, {}, {}", r(rd.to_reg()), r(*rn), r(*rm)) + } + Inst::BitRR { op, rd, rm } => { + alloc::format!("{} {}, {}", op.name(), r(rd.to_reg()), r(*rm)) + } + Inst::ExtRR { op, rd, rm } => { + alloc::format!("{} {}, {}", op.name(), r(rd.to_reg()), r(*rm)) + } + Inst::CSel { cond, rd, rn, rm } => { + let rd = r(rd.to_reg()); + alloc::format!("mov {rd}, {}; mov{} {rd}, {}", r(*rm), cond.name(), r(*rn)) + } + Inst::Bfc { rd, lsb, width } => { + alloc::format!("bfc {}, #{}, #{}", r(rd.to_reg()), lsb, width) + } + Inst::Bfi { rd, rn, lsb, width } => { + alloc::format!("bfi {}, {}, #{}, #{}", r(rd.to_reg()), r(*rn), lsb, width) + } + Inst::Bfx { + op, + rd, + rn, + lsb, + width, + } => alloc::format!( + "{} {}, {}, #{}, #{}", + op.name(), + r(rd.to_reg()), + r(*rn), + lsb, + width + ), + Inst::QAlu { op, rd, rm, rn } => { + alloc::format!("{} {}, {}, {}", op.name(), r(rd.to_reg()), r(*rm), r(*rn)) + } + Inst::Sat { + op, + rd, + sat_bits, + rm, + } => alloc::format!( + "{} {}, #{}, {}", + op.name(), + r(rd.to_reg()), + sat_bits, + r(*rm) + ), + Inst::ParAlu { op, rd, rn, rm } => { + alloc::format!("{} {}, {}, {}", op.name(), r(rd.to_reg()), r(*rn), r(*rm)) + } + Inst::ExtAdd { op, rd, rn, rm } => { + alloc::format!("{} {}, {}, {}", op.name(), r(rd.to_reg()), r(*rn), r(*rm)) + } + Inst::DspMul3 { op, rd, rn, rm } => { + alloc::format!("{} {}, {}, {}", op.name(), r(rd.to_reg()), r(*rn), r(*rm)) + } + Inst::DspMul4 { op, rd, rn, rm, ra } => alloc::format!( + "{} {}, {}, {}, {}", + op.name(), + r(rd.to_reg()), + r(*rn), + r(*rm), + r(*ra) + ), + Inst::DspMulL { + op, + rd_lo, + rd_hi, + rn, + rm, + } => alloc::format!( + "{} {}, {}, {}, {}", + op.name(), + r(rd_lo.to_reg()), + r(rd_hi.to_reg()), + r(*rn), + r(*rm) + ), + Inst::Sel { rd, rn, rm } => { + alloc::format!("sel {}, {}, {}", r(rd.to_reg()), r(*rn), r(*rm)) + } + Inst::Pkh { op, rd, rn, rm } => { + alloc::format!("{} {}, {}, {}", op.name(), r(rd.to_reg()), r(*rn), r(*rm)) + } + Inst::Rrx { rd, rm } => { + alloc::format!("rrx {}, {}", r(rd.to_reg()), r(*rm)) + } + Inst::Udf { code } => alloc::format!("udf ; {code}"), + Inst::TrapIf { cond, code } => { + alloc::format!("b{} 1f ; udf ; 1: ; {code}", cond.invert().name()) + } + Inst::LdmStm { + load, + rn, + writeback, + reg_list, + } => { + let names: Vec = (0..16) + .filter(|i| reg_list & (1 << i) != 0) + .map(|i| reg_name(xreg(i))) + .collect(); + alloc::format!( + "{} {}{}, {{{}}}", + if *load { "ldmia" } else { "stmia" }, + r(*rn), + if *writeback { "!" } else { "" }, + names.join(", ") + ) + } + Inst::LoadEx { + acquire, + size, + rt, + rn, + } => alloc::format!( + "{}{} {}, [{}]", + if *acquire { "ldaex" } else { "ldrex" }, + size.suffix(), + r(rt.to_reg()), + r(*rn) + ), + Inst::StoreEx { + release, + size, + rd, + rt, + rn, + } => alloc::format!( + "{}{} {}, {}, [{}]", + if *release { "stlex" } else { "strex" }, + size.suffix(), + r(rd.to_reg()), + r(*rt), + r(*rn) + ), + Inst::LoadAcq { size, rt, rn } => { + alloc::format!("lda{} {}, [{}]", size.suffix(), r(rt.to_reg()), r(*rn)) + } + Inst::StoreRel { size, rt, rn } => { + alloc::format!("stl{} {}, [{}]", size.suffix(), r(*rt), r(*rn)) + } + Inst::Barrier { op } => op.name().to_string(), + Inst::AtomicRmw { + op, + size, + rd, + addr, + operand, + .. + } => alloc::format!( + "atomic_rmw.{op:?}{} {}, [{}], {}", + size.suffix(), + r(rd.to_reg()), + r(*addr), + r(*operand) + ), + Inst::AtomicCas { + size, + rd, + addr, + expected, + new, + .. + } => alloc::format!( + "atomic_cas{} {}, [{}], {}, {}", + size.suffix(), + r(rd.to_reg()), + r(*addr), + r(*expected), + r(*new) + ), + Inst::AtomicRmw64 { + op, + rd_lo, + rd_hi, + addr, + operand_lo, + operand_hi, + .. + } => alloc::format!( + "atomic_rmw.{op:?}.i64 {}, {}, [{}], {}, {}", + r(rd_lo.to_reg()), + r(rd_hi.to_reg()), + r(*addr), + r(*operand_lo), + r(*operand_hi) + ), + Inst::AtomicCas64 { + rd_lo, + rd_hi, + addr, + expected_lo, + expected_hi, + new_lo, + new_hi, + .. + } => alloc::format!( + "atomic_cas.i64 {}, {}, [{}], {}, {}, {}, {}", + r(rd_lo.to_reg()), + r(rd_hi.to_reg()), + r(*addr), + r(*expected_lo), + r(*expected_hi), + r(*new_lo), + r(*new_hi) + ), + Inst::AtomicLoad64 { rd_lo, rd_hi, addr } => alloc::format!( + "atomic_load.i64 {}, {}, [{}]", + r(rd_lo.to_reg()), + r(rd_hi.to_reg()), + r(*addr) + ), + Inst::AtomicStore64 { + addr, + src_lo, + src_hi, + .. + } => alloc::format!( + "atomic_store.i64 {}, {}, [{}]", + r(*src_lo), + r(*src_hi), + r(*addr) + ), + + Inst::FpuRRR { + op, + size, + rd, + rn, + rm, + } => alloc::format!( + "{}.{} {}, {}, {}", + op.name(), + size.suffix(), + r(rd.to_reg()), + r(*rn), + r(*rm) + ), + Inst::FpuRR { op, size, rd, rm } => alloc::format!( + "{}.{} {}, {}", + op.name(), + size.suffix(), + r(rd.to_reg()), + r(*rm) + ), + Inst::FpuLoad { rd, mem, .. } => { + alloc::format!("vldr {}, {}", r(rd.to_reg()), mem.pretty_print()) + } + Inst::FpuStore { rt, mem, .. } => { + alloc::format!("vstr {}, {}", r(*rt), mem.pretty_print()) + } + Inst::MovToFpu32 { rd, rt } => { + alloc::format!("vmov {}, {}", r(rd.to_reg()), r(*rt)) + } + Inst::MovToFpu64 { rd, rt_lo, rt_hi } => { + alloc::format!("vmov {}, {}, {}", r(rd.to_reg()), r(*rt_lo), r(*rt_hi)) + } + Inst::MovFromFpu32 { rt, rm } => { + alloc::format!("vmov {}, {}", r(rt.to_reg()), r(*rm)) + } + Inst::MovFromFpu64 { rt_lo, rt_hi, rm } => { + alloc::format!( + "vmov {}, {}, {}", + r(rt_lo.to_reg()), + r(rt_hi.to_reg()), + r(*rm) + ) + } + Inst::VcmpMrs { size, rn, rm } => { + alloc::format!("vcmp.{} {}, {}; vmrs", size.suffix(), r(*rn), r(*rm)) + } + Inst::VcvtFF { to_f64, rd, rm } => { + let (dst, src) = if *to_f64 { + ("f64", "f32") + } else { + ("f32", "f64") + }; + alloc::format!("vcvt.{dst}.{src} {}, {}", r(rd.to_reg()), r(*rm)) + } + Inst::VcvtToInt { + signed, + size, + rd, + rm, + } => { + let int = if *signed { "s32" } else { "u32" }; + alloc::format!( + "vcvt.{int}.{} {}, {}", + size.suffix(), + r(rd.to_reg()), + r(*rm) + ) + } + Inst::VcvtFromInt { + signed, + size, + rd, + rm, + } => { + let int = if *signed { "s32" } else { "u32" }; + alloc::format!( + "vcvt.{}.{int} {}, {}", + size.suffix(), + r(rd.to_reg()), + r(*rm) + ) + } + Inst::FpuCSel { cond, rd, rn, rm } => { + let rd = r(rd.to_reg()); + alloc::format!( + "vmov {rd}, {}; vmov{} {rd}, {}", + r(*rm), + cond.name(), + r(*rn) + ) + } + Inst::FpuMinMax { + max, + size, + rd, + rn, + rm, + } => alloc::format!( + "{}.{} {}, {}, {}", + if *max { "vmaxnm" } else { "vminnm" }, + size.suffix(), + r(rd.to_reg()), + r(*rn), + r(*rm) + ), + Inst::CmpRR { op, rn, rm } => { + alloc::format!("{} {}, {}", op.name(), r(*rn), r(*rm)) + } + Inst::CmpRImm { op, rn, imm12 } => { + alloc::format!("{} {}, #{}", op.name(), r(*rn), decode_rotated_imm(*imm12)) + } + Inst::AdjustSp { amount } => { + if *amount < 0 { + alloc::format!("sub sp, sp, #{}", -*amount) + } else { + alloc::format!("add sp, sp, #{amount}") + } + } + Inst::Store { rt, mem, kind } => { + alloc::format!("{} {}, {}", kind.mnemonic(), r(*rt), mem.pretty_print()) + } + Inst::Load { rt, mem, kind } => { + alloc::format!( + "{} {}, {}", + kind.mnemonic(), + r(rt.to_reg()), + mem.pretty_print() + ) + } + Inst::Push { reg_list } => alloc::format!("push {}", reglist(*reg_list)), + Inst::Pop { reg_list } => alloc::format!("pop {}", reglist(*reg_list)), + Inst::Call { info } => alloc::format!("bl {}", info.dest.display(None)), + Inst::CallInd { info } => alloc::format!("blx {}", r(info.dest)), + Inst::LoadExtName { rd, name, offset } => { + alloc::format!("load_ext_name {}, {:?} + {}", r(rd.to_reg()), name, offset) + } + Inst::Jump { dest } => alloc::format!("b {}", dest.to_string()), + Inst::BrTable { index, targets, .. } => { + let (default, entries) = targets.split_first().unwrap(); + let entries: Vec = entries.iter().map(|l| l.to_string()).collect(); + alloc::format!( + "br_table {} [{}] default {}", + r(*index), + entries.join(", "), + default.to_string() + ) + } + Inst::CondBr { + cond, + taken, + not_taken, + } => alloc::format!( + "b{} {}; b {}", + cond.name(), + taken.to_string(), + not_taken.to_string() + ), + Inst::Args { args } => { + let mut s = "args".to_string(); + for arg in args { + write!(&mut s, " {}={}", r(arg.vreg.to_reg()), r(arg.preg)).unwrap(); + } + s + } + Inst::Rets { rets } => { + let mut s = "rets".to_string(); + for ret in rets { + write!(&mut s, " {}={}", r(ret.vreg), r(ret.preg)).unwrap(); + } + s + } + } + } +} + +//============================================================================= +// Label uses (branch fixups). + +/// A use of a label by an instruction, for branch fixups. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum LabelUse { + /// A 24-bit signed PC-relative branch offset, as used by the A32 `B`/`BL` + /// instructions. The stored immediate is the offset in units of 4 bytes, + /// relative to the branch instruction's address plus 8. + Branch26, +} + +impl MachInstLabelUse for LabelUse { + const ALIGN: CodeOffset = 4; + + fn max_pos_range(self) -> CodeOffset { + match self { + // 24-bit signed immediate, scaled by 4. + LabelUse::Branch26 => ((1 << 23) - 1) * 4, + } + } + + fn max_neg_range(self) -> CodeOffset { + match self { + LabelUse::Branch26 => (1 << 23) * 4, + } + } + + fn patch_size(self) -> CodeOffset { + 4 + } + + fn patch(self, buffer: &mut [u8], use_offset: CodeOffset, label_offset: CodeOffset) { + match self { + LabelUse::Branch26 => { + // The ARM PC reads as the instruction address plus 8. + let pc_base = use_offset as i64 + 8; + let offset = label_offset as i64 - pc_base; + debug_assert!(offset & 0b11 == 0); + let imm24 = ((offset >> 2) as u32) & 0x00ff_ffff; + let insn = u32::from_le_bytes(buffer[0..4].try_into().unwrap()); + let insn = (insn & 0xff00_0000) | imm24; + buffer[0..4].copy_from_slice(&insn.to_le_bytes()); + } + } + } + + fn supports_veneer(self) -> bool { + false + } + + fn veneer_size(self) -> CodeOffset { + 0 + } + + fn worst_case_veneer_size() -> CodeOffset { + 0 + } + + fn generate_veneer( + self, + _buffer: &mut [u8], + _veneer_offset: CodeOffset, + ) -> (CodeOffset, LabelUse) { + panic!("arm32 does not support branch veneers yet"); + } + + fn from_reloc(reloc: Reloc, _addend: Addend) -> Option { + match reloc { + Reloc::Arm32Call => Some(LabelUse::Branch26), + _ => None, + } + } +} diff --git a/cranelift/codegen/src/isa/arm32/inst/regs.rs b/cranelift/codegen/src/isa/arm32/inst/regs.rs new file mode 100644 index 000000000000..ea7f0ce7e812 --- /dev/null +++ b/cranelift/codegen/src/isa/arm32/inst/regs.rs @@ -0,0 +1,177 @@ +//! Arm32 (AArch32 / A32) ISA definitions: registers. +//! +//! We model the 16 general-purpose ARM registers r0-r15 as `RegClass::Int`. +//! Following the AAPCS calling convention: +//! +//! * r0-r3 : argument / caller-saved (scratch) +//! * r4-r10 : callee-saved +//! * r11 (fp): frame pointer +//! * r12 (ip): intra-procedure scratch / spill temporary (reserved) +//! * r13 (sp): stack pointer +//! * r14 (lr): link register +//! * r15 (pc): program counter +//! +//! Floating-point (VFP/NEON) registers are not modelled yet. + +use crate::machinst::{Reg, Writable}; +use regalloc2::{MachineEnv, PReg, RegClass, VReg}; + +/// Construct a virtual-ish `Reg` referencing GPR `enc` (0-15). +#[inline] +pub const fn xreg(enc: u8) -> Reg { + let p_reg = PReg::new(enc as usize, RegClass::Int); + let v_reg = VReg::new(p_reg.index(), p_reg.class()); + Reg::from_virtual_reg(v_reg) +} + +/// Construct a `PReg` referencing GPR `enc` (0-15). +pub const fn preg(enc: u8) -> PReg { + PReg::new(enc as usize, RegClass::Int) +} + +/// Construct a `Reg` referencing VFP double-register `D` (0-15). An `f32` +/// value uses the low single-register half (`S<2*enc>`) of its `D` register, so +/// each value owns a whole `D` register and the S/D register files never alias. +#[inline] +pub const fn dreg(enc: u8) -> Reg { + let p_reg = PReg::new(enc as usize, RegClass::Float); + let v_reg = VReg::new(p_reg.index(), p_reg.class()); + Reg::from_virtual_reg(v_reg) +} + +/// Construct a `PReg` referencing VFP double-register `D`. +pub const fn pdreg(enc: u8) -> PReg { + PReg::new(enc as usize, RegClass::Float) +} + +/// Get a writable reference to `D`. +#[inline] +#[allow( + dead_code, + reason = "part of the register API, used as the backend grows" +)] +pub fn writable_dreg(enc: u8) -> Writable { + Writable::from_reg(dreg(enc)) +} + +/// Get a writable reference to GPR `enc`. +#[inline] +pub fn writable_xreg(enc: u8) -> Writable { + Writable::from_reg(xreg(enc)) +} + +/// Frame pointer (r11). +#[inline] +pub fn fp_reg() -> Reg { + xreg(11) +} + +/// Writable frame pointer. +#[inline] +pub fn writable_fp_reg() -> Writable { + Writable::from_reg(fp_reg()) +} + +/// Intra-procedure-call scratch register (r12), used as the spill temporary. +#[inline] +pub fn spilltmp_reg() -> Reg { + xreg(12) +} + +/// Stack pointer (r13). +#[inline] +pub fn stack_reg() -> Reg { + xreg(13) +} + +/// Writable stack pointer. +#[inline] +#[allow( + dead_code, + reason = "part of the register API, used as the backend grows" +)] +pub fn writable_stack_reg() -> Writable { + Writable::from_reg(stack_reg()) +} + +/// Link register (r14). +#[inline] +#[allow( + dead_code, + reason = "part of the register API, used as the backend grows" +)] +pub fn link_reg() -> Reg { + xreg(14) +} + +/// Writable link register. +#[inline] +#[allow( + dead_code, + reason = "part of the register API, used as the backend grows" +)] +pub fn writable_link_reg() -> Writable { + Writable::from_reg(link_reg()) +} + +/// Program counter (r15). +#[inline] +#[allow( + dead_code, + reason = "part of the register API, used as the backend grows" +)] +pub fn pc_reg() -> Reg { + xreg(15) +} + +/// Pretty-print a register with its AAPCS / VFP name. +pub fn reg_name(reg: Reg) -> alloc::string::String { + use alloc::string::ToString; + match reg.to_real_reg() { + Some(real) => match real.class() { + RegClass::Float => alloc::format!("d{}", real.hw_enc()), + _ => match real.hw_enc() { + 11 => "fp".to_string(), + 12 => "ip".to_string(), + 13 => "sp".to_string(), + 14 => "lr".to_string(), + 15 => "pc".to_string(), + enc => alloc::format!("r{enc}"), + }, + }, + None => alloc::format!("{reg:?}"), + } +} + +/// Build the register environment describing which registers the allocator may +/// use, in preference order. +pub const fn create_reg_environment() -> MachineEnv { + // Preferred registers are the caller-saved (scratch) registers; the + // allocator reaches for these first since they need no save/restore. + let preferred_int = preg_set(RegClass::Int, &[0, 1, 2, 3]); + let non_preferred_int = preg_set(RegClass::Int, &[4, 5, 6, 7, 8, 9, 10]); + // AAPCS: D0-D7 are caller-saved, D8-D15 callee-saved. + let preferred_float = preg_set(RegClass::Float, &[0, 1, 2, 3, 4, 5, 6, 7]); + let non_preferred_float = preg_set(RegClass::Float, &[8, 9, 10, 11, 12, 13, 14, 15]); + + MachineEnv { + preferred_regs_by_class: [preferred_int, preferred_float, empty()], + non_preferred_regs_by_class: [non_preferred_int, non_preferred_float, empty()], + fixed_stack_slots: vec![], + scratch_by_class: [None, None, None], + } +} + +const fn preg_set(class: RegClass, encs: &[u8]) -> regalloc2::PRegSet { + let mut set = regalloc2::PRegSet::empty(); + let mut i = 0; + while i < encs.len() { + set = set.with(PReg::new(encs[i] as usize, class)); + i += 1; + } + set +} + +const fn empty() -> regalloc2::PRegSet { + regalloc2::PRegSet::empty() +} diff --git a/cranelift/codegen/src/isa/arm32/lower.isle b/cranelift/codegen/src/isa/arm32/lower.isle new file mode 100644 index 000000000000..f28c07ddb6bb --- /dev/null +++ b/cranelift/codegen/src/isa/arm32/lower.isle @@ -0,0 +1,511 @@ +;; arm32 (AArch32 / A32) lowering rules. + +;; The main lowering entry point. +(decl partial lower (Inst) InstOutput) + +;;;; `iconst` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(rule (lower (iconst (fits_in_32 ty) (u64_from_imm64 n))) + (imm ty n)) + +;;;; 64-bit integer operations (register pairs) ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; These use an explicit priority so they don't overlap the `fits_in_32` rules +;; (ISLE can't prove `$I64` and `fits_in_32` are disjoint on its own). + +(rule 4 (lower (iconst $I64 (u64_from_imm64 n))) + (value_regs (gen_constant n) (gen_constant (u64_high32 n)))) + +(rule 4 (lower (iadd $I64 x y)) + (add_i64 (put_in_regs x) (put_in_regs y))) +(rule 4 (lower (isub $I64 x y)) + (sub_i64 (put_in_regs x) (put_in_regs y))) + +(rule 4 (lower (band $I64 x y)) + (logical_i64 (ALUOp.And) (put_in_regs x) (put_in_regs y))) +(rule 4 (lower (bor $I64 x y)) + (logical_i64 (ALUOp.Orr) (put_in_regs x) (put_in_regs y))) +(rule 4 (lower (bxor $I64 x y)) + (logical_i64 (ALUOp.Eor) (put_in_regs x) (put_in_regs y))) +(rule 4 (lower (bnot $I64 x)) + (not_i64 (put_in_regs x))) + +;; `iconcat(lo, hi)` builds the pair; `isplit` takes it apart. +(rule 4 (lower (iconcat $I64 lo hi)) + (value_regs (put_in_reg lo) (put_in_reg hi))) +(rule 4 (lower (isplit $I64 x)) + (let ((regs ValueRegs (put_in_regs x))) + (output_pair (value_reg (vr_lo regs)) (value_reg (vr_hi regs))))) + +;; Widen/narrow between i32 and i64. +(rule 4 (lower (uextend $I64 x @ (value_type $I32))) + (value_regs (put_in_reg x) (gen_constant 0))) +(rule 4 (lower (sextend $I64 x @ (value_type $I32))) + (let ((lo Reg (put_in_reg x))) + (value_regs lo (gen_shift_imm (ShiftOp.Asr) lo 31)))) +(rule 4 (lower (ireduce $I32 x @ (value_type $I64))) + (vr_lo (put_in_regs x))) + +(rule 4 (lower (imul $I64 x y)) + (mul_i64 (put_in_regs x) (put_in_regs y))) + +;; 64-bit shifts by a constant amount. +(rule 4 (lower (ishl $I64 x (iconst _ (u64_from_imm64 n)))) + (gen_i64_shift_imm (ShiftOp.Lsl) (put_in_regs x) n)) +(rule 4 (lower (ushr $I64 x (iconst _ (u64_from_imm64 n)))) + (gen_i64_shift_imm (ShiftOp.Lsr) (put_in_regs x) n)) +(rule 4 (lower (sshr $I64 x (iconst _ (u64_from_imm64 n)))) + (gen_i64_shift_imm (ShiftOp.Asr) (put_in_regs x) n)) + +;; 64-bit loads/stores as two 32-bit accesses. +(rule 4 (lower (load $I64 flags addr offset)) + (value_regs (arm_load (amode addr offset) (LoadKind.Word)) + (arm_load (amode_hi addr offset) (LoadKind.Word)))) +(rule 4 (lower (store flags val @ (value_type $I64) addr offset)) + (let ((regs ValueRegs (put_in_regs val)) + (_ InstOutput (arm_store (vr_lo regs) (amode addr offset) (StoreKind.Word)))) + (arm_store (vr_hi regs) (amode_hi addr offset) (StoreKind.Word)))) + +;;;; `iadd` / `isub` / `ineg` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(rule 2 (lower (iadd (fits_in_32 _) x (iconst _ (u64_from_imm64 n)))) + (add_imm (put_in_reg x) n)) +(rule 1 (lower (iadd (fits_in_32 _) (iconst _ (u64_from_imm64 n)) y)) + (add_imm (put_in_reg y) n)) +(rule (lower (iadd (fits_in_32 _) x y)) + (add_reg (put_in_reg x) (put_in_reg y))) + +(rule 1 (lower (isub (fits_in_32 _) x (iconst _ (u64_from_imm64 n)))) + (sub_imm (put_in_reg x) n)) +(rule (lower (isub (fits_in_32 _) x y)) + (sub_reg (put_in_reg x) (put_in_reg y))) + +;; Negation is a reverse-subtract from zero. +(rule (lower (ineg (fits_in_32 _) x)) + (rsb_imm (put_in_reg x) 0)) + +;;;; Logical `band` / `bor` / `bxor` / `band_not` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(rule 1 (lower (band (fits_in_32 _) x (iconst _ (u64_from_imm64 n)))) + (alu_op_imm (ALUOp.And) (put_in_reg x) n)) +(rule (lower (band (fits_in_32 _) x y)) + (alu_rrr (ALUOp.And) (put_in_reg x) (put_in_reg y))) + +(rule 1 (lower (bor (fits_in_32 _) x (iconst _ (u64_from_imm64 n)))) + (alu_op_imm (ALUOp.Orr) (put_in_reg x) n)) +(rule (lower (bor (fits_in_32 _) x y)) + (alu_rrr (ALUOp.Orr) (put_in_reg x) (put_in_reg y))) + +(rule 1 (lower (bxor (fits_in_32 _) x (iconst _ (u64_from_imm64 n)))) + (alu_op_imm (ALUOp.Eor) (put_in_reg x) n)) +(rule (lower (bxor (fits_in_32 _) x y)) + (alu_rrr (ALUOp.Eor) (put_in_reg x) (put_in_reg y))) + +;; `x & ~y` maps directly to `bic` (Cranelift's `band_not` builder emits +;; `band` of `bnot`). +(rule 2 (lower (band (fits_in_32 _) x (bnot _ y))) + (alu_rrr (ALUOp.Bic) (put_in_reg x) (put_in_reg y))) +(rule 3 (lower (band (fits_in_32 _) (bnot _ y) x)) + (alu_rrr (ALUOp.Bic) (put_in_reg x) (put_in_reg y))) + +;; Bitwise NOT. +(rule (lower (bnot (fits_in_32 _) x)) + (mvn_reg (put_in_reg x))) + +;;;; `imul` and multiply-fused `isub` -> `mls` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(rule (lower (imul (fits_in_32 _) x y)) + (mul (put_in_reg x) (put_in_reg y))) + +;; `a - x*y` fuses into a single `mls`. +(rule 2 (lower (isub (fits_in_32 _) a (imul _ x y))) + (mls (put_in_reg x) (put_in_reg y) (put_in_reg a))) + +;;;; Traps ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(rule (lower (trap code)) + (side_effect (udf code))) + +;;;; Fences ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(rule (lower (fence)) + (side_effect (dmb))) + +;;;; Atomics (8/16/32-bit) ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(rule (lower (atomic_rmw (fits_in_32 ty) _flags op addr x)) + (atomic_rmw_size ty op (put_in_reg addr) (put_in_reg x))) +(rule (lower (atomic_cas (fits_in_32 ty) _flags addr e x)) + (atomic_cas_size ty (put_in_reg addr) (put_in_reg e) (put_in_reg x))) +(rule (lower (atomic_load (fits_in_32 ty) _flags addr)) + (atomic_load_size ty (put_in_reg addr))) +(rule (lower (atomic_store _flags src @ (value_type (fits_in_32 ty)) addr)) + (atomic_store_size ty (put_in_reg src) (put_in_reg addr))) + +;; 64-bit atomics use the paired exclusive instructions (priority 4 so `$I64` +;; wins over the `fits_in_32` rules, which ISLE can't prove disjoint). +(rule 4 (lower (atomic_rmw $I64 _flags op addr x)) + (atomic_rmw_i64 op (put_in_reg addr) (put_in_regs x))) +(rule 4 (lower (atomic_cas $I64 _flags addr e x)) + (atomic_cas_i64 (put_in_reg addr) (put_in_regs e) (put_in_regs x))) +(rule 4 (lower (atomic_load $I64 _flags addr)) + (atomic_load_i64 (put_in_reg addr))) +(rule 4 (lower (atomic_store _flags src @ (value_type $I64) addr)) + (atomic_store_i64 (put_in_regs src) (put_in_reg addr))) + +;;;; Floating point ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(rule (lower (f32const _ (u32_from_ieee32 n))) + (f32_const (u32_to_u64 n))) +(rule (lower (f64const _ (u64_from_ieee64 n))) + (f64_const n)) + +(rule (lower (fadd $F32 x y)) + (fpu_rrr (FpuOp3.Vadd) (FpuSize.F32) (put_in_reg x) (put_in_reg y))) +(rule (lower (fadd $F64 x y)) + (fpu_rrr (FpuOp3.Vadd) (FpuSize.F64) (put_in_reg x) (put_in_reg y))) +(rule (lower (fsub $F32 x y)) + (fpu_rrr (FpuOp3.Vsub) (FpuSize.F32) (put_in_reg x) (put_in_reg y))) +(rule (lower (fsub $F64 x y)) + (fpu_rrr (FpuOp3.Vsub) (FpuSize.F64) (put_in_reg x) (put_in_reg y))) +(rule (lower (fmul $F32 x y)) + (fpu_rrr (FpuOp3.Vmul) (FpuSize.F32) (put_in_reg x) (put_in_reg y))) +(rule (lower (fmul $F64 x y)) + (fpu_rrr (FpuOp3.Vmul) (FpuSize.F64) (put_in_reg x) (put_in_reg y))) +(rule (lower (fdiv $F32 x y)) + (fpu_rrr (FpuOp3.Vdiv) (FpuSize.F32) (put_in_reg x) (put_in_reg y))) +(rule (lower (fdiv $F64 x y)) + (fpu_rrr (FpuOp3.Vdiv) (FpuSize.F64) (put_in_reg x) (put_in_reg y))) + +(rule (lower (fneg $F32 x)) + (fpu_rr (FpuOp2.Vneg) (FpuSize.F32) (put_in_reg x))) +(rule (lower (fneg $F64 x)) + (fpu_rr (FpuOp2.Vneg) (FpuSize.F64) (put_in_reg x))) +(rule (lower (fabs $F32 x)) + (fpu_rr (FpuOp2.Vabs) (FpuSize.F32) (put_in_reg x))) +(rule (lower (fabs $F64 x)) + (fpu_rr (FpuOp2.Vabs) (FpuSize.F64) (put_in_reg x))) +(rule (lower (sqrt $F32 x)) + (fpu_rr (FpuOp2.Vsqrt) (FpuSize.F32) (put_in_reg x))) +(rule (lower (sqrt $F64 x)) + (fpu_rr (FpuOp2.Vsqrt) (FpuSize.F64) (put_in_reg x))) + +;; Priority 1 so the float result/value type wins over the integer load/store +;; rules (ISLE can't prove `$F32` disjoint from `fits_in_32`). +(rule 1 (lower (load $F32 flags addr offset)) + (fpu_load (FpuSize.F32) (amode addr offset))) +(rule 1 (lower (load $F64 flags addr offset)) + (fpu_load (FpuSize.F64) (amode addr offset))) +(rule 1 (lower (store flags val @ (value_type $F32) addr offset)) + (fpu_store (FpuSize.F32) (put_in_reg val) (amode addr offset))) +(rule 1 (lower (store flags val @ (value_type $F64) addr offset)) + (fpu_store (FpuSize.F64) (put_in_reg val) (amode addr offset))) + +;; Float comparisons. +(rule (lower (fcmp _ cc a @ (value_type $F32) b)) + (let ((_ Unit (emit_side_effect (vcmp (FpuSize.F32) (put_in_reg a) (put_in_reg b))))) + (csetv (cond_from_floatcc cc)))) +(rule (lower (fcmp _ cc a @ (value_type $F64) b)) + (let ((_ Unit (emit_side_effect (vcmp (FpuSize.F64) (put_in_reg a) (put_in_reg b))))) + (csetv (cond_from_floatcc cc)))) + +;; `one` (ordered and not-equal) and `ueq` (unordered or equal) can't be +;; expressed as a single ARM condition, so they get a two-condition sequence. +(rule 1 (lower (fcmp _ (FloatCC.OrderedNotEqual) a @ (value_type $F32) b)) + (gen_fcmp_one (FpuSize.F32) (put_in_reg a) (put_in_reg b))) +(rule 1 (lower (fcmp _ (FloatCC.OrderedNotEqual) a @ (value_type $F64) b)) + (gen_fcmp_one (FpuSize.F64) (put_in_reg a) (put_in_reg b))) +(rule 1 (lower (fcmp _ (FloatCC.UnorderedOrEqual) a @ (value_type $F32) b)) + (gen_fcmp_ueq (FpuSize.F32) (put_in_reg a) (put_in_reg b))) +(rule 1 (lower (fcmp _ (FloatCC.UnorderedOrEqual) a @ (value_type $F64) b)) + (gen_fcmp_ueq (FpuSize.F64) (put_in_reg a) (put_in_reg b))) + +;; f32 <-> f64. +(rule (lower (fpromote $F64 x)) + (vcvt_ff true (put_in_reg x))) +(rule (lower (fdemote $F32 x)) + (vcvt_ff false (put_in_reg x))) + +;; int -> float. +(rule (lower (fcvt_from_sint $F32 x @ (value_type $I32))) + (vcvt_from_int true (FpuSize.F32) (put_in_reg x))) +(rule (lower (fcvt_from_sint $F64 x @ (value_type $I32))) + (vcvt_from_int true (FpuSize.F64) (put_in_reg x))) +(rule (lower (fcvt_from_uint $F32 x @ (value_type $I32))) + (vcvt_from_int false (FpuSize.F32) (put_in_reg x))) +(rule (lower (fcvt_from_uint $F64 x @ (value_type $I32))) + (vcvt_from_int false (FpuSize.F64) (put_in_reg x))) + +;; float -> i32 (saturating). +(rule (lower (fcvt_to_sint_sat $I32 x @ (value_type $F32))) + (vcvt_to_int true (FpuSize.F32) (put_in_reg x))) +(rule (lower (fcvt_to_sint_sat $I32 x @ (value_type $F64))) + (vcvt_to_int true (FpuSize.F64) (put_in_reg x))) +(rule (lower (fcvt_to_uint_sat $I32 x @ (value_type $F32))) + (vcvt_to_int false (FpuSize.F32) (put_in_reg x))) +(rule (lower (fcvt_to_uint_sat $I32 x @ (value_type $F64))) + (vcvt_to_int false (FpuSize.F64) (put_in_reg x))) + +;; fmin/fmax (WebAssembly NaN-propagating). +(rule (lower (fmin $F32 x y)) + (gen_fminmax false (FpuSize.F32) (put_in_reg x) (put_in_reg y))) +(rule (lower (fmin $F64 x y)) + (gen_fminmax false (FpuSize.F64) (put_in_reg x) (put_in_reg y))) +(rule (lower (fmax $F32 x y)) + (gen_fminmax true (FpuSize.F32) (put_in_reg x) (put_in_reg y))) +(rule (lower (fmax $F64 x y)) + (gen_fminmax true (FpuSize.F64) (put_in_reg x) (put_in_reg y))) + +;; fcopysign. +(rule (lower (fcopysign $F32 x y)) + (gen_copysign_f32 (put_in_reg x) (put_in_reg y))) +(rule (lower (fcopysign $F64 x y)) + (gen_copysign_f64 (put_in_reg x) (put_in_reg y))) + +;; Float-valued select (priority 2 so it wins over the fits_in_32 rule for f32). +(rule 2 (lower (select (ty_scalar_float _) c a b)) + (let ((_ Unit (emit_side_effect (cmp_imm (put_in_reg c) 0)))) + (fpu_csel (Cond.Ne) (put_in_reg a) (put_in_reg b)))) + +;; Bitcasts between the integer and float register files. +(rule (lower (bitcast $I32 _ x @ (value_type $F32))) + (mov_from_fpu32 (put_in_reg x))) +(rule (lower (bitcast $F32 _ x @ (value_type $I32))) + (mov_to_fpu32 (put_in_reg x))) +(rule (lower (bitcast $I64 _ x @ (value_type $F64))) + (mov_from_fpu64 (put_in_reg x))) +(rule (lower (bitcast $F64 _ x @ (value_type $I64))) + (mov_to_fpu64 (put_in_regs x))) + +;;;; Divides (only when hardware `sdiv`/`udiv` is available) ;;;;;;;;;;;;;;;;;;; +;; `gen_sdiv`/`gen_udiv` emit the trap-on-zero check (and, for `sdiv`, the +;; `INT_MIN / -1` overflow check) before the divide. + +(rule (lower (sdiv $I32 x y)) + (if-let true (use_idiv)) + (gen_sdiv (put_in_reg x) (put_in_reg y))) +(rule (lower (udiv $I32 x y)) + (if-let true (use_idiv)) + (gen_udiv (put_in_reg x) (put_in_reg y))) + +;;;; Counting / byte-swap / bit-reverse ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(rule (lower (clz $I32 x)) + (bit_rr (BitOp.Clz) (put_in_reg x))) +(rule (lower (bswap $I32 x)) + (bit_rr (BitOp.Rev) (put_in_reg x))) +(rule (lower (bswap $I16 x)) + (bit_rr (BitOp.Rev16) (put_in_reg x))) +(rule (lower (bitrev $I32 x)) + (bit_rr (BitOp.Rbit) (put_in_reg x))) + +;;;; Integer extends / reduces ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(rule (lower (sextend $I32 x @ (value_type $I8))) + (ext_rr (ExtOp.Sxtb) (put_in_reg x))) +(rule (lower (sextend $I32 x @ (value_type $I16))) + (ext_rr (ExtOp.Sxth) (put_in_reg x))) +(rule (lower (uextend $I32 x @ (value_type $I8))) + (ext_rr (ExtOp.Uxtb) (put_in_reg x))) +(rule (lower (uextend $I32 x @ (value_type $I16))) + (ext_rr (ExtOp.Uxth) (put_in_reg x))) + +;; Reducing to a narrower integer is free: the value already lives in a +;; 32-bit register and consumers read only the low bits. +(rule (lower (ireduce (fits_in_32 _) x)) + (put_in_reg x)) + +;;;; `icmp` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +;; 32-bit comparison producing a 0/1 boolean. `emit_icmp_cmp` widens narrow +;; (i8/i16) operands to 32 bits per the comparison's signedness before the +;; `cmp`, since Cranelift leaves the high bits of narrow values undefined. +(rule (lower (icmp _ cc a @ (value_type (fits_in_32 _)) b)) + (csetv (emit_icmp_cmp cc a b))) + +;; 64-bit comparison producing a 0/1 boolean. +(rule 4 (lower (icmp _ cc a @ (value_type $I64) b)) + (csetv (lower_icmp_i64 cc (put_in_regs a) (put_in_regs b)))) + +;;;; `select` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +;; Fuse an `icmp` condition into the compare that precedes the select. +(rule 1 (lower (select (fits_in_32 _) (icmp _ cc x @ (value_type (fits_in_32 _)) y) a b)) + (let ((cond Cond (emit_icmp_cmp cc x y))) + (csel cond (put_in_reg a) (put_in_reg b)))) + +;; Generic select: branch on whether the condition value is non-zero. +(rule (lower (select (fits_in_32 _) c a b)) + (let ((_ Unit (emit_side_effect (cmp_imm (put_in_reg c) 0)))) + (csel (Cond.Ne) (put_in_reg a) (put_in_reg b)))) + +;; 64-bit select: pick each half of the pair. The operands are materialized +;; before the compare so nothing clobbers the flags in between. +(rule 4 (lower (select $I64 c a b)) + (let ((av ValueRegs (put_in_regs a)) + (bv ValueRegs (put_in_regs b)) + (_ Unit (emit_side_effect (cmp_imm (put_in_reg c) 0)))) + (value_regs (csel (Cond.Ne) (vr_lo av) (vr_lo bv)) + (csel (Cond.Ne) (vr_hi av) (vr_hi bv))))) + +;;;; Shifts: `ishl` / `ushr` / `sshr` / `rotr` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(rule 1 (lower (ishl $I32 x (iconst _ (u64_from_imm64 n)))) + (gen_shift_imm (ShiftOp.Lsl) (put_in_reg x) n)) +(rule (lower (ishl $I32 x y)) + (shift_reg (ShiftOp.Lsl) (put_in_reg x) y)) + +(rule 1 (lower (ushr $I32 x (iconst _ (u64_from_imm64 n)))) + (gen_shift_imm (ShiftOp.Lsr) (put_in_reg x) n)) +(rule (lower (ushr $I32 x y)) + (shift_reg (ShiftOp.Lsr) (put_in_reg x) y)) + +(rule 1 (lower (sshr $I32 x (iconst _ (u64_from_imm64 n)))) + (gen_shift_imm (ShiftOp.Asr) (put_in_reg x) n)) +(rule (lower (sshr $I32 x y)) + (shift_reg (ShiftOp.Asr) (put_in_reg x) y)) + +(rule 1 (lower (rotr $I32 x (iconst _ (u64_from_imm64 n)))) + (gen_shift_imm (ShiftOp.Ror) (put_in_reg x) n)) +(rule (lower (rotr $I32 x y)) + (shift_reg (ShiftOp.Ror) (put_in_reg x) y)) + +;;;; Loads ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +;; A plain `load` reads exactly `sizeof(ty)` bytes; narrow types zero-extend +;; into the 32-bit result register. +(rule 1 (lower (load $I8 flags addr offset)) + (arm_load (amode addr offset) (LoadKind.UByte))) +(rule 1 (lower (load $I16 flags addr offset)) + (arm_load (amode addr offset) (LoadKind.UHalf))) +(rule (lower (load (fits_in_32 _) flags addr offset)) + (arm_load (amode addr offset) (LoadKind.Word))) + +(rule (lower (uload8 _ flags addr offset)) + (arm_load (amode addr offset) (LoadKind.UByte))) +(rule (lower (sload8 _ flags addr offset)) + (arm_load (amode addr offset) (LoadKind.SByte))) +(rule (lower (uload16 _ flags addr offset)) + (arm_load (amode addr offset) (LoadKind.UHalf))) +(rule (lower (sload16 _ flags addr offset)) + (arm_load (amode addr offset) (LoadKind.SHalf))) + +;;;; Stores ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +;; A plain `store` writes exactly `sizeof(ty)` bytes. +(rule 1 (lower (store flags val @ (value_type $I8) addr offset)) + (arm_store (put_in_reg val) (amode addr offset) (StoreKind.Byte))) +(rule 1 (lower (store flags val @ (value_type $I16) addr offset)) + (arm_store (put_in_reg val) (amode addr offset) (StoreKind.Half))) +(rule (lower (store flags val @ (value_type (fits_in_32 _)) addr offset)) + (arm_store (put_in_reg val) (amode addr offset) (StoreKind.Word))) + +(rule (lower (istore8 flags val addr offset)) + (arm_store (put_in_reg val) (amode addr offset) (StoreKind.Byte))) +(rule (lower (istore16 flags val addr offset)) + (arm_store (put_in_reg val) (amode addr offset) (StoreKind.Half))) + +;;;; `return` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(rule (lower (return args)) + (lower_return args)) + +;;;; Calls ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +;; Direct call to an in-range function (`bl`). +(rule 1 (lower (call (func_ref_data sig_ref name (RelocDistance.Near) patchable) args)) + (let ((output ValueRegsVec (gen_call_output sig_ref)) + (abi Sig (abi_sig sig_ref)) + (uses CallArgList (gen_call_args abi args)) + (defs CallRetList (gen_call_rets abi output)) + (info BoxCallInfo (gen_call_info abi name uses defs (try_call_none) patchable)) + (_ Unit (emit_side_effect (call_impl info)))) + output)) + +;; Direct call to a far/non-colocated function: materialize the callee's +;; absolute address, then call it indirectly (`blx`). +(rule 0 (lower (call (func_ref_data sig_ref name _dist _patchable) args)) + (let ((output ValueRegsVec (gen_call_output sig_ref)) + (abi Sig (abi_sig sig_ref)) + (target Reg (load_ext_name name 0)) + (uses CallArgList (gen_call_args abi args)) + (defs CallRetList (gen_call_rets abi output)) + (info BoxCallIndInfo (gen_call_ind_info abi target uses defs (try_call_none))) + (_ Unit (emit_side_effect (call_ind_impl info)))) + output)) + +;; Indirect call through a register (`blx`). +(rule (lower (call_indirect sig_ref ptr args)) + (let ((output ValueRegsVec (gen_call_output sig_ref)) + (abi Sig (abi_sig sig_ref)) + (target Reg (put_in_reg ptr)) + (uses CallArgList (gen_call_args abi args)) + (defs CallRetList (gen_call_rets abi output)) + (info BoxCallIndInfo (gen_call_ind_info abi target uses defs (try_call_none))) + (_ Unit (emit_side_effect (call_ind_impl info)))) + output)) + +;; Address of a function symbol. +(rule (lower (func_addr _ (func_ref_data _ name _dist _))) + (load_ext_name name 0)) + +;; Address of a data/global symbol (plus a constant byte offset). +(rule (lower (symbol_value _ (symbol_value_data name _dist offset))) + (load_ext_name name offset)) + +;;;; Branches ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(decl partial lower_branch (Inst MachLabelSlice) Unit) + +(rule (lower_branch (jump _) (single_target label)) + (emit_side_effect (jump_impl label))) + +(rule (lower_branch (br_table index _) targets) + (lower_br_table (put_in_reg index) targets)) + +;; Fuse a 32-bit `icmp` feeding a `brif` into a compare plus conditional branch. +(rule 1 (lower_branch (brif (icmp _ cc a @ (value_type (fits_in_32 _)) b) _ _) (two_targets taken not_taken)) + (emit_side_effect (cond_br (emit_icmp_cmp cc a b) taken not_taken))) + +;; Fuse a 64-bit `icmp` feeding a `brif`. +(rule 2 (lower_branch (brif (icmp _ cc a @ (value_type $I64) b) _ _) (two_targets taken not_taken)) + (emit_side_effect + (cond_br (lower_icmp_i64 cc (put_in_regs a) (put_in_regs b)) taken not_taken))) + +;; `one`/`ueq` feeding a `brif`: materialize the two-condition boolean, then +;; branch on whether it is non-zero (priority 2 beats the generic fused rule, +;; whose single-condition mapping can't represent these). +(rule 2 (lower_branch (brif (fcmp _ (FloatCC.OrderedNotEqual) a @ (value_type $F32) b) _ _) (two_targets taken not_taken)) + (emit_side_effect (side_effect_concat + (cmp_imm (gen_fcmp_one (FpuSize.F32) (put_in_reg a) (put_in_reg b)) 0) + (cond_br (Cond.Ne) taken not_taken)))) +(rule 2 (lower_branch (brif (fcmp _ (FloatCC.OrderedNotEqual) a @ (value_type $F64) b) _ _) (two_targets taken not_taken)) + (emit_side_effect (side_effect_concat + (cmp_imm (gen_fcmp_one (FpuSize.F64) (put_in_reg a) (put_in_reg b)) 0) + (cond_br (Cond.Ne) taken not_taken)))) +(rule 2 (lower_branch (brif (fcmp _ (FloatCC.UnorderedOrEqual) a @ (value_type $F32) b) _ _) (two_targets taken not_taken)) + (emit_side_effect (side_effect_concat + (cmp_imm (gen_fcmp_ueq (FpuSize.F32) (put_in_reg a) (put_in_reg b)) 0) + (cond_br (Cond.Ne) taken not_taken)))) +(rule 2 (lower_branch (brif (fcmp _ (FloatCC.UnorderedOrEqual) a @ (value_type $F64) b) _ _) (two_targets taken not_taken)) + (emit_side_effect (side_effect_concat + (cmp_imm (gen_fcmp_ueq (FpuSize.F64) (put_in_reg a) (put_in_reg b)) 0) + (cond_br (Cond.Ne) taken not_taken)))) + +;; Fuse an `fcmp` feeding a `brif`. +(rule 1 (lower_branch (brif (fcmp _ cc a @ (value_type $F32) b) _ _) (two_targets taken not_taken)) + (emit_side_effect + (side_effect_concat + (vcmp (FpuSize.F32) (put_in_reg a) (put_in_reg b)) + (cond_br (cond_from_floatcc cc) taken not_taken)))) +(rule 1 (lower_branch (brif (fcmp _ cc a @ (value_type $F64) b) _ _) (two_targets taken not_taken)) + (emit_side_effect + (side_effect_concat + (vcmp (FpuSize.F64) (put_in_reg a) (put_in_reg b)) + (cond_br (cond_from_floatcc cc) taken not_taken)))) + +;; Generic `brif`: branch when the value is non-zero. +(rule (lower_branch (brif v _ _) (two_targets taken not_taken)) + (emit_side_effect + (side_effect_concat + (cmp_imm (put_in_reg v) 0) + (cond_br (Cond.Ne) taken not_taken)))) diff --git a/cranelift/codegen/src/isa/arm32/lower.rs b/cranelift/codegen/src/isa/arm32/lower.rs new file mode 100644 index 000000000000..d4b67ff5d323 --- /dev/null +++ b/cranelift/codegen/src/isa/arm32/lower.rs @@ -0,0 +1,30 @@ +//! Lowering rules for Arm32. + +use crate::ir::Inst as IRInst; +use crate::isa::arm32::Arm32Backend; +use crate::isa::arm32::inst::*; +use crate::machinst::lower::*; +use crate::machinst::*; + +pub mod isle; + +impl LowerBackend for Arm32Backend { + type MInst = Inst; + + fn lower(&self, ctx: &mut Lower, ir_inst: IRInst) -> Option { + isle::lower(ctx, self, ir_inst) + } + + fn lower_branch( + &self, + ctx: &mut Lower, + ir_inst: IRInst, + targets: &[MachLabel], + ) -> Option<()> { + isle::lower_branch(ctx, self, ir_inst, targets) + } + + fn maybe_pinned_reg(&self) -> Option { + None + } +} diff --git a/cranelift/codegen/src/isa/arm32/lower/isle.rs b/cranelift/codegen/src/isa/arm32/lower/isle.rs new file mode 100644 index 000000000000..409fd3243e27 --- /dev/null +++ b/cranelift/codegen/src/isa/arm32/lower/isle.rs @@ -0,0 +1,512 @@ +//! ISLE integration glue code for arm32 lowering. + +// Pull in the ISLE generated code. +pub mod generated_code; +use generated_code::MInst; + +// Types that the generated ISLE code refers to via `use super::*`. +use crate::ir::condcodes::{FloatCC, IntCC}; +use crate::ir::immediates::*; +use crate::ir::types::*; +use crate::ir::{ + BlockCall, ExternalName, Inst, InstructionData, MemFlags, Opcode, TrapCode, Value, ValueList, +}; +use crate::isa::arm32::Arm32Backend; +use crate::isa::arm32::inst::{ALUOp, CmpOp, Cond, ExtOp, ShiftOp, encode_rotated_imm}; +use crate::machinst::isle::*; +use crate::machinst::{ + ArgPair, CallArgList, CallInfo, CallRetList, InstOutput, Lower, MachInst, MachLabel, RetPair, + Sig, TryCallInfo, VCodeConstant, VCodeConstantData, VCodeInst, +}; +use alloc::boxed::Box; +use alloc::vec::Vec; +use regalloc2::PReg; + +type VecArgPair = Vec; +type VecRetPair = Vec; +type VecMachLabel = Vec; +type BoxCallInfo = Box>; +type BoxCallIndInfo = Box>; + +/// The ISLE lowering context for arm32. +pub(crate) struct Arm32IsleContext<'a, 'b, I, B> +where + I: VCodeInst, + B: LowerBackend, +{ + pub lower_ctx: &'a mut Lower<'b, I>, + pub backend: &'a B, +} + +impl<'a, 'b> Arm32IsleContext<'a, 'b, MInst, Arm32Backend> { + fn new(lower_ctx: &'a mut Lower<'b, MInst>, backend: &'a Arm32Backend) -> Self { + Self { lower_ctx, backend } + } + + pub(crate) fn dfg(&self) -> &crate::ir::DataFlowGraph { + &self.lower_ctx.f.dfg + } + + /// Emit a single 32-bit shift `op rd, rm, #amount` (1 <= amount <= 31). + fn shift_raw(&mut self, op: ShiftOp, rm: Reg, amount: u8) -> Reg { + let rd = self.lower_ctx.alloc_tmp(I32).only_reg().unwrap(); + self.lower_ctx.emit(MInst::ShiftImm { op, rd, rm, amount }); + rd.to_reg() + } + + /// Emit `orr rd, a, b`. + fn orr_raw(&mut self, a: Reg, b: Reg) -> Reg { + self.alu_raw(ALUOp::Orr, false, a, b) + } + + /// Emit `op rd, rn, #imm12` (immediate already rotated-encoded). + fn alu_imm_raw(&mut self, op: ALUOp, rn: Reg, imm12: u32) -> Reg { + let rd = self.lower_ctx.alloc_tmp(I32).only_reg().unwrap(); + self.lower_ctx.emit(MInst::AluRRImm { op, rd, rn, imm12 }); + rd.to_reg() + } + + /// `vmov r, s` — move an S register's bits into a GPR. + fn mov_from_s_raw(&mut self, s: Reg) -> Reg { + let rt = self.lower_ctx.alloc_tmp(I32).only_reg().unwrap(); + self.lower_ctx.emit(MInst::MovFromFpu32 { rt, rm: s }); + rt.to_reg() + } + + /// `vmov s, r` — move a GPR's bits into an S register. + fn mov_to_s_raw(&mut self, gpr: Reg) -> Reg { + let rd = self.lower_ctx.alloc_tmp(F32).only_reg().unwrap(); + self.lower_ctx.emit(MInst::MovToFpu32 { rd, rt: gpr }); + rd.to_reg() + } + + /// `vmov rlo, rhi, d` — move a D register's bits into two GPRs. + fn mov_from_d_raw(&mut self, d: Reg) -> (Reg, Reg) { + let lo = self.lower_ctx.alloc_tmp(I32).only_reg().unwrap(); + let hi = self.lower_ctx.alloc_tmp(I32).only_reg().unwrap(); + self.lower_ctx.emit(MInst::MovFromFpu64 { + rt_lo: lo, + rt_hi: hi, + rm: d, + }); + (lo.to_reg(), hi.to_reg()) + } + + /// `vmov d, rlo, rhi` — move two GPRs into a D register. + fn mov_to_d_raw(&mut self, lo: Reg, hi: Reg) -> Reg { + let rd = self.lower_ctx.alloc_tmp(F64).only_reg().unwrap(); + self.lower_ctx.emit(MInst::MovToFpu64 { + rd, + rt_lo: lo, + rt_hi: hi, + }); + rd.to_reg() + } + + /// Emit `op{s} rd, rn, rm` into a fresh register (which is returned). + fn alu_raw(&mut self, op: ALUOp, set_flags: bool, rn: Reg, rm: Reg) -> Reg { + let rd = self.lower_ctx.alloc_tmp(I32).only_reg().unwrap(); + let inst = if set_flags { + MInst::AluRRRFlags { op, rd, rn, rm } + } else { + MInst::AluRRR { op, rd, rn, rm } + }; + self.lower_ctx.emit(inst); + rd.to_reg() + } + + /// Widen a narrow (i8/i16) comparison operand to a full 32-bit register, + /// sign- or zero-extending per `signed`. i32 (and anything wider) is + /// returned unchanged. + fn extend_for_cmp(&mut self, ty: Type, signed: bool, r: Reg) -> Reg { + let op = match (ty, signed) { + (I8, true) => ExtOp::Sxtb, + (I8, false) => ExtOp::Uxtb, + (I16, true) => ExtOp::Sxth, + (I16, false) => ExtOp::Uxth, + _ => return r, + }; + let rd = self.lower_ctx.alloc_tmp(I32).only_reg().unwrap(); + self.lower_ctx.emit(MInst::ExtRR { op, rd, rm: r }); + rd.to_reg() + } + + /// Emit `cmp divisor, #0; trapif eq, int_divz`. + fn emit_divz_check(&mut self, divisor: Reg) { + self.lower_ctx.emit(MInst::CmpRImm { + op: CmpOp::Cmp, + rn: divisor, + imm12: 0, + }); + self.lower_ctx.emit(MInst::TrapIf { + cond: Cond::Eq, + code: TrapCode::INTEGER_DIVISION_BY_ZERO, + }); + } +} + +impl generated_code::Context for Arm32IsleContext<'_, '_, MInst, Arm32Backend> { + isle_lower_prelude_methods!(); + + fn emit(&mut self, inst: &MInst) -> Unit { + self.lower_ctx.emit(inst.clone()); + } + + fn lower_br_table(&mut self, index: Reg, targets: &[MachLabel]) -> Unit { + let tmp = self.lower_ctx.alloc_tmp(I32).only_reg().unwrap(); + self.lower_ctx.emit(MInst::BrTable { + index, + tmp, + targets: targets.to_vec(), + }); + } + + fn gen_call_info( + &mut self, + sig: Sig, + dest: ExternalName, + uses: CallArgList, + defs: CallRetList, + try_call_info: Option, + patchable: bool, + ) -> BoxCallInfo { + let stack_ret_space = self.lower_ctx.sigs()[sig].sized_stack_ret_space(); + let stack_arg_space = self.lower_ctx.sigs()[sig].sized_stack_arg_space(); + self.lower_ctx + .abi_mut() + .accumulate_outgoing_args_size(stack_ret_space + stack_arg_space); + Box::new( + self.lower_ctx + .gen_call_info(sig, dest, uses, defs, try_call_info, patchable), + ) + } + + fn gen_call_ind_info( + &mut self, + sig: Sig, + dest: Reg, + uses: CallArgList, + defs: CallRetList, + try_call_info: Option, + ) -> BoxCallIndInfo { + let stack_ret_space = self.lower_ctx.sigs()[sig].sized_stack_ret_space(); + let stack_arg_space = self.lower_ctx.sigs()[sig].sized_stack_arg_space(); + self.lower_ctx + .abi_mut() + .accumulate_outgoing_args_size(stack_ret_space + stack_arg_space); + Box::new( + self.lower_ctx + .gen_call_info(sig, dest, uses, defs, try_call_info, false), + ) + } + + /// Materialize a 32-bit constant into a register with the shortest sequence. + fn gen_constant(&mut self, val: u64) -> Reg { + let val = val as u32; + let rd = self.lower_ctx.alloc_tmp(I32).only_reg().unwrap(); + let inst = if let Some(imm12) = encode_rotated_imm(val) { + MInst::MovRotImm { rd, imm12 } + } else if let Some(imm12) = encode_rotated_imm(!val) { + MInst::MvnRotImm { rd, imm12 } + } else if val >> 16 == 0 { + MInst::Movw { rd, imm16: val } + } else { + MInst::MovImm { + rd, + imm: u64::from(val), + } + }; + self.lower_ctx.emit(inst); + rd.to_reg() + } + + /// Succeeds if the low 32 bits of `val` are encodable as a rotated imm12. + fn u64_from_rotated_imm12(&mut self, val: u64) -> Option { + encode_rotated_imm(val as u32) + } + + /// The high 32 bits of a 64-bit constant. + fn u64_high32(&mut self, val: u64) -> u64 { + val >> 32 + } + + /// Zero-extend a u32 to u64. + fn u32_to_u64(&mut self, val: u32) -> u64 { + u64::from(val) + } + + /// The offset of the high word of a 64-bit memory access. + fn offset32_plus4(&mut self, offset: Offset32) -> i32 { + i32::from(offset) + 4 + } + + /// Shift/rotate a 64-bit `(lo, hi)` pair by a constant amount, emitting the + /// funnel-shift sequence. Only Lsl/Lsr/Asr are supported. + fn gen_i64_shift_imm(&mut self, op: &ShiftOp, x: ValueRegs, amount: u64) -> ValueRegs { + let n = (amount & 63) as u8; + let xlo = x.regs()[0]; + let xhi = x.regs()[1]; + if n == 0 { + return ValueRegs::two(xlo, xhi); + } + match *op { + ShiftOp::Lsl => { + if n < 32 { + let lo = self.shift_raw(ShiftOp::Lsl, xlo, n); + let hi_hi = self.shift_raw(ShiftOp::Lsl, xhi, n); + let hi_lo = self.shift_raw(ShiftOp::Lsr, xlo, 32 - n); + let hi = self.orr_raw(hi_hi, hi_lo); + ValueRegs::two(lo, hi) + } else { + let zero = self.gen_constant(0); + let hi = if n == 32 { + xlo + } else { + self.shift_raw(ShiftOp::Lsl, xlo, n - 32) + }; + ValueRegs::two(zero, hi) + } + } + ShiftOp::Lsr => { + if n < 32 { + let hi = self.shift_raw(ShiftOp::Lsr, xhi, n); + let lo_lo = self.shift_raw(ShiftOp::Lsr, xlo, n); + let lo_hi = self.shift_raw(ShiftOp::Lsl, xhi, 32 - n); + let lo = self.orr_raw(lo_lo, lo_hi); + ValueRegs::two(lo, hi) + } else { + let zero = self.gen_constant(0); + let lo = if n == 32 { + xhi + } else { + self.shift_raw(ShiftOp::Lsr, xhi, n - 32) + }; + ValueRegs::two(lo, zero) + } + } + ShiftOp::Asr => { + if n < 32 { + let hi = self.shift_raw(ShiftOp::Asr, xhi, n); + let lo_lo = self.shift_raw(ShiftOp::Lsr, xlo, n); + let lo_hi = self.shift_raw(ShiftOp::Lsl, xhi, 32 - n); + let lo = self.orr_raw(lo_lo, lo_hi); + ValueRegs::two(lo, hi) + } else { + // The high word becomes all sign bits; the low word is the + // old high word arithmetically shifted by `n - 32`. + let hi = self.shift_raw(ShiftOp::Asr, xhi, 31); + let lo = if n == 32 { + xhi + } else { + self.shift_raw(ShiftOp::Asr, xhi, n - 32) + }; + ValueRegs::two(lo, hi) + } + } + ShiftOp::Ror => unimplemented!("arm32: 64-bit rotate is not implemented"), + } + } + + /// Shift/rotate by a constant, applying Cranelift's modulo-width semantics. + /// A masked amount of zero becomes a plain register move. + fn gen_shift_imm(&mut self, op: &ShiftOp, rm: Reg, amount: u64) -> Reg { + let amount = (amount & 31) as u8; + let rd = self.lower_ctx.alloc_tmp(I32).only_reg().unwrap(); + let inst = if amount == 0 { + MInst::MovReg { rd, rm } + } else { + MInst::ShiftImm { + op: *op, + rd, + rm, + amount, + } + }; + self.lower_ctx.emit(inst); + rd.to_reg() + } + + /// Whether the hardware integer-divide instructions may be used. + fn use_idiv(&mut self) -> bool { + self.backend.isa_flags.has_idiv() + } + + /// Emit the compare sequence for a 64-bit `icmp` and return the ARM + /// condition code that tests its result. + /// + /// A 64-bit `subs`/`sbcs` sets N, V and C correctly for the full value but + /// its Z flag only reflects the high word, so it's only usable for the + /// conditions that don't depend on Z (lt/ge/lo/hs). The gt/le/hi/ls forms + /// are obtained by swapping the operands, and eq/ne use a separate + /// `eor`/`eor`/`orrs` reduction. + fn lower_icmp_i64(&mut self, cc: &IntCC, a: ValueRegs, b: ValueRegs) -> Cond { + let (alo, ahi) = (a.regs()[0], a.regs()[1]); + let (blo, bhi) = (b.regs()[0], b.regs()[1]); + match cc { + IntCC::Equal | IntCC::NotEqual => { + let tlo = self.alu_raw(ALUOp::Eor, false, alo, blo); + let thi = self.alu_raw(ALUOp::Eor, false, ahi, bhi); + // `orrs` sets Z iff both halves are equal, i.e. a == b. + let _ = self.alu_raw(ALUOp::Orr, true, tlo, thi); + if matches!(cc, IntCC::Equal) { + Cond::Eq + } else { + Cond::Ne + } + } + _ => { + // (lo1, hi1) - (lo2, hi2), then test `cond`. + let (lo1, hi1, lo2, hi2, cond) = match cc { + IntCC::SignedLessThan => (alo, ahi, blo, bhi, Cond::Lt), + IntCC::SignedGreaterThanOrEqual => (alo, ahi, blo, bhi, Cond::Ge), + IntCC::SignedGreaterThan => (blo, bhi, alo, ahi, Cond::Lt), + IntCC::SignedLessThanOrEqual => (blo, bhi, alo, ahi, Cond::Ge), + IntCC::UnsignedLessThan => (alo, ahi, blo, bhi, Cond::Lo), + IntCC::UnsignedGreaterThanOrEqual => (alo, ahi, blo, bhi, Cond::Hs), + IntCC::UnsignedGreaterThan => (blo, bhi, alo, ahi, Cond::Lo), + IntCC::UnsignedLessThanOrEqual => (blo, bhi, alo, ahi, Cond::Hs), + _ => unreachable!(), + }; + let _ = self.alu_raw(ALUOp::Sub, true, lo1, lo2); // subs + let _ = self.alu_raw(ALUOp::Sbc, true, hi1, hi2); // sbcs + cond + } + } + } + + /// Map a `FloatCC` to the ARM condition testing the flags left by a `vcmp` + /// (`vmrs`). The `vcmp` sets N/Z/C/V per an IEEE ordered comparison, with an + /// unordered result giving C=1, V=1. + fn cond_from_floatcc(&mut self, cc: &FloatCC) -> Cond { + match cc { + FloatCC::Equal => Cond::Eq, + FloatCC::NotEqual => Cond::Ne, + FloatCC::LessThan => Cond::Mi, + FloatCC::LessThanOrEqual => Cond::Ls, + FloatCC::GreaterThan => Cond::Gt, + FloatCC::GreaterThanOrEqual => Cond::Ge, + FloatCC::Ordered => Cond::Vc, + FloatCC::Unordered => Cond::Vs, + FloatCC::UnorderedOrLessThan => Cond::Lt, + FloatCC::UnorderedOrLessThanOrEqual => Cond::Le, + FloatCC::UnorderedOrGreaterThan => Cond::Hi, + FloatCC::UnorderedOrGreaterThanOrEqual => Cond::Hs, + // These need a two-condition sequence and are handled by dedicated + // lowering rules (`gen_fcmp_one` / `gen_fcmp_ueq`) before reaching + // this single-condition mapping. + FloatCC::OrderedNotEqual | FloatCC::UnorderedOrEqual => { + unreachable!("arm32: fcmp {cc:?} is lowered via a two-condition sequence") + } + } + } + + /// `fcopysign` for f32: clear the sign bit of `a` and OR in `b`'s sign, + /// working on the raw bits in GPRs. + fn gen_copysign_f32(&mut self, a: Reg, b: Reg) -> Reg { + let sign_mask = encode_rotated_imm(0x8000_0000).unwrap(); + let a_bits = self.mov_from_s_raw(a); + let b_bits = self.mov_from_s_raw(b); + let sign = self.alu_imm_raw(ALUOp::And, b_bits, sign_mask); + let cleared = self.alu_imm_raw(ALUOp::Bic, a_bits, sign_mask); + let res = self.alu_raw(ALUOp::Orr, false, cleared, sign); + self.mov_to_s_raw(res) + } + + /// `fcopysign` for f64: the sign bit lives in the high word. + fn gen_copysign_f64(&mut self, a: Reg, b: Reg) -> Reg { + let sign_mask = encode_rotated_imm(0x8000_0000).unwrap(); + let (a_lo, a_hi) = self.mov_from_d_raw(a); + let (_b_lo, b_hi) = self.mov_from_d_raw(b); + let sign = self.alu_imm_raw(ALUOp::And, b_hi, sign_mask); + let cleared = self.alu_imm_raw(ALUOp::Bic, a_hi, sign_mask); + let res_hi = self.alu_raw(ALUOp::Orr, false, cleared, sign); + self.mov_to_d_raw(a_lo, res_hi) + } + + /// Emit the compare for an `icmp` (widening narrow operands per the + /// comparison's signedness) and return the ARM condition that tests it. + fn emit_icmp_cmp(&mut self, cc: &IntCC, a: Value, b: Value) -> Cond { + let ty = self.value_type(a); + let signed = matches!( + cc, + IntCC::SignedLessThan + | IntCC::SignedGreaterThanOrEqual + | IntCC::SignedGreaterThan + | IntCC::SignedLessThanOrEqual + ); + let ra = self.put_in_reg(a); + let ra = self.extend_for_cmp(ty, signed, ra); + let rb = self.put_in_reg(b); + let rb = self.extend_for_cmp(ty, signed, rb); + self.lower_ctx.emit(MInst::CmpRR { + op: CmpOp::Cmp, + rn: ra, + rm: rb, + }); + self.cond_from_intcc(cc) + } + + /// `udiv` with a divide-by-zero trap check. + fn gen_udiv(&mut self, x: Reg, y: Reg) -> Reg { + self.emit_divz_check(y); + let rd = self.lower_ctx.alloc_tmp(I32).only_reg().unwrap(); + self.lower_ctx.emit(MInst::UDiv { rd, rn: x, rm: y }); + rd.to_reg() + } + + /// `sdiv` with divide-by-zero and `INT_MIN / -1` overflow trap checks. + fn gen_sdiv(&mut self, x: Reg, y: Reg) -> Reg { + self.emit_divz_check(y); + // Overflow occurs only for `INT_MIN / -1`, i.e. when both + // `x ^ INT_MIN == 0` and `y + 1 == 0`; OR the two together and trap when + // the result is zero. + let int_min = encode_rotated_imm(0x8000_0000).unwrap(); + let t1 = self.alu_imm_raw(ALUOp::Eor, x, int_min); + let one = encode_rotated_imm(1).unwrap(); + let t2 = self.alu_imm_raw(ALUOp::Add, y, one); + let _ = self.alu_raw(ALUOp::Orr, true, t1, t2); // orrs (sets Z on overflow) + self.lower_ctx.emit(MInst::TrapIf { + cond: Cond::Eq, + code: TrapCode::INTEGER_OVERFLOW, + }); + let rd = self.lower_ctx.alloc_tmp(I32).only_reg().unwrap(); + self.lower_ctx.emit(MInst::SDiv { rd, rn: x, rm: y }); + rd.to_reg() + } + + fn cond_from_intcc(&mut self, cc: &IntCC) -> Cond { + match cc { + IntCC::Equal => Cond::Eq, + IntCC::NotEqual => Cond::Ne, + IntCC::SignedLessThan => Cond::Lt, + IntCC::SignedGreaterThanOrEqual => Cond::Ge, + IntCC::SignedGreaterThan => Cond::Gt, + IntCC::SignedLessThanOrEqual => Cond::Le, + IntCC::UnsignedLessThan => Cond::Lo, + IntCC::UnsignedGreaterThanOrEqual => Cond::Hs, + IntCC::UnsignedGreaterThan => Cond::Hi, + IntCC::UnsignedLessThanOrEqual => Cond::Ls, + } + } +} + +/// The main entry point for lowering with ISLE. +pub(crate) fn lower( + lower_ctx: &mut Lower, + backend: &Arm32Backend, + inst: Inst, +) -> Option { + let mut isle_ctx = Arm32IsleContext::new(lower_ctx, backend); + generated_code::constructor_lower(&mut isle_ctx, inst) +} + +/// The main entry point for branch lowering with ISLE. +pub(crate) fn lower_branch( + lower_ctx: &mut Lower, + backend: &Arm32Backend, + branch: Inst, + targets: &[MachLabel], +) -> Option<()> { + let mut isle_ctx = Arm32IsleContext::new(lower_ctx, backend); + generated_code::constructor_lower_branch(&mut isle_ctx, branch, targets) +} diff --git a/cranelift/codegen/src/isa/arm32/lower/isle/generated_code.rs b/cranelift/codegen/src/isa/arm32/lower/isle/generated_code.rs new file mode 100644 index 000000000000..e3ac3badcadf --- /dev/null +++ b/cranelift/codegen/src/isa/arm32/lower/isle/generated_code.rs @@ -0,0 +1,15 @@ +// See https://github.com/rust-lang/rust/issues/47995: we cannot use `#![...]` attributes inside of +// the generated ISLE source below because we include!() it. We must include!() it because its path +// depends on an environment variable; and also because of this, we can't do the `#[path = "..."] +// mod generated_code;` trick either. +#![allow( + dead_code, + unreachable_patterns, + unused_imports, + unused_variables, + irrefutable_let_patterns, + clippy::clone_on_copy, + reason = "generated code" +)] + +include!(concat!(env!("ISLE_DIR"), "/isle_arm32.rs")); diff --git a/cranelift/codegen/src/isa/arm32/mod.rs b/cranelift/codegen/src/isa/arm32/mod.rs new file mode 100644 index 000000000000..ccf3bbfc72a2 --- /dev/null +++ b/cranelift/codegen/src/isa/arm32/mod.rs @@ -0,0 +1,214 @@ +//! Arm32 Instruction Set Architecture. + +use crate::dominator_tree::DominatorTree; +use crate::ir::{self, Function, Type}; +use crate::isa::arm32::settings as arm32_settings; +use crate::isa::{ + Builder as IsaBuilder, FunctionAlignment, IsaFlagsHashKey, OwnedTargetIsa, TargetIsa, +}; +use crate::machinst::{ + CompiledCodeStencil, MachInst, MachTextSectionBuilder, Reg, SigSet, TextSectionBuilder, VCode, + compile, +}; +use crate::result::CodegenResult; +use crate::settings::{self as shared_settings, Flags}; +use alloc::string::String; +use alloc::{boxed::Box, vec::Vec}; +use core::fmt; +use cranelift_control::ControlPlane; +use target_lexicon::{Architecture, Triple}; + +mod abi; +pub(crate) mod inst; +mod lower; +mod settings; + +use self::inst::EmitInfo; + +/// An arm32 backend. +pub struct Arm32Backend { + triple: Triple, + flags: shared_settings::Flags, + isa_flags: arm32_settings::Flags, +} + +impl Arm32Backend { + /// Create a new arm32 backend with the given (shared) flags. + pub fn new_with_flags( + triple: Triple, + flags: shared_settings::Flags, + isa_flags: arm32_settings::Flags, + ) -> Arm32Backend { + Arm32Backend { + triple, + flags, + isa_flags, + } + } + + /// Lower to VCode, register-allocate, compute block layout and finalize + /// branches. The result is ready for binary emission. + fn compile_vcode( + &self, + func: &Function, + domtree: &DominatorTree, + ctrl_plane: &mut ControlPlane, + ) -> CodegenResult<(VCode, regalloc2::Output)> { + let emit_info = EmitInfo::new(self.flags.clone(), self.isa_flags.clone()); + let sigs = SigSet::new::(func, &self.flags)?; + let abi = abi::Arm32Callee::new(func, self, &self.isa_flags, &sigs)?; + compile::compile::(func, domtree, self, abi, emit_info, sigs, ctrl_plane) + } +} + +impl TargetIsa for Arm32Backend { + fn compile_function( + &self, + func: &Function, + domtree: &DominatorTree, + want_disasm: bool, + ctrl_plane: &mut ControlPlane, + ) -> CodegenResult { + let (vcode, regalloc_result) = self.compile_vcode(func, domtree, ctrl_plane)?; + + let want_disasm = want_disasm || log::log_enabled!(log::Level::Debug); + let emit_result = vcode.emit(®alloc_result, want_disasm, &self.flags, ctrl_plane); + let value_labels_ranges = emit_result.value_labels_ranges; + let buffer = emit_result.buffer; + + if let Some(disasm) = emit_result.disasm.as_ref() { + log::debug!("disassembly:\n{disasm}"); + } + + Ok(CompiledCodeStencil { + buffer, + vcode: emit_result.disasm, + value_labels_ranges, + bb_starts: emit_result.bb_offsets, + bb_edges: emit_result.bb_edges, + }) + } + + fn name(&self) -> &'static str { + "arm32" + } + + fn dynamic_vector_bytes(&self, _dynamic_ty: ir::Type) -> u32 { + 0 + } + + fn triple(&self) -> &Triple { + &self.triple + } + + fn flags(&self) -> &shared_settings::Flags { + &self.flags + } + + fn isa_flags(&self) -> Vec { + self.isa_flags.iter().collect() + } + + fn isa_flags_hash_key(&self) -> IsaFlagsHashKey<'_> { + IsaFlagsHashKey(self.isa_flags.hash_key()) + } + + #[cfg(feature = "unwind")] + fn emit_unwind_info( + &self, + _result: &crate::machinst::CompiledCode, + _kind: crate::isa::unwind::UnwindInfoKind, + ) -> CodegenResult> { + // Unwind info is not yet emitted for arm32. + Ok(None) + } + + fn text_section_builder(&self, num_funcs: usize) -> Box { + Box::new(MachTextSectionBuilder::::new(num_funcs)) + } + + fn function_alignment(&self) -> FunctionAlignment { + inst::Inst::function_alignment() + } + + fn page_size_align_log2(&self) -> u8 { + debug_assert_eq!(1 << 12, 0x1000); + 12 + } + + #[cfg(feature = "disas")] + fn to_capstone(&self) -> Result { + use capstone::prelude::*; + let mut cs = Capstone::new() + .arm() + .mode(arch::arm::ArchMode::Arm) + .build()?; + cs.set_skipdata(true)?; + Ok(cs) + } + + fn pretty_print_reg(&self, reg: Reg, _size: u8) -> String { + inst::reg_name(reg) + } + + fn has_native_fma(&self) -> bool { + false + } + + fn has_round(&self) -> bool { + false + } + + fn has_blendv_lowering(&self, _: Type) -> bool { + false + } + + fn has_x86_pshufb_lowering(&self) -> bool { + false + } + + fn has_x86_pmulhrsw_lowering(&self) -> bool { + false + } + + fn has_x86_pmaddubsw_lowering(&self) -> bool { + false + } + + fn default_argument_extension(&self) -> ir::ArgumentExtension { + ir::ArgumentExtension::None + } +} + +impl fmt::Display for Arm32Backend { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + f.debug_struct("MachBackend") + .field("name", &self.name()) + .field("triple", &self.triple()) + .field("flags", &format!("{}", self.flags())) + .finish() + } +} + +/// Create a new `isa::Builder`. +pub fn isa_builder(triple: Triple) -> IsaBuilder { + match triple.architecture { + Architecture::Arm(..) => {} + _ => unreachable!(), + } + IsaBuilder { + triple, + setup: arm32_settings::builder(), + constructor: isa_constructor, + } +} + +fn isa_constructor( + triple: Triple, + shared_flags: Flags, + builder: &shared_settings::Builder, +) -> CodegenResult { + let isa_flags = arm32_settings::Flags::new(&shared_flags, builder); + let backend = Arm32Backend::new_with_flags(triple, shared_flags, isa_flags); + Ok(backend.wrapped()) +} diff --git a/cranelift/codegen/src/isa/arm32/settings.rs b/cranelift/codegen/src/isa/arm32/settings.rs new file mode 100644 index 000000000000..a3a500c65025 --- /dev/null +++ b/cranelift/codegen/src/isa/arm32/settings.rs @@ -0,0 +1,9 @@ +//! arm32 Settings. + +use crate::settings::{self, Builder, Value, detail}; +use core::fmt; + +// Include code generated by `cranelift/codegen/meta/src/gen_settings.rs`. This +// file contains a public `Flags` struct with an impl for all of the settings +// defined in `cranelift/codegen/meta/src/isa/arm32.rs`. +include!(concat!(env!("OUT_DIR"), "/settings-arm32.rs")); diff --git a/cranelift/codegen/src/isa/mod.rs b/cranelift/codegen/src/isa/mod.rs index 5f33e632fb72..7706bc63ce40 100644 --- a/cranelift/codegen/src/isa/mod.rs +++ b/cranelift/codegen/src/isa/mod.rs @@ -70,6 +70,9 @@ pub mod x64; #[cfg(feature = "arm64")] pub mod aarch64; +#[cfg(feature = "arm32")] +pub mod arm32; + #[cfg(feature = "riscv64")] pub mod riscv64; @@ -111,6 +114,11 @@ pub fn lookup(triple: Triple) -> Result { isa_builder!(x64, (feature = "x86"), triple) } Architecture::Aarch64 { .. } => isa_builder!(aarch64, (feature = "arm64"), triple), + // The arm32 backend only emits A32 encodings, so Thumb-only targets are + // not supported. + Architecture::Arm(arm) if !arm.is_thumb() => { + isa_builder!(arm32, (feature = "arm32"), triple) + } Architecture::S390x { .. } => isa_builder!(s390x, (feature = "s390x"), triple), Architecture::Riscv64 { .. } => isa_builder!(riscv64, (feature = "riscv64"), triple), Architecture::Pulley32 | Architecture::Pulley32be => { @@ -126,7 +134,7 @@ pub fn lookup(triple: Triple) -> Result { /// The string names of all the supported, but possibly not enabled, architectures. The elements of /// this slice are suitable to be passed to the [lookup_by_name] function to obtain the default /// configuration for that architecture. -pub const ALL_ARCHITECTURES: &[&str] = &["x86_64", "aarch64", "s390x", "riscv64"]; +pub const ALL_ARCHITECTURES: &[&str] = &["x86_64", "aarch64", "arm", "s390x", "riscv64"]; /// Look for a supported ISA with the given `name`. /// Return a builder that can create a corresponding `TargetIsa`. diff --git a/cranelift/codegen/src/machinst/lower.rs b/cranelift/codegen/src/machinst/lower.rs index ad2ffd99e1f0..759d7833997d 100644 --- a/cranelift/codegen/src/machinst/lower.rs +++ b/cranelift/codegen/src/machinst/lower.rs @@ -1613,6 +1613,7 @@ impl<'func, I: VCodeInst> Lower<'func, I> { /// Increment the reference count for the Value, ensuring that it gets lowered. #[cfg(any( feature = "x86", + feature = "arm32", feature = "arm64", feature = "riscv64", feature = "s390x", diff --git a/cranelift/filetests/filetests/isa/arm32/arithmetic.clif b/cranelift/filetests/filetests/isa/arm32/arithmetic.clif new file mode 100644 index 000000000000..245e7955f028 --- /dev/null +++ b/cranelift/filetests/filetests/isa/arm32/arithmetic.clif @@ -0,0 +1,68 @@ +test compile precise-output +target arm + +function %add(i32, i32) -> i32 { +block0(v0: i32, v1: i32): + v2 = iadd v0, v1 + return v2 +} + +; VCode: +; block0: +; add r0, r0, r1 +; bx lr +; +; Disassembled: +; block0: ; offset 0x0 +; add r0, r0, r1 +; bx lr + +function %add_imm(i32) -> i32 { +block0(v0: i32): + v1 = iconst.i32 10 + v2 = iadd v0, v1 + return v2 +} + +; VCode: +; block0: +; add r0, r0, #10 +; bx lr +; +; Disassembled: +; block0: ; offset 0x0 +; add r0, r0, #0xa +; bx lr + +function %sub(i32, i32) -> i32 { +block0(v0: i32, v1: i32): + v2 = isub v0, v1 + return v2 +} + +; VCode: +; block0: +; sub r0, r0, r1 +; bx lr +; +; Disassembled: +; block0: ; offset 0x0 +; sub r0, r0, r1 +; bx lr + +function %neg(i32) -> i32 { +block0(v0: i32): + v1 = ineg v0 + return v1 +} + +; VCode: +; block0: +; rsb r0, r0, #0 +; bx lr +; +; Disassembled: +; block0: ; offset 0x0 +; rsb r0, r0, #0 +; bx lr + diff --git a/cranelift/filetests/filetests/isa/arm32/atomic.clif b/cranelift/filetests/filetests/isa/arm32/atomic.clif new file mode 100644 index 000000000000..4694514d9c69 --- /dev/null +++ b/cranelift/filetests/filetests/isa/arm32/atomic.clif @@ -0,0 +1,622 @@ +test compile precise-output +target arm + +function %rmw_add(i32, i32) -> i32 { +block0(v0: i32, v1: i32): + v2 = atomic_rmw.i32 add v0, v1 + return v2 +} + +; VCode: +; push {fp, lr} +; mov fp, sp +; sub sp, sp, #8 +; str r7, [sp, #4] +; block0: +; mov r2, r0 +; mov r7, r1 +; atomic_rmw.Add r3, [r2], r7 +; mov r0, r3 +; ldr r7, [sp, #4] +; add sp, sp, #8 +; pop {fp, lr} +; bx lr +; +; Disassembled: +; block0: ; offset 0x0 +; push {fp, lr} +; mov fp, sp +; sub sp, sp, #8 +; str r7, [sp, #4] +; block1: ; offset 0x10 +; mov r2, r0 +; mov r7, r1 +; ldaex r3, [r2] +; add r1, r3, r7 +; stlex r0, r1, [r2] +; cmp r0, #0 +; bne #0x18 +; mov r0, r3 +; ldr r7, [sp, #4] +; add sp, sp, #8 +; pop {fp, lr} +; bx lr + +function %rmw_umax(i32, i32) -> i32 { +block0(v0: i32, v1: i32): + v2 = atomic_rmw.i32 umax v0, v1 + return v2 +} + +; VCode: +; push {fp, lr} +; mov fp, sp +; sub sp, sp, #8 +; str r7, [sp, #4] +; block0: +; mov r2, r0 +; mov r7, r1 +; atomic_rmw.Umax r3, [r2], r7 +; mov r0, r3 +; ldr r7, [sp, #4] +; add sp, sp, #8 +; pop {fp, lr} +; bx lr +; +; Disassembled: +; block0: ; offset 0x0 +; push {fp, lr} +; mov fp, sp +; sub sp, sp, #8 +; str r7, [sp, #4] +; block1: ; offset 0x10 +; mov r2, r0 +; mov r7, r1 +; ldaex r3, [r2] +; mov r1, r7 +; cmp r3, r7 +; movhi r1, r3 +; stlex r0, r1, [r2] +; cmp r0, #0 +; bne #0x18 +; mov r0, r3 +; ldr r7, [sp, #4] +; add sp, sp, #8 +; pop {fp, lr} +; bx lr + +function %cas(i32, i32, i32) -> i32 { +block0(v0: i32, v1: i32, v2: i32): + v3 = atomic_cas.i32 v0, v1, v2 + return v3 +} + +; VCode: +; push {fp, lr} +; mov fp, sp +; sub sp, sp, #8 +; str r7, [sp, #4] +; block0: +; mov r3, r0 +; mov r7, r1 +; atomic_cas r1, [r3], r7, r2 +; mov r0, r1 +; ldr r7, [sp, #4] +; add sp, sp, #8 +; pop {fp, lr} +; bx lr +; +; Disassembled: +; block0: ; offset 0x0 +; push {fp, lr} +; mov fp, sp +; sub sp, sp, #8 +; str r7, [sp, #4] +; block1: ; offset 0x10 +; mov r3, r0 +; mov r7, r1 +; ldaex r1, [r3] +; cmp r1, r7 +; bne #0x30 +; stlex r0, r2, [r3] +; cmp r0, #0 +; bne #0x18 +; mov r0, r1 +; ldr r7, [sp, #4] +; add sp, sp, #8 +; pop {fp, lr} +; bx lr + +function %aload(i32) -> i32 { +block0(v0: i32): + v1 = atomic_load.i32 v0 + return v1 +} + +; VCode: +; block0: +; lda r0, [r0] +; bx lr +; +; Disassembled: +; block0: ; offset 0x0 +; lda r0, [r0] +; bx lr + +function %astore(i32, i32) { +block0(v0: i32, v1: i32): + atomic_store.i32 v1, v0 + return +} + +; VCode: +; block0: +; stl r1, [r0] +; bx lr +; +; Disassembled: +; block0: ; offset 0x0 +; stl r1, [r0] +; bx lr + +;; Sub-word: a simple op needs no extension (the store truncates). +function %rmw_add_i8(i32, i8) -> i8 { +block0(v0: i32, v1: i8): + v2 = atomic_rmw.i8 add v0, v1 + return v2 +} + +; VCode: +; push {fp, lr} +; mov fp, sp +; sub sp, sp, #8 +; str r7, [sp, #4] +; block0: +; mov r2, r0 +; mov r7, r1 +; atomic_rmw.Addb r3, [r2], r7 +; mov r0, r3 +; ldr r7, [sp, #4] +; add sp, sp, #8 +; pop {fp, lr} +; bx lr +; +; Disassembled: +; block0: ; offset 0x0 +; push {fp, lr} +; mov fp, sp +; sub sp, sp, #8 +; str r7, [sp, #4] +; block1: ; offset 0x10 +; mov r2, r0 +; mov r7, r1 +; ldaexb r3, [r2] +; add r1, r3, r7 +; stlexb r0, r1, [r2] +; cmp r0, #0 +; bne #0x18 +; mov r0, r3 +; ldr r7, [sp, #4] +; add sp, sp, #8 +; pop {fp, lr} +; bx lr + +;; Sub-word signed min: both inputs are sign-extended before the compare. +function %rmw_smax_i8(i32, i8) -> i8 { +block0(v0: i32, v1: i8): + v2 = atomic_rmw.i8 smax v0, v1 + return v2 +} + +; VCode: +; push {fp, lr} +; mov fp, sp +; sub sp, sp, #8 +; str r7, [sp, #4] +; block0: +; mov r2, r0 +; mov r7, r1 +; atomic_rmw.Smaxb r3, [r2], r7 +; mov r0, r3 +; ldr r7, [sp, #4] +; add sp, sp, #8 +; pop {fp, lr} +; bx lr +; +; Disassembled: +; block0: ; offset 0x0 +; push {fp, lr} +; mov fp, sp +; sub sp, sp, #8 +; str r7, [sp, #4] +; block1: ; offset 0x10 +; mov r2, r0 +; mov r7, r1 +; ldaexb r3, [r2] +; sxtb r1, r3 +; sxtb r0, r7 +; cmp r1, r0 +; mov r1, r7 +; movgt r1, r3 +; stlexb r0, r1, [r2] +; cmp r0, #0 +; bne #0x18 +; mov r0, r3 +; ldr r7, [sp, #4] +; add sp, sp, #8 +; pop {fp, lr} +; bx lr + +;; Sub-word unsigned min: both inputs are zero-extended before the compare. +function %rmw_umin_i16(i32, i16) -> i16 { +block0(v0: i32, v1: i16): + v2 = atomic_rmw.i16 umin v0, v1 + return v2 +} + +; VCode: +; push {fp, lr} +; mov fp, sp +; sub sp, sp, #8 +; str r7, [sp, #4] +; block0: +; mov r2, r0 +; mov r7, r1 +; atomic_rmw.Uminh r3, [r2], r7 +; mov r0, r3 +; ldr r7, [sp, #4] +; add sp, sp, #8 +; pop {fp, lr} +; bx lr +; +; Disassembled: +; block0: ; offset 0x0 +; push {fp, lr} +; mov fp, sp +; sub sp, sp, #8 +; str r7, [sp, #4] +; block1: ; offset 0x10 +; mov r2, r0 +; mov r7, r1 +; ldaexh r3, [r2] +; uxth r1, r3 +; uxth r0, r7 +; cmp r1, r0 +; mov r1, r7 +; movlo r1, r3 +; stlexh r0, r1, [r2] +; cmp r0, #0 +; bne #0x18 +; mov r0, r3 +; ldr r7, [sp, #4] +; add sp, sp, #8 +; pop {fp, lr} +; bx lr + +;; Sub-word compare-and-swap: `expected` is zero-extended for the compare. +function %cas_i16(i32, i16, i16) -> i16 { +block0(v0: i32, v1: i16, v2: i16): + v3 = atomic_cas.i16 v0, v1, v2 + return v3 +} + +; VCode: +; push {fp, lr} +; mov fp, sp +; sub sp, sp, #8 +; str r7, [sp, #4] +; block0: +; mov r3, r0 +; mov r7, r1 +; atomic_cash r1, [r3], r7, r2 +; mov r0, r1 +; ldr r7, [sp, #4] +; add sp, sp, #8 +; pop {fp, lr} +; bx lr +; +; Disassembled: +; block0: ; offset 0x0 +; push {fp, lr} +; mov fp, sp +; sub sp, sp, #8 +; str r7, [sp, #4] +; block1: ; offset 0x10 +; mov r3, r0 +; mov r7, r1 +; ldaexh r1, [r3] +; uxth r0, r7 +; cmp r1, r0 +; bne #0x34 +; stlexh r0, r2, [r3] +; cmp r0, #0 +; bne #0x18 +; mov r0, r1 +; ldr r7, [sp, #4] +; add sp, sp, #8 +; pop {fp, lr} +; bx lr + +function %aload_i8(i32) -> i8 { +block0(v0: i32): + v1 = atomic_load.i8 v0 + return v1 +} + +; VCode: +; block0: +; ldab r0, [r0] +; bx lr +; +; Disassembled: +; block0: ; offset 0x0 +; ldab r0, [r0] +; bx lr + +function %astore_i16(i32, i16) { +block0(v0: i32, v1: i16): + atomic_store.i16 v1, v0 + return +} + +; VCode: +; block0: +; stlh r1, [r0] +; bx lr +; +; Disassembled: +; block0: ; offset 0x0 +; stlh r1, [r0] +; bx lr + +;; 64-bit: paired exclusive loop with a carry chain. All registers are pinned +;; to a fixed even/odd layout (r0/r1 old, r2/r3 new, r4 addr, r5/r6 operand, +;; r7 result) because A32 `ldrexd`/`strexd` require consecutive pairs. +function %rmw_add_i64(i32, i64) -> i64 { +block0(v0: i32, v1: i64): + v2 = atomic_rmw.i64 add v0, v1 + return v2 +} + +; VCode: +; push {fp, lr} +; mov fp, sp +; sub sp, sp, #16 +; str r4, [sp, #12] +; str r5, [sp, #8] +; str r6, [sp, #4] +; str r7, [sp, #0] +; block0: +; mov r4, r0 +; mov r5, r2 +; mov r6, r3 +; atomic_rmw.Add.i64 r0, r1, [r4], r5, r6 +; ldr r4, [sp, #12] +; ldr r5, [sp, #8] +; ldr r6, [sp, #4] +; ldr r7, [sp, #0] +; add sp, sp, #16 +; pop {fp, lr} +; bx lr +; +; Disassembled: +; block0: ; offset 0x0 +; push {fp, lr} +; mov fp, sp +; sub sp, sp, #0x10 +; str r4, [sp, #0xc] +; str r5, [sp, #8] +; str r6, [sp, #4] +; str r7, [sp] +; block1: ; offset 0x1c +; mov r4, r0 +; mov r5, r2 +; mov r6, r3 +; ldaexd r0, r1, [r4] +; adds r2, r0, r5 +; adc r3, r1, r6 +; stlexd r7, r2, r3, [r4] +; cmp r7, #0 +; bne #0x28 +; ldr r4, [sp, #0xc] +; ldr r5, [sp, #8] +; ldr r6, [sp, #4] +; ldr r7, [sp] +; add sp, sp, #0x10 +; pop {fp, lr} +; bx lr + +;; 64-bit signed max: a subs/sbcs compare then a conditional select of both +;; halves. +function %rmw_smax_i64(i32, i64) -> i64 { +block0(v0: i32, v1: i64): + v2 = atomic_rmw.i64 smax v0, v1 + return v2 +} + +; VCode: +; push {fp, lr} +; mov fp, sp +; sub sp, sp, #16 +; str r4, [sp, #12] +; str r5, [sp, #8] +; str r6, [sp, #4] +; str r7, [sp, #0] +; block0: +; mov r4, r0 +; mov r5, r2 +; mov r6, r3 +; atomic_rmw.Smax.i64 r0, r1, [r4], r5, r6 +; ldr r4, [sp, #12] +; ldr r5, [sp, #8] +; ldr r6, [sp, #4] +; ldr r7, [sp, #0] +; add sp, sp, #16 +; pop {fp, lr} +; bx lr +; +; Disassembled: +; block0: ; offset 0x0 +; push {fp, lr} +; mov fp, sp +; sub sp, sp, #0x10 +; str r4, [sp, #0xc] +; str r5, [sp, #8] +; str r6, [sp, #4] +; str r7, [sp] +; block1: ; offset 0x1c +; mov r4, r0 +; mov r5, r2 +; mov r6, r3 +; ldaexd r0, r1, [r4] +; subs r2, r0, r5 +; sbcs r3, r1, r6 +; mov r2, r5 +; mov r3, r6 +; movge r2, r0 +; movge r3, r1 +; stlexd r7, r2, r3, [r4] +; cmp r7, #0 +; bne #0x28 +; ldr r4, [sp, #0xc] +; ldr r5, [sp, #8] +; ldr r6, [sp, #4] +; ldr r7, [sp] +; add sp, sp, #0x10 +; pop {fp, lr} +; bx lr + +function %cas_i64(i32, i64, i64) -> i64 { +block0(v0: i32, v1: i64, v2: i64): + v3 = atomic_cas.i64 v0, v1, v2 + return v3 +} + +; VCode: +; push {fp, lr} +; mov fp, sp +; sub sp, sp, #16 +; str r4, [sp, #12] +; str r5, [sp, #8] +; str r6, [sp, #4] +; str r7, [sp, #0] +; block0: +; mov r5, r2 +; mov r6, r3 +; ldr r2, [incoming_arg, #8] +; ldr r3, [incoming_arg, #4] +; mov r4, r0 +; atomic_cas.i64 r0, r1, [r4], r5, r6, r2, r3 +; ldr r4, [sp, #12] +; ldr r5, [sp, #8] +; ldr r6, [sp, #4] +; ldr r7, [sp, #0] +; add sp, sp, #16 +; pop {fp, lr} +; bx lr +; +; Disassembled: +; block0: ; offset 0x0 +; push {fp, lr} +; mov fp, sp +; sub sp, sp, #0x10 +; str r4, [sp, #0xc] +; str r5, [sp, #8] +; str r6, [sp, #4] +; str r7, [sp] +; block1: ; offset 0x1c +; mov r5, r2 +; mov r6, r3 +; ldr r2, [sp, #0x18] +; ldr r3, [sp, #0x1c] +; mov r4, r0 +; ldaexd r0, r1, [r4] +; cmp r0, r5 +; bne #0x50 +; cmp r1, r6 +; bne #0x50 +; stlexd r7, r2, r3, [r4] +; cmp r7, #0 +; bne #0x30 +; ldr r4, [sp, #0xc] +; ldr r5, [sp, #8] +; ldr r6, [sp, #4] +; ldr r7, [sp] +; add sp, sp, #0x10 +; pop {fp, lr} +; bx lr + +function %aload_i64(i32) -> i64 { +block0(v0: i32): + v1 = atomic_load.i64 v0 + return v1 +} + +; VCode: +; push {fp, lr} +; mov fp, sp +; sub sp, sp, #8 +; str r4, [sp, #4] +; block0: +; mov r4, r0 +; atomic_load.i64 r0, r1, [r4] +; ldr r4, [sp, #4] +; add sp, sp, #8 +; pop {fp, lr} +; bx lr +; +; Disassembled: +; block0: ; offset 0x0 +; push {fp, lr} +; mov fp, sp +; sub sp, sp, #8 +; str r4, [sp, #4] +; block1: ; offset 0x10 +; mov r4, r0 +; ldaexd r0, r1, [r4] +; clrex +; ldr r4, [sp, #4] +; add sp, sp, #8 +; pop {fp, lr} +; bx lr + +function %astore_i64(i32, i64) { +block0(v0: i32, v1: i64): + atomic_store.i64 v1, v0 + return +} + +; VCode: +; push {fp, lr} +; mov fp, sp +; sub sp, sp, #8 +; str r4, [sp, #4] +; str r7, [sp, #0] +; block0: +; mov r4, r0 +; atomic_store.i64 r2, r3, [r4] +; ldr r4, [sp, #4] +; ldr r7, [sp, #0] +; add sp, sp, #8 +; pop {fp, lr} +; bx lr +; +; Disassembled: +; block0: ; offset 0x0 +; push {fp, lr} +; mov fp, sp +; sub sp, sp, #8 +; str r4, [sp, #4] +; str r7, [sp] +; block1: ; offset 0x14 +; mov r4, r0 +; ldaexd r0, r1, [r4] +; stlexd r7, r2, r3, [r4] +; cmp r7, #0 +; bne #0x18 +; ldr r4, [sp, #4] +; ldr r7, [sp] +; add sp, sp, #8 +; pop {fp, lr} +; bx lr + diff --git a/cranelift/filetests/filetests/isa/arm32/bitops.clif b/cranelift/filetests/filetests/isa/arm32/bitops.clif new file mode 100644 index 000000000000..bcac6de99d14 --- /dev/null +++ b/cranelift/filetests/filetests/isa/arm32/bitops.clif @@ -0,0 +1,51 @@ +test compile precise-output +target arm + +function %clz(i32) -> i32 { +block0(v0: i32): + v1 = clz v0 + return v1 +} + +; VCode: +; block0: +; clz r0, r0 +; bx lr +; +; Disassembled: +; block0: ; offset 0x0 +; clz r0, r0 +; bx lr + +function %bswap(i32) -> i32 { +block0(v0: i32): + v1 = bswap v0 + return v1 +} + +; VCode: +; block0: +; rev r0, r0 +; bx lr +; +; Disassembled: +; block0: ; offset 0x0 +; rev r0, r0 +; bx lr + +function %bitrev(i32) -> i32 { +block0(v0: i32): + v1 = bitrev v0 + return v1 +} + +; VCode: +; block0: +; rbit r0, r0 +; bx lr +; +; Disassembled: +; block0: ; offset 0x0 +; rbit r0, r0 +; bx lr + diff --git a/cranelift/filetests/filetests/isa/arm32/br_table.clif b/cranelift/filetests/filetests/isa/arm32/br_table.clif new file mode 100644 index 000000000000..f942aea13e61 --- /dev/null +++ b/cranelift/filetests/filetests/isa/arm32/br_table.clif @@ -0,0 +1,64 @@ +test compile precise-output +target arm + +function %jt(i32) -> i32 { +block0(v0: i32): + br_table v0, block4, [block1, block2, block3] + +block1: + v1 = iconst.i32 10 + return v1 + +block2: + v2 = iconst.i32 20 + return v2 + +block3: + v3 = iconst.i32 30 + return v3 + +block4: + v4 = iconst.i32 99 + return v4 +} + +; VCode: +; block0: +; mov r1, r0 +; br_table r1 [label3, label2, label1] default label4 +; block1: +; mov r0, #30 +; bx lr +; block2: +; mov r0, #20 +; bx lr +; block3: +; mov r0, #10 +; bx lr +; block4: +; mov r0, #99 +; bx lr +; +; Disassembled: +; block0: ; offset 0x0 +; mov r1, r0 +; cmp r1, #3 +; bhs #0x38 +; add pc, pc, r1, lsl #2 +; nop +; b #0x30 +; b #0x28 +; b #0x20 +; block1: ; offset 0x20 +; mov r0, #0x1e +; bx lr +; block2: ; offset 0x28 +; mov r0, #0x14 +; bx lr +; block3: ; offset 0x30 +; mov r0, #0xa +; bx lr +; block4: ; offset 0x38 +; mov r0, #0x63 +; bx lr + diff --git a/cranelift/filetests/filetests/isa/arm32/branch.clif b/cranelift/filetests/filetests/isa/arm32/branch.clif new file mode 100644 index 000000000000..985216b32e9c --- /dev/null +++ b/cranelift/filetests/filetests/isa/arm32/branch.clif @@ -0,0 +1,39 @@ +test compile precise-output +target arm + +function %select_via_brif(i32, i32) -> i32 { +block0(v0: i32, v1: i32): + v2 = icmp slt v0, v1 + brif v2, block1, block2 + +block1: + v3 = iconst.i32 1 + return v3 + +block2: + v4 = iconst.i32 0 + return v4 +} + +; VCode: +; block0: +; cmp r0, r1 +; blt label2; b label1 +; block1: +; mov r0, #0 +; bx lr +; block2: +; mov r0, #1 +; bx lr +; +; Disassembled: +; block0: ; offset 0x0 +; cmp r0, r1 +; blt #0x10 +; block1: ; offset 0x8 +; mov r0, #0 +; bx lr +; block2: ; offset 0x10 +; mov r0, #1 +; bx lr + diff --git a/cranelift/filetests/filetests/isa/arm32/call.clif b/cranelift/filetests/filetests/isa/arm32/call.clif new file mode 100644 index 000000000000..a76d9436ce3a --- /dev/null +++ b/cranelift/filetests/filetests/isa/arm32/call.clif @@ -0,0 +1,57 @@ +test compile precise-output +target arm + +function %call_direct(i32) -> i32 { + fn0 = colocated %g(i32) -> i32 +block0(v0: i32): + v1 = call fn0(v0) + return v1 +} + +; VCode: +; push {fp, lr} +; mov fp, sp +; block0: +; bl %g +; pop {fp, lr} +; bx lr +; +; Disassembled: +; block0: ; offset 0x0 +; push {fp, lr} +; mov fp, sp +; block1: ; offset 0x8 +; bl #0x10 ; reloc_external Call %g 0 +; pop {fp, lr} +; bx lr + +function %call_indirect(i32, i32) -> i32 { + sig0 = (i32) -> i32 +block0(v0: i32, v1: i32): + v2 = call_indirect sig0, v0(v1) + return v2 +} + +; VCode: +; push {fp, lr} +; mov fp, sp +; block0: +; mov r2, r0 +; mov r0, r1 +; mov r1, r2 +; blx r1 +; pop {fp, lr} +; bx lr +; +; Disassembled: +; block0: ; offset 0x0 +; push {fp, lr} +; mov fp, sp +; block1: ; offset 0x8 +; mov r2, r0 +; mov r0, r1 +; mov r1, r2 +; blx r1 +; pop {fp, lr} +; bx lr + diff --git a/cranelift/filetests/filetests/isa/arm32/div.clif b/cranelift/filetests/filetests/isa/arm32/div.clif new file mode 100644 index 000000000000..d8bb6eeaf8a7 --- /dev/null +++ b/cranelift/filetests/filetests/isa/arm32/div.clif @@ -0,0 +1,54 @@ +test compile precise-output +target arm has_idiv + +function %sdiv(i32, i32) -> i32 { +block0(v0: i32, v1: i32): + v2 = sdiv v0, v1 + return v2 +} + +; VCode: +; block0: +; cmp r1, #0 +; bne 1f ; udf ; 1: ; int_divz +; eor r2, r0, #2147483648 +; add r3, r1, #1 +; orrs r2, r2, r3 +; bne 1f ; udf ; 1: ; int_ovf +; sdiv r0, r0, r1 +; bx lr +; +; Disassembled: +; block0: ; offset 0x0 +; cmp r1, #0 +; bne #0xc +; udf #0 ; trap: int_divz +; eor r2, r0, #0x80000000 +; add r3, r1, #1 +; orrs r2, r2, r3 +; bne #0x20 +; udf #0 ; trap: int_ovf +; sdiv r0, r0, r1 +; bx lr + +function %udiv(i32, i32) -> i32 { +block0(v0: i32, v1: i32): + v2 = udiv v0, v1 + return v2 +} + +; VCode: +; block0: +; cmp r1, #0 +; bne 1f ; udf ; 1: ; int_divz +; udiv r0, r0, r1 +; bx lr +; +; Disassembled: +; block0: ; offset 0x0 +; cmp r1, #0 +; bne #0xc +; udf #0 ; trap: int_divz +; udiv r0, r0, r1 +; bx lr + diff --git a/cranelift/filetests/filetests/isa/arm32/extend.clif b/cranelift/filetests/filetests/isa/arm32/extend.clif new file mode 100644 index 000000000000..6165525df5ab --- /dev/null +++ b/cranelift/filetests/filetests/isa/arm32/extend.clif @@ -0,0 +1,81 @@ +test compile precise-output +target arm + +function %sext8(i8) -> i32 { +block0(v0: i8): + v1 = sextend.i32 v0 + return v1 +} + +; VCode: +; block0: +; sxtb r0, r0 +; bx lr +; +; Disassembled: +; block0: ; offset 0x0 +; sxtb r0, r0 +; bx lr + +function %sext16(i16) -> i32 { +block0(v0: i16): + v1 = sextend.i32 v0 + return v1 +} + +; VCode: +; block0: +; sxth r0, r0 +; bx lr +; +; Disassembled: +; block0: ; offset 0x0 +; sxth r0, r0 +; bx lr + +function %uext8(i8) -> i32 { +block0(v0: i8): + v1 = uextend.i32 v0 + return v1 +} + +; VCode: +; block0: +; uxtb r0, r0 +; bx lr +; +; Disassembled: +; block0: ; offset 0x0 +; uxtb r0, r0 +; bx lr + +function %uext16(i16) -> i32 { +block0(v0: i16): + v1 = uextend.i32 v0 + return v1 +} + +; VCode: +; block0: +; uxth r0, r0 +; bx lr +; +; Disassembled: +; block0: ; offset 0x0 +; uxth r0, r0 +; bx lr + +function %reduce(i32) -> i8 { +block0(v0: i32): + v1 = ireduce.i8 v0 + return v1 +} + +; VCode: +; block0: +; bx lr +; +; Disassembled: +; block0: ; offset 0x0 +; bx lr + diff --git a/cranelift/filetests/filetests/isa/arm32/far-call.clif b/cranelift/filetests/filetests/isa/arm32/far-call.clif new file mode 100644 index 000000000000..5f7f7fd771a7 --- /dev/null +++ b/cranelift/filetests/filetests/isa/arm32/far-call.clif @@ -0,0 +1,73 @@ +test compile precise-output +target arm + +;; Direct call to a non-colocated (far) function: materialize the callee's +;; absolute address via an inline literal, then call it indirectly. +function %call_far(i32) -> i32 { + fn0 = %g(i32) -> i32 +block0(v0: i32): + v1 = call fn0(v0) + return v1 +} + +; VCode: +; push {fp, lr} +; mov fp, sp +; block0: +; load_ext_name r3, TestCase(%g) + 0 +; blx r3 +; pop {fp, lr} +; bx lr +; +; Disassembled: +; block0: ; offset 0x0 +; push {fp, lr} +; mov fp, sp +; block1: ; offset 0x8 +; ldr r3, [pc] +; b #0x14 +; andeq r0, r0, r0 ; reloc_external Abs4 %g 0 +; blx r3 +; pop {fp, lr} +; bx lr + +;; Address of a function symbol. +function %func_addr() -> i32 { + fn0 = %g(i32) -> i32 +block0: + v0 = func_addr.i32 fn0 + return v0 +} + +; VCode: +; block0: +; load_ext_name r0, TestCase(%g) + 0 +; bx lr +; +; Disassembled: +; block0: ; offset 0x0 +; ldr r0, [pc] +; b #0xc +; andeq r0, r0, r0 ; reloc_external Abs4 %g 0 +; bx lr + +;; Address of a data symbol plus a constant byte offset. +function %symbol_addr() -> i32 { + gv0 = symbol %sym+16 +block0: + v0 = symbol_value.i32 gv0 + return v0 +} + +; VCode: +; block0: +; load_ext_name r0, TestCase(%sym) + 16 +; bx lr +; +; Disassembled: +; block0: ; offset 0x0 +; ldr r0, [pc] +; b #0xc +; andeq r0, r0, r0 ; reloc_external Abs4 %sym 16 +; bx lr + diff --git a/cranelift/filetests/filetests/isa/arm32/fence.clif b/cranelift/filetests/filetests/isa/arm32/fence.clif new file mode 100644 index 000000000000..29658e86dfcc --- /dev/null +++ b/cranelift/filetests/filetests/isa/arm32/fence.clif @@ -0,0 +1,19 @@ +test compile precise-output +target arm + +function %fence() { +block0: + fence + return +} + +; VCode: +; block0: +; dmb sy +; bx lr +; +; Disassembled: +; block0: ; offset 0x0 +; dmb sy +; bx lr + diff --git a/cranelift/filetests/filetests/isa/arm32/float-cvt.clif b/cranelift/filetests/filetests/isa/arm32/float-cvt.clif new file mode 100644 index 000000000000..2bc07d680ef4 --- /dev/null +++ b/cranelift/filetests/filetests/isa/arm32/float-cvt.clif @@ -0,0 +1,130 @@ +test compile precise-output +target arm + +function %feq(f64, f64) -> i8 { +block0(v0: f64, v1: f64): + v2 = fcmp eq v0, v1 + return v2 +} + +; VCode: +; block0: +; vcmp.f64 d0, d1; vmrs +; mov r1, #1 +; mov r2, #0 +; mov r0, r2; moveq r0, r1 +; bx lr +; +; Disassembled: +; block0: ; offset 0x0 +; vcmp.f64 d0, d1 +; vmrs apsr_nzcv, fpscr +; mov r1, #1 +; mov r2, #0 +; mov r0, r2 +; moveq r0, r1 +; bx lr + +function %flt_br(f32, f32) -> i32 { +block0(v0: f32, v1: f32): + v2 = fcmp lt v0, v1 + brif v2, block1, block2 +block1: + v3 = iconst.i32 1 + return v3 +block2: + v4 = iconst.i32 0 + return v4 +} + +; VCode: +; block0: +; vcmp.f32 d0, d1; vmrs +; bmi label2; b label1 +; block1: +; mov r0, #0 +; bx lr +; block2: +; mov r0, #1 +; bx lr +; +; Disassembled: +; block0: ; offset 0x0 +; vcmp.f32 s0, s2 +; vmrs apsr_nzcv, fpscr +; bmi #0x14 +; block1: ; offset 0xc +; mov r0, #0 +; bx lr +; block2: ; offset 0x14 +; mov r0, #1 +; bx lr + +function %promote(f32) -> f64 { +block0(v0: f32): + v1 = fpromote.f64 v0 + return v1 +} + +; VCode: +; block0: +; vcvt.f64.f32 d0, d0 +; bx lr +; +; Disassembled: +; block0: ; offset 0x0 +; vcvt.f64.f32 d0, s0 +; bx lr + +function %f2i(f64) -> i32 { +block0(v0: f64): + v1 = fcvt_to_sint_sat.i32 v0 + return v1 +} + +; VCode: +; block0: +; vcvt.s32.f64 d2, d0 +; vmov r0, d2 +; bx lr +; +; Disassembled: +; block0: ; offset 0x0 +; vcvt.s32.f64 s4, d0 +; vmov r0, s4 +; bx lr + +function %i2f(i32) -> f64 { +block0(v0: i32): + v1 = fcvt_from_sint.f64 v0 + return v1 +} + +; VCode: +; block0: +; vmov d2, r0 +; vcvt.f64.s32 d0, d2 +; bx lr +; +; Disassembled: +; block0: ; offset 0x0 +; vmov s4, r0 +; vcvt.f64.s32 d0, s4 +; bx lr + +function %bitcast_f2i(f32) -> i32 { +block0(v0: f32): + v1 = bitcast.i32 v0 + return v1 +} + +; VCode: +; block0: +; vmov r0, d0 +; bx lr +; +; Disassembled: +; block0: ; offset 0x0 +; vmov r0, s0 +; bx lr + diff --git a/cranelift/filetests/filetests/isa/arm32/float-minmax.clif b/cranelift/filetests/filetests/isa/arm32/float-minmax.clif new file mode 100644 index 000000000000..5d9092e3da7c --- /dev/null +++ b/cranelift/filetests/filetests/isa/arm32/float-minmax.clif @@ -0,0 +1,124 @@ +test compile precise-output +target arm + +function %fmin64(f64, f64) -> f64 { +block0(v0: f64, v1: f64): + v2 = fmin v0, v1 + return v2 +} + +; VCode: +; block0: +; vminnm.f64 d3, d0, d1 +; vadd.f64 d5, d0, d1 +; vcmp.f64 d0, d1; vmrs +; vmov d0, d3; vmovvs d0, d5 +; bx lr +; +; Disassembled: +; block0: ; offset 0x0 +; vminnm.f64 d3, d0, d1 +; vadd.f64 d5, d0, d1 +; vcmp.f64 d0, d1 +; vmrs apsr_nzcv, fpscr +; vmov.f64 d0, d3 +; vmovvs.f64 d0, d5 +; bx lr + +function %fmax32(f32, f32) -> f32 { +block0(v0: f32, v1: f32): + v2 = fmax v0, v1 + return v2 +} + +; VCode: +; block0: +; vmaxnm.f32 d3, d0, d1 +; vadd.f32 d5, d0, d1 +; vcmp.f32 d0, d1; vmrs +; vmov d0, d3; vmovvs d0, d5 +; bx lr +; +; Disassembled: +; block0: ; offset 0x0 +; vmaxnm.f32 s6, s0, s2 +; vadd.f32 s10, s0, s2 +; vcmp.f32 s0, s2 +; vmrs apsr_nzcv, fpscr +; vmov.f64 d0, d3 +; vmovvs.f64 d0, d5 +; bx lr + +function %copysign32(f32, f32) -> f32 { +block0(v0: f32, v1: f32): + v2 = fcopysign v0, v1 + return v2 +} + +; VCode: +; block0: +; vmov r3, d0 +; vmov r0, d1 +; and r0, r0, #2147483648 +; bic r1, r3, #2147483648 +; orr r0, r1, r0 +; vmov d0, r0 +; bx lr +; +; Disassembled: +; block0: ; offset 0x0 +; vmov r3, s0 +; vmov r0, s2 +; and r0, r0, #0x80000000 +; bic r1, r3, #0x80000000 +; orr r0, r1, r0 +; vmov s0, r0 +; bx lr + +function %copysign64(f64, f64) -> f64 { +block0(v0: f64, v1: f64): + v2 = fcopysign v0, v1 + return v2 +} + +; VCode: +; block0: +; vmov r3, r0, d0 +; vmov r2, r1, d1 +; and r1, r1, #2147483648 +; bic r0, r0, #2147483648 +; orr r0, r0, r1 +; vmov d0, r3, r0 +; bx lr +; +; Disassembled: +; block0: ; offset 0x0 +; vmov r3, r0, d0 +; vmov r2, r1, d1 +; and r1, r1, #0x80000000 +; bic r0, r0, #0x80000000 +; orr r0, r0, r1 +; vmov d0, r3, r0 +; bx lr + +function %fsel(i32, f64, f64) -> f64 { +block0(v0: i32, v1: f64, v2: f64): + v3 = select v0, v1, v2 + return v3 +} + +; VCode: +; block0: +; vmov.f64 d6, d0 +; cmp r0, #0 +; vmov d0, d1; vmovne d0, d6 +; bx lr +; +; Disassembled: +; block0: ; offset 0x0 +; vmov.f64 d6, d0 +; cmp r0, #0 +; vmov.f64 d0, d1 +; vmovne.f64 d0, d6 +; bx lr + diff --git a/cranelift/filetests/filetests/isa/arm32/float.clif b/cranelift/filetests/filetests/isa/arm32/float.clif new file mode 100644 index 000000000000..e13f460e1fe0 --- /dev/null +++ b/cranelift/filetests/filetests/isa/arm32/float.clif @@ -0,0 +1,187 @@ +test compile precise-output +target arm + +function %fadd32(f32, f32) -> f32 { +block0(v0: f32, v1: f32): + v2 = fadd v0, v1 + return v2 +} + +; VCode: +; block0: +; vadd.f32 d0, d0, d1 +; bx lr +; +; Disassembled: +; block0: ; offset 0x0 +; vadd.f32 s0, s0, s2 +; bx lr + +function %fmul_fsub64(f64, f64, f64) -> f64 { +block0(v0: f64, v1: f64, v2: f64): + v3 = fmul v0, v1 + v4 = fsub v3, v2 + return v4 +} + +; VCode: +; block0: +; vmul.f64 d5, d0, d1 +; vsub.f64 d0, d5, d2 +; bx lr +; +; Disassembled: +; block0: ; offset 0x0 +; vmul.f64 d5, d0, d1 +; vsub.f64 d0, d5, d2 +; bx lr + +function %fdiv_sqrt(f64, f64) -> f64 { +block0(v0: f64, v1: f64): + v2 = fdiv v0, v1 + v3 = sqrt v2 + return v3 +} + +; VCode: +; block0: +; vdiv.f64 d4, d0, d1 +; vsqrt.f64 d0, d4 +; bx lr +; +; Disassembled: +; block0: ; offset 0x0 +; vdiv.f64 d4, d0, d1 +; vsqrt.f64 d0, d4 +; bx lr + +function %fneg_abs(f32) -> f32 { +block0(v0: f32): + v1 = fneg v0 + v2 = fabs v1 + return v2 +} + +; VCode: +; block0: +; vneg.f32 d3, d0 +; vabs.f32 d0, d3 +; bx lr +; +; Disassembled: +; block0: ; offset 0x0 +; vneg.f32 s6, s0 +; vabs.f32 s0, s6 +; bx lr + +function %fconst64() -> f64 { +block0: + v0 = f64const 0x1.8p3 + return v0 +} + +; VCode: +; block0: +; mov r0, #0 +; movw r2, #0; movt r2, #16424 +; vmov d0, r0, r2 +; bx lr +; +; Disassembled: +; block0: ; offset 0x0 +; mov r0, #0 +; movw r2, #0 +; movt r2, #0x4028 +; vmov d0, r0, r2 +; bx lr + +function %load_f32(i32) -> f32 { +block0(v0: i32): + v1 = load.f32 v0+8 + return v1 +} + +; VCode: +; block0: +; vldr d0, [r0, #8] +; bx lr +; +; Disassembled: +; block0: ; offset 0x0 +; vldr s0, [r0, #8] +; bx lr + +function %store_f64(i32, f64) { +block0(v0: i32, v1: f64): + store.f64 v1, v0 + return +} + +; VCode: +; block0: +; vstr d0, [r0, #0] +; bx lr +; +; Disassembled: +; block0: ; offset 0x0 +; vstr d0, [r0] +; bx lr + +function %fcmp_one_f32(f32, f32) -> i8 { +block0(v0: f32, v1: f32): + v2 = fcmp one v0, v1 + return v2 +} + +; VCode: +; block0: +; vcmp.f32 d0, d1; vmrs +; mov r1, #1 +; mov r0, #0 +; mov r2, r0; movne r2, r1 +; mov r1, #0 +; mov r0, r1; movvc r0, r2 +; bx lr +; +; Disassembled: +; block0: ; offset 0x0 +; vcmp.f32 s0, s2 +; vmrs apsr_nzcv, fpscr +; mov r1, #1 +; mov r0, #0 +; mov r2, r0 +; movne r2, r1 +; mov r1, #0 +; mov r0, r1 +; movvc r0, r2 +; bx lr + +function %fcmp_ueq_f64(f64, f64) -> i8 { +block0(v0: f64, v1: f64): + v2 = fcmp ueq v0, v1 + return v2 +} + +; VCode: +; block0: +; vcmp.f64 d0, d1; vmrs +; mov r2, #1 +; mov r1, #1 +; mov r0, #0 +; mov r3, r0; moveq r3, r1 +; mov r0, r3; movvs r0, r2 +; bx lr +; +; Disassembled: +; block0: ; offset 0x0 +; vcmp.f64 d0, d1 +; vmrs apsr_nzcv, fpscr +; mov r2, #1 +; mov r1, #1 +; mov r0, #0 +; mov r3, r0 +; moveq r3, r1 +; mov r0, r3 +; movvs r0, r2 +; bx lr + diff --git a/cranelift/filetests/filetests/isa/arm32/i64-cmp.clif b/cranelift/filetests/filetests/isa/arm32/i64-cmp.clif new file mode 100644 index 000000000000..33e92b2dedc0 --- /dev/null +++ b/cranelift/filetests/filetests/isa/arm32/i64-cmp.clif @@ -0,0 +1,116 @@ +test compile precise-output +target arm + +function %eq(i64, i64) -> i8 { +block0(v0: i64, v1: i64): + v2 = icmp eq v0, v1 + return v2 +} + +; VCode: +; block0: +; eor r0, r0, r2 +; eor r1, r1, r3 +; orrs r0, r0, r1 +; mov r1, #1 +; mov r2, #0 +; mov r0, r2; moveq r0, r1 +; bx lr +; +; Disassembled: +; block0: ; offset 0x0 +; eor r0, r0, r2 +; eor r1, r1, r3 +; orrs r0, r0, r1 +; mov r1, #1 +; mov r2, #0 +; mov r0, r2 +; moveq r0, r1 +; bx lr + +function %slt(i64, i64) -> i8 { +block0(v0: i64, v1: i64): + v2 = icmp slt v0, v1 + return v2 +} + +; VCode: +; block0: +; subs r0, r0, r2 +; sbcs r0, r1, r3 +; mov r1, #1 +; mov r2, #0 +; mov r0, r2; movlt r0, r1 +; bx lr +; +; Disassembled: +; block0: ; offset 0x0 +; subs r0, r0, r2 +; sbcs r0, r1, r3 +; mov r1, #1 +; mov r2, #0 +; mov r0, r2 +; movlt r0, r1 +; bx lr + +function %ugt(i64, i64) -> i8 { +block0(v0: i64, v1: i64): + v2 = icmp ugt v0, v1 + return v2 +} + +; VCode: +; block0: +; subs r0, r2, r0 +; sbcs r0, r3, r1 +; mov r1, #1 +; mov r2, #0 +; mov r0, r2; movlo r0, r1 +; bx lr +; +; Disassembled: +; block0: ; offset 0x0 +; subs r0, r2, r0 +; sbcs r0, r3, r1 +; mov r1, #1 +; mov r2, #0 +; mov r0, r2 +; movlo r0, r1 +; bx lr + +function %br_sle(i64, i64) -> i32 { +block0(v0: i64, v1: i64): + v2 = icmp sle v0, v1 + brif v2, block1, block2 +block1: + v3 = iconst.i32 1 + return v3 +block2: + v4 = iconst.i32 0 + return v4 +} + +; VCode: +; block0: +; subs r0, r2, r0 +; sbcs r0, r3, r1 +; bge label2; b label1 +; block1: +; mov r0, #0 +; bx lr +; block2: +; mov r0, #1 +; bx lr +; +; Disassembled: +; block0: ; offset 0x0 +; subs r0, r2, r0 +; sbcs r0, r3, r1 +; bge #0x14 +; block1: ; offset 0xc +; mov r0, #0 +; bx lr +; block2: ; offset 0x14 +; mov r0, #1 +; bx lr + diff --git a/cranelift/filetests/filetests/isa/arm32/i64.clif b/cranelift/filetests/filetests/isa/arm32/i64.clif new file mode 100644 index 000000000000..fd15d000af30 --- /dev/null +++ b/cranelift/filetests/filetests/isa/arm32/i64.clif @@ -0,0 +1,220 @@ +test compile precise-output +target arm + +function %add64(i64, i64) -> i64 { +block0(v0: i64, v1: i64): + v2 = iadd v0, v1 + return v2 +} + +; VCode: +; block0: +; adds r0, r0, r2 +; adc r1, r1, r3 +; bx lr +; +; Disassembled: +; block0: ; offset 0x0 +; adds r0, r0, r2 +; adc r1, r1, r3 +; bx lr + +function %sub64(i64, i64) -> i64 { +block0(v0: i64, v1: i64): + v2 = isub v0, v1 + return v2 +} + +; VCode: +; block0: +; subs r0, r0, r2 +; sbc r1, r1, r3 +; bx lr +; +; Disassembled: +; block0: ; offset 0x0 +; subs r0, r0, r2 +; sbc r1, r1, r3 +; bx lr + +function %const64() -> i64 { +block0: + v0 = iconst.i64 0x1234_5678_9abc_def0 + return v0 +} + +; VCode: +; block0: +; movw r0, #57072; movt r0, #39612 +; movw r1, #22136; movt r1, #4660 +; bx lr +; +; Disassembled: +; block0: ; offset 0x0 +; movw r0, #0xdef0 +; movt r0, #0x9abc +; movw r1, #0x5678 +; movt r1, #0x1234 +; bx lr + +function %uext(i32) -> i64 { +block0(v0: i32): + v1 = uextend.i64 v0 + return v1 +} + +; VCode: +; block0: +; mov r1, #0 +; bx lr +; +; Disassembled: +; block0: ; offset 0x0 +; mov r1, #0 +; bx lr + +function %sext(i32) -> i64 { +block0(v0: i32): + v1 = sextend.i64 v0 + return v1 +} + +; VCode: +; block0: +; asr r1, r0, #31 +; bx lr +; +; Disassembled: +; block0: ; offset 0x0 +; asr r1, r0, #0x1f +; bx lr + +function %load64(i32) -> i64 { +block0(v0: i32): + v1 = load.i64 v0 + return v1 +} + +; VCode: +; block0: +; ldr r2, [r0, #0] +; ldr r1, [r0, #4] +; mov r0, r2 +; bx lr +; +; Disassembled: +; block0: ; offset 0x0 +; ldr r2, [r0] +; ldr r1, [r0, #4] +; mov r0, r2 +; bx lr + + +function %mul64(i64, i64) -> i64 { +block0(v0: i64, v1: i64): + v2 = imul v0, v1 + return v2 +} + +; VCode: +; push {fp, lr} +; mov fp, sp +; sub sp, sp, #8 +; str r5, [sp, #4] +; str r6, [sp, #0] +; block0: +; umull r5, r6, r0, r2 +; mla r0, r0, r3, r6 +; mla r1, r1, r2, r0 +; mov r0, r5 +; ldr r5, [sp, #4] +; ldr r6, [sp, #0] +; add sp, sp, #8 +; pop {fp, lr} +; bx lr +; +; Disassembled: +; block0: ; offset 0x0 +; push {fp, lr} +; mov fp, sp +; sub sp, sp, #8 +; str r5, [sp, #4] +; str r6, [sp] +; block1: ; offset 0x14 +; umull r5, r6, r0, r2 +; mla r0, r0, r3, r6 +; mla r1, r1, r2, r0 +; mov r0, r5 +; ldr r5, [sp, #4] +; ldr r6, [sp] +; add sp, sp, #8 +; pop {fp, lr} +; bx lr + +function %shl64(i64) -> i64 { +block0(v0: i64): + v1 = iconst.i32 5 + v2 = ishl v0, v1 + return v2 +} + +; VCode: +; block0: +; lsl r3, r0, #5 +; lsl r1, r1, #5 +; lsr r0, r0, #27 +; orr r1, r1, r0 +; mov r0, r3 +; bx lr +; +; Disassembled: +; block0: ; offset 0x0 +; lsl r3, r0, #5 +; lsl r1, r1, #5 +; lsr r0, r0, #0x1b +; orr r1, r1, r0 +; mov r0, r3 +; bx lr + +function %shl64_big(i64) -> i64 { +block0(v0: i64): + v1 = iconst.i32 40 + v2 = ishl v0, v1 + return v2 +} + +; VCode: +; block0: +; mov r1, r0 +; mov r0, #0 +; lsl r1, r1, #8 +; bx lr +; +; Disassembled: +; block0: ; offset 0x0 +; mov r1, r0 +; mov r0, #0 +; lsl r1, r1, #8 +; bx lr + +function %sshr64(i64) -> i64 { +block0(v0: i64): + v1 = iconst.i32 33 + v2 = sshr v0, v1 + return v2 +} + +; VCode: +; block0: +; asr r3, r1, #31 +; asr r0, r1, #1 +; mov r1, r3 +; bx lr +; +; Disassembled: +; block0: ; offset 0x0 +; asr r3, r1, #0x1f +; asr r0, r1, #1 +; mov r1, r3 +; bx lr + diff --git a/cranelift/filetests/filetests/isa/arm32/icmp.clif b/cranelift/filetests/filetests/isa/arm32/icmp.clif new file mode 100644 index 000000000000..723fb66fe47c --- /dev/null +++ b/cranelift/filetests/filetests/isa/arm32/icmp.clif @@ -0,0 +1,82 @@ +test compile precise-output +target arm + +; Narrow (i8/i16) comparisons must widen their operands per the comparison's +; signedness, since Cranelift leaves the high bits of narrow values undefined. +function %icmp_slt_i8(i8, i8) -> i8 { +block0(v0: i8, v1: i8): + v2 = icmp slt v0, v1 + return v2 +} + +; VCode: +; block0: +; sxtb r3, r0 +; sxtb r0, r1 +; cmp r3, r0 +; mov r1, #1 +; mov r2, #0 +; mov r0, r2; movlt r0, r1 +; bx lr +; +; Disassembled: +; block0: ; offset 0x0 +; sxtb r3, r0 +; sxtb r0, r1 +; cmp r3, r0 +; mov r1, #1 +; mov r2, #0 +; mov r0, r2 +; movlt r0, r1 +; bx lr + +function %icmp_ult_i16(i16, i16) -> i8 { +block0(v0: i16, v1: i16): + v2 = icmp ult v0, v1 + return v2 +} + +; VCode: +; block0: +; uxth r3, r0 +; uxth r0, r1 +; cmp r3, r0 +; mov r1, #1 +; mov r2, #0 +; mov r0, r2; movlo r0, r1 +; bx lr +; +; Disassembled: +; block0: ; offset 0x0 +; uxth r3, r0 +; uxth r0, r1 +; cmp r3, r0 +; mov r1, #1 +; mov r2, #0 +; mov r0, r2 +; movlo r0, r1 +; bx lr + +function %icmp_eq_i32(i32, i32) -> i8 { +block0(v0: i32, v1: i32): + v2 = icmp eq v0, v1 + return v2 +} + +; VCode: +; block0: +; cmp r0, r1 +; mov r1, #1 +; mov r2, #0 +; mov r0, r2; moveq r0, r1 +; bx lr +; +; Disassembled: +; block0: ; offset 0x0 +; cmp r0, r1 +; mov r1, #1 +; mov r2, #0 +; mov r0, r2 +; moveq r0, r1 +; bx lr + diff --git a/cranelift/filetests/filetests/isa/arm32/iconst.clif b/cranelift/filetests/filetests/isa/arm32/iconst.clif new file mode 100644 index 000000000000..eddd8d854d41 --- /dev/null +++ b/cranelift/filetests/filetests/isa/arm32/iconst.clif @@ -0,0 +1,19 @@ +test compile precise-output +target arm + +function %f() -> i32 { +block0: + v0 = iconst.i32 42 + return v0 +} + +; VCode: +; block0: +; mov r0, #42 +; bx lr +; +; Disassembled: +; block0: ; offset 0x0 +; mov r0, #0x2a +; bx lr + diff --git a/cranelift/filetests/filetests/isa/arm32/logical.clif b/cranelift/filetests/filetests/isa/arm32/logical.clif new file mode 100644 index 000000000000..9f7c9a121b83 --- /dev/null +++ b/cranelift/filetests/filetests/isa/arm32/logical.clif @@ -0,0 +1,85 @@ +test compile precise-output +target arm + +function %and(i32, i32) -> i32 { +block0(v0: i32, v1: i32): + v2 = band v0, v1 + return v2 +} + +; VCode: +; block0: +; and r0, r0, r1 +; bx lr +; +; Disassembled: +; block0: ; offset 0x0 +; and r0, r0, r1 +; bx lr + +function %or_imm(i32) -> i32 { +block0(v0: i32): + v1 = iconst.i32 0xff + v2 = bor v0, v1 + return v2 +} + +; VCode: +; block0: +; orr r0, r0, #255 +; bx lr +; +; Disassembled: +; block0: ; offset 0x0 +; orr r0, r0, #0xff +; bx lr + +function %xor(i32, i32) -> i32 { +block0(v0: i32, v1: i32): + v2 = bxor v0, v1 + return v2 +} + +; VCode: +; block0: +; eor r0, r0, r1 +; bx lr +; +; Disassembled: +; block0: ; offset 0x0 +; eor r0, r0, r1 +; bx lr + +function %not(i32) -> i32 { +block0(v0: i32): + v1 = bnot v0 + return v1 +} + +; VCode: +; block0: +; mvn r0, r0 +; bx lr +; +; Disassembled: +; block0: ; offset 0x0 +; mvn r0, r0 +; bx lr + +function %and_not(i32, i32) -> i32 { +block0(v0: i32, v1: i32): + v2 = bnot v1 + v3 = band v0, v2 + return v3 +} + +; VCode: +; block0: +; bic r0, r0, r1 +; bx lr +; +; Disassembled: +; block0: ; offset 0x0 +; bic r0, r0, r1 +; bx lr + diff --git a/cranelift/filetests/filetests/isa/arm32/memory.clif b/cranelift/filetests/filetests/isa/arm32/memory.clif new file mode 100644 index 000000000000..4b8de0e625d1 --- /dev/null +++ b/cranelift/filetests/filetests/isa/arm32/memory.clif @@ -0,0 +1,131 @@ +test compile precise-output +target arm + +function %load_word(i32) -> i32 { +block0(v0: i32): + v1 = load.i32 v0+4 + return v1 +} + +; VCode: +; block0: +; ldr r0, [r0, #4] +; bx lr +; +; Disassembled: +; block0: ; offset 0x0 +; ldr r0, [r0, #4] +; bx lr + +function %store_word(i32, i32) { +block0(v0: i32, v1: i32): + store.i32 v1, v0+8 + return +} + +; VCode: +; block0: +; str r1, [r0, #8] +; bx lr +; +; Disassembled: +; block0: ; offset 0x0 +; str r1, [r0, #8] +; bx lr + +function %load_u8(i32) -> i32 { +block0(v0: i32): + v1 = uload8.i32 v0 + return v1 +} + +; VCode: +; block0: +; ldrb r0, [r0, #0] +; bx lr +; +; Disassembled: +; block0: ; offset 0x0 +; ldrb r0, [r0] +; bx lr + +function %store_u8(i32, i32) { +block0(v0: i32, v1: i32): + istore8 v1, v0 + return +} + +; VCode: +; block0: +; strb r1, [r0, #0] +; bx lr +; +; Disassembled: +; block0: ; offset 0x0 +; strb r1, [r0] +; bx lr + +function %load_s16(i32) -> i32 { +block0(v0: i32): + v1 = sload16.i32 v0+2 + return v1 +} + +; VCode: +; block0: +; ldrsh r0, [r0, #2] +; bx lr +; +; Disassembled: +; block0: ; offset 0x0 +; ldrsh r0, [r0, #2] +; bx lr + +function %load_i8(i32) -> i8 { +block0(v0: i32): + v1 = load.i8 v0 + return v1 +} + +; VCode: +; block0: +; ldrb r0, [r0, #0] +; bx lr +; +; Disassembled: +; block0: ; offset 0x0 +; ldrb r0, [r0] +; bx lr + +function %store_i8(i8, i32) { +block0(v0: i8, v1: i32): + store.i8 v0, v1 + return +} + +; VCode: +; block0: +; strb r0, [r1, #0] +; bx lr +; +; Disassembled: +; block0: ; offset 0x0 +; strb r0, [r1] +; bx lr + +function %load_i16(i32) -> i16 { +block0(v0: i32): + v1 = load.i16 v0 + return v1 +} + +; VCode: +; block0: +; ldrh r0, [r0, #0] +; bx lr +; +; Disassembled: +; block0: ; offset 0x0 +; ldrh r0, [r0] +; bx lr + diff --git a/cranelift/filetests/filetests/isa/arm32/mul.clif b/cranelift/filetests/filetests/isa/arm32/mul.clif new file mode 100644 index 000000000000..4bb6371f1ba7 --- /dev/null +++ b/cranelift/filetests/filetests/isa/arm32/mul.clif @@ -0,0 +1,36 @@ +test compile precise-output +target arm + +function %mul(i32, i32) -> i32 { +block0(v0: i32, v1: i32): + v2 = imul v0, v1 + return v2 +} + +; VCode: +; block0: +; mul r0, r0, r1 +; bx lr +; +; Disassembled: +; block0: ; offset 0x0 +; mul r0, r0, r1 +; bx lr + +function %mls(i32, i32, i32) -> i32 { +block0(v0: i32, v1: i32, v2: i32): + v3 = imul v1, v2 + v4 = isub v0, v3 + return v4 +} + +; VCode: +; block0: +; mls r0, r1, r2, r0 +; bx lr +; +; Disassembled: +; block0: ; offset 0x0 +; mls r0, r1, r2, r0 +; bx lr + diff --git a/cranelift/filetests/filetests/isa/arm32/select.clif b/cranelift/filetests/filetests/isa/arm32/select.clif new file mode 100644 index 000000000000..3296ae8639d3 --- /dev/null +++ b/cranelift/filetests/filetests/isa/arm32/select.clif @@ -0,0 +1,83 @@ +test compile precise-output +target arm + +function %select_icmp(i32, i32, i32, i32) -> i32 { +block0(v0: i32, v1: i32, v2: i32, v3: i32): + v4 = icmp slt v0, v1 + v5 = select v4, v2, v3 + return v5 +} + +; VCode: +; block0: +; cmp r0, r1 +; mov r0, r3; movlt r0, r2 +; bx lr +; +; Disassembled: +; block0: ; offset 0x0 +; cmp r0, r1 +; mov r0, r3 +; movlt r0, r2 +; bx lr + +function %select_val(i32, i32, i32) -> i32 { +block0(v0: i32, v1: i32, v2: i32): + v3 = select v0, v1, v2 + return v3 +} + +; VCode: +; block0: +; cmp r0, #0 +; mov r0, r2; movne r0, r1 +; bx lr +; +; Disassembled: +; block0: ; offset 0x0 +; cmp r0, #0 +; mov r0, r2 +; movne r0, r1 +; bx lr + +function %select_i64(i32, i64, i64) -> i64 { +block0(v0: i32, v1: i64, v2: i64): + v3 = select v0, v1, v2 + return v3 +} + +; VCode: +; push {fp, lr} +; mov fp, sp +; sub sp, sp, #8 +; str r6, [sp, #4] +; block0: +; ldr r1, [incoming_arg, #8] +; ldr r6, [incoming_arg, #4] +; cmp r0, #0 +; mov r0, r1; movne r0, r2 +; mov r1, r6; movne r1, r3 +; ldr r6, [sp, #4] +; add sp, sp, #8 +; pop {fp, lr} +; bx lr +; +; Disassembled: +; block0: ; offset 0x0 +; push {fp, lr} +; mov fp, sp +; sub sp, sp, #8 +; str r6, [sp, #4] +; block1: ; offset 0x10 +; ldr r1, [sp, #0x10] +; ldr r6, [sp, #0x14] +; cmp r0, #0 +; mov r0, r1 +; movne r0, r2 +; mov r1, r6 +; movne r1, r3 +; ldr r6, [sp, #4] +; add sp, sp, #8 +; pop {fp, lr} +; bx lr + diff --git a/cranelift/filetests/filetests/isa/arm32/shifts.clif b/cranelift/filetests/filetests/isa/arm32/shifts.clif new file mode 100644 index 000000000000..4c5ad3aaab86 --- /dev/null +++ b/cranelift/filetests/filetests/isa/arm32/shifts.clif @@ -0,0 +1,91 @@ +test compile precise-output +target arm + +function %shl_imm(i32) -> i32 { +block0(v0: i32): + v1 = iconst.i32 3 + v2 = ishl v0, v1 + return v2 +} + +; VCode: +; block0: +; lsl r0, r0, #3 +; bx lr +; +; Disassembled: +; block0: ; offset 0x0 +; lsl r0, r0, #3 +; bx lr + +function %shl_reg(i32, i32) -> i32 { +block0(v0: i32, v1: i32): + v2 = ishl v0, v1 + return v2 +} + +; VCode: +; block0: +; and r3, r1, #31 +; lsl r0, r0, r3 +; bx lr +; +; Disassembled: +; block0: ; offset 0x0 +; and r3, r1, #0x1f +; lsl r0, r0, r3 +; bx lr + +function %ushr(i32, i32) -> i32 { +block0(v0: i32, v1: i32): + v2 = ushr v0, v1 + return v2 +} + +; VCode: +; block0: +; and r3, r1, #31 +; lsr r0, r0, r3 +; bx lr +; +; Disassembled: +; block0: ; offset 0x0 +; and r3, r1, #0x1f +; lsr r0, r0, r3 +; bx lr + +function %sshr_imm(i32) -> i32 { +block0(v0: i32): + v1 = iconst.i32 5 + v2 = sshr v0, v1 + return v2 +} + +; VCode: +; block0: +; asr r0, r0, #5 +; bx lr +; +; Disassembled: +; block0: ; offset 0x0 +; asr r0, r0, #5 +; bx lr + +function %rotr(i32, i32) -> i32 { +block0(v0: i32, v1: i32): + v2 = rotr v0, v1 + return v2 +} + +; VCode: +; block0: +; and r3, r1, #31 +; ror r0, r0, r3 +; bx lr +; +; Disassembled: +; block0: ; offset 0x0 +; and r3, r1, #0x1f +; ror r0, r0, r3 +; bx lr + diff --git a/cranelift/filetests/filetests/isa/arm32/trap.clif b/cranelift/filetests/filetests/isa/arm32/trap.clif new file mode 100644 index 000000000000..cf667fcd36f7 --- /dev/null +++ b/cranelift/filetests/filetests/isa/arm32/trap.clif @@ -0,0 +1,16 @@ +test compile precise-output +target arm + +function %trap() { +block0: + trap user1 +} + +; VCode: +; block0: +; udf ; user1 +; +; Disassembled: +; block0: ; offset 0x0 +; udf #0 ; trap: user1 + diff --git a/cranelift/filetests/wast-arm-runtest/Cargo.toml b/cranelift/filetests/wast-arm-runtest/Cargo.toml new file mode 100644 index 000000000000..8dd0e2e29d5f --- /dev/null +++ b/cranelift/filetests/wast-arm-runtest/Cargo.toml @@ -0,0 +1,27 @@ +[package] +name = "wast-arm-runtest" +version.workspace = true +authors.workspace = true +edition.workspace = true +publish = false + +[dependencies] +anyhow.workspace = true +json-from-wast.workspace = true +wasmparser.workspace = true +serde.workspace = true +serde_json.workspace = true +wasmtime-environ = { path = "../../../crates/environ", features = ["compile"] } +wasmtime-cranelift = { path = "../../../crates/cranelift", package = "wasmtime-internal-cranelift" } +cranelift-codegen.workspace = true +cranelift-frontend.workspace = true +cranelift-module.workspace = true +cranelift-object.workspace = true +target-lexicon.workspace = true +tempfile.workspace = true +wat.workspace = true +wast.workspace = true + +[features] +default = [] +arm32 = ["cranelift-codegen/arm32"] diff --git a/cranelift/filetests/wast-arm-runtest/src/cases.rs b/cranelift/filetests/wast-arm-runtest/src/cases.rs new file mode 100644 index 000000000000..b5086b10c6c3 --- /dev/null +++ b/cranelift/filetests/wast-arm-runtest/src/cases.rs @@ -0,0 +1,259 @@ +use json_from_wast::{Const, CoreConst, FloatConst}; +use wasmparser::Parser; + +#[derive(Debug)] +pub struct Case { + pub module: Vec, + pub export: String, + pub args: Vec, + pub expected: CaseValue, +} + +impl Case { + pub fn try_build( + module: &[u8], + field: &str, + args: &[Const], + expected: &[Const], + ) -> Result, String> { + let args = args + .iter() + .map(CaseValue::from_const) + .collect::, _>>()?; + + if expected.is_empty() { + return Err("expected at least one result, got 0".to_string()); + } + if expected.len() != 1 { + return Err(format!( + "unsupported result arity: expected 1 result, got {}", + expected.len() + )); + } + let expected = CaseValue::from_const(&expected[0])?; + + if !module_exports_func(module, field) { + return Err(format!("no func export named {field:?}")); + } + + Ok(Some(Case { + module: module.to_vec(), + export: field.to_string(), + args, + expected, + })) + } +} + +#[derive(Debug, Clone, Copy)] +pub enum CaseValue { + I32(i32), + I64(i64), + F32(FloatConst), + F64(FloatConst), +} + +impl PartialEq for CaseValue { + fn eq(&self, other: &Self) -> bool { + match (self, other) { + (Self::I32(a), Self::I32(b)) => a == b, + (Self::I64(a), Self::I64(b)) => a == b, + (Self::F32(a), Self::F32(b)) => a.to_bits() == b.to_bits(), + (Self::F64(a), Self::F64(b)) => a.to_bits() == b.to_bits(), + _ => false, + } + } +} + +impl CaseValue { + fn from_const(value: &Const) -> Result { + match value { + Const::Core(CoreConst::I32 { value }) => Ok(Self::I32(value.0)), + Const::Core(CoreConst::I64 { value }) => Ok(Self::I64(value.0)), + Const::Core(CoreConst::F32 { value }) => Ok(Self::F32(*value)), + Const::Core(CoreConst::F64 { value }) => Ok(Self::F64(*value)), + other => Err(format!("unsupported value: {other:?}")), + } + } + + pub(crate) fn to_wast_literal(self) -> String { + match self { + Self::F32(value) => match value { + FloatConst::ArithmeticNan => "nan:arithmetic".to_string(), + FloatConst::CanonicalNan => "nan:canonical".to_string(), + FloatConst::Value(bits) => format_f32_literal(bits.to_bits()), + }, + Self::F64(value) => match value { + FloatConst::ArithmeticNan => "nan:arithmetic".to_string(), + FloatConst::CanonicalNan => "nan:canonical".to_string(), + FloatConst::Value(bits) => format_f64_literal(bits.to_bits()), + }, + Self::I32(value) => value.to_string(), + Self::I64(value) => value.to_string(), + } + } +} + +fn format_float_literal(sign: &str, exponent: i32, fraction_bits: &str) -> String { + let mut hex_digits = String::new(); + for chunk in fraction_bits.chars().collect::>().chunks(4) { + let nibble = chunk.iter().collect::(); + let value = u8::from_str_radix(&nibble, 2).unwrap_or_default(); + let digit = match value { + 0..=9 => char::from(b'0' + value), + 10 => 'a', + 11 => 'b', + 12 => 'c', + 13 => 'd', + 14 => 'e', + 15 => 'f', + _ => unreachable!(), + }; + hex_digits.push(digit); + } + + let trimmed = hex_digits.trim_start_matches('0'); + if trimmed.is_empty() { + format!("{sign}0x1p{exponent:+}") + } else { + format!("{sign}0x1.{trimmed}p{exponent:+}") + } +} + +#[derive(Debug, Clone, Copy)] +struct FloatLiteralFormatConfig { + sign_mask: u64, + exponent_mask: u64, + fraction_mask: u64, + exponent_shift: u32, + exponent_bias: i32, + fraction_width: usize, + quiet_nan_mask: u64, + exponent_all_ones: u64, +} + +const F32_FLOAT_LITERAL_CONFIG: FloatLiteralFormatConfig = FloatLiteralFormatConfig { + sign_mask: 0x8000_0000, + exponent_mask: 0x7f80_0000, + fraction_mask: 0x007f_ffff, + exponent_shift: 23, + exponent_bias: 127, + fraction_width: 23, + quiet_nan_mask: 0x0040_0000, + exponent_all_ones: 0x7f80_0000, +}; + +const F64_FLOAT_LITERAL_CONFIG: FloatLiteralFormatConfig = FloatLiteralFormatConfig { + sign_mask: 0x8000_0000_0000_0000, + exponent_mask: 0x7ff0_0000_0000_0000, + fraction_mask: 0x000f_ffff_ffff_ffff, + exponent_shift: 52, + exponent_bias: 1023, + fraction_width: 52, + quiet_nan_mask: 0x0008_0000_0000_0000, + exponent_all_ones: 0x7ff0_0000_0000_0000, +}; + +fn format_float_literal_from_bits(bits: u64, config: FloatLiteralFormatConfig) -> String { + if bits == 0 { + return "0x0p+0".to_string(); + } + if bits == config.sign_mask { + return "-0x0p+0".to_string(); + } + + let sign = if bits & config.sign_mask != 0 { + "-" + } else { + "" + }; + let signless = bits & !config.sign_mask; + + if signless == config.exponent_all_ones { + return format!("{sign}inf"); + } + if signless & config.exponent_all_ones == config.exponent_all_ones { + return if signless & config.quiet_nan_mask == 0 { + format!("{sign}nan:canonical") + } else { + format!("{sign}nan:arithmetic") + }; + } + + let exponent = + ((signless & config.exponent_mask) >> config.exponent_shift) as i32 - config.exponent_bias; + let fraction_bits = format!( + "{:0width$b}", + signless & config.fraction_mask, + width = config.fraction_width + ); + format_float_literal(sign, exponent, &fraction_bits) +} + +fn format_f32_literal(bits: u32) -> String { + format_float_literal_from_bits(bits as u64, F32_FLOAT_LITERAL_CONFIG) +} + +fn format_f64_literal(bits: u64) -> String { + format_float_literal_from_bits(bits, F64_FLOAT_LITERAL_CONFIG) +} + +fn module_exports_func(module: &[u8], name: &str) -> bool { + let parser = Parser::new(0); + for payload in parser.parse_all(module).flatten() { + if let wasmparser::Payload::ExportSection(reader) = payload { + for export in reader.into_iter().flatten() { + if export.kind == wasmparser::ExternalKind::Func && export.name == name { + return true; + } + } + } + } + false +} + +#[cfg(test)] +mod tests { + use super::*; + use json_from_wast::{Const, CoreConst, IntString}; + + #[test] + fn accepts_i64_args_and_results() { + let arg = Const::Core(CoreConst::I64 { + value: IntString(7), + }); + let expected = Const::Core(CoreConst::I64 { + value: IntString(8), + }); + + assert_eq!(CaseValue::from_const(&arg).unwrap(), CaseValue::I64(7)); + assert_eq!(CaseValue::from_const(&expected).unwrap(), CaseValue::I64(8)); + } + + #[test] + fn accepts_float_args_and_results() { + let arg = Const::Core(CoreConst::F32 { + value: FloatConst::Value(1.5_f32), + }); + let expected = Const::Core(CoreConst::F64 { + value: FloatConst::Value(2.5_f64), + }); + + assert_eq!( + CaseValue::from_const(&arg).unwrap(), + CaseValue::F32(FloatConst::Value(1.5_f32)) + ); + assert_eq!( + CaseValue::from_const(&expected).unwrap(), + CaseValue::F64(FloatConst::Value(2.5_f64)) + ); + } + + #[test] + fn formats_special_float_literals_consistently() { + assert_eq!(format_f32_literal(0x8000_0000), "-0x0p+0"); + assert_eq!(format_f32_literal(0x7fc0_0000), "nan:arithmetic"); + assert_eq!(format_f64_literal(0x8000_0000_0000_0000), "-0x0p+0"); + assert_eq!(format_f64_literal(0x7ff8_0000_0000_0000), "nan:arithmetic"); + } +} diff --git a/cranelift/filetests/wast-arm-runtest/src/compile.rs b/cranelift/filetests/wast-arm-runtest/src/compile.rs new file mode 100644 index 000000000000..7467a4fefaa4 --- /dev/null +++ b/cranelift/filetests/wast-arm-runtest/src/compile.rs @@ -0,0 +1,100 @@ +use anyhow::Result; +use std::str::FromStr; +use target_lexicon::Triple; +use wasmtime_cranelift::builder; +use wasmtime_environ::{Compiler, Tunables}; + +pub fn build_arm32_compiler() -> Result<(Box, Tunables)> { + let triple = Triple::from_str("armv7-unknown-linux-gnueabihf") + .map_err(|e| anyhow::anyhow!("failed to parse target triple: {}", e))?; + let tunables = Tunables::default_for_target(&triple) + .map_err(|e| anyhow::anyhow!("failed to create default tunables for target: {}", e))?; + let mut b = builder(Some(triple)) + .map_err(|e| anyhow::anyhow!("failed to create compiler builder: {}", e))?; + b.set_tunables(tunables.clone()) + .map_err(|e| anyhow::anyhow!("failed to set tunables: {}", e))?; + let compiler = b + .build() + .map_err(|e| anyhow::anyhow!("failed to build compiler: {}", e))?; + Ok((compiler, tunables)) +} + +/// Compile a wasm module's first function and extract the machine code. +/// Returns: (bytes, alignment, module_translation). +#[cfg(test)] +mod tests { + use super::*; + use cranelift_codegen::ir::{AbiParam, Function, InstBuilder, Signature, UserFuncName, types}; + use cranelift_codegen::isa::{CallConv, lookup}; + use cranelift_codegen::{Context, settings}; + use cranelift_frontend::{FunctionBuilder, FunctionBuilderContext}; + use cranelift_module::{Linkage, Module, default_libcall_names}; + use cranelift_object::{ObjectBuilder, ObjectModule}; + + #[test] + fn test_object_emission() { + let triple = Triple::from_str("armv7-unknown-linux-gnueabihf").unwrap(); + let Some(isa) = lookup(triple.clone()).ok() else { + return; + }; + let Ok(isa) = isa.finish(settings::Flags::new(settings::builder())) else { + return; + }; + let mut module = ObjectModule::new( + ObjectBuilder::new(isa.clone(), "test", default_libcall_names()).unwrap(), + ); + + let mut sig_a = Signature::new(CallConv::triple_default(&triple)); + sig_a.params.push(AbiParam::new(types::I32)); + sig_a.returns.push(AbiParam::new(types::I32)); + + let func_a_id = module + .declare_function("func_a", Linkage::Local, &sig_a) + .unwrap(); + + let mut ctx_a = Context::new(); + ctx_a.func = + Function::with_name_signature(UserFuncName::user(0, func_a_id.as_u32()), sig_a.clone()); + let mut func_ctx = FunctionBuilderContext::new(); + { + let mut bcx = FunctionBuilder::new(&mut ctx_a.func, &mut func_ctx); + let block = bcx.create_block(); + bcx.switch_to_block(block); + bcx.append_block_params_for_function_params(block); + let param = bcx.block_params(block)[0]; + let result = bcx.ins().iadd_imm_s(param, 1); + bcx.ins().return_(&[result]); + bcx.seal_all_blocks(); + bcx.finalize(isa.frontend_config()); + } + module.define_function(func_a_id, &mut ctx_a).unwrap(); + + let mut sig_b = Signature::new(CallConv::triple_default(&triple)); + sig_b.params.push(AbiParam::new(types::I32)); + sig_b.returns.push(AbiParam::new(types::I32)); + + let func_b_id = module + .declare_function("func_b", Linkage::Local, &sig_b) + .unwrap(); + + let mut ctx_b = Context::new(); + ctx_b.func = + Function::with_name_signature(UserFuncName::user(0, func_b_id.as_u32()), sig_b.clone()); + { + let mut bcx = FunctionBuilder::new(&mut ctx_b.func, &mut func_ctx); + let block = bcx.create_block(); + bcx.switch_to_block(block); + bcx.append_block_params_for_function_params(block); + let param = bcx.block_params(block)[0]; + let local_func = module.declare_func_in_func(func_a_id, bcx.func); + let result = bcx.ins().call(local_func, &[param]); + let return_val = bcx.inst_results(result)[0]; + bcx.ins().return_(&[return_val]); + bcx.seal_all_blocks(); + bcx.finalize(isa.frontend_config()); + } + module.define_function(func_b_id, &mut ctx_b).unwrap(); + + let _ = module.finish().emit().unwrap(); + } +} diff --git a/cranelift/filetests/wast-arm-runtest/src/main.rs b/cranelift/filetests/wast-arm-runtest/src/main.rs new file mode 100644 index 000000000000..8122b54fbfb0 --- /dev/null +++ b/cranelift/filetests/wast-arm-runtest/src/main.rs @@ -0,0 +1,136 @@ +mod cases; +mod compile; +mod parse; +mod report; +mod run; + +use anyhow::Result; +use parse::{ParsedWast, parse_wast_file}; +use report::Report; +use std::path::Path; + +fn parse_cli_args(args: I) -> (Option, bool, bool) +where + I: IntoIterator, + S: AsRef, +{ + let mut path: Option = None; + let mut verbose = false; + let mut retain_temp_files = false; + + for arg in args { + match arg.as_ref() { + "-v" | "--verbose" => verbose = true, + "--retain-temp-files" => retain_temp_files = true, + _ if path.is_none() => path = Some(arg.as_ref().to_string()), + _ => {} + } + } + + (path, verbose, retain_temp_files) +} + +fn main() { + let (path, verbose, retain_temp_files) = parse_cli_args(std::env::args().skip(1)); + + let path = match path { + Some(p) => p, + None => { + eprintln!("usage: wast-arm-runtest [--verbose] [--retain-temp-files] "); + std::process::exit(1); + } + }; + + println!("Processing: {}", path); + + let parsed = parse_wast_file(Path::new(&path), verbose).expect("Failed to parse file"); + + if verbose { + println!( + "Commands: {}, Modules: {}", + parsed.command_count, parsed.module_count + ); + } + + let report = + process_all_cases(&parsed, verbose, retain_temp_files).expect("Failed to process cases"); + report.print_and_exit(); +} + +fn should_keep_tempdir(stop_on_error: bool) -> bool { + stop_on_error +} + +fn process_all_cases( + parsed: &ParsedWast, + verbose: bool, + retain_temp_files: bool, +) -> Result { + let mut report = Report::default(); + let workdir = tempfile::tempdir().expect("Failed to create temp dir"); + let workdir_path = workdir.path().to_path_buf(); + + for (idx, case) in parsed.cases.iter().enumerate() { + if verbose { + eprintln!( + "Processing case {}: {} with args {:?}", + idx, case.export, case.args + ); + } + } + + match run::run_cases_batch(&parsed.cases, workdir.path(), verbose) { + Ok(result) => { + let failed = result.failed > 0; + report.add_passed(result.passed); + report.add_failed(result.failed); + report.add_skipped(result.skipped); + if !result.output.trim().is_empty() { + println!("{}", result.output); + } + if failed && retain_temp_files { + eprintln!("Retaining temp files due to --retain-temp-files flag"); + } + } + Err(e) => { + report.add_failed(1); + eprintln!("ERROR batch run - {}", e); + if verbose { + eprintln!(" Backtrace: {:?}", e.backtrace()); + } + } + } + + if should_keep_tempdir(retain_temp_files) { + let _ = workdir.keep(); + eprintln!( + "Preserving temp directory for debugging: {}", + workdir_path.display() + ); + } else { + let _ = std::fs::remove_dir_all(&workdir_path); + } + + Ok(report) +} + +#[cfg(test)] +mod tests { + use super::{parse_cli_args, should_keep_tempdir}; + + #[test] + fn parses_cli_flags() { + let (path, verbose, retain_temp_files) = + parse_cli_args(["--verbose", "--retain-temp-files", "foo.wast"]); + + assert_eq!(path.as_deref(), Some("foo.wast")); + assert!(verbose); + assert!(retain_temp_files); + } + + #[test] + fn keeps_tempdir_when_stop_on_error_is_enabled() { + assert!(should_keep_tempdir(true)); + assert!(!should_keep_tempdir(false)); + } +} diff --git a/cranelift/filetests/wast-arm-runtest/src/parse.rs b/cranelift/filetests/wast-arm-runtest/src/parse.rs new file mode 100644 index 000000000000..f656d29e761d --- /dev/null +++ b/cranelift/filetests/wast-arm-runtest/src/parse.rs @@ -0,0 +1,75 @@ +use crate::cases::Case; +use anyhow::{Context, Result}; +use std::collections::HashMap; +use std::path::Path; + +pub struct ParsedWast { + pub cases: Vec, + #[allow(dead_code)] + pub skipped: u32, + pub command_count: usize, + pub module_count: usize, +} + +pub fn parse_wast_file(path: &Path, verbose: bool) -> Result { + let bytes = + std::fs::read(path).with_context(|| format!("failed to read {}", path.display()))?; + let text = std::str::from_utf8(&bytes).context("wast file not utf-8")?; + + let buf = wast::parser::ParseBuffer::new(text).context("lex wast")?; + let ast = wast::parser::parse::(&buf).context("parse wast")?; + let filename = path.to_string_lossy(); + + // Convert and extract data while text is still alive + let wast = json_from_wast::Opts::default() + .convert(&filename, text, ast) + .map_err(|e| anyhow::anyhow!("convert wast: {e:?}"))?; + + // Move wasms (owned) into a name -> bytes map + let wasms: HashMap> = wast.wasms.into_iter().collect(); + + let command_count = wast.commands.len(); + let module_count = wasms.len(); + let mut cases = Vec::new(); + let mut skipped = 0u32; + let mut current: Option<&[u8]> = None; + + // Walk commands while borrowed data is alive + for cmd in &wast.commands { + match cmd { + json_from_wast::Command::Module { file, .. } => { + current = wasms.get(file.filename.as_ref()).map(|v| v.as_slice()); + } + json_from_wast::Command::AssertReturn { + action, expected, .. + } => { + let json_from_wast::Action::Invoke { field, args, .. } = action else { + skipped += 1; + continue; + }; + let Some(module) = current else { + skipped += 1; + continue; + }; + match Case::try_build(module, field.as_ref(), args, expected) { + Ok(Some(case)) => cases.push(case), + Ok(None) => skipped += 1, + Err(reason) => { + if verbose { + eprintln!("skip: {reason}"); + } + skipped += 1; + } + } + } + _ => skipped += 1, + } + } + + Ok(ParsedWast { + cases, + skipped, + command_count, + module_count, + }) +} diff --git a/cranelift/filetests/wast-arm-runtest/src/report.rs b/cranelift/filetests/wast-arm-runtest/src/report.rs new file mode 100644 index 000000000000..c7253cc3e2e2 --- /dev/null +++ b/cranelift/filetests/wast-arm-runtest/src/report.rs @@ -0,0 +1,34 @@ +#[derive(Default)] +pub struct Report { + passed: u32, + skipped: u32, + failed: u32, +} + +impl Report { + pub fn print_and_exit(&self) { + self.print(); + if self.failed > 0 { + std::process::exit(1); + } + } + + pub fn print(&self) { + println!( + "{} passed, {} failed, {} skipped", + self.passed, self.failed, self.skipped + ); + } + + pub fn add_passed(&mut self, n: u32) { + self.passed += n; + } + + pub fn add_skipped(&mut self, n: u32) { + self.skipped += n; + } + + pub fn add_failed(&mut self, n: u32) { + self.failed += n; + } +} diff --git a/cranelift/filetests/wast-arm-runtest/src/run.rs b/cranelift/filetests/wast-arm-runtest/src/run.rs new file mode 100644 index 000000000000..ab11df708b23 --- /dev/null +++ b/cranelift/filetests/wast-arm-runtest/src/run.rs @@ -0,0 +1,878 @@ +use anyhow::Result; +use json_from_wast::FloatConst; +use std::collections::{HashMap, HashSet}; +use std::path::{Path, PathBuf}; +use std::str::FromStr; + +use crate::{ + cases::{Case, CaseValue}, + compile, +}; + +pub struct BatchRunResult { + pub output: String, + pub passed: u32, + pub failed: u32, + pub skipped: u32, +} + +#[derive(Clone)] +struct PreparedCase { + index: usize, + symbol_name: String, + runner_name: String, + bytes: Vec, + alignment: u32, + args: Vec, + expected: CaseValue, + store_ctx_off: u32, +} + +pub fn run_cases_batch(cases: &[Case], workdir: &Path, verbose: bool) -> Result { + let (compiler, tunables) = compile::build_arm32_compiler()?; + + let mut compiled_cases = Vec::new(); + let mut compiled_cache: HashMap = HashMap::new(); + let mut output_lines = Vec::new(); + let mut passed = 0u32; + let mut failed = 0u32; + let skipped = 0u32; + + for (idx, case) in cases.iter().enumerate() { + let runner_name = make_case_runner_name(idx, &case.export); + let cache_key = make_compiled_case_key(case); + + let prepared = match compiled_cache.get(&cache_key) { + Some(compiled) => PreparedCase { + index: idx, + symbol_name: compiled.symbol_name.clone(), + runner_name: runner_name.clone(), + bytes: compiled.bytes.clone(), + alignment: compiled.alignment, + args: case.args.clone(), + expected: case.expected, + store_ctx_off: compiled.store_ctx_off, + }, + None => match std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + prepare_case( + &*compiler, + &tunables, + idx, + case, + &make_compiled_symbol_name(case), + &runner_name, + ) + })) { + Ok(Ok(result)) => { + compiled_cache.insert(cache_key.clone(), result.clone()); + result + } + Ok(Err(e)) => { + output_lines.push(format!("FAIL case {idx}: {e}")); + failed += 1; + continue; + } + Err(payload) => { + output_lines.push(format!("FAIL case {idx}: {payload:?}")); + failed += 1; + continue; + } + }, + }; + + compiled_cases.push(prepared); + } + + if compiled_cases.is_empty() { + return Ok(BatchRunResult { + output: output_lines.join("\n"), + passed, + failed, + skipped, + }); + } + + let obj_bytes = emit_object_file_with_trampoline(&compiled_cases)?; + let obj_path = workdir.join("module.o"); + std::fs::write(&obj_path, &obj_bytes)?; + + let c_source = generate_c_driver(&compiled_cases, verbose); + let driver_path = workdir.join("driver.c"); + std::fs::write(&driver_path, &c_source)?; + + let elf_path = workdir.join("program"); + let toolchain = locate_toolchain()?; + link_with_gcc( + &toolchain.compiler, + &toolchain.sysroot, + &obj_path, + &driver_path, + &elf_path, + )?; + + let (harness_output, exit_code) = + run_under_qemu(&toolchain.qemu, &toolchain.sysroot, &elf_path)?; + if let Some((summary_passed, summary_failed)) = parse_result_summary(&harness_output) { + passed += summary_passed; + failed += summary_failed; + } else { + let harness_failed = if exit_code == 0 { + 0u32 + } else { + exit_code as u32 + }; + let harness_passed = compiled_cases.len().saturating_sub(harness_failed as usize) as u32; + passed += harness_passed; + failed += harness_failed; + } + + output_lines.push(harness_output.trim().to_string()); + let output = output_lines + .into_iter() + .filter(|line| !line.trim().is_empty()) + .collect::>() + .join("\n"); + + Ok(BatchRunResult { + output, + passed, + failed, + skipped, + }) +} + +fn prepare_case( + compiler: &dyn wasmtime_environ::Compiler, + tunables: &wasmtime_environ::Tunables, + index: usize, + case: &Case, + symbol_name: &str, + runner_name: &str, +) -> Result { + let (bytes, alignment, translation) = + compile_wasm_function(compiler, tunables, &case.module, &case.export, symbol_name)?; + let offsets = get_vmctx_offsets(&translation); + Ok(PreparedCase { + index, + symbol_name: symbol_name.to_string(), + runner_name: runner_name.to_string(), + bytes, + alignment, + args: case.args.clone(), + expected: case.expected, + store_ctx_off: offsets.store_ctx_off, + }) +} + +fn make_case_symbol_name(index: usize, export_name: &str) -> String { + let sanitized = export_name + .chars() + .map(|ch| if ch.is_ascii_alphanumeric() { ch } else { '_' }) + .collect::(); + let sanitized = sanitized.trim_matches('_'); + if sanitized.is_empty() { + format!("case_{index}") + } else { + format!("case_{index}_{sanitized}") + } +} + +fn make_case_runner_name(index: usize, export_name: &str) -> String { + format!( + "run_case_{index}_{}", + make_case_symbol_name(index, export_name) + ) +} + +fn make_compiled_case_key(case: &Case) -> String { + let mut key = String::new(); + key.push_str(&case.export); + key.push('|'); + for arg in &case.args { + key.push_str(arg.type_key()); + key.push(','); + } + key.push_str(case.expected.type_key()); + key.push('|'); + key.push_str(&format!("{:016x}", hash_bytes(&case.module))); + key +} + +fn make_compiled_symbol_name(case: &Case) -> String { + let export_name = make_case_symbol_name(0, &case.export); + let key = make_compiled_case_key(case); + let hash = hash_bytes(key.as_bytes()); + format!("wast_arm_runtest_module_{export_name}_{hash:016x}") +} + +fn hash_bytes(bytes: &[u8]) -> u64 { + let mut hash = 0xcbf2_9ce4_8422_2325u64; + for &byte in bytes { + hash ^= u64::from(byte); + hash = hash.wrapping_mul(0x0100_0000_01b3_u64); + } + hash +} + +fn compile_wasm_function<'data>( + compiler: &dyn wasmtime_environ::Compiler, + tunables: &wasmtime_environ::Tunables, + wasm_bytes: &'data [u8], + export_name: &str, + symbol_name: &str, +) -> Result<(Vec, u32, wasmtime_environ::ModuleTranslation<'data>)> { + use wasmparser::{Parser, Validator}; + use wasmtime_environ::{FuncKey, ModuleEnvironment, ModuleTypesBuilder, StaticModuleIndex}; + + let mut validator = Validator::new(); + let mut types = ModuleTypesBuilder::new(&validator); + let env = ModuleEnvironment::new( + tunables, + &mut validator, + &mut types, + StaticModuleIndex::from_u32(0), + ); + + let mut translation = env + .translate(Parser::new(0), wasm_bytes) + .map_err(|e| anyhow::anyhow!("failed to translate module: {}", e))?; + + // Find the function index for the export + let func_index = find_export_func_index(&translation.module, export_name) + .ok_or_else(|| anyhow::anyhow!("export '{}' not found", export_name))?; + + // Move function bodies out - we need to clone the map but FunctionBodyData doesn't Clone + // So we use a workaround: iterate and collect the specific body we need + let bodies = std::mem::take(&mut translation.function_body_inputs); + + // Find the body for our function index by iterating + let body_data = bodies + .into_iter() + .find(|(idx, _)| *idx == func_index) + .map(|(_, data)| data) + .ok_or_else(|| anyhow::anyhow!("function body not found for index {:?}", func_index))?; + + let key = FuncKey::DefinedWasmFunction(StaticModuleIndex::from_u32(0), func_index); + + // Compile the function + let mut cfb = compiler.compile_function(&translation, key, body_data, &types, symbol_name)?; + + // Finish compiling + compiler + .inlining_compiler() + .ok_or_else(|| anyhow::anyhow!("compiler does not support inlining"))? + .finish_compiling(&mut cfb, None, symbol_name) + .map_err(|e| anyhow::anyhow!("failed to finish compiling: {}", e))?; + + // Extract machine code bytes + let cf = cfb + .code + .downcast_ref::() + .ok_or_else(|| anyhow::anyhow!("expected CompiledFunction"))?; + + let bytes = cf.buffer.data().to_vec(); + let alignment = cf.alignment; + + Ok((bytes, alignment, translation)) +} + +fn find_export_func_index( + module: &wasmtime_environ::Module, + name: &str, +) -> Option { + use wasmtime_environ::EntityIndex; + + for (atom, entity_idx) in &module.exports { + let atom_str = module.strings.get(*atom); + if atom_str == Some(name) { + match entity_idx { + EntityIndex::Function(idx) => { + // FuncIndex is a u32 wrapper, we need to convert to DefinedFuncIndex + // For now just use the raw value - this works because both are u32-based + return Some(wasmtime_environ::DefinedFuncIndex::from_u32(idx.as_u32())); + } + _ => continue, + } + } + } + None +} + +fn exported_symbol_name(symbol_name: &str) -> String { + if symbol_name.starts_with("wast_arm_runtest_module_") { + symbol_name.to_string() + } else { + format!("wast_arm_runtest_module_{symbol_name}") + } +} + +fn emit_object_file_with_trampoline(cases: &[PreparedCase]) -> Result> { + use cranelift_codegen::ir::{AbiParam, Signature, types}; + use cranelift_codegen::isa::{CallConv, lookup}; + use cranelift_codegen::settings; + use cranelift_module::{Linkage, Module, default_libcall_names}; + use cranelift_object::{ObjectBuilder, ObjectModule}; + + let triple_str = "armv7-unknown-linux-gnueabihf"; + let triple = target_lexicon::Triple::from_str(triple_str) + .map_err(|e| anyhow::anyhow!("failed to parse target triple: {}", e))?; + let isa = lookup(triple.clone())?.finish(settings::Flags::new(settings::builder()))?; + let builder = ObjectBuilder::new(isa, "module", default_libcall_names())?; + let mut module = ObjectModule::new(builder); + + let ptr = types::I32; + let mut emitted_symbols = HashSet::new(); + + for case in cases { + // Export the compiled wasm function directly so C can call it with the + // expected AAPCS-like register layout for (vmctx, caller_vmctx, args...). + let mut wasm_sig = Signature::new(CallConv::triple_default(&triple)); + wasm_sig.params.push(AbiParam::new(ptr)); // vmctx + wasm_sig.params.push(AbiParam::new(ptr)); // caller_vmctx + for arg in &case.args { + wasm_sig.params.push(AbiParam::new(arg.cranelift_type())); + } + wasm_sig + .returns + .push(AbiParam::new(case.expected.cranelift_type())); + + let symbol_name = exported_symbol_name(&case.symbol_name); + if emitted_symbols.contains(&symbol_name) { + continue; + } + emitted_symbols.insert(symbol_name.clone()); + + let wasm_id = module.declare_function(&symbol_name, Linkage::Export, &wasm_sig)?; + module.define_function_bytes(wasm_id, case.alignment as u64, &case.bytes, &[])?; + } + + Ok(module.finish().emit()?) +} + +fn get_vmctx_offsets(translation: &wasmtime_environ::ModuleTranslation<'_>) -> VmctxOffsets { + use wasmtime_environ::{PtrSize, VMOffsets}; + + let offsets = VMOffsets::new(4u8, &translation.module); + VmctxOffsets { + store_ctx_off: offsets.ptr.vmcontext_store_context() as u32, + stack_limit_off: offsets.ptr.vmstore_context_stack_limit() as u32, + } +} + +struct VmctxOffsets { + store_ctx_off: u32, + #[allow(dead_code)] + stack_limit_off: u32, +} + +impl CaseValue { + fn type_key(self) -> &'static str { + self.dispatch(|_| "i32", |_| "i64", |_| "f32", |_| "f64") + } + + fn dispatch( + self, + on_i32: impl FnOnce(i32) -> T, + on_i64: impl FnOnce(i64) -> T, + on_f32: impl FnOnce(FloatConst) -> T, + on_f64: impl FnOnce(FloatConst) -> T, + ) -> T { + match self { + Self::I32(value) => on_i32(value), + Self::I64(value) => on_i64(value), + Self::F32(value) => on_f32(value), + Self::F64(value) => on_f64(value), + } + } + + fn cranelift_type(self) -> cranelift_codegen::ir::Type { + self.dispatch( + |_| cranelift_codegen::ir::types::I32, + |_| cranelift_codegen::ir::types::I64, + |_| cranelift_codegen::ir::types::F32, + |_| cranelift_codegen::ir::types::F64, + ) + } + + fn c_type_name(self) -> &'static str { + self.dispatch(|_| "int", |_| "long long", |_| "float", |_| "double") + } + + fn c_assignment(self, index: usize) -> String { + self.dispatch( + |value| format!(" int a{index} = {value};"), + |value| format!(" long long a{index} = {value}LL;"), + |bits| { + format!( + " float a{index} = bits_to_f32(0x{:08x}u);", + bits.to_bits() + ) + }, + |bits| { + format!( + " double a{index} = bits_to_f64(0x{:016x}ULL);", + bits.to_bits() + ) + }, + ) + } + + fn c_compare_stmt(self, got_name: &str) -> String { + self.dispatch( + |value| format!(" int passed = ({got_name} == {value});"), + |value| format!(" int passed = ({got_name} == {value}LL);"), + |bits| { + format!( + " int passed = (f32_to_bits({got_name}) == 0x{:08x}u);", + bits.to_bits() + ) + }, + |bits| { + format!( + " int passed = (f64_to_bits({got_name}) == 0x{:016x}ULL);", + bits.to_bits() + ) + }, + ) + } + + fn c_expected_literal(self) -> String { + let literal = self.to_wast_literal(); + format!("\"{}\"", literal.replace('\\', "\\\\").replace('"', "\\\"")) + } + + fn c_got_literal_stmt(self) -> String { + self.dispatch( + |_| " char got_literal_buf[64];\n snprintf(got_literal_buf, sizeof(got_literal_buf), \"%d\", got);\n const char *got_literal = got_literal_buf;".to_string(), + |_| " char got_literal_buf[64];\n snprintf(got_literal_buf, sizeof(got_literal_buf), \"%lld\", got);\n const char *got_literal = got_literal_buf;".to_string(), + |_| " char got_literal_buf[64];\n snprintf(got_literal_buf, sizeof(got_literal_buf), \"%f\", got);\n const char *got_literal = got_literal_buf;".to_string(), + |_| " char got_literal_buf[64];\n snprintf(got_literal_buf, sizeof(got_literal_buf), \"%f\", got);\n const char *got_literal = got_literal_buf;".to_string(), + ) + } +} + +fn format_c_arg_params(args: &[CaseValue]) -> String { + if args.is_empty() { + String::new() + } else { + let params = args + .iter() + .enumerate() + .map(|(i, arg)| format!(" {} a{}", arg.c_type_name(), i)) + .collect::>() + .join(","); + format!(", {}", params) + } +} + +fn format_c_arg_calls(args: &[CaseValue]) -> String { + if args.is_empty() { + String::new() + } else { + let args_call = args + .iter() + .enumerate() + .map(|(i, _)| format!("a{i}")) + .collect::>() + .join(", "); + format!(", {}", args_call) + } +} + +fn c_driver_helpers(verbose: bool) -> String { + let verbose_define = if verbose { "1" } else { "0" }; + let mut helper = String::from( + "#include \n#include \n#include \n\n#define WAST_ARM_RUNTTEST_VERBOSE ", + ); + helper.push_str(verbose_define); + helper.push_str( + "\n\nstatic float bits_to_f32(uint32_t bits) {\n float out;\n memcpy(&out, &bits, sizeof(out));\n return out;\n}\n\nstatic double bits_to_f64(uint64_t bits) {\n double out;\n memcpy(&out, &bits, sizeof(out));\n return out;\n}\n\nstatic uint32_t f32_to_bits(float value) {\n uint32_t bits;\n memcpy(&bits, &value, sizeof(bits));\n return bits;\n}\n\nstatic uint64_t f64_to_bits(double value) {\n uint64_t bits;\n memcpy(&bits, &value, sizeof(bits));\n return bits;\n}\n\nstatic const char *f32_to_literal(uint32_t bits) {\n static char buf[64];\n float value = bits_to_f32(bits);\n if (bits == 0x00000000u) return \"0x0p+0\";\n if (bits == 0x80000000u) return \"-0x0p+0\";\n if ((bits & 0x7f800000u) == 0x7f800000u) {\n if ((bits & 0x00400000u) == 0) return \"nan:canonical\";\n return \"nan:arithmetic\";\n }\n snprintf(buf, sizeof(buf), \"%a\", (double)value);\n return buf;\n}\n\nstatic const char *f64_to_literal(uint64_t bits) {\n static char buf[64];\n double value = bits_to_f64(bits);\n if (bits == 0x0000000000000000ULL) return \"0x0p+0\";\n if (bits == 0x8000000000000000ULL) return \"-0x0p+0\";\n if ((bits & 0x7ff0000000000000ULL) == 0x7ff0000000000000ULL) {\n if ((bits & 0x0008000000000000ULL) == 0) return \"nan:canonical\";\n return \"nan:arithmetic\";\n }\n snprintf(buf, sizeof(buf), \"%a\", value);\n return buf;\n}\n", + ); + helper +} + +fn c_driver_prototype(ret_ty: &str, export_name: &str, args_params: &str) -> String { + format!("extern {ret_ty} {export_name}(void *vmctx, void *caller_vmctx{args_params});") +} + +struct CDriverMainContext<'a> { + store_ctx_off: u32, + sc_base: u32, + export_name: &'a str, + runner_name: &'a str, + case_label: &'a str, + arg_assignments: &'a str, + got_decl: &'a str, + args_call: &'a str, + compare_stmt: &'a str, + got_literal_stmt: &'a str, + expected_literal: &'a str, +} + +fn c_driver_main(context: CDriverMainContext<'_>) -> String { + let CDriverMainContext { + store_ctx_off, + sc_base, + export_name, + runner_name, + case_label, + arg_assignments, + got_decl, + args_call, + compare_stmt, + got_literal_stmt, + expected_literal, + } = context; + + format!( + "int {runner_name}(void) {{\n\ + unsigned char buf[256] = {{0}};\n\ + // VMContext.store_context (at STORE_CTX_OFF) -> points at buf+SC_BASE\n\ + *(uintptr_t*)(buf + {store_ctx_off}) = (uintptr_t)(buf + {sc_base});\n\ + // VMStoreContext.stack_limit at SC_BASE + STACK_LIMIT_OFF is 0 (buf is zero-initialized)\n\ + {arg_assignments}\n\ + {got_decl} = {export_name}((void*)buf, (void*)buf{args_call});\n\ + {got_literal_stmt}\n\ + {compare_stmt}\n\ + if (passed) {{\n\ + return 0;\n\ + }} else {{\n\ + printf(\"FAIL case {case_label}: expected %s, got %s\\n\", {expected_literal}, got_literal);\n\ + return 1;\n\ + }}\n\ + }}" + ) +} + +fn generate_c_driver(cases: &[PreparedCase], verbose: bool) -> String { + let sc_base = 64u32; + let mut driver = String::new(); + driver.push_str(&c_driver_helpers(verbose)); + driver.push_str("\n\n"); + + for case in cases { + let args_params_str = format_c_arg_params(&case.args); + let args_call_str = format_c_arg_calls(&case.args); + + let arg_assignments = case + .args + .iter() + .enumerate() + .map(|(i, arg)| arg.c_assignment(i)) + .collect::>() + .join("\n"); + + let ret_ty = case.expected.c_type_name(); + let got_decl = format!("{} got", ret_ty); + let compare_stmt = case.expected.c_compare_stmt("got"); + let got_literal_stmt = case.expected.c_got_literal_stmt(); + let expected_literal = case.expected.c_expected_literal(); + let case_label = format!("{}", case.index); + + let exported_name = exported_symbol_name(&case.symbol_name); + + driver.push_str(&c_driver_prototype( + ret_ty, + &exported_name, + &args_params_str, + )); + driver.push_str("\n\n"); + driver.push_str(&c_driver_main(CDriverMainContext { + store_ctx_off: case.store_ctx_off, + sc_base, + export_name: &exported_name, + runner_name: &case.runner_name, + case_label: &case_label, + arg_assignments: &arg_assignments, + got_decl: &got_decl, + args_call: &args_call_str, + compare_stmt: &compare_stmt, + got_literal_stmt: &got_literal_stmt, + expected_literal: &expected_literal, + })); + driver.push_str("\n\n"); + } + + driver.push_str("int main(void) {\n"); + driver.push_str(" int passed = 0;\n"); + driver.push_str(" int failed = 0;\n"); + for (idx, case) in cases.iter().enumerate() { + driver.push_str(&format!(" int result_{idx} = {}();\n", case.runner_name)); + driver.push_str(&format!(" if (result_{idx} == 0) {{\n")); + driver.push_str(" passed += 1;\n"); + driver.push_str( + " } else { +", + ); + driver.push_str(" failed += 1;\n"); + driver.push_str( + " } +", + ); + } + if verbose { + driver.push_str(" printf(\"RESULT passed=%d failed=%d\\n\", passed, failed);\n"); + } + driver.push_str(" return failed;\n}\n"); + driver +} + +fn parse_result_summary(output: &str) -> Option<(u32, u32)> { + output.lines().find_map(|line| { + let line = line.trim(); + let mut words = line.split_whitespace(); + if words.next()? != "RESULT" { + return None; + } + + let passed = words.next()?.strip_prefix("passed=")?.parse::().ok()?; + let failed = words.next()?.strip_prefix("failed=")?.parse::().ok()?; + Some((passed, failed)) + }) +} + +struct Toolchain { + compiler: std::path::PathBuf, + sysroot: Option, + qemu: std::path::PathBuf, +} + +fn locate_toolchain() -> Result { + let compiler_candidates = [ + "arm-linux-gnueabihf-gcc", + "arm-linux-gnueabihf-gcc-13", + "arm-linux-gnueabihf-gcc-12", + "arm-linux-gnueabihf-gcc-11", + "arm-none-eabi-gcc", + ]; + let qemu_candidates = ["qemu-arm-static", "qemu-arm"]; + + let compiler = compiler_candidates + .iter() + .find_map(|name| find_executable(name)) + .ok_or_else(|| { + anyhow::anyhow!( + "could not find an ARM cross-compiler; tried {:?}", + compiler_candidates + ) + })?; + let qemu = qemu_candidates + .iter() + .find_map(|name| find_executable(name)) + .ok_or_else(|| { + anyhow::anyhow!("could not find QEMU for ARM; tried {:?}", qemu_candidates) + })?; + + let sysroot = detect_sysroot(&compiler).or_else(detect_sysroot_from_env); + Ok(Toolchain { + compiler, + sysroot, + qemu, + }) +} + +fn find_executable(name: &str) -> Option { + std::env::var_os("PATH")?; + std::env::split_paths(&std::env::var_os("PATH")?).find_map(|dir| { + let path = dir.join(name); + path.is_file().then_some(path) + }) +} + +fn detect_sysroot(compiler: &std::path::Path) -> Option { + let output = std::process::Command::new(compiler) + .arg("-print-sysroot") + .output() + .ok()?; + let sysroot = String::from_utf8_lossy(&output.stdout).trim().to_string(); + let sysroot = if sysroot.is_empty() || sysroot == "/" { + None + } else { + Some(std::path::PathBuf::from(sysroot)) + }; + + sysroot.or_else(|| { + [ + std::path::PathBuf::from("/usr/arm-linux-gnueabihf"), + std::path::PathBuf::from("/usr/arm-linux-gnueabihf/lib"), + ] + .into_iter() + .find(|path| path.join("lib/ld-linux-armhf.so.3").is_file()) + }) +} + +fn detect_sysroot_from_env() -> Option { + std::env::var_os("ARM_LINUX_GNUEABIHF_SYSROOT") + .map(std::path::PathBuf::from) + .or_else(|| std::env::var_os("CROSS_SYSROOT").map(std::path::PathBuf::from)) +} + +fn link_with_gcc( + gcc_path: &Path, + sysroot: &Option, + obj_path: &Path, + driver_path: &Path, + elf_path: &Path, +) -> Result<()> { + let mut command = std::process::Command::new(gcc_path); + command + .arg("-marm") + .arg(driver_path) + .arg(obj_path) + .arg("-o") + .arg(elf_path); + if let Some(sysroot) = sysroot { + command.arg("-L").arg(sysroot.join("lib")); + } + let status = command.status()?; + + if !status.success() { + anyhow::bail!("linking failed with status: {}", status); + } + Ok(()) +} + +fn run_under_qemu( + qemu: &Path, + sysroot: &Option, + elf_path: &Path, +) -> Result<(String, i32)> { + let mut command = std::process::Command::new(qemu); + if let Some(sysroot) = sysroot { + command.arg("-L").arg(sysroot); + } + command.arg(elf_path); + let output = command.output()?; + + let result = String::from_utf8_lossy(&output.stdout); + let stderr = String::from_utf8_lossy(&output.stderr); + + let exit_code = output.status.code().unwrap_or(-1); + if exit_code < 0 { + anyhow::bail!( + "QEMU execution failed with status: {}, stderr: {}", + output.status, + stderr.trim() + ); + } + + Ok((result.trim().to_string(), exit_code)) +} + +#[cfg(test)] +mod tests { + use super::*; + use json_from_wast::FloatConst; + + #[test] + fn driver_uses_wast_style_float_literals() { + let prepared = PreparedCase { + index: 0, + symbol_name: "f".to_string(), + runner_name: "run_f".to_string(), + bytes: vec![], + alignment: 0, + args: vec![], + expected: CaseValue::F32(FloatConst::Value(1.0)), + store_ctx_off: 0, + }; + let driver = generate_c_driver(&[prepared], false); + assert!(driver.contains("#define WAST_ARM_RUNTTEST_VERBOSE 0")); + assert!(!driver.contains("RESULT")); + assert!(!driver.contains("PASS case")); + } + + #[test] + fn non_verbose_mode_suppresses_result_summary() { + let prepared = PreparedCase { + index: 0, + symbol_name: "f".to_string(), + runner_name: "run_f".to_string(), + bytes: vec![], + alignment: 0, + args: vec![], + expected: CaseValue::F32(FloatConst::Value(1.0)), + store_ctx_off: 0, + }; + let driver = generate_c_driver(&[prepared], false); + assert!(driver.contains("#define WAST_ARM_RUNTTEST_VERBOSE 0")); + assert!(!driver.contains("RESULT")); + } + + #[test] + fn verbose_mode_emits_result_summary() { + let prepared = PreparedCase { + index: 0, + symbol_name: "f".to_string(), + runner_name: "run_f".to_string(), + bytes: vec![], + alignment: 0, + args: vec![], + expected: CaseValue::F32(FloatConst::Value(1.0)), + store_ctx_off: 0, + }; + let driver = generate_c_driver(&[prepared], true); + assert!(driver.contains("#define WAST_ARM_RUNTTEST_VERBOSE 1")); + assert!(driver.contains("RESULT passed=%d failed=%d")); + } + + #[test] + fn generated_driver_uses_unique_result_names_per_case() { + let prepared_a = PreparedCase { + index: 0, + symbol_name: "f".to_string(), + runner_name: "run_f".to_string(), + bytes: vec![], + alignment: 0, + args: vec![], + expected: CaseValue::F32(FloatConst::Value(1.0)), + store_ctx_off: 0, + }; + let prepared_b = PreparedCase { + index: 1, + symbol_name: "g".to_string(), + runner_name: "run_g".to_string(), + bytes: vec![], + alignment: 0, + args: vec![], + expected: CaseValue::F32(FloatConst::Value(1.0)), + store_ctx_off: 0, + }; + let driver = generate_c_driver(&[prepared_a, prepared_b], false); + assert!(driver.contains("int result_0 = run_f();")); + assert!(driver.contains("int result_1 = run_g();")); + } + + #[test] + fn case_symbol_names_are_unique_and_c_safe() { + assert_eq!(make_case_symbol_name(0, "foo"), "case_0_foo"); + assert_eq!(make_case_symbol_name(1, "foo-bar"), "case_1_foo_bar"); + assert_eq!(make_case_symbol_name(2, "foo.bar"), "case_2_foo_bar"); + } + + #[test] + fn exported_symbols_are_prefixed_for_the_c_driver() { + assert_eq!( + exported_symbol_name("case_0_foo"), + "wast_arm_runtest_module_case_0_foo" + ); + assert_eq!( + exported_symbol_name("wast_arm_runtest_module_case_0_foo"), + "wast_arm_runtest_module_case_0_foo" + ); + } + + #[test] + fn unsupported_backend_errors_are_failed() { + let outcome = classify_case_error( + "Unsupported feature: should be implemented in ISLE: inst = `v4 = rotl.i32`", + ); + assert!(matches!(outcome, CaseOutcome::Fail(_))); + } + + #[test] + fn panic_errors_are_failed() { + let outcome = classify_case_error("panic while compiling wasm function"); + assert!(matches!(outcome, CaseOutcome::Fail(_))); + } +} diff --git a/cranelift/object/src/backend.rs b/cranelift/object/src/backend.rs index b8d7eb9c44b5..4675f7dc42fd 100644 --- a/cranelift/object/src/backend.rs +++ b/cranelift/object/src/backend.rs @@ -22,7 +22,7 @@ use std::collections::HashMap; use std::collections::hash_map::Entry; use std::fmt::Write as _; use std::mem; -use target_lexicon::{PointerWidth, Triple}; +use target_lexicon::{Environment, PointerWidth, Triple}; /// A builder for `ObjectModule`. pub struct ObjectBuilder { @@ -74,7 +74,29 @@ impl ObjectBuilder { let architecture = match isa.triple().architecture { target_lexicon::Architecture::X86_32(_) => object::Architecture::I386, target_lexicon::Architecture::X86_64 => object::Architecture::X86_64, - target_lexicon::Architecture::Arm(_) => object::Architecture::Arm, + target_lexicon::Architecture::Arm(_) => { + if binary_format != object::BinaryFormat::Elf { + return Err(ModuleError::Backend(anyhow!( + "binary format {binary_format:?} is not supported for arm", + ))); + } + + let mut e_flags = object::elf::EF_ARM_EABI_VER5; + if matches!( + isa.triple().environment, + Environment::Gnueabihf | Environment::Musleabihf + ) { + e_flags |= object::elf::EF_ARM_ABI_FLOAT_HARD; + } + + // ARM EABI version 5 is the standard for ARM Linux ELF objects. + file_flags = object::FileFlags::Elf { + os_abi: object::elf::ELFOSABI_NONE, + abi_version: 0, + e_flags, + }; + object::Architecture::Arm + } target_lexicon::Architecture::Aarch64(_) => object::Architecture::Aarch64, target_lexicon::Architecture::Riscv64(_) => { if binary_format != object::BinaryFormat::Elf { @@ -894,6 +916,16 @@ impl ObjectModule { encoding: RelocationEncoding::AArch64Call, size: 26, }, + Reloc::Arm32Call => match self.object.format() { + // The classic arm32 backend emits A32 (ARM-mode) `bl`, so the + // call site must use the ARM call relocation `R_ARM_CALL`. + // Using the Thumb relocation `R_ARM_THM_PC22` here would make + // the linker patch the instruction as Thumb and corrupt it. + object::BinaryFormat::Elf => RelocationFlags::Elf { + r_type: object::elf::R_ARM_CALL, + }, + _ => unimplemented!("Arm32Call is not supported for this file format"), + }, Reloc::ElfX86_64TlsGd => { assert_eq!( self.object.format(),