From 2e330917fc18b3834f58ea356681ce5424b076b1 Mon Sep 17 00:00:00 2001 From: WANG Rui Date: Thu, 28 May 2026 12:12:42 +0800 Subject: [PATCH] loongarch: Use `intrinsics::simd` for v{ld,st}[x] --- .../src/loongarch64/lasx/generated.rs | 40 --------- .../src/loongarch64/lasx/portable.rs | 8 ++ .../src/loongarch64/lsx/generated.rs | 40 --------- .../core_arch/src/loongarch64/lsx/portable.rs | 8 ++ crates/core_arch/src/loongarch64/simd.rs | 86 +++++++++++++++++++ crates/stdarch-gen-loongarch/lasx.spec | 4 + crates/stdarch-gen-loongarch/lsx.spec | 4 + .../src/portable-intrinsics.txt | 8 ++ 8 files changed, 118 insertions(+), 80 deletions(-) diff --git a/crates/core_arch/src/loongarch64/lasx/generated.rs b/crates/core_arch/src/loongarch64/lasx/generated.rs index 6a6a3ae924..6c0934b01d 100644 --- a/crates/core_arch/src/loongarch64/lasx/generated.rs +++ b/crates/core_arch/src/loongarch64/lasx/generated.rs @@ -455,10 +455,6 @@ unsafe extern "unadjusted" { fn __lasx_xvfrintrm_s(a: __v8f32) -> __v8f32; #[link_name = "llvm.loongarch.lasx.xvfrintrm.d"] fn __lasx_xvfrintrm_d(a: __v4f64) -> __v4f64; - #[link_name = "llvm.loongarch.lasx.xvld"] - fn __lasx_xvld(a: *const i8, b: i32) -> __v32i8; - #[link_name = "llvm.loongarch.lasx.xvst"] - fn __lasx_xvst(a: __v32i8, b: *mut i8, c: i32); #[link_name = "llvm.loongarch.lasx.xvstelm.b"] fn __lasx_xvstelm_b(a: __v32i8, b: *mut i8, c: i32, d: u32); #[link_name = "llvm.loongarch.lasx.xvstelm.h"] @@ -489,10 +485,6 @@ unsafe extern "unadjusted" { fn __lasx_xvssrln_w_d(a: __v4i64, b: __v4i64) -> __v8i32; #[link_name = "llvm.loongarch.lasx.xvldi"] fn __lasx_xvldi(a: i32) -> __v4i64; - #[link_name = "llvm.loongarch.lasx.xvldx"] - fn __lasx_xvldx(a: *const i8, b: i64) -> __v32i8; - #[link_name = "llvm.loongarch.lasx.xvstx"] - fn __lasx_xvstx(a: __v32i8, b: *mut i8, c: i64); #[link_name = "llvm.loongarch.lasx.xvextl.qu.du"] fn __lasx_xvextl_qu_du(a: __v4u64) -> __v4u64; #[link_name = "llvm.loongarch.lasx.vext2xv.h.b"] @@ -2665,24 +2657,6 @@ pub fn lasx_xvfrintrm_d(a: m256d) -> m256d { unsafe { transmute(__lasx_xvfrintrm_d(transmute(a))) } } -#[inline] -#[target_feature(enable = "lasx")] -#[rustc_legacy_const_generics(1)] -#[unstable(feature = "stdarch_loongarch", issue = "117427")] -pub unsafe fn lasx_xvld(mem_addr: *const i8) -> m256i { - static_assert_simm_bits!(IMM_S12, 12); - transmute(__lasx_xvld(mem_addr, IMM_S12)) -} - -#[inline] -#[target_feature(enable = "lasx")] -#[rustc_legacy_const_generics(2)] -#[unstable(feature = "stdarch_loongarch", issue = "117427")] -pub unsafe fn lasx_xvst(a: m256i, mem_addr: *mut i8) { - static_assert_simm_bits!(IMM_S12, 12); - __lasx_xvst(transmute(a), mem_addr, IMM_S12) -} - #[inline] #[target_feature(enable = "lasx")] #[rustc_legacy_const_generics(2, 3)] @@ -2810,20 +2784,6 @@ pub fn lasx_xvldi() -> m256i { unsafe { transmute(__lasx_xvldi(IMM_S13)) } } -#[inline] -#[target_feature(enable = "lasx")] -#[unstable(feature = "stdarch_loongarch", issue = "117427")] -pub unsafe fn lasx_xvldx(mem_addr: *const i8, b: i64) -> m256i { - transmute(__lasx_xvldx(mem_addr, transmute(b))) -} - -#[inline] -#[target_feature(enable = "lasx")] -#[unstable(feature = "stdarch_loongarch", issue = "117427")] -pub unsafe fn lasx_xvstx(a: m256i, mem_addr: *mut i8, b: i64) { - __lasx_xvstx(transmute(a), mem_addr, transmute(b)) -} - #[inline] #[target_feature(enable = "lasx")] #[unstable(feature = "stdarch_loongarch", issue = "117427")] diff --git a/crates/core_arch/src/loongarch64/lasx/portable.rs b/crates/core_arch/src/loongarch64/lasx/portable.rs index 4dfe80795f..b6f4fdcb25 100644 --- a/crates/core_arch/src/loongarch64/lasx/portable.rs +++ b/crates/core_arch/src/loongarch64/lasx/portable.rs @@ -339,6 +339,10 @@ impl_gv!("lasx", lasx_xvreplgr2vr_h, ls::simd_splat, m256i, i16x16, i32); impl_gv!("lasx", lasx_xvreplgr2vr_w, ls::simd_splat, m256i, i32x8, i32); impl_gv!("lasx", lasx_xvreplgr2vr_d, ls::simd_splat, m256i, i64x4, i64); +impl_ggv!("lasx", lasx_xvldx, simd_ldx, m256i, i8x32, *const i8, i64, unsafe); + +impl_gsv!("lasx", lasx_xvld, simd_ld, m256i, i8x32, *const i8, 12, const, unsafe); + impl_sv!("lasx", lasx_xvrepli_b, ls::simd_splat, m256i, i8x32, 10); impl_sv!("lasx", lasx_xvrepli_h, ls::simd_splat, m256i, i16x16, 10); impl_sv!("lasx", lasx_xvrepli_w, ls::simd_splat, m256i, i32x8, 10); @@ -499,6 +503,10 @@ impl_vvv!("lasx", lasx_xvpackod_h, simd_packod_h, m256i, i16x16); impl_vvv!("lasx", lasx_xvpackod_w, simd_packod_w, m256i, i32x8); impl_vvv!("lasx", lasx_xvpackod_d, simd_packod_d, m256i, i64x4); +impl_vgg!("lasx", lasx_xvstx, simd_stx, m256i, i8x32, *mut i8, i64, unsafe); + +impl_vgs!("lasx", lasx_xvst, simd_st, m256i, i8x32, *mut i8, 12, const, unsafe); + impl_vuv!("lasx", lasx_xvslli_b, is::simd_shl, m256i, i8x32); impl_vuv!("lasx", lasx_xvslli_h, is::simd_shl, m256i, i16x16); impl_vuv!("lasx", lasx_xvslli_w, is::simd_shl, m256i, i32x8); diff --git a/crates/core_arch/src/loongarch64/lsx/generated.rs b/crates/core_arch/src/loongarch64/lsx/generated.rs index 555866040e..fc79ce3fe6 100644 --- a/crates/core_arch/src/loongarch64/lsx/generated.rs +++ b/crates/core_arch/src/loongarch64/lsx/generated.rs @@ -789,10 +789,6 @@ unsafe extern "unadjusted" { fn __lsx_vssrarni_du_q(a: __v2u64, b: __v2i64, c: u32) -> __v2u64; #[link_name = "llvm.loongarch.lsx.vpermi.w"] fn __lsx_vpermi_w(a: __v4i32, b: __v4i32, c: u32) -> __v4i32; - #[link_name = "llvm.loongarch.lsx.vld"] - fn __lsx_vld(a: *const i8, b: i32) -> __v16i8; - #[link_name = "llvm.loongarch.lsx.vst"] - fn __lsx_vst(a: __v16i8, b: *mut i8, c: i32); #[link_name = "llvm.loongarch.lsx.vssrlrn.b.h"] fn __lsx_vssrlrn_b_h(a: __v8i16, b: __v8i16) -> __v16i8; #[link_name = "llvm.loongarch.lsx.vssrlrn.h.w"] @@ -809,10 +805,6 @@ unsafe extern "unadjusted" { fn __lsx_vldi(a: i32) -> __v2i64; #[link_name = "llvm.loongarch.lsx.vshuf.b"] fn __lsx_vshuf_b(a: __v16i8, b: __v16i8, c: __v16i8) -> __v16i8; - #[link_name = "llvm.loongarch.lsx.vldx"] - fn __lsx_vldx(a: *const i8, b: i64) -> __v16i8; - #[link_name = "llvm.loongarch.lsx.vstx"] - fn __lsx_vstx(a: __v16i8, b: *mut i8, c: i64); #[link_name = "llvm.loongarch.lsx.vextl.qu.du"] fn __lsx_vextl_qu_du(a: __v2u64) -> __v2u64; #[link_name = "llvm.loongarch.lsx.bnz.b"] @@ -3876,24 +3868,6 @@ pub fn lsx_vpermi_w(a: m128i, b: m128i) -> m128i { unsafe { transmute(__lsx_vpermi_w(transmute(a), transmute(b), IMM8)) } } -#[inline] -#[target_feature(enable = "lsx")] -#[rustc_legacy_const_generics(1)] -#[unstable(feature = "stdarch_loongarch", issue = "117427")] -pub unsafe fn lsx_vld(mem_addr: *const i8) -> m128i { - static_assert_simm_bits!(IMM_S12, 12); - transmute(__lsx_vld(mem_addr, IMM_S12)) -} - -#[inline] -#[target_feature(enable = "lsx")] -#[rustc_legacy_const_generics(2)] -#[unstable(feature = "stdarch_loongarch", issue = "117427")] -pub unsafe fn lsx_vst(a: m128i, mem_addr: *mut i8) { - static_assert_simm_bits!(IMM_S12, 12); - __lsx_vst(transmute(a), mem_addr, IMM_S12) -} - #[inline] #[target_feature(enable = "lsx")] #[unstable(feature = "stdarch_loongarch", issue = "117427")] @@ -3952,20 +3926,6 @@ pub fn lsx_vshuf_b(a: m128i, b: m128i, c: m128i) -> m128i { unsafe { transmute(__lsx_vshuf_b(transmute(a), transmute(b), transmute(c))) } } -#[inline] -#[target_feature(enable = "lsx")] -#[unstable(feature = "stdarch_loongarch", issue = "117427")] -pub unsafe fn lsx_vldx(mem_addr: *const i8, b: i64) -> m128i { - transmute(__lsx_vldx(mem_addr, transmute(b))) -} - -#[inline] -#[target_feature(enable = "lsx")] -#[unstable(feature = "stdarch_loongarch", issue = "117427")] -pub unsafe fn lsx_vstx(a: m128i, mem_addr: *mut i8, b: i64) { - __lsx_vstx(transmute(a), mem_addr, transmute(b)) -} - #[inline] #[target_feature(enable = "lsx")] #[unstable(feature = "stdarch_loongarch", issue = "117427")] diff --git a/crates/core_arch/src/loongarch64/lsx/portable.rs b/crates/core_arch/src/loongarch64/lsx/portable.rs index 0b0df11bbf..b7a21bc3fe 100644 --- a/crates/core_arch/src/loongarch64/lsx/portable.rs +++ b/crates/core_arch/src/loongarch64/lsx/portable.rs @@ -233,6 +233,10 @@ impl_gv!("lsx", lsx_vreplgr2vr_h, ls::simd_splat, m128i, i16x8, i32); impl_gv!("lsx", lsx_vreplgr2vr_w, ls::simd_splat, m128i, i32x4, i32); impl_gv!("lsx", lsx_vreplgr2vr_d, ls::simd_splat, m128i, i64x2, i64); +impl_ggv!("lsx", lsx_vldx, simd_ldx, m128i, i8x16, *const i8, i64, unsafe); + +impl_gsv!("lsx", lsx_vld, simd_ld, m128i, i8x16, *const i8, 12, const, unsafe); + impl_sv!("lsx", lsx_vrepli_b, ls::simd_splat, m128i, i8x16, 10); impl_sv!("lsx", lsx_vrepli_h, ls::simd_splat, m128i, i16x8, 10); impl_sv!("lsx", lsx_vrepli_w, ls::simd_splat, m128i, i32x4, 10); @@ -393,6 +397,10 @@ impl_vvv!("lsx", lsx_vpackod_h, simd_packod_h, m128i, i16x8); impl_vvv!("lsx", lsx_vpackod_w, simd_packod_w, m128i, i32x4); impl_vvv!("lsx", lsx_vpackod_d, simd_packod_d, m128i, i64x2); +impl_vgg!("lsx", lsx_vstx, simd_stx, m128i, i8x16, *mut i8, i64, unsafe); + +impl_vgs!("lsx", lsx_vst, simd_st, m128i, i8x16, *mut i8, 12, const, unsafe); + impl_vuv!("lsx", lsx_vslli_b, is::simd_shl, m128i, i8x16); impl_vuv!("lsx", lsx_vslli_h, is::simd_shl, m128i, i16x8); impl_vuv!("lsx", lsx_vslli_w, is::simd_shl, m128i, i32x4); diff --git a/crates/core_arch/src/loongarch64/simd.rs b/crates/core_arch/src/loongarch64/simd.rs index 2c4a0f8493..7ec670c54a 100644 --- a/crates/core_arch/src/loongarch64/simd.rs +++ b/crates/core_arch/src/loongarch64/simd.rs @@ -108,6 +108,20 @@ pub(super) const unsafe fn simd_fnmsub(a: T, b: T, c: T) -> T { is::simd_neg(ls::simd_fmsub(a, b, c)) } +#[inline(always)] +#[rustc_const_unstable(feature = "stdarch_const_helpers", issue = "none")] +pub(super) const unsafe fn simd_ld(a: *const i8) -> T { + let a = a.offset(I as isize) as *const T; + core::ptr::read_unaligned(a) +} + +#[inline(always)] +#[rustc_const_unstable(feature = "stdarch_const_helpers", issue = "none")] +pub(super) const unsafe fn simd_ldx(a: *const i8, b: i64) -> T { + let a = a.offset(b as isize) as *const T; + core::ptr::read_unaligned(a) +} + #[inline(always)] #[rustc_const_unstable(feature = "stdarch_const_helpers", issue = "none")] pub(super) const unsafe fn simd_madd(a: T, b: T, c: T) -> T { @@ -158,6 +172,20 @@ pub(super) const unsafe fn simd_splat(a: i64) -> T { T::splat(a) } +#[inline(always)] +#[rustc_const_unstable(feature = "stdarch_const_helpers", issue = "none")] +pub(super) const unsafe fn simd_st(a: T, b: *mut i8) { + let b = b.offset(I as isize) as *mut T; + core::ptr::write_unaligned(b, a); +} + +#[inline(always)] +#[rustc_const_unstable(feature = "stdarch_const_helpers", issue = "none")] +pub(super) const unsafe fn simd_stx(a: T, b: *mut i8, c: i64) { + let b = b.offset(c as isize) as *mut T; + core::ptr::write_unaligned(b, a); +} + macro_rules! impl_vv { ($ft:literal, $name:ident, $op:path, $oty:ty, $ity:ty) => { #[inline] @@ -191,6 +219,36 @@ macro_rules! impl_gv { pub(super) use impl_gv; +macro_rules! impl_ggv { + ($ft:literal, $name:ident, $op:path, $oty:ty, $ity:ident, $gty:ty, $xty:ty, unsafe) => { + #[inline] + #[target_feature(enable = $ft)] + #[unstable(feature = "stdarch_loongarch", issue = "117427")] + pub unsafe fn $name(a: $gty, b: $xty) -> $oty { + let r: $ity = $op(a, b); + transmute(r) + } + }; +} + +pub(super) use impl_ggv; + +macro_rules! impl_gsv { + ($ft:literal, $name:ident, $op:ident, $oty:ty, $ity:ident, $gty:ty, $ibs:expr, const, unsafe) => { + #[inline] + #[target_feature(enable = $ft)] + #[rustc_legacy_const_generics(1)] + #[unstable(feature = "stdarch_loongarch", issue = "117427")] + pub unsafe fn $name(a: $gty) -> $oty { + static_assert_simm_bits!(IMM, $ibs); + let r: $ity = $op::(a); + transmute(r) + } + }; +} + +pub(super) use impl_gsv; + macro_rules! impl_sv { ($ft:literal, $name:ident, $op:path, $oty:ty, $ity:ident, $ibs:expr) => { #[inline] @@ -227,6 +285,34 @@ macro_rules! impl_vvv { pub(super) use impl_vvv; +macro_rules! impl_vgg { + ($ft:literal, $name:ident, $op:path, $oty:ty, $ity:ident, $gty:ty, $xty:ty, unsafe) => { + #[inline] + #[target_feature(enable = $ft)] + #[unstable(feature = "stdarch_loongarch", issue = "117427")] + pub unsafe fn $name(a: $oty, b: $gty, c: $xty) { + $op(a, b, c); + } + }; +} + +pub(super) use impl_vgg; + +macro_rules! impl_vgs { + ($ft:literal, $name:ident, $op:ident, $oty:ty, $ity:ident, $gty:ty, $ibs:expr, const, unsafe) => { + #[inline] + #[target_feature(enable = $ft)] + #[rustc_legacy_const_generics(2)] + #[unstable(feature = "stdarch_loongarch", issue = "117427")] + pub unsafe fn $name(a: $oty, b: $gty) { + static_assert_simm_bits!(IMM, $ibs); + $op::(a, b); + } + }; +} + +pub(super) use impl_vgs; + macro_rules! impl_vuv { ($ft:literal, $name:ident, $op:path, $oty:ty, $ity:ident) => { #[inline] diff --git a/crates/stdarch-gen-loongarch/lasx.spec b/crates/stdarch-gen-loongarch/lasx.spec index 41432adf25..1a9710fda8 100644 --- a/crates/stdarch-gen-loongarch/lasx.spec +++ b/crates/stdarch-gen-loongarch/lasx.spec @@ -2590,11 +2590,13 @@ asm-fmts = xd, xj data-types = V4DF, V4DF /// lasx_xvld +impl = portable name = lasx_xvld asm-fmts = xd, rj, si12 data-types = V32QI, CVPOINTER, SI /// lasx_xvst +impl = portable name = lasx_xvst asm-fmts = xd, rj, si12 data-types = VOID, V32QI, CVPOINTER, SI @@ -2681,11 +2683,13 @@ asm-fmts = xd, i13 data-types = V4DI, HI /// lasx_xvldx +impl = portable name = lasx_xvldx asm-fmts = xd, rj, rk data-types = V32QI, CVPOINTER, DI /// lasx_xvstx +impl = portable name = lasx_xvstx asm-fmts = xd, rj, rk data-types = VOID, V32QI, CVPOINTER, DI diff --git a/crates/stdarch-gen-loongarch/lsx.spec b/crates/stdarch-gen-loongarch/lsx.spec index 211c3c0fcf..158db20263 100644 --- a/crates/stdarch-gen-loongarch/lsx.spec +++ b/crates/stdarch-gen-loongarch/lsx.spec @@ -3503,11 +3503,13 @@ asm-fmts = vd, vj, ui8 data-types = V4SI, V4SI, V4SI, USI /// lsx_vld +impl = portable name = lsx_vld asm-fmts = vd, rj, si12 data-types = V16QI, CVPOINTER, SI /// lsx_vst +impl = portable name = lsx_vst asm-fmts = vd, rj, si12 data-types = VOID, V16QI, CVPOINTER, SI @@ -3559,11 +3561,13 @@ asm-fmts = vd, vj, vk, va data-types = V16QI, V16QI, V16QI, V16QI /// lsx_vldx +impl = portable name = lsx_vldx asm-fmts = vd, rj, rk data-types = V16QI, CVPOINTER, DI /// lsx_vstx +impl = portable name = lsx_vstx asm-fmts = vd, rj, rk data-types = VOID, V16QI, CVPOINTER, DI diff --git a/crates/stdarch-gen-loongarch/src/portable-intrinsics.txt b/crates/stdarch-gen-loongarch/src/portable-intrinsics.txt index abbfcb3365..495ed916f5 100644 --- a/crates/stdarch-gen-loongarch/src/portable-intrinsics.txt +++ b/crates/stdarch-gen-loongarch/src/portable-intrinsics.txt @@ -263,6 +263,10 @@ lsx_vshuf4i_b lsx_vshuf4i_h lsx_vshuf4i_w lsx_vshuf4i_d +lsx_vld +lsx_vst +lsx_vldx +lsx_vstx # LASX intrinsics lasx_xvsll_b @@ -527,3 +531,7 @@ lasx_xvpackod_d lasx_xvshuf4i_b lasx_xvshuf4i_h lasx_xvshuf4i_w +lasx_xvld +lasx_xvst +lasx_xvldx +lasx_xvstx