From 58616a8149b89285bea545bc40cdaa2119881877 Mon Sep 17 00:00:00 2001 From: sh-zheng <2294474733@qq.com> Date: Wed, 24 Jun 2026 19:14:38 +0800 Subject: [PATCH 1/3] risc-v vector 1.0 support of fftw3 --- .gitignore | 2 + Makefile.am | 25 ++- api/version.c | 4 + cmake.config.h.in | 3 + configure.ac | 39 +++++ dft/codelet-dft.h | 10 ++ dft/conf.c | 22 +++ dft/simd/Makefile.am | 2 +- dft/simd/rvv1024/Makefile.am | 13 ++ dft/simd/rvv128/Makefile.am | 13 ++ dft/simd/rvv16384/Makefile.am | 13 ++ dft/simd/rvv2048/Makefile.am | 13 ++ dft/simd/rvv256/Makefile.am | 13 ++ dft/simd/rvv32768/Makefile.am | 13 ++ dft/simd/rvv4096/Makefile.am | 13 ++ dft/simd/rvv512/Makefile.am | 13 ++ dft/simd/rvv65536/Makefile.am | 13 ++ dft/simd/rvv8192/Makefile.am | 13 ++ doc/install.texi | 1 + doc/intro.texi | 2 +- doc/other.texi | 2 +- kernel/ifftw.h | 4 +- rdft/codelet-rdft.h | 10 ++ rdft/conf.c | 22 +++ rdft/simd/Makefile.am | 2 +- rdft/simd/rvv1024/Makefile.am | 13 ++ rdft/simd/rvv128/Makefile.am | 13 ++ rdft/simd/rvv16384/Makefile.am | 13 ++ rdft/simd/rvv2048/Makefile.am | 13 ++ rdft/simd/rvv256/Makefile.am | 13 ++ rdft/simd/rvv32768/Makefile.am | 13 ++ rdft/simd/rvv4096/Makefile.am | 13 ++ rdft/simd/rvv512/Makefile.am | 13 ++ rdft/simd/rvv65536/Makefile.am | 13 ++ rdft/simd/rvv8192/Makefile.am | 13 ++ simd-support/Makefile.am | 2 + simd-support/rvv.c | 31 ++++ simd-support/simd-common.h | 12 ++ simd-support/simd-rvv.h | 293 +++++++++++++++++++++++++++++++++ simd-support/simd-rvv1024.h | 29 ++++ simd-support/simd-rvv128.h | 29 ++++ simd-support/simd-rvv16384.h | 29 ++++ simd-support/simd-rvv2048.h | 29 ++++ simd-support/simd-rvv256.h | 29 ++++ simd-support/simd-rvv32768.h | 29 ++++ simd-support/simd-rvv4096.h | 29 ++++ simd-support/simd-rvv512.h | 29 ++++ simd-support/simd-rvv65536.h | 29 ++++ simd-support/simd-rvv8192.h | 29 ++++ 49 files changed, 1032 insertions(+), 6 deletions(-) create mode 100644 dft/simd/rvv1024/Makefile.am create mode 100644 dft/simd/rvv128/Makefile.am create mode 100644 dft/simd/rvv16384/Makefile.am create mode 100644 dft/simd/rvv2048/Makefile.am create mode 100644 dft/simd/rvv256/Makefile.am create mode 100644 dft/simd/rvv32768/Makefile.am create mode 100644 dft/simd/rvv4096/Makefile.am create mode 100644 dft/simd/rvv512/Makefile.am create mode 100644 dft/simd/rvv65536/Makefile.am create mode 100644 dft/simd/rvv8192/Makefile.am create mode 100644 rdft/simd/rvv1024/Makefile.am create mode 100644 rdft/simd/rvv128/Makefile.am create mode 100644 rdft/simd/rvv16384/Makefile.am create mode 100644 rdft/simd/rvv2048/Makefile.am create mode 100644 rdft/simd/rvv256/Makefile.am create mode 100644 rdft/simd/rvv32768/Makefile.am create mode 100644 rdft/simd/rvv4096/Makefile.am create mode 100644 rdft/simd/rvv512/Makefile.am create mode 100644 rdft/simd/rvv65536/Makefile.am create mode 100644 rdft/simd/rvv8192/Makefile.am create mode 100644 simd-support/rvv.c create mode 100644 simd-support/simd-rvv.h create mode 100644 simd-support/simd-rvv1024.h create mode 100644 simd-support/simd-rvv128.h create mode 100644 simd-support/simd-rvv16384.h create mode 100644 simd-support/simd-rvv2048.h create mode 100644 simd-support/simd-rvv256.h create mode 100644 simd-support/simd-rvv32768.h create mode 100644 simd-support/simd-rvv4096.h create mode 100644 simd-support/simd-rvv512.h create mode 100644 simd-support/simd-rvv65536.h create mode 100644 simd-support/simd-rvv8192.h diff --git a/.gitignore b/.gitignore index 5d1dc4cab..98c0a480a 100644 --- a/.gitignore +++ b/.gitignore @@ -29,6 +29,7 @@ rdft/simd/neon/*.c rdft/simd/sse2/*.c rdft/simd/sve*/*.c rdft/simd/vsx/*.c +rdft/simd/rvv*/*.c rdft/scalar/r2cb/*.c rdft/scalar/r2cf/*.c rdft/scalar/r2r/*.c @@ -48,6 +49,7 @@ dft/simd/neon/*.c dft/simd/sse2/*.c dft/simd/sve*/*.c dft/simd/vsx/*.c +dft/simd/rvv*/*.c # other generated files ChangeLog diff --git a/Makefile.am b/Makefile.am index b2ba29673..bb7b074aa 100644 --- a/Makefile.am +++ b/Makefile.am @@ -118,6 +118,29 @@ LASX_LIBS = dft/simd/lasx/libdft_lasx_codelets.la \ rdft/simd/lasx/librdft_lasx_codelets.la endif +if HAVE_RVV +RVV_LIBS = dft/simd/rvv128/libdft_rvv128_codelets.la \ +rdft/simd/rvv128/librdft_rvv128_codelets.la \ +dft/simd/rvv256/libdft_rvv256_codelets.la \ +rdft/simd/rvv256/librdft_rvv256_codelets.la \ +dft/simd/rvv512/libdft_rvv512_codelets.la \ +rdft/simd/rvv512/librdft_rvv512_codelets.la \ +dft/simd/rvv1024/libdft_rvv1024_codelets.la \ +rdft/simd/rvv1024/librdft_rvv1024_codelets.la \ +dft/simd/rvv2048/libdft_rvv2048_codelets.la \ +rdft/simd/rvv2048/librdft_rvv2048_codelets.la \ +dft/simd/rvv4096/libdft_rvv4096_codelets.la \ +rdft/simd/rvv4096/librdft_rvv4096_codelets.la \ +dft/simd/rvv8192/libdft_rvv8192_codelets.la \ +rdft/simd/rvv8192/librdft_rvv8192_codelets.la \ +dft/simd/rvv16384/libdft_rvv16384_codelets.la \ +rdft/simd/rvv16384/librdft_rvv16384_codelets.la \ +dft/simd/rvv32768/libdft_rvv32768_codelets.la \ +rdft/simd/rvv32768/librdft_rvv32768_codelets.la \ +dft/simd/rvv65536/libdft_rvv65536_codelets.la \ +rdft/simd/rvv65536/librdft_rvv65536_codelets.la +endif + if HAVE_GENERIC_SIMD128 GENERIC_SIMD128_LIBS = dft/simd/generic-simd128/libdft_generic_simd128_codelets.la \ rdft/simd/generic-simd128/librdft_generic_simd128_codelets.la @@ -151,7 +174,7 @@ libfftw3@PREC_SUFFIX@_la_LIBADD = \ $(SIMD_LIBS) $(SSE2_LIBS) $(AVX_LIBS) $(AVX_128_FMA_LIBS) \ $(AVX2_LIBS) $(ALTIVEC_LIBS) \ $(VSX_LIBS) $(NEON_LIBS) $(SVE_LIBS) $(KCVI_LIBS) $(AVX512_LIBS) \ - $(LSX_LIBS) $(LASX_LIBS) \ + $(LSX_LIBS) $(LASX_LIBS) $(RVV_LIBS) \ $(GENERIC_SIMD128_LIBS) $(GENERIC_SIMD256_LIBS) \ $(COMBINED_THREADLIBS) diff --git a/api/version.c b/api/version.c index da36f7d49..ef8d5ea56 100644 --- a/api/version.c +++ b/api/version.c @@ -89,6 +89,10 @@ const char X(version)[] = PACKAGE "-" PACKAGE_VERSION "-mlasx" #endif +#if HAVE_RVV + "-rvv" +#endif + #if defined(HAVE_GENERIC_SIMD128) "-generic_simd128" #endif diff --git a/cmake.config.h.in b/cmake.config.h.in index 1f4c50559..747568cd1 100644 --- a/cmake.config.h.in +++ b/cmake.config.h.in @@ -202,6 +202,9 @@ /* Define to enable ARM NEON optimizations. */ /* #undef HAVE_NEON */ +/* Define to enable RISC-V Vector optimizations. */ +/* #undef HAVE_RVV */ + /* Define if OpenMP is enabled */ #cmakedefine HAVE_OPENMP diff --git a/configure.ac b/configure.ac index a7f25263e..fed0ae680 100644 --- a/configure.ac +++ b/configure.ac @@ -254,6 +254,12 @@ if test "$have_sve" = "yes"; then fi AM_CONDITIONAL(HAVE_SVE, test "$have_sve" = "yes") +AC_ARG_ENABLE(rvv, [AC_HELP_STRING([--enable-rvv],[enable RISC-V V optimizations])], have_rvv=$enableval, have_rvv=no) +if test "$have_rvv" = "yes"; then + AC_DEFINE(HAVE_RVV,1,[Define to enable RISC-V V optimizations.]) +fi +AM_CONDITIONAL(HAVE_RVV, test "$have_rvv" = "yes") + dnl FIXME: dnl AC_ARG_ENABLE(mips-ps, [AS_HELP_STRING([--enable-mips-ps],[enable MIPS pair-single optimizations])], have_mips_ps=$enableval, have_mips_ps=no) dnl if test "$have_mips_ps" = "yes"; then @@ -446,6 +452,12 @@ case "${ax_cv_c_compiler_vendor}" in AX_CHECK_COMPILE_FLAG(-mlasx, [LASX_CFLAGS="-mlasx"], [AC_MSG_ERROR([Need a version of gcc with -mlasx])]) fi + + # RVV + if test "$have_rvv" = "yes" -a "x$RVV_CFLAGS" = x; then + AX_CHECK_COMPILE_FLAG(-march=rv64gcv, [RVV_CFLAGS="-march=rv64gcv"], + [AC_MSG_ERROR([Need a version of gcc with -march=rv64gcv])]) + fi ;; clang) @@ -486,6 +498,12 @@ case "${ax_cv_c_compiler_vendor}" in AX_CHECK_COMPILE_FLAG(-mlasx, [LASX_CFLAGS="-mlasx"], [AC_MSG_ERROR([Need a version of clang with -mlasx])]) fi + + # RVV + if test "$have_rvv" = "yes" -a "x$RVV_CFLAGS" = x; then + AX_CHECK_COMPILE_FLAG(-march=rv64gcv, [RVV_CFLAGS="-march=rv64gcv"], + [AC_MSG_ERROR([Need a version of clang with -march=rv64gcv])]) + fi case "${host_cpu}" in aarch64) @@ -523,6 +541,7 @@ AC_SUBST(NEON_CFLAGS) AC_SUBST(SVE_CFLAGS) AC_SUBST(LSX_CFLAGS) AC_SUBST(LASX_CFLAGS) +AC_SUBST(RVV_CFLAGS) dnl add stack alignment CFLAGS if so requested if test "$with_incoming_stack_boundary"x != "no"x; then @@ -823,6 +842,16 @@ AC_CONFIG_FILES([ dft/simd/sve2048/Makefile dft/simd/lsx/Makefile dft/simd/lasx/Makefile + dft/simd/rvv128/Makefile + dft/simd/rvv256/Makefile + dft/simd/rvv512/Makefile + dft/simd/rvv1024/Makefile + dft/simd/rvv2048/Makefile + dft/simd/rvv4096/Makefile + dft/simd/rvv8192/Makefile + dft/simd/rvv16384/Makefile + dft/simd/rvv32768/Makefile + dft/simd/rvv65536/Makefile dft/simd/generic-simd128/Makefile dft/simd/generic-simd256/Makefile @@ -850,6 +879,16 @@ AC_CONFIG_FILES([ rdft/simd/sve2048/Makefile rdft/simd/lsx/Makefile rdft/simd/lasx/Makefile + rdft/simd/rvv128/Makefile + rdft/simd/rvv256/Makefile + rdft/simd/rvv512/Makefile + rdft/simd/rvv1024/Makefile + rdft/simd/rvv2048/Makefile + rdft/simd/rvv4096/Makefile + rdft/simd/rvv8192/Makefile + rdft/simd/rvv16384/Makefile + rdft/simd/rvv32768/Makefile + rdft/simd/rvv65536/Makefile rdft/simd/generic-simd128/Makefile rdft/simd/generic-simd256/Makefile diff --git a/dft/codelet-dft.h b/dft/codelet-dft.h index 05cfee41e..c86eb2359 100644 --- a/dft/codelet-dft.h +++ b/dft/codelet-dft.h @@ -113,6 +113,16 @@ extern const solvtab X(solvtab_dft_sve1024); extern const solvtab X(solvtab_dft_sve2048); extern const solvtab X(solvtab_dft_lsx); extern const solvtab X(solvtab_dft_lasx); +extern const solvtab X(solvtab_dft_rvv128); +extern const solvtab X(solvtab_dft_rvv256); +extern const solvtab X(solvtab_dft_rvv512); +extern const solvtab X(solvtab_dft_rvv1024); +extern const solvtab X(solvtab_dft_rvv2048); +extern const solvtab X(solvtab_dft_rvv4096); +extern const solvtab X(solvtab_dft_rvv8192); +extern const solvtab X(solvtab_dft_rvv16384); +extern const solvtab X(solvtab_dft_rvv32768); +extern const solvtab X(solvtab_dft_rvv65536); extern const solvtab X(solvtab_dft_generic_simd128); extern const solvtab X(solvtab_dft_generic_simd256); diff --git a/dft/conf.c b/dft/conf.c index e6a9c5412..73501a8e0 100644 --- a/dft/conf.c +++ b/dft/conf.c @@ -99,6 +99,28 @@ void X(dft_conf_standard)(planner *p) if (X(have_simd_lasx)()) X(solvtab_exec)(X(solvtab_dft_lasx), p); #endif +#if HAVE_RVV + if (X(have_simd_rvv)(128)) + X(solvtab_exec)(X(solvtab_dft_rvv128), p); + if (X(have_simd_rvv)(256)) + X(solvtab_exec)(X(solvtab_dft_rvv256), p); + if (X(have_simd_rvv)(512)) + X(solvtab_exec)(X(solvtab_dft_rvv512), p); + if (X(have_simd_rvv)(1024)) + X(solvtab_exec)(X(solvtab_dft_rvv1024), p); + if (X(have_simd_rvv)(2048)) + X(solvtab_exec)(X(solvtab_dft_rvv2048), p); + if (X(have_simd_rvv)(4096)) + X(solvtab_exec)(X(solvtab_dft_rvv4096), p); + if (X(have_simd_rvv)(8192)) + X(solvtab_exec)(X(solvtab_dft_rvv8192), p); + if (X(have_simd_rvv)(16384)) + X(solvtab_exec)(X(solvtab_dft_rvv16384), p); + if (X(have_simd_rvv)(32768)) + X(solvtab_exec)(X(solvtab_dft_rvv32768), p); + if (X(have_simd_rvv)(65536)) + X(solvtab_exec)(X(solvtab_dft_rvv65536), p); +#endif #if HAVE_GENERIC_SIMD128 X(solvtab_exec)(X(solvtab_dft_generic_simd128), p); #endif diff --git a/dft/simd/Makefile.am b/dft/simd/Makefile.am index fe58c38f6..3f122ba95 100644 --- a/dft/simd/Makefile.am +++ b/dft/simd/Makefile.am @@ -1,4 +1,4 @@ AM_CPPFLAGS = -I $(top_srcdir) -SUBDIRS = common sse2 avx avx-128-fma avx2 avx2-128 avx512 kcvi altivec vsx neon sve128 sve256 sve512 sve1024 sve2048 lsx lasx generic-simd128 generic-simd256 +SUBDIRS = common sse2 avx avx-128-fma avx2 avx2-128 avx512 kcvi altivec vsx neon sve128 sve256 sve512 sve1024 sve2048 lsx lasx rvv128 rvv256 rvv512 rvv1024 rvv2048 rvv4096 rvv8192 rvv16384 rvv32768 rvv65536 generic-simd128 generic-simd256 EXTRA_DIST = n1b.h n1f.h n2b.h n2f.h n2s.h q1b.h q1f.h t1b.h t1bu.h \ t1f.h t1fu.h t2b.h t2f.h t3b.h t3f.h ts.h codlist.mk simd.mk diff --git a/dft/simd/rvv1024/Makefile.am b/dft/simd/rvv1024/Makefile.am new file mode 100644 index 000000000..71f8ef190 --- /dev/null +++ b/dft/simd/rvv1024/Makefile.am @@ -0,0 +1,13 @@ +AM_CFLAGS = $(RVV_CFLAGS) +SIMD_HEADER=simd-support/simd-rvv1024.h + +include $(top_srcdir)/dft/simd/codlist.mk +include $(top_srcdir)/dft/simd/simd.mk + +if HAVE_RVV + +BUILT_SOURCES = $(EXTRA_DIST) +noinst_LTLIBRARIES = libdft_rvv1024_codelets.la +libdft_rvv1024_codelets_la_SOURCES = $(BUILT_SOURCES) + +endif diff --git a/dft/simd/rvv128/Makefile.am b/dft/simd/rvv128/Makefile.am new file mode 100644 index 000000000..060c8a1e0 --- /dev/null +++ b/dft/simd/rvv128/Makefile.am @@ -0,0 +1,13 @@ +AM_CFLAGS = $(RVV_CFLAGS) +SIMD_HEADER=simd-support/simd-rvv128.h + +include $(top_srcdir)/dft/simd/codlist.mk +include $(top_srcdir)/dft/simd/simd.mk + +if HAVE_RVV + +BUILT_SOURCES = $(EXTRA_DIST) +noinst_LTLIBRARIES = libdft_rvv128_codelets.la +libdft_rvv128_codelets_la_SOURCES = $(BUILT_SOURCES) + +endif diff --git a/dft/simd/rvv16384/Makefile.am b/dft/simd/rvv16384/Makefile.am new file mode 100644 index 000000000..2dc83e48e --- /dev/null +++ b/dft/simd/rvv16384/Makefile.am @@ -0,0 +1,13 @@ +AM_CFLAGS = $(RVV_CFLAGS) +SIMD_HEADER=simd-support/simd-rvv16384.h + +include $(top_srcdir)/dft/simd/codlist.mk +include $(top_srcdir)/dft/simd/simd.mk + +if HAVE_RVV + +BUILT_SOURCES = $(EXTRA_DIST) +noinst_LTLIBRARIES = libdft_rvv16384_codelets.la +libdft_rvv16384_codelets_la_SOURCES = $(BUILT_SOURCES) + +endif diff --git a/dft/simd/rvv2048/Makefile.am b/dft/simd/rvv2048/Makefile.am new file mode 100644 index 000000000..0b8a79549 --- /dev/null +++ b/dft/simd/rvv2048/Makefile.am @@ -0,0 +1,13 @@ +AM_CFLAGS = $(RVV_CFLAGS) +SIMD_HEADER=simd-support/simd-rvv2048.h + +include $(top_srcdir)/dft/simd/codlist.mk +include $(top_srcdir)/dft/simd/simd.mk + +if HAVE_RVV + +BUILT_SOURCES = $(EXTRA_DIST) +noinst_LTLIBRARIES = libdft_rvv2048_codelets.la +libdft_rvv2048_codelets_la_SOURCES = $(BUILT_SOURCES) + +endif diff --git a/dft/simd/rvv256/Makefile.am b/dft/simd/rvv256/Makefile.am new file mode 100644 index 000000000..028d4d54b --- /dev/null +++ b/dft/simd/rvv256/Makefile.am @@ -0,0 +1,13 @@ +AM_CFLAGS = $(RVV_CFLAGS) +SIMD_HEADER=simd-support/simd-rvv256.h + +include $(top_srcdir)/dft/simd/codlist.mk +include $(top_srcdir)/dft/simd/simd.mk + +if HAVE_RVV + +BUILT_SOURCES = $(EXTRA_DIST) +noinst_LTLIBRARIES = libdft_rvv256_codelets.la +libdft_rvv256_codelets_la_SOURCES = $(BUILT_SOURCES) + +endif diff --git a/dft/simd/rvv32768/Makefile.am b/dft/simd/rvv32768/Makefile.am new file mode 100644 index 000000000..c7ccd251f --- /dev/null +++ b/dft/simd/rvv32768/Makefile.am @@ -0,0 +1,13 @@ +AM_CFLAGS = $(RVV_CFLAGS) +SIMD_HEADER=simd-support/simd-rvv32768.h + +include $(top_srcdir)/dft/simd/codlist.mk +include $(top_srcdir)/dft/simd/simd.mk + +if HAVE_RVV + +BUILT_SOURCES = $(EXTRA_DIST) +noinst_LTLIBRARIES = libdft_rvv32768_codelets.la +libdft_rvv32768_codelets_la_SOURCES = $(BUILT_SOURCES) + +endif diff --git a/dft/simd/rvv4096/Makefile.am b/dft/simd/rvv4096/Makefile.am new file mode 100644 index 000000000..8acd2828c --- /dev/null +++ b/dft/simd/rvv4096/Makefile.am @@ -0,0 +1,13 @@ +AM_CFLAGS = $(RVV_CFLAGS) +SIMD_HEADER=simd-support/simd-rvv4096.h + +include $(top_srcdir)/dft/simd/codlist.mk +include $(top_srcdir)/dft/simd/simd.mk + +if HAVE_RVV + +BUILT_SOURCES = $(EXTRA_DIST) +noinst_LTLIBRARIES = libdft_rvv4096_codelets.la +libdft_rvv4096_codelets_la_SOURCES = $(BUILT_SOURCES) + +endif diff --git a/dft/simd/rvv512/Makefile.am b/dft/simd/rvv512/Makefile.am new file mode 100644 index 000000000..739ec1290 --- /dev/null +++ b/dft/simd/rvv512/Makefile.am @@ -0,0 +1,13 @@ +AM_CFLAGS = $(RVV_CFLAGS) +SIMD_HEADER=simd-support/simd-rvv512.h + +include $(top_srcdir)/dft/simd/codlist.mk +include $(top_srcdir)/dft/simd/simd.mk + +if HAVE_RVV + +BUILT_SOURCES = $(EXTRA_DIST) +noinst_LTLIBRARIES = libdft_rvv512_codelets.la +libdft_rvv512_codelets_la_SOURCES = $(BUILT_SOURCES) + +endif diff --git a/dft/simd/rvv65536/Makefile.am b/dft/simd/rvv65536/Makefile.am new file mode 100644 index 000000000..45c1f0ad3 --- /dev/null +++ b/dft/simd/rvv65536/Makefile.am @@ -0,0 +1,13 @@ +AM_CFLAGS = $(RVV_CFLAGS) +SIMD_HEADER=simd-support/simd-rvv65536.h + +include $(top_srcdir)/dft/simd/codlist.mk +include $(top_srcdir)/dft/simd/simd.mk + +if HAVE_RVV + +BUILT_SOURCES = $(EXTRA_DIST) +noinst_LTLIBRARIES = libdft_rvv65536_codelets.la +libdft_rvv65536_codelets_la_SOURCES = $(BUILT_SOURCES) + +endif diff --git a/dft/simd/rvv8192/Makefile.am b/dft/simd/rvv8192/Makefile.am new file mode 100644 index 000000000..454e6a13b --- /dev/null +++ b/dft/simd/rvv8192/Makefile.am @@ -0,0 +1,13 @@ +AM_CFLAGS = $(RVV_CFLAGS) +SIMD_HEADER=simd-support/simd-rvv8192.h + +include $(top_srcdir)/dft/simd/codlist.mk +include $(top_srcdir)/dft/simd/simd.mk + +if HAVE_RVV + +BUILT_SOURCES = $(EXTRA_DIST) +noinst_LTLIBRARIES = libdft_rvv8192_codelets.la +libdft_rvv8192_codelets_la_SOURCES = $(BUILT_SOURCES) + +endif diff --git a/doc/install.texi b/doc/install.texi index 6ccac20b2..595fb548f 100644 --- a/doc/install.texi +++ b/doc/install.texi @@ -199,6 +199,7 @@ of the time). @xref{Cycle Counters}. @code{--enable-altivec} (single), @code{--enable-vsx} (single, double), @code{--enable-neon} (single, double on aarch64), +@code{--enable-rvv} (single, double on risc-v vector), @code{--enable-generic-simd128}, and @code{--enable-generic-simd256}: diff --git a/doc/intro.texi b/doc/intro.texi index b20d867f7..c560f6610 100644 --- a/doc/intro.texi +++ b/doc/intro.texi @@ -18,7 +18,7 @@ transform (DFT) and various special cases thereof. @item FFTW supports arbitrary multi-dimensional data. -@item FFTW supports the SSE, SSE2, AVX, AVX2, AVX512, KCVI, Altivec, VSX, and +@item FFTW supports the SSE, SSE2, AVX, AVX2, AVX512, KCVI, Altivec, VSX, RISC-V V, and NEON vector instruction sets. @item FFTW includes parallel (multi-threaded) transforms diff --git a/doc/other.texi b/doc/other.texi index b2d75ce9c..f01486545 100644 --- a/doc/other.texi +++ b/doc/other.texi @@ -16,7 +16,7 @@ special operations supported by some processors to perform a single operation on several numbers (usually 2 or 4) simultaneously. SIMD floating-point instructions are available on several popular CPUs: SSE/SSE2/AVX/AVX2/AVX512/KCVI on some x86/x86-64 processors, AltiVec and -VSX on some POWER/PowerPCs, NEON on some ARM models. FFTW can be +VSX on some POWER/PowerPCs, NEON on some ARM models, V extension on some RISC-V models. FFTW can be compiled to support the SIMD instructions on any of these systems. @cindex SIMD @cindex SSE diff --git a/kernel/ifftw.h b/kernel/ifftw.h index 29c15fa5f..e25cb49a0 100644 --- a/kernel/ifftw.h +++ b/kernel/ifftw.h @@ -114,6 +114,7 @@ extern void X(extract_reim)(int sign, R *c, R **r, R **i); defined(HAVE_SSE2)|| \ defined(HAVE_SVE)|| \ defined(HAVE_VSX)|| \ + defined(HAVE_RVV)|| \ defined(HAVE_GENERIC_SIMD128) || \ defined(HAVE_GENERIC_SIMD256) #define HAVE_SIMD 1 @@ -133,6 +134,7 @@ extern int X(have_simd_neon)(void); extern int X(have_simd_lsx)(void); extern int X(have_simd_lasx)(void); extern int X(have_simd_sve)(int minwidth); +extern int X(have_simd_rvv)(int minwidth); /* forward declarations */ typedef struct problem_s problem; @@ -874,7 +876,7 @@ enum { TW_COS = 0, TW_SIN = 1, TW_CEXP = 2, TW_NEXT = 3, typedef struct { unsigned char op; - signed char v; + short v; short i; } tw_instr; diff --git a/rdft/codelet-rdft.h b/rdft/codelet-rdft.h index 2081a2e80..9759db5ed 100644 --- a/rdft/codelet-rdft.h +++ b/rdft/codelet-rdft.h @@ -152,6 +152,16 @@ extern const solvtab X(solvtab_rdft_sve1024); extern const solvtab X(solvtab_rdft_sve2048); extern const solvtab X(solvtab_rdft_lsx); extern const solvtab X(solvtab_rdft_lasx); +extern const solvtab X(solvtab_rdft_rvv128); +extern const solvtab X(solvtab_rdft_rvv256); +extern const solvtab X(solvtab_rdft_rvv512); +extern const solvtab X(solvtab_rdft_rvv1024); +extern const solvtab X(solvtab_rdft_rvv2048); +extern const solvtab X(solvtab_rdft_rvv4096); +extern const solvtab X(solvtab_rdft_rvv8192); +extern const solvtab X(solvtab_rdft_rvv16384); +extern const solvtab X(solvtab_rdft_rvv32768); +extern const solvtab X(solvtab_rdft_rvv65536); extern const solvtab X(solvtab_rdft_generic_simd128); extern const solvtab X(solvtab_rdft_generic_simd256); diff --git a/rdft/conf.c b/rdft/conf.c index 2124ef87e..6a6c1f6a7 100644 --- a/rdft/conf.c +++ b/rdft/conf.c @@ -116,6 +116,28 @@ void X(rdft_conf_standard)(planner *p) if (X(have_simd_lasx)()) X(solvtab_exec)(X(solvtab_rdft_lasx), p); #endif +#if HAVE_RVV + if (X(have_simd_rvv)(128)) + X(solvtab_exec)(X(solvtab_rdft_rvv128), p); + if (X(have_simd_rvv)(256)) + X(solvtab_exec)(X(solvtab_rdft_rvv256), p); + if (X(have_simd_rvv)(512)) + X(solvtab_exec)(X(solvtab_rdft_rvv512), p); + if (X(have_simd_rvv)(1024)) + X(solvtab_exec)(X(solvtab_rdft_rvv1024), p); + if (X(have_simd_rvv)(2048)) + X(solvtab_exec)(X(solvtab_rdft_rvv2048), p); + if (X(have_simd_rvv)(4096)) + X(solvtab_exec)(X(solvtab_rdft_rvv4096), p); + if (X(have_simd_rvv)(8192)) + X(solvtab_exec)(X(solvtab_rdft_rvv8192), p); + if (X(have_simd_rvv)(16384)) + X(solvtab_exec)(X(solvtab_rdft_rvv16384), p); + if (X(have_simd_rvv)(32768)) + X(solvtab_exec)(X(solvtab_rdft_rvv32768), p); + if (X(have_simd_rvv)(65536)) + X(solvtab_exec)(X(solvtab_rdft_rvv65536), p); +#endif #if HAVE_GENERIC_SIMD128 X(solvtab_exec)(X(solvtab_rdft_generic_simd128), p); #endif diff --git a/rdft/simd/Makefile.am b/rdft/simd/Makefile.am index 19dc0fd5a..9931df9ae 100644 --- a/rdft/simd/Makefile.am +++ b/rdft/simd/Makefile.am @@ -1,4 +1,4 @@ AM_CPPFLAGS = -I $(top_srcdir) -SUBDIRS = common sse2 avx avx-128-fma avx2 avx2-128 avx512 kcvi altivec vsx neon sve128 sve256 sve512 sve1024 sve2048 lsx lasx generic-simd128 generic-simd256 +SUBDIRS = common sse2 avx avx-128-fma avx2 avx2-128 avx512 kcvi altivec vsx neon sve128 sve256 sve512 sve1024 sve2048 lsx lasx rvv128 rvv256 rvv512 rvv1024 rvv2048 rvv4096 rvv8192 rvv16384 rvv32768 rvv65536 generic-simd128 generic-simd256 EXTRA_DIST = hc2cbv.h hc2cfv.h codlist.mk simd.mk diff --git a/rdft/simd/rvv1024/Makefile.am b/rdft/simd/rvv1024/Makefile.am new file mode 100644 index 000000000..9d984d3f9 --- /dev/null +++ b/rdft/simd/rvv1024/Makefile.am @@ -0,0 +1,13 @@ +AM_CFLAGS = $(RVV_CFLAGS) +SIMD_HEADER=simd-support/simd-rvv1024.h + +include $(top_srcdir)/rdft/simd/codlist.mk +include $(top_srcdir)/rdft/simd/simd.mk + +if HAVE_RVV + +noinst_LTLIBRARIES = librdft_rvv1024_codelets.la +BUILT_SOURCES = $(EXTRA_DIST) +librdft_rvv1024_codelets_la_SOURCES = $(BUILT_SOURCES) + +endif diff --git a/rdft/simd/rvv128/Makefile.am b/rdft/simd/rvv128/Makefile.am new file mode 100644 index 000000000..8844665c9 --- /dev/null +++ b/rdft/simd/rvv128/Makefile.am @@ -0,0 +1,13 @@ +AM_CFLAGS = $(RVV_CFLAGS) +SIMD_HEADER=simd-support/simd-rvv128.h + +include $(top_srcdir)/rdft/simd/codlist.mk +include $(top_srcdir)/rdft/simd/simd.mk + +if HAVE_RVV + +noinst_LTLIBRARIES = librdft_rvv128_codelets.la +BUILT_SOURCES = $(EXTRA_DIST) +librdft_rvv128_codelets_la_SOURCES = $(BUILT_SOURCES) + +endif diff --git a/rdft/simd/rvv16384/Makefile.am b/rdft/simd/rvv16384/Makefile.am new file mode 100644 index 000000000..ffd0fe566 --- /dev/null +++ b/rdft/simd/rvv16384/Makefile.am @@ -0,0 +1,13 @@ +AM_CFLAGS = $(RVV_CFLAGS) +SIMD_HEADER=simd-support/simd-rvv16384.h + +include $(top_srcdir)/rdft/simd/codlist.mk +include $(top_srcdir)/rdft/simd/simd.mk + +if HAVE_RVV + +noinst_LTLIBRARIES = librdft_rvv16384_codelets.la +BUILT_SOURCES = $(EXTRA_DIST) +librdft_rvv16384_codelets_la_SOURCES = $(BUILT_SOURCES) + +endif diff --git a/rdft/simd/rvv2048/Makefile.am b/rdft/simd/rvv2048/Makefile.am new file mode 100644 index 000000000..5f31d9b7a --- /dev/null +++ b/rdft/simd/rvv2048/Makefile.am @@ -0,0 +1,13 @@ +AM_CFLAGS = $(RVV_CFLAGS) +SIMD_HEADER=simd-support/simd-rvv2048.h + +include $(top_srcdir)/rdft/simd/codlist.mk +include $(top_srcdir)/rdft/simd/simd.mk + +if HAVE_RVV + +noinst_LTLIBRARIES = librdft_rvv2048_codelets.la +BUILT_SOURCES = $(EXTRA_DIST) +librdft_rvv2048_codelets_la_SOURCES = $(BUILT_SOURCES) + +endif diff --git a/rdft/simd/rvv256/Makefile.am b/rdft/simd/rvv256/Makefile.am new file mode 100644 index 000000000..520d8638e --- /dev/null +++ b/rdft/simd/rvv256/Makefile.am @@ -0,0 +1,13 @@ +AM_CFLAGS = $(RVV_CFLAGS) +SIMD_HEADER=simd-support/simd-rvv256.h + +include $(top_srcdir)/rdft/simd/codlist.mk +include $(top_srcdir)/rdft/simd/simd.mk + +if HAVE_RVV + +noinst_LTLIBRARIES = librdft_rvv256_codelets.la +BUILT_SOURCES = $(EXTRA_DIST) +librdft_rvv256_codelets_la_SOURCES = $(BUILT_SOURCES) + +endif diff --git a/rdft/simd/rvv32768/Makefile.am b/rdft/simd/rvv32768/Makefile.am new file mode 100644 index 000000000..8183f6820 --- /dev/null +++ b/rdft/simd/rvv32768/Makefile.am @@ -0,0 +1,13 @@ +AM_CFLAGS = $(RVV_CFLAGS) +SIMD_HEADER=simd-support/simd-rvv32768.h + +include $(top_srcdir)/rdft/simd/codlist.mk +include $(top_srcdir)/rdft/simd/simd.mk + +if HAVE_RVV + +noinst_LTLIBRARIES = librdft_rvv32768_codelets.la +BUILT_SOURCES = $(EXTRA_DIST) +librdft_rvv32768_codelets_la_SOURCES = $(BUILT_SOURCES) + +endif diff --git a/rdft/simd/rvv4096/Makefile.am b/rdft/simd/rvv4096/Makefile.am new file mode 100644 index 000000000..1646022e5 --- /dev/null +++ b/rdft/simd/rvv4096/Makefile.am @@ -0,0 +1,13 @@ +AM_CFLAGS = $(RVV_CFLAGS) +SIMD_HEADER=simd-support/simd-rvv4096.h + +include $(top_srcdir)/rdft/simd/codlist.mk +include $(top_srcdir)/rdft/simd/simd.mk + +if HAVE_RVV + +noinst_LTLIBRARIES = librdft_rvv4096_codelets.la +BUILT_SOURCES = $(EXTRA_DIST) +librdft_rvv4096_codelets_la_SOURCES = $(BUILT_SOURCES) + +endif diff --git a/rdft/simd/rvv512/Makefile.am b/rdft/simd/rvv512/Makefile.am new file mode 100644 index 000000000..f8d4dd24e --- /dev/null +++ b/rdft/simd/rvv512/Makefile.am @@ -0,0 +1,13 @@ +AM_CFLAGS = $(RVV_CFLAGS) +SIMD_HEADER=simd-support/simd-rvv512.h + +include $(top_srcdir)/rdft/simd/codlist.mk +include $(top_srcdir)/rdft/simd/simd.mk + +if HAVE_RVV + +noinst_LTLIBRARIES = librdft_rvv512_codelets.la +BUILT_SOURCES = $(EXTRA_DIST) +librdft_rvv512_codelets_la_SOURCES = $(BUILT_SOURCES) + +endif diff --git a/rdft/simd/rvv65536/Makefile.am b/rdft/simd/rvv65536/Makefile.am new file mode 100644 index 000000000..1bfed0c10 --- /dev/null +++ b/rdft/simd/rvv65536/Makefile.am @@ -0,0 +1,13 @@ +AM_CFLAGS = $(RVV_CFLAGS) +SIMD_HEADER=simd-support/simd-rvv65536.h + +include $(top_srcdir)/rdft/simd/codlist.mk +include $(top_srcdir)/rdft/simd/simd.mk + +if HAVE_RVV + +noinst_LTLIBRARIES = librdft_rvv65536_codelets.la +BUILT_SOURCES = $(EXTRA_DIST) +librdft_rvv65536_codelets_la_SOURCES = $(BUILT_SOURCES) + +endif diff --git a/rdft/simd/rvv8192/Makefile.am b/rdft/simd/rvv8192/Makefile.am new file mode 100644 index 000000000..128226209 --- /dev/null +++ b/rdft/simd/rvv8192/Makefile.am @@ -0,0 +1,13 @@ +AM_CFLAGS = $(RVV_CFLAGS) +SIMD_HEADER=simd-support/simd-rvv8192.h + +include $(top_srcdir)/rdft/simd/codlist.mk +include $(top_srcdir)/rdft/simd/simd.mk + +if HAVE_RVV + +noinst_LTLIBRARIES = librdft_rvv8192_codelets.la +BUILT_SOURCES = $(EXTRA_DIST) +librdft_rvv8192_codelets_la_SOURCES = $(BUILT_SOURCES) + +endif diff --git a/simd-support/Makefile.am b/simd-support/Makefile.am index cc4fed936..7f196bb8f 100644 --- a/simd-support/Makefile.am +++ b/simd-support/Makefile.am @@ -16,6 +16,8 @@ neon.c simd-neon.h \ lsx.c simd-lsx.h \ lasx.c simd-lasx.h \ simd-generic128.h simd-generic256.h \ +rvv.c simd-rvv.h simd-rvv128.h simd-rvv256.h simd-rvv512.h simd-rvv1024.h \ +simd-rvv2048.h simd-rvv4096.h simd-rvv8192.h simd-rvv16384.h simd-rvv32768.h simd-rvv65536.h \ sve.c simd-maskedsve.h simd-maskedsve128.h simd-maskedsve256.h simd-maskedsve512.h simd-maskedsve1024.h simd-maskedsve2048.h # sve-getwidth.c goes into a separate library with SVE compilation flags diff --git a/simd-support/rvv.c b/simd-support/rvv.c new file mode 100644 index 000000000..0fad1ab8c --- /dev/null +++ b/simd-support/rvv.c @@ -0,0 +1,31 @@ +/* + * Copyright (c) 2003, 2007-14 Matteo Frigo + * Copyright (c) 2003, 2007-14 Massachusetts Institute of Technology + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + */ + + +#include "kernel/ifftw.h" + +#if HAVE_RVV +#include +/* don't know how to autodetect RVV; assume it is present */ + int X(have_simd_rvv)(int rs) + { + return __riscv_vsetvlmax_e64m1() == (rs / 64); + } +#endif diff --git a/simd-support/simd-common.h b/simd-support/simd-common.h index 2faa8486d..ff5cd3803 100644 --- a/simd-support/simd-common.h +++ b/simd-support/simd-common.h @@ -52,6 +52,14 @@ # define ALIGNMENT 16 # define ALIGNMENTA 16 # endif +#elif defined(HAVE_RVV) +# if defined(FFTW_SINGLE) +# define ALIGNMENT 8 +# define ALIGNMENTA 16 +# else +# define ALIGNMENT 16 +# define ALIGNMENTA 16 +# endif #elif defined(HAVE_GENERIC_SIMD256) # if defined(FFTW_SINGLE) # define ALIGNMENT 8 @@ -111,4 +119,8 @@ #define REPEAT_16(DEFX, v, x) REPEAT_8(DEFX, v, x), REPEAT_8(DEFX,(v)+8, x) #define REPEAT_32(DEFX, v, x) REPEAT_16(DEFX, v, x), REPEAT_16(DEFX, (v)+16, x) #define REPEAT_64(DEFX, v, x) REPEAT_32(DEFX, v, x), REPEAT_32(DEFX, (v)+32, x) +#define REPEAT_128(DEFX, v, x) REPEAT_64(DEFX, v, x), REPEAT_64(DEFX, (v)+64, x) +#define REPEAT_256(DEFX, v, x) REPEAT_128(DEFX, v, x), REPEAT_128(DEFX, (v)+128, x) +#define REPEAT_512(DEFX, v, x) REPEAT_256(DEFX, v, x), REPEAT_256(DEFX, (v)+256, x) +#define REPEAT_1024(DEFX, v, x) REPEAT_512(DEFX, v, x), REPEAT_512(DEFX, (v)+512, x) diff --git a/simd-support/simd-rvv.h b/simd-support/simd-rvv.h new file mode 100644 index 000000000..78ea45d0e --- /dev/null +++ b/simd-support/simd-rvv.h @@ -0,0 +1,293 @@ +/* + * Copyright (c) 2003, 2007-11 Matteo Frigo + * Copyright (c) 2003, 2007-11 Massachusetts Institute of Technology + * + * RISC-V V support implemented by Romain Dolbeau. (c) 2019 Romain Dolbeau + * Modified to support RVV spec v1.0 by Zheng Shuo. (c) 2026 Zheng Shuo + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + */ + +#if defined(FFTW_LDOUBLE) || defined(FFTW_QUAD) +#error "RISC-V V vector instructions only works in single or double precision" +#endif + +#ifdef FFTW_SINGLE +# define DS(d,s) s /* single-precision option */ +# define TYPE(name) __riscv_ ## name ## _f32m1 +# define TYPEUINT(name) __riscv_ ## name ## _u32m1 +# define TYPEINTERPRETF2U(name) __riscv_ ## name ## _f32m1_u32m1 +# define TYPEINTERPRETU2F(name) __riscv_ ## name ## _u32m1_f32m1 +# define TYPEMEM(name) __riscv_ ## name ## e32_v_f32m1 +#else +# define DS(d,s) d /* double-precision option */ +# define TYPE(name) __riscv_ ## name ## _f64m1 +# define TYPEUINT(name) __riscv_ ## name ## _u64m1 +# define TYPEINTERPRETF2U(name) __riscv_ ## name ## _f64m1_u64m1 +# define TYPEINTERPRETU2F(name) __riscv_ ## name ## _u64m1_f64m1 +# define TYPEMEM(name) __riscv_ ## name ## e64_v_f64m1 +#endif + +#if RVV_VLEN == 65536 +# define VL DS(512, 1024) /* SIMD complex vector length */ +#elif RVV_VLEN == 32768 +# define VL DS(256, 512) /* SIMD complex vector length */ +#elif RVV_VLEN == 16384 +# define VL DS(128, 256) /* SIMD complex vector length */ +#elif RVV_VLEN == 8192 +# define VL DS(64, 128) /* SIMD complex vector length */ +#elif RVV_VLEN == 4096 +# define VL DS(32, 64) /* SIMD complex vector length */ +#elif RVV_VLEN == 2048 +# define VL DS(16, 32) /* SIMD complex vector length */ +#elif RVV_VLEN == 1024 +# define VL DS(8, 16) /* SIMD complex vector length */ +#elif RVV_VLEN == 512 +# define VL DS(4, 8) /* SIMD complex vector length */ +#elif RVV_VLEN == 256 +# define VL DS(2, 4) /* SIMD complex vector length */ +#elif RVV_VLEN == 128 +# define VL DS(1, 2) /* SIMD complex vector length */ +#else +# error "RVV_VLEN must be a power of 2 between 128 and 65536 bits" +#endif /* RVV_VLEN */ + +#define SIMD_VSTRIDE_OKA(x) ((x) == 2) +#define SIMD_STRIDE_OKPAIR SIMD_STRIDE_OK + +#define ZERO DS(0.0, 0.0f) + +#include + +typedef DS(vfloat64m1_t, vfloat32m1_t) V; +typedef DS(vuint64m1_t, vuint32m1_t) Vuint; + +#define VNEG(a) TYPE(vfneg_v)(a, 2*VL) +#define VADD(a, b) TYPE(vfadd_vv)(a, b, 2*VL) +#define VSUB(a, b) TYPE(vfsub_vv)(a, b, 2*VL) +#define VMUL(a, b) TYPE(vfmul_vv)(a, b, 2*VL) + +// generate (all 1, 0, all 1, 0, ...) to split real and imagine parts +#define VPARTSPLIT TYPEUINT(vsub_vx)(TYPEUINT(vand_vx)(TYPEUINT(vid_v)(2*VL), 1, 2*VL), 1, 2*VL) + +static inline V VDUPL(const V x) +{ + Vuint partr = VPARTSPLIT; // (all 1, 0, all 1, 0, ...) + V xl = TYPEINTERPRETU2F(vreinterpret_v)(TYPEUINT(vand_vv)(TYPEINTERPRETF2U(vreinterpret_v)(x), partr, 2*VL)); // set odd elements to 0 + return VADD(TYPE(vfslide1up_vf)(xl, ZERO, 2*VL), xl); +} + +static inline V VDUPH(const V x) +{ + Vuint partr = VPARTSPLIT; // (all 1, 0, all 1, 0, ...) + Vuint parti = TYPEUINT(vnot_v)(partr, 2*VL); // (0, all 1, 0, all 1, ...) + V xh = TYPEINTERPRETU2F(vreinterpret_v)(TYPEUINT(vand_vv)(TYPEINTERPRETF2U(vreinterpret_v)(x), parti, 2*VL)); // set even elements to 0 + return VADD(TYPE(vfslide1down_vf)(xh, ZERO, 2*VL), xh); +} + +#define DVK(var, val) V var = TYPE(vfmv_v_f)(val, 2*VL) + +static inline V FLIP_RI(const V x) +{ + Vuint partr = VPARTSPLIT; // (all 1, 0, all 1, 0, ...) + V xl = TYPEINTERPRETU2F(vreinterpret_v)(TYPEUINT(vand_vv)(TYPEINTERPRETF2U(vreinterpret_v)(x), partr, 2*VL)); // set odd elements to 0 + Vuint parti = TYPEUINT(vnot_v)(partr, 2*VL); // (0, all 1, 0, all 1, ...) + V xh = TYPEINTERPRETU2F(vreinterpret_v)(TYPEUINT(vand_vv)(TYPEINTERPRETF2U(vreinterpret_v)(x), parti, 2*VL)); // set even elements to 0 + return VADD(TYPE(vfslide1down_vf)(xh, ZERO, 2*VL), TYPE(vfslide1up_vf)(xl, ZERO, 2*VL)); +} + +static inline V VCONJ(const V x) +{ + Vuint partr = VPARTSPLIT; // (all 1, 0, all 1, 0, ...) + V xl = TYPEINTERPRETU2F(vreinterpret_v)(TYPEUINT(vand_vv)(TYPEINTERPRETF2U(vreinterpret_v)(x), partr, 2*VL)); // set odd elements to 0 + Vuint parti = TYPEUINT(vnot_v)(partr, 2*VL); // (0, all 1, 0, all 1, ...) + V xh = TYPEINTERPRETU2F(vreinterpret_v)(TYPEUINT(vand_vv)(TYPEINTERPRETF2U(vreinterpret_v)(x), parti, 2*VL)); // set even elements to 0 + return VADD(xl, VNEG(xh)); +} + +static inline V VBYI(V x) +{ + Vuint partr = VPARTSPLIT; // (all 1, 0, all 1, 0, ...) + V xl = TYPEINTERPRETU2F(vreinterpret_v)(TYPEUINT(vand_vv)(TYPEINTERPRETF2U(vreinterpret_v)(x), partr, 2*VL)); // set odd elements to 0 + Vuint parti = TYPEUINT(vnot_v)(partr, 2*VL); // (0, all 1, 0, all 1, ...) + V xh = TYPEINTERPRETU2F(vreinterpret_v)(TYPEUINT(vand_vv)(TYPEINTERPRETF2U(vreinterpret_v)(VNEG(x)), parti, 2*VL)); // set elements to negative, then set even elements to 0 + return VADD(TYPE(vfslide1down_vf)(xh, ZERO, 2*VL), TYPE(vfslide1up_vf)(xl, ZERO, 2*VL)); +} + +#define LDK(x) x + +#define VFMA(a, b, c) TYPE(vfmacc_vv)(c, a, b, 2*VL) +#define VFMS(a, b, c) TYPE(vfmsac_vv)(c, a, b, 2*VL) +#define VFNMS(a, b, c) TYPE(vfnmsac_vv)(c, a, b, 2*VL) +#define VFMAI(b, c) VADD(c, VBYI(b)) +#define VFNMSI(b, c) VSUB(c, VBYI(b)) +#define VFMACONJ(b, c) VADD(VCONJ(b), c) +#define VFMSCONJ(b, c) VSUB(VCONJ(b), c) +#define VFNMSCONJ(b, c) VSUB(c, VCONJ(b)) + +static inline V VZMUL(V tx, V sr) +{ + V tr = VDUPL(tx); + V ti = VDUPH(tx); + tr = VMUL(sr, tr); + sr = VBYI(sr); + return VFMA(ti, sr, tr); +} + +static inline V VZMULJ(V tx, V sr) +{ + V tr = VDUPL(tx); + V ti = VDUPH(tx); + tr = VMUL(sr, tr); + sr = VBYI(sr); + return VFNMS(ti, sr, tr); +} + +static inline V VZMULI(V tx, V sr) +{ + V tr = VDUPL(tx); + V ti = VDUPH(tx); + ti = VMUL(ti, sr); + sr = VBYI(sr); + return VFMS(tr, sr, ti); +} + +static inline V VZMULIJ(V tx, V sr) +{ + V tr = VDUPL(tx); + V ti = VDUPH(tx); + ti = VMUL(ti, sr); + sr = VBYI(sr); + return VFMA(tr, sr, ti); +} + +static inline V LDA(const R *x, INT ivs, const R *aligned_like) { + (void)aligned_like; /* UNUSED */ + + return TYPEMEM(vl)(x, 2*VL); +} + +static inline void STA(R *x, V v, INT ovs, const R *aligned_like) { + (void)aligned_like; /* UNUSED */ + + TYPEMEM(vs)(x, v, 2*VL); +} + +static inline V LD(const R *x, INT ivs, const R *aligned_like) +{ + (void)aligned_like; /* UNUSED */ + + V xl = TYPEMEM(vls)(x, sizeof(R)*ivs, VL); + V xh = TYPEMEM(vls)(x+1, sizeof(R)*ivs, VL); + + Vuint idx = TYPEUINT(vid_v)(2*VL); // (0, 1, 2, 3, ...) + Vuint idx1 = TYPEUINT(vand_vx)(idx, -2, 2*VL); // (0, 0, 2, 2, ...) + Vuint idx2 = TYPEUINT(vsrl_vx)(idx1, 1, 2*VL); // (0, 0, 1, 1, ...) + + V xl1 = TYPE(vrgather_vv)(xl, idx2, 2*VL); + V xh1 = TYPE(vrgather_vv)(xh, idx2, 2*VL); + + Vuint idx3 = TYPEUINT(vand_vx)(idx, 1, 2*VL); // (0, 1, 0, 1, ...) + Vuint idx4 = TYPEUINT(vsub_vx)(idx3, 1, 2*VL); // (all 1, 0, all 1, 0, ...) + Vuint idx5 = TYPEUINT(vnot_v)(idx4, 2*VL); // (0, all 1, 0, all 1, ...) + + V xl2 = TYPEINTERPRETU2F(vreinterpret_v)(TYPEUINT(vand_vv)(TYPEINTERPRETF2U(vreinterpret_v)(xl1), idx4, 2*VL)); // set odd elements to 0 + V xh2 = TYPEINTERPRETU2F(vreinterpret_v)(TYPEUINT(vand_vv)(TYPEINTERPRETF2U(vreinterpret_v)(xh1), idx5, 2*VL)); // set even elements to 0 + return VADD(xl2, xh2); +} + +static inline void ST(R *x, V v, INT ovs, const R *aligned_like) +{ + (void)aligned_like; /* UNUSED */ + + Vuint idx = TYPEUINT(vid_v)(VL); // (0, 1, 2, 3, ...) + Vuint idx1 = TYPEUINT(vsll_vx)(idx, 1, VL); // (0, 2, 4, 6, ...) + + V vl = TYPE(vrgather_vv)(v, idx1, VL); + TYPEMEM(vss)(x, sizeof(R)*ovs, vl, ovs ? VL : 1); // if ovs=0, store the first element + + Vuint idx2 = TYPEUINT(vadd_vx)(idx1, 1, VL); // (1, 3, 5, 7, ...) + + V vh = TYPE(vrgather_vv)(v, idx2, VL); + TYPEMEM(vss)(x+1, sizeof(R)*ovs, vh, ovs ? VL : 1); // if ovs=0, store the first element +} + +// only one of STM2 and STN2 should be implemented, according to the hardware. Both micros occur in the code, and the implemented one does some operations, while the no-op one is skipped. +#define STM2(x, v, ovs, a) ST(x, v, ovs, a) + +#define STN2(x, v0, v1, ovs) /* no-op */ + +// only one of STM4 and STN4 should be implemented. See STM2 and STN2. +#define STM4(x, v, ovs, a) TYPEMEM(vss)(x, sizeof(R)*ovs, v, 2*VL) + +#define STN4(x, v0, v1, v2, v3, ovs) /* no-op */ + +/* twiddle storage #1: compact, slower */ +#define DEFVTW1(v, x) {TW_CEXP, v, x} +#define VTW1(v,x) CONCAT2(REPEAT_, VL)(DEFVTW1, v, x) +#define TWVL1 (VL) + +static inline V BYTW1(const R *t, V sr) +{ + return VZMUL(LDA(t, 2, t), sr); +} + +static inline V BYTWJ1(const R *t, V sr) +{ + return VZMULJ(LDA(t, 2, t), sr); +} + +/* twiddle storage #2: twice the space, faster (when in cache) */ +#define DEFVTW2_COS(v, x) {TW_COS, v, x}, {TW_COS, v, x} +#define DEFVTW2_SIN(v, x) {TW_SIN, v, -x}, {TW_SIN, v, x} +#define VTW2(v,x) CONCAT2(REPEAT_, VL)(DEFVTW2_COS, v, x), \ + CONCAT2(REPEAT_, VL)(DEFVTW2_SIN, v, x) +#define TWVL2 (2*VL) + +static inline V BYTW2(const R *t, V sr) +{ + V si = FLIP_RI(sr); + V ti = LDA(t+2*VL, 2, t+4*VL); + V tr = LDA(t, 2, t); + return VFMA(tr, sr, VMUL(ti, si)); +} + +static inline V BYTWJ2(const R *t, V sr) +{ + V si = FLIP_RI(sr); + V ti = LDA(t+2*VL, 2, t+4*VL); + V tr = LDA(t, 2, t); + return VFNMS(ti, si, VMUL(tr, sr)); +} + +/* twiddle storage #3 */ +#define VTW3(v,x) VTW1(v,x) +#define TWVL3 TWVL1 + +/* twiddle storage for split arrays */ +#define DEFVTWS_COS(v, x) {TW_COS, 2*(v), x}, {TW_COS, 2*(v)+1, x} +#define DEFVTWS_SIN(v, x) {TW_SIN, 2*(v), x}, {TW_SIN, 2*(v)+1, x} +#define VTWS(v,x) CONCAT2(REPEAT_, VL)(DEFVTWS_COS, v, x), \ + CONCAT2(REPEAT_, VL)(DEFVTWS_SIN, v, x) +#define TWVLS (2*VL) + +#define VLEAVE() /* nothing */ + +#include "simd-common.h" diff --git a/simd-support/simd-rvv1024.h b/simd-support/simd-rvv1024.h new file mode 100644 index 000000000..9301d1c72 --- /dev/null +++ b/simd-support/simd-rvv1024.h @@ -0,0 +1,29 @@ +/* + * Copyright (c) 2003, 2007-11 Matteo Frigo + * Copyright (c) 2003, 2007-11 Massachusetts Institute of Technology + * + * RISC-V V support implemented by Romain Dolbeau. (c) 2019 Romain Dolbeau + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + */ + +#define SIMD_SUFFIX _rvv1024 /* for renaming */ +#define RVV_VLEN 1024 +#include "simd-rvv.h" diff --git a/simd-support/simd-rvv128.h b/simd-support/simd-rvv128.h new file mode 100644 index 000000000..d1746cad6 --- /dev/null +++ b/simd-support/simd-rvv128.h @@ -0,0 +1,29 @@ +/* + * Copyright (c) 2003, 2007-11 Matteo Frigo + * Copyright (c) 2003, 2007-11 Massachusetts Institute of Technology + * + * RISC-V V support implemented by Romain Dolbeau. (c) 2019 Romain Dolbeau + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + */ + +#define SIMD_SUFFIX _rvv128 /* for renaming */ +#define RVV_VLEN 128 +#include "simd-rvv.h" diff --git a/simd-support/simd-rvv16384.h b/simd-support/simd-rvv16384.h new file mode 100644 index 000000000..c3963ff0d --- /dev/null +++ b/simd-support/simd-rvv16384.h @@ -0,0 +1,29 @@ +/* + * Copyright (c) 2003, 2007-11 Matteo Frigo + * Copyright (c) 2003, 2007-11 Massachusetts Institute of Technology + * + * RISC-V V support implemented by Romain Dolbeau. (c) 2019 Romain Dolbeau + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + */ + +#define SIMD_SUFFIX _rvv16384 /* for renaming */ +#define RVV_VLEN 16384 +#include "simd-rvv.h" diff --git a/simd-support/simd-rvv2048.h b/simd-support/simd-rvv2048.h new file mode 100644 index 000000000..a3a9ba6ec --- /dev/null +++ b/simd-support/simd-rvv2048.h @@ -0,0 +1,29 @@ +/* + * Copyright (c) 2003, 2007-11 Matteo Frigo + * Copyright (c) 2003, 2007-11 Massachusetts Institute of Technology + * + * RISC-V V support implemented by Romain Dolbeau. (c) 2019 Romain Dolbeau + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + */ + +#define SIMD_SUFFIX _rvv2048 /* for renaming */ +#define RVV_VLEN 2048 +#include "simd-rvv.h" diff --git a/simd-support/simd-rvv256.h b/simd-support/simd-rvv256.h new file mode 100644 index 000000000..5782a221d --- /dev/null +++ b/simd-support/simd-rvv256.h @@ -0,0 +1,29 @@ +/* + * Copyright (c) 2003, 2007-11 Matteo Frigo + * Copyright (c) 2003, 2007-11 Massachusetts Institute of Technology + * + * RISC-V V support implemented by Romain Dolbeau. (c) 2019 Romain Dolbeau + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + */ + +#define SIMD_SUFFIX _rvv256 /* for renaming */ +#define RVV_VLEN 256 +#include "simd-rvv.h" diff --git a/simd-support/simd-rvv32768.h b/simd-support/simd-rvv32768.h new file mode 100644 index 000000000..addf0fe67 --- /dev/null +++ b/simd-support/simd-rvv32768.h @@ -0,0 +1,29 @@ +/* + * Copyright (c) 2003, 2007-11 Matteo Frigo + * Copyright (c) 2003, 2007-11 Massachusetts Institute of Technology + * + * RISC-V V support implemented by Romain Dolbeau. (c) 2019 Romain Dolbeau + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + */ + +#define SIMD_SUFFIX _rvv32768 /* for renaming */ +#define RVV_VLEN 32768 +#include "simd-rvv.h" diff --git a/simd-support/simd-rvv4096.h b/simd-support/simd-rvv4096.h new file mode 100644 index 000000000..f961847c1 --- /dev/null +++ b/simd-support/simd-rvv4096.h @@ -0,0 +1,29 @@ +/* + * Copyright (c) 2003, 2007-11 Matteo Frigo + * Copyright (c) 2003, 2007-11 Massachusetts Institute of Technology + * + * RISC-V V support implemented by Romain Dolbeau. (c) 2019 Romain Dolbeau + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + */ + +#define SIMD_SUFFIX _rvv4096 /* for renaming */ +#define RVV_VLEN 4096 +#include "simd-rvv.h" diff --git a/simd-support/simd-rvv512.h b/simd-support/simd-rvv512.h new file mode 100644 index 000000000..146f1e53e --- /dev/null +++ b/simd-support/simd-rvv512.h @@ -0,0 +1,29 @@ +/* + * Copyright (c) 2003, 2007-11 Matteo Frigo + * Copyright (c) 2003, 2007-11 Massachusetts Institute of Technology + * + * RISC-V V support implemented by Romain Dolbeau. (c) 2019 Romain Dolbeau + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + */ + +#define SIMD_SUFFIX _rvv512 /* for renaming */ +#define RVV_VLEN 512 +#include "simd-rvv.h" diff --git a/simd-support/simd-rvv65536.h b/simd-support/simd-rvv65536.h new file mode 100644 index 000000000..febe838c8 --- /dev/null +++ b/simd-support/simd-rvv65536.h @@ -0,0 +1,29 @@ +/* + * Copyright (c) 2003, 2007-11 Matteo Frigo + * Copyright (c) 2003, 2007-11 Massachusetts Institute of Technology + * + * RISC-V V support implemented by Romain Dolbeau. (c) 2019 Romain Dolbeau + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + */ + +#define SIMD_SUFFIX _rvv65536 /* for renaming */ +#define RVV_VLEN 65536 +#include "simd-rvv.h" diff --git a/simd-support/simd-rvv8192.h b/simd-support/simd-rvv8192.h new file mode 100644 index 000000000..1ce98d44d --- /dev/null +++ b/simd-support/simd-rvv8192.h @@ -0,0 +1,29 @@ +/* + * Copyright (c) 2003, 2007-11 Matteo Frigo + * Copyright (c) 2003, 2007-11 Massachusetts Institute of Technology + * + * RISC-V V support implemented by Romain Dolbeau. (c) 2019 Romain Dolbeau + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + */ + +#define SIMD_SUFFIX _rvv8192 /* for renaming */ +#define RVV_VLEN 8192 +#include "simd-rvv.h" From 0ae452af4eba528f29e15a64fd182723e69eb1f3 Mon Sep 17 00:00:00 2001 From: sh-zheng <2294474733@qq.com> Date: Thu, 25 Jun 2026 00:20:48 +0800 Subject: [PATCH 2/3] Add specific rvv vlen detection for single precision mode --- simd-support/rvv.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/simd-support/rvv.c b/simd-support/rvv.c index 0fad1ab8c..a5d587aec 100644 --- a/simd-support/rvv.c +++ b/simd-support/rvv.c @@ -26,6 +26,10 @@ /* don't know how to autodetect RVV; assume it is present */ int X(have_simd_rvv)(int rs) { +#ifdef FFTW_SINGLE + return __riscv_vsetvlmax_e32m1() == (rs / 32); +#else return __riscv_vsetvlmax_e64m1() == (rs / 64); +#endif } #endif From 96036d90e03c7dff932c10006b4fe7cd7949633b Mon Sep 17 00:00:00 2001 From: sh-zheng <2294474733@qq.com> Date: Thu, 25 Jun 2026 21:57:37 +0800 Subject: [PATCH 3/3] Optimize the rvv kernel --- simd-support/simd-rvv.h | 44 ++++++++++++++++++++--------------------- 1 file changed, 21 insertions(+), 23 deletions(-) diff --git a/simd-support/simd-rvv.h b/simd-support/simd-rvv.h index 78ea45d0e..8f6b8ec07 100644 --- a/simd-support/simd-rvv.h +++ b/simd-support/simd-rvv.h @@ -3,7 +3,7 @@ * Copyright (c) 2003, 2007-11 Massachusetts Institute of Technology * * RISC-V V support implemented by Romain Dolbeau. (c) 2019 Romain Dolbeau - * Modified to support RVV spec v1.0 by Zheng Shuo. (c) 2026 Zheng Shuo + * Modified to support RVV spec v1.0 by Romain Dolbeau & Zheng Shuo. (c) 2024-2026 Romain Dolbeau & Zheng Shuo * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -33,15 +33,15 @@ # define DS(d,s) s /* single-precision option */ # define TYPE(name) __riscv_ ## name ## _f32m1 # define TYPEUINT(name) __riscv_ ## name ## _u32m1 -# define TYPEINTERPRETF2U(name) __riscv_ ## name ## _f32m1_u32m1 -# define TYPEINTERPRETU2F(name) __riscv_ ## name ## _u32m1_f32m1 +# define TYPEINTERPRETF2U __riscv_vreinterpret_v_f32m1_u32m1 +# define TYPEINTERPRETU2F __riscv_vreinterpret_v_u32m1_f32m1 # define TYPEMEM(name) __riscv_ ## name ## e32_v_f32m1 #else # define DS(d,s) d /* double-precision option */ # define TYPE(name) __riscv_ ## name ## _f64m1 # define TYPEUINT(name) __riscv_ ## name ## _u64m1 -# define TYPEINTERPRETF2U(name) __riscv_ ## name ## _f64m1_u64m1 -# define TYPEINTERPRETU2F(name) __riscv_ ## name ## _u64m1_f64m1 +# define TYPEINTERPRETF2U __riscv_vreinterpret_v_f64m1_u64m1 +# define TYPEINTERPRETU2F __riscv_vreinterpret_v_u64m1_f64m1 # define TYPEMEM(name) __riscv_ ## name ## e64_v_f64m1 #endif @@ -72,8 +72,6 @@ #define SIMD_VSTRIDE_OKA(x) ((x) == 2) #define SIMD_STRIDE_OKPAIR SIMD_STRIDE_OK -#define ZERO DS(0.0, 0.0f) - #include typedef DS(vfloat64m1_t, vfloat32m1_t) V; @@ -90,16 +88,16 @@ typedef DS(vuint64m1_t, vuint32m1_t) Vuint; static inline V VDUPL(const V x) { Vuint partr = VPARTSPLIT; // (all 1, 0, all 1, 0, ...) - V xl = TYPEINTERPRETU2F(vreinterpret_v)(TYPEUINT(vand_vv)(TYPEINTERPRETF2U(vreinterpret_v)(x), partr, 2*VL)); // set odd elements to 0 - return VADD(TYPE(vfslide1up_vf)(xl, ZERO, 2*VL), xl); + Vuint xl = TYPEUINT(vand_vv)(TYPEINTERPRETF2U(x), partr, 2*VL); // set odd elements to 0 + return TYPEINTERPRETU2F(TYPEUINT(vor_vv)(TYPEUINT(vslide1up_vx)(xl, 0, 2*VL), xl, 2*VL)); } static inline V VDUPH(const V x) { Vuint partr = VPARTSPLIT; // (all 1, 0, all 1, 0, ...) Vuint parti = TYPEUINT(vnot_v)(partr, 2*VL); // (0, all 1, 0, all 1, ...) - V xh = TYPEINTERPRETU2F(vreinterpret_v)(TYPEUINT(vand_vv)(TYPEINTERPRETF2U(vreinterpret_v)(x), parti, 2*VL)); // set even elements to 0 - return VADD(TYPE(vfslide1down_vf)(xh, ZERO, 2*VL), xh); + Vuint xh = TYPEUINT(vand_vv)(TYPEINTERPRETF2U(x), parti, 2*VL); // set even elements to 0 + return TYPEINTERPRETU2F(TYPEUINT(vor_vv)(TYPEUINT(vslide1down_vx)(xh, 0, 2*VL), xh, 2*VL)); } #define DVK(var, val) V var = TYPE(vfmv_v_f)(val, 2*VL) @@ -107,28 +105,28 @@ static inline V VDUPH(const V x) static inline V FLIP_RI(const V x) { Vuint partr = VPARTSPLIT; // (all 1, 0, all 1, 0, ...) - V xl = TYPEINTERPRETU2F(vreinterpret_v)(TYPEUINT(vand_vv)(TYPEINTERPRETF2U(vreinterpret_v)(x), partr, 2*VL)); // set odd elements to 0 + Vuint xl = TYPEUINT(vand_vv)(TYPEINTERPRETF2U(x), partr, 2*VL); // set odd elements to 0 Vuint parti = TYPEUINT(vnot_v)(partr, 2*VL); // (0, all 1, 0, all 1, ...) - V xh = TYPEINTERPRETU2F(vreinterpret_v)(TYPEUINT(vand_vv)(TYPEINTERPRETF2U(vreinterpret_v)(x), parti, 2*VL)); // set even elements to 0 - return VADD(TYPE(vfslide1down_vf)(xh, ZERO, 2*VL), TYPE(vfslide1up_vf)(xl, ZERO, 2*VL)); + Vuint xh = TYPEUINT(vand_vv)(TYPEINTERPRETF2U(x), parti, 2*VL); // set even elements to 0 + return TYPEINTERPRETU2F(TYPEUINT(vor_vv)(TYPEUINT(vslide1down_vx)(xh, 0, 2*VL), TYPEUINT(vslide1up_vx)(xl, 0, 2*VL), 2*VL)); } static inline V VCONJ(const V x) { Vuint partr = VPARTSPLIT; // (all 1, 0, all 1, 0, ...) - V xl = TYPEINTERPRETU2F(vreinterpret_v)(TYPEUINT(vand_vv)(TYPEINTERPRETF2U(vreinterpret_v)(x), partr, 2*VL)); // set odd elements to 0 + Vuint xl = TYPEUINT(vand_vv)(TYPEINTERPRETF2U(x), partr, 2*VL); // set odd elements to 0 Vuint parti = TYPEUINT(vnot_v)(partr, 2*VL); // (0, all 1, 0, all 1, ...) - V xh = TYPEINTERPRETU2F(vreinterpret_v)(TYPEUINT(vand_vv)(TYPEINTERPRETF2U(vreinterpret_v)(x), parti, 2*VL)); // set even elements to 0 - return VADD(xl, VNEG(xh)); + Vuint xh = TYPEUINT(vand_vv)(TYPEINTERPRETF2U(VNEG(x)), parti, 2*VL); // set even elements to 0 + return TYPEINTERPRETU2F(TYPEUINT(vor_vv)(xl, xh, 2*VL)); } static inline V VBYI(V x) { Vuint partr = VPARTSPLIT; // (all 1, 0, all 1, 0, ...) - V xl = TYPEINTERPRETU2F(vreinterpret_v)(TYPEUINT(vand_vv)(TYPEINTERPRETF2U(vreinterpret_v)(x), partr, 2*VL)); // set odd elements to 0 + Vuint xl = TYPEUINT(vand_vv)(TYPEINTERPRETF2U(x), partr, 2*VL); // set odd elements to 0 Vuint parti = TYPEUINT(vnot_v)(partr, 2*VL); // (0, all 1, 0, all 1, ...) - V xh = TYPEINTERPRETU2F(vreinterpret_v)(TYPEUINT(vand_vv)(TYPEINTERPRETF2U(vreinterpret_v)(VNEG(x)), parti, 2*VL)); // set elements to negative, then set even elements to 0 - return VADD(TYPE(vfslide1down_vf)(xh, ZERO, 2*VL), TYPE(vfslide1up_vf)(xl, ZERO, 2*VL)); + Vuint xh = TYPEUINT(vand_vv)(TYPEINTERPRETF2U(VNEG(x)), parti, 2*VL); // set elements to negative, then set even elements to 0 + return TYPEINTERPRETU2F(TYPEUINT(vor_vv)(TYPEUINT(vslide1down_vx)(xh, 0, 2*VL), TYPEUINT(vslide1up_vx)(xl, 0, 2*VL), 2*VL)); } #define LDK(x) x @@ -208,9 +206,9 @@ static inline V LD(const R *x, INT ivs, const R *aligned_like) Vuint idx4 = TYPEUINT(vsub_vx)(idx3, 1, 2*VL); // (all 1, 0, all 1, 0, ...) Vuint idx5 = TYPEUINT(vnot_v)(idx4, 2*VL); // (0, all 1, 0, all 1, ...) - V xl2 = TYPEINTERPRETU2F(vreinterpret_v)(TYPEUINT(vand_vv)(TYPEINTERPRETF2U(vreinterpret_v)(xl1), idx4, 2*VL)); // set odd elements to 0 - V xh2 = TYPEINTERPRETU2F(vreinterpret_v)(TYPEUINT(vand_vv)(TYPEINTERPRETF2U(vreinterpret_v)(xh1), idx5, 2*VL)); // set even elements to 0 - return VADD(xl2, xh2); + Vuint xl2 = TYPEUINT(vand_vv)(TYPEINTERPRETF2U(xl1), idx4, 2*VL); // set odd elements to 0 + Vuint xh2 = TYPEUINT(vand_vv)(TYPEINTERPRETF2U(xh1), idx5, 2*VL); // set even elements to 0 + return TYPEINTERPRETU2F(TYPEUINT(vor_vv)(xl2, xh2, 2*VL)); } static inline void ST(R *x, V v, INT ovs, const R *aligned_like)