diff --git a/.codespellrc b/.codespellrc
new file mode 100644
index 0000000000..068c4feba6
--- /dev/null
+++ b/.codespellrc
@@ -0,0 +1,12 @@
+# codespell configuration (read automatically from the repository root).
+#
+# "SME" is the Arm Scalable Matrix Extension, referenced throughout the
+# FrodoKEM Aarch64 code (comments, CPUID_..._SME macros, frodokem_..._sme
+# identifiers, "sve2+sme" build strings). codespell's dictionary treats it as
+# a misspelling of some/same/sms. The ignore-regex below drops the standalone
+# SME token (any case, including "_sme" / "+sme") before spell-checking, while
+# still flagging "sme" embedded in a larger alphabetic word - so genuine typos
+# are unaffected. Kept out of ignore-words-list so "sme" is not globally
+# whitelisted for every file.
+[codespell]
+ignore-regex = (?i)(?<![a-z])sme(?![a-z])
diff --git a/.github/workflows/pq-all.yml b/.github/workflows/pq-all.yml
index 50b7300529..6588d098a1 100644
--- a/.github/workflows/pq-all.yml
+++ b/.github/workflows/pq-all.yml
@@ -43,7 +43,7 @@ jobs:
       - name: Install dependencies
         uses: ./.github/actions/install-apt-deps
         with:
-          packages: autoconf automake libtool build-essential bubblewrap
+          packages: autoconf automake libtool build-essential bubblewrap crossbuild-essential-arm64 crossbuild-essential-armhf
           ghcr-debs-tag: ubuntu-24.04-minimal
 
       # ccache via the cross-platform composite; the script passes the
@@ -154,6 +154,46 @@ jobs:
             "--enable-dilithium=yes,no-ctx", "--enable-dual-alg-certs",
             "--disable-qt",
             "CPPFLAGS=-pedantic -Wdeclaration-after-statement -Wnull-dereference -DWOLFCRYPT_TEST_LINT -DNO_WOLFSSL_CIPHER_SUITE_TEST -DTEST_LIBWOLFSSL_SOURCES_INCLUSION_SEQUENCE"]},
+          {"name": "all-frodokem", "minutes": 3.5,
+           "configure": ["--enable-intelasm", "--enable-sp-asm",
+            "--enable-all", "--enable-testcert", "--enable-experimental",
+            "--enable-mlkem=yes,kyber,ml-kem",
+            "--enable-frodokem=aes,ephemeral", "--disable-qt",
+            "CPPFLAGS=-pedantic -Wdeclaration-after-statement -Wnull-dereference -DWOLFCRYPT_TEST_LINT -DNO_WOLFSSL_CIPHER_SUITE_TEST -DTEST_LIBWOLFSSL_SOURCES_INCLUSION_SEQUENCE"]},
+          {"name": "frodokem-small-smallstack-noasm", "minutes": 2.5,
+           "configure": ["--disable-intelasm", "--enable-smallstack",
+            "--enable-smallstackcache", "--enable-experimental",
+            "--enable-frodokem=aes,ephemeral,small",
+            "CPPFLAGS=-Wdeclaration-after-statement"]},
+          {"name": "frodokem-976-only-noasm", "minutes": 1.5,
+           "comment": "976-only builds D=16 (q=2^16) alone; no-asm the C path",
+           "configure": ["--disable-intelasm", "--enable-experimental",
+            "--enable-frodokem=aes,ephemeral,no-640,no-1344",
+            "CPPFLAGS=-Wdeclaration-after-statement"]},
+          {"name": "frodokem-640-shake-only", "minutes": 1.5,
+           "comment": "640-only builds D=15 (q=2^15); shake-only, with asm",
+           "configure": ["--enable-intelasm", "--enable-sp-asm",
+            "--enable-experimental",
+            "--enable-frodokem=no-976,no-1344,no-aes",
+            "CPPFLAGS=-Wdeclaration-after-statement"]},
+          {"name": "frodokem-aarch64-armasm", "minutes": 4, "check": false,
+           "cc": "ccache aarch64-linux-gnu-gcc",
+           "comment": "Cross-build only (no ARM runner): catches ARM asm link/compile breaks. sve+sme+aes A-gen all live in the .S.",
+           "configure": ["--host=aarch64-linux-gnu", "--enable-cryptonly",
+            "--enable-armasm", "--enable-experimental",
+            "--enable-frodokem=aes,ephemeral,sve,sme", "--disable-examples"]},
+          {"name": "frodokem-aarch64-armasm-inline", "minutes": 4,
+           "check": false, "cc": "ccache aarch64-linux-gnu-gcc",
+           "comment": "Inline armasm: sve/sme/aes are .S-only, so they must fall back to NEON/C; guards the inline-asm link break.",
+           "configure": ["--host=aarch64-linux-gnu", "--enable-cryptonly",
+            "--enable-armasm=inline", "--enable-experimental",
+            "--enable-frodokem=aes,ephemeral,sve,sme", "--disable-examples"]},
+          {"name": "frodokem-arm32-armasm", "minutes": 4, "check": false,
+           "cc": "ccache arm-linux-gnueabihf-gcc",
+           "comment": "Cross-build only: AArch32 NEON matrix ops + AES-crypto A-gen.",
+           "configure": ["--host=arm-linux-gnueabihf", "--enable-cryptonly",
+            "--enable-armasm", "--enable-experimental",
+            "--enable-frodokem=aes,ephemeral", "--disable-examples"]},
           {"name": "ubsan-mldsa-mlkem", "minutes": 2.5,
            "configure": ["--disable-shared", "--enable-dilithium",
             "--enable-mlkem",
diff --git a/.gitignore b/.gitignore
index ecc8bb7fea..d6a54acb81 100644
--- a/.gitignore
+++ b/.gitignore
@@ -7,6 +7,7 @@ ctaocrypt/src/src/
 *.d
 *.libs
 *.cache
+*.su
 .dirstamp
 *.user
 !*-VS2022.vcxproj.user
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 7345e9afb1..89e908b297 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -1037,6 +1037,56 @@ add_option(WOLFSSL_EXTRA_PQC_HYBRIDS
     "Enable extra PQ/T hybrid combinations (default: disabled)"
     "no" "yes;no")
 
+# FrodoKEM reference implementation (experimental)
+add_option(WOLFSSL_FRODOKEM
+    "Enable the wolfSSL FrodoKEM reference implementation (requires WOLFSSL_EXPERIMENTAL) (default: disabled)"
+    "no" "yes;no")
+# FrodoKEM matrix A generation with SHAKE-128 (default: enabled when FrodoKEM on)
+add_option(WOLFSSL_FRODOKEM_SHAKE
+    "Enable FrodoKEM SHAKE-128 matrix A generation (default: enabled)"
+    "yes" "yes;no")
+# FrodoKEM matrix A generation with AES-128 (default: disabled)
+add_option(WOLFSSL_FRODOKEM_AES
+    "Enable FrodoKEM AES-128 matrix A generation (default: disabled)"
+    "no" "yes;no")
+# eFrodoKEM (ephemeral, salt-less) variants (default: disabled)
+add_option(WOLFSSL_FRODOKEM_EPHEMERAL
+    "Enable eFrodoKEM (ephemeral) variants (default: disabled)"
+    "no" "yes;no")
+# FrodoKEM parameter sets (each default: enabled)
+add_option(WOLFSSL_FRODOKEM_640
+    "Enable the FrodoKEM-640 parameter set (default: enabled)"
+    "yes" "yes;no")
+add_option(WOLFSSL_FRODOKEM_976
+    "Enable the FrodoKEM-976 parameter set (default: enabled)"
+    "yes" "yes;no")
+add_option(WOLFSSL_FRODOKEM_1344
+    "Enable the FrodoKEM-1344 parameter set (default: enabled)"
+    "yes" "yes;no")
+# FrodoKEM operations (each default: enabled)
+add_option(WOLFSSL_FRODOKEM_MAKE_KEY
+    "Enable FrodoKEM key generation (default: enabled)"
+    "yes" "yes;no")
+add_option(WOLFSSL_FRODOKEM_ENCAPSULATE
+    "Enable FrodoKEM encapsulation (default: enabled)"
+    "yes" "yes;no")
+add_option(WOLFSSL_FRODOKEM_DECAPSULATE
+    "Enable FrodoKEM decapsulation (default: enabled)"
+    "yes" "yes;no")
+# Small-footprint FrodoKEM: looped (not unrolled) matrix arithmetic
+# (default: disabled)
+add_option(WOLFSSL_FRODOKEM_SMALL
+    "Enable small-footprint FrodoKEM (default: disabled)"
+    "no" "yes;no")
+
+# FrodoKEM AArch64 SVE / SME acceleration (default: disabled; AArch64 asm only)
+add_option(WOLFSSL_FRODOKEM_SVE
+    "Enable FrodoKEM AArch64 SVE acceleration (default: disabled)"
+    "no" "yes;no")
+add_option(WOLFSSL_FRODOKEM_SME
+    "Enable FrodoKEM AArch64 SME acceleration (default: disabled)"
+    "no" "yes;no")
+
 message(STATUS "Looking for WOLFSSL_EXPERIMENTAL")
 if (WOLFSSL_EXPERIMENTAL)
     message(STATUS "Looking for WOLFSSL_EXPERIMENTAL - found")
@@ -1092,6 +1142,98 @@ if (WOLFSSL_EXPERIMENTAL)
         message(STATUS "Looking for WOLFSSL_EXTRA_PQC_HYBRIDS - not found")
     endif()
 
+    # Checking for experimental feature: FrodoKEM
+    message(STATUS "Looking for WOLFSSL_FRODOKEM")
+    if (WOLFSSL_FRODOKEM)
+        set(WOLFSSL_FOUND_EXPERIMENTAL_FEATURE 1)
+        message(STATUS "Looking for WOLFSSL_FRODOKEM - found")
+
+        # At least one matrix A generation method must be enabled.
+        if (NOT WOLFSSL_FRODOKEM_SHAKE AND NOT WOLFSSL_FRODOKEM_AES)
+            message(FATAL_ERROR
+                "WOLFSSL_FRODOKEM requires WOLFSSL_FRODOKEM_SHAKE and/or WOLFSSL_FRODOKEM_AES.")
+        endif()
+        # At least one parameter set must be enabled.
+        if (NOT WOLFSSL_FRODOKEM_640 AND NOT WOLFSSL_FRODOKEM_976 AND
+            NOT WOLFSSL_FRODOKEM_1344)
+            message(FATAL_ERROR
+                "WOLFSSL_FRODOKEM requires at least one of WOLFSSL_FRODOKEM_640, WOLFSSL_FRODOKEM_976 or WOLFSSL_FRODOKEM_1344.")
+        endif()
+        # At least one operation must be enabled.
+        if (NOT WOLFSSL_FRODOKEM_MAKE_KEY AND NOT WOLFSSL_FRODOKEM_ENCAPSULATE
+            AND NOT WOLFSSL_FRODOKEM_DECAPSULATE)
+            message(FATAL_ERROR
+                "WOLFSSL_FRODOKEM requires at least one of WOLFSSL_FRODOKEM_MAKE_KEY, WOLFSSL_FRODOKEM_ENCAPSULATE or WOLFSSL_FRODOKEM_DECAPSULATE.")
+        endif()
+
+        list(APPEND WOLFSSL_DEFINITIONS "-DWOLFSSL_HAVE_FRODOKEM")
+        list(APPEND WOLFSSL_DEFINITIONS "-DWOLFSSL_SHA3")
+        list(APPEND WOLFSSL_DEFINITIONS "-DWOLFSSL_SHAKE128")
+        list(APPEND WOLFSSL_DEFINITIONS "-DWOLFSSL_SHAKE256")
+
+        set_wolfssl_definitions("WOLFSSL_HAVE_FRODOKEM" RESULT)
+        set_wolfssl_definitions("WOLFSSL_SHA3"          RESULT)
+        set_wolfssl_definitions("WOLFSSL_SHAKE128"      RESULT)
+        set_wolfssl_definitions("WOLFSSL_SHAKE256"      RESULT)
+
+        if (WOLFSSL_FRODOKEM_SHAKE)
+            list(APPEND WOLFSSL_DEFINITIONS "-DWOLFSSL_FRODOKEM_SHAKE")
+            set_wolfssl_definitions("WOLFSSL_FRODOKEM_SHAKE" RESULT)
+        endif()
+        if (WOLFSSL_FRODOKEM_AES)
+            list(APPEND WOLFSSL_DEFINITIONS "-DWOLFSSL_FRODOKEM_AES")
+            list(APPEND WOLFSSL_DEFINITIONS "-DWOLFSSL_AES_DIRECT")
+            list(APPEND WOLFSSL_DEFINITIONS "-DHAVE_AES_ECB")
+            set_wolfssl_definitions("WOLFSSL_FRODOKEM_AES" RESULT)
+            set_wolfssl_definitions("WOLFSSL_AES_DIRECT"   RESULT)
+            set_wolfssl_definitions("HAVE_AES_ECB"         RESULT)
+        endif()
+        if (WOLFSSL_FRODOKEM_EPHEMERAL)
+            list(APPEND WOLFSSL_DEFINITIONS "-DWOLFSSL_FRODOKEM_EPHEMERAL")
+            set_wolfssl_definitions("WOLFSSL_FRODOKEM_EPHEMERAL" RESULT)
+        endif()
+        if (WOLFSSL_FRODOKEM_SMALL)
+            list(APPEND WOLFSSL_DEFINITIONS "-DWOLFSSL_FRODOKEM_SMALL")
+            set_wolfssl_definitions("WOLFSSL_FRODOKEM_SMALL" RESULT)
+        endif()
+        if (WOLFSSL_FRODOKEM_SVE)
+            list(APPEND WOLFSSL_DEFINITIONS "-DWOLFSSL_FRODOKEM_SVE")
+            set_wolfssl_definitions("WOLFSSL_FRODOKEM_SVE" RESULT)
+        endif()
+        if (WOLFSSL_FRODOKEM_SME)
+            list(APPEND WOLFSSL_DEFINITIONS "-DWOLFSSL_FRODOKEM_SME")
+            set_wolfssl_definitions("WOLFSSL_FRODOKEM_SME" RESULT)
+        endif()
+        # Excluded parameter sets.
+        if (NOT WOLFSSL_FRODOKEM_640)
+            list(APPEND WOLFSSL_DEFINITIONS "-DWOLFSSL_NO_FRODOKEM_640")
+            set_wolfssl_definitions("WOLFSSL_NO_FRODOKEM_640" RESULT)
+        endif()
+        if (NOT WOLFSSL_FRODOKEM_976)
+            list(APPEND WOLFSSL_DEFINITIONS "-DWOLFSSL_NO_FRODOKEM_976")
+            set_wolfssl_definitions("WOLFSSL_NO_FRODOKEM_976" RESULT)
+        endif()
+        if (NOT WOLFSSL_FRODOKEM_1344)
+            list(APPEND WOLFSSL_DEFINITIONS "-DWOLFSSL_NO_FRODOKEM_1344")
+            set_wolfssl_definitions("WOLFSSL_NO_FRODOKEM_1344" RESULT)
+        endif()
+        # Excluded operations.
+        if (NOT WOLFSSL_FRODOKEM_MAKE_KEY)
+            list(APPEND WOLFSSL_DEFINITIONS "-DWOLFSSL_FRODOKEM_NO_MAKE_KEY")
+            set_wolfssl_definitions("WOLFSSL_FRODOKEM_NO_MAKE_KEY" RESULT)
+        endif()
+        if (NOT WOLFSSL_FRODOKEM_ENCAPSULATE)
+            list(APPEND WOLFSSL_DEFINITIONS "-DWOLFSSL_FRODOKEM_NO_ENCAPSULATE")
+            set_wolfssl_definitions("WOLFSSL_FRODOKEM_NO_ENCAPSULATE" RESULT)
+        endif()
+        if (NOT WOLFSSL_FRODOKEM_DECAPSULATE)
+            list(APPEND WOLFSSL_DEFINITIONS "-DWOLFSSL_FRODOKEM_NO_DECAPSULATE")
+            set_wolfssl_definitions("WOLFSSL_FRODOKEM_NO_DECAPSULATE" RESULT)
+        endif()
+    else()
+        message(STATUS "Looking for WOLFSSL_FRODOKEM - not found")
+    endif()
+
     # Other experimental feature detection can be added here...
 
     # Were any experimental features found? Display a message.
@@ -1110,6 +1252,9 @@ else()
     if (WOLFSSL_FALCON)
         message(FATAL_ERROR "Error: WOLFSSL_FALCON requires WOLFSSL_EXPERIMENTAL at this time.")
     endif()
+    if (WOLFSSL_FRODOKEM)
+        message(FATAL_ERROR "Error: WOLFSSL_FRODOKEM requires WOLFSSL_EXPERIMENTAL at this time.")
+    endif()
 endif()
 
 # LMS
@@ -4184,6 +4329,7 @@ if(WOLFSSL_EXAMPLES)
         tests/api/test_ed25519.c
         tests/api/test_curve448.c
         tests/api/test_ed448.c
+        tests/api/test_frodokem.c
         tests/api/test_mlkem.c
         tests/api/test_mldsa.c
         tests/api/test_mldsa_legacy.c
diff --git a/cmake/functions.cmake b/cmake/functions.cmake
index e53d44054c..a3d37f3de9 100644
--- a/cmake/functions.cmake
+++ b/cmake/functions.cmake
@@ -253,6 +253,9 @@ function(generate_build_flags)
     if(WOLFSSL_MLKEM OR WOLFSSL_USER_SETTINGS)
         set(BUILD_WC_MLKEM "yes" PARENT_SCOPE)
     endif()
+    if(WOLFSSL_FRODOKEM OR WOLFSSL_USER_SETTINGS)
+        set(BUILD_WC_FRODOKEM "yes" PARENT_SCOPE)
+    endif()
     if(WOLFSSL_MLDSA OR WOLFSSL_DILITHIUM OR WOLFSSL_USER_SETTINGS)
         set(BUILD_MLDSA "yes" PARENT_SCOPE)
     endif()
@@ -1175,6 +1178,35 @@ function(generate_lib_src_list LIB_SOURCES)
             endif()
         endif()
 
+        if(BUILD_WC_FRODOKEM)
+            list(APPEND LIB_SOURCES wolfcrypt/src/wc_frodokem.c)
+            list(APPEND LIB_SOURCES wolfcrypt/src/wc_frodokem_mat.c)
+
+            if(BUILD_ARMASM)
+                if(BUILD_ARMASM_INLINE)
+                    list(APPEND LIB_SOURCES
+                        wolfcrypt/src/port/arm/armv8-frodokem-asm_c.c
+                        wolfcrypt/src/port/arm/armv8-32-frodokem-asm_c.c)
+                else()
+                    list(APPEND LIB_SOURCES
+                        wolfcrypt/src/port/arm/armv8-frodokem-asm.S
+                        wolfcrypt/src/port/arm/armv8-32-frodokem-asm.S)
+                endif()
+
+                if(BUILD_ARMASM_INLINE AND BUILD_ARM_THUMB)
+                    list(APPEND LIB_SOURCES
+                        wolfcrypt/src/port/arm/thumb2-frodokem-asm_c.c)
+                elseif(BUILD_ARM_THUMB)
+                    list(APPEND LIB_SOURCES
+                        wolfcrypt/src/port/arm/thumb2-frodokem-asm.S)
+                endif()
+            endif()
+
+            if(BUILD_INTELASM)
+                list(APPEND LIB_SOURCES wolfcrypt/src/wc_frodokem_asm.S)
+            endif()
+        endif()
+
         if(BUILD_WC_LMS)
             list(APPEND LIB_SOURCES wolfcrypt/src/wc_lms.c)
             list(APPEND LIB_SOURCES wolfcrypt/src/wc_lms_impl.c)
diff --git a/configure.ac b/configure.ac
index 25028f5758..9efe1bb15f 100644
--- a/configure.ac
+++ b/configure.ac
@@ -1911,6 +1911,166 @@ AC_ARG_ENABLE([extra-pqc-hybrids],
     )
 
 
+# FrodoKEM (experimental reference implementation)
+AC_ARG_ENABLE([frodokem],
+    [AS_HELP_STRING([--enable-frodokem],[Enable FrodoKEM reference implementation, optionally with a comma-separated list of sub-options e.g. aes,ephemeral,small,no-640 (requires --enable-experimental) (default: disabled)])],
+    [ ENABLED_FRODOKEM=$enableval ],
+    [ ENABLED_FRODOKEM=no ]
+    )
+
+AS_IF([ test "$ENABLED_FRODOKEM" != "no" && test "$ENABLED_EXPERIMENTAL" != "yes" ],[ AC_MSG_ERROR([frodokem requires --enable-experimental.]) ])
+
+# FrodoKEM matrix A generation with SHAKE-128 (default: enabled with frodokem).
+AC_ARG_ENABLE([frodokem-shake],
+    [AS_HELP_STRING([--enable-frodokem-shake],[Enable FrodoKEM SHAKE-128 matrix A generation (default: enabled)])],
+    [ ENABLED_FRODOKEM_SHAKE=$enableval ],
+    [ ENABLED_FRODOKEM_SHAKE=default ]
+    )
+# FrodoKEM matrix A generation with AES-128 (default: disabled).
+AC_ARG_ENABLE([frodokem-aes],
+    [AS_HELP_STRING([--enable-frodokem-aes],[Enable FrodoKEM AES-128 matrix A generation (default: disabled)])],
+    [ ENABLED_FRODOKEM_AES=$enableval ],
+    [ ENABLED_FRODOKEM_AES=no ]
+    )
+# AArch64 SVE S*A accumulate, selected at run time by HWCAP (default: disabled).
+AC_ARG_ENABLE([frodokem-sve],
+    [AS_HELP_STRING([--enable-frodokem-sve],[Enable FrodoKEM AArch64 SVE matrix acceleration (default: disabled)])],
+    [ ENABLED_FRODOKEM_SVE=$enableval ],
+    [ ENABLED_FRODOKEM_SVE=no ]
+    )
+# AArch64 SME B*S via the ZA-tile UMOPA, selected at run time (default: disabled).
+AC_ARG_ENABLE([frodokem-sme],
+    [AS_HELP_STRING([--enable-frodokem-sme],[Enable FrodoKEM AArch64 SME matrix acceleration (default: disabled)])],
+    [ ENABLED_FRODOKEM_SME=$enableval ],
+    [ ENABLED_FRODOKEM_SME=no ]
+    )
+# eFrodoKEM (ephemeral, salt-less) variants (default: disabled).
+AC_ARG_ENABLE([frodokem-ephemeral],
+    [AS_HELP_STRING([--enable-frodokem-ephemeral],[Enable eFrodoKEM (ephemeral) variants (default: disabled)])],
+    [ ENABLED_FRODOKEM_EPHEMERAL=$enableval ],
+    [ ENABLED_FRODOKEM_EPHEMERAL=no ]
+    )
+# FrodoKEM parameter sets (each default: enabled with frodokem).
+AC_ARG_ENABLE([frodokem-640],
+    [AS_HELP_STRING([--enable-frodokem-640],[Enable the FrodoKEM-640 parameter set (default: enabled)])],
+    [ ENABLED_FRODOKEM_640=$enableval ],
+    [ ENABLED_FRODOKEM_640=yes ]
+    )
+AC_ARG_ENABLE([frodokem-976],
+    [AS_HELP_STRING([--enable-frodokem-976],[Enable the FrodoKEM-976 parameter set (default: enabled)])],
+    [ ENABLED_FRODOKEM_976=$enableval ],
+    [ ENABLED_FRODOKEM_976=yes ]
+    )
+AC_ARG_ENABLE([frodokem-1344],
+    [AS_HELP_STRING([--enable-frodokem-1344],[Enable the FrodoKEM-1344 parameter set (default: enabled)])],
+    [ ENABLED_FRODOKEM_1344=$enableval ],
+    [ ENABLED_FRODOKEM_1344=yes ]
+    )
+# FrodoKEM operations (each default: enabled with frodokem).
+AC_ARG_ENABLE([frodokem-make-key],
+    [AS_HELP_STRING([--enable-frodokem-make-key],[Enable FrodoKEM key generation (default: enabled)])],
+    [ ENABLED_FRODOKEM_MAKE_KEY=$enableval ],
+    [ ENABLED_FRODOKEM_MAKE_KEY=yes ]
+    )
+AC_ARG_ENABLE([frodokem-encapsulate],
+    [AS_HELP_STRING([--enable-frodokem-encapsulate],[Enable FrodoKEM encapsulation (default: enabled)])],
+    [ ENABLED_FRODOKEM_ENCAPSULATE=$enableval ],
+    [ ENABLED_FRODOKEM_ENCAPSULATE=yes ]
+    )
+AC_ARG_ENABLE([frodokem-decapsulate],
+    [AS_HELP_STRING([--enable-frodokem-decapsulate],[Enable FrodoKEM decapsulation (default: enabled)])],
+    [ ENABLED_FRODOKEM_DECAPSULATE=$enableval ],
+    [ ENABLED_FRODOKEM_DECAPSULATE=yes ]
+    )
+# Small-footprint FrodoKEM: use the looped (not fully unrolled) matrix
+# arithmetic (default: disabled).
+AC_ARG_ENABLE([frodokem-small],
+    [AS_HELP_STRING([--enable-frodokem-small],[Enable small-footprint FrodoKEM (default: disabled)])],
+    [ ENABLED_FRODOKEM_SMALL=$enableval ],
+    [ ENABLED_FRODOKEM_SMALL=no ]
+    )
+
+# The FrodoKEM sub-options may also be given as a comma-separated list to
+# --enable-frodokem, e.g. --enable-frodokem=aes,ephemeral,small,no-640. Each
+# token sets the matching --enable-frodokem-<option> value ("no-<option>"
+# disables). This is equivalent to passing the individual --enable-frodokem-*
+# options.
+case "$ENABLED_FRODOKEM" in
+no | yes | '')
+    ;;
+*)
+    frodokem_opts="$ENABLED_FRODOKEM"
+    ENABLED_FRODOKEM=yes
+    for frodokem_opt in $(echo "$frodokem_opts" | tr ',' ' ')
+    do
+        case "$frodokem_opt" in
+        yes)            ;;
+        small)          ENABLED_FRODOKEM_SMALL=yes ;;
+        no-small)       ENABLED_FRODOKEM_SMALL=no ;;
+        shake)          ENABLED_FRODOKEM_SHAKE=yes ;;
+        no-shake)       ENABLED_FRODOKEM_SHAKE=no ;;
+        aes)            ENABLED_FRODOKEM_AES=yes ;;
+        no-aes)         ENABLED_FRODOKEM_AES=no ;;
+        sve)            ENABLED_FRODOKEM_SVE=yes ;;
+        no-sve)         ENABLED_FRODOKEM_SVE=no ;;
+        sme)            ENABLED_FRODOKEM_SME=yes ;;
+        no-sme)         ENABLED_FRODOKEM_SME=no ;;
+        ephemeral)      ENABLED_FRODOKEM_EPHEMERAL=yes ;;
+        no-ephemeral)   ENABLED_FRODOKEM_EPHEMERAL=no ;;
+        640)            ENABLED_FRODOKEM_640=yes ;;
+        no-640)         ENABLED_FRODOKEM_640=no ;;
+        976)            ENABLED_FRODOKEM_976=yes ;;
+        no-976)         ENABLED_FRODOKEM_976=no ;;
+        1344)           ENABLED_FRODOKEM_1344=yes ;;
+        no-1344)        ENABLED_FRODOKEM_1344=no ;;
+        make-key)       ENABLED_FRODOKEM_MAKE_KEY=yes ;;
+        no-make-key)    ENABLED_FRODOKEM_MAKE_KEY=no ;;
+        encapsulate)    ENABLED_FRODOKEM_ENCAPSULATE=yes ;;
+        no-encapsulate) ENABLED_FRODOKEM_ENCAPSULATE=no ;;
+        decapsulate)    ENABLED_FRODOKEM_DECAPSULATE=yes ;;
+        no-decapsulate) ENABLED_FRODOKEM_DECAPSULATE=no ;;
+        *) AC_MSG_ERROR([Unknown --enable-frodokem option: $frodokem_opt]) ;;
+        esac
+    done
+    ;;
+esac
+
+AS_IF([ test "$ENABLED_FRODOKEM" = "no" &&
+        ( test "$ENABLED_FRODOKEM_AES" != "no" ||
+          test "$ENABLED_FRODOKEM_EPHEMERAL" != "no" ) ],
+      [ AC_MSG_ERROR([frodokem-aes and frodokem-ephemeral require --enable-frodokem.]) ])
+
+if test "$ENABLED_FRODOKEM" != "no"
+then
+    # SHAKE-128 matrix A generation is on by default unless explicitly disabled.
+    if test "$ENABLED_FRODOKEM_SHAKE" = "default"; then
+        ENABLED_FRODOKEM_SHAKE=yes
+    fi
+    # At least one matrix A generation method must be enabled.
+    if test "$ENABLED_FRODOKEM_SHAKE" = "no" && test "$ENABLED_FRODOKEM_AES" = "no"
+    then
+        AC_MSG_ERROR([frodokem requires at least one of --enable-frodokem-shake or --enable-frodokem-aes.])
+    fi
+    # At least one parameter set must be enabled.
+    if test "$ENABLED_FRODOKEM_640" = "no" && \
+       test "$ENABLED_FRODOKEM_976" = "no" && \
+       test "$ENABLED_FRODOKEM_1344" = "no"
+    then
+        AC_MSG_ERROR([frodokem requires at least one of --enable-frodokem-640, --enable-frodokem-976 or --enable-frodokem-1344.])
+    fi
+    # At least one operation must be enabled.
+    if test "$ENABLED_FRODOKEM_MAKE_KEY" = "no" && \
+       test "$ENABLED_FRODOKEM_ENCAPSULATE" = "no" && \
+       test "$ENABLED_FRODOKEM_DECAPSULATE" = "no"
+    then
+        AC_MSG_ERROR([frodokem requires at least one of --enable-frodokem-make-key, --enable-frodokem-encapsulate or --enable-frodokem-decapsulate.])
+    fi
+else
+    if test "$ENABLED_FRODOKEM_SHAKE" = "default"; then
+        ENABLED_FRODOKEM_SHAKE=no
+    fi
+fi
+
 # Dilithium
 #  - SHA3, Shake128 and Shake256
 AC_ARG_ENABLE([mldsa],
@@ -7615,6 +7775,12 @@ then
     ENABLED_SHAKE128=yes
     ENABLED_SHAKE256=yes
 fi
+if test "$ENABLED_FRODOKEM" != "no"
+then
+    ENABLED_SHA3=yes
+    ENABLED_SHAKE128=yes
+    ENABLED_SHAKE256=yes
+fi
 
 # Set SHA-3 flags
 if test "$ENABLED_SHA3" != "no"
@@ -11891,6 +12057,39 @@ AS_IF([test "x$ENABLED_SYS_CA_CERTS" = "xyes"],
 AS_IF([test "x$ENABLED_DUAL_ALG_CERTS" = "xyes"],
       [AM_CFLAGS="$AM_CFLAGS -DWOLFSSL_DUAL_ALG_CERTS"])
 
+AS_IF([test "x$ENABLED_FRODOKEM" = "xyes"],
+      [AM_CFLAGS="$AM_CFLAGS -DWOLFSSL_HAVE_FRODOKEM"
+       AM_CCASFLAGS="$AM_CCASFLAGS -DWOLFSSL_HAVE_FRODOKEM"])
+AS_IF([test "x$ENABLED_FRODOKEM_SHAKE" = "xyes"],
+      [AM_CFLAGS="$AM_CFLAGS -DWOLFSSL_FRODOKEM_SHAKE"])
+AS_IF([test "x$ENABLED_FRODOKEM_AES" = "xyes"],
+      [AM_CFLAGS="$AM_CFLAGS -DWOLFSSL_FRODOKEM_AES -DWOLFSSL_AES_DIRECT -DHAVE_AES_ECB"])
+AS_IF([test "x$ENABLED_FRODOKEM_SVE" = "xyes"],
+      [AM_CFLAGS="$AM_CFLAGS -DWOLFSSL_FRODOKEM_SVE"])
+AS_IF([test "x$ENABLED_FRODOKEM_SME" = "xyes"],
+      [AM_CFLAGS="$AM_CFLAGS -DWOLFSSL_FRODOKEM_SME"])
+AS_IF([test "x$ENABLED_FRODOKEM_EPHEMERAL" = "xyes"],
+      [AM_CFLAGS="$AM_CFLAGS -DWOLFSSL_FRODOKEM_EPHEMERAL"])
+AS_IF([test "x$ENABLED_FRODOKEM" = "xyes" && \
+       test "x$ENABLED_FRODOKEM_SMALL" = "xyes"],
+      [AM_CFLAGS="$AM_CFLAGS -DWOLFSSL_FRODOKEM_SMALL"])
+# Parameter sets and operations excluded from the build.
+AS_IF([test "x$ENABLED_FRODOKEM" = "xyes" && test "x$ENABLED_FRODOKEM_640" = "xno"],
+      [AM_CFLAGS="$AM_CFLAGS -DWOLFSSL_NO_FRODOKEM_640"])
+AS_IF([test "x$ENABLED_FRODOKEM" = "xyes" && test "x$ENABLED_FRODOKEM_976" = "xno"],
+      [AM_CFLAGS="$AM_CFLAGS -DWOLFSSL_NO_FRODOKEM_976"])
+AS_IF([test "x$ENABLED_FRODOKEM" = "xyes" && test "x$ENABLED_FRODOKEM_1344" = "xno"],
+      [AM_CFLAGS="$AM_CFLAGS -DWOLFSSL_NO_FRODOKEM_1344"])
+AS_IF([test "x$ENABLED_FRODOKEM" = "xyes" && \
+       test "x$ENABLED_FRODOKEM_MAKE_KEY" = "xno"],
+      [AM_CFLAGS="$AM_CFLAGS -DWOLFSSL_FRODOKEM_NO_MAKE_KEY"])
+AS_IF([test "x$ENABLED_FRODOKEM" = "xyes" && \
+       test "x$ENABLED_FRODOKEM_ENCAPSULATE" = "xno"],
+      [AM_CFLAGS="$AM_CFLAGS -DWOLFSSL_FRODOKEM_NO_ENCAPSULATE"])
+AS_IF([test "x$ENABLED_FRODOKEM" = "xyes" && \
+       test "x$ENABLED_FRODOKEM_DECAPSULATE" = "xno"],
+      [AM_CFLAGS="$AM_CFLAGS -DWOLFSSL_FRODOKEM_NO_DECAPSULATE"])
+
 AS_IF([test "x$ENABLED_RPK" = "xyes"],
       [AM_CFLAGS="$AM_CFLAGS -DHAVE_RPK"])
 
@@ -12556,6 +12755,7 @@ AM_CONDITIONAL([BUILD_WC_LMS],[test "x$ENABLED_LMS" != "xno" || test "x$ENABLED_
 AM_CONDITIONAL([BUILD_WC_XMSS],[test "x$ENABLED_XMSS" != "xno" || test "x$ENABLED_USERSETTINGS" = "xyes"])
 AM_CONDITIONAL([BUILD_WC_SLHDSA],[test "x$ENABLED_SLHDSA" != "xno" || test "x$ENABLED_USERSETTINGS" = "xyes"])
 AM_CONDITIONAL([BUILD_WC_MLKEM],[test "x$ENABLED_MLKEM" != "xno" || test "x$ENABLED_USERSETTINGS" = "xyes"])
+AM_CONDITIONAL([BUILD_WC_FRODOKEM],[test "x$ENABLED_FRODOKEM" = "xyes" || test "x$ENABLED_USERSETTINGS" = "xyes"])
 AM_CONDITIONAL([BUILD_MLDSA],[test "x$ENABLED_MLDSA" != "xno" || test "x$ENABLED_USERSETTINGS" = "xyes"])
 AM_CONDITIONAL([BUILD_ECCSI],[test "x$ENABLED_ECCSI" = "xyes" || test "x$ENABLED_USERSETTINGS" = "xyes"])
 AM_CONDITIONAL([BUILD_SAKKE],[test "x$ENABLED_SAKKE" = "xyes" || test "x$ENABLED_USERSETTINGS" = "xyes"])
@@ -13276,6 +13476,16 @@ echo "   * Maxim Integrated MAXQ10XX:  $ENABLED_MAXQ10XX"
 echo "   * PSA:                        $ENABLED_PSA"
 echo "   * System CA certs:            $ENABLED_SYS_CA_CERTS"
 echo "   * Dual alg cert support:      $ENABLED_DUAL_ALG_CERTS"
+echo "   * FrodoKEM (experimental):    $ENABLED_FRODOKEM"
+echo "   * FrodoKEM SHAKE-128 matrix A: $ENABLED_FRODOKEM_SHAKE"
+echo "   * FrodoKEM AES-128 matrix A:   $ENABLED_FRODOKEM_AES"
+echo "   * eFrodoKEM (ephemeral):      $ENABLED_FRODOKEM_EPHEMERAL"
+echo "   * FrodoKEM-640:               $ENABLED_FRODOKEM_640"
+echo "   * FrodoKEM-976:               $ENABLED_FRODOKEM_976"
+echo "   * FrodoKEM-1344:              $ENABLED_FRODOKEM_1344"
+echo "   * FrodoKEM make key:          $ENABLED_FRODOKEM_MAKE_KEY"
+echo "   * FrodoKEM encapsulate:       $ENABLED_FRODOKEM_ENCAPSULATE"
+echo "   * FrodoKEM decapsulate:       $ENABLED_FRODOKEM_DECAPSULATE"
 echo "   * ERR Queues per Thread:      $ENABLED_ERRORQUEUEPERTHREAD"
 echo "   * rwlock:                     $ENABLED_RWLOCK"
 echo "   * keylog export:              $ENABLED_KEYLOG_EXPORT"
diff --git a/doc/dox_comments/header_files/doxygen_groups.h b/doc/dox_comments/header_files/doxygen_groups.h
index b84a3e4909..35212c8401 100644
--- a/doc/dox_comments/header_files/doxygen_groups.h
+++ b/doc/dox_comments/header_files/doxygen_groups.h
@@ -39,6 +39,18 @@
     selected when the key is initialized via wc_MlKemKey_Init() or
     wc_MlKemKey_New().
 
+    \defgroup FRODO_KEM Algorithms - FrodoKEM
+    FrodoKEM is a conservative, unstructured-lattice (plain LWE) key
+    encapsulation mechanism. This is an experimental reference
+    implementation following the ISO/CFRG (salted) FrodoKEM and the
+    ephemeral eFrodoKEM (salt-less) variants. Three parameter sets are
+    provided - FrodoKEM-640 (NIST level 1), FrodoKEM-976 (level 3) and
+    FrodoKEM-1344 (level 5) - each with either SHAKE-128 or AES-128
+    generation of the public matrix A. The variant is selected by the
+    key type (a base parameter set optionally OR'd with FRODOKEM_AES
+    and/or FRODOKEM_EPHEMERAL) when the key is initialized via
+    wc_FrodoKemKey_Init() or wc_FrodoKemKey_New().
+
     \defgroup SLH_DSA Algorithms - SLH-DSA (FIPS 205)
     SLH-DSA (Stateless Hash-based Digital Signature Algorithm) is a
     quantum-resistant signature scheme standardized by NIST as
diff --git a/doc/dox_comments/header_files/wc_frodokem.h b/doc/dox_comments/header_files/wc_frodokem.h
new file mode 100644
index 0000000000..6fa6a9c980
--- /dev/null
+++ b/doc/dox_comments/header_files/wc_frodokem.h
@@ -0,0 +1,462 @@
+/*!
+    \ingroup FRODO_KEM
+
+    \brief Allocates and initializes a new FrodoKemKey on the heap. The
+    returned pointer must be released with wc_FrodoKemKey_Delete().
+
+    FrodoKEM is a conservative, unstructured-lattice key encapsulation
+    mechanism. The type parameter is a base parameter set -
+    WC_FRODOKEM_640_SHAKE (NIST level 1), WC_FRODOKEM_976_SHAKE
+    (level 3) or WC_FRODOKEM_1344_SHAKE (level 5) - optionally OR'd with
+    FRODOKEM_AES (generate matrix A with AES-128 instead of SHAKE-128)
+    and/or FRODOKEM_EPHEMERAL (eFrodoKEM: ephemeral, no salt). Named
+    combinations such as WC_FRODOKEM_640_AES and WC_EFRODOKEM_1344_SHAKE
+    are provided.
+
+    \return Pointer to a freshly allocated FrodoKemKey on success.
+    \return NULL on allocation failure or if type is invalid.
+
+    \param [in] type FrodoKEM key type: a base parameter set optionally
+    OR'd with FRODOKEM_AES and/or FRODOKEM_EPHEMERAL.
+    \param [in] heap Heap hint for dynamic memory allocation. May be
+    NULL.
+    \param [in] devId Device identifier for hardware crypto callbacks.
+    Use INVALID_DEVID for software-only.
+
+    _Example_
+    \code
+    FrodoKemKey* key = wc_FrodoKemKey_New(WC_FRODOKEM_640_SHAKE, NULL,
+        INVALID_DEVID);
+    if (key == NULL) {
+        // allocation failed
+    }
+    // ... use key ...
+    wc_FrodoKemKey_Delete(key, &key);
+    \endcode
+
+    \sa wc_FrodoKemKey_Delete
+    \sa wc_FrodoKemKey_Init
+*/
+FrodoKemKey* wc_FrodoKemKey_New(int type, void* heap, int devId);
+
+/*!
+    \ingroup FRODO_KEM
+
+    \brief Frees and zeros a heap-allocated FrodoKemKey previously
+    returned by wc_FrodoKemKey_New(). On success the caller's pointer
+    variable is set to NULL via key_p when key_p is not NULL.
+
+    \return 0 on success.
+    \return BAD_FUNC_ARG if key is NULL.
+
+    \param [in,out] key The FrodoKemKey to free.
+    \param [in,out] key_p Optional address of the caller's pointer
+    variable; when not NULL, it is set to NULL on success.
+
+    \sa wc_FrodoKemKey_New
+*/
+int wc_FrodoKemKey_Delete(FrodoKemKey* key, FrodoKemKey** key_p);
+
+/*!
+    \ingroup FRODO_KEM
+
+    \brief Initializes a FrodoKemKey object in place. The type parameter
+    selects the FrodoKEM variant: a base parameter set
+    (WC_FRODOKEM_640_SHAKE, WC_FRODOKEM_976_SHAKE or
+    WC_FRODOKEM_1344_SHAKE) optionally OR'd with FRODOKEM_AES and/or
+    FRODOKEM_EPHEMERAL.
+
+    \return 0 on success.
+    \return BAD_FUNC_ARG if key is NULL or type has bits set outside a
+    base parameter set, FRODOKEM_AES and FRODOKEM_EPHEMERAL.
+    \return NOT_COMPILED_IN if type names a valid variant whose
+    parameter set or matrix-A generation method was disabled at build
+    time.
+
+    \param [in,out] key Pointer to the FrodoKemKey to initialize.
+    \param [in] type FrodoKEM key type: a base parameter set optionally
+    OR'd with FRODOKEM_AES and/or FRODOKEM_EPHEMERAL.
+    \param [in] heap Heap hint for dynamic memory allocation. May be
+    NULL.
+    \param [in] devId Device identifier for hardware crypto callbacks.
+
+    _Example_
+    \code
+    FrodoKemKey key;
+    int ret;
+
+    ret = wc_FrodoKemKey_Init(&key, WC_FRODOKEM_640_SHAKE, NULL,
+        INVALID_DEVID);
+    if (ret != 0) {
+        // error initializing key
+    }
+    // ... use key ...
+    wc_FrodoKemKey_Free(&key);
+    \endcode
+
+    \sa wc_FrodoKemKey_Free
+    \sa wc_FrodoKemKey_MakeKey
+*/
+int wc_FrodoKemKey_Init(FrodoKemKey* key, int type, void* heap,
+    int devId);
+
+/*!
+    \ingroup FRODO_KEM
+
+    \brief Releases the resources held by a FrodoKemKey initialized with
+    wc_FrodoKemKey_Init() and zeroizes its secret material. The key
+    object itself is not freed (it was not allocated by Init).
+
+    \return 0 on success.
+    \return BAD_FUNC_ARG if key is NULL.
+
+    \param [in,out] key The FrodoKemKey to free.
+
+    \sa wc_FrodoKemKey_Init
+*/
+int wc_FrodoKemKey_Free(FrodoKemKey* key);
+
+/*!
+    \ingroup FRODO_KEM
+
+    \brief Generates a FrodoKEM key pair into an initialized key using
+    the provided random number generator. On success the key holds both
+    the public and private keys, which can be serialized with
+    wc_FrodoKemKey_EncodePublicKey() and wc_FrodoKemKey_EncodePrivateKey().
+
+    \return 0 on success.
+    \return BAD_FUNC_ARG if key or rng is NULL.
+    \return MEMORY_E on allocation failure.
+    \return A negative error code from the RNG or hashing on failure.
+
+    \param [in,out] key An initialized FrodoKemKey to hold the generated
+    key pair.
+    \param [in] rng An initialized random number generator.
+
+    _Example_
+    \code
+    FrodoKemKey key;
+    WC_RNG rng;
+    int ret;
+
+    wc_InitRng(&rng);
+    wc_FrodoKemKey_Init(&key, WC_FRODOKEM_640_SHAKE, NULL, INVALID_DEVID);
+
+    ret = wc_FrodoKemKey_MakeKey(&key, &rng);
+    if (ret != 0) {
+        // error generating key pair
+    }
+
+    wc_FrodoKemKey_Free(&key);
+    wc_FreeRng(&rng);
+    \endcode
+
+    \sa wc_FrodoKemKey_MakeKeyWithRandom
+    \sa wc_FrodoKemKey_Encapsulate
+    \sa wc_FrodoKemKey_EncodePublicKey
+*/
+int wc_FrodoKemKey_MakeKey(FrodoKemKey* key, WC_RNG* rng);
+
+/*!
+    \ingroup FRODO_KEM
+
+    \brief Generates a FrodoKEM key pair from caller-supplied random
+    bytes rather than an RNG. This is primarily for testing and known
+    answer tests; production code should use wc_FrodoKemKey_MakeKey().
+    The required length depends on the parameter set (for example
+    WC_FRODOKEM_640_MAKEKEY_RAND_SZ for FrodoKEM-640).
+
+    \return 0 on success.
+    \return BAD_FUNC_ARG if key or rand is NULL, or len is not the
+    length required for the key's parameter set.
+
+    \param [in,out] key An initialized FrodoKemKey to hold the generated
+    key pair.
+    \param [in] rand Buffer of random bytes.
+    \param [in] len Length of rand in bytes; must equal the parameter
+    set's WC_FRODOKEM_<set>_MAKEKEY_RAND_SZ.
+
+    \sa wc_FrodoKemKey_MakeKey
+*/
+int wc_FrodoKemKey_MakeKeyWithRandom(FrodoKemKey* key,
+    const unsigned char* rand, int len);
+
+/*!
+    \ingroup FRODO_KEM
+
+    \brief Returns, through len, the size in bytes of a ciphertext
+    (encapsulation) for the key's parameter set. Use this to size the
+    ct buffer passed to wc_FrodoKemKey_Encapsulate().
+
+    \return 0 on success.
+    \return BAD_FUNC_ARG if key or len is NULL.
+
+    \param [in] key An initialized FrodoKemKey.
+    \param [out] len Set to the ciphertext size in bytes.
+
+    \sa wc_FrodoKemKey_Encapsulate
+    \sa wc_FrodoKemKey_SharedSecretSize
+*/
+int wc_FrodoKemKey_CipherTextSize(FrodoKemKey* key, word32* len);
+
+/*!
+    \ingroup FRODO_KEM
+
+    \brief Returns, through len, the size in bytes of a shared secret
+    for the key's parameter set. Use this to size the ss buffer passed
+    to wc_FrodoKemKey_Encapsulate() and wc_FrodoKemKey_Decapsulate().
+
+    \return 0 on success.
+    \return BAD_FUNC_ARG if key or len is NULL.
+
+    \param [in] key An initialized FrodoKemKey.
+    \param [out] len Set to the shared secret size in bytes.
+
+    \sa wc_FrodoKemKey_CipherTextSize
+    \sa wc_FrodoKemKey_Encapsulate
+*/
+int wc_FrodoKemKey_SharedSecretSize(FrodoKemKey* key, word32* len);
+
+/*!
+    \ingroup FRODO_KEM
+
+    \brief Encapsulates to a public key: generates a random shared
+    secret and the ciphertext that encapsulates it under key's public
+    key. The key must hold a public key (from wc_FrodoKemKey_MakeKey()
+    or wc_FrodoKemKey_DecodePublicKey()). Size the ct and ss buffers with
+    wc_FrodoKemKey_CipherTextSize() and wc_FrodoKemKey_SharedSecretSize().
+
+    \return 0 on success.
+    \return BAD_FUNC_ARG if key, ct, ss or rng is NULL.
+    \return BAD_STATE_E if key does not hold a public key.
+    \return MEMORY_E on allocation failure.
+
+    \param [in] key A FrodoKemKey holding a public key.
+    \param [out] ct Buffer to receive the ciphertext.
+    \param [out] ss Buffer to receive the shared secret.
+    \param [in] rng An initialized random number generator.
+
+    _Example_
+    \code
+    FrodoKemKey key;
+    WC_RNG rng;
+    word32 ctSz, ssSz;
+    byte ct[WC_FRODOKEM_640_CIPHER_TEXT_SIZE];
+    byte ss[WC_FRODOKEM_640_SS_SIZE];
+    int ret;
+
+    // key holds a decoded/generated public key
+    wc_FrodoKemKey_CipherTextSize(&key, &ctSz);
+    wc_FrodoKemKey_SharedSecretSize(&key, &ssSz);
+
+    ret = wc_FrodoKemKey_Encapsulate(&key, ct, ss, &rng);
+    if (ret != 0) {
+        // error encapsulating
+    }
+    \endcode
+
+    \sa wc_FrodoKemKey_Decapsulate
+    \sa wc_FrodoKemKey_EncapsulateWithRandom
+    \sa wc_FrodoKemKey_CipherTextSize
+*/
+int wc_FrodoKemKey_Encapsulate(FrodoKemKey* key, unsigned char* ct,
+    unsigned char* ss, WC_RNG* rng);
+
+/*!
+    \ingroup FRODO_KEM
+
+    \brief Encapsulates using caller-supplied random bytes rather than
+    an RNG. This is primarily for testing and known answer tests;
+    production code should use wc_FrodoKemKey_Encapsulate(). The required
+    length depends on the parameter set (for example
+    WC_FRODOKEM_640_ENC_RAND_SZ for FrodoKEM-640).
+
+    \return 0 on success.
+    \return BAD_FUNC_ARG if key, ct, ss or rand is NULL, or len is not
+    the length required for the key's parameter set.
+    \return BAD_STATE_E if key does not hold a public key.
+
+    \param [in] key A FrodoKemKey holding a public key.
+    \param [out] ct Buffer to receive the ciphertext.
+    \param [out] ss Buffer to receive the shared secret.
+    \param [in] rand Buffer of random bytes.
+    \param [in] len Length of rand in bytes; must equal the parameter
+    set's WC_FRODOKEM_<set>_ENC_RAND_SZ.
+
+    \sa wc_FrodoKemKey_Encapsulate
+*/
+int wc_FrodoKemKey_EncapsulateWithRandom(FrodoKemKey* key,
+    unsigned char* ct, unsigned char* ss, const unsigned char* rand,
+    int len);
+
+/*!
+    \ingroup FRODO_KEM
+
+    \brief Decapsulates a ciphertext to recover the shared secret using
+    the key's private key. FrodoKEM is IND-CCA2 secure: a ciphertext
+    that does not decrypt consistently yields a pseudo-random shared
+    secret (implicit rejection) rather than an error, so decapsulation
+    of a valid ciphertext reproduces the shared secret from
+    encapsulation. Size the ss buffer with
+    wc_FrodoKemKey_SharedSecretSize().
+
+    \return 0 on success (a shared secret is always produced for a
+    correctly sized ciphertext).
+    \return BAD_FUNC_ARG if key, ss or ct is NULL, or len is not the
+    ciphertext size for the key's parameter set.
+    \return BAD_STATE_E if key does not hold a private key.
+    \return MEMORY_E on allocation failure.
+
+    \param [in] key A FrodoKemKey holding a private key.
+    \param [out] ss Buffer to receive the shared secret.
+    \param [in] ct The ciphertext to decapsulate.
+    \param [in] len Length of ct in bytes; must equal the ciphertext
+    size for the key's parameter set.
+
+    _Example_
+    \code
+    FrodoKemKey key;
+    word32 ssSz;
+    byte ss[WC_FRODOKEM_640_SS_SIZE];
+    int ret;
+
+    // key holds a decoded/generated private key
+    wc_FrodoKemKey_SharedSecretSize(&key, &ssSz);
+
+    ret = wc_FrodoKemKey_Decapsulate(&key, ss, ct, ctSz);
+    if (ret != 0) {
+        // error decapsulating
+    }
+    \endcode
+
+    \sa wc_FrodoKemKey_Encapsulate
+    \sa wc_FrodoKemKey_SharedSecretSize
+*/
+int wc_FrodoKemKey_Decapsulate(FrodoKemKey* key, unsigned char* ss,
+    const unsigned char* ct, word32 len);
+
+/*!
+    \ingroup FRODO_KEM
+
+    \brief Decodes (imports) a FrodoKEM private key from its serialized
+    form into an initialized key. The expected length matches
+    wc_FrodoKemKey_PrivateKeySize() for the key's parameter set.
+
+    \return 0 on success.
+    \return BAD_FUNC_ARG if key or in is NULL, or len is not the private
+    key size for the key's parameter set.
+
+    \param [in,out] key An initialized FrodoKemKey to hold the private
+    key.
+    \param [in] in The serialized private key.
+    \param [in] len Length of in in bytes.
+
+    \sa wc_FrodoKemKey_EncodePrivateKey
+    \sa wc_FrodoKemKey_PrivateKeySize
+    \sa wc_FrodoKemKey_Decapsulate
+*/
+int wc_FrodoKemKey_DecodePrivateKey(FrodoKemKey* key,
+    const unsigned char* in, word32 len);
+
+/*!
+    \ingroup FRODO_KEM
+
+    \brief Decodes (imports) a FrodoKEM public key from its serialized
+    form into an initialized key. The expected length matches
+    wc_FrodoKemKey_PublicKeySize() for the key's parameter set.
+
+    \return 0 on success.
+    \return BAD_FUNC_ARG if key or in is NULL, or len is not the public
+    key size for the key's parameter set.
+
+    \param [in,out] key An initialized FrodoKemKey to hold the public
+    key.
+    \param [in] in The serialized public key.
+    \param [in] len Length of in in bytes.
+
+    \sa wc_FrodoKemKey_EncodePublicKey
+    \sa wc_FrodoKemKey_PublicKeySize
+    \sa wc_FrodoKemKey_Encapsulate
+*/
+int wc_FrodoKemKey_DecodePublicKey(FrodoKemKey* key,
+    const unsigned char* in, word32 len);
+
+/*!
+    \ingroup FRODO_KEM
+
+    \brief Returns, through len, the size in bytes of a serialized
+    private key for the key's parameter set. Use this to size the buffer
+    passed to wc_FrodoKemKey_EncodePrivateKey().
+
+    \return 0 on success.
+    \return BAD_FUNC_ARG if key or len is NULL.
+
+    \param [in] key An initialized FrodoKemKey.
+    \param [out] len Set to the private key size in bytes.
+
+    \sa wc_FrodoKemKey_EncodePrivateKey
+    \sa wc_FrodoKemKey_DecodePrivateKey
+*/
+int wc_FrodoKemKey_PrivateKeySize(FrodoKemKey* key, word32* len);
+
+/*!
+    \ingroup FRODO_KEM
+
+    \brief Returns, through len, the size in bytes of a serialized
+    public key for the key's parameter set. Use this to size the buffer
+    passed to wc_FrodoKemKey_EncodePublicKey().
+
+    \return 0 on success.
+    \return BAD_FUNC_ARG if key or len is NULL.
+
+    \param [in] key An initialized FrodoKemKey.
+    \param [out] len Set to the public key size in bytes.
+
+    \sa wc_FrodoKemKey_EncodePublicKey
+    \sa wc_FrodoKemKey_DecodePublicKey
+*/
+int wc_FrodoKemKey_PublicKeySize(FrodoKemKey* key, word32* len);
+
+/*!
+    \ingroup FRODO_KEM
+
+    \brief Encodes (exports) the private key held by key into its
+    serialized form. The out buffer must be at least
+    wc_FrodoKemKey_PrivateKeySize() bytes.
+
+    \return 0 on success.
+    \return BAD_FUNC_ARG if key or out is NULL, or len is smaller than
+    the private key size for the key's parameter set.
+    \return BAD_STATE_E if key does not hold a private key.
+
+    \param [in] key A FrodoKemKey holding a private key.
+    \param [out] out Buffer to receive the serialized private key.
+    \param [in] len Length of the out buffer in bytes.
+
+    \sa wc_FrodoKemKey_DecodePrivateKey
+    \sa wc_FrodoKemKey_PrivateKeySize
+*/
+int wc_FrodoKemKey_EncodePrivateKey(FrodoKemKey* key,
+    unsigned char* out, word32 len);
+
+/*!
+    \ingroup FRODO_KEM
+
+    \brief Encodes (exports) the public key held by key into its
+    serialized form. The out buffer must be at least
+    wc_FrodoKemKey_PublicKeySize() bytes.
+
+    \return 0 on success.
+    \return BAD_FUNC_ARG if key or out is NULL, or len is smaller than
+    the public key size for the key's parameter set.
+    \return BAD_STATE_E if key does not hold a public key.
+
+    \param [in] key A FrodoKemKey holding a public key.
+    \param [out] out Buffer to receive the serialized public key.
+    \param [in] len Length of the out buffer in bytes.
+
+    \sa wc_FrodoKemKey_DecodePublicKey
+    \sa wc_FrodoKemKey_PublicKeySize
+*/
+int wc_FrodoKemKey_EncodePublicKey(FrodoKemKey* key,
+    unsigned char* out, word32 len);
diff --git a/src/include.am b/src/include.am
index 415141f66d..9623664e18 100644
--- a/src/include.am
+++ b/src/include.am
@@ -1830,6 +1830,43 @@ src_libwolfssl@LIBSUFFIX@_la_SOURCES += \
                wolfcrypt/src/logging.c \
                wolfcrypt/src/wc_port.c
 
+# FrodoKEM (experimental reference implementation)
+if BUILD_WC_FRODOKEM
+src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/wc_frodokem.c
+src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/wc_frodokem_mat.c
+if BUILD_INTELASM
+src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/wc_frodokem_asm.S
+endif BUILD_INTELASM
+# AArch64: NEON, plus the opt-in SVE and SME ops (guarded by WOLFSSL_FRODOKEM_SVE
+# / WOLFSSL_FRODOKEM_SME), all in the one armv8-frodokem-asm file.
+if BUILD_ARMASM_NEON
+if BUILD_ARMASM_INLINE
+src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/port/arm/armv8-frodokem-asm_c.c
+else
+src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/port/arm/armv8-frodokem-asm.S
+endif !BUILD_ARMASM_INLINE
+endif BUILD_ARMASM_NEON
+# AArch32: NEON and the 32-bit-SIMD (SMLAD/SADD16/...) NEON-absent fallback share
+# the one armv8-32-frodokem-asm file, built for ARM (non-Thumb) with or without
+# NEON. Thumb2 has its own file.
+if BUILD_ARMASM
+if BUILD_ARM_NONTHUMB
+if BUILD_ARMASM_INLINE
+src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/port/arm/armv8-32-frodokem-asm_c.c
+else
+src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/port/arm/armv8-32-frodokem-asm.S
+endif !BUILD_ARMASM_INLINE
+endif BUILD_ARM_NONTHUMB
+if BUILD_ARM_THUMB
+if BUILD_ARMASM_INLINE
+src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/port/arm/thumb2-frodokem-asm_c.c
+else
+src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/port/arm/thumb2-frodokem-asm.S
+endif !BUILD_ARMASM_INLINE
+endif BUILD_ARM_THUMB
+endif BUILD_ARMASM
+endif
+
 if BUILD_ERROR_STRINGS
 src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/error.c
 endif
diff --git a/tests/api.c b/tests/api.c
index eed600c040..299b3f3181 100644
--- a/tests/api.c
+++ b/tests/api.c
@@ -230,6 +230,7 @@
 #include <tests/api/test_curve448.h>
 #include <tests/api/test_ed448.h>
 #include <tests/api/test_mlkem.h>
+#include <tests/api/test_frodokem.h>
 #include <tests/api/test_mldsa.h>
 #include <tests/api/test_slhdsa.h>
 #include <tests/api/test_signature.h>
@@ -35795,6 +35796,7 @@ TEST_CASE testCases[] = {
     TEST_ED448_DECLS,
     /* Kyber */
     TEST_MLKEM_DECLS,
+    TEST_FRODOKEM_DECLS,
     /* Dilithium */
     TEST_MLDSA_DECLS,
     /* SLH-DSA */
diff --git a/tests/api/include.am b/tests/api/include.am
index fcb9720114..b3f46178e1 100644
--- a/tests/api/include.am
+++ b/tests/api/include.am
@@ -47,6 +47,7 @@ tests_unit_test_SOURCES += tests/api/test_ed25519.c
 tests_unit_test_SOURCES += tests/api/test_curve448.c
 tests_unit_test_SOURCES += tests/api/test_ed448.c
 tests_unit_test_SOURCES += tests/api/test_mlkem.c
+tests_unit_test_SOURCES += tests/api/test_frodokem.c
 tests_unit_test_SOURCES += tests/api/test_mldsa.c
 tests_unit_test_SOURCES += tests/api/test_mldsa_legacy.c
 tests_unit_test_SOURCES += tests/api/test_slhdsa.c
@@ -163,6 +164,7 @@ EXTRA_DIST += tests/api/test_ed25519.h
 EXTRA_DIST += tests/api/test_curve448.h
 EXTRA_DIST += tests/api/test_ed448.h
 EXTRA_DIST += tests/api/test_mlkem.h
+EXTRA_DIST += tests/api/test_frodokem.h
 EXTRA_DIST += tests/api/test_mldsa.h
 EXTRA_DIST += tests/api/test_slhdsa.h
 EXTRA_DIST += tests/api/test_signature.h
diff --git a/tests/api/test_frodokem.c b/tests/api/test_frodokem.c
new file mode 100644
index 0000000000..f02d64668a
--- /dev/null
+++ b/tests/api/test_frodokem.c
@@ -0,0 +1,1472 @@
+/* test_frodokem.c
+ *
+ * Copyright (C) 2006-2026 wolfSSL Inc.
+ *
+ * This file is part of wolfSSL.
+ *
+ * wolfSSL is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * wolfSSL is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335, USA
+ */
+
+#include <tests/unit.h>
+
+#ifdef NO_INLINE
+    #include <wolfssl/wolfcrypt/misc.h>
+#else
+    #define WOLFSSL_MISC_INCLUDED
+    #include <wolfcrypt/src/misc.c>
+#endif
+
+#ifdef WOLFSSL_HAVE_FRODOKEM
+    #include <wolfssl/wolfcrypt/wc_frodokem.h>
+    #include <wolfssl/wolfcrypt/hash.h>
+    #include <wolfssl/wolfcrypt/sha256.h>
+#endif
+#include <wolfssl/wolfcrypt/types.h>
+#include <tests/api/api.h>
+#include <tests/api/test_frodokem.h>
+
+#ifdef WOLFSSL_HAVE_FRODOKEM
+
+/* The set of compiled-in key types, used to drive the functional tests. */
+static const int frodokem_types[] = {
+#ifdef WOLFSSL_FRODOKEM_SHAKE
+#ifdef WOLFSSL_WC_FRODOKEM_640
+    WC_FRODOKEM_640_SHAKE,
+#endif
+#ifdef WOLFSSL_WC_FRODOKEM_976
+    WC_FRODOKEM_976_SHAKE,
+#endif
+#ifdef WOLFSSL_WC_FRODOKEM_1344
+    WC_FRODOKEM_1344_SHAKE,
+#endif
+#endif /* WOLFSSL_FRODOKEM_SHAKE */
+#ifdef WOLFSSL_FRODOKEM_AES
+#ifdef WOLFSSL_WC_FRODOKEM_640
+    WC_FRODOKEM_640_AES,
+#endif
+#ifdef WOLFSSL_WC_FRODOKEM_976
+    WC_FRODOKEM_976_AES,
+#endif
+#ifdef WOLFSSL_WC_FRODOKEM_1344
+    WC_FRODOKEM_1344_AES,
+#endif
+#endif /* WOLFSSL_FRODOKEM_AES */
+#ifdef WOLFSSL_FRODOKEM_EPHEMERAL
+#ifdef WOLFSSL_FRODOKEM_SHAKE
+#ifdef WOLFSSL_WC_FRODOKEM_640
+    WC_EFRODOKEM_640_SHAKE,
+#endif
+#ifdef WOLFSSL_WC_FRODOKEM_976
+    WC_EFRODOKEM_976_SHAKE,
+#endif
+#ifdef WOLFSSL_WC_FRODOKEM_1344
+    WC_EFRODOKEM_1344_SHAKE,
+#endif
+#endif /* WOLFSSL_FRODOKEM_SHAKE */
+#ifdef WOLFSSL_FRODOKEM_AES
+#ifdef WOLFSSL_WC_FRODOKEM_640
+    WC_EFRODOKEM_640_AES,
+#endif
+#ifdef WOLFSSL_WC_FRODOKEM_976
+    WC_EFRODOKEM_976_AES,
+#endif
+#ifdef WOLFSSL_WC_FRODOKEM_1344
+    WC_EFRODOKEM_1344_AES,
+#endif
+#endif /* WOLFSSL_FRODOKEM_AES */
+#endif /* WOLFSSL_FRODOKEM_EPHEMERAL */
+    0 /* sentinel so the array is never empty */
+};
+
+#define FRODOKEM_TYPE_CNT \
+    ((int)(sizeof(frodokem_types) / sizeof(frodokem_types[0])) - 1)
+
+/* The KAT data is only used by the make-key/encapsulate/decapsulate KAT tests,
+ * all of which need key generation to reconstruct the key. */
+#if !defined(NO_SHA256) && !defined(WOLFSSL_FRODOKEM_NO_MAKE_KEY)
+
+/* Known-answer test data derived from the official FrodoKEM and eFrodoKEM
+ * KAT vectors (PQCkemKAT_*.rsp, count 0), for both the SHAKE and AES matrix A
+ * generation variants. To keep the data compact the public key, secret key and
+ * ciphertext are stored as SHA-256 digests; the shared secret is stored in
+ * full. The seed randomness (kr = s || seedSE || z for key generation, er =
+ * u || salt for encapsulation) is what the NIST AES-CTR DRBG produces for KAT
+ * count 0. */
+
+#if defined(WOLFSSL_WC_FRODOKEM_640) && defined(WOLFSSL_FRODOKEM_SHAKE)
+static const byte frodokem_kr_640_SHAKE[] = {
+    0x7c,0x99,0x35,0xa0,0xb0,0x76,0x94,0xaa,0x0c,0x6d,0x10,0xe4,
+    0xdb,0x6b,0x1a,0xdd,0x2f,0xd8,0x1a,0x25,0xcc,0xb1,0x48,0x03,
+    0x2d,0xcd,0x73,0x99,0x36,0x73,0x7f,0x2d,0xb5,0x05,0xd7,0xcf,
+    0xad,0x1b,0x49,0x74,0x99,0x32,0x3c,0x86,0x86,0x32,0x5e,0x47,
+    0x92,0xf2,0x67,0xaa,0xfa,0x3f,0x87,0xca,0x60,0xd0,0x1c,0xb5,
+    0x4f,0x29,0x20,0x2a,
+};
+static const byte frodokem_er_640_SHAKE[] = {
+    0xeb,0x4a,0x7c,0x66,0xef,0x4e,0xba,0x2d,0xdb,0x38,0xc8,0x8d,
+    0x8b,0xc7,0x06,0xb1,0xd6,0x39,0x00,0x21,0x98,0x17,0x2a,0x7b,
+    0x19,0x42,0xec,0xa8,0xf6,0xc0,0x01,0xba,0x26,0x20,0x2b,0xee,
+    0x59,0xac,0x27,0x54,0x84,0xea,0x76,0x7d,0x41,0xd8,0xd3,0x57,
+};
+static const byte frodokem_hpk_640_SHAKE[] = {
+    0x10,0xe6,0x3e,0xfe,0x34,0x0a,0x73,0xd4,0x6d,0x78,0xf7,0x68,
+    0xcf,0xea,0x23,0x5d,0x0d,0x7d,0xa1,0xe9,0xc6,0x36,0xd6,0xed,
+    0xc3,0x2d,0x2a,0x4e,0xd4,0xb1,0x3c,0xdc,
+};
+static const byte frodokem_hsk_640_SHAKE[] = {
+    0x23,0x3a,0x52,0xe7,0x3b,0xf5,0xf1,0x6d,0xaa,0xd0,0x03,0xdd,
+    0x15,0xce,0xa2,0x8e,0x30,0xdb,0xe6,0x42,0x61,0x58,0xbe,0x48,
+    0x67,0x95,0x6a,0xff,0x03,0xd1,0x26,0x91,
+};
+static const byte frodokem_hct_640_SHAKE[] = {
+    0x27,0x11,0xc7,0xe8,0x63,0xc9,0xd8,0x1c,0x08,0x18,0x2b,0x3e,
+    0xe1,0xe1,0xc9,0x53,0xcb,0xff,0xb1,0x5d,0xd5,0x0b,0xff,0x88,
+    0x25,0xc6,0xc2,0xa7,0x1a,0x4f,0xd0,0x1d,
+};
+static const byte frodokem_ss_640_SHAKE[] = {
+    0x2e,0xd4,0x2c,0xe7,0xd5,0xdb,0xfb,0x11,0x5f,0x2e,0x2b,0xdc,
+    0xb6,0x50,0xb3,0xfa,
+};
+#define FKAT_640_SHAKE \
+    { WC_FRODOKEM_640_SHAKE, \
+      frodokem_kr_640_SHAKE, \
+      (word32)sizeof(frodokem_kr_640_SHAKE), \
+      frodokem_er_640_SHAKE, \
+      (word32)sizeof(frodokem_er_640_SHAKE), \
+      frodokem_hpk_640_SHAKE, \
+      frodokem_hsk_640_SHAKE, \
+      frodokem_hct_640_SHAKE, \
+      frodokem_ss_640_SHAKE, \
+      (word32)sizeof(frodokem_ss_640_SHAKE) },
+#else
+#define FKAT_640_SHAKE
+#endif
+
+#if defined(WOLFSSL_WC_FRODOKEM_976) && defined(WOLFSSL_FRODOKEM_SHAKE)
+static const byte frodokem_kr_976_SHAKE[] = {
+    0x7c,0x99,0x35,0xa0,0xb0,0x76,0x94,0xaa,0x0c,0x6d,0x10,0xe4,
+    0xdb,0x6b,0x1a,0xdd,0x2f,0xd8,0x1a,0x25,0xcc,0xb1,0x48,0x03,
+    0x2d,0xcd,0x73,0x99,0x36,0x73,0x7f,0x2d,0xb5,0x05,0xd7,0xcf,
+    0xad,0x1b,0x49,0x74,0x99,0x32,0x3c,0x86,0x86,0x32,0x5e,0x47,
+    0x92,0xf2,0x67,0xaa,0xfa,0x3f,0x87,0xca,0x60,0xd0,0x1c,0xb5,
+    0x4f,0x29,0x20,0x2a,0x3e,0x78,0x4c,0xcb,0x7e,0xbc,0xdc,0xfd,
+    0x45,0x54,0x2b,0x7f,0x6a,0xf7,0x78,0x74,0x2e,0x0f,0x44,0x79,
+    0x17,0x50,0x84,0xaa,
+};
+static const byte frodokem_er_976_SHAKE[] = {
+    0xee,0x71,0x67,0x62,0xc1,0x5e,0x3b,0x72,0xaa,0x76,0x50,0xa6,
+    0x3b,0x9a,0x51,0x00,0x40,0xb0,0x3c,0x0f,0xe7,0x04,0x75,0xc0,
+    0x46,0x3b,0xbc,0x45,0xa0,0xba,0x5b,0x79,0x80,0xdd,0x46,0xee,
+    0xf8,0x2f,0xb0,0x62,0x03,0x50,0x77,0xd0,0x42,0xf3,0x06,0xbb,
+    0x63,0x91,0x04,0x0e,0x0d,0xd9,0x65,0xf1,0xfd,0xa9,0xd1,0x83,
+    0xca,0x9f,0xcc,0xb4,0x8f,0xc0,0x10,0xb1,0x84,0xab,0x00,0x33,
+};
+static const byte frodokem_hpk_976_SHAKE[] = {
+    0x75,0xed,0x58,0xdd,0x9c,0xb2,0x50,0x1a,0xf1,0x3e,0x56,0xbb,
+    0xd8,0x57,0x9b,0x73,0x17,0x99,0x8d,0x82,0x8c,0xa2,0x93,0xe3,
+    0x98,0x09,0xf4,0x00,0x44,0x04,0xe7,0x89,
+};
+static const byte frodokem_hsk_976_SHAKE[] = {
+    0x2e,0x79,0x47,0x64,0xc6,0xa3,0xe7,0xc1,0xa1,0x6f,0x00,0xa3,
+    0xc3,0x45,0x59,0x1f,0xea,0x97,0x7c,0x21,0x3c,0x87,0x32,0xad,
+    0x37,0x1a,0xb9,0x60,0x3e,0x5d,0x63,0xa9,
+};
+static const byte frodokem_hct_976_SHAKE[] = {
+    0x6e,0x35,0x8f,0x98,0xcf,0x02,0xec,0xbf,0x38,0xb8,0xdf,0x50,
+    0x61,0x49,0x1d,0xec,0x1e,0xae,0x05,0x2e,0xa6,0x80,0x65,0x61,
+    0xc3,0xa4,0xaa,0x5e,0xd2,0x15,0xf7,0xa3,
+};
+static const byte frodokem_ss_976_SHAKE[] = {
+    0x5b,0x6e,0x5a,0x69,0xa3,0xd5,0xf8,0xe7,0x5e,0xea,0x3a,0x6e,
+    0x95,0x59,0x5e,0xd0,0x27,0x8d,0xa5,0x5b,0x8b,0x37,0x31,0x42,
+};
+#define FKAT_976_SHAKE \
+    { WC_FRODOKEM_976_SHAKE, \
+      frodokem_kr_976_SHAKE, \
+      (word32)sizeof(frodokem_kr_976_SHAKE), \
+      frodokem_er_976_SHAKE, \
+      (word32)sizeof(frodokem_er_976_SHAKE), \
+      frodokem_hpk_976_SHAKE, \
+      frodokem_hsk_976_SHAKE, \
+      frodokem_hct_976_SHAKE, \
+      frodokem_ss_976_SHAKE, \
+      (word32)sizeof(frodokem_ss_976_SHAKE) },
+#else
+#define FKAT_976_SHAKE
+#endif
+
+#if defined(WOLFSSL_WC_FRODOKEM_1344) && defined(WOLFSSL_FRODOKEM_SHAKE)
+static const byte frodokem_kr_1344_SHAKE[] = {
+    0x7c,0x99,0x35,0xa0,0xb0,0x76,0x94,0xaa,0x0c,0x6d,0x10,0xe4,
+    0xdb,0x6b,0x1a,0xdd,0x2f,0xd8,0x1a,0x25,0xcc,0xb1,0x48,0x03,
+    0x2d,0xcd,0x73,0x99,0x36,0x73,0x7f,0x2d,0xb5,0x05,0xd7,0xcf,
+    0xad,0x1b,0x49,0x74,0x99,0x32,0x3c,0x86,0x86,0x32,0x5e,0x47,
+    0x92,0xf2,0x67,0xaa,0xfa,0x3f,0x87,0xca,0x60,0xd0,0x1c,0xb5,
+    0x4f,0x29,0x20,0x2a,0x3e,0x78,0x4c,0xcb,0x7e,0xbc,0xdc,0xfd,
+    0x45,0x54,0x2b,0x7f,0x6a,0xf7,0x78,0x74,0x2e,0x0f,0x44,0x79,
+    0x17,0x50,0x84,0xaa,0x48,0x8b,0x3b,0x74,0x34,0x06,0x78,0xaa,
+    0x38,0xe2,0x2e,0x96,0x28,0xb0,0xa1,0x61,0xfd,0xeb,0x0b,0xd2,
+    0x52,0x17,0x3b,0x9c,
+};
+static const byte frodokem_er_1344_SHAKE[] = {
+    0x9f,0x08,0x58,0x76,0x87,0xff,0x66,0x76,0x5c,0x67,0x1d,0xe7,
+    0x3e,0x91,0x8d,0x28,0x23,0xca,0x57,0x3f,0xf4,0xe7,0xa3,0x1a,
+    0x91,0x60,0x32,0x40,0x26,0xe5,0x40,0xea,0xcb,0x3a,0x04,0xe0,
+    0xd5,0x4c,0x75,0xde,0xb9,0x70,0x5b,0xfd,0xfb,0xdf,0x93,0x5a,
+    0x75,0x28,0x80,0x2e,0xe6,0xe5,0xb0,0xc6,0xa7,0x3b,0x2b,0x76,
+    0x1d,0x9b,0xd0,0x84,0x8a,0x6e,0x4c,0xf3,0xfc,0x4c,0xa8,0x4f,
+    0x14,0xe0,0x33,0x1a,0xf3,0x5b,0xfe,0xf4,0x1e,0x42,0xb1,0x3a,
+    0x6d,0xae,0x6d,0xf9,0x37,0xf7,0x38,0xc1,0x85,0x7b,0xa1,0xca,
+};
+static const byte frodokem_hpk_1344_SHAKE[] = {
+    0x97,0x4d,0x55,0x14,0xfb,0x13,0x11,0x4b,0xb6,0x67,0x52,0x7e,
+    0x84,0x09,0x0b,0x6d,0xe0,0xdf,0xee,0xa0,0x14,0x35,0xc6,0x0a,
+    0x3e,0xdc,0x57,0x0d,0x33,0x1a,0x5b,0xd6,
+};
+static const byte frodokem_hsk_1344_SHAKE[] = {
+    0x02,0xd4,0xf7,0x10,0x96,0xb3,0x6f,0x46,0xa9,0x03,0xf7,0x7f,
+    0xcc,0x9b,0x77,0x64,0x36,0x6d,0x9a,0x8f,0x84,0xf0,0x6c,0x9f,
+    0x86,0xc4,0x64,0xcf,0x69,0x52,0x6d,0x8f,
+};
+static const byte frodokem_hct_1344_SHAKE[] = {
+    0xaa,0x07,0x79,0x97,0x59,0xe8,0x7a,0xd1,0xd0,0x80,0xb2,0xd1,
+    0xeb,0x31,0xd1,0x1e,0x3e,0xf9,0x3e,0x66,0x5d,0xf9,0xa8,0xba,
+    0xe5,0x02,0x7f,0x4e,0x40,0xfc,0x7a,0x46,
+};
+static const byte frodokem_ss_1344_SHAKE[] = {
+    0x8d,0x20,0xf9,0x71,0x46,0x4d,0xf1,0x9e,0x05,0x61,0xbd,0xd3,
+    0x85,0xaf,0xd0,0xe2,0xef,0x0c,0xe2,0x12,0xef,0xd4,0x5a,0x63,
+    0x2f,0x5d,0x2c,0x64,0xf3,0xd6,0x6a,0xac,
+};
+#define FKAT_1344_SHAKE \
+    { WC_FRODOKEM_1344_SHAKE, \
+      frodokem_kr_1344_SHAKE, \
+      (word32)sizeof(frodokem_kr_1344_SHAKE), \
+      frodokem_er_1344_SHAKE, \
+      (word32)sizeof(frodokem_er_1344_SHAKE), \
+      frodokem_hpk_1344_SHAKE, \
+      frodokem_hsk_1344_SHAKE, \
+      frodokem_hct_1344_SHAKE, \
+      frodokem_ss_1344_SHAKE, \
+      (word32)sizeof(frodokem_ss_1344_SHAKE) },
+#else
+#define FKAT_1344_SHAKE
+#endif
+
+#if defined(WOLFSSL_WC_FRODOKEM_640) && defined(WOLFSSL_FRODOKEM_AES)
+static const byte frodokem_kr_640_AES[] = {
+    0x7c,0x99,0x35,0xa0,0xb0,0x76,0x94,0xaa,0x0c,0x6d,0x10,0xe4,
+    0xdb,0x6b,0x1a,0xdd,0x2f,0xd8,0x1a,0x25,0xcc,0xb1,0x48,0x03,
+    0x2d,0xcd,0x73,0x99,0x36,0x73,0x7f,0x2d,0xb5,0x05,0xd7,0xcf,
+    0xad,0x1b,0x49,0x74,0x99,0x32,0x3c,0x86,0x86,0x32,0x5e,0x47,
+    0x92,0xf2,0x67,0xaa,0xfa,0x3f,0x87,0xca,0x60,0xd0,0x1c,0xb5,
+    0x4f,0x29,0x20,0x2a,
+};
+static const byte frodokem_er_640_AES[] = {
+    0xeb,0x4a,0x7c,0x66,0xef,0x4e,0xba,0x2d,0xdb,0x38,0xc8,0x8d,
+    0x8b,0xc7,0x06,0xb1,0xd6,0x39,0x00,0x21,0x98,0x17,0x2a,0x7b,
+    0x19,0x42,0xec,0xa8,0xf6,0xc0,0x01,0xba,0x26,0x20,0x2b,0xee,
+    0x59,0xac,0x27,0x54,0x84,0xea,0x76,0x7d,0x41,0xd8,0xd3,0x57,
+};
+static const byte frodokem_hpk_640_AES[] = {
+    0x44,0x19,0xf4,0xf1,0x4c,0x33,0x63,0x55,0x11,0x86,0xb9,0x93,
+    0x82,0xbf,0x3e,0x62,0xac,0x23,0x82,0xb7,0xb5,0xd4,0xca,0x70,
+    0x39,0x2d,0xda,0x58,0x69,0xc8,0xbd,0xa1,
+};
+static const byte frodokem_hsk_640_AES[] = {
+    0x1e,0xfb,0x89,0xe5,0xfc,0x52,0xd2,0xbb,0xab,0x4a,0xdb,0xbb,
+    0x65,0xda,0xfb,0x49,0x73,0xd8,0xbf,0x09,0x7f,0x86,0x7e,0xf8,
+    0x6d,0xbe,0xa2,0xfd,0x51,0xe5,0xe3,0xea,
+};
+static const byte frodokem_hct_640_AES[] = {
+    0x56,0xd2,0xbd,0xd0,0xc6,0xe9,0x2b,0xc3,0x65,0xf2,0x86,0xbe,
+    0x75,0x7e,0x46,0xa0,0xe0,0x40,0xc8,0x75,0xef,0x50,0x20,0xbb,
+    0xbf,0xab,0xee,0x00,0x28,0x52,0xe7,0xe4,
+};
+static const byte frodokem_ss_640_AES[] = {
+    0xee,0x5b,0xa8,0xce,0xbb,0x0b,0x41,0xe9,0x03,0x0c,0xa1,0xfb,
+    0xc3,0xbe,0xad,0xb9,
+};
+#define FKAT_640_AES \
+    { WC_FRODOKEM_640_AES, \
+      frodokem_kr_640_AES, \
+      (word32)sizeof(frodokem_kr_640_AES), \
+      frodokem_er_640_AES, \
+      (word32)sizeof(frodokem_er_640_AES), \
+      frodokem_hpk_640_AES, \
+      frodokem_hsk_640_AES, \
+      frodokem_hct_640_AES, \
+      frodokem_ss_640_AES, \
+      (word32)sizeof(frodokem_ss_640_AES) },
+#else
+#define FKAT_640_AES
+#endif
+
+#if defined(WOLFSSL_WC_FRODOKEM_976) && defined(WOLFSSL_FRODOKEM_AES)
+static const byte frodokem_kr_976_AES[] = {
+    0x7c,0x99,0x35,0xa0,0xb0,0x76,0x94,0xaa,0x0c,0x6d,0x10,0xe4,
+    0xdb,0x6b,0x1a,0xdd,0x2f,0xd8,0x1a,0x25,0xcc,0xb1,0x48,0x03,
+    0x2d,0xcd,0x73,0x99,0x36,0x73,0x7f,0x2d,0xb5,0x05,0xd7,0xcf,
+    0xad,0x1b,0x49,0x74,0x99,0x32,0x3c,0x86,0x86,0x32,0x5e,0x47,
+    0x92,0xf2,0x67,0xaa,0xfa,0x3f,0x87,0xca,0x60,0xd0,0x1c,0xb5,
+    0x4f,0x29,0x20,0x2a,0x3e,0x78,0x4c,0xcb,0x7e,0xbc,0xdc,0xfd,
+    0x45,0x54,0x2b,0x7f,0x6a,0xf7,0x78,0x74,0x2e,0x0f,0x44,0x79,
+    0x17,0x50,0x84,0xaa,
+};
+static const byte frodokem_er_976_AES[] = {
+    0xee,0x71,0x67,0x62,0xc1,0x5e,0x3b,0x72,0xaa,0x76,0x50,0xa6,
+    0x3b,0x9a,0x51,0x00,0x40,0xb0,0x3c,0x0f,0xe7,0x04,0x75,0xc0,
+    0x46,0x3b,0xbc,0x45,0xa0,0xba,0x5b,0x79,0x80,0xdd,0x46,0xee,
+    0xf8,0x2f,0xb0,0x62,0x03,0x50,0x77,0xd0,0x42,0xf3,0x06,0xbb,
+    0x63,0x91,0x04,0x0e,0x0d,0xd9,0x65,0xf1,0xfd,0xa9,0xd1,0x83,
+    0xca,0x9f,0xcc,0xb4,0x8f,0xc0,0x10,0xb1,0x84,0xab,0x00,0x33,
+};
+static const byte frodokem_hpk_976_AES[] = {
+    0x47,0x7d,0x38,0x49,0x0e,0x9c,0xf9,0x67,0xdd,0x03,0x8a,0xf6,
+    0xb9,0x34,0xd0,0x75,0x12,0x7c,0x0d,0x05,0x15,0x9d,0xe0,0x4f,
+    0xc9,0x37,0xb6,0x16,0x5b,0x3d,0xf2,0x1d,
+};
+static const byte frodokem_hsk_976_AES[] = {
+    0x01,0x6e,0xbc,0x6e,0x61,0xcd,0xcf,0x75,0xac,0xf2,0x17,0xe3,
+    0x7b,0xc9,0x2c,0xb5,0x57,0x7f,0x61,0xd1,0x4f,0x2c,0x83,0x8b,
+    0xd1,0xa2,0x67,0x5a,0x4d,0x34,0x26,0xe8,
+};
+static const byte frodokem_hct_976_AES[] = {
+    0x64,0x1f,0xa3,0x00,0x89,0xd3,0x97,0x82,0x8c,0xa9,0xd1,0xc0,
+    0xf2,0x80,0x7b,0xeb,0xe8,0x70,0xa3,0xbd,0xc7,0xdc,0x5b,0xa8,
+    0x86,0x6c,0x54,0x3e,0xbf,0x47,0xbc,0x98,
+};
+static const byte frodokem_ss_976_AES[] = {
+    0xd9,0x38,0x8d,0x1b,0x0f,0x97,0xc5,0xdb,0x5f,0x9b,0x4a,0x7e,
+    0x99,0x42,0x7c,0xfe,0x90,0xe3,0xe1,0x0c,0x5c,0x56,0x9d,0x02,
+};
+#define FKAT_976_AES \
+    { WC_FRODOKEM_976_AES, \
+      frodokem_kr_976_AES, \
+      (word32)sizeof(frodokem_kr_976_AES), \
+      frodokem_er_976_AES, \
+      (word32)sizeof(frodokem_er_976_AES), \
+      frodokem_hpk_976_AES, \
+      frodokem_hsk_976_AES, \
+      frodokem_hct_976_AES, \
+      frodokem_ss_976_AES, \
+      (word32)sizeof(frodokem_ss_976_AES) },
+#else
+#define FKAT_976_AES
+#endif
+
+#if defined(WOLFSSL_WC_FRODOKEM_1344) && defined(WOLFSSL_FRODOKEM_AES)
+static const byte frodokem_kr_1344_AES[] = {
+    0x7c,0x99,0x35,0xa0,0xb0,0x76,0x94,0xaa,0x0c,0x6d,0x10,0xe4,
+    0xdb,0x6b,0x1a,0xdd,0x2f,0xd8,0x1a,0x25,0xcc,0xb1,0x48,0x03,
+    0x2d,0xcd,0x73,0x99,0x36,0x73,0x7f,0x2d,0xb5,0x05,0xd7,0xcf,
+    0xad,0x1b,0x49,0x74,0x99,0x32,0x3c,0x86,0x86,0x32,0x5e,0x47,
+    0x92,0xf2,0x67,0xaa,0xfa,0x3f,0x87,0xca,0x60,0xd0,0x1c,0xb5,
+    0x4f,0x29,0x20,0x2a,0x3e,0x78,0x4c,0xcb,0x7e,0xbc,0xdc,0xfd,
+    0x45,0x54,0x2b,0x7f,0x6a,0xf7,0x78,0x74,0x2e,0x0f,0x44,0x79,
+    0x17,0x50,0x84,0xaa,0x48,0x8b,0x3b,0x74,0x34,0x06,0x78,0xaa,
+    0x38,0xe2,0x2e,0x96,0x28,0xb0,0xa1,0x61,0xfd,0xeb,0x0b,0xd2,
+    0x52,0x17,0x3b,0x9c,
+};
+static const byte frodokem_er_1344_AES[] = {
+    0x9f,0x08,0x58,0x76,0x87,0xff,0x66,0x76,0x5c,0x67,0x1d,0xe7,
+    0x3e,0x91,0x8d,0x28,0x23,0xca,0x57,0x3f,0xf4,0xe7,0xa3,0x1a,
+    0x91,0x60,0x32,0x40,0x26,0xe5,0x40,0xea,0xcb,0x3a,0x04,0xe0,
+    0xd5,0x4c,0x75,0xde,0xb9,0x70,0x5b,0xfd,0xfb,0xdf,0x93,0x5a,
+    0x75,0x28,0x80,0x2e,0xe6,0xe5,0xb0,0xc6,0xa7,0x3b,0x2b,0x76,
+    0x1d,0x9b,0xd0,0x84,0x8a,0x6e,0x4c,0xf3,0xfc,0x4c,0xa8,0x4f,
+    0x14,0xe0,0x33,0x1a,0xf3,0x5b,0xfe,0xf4,0x1e,0x42,0xb1,0x3a,
+    0x6d,0xae,0x6d,0xf9,0x37,0xf7,0x38,0xc1,0x85,0x7b,0xa1,0xca,
+};
+static const byte frodokem_hpk_1344_AES[] = {
+    0xe2,0x83,0xbf,0xd5,0x9b,0xcc,0xa4,0xda,0x53,0x80,0xbc,0xa9,
+    0xe4,0x3c,0x0d,0xfc,0xc3,0xb3,0xb2,0xb4,0xd3,0xf3,0xa0,0x2c,
+    0x64,0x2e,0x0d,0x43,0x92,0x03,0xee,0x81,
+};
+static const byte frodokem_hsk_1344_AES[] = {
+    0xee,0x4a,0x33,0x60,0x94,0x5b,0x71,0x8f,0xb1,0x04,0x74,0xe9,
+    0xc6,0x64,0xe2,0xc3,0x9b,0xd7,0x7d,0xfd,0x6b,0xfe,0xba,0x73,
+    0x3c,0x10,0xd2,0x69,0x50,0xf4,0x62,0xce,
+};
+static const byte frodokem_hct_1344_AES[] = {
+    0x4a,0x37,0x7d,0xba,0x12,0xe3,0x78,0xeb,0x9a,0x82,0x32,0x02,
+    0x81,0xc9,0xe8,0x61,0x08,0xd8,0x31,0xb2,0xde,0xba,0xe1,0xe1,
+    0x9c,0xc5,0x19,0x5f,0xe8,0x6d,0x73,0x20,
+};
+static const byte frodokem_ss_1344_AES[] = {
+    0x37,0x69,0x55,0x16,0x12,0x73,0xfc,0x66,0x7f,0x3f,0xea,0xe5,
+    0xec,0x98,0x68,0x18,0x20,0xdb,0xd7,0x59,0x97,0x1b,0xb0,0xa2,
+    0xd2,0xbe,0xc4,0x51,0x0f,0x55,0x7e,0x83,
+};
+#define FKAT_1344_AES \
+    { WC_FRODOKEM_1344_AES, \
+      frodokem_kr_1344_AES, \
+      (word32)sizeof(frodokem_kr_1344_AES), \
+      frodokem_er_1344_AES, \
+      (word32)sizeof(frodokem_er_1344_AES), \
+      frodokem_hpk_1344_AES, \
+      frodokem_hsk_1344_AES, \
+      frodokem_hct_1344_AES, \
+      frodokem_ss_1344_AES, \
+      (word32)sizeof(frodokem_ss_1344_AES) },
+#else
+#define FKAT_1344_AES
+#endif
+
+#if defined(WOLFSSL_WC_FRODOKEM_640) && defined(WOLFSSL_FRODOKEM_SHAKE) && \
+    defined(WOLFSSL_FRODOKEM_EPHEMERAL)
+static const byte frodokem_kr_E640_SHAKE[] = {
+    0x7c,0x99,0x35,0xa0,0xb0,0x76,0x94,0xaa,0x0c,0x6d,0x10,0xe4,
+    0xdb,0x6b,0x1a,0xdd,0x2f,0xd8,0x1a,0x25,0xcc,0xb1,0x48,0x03,
+    0x2d,0xcd,0x73,0x99,0x36,0x73,0x7f,0x2d,0xb5,0x05,0xd7,0xcf,
+    0xad,0x1b,0x49,0x74,0x99,0x32,0x3c,0x86,0x86,0x32,0x5e,0x47,
+};
+static const byte frodokem_er_E640_SHAKE[] = {
+    0x33,0xb3,0xc0,0x75,0x07,0xe4,0x20,0x17,0x48,0x49,0x4d,0x83,
+    0x2b,0x6e,0xe2,0xa6,
+};
+static const byte frodokem_hpk_E640_SHAKE[] = {
+    0x49,0xa3,0x03,0x46,0x0f,0x09,0x77,0x33,0x9c,0xbd,0x2c,0x1b,
+    0x5a,0xa0,0xe7,0xaf,0x44,0xc5,0x02,0x49,0x17,0xf5,0xf7,0xac,
+    0xed,0x47,0x25,0xe9,0xd4,0xa2,0xc6,0x14,
+};
+static const byte frodokem_hsk_E640_SHAKE[] = {
+    0x85,0xd0,0xe3,0x16,0x3f,0x8c,0xae,0xbe,0xe1,0xb5,0x48,0x37,
+    0xa4,0x70,0xe6,0xb0,0x49,0x83,0x78,0x4e,0x3c,0x3d,0x2b,0x2f,
+    0x03,0x6b,0x7f,0x95,0x87,0x55,0x08,0xaa,
+};
+static const byte frodokem_hct_E640_SHAKE[] = {
+    0x88,0xba,0x53,0x7b,0x53,0x6d,0x89,0xfd,0x0b,0x84,0xf8,0x60,
+    0x4a,0x60,0x40,0xa9,0xdd,0x96,0x11,0x4e,0xe6,0xad,0x43,0x23,
+    0x5c,0x95,0x1e,0xdf,0x42,0x6c,0x68,0x0d,
+};
+static const byte frodokem_ss_E640_SHAKE[] = {
+    0x72,0x97,0x80,0xfc,0x51,0x65,0x7e,0x21,0x35,0x7f,0x03,0xa3,
+    0x38,0x11,0x65,0x69,
+};
+#define FKAT_E640_SHAKE \
+    { WC_EFRODOKEM_640_SHAKE, \
+      frodokem_kr_E640_SHAKE, \
+      (word32)sizeof(frodokem_kr_E640_SHAKE), \
+      frodokem_er_E640_SHAKE, \
+      (word32)sizeof(frodokem_er_E640_SHAKE), \
+      frodokem_hpk_E640_SHAKE, \
+      frodokem_hsk_E640_SHAKE, \
+      frodokem_hct_E640_SHAKE, \
+      frodokem_ss_E640_SHAKE, \
+      (word32)sizeof(frodokem_ss_E640_SHAKE) },
+#else
+#define FKAT_E640_SHAKE
+#endif
+
+#if defined(WOLFSSL_WC_FRODOKEM_976) && defined(WOLFSSL_FRODOKEM_SHAKE) && \
+    defined(WOLFSSL_FRODOKEM_EPHEMERAL)
+static const byte frodokem_kr_E976_SHAKE[] = {
+    0x7c,0x99,0x35,0xa0,0xb0,0x76,0x94,0xaa,0x0c,0x6d,0x10,0xe4,
+    0xdb,0x6b,0x1a,0xdd,0x2f,0xd8,0x1a,0x25,0xcc,0xb1,0x48,0x03,
+    0x2d,0xcd,0x73,0x99,0x36,0x73,0x7f,0x2d,0xb5,0x05,0xd7,0xcf,
+    0xad,0x1b,0x49,0x74,0x99,0x32,0x3c,0x86,0x86,0x32,0x5e,0x47,
+    0x92,0xf2,0x67,0xaa,0xfa,0x3f,0x87,0xca,0x60,0xd0,0x1c,0xb5,
+    0x4f,0x29,0x20,0x2a,
+};
+static const byte frodokem_er_E976_SHAKE[] = {
+    0xeb,0x4a,0x7c,0x66,0xef,0x4e,0xba,0x2d,0xdb,0x38,0xc8,0x8d,
+    0x8b,0xc7,0x06,0xb1,0xd6,0x39,0x00,0x21,0x98,0x17,0x2a,0x7b,
+};
+static const byte frodokem_hpk_E976_SHAKE[] = {
+    0x94,0x9c,0xe7,0x6a,0xae,0x38,0xde,0xcd,0x41,0xe2,0x32,0x06,
+    0x23,0x62,0xad,0xb8,0x92,0x46,0xba,0xce,0xfe,0x22,0x1f,0x80,
+    0x32,0x5d,0x45,0x49,0x02,0x63,0x1c,0xe4,
+};
+static const byte frodokem_hsk_E976_SHAKE[] = {
+    0x3d,0x5a,0x3a,0x15,0x0f,0xee,0xd6,0x4a,0x86,0x1f,0x7e,0x87,
+    0xd4,0xee,0x83,0xb1,0x35,0xc8,0xb1,0xbe,0x43,0x9e,0x14,0x21,
+    0xc4,0x19,0xbf,0x86,0xac,0x15,0xdc,0x50,
+};
+static const byte frodokem_hct_E976_SHAKE[] = {
+    0xdf,0x84,0x22,0xbf,0xda,0xe4,0xe3,0x13,0x55,0x22,0x54,0xd4,
+    0xc2,0x91,0xeb,0x17,0xf0,0xc6,0x0d,0x5c,0x46,0x01,0x3a,0x26,
+    0x79,0x21,0x45,0xa4,0xc4,0x6f,0x4b,0x0e,
+};
+static const byte frodokem_ss_E976_SHAKE[] = {
+    0xa9,0x81,0x65,0x53,0x9a,0x4a,0xad,0x97,0x90,0x23,0xd6,0x7b,
+    0x43,0x5d,0x31,0x6f,0x00,0x7c,0x86,0xee,0xaf,0xdb,0x63,0xc7,
+};
+#define FKAT_E976_SHAKE \
+    { WC_EFRODOKEM_976_SHAKE, \
+      frodokem_kr_E976_SHAKE, \
+      (word32)sizeof(frodokem_kr_E976_SHAKE), \
+      frodokem_er_E976_SHAKE, \
+      (word32)sizeof(frodokem_er_E976_SHAKE), \
+      frodokem_hpk_E976_SHAKE, \
+      frodokem_hsk_E976_SHAKE, \
+      frodokem_hct_E976_SHAKE, \
+      frodokem_ss_E976_SHAKE, \
+      (word32)sizeof(frodokem_ss_E976_SHAKE) },
+#else
+#define FKAT_E976_SHAKE
+#endif
+
+#if defined(WOLFSSL_WC_FRODOKEM_1344) && defined(WOLFSSL_FRODOKEM_SHAKE) && \
+    defined(WOLFSSL_FRODOKEM_EPHEMERAL)
+static const byte frodokem_kr_E1344_SHAKE[] = {
+    0x7c,0x99,0x35,0xa0,0xb0,0x76,0x94,0xaa,0x0c,0x6d,0x10,0xe4,
+    0xdb,0x6b,0x1a,0xdd,0x2f,0xd8,0x1a,0x25,0xcc,0xb1,0x48,0x03,
+    0x2d,0xcd,0x73,0x99,0x36,0x73,0x7f,0x2d,0xb5,0x05,0xd7,0xcf,
+    0xad,0x1b,0x49,0x74,0x99,0x32,0x3c,0x86,0x86,0x32,0x5e,0x47,
+    0x92,0xf2,0x67,0xaa,0xfa,0x3f,0x87,0xca,0x60,0xd0,0x1c,0xb5,
+    0x4f,0x29,0x20,0x2a,0x3e,0x78,0x4c,0xcb,0x7e,0xbc,0xdc,0xfd,
+    0x45,0x54,0x2b,0x7f,0x6a,0xf7,0x78,0x74,
+};
+static const byte frodokem_er_E1344_SHAKE[] = {
+    0x8b,0xf0,0xf4,0x59,0xf0,0xfb,0x3e,0xa8,0xd3,0x27,0x64,0xc2,
+    0x59,0xae,0x63,0x11,0x78,0x97,0x6b,0xaf,0x36,0x83,0xd3,0x33,
+    0x83,0x18,0x8a,0x65,0xa4,0xc2,0x44,0x9b,
+};
+static const byte frodokem_hpk_E1344_SHAKE[] = {
+    0x45,0x4d,0x55,0x2f,0x3c,0x25,0xaa,0xf0,0xe5,0xac,0xc1,0xa4,
+    0xfc,0x43,0x10,0x98,0xb9,0x0c,0xe3,0x99,0x3e,0x86,0x42,0x68,
+    0xa2,0x79,0x5a,0xe7,0xff,0x3c,0xe5,0xca,
+};
+static const byte frodokem_hsk_E1344_SHAKE[] = {
+    0xbe,0x73,0xb2,0x19,0x69,0x05,0x4a,0x4f,0xd4,0x45,0x25,0x08,
+    0x23,0xbe,0xd8,0xec,0x91,0xe6,0xa2,0x52,0x1b,0xce,0x0a,0xd0,
+    0x97,0x49,0xdd,0x1e,0x6c,0xb8,0xd6,0x6f,
+};
+static const byte frodokem_hct_E1344_SHAKE[] = {
+    0x2c,0x76,0xc5,0x40,0xee,0xba,0x95,0x1c,0x7e,0x16,0x68,0x99,
+    0x3d,0x13,0xa3,0xe3,0xcc,0xfd,0xc8,0xbc,0xba,0x29,0x44,0x68,
+    0x54,0x80,0xc2,0xb8,0x82,0x50,0xcc,0xf9,
+};
+static const byte frodokem_ss_E1344_SHAKE[] = {
+    0x6d,0x69,0xdf,0x1a,0x90,0x96,0x8e,0xab,0xad,0xa6,0x9c,0xd3,
+    0x0e,0xc6,0x81,0x3a,0x44,0x06,0x30,0x9d,0xac,0x17,0x44,0x29,
+    0xa0,0x12,0x08,0x52,0xbf,0x82,0x64,0x60,
+};
+#define FKAT_E1344_SHAKE \
+    { WC_EFRODOKEM_1344_SHAKE, \
+      frodokem_kr_E1344_SHAKE, \
+      (word32)sizeof(frodokem_kr_E1344_SHAKE), \
+      frodokem_er_E1344_SHAKE, \
+      (word32)sizeof(frodokem_er_E1344_SHAKE), \
+      frodokem_hpk_E1344_SHAKE, \
+      frodokem_hsk_E1344_SHAKE, \
+      frodokem_hct_E1344_SHAKE, \
+      frodokem_ss_E1344_SHAKE, \
+      (word32)sizeof(frodokem_ss_E1344_SHAKE) },
+#else
+#define FKAT_E1344_SHAKE
+#endif
+
+#if defined(WOLFSSL_WC_FRODOKEM_640) && defined(WOLFSSL_FRODOKEM_AES) && \
+    defined(WOLFSSL_FRODOKEM_EPHEMERAL)
+static const byte frodokem_kr_E640_AES[] = {
+    0x7c,0x99,0x35,0xa0,0xb0,0x76,0x94,0xaa,0x0c,0x6d,0x10,0xe4,
+    0xdb,0x6b,0x1a,0xdd,0x2f,0xd8,0x1a,0x25,0xcc,0xb1,0x48,0x03,
+    0x2d,0xcd,0x73,0x99,0x36,0x73,0x7f,0x2d,0xb5,0x05,0xd7,0xcf,
+    0xad,0x1b,0x49,0x74,0x99,0x32,0x3c,0x86,0x86,0x32,0x5e,0x47,
+};
+static const byte frodokem_er_E640_AES[] = {
+    0x33,0xb3,0xc0,0x75,0x07,0xe4,0x20,0x17,0x48,0x49,0x4d,0x83,
+    0x2b,0x6e,0xe2,0xa6,
+};
+static const byte frodokem_hpk_E640_AES[] = {
+    0x3d,0x9e,0xc6,0x83,0x5b,0x6e,0x15,0xd8,0x15,0xc2,0xf8,0x45,
+    0x48,0x42,0x79,0xa7,0xcc,0x4b,0xf3,0xcc,0xc7,0x7d,0x00,0x29,
+    0xd3,0xb6,0x95,0x47,0x34,0x19,0xf3,0x29,
+};
+static const byte frodokem_hsk_E640_AES[] = {
+    0x80,0x96,0xbd,0x18,0x6c,0xf9,0x9d,0xfa,0x98,0x9a,0x78,0xe2,
+    0x95,0xe9,0x58,0xcc,0x83,0x39,0xaf,0x24,0x5d,0x7b,0xe3,0x8d,
+    0x88,0x70,0x4f,0x32,0x9a,0x00,0xba,0x66,
+};
+static const byte frodokem_hct_E640_AES[] = {
+    0xa2,0xdb,0xb4,0xa4,0xbc,0x4b,0x72,0x92,0xf3,0x68,0xe7,0x1a,
+    0x34,0x51,0xad,0x8d,0xcd,0x10,0x12,0x32,0x68,0x0c,0x24,0xa6,
+    0x92,0xcc,0xe8,0xca,0xd4,0x03,0x48,0xea,
+};
+static const byte frodokem_ss_E640_AES[] = {
+    0x9f,0x54,0x37,0x7d,0x45,0x20,0x90,0xf3,0x63,0x1e,0x45,0xb9,
+    0x39,0x9a,0x28,0x92,
+};
+#define FKAT_E640_AES \
+    { WC_EFRODOKEM_640_AES, \
+      frodokem_kr_E640_AES, \
+      (word32)sizeof(frodokem_kr_E640_AES), \
+      frodokem_er_E640_AES, \
+      (word32)sizeof(frodokem_er_E640_AES), \
+      frodokem_hpk_E640_AES, \
+      frodokem_hsk_E640_AES, \
+      frodokem_hct_E640_AES, \
+      frodokem_ss_E640_AES, \
+      (word32)sizeof(frodokem_ss_E640_AES) },
+#else
+#define FKAT_E640_AES
+#endif
+
+#if defined(WOLFSSL_WC_FRODOKEM_976) && defined(WOLFSSL_FRODOKEM_AES) && \
+    defined(WOLFSSL_FRODOKEM_EPHEMERAL)
+static const byte frodokem_kr_E976_AES[] = {
+    0x7c,0x99,0x35,0xa0,0xb0,0x76,0x94,0xaa,0x0c,0x6d,0x10,0xe4,
+    0xdb,0x6b,0x1a,0xdd,0x2f,0xd8,0x1a,0x25,0xcc,0xb1,0x48,0x03,
+    0x2d,0xcd,0x73,0x99,0x36,0x73,0x7f,0x2d,0xb5,0x05,0xd7,0xcf,
+    0xad,0x1b,0x49,0x74,0x99,0x32,0x3c,0x86,0x86,0x32,0x5e,0x47,
+    0x92,0xf2,0x67,0xaa,0xfa,0x3f,0x87,0xca,0x60,0xd0,0x1c,0xb5,
+    0x4f,0x29,0x20,0x2a,
+};
+static const byte frodokem_er_E976_AES[] = {
+    0xeb,0x4a,0x7c,0x66,0xef,0x4e,0xba,0x2d,0xdb,0x38,0xc8,0x8d,
+    0x8b,0xc7,0x06,0xb1,0xd6,0x39,0x00,0x21,0x98,0x17,0x2a,0x7b,
+};
+static const byte frodokem_hpk_E976_AES[] = {
+    0xc6,0xc9,0x25,0x07,0xc1,0xf4,0x8c,0x20,0x89,0x76,0x8d,0xe0,
+    0x14,0x22,0xa5,0x9c,0x2b,0xb8,0x5c,0xe5,0x6e,0x09,0x56,0x5c,
+    0x5f,0x07,0xa9,0x0f,0x84,0x9c,0x45,0xc0,
+};
+static const byte frodokem_hsk_E976_AES[] = {
+    0x07,0x74,0x6e,0x26,0x0b,0x4c,0x11,0x01,0x15,0x58,0xa9,0xb5,
+    0x5e,0x1c,0x27,0xa3,0x76,0x0a,0x3b,0x98,0x0e,0x76,0xab,0x22,
+    0x19,0xaf,0x01,0x85,0x41,0xff,0x00,0x45,
+};
+static const byte frodokem_hct_E976_AES[] = {
+    0x16,0x05,0x08,0xa0,0xd9,0x3e,0x28,0xcd,0xd7,0xd7,0xf3,0xac,
+    0x31,0x66,0x79,0x20,0x4e,0x24,0xf7,0x7d,0x57,0x1d,0x6d,0x31,
+    0x8f,0x41,0x41,0xf2,0x51,0x87,0xfd,0xc2,
+};
+static const byte frodokem_ss_E976_AES[] = {
+    0x59,0x4d,0xe8,0x44,0x73,0xb3,0x40,0x8e,0x35,0xf6,0xc4,0xd1,
+    0xf2,0xf2,0xec,0x3b,0x56,0xd2,0xdd,0xa9,0x6f,0xa2,0x34,0x96,
+};
+#define FKAT_E976_AES \
+    { WC_EFRODOKEM_976_AES, \
+      frodokem_kr_E976_AES, \
+      (word32)sizeof(frodokem_kr_E976_AES), \
+      frodokem_er_E976_AES, \
+      (word32)sizeof(frodokem_er_E976_AES), \
+      frodokem_hpk_E976_AES, \
+      frodokem_hsk_E976_AES, \
+      frodokem_hct_E976_AES, \
+      frodokem_ss_E976_AES, \
+      (word32)sizeof(frodokem_ss_E976_AES) },
+#else
+#define FKAT_E976_AES
+#endif
+
+#if defined(WOLFSSL_WC_FRODOKEM_1344) && defined(WOLFSSL_FRODOKEM_AES) && \
+    defined(WOLFSSL_FRODOKEM_EPHEMERAL)
+static const byte frodokem_kr_E1344_AES[] = {
+    0x7c,0x99,0x35,0xa0,0xb0,0x76,0x94,0xaa,0x0c,0x6d,0x10,0xe4,
+    0xdb,0x6b,0x1a,0xdd,0x2f,0xd8,0x1a,0x25,0xcc,0xb1,0x48,0x03,
+    0x2d,0xcd,0x73,0x99,0x36,0x73,0x7f,0x2d,0xb5,0x05,0xd7,0xcf,
+    0xad,0x1b,0x49,0x74,0x99,0x32,0x3c,0x86,0x86,0x32,0x5e,0x47,
+    0x92,0xf2,0x67,0xaa,0xfa,0x3f,0x87,0xca,0x60,0xd0,0x1c,0xb5,
+    0x4f,0x29,0x20,0x2a,0x3e,0x78,0x4c,0xcb,0x7e,0xbc,0xdc,0xfd,
+    0x45,0x54,0x2b,0x7f,0x6a,0xf7,0x78,0x74,
+};
+static const byte frodokem_er_E1344_AES[] = {
+    0x8b,0xf0,0xf4,0x59,0xf0,0xfb,0x3e,0xa8,0xd3,0x27,0x64,0xc2,
+    0x59,0xae,0x63,0x11,0x78,0x97,0x6b,0xaf,0x36,0x83,0xd3,0x33,
+    0x83,0x18,0x8a,0x65,0xa4,0xc2,0x44,0x9b,
+};
+static const byte frodokem_hpk_E1344_AES[] = {
+    0xf4,0xc8,0x3f,0xd6,0xbd,0xbd,0xd1,0x49,0x30,0x7f,0x72,0xd6,
+    0x7b,0xdd,0x83,0xaa,0x49,0xcd,0xa6,0xde,0xa6,0x9b,0xf3,0x64,
+    0xe7,0xef,0x27,0x76,0xc3,0xb7,0x21,0xff,
+};
+static const byte frodokem_hsk_E1344_AES[] = {
+    0xe4,0x99,0x44,0x64,0x5a,0xa1,0x19,0xb7,0x53,0xe1,0xdd,0x48,
+    0x68,0xb4,0x19,0xc4,0x21,0xe9,0x04,0x61,0xde,0x23,0xbb,0xdc,
+    0xd5,0x5e,0xa2,0x8c,0xa8,0xd1,0x59,0xf9,
+};
+static const byte frodokem_hct_E1344_AES[] = {
+    0x25,0x27,0xe2,0xb6,0x22,0xec,0xa7,0x6d,0xc8,0xea,0x48,0x9d,
+    0x51,0x6c,0x68,0x5c,0x4c,0x67,0x75,0x11,0x27,0xfd,0x3f,0x49,
+    0x25,0xc6,0x91,0x55,0x60,0xe9,0xe8,0x43,
+};
+static const byte frodokem_ss_E1344_AES[] = {
+    0xb2,0x43,0xfe,0x6d,0x7c,0x9b,0x38,0x29,0x25,0x2d,0x5a,0xec,
+    0x09,0x0a,0x47,0x09,0xf5,0xe3,0x96,0xfd,0xef,0xe4,0xef,0x1a,
+    0xa4,0xae,0x6c,0x94,0x98,0xcb,0xce,0x15,
+};
+#define FKAT_E1344_AES \
+    { WC_EFRODOKEM_1344_AES, \
+      frodokem_kr_E1344_AES, \
+      (word32)sizeof(frodokem_kr_E1344_AES), \
+      frodokem_er_E1344_AES, \
+      (word32)sizeof(frodokem_er_E1344_AES), \
+      frodokem_hpk_E1344_AES, \
+      frodokem_hsk_E1344_AES, \
+      frodokem_hct_E1344_AES, \
+      frodokem_ss_E1344_AES, \
+      (word32)sizeof(frodokem_ss_E1344_AES) },
+#else
+#define FKAT_E1344_AES
+#endif
+
+
+/* One known-answer entry per compiled parameter set / variant. */
+typedef struct FrodoKemKat {
+    int type;
+    const byte* kr;
+    word32 krLen;
+    const byte* er;
+    word32 erLen;
+    const byte* hpk;
+    const byte* hsk;
+    const byte* hct;
+    const byte* ss;
+    word32 ssLen;
+} FrodoKemKat;
+
+static const FrodoKemKat frodokem_kats[] = {
+    FKAT_640_SHAKE FKAT_976_SHAKE FKAT_1344_SHAKE
+    FKAT_640_AES FKAT_976_AES FKAT_1344_AES
+    FKAT_E640_SHAKE FKAT_E976_SHAKE FKAT_E1344_SHAKE
+    FKAT_E640_AES FKAT_E976_AES FKAT_E1344_AES
+};
+
+#define FRODOKEM_KAT_CNT \
+    ((int)(sizeof(frodokem_kats) / sizeof(frodokem_kats[0])))
+
+#endif /* !NO_SHA256 && !WOLFSSL_FRODOKEM_NO_MAKE_KEY */
+#endif /* WOLFSSL_HAVE_FRODOKEM */
+
+/* KAT: key generation. Regenerate each key from fixed randomness and check the
+ * encoded public and private keys against the expected SHA-256 digests. */
+int test_wc_frodokem_make_key_kats(void)
+{
+    EXPECT_DECLS;
+#if defined(WOLFSSL_HAVE_FRODOKEM) && !defined(NO_SHA256) && \
+    !defined(WOLFSSL_FRODOKEM_NO_MAKE_KEY)
+    int i;
+    FrodoKemKey* key = NULL;
+    byte* buf = NULL;
+    byte hash[WC_SHA256_DIGEST_SIZE];
+    word32 pkLen = 0;
+    word32 skLen = 0;
+
+    key = (FrodoKemKey*)XMALLOC(sizeof(*key), NULL, DYNAMIC_TYPE_TMP_BUFFER);
+    ExpectNotNull(key);
+    buf = (byte*)XMALLOC(FRODOKEM_MAX_PRIVATE_KEY_SIZE, NULL,
+        DYNAMIC_TYPE_TMP_BUFFER);
+    ExpectNotNull(buf);
+
+    for (i = 0; (i < FRODOKEM_KAT_CNT) && EXPECT_SUCCESS(); i++) {
+        const FrodoKemKat* k = &frodokem_kats[i];
+
+        ExpectIntEQ(wc_FrodoKemKey_Init(key, k->type, NULL, INVALID_DEVID), 0);
+        ExpectIntEQ(wc_FrodoKemKey_MakeKeyWithRandom(key, k->kr,
+            (int)k->krLen), 0);
+
+        ExpectIntEQ(wc_FrodoKemKey_PublicKeySize(key, &pkLen), 0);
+        ExpectIntEQ(wc_FrodoKemKey_EncodePublicKey(key, buf, pkLen), 0);
+        ExpectIntEQ(wc_Sha256Hash(buf, pkLen, hash), 0);
+        ExpectIntEQ(XMEMCMP(hash, k->hpk, WC_SHA256_DIGEST_SIZE), 0);
+
+        ExpectIntEQ(wc_FrodoKemKey_PrivateKeySize(key, &skLen), 0);
+        ExpectIntEQ(wc_FrodoKemKey_EncodePrivateKey(key, buf, skLen), 0);
+        ExpectIntEQ(wc_Sha256Hash(buf, skLen, hash), 0);
+        ExpectIntEQ(XMEMCMP(hash, k->hsk, WC_SHA256_DIGEST_SIZE), 0);
+
+        wc_FrodoKemKey_Free(key);
+    }
+
+    XFREE(buf, NULL, DYNAMIC_TYPE_TMP_BUFFER);
+    XFREE(key, NULL, DYNAMIC_TYPE_TMP_BUFFER);
+#endif
+    return EXPECT_RESULT();
+}
+
+/* KAT: encapsulation. Recreate the key, encapsulate with fixed randomness and
+ * check the ciphertext digest and shared secret against expected values. */
+int test_wc_frodokem_encapsulate_kats(void)
+{
+    EXPECT_DECLS;
+#if defined(WOLFSSL_HAVE_FRODOKEM) && !defined(NO_SHA256) && \
+    !defined(WOLFSSL_FRODOKEM_NO_MAKE_KEY) && \
+    !defined(WOLFSSL_FRODOKEM_NO_ENCAPSULATE)
+    int i;
+    FrodoKemKey* key = NULL;
+    byte* ct = NULL;
+    byte hash[WC_SHA256_DIGEST_SIZE];
+    byte ss[FRODOKEM_MAX_LENSEC];
+    word32 ctLen = 0;
+    word32 ssLen = 0;
+
+    key = (FrodoKemKey*)XMALLOC(sizeof(*key), NULL, DYNAMIC_TYPE_TMP_BUFFER);
+    ExpectNotNull(key);
+    ct = (byte*)XMALLOC(FRODOKEM_MAX_CIPHER_TEXT_SIZE, NULL,
+        DYNAMIC_TYPE_TMP_BUFFER);
+    ExpectNotNull(ct);
+
+    for (i = 0; (i < FRODOKEM_KAT_CNT) && EXPECT_SUCCESS(); i++) {
+        const FrodoKemKat* k = &frodokem_kats[i];
+
+        ExpectIntEQ(wc_FrodoKemKey_Init(key, k->type, NULL, INVALID_DEVID), 0);
+        ExpectIntEQ(wc_FrodoKemKey_MakeKeyWithRandom(key, k->kr,
+            (int)k->krLen), 0);
+        ExpectIntEQ(wc_FrodoKemKey_CipherTextSize(key, &ctLen), 0);
+        ExpectIntEQ(wc_FrodoKemKey_SharedSecretSize(key, &ssLen), 0);
+
+        ExpectIntEQ(wc_FrodoKemKey_EncapsulateWithRandom(key, ct, ss, k->er,
+            (int)k->erLen), 0);
+        ExpectIntEQ(wc_Sha256Hash(ct, ctLen, hash), 0);
+        ExpectIntEQ(XMEMCMP(hash, k->hct, WC_SHA256_DIGEST_SIZE), 0);
+        ExpectIntEQ(XMEMCMP(ss, k->ss, ssLen), 0);
+
+        wc_FrodoKemKey_Free(key);
+    }
+
+    XFREE(ct, NULL, DYNAMIC_TYPE_TMP_BUFFER);
+    XFREE(key, NULL, DYNAMIC_TYPE_TMP_BUFFER);
+#endif
+    return EXPECT_RESULT();
+}
+
+/* KAT: decapsulation. Recreate the key and ciphertext, decapsulate and check
+ * the recovered shared secret against the expected value. */
+int test_wc_frodokem_decapsulate_kats(void)
+{
+    EXPECT_DECLS;
+#if defined(WOLFSSL_HAVE_FRODOKEM) && !defined(NO_SHA256) && \
+    !defined(WOLFSSL_FRODOKEM_NO_MAKE_KEY) && \
+    !defined(WOLFSSL_FRODOKEM_NO_ENCAPSULATE) && \
+    !defined(WOLFSSL_FRODOKEM_NO_DECAPSULATE)
+    int i;
+    FrodoKemKey* key = NULL;
+    byte* ct = NULL;
+    byte ss[FRODOKEM_MAX_LENSEC];
+    byte ssDec[FRODOKEM_MAX_LENSEC];
+    word32 ctLen = 0;
+    word32 ssLen = 0;
+
+    key = (FrodoKemKey*)XMALLOC(sizeof(*key), NULL, DYNAMIC_TYPE_TMP_BUFFER);
+    ExpectNotNull(key);
+    ct = (byte*)XMALLOC(FRODOKEM_MAX_CIPHER_TEXT_SIZE, NULL,
+        DYNAMIC_TYPE_TMP_BUFFER);
+    ExpectNotNull(ct);
+
+    for (i = 0; (i < FRODOKEM_KAT_CNT) && EXPECT_SUCCESS(); i++) {
+        const FrodoKemKat* k = &frodokem_kats[i];
+
+        ExpectIntEQ(wc_FrodoKemKey_Init(key, k->type, NULL, INVALID_DEVID), 0);
+        ExpectIntEQ(wc_FrodoKemKey_MakeKeyWithRandom(key, k->kr,
+            (int)k->krLen), 0);
+        ExpectIntEQ(wc_FrodoKemKey_CipherTextSize(key, &ctLen), 0);
+        ExpectIntEQ(wc_FrodoKemKey_SharedSecretSize(key, &ssLen), 0);
+        ExpectIntEQ(wc_FrodoKemKey_EncapsulateWithRandom(key, ct, ss, k->er,
+            (int)k->erLen), 0);
+
+        ExpectIntEQ(wc_FrodoKemKey_Decapsulate(key, ssDec, ct, ctLen), 0);
+        ExpectIntEQ(XMEMCMP(ssDec, k->ss, ssLen), 0);
+
+        wc_FrodoKemKey_Free(key);
+    }
+
+    XFREE(ct, NULL, DYNAMIC_TYPE_TMP_BUFFER);
+    XFREE(key, NULL, DYNAMIC_TYPE_TMP_BUFFER);
+#endif
+    return EXPECT_RESULT();
+}
+
+/* Full RNG-based round trip for every compiled variant: the decapsulated
+ * shared secret must equal the encapsulated one. */
+int test_wc_frodokem_roundtrip(void)
+{
+    EXPECT_DECLS;
+#if defined(WOLFSSL_HAVE_FRODOKEM) && !defined(WC_NO_RNG) && \
+    !defined(WOLFSSL_FRODOKEM_NO_MAKE_KEY) && \
+    !defined(WOLFSSL_FRODOKEM_NO_ENCAPSULATE) && \
+    !defined(WOLFSSL_FRODOKEM_NO_DECAPSULATE)
+    int i;
+    FrodoKemKey* key = NULL;
+    WC_RNG rng;
+    byte* ct = NULL;
+    byte ss[FRODOKEM_MAX_LENSEC];
+    byte ssDec[FRODOKEM_MAX_LENSEC];
+    word32 ctLen = 0;
+    word32 ssLen = 0;
+    int rngInit = 0;
+
+    XMEMSET(&rng, 0, sizeof(rng));
+    key = (FrodoKemKey*)XMALLOC(sizeof(*key), NULL, DYNAMIC_TYPE_TMP_BUFFER);
+    ExpectNotNull(key);
+    ct = (byte*)XMALLOC(FRODOKEM_MAX_CIPHER_TEXT_SIZE, NULL,
+        DYNAMIC_TYPE_TMP_BUFFER);
+    ExpectNotNull(ct);
+    ExpectIntEQ(wc_InitRng(&rng), 0);
+    if (EXPECT_SUCCESS()) {
+        rngInit = 1;
+    }
+
+    for (i = 0; (i < FRODOKEM_TYPE_CNT) && EXPECT_SUCCESS(); i++) {
+        ExpectIntEQ(wc_FrodoKemKey_Init(key, frodokem_types[i], NULL,
+            INVALID_DEVID), 0);
+        ExpectIntEQ(wc_FrodoKemKey_CipherTextSize(key, &ctLen), 0);
+        ExpectIntEQ(wc_FrodoKemKey_SharedSecretSize(key, &ssLen), 0);
+        ExpectIntEQ(wc_FrodoKemKey_MakeKey(key, &rng), 0);
+        ExpectIntEQ(wc_FrodoKemKey_Encapsulate(key, ct, ss, &rng), 0);
+        ExpectIntEQ(wc_FrodoKemKey_Decapsulate(key, ssDec, ct, ctLen), 0);
+        ExpectIntEQ(XMEMCMP(ss, ssDec, ssLen), 0);
+        wc_FrodoKemKey_Free(key);
+    }
+
+    if (rngInit) {
+        DoExpectIntEQ(wc_FreeRng(&rng), 0);
+    }
+    XFREE(ct, NULL, DYNAMIC_TYPE_TMP_BUFFER);
+    XFREE(key, NULL, DYNAMIC_TYPE_TMP_BUFFER);
+#endif
+    return EXPECT_RESULT();
+}
+
+/* Encode then decode the public and private keys and confirm the decoded key
+ * pair still produces matching shared secrets. */
+int test_wc_frodokem_encode_decode(void)
+{
+    EXPECT_DECLS;
+#if defined(WOLFSSL_HAVE_FRODOKEM) && !defined(WC_NO_RNG) && \
+    !defined(WOLFSSL_FRODOKEM_NO_MAKE_KEY) && \
+    !defined(WOLFSSL_FRODOKEM_NO_ENCAPSULATE) && \
+    !defined(WOLFSSL_FRODOKEM_NO_DECAPSULATE)
+    int i;
+    FrodoKemKey* key = NULL;
+    FrodoKemKey* pubKey = NULL;
+    FrodoKemKey* privKey = NULL;
+    WC_RNG rng;
+    byte* pkBuf = NULL;
+    byte* skBuf = NULL;
+    byte* ct = NULL;
+    byte ss[FRODOKEM_MAX_LENSEC];
+    byte ssDec[FRODOKEM_MAX_LENSEC];
+    word32 pkLen = 0;
+    word32 skLen = 0;
+    word32 ctLen = 0;
+    word32 ssLen = 0;
+    int rngInit = 0;
+
+    XMEMSET(&rng, 0, sizeof(rng));
+    key = (FrodoKemKey*)XMALLOC(sizeof(*key), NULL, DYNAMIC_TYPE_TMP_BUFFER);
+    ExpectNotNull(key);
+    pubKey = (FrodoKemKey*)XMALLOC(sizeof(*pubKey), NULL,
+        DYNAMIC_TYPE_TMP_BUFFER);
+    ExpectNotNull(pubKey);
+    privKey = (FrodoKemKey*)XMALLOC(sizeof(*privKey), NULL,
+        DYNAMIC_TYPE_TMP_BUFFER);
+    ExpectNotNull(privKey);
+    pkBuf = (byte*)XMALLOC(FRODOKEM_MAX_PUBLIC_KEY_SIZE, NULL,
+        DYNAMIC_TYPE_TMP_BUFFER);
+    ExpectNotNull(pkBuf);
+    skBuf = (byte*)XMALLOC(FRODOKEM_MAX_PRIVATE_KEY_SIZE, NULL,
+        DYNAMIC_TYPE_TMP_BUFFER);
+    ExpectNotNull(skBuf);
+    ct = (byte*)XMALLOC(FRODOKEM_MAX_CIPHER_TEXT_SIZE, NULL,
+        DYNAMIC_TYPE_TMP_BUFFER);
+    ExpectNotNull(ct);
+    ExpectIntEQ(wc_InitRng(&rng), 0);
+    if (EXPECT_SUCCESS()) {
+        rngInit = 1;
+    }
+
+    for (i = 0; (i < FRODOKEM_TYPE_CNT) && EXPECT_SUCCESS(); i++) {
+        int type = frodokem_types[i];
+
+        ExpectIntEQ(wc_FrodoKemKey_Init(key, type, NULL, INVALID_DEVID), 0);
+        ExpectIntEQ(wc_FrodoKemKey_Init(pubKey, type, NULL, INVALID_DEVID), 0);
+        ExpectIntEQ(wc_FrodoKemKey_Init(privKey, type, NULL, INVALID_DEVID), 0);
+        ExpectIntEQ(wc_FrodoKemKey_PublicKeySize(key, &pkLen), 0);
+        ExpectIntEQ(wc_FrodoKemKey_PrivateKeySize(key, &skLen), 0);
+        ExpectIntEQ(wc_FrodoKemKey_CipherTextSize(key, &ctLen), 0);
+        ExpectIntEQ(wc_FrodoKemKey_SharedSecretSize(key, &ssLen), 0);
+
+        ExpectIntEQ(wc_FrodoKemKey_MakeKey(key, &rng), 0);
+        ExpectIntEQ(wc_FrodoKemKey_EncodePublicKey(key, pkBuf, pkLen), 0);
+        ExpectIntEQ(wc_FrodoKemKey_EncodePrivateKey(key, skBuf, skLen), 0);
+        ExpectIntEQ(wc_FrodoKemKey_DecodePublicKey(pubKey, pkBuf, pkLen), 0);
+        ExpectIntEQ(wc_FrodoKemKey_DecodePrivateKey(privKey, skBuf, skLen), 0);
+
+        /* Encapsulate to the decoded public key, decapsulate with the decoded
+         * private key. */
+        ExpectIntEQ(wc_FrodoKemKey_Encapsulate(pubKey, ct, ss, &rng), 0);
+        ExpectIntEQ(wc_FrodoKemKey_Decapsulate(privKey, ssDec, ct, ctLen), 0);
+        ExpectIntEQ(XMEMCMP(ss, ssDec, ssLen), 0);
+
+        wc_FrodoKemKey_Free(privKey);
+        wc_FrodoKemKey_Free(pubKey);
+        wc_FrodoKemKey_Free(key);
+    }
+
+    if (rngInit) {
+        DoExpectIntEQ(wc_FreeRng(&rng), 0);
+    }
+    XFREE(ct, NULL, DYNAMIC_TYPE_TMP_BUFFER);
+    XFREE(skBuf, NULL, DYNAMIC_TYPE_TMP_BUFFER);
+    XFREE(pkBuf, NULL, DYNAMIC_TYPE_TMP_BUFFER);
+    XFREE(privKey, NULL, DYNAMIC_TYPE_TMP_BUFFER);
+    XFREE(pubKey, NULL, DYNAMIC_TYPE_TMP_BUFFER);
+    XFREE(key, NULL, DYNAMIC_TYPE_TMP_BUFFER);
+#endif
+    return EXPECT_RESULT();
+}
+
+/* A modified ciphertext must decapsulate (implicit rejection) to a shared
+ * secret different from the one produced for the valid ciphertext. */
+int test_wc_frodokem_decap_implicit_reject(void)
+{
+    EXPECT_DECLS;
+#if defined(WOLFSSL_HAVE_FRODOKEM) && !defined(WC_NO_RNG) && \
+    !defined(WOLFSSL_FRODOKEM_NO_MAKE_KEY) && \
+    !defined(WOLFSSL_FRODOKEM_NO_ENCAPSULATE) && \
+    !defined(WOLFSSL_FRODOKEM_NO_DECAPSULATE)
+    int i;
+    FrodoKemKey* key = NULL;
+    WC_RNG rng;
+    byte* ct = NULL;
+    byte ss[FRODOKEM_MAX_LENSEC];
+    byte ssDec[FRODOKEM_MAX_LENSEC];
+    byte ssDec2[FRODOKEM_MAX_LENSEC];
+    byte exp[FRODOKEM_MAX_LENSEC];
+    wc_Shake sh;
+    word32 ctLen = 0;
+    word32 ssLen = 0;
+    int rngInit = 0;
+
+    XMEMSET(&rng, 0, sizeof(rng));
+    key = (FrodoKemKey*)XMALLOC(sizeof(*key), NULL, DYNAMIC_TYPE_TMP_BUFFER);
+    ExpectNotNull(key);
+    ct = (byte*)XMALLOC(FRODOKEM_MAX_CIPHER_TEXT_SIZE, NULL,
+        DYNAMIC_TYPE_TMP_BUFFER);
+    ExpectNotNull(ct);
+    ExpectIntEQ(wc_InitRng(&rng), 0);
+    if (EXPECT_SUCCESS()) {
+        rngInit = 1;
+    }
+
+    /* Every compiled variant: a modified ciphertext must decapsulate to an
+     * implicit-rejection secret, deterministically. */
+    for (i = 0; (i < FRODOKEM_TYPE_CNT) && EXPECT_SUCCESS(); i++) {
+        ExpectIntEQ(wc_FrodoKemKey_Init(key, frodokem_types[i], NULL,
+            INVALID_DEVID), 0);
+        ExpectIntEQ(wc_FrodoKemKey_CipherTextSize(key, &ctLen), 0);
+        ExpectIntEQ(wc_FrodoKemKey_SharedSecretSize(key, &ssLen), 0);
+        ExpectIntEQ(wc_FrodoKemKey_MakeKey(key, &rng), 0);
+        ExpectIntEQ(wc_FrodoKemKey_Encapsulate(key, ct, ss, &rng), 0);
+        /* Flip a bit in the ciphertext. */
+        if (EXPECT_SUCCESS()) {
+            ct[0] ^= 0xff;
+        }
+        ExpectIntEQ(wc_FrodoKemKey_Decapsulate(key, ssDec, ct, ctLen), 0);
+        ExpectIntNE(XMEMCMP(ss, ssDec, ssLen), 0);
+        /* Implicit rejection is deterministic: decapsulating the same modified
+         * ciphertext again yields the same shared secret (the SHAKE(c' || s)
+         * value), not uninitialized/garbage data. */
+        ExpectIntEQ(wc_FrodoKemKey_Decapsulate(key, ssDec2, ct, ctLen), 0);
+        ExpectIntEQ(XMEMCMP(ssDec, ssDec2, ssLen), 0);
+        /* Isolate the implicit-rejection property: the rejected secret must be
+         * SHAKE(c1 || c2 || salt || s) over the STORED rejection secret s (not
+         * k'). Recompute it (640 hashes with SHAKE128, 976/1344 with SHAKE256)
+         * and confirm it matches - a select that used k' would not. */
+        if ((frodokem_types[i] & FRODOKEM_BASE_MASK) == WC_FRODOKEM_640) {
+            ExpectIntEQ(wc_InitShake128(&sh, NULL, INVALID_DEVID), 0);
+            ExpectIntEQ(wc_Shake128_Update(&sh, ct, ctLen), 0);
+            ExpectIntEQ(wc_Shake128_Update(&sh, key->s, ssLen), 0);
+            ExpectIntEQ(wc_Shake128_Final(&sh, exp, ssLen), 0);
+            wc_Shake128_Free(&sh);
+        }
+        else {
+            ExpectIntEQ(wc_InitShake256(&sh, NULL, INVALID_DEVID), 0);
+            ExpectIntEQ(wc_Shake256_Update(&sh, ct, ctLen), 0);
+            ExpectIntEQ(wc_Shake256_Update(&sh, key->s, ssLen), 0);
+            ExpectIntEQ(wc_Shake256_Final(&sh, exp, ssLen), 0);
+            wc_Shake256_Free(&sh);
+        }
+        ExpectIntEQ(XMEMCMP(ssDec, exp, ssLen), 0);
+        wc_FrodoKemKey_Free(key);
+    }
+
+    if (rngInit) {
+        DoExpectIntEQ(wc_FreeRng(&rng), 0);
+    }
+    XFREE(ct, NULL, DYNAMIC_TYPE_TMP_BUFFER);
+    XFREE(key, NULL, DYNAMIC_TYPE_TMP_BUFFER);
+#endif
+    return EXPECT_RESULT();
+}
+
+/* Decapsulating with a public-key-only object must fail. */
+int test_wc_frodokem_decapsulate_pubonly_fails(void)
+{
+    EXPECT_DECLS;
+#if defined(WOLFSSL_HAVE_FRODOKEM) && !defined(WC_NO_RNG) && \
+    !defined(WOLFSSL_FRODOKEM_NO_MAKE_KEY) && \
+    !defined(WOLFSSL_FRODOKEM_NO_ENCAPSULATE) && \
+    !defined(WOLFSSL_FRODOKEM_NO_DECAPSULATE)
+    int i;
+    FrodoKemKey* key = NULL;
+    FrodoKemKey* pubKey = NULL;
+    WC_RNG rng;
+    byte* pkBuf = NULL;
+    byte* ct = NULL;
+    byte ss[FRODOKEM_MAX_LENSEC];
+    byte ssDec[FRODOKEM_MAX_LENSEC];
+    word32 pkLen = 0;
+    word32 ctLen = 0;
+    int rngInit = 0;
+
+    XMEMSET(&rng, 0, sizeof(rng));
+    key = (FrodoKemKey*)XMALLOC(sizeof(*key), NULL, DYNAMIC_TYPE_TMP_BUFFER);
+    ExpectNotNull(key);
+    pubKey = (FrodoKemKey*)XMALLOC(sizeof(*pubKey), NULL,
+        DYNAMIC_TYPE_TMP_BUFFER);
+    ExpectNotNull(pubKey);
+    pkBuf = (byte*)XMALLOC(FRODOKEM_MAX_PUBLIC_KEY_SIZE, NULL,
+        DYNAMIC_TYPE_TMP_BUFFER);
+    ExpectNotNull(pkBuf);
+    ct = (byte*)XMALLOC(FRODOKEM_MAX_CIPHER_TEXT_SIZE, NULL,
+        DYNAMIC_TYPE_TMP_BUFFER);
+    ExpectNotNull(ct);
+    ExpectIntEQ(wc_InitRng(&rng), 0);
+    if (EXPECT_SUCCESS()) {
+        rngInit = 1;
+    }
+
+    /* Every compiled variant: a public-key-only object cannot decapsulate. */
+    for (i = 0; (i < FRODOKEM_TYPE_CNT) && EXPECT_SUCCESS(); i++) {
+        ExpectIntEQ(wc_FrodoKemKey_Init(key, frodokem_types[i], NULL,
+            INVALID_DEVID), 0);
+        ExpectIntEQ(wc_FrodoKemKey_Init(pubKey, frodokem_types[i], NULL,
+            INVALID_DEVID), 0);
+        ExpectIntEQ(wc_FrodoKemKey_PublicKeySize(key, &pkLen), 0);
+        ExpectIntEQ(wc_FrodoKemKey_CipherTextSize(key, &ctLen), 0);
+        ExpectIntEQ(wc_FrodoKemKey_MakeKey(key, &rng), 0);
+        ExpectIntEQ(wc_FrodoKemKey_Encapsulate(key, ct, ss, &rng), 0);
+        ExpectIntEQ(wc_FrodoKemKey_EncodePublicKey(key, pkBuf, pkLen), 0);
+        ExpectIntEQ(wc_FrodoKemKey_DecodePublicKey(pubKey, pkBuf, pkLen), 0);
+        /* Public-key-only object cannot decapsulate. */
+        ExpectIntEQ(wc_FrodoKemKey_Decapsulate(pubKey, ssDec, ct, ctLen),
+            WC_NO_ERR_TRACE(BAD_STATE_E));
+        wc_FrodoKemKey_Free(pubKey);
+        wc_FrodoKemKey_Free(key);
+    }
+
+    if (rngInit) {
+        DoExpectIntEQ(wc_FreeRng(&rng), 0);
+    }
+    XFREE(ct, NULL, DYNAMIC_TYPE_TMP_BUFFER);
+    XFREE(pkBuf, NULL, DYNAMIC_TYPE_TMP_BUFFER);
+    XFREE(pubKey, NULL, DYNAMIC_TYPE_TMP_BUFFER);
+    XFREE(key, NULL, DYNAMIC_TYPE_TMP_BUFFER);
+#endif
+    return EXPECT_RESULT();
+}
+
+/* A private key whose embedded public-key hash does not match the public key
+ * must be rejected by decode. */
+int test_wc_frodokem_decode_privkey_bad_pkh(void)
+{
+    EXPECT_DECLS;
+#if defined(WOLFSSL_HAVE_FRODOKEM) && !defined(WC_NO_RNG) && \
+    !defined(WOLFSSL_FRODOKEM_NO_MAKE_KEY)
+    int i;
+    FrodoKemKey* key = NULL;
+    WC_RNG rng;
+    byte* sk = NULL;
+    word32 skLen = 0;
+    int rngInit = 0;
+
+    XMEMSET(&rng, 0, sizeof(rng));
+    key = (FrodoKemKey*)XMALLOC(sizeof(*key), NULL, DYNAMIC_TYPE_TMP_BUFFER);
+    ExpectNotNull(key);
+    sk = (byte*)XMALLOC(FRODOKEM_MAX_PRIVATE_KEY_SIZE, NULL,
+        DYNAMIC_TYPE_TMP_BUFFER);
+    ExpectNotNull(sk);
+    ExpectIntEQ(wc_InitRng(&rng), 0);
+    if (EXPECT_SUCCESS()) {
+        rngInit = 1;
+    }
+
+    /* Every compiled variant: a corrupted embedded pkh is rejected. */
+    for (i = 0; (i < FRODOKEM_TYPE_CNT) && EXPECT_SUCCESS(); i++) {
+        ExpectIntEQ(wc_FrodoKemKey_Init(key, frodokem_types[i], NULL,
+            INVALID_DEVID), 0);
+        ExpectIntEQ(wc_FrodoKemKey_PrivateKeySize(key, &skLen), 0);
+        ExpectIntEQ(wc_FrodoKemKey_MakeKey(key, &rng), 0);
+        ExpectIntEQ(wc_FrodoKemKey_EncodePrivateKey(key, sk, skLen), 0);
+
+        /* A correctly encoded private key decodes. */
+        ExpectIntEQ(wc_FrodoKemKey_DecodePrivateKey(key, sk, skLen), 0);
+
+        /* Corrupt the stored public-key hash (the final lenSec bytes). */
+        if (EXPECT_SUCCESS()) {
+            sk[skLen - 1] ^= 0xff;
+        }
+        ExpectIntEQ(wc_FrodoKemKey_DecodePrivateKey(key, sk, skLen),
+            WC_NO_ERR_TRACE(WC_KEY_MISMATCH_E));
+        wc_FrodoKemKey_Free(key);
+    }
+
+    if (rngInit) {
+        DoExpectIntEQ(wc_FreeRng(&rng), 0);
+    }
+    XFREE(sk, NULL, DYNAMIC_TYPE_TMP_BUFFER);
+    XFREE(key, NULL, DYNAMIC_TYPE_TMP_BUFFER);
+#endif
+    return EXPECT_RESULT();
+}
+
+/* Bad-argument and bad-length handling. */
+int test_wc_frodokem_bad_args(void)
+{
+    EXPECT_DECLS;
+#if defined(WOLFSSL_HAVE_FRODOKEM)
+    FrodoKemKey* key = NULL;
+    word32 len = 0;
+    word32 pkLen = 0;
+    word32 skLen = 0;
+    int type = frodokem_types[0];
+    byte small[8];
+
+    XMEMSET(small, 0, sizeof(small));
+    key = (FrodoKemKey*)XMALLOC(sizeof(*key), NULL, DYNAMIC_TYPE_TMP_BUFFER);
+    ExpectNotNull(key);
+
+    /* Constructor / destructor NULL handling. */
+#ifndef WC_NO_CONSTRUCTORS
+    ExpectIntEQ(wc_FrodoKemKey_Delete(NULL, NULL),
+        WC_NO_ERR_TRACE(BAD_FUNC_ARG));
+#endif
+    ExpectIntEQ(wc_FrodoKemKey_Free(NULL), WC_NO_ERR_TRACE(BAD_FUNC_ARG));
+
+    /* Init argument checks: NULL key, unknown type bits. */
+    ExpectIntEQ(wc_FrodoKemKey_Init(NULL, type, NULL, INVALID_DEVID),
+        WC_NO_ERR_TRACE(BAD_FUNC_ARG));
+    ExpectIntEQ(wc_FrodoKemKey_Init(key, -1, NULL, INVALID_DEVID),
+        WC_NO_ERR_TRACE(BAD_FUNC_ARG));
+
+    /* Size queries reject NULL key and NULL output. */
+    ExpectIntEQ(wc_FrodoKemKey_Init(key, type, NULL, INVALID_DEVID), 0);
+    ExpectIntEQ(wc_FrodoKemKey_PublicKeySize(NULL, &len),
+        WC_NO_ERR_TRACE(BAD_FUNC_ARG));
+    ExpectIntEQ(wc_FrodoKemKey_PublicKeySize(key, NULL),
+        WC_NO_ERR_TRACE(BAD_FUNC_ARG));
+    ExpectIntEQ(wc_FrodoKemKey_PrivateKeySize(NULL, &len),
+        WC_NO_ERR_TRACE(BAD_FUNC_ARG));
+    ExpectIntEQ(wc_FrodoKemKey_PrivateKeySize(key, NULL),
+        WC_NO_ERR_TRACE(BAD_FUNC_ARG));
+    ExpectIntEQ(wc_FrodoKemKey_CipherTextSize(NULL, &len),
+        WC_NO_ERR_TRACE(BAD_FUNC_ARG));
+    ExpectIntEQ(wc_FrodoKemKey_CipherTextSize(key, NULL),
+        WC_NO_ERR_TRACE(BAD_FUNC_ARG));
+    ExpectIntEQ(wc_FrodoKemKey_SharedSecretSize(NULL, &len),
+        WC_NO_ERR_TRACE(BAD_FUNC_ARG));
+    ExpectIntEQ(wc_FrodoKemKey_SharedSecretSize(key, NULL),
+        WC_NO_ERR_TRACE(BAD_FUNC_ARG));
+
+    ExpectIntEQ(wc_FrodoKemKey_PublicKeySize(key, &pkLen), 0);
+    ExpectIntEQ(wc_FrodoKemKey_PrivateKeySize(key, &skLen), 0);
+
+    /* Make key argument and length checks. */
+#ifndef WOLFSSL_FRODOKEM_NO_MAKE_KEY
+    ExpectIntEQ(wc_FrodoKemKey_MakeKeyWithRandom(NULL, small, 0),
+        WC_NO_ERR_TRACE(BAD_FUNC_ARG));
+    ExpectIntEQ(wc_FrodoKemKey_MakeKeyWithRandom(key, NULL, 0),
+        WC_NO_ERR_TRACE(BAD_FUNC_ARG));
+    ExpectIntEQ(wc_FrodoKemKey_MakeKeyWithRandom(key, small, 1),
+        WC_NO_ERR_TRACE(BUFFER_E));
+#ifndef WC_NO_RNG
+    ExpectIntEQ(wc_FrodoKemKey_MakeKey(NULL, NULL),
+        WC_NO_ERR_TRACE(BAD_FUNC_ARG));
+    ExpectIntEQ(wc_FrodoKemKey_MakeKey(key, NULL),
+        WC_NO_ERR_TRACE(BAD_FUNC_ARG));
+#endif
+#endif /* !WOLFSSL_FRODOKEM_NO_MAKE_KEY */
+
+    /* Encapsulate argument checks. */
+#ifndef WOLFSSL_FRODOKEM_NO_ENCAPSULATE
+#ifndef WC_NO_RNG
+    ExpectIntEQ(wc_FrodoKemKey_Encapsulate(NULL, small, small, NULL),
+        WC_NO_ERR_TRACE(BAD_FUNC_ARG));
+#endif
+    ExpectIntEQ(wc_FrodoKemKey_EncapsulateWithRandom(NULL, small, small,
+        small, 0), WC_NO_ERR_TRACE(BAD_FUNC_ARG));
+    /* Encapsulate on a key with no public key set reports bad state. */
+    ExpectIntEQ(wc_FrodoKemKey_EncapsulateWithRandom(key, small, small,
+        small, 0), WC_NO_ERR_TRACE(BAD_STATE_E));
+#endif /* !WOLFSSL_FRODOKEM_NO_ENCAPSULATE */
+
+    /* Decapsulate argument checks. */
+#ifndef WOLFSSL_FRODOKEM_NO_DECAPSULATE
+    ExpectIntEQ(wc_FrodoKemKey_Decapsulate(NULL, small, small, 0),
+        WC_NO_ERR_TRACE(BAD_FUNC_ARG));
+#endif
+
+    /* Encode / decode NULL handling. */
+    ExpectIntEQ(wc_FrodoKemKey_EncodePublicKey(NULL, small, pkLen),
+        WC_NO_ERR_TRACE(BAD_FUNC_ARG));
+    ExpectIntEQ(wc_FrodoKemKey_EncodePublicKey(key, NULL, pkLen),
+        WC_NO_ERR_TRACE(BAD_FUNC_ARG));
+    ExpectIntEQ(wc_FrodoKemKey_EncodePrivateKey(NULL, small, skLen),
+        WC_NO_ERR_TRACE(BAD_FUNC_ARG));
+    ExpectIntEQ(wc_FrodoKemKey_EncodePrivateKey(key, NULL, skLen),
+        WC_NO_ERR_TRACE(BAD_FUNC_ARG));
+    ExpectIntEQ(wc_FrodoKemKey_DecodePublicKey(NULL, small, pkLen),
+        WC_NO_ERR_TRACE(BAD_FUNC_ARG));
+    ExpectIntEQ(wc_FrodoKemKey_DecodePublicKey(key, NULL, pkLen),
+        WC_NO_ERR_TRACE(BAD_FUNC_ARG));
+    ExpectIntEQ(wc_FrodoKemKey_DecodePrivateKey(NULL, small, skLen),
+        WC_NO_ERR_TRACE(BAD_FUNC_ARG));
+    ExpectIntEQ(wc_FrodoKemKey_DecodePrivateKey(key, NULL, skLen),
+        WC_NO_ERR_TRACE(BAD_FUNC_ARG));
+
+    /* Encoding before any key material is set must report bad state. */
+    ExpectIntEQ(wc_FrodoKemKey_EncodePublicKey(key, small, pkLen),
+        WC_NO_ERR_TRACE(BAD_STATE_E));
+    ExpectIntEQ(wc_FrodoKemKey_EncodePrivateKey(key, small, skLen),
+        WC_NO_ERR_TRACE(BAD_STATE_E));
+
+    /* Wrong encoded lengths on decode must report a buffer error. */
+    ExpectIntEQ(wc_FrodoKemKey_DecodePublicKey(key, small, pkLen - 1),
+        WC_NO_ERR_TRACE(BUFFER_E));
+    ExpectIntEQ(wc_FrodoKemKey_DecodePrivateKey(key, small, skLen - 1),
+        WC_NO_ERR_TRACE(BUFFER_E));
+
+    wc_FrodoKemKey_Free(key);
+    XFREE(key, NULL, DYNAMIC_TYPE_TMP_BUFFER);
+#endif
+    return EXPECT_RESULT();
+}
+
+/* Encapsulate and decapsulate length checks on a fully made key. */
+int test_wc_frodokem_op_len_checks(void)
+{
+    EXPECT_DECLS;
+#if defined(WOLFSSL_HAVE_FRODOKEM) && !defined(WC_NO_RNG) && \
+    !defined(WOLFSSL_FRODOKEM_NO_MAKE_KEY) && \
+    !defined(WOLFSSL_FRODOKEM_NO_ENCAPSULATE) && \
+    !defined(WOLFSSL_FRODOKEM_NO_DECAPSULATE)
+    FrodoKemKey* key = NULL;
+    WC_RNG rng;
+    byte* ct = NULL;
+    byte ss[FRODOKEM_MAX_LENSEC];
+    byte rnd[8];
+    word32 ctLen = 0;
+    int rngInit = 0;
+
+    XMEMSET(&rng, 0, sizeof(rng));
+    XMEMSET(rnd, 0, sizeof(rnd));
+    key = (FrodoKemKey*)XMALLOC(sizeof(*key), NULL, DYNAMIC_TYPE_TMP_BUFFER);
+    ExpectNotNull(key);
+    ct = (byte*)XMALLOC(FRODOKEM_MAX_CIPHER_TEXT_SIZE, NULL,
+        DYNAMIC_TYPE_TMP_BUFFER);
+    ExpectNotNull(ct);
+    ExpectIntEQ(wc_InitRng(&rng), 0);
+    if (EXPECT_SUCCESS()) {
+        rngInit = 1;
+    }
+
+    ExpectIntEQ(wc_FrodoKemKey_Init(key, frodokem_types[0], NULL,
+        INVALID_DEVID), 0);
+    ExpectIntEQ(wc_FrodoKemKey_CipherTextSize(key, &ctLen), 0);
+    ExpectIntEQ(wc_FrodoKemKey_MakeKey(key, &rng), 0);
+
+    /* Encapsulate with the wrong random length reports a buffer error (the
+     * public key is set, so this reaches the length check, not bad state). */
+    ExpectIntEQ(wc_FrodoKemKey_EncapsulateWithRandom(key, ct, ss, rnd, 1),
+        WC_NO_ERR_TRACE(BUFFER_E));
+    /* Decapsulate with the wrong ciphertext length reports a buffer error. */
+    ExpectIntEQ(wc_FrodoKemKey_Decapsulate(key, ss, ct, ctLen - 1),
+        WC_NO_ERR_TRACE(BUFFER_E));
+
+    if (rngInit) {
+        DoExpectIntEQ(wc_FreeRng(&rng), 0);
+    }
+    wc_FrodoKemKey_Free(key);
+    XFREE(ct, NULL, DYNAMIC_TYPE_TMP_BUFFER);
+    XFREE(key, NULL, DYNAMIC_TYPE_TMP_BUFFER);
+#endif
+    return EXPECT_RESULT();
+}
+
+/* Constructor and destructor happy path. */
+int test_wc_frodokem_new_delete(void)
+{
+    EXPECT_DECLS;
+#if defined(WOLFSSL_HAVE_FRODOKEM) && !defined(WC_NO_CONSTRUCTORS)
+    FrodoKemKey* key = NULL;
+    word32 pkLen = 0;
+
+    /* New allocates and initializes the object. */
+    key = wc_FrodoKemKey_New(frodokem_types[0], NULL, INVALID_DEVID);
+    ExpectNotNull(key);
+    /* Initialized: a size query on it succeeds and returns a non-zero size. */
+    ExpectIntEQ(wc_FrodoKemKey_PublicKeySize(key, &pkLen), 0);
+    ExpectIntGT(pkLen, 0);
+    /* Delete frees the object and clears the caller's pointer. */
+    ExpectIntEQ(wc_FrodoKemKey_Delete(key, &key), 0);
+    ExpectNull(key);
+#endif
+    return EXPECT_RESULT();
+}
+
+/* A validly-formed type whose parameter set or A-generation method is not
+ * compiled in must report NOT_COMPILED_IN (an ill-formed type is
+ * BAD_FUNC_ARG). The compiled-variant list is the source of truth, so this
+ * exercises the not-compiled path in single-variant / method-restricted builds
+ * and is a self-consistent no-op when every variant is enabled. */
+int test_wc_frodokem_not_compiled_in(void)
+{
+    EXPECT_DECLS;
+#if defined(WOLFSSL_HAVE_FRODOKEM)
+    /* All twelve validly-formed variant types. These are bit combinations,
+     * always defined regardless of what is compiled in. */
+    static const int allTypes[] = {
+        WC_FRODOKEM_640_SHAKE, WC_FRODOKEM_976_SHAKE, WC_FRODOKEM_1344_SHAKE,
+        WC_FRODOKEM_640_AES, WC_FRODOKEM_976_AES, WC_FRODOKEM_1344_AES,
+        WC_EFRODOKEM_640_SHAKE, WC_EFRODOKEM_976_SHAKE, WC_EFRODOKEM_1344_SHAKE,
+        WC_EFRODOKEM_640_AES, WC_EFRODOKEM_976_AES, WC_EFRODOKEM_1344_AES
+    };
+    FrodoKemKey* key = NULL;
+    int i, j, compiled;
+
+    key = (FrodoKemKey*)XMALLOC(sizeof(*key), NULL, DYNAMIC_TYPE_TMP_BUFFER);
+    ExpectNotNull(key);
+
+    for (i = 0; (i < (int)(sizeof(allTypes) / sizeof(allTypes[0]))) &&
+            EXPECT_SUCCESS(); i++) {
+        /* Is this type one of the compiled-in variants? */
+        compiled = 0;
+        for (j = 0; j < FRODOKEM_TYPE_CNT; j++) {
+            if (frodokem_types[j] == allTypes[i]) {
+                compiled = 1;
+                break;
+            }
+        }
+        if (compiled) {
+            /* A compiled variant initializes. */
+            ExpectIntEQ(wc_FrodoKemKey_Init(key, allTypes[i], NULL,
+                INVALID_DEVID), 0);
+            wc_FrodoKemKey_Free(key);
+        }
+        else {
+            /* A valid but not-compiled variant reports NOT_COMPILED_IN. */
+            ExpectIntEQ(wc_FrodoKemKey_Init(key, allTypes[i], NULL,
+                INVALID_DEVID), WC_NO_ERR_TRACE(NOT_COMPILED_IN));
+        }
+    }
+
+    XFREE(key, NULL, DYNAMIC_TYPE_TMP_BUFFER);
+#endif
+    return EXPECT_RESULT();
+}
diff --git a/tests/api/test_frodokem.h b/tests/api/test_frodokem.h
new file mode 100644
index 0000000000..ffbdb66ccc
--- /dev/null
+++ b/tests/api/test_frodokem.h
@@ -0,0 +1,54 @@
+/* test_frodokem.h
+ *
+ * Copyright (C) 2006-2026 wolfSSL Inc.
+ *
+ * This file is part of wolfSSL.
+ *
+ * wolfSSL is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * wolfSSL is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335, USA
+ */
+
+#ifndef WOLFCRYPT_TEST_FRODOKEM_H
+#define WOLFCRYPT_TEST_FRODOKEM_H
+
+#include <tests/api/api_decl.h>
+
+int test_wc_frodokem_make_key_kats(void);
+int test_wc_frodokem_encapsulate_kats(void);
+int test_wc_frodokem_decapsulate_kats(void);
+int test_wc_frodokem_roundtrip(void);
+int test_wc_frodokem_encode_decode(void);
+int test_wc_frodokem_decap_implicit_reject(void);
+int test_wc_frodokem_decapsulate_pubonly_fails(void);
+int test_wc_frodokem_decode_privkey_bad_pkh(void);
+int test_wc_frodokem_bad_args(void);
+int test_wc_frodokem_op_len_checks(void);
+int test_wc_frodokem_new_delete(void);
+int test_wc_frodokem_not_compiled_in(void);
+
+#define TEST_FRODOKEM_DECLS                                                 \
+    TEST_DECL_GROUP("frodokem", test_wc_frodokem_make_key_kats),            \
+    TEST_DECL_GROUP("frodokem", test_wc_frodokem_encapsulate_kats),         \
+    TEST_DECL_GROUP("frodokem", test_wc_frodokem_decapsulate_kats),         \
+    TEST_DECL_GROUP("frodokem", test_wc_frodokem_roundtrip),                \
+    TEST_DECL_GROUP("frodokem", test_wc_frodokem_encode_decode),            \
+    TEST_DECL_GROUP("frodokem", test_wc_frodokem_decap_implicit_reject),    \
+    TEST_DECL_GROUP("frodokem", test_wc_frodokem_decapsulate_pubonly_fails),\
+    TEST_DECL_GROUP("frodokem", test_wc_frodokem_decode_privkey_bad_pkh),   \
+    TEST_DECL_GROUP("frodokem", test_wc_frodokem_bad_args),                 \
+    TEST_DECL_GROUP("frodokem", test_wc_frodokem_op_len_checks),            \
+    TEST_DECL_GROUP("frodokem", test_wc_frodokem_new_delete),               \
+    TEST_DECL_GROUP("frodokem", test_wc_frodokem_not_compiled_in)
+
+#endif /* WOLFCRYPT_TEST_FRODOKEM_H */
diff --git a/wolfcrypt/benchmark/benchmark.c b/wolfcrypt/benchmark/benchmark.c
index c8b5bf834f..db23e59101 100644
--- a/wolfcrypt/benchmark/benchmark.c
+++ b/wolfcrypt/benchmark/benchmark.c
@@ -171,6 +171,9 @@
 #ifdef WOLFSSL_HAVE_MLKEM
     #include <wolfssl/wolfcrypt/wc_mlkem.h>
 #endif
+#ifdef WOLFSSL_HAVE_FRODOKEM
+    #include <wolfssl/wolfcrypt/wc_frodokem.h>
+#endif
 #if defined(WOLFSSL_HAVE_LMS) && !defined(WOLFSSL_LMS_VERIFY_ONLY)
     #include <wolfssl/wolfcrypt/wc_lms.h>
 #endif
@@ -914,6 +917,12 @@ static WC_INLINE void bench_append_memory_info(char* buffer, size_t size,
 #define BENCH_ML_KEM_1024               0x00000080
 #define BENCH_ML_KEM                    (BENCH_ML_KEM_512 | BENCH_ML_KEM_768 | \
                                          BENCH_ML_KEM_1024)
+#define BENCH_FRODOKEM_640              0x00000100
+#define BENCH_FRODOKEM_976              0x00000200
+#define BENCH_FRODOKEM_1344             0x00000400
+#define BENCH_FRODOKEM                  (BENCH_FRODOKEM_640 | \
+                                         BENCH_FRODOKEM_976 | \
+                                         BENCH_FRODOKEM_1344)
 #define BENCH_FALCON_LEVEL1_SIGN        0x00000001
 #define BENCH_FALCON_LEVEL5_SIGN        0x00000002
 #define BENCH_ML_DSA_44_SIGN            0x04000000
@@ -1359,8 +1368,8 @@ static const bench_pq_hash_sig_alg bench_pq_hash_sig_opt[] = {
 #endif /* !WOLFSSL_BENCHMARK_ALL && !NO_MAIN_DRIVER */
 
 #if !defined(WOLFSSL_BENCHMARK_ALL) && !defined(MAIN_NO_ARGS)
-#if defined(WOLFSSL_HAVE_MLKEM) || defined(HAVE_FALCON) || \
-    defined(WOLFSSL_HAVE_MLDSA)
+#if defined(WOLFSSL_HAVE_MLKEM) || defined(WOLFSSL_HAVE_FRODOKEM) || \
+    defined(HAVE_FALCON) || defined(WOLFSSL_HAVE_MLDSA)
 /* The post-quantum-specific mapping of command line option to bit values and
  * OQS name. */
 typedef struct bench_pq_alg {
@@ -1384,6 +1393,12 @@ static const bench_pq_alg bench_pq_asym_opt[] = {
     { "-ml-kem-768",        BENCH_ML_KEM_768        },
     { "-ml-kem-1024",       BENCH_ML_KEM_1024       },
 #endif
+#ifdef WOLFSSL_HAVE_FRODOKEM
+    { "-frodokem",          BENCH_FRODOKEM          },
+    { "-frodokem-640",      BENCH_FRODOKEM_640      },
+    { "-frodokem-976",      BENCH_FRODOKEM_976      },
+    { "-frodokem-1344",     BENCH_FRODOKEM_1344     },
+#endif
 #if defined(HAVE_FALCON)
     { "-falcon_level1",     BENCH_FALCON_LEVEL1_SIGN },
     { "-falcon_level5",     BENCH_FALCON_LEVEL5_SIGN },
@@ -4458,6 +4473,26 @@ static void* benchmarks_do(void* args)
     }
 #endif
 
+#ifdef WOLFSSL_HAVE_FRODOKEM
+    if (bench_all || (bench_pq_asym_algs & BENCH_FRODOKEM)) {
+    #ifdef WOLFSSL_WC_FRODOKEM_640
+        if (bench_all || (bench_pq_asym_algs & BENCH_FRODOKEM_640)) {
+            bench_frodokem(WC_FRODOKEM_640);
+        }
+    #endif
+    #ifdef WOLFSSL_WC_FRODOKEM_976
+        if (bench_all || (bench_pq_asym_algs & BENCH_FRODOKEM_976)) {
+            bench_frodokem(WC_FRODOKEM_976);
+        }
+    #endif
+    #ifdef WOLFSSL_WC_FRODOKEM_1344
+        if (bench_all || (bench_pq_asym_algs & BENCH_FRODOKEM_1344)) {
+            bench_frodokem(WC_FRODOKEM_1344);
+        }
+    #endif
+    }
+#endif
+
 #if defined(WOLFSSL_HAVE_LMS) && !defined(WOLFSSL_LMS_VERIFY_ONLY)
     if (bench_all || (bench_pq_hash_sig_algs & BENCH_LMS_HSS)) {
         bench_lms();
@@ -11439,6 +11474,262 @@ void bench_mlkem(int type)
 }
 #endif
 
+#ifdef WOLFSSL_HAVE_FRODOKEM
+static void bench_frodokem_keygen(int type, const char* name, int keySize,
+    FrodoKemKey* key)
+{
+#ifndef WOLFSSL_FRODOKEM_NO_MAKE_KEY
+    int ret = 0, times, count, pending = 0;
+    double start;
+    const char**desc = bench_desc_words[lng_index];
+    DECLARE_MULTI_VALUE_STATS_VARS()
+
+    bench_stats_prepare();
+
+    /* FrodoKEM Make Key */
+    bench_stats_start(&count, &start);
+    do {
+        /* while free pending slots in queue, submit ops */
+        for (times = 0; times < agreeTimes || pending > 0; times++) {
+            wc_FrodoKemKey_Free(key);
+            ret = wc_FrodoKemKey_Init(key, type, HEAP_HINT, INVALID_DEVID);
+            if (ret != 0)
+                goto exit;
+
+            ret = wc_FrodoKemKey_MakeKey(key, &gRng);
+            if (ret != 0)
+                goto exit;
+            RECORD_MULTI_VALUE_STATS();
+        } /* for times */
+        count += times;
+    } while (bench_stats_check(start)
+#ifdef MULTI_VALUE_STATISTICS
+       || runs < minimum_runs
+#endif
+       );
+
+exit:
+    bench_stats_asym_finish(name, keySize, desc[2], 0, count, start, ret);
+#ifdef MULTI_VALUE_STATISTICS
+    bench_multi_value_stats(max, min, sum, squareSum, runs);
+#endif
+#else
+   (void)type;
+   (void)name;
+   (void)keySize;
+   (void)key;
+#endif /* !WOLFSSL_FRODOKEM_NO_MAKE_KEY */
+}
+
+#if !defined(WOLFSSL_FRODOKEM_NO_ENCAPSULATE) || \
+    !defined(WOLFSSL_FRODOKEM_NO_DECAPSULATE)
+static void bench_frodokem_encap(int type, const char* name, int keySize,
+    FrodoKemKey* key1, FrodoKemKey* key2)
+{
+    int ret = 0, times, count, pending = 0;
+    double start;
+    const char**desc = bench_desc_words[lng_index];
+    WC_DECLARE_VAR(ct, byte, FRODOKEM_MAX_CIPHER_TEXT_SIZE, HEAP_HINT);
+    WC_DECLARE_VAR(ss, byte, FRODOKEM_MAX_LENSEC, HEAP_HINT);
+    WC_DECLARE_VAR(pub, byte, FRODOKEM_MAX_PUBLIC_KEY_SIZE, HEAP_HINT);
+    word32 pubLen;
+    word32 ctSz;
+    DECLARE_MULTI_VALUE_STATS_VARS()
+
+    bench_stats_prepare();
+
+    WC_ALLOC_VAR(ct, byte, FRODOKEM_MAX_CIPHER_TEXT_SIZE, HEAP_HINT);
+    WC_ALLOC_VAR(ss, byte, FRODOKEM_MAX_LENSEC, HEAP_HINT);
+    WC_ALLOC_VAR(pub, byte, FRODOKEM_MAX_PUBLIC_KEY_SIZE, HEAP_HINT);
+
+    ret = wc_FrodoKemKey_PublicKeySize(key1, &pubLen);
+    if (ret != 0) {
+        goto exit;
+    }
+    ret = wc_FrodoKemKey_EncodePublicKey(key1, pub, pubLen);
+    if (ret != 0) {
+        goto exit;
+    }
+    ret = wc_FrodoKemKey_Init(key2, type, HEAP_HINT, INVALID_DEVID);
+    if (ret != 0) {
+        goto exit;
+    }
+    ret = wc_FrodoKemKey_DecodePublicKey(key2, pub, pubLen);
+    if (ret != 0) {
+        goto exit;
+    }
+
+    ret = wc_FrodoKemKey_CipherTextSize(key2, &ctSz);
+    if (ret != 0) {
+        goto exit;
+    }
+
+#ifndef WOLFSSL_FRODOKEM_NO_ENCAPSULATE
+    /* FrodoKEM Encapsulate */
+    bench_stats_start(&count, &start);
+    do {
+        /* while free pending slots in queue, submit ops */
+        for (times = 0; times < agreeTimes || pending > 0; times++) {
+            ret = wc_FrodoKemKey_Encapsulate(key2, ct, ss, &gRng);
+            if (ret != 0)
+                goto exit_encap;
+            RECORD_MULTI_VALUE_STATS();
+        } /* for times */
+        count += times;
+    } while (bench_stats_check(start)
+#ifdef MULTI_VALUE_STATISTICS
+       || runs < minimum_runs
+#endif
+       );
+
+exit_encap:
+    bench_stats_asym_finish(name, keySize, desc[9], 0, count, start, ret);
+#ifdef MULTI_VALUE_STATISTICS
+    bench_multi_value_stats(max, min, sum, squareSum, runs);
+#endif
+#endif
+
+#ifndef WOLFSSL_FRODOKEM_NO_DECAPSULATE
+    RESET_MULTI_VALUE_STATS_VARS();
+
+    /* FrodoKEM Decapsulate */
+    PRIVATE_KEY_UNLOCK();
+    bench_stats_start(&count, &start);
+    do {
+        /* while free pending slots in queue, submit ops */
+        for (times = 0; times < agreeTimes || pending > 0; times++) {
+            ret = wc_FrodoKemKey_Decapsulate(key1, ss, ct, ctSz);
+            if (ret != 0)
+                goto exit_decap;
+            RECORD_MULTI_VALUE_STATS();
+        } /* for times */
+        count += times;
+    } while (bench_stats_check(start)
+#ifdef MULTI_VALUE_STATISTICS
+       || runs < minimum_runs
+#endif
+       );
+
+exit_decap:
+    PRIVATE_KEY_LOCK();
+    bench_stats_asym_finish(name, keySize, desc[13], 0, count, start, ret);
+#ifdef MULTI_VALUE_STATISTICS
+    bench_multi_value_stats(max, min, sum, squareSum, runs);
+#endif
+
+#endif
+
+exit:
+
+    WC_FREE_VAR(ct, HEAP_HINT);
+    WC_FREE_VAR(ss, HEAP_HINT);
+    WC_FREE_VAR(pub, HEAP_HINT);
+
+    if (ret != 0)
+        printf("error: bench_frodokem_encap() failed with code %d.\n", ret);
+
+    return;
+}
+#endif
+
+void bench_frodokem(int type)
+{
+#ifdef WOLFSSL_SMALL_STACK
+    FrodoKemKey *key1 = NULL;
+    FrodoKemKey *key2 = NULL;
+#else
+    FrodoKemKey key1[1];
+    FrodoKemKey key2[1];
+#endif
+    /* Matrix-A method and eFrodoKEM modifiers to layer onto the base parameter
+     * set. Only combinations that are compiled in are listed, so every built
+     * variant of the requested parameter set is benchmarked. */
+    static const int mods[] = {
+#ifdef WOLFSSL_FRODOKEM_SHAKE
+        0,
+    #ifdef WOLFSSL_FRODOKEM_EPHEMERAL
+        FRODOKEM_EPHEMERAL,
+    #endif
+#endif
+#ifdef WOLFSSL_FRODOKEM_AES
+        FRODOKEM_AES,
+    #ifdef WOLFSSL_FRODOKEM_EPHEMERAL
+        FRODOKEM_AES | FRODOKEM_EPHEMERAL,
+    #endif
+#endif
+    };
+    int base = type & ~(FRODOKEM_AES | FRODOKEM_EPHEMERAL);
+    const char* baseName = NULL;
+    int keySize = 0;
+    size_t v;
+
+    switch (base) {
+#ifdef WOLFSSL_WC_FRODOKEM_640
+    case WC_FRODOKEM_640:
+        baseName = "640 ";
+        keySize = 128;
+        break;
+#endif
+#ifdef WOLFSSL_WC_FRODOKEM_976
+    case WC_FRODOKEM_976:
+        baseName = "976 ";
+        keySize = 192;
+        break;
+#endif
+#ifdef WOLFSSL_WC_FRODOKEM_1344
+    case WC_FRODOKEM_1344:
+        baseName = "1344";
+        keySize = 256;
+        break;
+#endif
+    default:
+        return;
+    }
+
+#ifdef WOLFSSL_SMALL_STACK
+    key1 = (FrodoKemKey *)XMALLOC(sizeof(*key1), HEAP_HINT,
+        DYNAMIC_TYPE_TMP_BUFFER);
+    if (key1 == NULL)
+        return;
+    key2 = (FrodoKemKey *)XMALLOC(sizeof(*key2), HEAP_HINT,
+        DYNAMIC_TYPE_TMP_BUFFER);
+    if (key2 == NULL) {
+        XFREE(key1, HEAP_HINT, DYNAMIC_TYPE_TMP_BUFFER);
+        return;
+    }
+#endif
+
+    /* Zero the key objects so the wc_FrodoKemKey_Free calls below are safe even
+     * if a benchmark helper returns before initializing its key. */
+    XMEMSET(key1, 0, sizeof(*key1));
+    XMEMSET(key2, 0, sizeof(*key2));
+
+    for (v = 0; v < sizeof(mods) / sizeof(mods[0]); v++) {
+        int mod = mods[v];
+        int fullType = base | mod;
+        /* e.g. "FRODOKEM 640  SHAKE" or "EFRODOKEM 1344 AES". */
+        char name[24];
+
+        (void)XSNPRINTF(name, sizeof(name), "%sFRODOKEM %s %s",
+            (mod & FRODOKEM_EPHEMERAL) ? "E" : "",
+            baseName,
+            (mod & FRODOKEM_AES) ? "AES" : "SHAKE");
+
+        bench_frodokem_keygen(fullType, name, keySize, key1);
+#if !defined(WOLFSSL_FRODOKEM_NO_ENCAPSULATE) || \
+    !defined(WOLFSSL_FRODOKEM_NO_DECAPSULATE)
+        bench_frodokem_encap(fullType, name, keySize, key1, key2);
+#endif
+    }
+
+    wc_FrodoKemKey_Free(key2);
+    wc_FrodoKemKey_Free(key1);
+
+    WC_FREE_VAR_EX(key1, HEAP_HINT, DYNAMIC_TYPE_TMP_BUFFER);
+    WC_FREE_VAR_EX(key2, HEAP_HINT, DYNAMIC_TYPE_TMP_BUFFER);
+}
+#endif
+
 #if defined(WOLFSSL_HAVE_LMS) && !defined(WOLFSSL_LMS_VERIFY_ONLY)
 #ifndef WOLFSSL_WC_LMS_SERIALIZE_STATE
 #ifndef WOLFSSL_NO_LMS_SHA256_256
diff --git a/wolfcrypt/benchmark/benchmark.h b/wolfcrypt/benchmark/benchmark.h
index 272e657843..3c97e0b300 100644
--- a/wolfcrypt/benchmark/benchmark.h
+++ b/wolfcrypt/benchmark/benchmark.h
@@ -104,6 +104,7 @@ void bench_rsa(int useDeviceID);
 void bench_rsa_key(int useDeviceID, word32 keySz);
 void bench_dh(int useDeviceID);
 void bench_mlkem(int type);
+void bench_frodokem(int type);
 void bench_lms(void);
 void bench_xmss(int hash);
 void bench_slhdsa(int param);
diff --git a/wolfcrypt/src/aes_gcm_asm.asm b/wolfcrypt/src/aes_gcm_asm.asm
index 54bcfc033e..c0de3a6f5d 100644
--- a/wolfcrypt/src/aes_gcm_asm.asm
+++ b/wolfcrypt/src/aes_gcm_asm.asm
@@ -6459,8 +6459,7 @@ _TEXT ENDS
 IFDEF WOLFSSL_AESGCM_SIV
 _DATA SEGMENT
 ALIGN 16
-L_aes_gcm_siv_bswap_mask QWORD \
-     08090a0b0c0d0e0fh,  0001020304050607h
+L_aes_gcm_siv_bswap_mask QWORD 08090a0b0c0d0e0fh, 0001020304050607h
 ptr_L_aes_gcm_siv_bswap_mask QWORD L_aes_gcm_siv_bswap_mask
 _DATA ENDS
 _TEXT SEGMENT READONLY PARA
@@ -6825,8 +6824,7 @@ AES_GCMSIV_polyval_aesni ENDP
 _TEXT ENDS
 _DATA SEGMENT
 ALIGN 16
-L_aes_gcmsiv_ctr_aesni_one QWORD \
-     0000000000000001h,  0000000000000000h
+L_aes_gcmsiv_ctr_aesni_one QWORD 0000000000000001h, 0000000000000000h
 ptr_L_aes_gcmsiv_ctr_aesni_one QWORD L_aes_gcmsiv_ctr_aesni_one
 _DATA ENDS
 _TEXT SEGMENT READONLY PARA
@@ -12471,8 +12469,7 @@ _TEXT ENDS
 IFDEF WOLFSSL_AESGCM_SIV
 _DATA SEGMENT
 ALIGN 16
-L_aes_gcm_siv_bswap_mask_avx1 QWORD \
-     08090a0b0c0d0e0fh,  0001020304050607h
+L_aes_gcm_siv_bswap_mask_avx1 QWORD 08090a0b0c0d0e0fh, 0001020304050607h
 ptr_L_aes_gcm_siv_bswap_mask_avx1 QWORD L_aes_gcm_siv_bswap_mask_avx1
 _DATA ENDS
 _TEXT SEGMENT READONLY PARA
@@ -12738,8 +12735,7 @@ AES_GCMSIV_polyval_avx1 ENDP
 _TEXT ENDS
 _DATA SEGMENT
 ALIGN 16
-L_aes_gcmsiv_ctr_avx1_one QWORD \
-     0000000000000001h,  0000000000000000h
+L_aes_gcmsiv_ctr_avx1_one QWORD 0000000000000001h, 0000000000000000h
 ptr_L_aes_gcmsiv_ctr_avx1_one QWORD L_aes_gcmsiv_ctr_avx1_one
 _DATA ENDS
 _TEXT SEGMENT READONLY PARA
@@ -25206,8 +25202,7 @@ _TEXT ENDS
 IFDEF WOLFSSL_AESGCM_SIV
 _DATA SEGMENT
 ALIGN 16
-L_aes_gcm_siv_bswap_mask_vaes QWORD \
-     08090a0b0c0d0e0fh,  0001020304050607h
+L_aes_gcm_siv_bswap_mask_vaes QWORD 08090a0b0c0d0e0fh, 0001020304050607h
 ptr_L_aes_gcm_siv_bswap_mask_vaes QWORD L_aes_gcm_siv_bswap_mask_vaes
 _DATA ENDS
 _TEXT SEGMENT READONLY PARA
@@ -25743,24 +25738,23 @@ AES_GCMSIV_polyval_vaes ENDP
 _TEXT ENDS
 _DATA SEGMENT
 ALIGN 16
-L_aes_gcmsiv_ctr_inc_vaes QWORD \
-     0000000000000000h,  0000000000000000h,
-     0000000000000001h,  0000000000000000h,
-     0000000000000002h,  0000000000000000h,
-     0000000000000003h,  0000000000000000h,
-     0000000000000004h,  0000000000000000h,
-     0000000000000005h,  0000000000000000h,
-     0000000000000006h,  0000000000000000h,
-     0000000000000007h,  0000000000000000h,
-     0000000000000008h,  0000000000000000h,
-     0000000000000009h,  0000000000000000h,
-     000000000000000ah,  0000000000000000h,
-     000000000000000bh,  0000000000000000h,
-     000000000000000ch,  0000000000000000h,
-     000000000000000dh,  0000000000000000h,
-     000000000000000eh,  0000000000000000h,
-     000000000000000fh,  0000000000000000h,
-     0000000000000010h,  0000000000000000h
+L_aes_gcmsiv_ctr_inc_vaes QWORD 0000000000000000h, 0000000000000000h
+        QWORD 0000000000000001h, 0000000000000000h
+        QWORD 0000000000000002h, 0000000000000000h
+        QWORD 0000000000000003h, 0000000000000000h
+        QWORD 0000000000000004h, 0000000000000000h
+        QWORD 0000000000000005h, 0000000000000000h
+        QWORD 0000000000000006h, 0000000000000000h
+        QWORD 0000000000000007h, 0000000000000000h
+        QWORD 0000000000000008h, 0000000000000000h
+        QWORD 0000000000000009h, 0000000000000000h
+        QWORD 000000000000000ah, 0000000000000000h
+        QWORD 000000000000000bh, 0000000000000000h
+        QWORD 000000000000000ch, 0000000000000000h
+        QWORD 000000000000000dh, 0000000000000000h
+        QWORD 000000000000000eh, 0000000000000000h
+        QWORD 000000000000000fh, 0000000000000000h
+        QWORD 0000000000000010h, 0000000000000000h
 ptr_L_aes_gcmsiv_ctr_inc_vaes QWORD L_aes_gcmsiv_ctr_inc_vaes
 _DATA ENDS
 _TEXT SEGMENT READONLY PARA
@@ -25777,10 +25771,10 @@ AES_GCMSIV_ctr_vaes PROC
         vmovdqu	OWORD PTR [rsp+48], xmm9
         vmovdqu	OWORD PTR [rsp+64], xmm10
         vmovdqu	OWORD PTR [rsp+80], xmm11
-        vbroadcasti128	ymm7, [r10]
-        vbroadcasti128	ymm8, [ptr_L_aes_gcmsiv_ctr_inc_vaes+128]
-        vbroadcasti128	ymm9, [ptr_L_aes_gcmsiv_ctr_inc_vaes+32]
-        vbroadcasti128	ymm10, [ptr_L_aes_gcmsiv_ctr_inc_vaes+16]
+        vbroadcasti128	ymm7, OWORD PTR [r10]
+        vbroadcasti128	ymm8, OWORD PTR [ptr_L_aes_gcmsiv_ctr_inc_vaes+128]
+        vbroadcasti128	ymm9, OWORD PTR [ptr_L_aes_gcmsiv_ctr_inc_vaes+32]
+        vbroadcasti128	ymm10, OWORD PTR [ptr_L_aes_gcmsiv_ctr_inc_vaes+16]
         xor	r11d, r11d
         cmp	r8d, 128
         mov	r12d, r8d
@@ -25803,81 +25797,81 @@ L_AES_GCMSIV_ctr_vaes_enc_128:
         vpaddd	ymm6, ymm6, ymm8
         vpaddd	ymm7, ymm7, ymm8
         ; aes_enc_block
-        vbroadcasti128	ymm11, [r9]
+        vbroadcasti128	ymm11, OWORD PTR [r9]
         vpxor	ymm0, ymm0, ymm11
         vpxor	ymm1, ymm1, ymm11
         vpxor	ymm2, ymm2, ymm11
         vpxor	ymm3, ymm3, ymm11
-        vbroadcasti128	ymm11, [r9+16]
+        vbroadcasti128	ymm11, OWORD PTR [r9+16]
         vaesenc	ymm0, ymm0, ymm11
         vaesenc	ymm1, ymm1, ymm11
         vaesenc	ymm2, ymm2, ymm11
         vaesenc	ymm3, ymm3, ymm11
-        vbroadcasti128	ymm11, [r9+32]
+        vbroadcasti128	ymm11, OWORD PTR [r9+32]
         vaesenc	ymm0, ymm0, ymm11
         vaesenc	ymm1, ymm1, ymm11
         vaesenc	ymm2, ymm2, ymm11
         vaesenc	ymm3, ymm3, ymm11
-        vbroadcasti128	ymm11, [r9+48]
+        vbroadcasti128	ymm11, OWORD PTR [r9+48]
         vaesenc	ymm0, ymm0, ymm11
         vaesenc	ymm1, ymm1, ymm11
         vaesenc	ymm2, ymm2, ymm11
         vaesenc	ymm3, ymm3, ymm11
-        vbroadcasti128	ymm11, [r9+64]
+        vbroadcasti128	ymm11, OWORD PTR [r9+64]
         vaesenc	ymm0, ymm0, ymm11
         vaesenc	ymm1, ymm1, ymm11
         vaesenc	ymm2, ymm2, ymm11
         vaesenc	ymm3, ymm3, ymm11
-        vbroadcasti128	ymm11, [r9+80]
+        vbroadcasti128	ymm11, OWORD PTR [r9+80]
         vaesenc	ymm0, ymm0, ymm11
         vaesenc	ymm1, ymm1, ymm11
         vaesenc	ymm2, ymm2, ymm11
         vaesenc	ymm3, ymm3, ymm11
-        vbroadcasti128	ymm11, [r9+96]
+        vbroadcasti128	ymm11, OWORD PTR [r9+96]
         vaesenc	ymm0, ymm0, ymm11
         vaesenc	ymm1, ymm1, ymm11
         vaesenc	ymm2, ymm2, ymm11
         vaesenc	ymm3, ymm3, ymm11
-        vbroadcasti128	ymm11, [r9+112]
+        vbroadcasti128	ymm11, OWORD PTR [r9+112]
         vaesenc	ymm0, ymm0, ymm11
         vaesenc	ymm1, ymm1, ymm11
         vaesenc	ymm2, ymm2, ymm11
         vaesenc	ymm3, ymm3, ymm11
-        vbroadcasti128	ymm11, [r9+128]
+        vbroadcasti128	ymm11, OWORD PTR [r9+128]
         vaesenc	ymm0, ymm0, ymm11
         vaesenc	ymm1, ymm1, ymm11
         vaesenc	ymm2, ymm2, ymm11
         vaesenc	ymm3, ymm3, ymm11
-        vbroadcasti128	ymm11, [r9+144]
+        vbroadcasti128	ymm11, OWORD PTR [r9+144]
         vaesenc	ymm0, ymm0, ymm11
         vaesenc	ymm1, ymm1, ymm11
         vaesenc	ymm2, ymm2, ymm11
         vaesenc	ymm3, ymm3, ymm11
         cmp	eax, 11
-        vbroadcasti128	ymm11, [r9+160]
+        vbroadcasti128	ymm11, OWORD PTR [r9+160]
         jl	L_AES_GCMSIV_ctr_vaes_128_aes_enc_block_last
         vaesenc	ymm0, ymm0, ymm11
         vaesenc	ymm1, ymm1, ymm11
         vaesenc	ymm2, ymm2, ymm11
         vaesenc	ymm3, ymm3, ymm11
-        vbroadcasti128	ymm11, [r9+176]
+        vbroadcasti128	ymm11, OWORD PTR [r9+176]
         vaesenc	ymm0, ymm0, ymm11
         vaesenc	ymm1, ymm1, ymm11
         vaesenc	ymm2, ymm2, ymm11
         vaesenc	ymm3, ymm3, ymm11
         cmp	eax, 13
-        vbroadcasti128	ymm11, [r9+192]
+        vbroadcasti128	ymm11, OWORD PTR [r9+192]
         jl	L_AES_GCMSIV_ctr_vaes_128_aes_enc_block_last
         vaesenc	ymm0, ymm0, ymm11
         vaesenc	ymm1, ymm1, ymm11
         vaesenc	ymm2, ymm2, ymm11
         vaesenc	ymm3, ymm3, ymm11
-        vbroadcasti128	ymm11, [r9+208]
+        vbroadcasti128	ymm11, OWORD PTR [r9+208]
         vaesenc	ymm0, ymm0, ymm11
         vaesenc	ymm1, ymm1, ymm11
         vaesenc	ymm2, ymm2, ymm11
         vaesenc	ymm3, ymm3, ymm11
-        vbroadcasti128	ymm11, [r9+224]
+        vbroadcasti128	ymm11, OWORD PTR [r9+224]
 L_AES_GCMSIV_ctr_vaes_128_aes_enc_block_last:
         vaesenclast	ymm0, ymm0, ymm11
         vaesenclast	ymm1, ymm1, ymm11
@@ -25908,39 +25902,39 @@ L_AES_GCMSIV_ctr_vaes_enc_32:
         vpaddd	ymm0, ymm7, [ptr_L_aes_gcmsiv_ctr_inc_vaes]
         vpaddd	ymm7, ymm7, ymm9
         ; aes_enc_block
-        vbroadcasti128	ymm11, [r9]
+        vbroadcasti128	ymm11, OWORD PTR [r9]
         vpxor	ymm0, ymm0, ymm11
-        vbroadcasti128	ymm11, [r9+16]
+        vbroadcasti128	ymm11, OWORD PTR [r9+16]
         vaesenc	ymm0, ymm0, ymm11
-        vbroadcasti128	ymm11, [r9+32]
+        vbroadcasti128	ymm11, OWORD PTR [r9+32]
         vaesenc	ymm0, ymm0, ymm11
-        vbroadcasti128	ymm11, [r9+48]
+        vbroadcasti128	ymm11, OWORD PTR [r9+48]
         vaesenc	ymm0, ymm0, ymm11
-        vbroadcasti128	ymm11, [r9+64]
+        vbroadcasti128	ymm11, OWORD PTR [r9+64]
         vaesenc	ymm0, ymm0, ymm11
-        vbroadcasti128	ymm11, [r9+80]
+        vbroadcasti128	ymm11, OWORD PTR [r9+80]
         vaesenc	ymm0, ymm0, ymm11
-        vbroadcasti128	ymm11, [r9+96]
+        vbroadcasti128	ymm11, OWORD PTR [r9+96]
         vaesenc	ymm0, ymm0, ymm11
-        vbroadcasti128	ymm11, [r9+112]
+        vbroadcasti128	ymm11, OWORD PTR [r9+112]
         vaesenc	ymm0, ymm0, ymm11
-        vbroadcasti128	ymm11, [r9+128]
+        vbroadcasti128	ymm11, OWORD PTR [r9+128]
         vaesenc	ymm0, ymm0, ymm11
-        vbroadcasti128	ymm11, [r9+144]
+        vbroadcasti128	ymm11, OWORD PTR [r9+144]
         vaesenc	ymm0, ymm0, ymm11
         cmp	eax, 11
-        vbroadcasti128	ymm11, [r9+160]
+        vbroadcasti128	ymm11, OWORD PTR [r9+160]
         jl	L_AES_GCMSIV_ctr_vaes_32_aes_enc_block_last
         vaesenc	ymm0, ymm0, ymm11
-        vbroadcasti128	ymm11, [r9+176]
+        vbroadcasti128	ymm11, OWORD PTR [r9+176]
         vaesenc	ymm0, ymm0, ymm11
         cmp	eax, 13
-        vbroadcasti128	ymm11, [r9+192]
+        vbroadcasti128	ymm11, OWORD PTR [r9+192]
         jl	L_AES_GCMSIV_ctr_vaes_32_aes_enc_block_last
         vaesenc	ymm0, ymm0, ymm11
-        vbroadcasti128	ymm11, [r9+208]
+        vbroadcasti128	ymm11, OWORD PTR [r9+208]
         vaesenc	ymm0, ymm0, ymm11
-        vbroadcasti128	ymm11, [r9+224]
+        vbroadcasti128	ymm11, OWORD PTR [r9+224]
 L_AES_GCMSIV_ctr_vaes_32_aes_enc_block_last:
         vaesenclast	ymm0, ymm0, ymm11
         vpxor	ymm0, ymm0, [r13]
@@ -35966,8 +35960,7 @@ _TEXT ENDS
 IFDEF WOLFSSL_AESGCM_SIV
 _DATA SEGMENT
 ALIGN 16
-L_aes_gcm_siv_bswap_mask_avx512 QWORD \
-     08090a0b0c0d0e0fh,  0001020304050607h
+L_aes_gcm_siv_bswap_mask_avx512 QWORD 08090a0b0c0d0e0fh, 0001020304050607h
 ptr_L_aes_gcm_siv_bswap_mask_avx512 QWORD L_aes_gcm_siv_bswap_mask_avx512
 _DATA ENDS
 _TEXT SEGMENT READONLY PARA
@@ -36702,24 +36695,23 @@ AES_GCMSIV_polyval_avx512 ENDP
 _TEXT ENDS
 _DATA SEGMENT
 ALIGN 16
-L_aes_gcmsiv_ctr_inc_avx512 QWORD \
-     0000000000000000h,  0000000000000000h,
-     0000000000000001h,  0000000000000000h,
-     0000000000000002h,  0000000000000000h,
-     0000000000000003h,  0000000000000000h,
-     0000000000000004h,  0000000000000000h,
-     0000000000000005h,  0000000000000000h,
-     0000000000000006h,  0000000000000000h,
-     0000000000000007h,  0000000000000000h,
-     0000000000000008h,  0000000000000000h,
-     0000000000000009h,  0000000000000000h,
-     000000000000000ah,  0000000000000000h,
-     000000000000000bh,  0000000000000000h,
-     000000000000000ch,  0000000000000000h,
-     000000000000000dh,  0000000000000000h,
-     000000000000000eh,  0000000000000000h,
-     000000000000000fh,  0000000000000000h,
-     0000000000000010h,  0000000000000000h
+L_aes_gcmsiv_ctr_inc_avx512 QWORD 0000000000000000h, 0000000000000000h
+        QWORD 0000000000000001h, 0000000000000000h
+        QWORD 0000000000000002h, 0000000000000000h
+        QWORD 0000000000000003h, 0000000000000000h
+        QWORD 0000000000000004h, 0000000000000000h
+        QWORD 0000000000000005h, 0000000000000000h
+        QWORD 0000000000000006h, 0000000000000000h
+        QWORD 0000000000000007h, 0000000000000000h
+        QWORD 0000000000000008h, 0000000000000000h
+        QWORD 0000000000000009h, 0000000000000000h
+        QWORD 000000000000000ah, 0000000000000000h
+        QWORD 000000000000000bh, 0000000000000000h
+        QWORD 000000000000000ch, 0000000000000000h
+        QWORD 000000000000000dh, 0000000000000000h
+        QWORD 000000000000000eh, 0000000000000000h
+        QWORD 000000000000000fh, 0000000000000000h
+        QWORD 0000000000000010h, 0000000000000000h
 ptr_L_aes_gcmsiv_ctr_inc_avx512 QWORD L_aes_gcmsiv_ctr_inc_avx512
 _DATA ENDS
 _TEXT SEGMENT READONLY PARA
@@ -36740,32 +36732,32 @@ AES_GCMSIV_ctr_avx512 PROC
         vmovdqu	OWORD PTR [rsp+112], xmm13
         vmovdqu	OWORD PTR [rsp+128], xmm14
         vmovdqu	OWORD PTR [rsp+144], xmm15
-        vbroadcasti32x4	zmm7, [r10]
-        vbroadcasti32x4	zmm8, [ptr_L_aes_gcmsiv_ctr_inc_avx512+256]
-        vbroadcasti32x4	zmm9, [ptr_L_aes_gcmsiv_ctr_inc_avx512+64]
-        vbroadcasti32x4	zmm10, [ptr_L_aes_gcmsiv_ctr_inc_avx512+16]
+        vbroadcasti32x4	zmm7, OWORD PTR [r10]
+        vbroadcasti32x4	zmm8, OWORD PTR [ptr_L_aes_gcmsiv_ctr_inc_avx512+256]
+        vbroadcasti32x4	zmm9, OWORD PTR [ptr_L_aes_gcmsiv_ctr_inc_avx512+64]
+        vbroadcasti32x4	zmm10, OWORD PTR [ptr_L_aes_gcmsiv_ctr_inc_avx512+16]
         xor	r11d, r11d
         cmp	r8d, 64
         jl	L_AES_GCMSIV_ctr_avx512_done_64
-        vbroadcasti32x4	zmm12, [r9]
-        vbroadcasti32x4	zmm13, [r9+16]
-        vbroadcasti32x4	zmm14, [r9+32]
-        vbroadcasti32x4	zmm15, [r9+48]
-        vbroadcasti32x4	zmm16, [r9+64]
-        vbroadcasti32x4	zmm17, [r9+80]
-        vbroadcasti32x4	zmm18, [r9+96]
-        vbroadcasti32x4	zmm19, [r9+112]
-        vbroadcasti32x4	zmm20, [r9+128]
-        vbroadcasti32x4	zmm21, [r9+144]
-        vbroadcasti32x4	zmm22, [r9+160]
+        vbroadcasti32x4	zmm12, OWORD PTR [r9]
+        vbroadcasti32x4	zmm13, OWORD PTR [r9+16]
+        vbroadcasti32x4	zmm14, OWORD PTR [r9+32]
+        vbroadcasti32x4	zmm15, OWORD PTR [r9+48]
+        vbroadcasti32x4	zmm16, OWORD PTR [r9+64]
+        vbroadcasti32x4	zmm17, OWORD PTR [r9+80]
+        vbroadcasti32x4	zmm18, OWORD PTR [r9+96]
+        vbroadcasti32x4	zmm19, OWORD PTR [r9+112]
+        vbroadcasti32x4	zmm20, OWORD PTR [r9+128]
+        vbroadcasti32x4	zmm21, OWORD PTR [r9+144]
+        vbroadcasti32x4	zmm22, OWORD PTR [r9+160]
         cmp	eax, 11
         jl	L_AES_GCMSIV_ctr_avx512_key_cached
-        vbroadcasti32x4	zmm23, [r9+176]
-        vbroadcasti32x4	zmm24, [r9+192]
+        vbroadcasti32x4	zmm23, OWORD PTR [r9+176]
+        vbroadcasti32x4	zmm24, OWORD PTR [r9+192]
         cmp	eax, 13
         jl	L_AES_GCMSIV_ctr_avx512_key_cached
-        vbroadcasti32x4	zmm25, [r9+208]
-        vbroadcasti32x4	zmm26, [r9+224]
+        vbroadcasti32x4	zmm25, OWORD PTR [r9+208]
+        vbroadcasti32x4	zmm26, OWORD PTR [r9+224]
 L_AES_GCMSIV_ctr_avx512_key_cached:
         cmp	r8d, 256
         mov	r12d, r8d
diff --git a/wolfcrypt/src/cpuid.c b/wolfcrypt/src/cpuid.c
index 545e6e927d..a39780c77b 100644
--- a/wolfcrypt/src/cpuid.c
+++ b/wolfcrypt/src/cpuid.c
@@ -154,7 +154,27 @@
 #define CPUID_AARCH64_FEAT_SHA3        ((word64)1 << 32)
 #define CPUID_AARCH64_FEAT_SM3         ((word64)1 << 36)
 #define CPUID_AARCH64_FEAT_SM4         ((word64)1 << 40)
+/* ID_AA64PFR0_EL1.AdvSIMD field [23:20]: 0xf means NEON is NOT implemented. */
 #define CPUID_AARCH64_FEAT_ASIMD       ((word64)0xf << 20)
+/* ID_AA64PFR0_EL1.SVE field [35:32]: non-zero means SVE is implemented. */
+#define CPUID_AARCH64_FEAT_SVE         ((word64)0xf << 32)
+/* ID_AA64PFR1_EL1.SME field [27:24]: non-zero means SME is implemented. */
+#define CPUID_AARCH64_FEAT_SME         ((word64)0xf << 24)
+
+/* SVE and/or SME are guaranteed present when the architecture the code is
+ * compiled for includes them (the ACLE __ARM_FEATURE_* macros). In that case
+ * the flag is set unconditionally: the runtime detection below can be
+ * unavailable or incomplete on some platforms, but the feature is known usable.
+ */
+#if defined(__ARM_FEATURE_SVE) && defined(__ARM_FEATURE_SME)
+    #define CPUID_AARCH64_COMPILED     (CPUID_SVE | CPUID_SME)
+#elif defined(__ARM_FEATURE_SVE)
+    #define CPUID_AARCH64_COMPILED     CPUID_SVE
+#elif defined(__ARM_FEATURE_SME)
+    #define CPUID_AARCH64_COMPILED     CPUID_SME
+#else
+    #define CPUID_AARCH64_COMPILED     0
+#endif
 
 #ifdef WOLFSSL_AARCH64_PRIVILEGE_MODE
     /* https://developer.arm.com/documentation/ddi0601/2024-09/AArch64-Registers
@@ -163,30 +183,39 @@
     static WC_INLINE void cpuid_set_flags(void)
     {
         if (WOLFSSL_ATOMIC_LOAD(cpuid_flags) == WC_CPUID_INITIALIZER) {
-            cpuid_flags_t new_cpuid_flags = 0,
+            cpuid_flags_t new_cpuid_flags = CPUID_AARCH64_COMPILED,
                 old_cpuid_flags = WC_CPUID_INITIALIZER;
             word64 features;
+            word64 pfr0;
+            word64 pfr1;
 
-        #ifndef WOLFSSL_ARMASM_NO_NEON
             __asm__ __volatile (
-                "mrs    %[feat], ID_AA64PFR0_EL1\n"
+                "mrs    %[feat], ID_AA64ISAR0_EL1\n"
                 : [feat] "=r" (features)
                 :
                 :
             );
-
-            if ((features & CPUID_AARCH64_FEAT_ASIMD) !=
-                    CPUID_AARCH64_FEAT_ASIMD)
-                new_cpuid_flags |= CPUID_ASIMD;
-        #endif
-
             __asm__ __volatile (
-                "mrs    %[feat], ID_AA64ISAR0_EL1\n"
-                : [feat] "=r" (features)
+                "mrs    %[feat], ID_AA64PFR0_EL1\n"
+                : [feat] "=r" (pfr0)
+                :
+                :
+            );
+            __asm__ __volatile (
+                "mrs    %[feat], ID_AA64PFR1_EL1\n"
+                : [feat] "=r" (pfr1)
                 :
                 :
             );
 
+        #ifndef WOLFSSL_ARMASM_NO_NEON
+            if ((pfr0 & CPUID_AARCH64_FEAT_ASIMD) != CPUID_AARCH64_FEAT_ASIMD)
+                new_cpuid_flags |= CPUID_ASIMD;
+        #endif
+            if (pfr0 & CPUID_AARCH64_FEAT_SVE)
+                new_cpuid_flags |= CPUID_SVE;
+            if (pfr1 & CPUID_AARCH64_FEAT_SME)
+                new_cpuid_flags |= CPUID_SME;
         #ifndef WOLFSSL_ARMASM_NO_HW_CRYPTO
             if (features & CPUID_AARCH64_FEAT_AES)
                 new_cpuid_flags |= CPUID_AES;
@@ -232,7 +261,7 @@
     static WC_INLINE void cpuid_set_flags(void)
     {
         if (WOLFSSL_ATOMIC_LOAD(cpuid_flags) == WC_CPUID_INITIALIZER) {
-            cpuid_flags_t new_cpuid_flags = 0,
+            cpuid_flags_t new_cpuid_flags = CPUID_AARCH64_COMPILED,
                 old_cpuid_flags = WC_CPUID_INITIALIZER;
             word64 hwcaps = getauxval(AT_HWCAP);
 
@@ -269,6 +298,15 @@
             if (hwcaps & HWCAP_SM4)
                 new_cpuid_flags |= CPUID_SM4;
         #endif
+        #ifdef HWCAP_SVE
+            if (hwcaps & HWCAP_SVE)
+                new_cpuid_flags |= CPUID_SVE;
+        #endif
+        #ifdef HWCAP2_SME
+            /* SME is reported in the second HWCAP word. */
+            if (getauxval(AT_HWCAP2) & HWCAP2_SME)
+                new_cpuid_flags |= CPUID_SME;
+        #endif
 
             (void)hwcaps;
             (void)wolfSSL_Atomic_Uint_CompareExchange
@@ -284,7 +322,7 @@
     static WC_INLINE void cpuid_set_flags(void)
     {
         if (WOLFSSL_ATOMIC_LOAD(cpuid_flags) == WC_CPUID_INITIALIZER) {
-            cpuid_flags_t new_cpuid_flags = 0,
+            cpuid_flags_t new_cpuid_flags = CPUID_AARCH64_COMPILED,
                 old_cpuid_flags = WC_CPUID_INITIALIZER;
             word64 features = android_getCpuFeatures();
 
@@ -324,7 +362,7 @@
     static WC_INLINE void cpuid_set_flags(void)
     {
         if (WOLFSSL_ATOMIC_LOAD(cpuid_flags) == WC_CPUID_INITIALIZER) {
-            cpuid_flags_t new_cpuid_flags = 0,
+            cpuid_flags_t new_cpuid_flags = CPUID_AARCH64_COMPILED,
                 old_cpuid_flags = WC_CPUID_INITIALIZER;
 
         #ifndef WOLFSSL_ARMASM_NO_NEON
@@ -370,7 +408,7 @@
     static WC_INLINE void cpuid_set_flags(void)
     {
         if (WOLFSSL_ATOMIC_LOAD(cpuid_flags) == WC_CPUID_INITIALIZER) {
-            cpuid_flags_t new_cpuid_flags = 0,
+            cpuid_flags_t new_cpuid_flags = CPUID_AARCH64_COMPILED,
                 old_cpuid_flags = WC_CPUID_INITIALIZER;
             word64 features = 0;
 
@@ -486,7 +524,7 @@
     static WC_INLINE void cpuid_set_flags(void)
     {
         if (WOLFSSL_ATOMIC_LOAD(cpuid_flags) == WC_CPUID_INITIALIZER) {
-            cpuid_flags_t new_cpuid_flags = 0,
+            cpuid_flags_t new_cpuid_flags = CPUID_AARCH64_COMPILED,
                 old_cpuid_flags = WC_CPUID_INITIALIZER;
         #ifndef WOLFSSL_ARMASM_NO_NEON
             new_cpuid_flags |= CPUID_ASIMD;
diff --git a/wolfcrypt/src/port/arm/armv8-32-frodokem-asm.S b/wolfcrypt/src/port/arm/armv8-32-frodokem-asm.S
new file mode 100644
index 0000000000..912b809b33
--- /dev/null
+++ b/wolfcrypt/src/port/arm/armv8-32-frodokem-asm.S
@@ -0,0 +1,690 @@
+/* armv8-32-frodokem-asm
+ *
+ * Copyright (C) 2006-2026 wolfSSL Inc.
+ *
+ * This file is part of wolfSSL.
+ *
+ * wolfSSL is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * wolfSSL is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335, USA
+ */
+
+/* Generated using (from wolfssl):
+ *   cd ../scripts
+ *   ruby ./frodokem/frodokem.rb arm32 \
+ *       ../wolfssl/wolfcrypt/src/port/arm/armv8-32-frodokem-asm.S
+ */
+
+#include <wolfssl/wolfcrypt/libwolfssl_sources_asm.h>
+
+#ifdef WOLFSSL_ARMASM
+#if !defined(__aarch64__) && !defined(WOLFSSL_ARMASM_THUMB2)
+#ifndef WOLFSSL_ARMASM_INLINE
+#ifdef WOLFSSL_HAVE_FRODOKEM
+#ifndef WOLFSSL_ARMASM_NO_NEON
+	.text
+	.align	4
+	.fpu	neon
+	.globl	frodokem_add_neon
+	.type	frodokem_add_neon, %function
+frodokem_add_neon:
+	vdup.16	q2, r2
+	mov	r3, #8
+L_frodokem_add_neon_blk:
+	vld1.16	{d0-d1}, [r0]
+	vld1.16	{d2-d3}, [r1]!
+	vadd.i16	q0, q0, q1
+	vand	q0, q0, q2
+	vst1.16	{d0-d1}, [r0]!
+	subs	r3, r3, #1
+	bne	L_frodokem_add_neon_blk
+	bx	lr
+	.size	frodokem_add_neon,.-frodokem_add_neon
+	.text
+	.align	4
+	.fpu	neon
+	.globl	frodokem_sample_neon
+	.type	frodokem_sample_neon, %function
+frodokem_sample_neon:
+	push	{r4, lr}
+	vpush	{d8-d11}
+L_frodokem_sample_neon_blk:
+	vld1.16	{d0-d1}, [r0]
+	vshr.u16	q1, q0, #1
+	vmov.i16	q2, #1
+	vand	q2, q2, q0
+	vmov.i16	q3, #0
+	mov	r12, r2
+	mov	lr, r3
+L_frodokem_sample_neon_cdf:
+	ldrh	r4, [r12]
+	add	r12, r12, #2
+	vdup.16	q4, r4
+	vsub.i16	q4, q4, q1
+	vshr.u16	q4, q4, #15
+	vadd.i16	q3, q3, q4
+	subs	lr, lr, #1
+	bne	L_frodokem_sample_neon_cdf
+	vneg.s16	q5, q2
+	veor	q3, q3, q5
+	vadd.i16	q3, q3, q2
+	vst1.16	{d6-d7}, [r0]!
+	subs	r1, r1, #8
+	bne	L_frodokem_sample_neon_blk
+	vpop	{d8-d11}
+	pop	{r4, pc}
+	.size	frodokem_sample_neon,.-frodokem_sample_neon
+	.text
+	.align	4
+	.fpu	neon
+	.globl	frodokem_sa_accum_neon
+	.type	frodokem_sa_accum_neon, %function
+frodokem_sa_accum_neon:
+	push	{r4, r5, r6, r7, r8, lr}
+	ldr	r12, [sp, #24]
+	lsl	lr, r12, #1
+	add	r4, r1, r3, lsl #1
+	mov	r7, #8
+L_frodokem_sa_accum_neon_i:
+	ldrh	r8, [r4]
+	vdup.16	q0, r8
+	add	r4, r4, lr
+	mov	r5, r2
+	lsr	r6, lr, #4
+L_frodokem_sa_accum_neon_k:
+	vld1.16	{d2-d3}, [r0]
+	vld1.16	{d4-d5}, [r5]!
+	vmla.i16	q1, q2, q0
+	vst1.16	{d2-d3}, [r0]!
+	subs	r6, r6, #1
+	bne	L_frodokem_sa_accum_neon_k
+	subs	r7, r7, #1
+	bne	L_frodokem_sa_accum_neon_i
+	pop	{r4, r5, r6, r7, r8, pc}
+	.size	frodokem_sa_accum_neon,.-frodokem_sa_accum_neon
+	.text
+	.align	4
+	.fpu	neon
+	.globl	frodokem_as_accum_neon
+	.type	frodokem_as_accum_neon, %function
+frodokem_as_accum_neon:
+	push	{r4, r5, r6, r7, r8, r9, lr}
+	vpush	{d8-d15}
+	ldr	r12, [sp, #92]
+	lsl	lr, r12, #1
+	add	r7, r0, r3, lsl #4
+	mov	r6, r2
+	mov	r4, r1
+	veor	q0, q0, q0
+	veor	q1, q1, q1
+	veor	q2, q2, q2
+	veor	q3, q3, q3
+	veor	q4, q4, q4
+	veor	q5, q5, q5
+	veor	q6, q6, q6
+	veor	q7, q7, q7
+	lsr	r8, lr, #4
+L_frodokem_as_accum_neon_k:
+	vld1.16	{d16-d17}, [r6]!
+	mov	r5, r4
+	vld1.16	{d18-d19}, [r5]
+	vmla.i16	q0, q8, q9
+	add	r5, r5, lr
+	vld1.16	{d18-d19}, [r5]
+	vmla.i16	q1, q8, q9
+	add	r5, r5, lr
+	vld1.16	{d18-d19}, [r5]
+	vmla.i16	q2, q8, q9
+	add	r5, r5, lr
+	vld1.16	{d18-d19}, [r5]
+	vmla.i16	q3, q8, q9
+	add	r5, r5, lr
+	vld1.16	{d18-d19}, [r5]
+	vmla.i16	q4, q8, q9
+	add	r5, r5, lr
+	vld1.16	{d18-d19}, [r5]
+	vmla.i16	q5, q8, q9
+	add	r5, r5, lr
+	vld1.16	{d18-d19}, [r5]
+	vmla.i16	q6, q8, q9
+	add	r5, r5, lr
+	vld1.16	{d18-d19}, [r5]
+	vmla.i16	q7, q8, q9
+	add	r5, r5, lr
+	add	r4, r4, #16
+	subs	r8, r8, #1
+	bne	L_frodokem_as_accum_neon_k
+	vadd.i16	d18, d0, d1
+	vpadd.i16	d18, d18, d18
+	vpadd.i16	d18, d18, d18
+	vmov.u16	r9, d18[0]
+	vmov.16	d20[0], r9
+	vadd.i16	d18, d2, d3
+	vpadd.i16	d18, d18, d18
+	vpadd.i16	d18, d18, d18
+	vmov.u16	r9, d18[0]
+	vmov.16	d20[1], r9
+	vadd.i16	d18, d4, d5
+	vpadd.i16	d18, d18, d18
+	vpadd.i16	d18, d18, d18
+	vmov.u16	r9, d18[0]
+	vmov.16	d20[2], r9
+	vadd.i16	d18, d6, d7
+	vpadd.i16	d18, d18, d18
+	vpadd.i16	d18, d18, d18
+	vmov.u16	r9, d18[0]
+	vmov.16	d20[3], r9
+	vadd.i16	d18, d8, d9
+	vpadd.i16	d18, d18, d18
+	vpadd.i16	d18, d18, d18
+	vmov.u16	r9, d18[0]
+	vmov.16	d21[0], r9
+	vadd.i16	d18, d10, d11
+	vpadd.i16	d18, d18, d18
+	vpadd.i16	d18, d18, d18
+	vmov.u16	r9, d18[0]
+	vmov.16	d21[1], r9
+	vadd.i16	d18, d12, d13
+	vpadd.i16	d18, d18, d18
+	vpadd.i16	d18, d18, d18
+	vmov.u16	r9, d18[0]
+	vmov.16	d21[2], r9
+	vadd.i16	d18, d14, d15
+	vpadd.i16	d18, d18, d18
+	vpadd.i16	d18, d18, d18
+	vmov.u16	r9, d18[0]
+	vmov.16	d21[3], r9
+	vld1.16	{d22-d23}, [r7]
+	vadd.i16	q10, q10, q11
+	vst1.16	{d20-d21}, [r7]
+	vpop	{d8-d15}
+	pop	{r4, r5, r6, r7, r8, r9, pc}
+	.size	frodokem_as_accum_neon,.-frodokem_as_accum_neon
+	.text
+	.align	4
+	.fpu	neon
+	.globl	frodokem_mul_bs_neon
+	.type	frodokem_mul_bs_neon, %function
+frodokem_mul_bs_neon:
+	push	{r4, r5, r6, r7, r8, r9, r10, lr}
+	vpush	{d8-d15}
+	ldr	lr, [sp, #96]
+	lsl	r12, r3, #1
+	vdup.16	q11, lr
+	mov	r7, r0
+	mov	r4, r1
+	mov	r9, #8
+L_frodokem_mul_bs_neon_r:
+	mov	r5, r2
+	veor	q0, q0, q0
+	veor	q1, q1, q1
+	veor	q2, q2, q2
+	veor	q3, q3, q3
+	veor	q4, q4, q4
+	veor	q5, q5, q5
+	veor	q6, q6, q6
+	veor	q7, q7, q7
+	lsr	r8, r12, #4
+L_frodokem_mul_bs_neon_k:
+	vld1.16	{d16-d17}, [r4]!
+	mov	r6, r5
+	vld1.16	{d18-d19}, [r6]
+	vmla.i16	q0, q8, q9
+	add	r6, r6, r12
+	vld1.16	{d18-d19}, [r6]
+	vmla.i16	q1, q8, q9
+	add	r6, r6, r12
+	vld1.16	{d18-d19}, [r6]
+	vmla.i16	q2, q8, q9
+	add	r6, r6, r12
+	vld1.16	{d18-d19}, [r6]
+	vmla.i16	q3, q8, q9
+	add	r6, r6, r12
+	vld1.16	{d18-d19}, [r6]
+	vmla.i16	q4, q8, q9
+	add	r6, r6, r12
+	vld1.16	{d18-d19}, [r6]
+	vmla.i16	q5, q8, q9
+	add	r6, r6, r12
+	vld1.16	{d18-d19}, [r6]
+	vmla.i16	q6, q8, q9
+	add	r6, r6, r12
+	vld1.16	{d18-d19}, [r6]
+	vmla.i16	q7, q8, q9
+	add	r6, r6, r12
+	add	r5, r5, #16
+	subs	r8, r8, #1
+	bne	L_frodokem_mul_bs_neon_k
+	vadd.i16	d18, d0, d1
+	vpadd.i16	d18, d18, d18
+	vpadd.i16	d18, d18, d18
+	vmov.u16	r10, d18[0]
+	vmov.16	d20[0], r10
+	vadd.i16	d18, d2, d3
+	vpadd.i16	d18, d18, d18
+	vpadd.i16	d18, d18, d18
+	vmov.u16	r10, d18[0]
+	vmov.16	d20[1], r10
+	vadd.i16	d18, d4, d5
+	vpadd.i16	d18, d18, d18
+	vpadd.i16	d18, d18, d18
+	vmov.u16	r10, d18[0]
+	vmov.16	d20[2], r10
+	vadd.i16	d18, d6, d7
+	vpadd.i16	d18, d18, d18
+	vpadd.i16	d18, d18, d18
+	vmov.u16	r10, d18[0]
+	vmov.16	d20[3], r10
+	vadd.i16	d18, d8, d9
+	vpadd.i16	d18, d18, d18
+	vpadd.i16	d18, d18, d18
+	vmov.u16	r10, d18[0]
+	vmov.16	d21[0], r10
+	vadd.i16	d18, d10, d11
+	vpadd.i16	d18, d18, d18
+	vpadd.i16	d18, d18, d18
+	vmov.u16	r10, d18[0]
+	vmov.16	d21[1], r10
+	vadd.i16	d18, d12, d13
+	vpadd.i16	d18, d18, d18
+	vpadd.i16	d18, d18, d18
+	vmov.u16	r10, d18[0]
+	vmov.16	d21[2], r10
+	vadd.i16	d18, d14, d15
+	vpadd.i16	d18, d18, d18
+	vpadd.i16	d18, d18, d18
+	vmov.u16	r10, d18[0]
+	vmov.16	d21[3], r10
+	vand	q10, q10, q11
+	vst1.16	{d20-d21}, [r7]!
+	subs	r9, r9, #1
+	bne	L_frodokem_mul_bs_neon_r
+	vpop	{d8-d15}
+	pop	{r4, r5, r6, r7, r8, r9, r10, pc}
+	.size	frodokem_mul_bs_neon,.-frodokem_mul_bs_neon
+	.text
+	.align	4
+	.fpu	neon
+	.globl	frodokem_mul_add_sb_plus_e_neon
+	.type	frodokem_mul_add_sb_plus_e_neon, %function
+frodokem_mul_add_sb_plus_e_neon:
+	push	{r4, r5, r6, r7, r8, r9, lr}
+	vpush	{d8-d15}
+	ldr	lr, [sp, #92]
+	lsl	r12, r3, #1
+	vdup.16	q10, lr
+	mov	r4, r1
+	mov	r5, r2
+	mov	r7, r0
+	vld1.16	{d0-d1}, [r7]!
+	vld1.16	{d2-d3}, [r7]!
+	vld1.16	{d4-d5}, [r7]!
+	vld1.16	{d6-d7}, [r7]!
+	vld1.16	{d8-d9}, [r7]!
+	vld1.16	{d10-d11}, [r7]!
+	vld1.16	{d12-d13}, [r7]!
+	vld1.16	{d14-d15}, [r7]!
+	mov	r8, r3
+L_frodokem_mul_add_sb_plus_e_neon_j:
+	vld1.16	{d16-d17}, [r4]!
+	mov	r6, r5
+	ldrh	r9, [r6]
+	vdup.16	q9, r9
+	vmla.i16	q0, q8, q9
+	add	r6, r6, r12
+	ldrh	r9, [r6]
+	vdup.16	q9, r9
+	vmla.i16	q1, q8, q9
+	add	r6, r6, r12
+	ldrh	r9, [r6]
+	vdup.16	q9, r9
+	vmla.i16	q2, q8, q9
+	add	r6, r6, r12
+	ldrh	r9, [r6]
+	vdup.16	q9, r9
+	vmla.i16	q3, q8, q9
+	add	r6, r6, r12
+	ldrh	r9, [r6]
+	vdup.16	q9, r9
+	vmla.i16	q4, q8, q9
+	add	r6, r6, r12
+	ldrh	r9, [r6]
+	vdup.16	q9, r9
+	vmla.i16	q5, q8, q9
+	add	r6, r6, r12
+	ldrh	r9, [r6]
+	vdup.16	q9, r9
+	vmla.i16	q6, q8, q9
+	add	r6, r6, r12
+	ldrh	r9, [r6]
+	vdup.16	q9, r9
+	vmla.i16	q7, q8, q9
+	add	r6, r6, r12
+	add	r5, r5, #2
+	subs	r8, r8, #1
+	bne	L_frodokem_mul_add_sb_plus_e_neon_j
+	mov	r7, r0
+	vand	q0, q0, q10
+	vst1.16	{d0-d1}, [r7]!
+	vand	q1, q1, q10
+	vst1.16	{d2-d3}, [r7]!
+	vand	q2, q2, q10
+	vst1.16	{d4-d5}, [r7]!
+	vand	q3, q3, q10
+	vst1.16	{d6-d7}, [r7]!
+	vand	q4, q4, q10
+	vst1.16	{d8-d9}, [r7]!
+	vand	q5, q5, q10
+	vst1.16	{d10-d11}, [r7]!
+	vand	q6, q6, q10
+	vst1.16	{d12-d13}, [r7]!
+	vand	q7, q7, q10
+	vst1.16	{d14-d15}, [r7]!
+	vpop	{d8-d15}
+	pop	{r4, r5, r6, r7, r8, r9, pc}
+	.size	frodokem_mul_add_sb_plus_e_neon,.-frodokem_mul_add_sb_plus_e_neon
+#endif /* WOLFSSL_ARMASM_NO_NEON */
+#endif /* WOLFSSL_HAVE_FRODOKEM */
+#ifdef WOLFSSL_HAVE_FRODOKEM
+#ifndef WOLFSSL_ARMASM_NO_NEON
+	.fpu	crypto-neon-fp-armv8
+	.text
+	.align	4
+	.globl	frodokem_gen_a_rows_aes_arm32
+	.type	frodokem_gen_a_rows_aes_arm32, %function
+frodokem_gen_a_rows_aes_arm32:
+	push	{r4, r5, r6, r7, r8, r9, lr}
+	vpush	{d8-d15}
+	ldr	lr, [sp, #96]
+	lsl	lr, lr, #1
+	ldr	r4, [sp, #92]
+	mul	r4, r4, lr
+	ldr	r12, [sp, #100]
+	vld1.8	{d0-d1}, [r2]!
+	vld1.8	{d2-d3}, [r2]!
+	vld1.8	{d4-d5}, [r2]!
+	vld1.8	{d6-d7}, [r2]!
+	vld1.8	{d8-d9}, [r2]!
+	vld1.8	{d10-d11}, [r2]!
+	vld1.8	{d12-d13}, [r2]!
+	vld1.8	{d14-d15}, [r2]!
+	vld1.8	{d16-d17}, [r2]!
+	vld1.8	{d18-d19}, [r2]!
+	vld1.8	{d20-d21}, [r2]!
+	vdup.16	q13, r12
+	movw	r8, #0
+#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7)
+	orr	r8, r8, #0x80000
+#else
+	movt	r8, #0x8
+#endif
+	mov	r9, #0
+	mov	r5, r0
+L_frodokem_gen_a_rows_aes_arm32_row:
+	mov	r7, r3
+	add	r6, r5, lr
+L_frodokem_gen_a_rows_aes_arm32_blk:
+	str	r7, [r5], #4
+	str	r9, [r5], #4
+	str	r9, [r5], #4
+	str	r9, [r5], #4
+	add	r7, r7, r8
+	cmp	r5, r6
+	blt	L_frodokem_gen_a_rows_aes_arm32_blk
+	add	r3, r3, #1
+	add	r6, r0, r4
+	cmp	r5, r6
+	blt	L_frodokem_gen_a_rows_aes_arm32_row
+L_frodokem_gen_a_rows_aes_arm32_aes:
+	vld1.8	{d22-d23}, [r0]!
+	vld1.8	{d24-d25}, [r0]!
+	aese.8	q11, q0
+	aese.8	q12, q0
+	aesmc.8	q11, q11
+	aesmc.8	q12, q12
+	aese.8	q11, q1
+	aese.8	q12, q1
+	aesmc.8	q11, q11
+	aesmc.8	q12, q12
+	aese.8	q11, q2
+	aese.8	q12, q2
+	aesmc.8	q11, q11
+	aesmc.8	q12, q12
+	aese.8	q11, q3
+	aese.8	q12, q3
+	aesmc.8	q11, q11
+	aesmc.8	q12, q12
+	aese.8	q11, q4
+	aese.8	q12, q4
+	aesmc.8	q11, q11
+	aesmc.8	q12, q12
+	aese.8	q11, q5
+	aese.8	q12, q5
+	aesmc.8	q11, q11
+	aesmc.8	q12, q12
+	aese.8	q11, q6
+	aese.8	q12, q6
+	aesmc.8	q11, q11
+	aesmc.8	q12, q12
+	aese.8	q11, q7
+	aese.8	q12, q7
+	aesmc.8	q11, q11
+	aesmc.8	q12, q12
+	aese.8	q11, q8
+	aese.8	q12, q8
+	aesmc.8	q11, q11
+	aesmc.8	q12, q12
+	aese.8	q11, q9
+	aese.8	q12, q9
+	veor	q11, q11, q10
+	veor	q12, q12, q10
+	vand	q11, q11, q13
+	vand	q12, q12, q13
+	vst1.8	{d22-d23}, [r1]!
+	vst1.8	{d24-d25}, [r1]!
+	subs	r4, r4, #32
+	bne	L_frodokem_gen_a_rows_aes_arm32_aes
+	vpop	{d8-d15}
+	pop	{r4, r5, r6, r7, r8, r9, pc}
+	.size	frodokem_gen_a_rows_aes_arm32,.-frodokem_gen_a_rows_aes_arm32
+#endif /* WOLFSSL_ARMASM_NO_NEON */
+#endif /* WOLFSSL_HAVE_FRODOKEM */
+#ifdef WOLFSSL_HAVE_FRODOKEM
+#ifdef __ARM_FEATURE_SIMD32
+#if (!defined(__ARM_NEON) && !defined(__ARM_NEON__)) || defined(WOLFSSL_ARMASM_NO_NEON)
+	.text
+	.align	4
+	.globl	frodokem_add_simd32
+	.type	frodokem_add_simd32, %function
+frodokem_add_simd32:
+	push	{lr}
+	orr	r2, r2, r2, lsl #16
+	mov	r3, #32
+L_frodokem_add_simd32_blk:
+	ldr	r12, [r0]
+	ldr	lr, [r1], #4
+	sadd16	r12, r12, lr
+	and	r12, r12, r2
+	str	r12, [r0], #4
+	subs	r3, r3, #1
+	bne	L_frodokem_add_simd32_blk
+	pop	{pc}
+	.size	frodokem_add_simd32,.-frodokem_add_simd32
+	.text
+	.align	4
+	.globl	frodokem_sa_accum_simd32
+	.type	frodokem_sa_accum_simd32, %function
+frodokem_sa_accum_simd32:
+	push	{r4, r5, r6, r7, r8, r9, r10, r11, lr}
+	ldr	r12, [sp, #36]
+	lsl	r12, r12, #1
+	add	lr, r1, r3, lsl #1
+	mov	r6, #8
+L_frodokem_sa_accum_simd32_i:
+	ldrh	r7, [lr]
+	add	lr, lr, r12
+	mov	r4, r2
+	lsr	r5, r12, #2
+L_frodokem_sa_accum_simd32_k:
+	ldr	r9, [r0]
+	ldr	r8, [r4], #4
+	smulbb	r10, r7, r8
+	smulbt	r11, r7, r8
+	pkhbt	r10, r10, r11, LSL #16
+	sadd16	r9, r9, r10
+	str	r9, [r0], #4
+	subs	r5, r5, #1
+	bne	L_frodokem_sa_accum_simd32_k
+	subs	r6, r6, #1
+	bne	L_frodokem_sa_accum_simd32_i
+	pop	{r4, r5, r6, r7, r8, r9, r10, r11, pc}
+	.size	frodokem_sa_accum_simd32,.-frodokem_sa_accum_simd32
+	.text
+	.align	4
+	.globl	frodokem_as_accum_simd32
+	.type	frodokem_as_accum_simd32, %function
+frodokem_as_accum_simd32:
+	push	{r4, r5, r6, r7, r8, r9, r10, r11, lr}
+	ldr	r12, [sp, #36]
+	lsl	r12, r12, #1
+	add	lr, r0, r3, lsl #4
+	mov	r4, r1
+	mov	r7, #8
+L_frodokem_as_accum_simd32_c:
+	mov	r9, #0
+	mov	r5, r2
+	mov	r6, r4
+	lsr	r8, r12, #2
+L_frodokem_as_accum_simd32_j:
+	ldr	r10, [r5], #4
+	ldr	r11, [r6], #4
+	smlad	r9, r10, r11, r9
+	subs	r8, r8, #1
+	bne	L_frodokem_as_accum_simd32_j
+	ldrh	r10, [lr]
+	add	r10, r10, r9
+	strh	r10, [lr]
+	add	lr, lr, #2
+	add	r4, r4, r12
+	subs	r7, r7, #1
+	bne	L_frodokem_as_accum_simd32_c
+	pop	{r4, r5, r6, r7, r8, r9, r10, r11, pc}
+	.size	frodokem_as_accum_simd32,.-frodokem_as_accum_simd32
+	.text
+	.align	4
+	.globl	frodokem_mul_bs_simd32
+	.type	frodokem_mul_bs_simd32, %function
+frodokem_mul_bs_simd32:
+	push	{r4, r5, r6, r7, r8, r9, r10, r11, lr}
+	mov	r12, r1
+	mov	r6, #8
+L_frodokem_mul_bs_simd32_i:
+	mov	lr, r2
+	mov	r7, #8
+L_frodokem_mul_bs_simd32_c:
+	mov	r9, #0
+	mov	r4, r12
+	mov	r5, lr
+	lsr	r8, r3, #1
+L_frodokem_mul_bs_simd32_j:
+	ldr	r10, [r4], #4
+	ldr	r11, [r5], #4
+	smlad	r9, r10, r11, r9
+	subs	r8, r8, #1
+	bne	L_frodokem_mul_bs_simd32_j
+	ldr	r10, [sp, #36]
+	orr	r10, r10, r10, lsl #16
+	and	r9, r9, r10
+	strh	r9, [r0]
+	add	r0, r0, #2
+	add	lr, lr, r3, lsl #1
+	subs	r7, r7, #1
+	bne	L_frodokem_mul_bs_simd32_c
+	add	r12, r12, r3, lsl #1
+	subs	r6, r6, #1
+	bne	L_frodokem_mul_bs_simd32_i
+	pop	{r4, r5, r6, r7, r8, r9, r10, r11, pc}
+	.size	frodokem_mul_bs_simd32,.-frodokem_mul_bs_simd32
+	.text
+	.align	4
+	.globl	frodokem_mul_add_sb_plus_e_simd32
+	.type	frodokem_mul_add_sb_plus_e_simd32, %function
+frodokem_mul_add_sb_plus_e_simd32:
+	push	{r4, r5, r6, r7, r8, r9, r10, r11, lr}
+	sub	sp, sp, #8
+	lsl	r9, r3, #1
+	str	r9, [sp, #4]
+	str	r1, [sp]
+	mov	r12, #8
+L_frodokem_mul_add_sb_plus_e_simd32_i:
+	ldr	r9, [sp, #4]
+	add	r3, r2, r9
+	ldr	r1, [sp]
+	mov	lr, r2
+	ldr	r4, [r0]
+	ldr	r5, [r0, #4]
+	ldr	r6, [r0, #8]
+	ldr	r7, [r0, #12]
+L_frodokem_mul_add_sb_plus_e_simd32_j:
+	ldrh	r8, [lr]
+	add	lr, lr, #2
+	ldr	r9, [r1]
+	smulbb	r10, r8, r9
+	smulbt	r11, r8, r9
+	pkhbt	r10, r10, r11, LSL #16
+	sadd16	r4, r4, r10
+	ldr	r9, [r1, #4]
+	smulbb	r10, r8, r9
+	smulbt	r11, r8, r9
+	pkhbt	r10, r10, r11, LSL #16
+	sadd16	r5, r5, r10
+	ldr	r9, [r1, #8]
+	smulbb	r10, r8, r9
+	smulbt	r11, r8, r9
+	pkhbt	r10, r10, r11, LSL #16
+	sadd16	r6, r6, r10
+	ldr	r9, [r1, #12]
+	smulbb	r10, r8, r9
+	smulbt	r11, r8, r9
+	pkhbt	r10, r10, r11, LSL #16
+	sadd16	r7, r7, r10
+	add	r1, r1, #16
+	cmp	lr, r3
+	bne	L_frodokem_mul_add_sb_plus_e_simd32_j
+	ldr	r9, [sp, #44]
+	orr	r9, r9, r9, lsl #16
+	and	r4, r4, r9
+	and	r5, r5, r9
+	and	r6, r6, r9
+	and	r7, r7, r9
+	str	r4, [r0]
+	str	r5, [r0, #4]
+	str	r6, [r0, #8]
+	str	r7, [r0, #12]
+	add	r0, r0, #16
+	mov	r2, r3
+	subs	r12, r12, #1
+	bne	L_frodokem_mul_add_sb_plus_e_simd32_i
+	add	sp, sp, #8
+	pop	{r4, r5, r6, r7, r8, r9, r10, r11, pc}
+	.size	frodokem_mul_add_sb_plus_e_simd32,.-frodokem_mul_add_sb_plus_e_simd32
+#endif /* (!defined(__ARM_NEON) && !defined(__ARM_NEON__)) || defined(WOLFSSL_ARMASM_NO_NEON) */
+#endif /* __ARM_FEATURE_SIMD32 */
+#endif /* WOLFSSL_HAVE_FRODOKEM */
+#endif /* !__aarch64__ && !WOLFSSL_ARMASM_THUMB2 */
+#endif /* WOLFSSL_ARMASM */
+
+#if defined(__linux__) && defined(__ELF__)
+.section	.note.GNU-stack,"",%progbits
+#endif
+#endif /* !WOLFSSL_ARMASM_INLINE */
diff --git a/wolfcrypt/src/port/arm/armv8-32-frodokem-asm_c.c b/wolfcrypt/src/port/arm/armv8-32-frodokem-asm_c.c
new file mode 100644
index 0000000000..f1a76c236c
--- /dev/null
+++ b/wolfcrypt/src/port/arm/armv8-32-frodokem-asm_c.c
@@ -0,0 +1,877 @@
+/* armv8-32-frodokem-asm
+ *
+ * Copyright (C) 2006-2026 wolfSSL Inc.
+ *
+ * This file is part of wolfSSL.
+ *
+ * wolfSSL is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * wolfSSL is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335, USA
+ */
+
+/* Generated using (from wolfssl):
+ *   cd ../scripts
+ *   ruby ./frodokem/frodokem.rb arm32 \
+ *       ../wolfssl/wolfcrypt/src/port/arm/armv8-32-frodokem-asm.c
+ */
+
+#include <wolfssl/wolfcrypt/libwolfssl_sources_asm.h>
+#include <wolfssl/wolfcrypt/error-crypt.h>
+
+#ifdef WOLFSSL_ARMASM
+#if !defined(__aarch64__) && !defined(WOLFSSL_ARMASM_THUMB2)
+#ifdef WOLFSSL_ARMASM_INLINE
+
+#ifdef __IAR_SYSTEMS_ICC__
+#define __asm__        asm
+#define __volatile__   volatile
+#define WOLFSSL_NO_VAR_ASSIGN_REG
+#endif /* __IAR_SYSTEMS_ICC__ */
+#ifdef __KEIL__
+#define __asm__        __asm
+#define __volatile__   volatile
+#endif /* __KEIL__ */
+#ifdef __ghs__
+#define __asm__        __asm
+#define __volatile__
+#define WOLFSSL_NO_VAR_ASSIGN_REG
+#endif /* __ghs__ */
+
+#include <wolfssl/wolfcrypt/wc_frodokem_mat.h>
+
+#ifdef WOLFSSL_HAVE_FRODOKEM
+#ifndef WOLFSSL_ARMASM_NO_NEON
+void frodokem_add_neon(word16* a_p, const word16* b_p, int qmask_p);
+#ifndef WOLFSSL_NO_VAR_ASSIGN_REG
+WC_OMIT_FRAME_POINTER void frodokem_add_neon(word16* a_p, const word16* b_p,
+    int qmask_p)
+#else
+WC_OMIT_FRAME_POINTER void frodokem_add_neon(word16* a, const word16* b,
+    int qmask)
+#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */
+{
+#ifndef WOLFSSL_NO_VAR_ASSIGN_REG
+    register word16* a __asm__ ("r0") = (word16*)a_p;
+    register const word16* b __asm__ ("r1") = (const word16*)b_p;
+    register int qmask __asm__ ("r2") = (int)qmask_p;
+#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */
+
+    __asm__ __volatile__ (
+        "vdup.16	q2, %[qmask]\n\t"
+        "mov	r3, #8\n\t"
+        "\n"
+    "L_frodokem_add_neon_blk_%=:\n\t"
+        "vld1.16	{d0-d1}, [%[a]]\n\t"
+        "vld1.16	{d2-d3}, [%[b]]!\n\t"
+        "vadd.i16	q0, q0, q1\n\t"
+        "vand	q0, q0, q2\n\t"
+        "vst1.16	{d0-d1}, [%[a]]!\n\t"
+        "subs	r3, r3, #1\n\t"
+        "bne	L_frodokem_add_neon_blk_%=\n\t"
+#ifndef WOLFSSL_NO_VAR_ASSIGN_REG
+        : [a] "+r" (a), [b] "+r" (b), [qmask] "+r" (qmask)
+        :
+#else
+        :
+        : [a] "r" (a), [b] "r" (b), [qmask] "r" (qmask)
+#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */
+        : "memory", "cc", "r3", "q0", "q1", "q2"
+    );
+}
+
+void frodokem_sample_neon(word16* mat_p, int cnt_p, const word16* cdf_p,
+    int cdflen_p);
+#ifndef WOLFSSL_NO_VAR_ASSIGN_REG
+WC_OMIT_FRAME_POINTER void frodokem_sample_neon(word16* mat_p, int cnt_p,
+    const word16* cdf_p, int cdflen_p)
+#else
+WC_OMIT_FRAME_POINTER void frodokem_sample_neon(word16* mat, int cnt,
+    const word16* cdf, int cdflen)
+#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */
+{
+#ifndef WOLFSSL_NO_VAR_ASSIGN_REG
+    register word16* mat __asm__ ("r0") = (word16*)mat_p;
+    register int cnt __asm__ ("r1") = (int)cnt_p;
+    register const word16* cdf __asm__ ("r2") = (const word16*)cdf_p;
+    register int cdflen __asm__ ("r3") = (int)cdflen_p;
+#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */
+
+    __asm__ __volatile__ (
+        "\n"
+    "L_frodokem_sample_neon_blk_%=:\n\t"
+        "vld1.16	{d0-d1}, [%[mat]]\n\t"
+        "vshr.u16	q1, q0, #1\n\t"
+        "vmov.i16	q2, #1\n\t"
+        "vand	q2, q2, q0\n\t"
+        "vmov.i16	q3, #0\n\t"
+        "mov	r12, %[cdf]\n\t"
+        "mov	lr, %[cdflen]\n\t"
+        "\n"
+    "L_frodokem_sample_neon_cdf_%=:\n\t"
+        "ldrh	r4, [r12]\n\t"
+        "add	r12, r12, #2\n\t"
+        "vdup.16	q4, r4\n\t"
+        "vsub.i16	q4, q4, q1\n\t"
+        "vshr.u16	q4, q4, #15\n\t"
+        "vadd.i16	q3, q3, q4\n\t"
+        "subs	lr, lr, #1\n\t"
+        "bne	L_frodokem_sample_neon_cdf_%=\n\t"
+        "vneg.s16	q5, q2\n\t"
+        "veor	q3, q3, q5\n\t"
+        "vadd.i16	q3, q3, q2\n\t"
+        "vst1.16	{d6-d7}, [%[mat]]!\n\t"
+        "subs	%[cnt], %[cnt], #8\n\t"
+        "bne	L_frodokem_sample_neon_blk_%=\n\t"
+#ifndef WOLFSSL_NO_VAR_ASSIGN_REG
+        : [mat] "+r" (mat), [cnt] "+r" (cnt), [cdf] "+r" (cdf),
+          [cdflen] "+r" (cdflen)
+        :
+#else
+        :
+        : [mat] "r" (mat), [cnt] "r" (cnt), [cdf] "r" (cdf),
+          [cdflen] "r" (cdflen)
+#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */
+        : "memory", "cc", "r12", "lr", "r4", "q0", "q1", "q2", "q3", "q4", "q5"
+    );
+}
+
+void frodokem_sa_accum_neon(word16* out_p, const word16* s_p,
+    const word16* row_p, int j_p, int n_p);
+#ifndef WOLFSSL_NO_VAR_ASSIGN_REG
+WC_OMIT_FRAME_POINTER void frodokem_sa_accum_neon(word16* out_p,
+    const word16* s_p, const word16* row_p, int j_p, int n_p)
+#else
+WC_OMIT_FRAME_POINTER void frodokem_sa_accum_neon(word16* out, const word16* s,
+    const word16* row, int j, int n)
+#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */
+{
+#ifndef WOLFSSL_NO_VAR_ASSIGN_REG
+    register word16* out __asm__ ("r0") = (word16*)out_p;
+    register const word16* s __asm__ ("r1") = (const word16*)s_p;
+    register const word16* row __asm__ ("r2") = (const word16*)row_p;
+    register int j __asm__ ("r3") = (int)j_p;
+    register int n __asm__ ("r12") = (int)n_p;
+#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */
+
+    __asm__ __volatile__ (
+        "push	{%[n]}\n\t"
+        "ldr	r12, [sp]\n\t"
+        "lsl	lr, r12, #1\n\t"
+        "add	r4, %[s], %[j], lsl #1\n\t"
+        "mov	r7, #8\n\t"
+        "\n"
+    "L_frodokem_sa_accum_neon_i_%=:\n\t"
+        "ldrh	r8, [r4]\n\t"
+        "vdup.16	q0, r8\n\t"
+        "add	r4, r4, lr\n\t"
+        "mov	r5, %[row]\n\t"
+        "lsr	r6, lr, #4\n\t"
+        "\n"
+    "L_frodokem_sa_accum_neon_k_%=:\n\t"
+        "vld1.16	{d2-d3}, [%[out]]\n\t"
+        "vld1.16	{d4-d5}, [r5]!\n\t"
+        "vmla.i16	q1, q2, q0\n\t"
+        "vst1.16	{d2-d3}, [%[out]]!\n\t"
+        "subs	r6, r6, #1\n\t"
+        "bne	L_frodokem_sa_accum_neon_k_%=\n\t"
+        "subs	r7, r7, #1\n\t"
+        "bne	L_frodokem_sa_accum_neon_i_%=\n\t"
+        "pop	{%[n]}\n\t"
+#ifndef WOLFSSL_NO_VAR_ASSIGN_REG
+        : [out] "+r" (out), [s] "+r" (s), [row] "+r" (row), [j] "+r" (j),
+          [n] "+r" (n)
+        :
+#else
+        :
+        : [out] "r" (out), [s] "r" (s), [row] "r" (row), [j] "r" (j),
+          [n] "r" (n)
+#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */
+        : "memory", "cc", "lr", "r4", "r5", "r6", "r7", "r8", "q0", "q1", "q2"
+    );
+}
+
+void frodokem_as_accum_neon(word16* out_p, const word16* s_p,
+    const word16* row_p, int i_p, int n_p);
+#ifndef WOLFSSL_NO_VAR_ASSIGN_REG
+WC_OMIT_FRAME_POINTER void frodokem_as_accum_neon(word16* out_p,
+    const word16* s_p, const word16* row_p, int i_p, int n_p)
+#else
+WC_OMIT_FRAME_POINTER void frodokem_as_accum_neon(word16* out, const word16* s,
+    const word16* row, int i, int n)
+#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */
+{
+#ifndef WOLFSSL_NO_VAR_ASSIGN_REG
+    register word16* out __asm__ ("r0") = (word16*)out_p;
+    register const word16* s __asm__ ("r1") = (const word16*)s_p;
+    register const word16* row __asm__ ("r2") = (const word16*)row_p;
+    register int i __asm__ ("r3") = (int)i_p;
+    register int n __asm__ ("r12") = (int)n_p;
+#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */
+
+    __asm__ __volatile__ (
+        "push	{%[n]}\n\t"
+        "ldr	r12, [sp]\n\t"
+        "lsl	lr, r12, #1\n\t"
+        "add	r7, %[out], %[i], lsl #4\n\t"
+        "mov	r6, %[row]\n\t"
+        "mov	r4, %[s]\n\t"
+        "veor	q0, q0, q0\n\t"
+        "veor	q1, q1, q1\n\t"
+        "veor	q2, q2, q2\n\t"
+        "veor	q3, q3, q3\n\t"
+        "veor	q4, q4, q4\n\t"
+        "veor	q5, q5, q5\n\t"
+        "veor	q6, q6, q6\n\t"
+        "veor	q7, q7, q7\n\t"
+        "lsr	r8, lr, #4\n\t"
+        "\n"
+    "L_frodokem_as_accum_neon_k_%=:\n\t"
+        "vld1.16	{d16-d17}, [r6]!\n\t"
+        "mov	r5, r4\n\t"
+        "vld1.16	{d18-d19}, [r5]\n\t"
+        "vmla.i16	q0, q8, q9\n\t"
+        "add	r5, r5, lr\n\t"
+        "vld1.16	{d18-d19}, [r5]\n\t"
+        "vmla.i16	q1, q8, q9\n\t"
+        "add	r5, r5, lr\n\t"
+        "vld1.16	{d18-d19}, [r5]\n\t"
+        "vmla.i16	q2, q8, q9\n\t"
+        "add	r5, r5, lr\n\t"
+        "vld1.16	{d18-d19}, [r5]\n\t"
+        "vmla.i16	q3, q8, q9\n\t"
+        "add	r5, r5, lr\n\t"
+        "vld1.16	{d18-d19}, [r5]\n\t"
+        "vmla.i16	q4, q8, q9\n\t"
+        "add	r5, r5, lr\n\t"
+        "vld1.16	{d18-d19}, [r5]\n\t"
+        "vmla.i16	q5, q8, q9\n\t"
+        "add	r5, r5, lr\n\t"
+        "vld1.16	{d18-d19}, [r5]\n\t"
+        "vmla.i16	q6, q8, q9\n\t"
+        "add	r5, r5, lr\n\t"
+        "vld1.16	{d18-d19}, [r5]\n\t"
+        "vmla.i16	q7, q8, q9\n\t"
+        "add	r5, r5, lr\n\t"
+        "add	r4, r4, #16\n\t"
+        "subs	r8, r8, #1\n\t"
+        "bne	L_frodokem_as_accum_neon_k_%=\n\t"
+        "vadd.i16	d18, d0, d1\n\t"
+        "vpadd.i16	d18, d18, d18\n\t"
+        "vpadd.i16	d18, d18, d18\n\t"
+        "vmov.u16	r9, d18[0]\n\t"
+        "vmov.16	d20[0], r9\n\t"
+        "vadd.i16	d18, d2, d3\n\t"
+        "vpadd.i16	d18, d18, d18\n\t"
+        "vpadd.i16	d18, d18, d18\n\t"
+        "vmov.u16	r9, d18[0]\n\t"
+        "vmov.16	d20[1], r9\n\t"
+        "vadd.i16	d18, d4, d5\n\t"
+        "vpadd.i16	d18, d18, d18\n\t"
+        "vpadd.i16	d18, d18, d18\n\t"
+        "vmov.u16	r9, d18[0]\n\t"
+        "vmov.16	d20[2], r9\n\t"
+        "vadd.i16	d18, d6, d7\n\t"
+        "vpadd.i16	d18, d18, d18\n\t"
+        "vpadd.i16	d18, d18, d18\n\t"
+        "vmov.u16	r9, d18[0]\n\t"
+        "vmov.16	d20[3], r9\n\t"
+        "vadd.i16	d18, d8, d9\n\t"
+        "vpadd.i16	d18, d18, d18\n\t"
+        "vpadd.i16	d18, d18, d18\n\t"
+        "vmov.u16	r9, d18[0]\n\t"
+        "vmov.16	d21[0], r9\n\t"
+        "vadd.i16	d18, d10, d11\n\t"
+        "vpadd.i16	d18, d18, d18\n\t"
+        "vpadd.i16	d18, d18, d18\n\t"
+        "vmov.u16	r9, d18[0]\n\t"
+        "vmov.16	d21[1], r9\n\t"
+        "vadd.i16	d18, d12, d13\n\t"
+        "vpadd.i16	d18, d18, d18\n\t"
+        "vpadd.i16	d18, d18, d18\n\t"
+        "vmov.u16	r9, d18[0]\n\t"
+        "vmov.16	d21[2], r9\n\t"
+        "vadd.i16	d18, d14, d15\n\t"
+        "vpadd.i16	d18, d18, d18\n\t"
+        "vpadd.i16	d18, d18, d18\n\t"
+        "vmov.u16	r9, d18[0]\n\t"
+        "vmov.16	d21[3], r9\n\t"
+        "vld1.16	{d22-d23}, [r7]\n\t"
+        "vadd.i16	q10, q10, q11\n\t"
+        "vst1.16	{d20-d21}, [r7]\n\t"
+        "pop	{%[n]}\n\t"
+#ifndef WOLFSSL_NO_VAR_ASSIGN_REG
+        : [out] "+r" (out), [s] "+r" (s), [row] "+r" (row), [i] "+r" (i),
+          [n] "+r" (n)
+        :
+#else
+        :
+        : [out] "r" (out), [s] "r" (s), [row] "r" (row), [i] "r" (i),
+          [n] "r" (n)
+#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */
+        : "memory", "cc", "lr", "r4", "r5", "r6", "r7", "r8", "r9", "q0", "q1",
+            "q2", "q3", "q4", "q5", "q6", "q7", "q8", "q9", "q10", "q11"
+    );
+}
+
+void frodokem_mul_bs_neon(word16* out_p, const word16* b_p, const word16* s_p,
+    int n_p, int qmask_p);
+#ifndef WOLFSSL_NO_VAR_ASSIGN_REG
+WC_OMIT_FRAME_POINTER void frodokem_mul_bs_neon(word16* out_p,
+    const word16* b_p, const word16* s_p, int n_p, int qmask_p)
+#else
+WC_OMIT_FRAME_POINTER void frodokem_mul_bs_neon(word16* out, const word16* b,
+    const word16* s, int n, int qmask)
+#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */
+{
+#ifndef WOLFSSL_NO_VAR_ASSIGN_REG
+    register word16* out __asm__ ("r0") = (word16*)out_p;
+    register const word16* b __asm__ ("r1") = (const word16*)b_p;
+    register const word16* s __asm__ ("r2") = (const word16*)s_p;
+    register int n __asm__ ("r3") = (int)n_p;
+    register int qmask __asm__ ("r12") = (int)qmask_p;
+#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */
+
+    __asm__ __volatile__ (
+        "push	{%[qmask]}\n\t"
+        "ldr	lr, [sp]\n\t"
+        "lsl	r12, %[n], #1\n\t"
+        "vdup.16	q11, lr\n\t"
+        "mov	r7, %[out]\n\t"
+        "mov	r4, %[b]\n\t"
+        "mov	r9, #8\n\t"
+        "\n"
+    "L_frodokem_mul_bs_neon_r_%=:\n\t"
+        "mov	r5, %[s]\n\t"
+        "veor	q0, q0, q0\n\t"
+        "veor	q1, q1, q1\n\t"
+        "veor	q2, q2, q2\n\t"
+        "veor	q3, q3, q3\n\t"
+        "veor	q4, q4, q4\n\t"
+        "veor	q5, q5, q5\n\t"
+        "veor	q6, q6, q6\n\t"
+        "veor	q7, q7, q7\n\t"
+        "lsr	r8, r12, #4\n\t"
+        "\n"
+    "L_frodokem_mul_bs_neon_k_%=:\n\t"
+        "vld1.16	{d16-d17}, [r4]!\n\t"
+        "mov	r6, r5\n\t"
+        "vld1.16	{d18-d19}, [r6]\n\t"
+        "vmla.i16	q0, q8, q9\n\t"
+        "add	r6, r6, r12\n\t"
+        "vld1.16	{d18-d19}, [r6]\n\t"
+        "vmla.i16	q1, q8, q9\n\t"
+        "add	r6, r6, r12\n\t"
+        "vld1.16	{d18-d19}, [r6]\n\t"
+        "vmla.i16	q2, q8, q9\n\t"
+        "add	r6, r6, r12\n\t"
+        "vld1.16	{d18-d19}, [r6]\n\t"
+        "vmla.i16	q3, q8, q9\n\t"
+        "add	r6, r6, r12\n\t"
+        "vld1.16	{d18-d19}, [r6]\n\t"
+        "vmla.i16	q4, q8, q9\n\t"
+        "add	r6, r6, r12\n\t"
+        "vld1.16	{d18-d19}, [r6]\n\t"
+        "vmla.i16	q5, q8, q9\n\t"
+        "add	r6, r6, r12\n\t"
+        "vld1.16	{d18-d19}, [r6]\n\t"
+        "vmla.i16	q6, q8, q9\n\t"
+        "add	r6, r6, r12\n\t"
+        "vld1.16	{d18-d19}, [r6]\n\t"
+        "vmla.i16	q7, q8, q9\n\t"
+        "add	r6, r6, r12\n\t"
+        "add	r5, r5, #16\n\t"
+        "subs	r8, r8, #1\n\t"
+        "bne	L_frodokem_mul_bs_neon_k_%=\n\t"
+        "vadd.i16	d18, d0, d1\n\t"
+        "vpadd.i16	d18, d18, d18\n\t"
+        "vpadd.i16	d18, d18, d18\n\t"
+        "vmov.u16	r10, d18[0]\n\t"
+        "vmov.16	d20[0], r10\n\t"
+        "vadd.i16	d18, d2, d3\n\t"
+        "vpadd.i16	d18, d18, d18\n\t"
+        "vpadd.i16	d18, d18, d18\n\t"
+        "vmov.u16	r10, d18[0]\n\t"
+        "vmov.16	d20[1], r10\n\t"
+        "vadd.i16	d18, d4, d5\n\t"
+        "vpadd.i16	d18, d18, d18\n\t"
+        "vpadd.i16	d18, d18, d18\n\t"
+        "vmov.u16	r10, d18[0]\n\t"
+        "vmov.16	d20[2], r10\n\t"
+        "vadd.i16	d18, d6, d7\n\t"
+        "vpadd.i16	d18, d18, d18\n\t"
+        "vpadd.i16	d18, d18, d18\n\t"
+        "vmov.u16	r10, d18[0]\n\t"
+        "vmov.16	d20[3], r10\n\t"
+        "vadd.i16	d18, d8, d9\n\t"
+        "vpadd.i16	d18, d18, d18\n\t"
+        "vpadd.i16	d18, d18, d18\n\t"
+        "vmov.u16	r10, d18[0]\n\t"
+        "vmov.16	d21[0], r10\n\t"
+        "vadd.i16	d18, d10, d11\n\t"
+        "vpadd.i16	d18, d18, d18\n\t"
+        "vpadd.i16	d18, d18, d18\n\t"
+        "vmov.u16	r10, d18[0]\n\t"
+        "vmov.16	d21[1], r10\n\t"
+        "vadd.i16	d18, d12, d13\n\t"
+        "vpadd.i16	d18, d18, d18\n\t"
+        "vpadd.i16	d18, d18, d18\n\t"
+        "vmov.u16	r10, d18[0]\n\t"
+        "vmov.16	d21[2], r10\n\t"
+        "vadd.i16	d18, d14, d15\n\t"
+        "vpadd.i16	d18, d18, d18\n\t"
+        "vpadd.i16	d18, d18, d18\n\t"
+        "vmov.u16	r10, d18[0]\n\t"
+        "vmov.16	d21[3], r10\n\t"
+        "vand	q10, q10, q11\n\t"
+        "vst1.16	{d20-d21}, [r7]!\n\t"
+        "subs	r9, r9, #1\n\t"
+        "bne	L_frodokem_mul_bs_neon_r_%=\n\t"
+        "pop	{%[qmask]}\n\t"
+#ifndef WOLFSSL_NO_VAR_ASSIGN_REG
+        : [out] "+r" (out), [b] "+r" (b), [s] "+r" (s), [n] "+r" (n),
+          [qmask] "+r" (qmask)
+        :
+#else
+        :
+        : [out] "r" (out), [b] "r" (b), [s] "r" (s), [n] "r" (n),
+          [qmask] "r" (qmask)
+#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */
+        : "memory", "cc", "lr", "r4", "r5", "r6", "r7", "r8", "r9", "r10", "q0",
+            "q1", "q2", "q3", "q4", "q5", "q6", "q7", "q8", "q9", "q10", "q11"
+    );
+}
+
+void frodokem_mul_add_sb_plus_e_neon(word16* out_p, const word16* b_p,
+    const word16* s_p, int n_p, int qmask_p);
+#ifndef WOLFSSL_NO_VAR_ASSIGN_REG
+WC_OMIT_FRAME_POINTER void frodokem_mul_add_sb_plus_e_neon(word16* out_p,
+    const word16* b_p, const word16* s_p, int n_p, int qmask_p)
+#else
+WC_OMIT_FRAME_POINTER void frodokem_mul_add_sb_plus_e_neon(word16* out,
+    const word16* b, const word16* s, int n, int qmask)
+#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */
+{
+#ifndef WOLFSSL_NO_VAR_ASSIGN_REG
+    register word16* out __asm__ ("r0") = (word16*)out_p;
+    register const word16* b __asm__ ("r1") = (const word16*)b_p;
+    register const word16* s __asm__ ("r2") = (const word16*)s_p;
+    register int n __asm__ ("r3") = (int)n_p;
+    register int qmask __asm__ ("r12") = (int)qmask_p;
+#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */
+
+    __asm__ __volatile__ (
+        "push	{%[qmask]}\n\t"
+        "ldr	lr, [sp]\n\t"
+        "lsl	r12, %[n], #1\n\t"
+        "vdup.16	q10, lr\n\t"
+        "mov	r4, %[b]\n\t"
+        "mov	r5, %[s]\n\t"
+        "mov	r7, %[out]\n\t"
+        "vld1.16	{d0-d1}, [r7]!\n\t"
+        "vld1.16	{d2-d3}, [r7]!\n\t"
+        "vld1.16	{d4-d5}, [r7]!\n\t"
+        "vld1.16	{d6-d7}, [r7]!\n\t"
+        "vld1.16	{d8-d9}, [r7]!\n\t"
+        "vld1.16	{d10-d11}, [r7]!\n\t"
+        "vld1.16	{d12-d13}, [r7]!\n\t"
+        "vld1.16	{d14-d15}, [r7]!\n\t"
+        "mov	r8, %[n]\n\t"
+        "\n"
+    "L_frodokem_mul_add_sb_plus_e_neon_j_%=:\n\t"
+        "vld1.16	{d16-d17}, [r4]!\n\t"
+        "mov	r6, r5\n\t"
+        "ldrh	r9, [r6]\n\t"
+        "vdup.16	q9, r9\n\t"
+        "vmla.i16	q0, q8, q9\n\t"
+        "add	r6, r6, r12\n\t"
+        "ldrh	r9, [r6]\n\t"
+        "vdup.16	q9, r9\n\t"
+        "vmla.i16	q1, q8, q9\n\t"
+        "add	r6, r6, r12\n\t"
+        "ldrh	r9, [r6]\n\t"
+        "vdup.16	q9, r9\n\t"
+        "vmla.i16	q2, q8, q9\n\t"
+        "add	r6, r6, r12\n\t"
+        "ldrh	r9, [r6]\n\t"
+        "vdup.16	q9, r9\n\t"
+        "vmla.i16	q3, q8, q9\n\t"
+        "add	r6, r6, r12\n\t"
+        "ldrh	r9, [r6]\n\t"
+        "vdup.16	q9, r9\n\t"
+        "vmla.i16	q4, q8, q9\n\t"
+        "add	r6, r6, r12\n\t"
+        "ldrh	r9, [r6]\n\t"
+        "vdup.16	q9, r9\n\t"
+        "vmla.i16	q5, q8, q9\n\t"
+        "add	r6, r6, r12\n\t"
+        "ldrh	r9, [r6]\n\t"
+        "vdup.16	q9, r9\n\t"
+        "vmla.i16	q6, q8, q9\n\t"
+        "add	r6, r6, r12\n\t"
+        "ldrh	r9, [r6]\n\t"
+        "vdup.16	q9, r9\n\t"
+        "vmla.i16	q7, q8, q9\n\t"
+        "add	r6, r6, r12\n\t"
+        "add	r5, r5, #2\n\t"
+        "subs	r8, r8, #1\n\t"
+        "bne	L_frodokem_mul_add_sb_plus_e_neon_j_%=\n\t"
+        "mov	r7, %[out]\n\t"
+        "vand	q0, q0, q10\n\t"
+        "vst1.16	{d0-d1}, [r7]!\n\t"
+        "vand	q1, q1, q10\n\t"
+        "vst1.16	{d2-d3}, [r7]!\n\t"
+        "vand	q2, q2, q10\n\t"
+        "vst1.16	{d4-d5}, [r7]!\n\t"
+        "vand	q3, q3, q10\n\t"
+        "vst1.16	{d6-d7}, [r7]!\n\t"
+        "vand	q4, q4, q10\n\t"
+        "vst1.16	{d8-d9}, [r7]!\n\t"
+        "vand	q5, q5, q10\n\t"
+        "vst1.16	{d10-d11}, [r7]!\n\t"
+        "vand	q6, q6, q10\n\t"
+        "vst1.16	{d12-d13}, [r7]!\n\t"
+        "vand	q7, q7, q10\n\t"
+        "vst1.16	{d14-d15}, [r7]!\n\t"
+        "pop	{%[qmask]}\n\t"
+#ifndef WOLFSSL_NO_VAR_ASSIGN_REG
+        : [out] "+r" (out), [b] "+r" (b), [s] "+r" (s), [n] "+r" (n),
+          [qmask] "+r" (qmask)
+        :
+#else
+        :
+        : [out] "r" (out), [b] "r" (b), [s] "r" (s), [n] "r" (n),
+          [qmask] "r" (qmask)
+#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */
+        : "memory", "cc", "lr", "r4", "r5", "r6", "r7", "r8", "r9", "q0", "q1",
+            "q2", "q3", "q4", "q5", "q6", "q7", "q8", "q9", "q10"
+    );
+}
+
+#endif /* WOLFSSL_ARMASM_NO_NEON */
+#endif /* WOLFSSL_HAVE_FRODOKEM */
+#ifdef WOLFSSL_HAVE_FRODOKEM
+#ifdef __ARM_FEATURE_SIMD32
+#if (!defined(__ARM_NEON) && !defined(__ARM_NEON__)) || \
+        defined(WOLFSSL_ARMASM_NO_NEON)
+#ifndef WOLFSSL_NO_VAR_ASSIGN_REG
+WC_OMIT_FRAME_POINTER void frodokem_add_simd32(word16* a_p, const word16* b_p,
+    int qmask_p)
+#else
+WC_OMIT_FRAME_POINTER void frodokem_add_simd32(word16* a, const word16* b,
+    int qmask)
+#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */
+{
+#ifndef WOLFSSL_NO_VAR_ASSIGN_REG
+    register word16* a __asm__ ("r0") = (word16*)a_p;
+    register const word16* b __asm__ ("r1") = (const word16*)b_p;
+    register int qmask __asm__ ("r2") = (int)qmask_p;
+#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */
+
+    __asm__ __volatile__ (
+        "orr	%[qmask], %[qmask], %[qmask], lsl #16\n\t"
+        "mov	r3, #32\n\t"
+        "\n"
+    "L_frodokem_add_simd32_blk_%=:\n\t"
+        "ldr	r12, [%[a]]\n\t"
+        "ldr	lr, [%[b]], #4\n\t"
+        "sadd16	r12, r12, lr\n\t"
+        "and	r12, r12, %[qmask]\n\t"
+        "str	r12, [%[a]], #4\n\t"
+        "subs	r3, r3, #1\n\t"
+        "bne	L_frodokem_add_simd32_blk_%=\n\t"
+#ifndef WOLFSSL_NO_VAR_ASSIGN_REG
+        : [a] "+r" (a), [b] "+r" (b), [qmask] "+r" (qmask)
+        :
+#else
+        :
+        : [a] "r" (a), [b] "r" (b), [qmask] "r" (qmask)
+#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */
+        : "memory", "cc", "r3", "r12", "lr"
+    );
+}
+
+#ifndef WOLFSSL_NO_VAR_ASSIGN_REG
+WC_OMIT_FRAME_POINTER void frodokem_sa_accum_simd32(word16* out_p,
+    const word16* s_p, const word16* row_p, int j_p, int n_p)
+#else
+WC_OMIT_FRAME_POINTER void frodokem_sa_accum_simd32(word16* out,
+    const word16* s, const word16* row, int j, int n)
+#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */
+{
+#ifndef WOLFSSL_NO_VAR_ASSIGN_REG
+    register word16* out __asm__ ("r0") = (word16*)out_p;
+    register const word16* s __asm__ ("r1") = (const word16*)s_p;
+    register const word16* row __asm__ ("r2") = (const word16*)row_p;
+    register int j __asm__ ("r3") = (int)j_p;
+    register int n __asm__ ("r12") = (int)n_p;
+#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */
+
+    __asm__ __volatile__ (
+        "push	{%[n]}\n\t"
+        "ldr	r12, [sp]\n\t"
+        "lsl	r12, r12, #1\n\t"
+        "add	lr, %[s], %[j], lsl #1\n\t"
+        "mov	r6, #8\n\t"
+        "\n"
+    "L_frodokem_sa_accum_simd32_i_%=:\n\t"
+        "ldrh	r7, [lr]\n\t"
+        "add	lr, lr, r12\n\t"
+        "mov	r4, %[row]\n\t"
+        "lsr	r5, r12, #2\n\t"
+        "\n"
+    "L_frodokem_sa_accum_simd32_k_%=:\n\t"
+        "ldr	r9, [%[out]]\n\t"
+        "ldr	r8, [r4], #4\n\t"
+        "smulbb	r10, r7, r8\n\t"
+        "smulbt	r11, r7, r8\n\t"
+        "pkhbt	r10, r10, r11, LSL #16\n\t"
+        "sadd16	r9, r9, r10\n\t"
+        "str	r9, [%[out]], #4\n\t"
+        "subs	r5, r5, #1\n\t"
+        "bne	L_frodokem_sa_accum_simd32_k_%=\n\t"
+        "subs	r6, r6, #1\n\t"
+        "bne	L_frodokem_sa_accum_simd32_i_%=\n\t"
+        "pop	{%[n]}\n\t"
+#ifndef WOLFSSL_NO_VAR_ASSIGN_REG
+        : [out] "+r" (out), [s] "+r" (s), [row] "+r" (row), [j] "+r" (j),
+          [n] "+r" (n)
+        :
+#else
+        :
+        : [out] "r" (out), [s] "r" (s), [row] "r" (row), [j] "r" (j),
+          [n] "r" (n)
+#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */
+        : "memory", "cc", "lr", "r4", "r5", "r6", "r7", "r8", "r9", "r10",
+            "r11"
+    );
+}
+
+#ifndef WOLFSSL_NO_VAR_ASSIGN_REG
+WC_OMIT_FRAME_POINTER void frodokem_as_accum_simd32(word16* out_p,
+    const word16* s_p, const word16* row_p, int i_p, int n_p)
+#else
+WC_OMIT_FRAME_POINTER void frodokem_as_accum_simd32(word16* out,
+    const word16* s, const word16* row, int i, int n)
+#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */
+{
+#ifndef WOLFSSL_NO_VAR_ASSIGN_REG
+    register word16* out __asm__ ("r0") = (word16*)out_p;
+    register const word16* s __asm__ ("r1") = (const word16*)s_p;
+    register const word16* row __asm__ ("r2") = (const word16*)row_p;
+    register int i __asm__ ("r3") = (int)i_p;
+    register int n __asm__ ("r12") = (int)n_p;
+#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */
+
+    __asm__ __volatile__ (
+        "push	{%[n]}\n\t"
+        "ldr	r12, [sp]\n\t"
+        "lsl	r12, r12, #1\n\t"
+        "add	lr, %[out], %[i], lsl #4\n\t"
+        "mov	r4, %[s]\n\t"
+        "mov	r7, #8\n\t"
+        "\n"
+    "L_frodokem_as_accum_simd32_c_%=:\n\t"
+        "mov	r9, #0\n\t"
+        "mov	r5, %[row]\n\t"
+        "mov	r6, r4\n\t"
+        "lsr	r8, r12, #2\n\t"
+        "\n"
+    "L_frodokem_as_accum_simd32_j_%=:\n\t"
+        "ldr	r10, [r5], #4\n\t"
+        "ldr	r11, [r6], #4\n\t"
+        "smlad	r9, r10, r11, r9\n\t"
+        "subs	r8, r8, #1\n\t"
+        "bne	L_frodokem_as_accum_simd32_j_%=\n\t"
+        "ldrh	r10, [lr]\n\t"
+        "add	r10, r10, r9\n\t"
+        "strh	r10, [lr]\n\t"
+        "add	lr, lr, #2\n\t"
+        "add	r4, r4, r12\n\t"
+        "subs	r7, r7, #1\n\t"
+        "bne	L_frodokem_as_accum_simd32_c_%=\n\t"
+        "pop	{%[n]}\n\t"
+#ifndef WOLFSSL_NO_VAR_ASSIGN_REG
+        : [out] "+r" (out), [s] "+r" (s), [row] "+r" (row), [i] "+r" (i),
+          [n] "+r" (n)
+        :
+#else
+        :
+        : [out] "r" (out), [s] "r" (s), [row] "r" (row), [i] "r" (i),
+          [n] "r" (n)
+#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */
+        : "memory", "cc", "lr", "r4", "r5", "r6", "r7", "r8", "r9", "r10",
+            "r11"
+    );
+}
+
+#ifndef WOLFSSL_NO_VAR_ASSIGN_REG
+WC_OMIT_FRAME_POINTER void frodokem_mul_bs_simd32(word16* out_p,
+    const word16* b_p, const word16* s_p, int n_p, int qmask_p)
+#else
+WC_OMIT_FRAME_POINTER void frodokem_mul_bs_simd32(word16* out, const word16* b,
+    const word16* s, int n, int qmask)
+#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */
+{
+#ifndef WOLFSSL_NO_VAR_ASSIGN_REG
+    register word16* out __asm__ ("r0") = (word16*)out_p;
+    register const word16* b __asm__ ("r1") = (const word16*)b_p;
+    register const word16* s __asm__ ("r2") = (const word16*)s_p;
+    register int n __asm__ ("r3") = (int)n_p;
+    register int qmask __asm__ ("r12") = (int)qmask_p;
+#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */
+
+    __asm__ __volatile__ (
+        "push	{%[qmask]}\n\t"
+        "mov	r12, %[b]\n\t"
+        "mov	r6, #8\n\t"
+        "\n"
+    "L_frodokem_mul_bs_simd32_i_%=:\n\t"
+        "mov	lr, %[s]\n\t"
+        "mov	r7, #8\n\t"
+        "\n"
+    "L_frodokem_mul_bs_simd32_c_%=:\n\t"
+        "mov	r9, #0\n\t"
+        "mov	r4, r12\n\t"
+        "mov	r5, lr\n\t"
+        "lsr	r8, %[n], #1\n\t"
+        "\n"
+    "L_frodokem_mul_bs_simd32_j_%=:\n\t"
+        "ldr	r10, [r4], #4\n\t"
+        "ldr	r11, [r5], #4\n\t"
+        "smlad	r9, r10, r11, r9\n\t"
+        "subs	r8, r8, #1\n\t"
+        "bne	L_frodokem_mul_bs_simd32_j_%=\n\t"
+        "ldr	r10, [sp]\n\t"
+        "orr	r10, r10, r10, lsl #16\n\t"
+        "and	r9, r9, r10\n\t"
+        "strh	r9, [%[out]]\n\t"
+        "add	%[out], %[out], #2\n\t"
+        "add	lr, lr, %[n], lsl #1\n\t"
+        "subs	r7, r7, #1\n\t"
+        "bne	L_frodokem_mul_bs_simd32_c_%=\n\t"
+        "add	r12, r12, %[n], lsl #1\n\t"
+        "subs	r6, r6, #1\n\t"
+        "bne	L_frodokem_mul_bs_simd32_i_%=\n\t"
+        "pop	{%[qmask]}\n\t"
+#ifndef WOLFSSL_NO_VAR_ASSIGN_REG
+        : [out] "+r" (out), [b] "+r" (b), [s] "+r" (s), [n] "+r" (n),
+          [qmask] "+r" (qmask)
+        :
+#else
+        :
+        : [out] "r" (out), [b] "r" (b), [s] "r" (s), [n] "r" (n),
+          [qmask] "r" (qmask)
+#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */
+        : "memory", "cc", "lr", "r4", "r5", "r6", "r7", "r8", "r9", "r10",
+            "r11"
+    );
+}
+
+#ifndef WOLFSSL_NO_VAR_ASSIGN_REG
+WC_OMIT_FRAME_POINTER void frodokem_mul_add_sb_plus_e_simd32(word16* out_p,
+    const word16* b_p, const word16* s_p, int n_p, int qmask_p)
+#else
+WC_OMIT_FRAME_POINTER void frodokem_mul_add_sb_plus_e_simd32(word16* out,
+    const word16* b, const word16* s, int n, int qmask)
+#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */
+{
+#ifndef WOLFSSL_NO_VAR_ASSIGN_REG
+    register word16* out __asm__ ("r0") = (word16*)out_p;
+    register const word16* b __asm__ ("r1") = (const word16*)b_p;
+    register const word16* s __asm__ ("r2") = (const word16*)s_p;
+    register int n __asm__ ("r3") = (int)n_p;
+    register int qmask __asm__ ("r12") = (int)qmask_p;
+#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */
+
+    __asm__ __volatile__ (
+        "push	{%[qmask]}\n\t"
+        "sub	sp, sp, #8\n\t"
+        "lsl	r9, %[n], #1\n\t"
+        "str	r9, [sp, #4]\n\t"
+        "str	%[b], [sp]\n\t"
+        "mov	r12, #8\n\t"
+        "\n"
+    "L_frodokem_mul_add_sb_plus_e_simd32_i_%=:\n\t"
+        "ldr	r9, [sp, #4]\n\t"
+        "add	%[n], %[s], r9\n\t"
+        "ldr	%[b], [sp]\n\t"
+        "mov	lr, %[s]\n\t"
+        "ldr	r4, [%[out]]\n\t"
+        "ldr	r5, [%[out], #4]\n\t"
+        "ldr	r6, [%[out], #8]\n\t"
+        "ldr	r7, [%[out], #12]\n\t"
+        "\n"
+    "L_frodokem_mul_add_sb_plus_e_simd32_j_%=:\n\t"
+        "ldrh	r8, [lr]\n\t"
+        "add	lr, lr, #2\n\t"
+        "ldr	r9, [%[b]]\n\t"
+        "smulbb	r10, r8, r9\n\t"
+        "smulbt	r11, r8, r9\n\t"
+        "pkhbt	r10, r10, r11, LSL #16\n\t"
+        "sadd16	r4, r4, r10\n\t"
+        "ldr	r9, [%[b], #4]\n\t"
+        "smulbb	r10, r8, r9\n\t"
+        "smulbt	r11, r8, r9\n\t"
+        "pkhbt	r10, r10, r11, LSL #16\n\t"
+        "sadd16	r5, r5, r10\n\t"
+        "ldr	r9, [%[b], #8]\n\t"
+        "smulbb	r10, r8, r9\n\t"
+        "smulbt	r11, r8, r9\n\t"
+        "pkhbt	r10, r10, r11, LSL #16\n\t"
+        "sadd16	r6, r6, r10\n\t"
+        "ldr	r9, [%[b], #12]\n\t"
+        "smulbb	r10, r8, r9\n\t"
+        "smulbt	r11, r8, r9\n\t"
+        "pkhbt	r10, r10, r11, LSL #16\n\t"
+        "sadd16	r7, r7, r10\n\t"
+        "add	%[b], %[b], #16\n\t"
+        "cmp	lr, %[n]\n\t"
+        "bne	L_frodokem_mul_add_sb_plus_e_simd32_j_%=\n\t"
+        "ldr	r9, [sp]\n\t"
+        "orr	r9, r9, r9, lsl #16\n\t"
+        "and	r4, r4, r9\n\t"
+        "and	r5, r5, r9\n\t"
+        "and	r6, r6, r9\n\t"
+        "and	r7, r7, r9\n\t"
+        "str	r4, [%[out]]\n\t"
+        "str	r5, [%[out], #4]\n\t"
+        "str	r6, [%[out], #8]\n\t"
+        "str	r7, [%[out], #12]\n\t"
+        "add	%[out], %[out], #16\n\t"
+        "mov	%[s], %[n]\n\t"
+        "subs	r12, r12, #1\n\t"
+        "bne	L_frodokem_mul_add_sb_plus_e_simd32_i_%=\n\t"
+        "add	sp, sp, #8\n\t"
+        "pop	{%[qmask]}\n\t"
+#ifndef WOLFSSL_NO_VAR_ASSIGN_REG
+        : [out] "+r" (out), [b] "+r" (b), [s] "+r" (s), [n] "+r" (n),
+          [qmask] "+r" (qmask)
+        :
+#else
+        :
+        : [out] "r" (out), [b] "r" (b), [s] "r" (s), [n] "r" (n),
+          [qmask] "r" (qmask)
+#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */
+        : "memory", "cc", "lr", "r4", "r5", "r6", "r7", "r8", "r9", "r10",
+            "r11"
+    );
+}
+
+#endif /* (!defined(__ARM_NEON) && !defined(__ARM_NEON__)) ||
+        * defined(WOLFSSL_ARMASM_NO_NEON) */
+#endif /* __ARM_FEATURE_SIMD32 */
+#endif /* WOLFSSL_HAVE_FRODOKEM */
+#endif /* !__aarch64__ && !WOLFSSL_ARMASM_THUMB2 */
+#endif /* WOLFSSL_ARMASM */
+
+#endif /* WOLFSSL_ARMASM_INLINE */
diff --git a/wolfcrypt/src/port/arm/armv8-aes-asm.asm b/wolfcrypt/src/port/arm/armv8-aes-asm.asm
index b6155ace79..1416b02bf5 100644
--- a/wolfcrypt/src/port/arm/armv8-aes-asm.asm
+++ b/wolfcrypt/src/port/arm/armv8-aes-asm.asm
@@ -42809,259 +42809,1306 @@ L_aes_xts_decrypt_arm64_crypto_done
 	ENDP
 	ENDIF
 	ENDIF
-	ENDIF
-	IF :LNOT::DEF:WOLFSSL_ARMASM_NO_NEON
-	IF :DEF:HAVE_AES_DECRYPT :LOR: :DEF:HAVE_AES_CBC :LOR: :DEF:HAVE_AESCCM :LOR: :DEF:HAVE_AESGCM :LOR: :DEF:WOLFSSL_AES_DIRECT :LOR: :DEF:WOLFSSL_AES_COUNTER
-	AREA	|.rodata|, DATA, READONLY, ALIGN=4
-	ALIGN	8
-L_AES_ARM64_NEON_te
-	DCB	0x63, 0x7c, 0x77, 0x7b, 0xf2, 0x6b, 0x6f, 0xc5
-	DCB	0x30, 0x01, 0x67, 0x2b, 0xfe, 0xd7, 0xab, 0x76
-	DCB	0xca, 0x82, 0xc9, 0x7d, 0xfa, 0x59, 0x47, 0xf0
-	DCB	0xad, 0xd4, 0xa2, 0xaf, 0x9c, 0xa4, 0x72, 0xc0
-	DCB	0xb7, 0xfd, 0x93, 0x26, 0x36, 0x3f, 0xf7, 0xcc
-	DCB	0x34, 0xa5, 0xe5, 0xf1, 0x71, 0xd8, 0x31, 0x15
-	DCB	0x04, 0xc7, 0x23, 0xc3, 0x18, 0x96, 0x05, 0x9a
-	DCB	0x07, 0x12, 0x80, 0xe2, 0xeb, 0x27, 0xb2, 0x75
-	DCB	0x09, 0x83, 0x2c, 0x1a, 0x1b, 0x6e, 0x5a, 0xa0
-	DCB	0x52, 0x3b, 0xd6, 0xb3, 0x29, 0xe3, 0x2f, 0x84
-	DCB	0x53, 0xd1, 0x00, 0xed, 0x20, 0xfc, 0xb1, 0x5b
-	DCB	0x6a, 0xcb, 0xbe, 0x39, 0x4a, 0x4c, 0x58, 0xcf
-	DCB	0xd0, 0xef, 0xaa, 0xfb, 0x43, 0x4d, 0x33, 0x85
-	DCB	0x45, 0xf9, 0x02, 0x7f, 0x50, 0x3c, 0x9f, 0xa8
-	DCB	0x51, 0xa3, 0x40, 0x8f, 0x92, 0x9d, 0x38, 0xf5
-	DCB	0xbc, 0xb6, 0xda, 0x21, 0x10, 0xff, 0xf3, 0xd2
-	DCB	0xcd, 0x0c, 0x13, 0xec, 0x5f, 0x97, 0x44, 0x17
-	DCB	0xc4, 0xa7, 0x7e, 0x3d, 0x64, 0x5d, 0x19, 0x73
-	DCB	0x60, 0x81, 0x4f, 0xdc, 0x22, 0x2a, 0x90, 0x88
-	DCB	0x46, 0xee, 0xb8, 0x14, 0xde, 0x5e, 0x0b, 0xdb
-	DCB	0xe0, 0x32, 0x3a, 0x0a, 0x49, 0x06, 0x24, 0x5c
-	DCB	0xc2, 0xd3, 0xac, 0x62, 0x91, 0x95, 0xe4, 0x79
-	DCB	0xe7, 0xc8, 0x37, 0x6d, 0x8d, 0xd5, 0x4e, 0xa9
-	DCB	0x6c, 0x56, 0xf4, 0xea, 0x65, 0x7a, 0xae, 0x08
-	DCB	0xba, 0x78, 0x25, 0x2e, 0x1c, 0xa6, 0xb4, 0xc6
-	DCB	0xe8, 0xdd, 0x74, 0x1f, 0x4b, 0xbd, 0x8b, 0x8a
-	DCB	0x70, 0x3e, 0xb5, 0x66, 0x48, 0x03, 0xf6, 0x0e
-	DCB	0x61, 0x35, 0x57, 0xb9, 0x86, 0xc1, 0x1d, 0x9e
-	DCB	0xe1, 0xf8, 0x98, 0x11, 0x69, 0xd9, 0x8e, 0x94
-	DCB	0x9b, 0x1e, 0x87, 0xe9, 0xce, 0x55, 0x28, 0xdf
-	DCB	0x8c, 0xa1, 0x89, 0x0d, 0xbf, 0xe6, 0x42, 0x68
-	DCB	0x41, 0x99, 0x2d, 0x0f, 0xb0, 0x54, 0xbb, 0x16
-	AREA	|.rodata|, DATA, READONLY, ALIGN=4
-	ALIGN	8
-L_AES_ARM64_NEON_shift_rows_shuffle
-	DCB	0x0c, 0x09, 0x06, 0x03, 0x00, 0x0d, 0x0a, 0x07
-	DCB	0x04, 0x01, 0x0e, 0x0b, 0x08, 0x05, 0x02, 0x0f
-	ENDIF
-	IF :DEF:HAVE_AES_DECRYPT
+	IF :DEF:WOLFSSL_AESGCM_SIV
 	AREA	|.text|, CODE, READONLY
 	ALIGN	4
-	EXPORT	AES_invert_key_NEON
-AES_invert_key_NEON PROC
-	add	x3, x0, x1, lsl 4
-	mov	x2, x0
-	mov	w4, w1
-L_AES_invert_key_NEON_loop
-	ld1	{V0.2D}, [x2]
-	ld1	{V1.2D}, [x3]
-	st1	{V0.2D}, [x3]
-	st1	{V1.2D}, [x2], #16
-	subs	w4, w4, #2
-	sub	x3, x3, #16
-	bne	L_AES_invert_key_NEON_loop
-	movi	V2.16B, #27
-	add	x2, x0, #16
-	sub	w4, w1, #1
-L_AES_invert_key_NEON_mix_loop
-	ld1	{V0.2D}, [x2]
-	sshr	V5.16B, V0.16B, #7
-	ushr	V6.16B, V0.16B, #6
-	ushr	V3.16B, V0.16B, #5
-	and	V5.16B, V5.16B, V2.16B
-	pmul	V6.16B, V6.16B, V2.16B
-	pmul	V3.16B, V3.16B, V2.16B
-	shl	V4.16B, V0.16B, #1
-	eor	V5.16B, V5.16B, V4.16B
-	shl	V4.16B, V0.16B, #3
-	eor	V3.16B, V3.16B, V4.16B
-	shl	V4.16B, V0.16B, #2
-	eor	V6.16B, V6.16B, V4.16B
-	eor	V4.16B, V5.16B, V3.16B
-	eor	V3.16B, V3.16B, V0.16B
-	eor	V5.16B, V6.16B, V3.16B
-	eor	V6.16B, V6.16B, V4.16B
-	eor	V4.16B, V4.16B, V0.16B
-	shl	V0.4S, V4.4S, #8
-	rev32	V5.8H, V5.8H
-	sri	V0.4S, V4.4S, #24
-	eor	V0.16B, V0.16B, V6.16B
-	shl	V4.4S, V3.4S, #24
-	eor	V0.16B, V0.16B, V5.16B
-	sri	V4.4S, V3.4S, #8
-	eor	V0.16B, V0.16B, V4.16B
-	st1	{V0.2D}, [x2], #16
-	subs	w4, w4, #1
-	bne	L_AES_invert_key_NEON_mix_loop
+	EXPORT	AES_GCMSIV_polyval_pmull
+AES_GCMSIV_polyval_pmull PROC
+	stp	x29, x30, [sp, #-80]!
+	add	x29, sp, #0
+	stp	D8, D9, [x29, #16]
+	stp	D10, D11, [x29, #32]
+	stp	D12, D13, [x29, #48]
+	stp	D14, D15, [x29, #64]
+	movi	V2.16B, #0
+	ld1	{V1.16B}, [x1]
+	ld1	{V0.16B}, [x0]
+	rev64	V0.16B, V0.16B
+	ext8	V0.16B, V0.16B, V0.16B, #8
+	pmull	V18.1Q, V1.1D, V1.1D
+	pmull2	V19.1Q, V1.2D, V1.2D
+	ext8	V22.16B, V1.16B, V1.16B, #8
+	pmull	V23.1Q, V22.1D, V1.1D
+	pmull2	V22.1Q, V22.2D, V1.2D
+	eor	V22.16B, V22.16B, V23.16B
+	ext8	V23.16B, V2.16B, V22.16B, #8
+	ext8	V22.16B, V22.16B, V2.16B, #8
+	eor	V18.16B, V18.16B, V23.16B
+	eor	V19.16B, V19.16B, V22.16B
+	ushr	V24.4S, V18.4S, #31
+	ushr	V25.4S, V19.4S, #31
+	shl	V18.4S, V18.4S, #1
+	shl	V19.4S, V19.4S, #1
+	ext8	V26.16B, V24.16B, V2.16B, #12
+	ext8	V25.16B, V2.16B, V25.16B, #12
+	ext8	V24.16B, V2.16B, V24.16B, #12
+	orr	V18.16B, V18.16B, V24.16B
+	orr	V19.16B, V19.16B, V25.16B
+	orr	V19.16B, V19.16B, V26.16B
+	shl	V24.4S, V18.4S, #31
+	shl	V25.4S, V18.4S, #30
+	shl	V26.4S, V18.4S, #25
+	eor	V24.16B, V24.16B, V25.16B
+	eor	V24.16B, V24.16B, V26.16B
+	ext8	V25.16B, V24.16B, V2.16B, #4
+	ext8	V24.16B, V2.16B, V24.16B, #4
+	eor	V18.16B, V18.16B, V24.16B
+	ushr	V24.4S, V18.4S, #1
+	ushr	V26.4S, V18.4S, #2
+	ushr	V27.4S, V18.4S, #7
+	eor	V24.16B, V24.16B, V26.16B
+	eor	V24.16B, V24.16B, V27.16B
+	eor	V24.16B, V24.16B, V25.16B
+	eor	V18.16B, V18.16B, V24.16B
+	eor	V3.16B, V19.16B, V18.16B
+	pmull	V18.1Q, V3.1D, V1.1D
+	pmull2	V19.1Q, V3.2D, V1.2D
+	ext8	V22.16B, V3.16B, V3.16B, #8
+	pmull	V23.1Q, V22.1D, V1.1D
+	pmull2	V22.1Q, V22.2D, V1.2D
+	eor	V22.16B, V22.16B, V23.16B
+	ext8	V23.16B, V2.16B, V22.16B, #8
+	ext8	V22.16B, V22.16B, V2.16B, #8
+	eor	V18.16B, V18.16B, V23.16B
+	eor	V19.16B, V19.16B, V22.16B
+	ushr	V24.4S, V18.4S, #31
+	ushr	V25.4S, V19.4S, #31
+	shl	V18.4S, V18.4S, #1
+	shl	V19.4S, V19.4S, #1
+	ext8	V26.16B, V24.16B, V2.16B, #12
+	ext8	V25.16B, V2.16B, V25.16B, #12
+	ext8	V24.16B, V2.16B, V24.16B, #12
+	orr	V18.16B, V18.16B, V24.16B
+	orr	V19.16B, V19.16B, V25.16B
+	orr	V19.16B, V19.16B, V26.16B
+	shl	V24.4S, V18.4S, #31
+	shl	V25.4S, V18.4S, #30
+	shl	V26.4S, V18.4S, #25
+	eor	V24.16B, V24.16B, V25.16B
+	eor	V24.16B, V24.16B, V26.16B
+	ext8	V25.16B, V24.16B, V2.16B, #4
+	ext8	V24.16B, V2.16B, V24.16B, #4
+	eor	V18.16B, V18.16B, V24.16B
+	ushr	V24.4S, V18.4S, #1
+	ushr	V26.4S, V18.4S, #2
+	ushr	V27.4S, V18.4S, #7
+	eor	V24.16B, V24.16B, V26.16B
+	eor	V24.16B, V24.16B, V27.16B
+	eor	V24.16B, V24.16B, V25.16B
+	eor	V18.16B, V18.16B, V24.16B
+	eor	V4.16B, V19.16B, V18.16B
+	pmull	V18.1Q, V3.1D, V3.1D
+	pmull2	V19.1Q, V3.2D, V3.2D
+	ext8	V22.16B, V3.16B, V3.16B, #8
+	pmull	V23.1Q, V22.1D, V3.1D
+	pmull2	V22.1Q, V22.2D, V3.2D
+	eor	V22.16B, V22.16B, V23.16B
+	ext8	V23.16B, V2.16B, V22.16B, #8
+	ext8	V22.16B, V22.16B, V2.16B, #8
+	eor	V18.16B, V18.16B, V23.16B
+	eor	V19.16B, V19.16B, V22.16B
+	ushr	V24.4S, V18.4S, #31
+	ushr	V25.4S, V19.4S, #31
+	shl	V18.4S, V18.4S, #1
+	shl	V19.4S, V19.4S, #1
+	ext8	V26.16B, V24.16B, V2.16B, #12
+	ext8	V25.16B, V2.16B, V25.16B, #12
+	ext8	V24.16B, V2.16B, V24.16B, #12
+	orr	V18.16B, V18.16B, V24.16B
+	orr	V19.16B, V19.16B, V25.16B
+	orr	V19.16B, V19.16B, V26.16B
+	shl	V24.4S, V18.4S, #31
+	shl	V25.4S, V18.4S, #30
+	shl	V26.4S, V18.4S, #25
+	eor	V24.16B, V24.16B, V25.16B
+	eor	V24.16B, V24.16B, V26.16B
+	ext8	V25.16B, V24.16B, V2.16B, #4
+	ext8	V24.16B, V2.16B, V24.16B, #4
+	eor	V18.16B, V18.16B, V24.16B
+	ushr	V24.4S, V18.4S, #1
+	ushr	V26.4S, V18.4S, #2
+	ushr	V27.4S, V18.4S, #7
+	eor	V24.16B, V24.16B, V26.16B
+	eor	V24.16B, V24.16B, V27.16B
+	eor	V24.16B, V24.16B, V25.16B
+	eor	V18.16B, V18.16B, V24.16B
+	eor	V5.16B, V19.16B, V18.16B
+	pmull	V18.1Q, V5.1D, V1.1D
+	pmull2	V19.1Q, V5.2D, V1.2D
+	ext8	V22.16B, V5.16B, V5.16B, #8
+	pmull	V23.1Q, V22.1D, V1.1D
+	pmull2	V22.1Q, V22.2D, V1.2D
+	eor	V22.16B, V22.16B, V23.16B
+	ext8	V23.16B, V2.16B, V22.16B, #8
+	ext8	V22.16B, V22.16B, V2.16B, #8
+	eor	V18.16B, V18.16B, V23.16B
+	eor	V19.16B, V19.16B, V22.16B
+	ushr	V24.4S, V18.4S, #31
+	ushr	V25.4S, V19.4S, #31
+	shl	V18.4S, V18.4S, #1
+	shl	V19.4S, V19.4S, #1
+	ext8	V26.16B, V24.16B, V2.16B, #12
+	ext8	V25.16B, V2.16B, V25.16B, #12
+	ext8	V24.16B, V2.16B, V24.16B, #12
+	orr	V18.16B, V18.16B, V24.16B
+	orr	V19.16B, V19.16B, V25.16B
+	orr	V19.16B, V19.16B, V26.16B
+	shl	V24.4S, V18.4S, #31
+	shl	V25.4S, V18.4S, #30
+	shl	V26.4S, V18.4S, #25
+	eor	V24.16B, V24.16B, V25.16B
+	eor	V24.16B, V24.16B, V26.16B
+	ext8	V25.16B, V24.16B, V2.16B, #4
+	ext8	V24.16B, V2.16B, V24.16B, #4
+	eor	V18.16B, V18.16B, V24.16B
+	ushr	V24.4S, V18.4S, #1
+	ushr	V26.4S, V18.4S, #2
+	ushr	V27.4S, V18.4S, #7
+	eor	V24.16B, V24.16B, V26.16B
+	eor	V24.16B, V24.16B, V27.16B
+	eor	V24.16B, V24.16B, V25.16B
+	eor	V18.16B, V18.16B, V24.16B
+	eor	V6.16B, V19.16B, V18.16B
+	pmull	V18.1Q, V5.1D, V3.1D
+	pmull2	V19.1Q, V5.2D, V3.2D
+	ext8	V22.16B, V5.16B, V5.16B, #8
+	pmull	V23.1Q, V22.1D, V3.1D
+	pmull2	V22.1Q, V22.2D, V3.2D
+	eor	V22.16B, V22.16B, V23.16B
+	ext8	V23.16B, V2.16B, V22.16B, #8
+	ext8	V22.16B, V22.16B, V2.16B, #8
+	eor	V18.16B, V18.16B, V23.16B
+	eor	V19.16B, V19.16B, V22.16B
+	ushr	V24.4S, V18.4S, #31
+	ushr	V25.4S, V19.4S, #31
+	shl	V18.4S, V18.4S, #1
+	shl	V19.4S, V19.4S, #1
+	ext8	V26.16B, V24.16B, V2.16B, #12
+	ext8	V25.16B, V2.16B, V25.16B, #12
+	ext8	V24.16B, V2.16B, V24.16B, #12
+	orr	V18.16B, V18.16B, V24.16B
+	orr	V19.16B, V19.16B, V25.16B
+	orr	V19.16B, V19.16B, V26.16B
+	shl	V24.4S, V18.4S, #31
+	shl	V25.4S, V18.4S, #30
+	shl	V26.4S, V18.4S, #25
+	eor	V24.16B, V24.16B, V25.16B
+	eor	V24.16B, V24.16B, V26.16B
+	ext8	V25.16B, V24.16B, V2.16B, #4
+	ext8	V24.16B, V2.16B, V24.16B, #4
+	eor	V18.16B, V18.16B, V24.16B
+	ushr	V24.4S, V18.4S, #1
+	ushr	V26.4S, V18.4S, #2
+	ushr	V27.4S, V18.4S, #7
+	eor	V24.16B, V24.16B, V26.16B
+	eor	V24.16B, V24.16B, V27.16B
+	eor	V24.16B, V24.16B, V25.16B
+	eor	V18.16B, V18.16B, V24.16B
+	eor	V7.16B, V19.16B, V18.16B
+	pmull	V18.1Q, V5.1D, V4.1D
+	pmull2	V19.1Q, V5.2D, V4.2D
+	ext8	V22.16B, V5.16B, V5.16B, #8
+	pmull	V23.1Q, V22.1D, V4.1D
+	pmull2	V22.1Q, V22.2D, V4.2D
+	eor	V22.16B, V22.16B, V23.16B
+	ext8	V23.16B, V2.16B, V22.16B, #8
+	ext8	V22.16B, V22.16B, V2.16B, #8
+	eor	V18.16B, V18.16B, V23.16B
+	eor	V19.16B, V19.16B, V22.16B
+	ushr	V24.4S, V18.4S, #31
+	ushr	V25.4S, V19.4S, #31
+	shl	V18.4S, V18.4S, #1
+	shl	V19.4S, V19.4S, #1
+	ext8	V26.16B, V24.16B, V2.16B, #12
+	ext8	V25.16B, V2.16B, V25.16B, #12
+	ext8	V24.16B, V2.16B, V24.16B, #12
+	orr	V18.16B, V18.16B, V24.16B
+	orr	V19.16B, V19.16B, V25.16B
+	orr	V19.16B, V19.16B, V26.16B
+	shl	V24.4S, V18.4S, #31
+	shl	V25.4S, V18.4S, #30
+	shl	V26.4S, V18.4S, #25
+	eor	V24.16B, V24.16B, V25.16B
+	eor	V24.16B, V24.16B, V26.16B
+	ext8	V25.16B, V24.16B, V2.16B, #4
+	ext8	V24.16B, V2.16B, V24.16B, #4
+	eor	V18.16B, V18.16B, V24.16B
+	ushr	V24.4S, V18.4S, #1
+	ushr	V26.4S, V18.4S, #2
+	ushr	V27.4S, V18.4S, #7
+	eor	V24.16B, V24.16B, V26.16B
+	eor	V24.16B, V24.16B, V27.16B
+	eor	V24.16B, V24.16B, V25.16B
+	eor	V18.16B, V18.16B, V24.16B
+	eor	V8.16B, V19.16B, V18.16B
+	pmull	V18.1Q, V5.1D, V5.1D
+	pmull2	V19.1Q, V5.2D, V5.2D
+	ext8	V22.16B, V5.16B, V5.16B, #8
+	pmull	V23.1Q, V22.1D, V5.1D
+	pmull2	V22.1Q, V22.2D, V5.2D
+	eor	V22.16B, V22.16B, V23.16B
+	ext8	V23.16B, V2.16B, V22.16B, #8
+	ext8	V22.16B, V22.16B, V2.16B, #8
+	eor	V18.16B, V18.16B, V23.16B
+	eor	V19.16B, V19.16B, V22.16B
+	ushr	V24.4S, V18.4S, #31
+	ushr	V25.4S, V19.4S, #31
+	shl	V18.4S, V18.4S, #1
+	shl	V19.4S, V19.4S, #1
+	ext8	V26.16B, V24.16B, V2.16B, #12
+	ext8	V25.16B, V2.16B, V25.16B, #12
+	ext8	V24.16B, V2.16B, V24.16B, #12
+	orr	V18.16B, V18.16B, V24.16B
+	orr	V19.16B, V19.16B, V25.16B
+	orr	V19.16B, V19.16B, V26.16B
+	shl	V24.4S, V18.4S, #31
+	shl	V25.4S, V18.4S, #30
+	shl	V26.4S, V18.4S, #25
+	eor	V24.16B, V24.16B, V25.16B
+	eor	V24.16B, V24.16B, V26.16B
+	ext8	V25.16B, V24.16B, V2.16B, #4
+	ext8	V24.16B, V2.16B, V24.16B, #4
+	eor	V18.16B, V18.16B, V24.16B
+	ushr	V24.4S, V18.4S, #1
+	ushr	V26.4S, V18.4S, #2
+	ushr	V27.4S, V18.4S, #7
+	eor	V24.16B, V24.16B, V26.16B
+	eor	V24.16B, V24.16B, V27.16B
+	eor	V24.16B, V24.16B, V25.16B
+	eor	V18.16B, V18.16B, V24.16B
+	eor	V9.16B, V19.16B, V18.16B
+L_AES_GCMSIV_polyval_pmull_loop8
+	cmp	w3, #8
+	blt	L_AES_GCMSIV_polyval_pmull_done8
+	ld1	{V10.16B, V11.16B, V12.16B, V13.16B}, [x2], #0x40
+	ld1	{V14.16B, V15.16B, V16.16B, V17.16B}, [x2], #0x40
+	eor	V10.16B, V10.16B, V0.16B
+	pmull	V18.1Q, V10.1D, V9.1D
+	pmull2	V19.1Q, V10.2D, V9.2D
+	ext8	V22.16B, V10.16B, V10.16B, #8
+	pmull	V23.1Q, V22.1D, V9.1D
+	pmull2	V22.1Q, V22.2D, V9.2D
+	eor	V22.16B, V22.16B, V23.16B
+	ext8	V23.16B, V2.16B, V22.16B, #8
+	ext8	V22.16B, V22.16B, V2.16B, #8
+	eor	V18.16B, V18.16B, V23.16B
+	eor	V19.16B, V19.16B, V22.16B
+	pmull	V20.1Q, V11.1D, V8.1D
+	pmull2	V21.1Q, V11.2D, V8.2D
+	ext8	V22.16B, V11.16B, V11.16B, #8
+	pmull	V23.1Q, V22.1D, V8.1D
+	pmull2	V22.1Q, V22.2D, V8.2D
+	eor	V22.16B, V22.16B, V23.16B
+	ext8	V23.16B, V2.16B, V22.16B, #8
+	ext8	V22.16B, V22.16B, V2.16B, #8
+	eor	V20.16B, V20.16B, V23.16B
+	eor	V21.16B, V21.16B, V22.16B
+	eor	V18.16B, V18.16B, V20.16B
+	eor	V19.16B, V19.16B, V21.16B
+	pmull	V20.1Q, V12.1D, V7.1D
+	pmull2	V21.1Q, V12.2D, V7.2D
+	ext8	V22.16B, V12.16B, V12.16B, #8
+	pmull	V23.1Q, V22.1D, V7.1D
+	pmull2	V22.1Q, V22.2D, V7.2D
+	eor	V22.16B, V22.16B, V23.16B
+	ext8	V23.16B, V2.16B, V22.16B, #8
+	ext8	V22.16B, V22.16B, V2.16B, #8
+	eor	V20.16B, V20.16B, V23.16B
+	eor	V21.16B, V21.16B, V22.16B
+	eor	V18.16B, V18.16B, V20.16B
+	eor	V19.16B, V19.16B, V21.16B
+	pmull	V20.1Q, V13.1D, V6.1D
+	pmull2	V21.1Q, V13.2D, V6.2D
+	ext8	V22.16B, V13.16B, V13.16B, #8
+	pmull	V23.1Q, V22.1D, V6.1D
+	pmull2	V22.1Q, V22.2D, V6.2D
+	eor	V22.16B, V22.16B, V23.16B
+	ext8	V23.16B, V2.16B, V22.16B, #8
+	ext8	V22.16B, V22.16B, V2.16B, #8
+	eor	V20.16B, V20.16B, V23.16B
+	eor	V21.16B, V21.16B, V22.16B
+	eor	V18.16B, V18.16B, V20.16B
+	eor	V19.16B, V19.16B, V21.16B
+	pmull	V20.1Q, V14.1D, V5.1D
+	pmull2	V21.1Q, V14.2D, V5.2D
+	ext8	V22.16B, V14.16B, V14.16B, #8
+	pmull	V23.1Q, V22.1D, V5.1D
+	pmull2	V22.1Q, V22.2D, V5.2D
+	eor	V22.16B, V22.16B, V23.16B
+	ext8	V23.16B, V2.16B, V22.16B, #8
+	ext8	V22.16B, V22.16B, V2.16B, #8
+	eor	V20.16B, V20.16B, V23.16B
+	eor	V21.16B, V21.16B, V22.16B
+	eor	V18.16B, V18.16B, V20.16B
+	eor	V19.16B, V19.16B, V21.16B
+	pmull	V20.1Q, V15.1D, V4.1D
+	pmull2	V21.1Q, V15.2D, V4.2D
+	ext8	V22.16B, V15.16B, V15.16B, #8
+	pmull	V23.1Q, V22.1D, V4.1D
+	pmull2	V22.1Q, V22.2D, V4.2D
+	eor	V22.16B, V22.16B, V23.16B
+	ext8	V23.16B, V2.16B, V22.16B, #8
+	ext8	V22.16B, V22.16B, V2.16B, #8
+	eor	V20.16B, V20.16B, V23.16B
+	eor	V21.16B, V21.16B, V22.16B
+	eor	V18.16B, V18.16B, V20.16B
+	eor	V19.16B, V19.16B, V21.16B
+	pmull	V20.1Q, V16.1D, V3.1D
+	pmull2	V21.1Q, V16.2D, V3.2D
+	ext8	V22.16B, V16.16B, V16.16B, #8
+	pmull	V23.1Q, V22.1D, V3.1D
+	pmull2	V22.1Q, V22.2D, V3.2D
+	eor	V22.16B, V22.16B, V23.16B
+	ext8	V23.16B, V2.16B, V22.16B, #8
+	ext8	V22.16B, V22.16B, V2.16B, #8
+	eor	V20.16B, V20.16B, V23.16B
+	eor	V21.16B, V21.16B, V22.16B
+	eor	V18.16B, V18.16B, V20.16B
+	eor	V19.16B, V19.16B, V21.16B
+	pmull	V20.1Q, V17.1D, V1.1D
+	pmull2	V21.1Q, V17.2D, V1.2D
+	ext8	V22.16B, V17.16B, V17.16B, #8
+	pmull	V23.1Q, V22.1D, V1.1D
+	pmull2	V22.1Q, V22.2D, V1.2D
+	eor	V22.16B, V22.16B, V23.16B
+	ext8	V23.16B, V2.16B, V22.16B, #8
+	ext8	V22.16B, V22.16B, V2.16B, #8
+	eor	V20.16B, V20.16B, V23.16B
+	eor	V21.16B, V21.16B, V22.16B
+	eor	V18.16B, V18.16B, V20.16B
+	eor	V19.16B, V19.16B, V21.16B
+	ushr	V24.4S, V18.4S, #31
+	ushr	V25.4S, V19.4S, #31
+	shl	V18.4S, V18.4S, #1
+	shl	V19.4S, V19.4S, #1
+	ext8	V26.16B, V24.16B, V2.16B, #12
+	ext8	V25.16B, V2.16B, V25.16B, #12
+	ext8	V24.16B, V2.16B, V24.16B, #12
+	orr	V18.16B, V18.16B, V24.16B
+	orr	V19.16B, V19.16B, V25.16B
+	orr	V19.16B, V19.16B, V26.16B
+	shl	V24.4S, V18.4S, #31
+	shl	V25.4S, V18.4S, #30
+	shl	V26.4S, V18.4S, #25
+	eor	V24.16B, V24.16B, V25.16B
+	eor	V24.16B, V24.16B, V26.16B
+	ext8	V25.16B, V24.16B, V2.16B, #4
+	ext8	V24.16B, V2.16B, V24.16B, #4
+	eor	V18.16B, V18.16B, V24.16B
+	ushr	V24.4S, V18.4S, #1
+	ushr	V26.4S, V18.4S, #2
+	ushr	V27.4S, V18.4S, #7
+	eor	V24.16B, V24.16B, V26.16B
+	eor	V24.16B, V24.16B, V27.16B
+	eor	V24.16B, V24.16B, V25.16B
+	eor	V18.16B, V18.16B, V24.16B
+	eor	V0.16B, V19.16B, V18.16B
+	sub	w3, w3, #8
+	b	L_AES_GCMSIV_polyval_pmull_loop8
+L_AES_GCMSIV_polyval_pmull_done8
+L_AES_GCMSIV_polyval_pmull_loop4
+	cmp	w3, #4
+	blt	L_AES_GCMSIV_polyval_pmull_done4
+	ld1	{V10.16B, V11.16B, V12.16B, V13.16B}, [x2], #0x40
+	eor	V10.16B, V10.16B, V0.16B
+	pmull	V18.1Q, V10.1D, V5.1D
+	pmull2	V19.1Q, V10.2D, V5.2D
+	ext8	V22.16B, V10.16B, V10.16B, #8
+	pmull	V23.1Q, V22.1D, V5.1D
+	pmull2	V22.1Q, V22.2D, V5.2D
+	eor	V22.16B, V22.16B, V23.16B
+	ext8	V23.16B, V2.16B, V22.16B, #8
+	ext8	V22.16B, V22.16B, V2.16B, #8
+	eor	V18.16B, V18.16B, V23.16B
+	eor	V19.16B, V19.16B, V22.16B
+	pmull	V20.1Q, V11.1D, V4.1D
+	pmull2	V21.1Q, V11.2D, V4.2D
+	ext8	V22.16B, V11.16B, V11.16B, #8
+	pmull	V23.1Q, V22.1D, V4.1D
+	pmull2	V22.1Q, V22.2D, V4.2D
+	eor	V22.16B, V22.16B, V23.16B
+	ext8	V23.16B, V2.16B, V22.16B, #8
+	ext8	V22.16B, V22.16B, V2.16B, #8
+	eor	V20.16B, V20.16B, V23.16B
+	eor	V21.16B, V21.16B, V22.16B
+	eor	V18.16B, V18.16B, V20.16B
+	eor	V19.16B, V19.16B, V21.16B
+	pmull	V20.1Q, V12.1D, V3.1D
+	pmull2	V21.1Q, V12.2D, V3.2D
+	ext8	V22.16B, V12.16B, V12.16B, #8
+	pmull	V23.1Q, V22.1D, V3.1D
+	pmull2	V22.1Q, V22.2D, V3.2D
+	eor	V22.16B, V22.16B, V23.16B
+	ext8	V23.16B, V2.16B, V22.16B, #8
+	ext8	V22.16B, V22.16B, V2.16B, #8
+	eor	V20.16B, V20.16B, V23.16B
+	eor	V21.16B, V21.16B, V22.16B
+	eor	V18.16B, V18.16B, V20.16B
+	eor	V19.16B, V19.16B, V21.16B
+	pmull	V20.1Q, V13.1D, V1.1D
+	pmull2	V21.1Q, V13.2D, V1.2D
+	ext8	V22.16B, V13.16B, V13.16B, #8
+	pmull	V23.1Q, V22.1D, V1.1D
+	pmull2	V22.1Q, V22.2D, V1.2D
+	eor	V22.16B, V22.16B, V23.16B
+	ext8	V23.16B, V2.16B, V22.16B, #8
+	ext8	V22.16B, V22.16B, V2.16B, #8
+	eor	V20.16B, V20.16B, V23.16B
+	eor	V21.16B, V21.16B, V22.16B
+	eor	V18.16B, V18.16B, V20.16B
+	eor	V19.16B, V19.16B, V21.16B
+	ushr	V24.4S, V18.4S, #31
+	ushr	V25.4S, V19.4S, #31
+	shl	V18.4S, V18.4S, #1
+	shl	V19.4S, V19.4S, #1
+	ext8	V26.16B, V24.16B, V2.16B, #12
+	ext8	V25.16B, V2.16B, V25.16B, #12
+	ext8	V24.16B, V2.16B, V24.16B, #12
+	orr	V18.16B, V18.16B, V24.16B
+	orr	V19.16B, V19.16B, V25.16B
+	orr	V19.16B, V19.16B, V26.16B
+	shl	V24.4S, V18.4S, #31
+	shl	V25.4S, V18.4S, #30
+	shl	V26.4S, V18.4S, #25
+	eor	V24.16B, V24.16B, V25.16B
+	eor	V24.16B, V24.16B, V26.16B
+	ext8	V25.16B, V24.16B, V2.16B, #4
+	ext8	V24.16B, V2.16B, V24.16B, #4
+	eor	V18.16B, V18.16B, V24.16B
+	ushr	V24.4S, V18.4S, #1
+	ushr	V26.4S, V18.4S, #2
+	ushr	V27.4S, V18.4S, #7
+	eor	V24.16B, V24.16B, V26.16B
+	eor	V24.16B, V24.16B, V27.16B
+	eor	V24.16B, V24.16B, V25.16B
+	eor	V18.16B, V18.16B, V24.16B
+	eor	V0.16B, V19.16B, V18.16B
+	sub	w3, w3, #4
+	b	L_AES_GCMSIV_polyval_pmull_loop4
+L_AES_GCMSIV_polyval_pmull_done4
+L_AES_GCMSIV_polyval_pmull_rem
+	cbz	w3, L_AES_GCMSIV_polyval_pmull_done
+	ld1	{V10.16B}, [x2], #16
+	eor	V0.16B, V0.16B, V10.16B
+	pmull	V18.1Q, V0.1D, V1.1D
+	pmull2	V19.1Q, V0.2D, V1.2D
+	ext8	V22.16B, V0.16B, V0.16B, #8
+	pmull	V23.1Q, V22.1D, V1.1D
+	pmull2	V22.1Q, V22.2D, V1.2D
+	eor	V22.16B, V22.16B, V23.16B
+	ext8	V23.16B, V2.16B, V22.16B, #8
+	ext8	V22.16B, V22.16B, V2.16B, #8
+	eor	V18.16B, V18.16B, V23.16B
+	eor	V19.16B, V19.16B, V22.16B
+	ushr	V24.4S, V18.4S, #31
+	ushr	V25.4S, V19.4S, #31
+	shl	V18.4S, V18.4S, #1
+	shl	V19.4S, V19.4S, #1
+	ext8	V26.16B, V24.16B, V2.16B, #12
+	ext8	V25.16B, V2.16B, V25.16B, #12
+	ext8	V24.16B, V2.16B, V24.16B, #12
+	orr	V18.16B, V18.16B, V24.16B
+	orr	V19.16B, V19.16B, V25.16B
+	orr	V19.16B, V19.16B, V26.16B
+	shl	V24.4S, V18.4S, #31
+	shl	V25.4S, V18.4S, #30
+	shl	V26.4S, V18.4S, #25
+	eor	V24.16B, V24.16B, V25.16B
+	eor	V24.16B, V24.16B, V26.16B
+	ext8	V25.16B, V24.16B, V2.16B, #4
+	ext8	V24.16B, V2.16B, V24.16B, #4
+	eor	V18.16B, V18.16B, V24.16B
+	ushr	V24.4S, V18.4S, #1
+	ushr	V26.4S, V18.4S, #2
+	ushr	V27.4S, V18.4S, #7
+	eor	V24.16B, V24.16B, V26.16B
+	eor	V24.16B, V24.16B, V27.16B
+	eor	V24.16B, V24.16B, V25.16B
+	eor	V18.16B, V18.16B, V24.16B
+	eor	V0.16B, V19.16B, V18.16B
+	subs	w3, w3, #1
+	bne	L_AES_GCMSIV_polyval_pmull_rem
+L_AES_GCMSIV_polyval_pmull_done
+	rev64	V0.16B, V0.16B
+	ext8	V0.16B, V0.16B, V0.16B, #8
+	st1	{V0.16B}, [x0]
+	ldp	D8, D9, [x29, #16]
+	ldp	D10, D11, [x29, #32]
+	ldp	D12, D13, [x29, #48]
+	ldp	D14, D15, [x29, #64]
+	ldp	x29, x30, [sp], #0x50
 	ret
 	ENDP
-	ENDIF
-	AREA	|.rodata|, DATA, READONLY, ALIGN=4
-	ALIGN	8
-L_AES_ARM64_NEON_rcon
-	DCD	0x01000000, 0x02000000, 0x04000000, 0x08000000
-	DCD	0x10000000, 0x20000000, 0x40000000, 0x80000000
-	DCD	0x1b000000, 0x36000000
 	AREA	|.text|, CODE, READONLY
 	ALIGN	4
-	EXPORT	AES_set_encrypt_key_NEON
-AES_set_encrypt_key_NEON PROC
+	EXPORT	AES_GCMSIV_ctr_aarch64
+AES_GCMSIV_ctr_aarch64 PROC
 	stp	x29, x30, [sp, #-80]!
 	add	x29, sp, #0
 	stp	D8, D9, [x29, #16]
 	stp	D10, D11, [x29, #32]
 	stp	D12, D13, [x29, #48]
 	stp	D14, D15, [x29, #64]
-	adrp	x4, L_AES_ARM64_NEON_rcon
-	add	x4, x4, L_AES_ARM64_NEON_rcon
-	adrp	x5, L_AES_ARM64_NEON_te
-	add	x5, x5, L_AES_ARM64_NEON_te
-	ld1	{V6.16B, V7.16B, V8.16B, V9.16B}, [x5], #0x40
-	ld1	{V10.16B, V11.16B, V12.16B, V13.16B}, [x5], #0x40
-	ld1	{V14.16B, V15.16B, V16.16B, V17.16B}, [x5], #0x40
-	ld1	{V18.16B, V19.16B, V20.16B, V21.16B}, [x5]
-	movi	V2.16B, #0x40
-	movi	V3.16B, #0x80
-	movi	V4.16B, #0xc0
-	movi	V5.16B, #27
-	eor	V26.16B, V26.16B, V26.16B
-	cmp	x1, #0x80
-	beq	L_AES_set_encrypt_key_NEON_start_128
-	cmp	x1, #0xc0
-	beq	L_AES_set_encrypt_key_NEON_start_192
-	ld1	{V0.16B}, [x0], #16
-	ld1	{V1.16B}, [x0]
-	rev32	V0.16B, V0.16B
-	rev32	V1.16B, V1.16B
-	st1	{V0.2D}, [x2], #16
-	st1	{V1.2D}, [x2], #16
-	mov	x3, #6
-L_AES_set_encrypt_key_NEON_loop_256
-	eor	V22.16B, V1.16B, V2.16B
-	eor	V23.16B, V1.16B, V3.16B
-	eor	V24.16B, V1.16B, V4.16B
-	tbl	V25.16B, {V6.16B, V7.16B, V8.16B, V9.16B}, V1.16B
-	tbl	V22.16B, {V10.16B, V11.16B, V12.16B, V13.16B}, V22.16B
-	tbl	V23.16B, {V14.16B, V15.16B, V16.16B, V17.16B}, V23.16B
-	tbl	V24.16B, {V18.16B, V19.16B, V20.16B, V21.16B}, V24.16B
-	orr	V25.16B, V25.16B, V22.16B
-	orr	V23.16B, V23.16B, V24.16B
-	orr	V25.16B, V25.16B, V23.16B
-	ext8	V25.16B, V25.16B, V26.16B, #12
-	shl	V22.4S, V25.4S, #8
-	sri	V22.4S, V25.4S, #24
-	eor	V0.16B, V0.16B, V22.16B
-	ld1r	{V25.4S}, [x4], #4
-	dup	V22.4S, V0.S[0]
-	dup	V23.2S, V0.S[1]
-	dup	V24.2S, V0.S[2]
-	ext8	V22.16B, V26.16B, V22.16B, #12
-	ext8	V23.16B, V26.16B, V23.16B, #8
-	eor	V0.16B, V0.16B, V22.16B
-	ext8	V24.16B, V26.16B, V24.16B, #4
-	eor	V0.16B, V0.16B, V23.16B
-	eor	V0.16B, V0.16B, V24.16B
-	eor	V0.16B, V0.16B, V25.16B
-	st1	{V0.2D}, [x2], #16
-	eor	V22.16B, V0.16B, V2.16B
-	eor	V23.16B, V0.16B, V3.16B
-	eor	V24.16B, V0.16B, V4.16B
-	tbl	V25.16B, {V6.16B, V7.16B, V8.16B, V9.16B}, V0.16B
-	tbl	V22.16B, {V10.16B, V11.16B, V12.16B, V13.16B}, V22.16B
-	tbl	V23.16B, {V14.16B, V15.16B, V16.16B, V17.16B}, V23.16B
-	tbl	V24.16B, {V18.16B, V19.16B, V20.16B, V21.16B}, V24.16B
-	orr	V25.16B, V25.16B, V22.16B
-	orr	V23.16B, V23.16B, V24.16B
-	orr	V25.16B, V25.16B, V23.16B
-	ext8	V25.16B, V25.16B, V26.16B, #12
-	eor	V1.16B, V1.16B, V25.16B
-	dup	V22.4S, V1.S[0]
-	dup	V23.2S, V1.S[1]
-	dup	V24.2S, V1.S[2]
-	ext8	V22.16B, V26.16B, V22.16B, #12
-	ext8	V23.16B, V26.16B, V23.16B, #8
-	eor	V1.16B, V1.16B, V22.16B
-	ext8	V24.16B, V26.16B, V24.16B, #4
-	eor	V1.16B, V1.16B, V23.16B
-	eor	V1.16B, V1.16B, V24.16B
-	st1	{V1.2D}, [x2], #16
-	subs	x3, x3, #1
-	bne	L_AES_set_encrypt_key_NEON_loop_256
-	eor	V22.16B, V1.16B, V2.16B
-	eor	V23.16B, V1.16B, V3.16B
-	eor	V24.16B, V1.16B, V4.16B
-	tbl	V25.16B, {V6.16B, V7.16B, V8.16B, V9.16B}, V1.16B
-	tbl	V22.16B, {V10.16B, V11.16B, V12.16B, V13.16B}, V22.16B
-	tbl	V23.16B, {V14.16B, V15.16B, V16.16B, V17.16B}, V23.16B
-	tbl	V24.16B, {V18.16B, V19.16B, V20.16B, V21.16B}, V24.16B
-	orr	V25.16B, V25.16B, V22.16B
-	orr	V23.16B, V23.16B, V24.16B
-	orr	V25.16B, V25.16B, V23.16B
-	ext8	V25.16B, V25.16B, V26.16B, #12
-	shl	V22.4S, V25.4S, #8
-	sri	V22.4S, V25.4S, #24
-	eor	V0.16B, V0.16B, V22.16B
-	ld1r	{V25.4S}, [x4], #4
-	dup	V22.4S, V0.S[0]
-	dup	V23.2S, V0.S[1]
-	dup	V24.2S, V0.S[2]
-	ext8	V22.16B, V26.16B, V22.16B, #12
-	ext8	V23.16B, V26.16B, V23.16B, #8
-	eor	V0.16B, V0.16B, V22.16B
-	ext8	V24.16B, V26.16B, V24.16B, #4
-	eor	V0.16B, V0.16B, V23.16B
-	eor	V0.16B, V0.16B, V24.16B
-	eor	V0.16B, V0.16B, V25.16B
-	st1	{V0.2D}, [x2], #16
-	b	L_AES_set_encrypt_key_NEON_end
-L_AES_set_encrypt_key_NEON_start_192
-	ld1	{V0.16B}, [x0], #16
-	ld1	{V1.8B}, [x0]
-	rev32	V0.16B, V0.16B
-	rev32	V1.8B, V1.8B
-	st1	{V0.16B}, [x2], #16
-	st1	{V1.8B}, [x2], #8
-	ext8	V1.16B, V1.16B, V1.16B, #8
-	mov	x3, #7
-L_AES_set_encrypt_key_NEON_loop_192
-	eor	V22.16B, V1.16B, V2.16B
-	eor	V23.16B, V1.16B, V3.16B
-	eor	V24.16B, V1.16B, V4.16B
-	tbl	V25.16B, {V6.16B, V7.16B, V8.16B, V9.16B}, V1.16B
-	tbl	V22.16B, {V10.16B, V11.16B, V12.16B, V13.16B}, V22.16B
-	tbl	V23.16B, {V14.16B, V15.16B, V16.16B, V17.16B}, V23.16B
-	tbl	V24.16B, {V18.16B, V19.16B, V20.16B, V21.16B}, V24.16B
-	orr	V25.16B, V25.16B, V22.16B
-	orr	V23.16B, V23.16B, V24.16B
-	orr	V25.16B, V25.16B, V23.16B
-	ext8	V25.16B, V25.16B, V26.16B, #12
-	shl	V22.4S, V25.4S, #8
-	sri	V22.4S, V25.4S, #24
-	eor	V0.16B, V0.16B, V22.16B
-	ld1r	{V25.4S}, [x4], #4
-	dup	V22.4S, V0.S[0]
-	dup	V23.2S, V0.S[1]
-	dup	V24.2S, V0.S[2]
-	ext8	V22.16B, V26.16B, V22.16B, #12
-	ext8	V23.16B, V26.16B, V23.16B, #8
-	eor	V0.16B, V0.16B, V22.16B
-	ext8	V24.16B, V26.16B, V24.16B, #4
-	eor	V0.16B, V0.16B, V23.16B
-	eor	V0.16B, V0.16B, V24.16B
-	eor	V0.16B, V0.16B, V25.16B
-	st1	{V0.2D}, [x2], #16
-	mov	V23.16B, V26.16B
-	mov	V23.S[2], V0.S[3]
-	eor	V1.16B, V1.16B, V23.16B
+	ld1	{V15.2D}, [x5]
+	mov	w7, V15.S[0]
+	lsr	x6, x2, #4
+	ld1	{V0.2D, V1.2D, V2.2D, V3.2D}, [x3], #0x40
+	ld1	{V4.2D, V5.2D, V6.2D, V7.2D}, [x3], #0x40
+	cmp	w4, #12
+	blt	L_AES_GCMSIV_ctr_aarch64_start_128
+	bgt	L_AES_GCMSIV_ctr_aarch64_start_256
+	; AES_GCMSIV_CTR_192
+	IF :LNOT::DEF:NO_AES_192
+	ld1	{V8.2D, V9.2D, V10.2D, V11.2D}, [x3], #0x40
+	ld1	{V12.2D}, [x3]
+L_AES_GCMSIV_ctr_aarch64_192_loop4
+	cmp	x6, #4
+	blt	L_AES_GCMSIV_ctr_aarch64_192_done4
+	mov	V16.16B, V15.16B
+	mov	V16.S[0], w7
+	mov	V17.16B, V15.16B
+	add	w8, w7, #1
+	mov	V17.S[0], w8
+	mov	V18.16B, V15.16B
+	add	w8, w7, #2
+	mov	V18.S[0], w8
+	mov	V19.16B, V15.16B
+	add	w8, w7, #3
+	mov	V19.S[0], w8
+	add	w7, w7, #4
+	ld1	{V20.16B, V21.16B, V22.16B, V23.16B}, [x0], #0x40
+	aese	V16.16B, V0.16B
+	aesmc	V16.16B, V16.16B
+	aese	V17.16B, V0.16B
+	aesmc	V17.16B, V17.16B
+	aese	V18.16B, V0.16B
+	aesmc	V18.16B, V18.16B
+	aese	V19.16B, V0.16B
+	aesmc	V19.16B, V19.16B
+	aese	V16.16B, V1.16B
+	aesmc	V16.16B, V16.16B
+	aese	V17.16B, V1.16B
+	aesmc	V17.16B, V17.16B
+	aese	V18.16B, V1.16B
+	aesmc	V18.16B, V18.16B
+	aese	V19.16B, V1.16B
+	aesmc	V19.16B, V19.16B
+	aese	V16.16B, V2.16B
+	aesmc	V16.16B, V16.16B
+	aese	V17.16B, V2.16B
+	aesmc	V17.16B, V17.16B
+	aese	V18.16B, V2.16B
+	aesmc	V18.16B, V18.16B
+	aese	V19.16B, V2.16B
+	aesmc	V19.16B, V19.16B
+	aese	V16.16B, V3.16B
+	aesmc	V16.16B, V16.16B
+	aese	V17.16B, V3.16B
+	aesmc	V17.16B, V17.16B
+	aese	V18.16B, V3.16B
+	aesmc	V18.16B, V18.16B
+	aese	V19.16B, V3.16B
+	aesmc	V19.16B, V19.16B
+	aese	V16.16B, V4.16B
+	aesmc	V16.16B, V16.16B
+	aese	V17.16B, V4.16B
+	aesmc	V17.16B, V17.16B
+	aese	V18.16B, V4.16B
+	aesmc	V18.16B, V18.16B
+	aese	V19.16B, V4.16B
+	aesmc	V19.16B, V19.16B
+	aese	V16.16B, V5.16B
+	aesmc	V16.16B, V16.16B
+	aese	V17.16B, V5.16B
+	aesmc	V17.16B, V17.16B
+	aese	V18.16B, V5.16B
+	aesmc	V18.16B, V18.16B
+	aese	V19.16B, V5.16B
+	aesmc	V19.16B, V19.16B
+	aese	V16.16B, V6.16B
+	aesmc	V16.16B, V16.16B
+	aese	V17.16B, V6.16B
+	aesmc	V17.16B, V17.16B
+	aese	V18.16B, V6.16B
+	aesmc	V18.16B, V18.16B
+	aese	V19.16B, V6.16B
+	aesmc	V19.16B, V19.16B
+	aese	V16.16B, V7.16B
+	aesmc	V16.16B, V16.16B
+	aese	V17.16B, V7.16B
+	aesmc	V17.16B, V17.16B
+	aese	V18.16B, V7.16B
+	aesmc	V18.16B, V18.16B
+	aese	V19.16B, V7.16B
+	aesmc	V19.16B, V19.16B
+	aese	V16.16B, V8.16B
+	aesmc	V16.16B, V16.16B
+	aese	V17.16B, V8.16B
+	aesmc	V17.16B, V17.16B
+	aese	V18.16B, V8.16B
+	aesmc	V18.16B, V18.16B
+	aese	V19.16B, V8.16B
+	aesmc	V19.16B, V19.16B
+	aese	V16.16B, V9.16B
+	aesmc	V16.16B, V16.16B
+	aese	V17.16B, V9.16B
+	aesmc	V17.16B, V17.16B
+	aese	V18.16B, V9.16B
+	aesmc	V18.16B, V18.16B
+	aese	V19.16B, V9.16B
+	aesmc	V19.16B, V19.16B
+	aese	V16.16B, V10.16B
+	aesmc	V16.16B, V16.16B
+	aese	V17.16B, V10.16B
+	aesmc	V17.16B, V17.16B
+	aese	V18.16B, V10.16B
+	aesmc	V18.16B, V18.16B
+	aese	V19.16B, V10.16B
+	aesmc	V19.16B, V19.16B
+	aese	V16.16B, V11.16B
+	eor	V16.16B, V16.16B, V12.16B
+	aese	V17.16B, V11.16B
+	eor	V17.16B, V17.16B, V12.16B
+	aese	V18.16B, V11.16B
+	eor	V18.16B, V18.16B, V12.16B
+	aese	V19.16B, V11.16B
+	eor	V19.16B, V19.16B, V12.16B
+	eor	V20.16B, V20.16B, V16.16B
+	eor	V21.16B, V21.16B, V17.16B
+	eor	V22.16B, V22.16B, V18.16B
+	eor	V23.16B, V23.16B, V19.16B
+	st1	{V20.16B, V21.16B, V22.16B, V23.16B}, [x1], #0x40
+	sub	x6, x6, #4
+	cmp	x6, #4
+	bge	L_AES_GCMSIV_ctr_aarch64_192_loop4
+L_AES_GCMSIV_ctr_aarch64_192_done4
+L_AES_GCMSIV_ctr_aarch64_192_loop1
+	cbz	x6, L_AES_GCMSIV_ctr_aarch64_192_done1
+	mov	V16.16B, V15.16B
+	mov	V16.S[0], w7
+	add	w7, w7, #1
+	ld1	{V20.16B}, [x0], #16
+	aese	V16.16B, V0.16B
+	aesmc	V16.16B, V16.16B
+	aese	V16.16B, V1.16B
+	aesmc	V16.16B, V16.16B
+	aese	V16.16B, V2.16B
+	aesmc	V16.16B, V16.16B
+	aese	V16.16B, V3.16B
+	aesmc	V16.16B, V16.16B
+	aese	V16.16B, V4.16B
+	aesmc	V16.16B, V16.16B
+	aese	V16.16B, V5.16B
+	aesmc	V16.16B, V16.16B
+	aese	V16.16B, V6.16B
+	aesmc	V16.16B, V16.16B
+	aese	V16.16B, V7.16B
+	aesmc	V16.16B, V16.16B
+	aese	V16.16B, V8.16B
+	aesmc	V16.16B, V16.16B
+	aese	V16.16B, V9.16B
+	aesmc	V16.16B, V16.16B
+	aese	V16.16B, V10.16B
+	aesmc	V16.16B, V16.16B
+	aese	V16.16B, V11.16B
+	eor	V16.16B, V16.16B, V12.16B
+	eor	V20.16B, V20.16B, V16.16B
+	st1	{V20.16B}, [x1], #16
+	sub	x6, x6, #1
+	b	L_AES_GCMSIV_ctr_aarch64_192_loop1
+L_AES_GCMSIV_ctr_aarch64_192_done1
+	ENDIF
+	b	L_AES_GCMSIV_ctr_aarch64_done
+	; AES_GCMSIV_CTR_256
+L_AES_GCMSIV_ctr_aarch64_start_256
+	IF :LNOT::DEF:NO_AES_256
+	ld1	{V8.2D, V9.2D, V10.2D, V11.2D}, [x3], #0x40
+	ld1	{V12.2D, V13.2D}, [x3], #32
+	ld1	{V14.2D}, [x3]
+L_AES_GCMSIV_ctr_aarch64_256_loop4
+	cmp	x6, #4
+	blt	L_AES_GCMSIV_ctr_aarch64_256_done4
+	mov	V16.16B, V15.16B
+	mov	V16.S[0], w7
+	mov	V17.16B, V15.16B
+	add	w8, w7, #1
+	mov	V17.S[0], w8
+	mov	V18.16B, V15.16B
+	add	w8, w7, #2
+	mov	V18.S[0], w8
+	mov	V19.16B, V15.16B
+	add	w8, w7, #3
+	mov	V19.S[0], w8
+	add	w7, w7, #4
+	ld1	{V20.16B, V21.16B, V22.16B, V23.16B}, [x0], #0x40
+	aese	V16.16B, V0.16B
+	aesmc	V16.16B, V16.16B
+	aese	V17.16B, V0.16B
+	aesmc	V17.16B, V17.16B
+	aese	V18.16B, V0.16B
+	aesmc	V18.16B, V18.16B
+	aese	V19.16B, V0.16B
+	aesmc	V19.16B, V19.16B
+	aese	V16.16B, V1.16B
+	aesmc	V16.16B, V16.16B
+	aese	V17.16B, V1.16B
+	aesmc	V17.16B, V17.16B
+	aese	V18.16B, V1.16B
+	aesmc	V18.16B, V18.16B
+	aese	V19.16B, V1.16B
+	aesmc	V19.16B, V19.16B
+	aese	V16.16B, V2.16B
+	aesmc	V16.16B, V16.16B
+	aese	V17.16B, V2.16B
+	aesmc	V17.16B, V17.16B
+	aese	V18.16B, V2.16B
+	aesmc	V18.16B, V18.16B
+	aese	V19.16B, V2.16B
+	aesmc	V19.16B, V19.16B
+	aese	V16.16B, V3.16B
+	aesmc	V16.16B, V16.16B
+	aese	V17.16B, V3.16B
+	aesmc	V17.16B, V17.16B
+	aese	V18.16B, V3.16B
+	aesmc	V18.16B, V18.16B
+	aese	V19.16B, V3.16B
+	aesmc	V19.16B, V19.16B
+	aese	V16.16B, V4.16B
+	aesmc	V16.16B, V16.16B
+	aese	V17.16B, V4.16B
+	aesmc	V17.16B, V17.16B
+	aese	V18.16B, V4.16B
+	aesmc	V18.16B, V18.16B
+	aese	V19.16B, V4.16B
+	aesmc	V19.16B, V19.16B
+	aese	V16.16B, V5.16B
+	aesmc	V16.16B, V16.16B
+	aese	V17.16B, V5.16B
+	aesmc	V17.16B, V17.16B
+	aese	V18.16B, V5.16B
+	aesmc	V18.16B, V18.16B
+	aese	V19.16B, V5.16B
+	aesmc	V19.16B, V19.16B
+	aese	V16.16B, V6.16B
+	aesmc	V16.16B, V16.16B
+	aese	V17.16B, V6.16B
+	aesmc	V17.16B, V17.16B
+	aese	V18.16B, V6.16B
+	aesmc	V18.16B, V18.16B
+	aese	V19.16B, V6.16B
+	aesmc	V19.16B, V19.16B
+	aese	V16.16B, V7.16B
+	aesmc	V16.16B, V16.16B
+	aese	V17.16B, V7.16B
+	aesmc	V17.16B, V17.16B
+	aese	V18.16B, V7.16B
+	aesmc	V18.16B, V18.16B
+	aese	V19.16B, V7.16B
+	aesmc	V19.16B, V19.16B
+	aese	V16.16B, V8.16B
+	aesmc	V16.16B, V16.16B
+	aese	V17.16B, V8.16B
+	aesmc	V17.16B, V17.16B
+	aese	V18.16B, V8.16B
+	aesmc	V18.16B, V18.16B
+	aese	V19.16B, V8.16B
+	aesmc	V19.16B, V19.16B
+	aese	V16.16B, V9.16B
+	aesmc	V16.16B, V16.16B
+	aese	V17.16B, V9.16B
+	aesmc	V17.16B, V17.16B
+	aese	V18.16B, V9.16B
+	aesmc	V18.16B, V18.16B
+	aese	V19.16B, V9.16B
+	aesmc	V19.16B, V19.16B
+	aese	V16.16B, V10.16B
+	aesmc	V16.16B, V16.16B
+	aese	V17.16B, V10.16B
+	aesmc	V17.16B, V17.16B
+	aese	V18.16B, V10.16B
+	aesmc	V18.16B, V18.16B
+	aese	V19.16B, V10.16B
+	aesmc	V19.16B, V19.16B
+	aese	V16.16B, V11.16B
+	aesmc	V16.16B, V16.16B
+	aese	V17.16B, V11.16B
+	aesmc	V17.16B, V17.16B
+	aese	V18.16B, V11.16B
+	aesmc	V18.16B, V18.16B
+	aese	V19.16B, V11.16B
+	aesmc	V19.16B, V19.16B
+	aese	V16.16B, V12.16B
+	aesmc	V16.16B, V16.16B
+	aese	V17.16B, V12.16B
+	aesmc	V17.16B, V17.16B
+	aese	V18.16B, V12.16B
+	aesmc	V18.16B, V18.16B
+	aese	V19.16B, V12.16B
+	aesmc	V19.16B, V19.16B
+	aese	V16.16B, V13.16B
+	eor	V16.16B, V16.16B, V14.16B
+	aese	V17.16B, V13.16B
+	eor	V17.16B, V17.16B, V14.16B
+	aese	V18.16B, V13.16B
+	eor	V18.16B, V18.16B, V14.16B
+	aese	V19.16B, V13.16B
+	eor	V19.16B, V19.16B, V14.16B
+	eor	V20.16B, V20.16B, V16.16B
+	eor	V21.16B, V21.16B, V17.16B
+	eor	V22.16B, V22.16B, V18.16B
+	eor	V23.16B, V23.16B, V19.16B
+	st1	{V20.16B, V21.16B, V22.16B, V23.16B}, [x1], #0x40
+	sub	x6, x6, #4
+	cmp	x6, #4
+	bge	L_AES_GCMSIV_ctr_aarch64_256_loop4
+L_AES_GCMSIV_ctr_aarch64_256_done4
+L_AES_GCMSIV_ctr_aarch64_256_loop1
+	cbz	x6, L_AES_GCMSIV_ctr_aarch64_256_done1
+	mov	V16.16B, V15.16B
+	mov	V16.S[0], w7
+	add	w7, w7, #1
+	ld1	{V20.16B}, [x0], #16
+	aese	V16.16B, V0.16B
+	aesmc	V16.16B, V16.16B
+	aese	V16.16B, V1.16B
+	aesmc	V16.16B, V16.16B
+	aese	V16.16B, V2.16B
+	aesmc	V16.16B, V16.16B
+	aese	V16.16B, V3.16B
+	aesmc	V16.16B, V16.16B
+	aese	V16.16B, V4.16B
+	aesmc	V16.16B, V16.16B
+	aese	V16.16B, V5.16B
+	aesmc	V16.16B, V16.16B
+	aese	V16.16B, V6.16B
+	aesmc	V16.16B, V16.16B
+	aese	V16.16B, V7.16B
+	aesmc	V16.16B, V16.16B
+	aese	V16.16B, V8.16B
+	aesmc	V16.16B, V16.16B
+	aese	V16.16B, V9.16B
+	aesmc	V16.16B, V16.16B
+	aese	V16.16B, V10.16B
+	aesmc	V16.16B, V16.16B
+	aese	V16.16B, V11.16B
+	aesmc	V16.16B, V16.16B
+	aese	V16.16B, V12.16B
+	aesmc	V16.16B, V16.16B
+	aese	V16.16B, V13.16B
+	eor	V16.16B, V16.16B, V14.16B
+	eor	V20.16B, V20.16B, V16.16B
+	st1	{V20.16B}, [x1], #16
+	sub	x6, x6, #1
+	b	L_AES_GCMSIV_ctr_aarch64_256_loop1
+L_AES_GCMSIV_ctr_aarch64_256_done1
+	ENDIF
+	b	L_AES_GCMSIV_ctr_aarch64_done
+	; AES_GCMSIV_CTR_128
+L_AES_GCMSIV_ctr_aarch64_start_128
+	IF :LNOT::DEF:NO_AES_128
+	ld1	{V8.2D, V9.2D}, [x3], #32
+	ld1	{V10.2D}, [x3]
+L_AES_GCMSIV_ctr_aarch64_128_loop4
+	cmp	x6, #4
+	blt	L_AES_GCMSIV_ctr_aarch64_128_done4
+	mov	V16.16B, V15.16B
+	mov	V16.S[0], w7
+	mov	V17.16B, V15.16B
+	add	w8, w7, #1
+	mov	V17.S[0], w8
+	mov	V18.16B, V15.16B
+	add	w8, w7, #2
+	mov	V18.S[0], w8
+	mov	V19.16B, V15.16B
+	add	w8, w7, #3
+	mov	V19.S[0], w8
+	add	w7, w7, #4
+	ld1	{V20.16B, V21.16B, V22.16B, V23.16B}, [x0], #0x40
+	aese	V16.16B, V0.16B
+	aesmc	V16.16B, V16.16B
+	aese	V17.16B, V0.16B
+	aesmc	V17.16B, V17.16B
+	aese	V18.16B, V0.16B
+	aesmc	V18.16B, V18.16B
+	aese	V19.16B, V0.16B
+	aesmc	V19.16B, V19.16B
+	aese	V16.16B, V1.16B
+	aesmc	V16.16B, V16.16B
+	aese	V17.16B, V1.16B
+	aesmc	V17.16B, V17.16B
+	aese	V18.16B, V1.16B
+	aesmc	V18.16B, V18.16B
+	aese	V19.16B, V1.16B
+	aesmc	V19.16B, V19.16B
+	aese	V16.16B, V2.16B
+	aesmc	V16.16B, V16.16B
+	aese	V17.16B, V2.16B
+	aesmc	V17.16B, V17.16B
+	aese	V18.16B, V2.16B
+	aesmc	V18.16B, V18.16B
+	aese	V19.16B, V2.16B
+	aesmc	V19.16B, V19.16B
+	aese	V16.16B, V3.16B
+	aesmc	V16.16B, V16.16B
+	aese	V17.16B, V3.16B
+	aesmc	V17.16B, V17.16B
+	aese	V18.16B, V3.16B
+	aesmc	V18.16B, V18.16B
+	aese	V19.16B, V3.16B
+	aesmc	V19.16B, V19.16B
+	aese	V16.16B, V4.16B
+	aesmc	V16.16B, V16.16B
+	aese	V17.16B, V4.16B
+	aesmc	V17.16B, V17.16B
+	aese	V18.16B, V4.16B
+	aesmc	V18.16B, V18.16B
+	aese	V19.16B, V4.16B
+	aesmc	V19.16B, V19.16B
+	aese	V16.16B, V5.16B
+	aesmc	V16.16B, V16.16B
+	aese	V17.16B, V5.16B
+	aesmc	V17.16B, V17.16B
+	aese	V18.16B, V5.16B
+	aesmc	V18.16B, V18.16B
+	aese	V19.16B, V5.16B
+	aesmc	V19.16B, V19.16B
+	aese	V16.16B, V6.16B
+	aesmc	V16.16B, V16.16B
+	aese	V17.16B, V6.16B
+	aesmc	V17.16B, V17.16B
+	aese	V18.16B, V6.16B
+	aesmc	V18.16B, V18.16B
+	aese	V19.16B, V6.16B
+	aesmc	V19.16B, V19.16B
+	aese	V16.16B, V7.16B
+	aesmc	V16.16B, V16.16B
+	aese	V17.16B, V7.16B
+	aesmc	V17.16B, V17.16B
+	aese	V18.16B, V7.16B
+	aesmc	V18.16B, V18.16B
+	aese	V19.16B, V7.16B
+	aesmc	V19.16B, V19.16B
+	aese	V16.16B, V8.16B
+	aesmc	V16.16B, V16.16B
+	aese	V17.16B, V8.16B
+	aesmc	V17.16B, V17.16B
+	aese	V18.16B, V8.16B
+	aesmc	V18.16B, V18.16B
+	aese	V19.16B, V8.16B
+	aesmc	V19.16B, V19.16B
+	aese	V16.16B, V9.16B
+	eor	V16.16B, V16.16B, V10.16B
+	aese	V17.16B, V9.16B
+	eor	V17.16B, V17.16B, V10.16B
+	aese	V18.16B, V9.16B
+	eor	V18.16B, V18.16B, V10.16B
+	aese	V19.16B, V9.16B
+	eor	V19.16B, V19.16B, V10.16B
+	eor	V20.16B, V20.16B, V16.16B
+	eor	V21.16B, V21.16B, V17.16B
+	eor	V22.16B, V22.16B, V18.16B
+	eor	V23.16B, V23.16B, V19.16B
+	st1	{V20.16B, V21.16B, V22.16B, V23.16B}, [x1], #0x40
+	sub	x6, x6, #4
+	cmp	x6, #4
+	bge	L_AES_GCMSIV_ctr_aarch64_128_loop4
+L_AES_GCMSIV_ctr_aarch64_128_done4
+L_AES_GCMSIV_ctr_aarch64_128_loop1
+	cbz	x6, L_AES_GCMSIV_ctr_aarch64_128_done1
+	mov	V16.16B, V15.16B
+	mov	V16.S[0], w7
+	add	w7, w7, #1
+	ld1	{V20.16B}, [x0], #16
+	aese	V16.16B, V0.16B
+	aesmc	V16.16B, V16.16B
+	aese	V16.16B, V1.16B
+	aesmc	V16.16B, V16.16B
+	aese	V16.16B, V2.16B
+	aesmc	V16.16B, V16.16B
+	aese	V16.16B, V3.16B
+	aesmc	V16.16B, V16.16B
+	aese	V16.16B, V4.16B
+	aesmc	V16.16B, V16.16B
+	aese	V16.16B, V5.16B
+	aesmc	V16.16B, V16.16B
+	aese	V16.16B, V6.16B
+	aesmc	V16.16B, V16.16B
+	aese	V16.16B, V7.16B
+	aesmc	V16.16B, V16.16B
+	aese	V16.16B, V8.16B
+	aesmc	V16.16B, V16.16B
+	aese	V16.16B, V9.16B
+	eor	V16.16B, V16.16B, V10.16B
+	eor	V20.16B, V20.16B, V16.16B
+	st1	{V20.16B}, [x1], #16
+	sub	x6, x6, #1
+	b	L_AES_GCMSIV_ctr_aarch64_128_loop1
+L_AES_GCMSIV_ctr_aarch64_128_done1
+	ENDIF
+L_AES_GCMSIV_ctr_aarch64_done
+	mov	V15.S[0], w7
+	st1	{V15.2D}, [x5]
+	ldp	D8, D9, [x29, #16]
+	ldp	D10, D11, [x29, #32]
+	ldp	D12, D13, [x29, #48]
+	ldp	D14, D15, [x29, #64]
+	ldp	x29, x30, [sp], #0x50
+	ret
+	ENDP
+	ENDIF
+	ENDIF
+	IF :LNOT::DEF:WOLFSSL_ARMASM_NO_NEON
+	IF :DEF:HAVE_AES_DECRYPT :LOR: :DEF:HAVE_AES_CBC :LOR: :DEF:HAVE_AESCCM :LOR: :DEF:HAVE_AESGCM :LOR: :DEF:WOLFSSL_AES_DIRECT :LOR: :DEF:WOLFSSL_AES_COUNTER
+	AREA	|.rodata|, DATA, READONLY, ALIGN=4
+	ALIGN	8
+L_AES_ARM64_NEON_te
+	DCB	0x63, 0x7c, 0x77, 0x7b, 0xf2, 0x6b, 0x6f, 0xc5
+	DCB	0x30, 0x01, 0x67, 0x2b, 0xfe, 0xd7, 0xab, 0x76
+	DCB	0xca, 0x82, 0xc9, 0x7d, 0xfa, 0x59, 0x47, 0xf0
+	DCB	0xad, 0xd4, 0xa2, 0xaf, 0x9c, 0xa4, 0x72, 0xc0
+	DCB	0xb7, 0xfd, 0x93, 0x26, 0x36, 0x3f, 0xf7, 0xcc
+	DCB	0x34, 0xa5, 0xe5, 0xf1, 0x71, 0xd8, 0x31, 0x15
+	DCB	0x04, 0xc7, 0x23, 0xc3, 0x18, 0x96, 0x05, 0x9a
+	DCB	0x07, 0x12, 0x80, 0xe2, 0xeb, 0x27, 0xb2, 0x75
+	DCB	0x09, 0x83, 0x2c, 0x1a, 0x1b, 0x6e, 0x5a, 0xa0
+	DCB	0x52, 0x3b, 0xd6, 0xb3, 0x29, 0xe3, 0x2f, 0x84
+	DCB	0x53, 0xd1, 0x00, 0xed, 0x20, 0xfc, 0xb1, 0x5b
+	DCB	0x6a, 0xcb, 0xbe, 0x39, 0x4a, 0x4c, 0x58, 0xcf
+	DCB	0xd0, 0xef, 0xaa, 0xfb, 0x43, 0x4d, 0x33, 0x85
+	DCB	0x45, 0xf9, 0x02, 0x7f, 0x50, 0x3c, 0x9f, 0xa8
+	DCB	0x51, 0xa3, 0x40, 0x8f, 0x92, 0x9d, 0x38, 0xf5
+	DCB	0xbc, 0xb6, 0xda, 0x21, 0x10, 0xff, 0xf3, 0xd2
+	DCB	0xcd, 0x0c, 0x13, 0xec, 0x5f, 0x97, 0x44, 0x17
+	DCB	0xc4, 0xa7, 0x7e, 0x3d, 0x64, 0x5d, 0x19, 0x73
+	DCB	0x60, 0x81, 0x4f, 0xdc, 0x22, 0x2a, 0x90, 0x88
+	DCB	0x46, 0xee, 0xb8, 0x14, 0xde, 0x5e, 0x0b, 0xdb
+	DCB	0xe0, 0x32, 0x3a, 0x0a, 0x49, 0x06, 0x24, 0x5c
+	DCB	0xc2, 0xd3, 0xac, 0x62, 0x91, 0x95, 0xe4, 0x79
+	DCB	0xe7, 0xc8, 0x37, 0x6d, 0x8d, 0xd5, 0x4e, 0xa9
+	DCB	0x6c, 0x56, 0xf4, 0xea, 0x65, 0x7a, 0xae, 0x08
+	DCB	0xba, 0x78, 0x25, 0x2e, 0x1c, 0xa6, 0xb4, 0xc6
+	DCB	0xe8, 0xdd, 0x74, 0x1f, 0x4b, 0xbd, 0x8b, 0x8a
+	DCB	0x70, 0x3e, 0xb5, 0x66, 0x48, 0x03, 0xf6, 0x0e
+	DCB	0x61, 0x35, 0x57, 0xb9, 0x86, 0xc1, 0x1d, 0x9e
+	DCB	0xe1, 0xf8, 0x98, 0x11, 0x69, 0xd9, 0x8e, 0x94
+	DCB	0x9b, 0x1e, 0x87, 0xe9, 0xce, 0x55, 0x28, 0xdf
+	DCB	0x8c, 0xa1, 0x89, 0x0d, 0xbf, 0xe6, 0x42, 0x68
+	DCB	0x41, 0x99, 0x2d, 0x0f, 0xb0, 0x54, 0xbb, 0x16
+	AREA	|.rodata|, DATA, READONLY, ALIGN=4
+	ALIGN	8
+L_AES_ARM64_NEON_shift_rows_shuffle
+	DCB	0x0c, 0x09, 0x06, 0x03, 0x00, 0x0d, 0x0a, 0x07
+	DCB	0x04, 0x01, 0x0e, 0x0b, 0x08, 0x05, 0x02, 0x0f
+	ENDIF
+	IF :DEF:HAVE_AES_DECRYPT
+	AREA	|.text|, CODE, READONLY
+	ALIGN	4
+	EXPORT	AES_invert_key_NEON
+AES_invert_key_NEON PROC
+	add	x3, x0, x1, lsl 4
+	mov	x2, x0
+	mov	w4, w1
+L_AES_invert_key_NEON_loop
+	ld1	{V0.2D}, [x2]
+	ld1	{V1.2D}, [x3]
+	st1	{V0.2D}, [x3]
+	st1	{V1.2D}, [x2], #16
+	subs	w4, w4, #2
+	sub	x3, x3, #16
+	bne	L_AES_invert_key_NEON_loop
+	movi	V2.16B, #27
+	add	x2, x0, #16
+	sub	w4, w1, #1
+L_AES_invert_key_NEON_mix_loop
+	ld1	{V0.2D}, [x2]
+	sshr	V5.16B, V0.16B, #7
+	ushr	V6.16B, V0.16B, #6
+	ushr	V3.16B, V0.16B, #5
+	and	V5.16B, V5.16B, V2.16B
+	pmul	V6.16B, V6.16B, V2.16B
+	pmul	V3.16B, V3.16B, V2.16B
+	shl	V4.16B, V0.16B, #1
+	eor	V5.16B, V5.16B, V4.16B
+	shl	V4.16B, V0.16B, #3
+	eor	V3.16B, V3.16B, V4.16B
+	shl	V4.16B, V0.16B, #2
+	eor	V6.16B, V6.16B, V4.16B
+	eor	V4.16B, V5.16B, V3.16B
+	eor	V3.16B, V3.16B, V0.16B
+	eor	V5.16B, V6.16B, V3.16B
+	eor	V6.16B, V6.16B, V4.16B
+	eor	V4.16B, V4.16B, V0.16B
+	shl	V0.4S, V4.4S, #8
+	rev32	V5.8H, V5.8H
+	sri	V0.4S, V4.4S, #24
+	eor	V0.16B, V0.16B, V6.16B
+	shl	V4.4S, V3.4S, #24
+	eor	V0.16B, V0.16B, V5.16B
+	sri	V4.4S, V3.4S, #8
+	eor	V0.16B, V0.16B, V4.16B
+	st1	{V0.2D}, [x2], #16
+	subs	w4, w4, #1
+	bne	L_AES_invert_key_NEON_mix_loop
+	ret
+	ENDP
+	ENDIF
+	AREA	|.rodata|, DATA, READONLY, ALIGN=4
+	ALIGN	8
+L_AES_ARM64_NEON_rcon
+	DCD	0x01000000, 0x02000000, 0x04000000, 0x08000000
+	DCD	0x10000000, 0x20000000, 0x40000000, 0x80000000
+	DCD	0x1b000000, 0x36000000
+	AREA	|.text|, CODE, READONLY
+	ALIGN	4
+	EXPORT	AES_set_encrypt_key_NEON
+AES_set_encrypt_key_NEON PROC
+	stp	x29, x30, [sp, #-80]!
+	add	x29, sp, #0
+	stp	D8, D9, [x29, #16]
+	stp	D10, D11, [x29, #32]
+	stp	D12, D13, [x29, #48]
+	stp	D14, D15, [x29, #64]
+	adrp	x4, L_AES_ARM64_NEON_rcon
+	add	x4, x4, L_AES_ARM64_NEON_rcon
+	adrp	x5, L_AES_ARM64_NEON_te
+	add	x5, x5, L_AES_ARM64_NEON_te
+	ld1	{V6.16B, V7.16B, V8.16B, V9.16B}, [x5], #0x40
+	ld1	{V10.16B, V11.16B, V12.16B, V13.16B}, [x5], #0x40
+	ld1	{V14.16B, V15.16B, V16.16B, V17.16B}, [x5], #0x40
+	ld1	{V18.16B, V19.16B, V20.16B, V21.16B}, [x5]
+	movi	V2.16B, #0x40
+	movi	V3.16B, #0x80
+	movi	V4.16B, #0xc0
+	movi	V5.16B, #27
+	eor	V26.16B, V26.16B, V26.16B
+	cmp	x1, #0x80
+	beq	L_AES_set_encrypt_key_NEON_start_128
+	cmp	x1, #0xc0
+	beq	L_AES_set_encrypt_key_NEON_start_192
+	ld1	{V0.16B}, [x0], #16
+	ld1	{V1.16B}, [x0]
+	rev32	V0.16B, V0.16B
+	rev32	V1.16B, V1.16B
+	st1	{V0.2D}, [x2], #16
+	st1	{V1.2D}, [x2], #16
+	mov	x3, #6
+L_AES_set_encrypt_key_NEON_loop_256
+	eor	V22.16B, V1.16B, V2.16B
+	eor	V23.16B, V1.16B, V3.16B
+	eor	V24.16B, V1.16B, V4.16B
+	tbl	V25.16B, {V6.16B, V7.16B, V8.16B, V9.16B}, V1.16B
+	tbl	V22.16B, {V10.16B, V11.16B, V12.16B, V13.16B}, V22.16B
+	tbl	V23.16B, {V14.16B, V15.16B, V16.16B, V17.16B}, V23.16B
+	tbl	V24.16B, {V18.16B, V19.16B, V20.16B, V21.16B}, V24.16B
+	orr	V25.16B, V25.16B, V22.16B
+	orr	V23.16B, V23.16B, V24.16B
+	orr	V25.16B, V25.16B, V23.16B
+	ext8	V25.16B, V25.16B, V26.16B, #12
+	shl	V22.4S, V25.4S, #8
+	sri	V22.4S, V25.4S, #24
+	eor	V0.16B, V0.16B, V22.16B
+	ld1r	{V25.4S}, [x4], #4
+	dup	V22.4S, V0.S[0]
+	dup	V23.2S, V0.S[1]
+	dup	V24.2S, V0.S[2]
+	ext8	V22.16B, V26.16B, V22.16B, #12
+	ext8	V23.16B, V26.16B, V23.16B, #8
+	eor	V0.16B, V0.16B, V22.16B
+	ext8	V24.16B, V26.16B, V24.16B, #4
+	eor	V0.16B, V0.16B, V23.16B
+	eor	V0.16B, V0.16B, V24.16B
+	eor	V0.16B, V0.16B, V25.16B
+	st1	{V0.2D}, [x2], #16
+	eor	V22.16B, V0.16B, V2.16B
+	eor	V23.16B, V0.16B, V3.16B
+	eor	V24.16B, V0.16B, V4.16B
+	tbl	V25.16B, {V6.16B, V7.16B, V8.16B, V9.16B}, V0.16B
+	tbl	V22.16B, {V10.16B, V11.16B, V12.16B, V13.16B}, V22.16B
+	tbl	V23.16B, {V14.16B, V15.16B, V16.16B, V17.16B}, V23.16B
+	tbl	V24.16B, {V18.16B, V19.16B, V20.16B, V21.16B}, V24.16B
+	orr	V25.16B, V25.16B, V22.16B
+	orr	V23.16B, V23.16B, V24.16B
+	orr	V25.16B, V25.16B, V23.16B
+	ext8	V25.16B, V25.16B, V26.16B, #12
+	eor	V1.16B, V1.16B, V25.16B
+	dup	V22.4S, V1.S[0]
+	dup	V23.2S, V1.S[1]
+	dup	V24.2S, V1.S[2]
+	ext8	V22.16B, V26.16B, V22.16B, #12
+	ext8	V23.16B, V26.16B, V23.16B, #8
+	eor	V1.16B, V1.16B, V22.16B
+	ext8	V24.16B, V26.16B, V24.16B, #4
+	eor	V1.16B, V1.16B, V23.16B
+	eor	V1.16B, V1.16B, V24.16B
+	st1	{V1.2D}, [x2], #16
+	subs	x3, x3, #1
+	bne	L_AES_set_encrypt_key_NEON_loop_256
+	eor	V22.16B, V1.16B, V2.16B
+	eor	V23.16B, V1.16B, V3.16B
+	eor	V24.16B, V1.16B, V4.16B
+	tbl	V25.16B, {V6.16B, V7.16B, V8.16B, V9.16B}, V1.16B
+	tbl	V22.16B, {V10.16B, V11.16B, V12.16B, V13.16B}, V22.16B
+	tbl	V23.16B, {V14.16B, V15.16B, V16.16B, V17.16B}, V23.16B
+	tbl	V24.16B, {V18.16B, V19.16B, V20.16B, V21.16B}, V24.16B
+	orr	V25.16B, V25.16B, V22.16B
+	orr	V23.16B, V23.16B, V24.16B
+	orr	V25.16B, V25.16B, V23.16B
+	ext8	V25.16B, V25.16B, V26.16B, #12
+	shl	V22.4S, V25.4S, #8
+	sri	V22.4S, V25.4S, #24
+	eor	V0.16B, V0.16B, V22.16B
+	ld1r	{V25.4S}, [x4], #4
+	dup	V22.4S, V0.S[0]
+	dup	V23.2S, V0.S[1]
+	dup	V24.2S, V0.S[2]
+	ext8	V22.16B, V26.16B, V22.16B, #12
+	ext8	V23.16B, V26.16B, V23.16B, #8
+	eor	V0.16B, V0.16B, V22.16B
+	ext8	V24.16B, V26.16B, V24.16B, #4
+	eor	V0.16B, V0.16B, V23.16B
+	eor	V0.16B, V0.16B, V24.16B
+	eor	V0.16B, V0.16B, V25.16B
+	st1	{V0.2D}, [x2], #16
+	b	L_AES_set_encrypt_key_NEON_end
+L_AES_set_encrypt_key_NEON_start_192
+	ld1	{V0.16B}, [x0], #16
+	ld1	{V1.8B}, [x0]
+	rev32	V0.16B, V0.16B
+	rev32	V1.8B, V1.8B
+	st1	{V0.16B}, [x2], #16
+	st1	{V1.8B}, [x2], #8
+	ext8	V1.16B, V1.16B, V1.16B, #8
+	mov	x3, #7
+L_AES_set_encrypt_key_NEON_loop_192
+	eor	V22.16B, V1.16B, V2.16B
+	eor	V23.16B, V1.16B, V3.16B
+	eor	V24.16B, V1.16B, V4.16B
+	tbl	V25.16B, {V6.16B, V7.16B, V8.16B, V9.16B}, V1.16B
+	tbl	V22.16B, {V10.16B, V11.16B, V12.16B, V13.16B}, V22.16B
+	tbl	V23.16B, {V14.16B, V15.16B, V16.16B, V17.16B}, V23.16B
+	tbl	V24.16B, {V18.16B, V19.16B, V20.16B, V21.16B}, V24.16B
+	orr	V25.16B, V25.16B, V22.16B
+	orr	V23.16B, V23.16B, V24.16B
+	orr	V25.16B, V25.16B, V23.16B
+	ext8	V25.16B, V25.16B, V26.16B, #12
+	shl	V22.4S, V25.4S, #8
+	sri	V22.4S, V25.4S, #24
+	eor	V0.16B, V0.16B, V22.16B
+	ld1r	{V25.4S}, [x4], #4
+	dup	V22.4S, V0.S[0]
+	dup	V23.2S, V0.S[1]
+	dup	V24.2S, V0.S[2]
+	ext8	V22.16B, V26.16B, V22.16B, #12
+	ext8	V23.16B, V26.16B, V23.16B, #8
+	eor	V0.16B, V0.16B, V22.16B
+	ext8	V24.16B, V26.16B, V24.16B, #4
+	eor	V0.16B, V0.16B, V23.16B
+	eor	V0.16B, V0.16B, V24.16B
+	eor	V0.16B, V0.16B, V25.16B
+	st1	{V0.2D}, [x2], #16
+	mov	V23.16B, V26.16B
+	mov	V23.S[2], V0.S[3]
+	eor	V1.16B, V1.16B, V23.16B
 	mov	V23.16B, V26.16B
 	mov	V23.S[3], V1.S[2]
 	eor	V1.16B, V1.16B, V23.16B
@@ -49055,7 +50102,162 @@ L_AES_XTS_encrypt_NEON_loop_nr_partial
 	orr	V9.16B, V9.16B, V10.16B
 	orr	V4.16B, V4.16B, V9.16B
 	tbl	V4.16B, {V4.16B}, V3.16B
-	ld1	{V0.2D}, [x22], #16
+	ld1	{V0.2D}, [x22], #16
+	sshr	V10.16B, V4.16B, #7
+	shl	V9.16B, V4.16B, #1
+	and	V10.16B, V10.16B, V15.16B
+	eor	V10.16B, V10.16B, V9.16B
+	rev32	V8.8H, V4.8H
+	eor	V11.16B, V10.16B, V4.16B
+	eor	V10.16B, V10.16B, V8.16B
+	shl	V9.4S, V4.4S, #24
+	shl	V8.4S, V11.4S, #8
+	;   XOR in Key Schedule
+	eor	V10.16B, V10.16B, V0.16B
+	sri	V9.4S, V4.4S, #8
+	sri	V8.4S, V11.4S, #24
+	eor	V4.16B, V10.16B, V9.16B
+	eor	V4.16B, V4.16B, V8.16B
+	eor	V8.16B, V4.16B, V12.16B
+	eor	V9.16B, V4.16B, V13.16B
+	eor	V10.16B, V4.16B, V14.16B
+	tbl	V0.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V4.16B
+	tbl	V8.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V8.16B
+	tbl	V9.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V9.16B
+	tbl	V10.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V10.16B
+	orr	V0.16B, V0.16B, V8.16B
+	orr	V9.16B, V9.16B, V10.16B
+	orr	V0.16B, V0.16B, V9.16B
+	tbl	V0.16B, {V0.16B}, V3.16B
+	ld1	{V4.2D}, [x22], #16
+	sshr	V10.16B, V0.16B, #7
+	shl	V9.16B, V0.16B, #1
+	and	V10.16B, V10.16B, V15.16B
+	eor	V10.16B, V10.16B, V9.16B
+	rev32	V8.8H, V0.8H
+	eor	V11.16B, V10.16B, V0.16B
+	eor	V10.16B, V10.16B, V8.16B
+	shl	V9.4S, V0.4S, #24
+	shl	V8.4S, V11.4S, #8
+	;   XOR in Key Schedule
+	eor	V10.16B, V10.16B, V4.16B
+	sri	V9.4S, V0.4S, #8
+	sri	V8.4S, V11.4S, #24
+	eor	V0.16B, V10.16B, V9.16B
+	eor	V0.16B, V0.16B, V8.16B
+	subs	w21, w21, #2
+	bne	L_AES_XTS_encrypt_NEON_loop_nr_partial
+	eor	V8.16B, V0.16B, V12.16B
+	eor	V9.16B, V0.16B, V13.16B
+	eor	V10.16B, V0.16B, V14.16B
+	tbl	V4.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V0.16B
+	tbl	V8.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V8.16B
+	tbl	V9.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V9.16B
+	tbl	V10.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V10.16B
+	orr	V4.16B, V4.16B, V8.16B
+	orr	V9.16B, V9.16B, V10.16B
+	orr	V4.16B, V4.16B, V9.16B
+	tbl	V4.16B, {V4.16B}, V3.16B
+	ld1	{V0.2D}, [x22], #16
+	sshr	V10.16B, V4.16B, #7
+	shl	V9.16B, V4.16B, #1
+	and	V10.16B, V10.16B, V15.16B
+	eor	V10.16B, V10.16B, V9.16B
+	rev32	V8.8H, V4.8H
+	eor	V11.16B, V10.16B, V4.16B
+	eor	V10.16B, V10.16B, V8.16B
+	shl	V9.4S, V4.4S, #24
+	shl	V8.4S, V11.4S, #8
+	;   XOR in Key Schedule
+	eor	V10.16B, V10.16B, V0.16B
+	sri	V9.4S, V4.4S, #8
+	sri	V8.4S, V11.4S, #24
+	eor	V4.16B, V10.16B, V9.16B
+	eor	V4.16B, V4.16B, V8.16B
+	eor	V8.16B, V4.16B, V12.16B
+	eor	V9.16B, V4.16B, V13.16B
+	eor	V10.16B, V4.16B, V14.16B
+	tbl	V0.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V4.16B
+	tbl	V8.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V8.16B
+	tbl	V9.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V9.16B
+	tbl	V10.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V10.16B
+	orr	V0.16B, V0.16B, V8.16B
+	orr	V9.16B, V9.16B, V10.16B
+	orr	V0.16B, V0.16B, V9.16B
+	tbl	V0.16B, {V0.16B}, V3.16B
+	ld1	{V4.2D}, [x22], #16
+	;   XOR in Key Schedule
+	eor	V0.16B, V0.16B, V4.16B
+	rev32	V0.16B, V0.16B
+	eor	V0.16B, V0.16B, V2.16B
+	st1	{V0.16B}, [x1]
+L_AES_XTS_encrypt_NEON_data_done
+	ldp	x17, x19, [x29, #24]
+	ldp	x20, x21, [x29, #40]
+	ldr	x22, [x29, #56]
+	ldp	D8, D9, [x29, #64]
+	ldp	D10, D11, [x29, #80]
+	ldp	D12, D13, [x29, #96]
+	ldp	D14, D15, [x29, #112]
+	ldp	x29, x30, [sp], #0x80
+	ret
+	ENDP
+	IF :DEF:HAVE_AES_DECRYPT
+	AREA	|.text|, CODE, READONLY
+	ALIGN	4
+	EXPORT	AES_XTS_decrypt_NEON
+AES_XTS_decrypt_NEON PROC
+	stp	x29, x30, [sp, #-144]!
+	add	x29, sp, #0
+	stp	x17, x19, [x29, #16]
+	stp	x20, x21, [x29, #32]
+	stp	x22, x23, [x29, #48]
+	stp	x24, x25, [x29, #64]
+	stp	D8, D9, [x29, #80]
+	stp	D10, D11, [x29, #96]
+	stp	D12, D13, [x29, #112]
+	stp	D14, D15, [x29, #128]
+	adrp	x20, L_AES_ARM64_NEON_te
+	add	x20, x20, L_AES_ARM64_NEON_te
+	adrp	x21, L_AES_ARM64_NEON_td
+	add	x21, x21, L_AES_ARM64_NEON_td
+	adrp	x22, L_AES_ARM64_NEON_shift_rows_shuffle
+	add	x22, x22, L_AES_ARM64_NEON_shift_rows_shuffle
+	adrp	x23, L_AES_ARM64_NEON_shift_rows_invshuffle
+	add	x23, x23, L_AES_ARM64_NEON_shift_rows_invshuffle
+	ld1	{V16.16B, V17.16B, V18.16B, V19.16B}, [x20], #0x40
+	ld1	{V20.16B, V21.16B, V22.16B, V23.16B}, [x20], #0x40
+	ld1	{V24.16B, V25.16B, V26.16B, V27.16B}, [x20], #0x40
+	ld1	{V28.16B, V29.16B, V30.16B, V31.16B}, [x20]
+	movi	V12.16B, #0x40
+	movi	V13.16B, #0x80
+	movi	V14.16B, #0xc0
+	movi	V15.16B, #27
+	ld1	{V3.2D}, [x22]
+	mov	x17, #0x87
+	ands	w19, w2, #15
+	cset	w16, ne
+	sub	w2, w2, w16, lsl 4
+	ld1	{V2.2D}, [x3]
+	ld1	{V4.2D}, [x5]
+	rev32	V2.16B, V2.16B
+	add	x25, x5, #16
+	; Round: 0 - XOR in key schedule
+	eor	V2.16B, V2.16B, V4.16B
+	sub	w24, w7, #2
+L_AES_XTS_decrypt_NEON_loop_nr_tweak
+	eor	V8.16B, V2.16B, V12.16B
+	eor	V9.16B, V2.16B, V13.16B
+	eor	V10.16B, V2.16B, V14.16B
+	tbl	V4.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V2.16B
+	tbl	V8.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V8.16B
+	tbl	V9.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V9.16B
+	tbl	V10.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V10.16B
+	orr	V4.16B, V4.16B, V8.16B
+	orr	V9.16B, V9.16B, V10.16B
+	orr	V4.16B, V4.16B, V9.16B
+	tbl	V4.16B, {V4.16B}, V3.16B
+	ld1	{V2.2D}, [x25], #16
 	sshr	V10.16B, V4.16B, #7
 	shl	V9.16B, V4.16B, #1
 	and	V10.16B, V10.16B, V15.16B
@@ -49066,7 +50268,7 @@ L_AES_XTS_encrypt_NEON_loop_nr_partial
 	shl	V9.4S, V4.4S, #24
 	shl	V8.4S, V11.4S, #8
 	;   XOR in Key Schedule
-	eor	V10.16B, V10.16B, V0.16B
+	eor	V10.16B, V10.16B, V2.16B
 	sri	V9.4S, V4.4S, #8
 	sri	V8.4S, V11.4S, #24
 	eor	V4.16B, V10.16B, V9.16B
@@ -49074,36 +50276,36 @@ L_AES_XTS_encrypt_NEON_loop_nr_partial
 	eor	V8.16B, V4.16B, V12.16B
 	eor	V9.16B, V4.16B, V13.16B
 	eor	V10.16B, V4.16B, V14.16B
-	tbl	V0.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V4.16B
+	tbl	V2.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V4.16B
 	tbl	V8.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V8.16B
 	tbl	V9.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V9.16B
 	tbl	V10.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V10.16B
-	orr	V0.16B, V0.16B, V8.16B
+	orr	V2.16B, V2.16B, V8.16B
 	orr	V9.16B, V9.16B, V10.16B
-	orr	V0.16B, V0.16B, V9.16B
-	tbl	V0.16B, {V0.16B}, V3.16B
-	ld1	{V4.2D}, [x22], #16
-	sshr	V10.16B, V0.16B, #7
-	shl	V9.16B, V0.16B, #1
+	orr	V2.16B, V2.16B, V9.16B
+	tbl	V2.16B, {V2.16B}, V3.16B
+	ld1	{V4.2D}, [x25], #16
+	sshr	V10.16B, V2.16B, #7
+	shl	V9.16B, V2.16B, #1
 	and	V10.16B, V10.16B, V15.16B
 	eor	V10.16B, V10.16B, V9.16B
-	rev32	V8.8H, V0.8H
-	eor	V11.16B, V10.16B, V0.16B
+	rev32	V8.8H, V2.8H
+	eor	V11.16B, V10.16B, V2.16B
 	eor	V10.16B, V10.16B, V8.16B
-	shl	V9.4S, V0.4S, #24
+	shl	V9.4S, V2.4S, #24
 	shl	V8.4S, V11.4S, #8
 	;   XOR in Key Schedule
 	eor	V10.16B, V10.16B, V4.16B
-	sri	V9.4S, V0.4S, #8
+	sri	V9.4S, V2.4S, #8
 	sri	V8.4S, V11.4S, #24
-	eor	V0.16B, V10.16B, V9.16B
-	eor	V0.16B, V0.16B, V8.16B
-	subs	w21, w21, #2
-	bne	L_AES_XTS_encrypt_NEON_loop_nr_partial
-	eor	V8.16B, V0.16B, V12.16B
-	eor	V9.16B, V0.16B, V13.16B
-	eor	V10.16B, V0.16B, V14.16B
-	tbl	V4.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V0.16B
+	eor	V2.16B, V10.16B, V9.16B
+	eor	V2.16B, V2.16B, V8.16B
+	subs	w24, w24, #2
+	bne	L_AES_XTS_decrypt_NEON_loop_nr_tweak
+	eor	V8.16B, V2.16B, V12.16B
+	eor	V9.16B, V2.16B, V13.16B
+	eor	V10.16B, V2.16B, V14.16B
+	tbl	V4.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V2.16B
 	tbl	V8.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V8.16B
 	tbl	V9.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V9.16B
 	tbl	V10.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V10.16B
@@ -49111,7 +50313,7 @@ L_AES_XTS_encrypt_NEON_loop_nr_partial
 	orr	V9.16B, V9.16B, V10.16B
 	orr	V4.16B, V4.16B, V9.16B
 	tbl	V4.16B, {V4.16B}, V3.16B
-	ld1	{V0.2D}, [x22], #16
+	ld1	{V2.2D}, [x25], #16
 	sshr	V10.16B, V4.16B, #7
 	shl	V9.16B, V4.16B, #1
 	and	V10.16B, V10.16B, V15.16B
@@ -49122,203 +50324,602 @@ L_AES_XTS_encrypt_NEON_loop_nr_partial
 	shl	V9.4S, V4.4S, #24
 	shl	V8.4S, V11.4S, #8
 	;   XOR in Key Schedule
-	eor	V10.16B, V10.16B, V0.16B
-	sri	V9.4S, V4.4S, #8
-	sri	V8.4S, V11.4S, #24
-	eor	V4.16B, V10.16B, V9.16B
-	eor	V4.16B, V4.16B, V8.16B
-	eor	V8.16B, V4.16B, V12.16B
-	eor	V9.16B, V4.16B, V13.16B
-	eor	V10.16B, V4.16B, V14.16B
+	eor	V10.16B, V10.16B, V2.16B
+	sri	V9.4S, V4.4S, #8
+	sri	V8.4S, V11.4S, #24
+	eor	V4.16B, V10.16B, V9.16B
+	eor	V4.16B, V4.16B, V8.16B
+	eor	V8.16B, V4.16B, V12.16B
+	eor	V9.16B, V4.16B, V13.16B
+	eor	V10.16B, V4.16B, V14.16B
+	tbl	V2.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V4.16B
+	tbl	V8.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V8.16B
+	tbl	V9.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V9.16B
+	tbl	V10.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V10.16B
+	orr	V2.16B, V2.16B, V8.16B
+	orr	V9.16B, V9.16B, V10.16B
+	orr	V2.16B, V2.16B, V9.16B
+	tbl	V2.16B, {V2.16B}, V3.16B
+	ld1	{V4.2D}, [x25], #16
+	;   XOR in Key Schedule
+	eor	V2.16B, V2.16B, V4.16B
+	rev32	V2.16B, V2.16B
+	mov	x8, V2.D[0]
+	mov	x9, V2.D[1]
+	ld1	{V16.16B, V17.16B, V18.16B, V19.16B}, [x21], #0x40
+	ld1	{V20.16B, V21.16B, V22.16B, V23.16B}, [x21], #0x40
+	ld1	{V24.16B, V25.16B, V26.16B, V27.16B}, [x21], #0x40
+	ld1	{V28.16B, V29.16B, V30.16B, V31.16B}, [x21]
+	ld1	{V3.2D}, [x23]
+	cmp	w2, #0x40
+	blt	L_AES_XTS_decrypt_NEON_start_2
+L_AES_XTS_decrypt_NEON_loop_4
+	mov	x25, x4
+	ld1	{V0.16B, V1.16B, V2.16B, V3.16B}, [x0], #0x40
+	ld1	{V4.16B}, [x25], #16
+	and	x16, x17, x9, asr 63
+	extr	x11, x9, x8, #63
+	eor	x10, x16, x8, lsl 1
+	and	x16, x17, x11, asr 63
+	extr	x13, x11, x10, #63
+	eor	x12, x16, x10, lsl 1
+	and	x16, x17, x13, asr 63
+	extr	x15, x13, x12, #63
+	eor	x14, x16, x12, lsl 1
+	mov	V8.D[0], x8
+	mov	V8.D[1], x9
+	mov	V9.D[0], x10
+	mov	V9.D[1], x11
+	mov	V10.D[0], x12
+	mov	V10.D[1], x13
+	mov	V11.D[0], x14
+	mov	V11.D[1], x15
+	eor	V0.16B, V0.16B, V8.16B
+	eor	V1.16B, V1.16B, V9.16B
+	eor	V2.16B, V2.16B, V10.16B
+	eor	V3.16B, V3.16B, V11.16B
+	rev32	V0.16B, V0.16B
+	rev32	V1.16B, V1.16B
+	rev32	V2.16B, V2.16B
+	rev32	V3.16B, V3.16B
+	eor	V0.16B, V0.16B, V4.16B
+	eor	V1.16B, V1.16B, V4.16B
+	eor	V2.16B, V2.16B, V4.16B
+	eor	V3.16B, V3.16B, V4.16B
+	sub	w24, w7, #2
+L_AES_XTS_decrypt_NEON_loop_nr_4
+	tbl	V4.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V0.16B
+	tbl	V5.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V1.16B
+	tbl	V6.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V2.16B
+	tbl	V7.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V3.16B
+	movi	V12.16B, #0x40
+	movi	V13.16B, #0x80
+	movi	V14.16B, #0xc0
+	eor	V8.16B, V0.16B, V12.16B
+	eor	V9.16B, V1.16B, V12.16B
+	eor	V10.16B, V2.16B, V12.16B
+	eor	V11.16B, V3.16B, V12.16B
+	tbl	V8.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V8.16B
+	tbl	V9.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V9.16B
+	tbl	V10.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V10.16B
+	tbl	V11.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V11.16B
+	orr	V4.16B, V4.16B, V8.16B
+	orr	V5.16B, V5.16B, V9.16B
+	orr	V6.16B, V6.16B, V10.16B
+	orr	V7.16B, V7.16B, V11.16B
+	eor	V8.16B, V0.16B, V13.16B
+	eor	V9.16B, V1.16B, V13.16B
+	eor	V10.16B, V2.16B, V13.16B
+	eor	V11.16B, V3.16B, V13.16B
+	tbl	V8.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V8.16B
+	tbl	V9.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V9.16B
+	tbl	V10.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V10.16B
+	tbl	V11.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V11.16B
+	orr	V4.16B, V4.16B, V8.16B
+	orr	V5.16B, V5.16B, V9.16B
+	orr	V6.16B, V6.16B, V10.16B
+	orr	V7.16B, V7.16B, V11.16B
+	eor	V8.16B, V0.16B, V14.16B
+	eor	V9.16B, V1.16B, V14.16B
+	eor	V10.16B, V2.16B, V14.16B
+	eor	V11.16B, V3.16B, V14.16B
+	tbl	V8.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V8.16B
+	tbl	V9.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V9.16B
+	tbl	V10.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V10.16B
+	tbl	V11.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V11.16B
+	orr	V4.16B, V4.16B, V8.16B
+	orr	V5.16B, V5.16B, V9.16B
+	orr	V6.16B, V6.16B, V10.16B
+	orr	V7.16B, V7.16B, V11.16B
+	ld1	{V0.16B}, [x23]
+	tbl	V4.16B, {V4.16B}, V0.16B
+	tbl	V5.16B, {V5.16B}, V0.16B
+	tbl	V6.16B, {V6.16B}, V0.16B
+	tbl	V7.16B, {V7.16B}, V0.16B
+	movi	V28.16B, #27
+	sshr	V8.16B, V4.16B, #7
+	sshr	V9.16B, V5.16B, #7
+	sshr	V10.16B, V6.16B, #7
+	sshr	V11.16B, V7.16B, #7
+	shl	V12.16B, V4.16B, #1
+	shl	V13.16B, V5.16B, #1
+	shl	V14.16B, V6.16B, #1
+	shl	V15.16B, V7.16B, #1
+	and	V8.16B, V8.16B, V28.16B
+	and	V9.16B, V9.16B, V28.16B
+	and	V10.16B, V10.16B, V28.16B
+	and	V11.16B, V11.16B, V28.16B
+	eor	V8.16B, V8.16B, V12.16B
+	eor	V9.16B, V9.16B, V13.16B
+	eor	V10.16B, V10.16B, V14.16B
+	eor	V11.16B, V11.16B, V15.16B
+	ushr	V12.16B, V4.16B, #6
+	ushr	V13.16B, V5.16B, #6
+	ushr	V14.16B, V6.16B, #6
+	ushr	V15.16B, V7.16B, #6
+	shl	V0.16B, V4.16B, #2
+	shl	V1.16B, V5.16B, #2
+	shl	V2.16B, V6.16B, #2
+	shl	V3.16B, V7.16B, #2
+	pmul	V12.16B, V12.16B, V28.16B
+	pmul	V13.16B, V13.16B, V28.16B
+	pmul	V14.16B, V14.16B, V28.16B
+	pmul	V15.16B, V15.16B, V28.16B
+	eor	V12.16B, V12.16B, V0.16B
+	eor	V13.16B, V13.16B, V1.16B
+	eor	V14.16B, V14.16B, V2.16B
+	eor	V15.16B, V15.16B, V3.16B
+	ushr	V0.16B, V4.16B, #5
+	ushr	V1.16B, V5.16B, #5
+	ushr	V2.16B, V6.16B, #5
+	ushr	V3.16B, V7.16B, #5
+	pmul	V0.16B, V0.16B, V28.16B
+	pmul	V1.16B, V1.16B, V28.16B
+	pmul	V2.16B, V2.16B, V28.16B
+	pmul	V3.16B, V3.16B, V28.16B
+	shl	V28.16B, V4.16B, #3
+	shl	V29.16B, V5.16B, #3
+	shl	V30.16B, V6.16B, #3
+	shl	V31.16B, V7.16B, #3
+	eor	V0.16B, V0.16B, V28.16B
+	eor	V1.16B, V1.16B, V29.16B
+	eor	V2.16B, V2.16B, V30.16B
+	eor	V3.16B, V3.16B, V31.16B
+	eor	V28.16B, V8.16B, V0.16B
+	eor	V29.16B, V9.16B, V1.16B
+	eor	V30.16B, V10.16B, V2.16B
+	eor	V31.16B, V11.16B, V3.16B
+	eor	V0.16B, V0.16B, V4.16B
+	eor	V1.16B, V1.16B, V5.16B
+	eor	V2.16B, V2.16B, V6.16B
+	eor	V3.16B, V3.16B, V7.16B
+	eor	V8.16B, V12.16B, V0.16B
+	eor	V9.16B, V13.16B, V1.16B
+	eor	V10.16B, V14.16B, V2.16B
+	eor	V11.16B, V15.16B, V3.16B
+	eor	V12.16B, V12.16B, V28.16B
+	eor	V13.16B, V13.16B, V29.16B
+	eor	V14.16B, V14.16B, V30.16B
+	eor	V15.16B, V15.16B, V31.16B
+	eor	V28.16B, V28.16B, V4.16B
+	eor	V29.16B, V29.16B, V5.16B
+	eor	V30.16B, V30.16B, V6.16B
+	eor	V31.16B, V31.16B, V7.16B
+	shl	V4.4S, V28.4S, #8
+	shl	V5.4S, V29.4S, #8
+	shl	V6.4S, V30.4S, #8
+	shl	V7.4S, V31.4S, #8
+	rev32	V8.8H, V8.8H
+	rev32	V9.8H, V9.8H
+	rev32	V10.8H, V10.8H
+	rev32	V11.8H, V11.8H
+	sri	V4.4S, V28.4S, #24
+	sri	V5.4S, V29.4S, #24
+	sri	V6.4S, V30.4S, #24
+	sri	V7.4S, V31.4S, #24
+	eor	V4.16B, V4.16B, V12.16B
+	eor	V5.16B, V5.16B, V13.16B
+	eor	V6.16B, V6.16B, V14.16B
+	eor	V7.16B, V7.16B, V15.16B
+	shl	V28.4S, V0.4S, #24
+	shl	V29.4S, V1.4S, #24
+	shl	V30.4S, V2.4S, #24
+	shl	V31.4S, V3.4S, #24
+	eor	V4.16B, V4.16B, V8.16B
+	eor	V5.16B, V5.16B, V9.16B
+	eor	V6.16B, V6.16B, V10.16B
+	eor	V7.16B, V7.16B, V11.16B
+	sri	V28.4S, V0.4S, #8
+	sri	V29.4S, V1.4S, #8
+	sri	V30.4S, V2.4S, #8
+	sri	V31.4S, V3.4S, #8
+	eor	V4.16B, V4.16B, V28.16B
+	eor	V5.16B, V5.16B, V29.16B
+	eor	V6.16B, V6.16B, V30.16B
+	eor	V7.16B, V7.16B, V31.16B
+	ld1	{V28.16B, V29.16B, V30.16B, V31.16B}, [x21]
+	;   XOR in Key Schedule
+	ld1	{V0.2D}, [x25], #16
+	eor	V4.16B, V4.16B, V0.16B
+	eor	V5.16B, V5.16B, V0.16B
+	eor	V6.16B, V6.16B, V0.16B
+	eor	V7.16B, V7.16B, V0.16B
+	; Round Done
 	tbl	V0.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V4.16B
-	tbl	V8.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V8.16B
-	tbl	V9.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V9.16B
-	tbl	V10.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V10.16B
-	orr	V0.16B, V0.16B, V8.16B
-	orr	V9.16B, V9.16B, V10.16B
-	orr	V0.16B, V0.16B, V9.16B
-	tbl	V0.16B, {V0.16B}, V3.16B
-	ld1	{V4.2D}, [x22], #16
-	;   XOR in Key Schedule
-	eor	V0.16B, V0.16B, V4.16B
-	rev32	V0.16B, V0.16B
-	eor	V0.16B, V0.16B, V2.16B
-	st1	{V0.16B}, [x1]
-L_AES_XTS_encrypt_NEON_data_done
-	ldp	x17, x19, [x29, #24]
-	ldp	x20, x21, [x29, #40]
-	ldr	x22, [x29, #56]
-	ldp	D8, D9, [x29, #64]
-	ldp	D10, D11, [x29, #80]
-	ldp	D12, D13, [x29, #96]
-	ldp	D14, D15, [x29, #112]
-	ldp	x29, x30, [sp], #0x80
-	ret
-	ENDP
-	IF :DEF:HAVE_AES_DECRYPT
-	AREA	|.text|, CODE, READONLY
-	ALIGN	4
-	EXPORT	AES_XTS_decrypt_NEON
-AES_XTS_decrypt_NEON PROC
-	stp	x29, x30, [sp, #-144]!
-	add	x29, sp, #0
-	stp	x17, x19, [x29, #16]
-	stp	x20, x21, [x29, #32]
-	stp	x22, x23, [x29, #48]
-	stp	x24, x25, [x29, #64]
-	stp	D8, D9, [x29, #80]
-	stp	D10, D11, [x29, #96]
-	stp	D12, D13, [x29, #112]
-	stp	D14, D15, [x29, #128]
-	adrp	x20, L_AES_ARM64_NEON_te
-	add	x20, x20, L_AES_ARM64_NEON_te
-	adrp	x21, L_AES_ARM64_NEON_td
-	add	x21, x21, L_AES_ARM64_NEON_td
-	adrp	x22, L_AES_ARM64_NEON_shift_rows_shuffle
-	add	x22, x22, L_AES_ARM64_NEON_shift_rows_shuffle
-	adrp	x23, L_AES_ARM64_NEON_shift_rows_invshuffle
-	add	x23, x23, L_AES_ARM64_NEON_shift_rows_invshuffle
-	ld1	{V16.16B, V17.16B, V18.16B, V19.16B}, [x20], #0x40
-	ld1	{V20.16B, V21.16B, V22.16B, V23.16B}, [x20], #0x40
-	ld1	{V24.16B, V25.16B, V26.16B, V27.16B}, [x20], #0x40
-	ld1	{V28.16B, V29.16B, V30.16B, V31.16B}, [x20]
+	tbl	V1.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V5.16B
+	tbl	V2.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V6.16B
+	tbl	V3.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V7.16B
 	movi	V12.16B, #0x40
 	movi	V13.16B, #0x80
 	movi	V14.16B, #0xc0
-	movi	V15.16B, #27
-	ld1	{V3.2D}, [x22]
-	mov	x17, #0x87
-	ands	w19, w2, #15
-	cset	w16, ne
-	sub	w2, w2, w16, lsl 4
-	ld1	{V2.2D}, [x3]
-	ld1	{V4.2D}, [x5]
-	rev32	V2.16B, V2.16B
-	add	x25, x5, #16
-	; Round: 0 - XOR in key schedule
-	eor	V2.16B, V2.16B, V4.16B
-	sub	w24, w7, #2
-L_AES_XTS_decrypt_NEON_loop_nr_tweak
-	eor	V8.16B, V2.16B, V12.16B
-	eor	V9.16B, V2.16B, V13.16B
-	eor	V10.16B, V2.16B, V14.16B
-	tbl	V4.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V2.16B
-	tbl	V8.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V8.16B
-	tbl	V9.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V9.16B
-	tbl	V10.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V10.16B
-	orr	V4.16B, V4.16B, V8.16B
-	orr	V9.16B, V9.16B, V10.16B
-	orr	V4.16B, V4.16B, V9.16B
-	tbl	V4.16B, {V4.16B}, V3.16B
-	ld1	{V2.2D}, [x25], #16
-	sshr	V10.16B, V4.16B, #7
-	shl	V9.16B, V4.16B, #1
-	and	V10.16B, V10.16B, V15.16B
-	eor	V10.16B, V10.16B, V9.16B
-	rev32	V8.8H, V4.8H
-	eor	V11.16B, V10.16B, V4.16B
-	eor	V10.16B, V10.16B, V8.16B
-	shl	V9.4S, V4.4S, #24
-	shl	V8.4S, V11.4S, #8
-	;   XOR in Key Schedule
-	eor	V10.16B, V10.16B, V2.16B
-	sri	V9.4S, V4.4S, #8
-	sri	V8.4S, V11.4S, #24
-	eor	V4.16B, V10.16B, V9.16B
-	eor	V4.16B, V4.16B, V8.16B
 	eor	V8.16B, V4.16B, V12.16B
-	eor	V9.16B, V4.16B, V13.16B
-	eor	V10.16B, V4.16B, V14.16B
-	tbl	V2.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V4.16B
+	eor	V9.16B, V5.16B, V12.16B
+	eor	V10.16B, V6.16B, V12.16B
+	eor	V11.16B, V7.16B, V12.16B
 	tbl	V8.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V8.16B
+	tbl	V9.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V9.16B
+	tbl	V10.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V10.16B
+	tbl	V11.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V11.16B
+	orr	V0.16B, V0.16B, V8.16B
+	orr	V1.16B, V1.16B, V9.16B
+	orr	V2.16B, V2.16B, V10.16B
+	orr	V3.16B, V3.16B, V11.16B
+	eor	V8.16B, V4.16B, V13.16B
+	eor	V9.16B, V5.16B, V13.16B
+	eor	V10.16B, V6.16B, V13.16B
+	eor	V11.16B, V7.16B, V13.16B
+	tbl	V8.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V8.16B
 	tbl	V9.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V9.16B
+	tbl	V10.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V10.16B
+	tbl	V11.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V11.16B
+	orr	V0.16B, V0.16B, V8.16B
+	orr	V1.16B, V1.16B, V9.16B
+	orr	V2.16B, V2.16B, V10.16B
+	orr	V3.16B, V3.16B, V11.16B
+	eor	V8.16B, V4.16B, V14.16B
+	eor	V9.16B, V5.16B, V14.16B
+	eor	V10.16B, V6.16B, V14.16B
+	eor	V11.16B, V7.16B, V14.16B
+	tbl	V8.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V8.16B
+	tbl	V9.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V9.16B
 	tbl	V10.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V10.16B
-	orr	V2.16B, V2.16B, V8.16B
-	orr	V9.16B, V9.16B, V10.16B
-	orr	V2.16B, V2.16B, V9.16B
-	tbl	V2.16B, {V2.16B}, V3.16B
-	ld1	{V4.2D}, [x25], #16
+	tbl	V11.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V11.16B
+	orr	V0.16B, V0.16B, V8.16B
+	orr	V1.16B, V1.16B, V9.16B
+	orr	V2.16B, V2.16B, V10.16B
+	orr	V3.16B, V3.16B, V11.16B
+	ld1	{V4.16B}, [x23]
+	tbl	V0.16B, {V0.16B}, V4.16B
+	tbl	V1.16B, {V1.16B}, V4.16B
+	tbl	V2.16B, {V2.16B}, V4.16B
+	tbl	V3.16B, {V3.16B}, V4.16B
+	movi	V28.16B, #27
+	sshr	V8.16B, V0.16B, #7
+	sshr	V9.16B, V1.16B, #7
 	sshr	V10.16B, V2.16B, #7
-	shl	V9.16B, V2.16B, #1
-	and	V10.16B, V10.16B, V15.16B
-	eor	V10.16B, V10.16B, V9.16B
-	rev32	V8.8H, V2.8H
-	eor	V11.16B, V10.16B, V2.16B
-	eor	V10.16B, V10.16B, V8.16B
-	shl	V9.4S, V2.4S, #24
-	shl	V8.4S, V11.4S, #8
+	sshr	V11.16B, V3.16B, #7
+	shl	V12.16B, V0.16B, #1
+	shl	V13.16B, V1.16B, #1
+	shl	V14.16B, V2.16B, #1
+	shl	V15.16B, V3.16B, #1
+	and	V8.16B, V8.16B, V28.16B
+	and	V9.16B, V9.16B, V28.16B
+	and	V10.16B, V10.16B, V28.16B
+	and	V11.16B, V11.16B, V28.16B
+	eor	V8.16B, V8.16B, V12.16B
+	eor	V9.16B, V9.16B, V13.16B
+	eor	V10.16B, V10.16B, V14.16B
+	eor	V11.16B, V11.16B, V15.16B
+	ushr	V12.16B, V0.16B, #6
+	ushr	V13.16B, V1.16B, #6
+	ushr	V14.16B, V2.16B, #6
+	ushr	V15.16B, V3.16B, #6
+	shl	V4.16B, V0.16B, #2
+	shl	V5.16B, V1.16B, #2
+	shl	V6.16B, V2.16B, #2
+	shl	V7.16B, V3.16B, #2
+	pmul	V12.16B, V12.16B, V28.16B
+	pmul	V13.16B, V13.16B, V28.16B
+	pmul	V14.16B, V14.16B, V28.16B
+	pmul	V15.16B, V15.16B, V28.16B
+	eor	V12.16B, V12.16B, V4.16B
+	eor	V13.16B, V13.16B, V5.16B
+	eor	V14.16B, V14.16B, V6.16B
+	eor	V15.16B, V15.16B, V7.16B
+	ushr	V4.16B, V0.16B, #5
+	ushr	V5.16B, V1.16B, #5
+	ushr	V6.16B, V2.16B, #5
+	ushr	V7.16B, V3.16B, #5
+	pmul	V4.16B, V4.16B, V28.16B
+	pmul	V5.16B, V5.16B, V28.16B
+	pmul	V6.16B, V6.16B, V28.16B
+	pmul	V7.16B, V7.16B, V28.16B
+	shl	V28.16B, V0.16B, #3
+	shl	V29.16B, V1.16B, #3
+	shl	V30.16B, V2.16B, #3
+	shl	V31.16B, V3.16B, #3
+	eor	V4.16B, V4.16B, V28.16B
+	eor	V5.16B, V5.16B, V29.16B
+	eor	V6.16B, V6.16B, V30.16B
+	eor	V7.16B, V7.16B, V31.16B
+	eor	V28.16B, V8.16B, V4.16B
+	eor	V29.16B, V9.16B, V5.16B
+	eor	V30.16B, V10.16B, V6.16B
+	eor	V31.16B, V11.16B, V7.16B
+	eor	V4.16B, V4.16B, V0.16B
+	eor	V5.16B, V5.16B, V1.16B
+	eor	V6.16B, V6.16B, V2.16B
+	eor	V7.16B, V7.16B, V3.16B
+	eor	V8.16B, V12.16B, V4.16B
+	eor	V9.16B, V13.16B, V5.16B
+	eor	V10.16B, V14.16B, V6.16B
+	eor	V11.16B, V15.16B, V7.16B
+	eor	V12.16B, V12.16B, V28.16B
+	eor	V13.16B, V13.16B, V29.16B
+	eor	V14.16B, V14.16B, V30.16B
+	eor	V15.16B, V15.16B, V31.16B
+	eor	V28.16B, V28.16B, V0.16B
+	eor	V29.16B, V29.16B, V1.16B
+	eor	V30.16B, V30.16B, V2.16B
+	eor	V31.16B, V31.16B, V3.16B
+	shl	V0.4S, V28.4S, #8
+	shl	V1.4S, V29.4S, #8
+	shl	V2.4S, V30.4S, #8
+	shl	V3.4S, V31.4S, #8
+	rev32	V8.8H, V8.8H
+	rev32	V9.8H, V9.8H
+	rev32	V10.8H, V10.8H
+	rev32	V11.8H, V11.8H
+	sri	V0.4S, V28.4S, #24
+	sri	V1.4S, V29.4S, #24
+	sri	V2.4S, V30.4S, #24
+	sri	V3.4S, V31.4S, #24
+	eor	V0.16B, V0.16B, V12.16B
+	eor	V1.16B, V1.16B, V13.16B
+	eor	V2.16B, V2.16B, V14.16B
+	eor	V3.16B, V3.16B, V15.16B
+	shl	V28.4S, V4.4S, #24
+	shl	V29.4S, V5.4S, #24
+	shl	V30.4S, V6.4S, #24
+	shl	V31.4S, V7.4S, #24
+	eor	V0.16B, V0.16B, V8.16B
+	eor	V1.16B, V1.16B, V9.16B
+	eor	V2.16B, V2.16B, V10.16B
+	eor	V3.16B, V3.16B, V11.16B
+	sri	V28.4S, V4.4S, #8
+	sri	V29.4S, V5.4S, #8
+	sri	V30.4S, V6.4S, #8
+	sri	V31.4S, V7.4S, #8
+	eor	V0.16B, V0.16B, V28.16B
+	eor	V1.16B, V1.16B, V29.16B
+	eor	V2.16B, V2.16B, V30.16B
+	eor	V3.16B, V3.16B, V31.16B
+	ld1	{V28.16B, V29.16B, V30.16B, V31.16B}, [x21]
 	;   XOR in Key Schedule
-	eor	V10.16B, V10.16B, V4.16B
-	sri	V9.4S, V2.4S, #8
-	sri	V8.4S, V11.4S, #24
-	eor	V2.16B, V10.16B, V9.16B
-	eor	V2.16B, V2.16B, V8.16B
+	ld1	{V4.2D}, [x25], #16
+	eor	V0.16B, V0.16B, V4.16B
+	eor	V1.16B, V1.16B, V4.16B
+	eor	V2.16B, V2.16B, V4.16B
+	eor	V3.16B, V3.16B, V4.16B
+	; Round Done
 	subs	w24, w24, #2
-	bne	L_AES_XTS_decrypt_NEON_loop_nr_tweak
-	eor	V8.16B, V2.16B, V12.16B
-	eor	V9.16B, V2.16B, V13.16B
-	eor	V10.16B, V2.16B, V14.16B
-	tbl	V4.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V2.16B
+	bne	L_AES_XTS_decrypt_NEON_loop_nr_4
+	tbl	V4.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V0.16B
+	tbl	V5.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V1.16B
+	tbl	V6.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V2.16B
+	tbl	V7.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V3.16B
+	movi	V12.16B, #0x40
+	movi	V13.16B, #0x80
+	movi	V14.16B, #0xc0
+	eor	V8.16B, V0.16B, V12.16B
+	eor	V9.16B, V1.16B, V12.16B
+	eor	V10.16B, V2.16B, V12.16B
+	eor	V11.16B, V3.16B, V12.16B
 	tbl	V8.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V8.16B
+	tbl	V9.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V9.16B
+	tbl	V10.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V10.16B
+	tbl	V11.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V11.16B
+	orr	V4.16B, V4.16B, V8.16B
+	orr	V5.16B, V5.16B, V9.16B
+	orr	V6.16B, V6.16B, V10.16B
+	orr	V7.16B, V7.16B, V11.16B
+	eor	V8.16B, V0.16B, V13.16B
+	eor	V9.16B, V1.16B, V13.16B
+	eor	V10.16B, V2.16B, V13.16B
+	eor	V11.16B, V3.16B, V13.16B
+	tbl	V8.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V8.16B
 	tbl	V9.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V9.16B
+	tbl	V10.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V10.16B
+	tbl	V11.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V11.16B
+	orr	V4.16B, V4.16B, V8.16B
+	orr	V5.16B, V5.16B, V9.16B
+	orr	V6.16B, V6.16B, V10.16B
+	orr	V7.16B, V7.16B, V11.16B
+	eor	V8.16B, V0.16B, V14.16B
+	eor	V9.16B, V1.16B, V14.16B
+	eor	V10.16B, V2.16B, V14.16B
+	eor	V11.16B, V3.16B, V14.16B
+	tbl	V8.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V8.16B
+	tbl	V9.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V9.16B
 	tbl	V10.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V10.16B
+	tbl	V11.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V11.16B
 	orr	V4.16B, V4.16B, V8.16B
-	orr	V9.16B, V9.16B, V10.16B
-	orr	V4.16B, V4.16B, V9.16B
-	tbl	V4.16B, {V4.16B}, V3.16B
-	ld1	{V2.2D}, [x25], #16
-	sshr	V10.16B, V4.16B, #7
-	shl	V9.16B, V4.16B, #1
-	and	V10.16B, V10.16B, V15.16B
-	eor	V10.16B, V10.16B, V9.16B
-	rev32	V8.8H, V4.8H
-	eor	V11.16B, V10.16B, V4.16B
-	eor	V10.16B, V10.16B, V8.16B
-	shl	V9.4S, V4.4S, #24
-	shl	V8.4S, V11.4S, #8
-	;   XOR in Key Schedule
-	eor	V10.16B, V10.16B, V2.16B
-	sri	V9.4S, V4.4S, #8
-	sri	V8.4S, V11.4S, #24
-	eor	V4.16B, V10.16B, V9.16B
+	orr	V5.16B, V5.16B, V9.16B
+	orr	V6.16B, V6.16B, V10.16B
+	orr	V7.16B, V7.16B, V11.16B
+	ld1	{V0.16B}, [x23]
+	tbl	V4.16B, {V4.16B}, V0.16B
+	tbl	V5.16B, {V5.16B}, V0.16B
+	tbl	V6.16B, {V6.16B}, V0.16B
+	tbl	V7.16B, {V7.16B}, V0.16B
+	movi	V28.16B, #27
+	sshr	V8.16B, V4.16B, #7
+	sshr	V9.16B, V5.16B, #7
+	sshr	V10.16B, V6.16B, #7
+	sshr	V11.16B, V7.16B, #7
+	shl	V12.16B, V4.16B, #1
+	shl	V13.16B, V5.16B, #1
+	shl	V14.16B, V6.16B, #1
+	shl	V15.16B, V7.16B, #1
+	and	V8.16B, V8.16B, V28.16B
+	and	V9.16B, V9.16B, V28.16B
+	and	V10.16B, V10.16B, V28.16B
+	and	V11.16B, V11.16B, V28.16B
+	eor	V8.16B, V8.16B, V12.16B
+	eor	V9.16B, V9.16B, V13.16B
+	eor	V10.16B, V10.16B, V14.16B
+	eor	V11.16B, V11.16B, V15.16B
+	ushr	V12.16B, V4.16B, #6
+	ushr	V13.16B, V5.16B, #6
+	ushr	V14.16B, V6.16B, #6
+	ushr	V15.16B, V7.16B, #6
+	shl	V0.16B, V4.16B, #2
+	shl	V1.16B, V5.16B, #2
+	shl	V2.16B, V6.16B, #2
+	shl	V3.16B, V7.16B, #2
+	pmul	V12.16B, V12.16B, V28.16B
+	pmul	V13.16B, V13.16B, V28.16B
+	pmul	V14.16B, V14.16B, V28.16B
+	pmul	V15.16B, V15.16B, V28.16B
+	eor	V12.16B, V12.16B, V0.16B
+	eor	V13.16B, V13.16B, V1.16B
+	eor	V14.16B, V14.16B, V2.16B
+	eor	V15.16B, V15.16B, V3.16B
+	ushr	V0.16B, V4.16B, #5
+	ushr	V1.16B, V5.16B, #5
+	ushr	V2.16B, V6.16B, #5
+	ushr	V3.16B, V7.16B, #5
+	pmul	V0.16B, V0.16B, V28.16B
+	pmul	V1.16B, V1.16B, V28.16B
+	pmul	V2.16B, V2.16B, V28.16B
+	pmul	V3.16B, V3.16B, V28.16B
+	shl	V28.16B, V4.16B, #3
+	shl	V29.16B, V5.16B, #3
+	shl	V30.16B, V6.16B, #3
+	shl	V31.16B, V7.16B, #3
+	eor	V0.16B, V0.16B, V28.16B
+	eor	V1.16B, V1.16B, V29.16B
+	eor	V2.16B, V2.16B, V30.16B
+	eor	V3.16B, V3.16B, V31.16B
+	eor	V28.16B, V8.16B, V0.16B
+	eor	V29.16B, V9.16B, V1.16B
+	eor	V30.16B, V10.16B, V2.16B
+	eor	V31.16B, V11.16B, V3.16B
+	eor	V0.16B, V0.16B, V4.16B
+	eor	V1.16B, V1.16B, V5.16B
+	eor	V2.16B, V2.16B, V6.16B
+	eor	V3.16B, V3.16B, V7.16B
+	eor	V8.16B, V12.16B, V0.16B
+	eor	V9.16B, V13.16B, V1.16B
+	eor	V10.16B, V14.16B, V2.16B
+	eor	V11.16B, V15.16B, V3.16B
+	eor	V12.16B, V12.16B, V28.16B
+	eor	V13.16B, V13.16B, V29.16B
+	eor	V14.16B, V14.16B, V30.16B
+	eor	V15.16B, V15.16B, V31.16B
+	eor	V28.16B, V28.16B, V4.16B
+	eor	V29.16B, V29.16B, V5.16B
+	eor	V30.16B, V30.16B, V6.16B
+	eor	V31.16B, V31.16B, V7.16B
+	shl	V4.4S, V28.4S, #8
+	shl	V5.4S, V29.4S, #8
+	shl	V6.4S, V30.4S, #8
+	shl	V7.4S, V31.4S, #8
+	rev32	V8.8H, V8.8H
+	rev32	V9.8H, V9.8H
+	rev32	V10.8H, V10.8H
+	rev32	V11.8H, V11.8H
+	sri	V4.4S, V28.4S, #24
+	sri	V5.4S, V29.4S, #24
+	sri	V6.4S, V30.4S, #24
+	sri	V7.4S, V31.4S, #24
+	eor	V4.16B, V4.16B, V12.16B
+	eor	V5.16B, V5.16B, V13.16B
+	eor	V6.16B, V6.16B, V14.16B
+	eor	V7.16B, V7.16B, V15.16B
+	shl	V28.4S, V0.4S, #24
+	shl	V29.4S, V1.4S, #24
+	shl	V30.4S, V2.4S, #24
+	shl	V31.4S, V3.4S, #24
 	eor	V4.16B, V4.16B, V8.16B
+	eor	V5.16B, V5.16B, V9.16B
+	eor	V6.16B, V6.16B, V10.16B
+	eor	V7.16B, V7.16B, V11.16B
+	sri	V28.4S, V0.4S, #8
+	sri	V29.4S, V1.4S, #8
+	sri	V30.4S, V2.4S, #8
+	sri	V31.4S, V3.4S, #8
+	eor	V4.16B, V4.16B, V28.16B
+	eor	V5.16B, V5.16B, V29.16B
+	eor	V6.16B, V6.16B, V30.16B
+	eor	V7.16B, V7.16B, V31.16B
+	ld1	{V28.16B, V29.16B, V30.16B, V31.16B}, [x21]
+	;   XOR in Key Schedule
+	ld1	{V0.2D}, [x25], #16
+	eor	V4.16B, V4.16B, V0.16B
+	eor	V5.16B, V5.16B, V0.16B
+	eor	V6.16B, V6.16B, V0.16B
+	eor	V7.16B, V7.16B, V0.16B
+	; Round Done
+	tbl	V0.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V4.16B
+	tbl	V1.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V5.16B
+	tbl	V2.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V6.16B
+	tbl	V3.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V7.16B
+	movi	V12.16B, #0x40
+	movi	V13.16B, #0x80
+	movi	V14.16B, #0xc0
 	eor	V8.16B, V4.16B, V12.16B
-	eor	V9.16B, V4.16B, V13.16B
-	eor	V10.16B, V4.16B, V14.16B
-	tbl	V2.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V4.16B
+	eor	V9.16B, V5.16B, V12.16B
+	eor	V10.16B, V6.16B, V12.16B
+	eor	V11.16B, V7.16B, V12.16B
 	tbl	V8.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V8.16B
+	tbl	V9.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V9.16B
+	tbl	V10.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V10.16B
+	tbl	V11.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V11.16B
+	orr	V0.16B, V0.16B, V8.16B
+	orr	V1.16B, V1.16B, V9.16B
+	orr	V2.16B, V2.16B, V10.16B
+	orr	V3.16B, V3.16B, V11.16B
+	eor	V8.16B, V4.16B, V13.16B
+	eor	V9.16B, V5.16B, V13.16B
+	eor	V10.16B, V6.16B, V13.16B
+	eor	V11.16B, V7.16B, V13.16B
+	tbl	V8.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V8.16B
 	tbl	V9.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V9.16B
+	tbl	V10.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V10.16B
+	tbl	V11.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V11.16B
+	orr	V0.16B, V0.16B, V8.16B
+	orr	V1.16B, V1.16B, V9.16B
+	orr	V2.16B, V2.16B, V10.16B
+	orr	V3.16B, V3.16B, V11.16B
+	eor	V8.16B, V4.16B, V14.16B
+	eor	V9.16B, V5.16B, V14.16B
+	eor	V10.16B, V6.16B, V14.16B
+	eor	V11.16B, V7.16B, V14.16B
+	tbl	V8.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V8.16B
+	tbl	V9.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V9.16B
 	tbl	V10.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V10.16B
-	orr	V2.16B, V2.16B, V8.16B
-	orr	V9.16B, V9.16B, V10.16B
-	orr	V2.16B, V2.16B, V9.16B
-	tbl	V2.16B, {V2.16B}, V3.16B
-	ld1	{V4.2D}, [x25], #16
+	tbl	V11.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V11.16B
+	orr	V0.16B, V0.16B, V8.16B
+	orr	V1.16B, V1.16B, V9.16B
+	orr	V2.16B, V2.16B, V10.16B
+	orr	V3.16B, V3.16B, V11.16B
+	ld1	{V4.16B}, [x23]
+	tbl	V0.16B, {V0.16B}, V4.16B
+	tbl	V1.16B, {V1.16B}, V4.16B
+	tbl	V2.16B, {V2.16B}, V4.16B
+	tbl	V3.16B, {V3.16B}, V4.16B
 	;   XOR in Key Schedule
+	ld1	{V4.2D}, [x25], #16
+	eor	V0.16B, V0.16B, V4.16B
+	eor	V1.16B, V1.16B, V4.16B
 	eor	V2.16B, V2.16B, V4.16B
+	eor	V3.16B, V3.16B, V4.16B
+	; Round Done
+	rev32	V0.16B, V0.16B
+	rev32	V1.16B, V1.16B
 	rev32	V2.16B, V2.16B
-	mov	x8, V2.D[0]
-	mov	x9, V2.D[1]
-	ld1	{V16.16B, V17.16B, V18.16B, V19.16B}, [x21], #0x40
-	ld1	{V20.16B, V21.16B, V22.16B, V23.16B}, [x21], #0x40
-	ld1	{V24.16B, V25.16B, V26.16B, V27.16B}, [x21], #0x40
-	ld1	{V28.16B, V29.16B, V30.16B, V31.16B}, [x21]
-	ld1	{V3.2D}, [x23]
-	cmp	w2, #0x40
-	blt	L_AES_XTS_decrypt_NEON_start_2
-L_AES_XTS_decrypt_NEON_loop_4
-	mov	x25, x4
-	ld1	{V0.16B, V1.16B, V2.16B, V3.16B}, [x0], #0x40
-	ld1	{V4.16B}, [x25], #16
-	and	x16, x17, x9, asr 63
-	extr	x11, x9, x8, #63
-	eor	x10, x16, x8, lsl 1
-	and	x16, x17, x11, asr 63
-	extr	x13, x11, x10, #63
-	eor	x12, x16, x10, lsl 1
-	and	x16, x17, x13, asr 63
-	extr	x15, x13, x12, #63
-	eor	x14, x16, x12, lsl 1
+	rev32	V3.16B, V3.16B
 	mov	V8.D[0], x8
 	mov	V8.D[1], x9
 	mov	V9.D[0], x10
@@ -49331,1212 +50932,1863 @@ L_AES_XTS_decrypt_NEON_loop_4
 	eor	V1.16B, V1.16B, V9.16B
 	eor	V2.16B, V2.16B, V10.16B
 	eor	V3.16B, V3.16B, V11.16B
+	st1	{V0.16B, V1.16B, V2.16B, V3.16B}, [x1], #0x40
+	and	x16, x17, x15, asr 63
+	extr	x9, x15, x14, #63
+	eor	x8, x16, x14, lsl 1
+	sub	w2, w2, #0x40
+	cmp	w2, #0x40
+	bge	L_AES_XTS_decrypt_NEON_loop_4
+	movi	V12.16B, #0x40
+	movi	V13.16B, #0x80
+	movi	V14.16B, #0xc0
+	movi	V15.16B, #27
+L_AES_XTS_decrypt_NEON_start_2
+	cmp	w2, #32
+	blt	L_AES_XTS_decrypt_NEON_start_1
+	mov	x25, x4
+	ld1	{V0.16B, V1.16B}, [x0], #32
+	ld1	{V4.16B}, [x25], #16
+	and	x16, x17, x9, asr 63
+	extr	x11, x9, x8, #63
+	eor	x10, x16, x8, lsl 1
+	and	x16, x17, x11, asr 63
+	extr	x13, x11, x10, #63
+	eor	x12, x16, x10, lsl 1
+	mov	V2.D[0], x8
+	mov	V2.D[1], x9
+	mov	V3.D[0], x10
+	mov	V3.D[1], x11
+	eor	V0.16B, V0.16B, V2.16B
+	eor	V1.16B, V1.16B, V3.16B
 	rev32	V0.16B, V0.16B
 	rev32	V1.16B, V1.16B
-	rev32	V2.16B, V2.16B
-	rev32	V3.16B, V3.16B
 	eor	V0.16B, V0.16B, V4.16B
 	eor	V1.16B, V1.16B, V4.16B
-	eor	V2.16B, V2.16B, V4.16B
-	eor	V3.16B, V3.16B, V4.16B
 	sub	w24, w7, #2
-L_AES_XTS_decrypt_NEON_loop_nr_4
-	tbl	V4.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V0.16B
-	tbl	V5.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V1.16B
-	tbl	V6.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V2.16B
-	tbl	V7.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V3.16B
+L_AES_XTS_decrypt_NEON_loop_nr_2
 	movi	V12.16B, #0x40
 	movi	V13.16B, #0x80
 	movi	V14.16B, #0xc0
 	eor	V8.16B, V0.16B, V12.16B
 	eor	V9.16B, V1.16B, V12.16B
-	eor	V10.16B, V2.16B, V12.16B
-	eor	V11.16B, V3.16B, V12.16B
+	tbl	V4.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V0.16B
+	tbl	V5.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V1.16B
 	tbl	V8.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V8.16B
 	tbl	V9.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V9.16B
-	tbl	V10.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V10.16B
-	tbl	V11.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V11.16B
+	eor	V10.16B, V0.16B, V13.16B
+	eor	V11.16B, V1.16B, V13.16B
+	tbl	V10.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V10.16B
+	tbl	V11.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V11.16B
 	orr	V4.16B, V4.16B, V8.16B
 	orr	V5.16B, V5.16B, V9.16B
-	orr	V6.16B, V6.16B, V10.16B
-	orr	V7.16B, V7.16B, V11.16B
-	eor	V8.16B, V0.16B, V13.16B
-	eor	V9.16B, V1.16B, V13.16B
-	eor	V10.16B, V2.16B, V13.16B
-	eor	V11.16B, V3.16B, V13.16B
-	tbl	V8.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V8.16B
-	tbl	V9.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V9.16B
+	eor	V8.16B, V0.16B, V14.16B
+	eor	V9.16B, V1.16B, V14.16B
+	orr	V4.16B, V4.16B, V10.16B
+	orr	V5.16B, V5.16B, V11.16B
+	tbl	V8.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V8.16B
+	tbl	V9.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V9.16B
+	orr	V4.16B, V4.16B, V8.16B
+	orr	V5.16B, V5.16B, V9.16B
+	ld1	{V0.16B}, [x23]
+	tbl	V4.16B, {V4.16B}, V0.16B
+	tbl	V5.16B, {V5.16B}, V0.16B
+	movi	V10.16B, #27
+	sshr	V8.16B, V4.16B, #7
+	sshr	V9.16B, V5.16B, #7
+	shl	V12.16B, V4.16B, #1
+	shl	V13.16B, V5.16B, #1
+	and	V8.16B, V8.16B, V10.16B
+	and	V9.16B, V9.16B, V10.16B
+	eor	V8.16B, V8.16B, V12.16B
+	eor	V9.16B, V9.16B, V13.16B
+	ushr	V12.16B, V4.16B, #6
+	ushr	V13.16B, V5.16B, #6
+	shl	V0.16B, V4.16B, #2
+	shl	V1.16B, V5.16B, #2
+	pmul	V12.16B, V12.16B, V10.16B
+	pmul	V13.16B, V13.16B, V10.16B
+	eor	V12.16B, V12.16B, V0.16B
+	eor	V13.16B, V13.16B, V1.16B
+	ushr	V0.16B, V4.16B, #5
+	ushr	V1.16B, V5.16B, #5
+	pmul	V0.16B, V0.16B, V10.16B
+	pmul	V1.16B, V1.16B, V10.16B
+	shl	V10.16B, V4.16B, #3
+	shl	V11.16B, V5.16B, #3
+	eor	V0.16B, V0.16B, V10.16B
+	eor	V1.16B, V1.16B, V11.16B
+	eor	V10.16B, V8.16B, V0.16B
+	eor	V11.16B, V9.16B, V1.16B
+	eor	V0.16B, V0.16B, V4.16B
+	eor	V1.16B, V1.16B, V5.16B
+	eor	V8.16B, V12.16B, V0.16B
+	eor	V9.16B, V13.16B, V1.16B
+	eor	V12.16B, V12.16B, V10.16B
+	eor	V13.16B, V13.16B, V11.16B
+	eor	V10.16B, V10.16B, V4.16B
+	eor	V11.16B, V11.16B, V5.16B
+	shl	V4.4S, V10.4S, #8
+	shl	V5.4S, V11.4S, #8
+	rev32	V8.8H, V8.8H
+	rev32	V9.8H, V9.8H
+	sri	V4.4S, V10.4S, #24
+	sri	V5.4S, V11.4S, #24
+	eor	V4.16B, V4.16B, V12.16B
+	eor	V5.16B, V5.16B, V13.16B
+	shl	V10.4S, V0.4S, #24
+	shl	V11.4S, V1.4S, #24
+	eor	V4.16B, V4.16B, V8.16B
+	eor	V5.16B, V5.16B, V9.16B
+	sri	V10.4S, V0.4S, #8
+	sri	V11.4S, V1.4S, #8
+	eor	V4.16B, V4.16B, V10.16B
+	eor	V5.16B, V5.16B, V11.16B
+	;   XOR in Key Schedule
+	ld1	{V0.2D}, [x25], #16
+	eor	V4.16B, V4.16B, V0.16B
+	eor	V5.16B, V5.16B, V0.16B
+	; Round Done
+	movi	V12.16B, #0x40
+	movi	V13.16B, #0x80
+	movi	V14.16B, #0xc0
+	eor	V8.16B, V4.16B, V12.16B
+	eor	V9.16B, V5.16B, V12.16B
+	tbl	V0.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V4.16B
+	tbl	V1.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V5.16B
+	tbl	V8.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V8.16B
+	tbl	V9.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V9.16B
+	eor	V10.16B, V4.16B, V13.16B
+	eor	V11.16B, V5.16B, V13.16B
+	tbl	V10.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V10.16B
+	tbl	V11.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V11.16B
+	orr	V0.16B, V0.16B, V8.16B
+	orr	V1.16B, V1.16B, V9.16B
+	eor	V8.16B, V4.16B, V14.16B
+	eor	V9.16B, V5.16B, V14.16B
+	orr	V0.16B, V0.16B, V10.16B
+	orr	V1.16B, V1.16B, V11.16B
+	tbl	V8.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V8.16B
+	tbl	V9.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V9.16B
+	orr	V0.16B, V0.16B, V8.16B
+	orr	V1.16B, V1.16B, V9.16B
+	ld1	{V4.16B}, [x23]
+	tbl	V0.16B, {V0.16B}, V4.16B
+	tbl	V1.16B, {V1.16B}, V4.16B
+	movi	V10.16B, #27
+	sshr	V8.16B, V0.16B, #7
+	sshr	V9.16B, V1.16B, #7
+	shl	V12.16B, V0.16B, #1
+	shl	V13.16B, V1.16B, #1
+	and	V8.16B, V8.16B, V10.16B
+	and	V9.16B, V9.16B, V10.16B
+	eor	V8.16B, V8.16B, V12.16B
+	eor	V9.16B, V9.16B, V13.16B
+	ushr	V12.16B, V0.16B, #6
+	ushr	V13.16B, V1.16B, #6
+	shl	V4.16B, V0.16B, #2
+	shl	V5.16B, V1.16B, #2
+	pmul	V12.16B, V12.16B, V10.16B
+	pmul	V13.16B, V13.16B, V10.16B
+	eor	V12.16B, V12.16B, V4.16B
+	eor	V13.16B, V13.16B, V5.16B
+	ushr	V4.16B, V0.16B, #5
+	ushr	V5.16B, V1.16B, #5
+	pmul	V4.16B, V4.16B, V10.16B
+	pmul	V5.16B, V5.16B, V10.16B
+	shl	V10.16B, V0.16B, #3
+	shl	V11.16B, V1.16B, #3
+	eor	V4.16B, V4.16B, V10.16B
+	eor	V5.16B, V5.16B, V11.16B
+	eor	V10.16B, V8.16B, V4.16B
+	eor	V11.16B, V9.16B, V5.16B
+	eor	V4.16B, V4.16B, V0.16B
+	eor	V5.16B, V5.16B, V1.16B
+	eor	V8.16B, V12.16B, V4.16B
+	eor	V9.16B, V13.16B, V5.16B
+	eor	V12.16B, V12.16B, V10.16B
+	eor	V13.16B, V13.16B, V11.16B
+	eor	V10.16B, V10.16B, V0.16B
+	eor	V11.16B, V11.16B, V1.16B
+	shl	V0.4S, V10.4S, #8
+	shl	V1.4S, V11.4S, #8
+	rev32	V8.8H, V8.8H
+	rev32	V9.8H, V9.8H
+	sri	V0.4S, V10.4S, #24
+	sri	V1.4S, V11.4S, #24
+	eor	V0.16B, V0.16B, V12.16B
+	eor	V1.16B, V1.16B, V13.16B
+	shl	V10.4S, V4.4S, #24
+	shl	V11.4S, V5.4S, #24
+	eor	V0.16B, V0.16B, V8.16B
+	eor	V1.16B, V1.16B, V9.16B
+	sri	V10.4S, V4.4S, #8
+	sri	V11.4S, V5.4S, #8
+	eor	V0.16B, V0.16B, V10.16B
+	eor	V1.16B, V1.16B, V11.16B
+	;   XOR in Key Schedule
+	ld1	{V4.2D}, [x25], #16
+	eor	V0.16B, V0.16B, V4.16B
+	eor	V1.16B, V1.16B, V4.16B
+	; Round Done
+	subs	w24, w24, #2
+	bne	L_AES_XTS_decrypt_NEON_loop_nr_2
+	movi	V12.16B, #0x40
+	movi	V13.16B, #0x80
+	movi	V14.16B, #0xc0
+	eor	V8.16B, V0.16B, V12.16B
+	eor	V9.16B, V1.16B, V12.16B
+	tbl	V4.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V0.16B
+	tbl	V5.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V1.16B
+	tbl	V8.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V8.16B
+	tbl	V9.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V9.16B
+	eor	V10.16B, V0.16B, V13.16B
+	eor	V11.16B, V1.16B, V13.16B
 	tbl	V10.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V10.16B
 	tbl	V11.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V11.16B
 	orr	V4.16B, V4.16B, V8.16B
 	orr	V5.16B, V5.16B, V9.16B
-	orr	V6.16B, V6.16B, V10.16B
-	orr	V7.16B, V7.16B, V11.16B
 	eor	V8.16B, V0.16B, V14.16B
 	eor	V9.16B, V1.16B, V14.16B
-	eor	V10.16B, V2.16B, V14.16B
-	eor	V11.16B, V3.16B, V14.16B
+	orr	V4.16B, V4.16B, V10.16B
+	orr	V5.16B, V5.16B, V11.16B
 	tbl	V8.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V8.16B
 	tbl	V9.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V9.16B
-	tbl	V10.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V10.16B
-	tbl	V11.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V11.16B
 	orr	V4.16B, V4.16B, V8.16B
 	orr	V5.16B, V5.16B, V9.16B
-	orr	V6.16B, V6.16B, V10.16B
-	orr	V7.16B, V7.16B, V11.16B
 	ld1	{V0.16B}, [x23]
 	tbl	V4.16B, {V4.16B}, V0.16B
 	tbl	V5.16B, {V5.16B}, V0.16B
-	tbl	V6.16B, {V6.16B}, V0.16B
-	tbl	V7.16B, {V7.16B}, V0.16B
-	movi	V28.16B, #27
+	movi	V10.16B, #27
 	sshr	V8.16B, V4.16B, #7
 	sshr	V9.16B, V5.16B, #7
-	sshr	V10.16B, V6.16B, #7
-	sshr	V11.16B, V7.16B, #7
 	shl	V12.16B, V4.16B, #1
 	shl	V13.16B, V5.16B, #1
-	shl	V14.16B, V6.16B, #1
-	shl	V15.16B, V7.16B, #1
-	and	V8.16B, V8.16B, V28.16B
-	and	V9.16B, V9.16B, V28.16B
-	and	V10.16B, V10.16B, V28.16B
-	and	V11.16B, V11.16B, V28.16B
+	and	V8.16B, V8.16B, V10.16B
+	and	V9.16B, V9.16B, V10.16B
 	eor	V8.16B, V8.16B, V12.16B
 	eor	V9.16B, V9.16B, V13.16B
-	eor	V10.16B, V10.16B, V14.16B
-	eor	V11.16B, V11.16B, V15.16B
 	ushr	V12.16B, V4.16B, #6
 	ushr	V13.16B, V5.16B, #6
-	ushr	V14.16B, V6.16B, #6
-	ushr	V15.16B, V7.16B, #6
 	shl	V0.16B, V4.16B, #2
 	shl	V1.16B, V5.16B, #2
-	shl	V2.16B, V6.16B, #2
-	shl	V3.16B, V7.16B, #2
-	pmul	V12.16B, V12.16B, V28.16B
-	pmul	V13.16B, V13.16B, V28.16B
-	pmul	V14.16B, V14.16B, V28.16B
-	pmul	V15.16B, V15.16B, V28.16B
+	pmul	V12.16B, V12.16B, V10.16B
+	pmul	V13.16B, V13.16B, V10.16B
 	eor	V12.16B, V12.16B, V0.16B
 	eor	V13.16B, V13.16B, V1.16B
-	eor	V14.16B, V14.16B, V2.16B
-	eor	V15.16B, V15.16B, V3.16B
 	ushr	V0.16B, V4.16B, #5
 	ushr	V1.16B, V5.16B, #5
-	ushr	V2.16B, V6.16B, #5
-	ushr	V3.16B, V7.16B, #5
-	pmul	V0.16B, V0.16B, V28.16B
-	pmul	V1.16B, V1.16B, V28.16B
-	pmul	V2.16B, V2.16B, V28.16B
-	pmul	V3.16B, V3.16B, V28.16B
-	shl	V28.16B, V4.16B, #3
-	shl	V29.16B, V5.16B, #3
-	shl	V30.16B, V6.16B, #3
-	shl	V31.16B, V7.16B, #3
-	eor	V0.16B, V0.16B, V28.16B
-	eor	V1.16B, V1.16B, V29.16B
-	eor	V2.16B, V2.16B, V30.16B
-	eor	V3.16B, V3.16B, V31.16B
-	eor	V28.16B, V8.16B, V0.16B
-	eor	V29.16B, V9.16B, V1.16B
-	eor	V30.16B, V10.16B, V2.16B
-	eor	V31.16B, V11.16B, V3.16B
+	pmul	V0.16B, V0.16B, V10.16B
+	pmul	V1.16B, V1.16B, V10.16B
+	shl	V10.16B, V4.16B, #3
+	shl	V11.16B, V5.16B, #3
+	eor	V0.16B, V0.16B, V10.16B
+	eor	V1.16B, V1.16B, V11.16B
+	eor	V10.16B, V8.16B, V0.16B
+	eor	V11.16B, V9.16B, V1.16B
 	eor	V0.16B, V0.16B, V4.16B
 	eor	V1.16B, V1.16B, V5.16B
-	eor	V2.16B, V2.16B, V6.16B
-	eor	V3.16B, V3.16B, V7.16B
 	eor	V8.16B, V12.16B, V0.16B
 	eor	V9.16B, V13.16B, V1.16B
-	eor	V10.16B, V14.16B, V2.16B
-	eor	V11.16B, V15.16B, V3.16B
-	eor	V12.16B, V12.16B, V28.16B
-	eor	V13.16B, V13.16B, V29.16B
-	eor	V14.16B, V14.16B, V30.16B
-	eor	V15.16B, V15.16B, V31.16B
-	eor	V28.16B, V28.16B, V4.16B
-	eor	V29.16B, V29.16B, V5.16B
-	eor	V30.16B, V30.16B, V6.16B
-	eor	V31.16B, V31.16B, V7.16B
-	shl	V4.4S, V28.4S, #8
-	shl	V5.4S, V29.4S, #8
-	shl	V6.4S, V30.4S, #8
-	shl	V7.4S, V31.4S, #8
+	eor	V12.16B, V12.16B, V10.16B
+	eor	V13.16B, V13.16B, V11.16B
+	eor	V10.16B, V10.16B, V4.16B
+	eor	V11.16B, V11.16B, V5.16B
+	shl	V4.4S, V10.4S, #8
+	shl	V5.4S, V11.4S, #8
 	rev32	V8.8H, V8.8H
 	rev32	V9.8H, V9.8H
-	rev32	V10.8H, V10.8H
-	rev32	V11.8H, V11.8H
-	sri	V4.4S, V28.4S, #24
-	sri	V5.4S, V29.4S, #24
-	sri	V6.4S, V30.4S, #24
-	sri	V7.4S, V31.4S, #24
+	sri	V4.4S, V10.4S, #24
+	sri	V5.4S, V11.4S, #24
 	eor	V4.16B, V4.16B, V12.16B
 	eor	V5.16B, V5.16B, V13.16B
-	eor	V6.16B, V6.16B, V14.16B
-	eor	V7.16B, V7.16B, V15.16B
-	shl	V28.4S, V0.4S, #24
-	shl	V29.4S, V1.4S, #24
-	shl	V30.4S, V2.4S, #24
-	shl	V31.4S, V3.4S, #24
+	shl	V10.4S, V0.4S, #24
+	shl	V11.4S, V1.4S, #24
 	eor	V4.16B, V4.16B, V8.16B
 	eor	V5.16B, V5.16B, V9.16B
-	eor	V6.16B, V6.16B, V10.16B
-	eor	V7.16B, V7.16B, V11.16B
-	sri	V28.4S, V0.4S, #8
-	sri	V29.4S, V1.4S, #8
-	sri	V30.4S, V2.4S, #8
-	sri	V31.4S, V3.4S, #8
-	eor	V4.16B, V4.16B, V28.16B
-	eor	V5.16B, V5.16B, V29.16B
-	eor	V6.16B, V6.16B, V30.16B
-	eor	V7.16B, V7.16B, V31.16B
-	ld1	{V28.16B, V29.16B, V30.16B, V31.16B}, [x21]
+	sri	V10.4S, V0.4S, #8
+	sri	V11.4S, V1.4S, #8
+	eor	V4.16B, V4.16B, V10.16B
+	eor	V5.16B, V5.16B, V11.16B
+	;   XOR in Key Schedule
+	ld1	{V0.2D}, [x25], #16
+	eor	V4.16B, V4.16B, V0.16B
+	eor	V5.16B, V5.16B, V0.16B
+	; Round Done
+	movi	V12.16B, #0x40
+	movi	V13.16B, #0x80
+	movi	V14.16B, #0xc0
+	eor	V8.16B, V4.16B, V12.16B
+	eor	V9.16B, V5.16B, V12.16B
+	tbl	V0.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V4.16B
+	tbl	V1.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V5.16B
+	tbl	V8.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V8.16B
+	tbl	V9.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V9.16B
+	eor	V10.16B, V4.16B, V13.16B
+	eor	V11.16B, V5.16B, V13.16B
+	tbl	V10.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V10.16B
+	tbl	V11.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V11.16B
+	orr	V0.16B, V0.16B, V8.16B
+	orr	V1.16B, V1.16B, V9.16B
+	eor	V8.16B, V4.16B, V14.16B
+	eor	V9.16B, V5.16B, V14.16B
+	orr	V0.16B, V0.16B, V10.16B
+	orr	V1.16B, V1.16B, V11.16B
+	tbl	V8.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V8.16B
+	tbl	V9.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V9.16B
+	orr	V0.16B, V0.16B, V8.16B
+	orr	V1.16B, V1.16B, V9.16B
+	ld1	{V4.16B}, [x23]
+	tbl	V0.16B, {V0.16B}, V4.16B
+	tbl	V1.16B, {V1.16B}, V4.16B
+	;   XOR in Key Schedule
+	ld1	{V4.2D}, [x25], #16
+	eor	V0.16B, V0.16B, V4.16B
+	eor	V1.16B, V1.16B, V4.16B
+	; Round Done
+	rev32	V0.16B, V0.16B
+	rev32	V1.16B, V1.16B
+	eor	V0.16B, V0.16B, V2.16B
+	eor	V1.16B, V1.16B, V3.16B
+	st1	{V0.16B, V1.16B}, [x1], #32
+	and	x16, x17, x11, asr 63
+	extr	x9, x11, x10, #63
+	eor	x8, x16, x10, lsl 1
+	sub	w2, w2, #32
+L_AES_XTS_decrypt_NEON_start_1
+	ld1	{V3.2D}, [x23]
+	mov	V2.D[0], x8
+	mov	V2.D[1], x9
+	cmp	w2, #16
+	blt	L_AES_XTS_decrypt_NEON_start_partial
+	mov	x25, x4
+	ld1	{V0.16B}, [x0], #16
+	ld1	{V4.2D}, [x25], #16
+	eor	V0.16B, V0.16B, V2.16B
+	rev32	V0.16B, V0.16B
+	eor	V0.16B, V0.16B, V4.16B
+	sub	w24, w7, #2
+L_AES_XTS_decrypt_NEON_loop_nr_1
+	eor	V8.16B, V0.16B, V12.16B
+	eor	V9.16B, V0.16B, V13.16B
+	eor	V10.16B, V0.16B, V14.16B
+	tbl	V4.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V0.16B
+	tbl	V8.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V8.16B
+	tbl	V9.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V9.16B
+	tbl	V10.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V10.16B
+	orr	V4.16B, V4.16B, V8.16B
+	orr	V9.16B, V9.16B, V10.16B
+	orr	V4.16B, V4.16B, V9.16B
+	tbl	V4.16B, {V4.16B}, V3.16B
+	sshr	V10.16B, V4.16B, #7
+	ushr	V11.16B, V4.16B, #6
+	ushr	V8.16B, V4.16B, #5
+	and	V10.16B, V10.16B, V15.16B
+	pmul	V11.16B, V11.16B, V15.16B
+	pmul	V8.16B, V8.16B, V15.16B
+	shl	V9.16B, V4.16B, #1
+	eor	V10.16B, V10.16B, V9.16B
+	shl	V9.16B, V4.16B, #3
+	eor	V8.16B, V8.16B, V9.16B
+	shl	V9.16B, V4.16B, #2
+	eor	V11.16B, V11.16B, V9.16B
+	eor	V9.16B, V10.16B, V8.16B
+	eor	V8.16B, V8.16B, V4.16B
+	eor	V10.16B, V11.16B, V8.16B
+	eor	V11.16B, V11.16B, V9.16B
+	eor	V9.16B, V9.16B, V4.16B
+	shl	V4.4S, V9.4S, #8
+	rev32	V10.8H, V10.8H
+	sri	V4.4S, V9.4S, #24
+	eor	V4.16B, V4.16B, V11.16B
+	shl	V9.4S, V8.4S, #24
+	eor	V4.16B, V4.16B, V10.16B
+	sri	V9.4S, V8.4S, #8
+	eor	V4.16B, V4.16B, V9.16B
+	ld1	{V0.2D}, [x25], #16
+	;   XOR in Key Schedule
+	eor	V4.16B, V4.16B, V0.16B
+	eor	V8.16B, V4.16B, V12.16B
+	eor	V9.16B, V4.16B, V13.16B
+	eor	V10.16B, V4.16B, V14.16B
+	tbl	V0.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V4.16B
+	tbl	V8.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V8.16B
+	tbl	V9.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V9.16B
+	tbl	V10.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V10.16B
+	orr	V0.16B, V0.16B, V8.16B
+	orr	V9.16B, V9.16B, V10.16B
+	orr	V0.16B, V0.16B, V9.16B
+	tbl	V0.16B, {V0.16B}, V3.16B
+	sshr	V10.16B, V0.16B, #7
+	ushr	V11.16B, V0.16B, #6
+	ushr	V8.16B, V0.16B, #5
+	and	V10.16B, V10.16B, V15.16B
+	pmul	V11.16B, V11.16B, V15.16B
+	pmul	V8.16B, V8.16B, V15.16B
+	shl	V9.16B, V0.16B, #1
+	eor	V10.16B, V10.16B, V9.16B
+	shl	V9.16B, V0.16B, #3
+	eor	V8.16B, V8.16B, V9.16B
+	shl	V9.16B, V0.16B, #2
+	eor	V11.16B, V11.16B, V9.16B
+	eor	V9.16B, V10.16B, V8.16B
+	eor	V8.16B, V8.16B, V0.16B
+	eor	V10.16B, V11.16B, V8.16B
+	eor	V11.16B, V11.16B, V9.16B
+	eor	V9.16B, V9.16B, V0.16B
+	shl	V0.4S, V9.4S, #8
+	rev32	V10.8H, V10.8H
+	sri	V0.4S, V9.4S, #24
+	eor	V0.16B, V0.16B, V11.16B
+	shl	V9.4S, V8.4S, #24
+	eor	V0.16B, V0.16B, V10.16B
+	sri	V9.4S, V8.4S, #8
+	eor	V0.16B, V0.16B, V9.16B
+	ld1	{V4.2D}, [x25], #16
 	;   XOR in Key Schedule
+	eor	V0.16B, V0.16B, V4.16B
+	subs	w24, w24, #2
+	bne	L_AES_XTS_decrypt_NEON_loop_nr_1
+	eor	V8.16B, V0.16B, V12.16B
+	eor	V9.16B, V0.16B, V13.16B
+	eor	V10.16B, V0.16B, V14.16B
+	tbl	V4.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V0.16B
+	tbl	V8.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V8.16B
+	tbl	V9.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V9.16B
+	tbl	V10.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V10.16B
+	orr	V4.16B, V4.16B, V8.16B
+	orr	V9.16B, V9.16B, V10.16B
+	orr	V4.16B, V4.16B, V9.16B
+	tbl	V4.16B, {V4.16B}, V3.16B
+	sshr	V10.16B, V4.16B, #7
+	ushr	V11.16B, V4.16B, #6
+	ushr	V8.16B, V4.16B, #5
+	and	V10.16B, V10.16B, V15.16B
+	pmul	V11.16B, V11.16B, V15.16B
+	pmul	V8.16B, V8.16B, V15.16B
+	shl	V9.16B, V4.16B, #1
+	eor	V10.16B, V10.16B, V9.16B
+	shl	V9.16B, V4.16B, #3
+	eor	V8.16B, V8.16B, V9.16B
+	shl	V9.16B, V4.16B, #2
+	eor	V11.16B, V11.16B, V9.16B
+	eor	V9.16B, V10.16B, V8.16B
+	eor	V8.16B, V8.16B, V4.16B
+	eor	V10.16B, V11.16B, V8.16B
+	eor	V11.16B, V11.16B, V9.16B
+	eor	V9.16B, V9.16B, V4.16B
+	shl	V4.4S, V9.4S, #8
+	rev32	V10.8H, V10.8H
+	sri	V4.4S, V9.4S, #24
+	eor	V4.16B, V4.16B, V11.16B
+	shl	V9.4S, V8.4S, #24
+	eor	V4.16B, V4.16B, V10.16B
+	sri	V9.4S, V8.4S, #8
+	eor	V4.16B, V4.16B, V9.16B
 	ld1	{V0.2D}, [x25], #16
+	;   XOR in Key Schedule
 	eor	V4.16B, V4.16B, V0.16B
-	eor	V5.16B, V5.16B, V0.16B
-	eor	V6.16B, V6.16B, V0.16B
-	eor	V7.16B, V7.16B, V0.16B
-	; Round Done
-	tbl	V0.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V4.16B
-	tbl	V1.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V5.16B
-	tbl	V2.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V6.16B
-	tbl	V3.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V7.16B
-	movi	V12.16B, #0x40
-	movi	V13.16B, #0x80
-	movi	V14.16B, #0xc0
 	eor	V8.16B, V4.16B, V12.16B
-	eor	V9.16B, V5.16B, V12.16B
-	eor	V10.16B, V6.16B, V12.16B
-	eor	V11.16B, V7.16B, V12.16B
+	eor	V9.16B, V4.16B, V13.16B
+	eor	V10.16B, V4.16B, V14.16B
+	tbl	V0.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V4.16B
 	tbl	V8.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V8.16B
-	tbl	V9.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V9.16B
-	tbl	V10.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V10.16B
-	tbl	V11.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V11.16B
-	orr	V0.16B, V0.16B, V8.16B
-	orr	V1.16B, V1.16B, V9.16B
-	orr	V2.16B, V2.16B, V10.16B
-	orr	V3.16B, V3.16B, V11.16B
-	eor	V8.16B, V4.16B, V13.16B
-	eor	V9.16B, V5.16B, V13.16B
-	eor	V10.16B, V6.16B, V13.16B
-	eor	V11.16B, V7.16B, V13.16B
-	tbl	V8.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V8.16B
 	tbl	V9.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V9.16B
-	tbl	V10.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V10.16B
-	tbl	V11.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V11.16B
-	orr	V0.16B, V0.16B, V8.16B
-	orr	V1.16B, V1.16B, V9.16B
-	orr	V2.16B, V2.16B, V10.16B
-	orr	V3.16B, V3.16B, V11.16B
-	eor	V8.16B, V4.16B, V14.16B
-	eor	V9.16B, V5.16B, V14.16B
-	eor	V10.16B, V6.16B, V14.16B
-	eor	V11.16B, V7.16B, V14.16B
-	tbl	V8.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V8.16B
-	tbl	V9.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V9.16B
 	tbl	V10.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V10.16B
-	tbl	V11.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V11.16B
 	orr	V0.16B, V0.16B, V8.16B
-	orr	V1.16B, V1.16B, V9.16B
-	orr	V2.16B, V2.16B, V10.16B
-	orr	V3.16B, V3.16B, V11.16B
-	ld1	{V4.16B}, [x23]
-	tbl	V0.16B, {V0.16B}, V4.16B
-	tbl	V1.16B, {V1.16B}, V4.16B
-	tbl	V2.16B, {V2.16B}, V4.16B
-	tbl	V3.16B, {V3.16B}, V4.16B
-	movi	V28.16B, #27
-	sshr	V8.16B, V0.16B, #7
-	sshr	V9.16B, V1.16B, #7
-	sshr	V10.16B, V2.16B, #7
-	sshr	V11.16B, V3.16B, #7
-	shl	V12.16B, V0.16B, #1
-	shl	V13.16B, V1.16B, #1
-	shl	V14.16B, V2.16B, #1
-	shl	V15.16B, V3.16B, #1
-	and	V8.16B, V8.16B, V28.16B
-	and	V9.16B, V9.16B, V28.16B
-	and	V10.16B, V10.16B, V28.16B
-	and	V11.16B, V11.16B, V28.16B
-	eor	V8.16B, V8.16B, V12.16B
-	eor	V9.16B, V9.16B, V13.16B
-	eor	V10.16B, V10.16B, V14.16B
-	eor	V11.16B, V11.16B, V15.16B
-	ushr	V12.16B, V0.16B, #6
-	ushr	V13.16B, V1.16B, #6
-	ushr	V14.16B, V2.16B, #6
-	ushr	V15.16B, V3.16B, #6
-	shl	V4.16B, V0.16B, #2
-	shl	V5.16B, V1.16B, #2
-	shl	V6.16B, V2.16B, #2
-	shl	V7.16B, V3.16B, #2
-	pmul	V12.16B, V12.16B, V28.16B
-	pmul	V13.16B, V13.16B, V28.16B
-	pmul	V14.16B, V14.16B, V28.16B
-	pmul	V15.16B, V15.16B, V28.16B
-	eor	V12.16B, V12.16B, V4.16B
-	eor	V13.16B, V13.16B, V5.16B
-	eor	V14.16B, V14.16B, V6.16B
-	eor	V15.16B, V15.16B, V7.16B
-	ushr	V4.16B, V0.16B, #5
-	ushr	V5.16B, V1.16B, #5
-	ushr	V6.16B, V2.16B, #5
-	ushr	V7.16B, V3.16B, #5
-	pmul	V4.16B, V4.16B, V28.16B
-	pmul	V5.16B, V5.16B, V28.16B
-	pmul	V6.16B, V6.16B, V28.16B
-	pmul	V7.16B, V7.16B, V28.16B
-	shl	V28.16B, V0.16B, #3
-	shl	V29.16B, V1.16B, #3
-	shl	V30.16B, V2.16B, #3
-	shl	V31.16B, V3.16B, #3
-	eor	V4.16B, V4.16B, V28.16B
-	eor	V5.16B, V5.16B, V29.16B
-	eor	V6.16B, V6.16B, V30.16B
-	eor	V7.16B, V7.16B, V31.16B
-	eor	V28.16B, V8.16B, V4.16B
-	eor	V29.16B, V9.16B, V5.16B
-	eor	V30.16B, V10.16B, V6.16B
-	eor	V31.16B, V11.16B, V7.16B
-	eor	V4.16B, V4.16B, V0.16B
-	eor	V5.16B, V5.16B, V1.16B
-	eor	V6.16B, V6.16B, V2.16B
-	eor	V7.16B, V7.16B, V3.16B
-	eor	V8.16B, V12.16B, V4.16B
-	eor	V9.16B, V13.16B, V5.16B
-	eor	V10.16B, V14.16B, V6.16B
-	eor	V11.16B, V15.16B, V7.16B
-	eor	V12.16B, V12.16B, V28.16B
-	eor	V13.16B, V13.16B, V29.16B
-	eor	V14.16B, V14.16B, V30.16B
-	eor	V15.16B, V15.16B, V31.16B
-	eor	V28.16B, V28.16B, V0.16B
-	eor	V29.16B, V29.16B, V1.16B
-	eor	V30.16B, V30.16B, V2.16B
-	eor	V31.16B, V31.16B, V3.16B
-	shl	V0.4S, V28.4S, #8
-	shl	V1.4S, V29.4S, #8
-	shl	V2.4S, V30.4S, #8
-	shl	V3.4S, V31.4S, #8
-	rev32	V8.8H, V8.8H
-	rev32	V9.8H, V9.8H
+	orr	V9.16B, V9.16B, V10.16B
+	orr	V0.16B, V0.16B, V9.16B
+	tbl	V0.16B, {V0.16B}, V3.16B
+	ld1	{V4.2D}, [x25], #16
+	;   XOR in Key Schedule
+	eor	V0.16B, V0.16B, V4.16B
+	rev32	V0.16B, V0.16B
+	eor	V0.16B, V0.16B, V2.16B
+	st1	{V0.16B}, [x1], #16
+	sub	w2, w2, #16
+	cbz	w19, L_AES_XTS_decrypt_NEON_data_done
+	and	x16, x17, x9, asr 63
+	extr	x9, x9, x8, #63
+	eor	x8, x16, x8, lsl 1
+L_AES_XTS_decrypt_NEON_start_partial
+	mov	w2, w19
+	cbz	w2, L_AES_XTS_decrypt_NEON_data_done
+	mov	V2.D[0], x8
+	mov	V2.D[1], x9
+	and	x16, x17, x9, asr 63
+	extr	x11, x9, x8, #63
+	eor	x10, x16, x8, lsl 1
+	mov	V1.D[0], x10
+	mov	V1.D[1], x11
+	mov	x25, x4
+	ld1	{V0.16B}, [x0], #16
+	ld1	{V4.2D}, [x25], #16
+	eor	V0.16B, V0.16B, V1.16B
+	rev32	V0.16B, V0.16B
+	eor	V0.16B, V0.16B, V4.16B
+	sub	w24, w7, #2
+L_AES_XTS_decrypt_NEON_loop_nr_partial_1
+	eor	V8.16B, V0.16B, V12.16B
+	eor	V9.16B, V0.16B, V13.16B
+	eor	V10.16B, V0.16B, V14.16B
+	tbl	V4.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V0.16B
+	tbl	V8.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V8.16B
+	tbl	V9.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V9.16B
+	tbl	V10.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V10.16B
+	orr	V4.16B, V4.16B, V8.16B
+	orr	V9.16B, V9.16B, V10.16B
+	orr	V4.16B, V4.16B, V9.16B
+	tbl	V4.16B, {V4.16B}, V3.16B
+	sshr	V10.16B, V4.16B, #7
+	ushr	V11.16B, V4.16B, #6
+	ushr	V8.16B, V4.16B, #5
+	and	V10.16B, V10.16B, V15.16B
+	pmul	V11.16B, V11.16B, V15.16B
+	pmul	V8.16B, V8.16B, V15.16B
+	shl	V9.16B, V4.16B, #1
+	eor	V10.16B, V10.16B, V9.16B
+	shl	V9.16B, V4.16B, #3
+	eor	V8.16B, V8.16B, V9.16B
+	shl	V9.16B, V4.16B, #2
+	eor	V11.16B, V11.16B, V9.16B
+	eor	V9.16B, V10.16B, V8.16B
+	eor	V8.16B, V8.16B, V4.16B
+	eor	V10.16B, V11.16B, V8.16B
+	eor	V11.16B, V11.16B, V9.16B
+	eor	V9.16B, V9.16B, V4.16B
+	shl	V4.4S, V9.4S, #8
 	rev32	V10.8H, V10.8H
-	rev32	V11.8H, V11.8H
-	sri	V0.4S, V28.4S, #24
-	sri	V1.4S, V29.4S, #24
-	sri	V2.4S, V30.4S, #24
-	sri	V3.4S, V31.4S, #24
-	eor	V0.16B, V0.16B, V12.16B
-	eor	V1.16B, V1.16B, V13.16B
-	eor	V2.16B, V2.16B, V14.16B
-	eor	V3.16B, V3.16B, V15.16B
-	shl	V28.4S, V4.4S, #24
-	shl	V29.4S, V5.4S, #24
-	shl	V30.4S, V6.4S, #24
-	shl	V31.4S, V7.4S, #24
-	eor	V0.16B, V0.16B, V8.16B
-	eor	V1.16B, V1.16B, V9.16B
-	eor	V2.16B, V2.16B, V10.16B
-	eor	V3.16B, V3.16B, V11.16B
-	sri	V28.4S, V4.4S, #8
-	sri	V29.4S, V5.4S, #8
-	sri	V30.4S, V6.4S, #8
-	sri	V31.4S, V7.4S, #8
-	eor	V0.16B, V0.16B, V28.16B
-	eor	V1.16B, V1.16B, V29.16B
-	eor	V2.16B, V2.16B, V30.16B
-	eor	V3.16B, V3.16B, V31.16B
-	ld1	{V28.16B, V29.16B, V30.16B, V31.16B}, [x21]
+	sri	V4.4S, V9.4S, #24
+	eor	V4.16B, V4.16B, V11.16B
+	shl	V9.4S, V8.4S, #24
+	eor	V4.16B, V4.16B, V10.16B
+	sri	V9.4S, V8.4S, #8
+	eor	V4.16B, V4.16B, V9.16B
+	ld1	{V0.2D}, [x25], #16
 	;   XOR in Key Schedule
+	eor	V4.16B, V4.16B, V0.16B
+	eor	V8.16B, V4.16B, V12.16B
+	eor	V9.16B, V4.16B, V13.16B
+	eor	V10.16B, V4.16B, V14.16B
+	tbl	V0.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V4.16B
+	tbl	V8.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V8.16B
+	tbl	V9.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V9.16B
+	tbl	V10.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V10.16B
+	orr	V0.16B, V0.16B, V8.16B
+	orr	V9.16B, V9.16B, V10.16B
+	orr	V0.16B, V0.16B, V9.16B
+	tbl	V0.16B, {V0.16B}, V3.16B
+	sshr	V10.16B, V0.16B, #7
+	ushr	V11.16B, V0.16B, #6
+	ushr	V8.16B, V0.16B, #5
+	and	V10.16B, V10.16B, V15.16B
+	pmul	V11.16B, V11.16B, V15.16B
+	pmul	V8.16B, V8.16B, V15.16B
+	shl	V9.16B, V0.16B, #1
+	eor	V10.16B, V10.16B, V9.16B
+	shl	V9.16B, V0.16B, #3
+	eor	V8.16B, V8.16B, V9.16B
+	shl	V9.16B, V0.16B, #2
+	eor	V11.16B, V11.16B, V9.16B
+	eor	V9.16B, V10.16B, V8.16B
+	eor	V8.16B, V8.16B, V0.16B
+	eor	V10.16B, V11.16B, V8.16B
+	eor	V11.16B, V11.16B, V9.16B
+	eor	V9.16B, V9.16B, V0.16B
+	shl	V0.4S, V9.4S, #8
+	rev32	V10.8H, V10.8H
+	sri	V0.4S, V9.4S, #24
+	eor	V0.16B, V0.16B, V11.16B
+	shl	V9.4S, V8.4S, #24
+	eor	V0.16B, V0.16B, V10.16B
+	sri	V9.4S, V8.4S, #8
+	eor	V0.16B, V0.16B, V9.16B
 	ld1	{V4.2D}, [x25], #16
+	;   XOR in Key Schedule
 	eor	V0.16B, V0.16B, V4.16B
-	eor	V1.16B, V1.16B, V4.16B
-	eor	V2.16B, V2.16B, V4.16B
-	eor	V3.16B, V3.16B, V4.16B
-	; Round Done
 	subs	w24, w24, #2
-	bne	L_AES_XTS_decrypt_NEON_loop_nr_4
-	tbl	V4.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V0.16B
-	tbl	V5.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V1.16B
-	tbl	V6.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V2.16B
-	tbl	V7.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V3.16B
-	movi	V12.16B, #0x40
-	movi	V13.16B, #0x80
-	movi	V14.16B, #0xc0
+	bne	L_AES_XTS_decrypt_NEON_loop_nr_partial_1
 	eor	V8.16B, V0.16B, V12.16B
-	eor	V9.16B, V1.16B, V12.16B
-	eor	V10.16B, V2.16B, V12.16B
-	eor	V11.16B, V3.16B, V12.16B
+	eor	V9.16B, V0.16B, V13.16B
+	eor	V10.16B, V0.16B, V14.16B
+	tbl	V4.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V0.16B
 	tbl	V8.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V8.16B
-	tbl	V9.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V9.16B
-	tbl	V10.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V10.16B
-	tbl	V11.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V11.16B
-	orr	V4.16B, V4.16B, V8.16B
-	orr	V5.16B, V5.16B, V9.16B
-	orr	V6.16B, V6.16B, V10.16B
-	orr	V7.16B, V7.16B, V11.16B
-	eor	V8.16B, V0.16B, V13.16B
-	eor	V9.16B, V1.16B, V13.16B
-	eor	V10.16B, V2.16B, V13.16B
-	eor	V11.16B, V3.16B, V13.16B
-	tbl	V8.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V8.16B
 	tbl	V9.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V9.16B
-	tbl	V10.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V10.16B
-	tbl	V11.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V11.16B
-	orr	V4.16B, V4.16B, V8.16B
-	orr	V5.16B, V5.16B, V9.16B
-	orr	V6.16B, V6.16B, V10.16B
-	orr	V7.16B, V7.16B, V11.16B
-	eor	V8.16B, V0.16B, V14.16B
-	eor	V9.16B, V1.16B, V14.16B
-	eor	V10.16B, V2.16B, V14.16B
-	eor	V11.16B, V3.16B, V14.16B
-	tbl	V8.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V8.16B
-	tbl	V9.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V9.16B
 	tbl	V10.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V10.16B
-	tbl	V11.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V11.16B
 	orr	V4.16B, V4.16B, V8.16B
-	orr	V5.16B, V5.16B, V9.16B
-	orr	V6.16B, V6.16B, V10.16B
-	orr	V7.16B, V7.16B, V11.16B
-	ld1	{V0.16B}, [x23]
-	tbl	V4.16B, {V4.16B}, V0.16B
-	tbl	V5.16B, {V5.16B}, V0.16B
-	tbl	V6.16B, {V6.16B}, V0.16B
-	tbl	V7.16B, {V7.16B}, V0.16B
-	movi	V28.16B, #27
-	sshr	V8.16B, V4.16B, #7
-	sshr	V9.16B, V5.16B, #7
-	sshr	V10.16B, V6.16B, #7
-	sshr	V11.16B, V7.16B, #7
-	shl	V12.16B, V4.16B, #1
-	shl	V13.16B, V5.16B, #1
-	shl	V14.16B, V6.16B, #1
-	shl	V15.16B, V7.16B, #1
-	and	V8.16B, V8.16B, V28.16B
-	and	V9.16B, V9.16B, V28.16B
-	and	V10.16B, V10.16B, V28.16B
-	and	V11.16B, V11.16B, V28.16B
-	eor	V8.16B, V8.16B, V12.16B
-	eor	V9.16B, V9.16B, V13.16B
-	eor	V10.16B, V10.16B, V14.16B
-	eor	V11.16B, V11.16B, V15.16B
-	ushr	V12.16B, V4.16B, #6
-	ushr	V13.16B, V5.16B, #6
-	ushr	V14.16B, V6.16B, #6
-	ushr	V15.16B, V7.16B, #6
-	shl	V0.16B, V4.16B, #2
-	shl	V1.16B, V5.16B, #2
-	shl	V2.16B, V6.16B, #2
-	shl	V3.16B, V7.16B, #2
-	pmul	V12.16B, V12.16B, V28.16B
-	pmul	V13.16B, V13.16B, V28.16B
-	pmul	V14.16B, V14.16B, V28.16B
-	pmul	V15.16B, V15.16B, V28.16B
-	eor	V12.16B, V12.16B, V0.16B
-	eor	V13.16B, V13.16B, V1.16B
-	eor	V14.16B, V14.16B, V2.16B
-	eor	V15.16B, V15.16B, V3.16B
-	ushr	V0.16B, V4.16B, #5
-	ushr	V1.16B, V5.16B, #5
-	ushr	V2.16B, V6.16B, #5
-	ushr	V3.16B, V7.16B, #5
-	pmul	V0.16B, V0.16B, V28.16B
-	pmul	V1.16B, V1.16B, V28.16B
-	pmul	V2.16B, V2.16B, V28.16B
-	pmul	V3.16B, V3.16B, V28.16B
-	shl	V28.16B, V4.16B, #3
-	shl	V29.16B, V5.16B, #3
-	shl	V30.16B, V6.16B, #3
-	shl	V31.16B, V7.16B, #3
-	eor	V0.16B, V0.16B, V28.16B
-	eor	V1.16B, V1.16B, V29.16B
-	eor	V2.16B, V2.16B, V30.16B
-	eor	V3.16B, V3.16B, V31.16B
-	eor	V28.16B, V8.16B, V0.16B
-	eor	V29.16B, V9.16B, V1.16B
-	eor	V30.16B, V10.16B, V2.16B
-	eor	V31.16B, V11.16B, V3.16B
+	orr	V9.16B, V9.16B, V10.16B
+	orr	V4.16B, V4.16B, V9.16B
+	tbl	V4.16B, {V4.16B}, V3.16B
+	sshr	V10.16B, V4.16B, #7
+	ushr	V11.16B, V4.16B, #6
+	ushr	V8.16B, V4.16B, #5
+	and	V10.16B, V10.16B, V15.16B
+	pmul	V11.16B, V11.16B, V15.16B
+	pmul	V8.16B, V8.16B, V15.16B
+	shl	V9.16B, V4.16B, #1
+	eor	V10.16B, V10.16B, V9.16B
+	shl	V9.16B, V4.16B, #3
+	eor	V8.16B, V8.16B, V9.16B
+	shl	V9.16B, V4.16B, #2
+	eor	V11.16B, V11.16B, V9.16B
+	eor	V9.16B, V10.16B, V8.16B
+	eor	V8.16B, V8.16B, V4.16B
+	eor	V10.16B, V11.16B, V8.16B
+	eor	V11.16B, V11.16B, V9.16B
+	eor	V9.16B, V9.16B, V4.16B
+	shl	V4.4S, V9.4S, #8
+	rev32	V10.8H, V10.8H
+	sri	V4.4S, V9.4S, #24
+	eor	V4.16B, V4.16B, V11.16B
+	shl	V9.4S, V8.4S, #24
+	eor	V4.16B, V4.16B, V10.16B
+	sri	V9.4S, V8.4S, #8
+	eor	V4.16B, V4.16B, V9.16B
+	ld1	{V0.2D}, [x25], #16
+	;   XOR in Key Schedule
+	eor	V4.16B, V4.16B, V0.16B
+	eor	V8.16B, V4.16B, V12.16B
+	eor	V9.16B, V4.16B, V13.16B
+	eor	V10.16B, V4.16B, V14.16B
+	tbl	V0.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V4.16B
+	tbl	V8.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V8.16B
+	tbl	V9.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V9.16B
+	tbl	V10.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V10.16B
+	orr	V0.16B, V0.16B, V8.16B
+	orr	V9.16B, V9.16B, V10.16B
+	orr	V0.16B, V0.16B, V9.16B
+	tbl	V0.16B, {V0.16B}, V3.16B
+	ld1	{V4.2D}, [x25], #16
+	;   XOR in Key Schedule
 	eor	V0.16B, V0.16B, V4.16B
-	eor	V1.16B, V1.16B, V5.16B
-	eor	V2.16B, V2.16B, V6.16B
-	eor	V3.16B, V3.16B, V7.16B
-	eor	V8.16B, V12.16B, V0.16B
-	eor	V9.16B, V13.16B, V1.16B
-	eor	V10.16B, V14.16B, V2.16B
-	eor	V11.16B, V15.16B, V3.16B
-	eor	V12.16B, V12.16B, V28.16B
-	eor	V13.16B, V13.16B, V29.16B
-	eor	V14.16B, V14.16B, V30.16B
-	eor	V15.16B, V15.16B, V31.16B
-	eor	V28.16B, V28.16B, V4.16B
-	eor	V29.16B, V29.16B, V5.16B
-	eor	V30.16B, V30.16B, V6.16B
-	eor	V31.16B, V31.16B, V7.16B
-	shl	V4.4S, V28.4S, #8
-	shl	V5.4S, V29.4S, #8
-	shl	V6.4S, V30.4S, #8
-	shl	V7.4S, V31.4S, #8
-	rev32	V8.8H, V8.8H
-	rev32	V9.8H, V9.8H
+	rev32	V0.16B, V0.16B
+	eor	V0.16B, V0.16B, V1.16B
+	st1	{V0.2D}, [x6]
+	add	x1, x1, #16
+	mov	w16, w2
+L_AES_XTS_decrypt_NEON_start_byte
+	ldrb	w10, [x6]
+	ldrb	w11, [x0], #1
+	strb	w10, [x1], #1
+	strb	w11, [x6], #1
+	subs	w16, w16, #1
+	bgt	L_AES_XTS_decrypt_NEON_start_byte
+	sub	x1, x1, x2
+	sub	x6, x6, x2
+	sub	x1, x1, #16
+	mov	x25, x4
+	ld1	{V0.2D}, [x6]
+	ld1	{V4.2D}, [x25], #16
+	eor	V0.16B, V0.16B, V2.16B
+	rev32	V0.16B, V0.16B
+	eor	V0.16B, V0.16B, V4.16B
+	sub	w24, w7, #2
+L_AES_XTS_decrypt_NEON_loop_nr_partial_2
+	eor	V8.16B, V0.16B, V12.16B
+	eor	V9.16B, V0.16B, V13.16B
+	eor	V10.16B, V0.16B, V14.16B
+	tbl	V4.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V0.16B
+	tbl	V8.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V8.16B
+	tbl	V9.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V9.16B
+	tbl	V10.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V10.16B
+	orr	V4.16B, V4.16B, V8.16B
+	orr	V9.16B, V9.16B, V10.16B
+	orr	V4.16B, V4.16B, V9.16B
+	tbl	V4.16B, {V4.16B}, V3.16B
+	sshr	V10.16B, V4.16B, #7
+	ushr	V11.16B, V4.16B, #6
+	ushr	V8.16B, V4.16B, #5
+	and	V10.16B, V10.16B, V15.16B
+	pmul	V11.16B, V11.16B, V15.16B
+	pmul	V8.16B, V8.16B, V15.16B
+	shl	V9.16B, V4.16B, #1
+	eor	V10.16B, V10.16B, V9.16B
+	shl	V9.16B, V4.16B, #3
+	eor	V8.16B, V8.16B, V9.16B
+	shl	V9.16B, V4.16B, #2
+	eor	V11.16B, V11.16B, V9.16B
+	eor	V9.16B, V10.16B, V8.16B
+	eor	V8.16B, V8.16B, V4.16B
+	eor	V10.16B, V11.16B, V8.16B
+	eor	V11.16B, V11.16B, V9.16B
+	eor	V9.16B, V9.16B, V4.16B
+	shl	V4.4S, V9.4S, #8
+	rev32	V10.8H, V10.8H
+	sri	V4.4S, V9.4S, #24
+	eor	V4.16B, V4.16B, V11.16B
+	shl	V9.4S, V8.4S, #24
+	eor	V4.16B, V4.16B, V10.16B
+	sri	V9.4S, V8.4S, #8
+	eor	V4.16B, V4.16B, V9.16B
+	ld1	{V0.2D}, [x25], #16
+	;   XOR in Key Schedule
+	eor	V4.16B, V4.16B, V0.16B
+	eor	V8.16B, V4.16B, V12.16B
+	eor	V9.16B, V4.16B, V13.16B
+	eor	V10.16B, V4.16B, V14.16B
+	tbl	V0.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V4.16B
+	tbl	V8.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V8.16B
+	tbl	V9.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V9.16B
+	tbl	V10.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V10.16B
+	orr	V0.16B, V0.16B, V8.16B
+	orr	V9.16B, V9.16B, V10.16B
+	orr	V0.16B, V0.16B, V9.16B
+	tbl	V0.16B, {V0.16B}, V3.16B
+	sshr	V10.16B, V0.16B, #7
+	ushr	V11.16B, V0.16B, #6
+	ushr	V8.16B, V0.16B, #5
+	and	V10.16B, V10.16B, V15.16B
+	pmul	V11.16B, V11.16B, V15.16B
+	pmul	V8.16B, V8.16B, V15.16B
+	shl	V9.16B, V0.16B, #1
+	eor	V10.16B, V10.16B, V9.16B
+	shl	V9.16B, V0.16B, #3
+	eor	V8.16B, V8.16B, V9.16B
+	shl	V9.16B, V0.16B, #2
+	eor	V11.16B, V11.16B, V9.16B
+	eor	V9.16B, V10.16B, V8.16B
+	eor	V8.16B, V8.16B, V0.16B
+	eor	V10.16B, V11.16B, V8.16B
+	eor	V11.16B, V11.16B, V9.16B
+	eor	V9.16B, V9.16B, V0.16B
+	shl	V0.4S, V9.4S, #8
 	rev32	V10.8H, V10.8H
-	rev32	V11.8H, V11.8H
-	sri	V4.4S, V28.4S, #24
-	sri	V5.4S, V29.4S, #24
-	sri	V6.4S, V30.4S, #24
-	sri	V7.4S, V31.4S, #24
-	eor	V4.16B, V4.16B, V12.16B
-	eor	V5.16B, V5.16B, V13.16B
-	eor	V6.16B, V6.16B, V14.16B
-	eor	V7.16B, V7.16B, V15.16B
-	shl	V28.4S, V0.4S, #24
-	shl	V29.4S, V1.4S, #24
-	shl	V30.4S, V2.4S, #24
-	shl	V31.4S, V3.4S, #24
-	eor	V4.16B, V4.16B, V8.16B
-	eor	V5.16B, V5.16B, V9.16B
-	eor	V6.16B, V6.16B, V10.16B
-	eor	V7.16B, V7.16B, V11.16B
-	sri	V28.4S, V0.4S, #8
-	sri	V29.4S, V1.4S, #8
-	sri	V30.4S, V2.4S, #8
-	sri	V31.4S, V3.4S, #8
-	eor	V4.16B, V4.16B, V28.16B
-	eor	V5.16B, V5.16B, V29.16B
-	eor	V6.16B, V6.16B, V30.16B
-	eor	V7.16B, V7.16B, V31.16B
-	ld1	{V28.16B, V29.16B, V30.16B, V31.16B}, [x21]
+	sri	V0.4S, V9.4S, #24
+	eor	V0.16B, V0.16B, V11.16B
+	shl	V9.4S, V8.4S, #24
+	eor	V0.16B, V0.16B, V10.16B
+	sri	V9.4S, V8.4S, #8
+	eor	V0.16B, V0.16B, V9.16B
+	ld1	{V4.2D}, [x25], #16
 	;   XOR in Key Schedule
+	eor	V0.16B, V0.16B, V4.16B
+	subs	w24, w24, #2
+	bne	L_AES_XTS_decrypt_NEON_loop_nr_partial_2
+	eor	V8.16B, V0.16B, V12.16B
+	eor	V9.16B, V0.16B, V13.16B
+	eor	V10.16B, V0.16B, V14.16B
+	tbl	V4.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V0.16B
+	tbl	V8.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V8.16B
+	tbl	V9.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V9.16B
+	tbl	V10.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V10.16B
+	orr	V4.16B, V4.16B, V8.16B
+	orr	V9.16B, V9.16B, V10.16B
+	orr	V4.16B, V4.16B, V9.16B
+	tbl	V4.16B, {V4.16B}, V3.16B
+	sshr	V10.16B, V4.16B, #7
+	ushr	V11.16B, V4.16B, #6
+	ushr	V8.16B, V4.16B, #5
+	and	V10.16B, V10.16B, V15.16B
+	pmul	V11.16B, V11.16B, V15.16B
+	pmul	V8.16B, V8.16B, V15.16B
+	shl	V9.16B, V4.16B, #1
+	eor	V10.16B, V10.16B, V9.16B
+	shl	V9.16B, V4.16B, #3
+	eor	V8.16B, V8.16B, V9.16B
+	shl	V9.16B, V4.16B, #2
+	eor	V11.16B, V11.16B, V9.16B
+	eor	V9.16B, V10.16B, V8.16B
+	eor	V8.16B, V8.16B, V4.16B
+	eor	V10.16B, V11.16B, V8.16B
+	eor	V11.16B, V11.16B, V9.16B
+	eor	V9.16B, V9.16B, V4.16B
+	shl	V4.4S, V9.4S, #8
+	rev32	V10.8H, V10.8H
+	sri	V4.4S, V9.4S, #24
+	eor	V4.16B, V4.16B, V11.16B
+	shl	V9.4S, V8.4S, #24
+	eor	V4.16B, V4.16B, V10.16B
+	sri	V9.4S, V8.4S, #8
+	eor	V4.16B, V4.16B, V9.16B
 	ld1	{V0.2D}, [x25], #16
+	;   XOR in Key Schedule
 	eor	V4.16B, V4.16B, V0.16B
-	eor	V5.16B, V5.16B, V0.16B
-	eor	V6.16B, V6.16B, V0.16B
-	eor	V7.16B, V7.16B, V0.16B
-	; Round Done
-	tbl	V0.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V4.16B
-	tbl	V1.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V5.16B
-	tbl	V2.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V6.16B
-	tbl	V3.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V7.16B
-	movi	V12.16B, #0x40
-	movi	V13.16B, #0x80
-	movi	V14.16B, #0xc0
 	eor	V8.16B, V4.16B, V12.16B
-	eor	V9.16B, V5.16B, V12.16B
-	eor	V10.16B, V6.16B, V12.16B
-	eor	V11.16B, V7.16B, V12.16B
+	eor	V9.16B, V4.16B, V13.16B
+	eor	V10.16B, V4.16B, V14.16B
+	tbl	V0.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V4.16B
 	tbl	V8.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V8.16B
-	tbl	V9.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V9.16B
-	tbl	V10.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V10.16B
-	tbl	V11.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V11.16B
-	orr	V0.16B, V0.16B, V8.16B
-	orr	V1.16B, V1.16B, V9.16B
-	orr	V2.16B, V2.16B, V10.16B
-	orr	V3.16B, V3.16B, V11.16B
-	eor	V8.16B, V4.16B, V13.16B
-	eor	V9.16B, V5.16B, V13.16B
-	eor	V10.16B, V6.16B, V13.16B
-	eor	V11.16B, V7.16B, V13.16B
-	tbl	V8.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V8.16B
 	tbl	V9.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V9.16B
-	tbl	V10.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V10.16B
-	tbl	V11.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V11.16B
-	orr	V0.16B, V0.16B, V8.16B
-	orr	V1.16B, V1.16B, V9.16B
-	orr	V2.16B, V2.16B, V10.16B
-	orr	V3.16B, V3.16B, V11.16B
-	eor	V8.16B, V4.16B, V14.16B
-	eor	V9.16B, V5.16B, V14.16B
-	eor	V10.16B, V6.16B, V14.16B
-	eor	V11.16B, V7.16B, V14.16B
-	tbl	V8.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V8.16B
-	tbl	V9.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V9.16B
 	tbl	V10.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V10.16B
-	tbl	V11.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V11.16B
 	orr	V0.16B, V0.16B, V8.16B
-	orr	V1.16B, V1.16B, V9.16B
-	orr	V2.16B, V2.16B, V10.16B
-	orr	V3.16B, V3.16B, V11.16B
-	ld1	{V4.16B}, [x23]
-	tbl	V0.16B, {V0.16B}, V4.16B
-	tbl	V1.16B, {V1.16B}, V4.16B
-	tbl	V2.16B, {V2.16B}, V4.16B
-	tbl	V3.16B, {V3.16B}, V4.16B
-	;   XOR in Key Schedule
+	orr	V9.16B, V9.16B, V10.16B
+	orr	V0.16B, V0.16B, V9.16B
+	tbl	V0.16B, {V0.16B}, V3.16B
 	ld1	{V4.2D}, [x25], #16
+	;   XOR in Key Schedule
 	eor	V0.16B, V0.16B, V4.16B
-	eor	V1.16B, V1.16B, V4.16B
-	eor	V2.16B, V2.16B, V4.16B
-	eor	V3.16B, V3.16B, V4.16B
-	; Round Done
 	rev32	V0.16B, V0.16B
+	eor	V0.16B, V0.16B, V2.16B
+	st1	{V0.16B}, [x1]
+L_AES_XTS_decrypt_NEON_data_done
+	ldp	x17, x19, [x29, #16]
+	ldp	x20, x21, [x29, #32]
+	ldp	x22, x23, [x29, #48]
+	ldp	x24, x25, [x29, #64]
+	ldp	D8, D9, [x29, #80]
+	ldp	D10, D11, [x29, #96]
+	ldp	D12, D13, [x29, #112]
+	ldp	D14, D15, [x29, #128]
+	ldp	x29, x30, [sp], #0x90
+	ret
+	ENDP
+	ENDIF
+	ENDIF
+	IF :DEF:WOLFSSL_AESGCM_SIV
+	AREA	|.text|, CODE, READONLY
+	ALIGN	4
+	EXPORT	AES_GCMSIV_polyval_neon
+AES_GCMSIV_polyval_neon PROC
+	stp	x29, x30, [sp, #-80]!
+	add	x29, sp, #0
+	stp	D8, D9, [x29, #16]
+	stp	D10, D11, [x29, #32]
+	stp	D12, D13, [x29, #48]
+	stp	D14, D15, [x29, #64]
+	ld1	{V18.2D}, [x0]
+	ld1	{V10.2D}, [x1]
+	movi	V19.16B, #15
+	eor	V20.16B, V20.16B, V20.16B
+	rev64	V10.16B, V10.16B
+	ext8	V10.16B, V10.16B, V10.16B, #8
+	rbit	V10.16B, V10.16B
+	rbit	V18.16B, V18.16B
+	and	V12.16B, V10.16B, V19.16B
+	ushr	V13.16B, V10.16B, #4
+	eor	V14.16B, V12.16B, V13.16B
+	cbz	w3, L_AES_GCMSIV_polyval_neon_done
+L_AES_GCMSIV_polyval_neon_loop
+	ld1	{V0.16B}, [x2], #16
+	rev64	V0.16B, V0.16B
+	ext8	V0.16B, V0.16B, V0.16B, #8
+	rbit	V0.16B, V0.16B
+	eor	V18.16B, V18.16B, V0.16B
+	and	V15.16B, V18.16B, V19.16B
+	ushr	V16.16B, V18.16B, #4
+	eor	V17.16B, V15.16B, V16.16B
+	dup	V0.16B, V12.B[0]
+	dup	V2.16B, V14.B[0]
+	dup	V1.16B, V13.B[0]
+	pmul	V8.16B, V15.16B, V0.16B
+	pmul	V5.16B, V17.16B, V2.16B
+	pmul	V4.16B, V16.16B, V1.16B
+	eor	V5.16B, V5.16B, V8.16B
+	eor	V5.16B, V5.16B, V4.16B
+	shl	V6.16B, V5.16B, #4
+	ushr	V7.16B, V5.16B, #4
+	eor	V8.16B, V8.16B, V6.16B
+	eor	V11.16B, V4.16B, V7.16B
+	dup	V0.16B, V12.B[1]
+	dup	V2.16B, V14.B[1]
+	dup	V1.16B, V13.B[1]
+	pmul	V3.16B, V15.16B, V0.16B
+	pmul	V5.16B, V17.16B, V2.16B
+	pmul	V4.16B, V16.16B, V1.16B
+	eor	V5.16B, V5.16B, V3.16B
+	eor	V5.16B, V5.16B, V4.16B
+	eor	V3.16B, V3.16B, V11.16B
+	shl	V6.16B, V5.16B, #4
+	ushr	V7.16B, V5.16B, #4
+	eor	V3.16B, V3.16B, V6.16B
+	eor	V11.16B, V4.16B, V7.16B
+	ext8	V6.16B, V20.16B, V3.16B, #15
+	ext8	V9.16B, V3.16B, V20.16B, #15
+	eor	V8.16B, V8.16B, V6.16B
+	dup	V0.16B, V12.B[2]
+	dup	V2.16B, V14.B[2]
+	dup	V1.16B, V13.B[2]
+	pmul	V3.16B, V15.16B, V0.16B
+	pmul	V5.16B, V17.16B, V2.16B
+	pmul	V4.16B, V16.16B, V1.16B
+	eor	V5.16B, V5.16B, V3.16B
+	eor	V5.16B, V5.16B, V4.16B
+	eor	V3.16B, V3.16B, V11.16B
+	shl	V6.16B, V5.16B, #4
+	ushr	V7.16B, V5.16B, #4
+	eor	V3.16B, V3.16B, V6.16B
+	eor	V11.16B, V4.16B, V7.16B
+	ext8	V7.16B, V3.16B, V20.16B, #14
+	ext8	V6.16B, V20.16B, V3.16B, #14
+	eor	V9.16B, V9.16B, V7.16B
+	eor	V8.16B, V8.16B, V6.16B
+	dup	V0.16B, V12.B[3]
+	dup	V2.16B, V14.B[3]
+	dup	V1.16B, V13.B[3]
+	pmul	V3.16B, V15.16B, V0.16B
+	pmul	V5.16B, V17.16B, V2.16B
+	pmul	V4.16B, V16.16B, V1.16B
+	eor	V5.16B, V5.16B, V3.16B
+	eor	V5.16B, V5.16B, V4.16B
+	eor	V3.16B, V3.16B, V11.16B
+	shl	V6.16B, V5.16B, #4
+	ushr	V7.16B, V5.16B, #4
+	eor	V3.16B, V3.16B, V6.16B
+	eor	V11.16B, V4.16B, V7.16B
+	ext8	V7.16B, V3.16B, V20.16B, #13
+	ext8	V6.16B, V20.16B, V3.16B, #13
+	eor	V9.16B, V9.16B, V7.16B
+	eor	V8.16B, V8.16B, V6.16B
+	dup	V0.16B, V12.B[4]
+	dup	V2.16B, V14.B[4]
+	dup	V1.16B, V13.B[4]
+	pmul	V3.16B, V15.16B, V0.16B
+	pmul	V5.16B, V17.16B, V2.16B
+	pmul	V4.16B, V16.16B, V1.16B
+	eor	V5.16B, V5.16B, V3.16B
+	eor	V5.16B, V5.16B, V4.16B
+	eor	V3.16B, V3.16B, V11.16B
+	shl	V6.16B, V5.16B, #4
+	ushr	V7.16B, V5.16B, #4
+	eor	V3.16B, V3.16B, V6.16B
+	eor	V11.16B, V4.16B, V7.16B
+	ext8	V7.16B, V3.16B, V20.16B, #12
+	ext8	V6.16B, V20.16B, V3.16B, #12
+	eor	V9.16B, V9.16B, V7.16B
+	eor	V8.16B, V8.16B, V6.16B
+	dup	V0.16B, V12.B[5]
+	dup	V2.16B, V14.B[5]
+	dup	V1.16B, V13.B[5]
+	pmul	V3.16B, V15.16B, V0.16B
+	pmul	V5.16B, V17.16B, V2.16B
+	pmul	V4.16B, V16.16B, V1.16B
+	eor	V5.16B, V5.16B, V3.16B
+	eor	V5.16B, V5.16B, V4.16B
+	eor	V3.16B, V3.16B, V11.16B
+	shl	V6.16B, V5.16B, #4
+	ushr	V7.16B, V5.16B, #4
+	eor	V3.16B, V3.16B, V6.16B
+	eor	V11.16B, V4.16B, V7.16B
+	ext8	V7.16B, V3.16B, V20.16B, #11
+	ext8	V6.16B, V20.16B, V3.16B, #11
+	eor	V9.16B, V9.16B, V7.16B
+	eor	V8.16B, V8.16B, V6.16B
+	dup	V0.16B, V12.B[6]
+	dup	V2.16B, V14.B[6]
+	dup	V1.16B, V13.B[6]
+	pmul	V3.16B, V15.16B, V0.16B
+	pmul	V5.16B, V17.16B, V2.16B
+	pmul	V4.16B, V16.16B, V1.16B
+	eor	V5.16B, V5.16B, V3.16B
+	eor	V5.16B, V5.16B, V4.16B
+	eor	V3.16B, V3.16B, V11.16B
+	shl	V6.16B, V5.16B, #4
+	ushr	V7.16B, V5.16B, #4
+	eor	V3.16B, V3.16B, V6.16B
+	eor	V11.16B, V4.16B, V7.16B
+	ext8	V7.16B, V3.16B, V20.16B, #10
+	ext8	V6.16B, V20.16B, V3.16B, #10
+	eor	V9.16B, V9.16B, V7.16B
+	eor	V8.16B, V8.16B, V6.16B
+	dup	V0.16B, V12.B[7]
+	dup	V2.16B, V14.B[7]
+	dup	V1.16B, V13.B[7]
+	pmul	V3.16B, V15.16B, V0.16B
+	pmul	V5.16B, V17.16B, V2.16B
+	pmul	V4.16B, V16.16B, V1.16B
+	eor	V5.16B, V5.16B, V3.16B
+	eor	V5.16B, V5.16B, V4.16B
+	eor	V3.16B, V3.16B, V11.16B
+	shl	V6.16B, V5.16B, #4
+	ushr	V7.16B, V5.16B, #4
+	eor	V3.16B, V3.16B, V6.16B
+	eor	V11.16B, V4.16B, V7.16B
+	ext8	V7.16B, V3.16B, V20.16B, #9
+	ext8	V6.16B, V20.16B, V3.16B, #9
+	eor	V9.16B, V9.16B, V7.16B
+	eor	V8.16B, V8.16B, V6.16B
+	dup	V0.16B, V12.B[8]
+	dup	V2.16B, V14.B[8]
+	dup	V1.16B, V13.B[8]
+	pmul	V3.16B, V15.16B, V0.16B
+	pmul	V5.16B, V17.16B, V2.16B
+	pmul	V4.16B, V16.16B, V1.16B
+	eor	V5.16B, V5.16B, V3.16B
+	eor	V5.16B, V5.16B, V4.16B
+	eor	V3.16B, V3.16B, V11.16B
+	shl	V6.16B, V5.16B, #4
+	ushr	V7.16B, V5.16B, #4
+	eor	V3.16B, V3.16B, V6.16B
+	eor	V11.16B, V4.16B, V7.16B
+	ext8	V7.16B, V3.16B, V20.16B, #8
+	ext8	V6.16B, V20.16B, V3.16B, #8
+	eor	V9.16B, V9.16B, V7.16B
+	eor	V8.16B, V8.16B, V6.16B
+	dup	V0.16B, V12.B[9]
+	dup	V2.16B, V14.B[9]
+	dup	V1.16B, V13.B[9]
+	pmul	V3.16B, V15.16B, V0.16B
+	pmul	V5.16B, V17.16B, V2.16B
+	pmul	V4.16B, V16.16B, V1.16B
+	eor	V5.16B, V5.16B, V3.16B
+	eor	V5.16B, V5.16B, V4.16B
+	eor	V3.16B, V3.16B, V11.16B
+	shl	V6.16B, V5.16B, #4
+	ushr	V7.16B, V5.16B, #4
+	eor	V3.16B, V3.16B, V6.16B
+	eor	V11.16B, V4.16B, V7.16B
+	ext8	V7.16B, V3.16B, V20.16B, #7
+	ext8	V6.16B, V20.16B, V3.16B, #7
+	eor	V9.16B, V9.16B, V7.16B
+	eor	V8.16B, V8.16B, V6.16B
+	dup	V0.16B, V12.B[10]
+	dup	V2.16B, V14.B[10]
+	dup	V1.16B, V13.B[10]
+	pmul	V3.16B, V15.16B, V0.16B
+	pmul	V5.16B, V17.16B, V2.16B
+	pmul	V4.16B, V16.16B, V1.16B
+	eor	V5.16B, V5.16B, V3.16B
+	eor	V5.16B, V5.16B, V4.16B
+	eor	V3.16B, V3.16B, V11.16B
+	shl	V6.16B, V5.16B, #4
+	ushr	V7.16B, V5.16B, #4
+	eor	V3.16B, V3.16B, V6.16B
+	eor	V11.16B, V4.16B, V7.16B
+	ext8	V7.16B, V3.16B, V20.16B, #6
+	ext8	V6.16B, V20.16B, V3.16B, #6
+	eor	V9.16B, V9.16B, V7.16B
+	eor	V8.16B, V8.16B, V6.16B
+	dup	V0.16B, V12.B[11]
+	dup	V2.16B, V14.B[11]
+	dup	V1.16B, V13.B[11]
+	pmul	V3.16B, V15.16B, V0.16B
+	pmul	V5.16B, V17.16B, V2.16B
+	pmul	V4.16B, V16.16B, V1.16B
+	eor	V5.16B, V5.16B, V3.16B
+	eor	V5.16B, V5.16B, V4.16B
+	eor	V3.16B, V3.16B, V11.16B
+	shl	V6.16B, V5.16B, #4
+	ushr	V7.16B, V5.16B, #4
+	eor	V3.16B, V3.16B, V6.16B
+	eor	V11.16B, V4.16B, V7.16B
+	ext8	V7.16B, V3.16B, V20.16B, #5
+	ext8	V6.16B, V20.16B, V3.16B, #5
+	eor	V9.16B, V9.16B, V7.16B
+	eor	V8.16B, V8.16B, V6.16B
+	dup	V0.16B, V12.B[12]
+	dup	V2.16B, V14.B[12]
+	dup	V1.16B, V13.B[12]
+	pmul	V3.16B, V15.16B, V0.16B
+	pmul	V5.16B, V17.16B, V2.16B
+	pmul	V4.16B, V16.16B, V1.16B
+	eor	V5.16B, V5.16B, V3.16B
+	eor	V5.16B, V5.16B, V4.16B
+	eor	V3.16B, V3.16B, V11.16B
+	shl	V6.16B, V5.16B, #4
+	ushr	V7.16B, V5.16B, #4
+	eor	V3.16B, V3.16B, V6.16B
+	eor	V11.16B, V4.16B, V7.16B
+	ext8	V7.16B, V3.16B, V20.16B, #4
+	ext8	V6.16B, V20.16B, V3.16B, #4
+	eor	V9.16B, V9.16B, V7.16B
+	eor	V8.16B, V8.16B, V6.16B
+	dup	V0.16B, V12.B[13]
+	dup	V2.16B, V14.B[13]
+	dup	V1.16B, V13.B[13]
+	pmul	V3.16B, V15.16B, V0.16B
+	pmul	V5.16B, V17.16B, V2.16B
+	pmul	V4.16B, V16.16B, V1.16B
+	eor	V5.16B, V5.16B, V3.16B
+	eor	V5.16B, V5.16B, V4.16B
+	eor	V3.16B, V3.16B, V11.16B
+	shl	V6.16B, V5.16B, #4
+	ushr	V7.16B, V5.16B, #4
+	eor	V3.16B, V3.16B, V6.16B
+	eor	V11.16B, V4.16B, V7.16B
+	ext8	V7.16B, V3.16B, V20.16B, #3
+	ext8	V6.16B, V20.16B, V3.16B, #3
+	eor	V9.16B, V9.16B, V7.16B
+	eor	V8.16B, V8.16B, V6.16B
+	dup	V0.16B, V12.B[14]
+	dup	V2.16B, V14.B[14]
+	dup	V1.16B, V13.B[14]
+	pmul	V3.16B, V15.16B, V0.16B
+	pmul	V5.16B, V17.16B, V2.16B
+	pmul	V4.16B, V16.16B, V1.16B
+	eor	V5.16B, V5.16B, V3.16B
+	eor	V5.16B, V5.16B, V4.16B
+	eor	V3.16B, V3.16B, V11.16B
+	shl	V6.16B, V5.16B, #4
+	ushr	V7.16B, V5.16B, #4
+	eor	V3.16B, V3.16B, V6.16B
+	eor	V11.16B, V4.16B, V7.16B
+	ext8	V7.16B, V3.16B, V20.16B, #2
+	ext8	V6.16B, V20.16B, V3.16B, #2
+	eor	V9.16B, V9.16B, V7.16B
+	eor	V8.16B, V8.16B, V6.16B
+	dup	V0.16B, V12.B[15]
+	dup	V2.16B, V14.B[15]
+	dup	V1.16B, V13.B[15]
+	pmul	V3.16B, V15.16B, V0.16B
+	pmul	V5.16B, V17.16B, V2.16B
+	pmul	V4.16B, V16.16B, V1.16B
+	eor	V5.16B, V5.16B, V3.16B
+	eor	V5.16B, V5.16B, V4.16B
+	eor	V3.16B, V3.16B, V11.16B
+	shl	V6.16B, V5.16B, #4
+	ushr	V7.16B, V5.16B, #4
+	eor	V3.16B, V3.16B, V6.16B
+	eor	V11.16B, V4.16B, V7.16B
+	ext8	V7.16B, V3.16B, V20.16B, #1
+	ext8	V6.16B, V20.16B, V3.16B, #1
+	eor	V9.16B, V9.16B, V7.16B
+	eor	V8.16B, V8.16B, V6.16B
+	eor	V9.16B, V9.16B, V11.16B
+	shl	V0.16B, V9.16B, #1
+	shl	V1.16B, V9.16B, #2
+	shl	V2.16B, V9.16B, #7
+	ushr	V3.16B, V9.16B, #7
+	ushr	V4.16B, V9.16B, #6
+	ushr	V5.16B, V9.16B, #1
+	eor	V0.16B, V0.16B, V9.16B
+	eor	V1.16B, V1.16B, V2.16B
+	eor	V0.16B, V0.16B, V1.16B
+	eor	V8.16B, V8.16B, V0.16B
+	ext8	V0.16B, V20.16B, V3.16B, #15
+	ext8	V1.16B, V20.16B, V4.16B, #15
+	ext8	V2.16B, V20.16B, V5.16B, #15
+	ext8	V4.16B, V4.16B, V20.16B, #15
+	ext8	V5.16B, V5.16B, V20.16B, #15
+	eor	V0.16B, V0.16B, V1.16B
+	eor	V8.16B, V8.16B, V2.16B
+	eor	V8.16B, V8.16B, V0.16B
+	eor	V3.16B, V4.16B, V5.16B
+	shl	V0.2D, V3.2D, #1
+	shl	V1.2D, V3.2D, #2
+	shl	V2.2D, V3.2D, #7
+	eor	V3.16B, V3.16B, V0.16B
+	eor	V1.16B, V1.16B, V2.16B
+	eor	V8.16B, V8.16B, V3.16B
+	eor	V18.16B, V8.16B, V1.16B
+	subs	w3, w3, #1
+	bne	L_AES_GCMSIV_polyval_neon_loop
+L_AES_GCMSIV_polyval_neon_done
+	rbit	V18.16B, V18.16B
+	st1	{V18.2D}, [x0]
+	ldp	D8, D9, [x29, #16]
+	ldp	D10, D11, [x29, #32]
+	ldp	D12, D13, [x29, #48]
+	ldp	D14, D15, [x29, #64]
+	ldp	x29, x30, [sp], #0x50
+	ret
+	ENDP
+	AREA	|.rodata|, DATA, READONLY, ALIGN=4
+	ALIGN	8
+L_AES_GCMSIV_ctr_neon_te
+	DCB	0x63, 0x7c, 0x77, 0x7b, 0xf2, 0x6b, 0x6f, 0xc5
+	DCB	0x30, 0x01, 0x67, 0x2b, 0xfe, 0xd7, 0xab, 0x76
+	DCB	0xca, 0x82, 0xc9, 0x7d, 0xfa, 0x59, 0x47, 0xf0
+	DCB	0xad, 0xd4, 0xa2, 0xaf, 0x9c, 0xa4, 0x72, 0xc0
+	DCB	0xb7, 0xfd, 0x93, 0x26, 0x36, 0x3f, 0xf7, 0xcc
+	DCB	0x34, 0xa5, 0xe5, 0xf1, 0x71, 0xd8, 0x31, 0x15
+	DCB	0x04, 0xc7, 0x23, 0xc3, 0x18, 0x96, 0x05, 0x9a
+	DCB	0x07, 0x12, 0x80, 0xe2, 0xeb, 0x27, 0xb2, 0x75
+	DCB	0x09, 0x83, 0x2c, 0x1a, 0x1b, 0x6e, 0x5a, 0xa0
+	DCB	0x52, 0x3b, 0xd6, 0xb3, 0x29, 0xe3, 0x2f, 0x84
+	DCB	0x53, 0xd1, 0x00, 0xed, 0x20, 0xfc, 0xb1, 0x5b
+	DCB	0x6a, 0xcb, 0xbe, 0x39, 0x4a, 0x4c, 0x58, 0xcf
+	DCB	0xd0, 0xef, 0xaa, 0xfb, 0x43, 0x4d, 0x33, 0x85
+	DCB	0x45, 0xf9, 0x02, 0x7f, 0x50, 0x3c, 0x9f, 0xa8
+	DCB	0x51, 0xa3, 0x40, 0x8f, 0x92, 0x9d, 0x38, 0xf5
+	DCB	0xbc, 0xb6, 0xda, 0x21, 0x10, 0xff, 0xf3, 0xd2
+	DCB	0xcd, 0x0c, 0x13, 0xec, 0x5f, 0x97, 0x44, 0x17
+	DCB	0xc4, 0xa7, 0x7e, 0x3d, 0x64, 0x5d, 0x19, 0x73
+	DCB	0x60, 0x81, 0x4f, 0xdc, 0x22, 0x2a, 0x90, 0x88
+	DCB	0x46, 0xee, 0xb8, 0x14, 0xde, 0x5e, 0x0b, 0xdb
+	DCB	0xe0, 0x32, 0x3a, 0x0a, 0x49, 0x06, 0x24, 0x5c
+	DCB	0xc2, 0xd3, 0xac, 0x62, 0x91, 0x95, 0xe4, 0x79
+	DCB	0xe7, 0xc8, 0x37, 0x6d, 0x8d, 0xd5, 0x4e, 0xa9
+	DCB	0x6c, 0x56, 0xf4, 0xea, 0x65, 0x7a, 0xae, 0x08
+	DCB	0xba, 0x78, 0x25, 0x2e, 0x1c, 0xa6, 0xb4, 0xc6
+	DCB	0xe8, 0xdd, 0x74, 0x1f, 0x4b, 0xbd, 0x8b, 0x8a
+	DCB	0x70, 0x3e, 0xb5, 0x66, 0x48, 0x03, 0xf6, 0x0e
+	DCB	0x61, 0x35, 0x57, 0xb9, 0x86, 0xc1, 0x1d, 0x9e
+	DCB	0xe1, 0xf8, 0x98, 0x11, 0x69, 0xd9, 0x8e, 0x94
+	DCB	0x9b, 0x1e, 0x87, 0xe9, 0xce, 0x55, 0x28, 0xdf
+	DCB	0x8c, 0xa1, 0x89, 0x0d, 0xbf, 0xe6, 0x42, 0x68
+	DCB	0x41, 0x99, 0x2d, 0x0f, 0xb0, 0x54, 0xbb, 0x16
+	AREA	|.rodata|, DATA, READONLY, ALIGN=4
+	ALIGN	8
+L_AES_GCMSIV_ctr_neon_shuffle
+	DCB	0x0c, 0x09, 0x06, 0x03, 0x00, 0x0d, 0x0a, 0x07
+	DCB	0x04, 0x01, 0x0e, 0x0b, 0x08, 0x05, 0x02, 0x0f
+	AREA	|.text|, CODE, READONLY
+	ALIGN	4
+	EXPORT	AES_GCMSIV_ctr_neon
+AES_GCMSIV_ctr_neon PROC
+	stp	x29, x30, [sp, #-80]!
+	add	x29, sp, #0
+	stp	D8, D9, [x29, #16]
+	stp	D10, D11, [x29, #32]
+	stp	D12, D13, [x29, #48]
+	stp	D14, D15, [x29, #64]
+	adrp	x6, L_AES_GCMSIV_ctr_neon_te
+	add	x6, x6, L_AES_GCMSIV_ctr_neon_te
+	adrp	x7, L_AES_GCMSIV_ctr_neon_shuffle
+	add	x7, x7, L_AES_GCMSIV_ctr_neon_shuffle
+	ld1	{V16.16B, V17.16B, V18.16B, V19.16B}, [x6], #0x40
+	ld1	{V20.16B, V21.16B, V22.16B, V23.16B}, [x6], #0x40
+	ld1	{V24.16B, V25.16B, V26.16B, V27.16B}, [x6], #0x40
+	ld1	{V28.16B, V29.16B, V30.16B, V31.16B}, [x6]
+	ldr	w10, [x5]
+	cmp	x2, #0x40
+	blt	L_AES_GCMSIV_ctr_neon_start_2
+L_AES_GCMSIV_ctr_neon_loop_4
+	mov	x9, x3
+	ld1	{V4.2D}, [x9], #16
+	; Round: 0 - build counters and XOR in key schedule
+	ld1	{V0.2D}, [x5]
+	mov	V0.S[0], w10
+	rev32	V0.16B, V0.16B
+	eor	V0.16B, V0.16B, V4.16B
+	ld1	{V1.2D}, [x5]
+	add	w8, w10, #1
+	mov	V1.S[0], w8
 	rev32	V1.16B, V1.16B
+	eor	V1.16B, V1.16B, V4.16B
+	ld1	{V2.2D}, [x5]
+	add	w8, w10, #2
+	mov	V2.S[0], w8
 	rev32	V2.16B, V2.16B
+	eor	V2.16B, V2.16B, V4.16B
+	ld1	{V3.2D}, [x5]
+	add	w8, w10, #3
+	mov	V3.S[0], w8
 	rev32	V3.16B, V3.16B
-	mov	V8.D[0], x8
-	mov	V8.D[1], x9
-	mov	V9.D[0], x10
-	mov	V9.D[1], x11
-	mov	V10.D[0], x12
-	mov	V10.D[1], x13
-	mov	V11.D[0], x14
-	mov	V11.D[1], x15
-	eor	V0.16B, V0.16B, V8.16B
-	eor	V1.16B, V1.16B, V9.16B
-	eor	V2.16B, V2.16B, V10.16B
-	eor	V3.16B, V3.16B, V11.16B
-	st1	{V0.16B, V1.16B, V2.16B, V3.16B}, [x1], #0x40
-	and	x16, x17, x15, asr 63
-	extr	x9, x15, x14, #63
-	eor	x8, x16, x14, lsl 1
-	sub	w2, w2, #0x40
-	cmp	w2, #0x40
-	bge	L_AES_XTS_decrypt_NEON_loop_4
-	movi	V12.16B, #0x40
-	movi	V13.16B, #0x80
-	movi	V14.16B, #0xc0
-	movi	V15.16B, #27
-L_AES_XTS_decrypt_NEON_start_2
-	cmp	w2, #32
-	blt	L_AES_XTS_decrypt_NEON_start_1
-	mov	x25, x4
-	ld1	{V0.16B, V1.16B}, [x0], #32
-	ld1	{V4.16B}, [x25], #16
-	and	x16, x17, x9, asr 63
-	extr	x11, x9, x8, #63
-	eor	x10, x16, x8, lsl 1
-	and	x16, x17, x11, asr 63
-	extr	x13, x11, x10, #63
-	eor	x12, x16, x10, lsl 1
-	mov	V2.D[0], x8
-	mov	V2.D[1], x9
-	mov	V3.D[0], x10
-	mov	V3.D[1], x11
-	eor	V0.16B, V0.16B, V2.16B
-	eor	V1.16B, V1.16B, V3.16B
-	rev32	V0.16B, V0.16B
-	rev32	V1.16B, V1.16B
-	eor	V0.16B, V0.16B, V4.16B
-	eor	V1.16B, V1.16B, V4.16B
-	sub	w24, w7, #2
-L_AES_XTS_decrypt_NEON_loop_nr_2
+	eor	V3.16B, V3.16B, V4.16B
+	add	w10, w10, #4
+	sub	w8, w4, #2
+L_AES_GCMSIV_ctr_neon_loop_nr_4
+	tbl	V4.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V0.16B
+	tbl	V5.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V1.16B
+	tbl	V6.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V2.16B
+	tbl	V7.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V3.16B
 	movi	V12.16B, #0x40
 	movi	V13.16B, #0x80
 	movi	V14.16B, #0xc0
 	eor	V8.16B, V0.16B, V12.16B
 	eor	V9.16B, V1.16B, V12.16B
-	tbl	V4.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V0.16B
-	tbl	V5.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V1.16B
+	eor	V10.16B, V2.16B, V12.16B
+	eor	V11.16B, V3.16B, V12.16B
 	tbl	V8.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V8.16B
 	tbl	V9.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V9.16B
-	eor	V10.16B, V0.16B, V13.16B
-	eor	V11.16B, V1.16B, V13.16B
+	tbl	V10.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V10.16B
+	tbl	V11.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V11.16B
+	orr	V4.16B, V4.16B, V8.16B
+	orr	V5.16B, V5.16B, V9.16B
+	orr	V6.16B, V6.16B, V10.16B
+	orr	V7.16B, V7.16B, V11.16B
+	eor	V8.16B, V0.16B, V13.16B
+	eor	V9.16B, V1.16B, V13.16B
+	eor	V10.16B, V2.16B, V13.16B
+	eor	V11.16B, V3.16B, V13.16B
+	tbl	V8.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V8.16B
+	tbl	V9.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V9.16B
 	tbl	V10.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V10.16B
 	tbl	V11.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V11.16B
 	orr	V4.16B, V4.16B, V8.16B
 	orr	V5.16B, V5.16B, V9.16B
+	orr	V6.16B, V6.16B, V10.16B
+	orr	V7.16B, V7.16B, V11.16B
 	eor	V8.16B, V0.16B, V14.16B
 	eor	V9.16B, V1.16B, V14.16B
-	orr	V4.16B, V4.16B, V10.16B
-	orr	V5.16B, V5.16B, V11.16B
+	eor	V10.16B, V2.16B, V14.16B
+	eor	V11.16B, V3.16B, V14.16B
 	tbl	V8.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V8.16B
 	tbl	V9.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V9.16B
+	tbl	V10.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V10.16B
+	tbl	V11.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V11.16B
 	orr	V4.16B, V4.16B, V8.16B
 	orr	V5.16B, V5.16B, V9.16B
-	ld1	{V0.16B}, [x23]
+	orr	V6.16B, V6.16B, V10.16B
+	orr	V7.16B, V7.16B, V11.16B
+	ld1	{V0.16B}, [x7]
 	tbl	V4.16B, {V4.16B}, V0.16B
 	tbl	V5.16B, {V5.16B}, V0.16B
-	movi	V10.16B, #27
+	tbl	V6.16B, {V6.16B}, V0.16B
+	tbl	V7.16B, {V7.16B}, V0.16B
 	sshr	V8.16B, V4.16B, #7
 	sshr	V9.16B, V5.16B, #7
+	sshr	V10.16B, V6.16B, #7
+	sshr	V11.16B, V7.16B, #7
 	shl	V12.16B, V4.16B, #1
 	shl	V13.16B, V5.16B, #1
-	and	V8.16B, V8.16B, V10.16B
-	and	V9.16B, V9.16B, V10.16B
+	shl	V14.16B, V6.16B, #1
+	shl	V15.16B, V7.16B, #1
+	movi	V0.16B, #27
+	and	V8.16B, V8.16B, V0.16B
+	and	V9.16B, V9.16B, V0.16B
+	and	V10.16B, V10.16B, V0.16B
+	and	V11.16B, V11.16B, V0.16B
 	eor	V8.16B, V8.16B, V12.16B
 	eor	V9.16B, V9.16B, V13.16B
-	ushr	V12.16B, V4.16B, #6
-	ushr	V13.16B, V5.16B, #6
-	shl	V0.16B, V4.16B, #2
-	shl	V1.16B, V5.16B, #2
-	pmul	V12.16B, V12.16B, V10.16B
-	pmul	V13.16B, V13.16B, V10.16B
-	eor	V12.16B, V12.16B, V0.16B
-	eor	V13.16B, V13.16B, V1.16B
-	ushr	V0.16B, V4.16B, #5
-	ushr	V1.16B, V5.16B, #5
-	pmul	V0.16B, V0.16B, V10.16B
-	pmul	V1.16B, V1.16B, V10.16B
-	shl	V10.16B, V4.16B, #3
-	shl	V11.16B, V5.16B, #3
-	eor	V0.16B, V0.16B, V10.16B
-	eor	V1.16B, V1.16B, V11.16B
-	eor	V10.16B, V8.16B, V0.16B
-	eor	V11.16B, V9.16B, V1.16B
-	eor	V0.16B, V0.16B, V4.16B
-	eor	V1.16B, V1.16B, V5.16B
-	eor	V8.16B, V12.16B, V0.16B
-	eor	V9.16B, V13.16B, V1.16B
-	eor	V12.16B, V12.16B, V10.16B
-	eor	V13.16B, V13.16B, V11.16B
-	eor	V10.16B, V10.16B, V4.16B
-	eor	V11.16B, V11.16B, V5.16B
-	shl	V4.4S, V10.4S, #8
-	shl	V5.4S, V11.4S, #8
-	rev32	V8.8H, V8.8H
-	rev32	V9.8H, V9.8H
-	sri	V4.4S, V10.4S, #24
-	sri	V5.4S, V11.4S, #24
-	eor	V4.16B, V4.16B, V12.16B
-	eor	V5.16B, V5.16B, V13.16B
-	shl	V10.4S, V0.4S, #24
-	shl	V11.4S, V1.4S, #24
+	eor	V10.16B, V10.16B, V14.16B
+	eor	V11.16B, V11.16B, V15.16B
+	eor	V0.16B, V8.16B, V4.16B
+	eor	V1.16B, V9.16B, V5.16B
+	eor	V2.16B, V10.16B, V6.16B
+	eor	V3.16B, V11.16B, V7.16B
+	shl	V12.4S, V0.4S, #8
+	shl	V13.4S, V1.4S, #8
+	shl	V14.4S, V2.4S, #8
+	shl	V15.4S, V3.4S, #8
+	sri	V12.4S, V0.4S, #24
+	sri	V13.4S, V1.4S, #24
+	sri	V14.4S, V2.4S, #24
+	sri	V15.4S, V3.4S, #24
+	shl	V0.4S, V4.4S, #24
+	shl	V1.4S, V5.4S, #24
+	shl	V2.4S, V6.4S, #24
+	shl	V3.4S, V7.4S, #24
+	sri	V0.4S, V4.4S, #8
+	sri	V1.4S, V5.4S, #8
+	sri	V2.4S, V6.4S, #8
+	sri	V3.4S, V7.4S, #8
+	rev32	V4.8H, V4.8H
+	rev32	V5.8H, V5.8H
+	rev32	V6.8H, V6.8H
+	rev32	V7.8H, V7.8H
+	eor	V4.16B, V4.16B, V0.16B
+	eor	V5.16B, V5.16B, V1.16B
+	eor	V6.16B, V6.16B, V2.16B
+	eor	V7.16B, V7.16B, V3.16B
+	;   XOR in Key Schedule
+	ld1	{V0.2D}, [x9], #16
 	eor	V4.16B, V4.16B, V8.16B
 	eor	V5.16B, V5.16B, V9.16B
-	sri	V10.4S, V0.4S, #8
-	sri	V11.4S, V1.4S, #8
-	eor	V4.16B, V4.16B, V10.16B
-	eor	V5.16B, V5.16B, V11.16B
-	;   XOR in Key Schedule
-	ld1	{V0.2D}, [x25], #16
+	eor	V6.16B, V6.16B, V10.16B
+	eor	V7.16B, V7.16B, V11.16B
 	eor	V4.16B, V4.16B, V0.16B
 	eor	V5.16B, V5.16B, V0.16B
+	eor	V6.16B, V6.16B, V0.16B
+	eor	V7.16B, V7.16B, V0.16B
+	eor	V4.16B, V4.16B, V12.16B
+	eor	V5.16B, V5.16B, V13.16B
+	eor	V6.16B, V6.16B, V14.16B
+	eor	V7.16B, V7.16B, V15.16B
 	; Round Done
+	tbl	V0.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V4.16B
+	tbl	V1.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V5.16B
+	tbl	V2.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V6.16B
+	tbl	V3.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V7.16B
 	movi	V12.16B, #0x40
 	movi	V13.16B, #0x80
 	movi	V14.16B, #0xc0
 	eor	V8.16B, V4.16B, V12.16B
 	eor	V9.16B, V5.16B, V12.16B
-	tbl	V0.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V4.16B
-	tbl	V1.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V5.16B
+	eor	V10.16B, V6.16B, V12.16B
+	eor	V11.16B, V7.16B, V12.16B
 	tbl	V8.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V8.16B
 	tbl	V9.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V9.16B
-	eor	V10.16B, V4.16B, V13.16B
-	eor	V11.16B, V5.16B, V13.16B
+	tbl	V10.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V10.16B
+	tbl	V11.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V11.16B
+	orr	V0.16B, V0.16B, V8.16B
+	orr	V1.16B, V1.16B, V9.16B
+	orr	V2.16B, V2.16B, V10.16B
+	orr	V3.16B, V3.16B, V11.16B
+	eor	V8.16B, V4.16B, V13.16B
+	eor	V9.16B, V5.16B, V13.16B
+	eor	V10.16B, V6.16B, V13.16B
+	eor	V11.16B, V7.16B, V13.16B
+	tbl	V8.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V8.16B
+	tbl	V9.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V9.16B
 	tbl	V10.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V10.16B
 	tbl	V11.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V11.16B
 	orr	V0.16B, V0.16B, V8.16B
 	orr	V1.16B, V1.16B, V9.16B
+	orr	V2.16B, V2.16B, V10.16B
+	orr	V3.16B, V3.16B, V11.16B
 	eor	V8.16B, V4.16B, V14.16B
 	eor	V9.16B, V5.16B, V14.16B
-	orr	V0.16B, V0.16B, V10.16B
-	orr	V1.16B, V1.16B, V11.16B
+	eor	V10.16B, V6.16B, V14.16B
+	eor	V11.16B, V7.16B, V14.16B
 	tbl	V8.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V8.16B
 	tbl	V9.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V9.16B
+	tbl	V10.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V10.16B
+	tbl	V11.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V11.16B
 	orr	V0.16B, V0.16B, V8.16B
 	orr	V1.16B, V1.16B, V9.16B
-	ld1	{V4.16B}, [x23]
+	orr	V2.16B, V2.16B, V10.16B
+	orr	V3.16B, V3.16B, V11.16B
+	ld1	{V4.16B}, [x7]
 	tbl	V0.16B, {V0.16B}, V4.16B
 	tbl	V1.16B, {V1.16B}, V4.16B
-	movi	V10.16B, #27
+	tbl	V2.16B, {V2.16B}, V4.16B
+	tbl	V3.16B, {V3.16B}, V4.16B
 	sshr	V8.16B, V0.16B, #7
 	sshr	V9.16B, V1.16B, #7
+	sshr	V10.16B, V2.16B, #7
+	sshr	V11.16B, V3.16B, #7
 	shl	V12.16B, V0.16B, #1
 	shl	V13.16B, V1.16B, #1
-	and	V8.16B, V8.16B, V10.16B
-	and	V9.16B, V9.16B, V10.16B
+	shl	V14.16B, V2.16B, #1
+	shl	V15.16B, V3.16B, #1
+	movi	V4.16B, #27
+	and	V8.16B, V8.16B, V4.16B
+	and	V9.16B, V9.16B, V4.16B
+	and	V10.16B, V10.16B, V4.16B
+	and	V11.16B, V11.16B, V4.16B
 	eor	V8.16B, V8.16B, V12.16B
 	eor	V9.16B, V9.16B, V13.16B
-	ushr	V12.16B, V0.16B, #6
-	ushr	V13.16B, V1.16B, #6
-	shl	V4.16B, V0.16B, #2
-	shl	V5.16B, V1.16B, #2
-	pmul	V12.16B, V12.16B, V10.16B
-	pmul	V13.16B, V13.16B, V10.16B
-	eor	V12.16B, V12.16B, V4.16B
-	eor	V13.16B, V13.16B, V5.16B
-	ushr	V4.16B, V0.16B, #5
-	ushr	V5.16B, V1.16B, #5
-	pmul	V4.16B, V4.16B, V10.16B
-	pmul	V5.16B, V5.16B, V10.16B
-	shl	V10.16B, V0.16B, #3
-	shl	V11.16B, V1.16B, #3
-	eor	V4.16B, V4.16B, V10.16B
-	eor	V5.16B, V5.16B, V11.16B
-	eor	V10.16B, V8.16B, V4.16B
-	eor	V11.16B, V9.16B, V5.16B
-	eor	V4.16B, V4.16B, V0.16B
-	eor	V5.16B, V5.16B, V1.16B
-	eor	V8.16B, V12.16B, V4.16B
-	eor	V9.16B, V13.16B, V5.16B
-	eor	V12.16B, V12.16B, V10.16B
-	eor	V13.16B, V13.16B, V11.16B
-	eor	V10.16B, V10.16B, V0.16B
-	eor	V11.16B, V11.16B, V1.16B
-	shl	V0.4S, V10.4S, #8
-	shl	V1.4S, V11.4S, #8
-	rev32	V8.8H, V8.8H
-	rev32	V9.8H, V9.8H
-	sri	V0.4S, V10.4S, #24
-	sri	V1.4S, V11.4S, #24
-	eor	V0.16B, V0.16B, V12.16B
-	eor	V1.16B, V1.16B, V13.16B
-	shl	V10.4S, V4.4S, #24
-	shl	V11.4S, V5.4S, #24
+	eor	V10.16B, V10.16B, V14.16B
+	eor	V11.16B, V11.16B, V15.16B
+	eor	V4.16B, V8.16B, V0.16B
+	eor	V5.16B, V9.16B, V1.16B
+	eor	V6.16B, V10.16B, V2.16B
+	eor	V7.16B, V11.16B, V3.16B
+	shl	V12.4S, V4.4S, #8
+	shl	V13.4S, V5.4S, #8
+	shl	V14.4S, V6.4S, #8
+	shl	V15.4S, V7.4S, #8
+	sri	V12.4S, V4.4S, #24
+	sri	V13.4S, V5.4S, #24
+	sri	V14.4S, V6.4S, #24
+	sri	V15.4S, V7.4S, #24
+	shl	V4.4S, V0.4S, #24
+	shl	V5.4S, V1.4S, #24
+	shl	V6.4S, V2.4S, #24
+	shl	V7.4S, V3.4S, #24
+	sri	V4.4S, V0.4S, #8
+	sri	V5.4S, V1.4S, #8
+	sri	V6.4S, V2.4S, #8
+	sri	V7.4S, V3.4S, #8
+	rev32	V0.8H, V0.8H
+	rev32	V1.8H, V1.8H
+	rev32	V2.8H, V2.8H
+	rev32	V3.8H, V3.8H
+	eor	V0.16B, V0.16B, V4.16B
+	eor	V1.16B, V1.16B, V5.16B
+	eor	V2.16B, V2.16B, V6.16B
+	eor	V3.16B, V3.16B, V7.16B
+	;   XOR in Key Schedule
+	ld1	{V4.2D}, [x9], #16
 	eor	V0.16B, V0.16B, V8.16B
 	eor	V1.16B, V1.16B, V9.16B
-	sri	V10.4S, V4.4S, #8
-	sri	V11.4S, V5.4S, #8
-	eor	V0.16B, V0.16B, V10.16B
-	eor	V1.16B, V1.16B, V11.16B
-	;   XOR in Key Schedule
-	ld1	{V4.2D}, [x25], #16
+	eor	V2.16B, V2.16B, V10.16B
+	eor	V3.16B, V3.16B, V11.16B
 	eor	V0.16B, V0.16B, V4.16B
 	eor	V1.16B, V1.16B, V4.16B
+	eor	V2.16B, V2.16B, V4.16B
+	eor	V3.16B, V3.16B, V4.16B
+	eor	V0.16B, V0.16B, V12.16B
+	eor	V1.16B, V1.16B, V13.16B
+	eor	V2.16B, V2.16B, V14.16B
+	eor	V3.16B, V3.16B, V15.16B
 	; Round Done
-	subs	w24, w24, #2
-	bne	L_AES_XTS_decrypt_NEON_loop_nr_2
+	subs	w8, w8, #2
+	bne	L_AES_GCMSIV_ctr_neon_loop_nr_4
+	tbl	V4.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V0.16B
+	tbl	V5.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V1.16B
+	tbl	V6.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V2.16B
+	tbl	V7.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V3.16B
 	movi	V12.16B, #0x40
 	movi	V13.16B, #0x80
 	movi	V14.16B, #0xc0
 	eor	V8.16B, V0.16B, V12.16B
 	eor	V9.16B, V1.16B, V12.16B
-	tbl	V4.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V0.16B
-	tbl	V5.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V1.16B
+	eor	V10.16B, V2.16B, V12.16B
+	eor	V11.16B, V3.16B, V12.16B
 	tbl	V8.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V8.16B
 	tbl	V9.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V9.16B
-	eor	V10.16B, V0.16B, V13.16B
-	eor	V11.16B, V1.16B, V13.16B
+	tbl	V10.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V10.16B
+	tbl	V11.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V11.16B
+	orr	V4.16B, V4.16B, V8.16B
+	orr	V5.16B, V5.16B, V9.16B
+	orr	V6.16B, V6.16B, V10.16B
+	orr	V7.16B, V7.16B, V11.16B
+	eor	V8.16B, V0.16B, V13.16B
+	eor	V9.16B, V1.16B, V13.16B
+	eor	V10.16B, V2.16B, V13.16B
+	eor	V11.16B, V3.16B, V13.16B
+	tbl	V8.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V8.16B
+	tbl	V9.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V9.16B
 	tbl	V10.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V10.16B
 	tbl	V11.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V11.16B
 	orr	V4.16B, V4.16B, V8.16B
 	orr	V5.16B, V5.16B, V9.16B
+	orr	V6.16B, V6.16B, V10.16B
+	orr	V7.16B, V7.16B, V11.16B
 	eor	V8.16B, V0.16B, V14.16B
 	eor	V9.16B, V1.16B, V14.16B
-	orr	V4.16B, V4.16B, V10.16B
-	orr	V5.16B, V5.16B, V11.16B
+	eor	V10.16B, V2.16B, V14.16B
+	eor	V11.16B, V3.16B, V14.16B
 	tbl	V8.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V8.16B
 	tbl	V9.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V9.16B
+	tbl	V10.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V10.16B
+	tbl	V11.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V11.16B
 	orr	V4.16B, V4.16B, V8.16B
 	orr	V5.16B, V5.16B, V9.16B
-	ld1	{V0.16B}, [x23]
+	orr	V6.16B, V6.16B, V10.16B
+	orr	V7.16B, V7.16B, V11.16B
+	ld1	{V0.16B}, [x7]
 	tbl	V4.16B, {V4.16B}, V0.16B
 	tbl	V5.16B, {V5.16B}, V0.16B
-	movi	V10.16B, #27
+	tbl	V6.16B, {V6.16B}, V0.16B
+	tbl	V7.16B, {V7.16B}, V0.16B
 	sshr	V8.16B, V4.16B, #7
 	sshr	V9.16B, V5.16B, #7
+	sshr	V10.16B, V6.16B, #7
+	sshr	V11.16B, V7.16B, #7
 	shl	V12.16B, V4.16B, #1
 	shl	V13.16B, V5.16B, #1
-	and	V8.16B, V8.16B, V10.16B
-	and	V9.16B, V9.16B, V10.16B
+	shl	V14.16B, V6.16B, #1
+	shl	V15.16B, V7.16B, #1
+	movi	V0.16B, #27
+	and	V8.16B, V8.16B, V0.16B
+	and	V9.16B, V9.16B, V0.16B
+	and	V10.16B, V10.16B, V0.16B
+	and	V11.16B, V11.16B, V0.16B
 	eor	V8.16B, V8.16B, V12.16B
 	eor	V9.16B, V9.16B, V13.16B
-	ushr	V12.16B, V4.16B, #6
-	ushr	V13.16B, V5.16B, #6
-	shl	V0.16B, V4.16B, #2
-	shl	V1.16B, V5.16B, #2
-	pmul	V12.16B, V12.16B, V10.16B
-	pmul	V13.16B, V13.16B, V10.16B
-	eor	V12.16B, V12.16B, V0.16B
-	eor	V13.16B, V13.16B, V1.16B
-	ushr	V0.16B, V4.16B, #5
-	ushr	V1.16B, V5.16B, #5
-	pmul	V0.16B, V0.16B, V10.16B
-	pmul	V1.16B, V1.16B, V10.16B
-	shl	V10.16B, V4.16B, #3
-	shl	V11.16B, V5.16B, #3
-	eor	V0.16B, V0.16B, V10.16B
-	eor	V1.16B, V1.16B, V11.16B
-	eor	V10.16B, V8.16B, V0.16B
-	eor	V11.16B, V9.16B, V1.16B
-	eor	V0.16B, V0.16B, V4.16B
-	eor	V1.16B, V1.16B, V5.16B
-	eor	V8.16B, V12.16B, V0.16B
-	eor	V9.16B, V13.16B, V1.16B
-	eor	V12.16B, V12.16B, V10.16B
-	eor	V13.16B, V13.16B, V11.16B
-	eor	V10.16B, V10.16B, V4.16B
-	eor	V11.16B, V11.16B, V5.16B
-	shl	V4.4S, V10.4S, #8
-	shl	V5.4S, V11.4S, #8
-	rev32	V8.8H, V8.8H
-	rev32	V9.8H, V9.8H
-	sri	V4.4S, V10.4S, #24
-	sri	V5.4S, V11.4S, #24
-	eor	V4.16B, V4.16B, V12.16B
-	eor	V5.16B, V5.16B, V13.16B
-	shl	V10.4S, V0.4S, #24
-	shl	V11.4S, V1.4S, #24
+	eor	V10.16B, V10.16B, V14.16B
+	eor	V11.16B, V11.16B, V15.16B
+	eor	V0.16B, V8.16B, V4.16B
+	eor	V1.16B, V9.16B, V5.16B
+	eor	V2.16B, V10.16B, V6.16B
+	eor	V3.16B, V11.16B, V7.16B
+	shl	V12.4S, V0.4S, #8
+	shl	V13.4S, V1.4S, #8
+	shl	V14.4S, V2.4S, #8
+	shl	V15.4S, V3.4S, #8
+	sri	V12.4S, V0.4S, #24
+	sri	V13.4S, V1.4S, #24
+	sri	V14.4S, V2.4S, #24
+	sri	V15.4S, V3.4S, #24
+	shl	V0.4S, V4.4S, #24
+	shl	V1.4S, V5.4S, #24
+	shl	V2.4S, V6.4S, #24
+	shl	V3.4S, V7.4S, #24
+	sri	V0.4S, V4.4S, #8
+	sri	V1.4S, V5.4S, #8
+	sri	V2.4S, V6.4S, #8
+	sri	V3.4S, V7.4S, #8
+	rev32	V4.8H, V4.8H
+	rev32	V5.8H, V5.8H
+	rev32	V6.8H, V6.8H
+	rev32	V7.8H, V7.8H
+	eor	V4.16B, V4.16B, V0.16B
+	eor	V5.16B, V5.16B, V1.16B
+	eor	V6.16B, V6.16B, V2.16B
+	eor	V7.16B, V7.16B, V3.16B
+	;   XOR in Key Schedule
+	ld1	{V0.2D}, [x9], #16
 	eor	V4.16B, V4.16B, V8.16B
 	eor	V5.16B, V5.16B, V9.16B
-	sri	V10.4S, V0.4S, #8
-	sri	V11.4S, V1.4S, #8
-	eor	V4.16B, V4.16B, V10.16B
-	eor	V5.16B, V5.16B, V11.16B
-	;   XOR in Key Schedule
-	ld1	{V0.2D}, [x25], #16
+	eor	V6.16B, V6.16B, V10.16B
+	eor	V7.16B, V7.16B, V11.16B
 	eor	V4.16B, V4.16B, V0.16B
 	eor	V5.16B, V5.16B, V0.16B
+	eor	V6.16B, V6.16B, V0.16B
+	eor	V7.16B, V7.16B, V0.16B
+	eor	V4.16B, V4.16B, V12.16B
+	eor	V5.16B, V5.16B, V13.16B
+	eor	V6.16B, V6.16B, V14.16B
+	eor	V7.16B, V7.16B, V15.16B
 	; Round Done
+	tbl	V0.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V4.16B
+	tbl	V1.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V5.16B
+	tbl	V2.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V6.16B
+	tbl	V3.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V7.16B
 	movi	V12.16B, #0x40
 	movi	V13.16B, #0x80
 	movi	V14.16B, #0xc0
 	eor	V8.16B, V4.16B, V12.16B
 	eor	V9.16B, V5.16B, V12.16B
-	tbl	V0.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V4.16B
-	tbl	V1.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V5.16B
+	eor	V10.16B, V6.16B, V12.16B
+	eor	V11.16B, V7.16B, V12.16B
 	tbl	V8.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V8.16B
 	tbl	V9.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V9.16B
-	eor	V10.16B, V4.16B, V13.16B
-	eor	V11.16B, V5.16B, V13.16B
+	tbl	V10.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V10.16B
+	tbl	V11.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V11.16B
+	orr	V0.16B, V0.16B, V8.16B
+	orr	V1.16B, V1.16B, V9.16B
+	orr	V2.16B, V2.16B, V10.16B
+	orr	V3.16B, V3.16B, V11.16B
+	eor	V8.16B, V4.16B, V13.16B
+	eor	V9.16B, V5.16B, V13.16B
+	eor	V10.16B, V6.16B, V13.16B
+	eor	V11.16B, V7.16B, V13.16B
+	tbl	V8.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V8.16B
+	tbl	V9.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V9.16B
 	tbl	V10.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V10.16B
 	tbl	V11.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V11.16B
 	orr	V0.16B, V0.16B, V8.16B
 	orr	V1.16B, V1.16B, V9.16B
+	orr	V2.16B, V2.16B, V10.16B
+	orr	V3.16B, V3.16B, V11.16B
 	eor	V8.16B, V4.16B, V14.16B
 	eor	V9.16B, V5.16B, V14.16B
-	orr	V0.16B, V0.16B, V10.16B
-	orr	V1.16B, V1.16B, V11.16B
+	eor	V10.16B, V6.16B, V14.16B
+	eor	V11.16B, V7.16B, V14.16B
 	tbl	V8.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V8.16B
 	tbl	V9.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V9.16B
+	tbl	V10.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V10.16B
+	tbl	V11.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V11.16B
 	orr	V0.16B, V0.16B, V8.16B
 	orr	V1.16B, V1.16B, V9.16B
-	ld1	{V4.16B}, [x23]
+	orr	V2.16B, V2.16B, V10.16B
+	orr	V3.16B, V3.16B, V11.16B
+	ld1	{V4.16B}, [x7]
 	tbl	V0.16B, {V0.16B}, V4.16B
 	tbl	V1.16B, {V1.16B}, V4.16B
+	tbl	V2.16B, {V2.16B}, V4.16B
+	tbl	V3.16B, {V3.16B}, V4.16B
 	;   XOR in Key Schedule
-	ld1	{V4.2D}, [x25], #16
+	ld1	{V4.2D}, [x9], #16
 	eor	V0.16B, V0.16B, V4.16B
 	eor	V1.16B, V1.16B, V4.16B
+	eor	V2.16B, V2.16B, V4.16B
+	eor	V3.16B, V3.16B, V4.16B
 	; Round Done
 	rev32	V0.16B, V0.16B
 	rev32	V1.16B, V1.16B
-	eor	V0.16B, V0.16B, V2.16B
-	eor	V1.16B, V1.16B, V3.16B
-	st1	{V0.16B, V1.16B}, [x1], #32
-	and	x16, x17, x11, asr 63
-	extr	x9, x11, x10, #63
-	eor	x8, x16, x10, lsl 1
-	sub	w2, w2, #32
-L_AES_XTS_decrypt_NEON_start_1
-	ld1	{V3.2D}, [x23]
-	mov	V2.D[0], x8
-	mov	V2.D[1], x9
-	cmp	w2, #16
-	blt	L_AES_XTS_decrypt_NEON_start_partial
-	mov	x25, x4
-	ld1	{V0.16B}, [x0], #16
-	ld1	{V4.2D}, [x25], #16
-	eor	V0.16B, V0.16B, V2.16B
-	rev32	V0.16B, V0.16B
-	eor	V0.16B, V0.16B, V4.16B
-	sub	w24, w7, #2
-L_AES_XTS_decrypt_NEON_loop_nr_1
-	eor	V8.16B, V0.16B, V12.16B
-	eor	V9.16B, V0.16B, V13.16B
-	eor	V10.16B, V0.16B, V14.16B
-	tbl	V4.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V0.16B
-	tbl	V8.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V8.16B
-	tbl	V9.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V9.16B
-	tbl	V10.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V10.16B
-	orr	V4.16B, V4.16B, V8.16B
-	orr	V9.16B, V9.16B, V10.16B
-	orr	V4.16B, V4.16B, V9.16B
-	tbl	V4.16B, {V4.16B}, V3.16B
-	sshr	V10.16B, V4.16B, #7
-	ushr	V11.16B, V4.16B, #6
-	ushr	V8.16B, V4.16B, #5
-	and	V10.16B, V10.16B, V15.16B
-	pmul	V11.16B, V11.16B, V15.16B
-	pmul	V8.16B, V8.16B, V15.16B
-	shl	V9.16B, V4.16B, #1
-	eor	V10.16B, V10.16B, V9.16B
-	shl	V9.16B, V4.16B, #3
-	eor	V8.16B, V8.16B, V9.16B
-	shl	V9.16B, V4.16B, #2
-	eor	V11.16B, V11.16B, V9.16B
-	eor	V9.16B, V10.16B, V8.16B
-	eor	V8.16B, V8.16B, V4.16B
-	eor	V10.16B, V11.16B, V8.16B
-	eor	V11.16B, V11.16B, V9.16B
-	eor	V9.16B, V9.16B, V4.16B
-	shl	V4.4S, V9.4S, #8
-	rev32	V10.8H, V10.8H
-	sri	V4.4S, V9.4S, #24
-	eor	V4.16B, V4.16B, V11.16B
-	shl	V9.4S, V8.4S, #24
-	eor	V4.16B, V4.16B, V10.16B
-	sri	V9.4S, V8.4S, #8
-	eor	V4.16B, V4.16B, V9.16B
-	ld1	{V0.2D}, [x25], #16
-	;   XOR in Key Schedule
-	eor	V4.16B, V4.16B, V0.16B
-	eor	V8.16B, V4.16B, V12.16B
-	eor	V9.16B, V4.16B, V13.16B
-	eor	V10.16B, V4.16B, V14.16B
-	tbl	V0.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V4.16B
-	tbl	V8.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V8.16B
-	tbl	V9.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V9.16B
-	tbl	V10.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V10.16B
-	orr	V0.16B, V0.16B, V8.16B
-	orr	V9.16B, V9.16B, V10.16B
-	orr	V0.16B, V0.16B, V9.16B
-	tbl	V0.16B, {V0.16B}, V3.16B
-	sshr	V10.16B, V0.16B, #7
-	ushr	V11.16B, V0.16B, #6
-	ushr	V8.16B, V0.16B, #5
-	and	V10.16B, V10.16B, V15.16B
-	pmul	V11.16B, V11.16B, V15.16B
-	pmul	V8.16B, V8.16B, V15.16B
-	shl	V9.16B, V0.16B, #1
-	eor	V10.16B, V10.16B, V9.16B
-	shl	V9.16B, V0.16B, #3
-	eor	V8.16B, V8.16B, V9.16B
-	shl	V9.16B, V0.16B, #2
-	eor	V11.16B, V11.16B, V9.16B
-	eor	V9.16B, V10.16B, V8.16B
-	eor	V8.16B, V8.16B, V0.16B
-	eor	V10.16B, V11.16B, V8.16B
-	eor	V11.16B, V11.16B, V9.16B
-	eor	V9.16B, V9.16B, V0.16B
-	shl	V0.4S, V9.4S, #8
-	rev32	V10.8H, V10.8H
-	sri	V0.4S, V9.4S, #24
-	eor	V0.16B, V0.16B, V11.16B
-	shl	V9.4S, V8.4S, #24
-	eor	V0.16B, V0.16B, V10.16B
-	sri	V9.4S, V8.4S, #8
-	eor	V0.16B, V0.16B, V9.16B
-	ld1	{V4.2D}, [x25], #16
-	;   XOR in Key Schedule
-	eor	V0.16B, V0.16B, V4.16B
-	subs	w24, w24, #2
-	bne	L_AES_XTS_decrypt_NEON_loop_nr_1
-	eor	V8.16B, V0.16B, V12.16B
-	eor	V9.16B, V0.16B, V13.16B
-	eor	V10.16B, V0.16B, V14.16B
-	tbl	V4.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V0.16B
-	tbl	V8.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V8.16B
-	tbl	V9.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V9.16B
-	tbl	V10.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V10.16B
-	orr	V4.16B, V4.16B, V8.16B
-	orr	V9.16B, V9.16B, V10.16B
-	orr	V4.16B, V4.16B, V9.16B
-	tbl	V4.16B, {V4.16B}, V3.16B
-	sshr	V10.16B, V4.16B, #7
-	ushr	V11.16B, V4.16B, #6
-	ushr	V8.16B, V4.16B, #5
-	and	V10.16B, V10.16B, V15.16B
-	pmul	V11.16B, V11.16B, V15.16B
-	pmul	V8.16B, V8.16B, V15.16B
-	shl	V9.16B, V4.16B, #1
-	eor	V10.16B, V10.16B, V9.16B
-	shl	V9.16B, V4.16B, #3
-	eor	V8.16B, V8.16B, V9.16B
-	shl	V9.16B, V4.16B, #2
-	eor	V11.16B, V11.16B, V9.16B
-	eor	V9.16B, V10.16B, V8.16B
-	eor	V8.16B, V8.16B, V4.16B
-	eor	V10.16B, V11.16B, V8.16B
-	eor	V11.16B, V11.16B, V9.16B
-	eor	V9.16B, V9.16B, V4.16B
-	shl	V4.4S, V9.4S, #8
-	rev32	V10.8H, V10.8H
-	sri	V4.4S, V9.4S, #24
-	eor	V4.16B, V4.16B, V11.16B
-	shl	V9.4S, V8.4S, #24
-	eor	V4.16B, V4.16B, V10.16B
-	sri	V9.4S, V8.4S, #8
-	eor	V4.16B, V4.16B, V9.16B
-	ld1	{V0.2D}, [x25], #16
-	;   XOR in Key Schedule
-	eor	V4.16B, V4.16B, V0.16B
-	eor	V8.16B, V4.16B, V12.16B
-	eor	V9.16B, V4.16B, V13.16B
-	eor	V10.16B, V4.16B, V14.16B
-	tbl	V0.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V4.16B
-	tbl	V8.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V8.16B
-	tbl	V9.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V9.16B
-	tbl	V10.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V10.16B
-	orr	V0.16B, V0.16B, V8.16B
-	orr	V9.16B, V9.16B, V10.16B
-	orr	V0.16B, V0.16B, V9.16B
-	tbl	V0.16B, {V0.16B}, V3.16B
-	ld1	{V4.2D}, [x25], #16
-	;   XOR in Key Schedule
+	rev32	V2.16B, V2.16B
+	rev32	V3.16B, V3.16B
+	ld1	{V4.16B, V5.16B, V6.16B, V7.16B}, [x0], #0x40
 	eor	V0.16B, V0.16B, V4.16B
-	rev32	V0.16B, V0.16B
-	eor	V0.16B, V0.16B, V2.16B
-	st1	{V0.16B}, [x1], #16
-	sub	w2, w2, #16
-	cbz	w19, L_AES_XTS_decrypt_NEON_data_done
-	and	x16, x17, x9, asr 63
-	extr	x9, x9, x8, #63
-	eor	x8, x16, x8, lsl 1
-L_AES_XTS_decrypt_NEON_start_partial
-	mov	w2, w19
-	cbz	w2, L_AES_XTS_decrypt_NEON_data_done
-	mov	V2.D[0], x8
-	mov	V2.D[1], x9
-	and	x16, x17, x9, asr 63
-	extr	x11, x9, x8, #63
-	eor	x10, x16, x8, lsl 1
-	mov	V1.D[0], x10
-	mov	V1.D[1], x11
-	mov	x25, x4
-	ld1	{V0.16B}, [x0], #16
-	ld1	{V4.2D}, [x25], #16
-	eor	V0.16B, V0.16B, V1.16B
+	eor	V1.16B, V1.16B, V5.16B
+	eor	V2.16B, V2.16B, V6.16B
+	eor	V3.16B, V3.16B, V7.16B
+	st1	{V0.16B, V1.16B, V2.16B, V3.16B}, [x1], #0x40
+	sub	x2, x2, #0x40
+	cmp	x2, #0x40
+	bge	L_AES_GCMSIV_ctr_neon_loop_4
+L_AES_GCMSIV_ctr_neon_start_2
+	movi	V12.16B, #0x40
+	movi	V13.16B, #0x80
+	movi	V14.16B, #0xc0
+	movi	V15.16B, #27
+	cmp	x2, #16
+	beq	L_AES_GCMSIV_ctr_neon_start_1
+	blt	L_AES_GCMSIV_ctr_neon_data_done
+L_AES_GCMSIV_ctr_neon_loop_2
+	mov	x9, x3
+	ld1	{V4.2D}, [x9], #16
+	; Round: 0 - build counters and XOR in key schedule
+	ld1	{V0.2D}, [x5]
+	mov	V0.S[0], w10
 	rev32	V0.16B, V0.16B
 	eor	V0.16B, V0.16B, V4.16B
-	sub	w24, w7, #2
-L_AES_XTS_decrypt_NEON_loop_nr_partial_1
+	ld1	{V1.2D}, [x5]
+	add	w8, w10, #1
+	mov	V1.S[0], w8
+	rev32	V1.16B, V1.16B
+	eor	V1.16B, V1.16B, V4.16B
+	add	w10, w10, #2
+	sub	w8, w4, #2
+L_AES_GCMSIV_ctr_neon_loop_nr_2
 	eor	V8.16B, V0.16B, V12.16B
-	eor	V9.16B, V0.16B, V13.16B
-	eor	V10.16B, V0.16B, V14.16B
+	eor	V9.16B, V1.16B, V12.16B
 	tbl	V4.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V0.16B
-	tbl	V8.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V8.16B
-	tbl	V9.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V9.16B
-	tbl	V10.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V10.16B
-	orr	V4.16B, V4.16B, V8.16B
-	orr	V9.16B, V9.16B, V10.16B
-	orr	V4.16B, V4.16B, V9.16B
-	tbl	V4.16B, {V4.16B}, V3.16B
-	sshr	V10.16B, V4.16B, #7
-	ushr	V11.16B, V4.16B, #6
-	ushr	V8.16B, V4.16B, #5
-	and	V10.16B, V10.16B, V15.16B
-	pmul	V11.16B, V11.16B, V15.16B
-	pmul	V8.16B, V8.16B, V15.16B
-	shl	V9.16B, V4.16B, #1
-	eor	V10.16B, V10.16B, V9.16B
-	shl	V9.16B, V4.16B, #3
-	eor	V8.16B, V8.16B, V9.16B
-	shl	V9.16B, V4.16B, #2
-	eor	V11.16B, V11.16B, V9.16B
-	eor	V9.16B, V10.16B, V8.16B
-	eor	V8.16B, V8.16B, V4.16B
-	eor	V10.16B, V11.16B, V8.16B
-	eor	V11.16B, V11.16B, V9.16B
-	eor	V9.16B, V9.16B, V4.16B
-	shl	V4.4S, V9.4S, #8
-	rev32	V10.8H, V10.8H
-	sri	V4.4S, V9.4S, #24
-	eor	V4.16B, V4.16B, V11.16B
-	shl	V9.4S, V8.4S, #24
-	eor	V4.16B, V4.16B, V10.16B
-	sri	V9.4S, V8.4S, #8
-	eor	V4.16B, V4.16B, V9.16B
-	ld1	{V0.2D}, [x25], #16
+	tbl	V5.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V1.16B
+	tbl	V8.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V8.16B
+	tbl	V9.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V9.16B
+	eor	V10.16B, V0.16B, V13.16B
+	eor	V11.16B, V1.16B, V13.16B
+	tbl	V10.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V10.16B
+	tbl	V11.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V11.16B
+	orr	V4.16B, V4.16B, V8.16B
+	orr	V5.16B, V5.16B, V9.16B
+	eor	V8.16B, V0.16B, V14.16B
+	eor	V9.16B, V1.16B, V14.16B
+	orr	V4.16B, V4.16B, V10.16B
+	orr	V5.16B, V5.16B, V11.16B
+	tbl	V8.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V8.16B
+	tbl	V9.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V9.16B
+	orr	V4.16B, V4.16B, V8.16B
+	orr	V5.16B, V5.16B, V9.16B
+	ld1	{V0.16B}, [x7]
+	tbl	V4.16B, {V4.16B}, V0.16B
+	tbl	V5.16B, {V5.16B}, V0.16B
+	sshr	V8.16B, V4.16B, #7
+	sshr	V9.16B, V5.16B, #7
+	shl	V10.16B, V4.16B, #1
+	shl	V11.16B, V5.16B, #1
+	and	V8.16B, V8.16B, V15.16B
+	and	V9.16B, V9.16B, V15.16B
+	eor	V8.16B, V8.16B, V10.16B
+	eor	V9.16B, V9.16B, V11.16B
+	eor	V0.16B, V8.16B, V4.16B
+	eor	V1.16B, V9.16B, V5.16B
+	shl	V10.4S, V0.4S, #8
+	shl	V11.4S, V1.4S, #8
+	sri	V10.4S, V0.4S, #24
+	sri	V11.4S, V1.4S, #24
+	shl	V0.4S, V4.4S, #24
+	shl	V1.4S, V5.4S, #24
+	sri	V0.4S, V4.4S, #8
+	sri	V1.4S, V5.4S, #8
+	rev32	V4.8H, V4.8H
+	rev32	V5.8H, V5.8H
+	eor	V4.16B, V4.16B, V0.16B
+	eor	V5.16B, V5.16B, V1.16B
 	;   XOR in Key Schedule
+	ld1	{V0.2D}, [x9], #16
+	eor	V4.16B, V4.16B, V8.16B
+	eor	V5.16B, V5.16B, V9.16B
 	eor	V4.16B, V4.16B, V0.16B
+	eor	V5.16B, V5.16B, V0.16B
+	eor	V4.16B, V4.16B, V10.16B
+	eor	V5.16B, V5.16B, V11.16B
+	; Round Done
 	eor	V8.16B, V4.16B, V12.16B
-	eor	V9.16B, V4.16B, V13.16B
-	eor	V10.16B, V4.16B, V14.16B
+	eor	V9.16B, V5.16B, V12.16B
 	tbl	V0.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V4.16B
+	tbl	V1.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V5.16B
 	tbl	V8.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V8.16B
-	tbl	V9.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V9.16B
-	tbl	V10.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V10.16B
+	tbl	V9.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V9.16B
+	eor	V10.16B, V4.16B, V13.16B
+	eor	V11.16B, V5.16B, V13.16B
+	tbl	V10.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V10.16B
+	tbl	V11.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V11.16B
 	orr	V0.16B, V0.16B, V8.16B
-	orr	V9.16B, V9.16B, V10.16B
-	orr	V0.16B, V0.16B, V9.16B
-	tbl	V0.16B, {V0.16B}, V3.16B
-	sshr	V10.16B, V0.16B, #7
-	ushr	V11.16B, V0.16B, #6
-	ushr	V8.16B, V0.16B, #5
-	and	V10.16B, V10.16B, V15.16B
-	pmul	V11.16B, V11.16B, V15.16B
-	pmul	V8.16B, V8.16B, V15.16B
-	shl	V9.16B, V0.16B, #1
-	eor	V10.16B, V10.16B, V9.16B
-	shl	V9.16B, V0.16B, #3
-	eor	V8.16B, V8.16B, V9.16B
-	shl	V9.16B, V0.16B, #2
-	eor	V11.16B, V11.16B, V9.16B
-	eor	V9.16B, V10.16B, V8.16B
-	eor	V8.16B, V8.16B, V0.16B
-	eor	V10.16B, V11.16B, V8.16B
-	eor	V11.16B, V11.16B, V9.16B
-	eor	V9.16B, V9.16B, V0.16B
-	shl	V0.4S, V9.4S, #8
-	rev32	V10.8H, V10.8H
-	sri	V0.4S, V9.4S, #24
-	eor	V0.16B, V0.16B, V11.16B
-	shl	V9.4S, V8.4S, #24
-	eor	V0.16B, V0.16B, V10.16B
-	sri	V9.4S, V8.4S, #8
-	eor	V0.16B, V0.16B, V9.16B
-	ld1	{V4.2D}, [x25], #16
+	orr	V1.16B, V1.16B, V9.16B
+	eor	V8.16B, V4.16B, V14.16B
+	eor	V9.16B, V5.16B, V14.16B
+	orr	V0.16B, V0.16B, V10.16B
+	orr	V1.16B, V1.16B, V11.16B
+	tbl	V8.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V8.16B
+	tbl	V9.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V9.16B
+	orr	V0.16B, V0.16B, V8.16B
+	orr	V1.16B, V1.16B, V9.16B
+	ld1	{V4.16B}, [x7]
+	tbl	V0.16B, {V0.16B}, V4.16B
+	tbl	V1.16B, {V1.16B}, V4.16B
+	sshr	V8.16B, V0.16B, #7
+	sshr	V9.16B, V1.16B, #7
+	shl	V10.16B, V0.16B, #1
+	shl	V11.16B, V1.16B, #1
+	and	V8.16B, V8.16B, V15.16B
+	and	V9.16B, V9.16B, V15.16B
+	eor	V8.16B, V8.16B, V10.16B
+	eor	V9.16B, V9.16B, V11.16B
+	eor	V4.16B, V8.16B, V0.16B
+	eor	V5.16B, V9.16B, V1.16B
+	shl	V10.4S, V4.4S, #8
+	shl	V11.4S, V5.4S, #8
+	sri	V10.4S, V4.4S, #24
+	sri	V11.4S, V5.4S, #24
+	shl	V4.4S, V0.4S, #24
+	shl	V5.4S, V1.4S, #24
+	sri	V4.4S, V0.4S, #8
+	sri	V5.4S, V1.4S, #8
+	rev32	V0.8H, V0.8H
+	rev32	V1.8H, V1.8H
+	eor	V0.16B, V0.16B, V4.16B
+	eor	V1.16B, V1.16B, V5.16B
 	;   XOR in Key Schedule
+	ld1	{V4.2D}, [x9], #16
+	eor	V0.16B, V0.16B, V8.16B
+	eor	V1.16B, V1.16B, V9.16B
 	eor	V0.16B, V0.16B, V4.16B
-	subs	w24, w24, #2
-	bne	L_AES_XTS_decrypt_NEON_loop_nr_partial_1
+	eor	V1.16B, V1.16B, V4.16B
+	eor	V0.16B, V0.16B, V10.16B
+	eor	V1.16B, V1.16B, V11.16B
+	; Round Done
+	subs	w8, w8, #2
+	bne	L_AES_GCMSIV_ctr_neon_loop_nr_2
 	eor	V8.16B, V0.16B, V12.16B
-	eor	V9.16B, V0.16B, V13.16B
-	eor	V10.16B, V0.16B, V14.16B
+	eor	V9.16B, V1.16B, V12.16B
 	tbl	V4.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V0.16B
+	tbl	V5.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V1.16B
 	tbl	V8.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V8.16B
-	tbl	V9.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V9.16B
-	tbl	V10.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V10.16B
+	tbl	V9.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V9.16B
+	eor	V10.16B, V0.16B, V13.16B
+	eor	V11.16B, V1.16B, V13.16B
+	tbl	V10.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V10.16B
+	tbl	V11.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V11.16B
 	orr	V4.16B, V4.16B, V8.16B
-	orr	V9.16B, V9.16B, V10.16B
-	orr	V4.16B, V4.16B, V9.16B
-	tbl	V4.16B, {V4.16B}, V3.16B
-	sshr	V10.16B, V4.16B, #7
-	ushr	V11.16B, V4.16B, #6
-	ushr	V8.16B, V4.16B, #5
-	and	V10.16B, V10.16B, V15.16B
-	pmul	V11.16B, V11.16B, V15.16B
-	pmul	V8.16B, V8.16B, V15.16B
-	shl	V9.16B, V4.16B, #1
-	eor	V10.16B, V10.16B, V9.16B
-	shl	V9.16B, V4.16B, #3
-	eor	V8.16B, V8.16B, V9.16B
-	shl	V9.16B, V4.16B, #2
-	eor	V11.16B, V11.16B, V9.16B
-	eor	V9.16B, V10.16B, V8.16B
-	eor	V8.16B, V8.16B, V4.16B
-	eor	V10.16B, V11.16B, V8.16B
-	eor	V11.16B, V11.16B, V9.16B
-	eor	V9.16B, V9.16B, V4.16B
-	shl	V4.4S, V9.4S, #8
-	rev32	V10.8H, V10.8H
-	sri	V4.4S, V9.4S, #24
-	eor	V4.16B, V4.16B, V11.16B
-	shl	V9.4S, V8.4S, #24
-	eor	V4.16B, V4.16B, V10.16B
-	sri	V9.4S, V8.4S, #8
-	eor	V4.16B, V4.16B, V9.16B
-	ld1	{V0.2D}, [x25], #16
+	orr	V5.16B, V5.16B, V9.16B
+	eor	V8.16B, V0.16B, V14.16B
+	eor	V9.16B, V1.16B, V14.16B
+	orr	V4.16B, V4.16B, V10.16B
+	orr	V5.16B, V5.16B, V11.16B
+	tbl	V8.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V8.16B
+	tbl	V9.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V9.16B
+	orr	V4.16B, V4.16B, V8.16B
+	orr	V5.16B, V5.16B, V9.16B
+	ld1	{V0.16B}, [x7]
+	tbl	V4.16B, {V4.16B}, V0.16B
+	tbl	V5.16B, {V5.16B}, V0.16B
+	sshr	V8.16B, V4.16B, #7
+	sshr	V9.16B, V5.16B, #7
+	shl	V10.16B, V4.16B, #1
+	shl	V11.16B, V5.16B, #1
+	and	V8.16B, V8.16B, V15.16B
+	and	V9.16B, V9.16B, V15.16B
+	eor	V8.16B, V8.16B, V10.16B
+	eor	V9.16B, V9.16B, V11.16B
+	eor	V0.16B, V8.16B, V4.16B
+	eor	V1.16B, V9.16B, V5.16B
+	shl	V10.4S, V0.4S, #8
+	shl	V11.4S, V1.4S, #8
+	sri	V10.4S, V0.4S, #24
+	sri	V11.4S, V1.4S, #24
+	shl	V0.4S, V4.4S, #24
+	shl	V1.4S, V5.4S, #24
+	sri	V0.4S, V4.4S, #8
+	sri	V1.4S, V5.4S, #8
+	rev32	V4.8H, V4.8H
+	rev32	V5.8H, V5.8H
+	eor	V4.16B, V4.16B, V0.16B
+	eor	V5.16B, V5.16B, V1.16B
 	;   XOR in Key Schedule
+	ld1	{V0.2D}, [x9], #16
+	eor	V4.16B, V4.16B, V8.16B
+	eor	V5.16B, V5.16B, V9.16B
 	eor	V4.16B, V4.16B, V0.16B
+	eor	V5.16B, V5.16B, V0.16B
+	eor	V4.16B, V4.16B, V10.16B
+	eor	V5.16B, V5.16B, V11.16B
+	; Round Done
 	eor	V8.16B, V4.16B, V12.16B
-	eor	V9.16B, V4.16B, V13.16B
-	eor	V10.16B, V4.16B, V14.16B
+	eor	V9.16B, V5.16B, V12.16B
 	tbl	V0.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V4.16B
+	tbl	V1.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V5.16B
 	tbl	V8.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V8.16B
-	tbl	V9.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V9.16B
-	tbl	V10.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V10.16B
+	tbl	V9.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V9.16B
+	eor	V10.16B, V4.16B, V13.16B
+	eor	V11.16B, V5.16B, V13.16B
+	tbl	V10.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V10.16B
+	tbl	V11.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V11.16B
 	orr	V0.16B, V0.16B, V8.16B
-	orr	V9.16B, V9.16B, V10.16B
-	orr	V0.16B, V0.16B, V9.16B
-	tbl	V0.16B, {V0.16B}, V3.16B
-	ld1	{V4.2D}, [x25], #16
-	;   XOR in Key Schedule
-	eor	V0.16B, V0.16B, V4.16B
-	rev32	V0.16B, V0.16B
-	eor	V0.16B, V0.16B, V1.16B
-	st1	{V0.2D}, [x6]
-	add	x1, x1, #16
-	mov	w16, w2
-L_AES_XTS_decrypt_NEON_start_byte
-	ldrb	w10, [x6]
-	ldrb	w11, [x0], #1
-	strb	w10, [x1], #1
-	strb	w11, [x6], #1
-	subs	w16, w16, #1
-	bgt	L_AES_XTS_decrypt_NEON_start_byte
-	sub	x1, x1, x2
-	sub	x6, x6, x2
-	sub	x1, x1, #16
-	mov	x25, x4
-	ld1	{V0.2D}, [x6]
-	ld1	{V4.2D}, [x25], #16
-	eor	V0.16B, V0.16B, V2.16B
+	orr	V1.16B, V1.16B, V9.16B
+	eor	V8.16B, V4.16B, V14.16B
+	eor	V9.16B, V5.16B, V14.16B
+	orr	V0.16B, V0.16B, V10.16B
+	orr	V1.16B, V1.16B, V11.16B
+	tbl	V8.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V8.16B
+	tbl	V9.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V9.16B
+	orr	V0.16B, V0.16B, V8.16B
+	orr	V1.16B, V1.16B, V9.16B
+	ld1	{V4.16B}, [x7]
+	tbl	V0.16B, {V0.16B}, V4.16B
+	tbl	V1.16B, {V1.16B}, V4.16B
+	;   XOR in Key Schedule
+	ld1	{V4.2D}, [x9], #16
+	eor	V0.16B, V0.16B, V4.16B
+	eor	V1.16B, V1.16B, V4.16B
+	; Round Done
 	rev32	V0.16B, V0.16B
+	rev32	V1.16B, V1.16B
+	ld1	{V4.16B, V5.16B}, [x0], #32
 	eor	V0.16B, V0.16B, V4.16B
-	sub	w24, w7, #2
-L_AES_XTS_decrypt_NEON_loop_nr_partial_2
+	eor	V1.16B, V1.16B, V5.16B
+	st1	{V0.16B, V1.16B}, [x1], #32
+	sub	x2, x2, #32
+	cmp	x2, #0
+	beq	L_AES_GCMSIV_ctr_neon_data_done
+L_AES_GCMSIV_ctr_neon_start_1
+	ld1	{V3.2D}, [x7]
+	mov	x9, x3
+	ld1	{V4.2D}, [x9], #16
+	; Round: 0 - build counter and XOR in key schedule
+	ld1	{V0.2D}, [x5]
+	mov	V0.S[0], w10
+	rev32	V0.16B, V0.16B
+	eor	V0.16B, V0.16B, V4.16B
+	add	w10, w10, #1
+	sub	w8, w4, #2
+L_AES_GCMSIV_ctr_neon_loop_nr_1
 	eor	V8.16B, V0.16B, V12.16B
 	eor	V9.16B, V0.16B, V13.16B
 	eor	V10.16B, V0.16B, V14.16B
@@ -50548,34 +52800,22 @@ L_AES_XTS_decrypt_NEON_loop_nr_partial_2
 	orr	V9.16B, V9.16B, V10.16B
 	orr	V4.16B, V4.16B, V9.16B
 	tbl	V4.16B, {V4.16B}, V3.16B
+	ld1	{V0.2D}, [x9], #16
 	sshr	V10.16B, V4.16B, #7
-	ushr	V11.16B, V4.16B, #6
-	ushr	V8.16B, V4.16B, #5
-	and	V10.16B, V10.16B, V15.16B
-	pmul	V11.16B, V11.16B, V15.16B
-	pmul	V8.16B, V8.16B, V15.16B
 	shl	V9.16B, V4.16B, #1
+	and	V10.16B, V10.16B, V15.16B
 	eor	V10.16B, V10.16B, V9.16B
-	shl	V9.16B, V4.16B, #3
-	eor	V8.16B, V8.16B, V9.16B
-	shl	V9.16B, V4.16B, #2
-	eor	V11.16B, V11.16B, V9.16B
-	eor	V9.16B, V10.16B, V8.16B
-	eor	V8.16B, V8.16B, V4.16B
-	eor	V10.16B, V11.16B, V8.16B
-	eor	V11.16B, V11.16B, V9.16B
-	eor	V9.16B, V9.16B, V4.16B
-	shl	V4.4S, V9.4S, #8
-	rev32	V10.8H, V10.8H
-	sri	V4.4S, V9.4S, #24
-	eor	V4.16B, V4.16B, V11.16B
-	shl	V9.4S, V8.4S, #24
-	eor	V4.16B, V4.16B, V10.16B
-	sri	V9.4S, V8.4S, #8
-	eor	V4.16B, V4.16B, V9.16B
-	ld1	{V0.2D}, [x25], #16
+	rev32	V8.8H, V4.8H
+	eor	V11.16B, V10.16B, V4.16B
+	eor	V10.16B, V10.16B, V8.16B
+	shl	V9.4S, V4.4S, #24
+	shl	V8.4S, V11.4S, #8
 	;   XOR in Key Schedule
-	eor	V4.16B, V4.16B, V0.16B
+	eor	V10.16B, V10.16B, V0.16B
+	sri	V9.4S, V4.4S, #8
+	sri	V8.4S, V11.4S, #24
+	eor	V4.16B, V10.16B, V9.16B
+	eor	V4.16B, V4.16B, V8.16B
 	eor	V8.16B, V4.16B, V12.16B
 	eor	V9.16B, V4.16B, V13.16B
 	eor	V10.16B, V4.16B, V14.16B
@@ -50587,36 +52827,24 @@ L_AES_XTS_decrypt_NEON_loop_nr_partial_2
 	orr	V9.16B, V9.16B, V10.16B
 	orr	V0.16B, V0.16B, V9.16B
 	tbl	V0.16B, {V0.16B}, V3.16B
+	ld1	{V4.2D}, [x9], #16
 	sshr	V10.16B, V0.16B, #7
-	ushr	V11.16B, V0.16B, #6
-	ushr	V8.16B, V0.16B, #5
-	and	V10.16B, V10.16B, V15.16B
-	pmul	V11.16B, V11.16B, V15.16B
-	pmul	V8.16B, V8.16B, V15.16B
 	shl	V9.16B, V0.16B, #1
+	and	V10.16B, V10.16B, V15.16B
 	eor	V10.16B, V10.16B, V9.16B
-	shl	V9.16B, V0.16B, #3
-	eor	V8.16B, V8.16B, V9.16B
-	shl	V9.16B, V0.16B, #2
-	eor	V11.16B, V11.16B, V9.16B
-	eor	V9.16B, V10.16B, V8.16B
-	eor	V8.16B, V8.16B, V0.16B
-	eor	V10.16B, V11.16B, V8.16B
-	eor	V11.16B, V11.16B, V9.16B
-	eor	V9.16B, V9.16B, V0.16B
-	shl	V0.4S, V9.4S, #8
-	rev32	V10.8H, V10.8H
-	sri	V0.4S, V9.4S, #24
-	eor	V0.16B, V0.16B, V11.16B
-	shl	V9.4S, V8.4S, #24
-	eor	V0.16B, V0.16B, V10.16B
-	sri	V9.4S, V8.4S, #8
-	eor	V0.16B, V0.16B, V9.16B
-	ld1	{V4.2D}, [x25], #16
+	rev32	V8.8H, V0.8H
+	eor	V11.16B, V10.16B, V0.16B
+	eor	V10.16B, V10.16B, V8.16B
+	shl	V9.4S, V0.4S, #24
+	shl	V8.4S, V11.4S, #8
 	;   XOR in Key Schedule
-	eor	V0.16B, V0.16B, V4.16B
-	subs	w24, w24, #2
-	bne	L_AES_XTS_decrypt_NEON_loop_nr_partial_2
+	eor	V10.16B, V10.16B, V4.16B
+	sri	V9.4S, V0.4S, #8
+	sri	V8.4S, V11.4S, #24
+	eor	V0.16B, V10.16B, V9.16B
+	eor	V0.16B, V0.16B, V8.16B
+	subs	w8, w8, #2
+	bne	L_AES_GCMSIV_ctr_neon_loop_nr_1
 	eor	V8.16B, V0.16B, V12.16B
 	eor	V9.16B, V0.16B, V13.16B
 	eor	V10.16B, V0.16B, V14.16B
@@ -50628,34 +52856,22 @@ L_AES_XTS_decrypt_NEON_loop_nr_partial_2
 	orr	V9.16B, V9.16B, V10.16B
 	orr	V4.16B, V4.16B, V9.16B
 	tbl	V4.16B, {V4.16B}, V3.16B
+	ld1	{V0.2D}, [x9], #16
 	sshr	V10.16B, V4.16B, #7
-	ushr	V11.16B, V4.16B, #6
-	ushr	V8.16B, V4.16B, #5
-	and	V10.16B, V10.16B, V15.16B
-	pmul	V11.16B, V11.16B, V15.16B
-	pmul	V8.16B, V8.16B, V15.16B
 	shl	V9.16B, V4.16B, #1
+	and	V10.16B, V10.16B, V15.16B
 	eor	V10.16B, V10.16B, V9.16B
-	shl	V9.16B, V4.16B, #3
-	eor	V8.16B, V8.16B, V9.16B
-	shl	V9.16B, V4.16B, #2
-	eor	V11.16B, V11.16B, V9.16B
-	eor	V9.16B, V10.16B, V8.16B
-	eor	V8.16B, V8.16B, V4.16B
-	eor	V10.16B, V11.16B, V8.16B
-	eor	V11.16B, V11.16B, V9.16B
-	eor	V9.16B, V9.16B, V4.16B
-	shl	V4.4S, V9.4S, #8
-	rev32	V10.8H, V10.8H
-	sri	V4.4S, V9.4S, #24
-	eor	V4.16B, V4.16B, V11.16B
-	shl	V9.4S, V8.4S, #24
-	eor	V4.16B, V4.16B, V10.16B
-	sri	V9.4S, V8.4S, #8
-	eor	V4.16B, V4.16B, V9.16B
-	ld1	{V0.2D}, [x25], #16
+	rev32	V8.8H, V4.8H
+	eor	V11.16B, V10.16B, V4.16B
+	eor	V10.16B, V10.16B, V8.16B
+	shl	V9.4S, V4.4S, #24
+	shl	V8.4S, V11.4S, #8
 	;   XOR in Key Schedule
-	eor	V4.16B, V4.16B, V0.16B
+	eor	V10.16B, V10.16B, V0.16B
+	sri	V9.4S, V4.4S, #8
+	sri	V8.4S, V11.4S, #24
+	eor	V4.16B, V10.16B, V9.16B
+	eor	V4.16B, V4.16B, V8.16B
 	eor	V8.16B, V4.16B, V12.16B
 	eor	V9.16B, V4.16B, V13.16B
 	eor	V10.16B, V4.16B, V14.16B
@@ -50667,27 +52883,24 @@ L_AES_XTS_decrypt_NEON_loop_nr_partial_2
 	orr	V9.16B, V9.16B, V10.16B
 	orr	V0.16B, V0.16B, V9.16B
 	tbl	V0.16B, {V0.16B}, V3.16B
-	ld1	{V4.2D}, [x25], #16
+	ld1	{V4.2D}, [x9], #16
 	;   XOR in Key Schedule
 	eor	V0.16B, V0.16B, V4.16B
 	rev32	V0.16B, V0.16B
-	eor	V0.16B, V0.16B, V2.16B
-	st1	{V0.16B}, [x1]
-L_AES_XTS_decrypt_NEON_data_done
-	ldp	x17, x19, [x29, #16]
-	ldp	x20, x21, [x29, #32]
-	ldp	x22, x23, [x29, #48]
-	ldp	x24, x25, [x29, #64]
-	ldp	D8, D9, [x29, #80]
-	ldp	D10, D11, [x29, #96]
-	ldp	D12, D13, [x29, #112]
-	ldp	D14, D15, [x29, #128]
-	ldp	x29, x30, [sp], #0x90
+	ld1	{V4.16B}, [x0], #16
+	eor	V0.16B, V0.16B, V4.16B
+	st1	{V0.16B}, [x1], #16
+L_AES_GCMSIV_ctr_neon_data_done
+	str	w10, [x5]
+	ldp	D8, D9, [x29, #16]
+	ldp	D10, D11, [x29, #32]
+	ldp	D12, D13, [x29, #48]
+	ldp	D14, D15, [x29, #64]
+	ldp	x29, x30, [sp], #0x50
 	ret
 	ENDP
 	ENDIF
 	ENDIF
-	ENDIF
 	IF :LNOT::DEF:WOLFSSL_ARMASM_NEON_NO_TABLE_LOOKUP
 	IF :DEF:HAVE_AES_DECRYPT
 	AREA	|.rodata|, DATA, READONLY, ALIGN=4
@@ -53785,122 +55998,419 @@ L_AES_GCM_encrypt_loop_nr
 	eor	w9, w9, w15, lsl 16
 	bfi	x7, x9, #32, #32
 	;   XOR in Key Schedule
-	eor	x6, x6, x10
-	eor	x7, x7, x11
-	rev32	x6, x6
-	rev32	x7, x7
-	ldr	x10, [x0]
-	ldr	x11, [x0, #8]
-	eor	x6, x6, x10
-	eor	x7, x7, x11
-	str	x6, [x1]
-	str	x7, [x1, #8]
-	subs	x2, x2, #16
-	add	x0, x0, #16
-	add	x1, x1, #16
-	bne	L_AES_GCM_encrypt_loop_block
-	rev32	x16, x16
-	rev32	x17, x17
-	stp	x16, x17, [x5]
-	ldp	x17, x19, [x29, #16]
-	ldp	x20, x21, [x29, #32]
-	ldp	x29, x30, [sp], #48
-	ret
-	ENDP
-	ENDIF
-	IF :DEF:WOLFSSL_AES_XTS
-	AREA	|.text|, CODE, READONLY
-	ALIGN	4
-	EXPORT	AES_XTS_encrypt
-AES_XTS_encrypt PROC
-	stp	x29, x30, [sp, #-96]!
-	add	x29, sp, #0
-	stp	x17, x19, [x29, #24]
-	stp	x20, x21, [x29, #40]
-	stp	x22, x23, [x29, #56]
-	stp	x24, x25, [x29, #72]
-	str	x26, [x29, #88]
-	adrp	x8, L_AES_ARM64_te
-	add	x8, x8, L_AES_ARM64_te
-	mov	x9, #0x87
-	mov	x26, x5
-	ldp	x21, x22, [x3]
+	eor	x6, x6, x10
+	eor	x7, x7, x11
+	rev32	x6, x6
+	rev32	x7, x7
+	ldr	x10, [x0]
+	ldr	x11, [x0, #8]
+	eor	x6, x6, x10
+	eor	x7, x7, x11
+	str	x6, [x1]
+	str	x7, [x1, #8]
+	subs	x2, x2, #16
+	add	x0, x0, #16
+	add	x1, x1, #16
+	bne	L_AES_GCM_encrypt_loop_block
+	rev32	x16, x16
+	rev32	x17, x17
+	stp	x16, x17, [x5]
+	ldp	x17, x19, [x29, #16]
+	ldp	x20, x21, [x29, #32]
+	ldp	x29, x30, [sp], #48
+	ret
+	ENDP
+	ENDIF
+	IF :DEF:WOLFSSL_AES_XTS
+	AREA	|.text|, CODE, READONLY
+	ALIGN	4
+	EXPORT	AES_XTS_encrypt
+AES_XTS_encrypt PROC
+	stp	x29, x30, [sp, #-96]!
+	add	x29, sp, #0
+	stp	x17, x19, [x29, #24]
+	stp	x20, x21, [x29, #40]
+	stp	x22, x23, [x29, #56]
+	stp	x24, x25, [x29, #72]
+	str	x26, [x29, #88]
+	adrp	x8, L_AES_ARM64_te
+	add	x8, x8, L_AES_ARM64_te
+	mov	x9, #0x87
+	mov	x26, x5
+	ldp	x21, x22, [x3]
+	ldp	x14, x15, [x26], #16
+	rev32	x21, x21
+	rev32	x22, x22
+	; Round: 0 - XOR in key schedule
+	eor	x21, x21, x14
+	eor	x22, x22, x15
+	sub	w25, w7, #2
+L_AES_XTS_encrypt_loop_nr_tweak
+	ubfx	x14, x21, #48, #8
+	ubfx	x17, x21, #24, #8
+	ubfx	x19, x22, #8, #8
+	ubfx	x20, x22, #32, #8
+	ldr	x23, [x8]
+	ldr	x23, [x8, #64]
+	ldr	x23, [x8, #128]
+	ldr	x23, [x8, #192]
+	ldr	x23, [x8, #256]
+	ldr	x23, [x8, #320]
+	ldr	x23, [x8, #384]
+	ldr	x23, [x8, #448]
+	ldr	x23, [x8, #512]
+	ldr	x23, [x8, #576]
+	ldr	x23, [x8, #640]
+	ldr	x23, [x8, #704]
+	ldr	x23, [x8, #768]
+	ldr	x23, [x8, #832]
+	ldr	x23, [x8, #896]
+	ldr	x23, [x8, #960]
+	ldr	w14, [x8, x14, LSL 2]
+	ldr	w17, [x8, x17, LSL 2]
+	ldr	w19, [x8, x19, LSL 2]
+	ldr	w20, [x8, x20, LSL 2]
+	ubfx	x15, x22, #16, #8
+	eor	w14, w14, w17, ror 24
+	ubfx	x17, x21, #56, #8
+	eor	w14, w14, w19, ror 8
+	ubfx	x19, x22, #40, #8
+	eor	w14, w14, w20, ror 16
+	ubfx	x20, x21, #0, #8
+	ldr	w15, [x8, x15, LSL 2]
+	ldr	w17, [x8, x17, LSL 2]
+	ldr	w19, [x8, x19, LSL 2]
+	ldr	w20, [x8, x20, LSL 2]
+	ubfx	x16, x22, #48, #8
+	eor	w15, w15, w17, ror 24
+	ubfx	x17, x22, #24, #8
+	eor	w15, w15, w19, ror 8
+	ubfx	x19, x21, #8, #8
+	eor	w15, w15, w20, ror 16
+	ubfx	x20, x21, #32, #8
+	bfi	x14, x15, #32, #32
+	ldr	w16, [x8, x16, LSL 2]
+	ldr	w17, [x8, x17, LSL 2]
+	ldr	w19, [x8, x19, LSL 2]
+	ldr	w20, [x8, x20, LSL 2]
+	ubfx	x23, x22, #0, #8
+	eor	w16, w16, w17, ror 24
+	ubfx	x17, x21, #16, #8
+	eor	w16, w16, w19, ror 8
+	ubfx	x19, x22, #56, #8
+	eor	w15, w16, w20, ror 16
+	ubfx	x20, x21, #40, #8
+	ldr	w23, [x8, x23, LSL 2]
+	ldr	w19, [x8, x19, LSL 2]
+	ldr	w17, [x8, x17, LSL 2]
+	ldr	w20, [x8, x20, LSL 2]
+	eor	w19, w19, w23, ror 24
+	ldp	x21, x22, [x26], #16
+	eor	w17, w17, w19, ror 24
+	eor	w17, w17, w20, ror 8
+	bfi	x15, x17, #32, #32
+	;   XOR in Key Schedule
+	eor	x14, x14, x21
+	eor	x15, x15, x22
+	ubfx	x21, x14, #48, #8
+	ubfx	x24, x14, #24, #8
+	ubfx	x19, x15, #8, #8
+	ubfx	x20, x15, #32, #8
+	ldr	x16, [x8]
+	ldr	x16, [x8, #64]
+	ldr	x16, [x8, #128]
+	ldr	x16, [x8, #192]
+	ldr	x16, [x8, #256]
+	ldr	x16, [x8, #320]
+	ldr	x16, [x8, #384]
+	ldr	x16, [x8, #448]
+	ldr	x16, [x8, #512]
+	ldr	x16, [x8, #576]
+	ldr	x16, [x8, #640]
+	ldr	x16, [x8, #704]
+	ldr	x16, [x8, #768]
+	ldr	x16, [x8, #832]
+	ldr	x16, [x8, #896]
+	ldr	x16, [x8, #960]
+	ldr	w21, [x8, x21, LSL 2]
+	ldr	w24, [x8, x24, LSL 2]
+	ldr	w19, [x8, x19, LSL 2]
+	ldr	w20, [x8, x20, LSL 2]
+	ubfx	x22, x15, #16, #8
+	eor	w21, w21, w24, ror 24
+	ubfx	x24, x14, #56, #8
+	eor	w21, w21, w19, ror 8
+	ubfx	x19, x15, #40, #8
+	eor	w21, w21, w20, ror 16
+	ubfx	x20, x14, #0, #8
+	ldr	w22, [x8, x22, LSL 2]
+	ldr	w24, [x8, x24, LSL 2]
+	ldr	w19, [x8, x19, LSL 2]
+	ldr	w20, [x8, x20, LSL 2]
+	ubfx	x23, x15, #48, #8
+	eor	w22, w22, w24, ror 24
+	ubfx	x24, x15, #24, #8
+	eor	w22, w22, w19, ror 8
+	ubfx	x19, x14, #8, #8
+	eor	w22, w22, w20, ror 16
+	ubfx	x20, x14, #32, #8
+	bfi	x21, x22, #32, #32
+	ldr	w23, [x8, x23, LSL 2]
+	ldr	w24, [x8, x24, LSL 2]
+	ldr	w19, [x8, x19, LSL 2]
+	ldr	w20, [x8, x20, LSL 2]
+	ubfx	x16, x15, #0, #8
+	eor	w23, w23, w24, ror 24
+	ubfx	x24, x14, #16, #8
+	eor	w23, w23, w19, ror 8
+	ubfx	x19, x15, #56, #8
+	eor	w22, w23, w20, ror 16
+	ubfx	x20, x14, #40, #8
+	ldr	w16, [x8, x16, LSL 2]
+	ldr	w19, [x8, x19, LSL 2]
+	ldr	w24, [x8, x24, LSL 2]
+	ldr	w20, [x8, x20, LSL 2]
+	eor	w19, w19, w16, ror 24
 	ldp	x14, x15, [x26], #16
+	eor	w24, w24, w19, ror 24
+	eor	w24, w24, w20, ror 8
+	bfi	x22, x24, #32, #32
+	;   XOR in Key Schedule
+	eor	x21, x21, x14
+	eor	x22, x22, x15
+	subs	w25, w25, #2
+	bne	L_AES_XTS_encrypt_loop_nr_tweak
+	ubfx	x14, x21, #48, #8
+	ubfx	x17, x21, #24, #8
+	ubfx	x19, x22, #8, #8
+	ubfx	x20, x22, #32, #8
+	ldr	x23, [x8]
+	ldr	x23, [x8, #64]
+	ldr	x23, [x8, #128]
+	ldr	x23, [x8, #192]
+	ldr	x23, [x8, #256]
+	ldr	x23, [x8, #320]
+	ldr	x23, [x8, #384]
+	ldr	x23, [x8, #448]
+	ldr	x23, [x8, #512]
+	ldr	x23, [x8, #576]
+	ldr	x23, [x8, #640]
+	ldr	x23, [x8, #704]
+	ldr	x23, [x8, #768]
+	ldr	x23, [x8, #832]
+	ldr	x23, [x8, #896]
+	ldr	x23, [x8, #960]
+	ldr	w14, [x8, x14, LSL 2]
+	ldr	w17, [x8, x17, LSL 2]
+	ldr	w19, [x8, x19, LSL 2]
+	ldr	w20, [x8, x20, LSL 2]
+	ubfx	x15, x22, #16, #8
+	eor	w14, w14, w17, ror 24
+	ubfx	x17, x21, #56, #8
+	eor	w14, w14, w19, ror 8
+	ubfx	x19, x22, #40, #8
+	eor	w14, w14, w20, ror 16
+	ubfx	x20, x21, #0, #8
+	ldr	w15, [x8, x15, LSL 2]
+	ldr	w17, [x8, x17, LSL 2]
+	ldr	w19, [x8, x19, LSL 2]
+	ldr	w20, [x8, x20, LSL 2]
+	ubfx	x16, x22, #48, #8
+	eor	w15, w15, w17, ror 24
+	ubfx	x17, x22, #24, #8
+	eor	w15, w15, w19, ror 8
+	ubfx	x19, x21, #8, #8
+	eor	w15, w15, w20, ror 16
+	ubfx	x20, x21, #32, #8
+	bfi	x14, x15, #32, #32
+	ldr	w16, [x8, x16, LSL 2]
+	ldr	w17, [x8, x17, LSL 2]
+	ldr	w19, [x8, x19, LSL 2]
+	ldr	w20, [x8, x20, LSL 2]
+	ubfx	x23, x22, #0, #8
+	eor	w16, w16, w17, ror 24
+	ubfx	x17, x21, #16, #8
+	eor	w16, w16, w19, ror 8
+	ubfx	x19, x22, #56, #8
+	eor	w15, w16, w20, ror 16
+	ubfx	x20, x21, #40, #8
+	ldr	w23, [x8, x23, LSL 2]
+	ldr	w19, [x8, x19, LSL 2]
+	ldr	w17, [x8, x17, LSL 2]
+	ldr	w20, [x8, x20, LSL 2]
+	eor	w19, w19, w23, ror 24
+	ldp	x21, x22, [x26], #16
+	eor	w17, w17, w19, ror 24
+	eor	w17, w17, w20, ror 8
+	bfi	x15, x17, #32, #32
+	;   XOR in Key Schedule
+	eor	x14, x14, x21
+	eor	x15, x15, x22
+	ubfx	x21, x15, #32, #8
+	ubfx	x24, x15, #8, #8
+	ubfx	x19, x14, #48, #8
+	ubfx	x20, x14, #24, #8
+	lsl	w21, w21, #2
+	lsl	w24, w24, #2
+	lsl	w19, w19, #2
+	lsl	w20, w20, #2
+	ldr	x17, [x8]
+	ldr	x17, [x8, #64]
+	ldr	x17, [x8, #128]
+	ldr	x17, [x8, #192]
+	ldr	x17, [x8, #256]
+	ldr	x17, [x8, #320]
+	ldr	x17, [x8, #384]
+	ldr	x17, [x8, #448]
+	ldr	x17, [x8, #512]
+	ldr	x17, [x8, #576]
+	ldr	x17, [x8, #640]
+	ldr	x17, [x8, #704]
+	ldr	x17, [x8, #768]
+	ldr	x17, [x8, #832]
+	ldr	x17, [x8, #896]
+	ldr	x17, [x8, #960]
+	ldrb	w21, [x8, x21, LSL 0]
+	ldrb	w24, [x8, x24, LSL 0]
+	ldrb	w19, [x8, x19, LSL 0]
+	ldrb	w20, [x8, x20, LSL 0]
+	ubfx	x22, x14, #0, #8
+	eor	w21, w21, w24, lsl 8
+	ubfx	x24, x15, #40, #8
+	eor	w21, w21, w19, lsl 16
+	ubfx	x19, x15, #16, #8
+	eor	w21, w21, w20, lsl 24
+	ubfx	x20, x14, #56, #8
+	lsl	w22, w22, #2
+	lsl	w24, w24, #2
+	lsl	w19, w19, #2
+	lsl	w20, w20, #2
+	ldrb	w22, [x8, x22, LSL 0]
+	ldrb	w24, [x8, x24, LSL 0]
+	ldrb	w19, [x8, x19, LSL 0]
+	ldrb	w20, [x8, x20, LSL 0]
+	ubfx	x23, x14, #32, #8
+	eor	w22, w22, w24, lsl 8
+	ubfx	x24, x14, #8, #8
+	eor	w22, w22, w19, lsl 16
+	ubfx	x19, x15, #48, #8
+	eor	w22, w22, w20, lsl 24
+	ubfx	x20, x15, #24, #8
+	bfi	x21, x22, #32, #32
+	lsl	w23, w23, #2
+	lsl	w24, w24, #2
+	lsl	w19, w19, #2
+	lsl	w20, w20, #2
+	ldrb	w23, [x8, x23, LSL 0]
+	ldrb	w24, [x8, x24, LSL 0]
+	ldrb	w19, [x8, x19, LSL 0]
+	ldrb	w20, [x8, x20, LSL 0]
+	ubfx	x17, x15, #56, #8
+	eor	w23, w23, w24, lsl 8
+	ubfx	x24, x15, #0, #8
+	eor	w23, w23, w19, lsl 16
+	ubfx	x19, x14, #40, #8
+	eor	w22, w23, w20, lsl 24
+	ubfx	x20, x14, #16, #8
+	lsl	w17, w17, #2
+	lsl	w24, w24, #2
+	lsl	w19, w19, #2
+	lsl	w20, w20, #2
+	ldrb	w17, [x8, x17, LSL 0]
+	ldrb	w24, [x8, x24, LSL 0]
+	ldrb	w19, [x8, x19, LSL 0]
+	ldrb	w20, [x8, x20, LSL 0]
+	eor	w19, w19, w17, lsl 16
+	ldp	x14, x15, [x26]
+	eor	w24, w24, w19, lsl 8
+	eor	w24, w24, w20, lsl 16
+	bfi	x22, x24, #32, #32
+	;   XOR in Key Schedule
+	eor	x21, x21, x14
+	eor	x22, x22, x15
 	rev32	x21, x21
 	rev32	x22, x22
+L_AES_XTS_encrypt_loop_block
+	mov	x26, x4
+	ldp	x10, x11, [x0]
+	ldp	x14, x15, [x26], #16
+	eor	x10, x10, x21
+	eor	x11, x11, x22
+	rev32	x10, x10
+	rev32	x11, x11
 	; Round: 0 - XOR in key schedule
-	eor	x21, x21, x14
-	eor	x22, x22, x15
+	eor	x10, x10, x14
+	eor	x11, x11, x15
 	sub	w25, w7, #2
-L_AES_XTS_encrypt_loop_nr_tweak
-	ubfx	x14, x21, #48, #8
-	ubfx	x17, x21, #24, #8
-	ubfx	x19, x22, #8, #8
-	ubfx	x20, x22, #32, #8
-	ldr	x23, [x8]
-	ldr	x23, [x8, #64]
-	ldr	x23, [x8, #128]
-	ldr	x23, [x8, #192]
-	ldr	x23, [x8, #256]
-	ldr	x23, [x8, #320]
-	ldr	x23, [x8, #384]
-	ldr	x23, [x8, #448]
-	ldr	x23, [x8, #512]
-	ldr	x23, [x8, #576]
-	ldr	x23, [x8, #640]
-	ldr	x23, [x8, #704]
-	ldr	x23, [x8, #768]
-	ldr	x23, [x8, #832]
-	ldr	x23, [x8, #896]
-	ldr	x23, [x8, #960]
+L_AES_XTS_encrypt_loop_nr
+	ubfx	x14, x10, #48, #8
+	ubfx	x17, x10, #24, #8
+	ubfx	x19, x11, #8, #8
+	ubfx	x20, x11, #32, #8
+	ldr	x12, [x8]
+	ldr	x12, [x8, #64]
+	ldr	x12, [x8, #128]
+	ldr	x12, [x8, #192]
+	ldr	x12, [x8, #256]
+	ldr	x12, [x8, #320]
+	ldr	x12, [x8, #384]
+	ldr	x12, [x8, #448]
+	ldr	x12, [x8, #512]
+	ldr	x12, [x8, #576]
+	ldr	x12, [x8, #640]
+	ldr	x12, [x8, #704]
+	ldr	x12, [x8, #768]
+	ldr	x12, [x8, #832]
+	ldr	x12, [x8, #896]
+	ldr	x12, [x8, #960]
 	ldr	w14, [x8, x14, LSL 2]
 	ldr	w17, [x8, x17, LSL 2]
 	ldr	w19, [x8, x19, LSL 2]
 	ldr	w20, [x8, x20, LSL 2]
-	ubfx	x15, x22, #16, #8
+	ubfx	x15, x11, #16, #8
 	eor	w14, w14, w17, ror 24
-	ubfx	x17, x21, #56, #8
+	ubfx	x17, x10, #56, #8
 	eor	w14, w14, w19, ror 8
-	ubfx	x19, x22, #40, #8
+	ubfx	x19, x11, #40, #8
 	eor	w14, w14, w20, ror 16
-	ubfx	x20, x21, #0, #8
+	ubfx	x20, x10, #0, #8
 	ldr	w15, [x8, x15, LSL 2]
 	ldr	w17, [x8, x17, LSL 2]
 	ldr	w19, [x8, x19, LSL 2]
 	ldr	w20, [x8, x20, LSL 2]
-	ubfx	x16, x22, #48, #8
+	ubfx	x16, x11, #48, #8
 	eor	w15, w15, w17, ror 24
-	ubfx	x17, x22, #24, #8
+	ubfx	x17, x11, #24, #8
 	eor	w15, w15, w19, ror 8
-	ubfx	x19, x21, #8, #8
+	ubfx	x19, x10, #8, #8
 	eor	w15, w15, w20, ror 16
-	ubfx	x20, x21, #32, #8
+	ubfx	x20, x10, #32, #8
 	bfi	x14, x15, #32, #32
 	ldr	w16, [x8, x16, LSL 2]
 	ldr	w17, [x8, x17, LSL 2]
 	ldr	w19, [x8, x19, LSL 2]
 	ldr	w20, [x8, x20, LSL 2]
-	ubfx	x23, x22, #0, #8
+	ubfx	x12, x11, #0, #8
 	eor	w16, w16, w17, ror 24
-	ubfx	x17, x21, #16, #8
+	ubfx	x17, x10, #16, #8
 	eor	w16, w16, w19, ror 8
-	ubfx	x19, x22, #56, #8
+	ubfx	x19, x11, #56, #8
 	eor	w15, w16, w20, ror 16
-	ubfx	x20, x21, #40, #8
-	ldr	w23, [x8, x23, LSL 2]
+	ubfx	x20, x10, #40, #8
+	ldr	w12, [x8, x12, LSL 2]
 	ldr	w19, [x8, x19, LSL 2]
 	ldr	w17, [x8, x17, LSL 2]
 	ldr	w20, [x8, x20, LSL 2]
-	eor	w19, w19, w23, ror 24
-	ldp	x21, x22, [x26], #16
+	eor	w19, w19, w12, ror 24
+	ldp	x10, x11, [x26], #16
 	eor	w17, w17, w19, ror 24
 	eor	w17, w17, w20, ror 8
 	bfi	x15, x17, #32, #32
 	;   XOR in Key Schedule
-	eor	x14, x14, x21
-	eor	x15, x15, x22
-	ubfx	x21, x14, #48, #8
-	ubfx	x24, x14, #24, #8
+	eor	x14, x14, x10
+	eor	x15, x15, x11
+	ubfx	x10, x14, #48, #8
+	ubfx	x13, x14, #24, #8
 	ubfx	x19, x15, #8, #8
 	ubfx	x20, x15, #32, #8
 	ldr	x16, [x8]
@@ -53919,126 +56429,126 @@ L_AES_XTS_encrypt_loop_nr_tweak
 	ldr	x16, [x8, #832]
 	ldr	x16, [x8, #896]
 	ldr	x16, [x8, #960]
-	ldr	w21, [x8, x21, LSL 2]
-	ldr	w24, [x8, x24, LSL 2]
+	ldr	w10, [x8, x10, LSL 2]
+	ldr	w13, [x8, x13, LSL 2]
 	ldr	w19, [x8, x19, LSL 2]
 	ldr	w20, [x8, x20, LSL 2]
-	ubfx	x22, x15, #16, #8
-	eor	w21, w21, w24, ror 24
-	ubfx	x24, x14, #56, #8
-	eor	w21, w21, w19, ror 8
+	ubfx	x11, x15, #16, #8
+	eor	w10, w10, w13, ror 24
+	ubfx	x13, x14, #56, #8
+	eor	w10, w10, w19, ror 8
 	ubfx	x19, x15, #40, #8
-	eor	w21, w21, w20, ror 16
+	eor	w10, w10, w20, ror 16
 	ubfx	x20, x14, #0, #8
-	ldr	w22, [x8, x22, LSL 2]
-	ldr	w24, [x8, x24, LSL 2]
+	ldr	w11, [x8, x11, LSL 2]
+	ldr	w13, [x8, x13, LSL 2]
 	ldr	w19, [x8, x19, LSL 2]
 	ldr	w20, [x8, x20, LSL 2]
-	ubfx	x23, x15, #48, #8
-	eor	w22, w22, w24, ror 24
-	ubfx	x24, x15, #24, #8
-	eor	w22, w22, w19, ror 8
+	ubfx	x12, x15, #48, #8
+	eor	w11, w11, w13, ror 24
+	ubfx	x13, x15, #24, #8
+	eor	w11, w11, w19, ror 8
 	ubfx	x19, x14, #8, #8
-	eor	w22, w22, w20, ror 16
+	eor	w11, w11, w20, ror 16
 	ubfx	x20, x14, #32, #8
-	bfi	x21, x22, #32, #32
-	ldr	w23, [x8, x23, LSL 2]
-	ldr	w24, [x8, x24, LSL 2]
+	bfi	x10, x11, #32, #32
+	ldr	w12, [x8, x12, LSL 2]
+	ldr	w13, [x8, x13, LSL 2]
 	ldr	w19, [x8, x19, LSL 2]
 	ldr	w20, [x8, x20, LSL 2]
 	ubfx	x16, x15, #0, #8
-	eor	w23, w23, w24, ror 24
-	ubfx	x24, x14, #16, #8
-	eor	w23, w23, w19, ror 8
+	eor	w12, w12, w13, ror 24
+	ubfx	x13, x14, #16, #8
+	eor	w12, w12, w19, ror 8
 	ubfx	x19, x15, #56, #8
-	eor	w22, w23, w20, ror 16
+	eor	w11, w12, w20, ror 16
 	ubfx	x20, x14, #40, #8
 	ldr	w16, [x8, x16, LSL 2]
 	ldr	w19, [x8, x19, LSL 2]
-	ldr	w24, [x8, x24, LSL 2]
+	ldr	w13, [x8, x13, LSL 2]
 	ldr	w20, [x8, x20, LSL 2]
 	eor	w19, w19, w16, ror 24
 	ldp	x14, x15, [x26], #16
-	eor	w24, w24, w19, ror 24
-	eor	w24, w24, w20, ror 8
-	bfi	x22, x24, #32, #32
+	eor	w13, w13, w19, ror 24
+	eor	w13, w13, w20, ror 8
+	bfi	x11, x13, #32, #32
 	;   XOR in Key Schedule
-	eor	x21, x21, x14
-	eor	x22, x22, x15
+	eor	x10, x10, x14
+	eor	x11, x11, x15
 	subs	w25, w25, #2
-	bne	L_AES_XTS_encrypt_loop_nr_tweak
-	ubfx	x14, x21, #48, #8
-	ubfx	x17, x21, #24, #8
-	ubfx	x19, x22, #8, #8
-	ubfx	x20, x22, #32, #8
-	ldr	x23, [x8]
-	ldr	x23, [x8, #64]
-	ldr	x23, [x8, #128]
-	ldr	x23, [x8, #192]
-	ldr	x23, [x8, #256]
-	ldr	x23, [x8, #320]
-	ldr	x23, [x8, #384]
-	ldr	x23, [x8, #448]
-	ldr	x23, [x8, #512]
-	ldr	x23, [x8, #576]
-	ldr	x23, [x8, #640]
-	ldr	x23, [x8, #704]
-	ldr	x23, [x8, #768]
-	ldr	x23, [x8, #832]
-	ldr	x23, [x8, #896]
-	ldr	x23, [x8, #960]
+	bne	L_AES_XTS_encrypt_loop_nr
+	ubfx	x14, x10, #48, #8
+	ubfx	x17, x10, #24, #8
+	ubfx	x19, x11, #8, #8
+	ubfx	x20, x11, #32, #8
+	ldr	x12, [x8]
+	ldr	x12, [x8, #64]
+	ldr	x12, [x8, #128]
+	ldr	x12, [x8, #192]
+	ldr	x12, [x8, #256]
+	ldr	x12, [x8, #320]
+	ldr	x12, [x8, #384]
+	ldr	x12, [x8, #448]
+	ldr	x12, [x8, #512]
+	ldr	x12, [x8, #576]
+	ldr	x12, [x8, #640]
+	ldr	x12, [x8, #704]
+	ldr	x12, [x8, #768]
+	ldr	x12, [x8, #832]
+	ldr	x12, [x8, #896]
+	ldr	x12, [x8, #960]
 	ldr	w14, [x8, x14, LSL 2]
 	ldr	w17, [x8, x17, LSL 2]
 	ldr	w19, [x8, x19, LSL 2]
 	ldr	w20, [x8, x20, LSL 2]
-	ubfx	x15, x22, #16, #8
+	ubfx	x15, x11, #16, #8
 	eor	w14, w14, w17, ror 24
-	ubfx	x17, x21, #56, #8
+	ubfx	x17, x10, #56, #8
 	eor	w14, w14, w19, ror 8
-	ubfx	x19, x22, #40, #8
+	ubfx	x19, x11, #40, #8
 	eor	w14, w14, w20, ror 16
-	ubfx	x20, x21, #0, #8
+	ubfx	x20, x10, #0, #8
 	ldr	w15, [x8, x15, LSL 2]
 	ldr	w17, [x8, x17, LSL 2]
 	ldr	w19, [x8, x19, LSL 2]
 	ldr	w20, [x8, x20, LSL 2]
-	ubfx	x16, x22, #48, #8
+	ubfx	x16, x11, #48, #8
 	eor	w15, w15, w17, ror 24
-	ubfx	x17, x22, #24, #8
+	ubfx	x17, x11, #24, #8
 	eor	w15, w15, w19, ror 8
-	ubfx	x19, x21, #8, #8
+	ubfx	x19, x10, #8, #8
 	eor	w15, w15, w20, ror 16
-	ubfx	x20, x21, #32, #8
+	ubfx	x20, x10, #32, #8
 	bfi	x14, x15, #32, #32
 	ldr	w16, [x8, x16, LSL 2]
 	ldr	w17, [x8, x17, LSL 2]
 	ldr	w19, [x8, x19, LSL 2]
 	ldr	w20, [x8, x20, LSL 2]
-	ubfx	x23, x22, #0, #8
+	ubfx	x12, x11, #0, #8
 	eor	w16, w16, w17, ror 24
-	ubfx	x17, x21, #16, #8
+	ubfx	x17, x10, #16, #8
 	eor	w16, w16, w19, ror 8
-	ubfx	x19, x22, #56, #8
+	ubfx	x19, x11, #56, #8
 	eor	w15, w16, w20, ror 16
-	ubfx	x20, x21, #40, #8
-	ldr	w23, [x8, x23, LSL 2]
+	ubfx	x20, x10, #40, #8
+	ldr	w12, [x8, x12, LSL 2]
 	ldr	w19, [x8, x19, LSL 2]
 	ldr	w17, [x8, x17, LSL 2]
 	ldr	w20, [x8, x20, LSL 2]
-	eor	w19, w19, w23, ror 24
-	ldp	x21, x22, [x26], #16
+	eor	w19, w19, w12, ror 24
+	ldp	x10, x11, [x26], #16
 	eor	w17, w17, w19, ror 24
 	eor	w17, w17, w20, ror 8
 	bfi	x15, x17, #32, #32
 	;   XOR in Key Schedule
-	eor	x14, x14, x21
-	eor	x15, x15, x22
-	ubfx	x21, x15, #32, #8
-	ubfx	x24, x15, #8, #8
+	eor	x14, x14, x10
+	eor	x15, x15, x11
+	ubfx	x10, x15, #32, #8
+	ubfx	x13, x15, #8, #8
 	ubfx	x19, x14, #48, #8
 	ubfx	x20, x14, #24, #8
-	lsl	w21, w21, #2
-	lsl	w24, w24, #2
+	lsl	w10, w10, #2
+	lsl	w13, w13, #2
 	lsl	w19, w19, #2
 	lsl	w20, w20, #2
 	ldr	x17, [x8]
@@ -54057,69 +56567,94 @@ L_AES_XTS_encrypt_loop_nr_tweak
 	ldr	x17, [x8, #832]
 	ldr	x17, [x8, #896]
 	ldr	x17, [x8, #960]
-	ldrb	w21, [x8, x21, LSL 0]
-	ldrb	w24, [x8, x24, LSL 0]
+	ldrb	w10, [x8, x10, LSL 0]
+	ldrb	w13, [x8, x13, LSL 0]
 	ldrb	w19, [x8, x19, LSL 0]
-	ldrb	w20, [x8, x20, LSL 0]
-	ubfx	x22, x14, #0, #8
-	eor	w21, w21, w24, lsl 8
-	ubfx	x24, x15, #40, #8
-	eor	w21, w21, w19, lsl 16
+	ldrb	w20, [x8, x20, LSL 0]
+	ubfx	x11, x14, #0, #8
+	eor	w10, w10, w13, lsl 8
+	ubfx	x13, x15, #40, #8
+	eor	w10, w10, w19, lsl 16
 	ubfx	x19, x15, #16, #8
-	eor	w21, w21, w20, lsl 24
+	eor	w10, w10, w20, lsl 24
 	ubfx	x20, x14, #56, #8
-	lsl	w22, w22, #2
-	lsl	w24, w24, #2
+	lsl	w11, w11, #2
+	lsl	w13, w13, #2
 	lsl	w19, w19, #2
 	lsl	w20, w20, #2
-	ldrb	w22, [x8, x22, LSL 0]
-	ldrb	w24, [x8, x24, LSL 0]
+	ldrb	w11, [x8, x11, LSL 0]
+	ldrb	w13, [x8, x13, LSL 0]
 	ldrb	w19, [x8, x19, LSL 0]
 	ldrb	w20, [x8, x20, LSL 0]
-	ubfx	x23, x14, #32, #8
-	eor	w22, w22, w24, lsl 8
-	ubfx	x24, x14, #8, #8
-	eor	w22, w22, w19, lsl 16
+	ubfx	x12, x14, #32, #8
+	eor	w11, w11, w13, lsl 8
+	ubfx	x13, x14, #8, #8
+	eor	w11, w11, w19, lsl 16
 	ubfx	x19, x15, #48, #8
-	eor	w22, w22, w20, lsl 24
+	eor	w11, w11, w20, lsl 24
 	ubfx	x20, x15, #24, #8
-	bfi	x21, x22, #32, #32
-	lsl	w23, w23, #2
-	lsl	w24, w24, #2
+	bfi	x10, x11, #32, #32
+	lsl	w12, w12, #2
+	lsl	w13, w13, #2
 	lsl	w19, w19, #2
 	lsl	w20, w20, #2
-	ldrb	w23, [x8, x23, LSL 0]
-	ldrb	w24, [x8, x24, LSL 0]
+	ldrb	w12, [x8, x12, LSL 0]
+	ldrb	w13, [x8, x13, LSL 0]
 	ldrb	w19, [x8, x19, LSL 0]
 	ldrb	w20, [x8, x20, LSL 0]
 	ubfx	x17, x15, #56, #8
-	eor	w23, w23, w24, lsl 8
-	ubfx	x24, x15, #0, #8
-	eor	w23, w23, w19, lsl 16
+	eor	w12, w12, w13, lsl 8
+	ubfx	x13, x15, #0, #8
+	eor	w12, w12, w19, lsl 16
 	ubfx	x19, x14, #40, #8
-	eor	w22, w23, w20, lsl 24
+	eor	w11, w12, w20, lsl 24
 	ubfx	x20, x14, #16, #8
 	lsl	w17, w17, #2
-	lsl	w24, w24, #2
+	lsl	w13, w13, #2
 	lsl	w19, w19, #2
 	lsl	w20, w20, #2
 	ldrb	w17, [x8, x17, LSL 0]
-	ldrb	w24, [x8, x24, LSL 0]
+	ldrb	w13, [x8, x13, LSL 0]
 	ldrb	w19, [x8, x19, LSL 0]
 	ldrb	w20, [x8, x20, LSL 0]
 	eor	w19, w19, w17, lsl 16
 	ldp	x14, x15, [x26]
-	eor	w24, w24, w19, lsl 8
-	eor	w24, w24, w20, lsl 16
-	bfi	x22, x24, #32, #32
+	eor	w13, w13, w19, lsl 8
+	eor	w13, w13, w20, lsl 16
+	bfi	x11, x13, #32, #32
 	;   XOR in Key Schedule
-	eor	x21, x21, x14
-	eor	x22, x22, x15
-	rev32	x21, x21
-	rev32	x22, x22
-L_AES_XTS_encrypt_loop_block
+	eor	x10, x10, x14
+	eor	x11, x11, x15
+	rev32	x10, x10
+	rev32	x11, x11
+	eor	x10, x10, x21
+	eor	x11, x11, x22
+	stp	x10, x11, [x1]
+	and	x19, x9, x22, asr 63
+	extr	x22, x22, x21, #63
+	eor	x21, x19, x21, lsl 1
+	sub	w2, w2, #16
+	add	x0, x0, #16
+	add	x1, x1, #16
+	cmp	w2, #16
+	bge	L_AES_XTS_encrypt_loop_block
+	cbz	w2, L_AES_XTS_encrypt_done_data
 	mov	x26, x4
-	ldp	x10, x11, [x0]
+	sub	x1, x1, #16
+	ldp	x10, x11, [x1], #16
+	stp	x10, x11, [x6]
+	mov	w14, w2
+L_AES_XTS_encrypt_start_byte
+	ldrb	w19, [x6]
+	ldrb	w20, [x0], #1
+	strb	w19, [x1], #1
+	strb	w20, [x6], #1
+	subs	w14, w14, #1
+	bgt	L_AES_XTS_encrypt_start_byte
+	sub	x1, x1, x2
+	sub	x6, x6, x2
+	sub	x1, x1, #16
+	ldp	x10, x11, [x6]
 	ldp	x14, x15, [x26], #16
 	eor	x10, x10, x21
 	eor	x11, x11, x22
@@ -54129,7 +56664,7 @@ L_AES_XTS_encrypt_loop_block
 	eor	x10, x10, x14
 	eor	x11, x11, x15
 	sub	w25, w7, #2
-L_AES_XTS_encrypt_loop_nr
+L_AES_XTS_encrypt_loop_nr_partial
 	ubfx	x14, x10, #48, #8
 	ubfx	x17, x10, #24, #8
 	ubfx	x19, x11, #8, #8
@@ -54263,7 +56798,7 @@ L_AES_XTS_encrypt_loop_nr
 	eor	x10, x10, x14
 	eor	x11, x11, x15
 	subs	w25, w25, #2
-	bne	L_AES_XTS_encrypt_loop_nr
+	bne	L_AES_XTS_encrypt_loop_nr_partial
 	ubfx	x14, x10, #48, #8
 	ubfx	x17, x10, #24, #8
 	ubfx	x19, x11, #8, #8
@@ -54417,669 +56952,631 @@ L_AES_XTS_encrypt_loop_nr
 	eor	x10, x10, x21
 	eor	x11, x11, x22
 	stp	x10, x11, [x1]
-	and	x19, x9, x22, asr 63
-	extr	x22, x22, x21, #63
-	eor	x21, x19, x21, lsl 1
-	sub	w2, w2, #16
-	add	x0, x0, #16
-	add	x1, x1, #16
-	cmp	w2, #16
-	bge	L_AES_XTS_encrypt_loop_block
-	cbz	w2, L_AES_XTS_encrypt_done_data
-	mov	x26, x4
-	sub	x1, x1, #16
-	ldp	x10, x11, [x1], #16
-	stp	x10, x11, [x6]
-	mov	w14, w2
-L_AES_XTS_encrypt_start_byte
-	ldrb	w19, [x6]
-	ldrb	w20, [x0], #1
-	strb	w19, [x1], #1
-	strb	w20, [x6], #1
-	subs	w14, w14, #1
-	bgt	L_AES_XTS_encrypt_start_byte
-	sub	x1, x1, x2
-	sub	x6, x6, x2
-	sub	x1, x1, #16
-	ldp	x10, x11, [x6]
-	ldp	x14, x15, [x26], #16
-	eor	x10, x10, x21
-	eor	x11, x11, x22
-	rev32	x10, x10
-	rev32	x11, x11
+L_AES_XTS_encrypt_done_data
+	ldp	x17, x19, [x29, #24]
+	ldp	x20, x21, [x29, #40]
+	ldp	x22, x23, [x29, #56]
+	ldp	x24, x25, [x29, #72]
+	ldr	x26, [x29, #88]
+	ldp	x29, x30, [sp], #0x60
+	ret
+	ENDP
+	IF :DEF:HAVE_AES_DECRYPT
+	AREA	|.text|, CODE, READONLY
+	ALIGN	4
+	EXPORT	AES_XTS_decrypt
+AES_XTS_decrypt PROC
+	stp	x29, x30, [sp, #-112]!
+	add	x29, sp, #0
+	stp	x17, x19, [x29, #24]
+	stp	x20, x21, [x29, #40]
+	stp	x22, x23, [x29, #56]
+	stp	x24, x25, [x29, #72]
+	stp	x26, x27, [x29, #88]
+	str	x28, [x29, #104]
+	adrp	x8, L_AES_ARM64_td
+	add	x8, x8, L_AES_ARM64_td
+	adrp	x9, L_AES_ARM64_td4
+	add	x9, x9, L_AES_ARM64_td4
+	adrp	x10, L_AES_ARM64_te
+	add	x10, x10, L_AES_ARM64_te
+	ands	w11, w2, #15
+	cset	w11, ne
+	sub	w2, w2, w11, lsl 4
+	mov	x11, #0x87
+	mov	x28, x5
+	ldp	x23, x24, [x3]
+	ldp	x16, x17, [x28], #16
+	rev32	x23, x23
+	rev32	x24, x24
 	; Round: 0 - XOR in key schedule
-	eor	x10, x10, x14
-	eor	x11, x11, x15
-	sub	w25, w7, #2
-L_AES_XTS_encrypt_loop_nr_partial
-	ubfx	x14, x10, #48, #8
-	ubfx	x17, x10, #24, #8
-	ubfx	x19, x11, #8, #8
-	ubfx	x20, x11, #32, #8
-	ldr	x12, [x8]
-	ldr	x12, [x8, #64]
-	ldr	x12, [x8, #128]
-	ldr	x12, [x8, #192]
-	ldr	x12, [x8, #256]
-	ldr	x12, [x8, #320]
-	ldr	x12, [x8, #384]
-	ldr	x12, [x8, #448]
-	ldr	x12, [x8, #512]
-	ldr	x12, [x8, #576]
-	ldr	x12, [x8, #640]
-	ldr	x12, [x8, #704]
-	ldr	x12, [x8, #768]
-	ldr	x12, [x8, #832]
-	ldr	x12, [x8, #896]
-	ldr	x12, [x8, #960]
-	ldr	w14, [x8, x14, LSL 2]
-	ldr	w17, [x8, x17, LSL 2]
-	ldr	w19, [x8, x19, LSL 2]
-	ldr	w20, [x8, x20, LSL 2]
-	ubfx	x15, x11, #16, #8
-	eor	w14, w14, w17, ror 24
-	ubfx	x17, x10, #56, #8
-	eor	w14, w14, w19, ror 8
-	ubfx	x19, x11, #40, #8
-	eor	w14, w14, w20, ror 16
-	ubfx	x20, x10, #0, #8
-	ldr	w15, [x8, x15, LSL 2]
-	ldr	w17, [x8, x17, LSL 2]
-	ldr	w19, [x8, x19, LSL 2]
-	ldr	w20, [x8, x20, LSL 2]
-	ubfx	x16, x11, #48, #8
-	eor	w15, w15, w17, ror 24
-	ubfx	x17, x11, #24, #8
-	eor	w15, w15, w19, ror 8
-	ubfx	x19, x10, #8, #8
-	eor	w15, w15, w20, ror 16
-	ubfx	x20, x10, #32, #8
-	bfi	x14, x15, #32, #32
-	ldr	w16, [x8, x16, LSL 2]
-	ldr	w17, [x8, x17, LSL 2]
-	ldr	w19, [x8, x19, LSL 2]
-	ldr	w20, [x8, x20, LSL 2]
-	ubfx	x12, x11, #0, #8
-	eor	w16, w16, w17, ror 24
-	ubfx	x17, x10, #16, #8
-	eor	w16, w16, w19, ror 8
-	ubfx	x19, x11, #56, #8
-	eor	w15, w16, w20, ror 16
-	ubfx	x20, x10, #40, #8
-	ldr	w12, [x8, x12, LSL 2]
-	ldr	w19, [x8, x19, LSL 2]
-	ldr	w17, [x8, x17, LSL 2]
-	ldr	w20, [x8, x20, LSL 2]
-	eor	w19, w19, w12, ror 24
-	ldp	x10, x11, [x26], #16
-	eor	w17, w17, w19, ror 24
-	eor	w17, w17, w20, ror 8
-	bfi	x15, x17, #32, #32
+	eor	x23, x23, x16
+	eor	x24, x24, x17
+	sub	w27, w7, #2
+L_AES_XTS_decrypt_loop_nr_tweak
+	ubfx	x16, x23, #48, #8
+	ubfx	x20, x23, #24, #8
+	ubfx	x21, x24, #8, #8
+	ubfx	x22, x24, #32, #8
+	ldr	x25, [x10]
+	ldr	x25, [x10, #64]
+	ldr	x25, [x10, #128]
+	ldr	x25, [x10, #192]
+	ldr	x25, [x10, #256]
+	ldr	x25, [x10, #320]
+	ldr	x25, [x10, #384]
+	ldr	x25, [x10, #448]
+	ldr	x25, [x10, #512]
+	ldr	x25, [x10, #576]
+	ldr	x25, [x10, #640]
+	ldr	x25, [x10, #704]
+	ldr	x25, [x10, #768]
+	ldr	x25, [x10, #832]
+	ldr	x25, [x10, #896]
+	ldr	x25, [x10, #960]
+	ldr	w16, [x10, x16, LSL 2]
+	ldr	w20, [x10, x20, LSL 2]
+	ldr	w21, [x10, x21, LSL 2]
+	ldr	w22, [x10, x22, LSL 2]
+	ubfx	x17, x24, #16, #8
+	eor	w16, w16, w20, ror 24
+	ubfx	x20, x23, #56, #8
+	eor	w16, w16, w21, ror 8
+	ubfx	x21, x24, #40, #8
+	eor	w16, w16, w22, ror 16
+	ubfx	x22, x23, #0, #8
+	ldr	w17, [x10, x17, LSL 2]
+	ldr	w20, [x10, x20, LSL 2]
+	ldr	w21, [x10, x21, LSL 2]
+	ldr	w22, [x10, x22, LSL 2]
+	ubfx	x19, x24, #48, #8
+	eor	w17, w17, w20, ror 24
+	ubfx	x20, x24, #24, #8
+	eor	w17, w17, w21, ror 8
+	ubfx	x21, x23, #8, #8
+	eor	w17, w17, w22, ror 16
+	ubfx	x22, x23, #32, #8
+	bfi	x16, x17, #32, #32
+	ldr	w19, [x10, x19, LSL 2]
+	ldr	w20, [x10, x20, LSL 2]
+	ldr	w21, [x10, x21, LSL 2]
+	ldr	w22, [x10, x22, LSL 2]
+	ubfx	x25, x24, #0, #8
+	eor	w19, w19, w20, ror 24
+	ubfx	x20, x23, #16, #8
+	eor	w19, w19, w21, ror 8
+	ubfx	x21, x24, #56, #8
+	eor	w17, w19, w22, ror 16
+	ubfx	x22, x23, #40, #8
+	ldr	w25, [x10, x25, LSL 2]
+	ldr	w21, [x10, x21, LSL 2]
+	ldr	w20, [x10, x20, LSL 2]
+	ldr	w22, [x10, x22, LSL 2]
+	eor	w21, w21, w25, ror 24
+	ldp	x23, x24, [x28], #16
+	eor	w20, w20, w21, ror 24
+	eor	w20, w20, w22, ror 8
+	bfi	x17, x20, #32, #32
 	;   XOR in Key Schedule
-	eor	x14, x14, x10
-	eor	x15, x15, x11
-	ubfx	x10, x14, #48, #8
-	ubfx	x13, x14, #24, #8
-	ubfx	x19, x15, #8, #8
-	ubfx	x20, x15, #32, #8
-	ldr	x16, [x8]
-	ldr	x16, [x8, #64]
-	ldr	x16, [x8, #128]
-	ldr	x16, [x8, #192]
-	ldr	x16, [x8, #256]
-	ldr	x16, [x8, #320]
-	ldr	x16, [x8, #384]
-	ldr	x16, [x8, #448]
-	ldr	x16, [x8, #512]
-	ldr	x16, [x8, #576]
-	ldr	x16, [x8, #640]
-	ldr	x16, [x8, #704]
-	ldr	x16, [x8, #768]
-	ldr	x16, [x8, #832]
-	ldr	x16, [x8, #896]
-	ldr	x16, [x8, #960]
-	ldr	w10, [x8, x10, LSL 2]
-	ldr	w13, [x8, x13, LSL 2]
-	ldr	w19, [x8, x19, LSL 2]
-	ldr	w20, [x8, x20, LSL 2]
-	ubfx	x11, x15, #16, #8
-	eor	w10, w10, w13, ror 24
-	ubfx	x13, x14, #56, #8
-	eor	w10, w10, w19, ror 8
-	ubfx	x19, x15, #40, #8
-	eor	w10, w10, w20, ror 16
-	ubfx	x20, x14, #0, #8
-	ldr	w11, [x8, x11, LSL 2]
-	ldr	w13, [x8, x13, LSL 2]
-	ldr	w19, [x8, x19, LSL 2]
-	ldr	w20, [x8, x20, LSL 2]
-	ubfx	x12, x15, #48, #8
-	eor	w11, w11, w13, ror 24
-	ubfx	x13, x15, #24, #8
-	eor	w11, w11, w19, ror 8
-	ubfx	x19, x14, #8, #8
-	eor	w11, w11, w20, ror 16
-	ubfx	x20, x14, #32, #8
-	bfi	x10, x11, #32, #32
-	ldr	w12, [x8, x12, LSL 2]
-	ldr	w13, [x8, x13, LSL 2]
-	ldr	w19, [x8, x19, LSL 2]
-	ldr	w20, [x8, x20, LSL 2]
-	ubfx	x16, x15, #0, #8
-	eor	w12, w12, w13, ror 24
-	ubfx	x13, x14, #16, #8
-	eor	w12, w12, w19, ror 8
-	ubfx	x19, x15, #56, #8
-	eor	w11, w12, w20, ror 16
-	ubfx	x20, x14, #40, #8
-	ldr	w16, [x8, x16, LSL 2]
-	ldr	w19, [x8, x19, LSL 2]
-	ldr	w13, [x8, x13, LSL 2]
-	ldr	w20, [x8, x20, LSL 2]
-	eor	w19, w19, w16, ror 24
-	ldp	x14, x15, [x26], #16
-	eor	w13, w13, w19, ror 24
-	eor	w13, w13, w20, ror 8
-	bfi	x11, x13, #32, #32
+	eor	x16, x16, x23
+	eor	x17, x17, x24
+	ubfx	x23, x16, #48, #8
+	ubfx	x26, x16, #24, #8
+	ubfx	x21, x17, #8, #8
+	ubfx	x22, x17, #32, #8
+	ldr	x19, [x10]
+	ldr	x19, [x10, #64]
+	ldr	x19, [x10, #128]
+	ldr	x19, [x10, #192]
+	ldr	x19, [x10, #256]
+	ldr	x19, [x10, #320]
+	ldr	x19, [x10, #384]
+	ldr	x19, [x10, #448]
+	ldr	x19, [x10, #512]
+	ldr	x19, [x10, #576]
+	ldr	x19, [x10, #640]
+	ldr	x19, [x10, #704]
+	ldr	x19, [x10, #768]
+	ldr	x19, [x10, #832]
+	ldr	x19, [x10, #896]
+	ldr	x19, [x10, #960]
+	ldr	w23, [x10, x23, LSL 2]
+	ldr	w26, [x10, x26, LSL 2]
+	ldr	w21, [x10, x21, LSL 2]
+	ldr	w22, [x10, x22, LSL 2]
+	ubfx	x24, x17, #16, #8
+	eor	w23, w23, w26, ror 24
+	ubfx	x26, x16, #56, #8
+	eor	w23, w23, w21, ror 8
+	ubfx	x21, x17, #40, #8
+	eor	w23, w23, w22, ror 16
+	ubfx	x22, x16, #0, #8
+	ldr	w24, [x10, x24, LSL 2]
+	ldr	w26, [x10, x26, LSL 2]
+	ldr	w21, [x10, x21, LSL 2]
+	ldr	w22, [x10, x22, LSL 2]
+	ubfx	x25, x17, #48, #8
+	eor	w24, w24, w26, ror 24
+	ubfx	x26, x17, #24, #8
+	eor	w24, w24, w21, ror 8
+	ubfx	x21, x16, #8, #8
+	eor	w24, w24, w22, ror 16
+	ubfx	x22, x16, #32, #8
+	bfi	x23, x24, #32, #32
+	ldr	w25, [x10, x25, LSL 2]
+	ldr	w26, [x10, x26, LSL 2]
+	ldr	w21, [x10, x21, LSL 2]
+	ldr	w22, [x10, x22, LSL 2]
+	ubfx	x19, x17, #0, #8
+	eor	w25, w25, w26, ror 24
+	ubfx	x26, x16, #16, #8
+	eor	w25, w25, w21, ror 8
+	ubfx	x21, x17, #56, #8
+	eor	w24, w25, w22, ror 16
+	ubfx	x22, x16, #40, #8
+	ldr	w19, [x10, x19, LSL 2]
+	ldr	w21, [x10, x21, LSL 2]
+	ldr	w26, [x10, x26, LSL 2]
+	ldr	w22, [x10, x22, LSL 2]
+	eor	w21, w21, w19, ror 24
+	ldp	x16, x17, [x28], #16
+	eor	w26, w26, w21, ror 24
+	eor	w26, w26, w22, ror 8
+	bfi	x24, x26, #32, #32
 	;   XOR in Key Schedule
-	eor	x10, x10, x14
-	eor	x11, x11, x15
-	subs	w25, w25, #2
-	bne	L_AES_XTS_encrypt_loop_nr_partial
-	ubfx	x14, x10, #48, #8
-	ubfx	x17, x10, #24, #8
-	ubfx	x19, x11, #8, #8
-	ubfx	x20, x11, #32, #8
-	ldr	x12, [x8]
-	ldr	x12, [x8, #64]
-	ldr	x12, [x8, #128]
-	ldr	x12, [x8, #192]
-	ldr	x12, [x8, #256]
-	ldr	x12, [x8, #320]
-	ldr	x12, [x8, #384]
-	ldr	x12, [x8, #448]
-	ldr	x12, [x8, #512]
-	ldr	x12, [x8, #576]
-	ldr	x12, [x8, #640]
-	ldr	x12, [x8, #704]
-	ldr	x12, [x8, #768]
-	ldr	x12, [x8, #832]
-	ldr	x12, [x8, #896]
-	ldr	x12, [x8, #960]
-	ldr	w14, [x8, x14, LSL 2]
-	ldr	w17, [x8, x17, LSL 2]
-	ldr	w19, [x8, x19, LSL 2]
-	ldr	w20, [x8, x20, LSL 2]
-	ubfx	x15, x11, #16, #8
-	eor	w14, w14, w17, ror 24
-	ubfx	x17, x10, #56, #8
-	eor	w14, w14, w19, ror 8
-	ubfx	x19, x11, #40, #8
-	eor	w14, w14, w20, ror 16
-	ubfx	x20, x10, #0, #8
-	ldr	w15, [x8, x15, LSL 2]
-	ldr	w17, [x8, x17, LSL 2]
-	ldr	w19, [x8, x19, LSL 2]
-	ldr	w20, [x8, x20, LSL 2]
-	ubfx	x16, x11, #48, #8
-	eor	w15, w15, w17, ror 24
-	ubfx	x17, x11, #24, #8
-	eor	w15, w15, w19, ror 8
-	ubfx	x19, x10, #8, #8
-	eor	w15, w15, w20, ror 16
-	ubfx	x20, x10, #32, #8
-	bfi	x14, x15, #32, #32
-	ldr	w16, [x8, x16, LSL 2]
-	ldr	w17, [x8, x17, LSL 2]
-	ldr	w19, [x8, x19, LSL 2]
-	ldr	w20, [x8, x20, LSL 2]
-	ubfx	x12, x11, #0, #8
-	eor	w16, w16, w17, ror 24
-	ubfx	x17, x10, #16, #8
-	eor	w16, w16, w19, ror 8
-	ubfx	x19, x11, #56, #8
-	eor	w15, w16, w20, ror 16
-	ubfx	x20, x10, #40, #8
-	ldr	w12, [x8, x12, LSL 2]
-	ldr	w19, [x8, x19, LSL 2]
-	ldr	w17, [x8, x17, LSL 2]
-	ldr	w20, [x8, x20, LSL 2]
-	eor	w19, w19, w12, ror 24
-	ldp	x10, x11, [x26], #16
-	eor	w17, w17, w19, ror 24
-	eor	w17, w17, w20, ror 8
-	bfi	x15, x17, #32, #32
+	eor	x23, x23, x16
+	eor	x24, x24, x17
+	subs	w27, w27, #2
+	bne	L_AES_XTS_decrypt_loop_nr_tweak
+	ubfx	x16, x23, #48, #8
+	ubfx	x20, x23, #24, #8
+	ubfx	x21, x24, #8, #8
+	ubfx	x22, x24, #32, #8
+	ldr	x25, [x10]
+	ldr	x25, [x10, #64]
+	ldr	x25, [x10, #128]
+	ldr	x25, [x10, #192]
+	ldr	x25, [x10, #256]
+	ldr	x25, [x10, #320]
+	ldr	x25, [x10, #384]
+	ldr	x25, [x10, #448]
+	ldr	x25, [x10, #512]
+	ldr	x25, [x10, #576]
+	ldr	x25, [x10, #640]
+	ldr	x25, [x10, #704]
+	ldr	x25, [x10, #768]
+	ldr	x25, [x10, #832]
+	ldr	x25, [x10, #896]
+	ldr	x25, [x10, #960]
+	ldr	w16, [x10, x16, LSL 2]
+	ldr	w20, [x10, x20, LSL 2]
+	ldr	w21, [x10, x21, LSL 2]
+	ldr	w22, [x10, x22, LSL 2]
+	ubfx	x17, x24, #16, #8
+	eor	w16, w16, w20, ror 24
+	ubfx	x20, x23, #56, #8
+	eor	w16, w16, w21, ror 8
+	ubfx	x21, x24, #40, #8
+	eor	w16, w16, w22, ror 16
+	ubfx	x22, x23, #0, #8
+	ldr	w17, [x10, x17, LSL 2]
+	ldr	w20, [x10, x20, LSL 2]
+	ldr	w21, [x10, x21, LSL 2]
+	ldr	w22, [x10, x22, LSL 2]
+	ubfx	x19, x24, #48, #8
+	eor	w17, w17, w20, ror 24
+	ubfx	x20, x24, #24, #8
+	eor	w17, w17, w21, ror 8
+	ubfx	x21, x23, #8, #8
+	eor	w17, w17, w22, ror 16
+	ubfx	x22, x23, #32, #8
+	bfi	x16, x17, #32, #32
+	ldr	w19, [x10, x19, LSL 2]
+	ldr	w20, [x10, x20, LSL 2]
+	ldr	w21, [x10, x21, LSL 2]
+	ldr	w22, [x10, x22, LSL 2]
+	ubfx	x25, x24, #0, #8
+	eor	w19, w19, w20, ror 24
+	ubfx	x20, x23, #16, #8
+	eor	w19, w19, w21, ror 8
+	ubfx	x21, x24, #56, #8
+	eor	w17, w19, w22, ror 16
+	ubfx	x22, x23, #40, #8
+	ldr	w25, [x10, x25, LSL 2]
+	ldr	w21, [x10, x21, LSL 2]
+	ldr	w20, [x10, x20, LSL 2]
+	ldr	w22, [x10, x22, LSL 2]
+	eor	w21, w21, w25, ror 24
+	ldp	x23, x24, [x28], #16
+	eor	w20, w20, w21, ror 24
+	eor	w20, w20, w22, ror 8
+	bfi	x17, x20, #32, #32
 	;   XOR in Key Schedule
-	eor	x14, x14, x10
-	eor	x15, x15, x11
-	ubfx	x10, x15, #32, #8
-	ubfx	x13, x15, #8, #8
-	ubfx	x19, x14, #48, #8
-	ubfx	x20, x14, #24, #8
-	lsl	w10, w10, #2
-	lsl	w13, w13, #2
-	lsl	w19, w19, #2
-	lsl	w20, w20, #2
-	ldr	x17, [x8]
-	ldr	x17, [x8, #64]
-	ldr	x17, [x8, #128]
-	ldr	x17, [x8, #192]
-	ldr	x17, [x8, #256]
-	ldr	x17, [x8, #320]
-	ldr	x17, [x8, #384]
-	ldr	x17, [x8, #448]
-	ldr	x17, [x8, #512]
-	ldr	x17, [x8, #576]
-	ldr	x17, [x8, #640]
-	ldr	x17, [x8, #704]
-	ldr	x17, [x8, #768]
-	ldr	x17, [x8, #832]
-	ldr	x17, [x8, #896]
-	ldr	x17, [x8, #960]
-	ldrb	w10, [x8, x10, LSL 0]
-	ldrb	w13, [x8, x13, LSL 0]
-	ldrb	w19, [x8, x19, LSL 0]
-	ldrb	w20, [x8, x20, LSL 0]
-	ubfx	x11, x14, #0, #8
-	eor	w10, w10, w13, lsl 8
-	ubfx	x13, x15, #40, #8
-	eor	w10, w10, w19, lsl 16
-	ubfx	x19, x15, #16, #8
-	eor	w10, w10, w20, lsl 24
-	ubfx	x20, x14, #56, #8
-	lsl	w11, w11, #2
-	lsl	w13, w13, #2
-	lsl	w19, w19, #2
-	lsl	w20, w20, #2
-	ldrb	w11, [x8, x11, LSL 0]
-	ldrb	w13, [x8, x13, LSL 0]
-	ldrb	w19, [x8, x19, LSL 0]
-	ldrb	w20, [x8, x20, LSL 0]
-	ubfx	x12, x14, #32, #8
-	eor	w11, w11, w13, lsl 8
-	ubfx	x13, x14, #8, #8
-	eor	w11, w11, w19, lsl 16
-	ubfx	x19, x15, #48, #8
-	eor	w11, w11, w20, lsl 24
-	ubfx	x20, x15, #24, #8
-	bfi	x10, x11, #32, #32
-	lsl	w12, w12, #2
-	lsl	w13, w13, #2
-	lsl	w19, w19, #2
-	lsl	w20, w20, #2
-	ldrb	w12, [x8, x12, LSL 0]
-	ldrb	w13, [x8, x13, LSL 0]
-	ldrb	w19, [x8, x19, LSL 0]
-	ldrb	w20, [x8, x20, LSL 0]
-	ubfx	x17, x15, #56, #8
-	eor	w12, w12, w13, lsl 8
-	ubfx	x13, x15, #0, #8
-	eor	w12, w12, w19, lsl 16
-	ubfx	x19, x14, #40, #8
-	eor	w11, w12, w20, lsl 24
-	ubfx	x20, x14, #16, #8
-	lsl	w17, w17, #2
-	lsl	w13, w13, #2
-	lsl	w19, w19, #2
+	eor	x16, x16, x23
+	eor	x17, x17, x24
+	ubfx	x23, x17, #32, #8
+	ubfx	x26, x17, #8, #8
+	ubfx	x21, x16, #48, #8
+	ubfx	x22, x16, #24, #8
+	lsl	w23, w23, #2
+	lsl	w26, w26, #2
+	lsl	w21, w21, #2
+	lsl	w22, w22, #2
+	ldr	x20, [x10]
+	ldr	x20, [x10, #64]
+	ldr	x20, [x10, #128]
+	ldr	x20, [x10, #192]
+	ldr	x20, [x10, #256]
+	ldr	x20, [x10, #320]
+	ldr	x20, [x10, #384]
+	ldr	x20, [x10, #448]
+	ldr	x20, [x10, #512]
+	ldr	x20, [x10, #576]
+	ldr	x20, [x10, #640]
+	ldr	x20, [x10, #704]
+	ldr	x20, [x10, #768]
+	ldr	x20, [x10, #832]
+	ldr	x20, [x10, #896]
+	ldr	x20, [x10, #960]
+	ldrb	w23, [x10, x23, LSL 0]
+	ldrb	w26, [x10, x26, LSL 0]
+	ldrb	w21, [x10, x21, LSL 0]
+	ldrb	w22, [x10, x22, LSL 0]
+	ubfx	x24, x16, #0, #8
+	eor	w23, w23, w26, lsl 8
+	ubfx	x26, x17, #40, #8
+	eor	w23, w23, w21, lsl 16
+	ubfx	x21, x17, #16, #8
+	eor	w23, w23, w22, lsl 24
+	ubfx	x22, x16, #56, #8
+	lsl	w24, w24, #2
+	lsl	w26, w26, #2
+	lsl	w21, w21, #2
+	lsl	w22, w22, #2
+	ldrb	w24, [x10, x24, LSL 0]
+	ldrb	w26, [x10, x26, LSL 0]
+	ldrb	w21, [x10, x21, LSL 0]
+	ldrb	w22, [x10, x22, LSL 0]
+	ubfx	x25, x16, #32, #8
+	eor	w24, w24, w26, lsl 8
+	ubfx	x26, x16, #8, #8
+	eor	w24, w24, w21, lsl 16
+	ubfx	x21, x17, #48, #8
+	eor	w24, w24, w22, lsl 24
+	ubfx	x22, x17, #24, #8
+	bfi	x23, x24, #32, #32
+	lsl	w25, w25, #2
+	lsl	w26, w26, #2
+	lsl	w21, w21, #2
+	lsl	w22, w22, #2
+	ldrb	w25, [x10, x25, LSL 0]
+	ldrb	w26, [x10, x26, LSL 0]
+	ldrb	w21, [x10, x21, LSL 0]
+	ldrb	w22, [x10, x22, LSL 0]
+	ubfx	x20, x17, #56, #8
+	eor	w25, w25, w26, lsl 8
+	ubfx	x26, x17, #0, #8
+	eor	w25, w25, w21, lsl 16
+	ubfx	x21, x16, #40, #8
+	eor	w24, w25, w22, lsl 24
+	ubfx	x22, x16, #16, #8
 	lsl	w20, w20, #2
-	ldrb	w17, [x8, x17, LSL 0]
-	ldrb	w13, [x8, x13, LSL 0]
-	ldrb	w19, [x8, x19, LSL 0]
-	ldrb	w20, [x8, x20, LSL 0]
-	eor	w19, w19, w17, lsl 16
-	ldp	x14, x15, [x26]
-	eor	w13, w13, w19, lsl 8
-	eor	w13, w13, w20, lsl 16
-	bfi	x11, x13, #32, #32
+	lsl	w26, w26, #2
+	lsl	w21, w21, #2
+	lsl	w22, w22, #2
+	ldrb	w20, [x10, x20, LSL 0]
+	ldrb	w26, [x10, x26, LSL 0]
+	ldrb	w21, [x10, x21, LSL 0]
+	ldrb	w22, [x10, x22, LSL 0]
+	eor	w21, w21, w20, lsl 16
+	ldp	x16, x17, [x28]
+	eor	w26, w26, w21, lsl 8
+	eor	w26, w26, w22, lsl 16
+	bfi	x24, x26, #32, #32
 	;   XOR in Key Schedule
-	eor	x10, x10, x14
-	eor	x11, x11, x15
-	rev32	x10, x10
-	rev32	x11, x11
-	eor	x10, x10, x21
-	eor	x11, x11, x22
-	stp	x10, x11, [x1]
-L_AES_XTS_encrypt_done_data
-	ldp	x17, x19, [x29, #24]
-	ldp	x20, x21, [x29, #40]
-	ldp	x22, x23, [x29, #56]
-	ldp	x24, x25, [x29, #72]
-	ldr	x26, [x29, #88]
-	ldp	x29, x30, [sp], #0x60
-	ret
-	ENDP
-	IF :DEF:HAVE_AES_DECRYPT
-	AREA	|.text|, CODE, READONLY
-	ALIGN	4
-	EXPORT	AES_XTS_decrypt
-AES_XTS_decrypt PROC
-	stp	x29, x30, [sp, #-112]!
-	add	x29, sp, #0
-	stp	x17, x19, [x29, #24]
-	stp	x20, x21, [x29, #40]
-	stp	x22, x23, [x29, #56]
-	stp	x24, x25, [x29, #72]
-	stp	x26, x27, [x29, #88]
-	str	x28, [x29, #104]
-	adrp	x8, L_AES_ARM64_td
-	add	x8, x8, L_AES_ARM64_td
-	adrp	x9, L_AES_ARM64_td4
-	add	x9, x9, L_AES_ARM64_td4
-	adrp	x10, L_AES_ARM64_te
-	add	x10, x10, L_AES_ARM64_te
-	ands	w11, w2, #15
-	cset	w11, ne
-	sub	w2, w2, w11, lsl 4
-	mov	x11, #0x87
-	mov	x28, x5
-	ldp	x23, x24, [x3]
+	eor	x23, x23, x16
+	eor	x24, x24, x17
+	rev32	x23, x23
+	rev32	x24, x24
+	cmp	w2, #16
+	blt	L_AES_XTS_decrypt_start_partail
+L_AES_XTS_decrypt_loop_block
+	mov	x28, x4
+	ldp	x12, x13, [x0]
 	ldp	x16, x17, [x28], #16
-	rev32	x23, x23
-	rev32	x24, x24
+	eor	x12, x12, x23
+	eor	x13, x13, x24
+	rev32	x12, x12
+	rev32	x13, x13
 	; Round: 0 - XOR in key schedule
-	eor	x23, x23, x16
-	eor	x24, x24, x17
+	eor	x12, x12, x16
+	eor	x13, x13, x17
 	sub	w27, w7, #2
-L_AES_XTS_decrypt_loop_nr_tweak
-	ubfx	x16, x23, #48, #8
-	ubfx	x20, x23, #24, #8
-	ubfx	x21, x24, #8, #8
-	ubfx	x22, x24, #32, #8
-	ldr	x25, [x10]
-	ldr	x25, [x10, #64]
-	ldr	x25, [x10, #128]
-	ldr	x25, [x10, #192]
-	ldr	x25, [x10, #256]
-	ldr	x25, [x10, #320]
-	ldr	x25, [x10, #384]
-	ldr	x25, [x10, #448]
-	ldr	x25, [x10, #512]
-	ldr	x25, [x10, #576]
-	ldr	x25, [x10, #640]
-	ldr	x25, [x10, #704]
-	ldr	x25, [x10, #768]
-	ldr	x25, [x10, #832]
-	ldr	x25, [x10, #896]
-	ldr	x25, [x10, #960]
-	ldr	w16, [x10, x16, LSL 2]
-	ldr	w20, [x10, x20, LSL 2]
-	ldr	w21, [x10, x21, LSL 2]
-	ldr	w22, [x10, x22, LSL 2]
-	ubfx	x17, x24, #16, #8
+L_AES_XTS_decrypt_loop_nr
+	ubfx	x16, x13, #48, #8
+	ubfx	x20, x12, #24, #8
+	ubfx	x21, x13, #8, #8
+	ubfx	x22, x12, #32, #8
+	ldr	x14, [x8]
+	ldr	x14, [x8, #64]
+	ldr	x14, [x8, #128]
+	ldr	x14, [x8, #192]
+	ldr	x14, [x8, #256]
+	ldr	x14, [x8, #320]
+	ldr	x14, [x8, #384]
+	ldr	x14, [x8, #448]
+	ldr	x14, [x8, #512]
+	ldr	x14, [x8, #576]
+	ldr	x14, [x8, #640]
+	ldr	x14, [x8, #704]
+	ldr	x14, [x8, #768]
+	ldr	x14, [x8, #832]
+	ldr	x14, [x8, #896]
+	ldr	x14, [x8, #960]
+	ldr	w16, [x8, x16, LSL 2]
+	ldr	w20, [x8, x20, LSL 2]
+	ldr	w21, [x8, x21, LSL 2]
+	ldr	w22, [x8, x22, LSL 2]
+	ubfx	x17, x12, #16, #8
 	eor	w16, w16, w20, ror 24
-	ubfx	x20, x23, #56, #8
+	ubfx	x20, x12, #56, #8
 	eor	w16, w16, w21, ror 8
-	ubfx	x21, x24, #40, #8
+	ubfx	x21, x13, #40, #8
 	eor	w16, w16, w22, ror 16
-	ubfx	x22, x23, #0, #8
-	ldr	w17, [x10, x17, LSL 2]
-	ldr	w20, [x10, x20, LSL 2]
-	ldr	w21, [x10, x21, LSL 2]
-	ldr	w22, [x10, x22, LSL 2]
-	ubfx	x19, x24, #48, #8
+	ubfx	x22, x13, #0, #8
+	ldr	w17, [x8, x17, LSL 2]
+	ldr	w20, [x8, x20, LSL 2]
+	ldr	w21, [x8, x21, LSL 2]
+	ldr	w22, [x8, x22, LSL 2]
+	ubfx	x19, x12, #48, #8
 	eor	w17, w17, w20, ror 24
-	ubfx	x20, x24, #24, #8
+	ubfx	x20, x13, #24, #8
 	eor	w17, w17, w21, ror 8
-	ubfx	x21, x23, #8, #8
+	ubfx	x21, x12, #8, #8
 	eor	w17, w17, w22, ror 16
-	ubfx	x22, x23, #32, #8
+	ubfx	x22, x13, #32, #8
 	bfi	x16, x17, #32, #32
-	ldr	w19, [x10, x19, LSL 2]
-	ldr	w20, [x10, x20, LSL 2]
-	ldr	w21, [x10, x21, LSL 2]
-	ldr	w22, [x10, x22, LSL 2]
-	ubfx	x25, x24, #0, #8
+	ldr	w19, [x8, x19, LSL 2]
+	ldr	w20, [x8, x20, LSL 2]
+	ldr	w21, [x8, x21, LSL 2]
+	ldr	w22, [x8, x22, LSL 2]
+	ubfx	x14, x12, #0, #8
 	eor	w19, w19, w20, ror 24
-	ubfx	x20, x23, #16, #8
+	ubfx	x20, x13, #16, #8
 	eor	w19, w19, w21, ror 8
-	ubfx	x21, x24, #56, #8
+	ubfx	x21, x13, #56, #8
 	eor	w17, w19, w22, ror 16
-	ubfx	x22, x23, #40, #8
-	ldr	w25, [x10, x25, LSL 2]
-	ldr	w21, [x10, x21, LSL 2]
-	ldr	w20, [x10, x20, LSL 2]
-	ldr	w22, [x10, x22, LSL 2]
-	eor	w21, w21, w25, ror 24
-	ldp	x23, x24, [x28], #16
-	eor	w20, w20, w21, ror 24
+	ubfx	x22, x12, #40, #8
+	ldr	w14, [x8, x14, LSL 2]
+	ldr	w21, [x8, x21, LSL 2]
+	ldr	w20, [x8, x20, LSL 2]
+	ldr	w22, [x8, x22, LSL 2]
+	eor	w21, w21, w14, ror 24
+	ldp	x12, x13, [x28], #16
 	eor	w20, w20, w22, ror 8
+	eor	w20, w20, w21, ror 24
 	bfi	x17, x20, #32, #32
 	;   XOR in Key Schedule
-	eor	x16, x16, x23
-	eor	x17, x17, x24
-	ubfx	x23, x16, #48, #8
-	ubfx	x26, x16, #24, #8
+	eor	x16, x16, x12
+	eor	x17, x17, x13
+	ubfx	x12, x17, #48, #8
+	ubfx	x15, x16, #24, #8
 	ubfx	x21, x17, #8, #8
-	ubfx	x22, x17, #32, #8
-	ldr	x19, [x10]
-	ldr	x19, [x10, #64]
-	ldr	x19, [x10, #128]
-	ldr	x19, [x10, #192]
-	ldr	x19, [x10, #256]
-	ldr	x19, [x10, #320]
-	ldr	x19, [x10, #384]
-	ldr	x19, [x10, #448]
-	ldr	x19, [x10, #512]
-	ldr	x19, [x10, #576]
-	ldr	x19, [x10, #640]
-	ldr	x19, [x10, #704]
-	ldr	x19, [x10, #768]
-	ldr	x19, [x10, #832]
-	ldr	x19, [x10, #896]
-	ldr	x19, [x10, #960]
-	ldr	w23, [x10, x23, LSL 2]
-	ldr	w26, [x10, x26, LSL 2]
-	ldr	w21, [x10, x21, LSL 2]
-	ldr	w22, [x10, x22, LSL 2]
-	ubfx	x24, x17, #16, #8
-	eor	w23, w23, w26, ror 24
-	ubfx	x26, x16, #56, #8
-	eor	w23, w23, w21, ror 8
+	ubfx	x22, x16, #32, #8
+	ldr	x19, [x8]
+	ldr	x19, [x8, #64]
+	ldr	x19, [x8, #128]
+	ldr	x19, [x8, #192]
+	ldr	x19, [x8, #256]
+	ldr	x19, [x8, #320]
+	ldr	x19, [x8, #384]
+	ldr	x19, [x8, #448]
+	ldr	x19, [x8, #512]
+	ldr	x19, [x8, #576]
+	ldr	x19, [x8, #640]
+	ldr	x19, [x8, #704]
+	ldr	x19, [x8, #768]
+	ldr	x19, [x8, #832]
+	ldr	x19, [x8, #896]
+	ldr	x19, [x8, #960]
+	ldr	w12, [x8, x12, LSL 2]
+	ldr	w15, [x8, x15, LSL 2]
+	ldr	w21, [x8, x21, LSL 2]
+	ldr	w22, [x8, x22, LSL 2]
+	ubfx	x13, x16, #16, #8
+	eor	w12, w12, w15, ror 24
+	ubfx	x15, x16, #56, #8
+	eor	w12, w12, w21, ror 8
 	ubfx	x21, x17, #40, #8
-	eor	w23, w23, w22, ror 16
-	ubfx	x22, x16, #0, #8
-	ldr	w24, [x10, x24, LSL 2]
-	ldr	w26, [x10, x26, LSL 2]
-	ldr	w21, [x10, x21, LSL 2]
-	ldr	w22, [x10, x22, LSL 2]
-	ubfx	x25, x17, #48, #8
-	eor	w24, w24, w26, ror 24
-	ubfx	x26, x17, #24, #8
-	eor	w24, w24, w21, ror 8
+	eor	w12, w12, w22, ror 16
+	ubfx	x22, x17, #0, #8
+	ldr	w13, [x8, x13, LSL 2]
+	ldr	w15, [x8, x15, LSL 2]
+	ldr	w21, [x8, x21, LSL 2]
+	ldr	w22, [x8, x22, LSL 2]
+	ubfx	x14, x16, #48, #8
+	eor	w13, w13, w15, ror 24
+	ubfx	x15, x17, #24, #8
+	eor	w13, w13, w21, ror 8
 	ubfx	x21, x16, #8, #8
-	eor	w24, w24, w22, ror 16
-	ubfx	x22, x16, #32, #8
-	bfi	x23, x24, #32, #32
-	ldr	w25, [x10, x25, LSL 2]
-	ldr	w26, [x10, x26, LSL 2]
-	ldr	w21, [x10, x21, LSL 2]
-	ldr	w22, [x10, x22, LSL 2]
-	ubfx	x19, x17, #0, #8
-	eor	w25, w25, w26, ror 24
-	ubfx	x26, x16, #16, #8
-	eor	w25, w25, w21, ror 8
+	eor	w13, w13, w22, ror 16
+	ubfx	x22, x17, #32, #8
+	bfi	x12, x13, #32, #32
+	ldr	w14, [x8, x14, LSL 2]
+	ldr	w15, [x8, x15, LSL 2]
+	ldr	w21, [x8, x21, LSL 2]
+	ldr	w22, [x8, x22, LSL 2]
+	ubfx	x19, x16, #0, #8
+	eor	w14, w14, w15, ror 24
+	ubfx	x15, x17, #16, #8
+	eor	w14, w14, w21, ror 8
 	ubfx	x21, x17, #56, #8
-	eor	w24, w25, w22, ror 16
+	eor	w13, w14, w22, ror 16
 	ubfx	x22, x16, #40, #8
-	ldr	w19, [x10, x19, LSL 2]
-	ldr	w21, [x10, x21, LSL 2]
-	ldr	w26, [x10, x26, LSL 2]
-	ldr	w22, [x10, x22, LSL 2]
-	eor	w21, w21, w19, ror 24
-	ldp	x16, x17, [x28], #16
-	eor	w26, w26, w21, ror 24
-	eor	w26, w26, w22, ror 8
-	bfi	x24, x26, #32, #32
-	;   XOR in Key Schedule
-	eor	x23, x23, x16
-	eor	x24, x24, x17
-	subs	w27, w27, #2
-	bne	L_AES_XTS_decrypt_loop_nr_tweak
-	ubfx	x16, x23, #48, #8
-	ubfx	x20, x23, #24, #8
-	ubfx	x21, x24, #8, #8
-	ubfx	x22, x24, #32, #8
-	ldr	x25, [x10]
-	ldr	x25, [x10, #64]
-	ldr	x25, [x10, #128]
-	ldr	x25, [x10, #192]
-	ldr	x25, [x10, #256]
-	ldr	x25, [x10, #320]
-	ldr	x25, [x10, #384]
-	ldr	x25, [x10, #448]
-	ldr	x25, [x10, #512]
-	ldr	x25, [x10, #576]
-	ldr	x25, [x10, #640]
-	ldr	x25, [x10, #704]
-	ldr	x25, [x10, #768]
-	ldr	x25, [x10, #832]
-	ldr	x25, [x10, #896]
-	ldr	x25, [x10, #960]
-	ldr	w16, [x10, x16, LSL 2]
-	ldr	w20, [x10, x20, LSL 2]
-	ldr	w21, [x10, x21, LSL 2]
-	ldr	w22, [x10, x22, LSL 2]
-	ubfx	x17, x24, #16, #8
+	ldr	w19, [x8, x19, LSL 2]
+	ldr	w21, [x8, x21, LSL 2]
+	ldr	w15, [x8, x15, LSL 2]
+	ldr	w22, [x8, x22, LSL 2]
+	eor	w21, w21, w19, ror 24
+	ldp	x16, x17, [x28], #16
+	eor	w15, w15, w22, ror 8
+	eor	w15, w15, w21, ror 24
+	bfi	x13, x15, #32, #32
+	;   XOR in Key Schedule
+	eor	x12, x12, x16
+	eor	x13, x13, x17
+	subs	w27, w27, #2
+	bne	L_AES_XTS_decrypt_loop_nr
+	ubfx	x16, x13, #48, #8
+	ubfx	x20, x12, #24, #8
+	ubfx	x21, x13, #8, #8
+	ubfx	x22, x12, #32, #8
+	ldr	x14, [x8]
+	ldr	x14, [x8, #64]
+	ldr	x14, [x8, #128]
+	ldr	x14, [x8, #192]
+	ldr	x14, [x8, #256]
+	ldr	x14, [x8, #320]
+	ldr	x14, [x8, #384]
+	ldr	x14, [x8, #448]
+	ldr	x14, [x8, #512]
+	ldr	x14, [x8, #576]
+	ldr	x14, [x8, #640]
+	ldr	x14, [x8, #704]
+	ldr	x14, [x8, #768]
+	ldr	x14, [x8, #832]
+	ldr	x14, [x8, #896]
+	ldr	x14, [x8, #960]
+	ldr	w16, [x8, x16, LSL 2]
+	ldr	w20, [x8, x20, LSL 2]
+	ldr	w21, [x8, x21, LSL 2]
+	ldr	w22, [x8, x22, LSL 2]
+	ubfx	x17, x12, #16, #8
 	eor	w16, w16, w20, ror 24
-	ubfx	x20, x23, #56, #8
+	ubfx	x20, x12, #56, #8
 	eor	w16, w16, w21, ror 8
-	ubfx	x21, x24, #40, #8
+	ubfx	x21, x13, #40, #8
 	eor	w16, w16, w22, ror 16
-	ubfx	x22, x23, #0, #8
-	ldr	w17, [x10, x17, LSL 2]
-	ldr	w20, [x10, x20, LSL 2]
-	ldr	w21, [x10, x21, LSL 2]
-	ldr	w22, [x10, x22, LSL 2]
-	ubfx	x19, x24, #48, #8
+	ubfx	x22, x13, #0, #8
+	ldr	w17, [x8, x17, LSL 2]
+	ldr	w20, [x8, x20, LSL 2]
+	ldr	w21, [x8, x21, LSL 2]
+	ldr	w22, [x8, x22, LSL 2]
+	ubfx	x19, x12, #48, #8
 	eor	w17, w17, w20, ror 24
-	ubfx	x20, x24, #24, #8
+	ubfx	x20, x13, #24, #8
 	eor	w17, w17, w21, ror 8
-	ubfx	x21, x23, #8, #8
+	ubfx	x21, x12, #8, #8
 	eor	w17, w17, w22, ror 16
-	ubfx	x22, x23, #32, #8
+	ubfx	x22, x13, #32, #8
 	bfi	x16, x17, #32, #32
-	ldr	w19, [x10, x19, LSL 2]
-	ldr	w20, [x10, x20, LSL 2]
-	ldr	w21, [x10, x21, LSL 2]
-	ldr	w22, [x10, x22, LSL 2]
-	ubfx	x25, x24, #0, #8
+	ldr	w19, [x8, x19, LSL 2]
+	ldr	w20, [x8, x20, LSL 2]
+	ldr	w21, [x8, x21, LSL 2]
+	ldr	w22, [x8, x22, LSL 2]
+	ubfx	x14, x12, #0, #8
 	eor	w19, w19, w20, ror 24
-	ubfx	x20, x23, #16, #8
+	ubfx	x20, x13, #16, #8
 	eor	w19, w19, w21, ror 8
-	ubfx	x21, x24, #56, #8
+	ubfx	x21, x13, #56, #8
 	eor	w17, w19, w22, ror 16
-	ubfx	x22, x23, #40, #8
-	ldr	w25, [x10, x25, LSL 2]
-	ldr	w21, [x10, x21, LSL 2]
-	ldr	w20, [x10, x20, LSL 2]
-	ldr	w22, [x10, x22, LSL 2]
-	eor	w21, w21, w25, ror 24
-	ldp	x23, x24, [x28], #16
-	eor	w20, w20, w21, ror 24
+	ubfx	x22, x12, #40, #8
+	ldr	w14, [x8, x14, LSL 2]
+	ldr	w21, [x8, x21, LSL 2]
+	ldr	w20, [x8, x20, LSL 2]
+	ldr	w22, [x8, x22, LSL 2]
+	eor	w21, w21, w14, ror 24
+	ldp	x12, x13, [x28], #16
 	eor	w20, w20, w22, ror 8
+	eor	w20, w20, w21, ror 24
 	bfi	x17, x20, #32, #32
 	;   XOR in Key Schedule
-	eor	x16, x16, x23
-	eor	x17, x17, x24
-	ubfx	x23, x17, #32, #8
-	ubfx	x26, x17, #8, #8
-	ubfx	x21, x16, #48, #8
+	eor	x16, x16, x12
+	eor	x17, x17, x13
+	ubfx	x12, x16, #32, #8
+	ubfx	x15, x17, #8, #8
+	ubfx	x21, x17, #48, #8
 	ubfx	x22, x16, #24, #8
-	lsl	w23, w23, #2
-	lsl	w26, w26, #2
-	lsl	w21, w21, #2
-	lsl	w22, w22, #2
-	ldr	x20, [x10]
-	ldr	x20, [x10, #64]
-	ldr	x20, [x10, #128]
-	ldr	x20, [x10, #192]
-	ldr	x20, [x10, #256]
-	ldr	x20, [x10, #320]
-	ldr	x20, [x10, #384]
-	ldr	x20, [x10, #448]
-	ldr	x20, [x10, #512]
-	ldr	x20, [x10, #576]
-	ldr	x20, [x10, #640]
-	ldr	x20, [x10, #704]
-	ldr	x20, [x10, #768]
-	ldr	x20, [x10, #832]
-	ldr	x20, [x10, #896]
-	ldr	x20, [x10, #960]
-	ldrb	w23, [x10, x23, LSL 0]
-	ldrb	w26, [x10, x26, LSL 0]
-	ldrb	w21, [x10, x21, LSL 0]
-	ldrb	w22, [x10, x22, LSL 0]
-	ubfx	x24, x16, #0, #8
-	eor	w23, w23, w26, lsl 8
-	ubfx	x26, x17, #40, #8
-	eor	w23, w23, w21, lsl 16
-	ubfx	x21, x17, #16, #8
-	eor	w23, w23, w22, lsl 24
+	ldr	x20, [x9]
+	ldr	x20, [x9, #64]
+	ldr	x20, [x9, #128]
+	ldr	x20, [x9, #192]
+	ldrb	w12, [x9, x12, LSL 0]
+	ldrb	w15, [x9, x15, LSL 0]
+	ldrb	w21, [x9, x21, LSL 0]
+	ldrb	w22, [x9, x22, LSL 0]
+	ubfx	x13, x17, #0, #8
+	eor	w12, w12, w15, lsl 8
+	ubfx	x15, x17, #40, #8
+	eor	w12, w12, w21, lsl 16
+	ubfx	x21, x16, #16, #8
+	eor	w12, w12, w22, lsl 24
 	ubfx	x22, x16, #56, #8
-	lsl	w24, w24, #2
-	lsl	w26, w26, #2
-	lsl	w21, w21, #2
-	lsl	w22, w22, #2
-	ldrb	w24, [x10, x24, LSL 0]
-	ldrb	w26, [x10, x26, LSL 0]
-	ldrb	w21, [x10, x21, LSL 0]
-	ldrb	w22, [x10, x22, LSL 0]
-	ubfx	x25, x16, #32, #8
-	eor	w24, w24, w26, lsl 8
-	ubfx	x26, x16, #8, #8
-	eor	w24, w24, w21, lsl 16
-	ubfx	x21, x17, #48, #8
-	eor	w24, w24, w22, lsl 24
+	ldrb	w15, [x9, x15, LSL 0]
+	ldrb	w22, [x9, x22, LSL 0]
+	ldrb	w13, [x9, x13, LSL 0]
+	ldrb	w21, [x9, x21, LSL 0]
+	ubfx	x14, x17, #32, #8
+	eor	w13, w13, w15, lsl 8
+	ubfx	x15, x16, #8, #8
+	eor	w13, w13, w21, lsl 16
+	ubfx	x21, x16, #48, #8
+	eor	w13, w13, w22, lsl 24
 	ubfx	x22, x17, #24, #8
-	bfi	x23, x24, #32, #32
-	lsl	w25, w25, #2
-	lsl	w26, w26, #2
-	lsl	w21, w21, #2
-	lsl	w22, w22, #2
-	ldrb	w25, [x10, x25, LSL 0]
-	ldrb	w26, [x10, x26, LSL 0]
-	ldrb	w21, [x10, x21, LSL 0]
-	ldrb	w22, [x10, x22, LSL 0]
+	bfi	x12, x13, #32, #32
+	ldrb	w15, [x9, x15, LSL 0]
+	ldrb	w22, [x9, x22, LSL 0]
+	ldrb	w14, [x9, x14, LSL 0]
+	ldrb	w21, [x9, x21, LSL 0]
 	ubfx	x20, x17, #56, #8
-	eor	w25, w25, w26, lsl 8
-	ubfx	x26, x17, #0, #8
-	eor	w25, w25, w21, lsl 16
+	eor	w14, w14, w15, lsl 8
+	ubfx	x15, x16, #0, #8
+	eor	w14, w14, w21, lsl 16
 	ubfx	x21, x16, #40, #8
-	eor	w24, w25, w22, lsl 24
-	ubfx	x22, x16, #16, #8
-	lsl	w20, w20, #2
-	lsl	w26, w26, #2
-	lsl	w21, w21, #2
-	lsl	w22, w22, #2
-	ldrb	w20, [x10, x20, LSL 0]
-	ldrb	w26, [x10, x26, LSL 0]
-	ldrb	w21, [x10, x21, LSL 0]
-	ldrb	w22, [x10, x22, LSL 0]
+	eor	w13, w14, w22, lsl 24
+	ubfx	x22, x17, #16, #8
+	ldrb	w20, [x9, x20, LSL 0]
+	ldrb	w21, [x9, x21, LSL 0]
+	ldrb	w15, [x9, x15, LSL 0]
+	ldrb	w22, [x9, x22, LSL 0]
 	eor	w21, w21, w20, lsl 16
 	ldp	x16, x17, [x28]
-	eor	w26, w26, w21, lsl 8
-	eor	w26, w26, w22, lsl 16
-	bfi	x24, x26, #32, #32
+	eor	w15, w15, w21, lsl 8
+	eor	w15, w15, w22, lsl 16
+	bfi	x13, x15, #32, #32
 	;   XOR in Key Schedule
-	eor	x23, x23, x16
-	eor	x24, x24, x17
-	rev32	x23, x23
-	rev32	x24, x24
+	eor	x12, x12, x16
+	eor	x13, x13, x17
+	rev32	x12, x12
+	rev32	x13, x13
+	eor	x12, x12, x23
+	eor	x13, x13, x24
+	stp	x12, x13, [x1]
+	and	x21, x11, x24, asr 63
+	extr	x24, x24, x23, #63
+	eor	x23, x21, x23, lsl 1
+	sub	w2, w2, #16
+	add	x0, x0, #16
+	add	x1, x1, #16
 	cmp	w2, #16
-	blt	L_AES_XTS_decrypt_start_partail
-L_AES_XTS_decrypt_loop_block
+	bge	L_AES_XTS_decrypt_loop_block
+	cbz	w2, L_AES_XTS_decrypt_done_data
+L_AES_XTS_decrypt_start_partail
+	and	x21, x11, x24, asr 63
+	extr	x26, x24, x23, #63
+	eor	x25, x21, x23, lsl 1
 	mov	x28, x4
-	ldp	x12, x13, [x0]
+	ldp	x12, x13, [x0], #16
 	ldp	x16, x17, [x28], #16
-	eor	x12, x12, x23
-	eor	x13, x13, x24
+	eor	x12, x12, x25
+	eor	x13, x13, x26
 	rev32	x12, x12
 	rev32	x13, x13
 	; Round: 0 - XOR in key schedule
 	eor	x12, x12, x16
 	eor	x13, x13, x17
 	sub	w27, w7, #2
-L_AES_XTS_decrypt_loop_nr
+L_AES_XTS_decrypt_loop_nr_partial_1
 	ubfx	x16, x13, #48, #8
 	ubfx	x20, x12, #24, #8
 	ubfx	x21, x13, #8, #8
@@ -55213,7 +57710,7 @@ L_AES_XTS_decrypt_loop_nr
 	eor	x12, x12, x16
 	eor	x13, x13, x17
 	subs	w27, w27, #2
-	bne	L_AES_XTS_decrypt_loop_nr
+	bne	L_AES_XTS_decrypt_loop_nr_partial_1
 	ubfx	x16, x13, #48, #8
 	ubfx	x20, x12, #24, #8
 	ubfx	x21, x13, #8, #8
@@ -55334,36 +57831,35 @@ L_AES_XTS_decrypt_loop_nr
 	;   XOR in Key Schedule
 	eor	x12, x12, x16
 	eor	x13, x13, x17
-	rev32	x12, x12
-	rev32	x13, x13
-	eor	x12, x12, x23
-	eor	x13, x13, x24
-	stp	x12, x13, [x1]
-	and	x21, x11, x24, asr 63
-	extr	x24, x24, x23, #63
-	eor	x23, x21, x23, lsl 1
-	sub	w2, w2, #16
-	add	x0, x0, #16
+	rev32	x12, x12
+	rev32	x13, x13
+	eor	x12, x12, x25
+	eor	x13, x13, x26
+	stp	x12, x13, [x6]
 	add	x1, x1, #16
-	cmp	w2, #16
-	bge	L_AES_XTS_decrypt_loop_block
-	cbz	w2, L_AES_XTS_decrypt_done_data
-L_AES_XTS_decrypt_start_partail
-	and	x21, x11, x24, asr 63
-	extr	x26, x24, x23, #63
-	eor	x25, x21, x23, lsl 1
+	mov	w16, w2
+L_AES_XTS_decrypt_start_byte
+	ldrb	w21, [x6]
+	ldrb	w22, [x0], #1
+	strb	w21, [x1], #1
+	strb	w22, [x6], #1
+	subs	w16, w16, #1
+	bgt	L_AES_XTS_decrypt_start_byte
+	sub	x1, x1, x2
+	sub	x6, x6, x2
+	sub	x1, x1, #16
 	mov	x28, x4
-	ldp	x12, x13, [x0], #16
+	ldp	x12, x13, [x6]
 	ldp	x16, x17, [x28], #16
-	eor	x12, x12, x25
-	eor	x13, x13, x26
+	eor	x12, x12, x23
+	eor	x13, x13, x24
 	rev32	x12, x12
 	rev32	x13, x13
 	; Round: 0 - XOR in key schedule
 	eor	x12, x12, x16
 	eor	x13, x13, x17
 	sub	w27, w7, #2
-L_AES_XTS_decrypt_loop_nr_partial_1
+L_AES_XTS_decrypt_loop_nr_partial_2
 	ubfx	x16, x13, #48, #8
 	ubfx	x20, x12, #24, #8
 	ubfx	x21, x13, #8, #8
@@ -55497,7 +57993,7 @@ L_AES_XTS_decrypt_loop_nr_partial_1
 	eor	x12, x12, x16
 	eor	x13, x13, x17
 	subs	w27, w27, #2
-	bne	L_AES_XTS_decrypt_loop_nr_partial_1
+	bne	L_AES_XTS_decrypt_loop_nr_partial_2
 	ubfx	x16, x13, #48, #8
 	ubfx	x20, x12, #24, #8
 	ubfx	x21, x13, #8, #8
@@ -55616,308 +58112,805 @@ L_AES_XTS_decrypt_loop_nr_partial_1
 	eor	w15, w15, w22, lsl 16
 	bfi	x13, x15, #32, #32
 	;   XOR in Key Schedule
-	eor	x12, x12, x16
-	eor	x13, x13, x17
-	rev32	x12, x12
-	rev32	x13, x13
-	eor	x12, x12, x25
-	eor	x13, x13, x26
-	stp	x12, x13, [x6]
-	add	x1, x1, #16
-	mov	w16, w2
-L_AES_XTS_decrypt_start_byte
-	ldrb	w21, [x6]
-	ldrb	w22, [x0], #1
-	strb	w21, [x1], #1
-	strb	w22, [x6], #1
-	subs	w16, w16, #1
-	bgt	L_AES_XTS_decrypt_start_byte
-	sub	x1, x1, x2
-	sub	x6, x6, x2
-	sub	x1, x1, #16
-	mov	x28, x4
-	ldp	x12, x13, [x6]
-	ldp	x16, x17, [x28], #16
-	eor	x12, x12, x23
-	eor	x13, x13, x24
-	rev32	x12, x12
-	rev32	x13, x13
-	; Round: 0 - XOR in key schedule
-	eor	x12, x12, x16
-	eor	x13, x13, x17
-	sub	w27, w7, #2
-L_AES_XTS_decrypt_loop_nr_partial_2
-	ubfx	x16, x13, #48, #8
-	ubfx	x20, x12, #24, #8
-	ubfx	x21, x13, #8, #8
-	ubfx	x22, x12, #32, #8
-	ldr	x14, [x8]
-	ldr	x14, [x8, #64]
-	ldr	x14, [x8, #128]
-	ldr	x14, [x8, #192]
-	ldr	x14, [x8, #256]
-	ldr	x14, [x8, #320]
-	ldr	x14, [x8, #384]
-	ldr	x14, [x8, #448]
-	ldr	x14, [x8, #512]
-	ldr	x14, [x8, #576]
-	ldr	x14, [x8, #640]
-	ldr	x14, [x8, #704]
-	ldr	x14, [x8, #768]
-	ldr	x14, [x8, #832]
-	ldr	x14, [x8, #896]
-	ldr	x14, [x8, #960]
-	ldr	w16, [x8, x16, LSL 2]
-	ldr	w20, [x8, x20, LSL 2]
-	ldr	w21, [x8, x21, LSL 2]
-	ldr	w22, [x8, x22, LSL 2]
-	ubfx	x17, x12, #16, #8
-	eor	w16, w16, w20, ror 24
-	ubfx	x20, x12, #56, #8
-	eor	w16, w16, w21, ror 8
-	ubfx	x21, x13, #40, #8
-	eor	w16, w16, w22, ror 16
-	ubfx	x22, x13, #0, #8
-	ldr	w17, [x8, x17, LSL 2]
-	ldr	w20, [x8, x20, LSL 2]
-	ldr	w21, [x8, x21, LSL 2]
-	ldr	w22, [x8, x22, LSL 2]
-	ubfx	x19, x12, #48, #8
-	eor	w17, w17, w20, ror 24
-	ubfx	x20, x13, #24, #8
-	eor	w17, w17, w21, ror 8
-	ubfx	x21, x12, #8, #8
-	eor	w17, w17, w22, ror 16
-	ubfx	x22, x13, #32, #8
-	bfi	x16, x17, #32, #32
-	ldr	w19, [x8, x19, LSL 2]
-	ldr	w20, [x8, x20, LSL 2]
-	ldr	w21, [x8, x21, LSL 2]
-	ldr	w22, [x8, x22, LSL 2]
-	ubfx	x14, x12, #0, #8
-	eor	w19, w19, w20, ror 24
-	ubfx	x20, x13, #16, #8
-	eor	w19, w19, w21, ror 8
-	ubfx	x21, x13, #56, #8
-	eor	w17, w19, w22, ror 16
-	ubfx	x22, x12, #40, #8
-	ldr	w14, [x8, x14, LSL 2]
-	ldr	w21, [x8, x21, LSL 2]
-	ldr	w20, [x8, x20, LSL 2]
-	ldr	w22, [x8, x22, LSL 2]
-	eor	w21, w21, w14, ror 24
-	ldp	x12, x13, [x28], #16
-	eor	w20, w20, w22, ror 8
-	eor	w20, w20, w21, ror 24
-	bfi	x17, x20, #32, #32
+	eor	x12, x12, x16
+	eor	x13, x13, x17
+	rev32	x12, x12
+	rev32	x13, x13
+	eor	x12, x12, x23
+	eor	x13, x13, x24
+	stp	x12, x13, [x1]
+L_AES_XTS_decrypt_done_data
+	ldp	x17, x19, [x29, #24]
+	ldp	x20, x21, [x29, #40]
+	ldp	x22, x23, [x29, #56]
+	ldp	x24, x25, [x29, #72]
+	ldp	x26, x27, [x29, #88]
+	ldr	x28, [x29, #104]
+	ldp	x29, x30, [sp], #0x70
+	ret
+	ENDP
+	ENDIF
+	ENDIF
+	IF :DEF:WOLFSSL_AESGCM_SIV
+	AREA	|.rodata|, DATA, READONLY, ALIGN=4
+	ALIGN	16
+L_AES_GCMSIV_polyval_base_r
+	DCQ	0x0000000000000000, 0x1c20000000000000
+	DCQ	0x3840000000000000, 0x2460000000000000
+	DCQ	0x7080000000000000, 0x6ca0000000000000
+	DCQ	0x48c0000000000000, 0x54e0000000000000
+	DCQ	0xe100000000000000, 0xfd20000000000000
+	DCQ	0xd940000000000000, 0xc560000000000000
+	DCQ	0x9180000000000000, 0x8da0000000000000
+	DCQ	0xa9c0000000000000, 0xb5e0000000000000
+	AREA	|.text|, CODE, READONLY
+	ALIGN	4
+	EXPORT	AES_GCMSIV_polyval_base
+AES_GCMSIV_polyval_base PROC
+	adrp	x15, L_AES_GCMSIV_polyval_base_r
+	add	x15, x15, L_AES_GCMSIV_polyval_base_r
+	cbz	w3, L_AES_GCMSIV_polyval_base_done
+L_AES_GCMSIV_polyval_base_loop
+	ldp	x6, x7, [x2]
+	ldp	x4, x5, [x0]
+	rev	x6, x6
+	rev	x7, x7
+	eor	x4, x4, x7
+	eor	x5, x5, x6
+	eor	x8, x8, x8
+	eor	x9, x9, x9
+	ubfx	x12, x5, #56, #4
+	add	x14, x1, x12, lsl 4
+	ldp	x10, x11, [x14]
+	eor	x8, x8, x10
+	eor	x9, x9, x11
+	and	x13, x9, #15
+	lsr	x9, x9, #4
+	orr	x9, x9, x8, lsl 60
+	lsr	x8, x8, #4
+	ldr	x10, [x15, x13, LSL 3]
+	eor	x8, x8, x10
+	ubfx	x12, x5, #60, #4
+	add	x14, x1, x12, lsl 4
+	ldp	x10, x11, [x14]
+	eor	x8, x8, x10
+	eor	x9, x9, x11
+	and	x13, x9, #15
+	lsr	x9, x9, #4
+	orr	x9, x9, x8, lsl 60
+	lsr	x8, x8, #4
+	ldr	x10, [x15, x13, LSL 3]
+	eor	x8, x8, x10
+	ubfx	x12, x5, #48, #4
+	add	x14, x1, x12, lsl 4
+	ldp	x10, x11, [x14]
+	eor	x8, x8, x10
+	eor	x9, x9, x11
+	and	x13, x9, #15
+	lsr	x9, x9, #4
+	orr	x9, x9, x8, lsl 60
+	lsr	x8, x8, #4
+	ldr	x10, [x15, x13, LSL 3]
+	eor	x8, x8, x10
+	ubfx	x12, x5, #52, #4
+	add	x14, x1, x12, lsl 4
+	ldp	x10, x11, [x14]
+	eor	x8, x8, x10
+	eor	x9, x9, x11
+	and	x13, x9, #15
+	lsr	x9, x9, #4
+	orr	x9, x9, x8, lsl 60
+	lsr	x8, x8, #4
+	ldr	x10, [x15, x13, LSL 3]
+	eor	x8, x8, x10
+	ubfx	x12, x5, #40, #4
+	add	x14, x1, x12, lsl 4
+	ldp	x10, x11, [x14]
+	eor	x8, x8, x10
+	eor	x9, x9, x11
+	and	x13, x9, #15
+	lsr	x9, x9, #4
+	orr	x9, x9, x8, lsl 60
+	lsr	x8, x8, #4
+	ldr	x10, [x15, x13, LSL 3]
+	eor	x8, x8, x10
+	ubfx	x12, x5, #44, #4
+	add	x14, x1, x12, lsl 4
+	ldp	x10, x11, [x14]
+	eor	x8, x8, x10
+	eor	x9, x9, x11
+	and	x13, x9, #15
+	lsr	x9, x9, #4
+	orr	x9, x9, x8, lsl 60
+	lsr	x8, x8, #4
+	ldr	x10, [x15, x13, LSL 3]
+	eor	x8, x8, x10
+	ubfx	x12, x5, #32, #4
+	add	x14, x1, x12, lsl 4
+	ldp	x10, x11, [x14]
+	eor	x8, x8, x10
+	eor	x9, x9, x11
+	and	x13, x9, #15
+	lsr	x9, x9, #4
+	orr	x9, x9, x8, lsl 60
+	lsr	x8, x8, #4
+	ldr	x10, [x15, x13, LSL 3]
+	eor	x8, x8, x10
+	ubfx	x12, x5, #36, #4
+	add	x14, x1, x12, lsl 4
+	ldp	x10, x11, [x14]
+	eor	x8, x8, x10
+	eor	x9, x9, x11
+	and	x13, x9, #15
+	lsr	x9, x9, #4
+	orr	x9, x9, x8, lsl 60
+	lsr	x8, x8, #4
+	ldr	x10, [x15, x13, LSL 3]
+	eor	x8, x8, x10
+	ubfx	x12, x5, #24, #4
+	add	x14, x1, x12, lsl 4
+	ldp	x10, x11, [x14]
+	eor	x8, x8, x10
+	eor	x9, x9, x11
+	and	x13, x9, #15
+	lsr	x9, x9, #4
+	orr	x9, x9, x8, lsl 60
+	lsr	x8, x8, #4
+	ldr	x10, [x15, x13, LSL 3]
+	eor	x8, x8, x10
+	ubfx	x12, x5, #28, #4
+	add	x14, x1, x12, lsl 4
+	ldp	x10, x11, [x14]
+	eor	x8, x8, x10
+	eor	x9, x9, x11
+	and	x13, x9, #15
+	lsr	x9, x9, #4
+	orr	x9, x9, x8, lsl 60
+	lsr	x8, x8, #4
+	ldr	x10, [x15, x13, LSL 3]
+	eor	x8, x8, x10
+	ubfx	x12, x5, #16, #4
+	add	x14, x1, x12, lsl 4
+	ldp	x10, x11, [x14]
+	eor	x8, x8, x10
+	eor	x9, x9, x11
+	and	x13, x9, #15
+	lsr	x9, x9, #4
+	orr	x9, x9, x8, lsl 60
+	lsr	x8, x8, #4
+	ldr	x10, [x15, x13, LSL 3]
+	eor	x8, x8, x10
+	ubfx	x12, x5, #20, #4
+	add	x14, x1, x12, lsl 4
+	ldp	x10, x11, [x14]
+	eor	x8, x8, x10
+	eor	x9, x9, x11
+	and	x13, x9, #15
+	lsr	x9, x9, #4
+	orr	x9, x9, x8, lsl 60
+	lsr	x8, x8, #4
+	ldr	x10, [x15, x13, LSL 3]
+	eor	x8, x8, x10
+	ubfx	x12, x5, #8, #4
+	add	x14, x1, x12, lsl 4
+	ldp	x10, x11, [x14]
+	eor	x8, x8, x10
+	eor	x9, x9, x11
+	and	x13, x9, #15
+	lsr	x9, x9, #4
+	orr	x9, x9, x8, lsl 60
+	lsr	x8, x8, #4
+	ldr	x10, [x15, x13, LSL 3]
+	eor	x8, x8, x10
+	ubfx	x12, x5, #12, #4
+	add	x14, x1, x12, lsl 4
+	ldp	x10, x11, [x14]
+	eor	x8, x8, x10
+	eor	x9, x9, x11
+	and	x13, x9, #15
+	lsr	x9, x9, #4
+	orr	x9, x9, x8, lsl 60
+	lsr	x8, x8, #4
+	ldr	x10, [x15, x13, LSL 3]
+	eor	x8, x8, x10
+	ubfx	x12, x5, #0, #4
+	add	x14, x1, x12, lsl 4
+	ldp	x10, x11, [x14]
+	eor	x8, x8, x10
+	eor	x9, x9, x11
+	and	x13, x9, #15
+	lsr	x9, x9, #4
+	orr	x9, x9, x8, lsl 60
+	lsr	x8, x8, #4
+	ldr	x10, [x15, x13, LSL 3]
+	eor	x8, x8, x10
+	ubfx	x12, x5, #4, #4
+	add	x14, x1, x12, lsl 4
+	ldp	x10, x11, [x14]
+	eor	x8, x8, x10
+	eor	x9, x9, x11
+	and	x13, x9, #15
+	lsr	x9, x9, #4
+	orr	x9, x9, x8, lsl 60
+	lsr	x8, x8, #4
+	ldr	x10, [x15, x13, LSL 3]
+	eor	x8, x8, x10
+	ubfx	x12, x4, #56, #4
+	add	x14, x1, x12, lsl 4
+	ldp	x10, x11, [x14]
+	eor	x8, x8, x10
+	eor	x9, x9, x11
+	and	x13, x9, #15
+	lsr	x9, x9, #4
+	orr	x9, x9, x8, lsl 60
+	lsr	x8, x8, #4
+	ldr	x10, [x15, x13, LSL 3]
+	eor	x8, x8, x10
+	ubfx	x12, x4, #60, #4
+	add	x14, x1, x12, lsl 4
+	ldp	x10, x11, [x14]
+	eor	x8, x8, x10
+	eor	x9, x9, x11
+	and	x13, x9, #15
+	lsr	x9, x9, #4
+	orr	x9, x9, x8, lsl 60
+	lsr	x8, x8, #4
+	ldr	x10, [x15, x13, LSL 3]
+	eor	x8, x8, x10
+	ubfx	x12, x4, #48, #4
+	add	x14, x1, x12, lsl 4
+	ldp	x10, x11, [x14]
+	eor	x8, x8, x10
+	eor	x9, x9, x11
+	and	x13, x9, #15
+	lsr	x9, x9, #4
+	orr	x9, x9, x8, lsl 60
+	lsr	x8, x8, #4
+	ldr	x10, [x15, x13, LSL 3]
+	eor	x8, x8, x10
+	ubfx	x12, x4, #52, #4
+	add	x14, x1, x12, lsl 4
+	ldp	x10, x11, [x14]
+	eor	x8, x8, x10
+	eor	x9, x9, x11
+	and	x13, x9, #15
+	lsr	x9, x9, #4
+	orr	x9, x9, x8, lsl 60
+	lsr	x8, x8, #4
+	ldr	x10, [x15, x13, LSL 3]
+	eor	x8, x8, x10
+	ubfx	x12, x4, #40, #4
+	add	x14, x1, x12, lsl 4
+	ldp	x10, x11, [x14]
+	eor	x8, x8, x10
+	eor	x9, x9, x11
+	and	x13, x9, #15
+	lsr	x9, x9, #4
+	orr	x9, x9, x8, lsl 60
+	lsr	x8, x8, #4
+	ldr	x10, [x15, x13, LSL 3]
+	eor	x8, x8, x10
+	ubfx	x12, x4, #44, #4
+	add	x14, x1, x12, lsl 4
+	ldp	x10, x11, [x14]
+	eor	x8, x8, x10
+	eor	x9, x9, x11
+	and	x13, x9, #15
+	lsr	x9, x9, #4
+	orr	x9, x9, x8, lsl 60
+	lsr	x8, x8, #4
+	ldr	x10, [x15, x13, LSL 3]
+	eor	x8, x8, x10
+	ubfx	x12, x4, #32, #4
+	add	x14, x1, x12, lsl 4
+	ldp	x10, x11, [x14]
+	eor	x8, x8, x10
+	eor	x9, x9, x11
+	and	x13, x9, #15
+	lsr	x9, x9, #4
+	orr	x9, x9, x8, lsl 60
+	lsr	x8, x8, #4
+	ldr	x10, [x15, x13, LSL 3]
+	eor	x8, x8, x10
+	ubfx	x12, x4, #36, #4
+	add	x14, x1, x12, lsl 4
+	ldp	x10, x11, [x14]
+	eor	x8, x8, x10
+	eor	x9, x9, x11
+	and	x13, x9, #15
+	lsr	x9, x9, #4
+	orr	x9, x9, x8, lsl 60
+	lsr	x8, x8, #4
+	ldr	x10, [x15, x13, LSL 3]
+	eor	x8, x8, x10
+	ubfx	x12, x4, #24, #4
+	add	x14, x1, x12, lsl 4
+	ldp	x10, x11, [x14]
+	eor	x8, x8, x10
+	eor	x9, x9, x11
+	and	x13, x9, #15
+	lsr	x9, x9, #4
+	orr	x9, x9, x8, lsl 60
+	lsr	x8, x8, #4
+	ldr	x10, [x15, x13, LSL 3]
+	eor	x8, x8, x10
+	ubfx	x12, x4, #28, #4
+	add	x14, x1, x12, lsl 4
+	ldp	x10, x11, [x14]
+	eor	x8, x8, x10
+	eor	x9, x9, x11
+	and	x13, x9, #15
+	lsr	x9, x9, #4
+	orr	x9, x9, x8, lsl 60
+	lsr	x8, x8, #4
+	ldr	x10, [x15, x13, LSL 3]
+	eor	x8, x8, x10
+	ubfx	x12, x4, #16, #4
+	add	x14, x1, x12, lsl 4
+	ldp	x10, x11, [x14]
+	eor	x8, x8, x10
+	eor	x9, x9, x11
+	and	x13, x9, #15
+	lsr	x9, x9, #4
+	orr	x9, x9, x8, lsl 60
+	lsr	x8, x8, #4
+	ldr	x10, [x15, x13, LSL 3]
+	eor	x8, x8, x10
+	ubfx	x12, x4, #20, #4
+	add	x14, x1, x12, lsl 4
+	ldp	x10, x11, [x14]
+	eor	x8, x8, x10
+	eor	x9, x9, x11
+	and	x13, x9, #15
+	lsr	x9, x9, #4
+	orr	x9, x9, x8, lsl 60
+	lsr	x8, x8, #4
+	ldr	x10, [x15, x13, LSL 3]
+	eor	x8, x8, x10
+	ubfx	x12, x4, #8, #4
+	add	x14, x1, x12, lsl 4
+	ldp	x10, x11, [x14]
+	eor	x8, x8, x10
+	eor	x9, x9, x11
+	and	x13, x9, #15
+	lsr	x9, x9, #4
+	orr	x9, x9, x8, lsl 60
+	lsr	x8, x8, #4
+	ldr	x10, [x15, x13, LSL 3]
+	eor	x8, x8, x10
+	ubfx	x12, x4, #12, #4
+	add	x14, x1, x12, lsl 4
+	ldp	x10, x11, [x14]
+	eor	x8, x8, x10
+	eor	x9, x9, x11
+	and	x13, x9, #15
+	lsr	x9, x9, #4
+	orr	x9, x9, x8, lsl 60
+	lsr	x8, x8, #4
+	ldr	x10, [x15, x13, LSL 3]
+	eor	x8, x8, x10
+	ubfx	x12, x4, #0, #4
+	add	x14, x1, x12, lsl 4
+	ldp	x10, x11, [x14]
+	eor	x8, x8, x10
+	eor	x9, x9, x11
+	and	x13, x9, #15
+	lsr	x9, x9, #4
+	orr	x9, x9, x8, lsl 60
+	lsr	x8, x8, #4
+	ldr	x10, [x15, x13, LSL 3]
+	eor	x8, x8, x10
+	ubfx	x12, x4, #4, #4
+	add	x14, x1, x12, lsl 4
+	ldp	x10, x11, [x14]
+	eor	x8, x8, x10
+	eor	x9, x9, x11
+	rev	x8, x8
+	rev	x9, x9
+	stp	x8, x9, [x0]
+	subs	w3, w3, #1
+	add	x2, x2, #16
+	bne	L_AES_GCMSIV_polyval_base_loop
+L_AES_GCMSIV_polyval_base_done
+	ret
+	ENDP
+	AREA	|.rodata|, DATA, READONLY, ALIGN=4
+	ALIGN	8
+L_AES_GCMSIV_ctr_base_te
+	DCD	0xa5c66363, 0x84f87c7c, 0x99ee7777, 0x8df67b7b
+	DCD	0x0dfff2f2, 0xbdd66b6b, 0xb1de6f6f, 0x5491c5c5
+	DCD	0x50603030, 0x03020101, 0xa9ce6767, 0x7d562b2b
+	DCD	0x19e7fefe, 0x62b5d7d7, 0xe64dabab, 0x9aec7676
+	DCD	0x458fcaca, 0x9d1f8282, 0x4089c9c9, 0x87fa7d7d
+	DCD	0x15effafa, 0xebb25959, 0xc98e4747, 0x0bfbf0f0
+	DCD	0xec41adad, 0x67b3d4d4, 0xfd5fa2a2, 0xea45afaf
+	DCD	0xbf239c9c, 0xf753a4a4, 0x96e47272, 0x5b9bc0c0
+	DCD	0xc275b7b7, 0x1ce1fdfd, 0xae3d9393, 0x6a4c2626
+	DCD	0x5a6c3636, 0x417e3f3f, 0x02f5f7f7, 0x4f83cccc
+	DCD	0x5c683434, 0xf451a5a5, 0x34d1e5e5, 0x08f9f1f1
+	DCD	0x93e27171, 0x73abd8d8, 0x53623131, 0x3f2a1515
+	DCD	0x0c080404, 0x5295c7c7, 0x65462323, 0x5e9dc3c3
+	DCD	0x28301818, 0xa1379696, 0x0f0a0505, 0xb52f9a9a
+	DCD	0x090e0707, 0x36241212, 0x9b1b8080, 0x3ddfe2e2
+	DCD	0x26cdebeb, 0x694e2727, 0xcd7fb2b2, 0x9fea7575
+	DCD	0x1b120909, 0x9e1d8383, 0x74582c2c, 0x2e341a1a
+	DCD	0x2d361b1b, 0xb2dc6e6e, 0xeeb45a5a, 0xfb5ba0a0
+	DCD	0xf6a45252, 0x4d763b3b, 0x61b7d6d6, 0xce7db3b3
+	DCD	0x7b522929, 0x3edde3e3, 0x715e2f2f, 0x97138484
+	DCD	0xf5a65353, 0x68b9d1d1, 0x00000000, 0x2cc1eded
+	DCD	0x60402020, 0x1fe3fcfc, 0xc879b1b1, 0xedb65b5b
+	DCD	0xbed46a6a, 0x468dcbcb, 0xd967bebe, 0x4b723939
+	DCD	0xde944a4a, 0xd4984c4c, 0xe8b05858, 0x4a85cfcf
+	DCD	0x6bbbd0d0, 0x2ac5efef, 0xe54faaaa, 0x16edfbfb
+	DCD	0xc5864343, 0xd79a4d4d, 0x55663333, 0x94118585
+	DCD	0xcf8a4545, 0x10e9f9f9, 0x06040202, 0x81fe7f7f
+	DCD	0xf0a05050, 0x44783c3c, 0xba259f9f, 0xe34ba8a8
+	DCD	0xf3a25151, 0xfe5da3a3, 0xc0804040, 0x8a058f8f
+	DCD	0xad3f9292, 0xbc219d9d, 0x48703838, 0x04f1f5f5
+	DCD	0xdf63bcbc, 0xc177b6b6, 0x75afdada, 0x63422121
+	DCD	0x30201010, 0x1ae5ffff, 0x0efdf3f3, 0x6dbfd2d2
+	DCD	0x4c81cdcd, 0x14180c0c, 0x35261313, 0x2fc3ecec
+	DCD	0xe1be5f5f, 0xa2359797, 0xcc884444, 0x392e1717
+	DCD	0x5793c4c4, 0xf255a7a7, 0x82fc7e7e, 0x477a3d3d
+	DCD	0xacc86464, 0xe7ba5d5d, 0x2b321919, 0x95e67373
+	DCD	0xa0c06060, 0x98198181, 0xd19e4f4f, 0x7fa3dcdc
+	DCD	0x66442222, 0x7e542a2a, 0xab3b9090, 0x830b8888
+	DCD	0xca8c4646, 0x29c7eeee, 0xd36bb8b8, 0x3c281414
+	DCD	0x79a7dede, 0xe2bc5e5e, 0x1d160b0b, 0x76addbdb
+	DCD	0x3bdbe0e0, 0x56643232, 0x4e743a3a, 0x1e140a0a
+	DCD	0xdb924949, 0x0a0c0606, 0x6c482424, 0xe4b85c5c
+	DCD	0x5d9fc2c2, 0x6ebdd3d3, 0xef43acac, 0xa6c46262
+	DCD	0xa8399191, 0xa4319595, 0x37d3e4e4, 0x8bf27979
+	DCD	0x32d5e7e7, 0x438bc8c8, 0x596e3737, 0xb7da6d6d
+	DCD	0x8c018d8d, 0x64b1d5d5, 0xd29c4e4e, 0xe049a9a9
+	DCD	0xb4d86c6c, 0xfaac5656, 0x07f3f4f4, 0x25cfeaea
+	DCD	0xafca6565, 0x8ef47a7a, 0xe947aeae, 0x18100808
+	DCD	0xd56fbaba, 0x88f07878, 0x6f4a2525, 0x725c2e2e
+	DCD	0x24381c1c, 0xf157a6a6, 0xc773b4b4, 0x5197c6c6
+	DCD	0x23cbe8e8, 0x7ca1dddd, 0x9ce87474, 0x213e1f1f
+	DCD	0xdd964b4b, 0xdc61bdbd, 0x860d8b8b, 0x850f8a8a
+	DCD	0x90e07070, 0x427c3e3e, 0xc471b5b5, 0xaacc6666
+	DCD	0xd8904848, 0x05060303, 0x01f7f6f6, 0x121c0e0e
+	DCD	0xa3c26161, 0x5f6a3535, 0xf9ae5757, 0xd069b9b9
+	DCD	0x91178686, 0x5899c1c1, 0x273a1d1d, 0xb9279e9e
+	DCD	0x38d9e1e1, 0x13ebf8f8, 0xb32b9898, 0x33221111
+	DCD	0xbbd26969, 0x70a9d9d9, 0x89078e8e, 0xa7339494
+	DCD	0xb62d9b9b, 0x223c1e1e, 0x92158787, 0x20c9e9e9
+	DCD	0x4987cece, 0xffaa5555, 0x78502828, 0x7aa5dfdf
+	DCD	0x8f038c8c, 0xf859a1a1, 0x80098989, 0x171a0d0d
+	DCD	0xda65bfbf, 0x31d7e6e6, 0xc6844242, 0xb8d06868
+	DCD	0xc3824141, 0xb0299999, 0x775a2d2d, 0x111e0f0f
+	DCD	0xcb7bb0b0, 0xfca85454, 0xd66dbbbb, 0x3a2c1616
+	AREA	|.text|, CODE, READONLY
+	ALIGN	4
+	EXPORT	AES_GCMSIV_ctr_base
+AES_GCMSIV_ctr_base PROC
+	stp	x29, x30, [sp, #-64]!
+	add	x29, sp, #0
+	stp	x17, x19, [x29, #24]
+	stp	x20, x21, [x29, #40]
+	str	x22, [x29, #56]
+	adrp	x6, L_AES_GCMSIV_ctr_base_te
+	add	x6, x6, L_AES_GCMSIV_ctr_base_te
+	ldp	x15, x16, [x5]
+	mov	w17, w15
+	cbz	x2, L_AES_GCMSIV_ctr_base_done
+L_AES_GCMSIV_ctr_base_loop_block
+	mov	x22, x3
+	ldp	x11, x12, [x22], #16
+	; Round: 0 - set counter, XOR in key schedule
+	bfi	x15, x17, #0, #32
+	rev32	x7, x15
+	rev32	x8, x16
+	eor	x7, x7, x11
+	eor	x8, x8, x12
+	sub	w21, w4, #2
+L_AES_GCMSIV_ctr_base_loop_nr
+	ubfx	x11, x7, #48, #8
+	ubfx	x14, x7, #24, #8
+	ubfx	x19, x8, #8, #8
+	ubfx	x20, x8, #32, #8
+	ldr	x9, [x6]
+	ldr	x9, [x6, #64]
+	ldr	x9, [x6, #128]
+	ldr	x9, [x6, #192]
+	ldr	x9, [x6, #256]
+	ldr	x9, [x6, #320]
+	ldr	x9, [x6, #384]
+	ldr	x9, [x6, #448]
+	ldr	x9, [x6, #512]
+	ldr	x9, [x6, #576]
+	ldr	x9, [x6, #640]
+	ldr	x9, [x6, #704]
+	ldr	x9, [x6, #768]
+	ldr	x9, [x6, #832]
+	ldr	x9, [x6, #896]
+	ldr	x9, [x6, #960]
+	ldr	w11, [x6, x11, LSL 2]
+	ldr	w14, [x6, x14, LSL 2]
+	ldr	w19, [x6, x19, LSL 2]
+	ldr	w20, [x6, x20, LSL 2]
+	ubfx	x12, x8, #16, #8
+	eor	w11, w11, w14, ror 24
+	ubfx	x14, x7, #56, #8
+	eor	w11, w11, w19, ror 8
+	ubfx	x19, x8, #40, #8
+	eor	w11, w11, w20, ror 16
+	ubfx	x20, x7, #0, #8
+	ldr	w12, [x6, x12, LSL 2]
+	ldr	w14, [x6, x14, LSL 2]
+	ldr	w19, [x6, x19, LSL 2]
+	ldr	w20, [x6, x20, LSL 2]
+	ubfx	x13, x8, #48, #8
+	eor	w12, w12, w14, ror 24
+	ubfx	x14, x8, #24, #8
+	eor	w12, w12, w19, ror 8
+	ubfx	x19, x7, #8, #8
+	eor	w12, w12, w20, ror 16
+	ubfx	x20, x7, #32, #8
+	bfi	x11, x12, #32, #32
+	ldr	w13, [x6, x13, LSL 2]
+	ldr	w14, [x6, x14, LSL 2]
+	ldr	w19, [x6, x19, LSL 2]
+	ldr	w20, [x6, x20, LSL 2]
+	ubfx	x9, x8, #0, #8
+	eor	w13, w13, w14, ror 24
+	ubfx	x14, x7, #16, #8
+	eor	w13, w13, w19, ror 8
+	ubfx	x19, x8, #56, #8
+	eor	w12, w13, w20, ror 16
+	ubfx	x20, x7, #40, #8
+	ldr	w9, [x6, x9, LSL 2]
+	ldr	w19, [x6, x19, LSL 2]
+	ldr	w14, [x6, x14, LSL 2]
+	ldr	w20, [x6, x20, LSL 2]
+	eor	w19, w19, w9, ror 24
+	ldp	x7, x8, [x22], #16
+	eor	w14, w14, w19, ror 24
+	eor	w14, w14, w20, ror 8
+	bfi	x12, x14, #32, #32
+	;   XOR in Key Schedule
+	eor	x11, x11, x7
+	eor	x12, x12, x8
+	ubfx	x7, x11, #48, #8
+	ubfx	x10, x11, #24, #8
+	ubfx	x19, x12, #8, #8
+	ubfx	x20, x12, #32, #8
+	ldr	x13, [x6]
+	ldr	x13, [x6, #64]
+	ldr	x13, [x6, #128]
+	ldr	x13, [x6, #192]
+	ldr	x13, [x6, #256]
+	ldr	x13, [x6, #320]
+	ldr	x13, [x6, #384]
+	ldr	x13, [x6, #448]
+	ldr	x13, [x6, #512]
+	ldr	x13, [x6, #576]
+	ldr	x13, [x6, #640]
+	ldr	x13, [x6, #704]
+	ldr	x13, [x6, #768]
+	ldr	x13, [x6, #832]
+	ldr	x13, [x6, #896]
+	ldr	x13, [x6, #960]
+	ldr	w7, [x6, x7, LSL 2]
+	ldr	w10, [x6, x10, LSL 2]
+	ldr	w19, [x6, x19, LSL 2]
+	ldr	w20, [x6, x20, LSL 2]
+	ubfx	x8, x12, #16, #8
+	eor	w7, w7, w10, ror 24
+	ubfx	x10, x11, #56, #8
+	eor	w7, w7, w19, ror 8
+	ubfx	x19, x12, #40, #8
+	eor	w7, w7, w20, ror 16
+	ubfx	x20, x11, #0, #8
+	ldr	w8, [x6, x8, LSL 2]
+	ldr	w10, [x6, x10, LSL 2]
+	ldr	w19, [x6, x19, LSL 2]
+	ldr	w20, [x6, x20, LSL 2]
+	ubfx	x9, x12, #48, #8
+	eor	w8, w8, w10, ror 24
+	ubfx	x10, x12, #24, #8
+	eor	w8, w8, w19, ror 8
+	ubfx	x19, x11, #8, #8
+	eor	w8, w8, w20, ror 16
+	ubfx	x20, x11, #32, #8
+	bfi	x7, x8, #32, #32
+	ldr	w9, [x6, x9, LSL 2]
+	ldr	w10, [x6, x10, LSL 2]
+	ldr	w19, [x6, x19, LSL 2]
+	ldr	w20, [x6, x20, LSL 2]
+	ubfx	x13, x12, #0, #8
+	eor	w9, w9, w10, ror 24
+	ubfx	x10, x11, #16, #8
+	eor	w9, w9, w19, ror 8
+	ubfx	x19, x12, #56, #8
+	eor	w8, w9, w20, ror 16
+	ubfx	x20, x11, #40, #8
+	ldr	w13, [x6, x13, LSL 2]
+	ldr	w19, [x6, x19, LSL 2]
+	ldr	w10, [x6, x10, LSL 2]
+	ldr	w20, [x6, x20, LSL 2]
+	eor	w19, w19, w13, ror 24
+	ldp	x11, x12, [x22], #16
+	eor	w10, w10, w19, ror 24
+	eor	w10, w10, w20, ror 8
+	bfi	x8, x10, #32, #32
 	;   XOR in Key Schedule
-	eor	x16, x16, x12
-	eor	x17, x17, x13
-	ubfx	x12, x17, #48, #8
-	ubfx	x15, x16, #24, #8
-	ubfx	x21, x17, #8, #8
-	ubfx	x22, x16, #32, #8
-	ldr	x19, [x8]
-	ldr	x19, [x8, #64]
-	ldr	x19, [x8, #128]
-	ldr	x19, [x8, #192]
-	ldr	x19, [x8, #256]
-	ldr	x19, [x8, #320]
-	ldr	x19, [x8, #384]
-	ldr	x19, [x8, #448]
-	ldr	x19, [x8, #512]
-	ldr	x19, [x8, #576]
-	ldr	x19, [x8, #640]
-	ldr	x19, [x8, #704]
-	ldr	x19, [x8, #768]
-	ldr	x19, [x8, #832]
-	ldr	x19, [x8, #896]
-	ldr	x19, [x8, #960]
-	ldr	w12, [x8, x12, LSL 2]
-	ldr	w15, [x8, x15, LSL 2]
-	ldr	w21, [x8, x21, LSL 2]
-	ldr	w22, [x8, x22, LSL 2]
-	ubfx	x13, x16, #16, #8
-	eor	w12, w12, w15, ror 24
-	ubfx	x15, x16, #56, #8
-	eor	w12, w12, w21, ror 8
-	ubfx	x21, x17, #40, #8
-	eor	w12, w12, w22, ror 16
-	ubfx	x22, x17, #0, #8
-	ldr	w13, [x8, x13, LSL 2]
-	ldr	w15, [x8, x15, LSL 2]
-	ldr	w21, [x8, x21, LSL 2]
-	ldr	w22, [x8, x22, LSL 2]
-	ubfx	x14, x16, #48, #8
-	eor	w13, w13, w15, ror 24
-	ubfx	x15, x17, #24, #8
-	eor	w13, w13, w21, ror 8
-	ubfx	x21, x16, #8, #8
-	eor	w13, w13, w22, ror 16
-	ubfx	x22, x17, #32, #8
-	bfi	x12, x13, #32, #32
-	ldr	w14, [x8, x14, LSL 2]
-	ldr	w15, [x8, x15, LSL 2]
-	ldr	w21, [x8, x21, LSL 2]
-	ldr	w22, [x8, x22, LSL 2]
-	ubfx	x19, x16, #0, #8
-	eor	w14, w14, w15, ror 24
-	ubfx	x15, x17, #16, #8
-	eor	w14, w14, w21, ror 8
-	ubfx	x21, x17, #56, #8
-	eor	w13, w14, w22, ror 16
-	ubfx	x22, x16, #40, #8
-	ldr	w19, [x8, x19, LSL 2]
-	ldr	w21, [x8, x21, LSL 2]
-	ldr	w15, [x8, x15, LSL 2]
-	ldr	w22, [x8, x22, LSL 2]
-	eor	w21, w21, w19, ror 24
-	ldp	x16, x17, [x28], #16
-	eor	w15, w15, w22, ror 8
-	eor	w15, w15, w21, ror 24
-	bfi	x13, x15, #32, #32
+	eor	x7, x7, x11
+	eor	x8, x8, x12
+	subs	w21, w21, #2
+	bne	L_AES_GCMSIV_ctr_base_loop_nr
+	ubfx	x11, x7, #48, #8
+	ubfx	x14, x7, #24, #8
+	ubfx	x19, x8, #8, #8
+	ubfx	x20, x8, #32, #8
+	ldr	x9, [x6]
+	ldr	x9, [x6, #64]
+	ldr	x9, [x6, #128]
+	ldr	x9, [x6, #192]
+	ldr	x9, [x6, #256]
+	ldr	x9, [x6, #320]
+	ldr	x9, [x6, #384]
+	ldr	x9, [x6, #448]
+	ldr	x9, [x6, #512]
+	ldr	x9, [x6, #576]
+	ldr	x9, [x6, #640]
+	ldr	x9, [x6, #704]
+	ldr	x9, [x6, #768]
+	ldr	x9, [x6, #832]
+	ldr	x9, [x6, #896]
+	ldr	x9, [x6, #960]
+	ldr	w11, [x6, x11, LSL 2]
+	ldr	w14, [x6, x14, LSL 2]
+	ldr	w19, [x6, x19, LSL 2]
+	ldr	w20, [x6, x20, LSL 2]
+	ubfx	x12, x8, #16, #8
+	eor	w11, w11, w14, ror 24
+	ubfx	x14, x7, #56, #8
+	eor	w11, w11, w19, ror 8
+	ubfx	x19, x8, #40, #8
+	eor	w11, w11, w20, ror 16
+	ubfx	x20, x7, #0, #8
+	ldr	w12, [x6, x12, LSL 2]
+	ldr	w14, [x6, x14, LSL 2]
+	ldr	w19, [x6, x19, LSL 2]
+	ldr	w20, [x6, x20, LSL 2]
+	ubfx	x13, x8, #48, #8
+	eor	w12, w12, w14, ror 24
+	ubfx	x14, x8, #24, #8
+	eor	w12, w12, w19, ror 8
+	ubfx	x19, x7, #8, #8
+	eor	w12, w12, w20, ror 16
+	ubfx	x20, x7, #32, #8
+	bfi	x11, x12, #32, #32
+	ldr	w13, [x6, x13, LSL 2]
+	ldr	w14, [x6, x14, LSL 2]
+	ldr	w19, [x6, x19, LSL 2]
+	ldr	w20, [x6, x20, LSL 2]
+	ubfx	x9, x8, #0, #8
+	eor	w13, w13, w14, ror 24
+	ubfx	x14, x7, #16, #8
+	eor	w13, w13, w19, ror 8
+	ubfx	x19, x8, #56, #8
+	eor	w12, w13, w20, ror 16
+	ubfx	x20, x7, #40, #8
+	ldr	w9, [x6, x9, LSL 2]
+	ldr	w19, [x6, x19, LSL 2]
+	ldr	w14, [x6, x14, LSL 2]
+	ldr	w20, [x6, x20, LSL 2]
+	eor	w19, w19, w9, ror 24
+	ldp	x7, x8, [x22], #16
+	eor	w14, w14, w19, ror 24
+	eor	w14, w14, w20, ror 8
+	bfi	x12, x14, #32, #32
 	;   XOR in Key Schedule
-	eor	x12, x12, x16
-	eor	x13, x13, x17
-	subs	w27, w27, #2
-	bne	L_AES_XTS_decrypt_loop_nr_partial_2
-	ubfx	x16, x13, #48, #8
-	ubfx	x20, x12, #24, #8
-	ubfx	x21, x13, #8, #8
-	ubfx	x22, x12, #32, #8
-	ldr	x14, [x8]
-	ldr	x14, [x8, #64]
-	ldr	x14, [x8, #128]
-	ldr	x14, [x8, #192]
-	ldr	x14, [x8, #256]
-	ldr	x14, [x8, #320]
-	ldr	x14, [x8, #384]
-	ldr	x14, [x8, #448]
-	ldr	x14, [x8, #512]
-	ldr	x14, [x8, #576]
-	ldr	x14, [x8, #640]
-	ldr	x14, [x8, #704]
-	ldr	x14, [x8, #768]
-	ldr	x14, [x8, #832]
-	ldr	x14, [x8, #896]
-	ldr	x14, [x8, #960]
-	ldr	w16, [x8, x16, LSL 2]
-	ldr	w20, [x8, x20, LSL 2]
-	ldr	w21, [x8, x21, LSL 2]
-	ldr	w22, [x8, x22, LSL 2]
-	ubfx	x17, x12, #16, #8
-	eor	w16, w16, w20, ror 24
-	ubfx	x20, x12, #56, #8
-	eor	w16, w16, w21, ror 8
-	ubfx	x21, x13, #40, #8
-	eor	w16, w16, w22, ror 16
-	ubfx	x22, x13, #0, #8
-	ldr	w17, [x8, x17, LSL 2]
-	ldr	w20, [x8, x20, LSL 2]
-	ldr	w21, [x8, x21, LSL 2]
-	ldr	w22, [x8, x22, LSL 2]
+	eor	x11, x11, x7
+	eor	x12, x12, x8
+	ubfx	x7, x12, #32, #8
+	ubfx	x10, x12, #8, #8
+	ubfx	x19, x11, #48, #8
+	ubfx	x20, x11, #24, #8
+	lsl	w7, w7, #2
+	lsl	w10, w10, #2
+	lsl	w19, w19, #2
+	lsl	w20, w20, #2
+	ldr	x14, [x6]
+	ldr	x14, [x6, #64]
+	ldr	x14, [x6, #128]
+	ldr	x14, [x6, #192]
+	ldr	x14, [x6, #256]
+	ldr	x14, [x6, #320]
+	ldr	x14, [x6, #384]
+	ldr	x14, [x6, #448]
+	ldr	x14, [x6, #512]
+	ldr	x14, [x6, #576]
+	ldr	x14, [x6, #640]
+	ldr	x14, [x6, #704]
+	ldr	x14, [x6, #768]
+	ldr	x14, [x6, #832]
+	ldr	x14, [x6, #896]
+	ldr	x14, [x6, #960]
+	ldrb	w7, [x6, x7, LSL 0]
+	ldrb	w10, [x6, x10, LSL 0]
+	ldrb	w19, [x6, x19, LSL 0]
+	ldrb	w20, [x6, x20, LSL 0]
+	ubfx	x8, x11, #0, #8
+	eor	w7, w7, w10, lsl 8
+	ubfx	x10, x12, #40, #8
+	eor	w7, w7, w19, lsl 16
+	ubfx	x19, x12, #16, #8
+	eor	w7, w7, w20, lsl 24
+	ubfx	x20, x11, #56, #8
+	lsl	w8, w8, #2
+	lsl	w10, w10, #2
+	lsl	w19, w19, #2
+	lsl	w20, w20, #2
+	ldrb	w8, [x6, x8, LSL 0]
+	ldrb	w10, [x6, x10, LSL 0]
+	ldrb	w19, [x6, x19, LSL 0]
+	ldrb	w20, [x6, x20, LSL 0]
+	ubfx	x9, x11, #32, #8
+	eor	w8, w8, w10, lsl 8
+	ubfx	x10, x11, #8, #8
+	eor	w8, w8, w19, lsl 16
 	ubfx	x19, x12, #48, #8
-	eor	w17, w17, w20, ror 24
-	ubfx	x20, x13, #24, #8
-	eor	w17, w17, w21, ror 8
-	ubfx	x21, x12, #8, #8
-	eor	w17, w17, w22, ror 16
-	ubfx	x22, x13, #32, #8
-	bfi	x16, x17, #32, #32
-	ldr	w19, [x8, x19, LSL 2]
-	ldr	w20, [x8, x20, LSL 2]
-	ldr	w21, [x8, x21, LSL 2]
-	ldr	w22, [x8, x22, LSL 2]
-	ubfx	x14, x12, #0, #8
-	eor	w19, w19, w20, ror 24
-	ubfx	x20, x13, #16, #8
-	eor	w19, w19, w21, ror 8
-	ubfx	x21, x13, #56, #8
-	eor	w17, w19, w22, ror 16
-	ubfx	x22, x12, #40, #8
-	ldr	w14, [x8, x14, LSL 2]
-	ldr	w21, [x8, x21, LSL 2]
-	ldr	w20, [x8, x20, LSL 2]
-	ldr	w22, [x8, x22, LSL 2]
-	eor	w21, w21, w14, ror 24
-	ldp	x12, x13, [x28], #16
-	eor	w20, w20, w22, ror 8
-	eor	w20, w20, w21, ror 24
-	bfi	x17, x20, #32, #32
-	;   XOR in Key Schedule
-	eor	x16, x16, x12
-	eor	x17, x17, x13
-	ubfx	x12, x16, #32, #8
-	ubfx	x15, x17, #8, #8
-	ubfx	x21, x17, #48, #8
-	ubfx	x22, x16, #24, #8
-	ldr	x20, [x9]
-	ldr	x20, [x9, #64]
-	ldr	x20, [x9, #128]
-	ldr	x20, [x9, #192]
-	ldrb	w12, [x9, x12, LSL 0]
-	ldrb	w15, [x9, x15, LSL 0]
-	ldrb	w21, [x9, x21, LSL 0]
-	ldrb	w22, [x9, x22, LSL 0]
-	ubfx	x13, x17, #0, #8
-	eor	w12, w12, w15, lsl 8
-	ubfx	x15, x17, #40, #8
-	eor	w12, w12, w21, lsl 16
-	ubfx	x21, x16, #16, #8
-	eor	w12, w12, w22, lsl 24
-	ubfx	x22, x16, #56, #8
-	ldrb	w15, [x9, x15, LSL 0]
-	ldrb	w22, [x9, x22, LSL 0]
-	ldrb	w13, [x9, x13, LSL 0]
-	ldrb	w21, [x9, x21, LSL 0]
-	ubfx	x14, x17, #32, #8
-	eor	w13, w13, w15, lsl 8
-	ubfx	x15, x16, #8, #8
-	eor	w13, w13, w21, lsl 16
-	ubfx	x21, x16, #48, #8
-	eor	w13, w13, w22, lsl 24
-	ubfx	x22, x17, #24, #8
-	bfi	x12, x13, #32, #32
-	ldrb	w15, [x9, x15, LSL 0]
-	ldrb	w22, [x9, x22, LSL 0]
-	ldrb	w14, [x9, x14, LSL 0]
-	ldrb	w21, [x9, x21, LSL 0]
-	ubfx	x20, x17, #56, #8
-	eor	w14, w14, w15, lsl 8
-	ubfx	x15, x16, #0, #8
-	eor	w14, w14, w21, lsl 16
-	ubfx	x21, x16, #40, #8
-	eor	w13, w14, w22, lsl 24
-	ubfx	x22, x17, #16, #8
-	ldrb	w20, [x9, x20, LSL 0]
-	ldrb	w21, [x9, x21, LSL 0]
-	ldrb	w15, [x9, x15, LSL 0]
-	ldrb	w22, [x9, x22, LSL 0]
-	eor	w21, w21, w20, lsl 16
-	ldp	x16, x17, [x28]
-	eor	w15, w15, w21, lsl 8
-	eor	w15, w15, w22, lsl 16
-	bfi	x13, x15, #32, #32
+	eor	w8, w8, w20, lsl 24
+	ubfx	x20, x12, #24, #8
+	bfi	x7, x8, #32, #32
+	lsl	w9, w9, #2
+	lsl	w10, w10, #2
+	lsl	w19, w19, #2
+	lsl	w20, w20, #2
+	ldrb	w9, [x6, x9, LSL 0]
+	ldrb	w10, [x6, x10, LSL 0]
+	ldrb	w19, [x6, x19, LSL 0]
+	ldrb	w20, [x6, x20, LSL 0]
+	ubfx	x14, x12, #56, #8
+	eor	w9, w9, w10, lsl 8
+	ubfx	x10, x12, #0, #8
+	eor	w9, w9, w19, lsl 16
+	ubfx	x19, x11, #40, #8
+	eor	w8, w9, w20, lsl 24
+	ubfx	x20, x11, #16, #8
+	lsl	w14, w14, #2
+	lsl	w10, w10, #2
+	lsl	w19, w19, #2
+	lsl	w20, w20, #2
+	ldrb	w14, [x6, x14, LSL 0]
+	ldrb	w10, [x6, x10, LSL 0]
+	ldrb	w19, [x6, x19, LSL 0]
+	ldrb	w20, [x6, x20, LSL 0]
+	eor	w19, w19, w14, lsl 16
+	ldp	x11, x12, [x22]
+	eor	w10, w10, w19, lsl 8
+	eor	w10, w10, w20, lsl 16
+	bfi	x8, x10, #32, #32
 	;   XOR in Key Schedule
-	eor	x12, x12, x16
-	eor	x13, x13, x17
-	rev32	x12, x12
-	rev32	x13, x13
-	eor	x12, x12, x23
-	eor	x13, x13, x24
-	stp	x12, x13, [x1]
-L_AES_XTS_decrypt_done_data
+	eor	x7, x7, x11
+	eor	x8, x8, x12
+	rev32	x7, x7
+	rev32	x8, x8
+	ldr	x11, [x0]
+	ldr	x12, [x0, #8]
+	eor	x7, x7, x11
+	eor	x8, x8, x12
+	str	x7, [x1]
+	str	x8, [x1, #8]
+	add	w17, w17, #1
+	subs	x2, x2, #16
+	add	x0, x0, #16
+	add	x1, x1, #16
+	bne	L_AES_GCMSIV_ctr_base_loop_block
+L_AES_GCMSIV_ctr_base_done
+	bfi	x15, x17, #0, #32
+	stp	x15, x16, [x5]
 	ldp	x17, x19, [x29, #24]
 	ldp	x20, x21, [x29, #40]
-	ldp	x22, x23, [x29, #56]
-	ldp	x24, x25, [x29, #72]
-	ldp	x26, x27, [x29, #88]
-	ldr	x28, [x29, #104]
-	ldp	x29, x30, [sp], #0x70
+	ldr	x22, [x29, #56]
+	ldp	x29, x30, [sp], #0x40
 	ret
 	ENDP
 	ENDIF
 	ENDIF
 	ENDIF
-	ENDIF
 	END
diff --git a/wolfcrypt/src/port/arm/armv8-frodokem-asm.S b/wolfcrypt/src/port/arm/armv8-frodokem-asm.S
new file mode 100644
index 0000000000..6f0fd9bb6a
--- /dev/null
+++ b/wolfcrypt/src/port/arm/armv8-frodokem-asm.S
@@ -0,0 +1,2347 @@
+/* armv8-frodokem-asm
+ *
+ * Copyright (C) 2006-2026 wolfSSL Inc.
+ *
+ * This file is part of wolfSSL.
+ *
+ * wolfSSL is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * wolfSSL is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335, USA
+ */
+
+#include <wolfssl/wolfcrypt/libwolfssl_sources_asm.h>
+
+/* Generated using (from wolfssl):
+ *   cd ../scripts
+ *   ruby ./frodokem/frodokem.rb arm64 \
+ *       ../wolfssl/wolfcrypt/src/port/arm/armv8-frodokem-asm.S
+ */
+#ifdef WOLFSSL_ARMASM
+#ifdef __aarch64__
+#ifndef WOLFSSL_ARMASM_INLINE
+#ifdef WOLFSSL_HAVE_FRODOKEM
+#ifndef __APPLE__
+	.text
+	.section	.rodata
+	.type	L_sha3_aarch64_r, %object
+	.size	L_sha3_aarch64_r, 192
+#else
+	.section	__DATA,__data
+#endif /* __APPLE__ */
+	# 16-byte aligned, 128-bit aligned
+#ifndef __APPLE__
+	.align	4
+#else
+	.p2align	4
+#endif /* __APPLE__ */
+L_sha3_aarch64_r:
+	.quad	0x0000000000000001,0x0000000000008082
+	.quad	0x800000000000808a,0x8000000080008000
+	.quad	0x000000000000808b,0x0000000080000001
+	.quad	0x8000000080008081,0x8000000000008009
+	.quad	0x000000000000008a,0x0000000000000088
+	.quad	0x0000000080008009,0x000000008000000a
+	.quad	0x000000008000808b,0x800000000000008b
+	.quad	0x8000000000008089,0x8000000000008003
+	.quad	0x8000000000008002,0x8000000000000080
+	.quad	0x000000000000800a,0x800000008000000a
+	.quad	0x8000000080008081,0x8000000000008080
+	.quad	0x0000000080000001,0x8000000080008008
+#ifndef __APPLE__
+.text
+.globl	frodokem_sha3_x2_neon
+.type	frodokem_sha3_x2_neon,@function
+.align	2
+frodokem_sha3_x2_neon:
+#else
+.section	__TEXT,__text
+.globl	_frodokem_sha3_x2_neon
+.p2align	2
+_frodokem_sha3_x2_neon:
+#endif /* __APPLE__ */
+	stp	x29, x30, [sp, #-80]!
+	add	x29, sp, #0
+	stp	d8, d9, [x29, #16]
+	stp	d10, d11, [x29, #32]
+	stp	d12, d13, [x29, #48]
+	stp	d14, d15, [x29, #64]
+#ifndef __APPLE__
+	adrp x1, L_sha3_aarch64_r
+	add  x1, x1, :lo12:L_sha3_aarch64_r
+#else
+	adrp x1, L_sha3_aarch64_r@PAGE
+	add  x1, x1, L_sha3_aarch64_r@PAGEOFF
+#endif /* __APPLE__ */
+	ld4	{v0.d, v1.d, v2.d, v3.d}[0], [x0], #32
+	ld4	{v4.d, v5.d, v6.d, v7.d}[0], [x0], #32
+	ld4	{v8.d, v9.d, v10.d, v11.d}[0], [x0], #32
+	ld4	{v12.d, v13.d, v14.d, v15.d}[0], [x0], #32
+	ld4	{v16.d, v17.d, v18.d, v19.d}[0], [x0], #32
+	ld4	{v20.d, v21.d, v22.d, v23.d}[0], [x0], #32
+	ld1	{v24.d}[0], [x0]
+	add	x0, x0, #8
+	ld4	{v0.d, v1.d, v2.d, v3.d}[1], [x0], #32
+	ld4	{v4.d, v5.d, v6.d, v7.d}[1], [x0], #32
+	ld4	{v8.d, v9.d, v10.d, v11.d}[1], [x0], #32
+	ld4	{v12.d, v13.d, v14.d, v15.d}[1], [x0], #32
+	ld4	{v16.d, v17.d, v18.d, v19.d}[1], [x0], #32
+	ld4	{v20.d, v21.d, v22.d, v23.d}[1], [x0], #32
+	ld1	{v24.d}[1], [x0]
+	add	x0, x0, #8
+	sub	x0, x0, #0x190
+	mov	x2, #24
+L_frodokem_sha3_x2_neon_begin:
+	# Col Mix NEON
+	eor	v30.16b, v4.16b, v9.16b
+	eor	v27.16b, v1.16b, v6.16b
+	eor	v30.16b, v30.16b, v14.16b
+	eor	v27.16b, v27.16b, v11.16b
+	eor	v30.16b, v30.16b, v19.16b
+	eor	v27.16b, v27.16b, v16.16b
+	eor	v30.16b, v30.16b, v24.16b
+	eor	v27.16b, v27.16b, v21.16b
+	ushr	v25.2d, v27.2d, #63
+	sli	v25.2d, v27.2d, #1
+	eor	v25.16b, v25.16b, v30.16b
+	eor	v31.16b, v0.16b, v5.16b
+	eor	v28.16b, v2.16b, v7.16b
+	eor	v31.16b, v31.16b, v10.16b
+	eor	v28.16b, v28.16b, v12.16b
+	eor	v31.16b, v31.16b, v15.16b
+	eor	v28.16b, v28.16b, v17.16b
+	eor	v31.16b, v31.16b, v20.16b
+	eor	v28.16b, v28.16b, v22.16b
+	ushr	v29.2d, v30.2d, #63
+	ushr	v26.2d, v28.2d, #63
+	sli	v29.2d, v30.2d, #1
+	sli	v26.2d, v28.2d, #1
+	eor	v28.16b, v28.16b, v29.16b
+	eor	v29.16b, v3.16b, v8.16b
+	eor	v26.16b, v26.16b, v31.16b
+	eor	v29.16b, v29.16b, v13.16b
+	eor	v29.16b, v29.16b, v18.16b
+	eor	v29.16b, v29.16b, v23.16b
+	ushr	v30.2d, v29.2d, #63
+	sli	v30.2d, v29.2d, #1
+	eor	v27.16b, v27.16b, v30.16b
+	ushr	v30.2d, v31.2d, #63
+	sli	v30.2d, v31.2d, #1
+	eor	v29.16b, v29.16b, v30.16b
+	# Swap Rotate NEON
+	eor	v0.16b, v0.16b, v25.16b
+	eor	v31.16b, v1.16b, v26.16b
+	eor	v6.16b, v6.16b, v26.16b
+	ushr	v30.2d, v31.2d, #63
+	ushr	v1.2d, v6.2d, #20
+	sli	v30.2d, v31.2d, #1
+	sli	v1.2d, v6.2d, #44
+	eor	v31.16b, v9.16b, v29.16b
+	eor	v22.16b, v22.16b, v27.16b
+	ushr	v6.2d, v31.2d, #44
+	ushr	v9.2d, v22.2d, #3
+	sli	v6.2d, v31.2d, #20
+	sli	v9.2d, v22.2d, #61
+	eor	v31.16b, v14.16b, v29.16b
+	eor	v20.16b, v20.16b, v25.16b
+	ushr	v22.2d, v31.2d, #25
+	ushr	v14.2d, v20.2d, #46
+	sli	v22.2d, v31.2d, #39
+	sli	v14.2d, v20.2d, #18
+	eor	v31.16b, v2.16b, v27.16b
+	eor	v12.16b, v12.16b, v27.16b
+	ushr	v20.2d, v31.2d, #2
+	ushr	v2.2d, v12.2d, #21
+	sli	v20.2d, v31.2d, #62
+	sli	v2.2d, v12.2d, #43
+	eor	v31.16b, v13.16b, v28.16b
+	eor	v19.16b, v19.16b, v29.16b
+	ushr	v12.2d, v31.2d, #39
+	ushr	v13.2d, v19.2d, #56
+	sli	v12.2d, v31.2d, #25
+	sli	v13.2d, v19.2d, #8
+	eor	v31.16b, v23.16b, v28.16b
+	eor	v15.16b, v15.16b, v25.16b
+	ushr	v19.2d, v31.2d, #8
+	ushr	v23.2d, v15.2d, #23
+	sli	v19.2d, v31.2d, #56
+	sli	v23.2d, v15.2d, #41
+	eor	v31.16b, v4.16b, v29.16b
+	eor	v24.16b, v24.16b, v29.16b
+	ushr	v15.2d, v31.2d, #37
+	ushr	v4.2d, v24.2d, #50
+	sli	v15.2d, v31.2d, #27
+	sli	v4.2d, v24.2d, #14
+	eor	v31.16b, v21.16b, v26.16b
+	eor	v8.16b, v8.16b, v28.16b
+	ushr	v24.2d, v31.2d, #62
+	ushr	v21.2d, v8.2d, #9
+	sli	v24.2d, v31.2d, #2
+	sli	v21.2d, v8.2d, #55
+	eor	v31.16b, v16.16b, v26.16b
+	eor	v5.16b, v5.16b, v25.16b
+	ushr	v8.2d, v31.2d, #19
+	ushr	v16.2d, v5.2d, #28
+	sli	v8.2d, v31.2d, #45
+	sli	v16.2d, v5.2d, #36
+	eor	v31.16b, v3.16b, v28.16b
+	eor	v18.16b, v18.16b, v28.16b
+	ushr	v5.2d, v31.2d, #36
+	ushr	v3.2d, v18.2d, #43
+	sli	v5.2d, v31.2d, #28
+	sli	v3.2d, v18.2d, #21
+	eor	v31.16b, v17.16b, v27.16b
+	eor	v11.16b, v11.16b, v26.16b
+	ushr	v18.2d, v31.2d, #49
+	ushr	v17.2d, v11.2d, #54
+	sli	v18.2d, v31.2d, #15
+	sli	v17.2d, v11.2d, #10
+	eor	v31.16b, v7.16b, v27.16b
+	eor	v10.16b, v10.16b, v25.16b
+	ushr	v11.2d, v31.2d, #58
+	ushr	v7.2d, v10.2d, #61
+	sli	v11.2d, v31.2d, #6
+	sli	v7.2d, v10.2d, #3
+	# Row Mix NEON
+	bic	v25.16b, v2.16b, v1.16b
+	bic	v26.16b, v3.16b, v2.16b
+	bic	v27.16b, v4.16b, v3.16b
+	bic	v28.16b, v0.16b, v4.16b
+	bic	v29.16b, v1.16b, v0.16b
+	eor	v0.16b, v0.16b, v25.16b
+	eor	v1.16b, v1.16b, v26.16b
+	eor	v2.16b, v2.16b, v27.16b
+	eor	v3.16b, v3.16b, v28.16b
+	eor	v4.16b, v4.16b, v29.16b
+	bic	v25.16b, v7.16b, v6.16b
+	bic	v26.16b, v8.16b, v7.16b
+	bic	v27.16b, v9.16b, v8.16b
+	bic	v28.16b, v5.16b, v9.16b
+	bic	v29.16b, v6.16b, v5.16b
+	eor	v5.16b, v5.16b, v25.16b
+	eor	v6.16b, v6.16b, v26.16b
+	eor	v7.16b, v7.16b, v27.16b
+	eor	v8.16b, v8.16b, v28.16b
+	eor	v9.16b, v9.16b, v29.16b
+	bic	v25.16b, v12.16b, v11.16b
+	bic	v26.16b, v13.16b, v12.16b
+	bic	v27.16b, v14.16b, v13.16b
+	bic	v28.16b, v30.16b, v14.16b
+	bic	v29.16b, v11.16b, v30.16b
+	eor	v10.16b, v30.16b, v25.16b
+	eor	v11.16b, v11.16b, v26.16b
+	eor	v12.16b, v12.16b, v27.16b
+	eor	v13.16b, v13.16b, v28.16b
+	eor	v14.16b, v14.16b, v29.16b
+	bic	v25.16b, v17.16b, v16.16b
+	bic	v26.16b, v18.16b, v17.16b
+	bic	v27.16b, v19.16b, v18.16b
+	bic	v28.16b, v15.16b, v19.16b
+	bic	v29.16b, v16.16b, v15.16b
+	eor	v15.16b, v15.16b, v25.16b
+	eor	v16.16b, v16.16b, v26.16b
+	eor	v17.16b, v17.16b, v27.16b
+	eor	v18.16b, v18.16b, v28.16b
+	eor	v19.16b, v19.16b, v29.16b
+	bic	v25.16b, v22.16b, v21.16b
+	bic	v26.16b, v23.16b, v22.16b
+	bic	v27.16b, v24.16b, v23.16b
+	bic	v28.16b, v20.16b, v24.16b
+	bic	v29.16b, v21.16b, v20.16b
+	eor	v20.16b, v20.16b, v25.16b
+	eor	v21.16b, v21.16b, v26.16b
+	eor	v22.16b, v22.16b, v27.16b
+	eor	v23.16b, v23.16b, v28.16b
+	eor	v24.16b, v24.16b, v29.16b
+	ldr	x3, [x1], #8
+	subs	x2, x2, #1
+	mov	v30.d[0], x3
+	mov	v30.d[1], x3
+	eor	v0.16b, v0.16b, v30.16b
+	bne	L_frodokem_sha3_x2_neon_begin
+	st4	{v0.d, v1.d, v2.d, v3.d}[0], [x0], #32
+	st4	{v4.d, v5.d, v6.d, v7.d}[0], [x0], #32
+	st4	{v8.d, v9.d, v10.d, v11.d}[0], [x0], #32
+	st4	{v12.d, v13.d, v14.d, v15.d}[0], [x0], #32
+	st4	{v16.d, v17.d, v18.d, v19.d}[0], [x0], #32
+	st4	{v20.d, v21.d, v22.d, v23.d}[0], [x0], #32
+	st1	{v24.d}[0], [x0]
+	add	x0, x0, #8
+	st4	{v0.d, v1.d, v2.d, v3.d}[1], [x0], #32
+	st4	{v4.d, v5.d, v6.d, v7.d}[1], [x0], #32
+	st4	{v8.d, v9.d, v10.d, v11.d}[1], [x0], #32
+	st4	{v12.d, v13.d, v14.d, v15.d}[1], [x0], #32
+	st4	{v16.d, v17.d, v18.d, v19.d}[1], [x0], #32
+	st4	{v20.d, v21.d, v22.d, v23.d}[1], [x0], #32
+	st1	{v24.d}[1], [x0]
+	add	x0, x0, #8
+	ldp	d8, d9, [x29, #16]
+	ldp	d10, d11, [x29, #32]
+	ldp	d12, d13, [x29, #48]
+	ldp	d14, d15, [x29, #64]
+	ldp	x29, x30, [sp], #0x50
+	ret
+#ifndef __APPLE__
+	.size	frodokem_sha3_x2_neon,.-frodokem_sha3_x2_neon
+#endif /* __APPLE__ */
+#ifdef WOLFSSL_ARMASM_CRYPTO_SHA3
+	.arch_extension	sha3
+#ifndef __APPLE__
+.text
+.globl	frodokem_sha3_x2_crypto
+.type	frodokem_sha3_x2_crypto,@function
+.align	2
+frodokem_sha3_x2_crypto:
+#else
+.section	__TEXT,__text
+.globl	_frodokem_sha3_x2_crypto
+.p2align	2
+_frodokem_sha3_x2_crypto:
+#endif /* __APPLE__ */
+	stp	x29, x30, [sp, #-80]!
+	add	x29, sp, #0
+	stp	d8, d9, [x29, #16]
+	stp	d10, d11, [x29, #32]
+	stp	d12, d13, [x29, #48]
+	stp	d14, d15, [x29, #64]
+#ifndef __APPLE__
+	adrp x1, L_sha3_aarch64_r
+	add  x1, x1, :lo12:L_sha3_aarch64_r
+#else
+	adrp x1, L_sha3_aarch64_r@PAGE
+	add  x1, x1, L_sha3_aarch64_r@PAGEOFF
+#endif /* __APPLE__ */
+	ld4	{v0.d, v1.d, v2.d, v3.d}[0], [x0], #32
+	ld4	{v4.d, v5.d, v6.d, v7.d}[0], [x0], #32
+	ld4	{v8.d, v9.d, v10.d, v11.d}[0], [x0], #32
+	ld4	{v12.d, v13.d, v14.d, v15.d}[0], [x0], #32
+	ld4	{v16.d, v17.d, v18.d, v19.d}[0], [x0], #32
+	ld4	{v20.d, v21.d, v22.d, v23.d}[0], [x0], #32
+	ld1	{v24.d}[0], [x0]
+	add	x0, x0, #8
+	ld4	{v0.d, v1.d, v2.d, v3.d}[1], [x0], #32
+	ld4	{v4.d, v5.d, v6.d, v7.d}[1], [x0], #32
+	ld4	{v8.d, v9.d, v10.d, v11.d}[1], [x0], #32
+	ld4	{v12.d, v13.d, v14.d, v15.d}[1], [x0], #32
+	ld4	{v16.d, v17.d, v18.d, v19.d}[1], [x0], #32
+	ld4	{v20.d, v21.d, v22.d, v23.d}[1], [x0], #32
+	ld1	{v24.d}[1], [x0]
+	add	x0, x0, #8
+	sub	x0, x0, #0x190
+	mov	x2, #24
+L_frodokem_sha3_x2_crypto_begin:
+	# Col Mix
+	eor3	v31.16b, v0.16b, v5.16b, v10.16b
+	eor3	v27.16b, v1.16b, v6.16b, v11.16b
+	eor3	v28.16b, v2.16b, v7.16b, v12.16b
+	eor3	v29.16b, v3.16b, v8.16b, v13.16b
+	eor3	v30.16b, v4.16b, v9.16b, v14.16b
+	eor3	v31.16b, v31.16b, v15.16b, v20.16b
+	eor3	v27.16b, v27.16b, v16.16b, v21.16b
+	eor3	v28.16b, v28.16b, v17.16b, v22.16b
+	eor3	v29.16b, v29.16b, v18.16b, v23.16b
+	eor3	v30.16b, v30.16b, v19.16b, v24.16b
+	rax1	v25.2d, v30.2d, v27.2d
+	rax1	v26.2d, v31.2d, v28.2d
+	rax1	v27.2d, v27.2d, v29.2d
+	rax1	v28.2d, v28.2d, v30.2d
+	rax1	v29.2d, v29.2d, v31.2d
+	eor	v0.16b, v0.16b, v25.16b
+	xar	v30.2d, v1.2d, v26.2d, #63
+	xar	v1.2d, v6.2d, v26.2d, #20
+	xar	v6.2d, v9.2d, v29.2d, #44
+	xar	v9.2d, v22.2d, v27.2d, #3
+	xar	v22.2d, v14.2d, v29.2d, #25
+	xar	v14.2d, v20.2d, v25.2d, #46
+	xar	v20.2d, v2.2d, v27.2d, #2
+	xar	v2.2d, v12.2d, v27.2d, #21
+	xar	v12.2d, v13.2d, v28.2d, #39
+	xar	v13.2d, v19.2d, v29.2d, #56
+	xar	v19.2d, v23.2d, v28.2d, #8
+	xar	v23.2d, v15.2d, v25.2d, #23
+	xar	v15.2d, v4.2d, v29.2d, #37
+	xar	v4.2d, v24.2d, v29.2d, #50
+	xar	v24.2d, v21.2d, v26.2d, #62
+	xar	v21.2d, v8.2d, v28.2d, #9
+	xar	v8.2d, v16.2d, v26.2d, #19
+	xar	v16.2d, v5.2d, v25.2d, #28
+	xar	v5.2d, v3.2d, v28.2d, #36
+	xar	v3.2d, v18.2d, v28.2d, #43
+	xar	v18.2d, v17.2d, v27.2d, #49
+	xar	v17.2d, v11.2d, v26.2d, #54
+	xar	v11.2d, v7.2d, v27.2d, #58
+	xar	v7.2d, v10.2d, v25.2d, #61
+	# Row Mix
+	mov	v25.16b, v0.16b
+	mov	v26.16b, v1.16b
+	bcax	v0.16b, v25.16b, v2.16b, v26.16b
+	bcax	v1.16b, v26.16b, v3.16b, v2.16b
+	bcax	v2.16b, v2.16b, v4.16b, v3.16b
+	bcax	v3.16b, v3.16b, v25.16b, v4.16b
+	bcax	v4.16b, v4.16b, v26.16b, v25.16b
+	mov	v25.16b, v5.16b
+	mov	v26.16b, v6.16b
+	bcax	v5.16b, v25.16b, v7.16b, v26.16b
+	bcax	v6.16b, v26.16b, v8.16b, v7.16b
+	bcax	v7.16b, v7.16b, v9.16b, v8.16b
+	bcax	v8.16b, v8.16b, v25.16b, v9.16b
+	bcax	v9.16b, v9.16b, v26.16b, v25.16b
+	mov	v26.16b, v11.16b
+	bcax	v10.16b, v30.16b, v12.16b, v26.16b
+	bcax	v11.16b, v26.16b, v13.16b, v12.16b
+	bcax	v12.16b, v12.16b, v14.16b, v13.16b
+	bcax	v13.16b, v13.16b, v30.16b, v14.16b
+	bcax	v14.16b, v14.16b, v26.16b, v30.16b
+	mov	v25.16b, v15.16b
+	mov	v26.16b, v16.16b
+	bcax	v15.16b, v25.16b, v17.16b, v26.16b
+	bcax	v16.16b, v26.16b, v18.16b, v17.16b
+	bcax	v17.16b, v17.16b, v19.16b, v18.16b
+	bcax	v18.16b, v18.16b, v25.16b, v19.16b
+	bcax	v19.16b, v19.16b, v26.16b, v25.16b
+	mov	v25.16b, v20.16b
+	mov	v26.16b, v21.16b
+	bcax	v20.16b, v25.16b, v22.16b, v26.16b
+	bcax	v21.16b, v26.16b, v23.16b, v22.16b
+	bcax	v22.16b, v22.16b, v24.16b, v23.16b
+	bcax	v23.16b, v23.16b, v25.16b, v24.16b
+	bcax	v24.16b, v24.16b, v26.16b, v25.16b
+	ldr	x3, [x1], #8
+	subs	x2, x2, #1
+	mov	v30.d[0], x3
+	mov	v30.d[1], x3
+	eor	v0.16b, v0.16b, v30.16b
+	bne	L_frodokem_sha3_x2_crypto_begin
+	st4	{v0.d, v1.d, v2.d, v3.d}[0], [x0], #32
+	st4	{v4.d, v5.d, v6.d, v7.d}[0], [x0], #32
+	st4	{v8.d, v9.d, v10.d, v11.d}[0], [x0], #32
+	st4	{v12.d, v13.d, v14.d, v15.d}[0], [x0], #32
+	st4	{v16.d, v17.d, v18.d, v19.d}[0], [x0], #32
+	st4	{v20.d, v21.d, v22.d, v23.d}[0], [x0], #32
+	st1	{v24.d}[0], [x0]
+	add	x0, x0, #8
+	st4	{v0.d, v1.d, v2.d, v3.d}[1], [x0], #32
+	st4	{v4.d, v5.d, v6.d, v7.d}[1], [x0], #32
+	st4	{v8.d, v9.d, v10.d, v11.d}[1], [x0], #32
+	st4	{v12.d, v13.d, v14.d, v15.d}[1], [x0], #32
+	st4	{v16.d, v17.d, v18.d, v19.d}[1], [x0], #32
+	st4	{v20.d, v21.d, v22.d, v23.d}[1], [x0], #32
+	st1	{v24.d}[1], [x0]
+	add	x0, x0, #8
+	ldp	d8, d9, [x29, #16]
+	ldp	d10, d11, [x29, #32]
+	ldp	d12, d13, [x29, #48]
+	ldp	d14, d15, [x29, #64]
+	ldp	x29, x30, [sp], #0x50
+	ret
+#ifndef __APPLE__
+	.size	frodokem_sha3_x2_crypto,.-frodokem_sha3_x2_crypto
+#endif /* __APPLE__ */
+#endif /* WOLFSSL_ARMASM_CRYPTO_SHA3 */
+#ifndef __APPLE__
+.text
+.globl	frodokem_add_neon
+.type	frodokem_add_neon,@function
+.align	2
+frodokem_add_neon:
+#else
+.section	__TEXT,__text
+.globl	_frodokem_add_neon
+.p2align	2
+_frodokem_add_neon:
+#endif /* __APPLE__ */
+	stp	x29, x30, [sp, #-80]!
+	add	x29, sp, #0
+	stp	d8, d9, [x29, #16]
+	stp	d10, d11, [x29, #32]
+	stp	d12, d13, [x29, #48]
+	stp	d14, d15, [x29, #64]
+	dup	v16.8h, w2
+	ld1	{v0.8h, v1.8h, v2.8h, v3.8h}, [x0], #0x40
+	ld1	{v4.8h, v5.8h, v6.8h, v7.8h}, [x0], #0x40
+	sub	x0, x0, #0x80
+	ld1	{v8.8h, v9.8h, v10.8h, v11.8h}, [x1], #0x40
+	ld1	{v12.8h, v13.8h, v14.8h, v15.8h}, [x1], #0x40
+	add	v0.8h, v0.8h, v8.8h
+	and	v0.16b, v0.16b, v16.16b
+	add	v1.8h, v1.8h, v9.8h
+	and	v1.16b, v1.16b, v16.16b
+	add	v2.8h, v2.8h, v10.8h
+	and	v2.16b, v2.16b, v16.16b
+	add	v3.8h, v3.8h, v11.8h
+	and	v3.16b, v3.16b, v16.16b
+	add	v4.8h, v4.8h, v12.8h
+	and	v4.16b, v4.16b, v16.16b
+	add	v5.8h, v5.8h, v13.8h
+	and	v5.16b, v5.16b, v16.16b
+	add	v6.8h, v6.8h, v14.8h
+	and	v6.16b, v6.16b, v16.16b
+	add	v7.8h, v7.8h, v15.8h
+	and	v7.16b, v7.16b, v16.16b
+	st1	{v0.8h, v1.8h, v2.8h, v3.8h}, [x0], #0x40
+	st1	{v4.8h, v5.8h, v6.8h, v7.8h}, [x0], #0x40
+	ldp	d8, d9, [x29, #16]
+	ldp	d10, d11, [x29, #32]
+	ldp	d12, d13, [x29, #48]
+	ldp	d14, d15, [x29, #64]
+	ldp	x29, x30, [sp], #0x50
+	ret
+#ifndef __APPLE__
+	.size	frodokem_add_neon,.-frodokem_add_neon
+#endif /* __APPLE__ */
+#ifndef __APPLE__
+.text
+.globl	frodokem_sample_neon
+.type	frodokem_sample_neon,@function
+.align	2
+frodokem_sample_neon:
+#else
+.section	__TEXT,__text
+.globl	_frodokem_sample_neon
+.p2align	2
+_frodokem_sample_neon:
+#endif /* __APPLE__ */
+	stp	x29, x30, [sp, #-64]!
+	add	x29, sp, #0
+	stp	d8, d9, [x29, #16]
+	stp	d10, d11, [x29, #32]
+	stp	d12, d13, [x29, #48]
+	cmp	w1, #16
+	blt	L_frodokem_sample_neon_rem
+L_frodokem_sample_neon_blk16:
+	ld1	{v0.8h}, [x0], #16
+	ld1	{v1.8h}, [x0], #16
+	sub	x0, x0, #32
+	ushr	v2.8h, v0.8h, #1
+	movi	v4.8h, #1
+	and	v4.16b, v4.16b, v0.16b
+	movi	v6.8h, #0
+	ushr	v3.8h, v1.8h, #1
+	movi	v5.8h, #1
+	and	v5.16b, v5.16b, v1.16b
+	movi	v7.8h, #0
+	mov	x4, x2
+	mov	w5, w3
+L_frodokem_sample_neon_cdf16:
+	ld1r	{v12.8h}, [x4], #2
+	sub	v10.8h, v12.8h, v2.8h
+	ushr	v10.8h, v10.8h, #15
+	add	v6.8h, v6.8h, v10.8h
+	sub	v11.8h, v12.8h, v3.8h
+	ushr	v11.8h, v11.8h, #15
+	add	v7.8h, v7.8h, v11.8h
+	subs	w5, w5, #1
+	bne	L_frodokem_sample_neon_cdf16
+	neg	v8.8h, v4.8h
+	eor	v6.16b, v6.16b, v8.16b
+	add	v6.8h, v6.8h, v4.8h
+	st1	{v6.8h}, [x0], #16
+	neg	v9.8h, v5.8h
+	eor	v7.16b, v7.16b, v9.16b
+	add	v7.8h, v7.8h, v5.8h
+	st1	{v7.8h}, [x0], #16
+	subs	w1, w1, #16
+	cmp	w1, #16
+	bge	L_frodokem_sample_neon_blk16
+L_frodokem_sample_neon_rem:
+	cmp	w1, #0
+	beq	L_frodokem_sample_neon_done
+L_frodokem_sample_neon_blk:
+	ld1	{v0.8h}, [x0]
+	ushr	v2.8h, v0.8h, #1
+	movi	v4.8h, #1
+	and	v4.16b, v4.16b, v0.16b
+	movi	v6.8h, #0
+	mov	x4, x2
+	mov	w5, w3
+L_frodokem_sample_neon_cdf:
+	ld1r	{v12.8h}, [x4], #2
+	sub	v12.8h, v12.8h, v2.8h
+	ushr	v12.8h, v12.8h, #15
+	add	v6.8h, v6.8h, v12.8h
+	subs	w5, w5, #1
+	bne	L_frodokem_sample_neon_cdf
+	neg	v8.8h, v4.8h
+	eor	v6.16b, v6.16b, v8.16b
+	add	v6.8h, v6.8h, v4.8h
+	st1	{v6.8h}, [x0], #16
+	subs	w1, w1, #8
+	bne	L_frodokem_sample_neon_blk
+L_frodokem_sample_neon_done:
+	ldp	d8, d9, [x29, #16]
+	ldp	d10, d11, [x29, #32]
+	ldp	d12, d13, [x29, #48]
+	ldp	x29, x30, [sp], #0x40
+	ret
+#ifndef __APPLE__
+	.size	frodokem_sample_neon,.-frodokem_sample_neon
+#endif /* __APPLE__ */
+#ifndef __APPLE__
+.text
+.globl	frodokem_sa_accum_neon
+.type	frodokem_sa_accum_neon,@function
+.align	2
+frodokem_sa_accum_neon:
+#else
+.section	__TEXT,__text
+.globl	_frodokem_sa_accum_neon
+.p2align	2
+_frodokem_sa_accum_neon:
+#endif /* __APPLE__ */
+	stp	x29, x30, [sp, #-80]!
+	add	x29, sp, #0
+	str	x17, [x29, #24]
+	stp	d8, d9, [x29, #32]
+	stp	d10, d11, [x29, #48]
+	stp	d12, d13, [x29, #64]
+	sxtw	x5, w4
+	lsl	x5, x5, #1
+	lsl	x6, x5, #2
+	sxtw	x7, w3
+	lsl	x7, x7, #1
+	add	x7, x1, x7
+	mov	x8, x0
+	mov	x12, #2
+L_frodokem_sa_accum_neon_t:
+	ldr	w13, [x7]
+	dup	v0.8h, w13
+	lsr	w13, w13, #16
+	dup	v4.8h, w13
+	add	x7, x7, x5
+	ldr	w13, [x7]
+	dup	v1.8h, w13
+	lsr	w13, w13, #16
+	dup	v5.8h, w13
+	add	x7, x7, x5
+	ldr	w13, [x7]
+	dup	v2.8h, w13
+	lsr	w13, w13, #16
+	dup	v6.8h, w13
+	add	x7, x7, x5
+	ldr	w13, [x7]
+	dup	v3.8h, w13
+	lsr	w13, w13, #16
+	dup	v7.8h, w13
+	add	x7, x7, x5
+	mov	x14, x8
+	add	x15, x14, x5
+	add	x16, x15, x5
+	add	x17, x16, x5
+	mov	x9, x2
+	add	x10, x2, x5
+	lsr	x11, x5, #4
+L_frodokem_sa_accum_neon_k:
+	ld1	{v8.8h}, [x9], #16
+	ld1	{v9.8h}, [x10], #16
+	ld1	{v10.8h}, [x14]
+	mla	v10.8h, v8.8h, v0.8h
+	mla	v10.8h, v9.8h, v4.8h
+	st1	{v10.8h}, [x14], #16
+	ld1	{v11.8h}, [x15]
+	mla	v11.8h, v8.8h, v1.8h
+	mla	v11.8h, v9.8h, v5.8h
+	st1	{v11.8h}, [x15], #16
+	ld1	{v12.8h}, [x16]
+	mla	v12.8h, v8.8h, v2.8h
+	mla	v12.8h, v9.8h, v6.8h
+	st1	{v12.8h}, [x16], #16
+	ld1	{v13.8h}, [x17]
+	mla	v13.8h, v8.8h, v3.8h
+	mla	v13.8h, v9.8h, v7.8h
+	st1	{v13.8h}, [x17], #16
+	subs	x11, x11, #1
+	bne	L_frodokem_sa_accum_neon_k
+	add	x8, x8, x6
+	subs	x12, x12, #1
+	bne	L_frodokem_sa_accum_neon_t
+	ldr	x17, [x29, #24]
+	ldp	d8, d9, [x29, #32]
+	ldp	d10, d11, [x29, #48]
+	ldp	d12, d13, [x29, #64]
+	ldp	x29, x30, [sp], #0x50
+	ret
+#ifndef __APPLE__
+	.size	frodokem_sa_accum_neon,.-frodokem_sa_accum_neon
+#endif /* __APPLE__ */
+#ifndef __APPLE__
+.text
+.globl	frodokem_sa_accum_x4_neon
+.type	frodokem_sa_accum_x4_neon,@function
+.align	2
+frodokem_sa_accum_x4_neon:
+#else
+.section	__TEXT,__text
+.globl	_frodokem_sa_accum_x4_neon
+.p2align	2
+_frodokem_sa_accum_x4_neon:
+#endif /* __APPLE__ */
+	stp	x29, x30, [sp, #-112]!
+	add	x29, sp, #0
+	stp	x17, x19, [x29, #24]
+	str	x20, [x29, #40]
+	stp	d8, d9, [x29, #48]
+	stp	d10, d11, [x29, #64]
+	stp	d12, d13, [x29, #80]
+	stp	d14, d15, [x29, #96]
+	sxtw	x5, w4
+	lsl	x5, x5, #1
+	lsl	x6, x5, #2
+	sxtw	x7, w3
+	lsl	x7, x7, #1
+	add	x7, x1, x7
+	mov	x8, x0
+	mov	x14, #2
+L_frodokem_sa_accum_x4_neon_t:
+	ldr	x15, [x7]
+	dup	v0.8h, w15
+	lsr	x15, x15, #16
+	dup	v1.8h, w15
+	lsr	x15, x15, #16
+	dup	v2.8h, w15
+	lsr	x15, x15, #16
+	dup	v3.8h, w15
+	add	x7, x7, x5
+	ldr	x15, [x7]
+	dup	v4.8h, w15
+	lsr	x15, x15, #16
+	dup	v5.8h, w15
+	lsr	x15, x15, #16
+	dup	v6.8h, w15
+	lsr	x15, x15, #16
+	dup	v7.8h, w15
+	add	x7, x7, x5
+	ldr	x15, [x7]
+	dup	v8.8h, w15
+	lsr	x15, x15, #16
+	dup	v9.8h, w15
+	lsr	x15, x15, #16
+	dup	v10.8h, w15
+	lsr	x15, x15, #16
+	dup	v11.8h, w15
+	add	x7, x7, x5
+	ldr	x15, [x7]
+	dup	v12.8h, w15
+	lsr	x15, x15, #16
+	dup	v13.8h, w15
+	lsr	x15, x15, #16
+	dup	v14.8h, w15
+	lsr	x15, x15, #16
+	dup	v15.8h, w15
+	add	x7, x7, x5
+	mov	x16, x8
+	add	x17, x16, x5
+	add	x19, x17, x5
+	add	x20, x19, x5
+	mov	x9, x2
+	add	x10, x9, x5
+	add	x11, x10, x5
+	add	x12, x11, x5
+	lsr	x13, x5, #4
+L_frodokem_sa_accum_x4_neon_k:
+	ld1	{v16.8h}, [x9], #16
+	ld1	{v17.8h}, [x10], #16
+	ld1	{v18.8h}, [x11], #16
+	ld1	{v19.8h}, [x12], #16
+	ld1	{v20.8h}, [x16]
+	mla	v20.8h, v16.8h, v0.8h
+	mla	v20.8h, v17.8h, v1.8h
+	mla	v20.8h, v18.8h, v2.8h
+	mla	v20.8h, v19.8h, v3.8h
+	st1	{v20.8h}, [x16], #16
+	ld1	{v21.8h}, [x17]
+	mla	v21.8h, v16.8h, v4.8h
+	mla	v21.8h, v17.8h, v5.8h
+	mla	v21.8h, v18.8h, v6.8h
+	mla	v21.8h, v19.8h, v7.8h
+	st1	{v21.8h}, [x17], #16
+	ld1	{v22.8h}, [x19]
+	mla	v22.8h, v16.8h, v8.8h
+	mla	v22.8h, v17.8h, v9.8h
+	mla	v22.8h, v18.8h, v10.8h
+	mla	v22.8h, v19.8h, v11.8h
+	st1	{v22.8h}, [x19], #16
+	ld1	{v23.8h}, [x20]
+	mla	v23.8h, v16.8h, v12.8h
+	mla	v23.8h, v17.8h, v13.8h
+	mla	v23.8h, v18.8h, v14.8h
+	mla	v23.8h, v19.8h, v15.8h
+	st1	{v23.8h}, [x20], #16
+	subs	x13, x13, #1
+	bne	L_frodokem_sa_accum_x4_neon_k
+	add	x8, x8, x6
+	subs	x14, x14, #1
+	bne	L_frodokem_sa_accum_x4_neon_t
+	ldp	x17, x19, [x29, #24]
+	ldr	x20, [x29, #40]
+	ldp	d8, d9, [x29, #48]
+	ldp	d10, d11, [x29, #64]
+	ldp	d12, d13, [x29, #80]
+	ldp	d14, d15, [x29, #96]
+	ldp	x29, x30, [sp], #0x70
+	ret
+#ifndef __APPLE__
+	.size	frodokem_sa_accum_x4_neon,.-frodokem_sa_accum_x4_neon
+#endif /* __APPLE__ */
+#ifndef __APPLE__
+.text
+.globl	frodokem_as_accum_neon
+.type	frodokem_as_accum_neon,@function
+.align	2
+frodokem_as_accum_neon:
+#else
+.section	__TEXT,__text
+.globl	_frodokem_as_accum_neon
+.p2align	2
+_frodokem_as_accum_neon:
+#endif /* __APPLE__ */
+	stp	x29, x30, [sp, #-96]!
+	add	x29, sp, #0
+	stp	x17, x19, [x29, #24]
+	str	x20, [x29, #40]
+	stp	d8, d9, [x29, #48]
+	stp	d10, d11, [x29, #64]
+	stp	d12, d13, [x29, #80]
+	sxtw	x5, w4
+	lsl	x5, x5, #1
+	lsl	x6, x5, #2
+	sxtw	x15, w3
+	lsl	x15, x15, #4
+	add	x7, x0, x15
+	add	x8, x7, #16
+	mov	x9, x2
+	add	x10, x2, x5
+	mov	x13, x1
+	mov	x16, x13
+	add	x17, x16, x5
+	add	x19, x17, x5
+	add	x20, x19, x5
+	mov	x11, x9
+	mov	x12, x10
+	movi	v0.8h, #0
+	movi	v1.8h, #0
+	movi	v2.8h, #0
+	movi	v3.8h, #0
+	movi	v4.8h, #0
+	movi	v5.8h, #0
+	movi	v6.8h, #0
+	movi	v7.8h, #0
+	lsr	x14, x5, #4
+L_frodokem_as_accum_neon_k0:
+	ld1	{v8.8h}, [x11], #16
+	ld1	{v9.8h}, [x12], #16
+	ld1	{v10.8h}, [x16], #16
+	mla	v0.8h, v8.8h, v10.8h
+	mla	v4.8h, v9.8h, v10.8h
+	ld1	{v10.8h}, [x17], #16
+	mla	v1.8h, v8.8h, v10.8h
+	mla	v5.8h, v9.8h, v10.8h
+	ld1	{v10.8h}, [x19], #16
+	mla	v2.8h, v8.8h, v10.8h
+	mla	v6.8h, v9.8h, v10.8h
+	ld1	{v10.8h}, [x20], #16
+	mla	v3.8h, v8.8h, v10.8h
+	mla	v7.8h, v9.8h, v10.8h
+	subs	x14, x14, #1
+	bne	L_frodokem_as_accum_neon_k0
+	addv	h13, v0.8h
+	ins	v11.h[0], v13.h[0]
+	addv	h13, v4.8h
+	ins	v12.h[0], v13.h[0]
+	addv	h13, v1.8h
+	ins	v11.h[1], v13.h[0]
+	addv	h13, v5.8h
+	ins	v12.h[1], v13.h[0]
+	addv	h13, v2.8h
+	ins	v11.h[2], v13.h[0]
+	addv	h13, v6.8h
+	ins	v12.h[2], v13.h[0]
+	addv	h13, v3.8h
+	ins	v11.h[3], v13.h[0]
+	addv	h13, v7.8h
+	ins	v12.h[3], v13.h[0]
+	add	x13, x13, x6
+	mov	x16, x13
+	add	x17, x16, x5
+	add	x19, x17, x5
+	add	x20, x19, x5
+	mov	x11, x9
+	mov	x12, x10
+	movi	v0.8h, #0
+	movi	v1.8h, #0
+	movi	v2.8h, #0
+	movi	v3.8h, #0
+	movi	v4.8h, #0
+	movi	v5.8h, #0
+	movi	v6.8h, #0
+	movi	v7.8h, #0
+	lsr	x14, x5, #4
+L_frodokem_as_accum_neon_k1:
+	ld1	{v8.8h}, [x11], #16
+	ld1	{v9.8h}, [x12], #16
+	ld1	{v10.8h}, [x16], #16
+	mla	v0.8h, v8.8h, v10.8h
+	mla	v4.8h, v9.8h, v10.8h
+	ld1	{v10.8h}, [x17], #16
+	mla	v1.8h, v8.8h, v10.8h
+	mla	v5.8h, v9.8h, v10.8h
+	ld1	{v10.8h}, [x19], #16
+	mla	v2.8h, v8.8h, v10.8h
+	mla	v6.8h, v9.8h, v10.8h
+	ld1	{v10.8h}, [x20], #16
+	mla	v3.8h, v8.8h, v10.8h
+	mla	v7.8h, v9.8h, v10.8h
+	subs	x14, x14, #1
+	bne	L_frodokem_as_accum_neon_k1
+	addv	h13, v0.8h
+	ins	v11.h[4], v13.h[0]
+	addv	h13, v4.8h
+	ins	v12.h[4], v13.h[0]
+	addv	h13, v1.8h
+	ins	v11.h[5], v13.h[0]
+	addv	h13, v5.8h
+	ins	v12.h[5], v13.h[0]
+	addv	h13, v2.8h
+	ins	v11.h[6], v13.h[0]
+	addv	h13, v6.8h
+	ins	v12.h[6], v13.h[0]
+	addv	h13, v3.8h
+	ins	v11.h[7], v13.h[0]
+	addv	h13, v7.8h
+	ins	v12.h[7], v13.h[0]
+	add	x13, x13, x6
+	ld1	{v8.8h}, [x7]
+	add	v11.8h, v11.8h, v8.8h
+	st1	{v11.8h}, [x7], #16
+	ld1	{v9.8h}, [x8]
+	add	v12.8h, v12.8h, v9.8h
+	st1	{v12.8h}, [x8], #16
+	ldp	x17, x19, [x29, #24]
+	ldr	x20, [x29, #40]
+	ldp	d8, d9, [x29, #48]
+	ldp	d10, d11, [x29, #64]
+	ldp	d12, d13, [x29, #80]
+	ldp	x29, x30, [sp], #0x60
+	ret
+#ifndef __APPLE__
+	.size	frodokem_as_accum_neon,.-frodokem_as_accum_neon
+#endif /* __APPLE__ */
+#ifndef __APPLE__
+.text
+.globl	frodokem_as_accum_x4_neon
+.type	frodokem_as_accum_x4_neon,@function
+.align	2
+frodokem_as_accum_x4_neon:
+#else
+.section	__TEXT,__text
+.globl	_frodokem_as_accum_x4_neon
+.p2align	2
+_frodokem_as_accum_x4_neon:
+#endif /* __APPLE__ */
+	stp	x29, x30, [sp, #-160]!
+	add	x29, sp, #0
+	stp	x17, x19, [x29, #24]
+	stp	x20, x21, [x29, #40]
+	stp	x22, x23, [x29, #56]
+	stp	x24, x25, [x29, #72]
+	str	x26, [x29, #88]
+	stp	d8, d9, [x29, #96]
+	stp	d10, d11, [x29, #112]
+	stp	d12, d13, [x29, #128]
+	stp	d14, d15, [x29, #144]
+	sxtw	x5, w4
+	lsl	x5, x5, #1
+	lsl	x6, x5, #2
+	sxtw	x22, w3
+	lsl	x22, x22, #4
+	add	x7, x0, x22
+	add	x8, x7, #16
+	add	x9, x8, #16
+	add	x10, x9, #16
+	mov	x11, x2
+	add	x12, x11, x5
+	add	x13, x12, x5
+	add	x14, x13, x5
+	mov	x20, x1
+	mov	x23, x20
+	add	x24, x23, x5
+	add	x25, x24, x5
+	add	x26, x25, x5
+	mov	x15, x11
+	mov	x16, x12
+	mov	x17, x13
+	mov	x19, x14
+	movi	v0.8h, #0
+	movi	v1.8h, #0
+	movi	v2.8h, #0
+	movi	v3.8h, #0
+	movi	v4.8h, #0
+	movi	v5.8h, #0
+	movi	v6.8h, #0
+	movi	v7.8h, #0
+	movi	v8.8h, #0
+	movi	v9.8h, #0
+	movi	v10.8h, #0
+	movi	v11.8h, #0
+	movi	v12.8h, #0
+	movi	v13.8h, #0
+	movi	v14.8h, #0
+	movi	v15.8h, #0
+	lsr	x21, x5, #4
+L_frodokem_as_accum_x4_neon_k0:
+	ld1	{v16.8h}, [x15], #16
+	ld1	{v17.8h}, [x16], #16
+	ld1	{v18.8h}, [x17], #16
+	ld1	{v19.8h}, [x19], #16
+	ld1	{v20.8h}, [x23], #16
+	mla	v0.8h, v16.8h, v20.8h
+	mla	v4.8h, v17.8h, v20.8h
+	mla	v8.8h, v18.8h, v20.8h
+	mla	v12.8h, v19.8h, v20.8h
+	ld1	{v20.8h}, [x24], #16
+	mla	v1.8h, v16.8h, v20.8h
+	mla	v5.8h, v17.8h, v20.8h
+	mla	v9.8h, v18.8h, v20.8h
+	mla	v13.8h, v19.8h, v20.8h
+	ld1	{v20.8h}, [x25], #16
+	mla	v2.8h, v16.8h, v20.8h
+	mla	v6.8h, v17.8h, v20.8h
+	mla	v10.8h, v18.8h, v20.8h
+	mla	v14.8h, v19.8h, v20.8h
+	ld1	{v20.8h}, [x26], #16
+	mla	v3.8h, v16.8h, v20.8h
+	mla	v7.8h, v17.8h, v20.8h
+	mla	v11.8h, v18.8h, v20.8h
+	mla	v15.8h, v19.8h, v20.8h
+	subs	x21, x21, #1
+	bne	L_frodokem_as_accum_x4_neon_k0
+	addv	h25, v0.8h
+	ins	v21.h[0], v25.h[0]
+	addv	h25, v4.8h
+	ins	v22.h[0], v25.h[0]
+	addv	h25, v8.8h
+	ins	v23.h[0], v25.h[0]
+	addv	h25, v12.8h
+	ins	v24.h[0], v25.h[0]
+	addv	h25, v1.8h
+	ins	v21.h[1], v25.h[0]
+	addv	h25, v5.8h
+	ins	v22.h[1], v25.h[0]
+	addv	h25, v9.8h
+	ins	v23.h[1], v25.h[0]
+	addv	h25, v13.8h
+	ins	v24.h[1], v25.h[0]
+	addv	h25, v2.8h
+	ins	v21.h[2], v25.h[0]
+	addv	h25, v6.8h
+	ins	v22.h[2], v25.h[0]
+	addv	h25, v10.8h
+	ins	v23.h[2], v25.h[0]
+	addv	h25, v14.8h
+	ins	v24.h[2], v25.h[0]
+	addv	h25, v3.8h
+	ins	v21.h[3], v25.h[0]
+	addv	h25, v7.8h
+	ins	v22.h[3], v25.h[0]
+	addv	h25, v11.8h
+	ins	v23.h[3], v25.h[0]
+	addv	h25, v15.8h
+	ins	v24.h[3], v25.h[0]
+	add	x20, x20, x6
+	mov	x23, x20
+	add	x24, x23, x5
+	add	x25, x24, x5
+	add	x26, x25, x5
+	mov	x15, x11
+	mov	x16, x12
+	mov	x17, x13
+	mov	x19, x14
+	movi	v0.8h, #0
+	movi	v1.8h, #0
+	movi	v2.8h, #0
+	movi	v3.8h, #0
+	movi	v4.8h, #0
+	movi	v5.8h, #0
+	movi	v6.8h, #0
+	movi	v7.8h, #0
+	movi	v8.8h, #0
+	movi	v9.8h, #0
+	movi	v10.8h, #0
+	movi	v11.8h, #0
+	movi	v12.8h, #0
+	movi	v13.8h, #0
+	movi	v14.8h, #0
+	movi	v15.8h, #0
+	lsr	x21, x5, #4
+L_frodokem_as_accum_x4_neon_k1:
+	ld1	{v16.8h}, [x15], #16
+	ld1	{v17.8h}, [x16], #16
+	ld1	{v18.8h}, [x17], #16
+	ld1	{v19.8h}, [x19], #16
+	ld1	{v20.8h}, [x23], #16
+	mla	v0.8h, v16.8h, v20.8h
+	mla	v4.8h, v17.8h, v20.8h
+	mla	v8.8h, v18.8h, v20.8h
+	mla	v12.8h, v19.8h, v20.8h
+	ld1	{v20.8h}, [x24], #16
+	mla	v1.8h, v16.8h, v20.8h
+	mla	v5.8h, v17.8h, v20.8h
+	mla	v9.8h, v18.8h, v20.8h
+	mla	v13.8h, v19.8h, v20.8h
+	ld1	{v20.8h}, [x25], #16
+	mla	v2.8h, v16.8h, v20.8h
+	mla	v6.8h, v17.8h, v20.8h
+	mla	v10.8h, v18.8h, v20.8h
+	mla	v14.8h, v19.8h, v20.8h
+	ld1	{v20.8h}, [x26], #16
+	mla	v3.8h, v16.8h, v20.8h
+	mla	v7.8h, v17.8h, v20.8h
+	mla	v11.8h, v18.8h, v20.8h
+	mla	v15.8h, v19.8h, v20.8h
+	subs	x21, x21, #1
+	bne	L_frodokem_as_accum_x4_neon_k1
+	addv	h25, v0.8h
+	ins	v21.h[4], v25.h[0]
+	addv	h25, v4.8h
+	ins	v22.h[4], v25.h[0]
+	addv	h25, v8.8h
+	ins	v23.h[4], v25.h[0]
+	addv	h25, v12.8h
+	ins	v24.h[4], v25.h[0]
+	addv	h25, v1.8h
+	ins	v21.h[5], v25.h[0]
+	addv	h25, v5.8h
+	ins	v22.h[5], v25.h[0]
+	addv	h25, v9.8h
+	ins	v23.h[5], v25.h[0]
+	addv	h25, v13.8h
+	ins	v24.h[5], v25.h[0]
+	addv	h25, v2.8h
+	ins	v21.h[6], v25.h[0]
+	addv	h25, v6.8h
+	ins	v22.h[6], v25.h[0]
+	addv	h25, v10.8h
+	ins	v23.h[6], v25.h[0]
+	addv	h25, v14.8h
+	ins	v24.h[6], v25.h[0]
+	addv	h25, v3.8h
+	ins	v21.h[7], v25.h[0]
+	addv	h25, v7.8h
+	ins	v22.h[7], v25.h[0]
+	addv	h25, v11.8h
+	ins	v23.h[7], v25.h[0]
+	addv	h25, v15.8h
+	ins	v24.h[7], v25.h[0]
+	add	x20, x20, x6
+	ld1	{v16.8h}, [x7]
+	add	v21.8h, v21.8h, v16.8h
+	st1	{v21.8h}, [x7], #16
+	ld1	{v16.8h}, [x8]
+	add	v22.8h, v22.8h, v16.8h
+	st1	{v22.8h}, [x8], #16
+	ld1	{v16.8h}, [x9]
+	add	v23.8h, v23.8h, v16.8h
+	st1	{v23.8h}, [x9], #16
+	ld1	{v16.8h}, [x10]
+	add	v24.8h, v24.8h, v16.8h
+	st1	{v24.8h}, [x10], #16
+	ldp	x17, x19, [x29, #24]
+	ldp	x20, x21, [x29, #40]
+	ldp	x22, x23, [x29, #56]
+	ldp	x24, x25, [x29, #72]
+	ldr	x26, [x29, #88]
+	ldp	d8, d9, [x29, #96]
+	ldp	d10, d11, [x29, #112]
+	ldp	d12, d13, [x29, #128]
+	ldp	d14, d15, [x29, #144]
+	ldp	x29, x30, [sp], #0xa0
+	ret
+#ifndef __APPLE__
+	.size	frodokem_as_accum_x4_neon,.-frodokem_as_accum_x4_neon
+#endif /* __APPLE__ */
+#ifndef __APPLE__
+.text
+.globl	frodokem_mul_bs_neon
+.type	frodokem_mul_bs_neon,@function
+.align	2
+frodokem_mul_bs_neon:
+#else
+.section	__TEXT,__text
+.globl	_frodokem_mul_bs_neon
+.p2align	2
+_frodokem_mul_bs_neon:
+#endif /* __APPLE__ */
+	stp	x29, x30, [sp, #-160]!
+	add	x29, sp, #0
+	stp	x17, x19, [x29, #16]
+	stp	x20, x21, [x29, #32]
+	stp	x22, x23, [x29, #48]
+	stp	x24, x25, [x29, #64]
+	stp	x26, x27, [x29, #80]
+	stp	d8, d9, [x29, #96]
+	stp	d10, d11, [x29, #112]
+	stp	d12, d13, [x29, #128]
+	stp	d14, d15, [x29, #144]
+	sxtw	x5, w3
+	lsl	x5, x5, #1
+	lsl	x6, x5, #2
+	dup	v26.8h, w4
+	mov	x7, x0
+	add	x8, x7, #16
+	add	x9, x8, #16
+	add	x10, x9, #16
+	mov	x11, x1
+	mov	x23, #2
+L_frodokem_mul_bs_neon_p:
+	mov	x12, x11
+	add	x13, x12, x5
+	add	x14, x13, x5
+	add	x15, x14, x5
+	mov	x21, x2
+	mov	x24, x21
+	add	x25, x24, x5
+	add	x26, x25, x5
+	add	x27, x26, x5
+	mov	x16, x12
+	mov	x17, x13
+	mov	x19, x14
+	mov	x20, x15
+	movi	v0.8h, #0
+	movi	v1.8h, #0
+	movi	v2.8h, #0
+	movi	v3.8h, #0
+	movi	v4.8h, #0
+	movi	v5.8h, #0
+	movi	v6.8h, #0
+	movi	v7.8h, #0
+	movi	v8.8h, #0
+	movi	v9.8h, #0
+	movi	v10.8h, #0
+	movi	v11.8h, #0
+	movi	v12.8h, #0
+	movi	v13.8h, #0
+	movi	v14.8h, #0
+	movi	v15.8h, #0
+	lsr	x22, x5, #4
+L_frodokem_mul_bs_neon_k0:
+	ld1	{v16.8h}, [x16], #16
+	ld1	{v17.8h}, [x17], #16
+	ld1	{v18.8h}, [x19], #16
+	ld1	{v19.8h}, [x20], #16
+	ld1	{v20.8h}, [x24], #16
+	mla	v0.8h, v16.8h, v20.8h
+	mla	v4.8h, v17.8h, v20.8h
+	mla	v8.8h, v18.8h, v20.8h
+	mla	v12.8h, v19.8h, v20.8h
+	ld1	{v20.8h}, [x25], #16
+	mla	v1.8h, v16.8h, v20.8h
+	mla	v5.8h, v17.8h, v20.8h
+	mla	v9.8h, v18.8h, v20.8h
+	mla	v13.8h, v19.8h, v20.8h
+	ld1	{v20.8h}, [x26], #16
+	mla	v2.8h, v16.8h, v20.8h
+	mla	v6.8h, v17.8h, v20.8h
+	mla	v10.8h, v18.8h, v20.8h
+	mla	v14.8h, v19.8h, v20.8h
+	ld1	{v20.8h}, [x27], #16
+	mla	v3.8h, v16.8h, v20.8h
+	mla	v7.8h, v17.8h, v20.8h
+	mla	v11.8h, v18.8h, v20.8h
+	mla	v15.8h, v19.8h, v20.8h
+	subs	x22, x22, #1
+	bne	L_frodokem_mul_bs_neon_k0
+	addv	h25, v0.8h
+	ins	v21.h[0], v25.h[0]
+	addv	h25, v4.8h
+	ins	v22.h[0], v25.h[0]
+	addv	h25, v8.8h
+	ins	v23.h[0], v25.h[0]
+	addv	h25, v12.8h
+	ins	v24.h[0], v25.h[0]
+	addv	h25, v1.8h
+	ins	v21.h[1], v25.h[0]
+	addv	h25, v5.8h
+	ins	v22.h[1], v25.h[0]
+	addv	h25, v9.8h
+	ins	v23.h[1], v25.h[0]
+	addv	h25, v13.8h
+	ins	v24.h[1], v25.h[0]
+	addv	h25, v2.8h
+	ins	v21.h[2], v25.h[0]
+	addv	h25, v6.8h
+	ins	v22.h[2], v25.h[0]
+	addv	h25, v10.8h
+	ins	v23.h[2], v25.h[0]
+	addv	h25, v14.8h
+	ins	v24.h[2], v25.h[0]
+	addv	h25, v3.8h
+	ins	v21.h[3], v25.h[0]
+	addv	h25, v7.8h
+	ins	v22.h[3], v25.h[0]
+	addv	h25, v11.8h
+	ins	v23.h[3], v25.h[0]
+	addv	h25, v15.8h
+	ins	v24.h[3], v25.h[0]
+	add	x21, x21, x6
+	mov	x24, x21
+	add	x25, x24, x5
+	add	x26, x25, x5
+	add	x27, x26, x5
+	mov	x16, x12
+	mov	x17, x13
+	mov	x19, x14
+	mov	x20, x15
+	movi	v0.8h, #0
+	movi	v1.8h, #0
+	movi	v2.8h, #0
+	movi	v3.8h, #0
+	movi	v4.8h, #0
+	movi	v5.8h, #0
+	movi	v6.8h, #0
+	movi	v7.8h, #0
+	movi	v8.8h, #0
+	movi	v9.8h, #0
+	movi	v10.8h, #0
+	movi	v11.8h, #0
+	movi	v12.8h, #0
+	movi	v13.8h, #0
+	movi	v14.8h, #0
+	movi	v15.8h, #0
+	lsr	x22, x5, #4
+L_frodokem_mul_bs_neon_k1:
+	ld1	{v16.8h}, [x16], #16
+	ld1	{v17.8h}, [x17], #16
+	ld1	{v18.8h}, [x19], #16
+	ld1	{v19.8h}, [x20], #16
+	ld1	{v20.8h}, [x24], #16
+	mla	v0.8h, v16.8h, v20.8h
+	mla	v4.8h, v17.8h, v20.8h
+	mla	v8.8h, v18.8h, v20.8h
+	mla	v12.8h, v19.8h, v20.8h
+	ld1	{v20.8h}, [x25], #16
+	mla	v1.8h, v16.8h, v20.8h
+	mla	v5.8h, v17.8h, v20.8h
+	mla	v9.8h, v18.8h, v20.8h
+	mla	v13.8h, v19.8h, v20.8h
+	ld1	{v20.8h}, [x26], #16
+	mla	v2.8h, v16.8h, v20.8h
+	mla	v6.8h, v17.8h, v20.8h
+	mla	v10.8h, v18.8h, v20.8h
+	mla	v14.8h, v19.8h, v20.8h
+	ld1	{v20.8h}, [x27], #16
+	mla	v3.8h, v16.8h, v20.8h
+	mla	v7.8h, v17.8h, v20.8h
+	mla	v11.8h, v18.8h, v20.8h
+	mla	v15.8h, v19.8h, v20.8h
+	subs	x22, x22, #1
+	bne	L_frodokem_mul_bs_neon_k1
+	addv	h25, v0.8h
+	ins	v21.h[4], v25.h[0]
+	addv	h25, v4.8h
+	ins	v22.h[4], v25.h[0]
+	addv	h25, v8.8h
+	ins	v23.h[4], v25.h[0]
+	addv	h25, v12.8h
+	ins	v24.h[4], v25.h[0]
+	addv	h25, v1.8h
+	ins	v21.h[5], v25.h[0]
+	addv	h25, v5.8h
+	ins	v22.h[5], v25.h[0]
+	addv	h25, v9.8h
+	ins	v23.h[5], v25.h[0]
+	addv	h25, v13.8h
+	ins	v24.h[5], v25.h[0]
+	addv	h25, v2.8h
+	ins	v21.h[6], v25.h[0]
+	addv	h25, v6.8h
+	ins	v22.h[6], v25.h[0]
+	addv	h25, v10.8h
+	ins	v23.h[6], v25.h[0]
+	addv	h25, v14.8h
+	ins	v24.h[6], v25.h[0]
+	addv	h25, v3.8h
+	ins	v21.h[7], v25.h[0]
+	addv	h25, v7.8h
+	ins	v22.h[7], v25.h[0]
+	addv	h25, v11.8h
+	ins	v23.h[7], v25.h[0]
+	addv	h25, v15.8h
+	ins	v24.h[7], v25.h[0]
+	add	x21, x21, x6
+	and	v21.16b, v21.16b, v26.16b
+	st1	{v21.8h}, [x7]
+	and	v22.16b, v22.16b, v26.16b
+	st1	{v22.8h}, [x8]
+	and	v23.16b, v23.16b, v26.16b
+	st1	{v23.8h}, [x9]
+	and	v24.16b, v24.16b, v26.16b
+	st1	{v24.8h}, [x10]
+	add	x11, x11, x6
+	add	x7, x7, #0x40
+	add	x8, x8, #0x40
+	add	x9, x9, #0x40
+	add	x10, x10, #0x40
+	subs	x23, x23, #1
+	bne	L_frodokem_mul_bs_neon_p
+	ldp	x17, x19, [x29, #16]
+	ldp	x20, x21, [x29, #32]
+	ldp	x22, x23, [x29, #48]
+	ldp	x24, x25, [x29, #64]
+	ldp	x26, x27, [x29, #80]
+	ldp	d8, d9, [x29, #96]
+	ldp	d10, d11, [x29, #112]
+	ldp	d12, d13, [x29, #128]
+	ldp	d14, d15, [x29, #144]
+	ldp	x29, x30, [sp], #0xa0
+	ret
+#ifndef __APPLE__
+	.size	frodokem_mul_bs_neon,.-frodokem_mul_bs_neon
+#endif /* __APPLE__ */
+#ifndef __APPLE__
+.text
+.globl	frodokem_mul_add_sb_plus_e_neon
+.type	frodokem_mul_add_sb_plus_e_neon,@function
+.align	2
+frodokem_mul_add_sb_plus_e_neon:
+#else
+.section	__TEXT,__text
+.globl	_frodokem_mul_add_sb_plus_e_neon
+.p2align	2
+_frodokem_mul_add_sb_plus_e_neon:
+#endif /* __APPLE__ */
+	stp	x29, x30, [sp, #-48]!
+	add	x29, sp, #0
+	stp	d8, d9, [x29, #16]
+	stp	d10, d11, [x29, #32]
+	sxtw	x5, w3
+	lsl	x5, x5, #1
+	dup	v10.8h, w4
+	mov	x6, x2
+	add	x7, x6, x5
+	add	x8, x7, x5
+	add	x9, x8, x5
+	add	x10, x9, x5
+	add	x11, x10, x5
+	add	x12, x11, x5
+	add	x13, x12, x5
+	mov	x14, x1
+	mov	x16, x0
+	ld1	{v0.8h, v1.8h, v2.8h, v3.8h}, [x16], #0x40
+	ld1	{v4.8h, v5.8h, v6.8h, v7.8h}, [x16], #0x40
+	sub	x16, x16, #0x80
+	sxtw	x15, w3
+L_frodokem_mul_add_sb_plus_e_neon_j:
+	ld1	{v8.8h}, [x14], #16
+	ld1r	{v9.8h}, [x6], #2
+	mla	v0.8h, v8.8h, v9.8h
+	ld1r	{v9.8h}, [x7], #2
+	mla	v1.8h, v8.8h, v9.8h
+	ld1r	{v9.8h}, [x8], #2
+	mla	v2.8h, v8.8h, v9.8h
+	ld1r	{v9.8h}, [x9], #2
+	mla	v3.8h, v8.8h, v9.8h
+	ld1r	{v9.8h}, [x10], #2
+	mla	v4.8h, v8.8h, v9.8h
+	ld1r	{v9.8h}, [x11], #2
+	mla	v5.8h, v8.8h, v9.8h
+	ld1r	{v9.8h}, [x12], #2
+	mla	v6.8h, v8.8h, v9.8h
+	ld1r	{v9.8h}, [x13], #2
+	mla	v7.8h, v8.8h, v9.8h
+	subs	x15, x15, #1
+	bne	L_frodokem_mul_add_sb_plus_e_neon_j
+	and	v0.16b, v0.16b, v10.16b
+	and	v1.16b, v1.16b, v10.16b
+	and	v2.16b, v2.16b, v10.16b
+	and	v3.16b, v3.16b, v10.16b
+	and	v4.16b, v4.16b, v10.16b
+	and	v5.16b, v5.16b, v10.16b
+	and	v6.16b, v6.16b, v10.16b
+	and	v7.16b, v7.16b, v10.16b
+	st1	{v0.8h, v1.8h, v2.8h, v3.8h}, [x16], #0x40
+	st1	{v4.8h, v5.8h, v6.8h, v7.8h}, [x16], #0x40
+	ldp	d8, d9, [x29, #16]
+	ldp	d10, d11, [x29, #32]
+	ldp	x29, x30, [sp], #48
+	ret
+#ifndef __APPLE__
+	.size	frodokem_mul_add_sb_plus_e_neon,.-frodokem_mul_add_sb_plus_e_neon
+#endif /* __APPLE__ */
+#endif /* WOLFSSL_HAVE_FRODOKEM */
+#ifdef WOLFSSL_HAVE_FRODOKEM
+	.arch	armv8-a+crypto
+#ifndef __APPLE__
+.text
+.globl	frodokem_gen_a_rows_aes_arm64
+.type	frodokem_gen_a_rows_aes_arm64,@function
+.align	2
+frodokem_gen_a_rows_aes_arm64:
+#else
+.section	__TEXT,__text
+.globl	_frodokem_gen_a_rows_aes_arm64
+.p2align	2
+_frodokem_gen_a_rows_aes_arm64:
+#endif /* __APPLE__ */
+	stp	x29, x30, [sp, #-80]!
+	add	x29, sp, #0
+	stp	d8, d9, [x29, #16]
+	stp	d10, d11, [x29, #32]
+	stp	d12, d13, [x29, #48]
+	stp	d14, d15, [x29, #64]
+	ld1	{v0.16b, v1.16b, v2.16b, v3.16b}, [x2], #0x40
+	ld1	{v4.16b, v5.16b, v6.16b, v7.16b}, [x2], #0x40
+	ld1	{v8.16b, v9.16b, v10.16b}, [x2], #48
+	dup	v19.8h, w6
+	sxtw	x13, w3
+	sxtw	x7, w5
+	lsl	x7, x7, #1
+	sxtw	x8, w4
+	mul	x8, x8, x7
+	movz	w14, #8, lsl 16
+	mov	x9, x0
+L_frodokem_gen_a_rows_aes_arm64_row:
+	mov	w11, w13
+	add	x10, x9, x7
+L_frodokem_gen_a_rows_aes_arm64_blk:
+	add	x12, x11, x14
+	stp	x11, xzr, [x9], #16
+	stp	x12, xzr, [x9], #16
+	add	x11, x12, x14
+	cmp	x9, x10
+	blt	L_frodokem_gen_a_rows_aes_arm64_blk
+	add	x13, x13, #1
+	add	x10, x0, x8
+	cmp	x9, x10
+	blt	L_frodokem_gen_a_rows_aes_arm64_row
+	cmp	x8, #0x80
+	blt	L_frodokem_gen_a_rows_aes_arm64_tail
+L_frodokem_gen_a_rows_aes_arm64_aes:
+	ld1	{v11.16b, v12.16b, v13.16b, v14.16b}, [x0], #0x40
+	ld1	{v15.16b, v16.16b, v17.16b, v18.16b}, [x0], #0x40
+	aese	v11.16b, v0.16b
+	aesmc	v11.16b, v11.16b
+	aese	v12.16b, v0.16b
+	aesmc	v12.16b, v12.16b
+	aese	v13.16b, v0.16b
+	aesmc	v13.16b, v13.16b
+	aese	v14.16b, v0.16b
+	aesmc	v14.16b, v14.16b
+	aese	v15.16b, v0.16b
+	aesmc	v15.16b, v15.16b
+	aese	v16.16b, v0.16b
+	aesmc	v16.16b, v16.16b
+	aese	v17.16b, v0.16b
+	aesmc	v17.16b, v17.16b
+	aese	v18.16b, v0.16b
+	aesmc	v18.16b, v18.16b
+	aese	v11.16b, v1.16b
+	aesmc	v11.16b, v11.16b
+	aese	v12.16b, v1.16b
+	aesmc	v12.16b, v12.16b
+	aese	v13.16b, v1.16b
+	aesmc	v13.16b, v13.16b
+	aese	v14.16b, v1.16b
+	aesmc	v14.16b, v14.16b
+	aese	v15.16b, v1.16b
+	aesmc	v15.16b, v15.16b
+	aese	v16.16b, v1.16b
+	aesmc	v16.16b, v16.16b
+	aese	v17.16b, v1.16b
+	aesmc	v17.16b, v17.16b
+	aese	v18.16b, v1.16b
+	aesmc	v18.16b, v18.16b
+	aese	v11.16b, v2.16b
+	aesmc	v11.16b, v11.16b
+	aese	v12.16b, v2.16b
+	aesmc	v12.16b, v12.16b
+	aese	v13.16b, v2.16b
+	aesmc	v13.16b, v13.16b
+	aese	v14.16b, v2.16b
+	aesmc	v14.16b, v14.16b
+	aese	v15.16b, v2.16b
+	aesmc	v15.16b, v15.16b
+	aese	v16.16b, v2.16b
+	aesmc	v16.16b, v16.16b
+	aese	v17.16b, v2.16b
+	aesmc	v17.16b, v17.16b
+	aese	v18.16b, v2.16b
+	aesmc	v18.16b, v18.16b
+	aese	v11.16b, v3.16b
+	aesmc	v11.16b, v11.16b
+	aese	v12.16b, v3.16b
+	aesmc	v12.16b, v12.16b
+	aese	v13.16b, v3.16b
+	aesmc	v13.16b, v13.16b
+	aese	v14.16b, v3.16b
+	aesmc	v14.16b, v14.16b
+	aese	v15.16b, v3.16b
+	aesmc	v15.16b, v15.16b
+	aese	v16.16b, v3.16b
+	aesmc	v16.16b, v16.16b
+	aese	v17.16b, v3.16b
+	aesmc	v17.16b, v17.16b
+	aese	v18.16b, v3.16b
+	aesmc	v18.16b, v18.16b
+	aese	v11.16b, v4.16b
+	aesmc	v11.16b, v11.16b
+	aese	v12.16b, v4.16b
+	aesmc	v12.16b, v12.16b
+	aese	v13.16b, v4.16b
+	aesmc	v13.16b, v13.16b
+	aese	v14.16b, v4.16b
+	aesmc	v14.16b, v14.16b
+	aese	v15.16b, v4.16b
+	aesmc	v15.16b, v15.16b
+	aese	v16.16b, v4.16b
+	aesmc	v16.16b, v16.16b
+	aese	v17.16b, v4.16b
+	aesmc	v17.16b, v17.16b
+	aese	v18.16b, v4.16b
+	aesmc	v18.16b, v18.16b
+	aese	v11.16b, v5.16b
+	aesmc	v11.16b, v11.16b
+	aese	v12.16b, v5.16b
+	aesmc	v12.16b, v12.16b
+	aese	v13.16b, v5.16b
+	aesmc	v13.16b, v13.16b
+	aese	v14.16b, v5.16b
+	aesmc	v14.16b, v14.16b
+	aese	v15.16b, v5.16b
+	aesmc	v15.16b, v15.16b
+	aese	v16.16b, v5.16b
+	aesmc	v16.16b, v16.16b
+	aese	v17.16b, v5.16b
+	aesmc	v17.16b, v17.16b
+	aese	v18.16b, v5.16b
+	aesmc	v18.16b, v18.16b
+	aese	v11.16b, v6.16b
+	aesmc	v11.16b, v11.16b
+	aese	v12.16b, v6.16b
+	aesmc	v12.16b, v12.16b
+	aese	v13.16b, v6.16b
+	aesmc	v13.16b, v13.16b
+	aese	v14.16b, v6.16b
+	aesmc	v14.16b, v14.16b
+	aese	v15.16b, v6.16b
+	aesmc	v15.16b, v15.16b
+	aese	v16.16b, v6.16b
+	aesmc	v16.16b, v16.16b
+	aese	v17.16b, v6.16b
+	aesmc	v17.16b, v17.16b
+	aese	v18.16b, v6.16b
+	aesmc	v18.16b, v18.16b
+	aese	v11.16b, v7.16b
+	aesmc	v11.16b, v11.16b
+	aese	v12.16b, v7.16b
+	aesmc	v12.16b, v12.16b
+	aese	v13.16b, v7.16b
+	aesmc	v13.16b, v13.16b
+	aese	v14.16b, v7.16b
+	aesmc	v14.16b, v14.16b
+	aese	v15.16b, v7.16b
+	aesmc	v15.16b, v15.16b
+	aese	v16.16b, v7.16b
+	aesmc	v16.16b, v16.16b
+	aese	v17.16b, v7.16b
+	aesmc	v17.16b, v17.16b
+	aese	v18.16b, v7.16b
+	aesmc	v18.16b, v18.16b
+	aese	v11.16b, v8.16b
+	aesmc	v11.16b, v11.16b
+	aese	v12.16b, v8.16b
+	aesmc	v12.16b, v12.16b
+	aese	v13.16b, v8.16b
+	aesmc	v13.16b, v13.16b
+	aese	v14.16b, v8.16b
+	aesmc	v14.16b, v14.16b
+	aese	v15.16b, v8.16b
+	aesmc	v15.16b, v15.16b
+	aese	v16.16b, v8.16b
+	aesmc	v16.16b, v16.16b
+	aese	v17.16b, v8.16b
+	aesmc	v17.16b, v17.16b
+	aese	v18.16b, v8.16b
+	aesmc	v18.16b, v18.16b
+	aese	v11.16b, v9.16b
+	aese	v12.16b, v9.16b
+	aese	v13.16b, v9.16b
+	aese	v14.16b, v9.16b
+	aese	v15.16b, v9.16b
+	aese	v16.16b, v9.16b
+	aese	v17.16b, v9.16b
+	aese	v18.16b, v9.16b
+	eor	v11.16b, v11.16b, v10.16b
+	eor	v12.16b, v12.16b, v10.16b
+	eor	v13.16b, v13.16b, v10.16b
+	eor	v14.16b, v14.16b, v10.16b
+	eor	v15.16b, v15.16b, v10.16b
+	eor	v16.16b, v16.16b, v10.16b
+	eor	v17.16b, v17.16b, v10.16b
+	eor	v18.16b, v18.16b, v10.16b
+	and	v11.16b, v11.16b, v19.16b
+	and	v12.16b, v12.16b, v19.16b
+	and	v13.16b, v13.16b, v19.16b
+	and	v14.16b, v14.16b, v19.16b
+	and	v15.16b, v15.16b, v19.16b
+	and	v16.16b, v16.16b, v19.16b
+	and	v17.16b, v17.16b, v19.16b
+	and	v18.16b, v18.16b, v19.16b
+	st1	{v11.16b, v12.16b, v13.16b, v14.16b}, [x1], #0x40
+	st1	{v15.16b, v16.16b, v17.16b, v18.16b}, [x1], #0x40
+	subs	x8, x8, #0x80
+	cmp	x8, #0x80
+	bge	L_frodokem_gen_a_rows_aes_arm64_aes
+L_frodokem_gen_a_rows_aes_arm64_tail:
+	cmp	x8, #0
+	beq	L_frodokem_gen_a_rows_aes_arm64_done
+	ld1	{v11.16b, v12.16b, v13.16b, v14.16b}, [x0], #0x40
+	aese	v11.16b, v0.16b
+	aesmc	v11.16b, v11.16b
+	aese	v12.16b, v0.16b
+	aesmc	v12.16b, v12.16b
+	aese	v13.16b, v0.16b
+	aesmc	v13.16b, v13.16b
+	aese	v14.16b, v0.16b
+	aesmc	v14.16b, v14.16b
+	aese	v11.16b, v1.16b
+	aesmc	v11.16b, v11.16b
+	aese	v12.16b, v1.16b
+	aesmc	v12.16b, v12.16b
+	aese	v13.16b, v1.16b
+	aesmc	v13.16b, v13.16b
+	aese	v14.16b, v1.16b
+	aesmc	v14.16b, v14.16b
+	aese	v11.16b, v2.16b
+	aesmc	v11.16b, v11.16b
+	aese	v12.16b, v2.16b
+	aesmc	v12.16b, v12.16b
+	aese	v13.16b, v2.16b
+	aesmc	v13.16b, v13.16b
+	aese	v14.16b, v2.16b
+	aesmc	v14.16b, v14.16b
+	aese	v11.16b, v3.16b
+	aesmc	v11.16b, v11.16b
+	aese	v12.16b, v3.16b
+	aesmc	v12.16b, v12.16b
+	aese	v13.16b, v3.16b
+	aesmc	v13.16b, v13.16b
+	aese	v14.16b, v3.16b
+	aesmc	v14.16b, v14.16b
+	aese	v11.16b, v4.16b
+	aesmc	v11.16b, v11.16b
+	aese	v12.16b, v4.16b
+	aesmc	v12.16b, v12.16b
+	aese	v13.16b, v4.16b
+	aesmc	v13.16b, v13.16b
+	aese	v14.16b, v4.16b
+	aesmc	v14.16b, v14.16b
+	aese	v11.16b, v5.16b
+	aesmc	v11.16b, v11.16b
+	aese	v12.16b, v5.16b
+	aesmc	v12.16b, v12.16b
+	aese	v13.16b, v5.16b
+	aesmc	v13.16b, v13.16b
+	aese	v14.16b, v5.16b
+	aesmc	v14.16b, v14.16b
+	aese	v11.16b, v6.16b
+	aesmc	v11.16b, v11.16b
+	aese	v12.16b, v6.16b
+	aesmc	v12.16b, v12.16b
+	aese	v13.16b, v6.16b
+	aesmc	v13.16b, v13.16b
+	aese	v14.16b, v6.16b
+	aesmc	v14.16b, v14.16b
+	aese	v11.16b, v7.16b
+	aesmc	v11.16b, v11.16b
+	aese	v12.16b, v7.16b
+	aesmc	v12.16b, v12.16b
+	aese	v13.16b, v7.16b
+	aesmc	v13.16b, v13.16b
+	aese	v14.16b, v7.16b
+	aesmc	v14.16b, v14.16b
+	aese	v11.16b, v8.16b
+	aesmc	v11.16b, v11.16b
+	aese	v12.16b, v8.16b
+	aesmc	v12.16b, v12.16b
+	aese	v13.16b, v8.16b
+	aesmc	v13.16b, v13.16b
+	aese	v14.16b, v8.16b
+	aesmc	v14.16b, v14.16b
+	aese	v11.16b, v9.16b
+	aese	v12.16b, v9.16b
+	aese	v13.16b, v9.16b
+	aese	v14.16b, v9.16b
+	eor	v11.16b, v11.16b, v10.16b
+	eor	v12.16b, v12.16b, v10.16b
+	eor	v13.16b, v13.16b, v10.16b
+	eor	v14.16b, v14.16b, v10.16b
+	and	v11.16b, v11.16b, v19.16b
+	and	v12.16b, v12.16b, v19.16b
+	and	v13.16b, v13.16b, v19.16b
+	and	v14.16b, v14.16b, v19.16b
+	st1	{v11.16b, v12.16b, v13.16b, v14.16b}, [x1], #0x40
+L_frodokem_gen_a_rows_aes_arm64_done:
+	ldp	d8, d9, [x29, #16]
+	ldp	d10, d11, [x29, #32]
+	ldp	d12, d13, [x29, #48]
+	ldp	d14, d15, [x29, #64]
+	ldp	x29, x30, [sp], #0x50
+	ret
+#ifndef __APPLE__
+	.size	frodokem_gen_a_rows_aes_arm64,.-frodokem_gen_a_rows_aes_arm64
+#endif /* __APPLE__ */
+#endif /* WOLFSSL_HAVE_FRODOKEM */
+#ifdef WOLFSSL_HAVE_FRODOKEM
+#ifdef WOLFSSL_FRODOKEM_SVE
+	.arch	armv9-a+sve2
+#ifndef __APPLE__
+.text
+.globl	frodokem_sa_accum_sve
+.type	frodokem_sa_accum_sve,@function
+.align	2
+frodokem_sa_accum_sve:
+#else
+.section	__TEXT,__text
+.globl	_frodokem_sa_accum_sve
+.p2align	2
+_frodokem_sa_accum_sve:
+#endif /* __APPLE__ */
+	sxtw	x5, w4
+	lsl	x6, x5, #1
+	add	x7, x2, x6
+	sxtw	x8, w3
+	lsl	x8, x8, #1
+	add	x8, x1, x8
+	mov	x9, #8
+L_frodokem_sa_accum_sve_i:
+	ldr	w11, [x8]
+	mov	z0.h, w11
+	lsr	w11, w11, #16
+	mov	z1.h, w11
+	add	x8, x8, x6
+	mov	x10, #0
+L_frodokem_sa_accum_sve_k:
+	whilelt	p0.h, x10, x5
+	ld1h	{z2.h}, p0/z, [x0, x10, lsl #1]
+	ld1h	{z3.h}, p0/z, [x2, x10, lsl #1]
+	ld1h	{z4.h}, p0/z, [x7, x10, lsl #1]
+	mla	z2.h, p0/m, z3.h, z0.h
+	mla	z2.h, p0/m, z4.h, z1.h
+	st1h	{z2.h}, p0, [x0, x10, lsl #1]
+	inch	x10
+	cmp	x10, x5
+	blt	L_frodokem_sa_accum_sve_k
+	add	x0, x0, x6
+	subs	x9, x9, #1
+	bne	L_frodokem_sa_accum_sve_i
+	ret
+#ifndef __APPLE__
+	.size	frodokem_sa_accum_sve,.-frodokem_sa_accum_sve
+#endif /* __APPLE__ */
+#ifndef __APPLE__
+.text
+.globl	frodokem_as_accum_sve
+.type	frodokem_as_accum_sve,@function
+.align	2
+frodokem_as_accum_sve:
+#else
+.section	__TEXT,__text
+.globl	_frodokem_as_accum_sve
+.p2align	2
+_frodokem_as_accum_sve:
+#endif /* __APPLE__ */
+	stp	x29, x30, [sp, #-112]!
+	add	x29, sp, #0
+	stp	x17, x19, [x29, #24]
+	str	x20, [x29, #40]
+	stp	d8, d9, [x29, #48]
+	stp	d10, d11, [x29, #64]
+	stp	d12, d13, [x29, #80]
+	stp	d14, d15, [x29, #96]
+	sxtw	x5, w4
+	lsl	x6, x5, #1
+	add	x7, x2, x6
+	mov	x12, x1
+	add	x13, x12, x6
+	add	x14, x13, x6
+	add	x15, x14, x6
+	add	x16, x15, x6
+	add	x17, x16, x6
+	add	x19, x17, x6
+	add	x20, x19, x6
+	mov	z0.h, #0
+	mov	z1.h, #0
+	mov	z2.h, #0
+	mov	z3.h, #0
+	mov	z4.h, #0
+	mov	z5.h, #0
+	mov	z6.h, #0
+	mov	z7.h, #0
+	mov	z8.h, #0
+	mov	z9.h, #0
+	mov	z10.h, #0
+	mov	z11.h, #0
+	mov	z12.h, #0
+	mov	z13.h, #0
+	mov	z14.h, #0
+	mov	z15.h, #0
+	mov	x9, #0
+L_frodokem_as_accum_sve_k:
+	whilelt	p0.h, x9, x5
+	ld1h	{z16.h}, p0/z, [x2, x9, lsl #1]
+	ld1h	{z17.h}, p0/z, [x7, x9, lsl #1]
+	ld1h	{z18.h}, p0/z, [x12, x9, lsl #1]
+	mla	z0.h, p0/m, z16.h, z18.h
+	mla	z8.h, p0/m, z17.h, z18.h
+	ld1h	{z18.h}, p0/z, [x13, x9, lsl #1]
+	mla	z1.h, p0/m, z16.h, z18.h
+	mla	z9.h, p0/m, z17.h, z18.h
+	ld1h	{z18.h}, p0/z, [x14, x9, lsl #1]
+	mla	z2.h, p0/m, z16.h, z18.h
+	mla	z10.h, p0/m, z17.h, z18.h
+	ld1h	{z18.h}, p0/z, [x15, x9, lsl #1]
+	mla	z3.h, p0/m, z16.h, z18.h
+	mla	z11.h, p0/m, z17.h, z18.h
+	ld1h	{z18.h}, p0/z, [x16, x9, lsl #1]
+	mla	z4.h, p0/m, z16.h, z18.h
+	mla	z12.h, p0/m, z17.h, z18.h
+	ld1h	{z18.h}, p0/z, [x17, x9, lsl #1]
+	mla	z5.h, p0/m, z16.h, z18.h
+	mla	z13.h, p0/m, z17.h, z18.h
+	ld1h	{z18.h}, p0/z, [x19, x9, lsl #1]
+	mla	z6.h, p0/m, z16.h, z18.h
+	mla	z14.h, p0/m, z17.h, z18.h
+	ld1h	{z18.h}, p0/z, [x20, x9, lsl #1]
+	mla	z7.h, p0/m, z16.h, z18.h
+	mla	z15.h, p0/m, z17.h, z18.h
+	inch	x9
+	cmp	x9, x5
+	blt	L_frodokem_as_accum_sve_k
+	ptrue	p1.h
+	sxtw	x10, w3
+	lsl	x10, x10, #4
+	add	x8, x0, x10
+	uaddv	d19, p1, z0.h
+	umov	w11, v19.h[0]
+	ldrh	w10, [x8]
+	add	w11, w11, w10
+	strh	w11, [x8]
+	add	x8, x8, #2
+	uaddv	d19, p1, z1.h
+	umov	w11, v19.h[0]
+	ldrh	w10, [x8]
+	add	w11, w11, w10
+	strh	w11, [x8]
+	add	x8, x8, #2
+	uaddv	d19, p1, z2.h
+	umov	w11, v19.h[0]
+	ldrh	w10, [x8]
+	add	w11, w11, w10
+	strh	w11, [x8]
+	add	x8, x8, #2
+	uaddv	d19, p1, z3.h
+	umov	w11, v19.h[0]
+	ldrh	w10, [x8]
+	add	w11, w11, w10
+	strh	w11, [x8]
+	add	x8, x8, #2
+	uaddv	d19, p1, z4.h
+	umov	w11, v19.h[0]
+	ldrh	w10, [x8]
+	add	w11, w11, w10
+	strh	w11, [x8]
+	add	x8, x8, #2
+	uaddv	d19, p1, z5.h
+	umov	w11, v19.h[0]
+	ldrh	w10, [x8]
+	add	w11, w11, w10
+	strh	w11, [x8]
+	add	x8, x8, #2
+	uaddv	d19, p1, z6.h
+	umov	w11, v19.h[0]
+	ldrh	w10, [x8]
+	add	w11, w11, w10
+	strh	w11, [x8]
+	add	x8, x8, #2
+	uaddv	d19, p1, z7.h
+	umov	w11, v19.h[0]
+	ldrh	w10, [x8]
+	add	w11, w11, w10
+	strh	w11, [x8]
+	add	x8, x8, #2
+	uaddv	d19, p1, z8.h
+	umov	w11, v19.h[0]
+	ldrh	w10, [x8]
+	add	w11, w11, w10
+	strh	w11, [x8]
+	add	x8, x8, #2
+	uaddv	d19, p1, z9.h
+	umov	w11, v19.h[0]
+	ldrh	w10, [x8]
+	add	w11, w11, w10
+	strh	w11, [x8]
+	add	x8, x8, #2
+	uaddv	d19, p1, z10.h
+	umov	w11, v19.h[0]
+	ldrh	w10, [x8]
+	add	w11, w11, w10
+	strh	w11, [x8]
+	add	x8, x8, #2
+	uaddv	d19, p1, z11.h
+	umov	w11, v19.h[0]
+	ldrh	w10, [x8]
+	add	w11, w11, w10
+	strh	w11, [x8]
+	add	x8, x8, #2
+	uaddv	d19, p1, z12.h
+	umov	w11, v19.h[0]
+	ldrh	w10, [x8]
+	add	w11, w11, w10
+	strh	w11, [x8]
+	add	x8, x8, #2
+	uaddv	d19, p1, z13.h
+	umov	w11, v19.h[0]
+	ldrh	w10, [x8]
+	add	w11, w11, w10
+	strh	w11, [x8]
+	add	x8, x8, #2
+	uaddv	d19, p1, z14.h
+	umov	w11, v19.h[0]
+	ldrh	w10, [x8]
+	add	w11, w11, w10
+	strh	w11, [x8]
+	add	x8, x8, #2
+	uaddv	d19, p1, z15.h
+	umov	w11, v19.h[0]
+	ldrh	w10, [x8]
+	add	w11, w11, w10
+	strh	w11, [x8]
+	add	x8, x8, #2
+	ldp	x17, x19, [x29, #24]
+	ldr	x20, [x29, #40]
+	ldp	d8, d9, [x29, #48]
+	ldp	d10, d11, [x29, #64]
+	ldp	d12, d13, [x29, #80]
+	ldp	d14, d15, [x29, #96]
+	ldp	x29, x30, [sp], #0x70
+	ret
+#ifndef __APPLE__
+	.size	frodokem_as_accum_sve,.-frodokem_as_accum_sve
+#endif /* __APPLE__ */
+#ifndef __APPLE__
+.text
+.globl	frodokem_mul_bs_sve
+.type	frodokem_mul_bs_sve,@function
+.align	2
+frodokem_mul_bs_sve:
+#else
+.section	__TEXT,__text
+.globl	_frodokem_mul_bs_sve
+.p2align	2
+_frodokem_mul_bs_sve:
+#endif /* __APPLE__ */
+	stp	x29, x30, [sp, #-80]!
+	add	x29, sp, #0
+	stp	x17, x19, [x29, #24]
+	str	x20, [x29, #40]
+	stp	d8, d9, [x29, #48]
+	stp	d10, d11, [x29, #64]
+	sxtw	x5, w3
+	lsl	x6, x5, #1
+	mov	x12, x2
+	add	x13, x12, x6
+	add	x14, x13, x6
+	add	x15, x14, x6
+	add	x16, x15, x6
+	add	x17, x16, x6
+	add	x19, x17, x6
+	add	x20, x19, x6
+	mov	x7, x1
+	mov	x8, x0
+	ptrue	p1.h
+	mov	x9, #8
+L_frodokem_mul_bs_sve_i:
+	mov	z0.h, #0
+	mov	z1.h, #0
+	mov	z2.h, #0
+	mov	z3.h, #0
+	mov	z4.h, #0
+	mov	z5.h, #0
+	mov	z6.h, #0
+	mov	z7.h, #0
+	mov	x10, #0
+L_frodokem_mul_bs_sve_k:
+	whilelt	p0.h, x10, x5
+	ld1h	{z8.h}, p0/z, [x7, x10, lsl #1]
+	ld1h	{z9.h}, p0/z, [x12, x10, lsl #1]
+	mla	z0.h, p0/m, z8.h, z9.h
+	ld1h	{z9.h}, p0/z, [x13, x10, lsl #1]
+	mla	z1.h, p0/m, z8.h, z9.h
+	ld1h	{z9.h}, p0/z, [x14, x10, lsl #1]
+	mla	z2.h, p0/m, z8.h, z9.h
+	ld1h	{z9.h}, p0/z, [x15, x10, lsl #1]
+	mla	z3.h, p0/m, z8.h, z9.h
+	ld1h	{z9.h}, p0/z, [x16, x10, lsl #1]
+	mla	z4.h, p0/m, z8.h, z9.h
+	ld1h	{z9.h}, p0/z, [x17, x10, lsl #1]
+	mla	z5.h, p0/m, z8.h, z9.h
+	ld1h	{z9.h}, p0/z, [x19, x10, lsl #1]
+	mla	z6.h, p0/m, z8.h, z9.h
+	ld1h	{z9.h}, p0/z, [x20, x10, lsl #1]
+	mla	z7.h, p0/m, z8.h, z9.h
+	inch	x10
+	cmp	x10, x5
+	blt	L_frodokem_mul_bs_sve_k
+	uaddv	d10, p1, z0.h
+	umov	w11, v10.h[0]
+	and	w11, w11, w4
+	strh	w11, [x8]
+	add	x8, x8, #2
+	uaddv	d10, p1, z1.h
+	umov	w11, v10.h[0]
+	and	w11, w11, w4
+	strh	w11, [x8]
+	add	x8, x8, #2
+	uaddv	d10, p1, z2.h
+	umov	w11, v10.h[0]
+	and	w11, w11, w4
+	strh	w11, [x8]
+	add	x8, x8, #2
+	uaddv	d10, p1, z3.h
+	umov	w11, v10.h[0]
+	and	w11, w11, w4
+	strh	w11, [x8]
+	add	x8, x8, #2
+	uaddv	d10, p1, z4.h
+	umov	w11, v10.h[0]
+	and	w11, w11, w4
+	strh	w11, [x8]
+	add	x8, x8, #2
+	uaddv	d10, p1, z5.h
+	umov	w11, v10.h[0]
+	and	w11, w11, w4
+	strh	w11, [x8]
+	add	x8, x8, #2
+	uaddv	d10, p1, z6.h
+	umov	w11, v10.h[0]
+	and	w11, w11, w4
+	strh	w11, [x8]
+	add	x8, x8, #2
+	uaddv	d10, p1, z7.h
+	umov	w11, v10.h[0]
+	and	w11, w11, w4
+	strh	w11, [x8]
+	add	x8, x8, #2
+	add	x7, x7, x6
+	subs	x9, x9, #1
+	bne	L_frodokem_mul_bs_sve_i
+	ldp	x17, x19, [x29, #24]
+	ldr	x20, [x29, #40]
+	ldp	d8, d9, [x29, #48]
+	ldp	d10, d11, [x29, #64]
+	ldp	x29, x30, [sp], #0x50
+	ret
+#ifndef __APPLE__
+	.size	frodokem_mul_bs_sve,.-frodokem_mul_bs_sve
+#endif /* __APPLE__ */
+#ifndef __APPLE__
+.text
+.globl	frodokem_add_sve
+.type	frodokem_add_sve,@function
+.align	2
+frodokem_add_sve:
+#else
+.section	__TEXT,__text
+.globl	_frodokem_add_sve
+.p2align	2
+_frodokem_add_sve:
+#endif /* __APPLE__ */
+	mov	z2.h, w2
+	mov	x3, #0x40
+	mov	x4, #0
+L_frodokem_add_sve_k:
+	whilelt	p0.h, x4, x3
+	ld1h	{z0.h}, p0/z, [x0, x4, lsl #1]
+	ld1h	{z1.h}, p0/z, [x1, x4, lsl #1]
+	add	z0.h, z0.h, z1.h
+	and	z0.d, z0.d, z2.d
+	st1h	{z0.h}, p0, [x0, x4, lsl #1]
+	inch	x4
+	cmp	x4, x3
+	blt	L_frodokem_add_sve_k
+	ret
+#ifndef __APPLE__
+	.size	frodokem_add_sve,.-frodokem_add_sve
+#endif /* __APPLE__ */
+#endif /* WOLFSSL_FRODOKEM_SVE */
+#endif /* WOLFSSL_HAVE_FRODOKEM */
+#ifdef WOLFSSL_HAVE_FRODOKEM
+#ifdef WOLFSSL_FRODOKEM_SME
+	.arch	armv9-a+sve2+sme+sme2
+#ifndef __APPLE__
+.text
+.globl	frodokem_mul_bs_sme
+.type	frodokem_mul_bs_sme,@function
+.align	2
+frodokem_mul_bs_sme:
+#else
+.section	__TEXT,__text
+.globl	_frodokem_mul_bs_sme
+.p2align	2
+_frodokem_mul_bs_sme:
+#endif /* __APPLE__ */
+	smstart
+	zero	{za}
+	mov	x6, #16
+	whilelt	p0.h, xzr, x6
+	sxtw	x5, w3
+	lsr	x5, x5, #1
+L_frodokem_mul_bs_sme_k:
+	ld1h	{z0.h}, p0/z, [x1]
+	ld1h	{z1.h}, p0/z, [x2]
+	umopa	za0.s, p0/m, p0/m, z0.h, z1.h
+	add	x1, x1, #32
+	add	x2, x2, #32
+	subs	x5, x5, #1
+	bne	L_frodokem_mul_bs_sme_k
+	mov	z3.h, w4
+	mov	x6, #8
+	whilelt	p1.h, xzr, x6
+	whilelt	p2.s, xzr, x6
+	mov	w12, #0
+L_frodokem_mul_bs_sme_e:
+	mova	z2.s, p2/m, za0h.s[w12, 0]
+	uzp1	z2.h, z2.h, z2.h
+	and	z2.d, z2.d, z3.d
+	st1h	{z2.h}, p1, [x0]
+	add	x0, x0, #16
+	add	w12, w12, #1
+	cmp	w12, #8
+	bne	L_frodokem_mul_bs_sme_e
+	smstop
+	ret
+#ifndef __APPLE__
+	.size	frodokem_mul_bs_sme,.-frodokem_mul_bs_sme
+#endif /* __APPLE__ */
+#ifndef __APPLE__
+.text
+.globl	frodokem_sa_accum_sme
+.type	frodokem_sa_accum_sme,@function
+.align	2
+frodokem_sa_accum_sme:
+#else
+.section	__TEXT,__text
+.globl	_frodokem_sa_accum_sme
+.p2align	2
+_frodokem_sa_accum_sme:
+#endif /* __APPLE__ */
+	smstart
+	mov	x10, #16
+	whilelt	p0.h, xzr, x10
+	mov	x10, #8
+	whilelt	p1.h, xzr, x10
+	whilelt	p2.s, xzr, x10
+	ld1h	{z0.h}, p0/z, [x1]
+	sxtw	x10, w3
+	lsr	x9, x10, #3
+	lsl	x4, x10, #1
+	mov	x5, x2
+	add	x6, x2, x4
+	mov	x7, x0
+L_frodokem_sa_accum_sme_k:
+	zero	{za}
+	ld1h	{z1.h}, p1/z, [x5]
+	ld1h	{z2.h}, p1/z, [x6]
+	zip1	z3.h, z1.h, z2.h
+	umopa	za0.s, p0/m, p0/m, z0.h, z3.h
+	add	x5, x5, #16
+	add	x6, x6, #16
+	mov	x8, x7
+	mov	w12, #0
+L_frodokem_sa_accum_sme_e:
+	mova	z4.s, p2/m, za0h.s[w12, 0]
+	uzp1	z4.h, z4.h, z4.h
+	ld1h	{z5.h}, p1/z, [x8]
+	add	z4.h, z4.h, z5.h
+	st1h	{z4.h}, p1, [x8]
+	add	x8, x8, x4
+	add	w12, w12, #1
+	cmp	w12, #8
+	blt	L_frodokem_sa_accum_sme_e
+	add	x7, x7, #16
+	subs	x9, x9, #1
+	bne	L_frodokem_sa_accum_sme_k
+	smstop
+	ret
+#ifndef __APPLE__
+	.size	frodokem_sa_accum_sme,.-frodokem_sa_accum_sme
+#endif /* __APPLE__ */
+#ifndef __APPLE__
+.text
+.globl	frodokem_as_accum_sme
+.type	frodokem_as_accum_sme,@function
+.align	2
+frodokem_as_accum_sme:
+#else
+.section	__TEXT,__text
+.globl	_frodokem_as_accum_sme
+.p2align	2
+_frodokem_as_accum_sme:
+#endif /* __APPLE__ */
+	smstart
+	zero	{za}
+	mov	x8, #4
+	whilelt	p0.h, xzr, x8
+	mov	x8, #16
+	whilelt	p1.h, xzr, x8
+	sxtw	x7, w4
+	lsr	x7, x7, #1
+L_frodokem_as_accum_sme_k:
+	ld1h	{z0.h}, p0/z, [x1]
+	ld1h	{z1.h}, p1/z, [x2]
+	umopa	za0.s, p0/m, p1/m, z0.h, z1.h
+	add	x1, x1, #8
+	add	x2, x2, #32
+	subs	x7, x7, #1
+	bne	L_frodokem_as_accum_sme_k
+	mov	x8, #8
+	whilelt	p2.h, xzr, x8
+	whilelt	p3.s, xzr, x8
+	sxtw	x5, w3
+	lsl	x5, x5, #4
+	add	x5, x0, x5
+	add	x6, x5, #16
+	mov	w12, #0
+	mova	z2.s, p3/m, za0h.s[w12, 0]
+	uzp1	z2.h, z2.h, z2.h
+	ld1h	{z3.h}, p2/z, [x5]
+	add	z2.h, z2.h, z3.h
+	st1h	{z2.h}, p2, [x5]
+	mov	w12, #1
+	mova	z2.s, p3/m, za0h.s[w12, 0]
+	uzp1	z2.h, z2.h, z2.h
+	ld1h	{z3.h}, p2/z, [x6]
+	add	z2.h, z2.h, z3.h
+	st1h	{z2.h}, p2, [x6]
+	smstop
+	ret
+#ifndef __APPLE__
+	.size	frodokem_as_accum_sme,.-frodokem_as_accum_sme
+#endif /* __APPLE__ */
+#endif /* WOLFSSL_FRODOKEM_SME */
+#endif /* WOLFSSL_HAVE_FRODOKEM */
+#endif /* __aarch64__ */
+#endif /* WOLFSSL_ARMASM */
+
+#if defined(__linux__) && defined(__ELF__)
+.section	.note.GNU-stack,"",%progbits
+#endif
+#endif /* !WOLFSSL_ARMASM_INLINE */
diff --git a/wolfcrypt/src/port/arm/armv8-frodokem-asm.asm b/wolfcrypt/src/port/arm/armv8-frodokem-asm.asm
new file mode 100644
index 0000000000..8dbf29e94e
--- /dev/null
+++ b/wolfcrypt/src/port/arm/armv8-frodokem-asm.asm
@@ -0,0 +1,1665 @@
+; /* armv8-frodokem-asm
+;  *
+;  * Copyright (C) 2006-2026 wolfSSL Inc.
+;  *
+;  * This file is part of wolfSSL.
+;  *
+;  * wolfSSL is free software; you can redistribute it and/or modify
+;  * it under the terms of the GNU General Public License as published by
+;  * the Free Software Foundation; either version 3 of the License, or
+;  * (at your option) any later version.
+;  *
+;  * wolfSSL is distributed in the hope that it will be useful,
+;  * but WITHOUT ANY WARRANTY; without even the implied warranty of
+;  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+;  * GNU General Public License for more details.
+;  *
+;  * You should have received a copy of the GNU General Public License
+;  * along with this program; if not, write to the Free Software
+;  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335, USA
+;  */
+
+
+; Generated using (from wolfssl):
+;   cd ../scripts
+;   ruby ./frodokem/frodokem.rb arm64 \
+;       ../wolfssl/wolfcrypt/src/port/arm/armv8-frodokem-asm.asm
+	IF :DEF:WOLFSSL_HAVE_FRODOKEM
+	AREA	|.rodata|, DATA, READONLY, ALIGN=4
+	ALIGN	16
+L_sha3_aarch64_r
+	DCQ	0x0000000000000001, 0x0000000000008082
+	DCQ	0x800000000000808a, 0x8000000080008000
+	DCQ	0x000000000000808b, 0x0000000080000001
+	DCQ	0x8000000080008081, 0x8000000000008009
+	DCQ	0x000000000000008a, 0x0000000000000088
+	DCQ	0x0000000080008009, 0x000000008000000a
+	DCQ	0x000000008000808b, 0x800000000000008b
+	DCQ	0x8000000000008089, 0x8000000000008003
+	DCQ	0x8000000000008002, 0x8000000000000080
+	DCQ	0x000000000000800a, 0x800000008000000a
+	DCQ	0x8000000080008081, 0x8000000000008080
+	DCQ	0x0000000080000001, 0x8000000080008008
+	AREA	|.text|, CODE, READONLY
+	ALIGN	4
+	EXPORT	frodokem_sha3_x2_neon
+frodokem_sha3_x2_neon PROC
+	stp	x29, x30, [sp, #-80]!
+	add	x29, sp, #0
+	stp	D8, D9, [x29, #16]
+	stp	D10, D11, [x29, #32]
+	stp	D12, D13, [x29, #48]
+	stp	D14, D15, [x29, #64]
+	adrp	x1, L_sha3_aarch64_r
+	add	x1, x1, L_sha3_aarch64_r
+	ld4	{V0.D, V1.D, V2.D, V3.D}[0], [x0], #32
+	ld4	{V4.D, V5.D, V6.D, V7.D}[0], [x0], #32
+	ld4	{V8.D, V9.D, V10.D, V11.D}[0], [x0], #32
+	ld4	{V12.D, V13.D, V14.D, V15.D}[0], [x0], #32
+	ld4	{V16.D, V17.D, V18.D, V19.D}[0], [x0], #32
+	ld4	{V20.D, V21.D, V22.D, V23.D}[0], [x0], #32
+	ld1	{V24.D}[0], [x0]
+	add	x0, x0, #8
+	ld4	{V0.D, V1.D, V2.D, V3.D}[1], [x0], #32
+	ld4	{V4.D, V5.D, V6.D, V7.D}[1], [x0], #32
+	ld4	{V8.D, V9.D, V10.D, V11.D}[1], [x0], #32
+	ld4	{V12.D, V13.D, V14.D, V15.D}[1], [x0], #32
+	ld4	{V16.D, V17.D, V18.D, V19.D}[1], [x0], #32
+	ld4	{V20.D, V21.D, V22.D, V23.D}[1], [x0], #32
+	ld1	{V24.D}[1], [x0]
+	add	x0, x0, #8
+	sub	x0, x0, #0x190
+	mov	x2, #24
+L_frodokem_sha3_x2_neon_begin
+	; Col Mix NEON
+	eor	V30.16B, V4.16B, V9.16B
+	eor	V27.16B, V1.16B, V6.16B
+	eor	V30.16B, V30.16B, V14.16B
+	eor	V27.16B, V27.16B, V11.16B
+	eor	V30.16B, V30.16B, V19.16B
+	eor	V27.16B, V27.16B, V16.16B
+	eor	V30.16B, V30.16B, V24.16B
+	eor	V27.16B, V27.16B, V21.16B
+	ushr	V25.2D, V27.2D, #63
+	sli	V25.2D, V27.2D, #1
+	eor	V25.16B, V25.16B, V30.16B
+	eor	V31.16B, V0.16B, V5.16B
+	eor	V28.16B, V2.16B, V7.16B
+	eor	V31.16B, V31.16B, V10.16B
+	eor	V28.16B, V28.16B, V12.16B
+	eor	V31.16B, V31.16B, V15.16B
+	eor	V28.16B, V28.16B, V17.16B
+	eor	V31.16B, V31.16B, V20.16B
+	eor	V28.16B, V28.16B, V22.16B
+	ushr	V29.2D, V30.2D, #63
+	ushr	V26.2D, V28.2D, #63
+	sli	V29.2D, V30.2D, #1
+	sli	V26.2D, V28.2D, #1
+	eor	V28.16B, V28.16B, V29.16B
+	eor	V29.16B, V3.16B, V8.16B
+	eor	V26.16B, V26.16B, V31.16B
+	eor	V29.16B, V29.16B, V13.16B
+	eor	V29.16B, V29.16B, V18.16B
+	eor	V29.16B, V29.16B, V23.16B
+	ushr	V30.2D, V29.2D, #63
+	sli	V30.2D, V29.2D, #1
+	eor	V27.16B, V27.16B, V30.16B
+	ushr	V30.2D, V31.2D, #63
+	sli	V30.2D, V31.2D, #1
+	eor	V29.16B, V29.16B, V30.16B
+	; Swap Rotate NEON
+	eor	V0.16B, V0.16B, V25.16B
+	eor	V31.16B, V1.16B, V26.16B
+	eor	V6.16B, V6.16B, V26.16B
+	ushr	V30.2D, V31.2D, #63
+	ushr	V1.2D, V6.2D, #20
+	sli	V30.2D, V31.2D, #1
+	sli	V1.2D, V6.2D, #44
+	eor	V31.16B, V9.16B, V29.16B
+	eor	V22.16B, V22.16B, V27.16B
+	ushr	V6.2D, V31.2D, #44
+	ushr	V9.2D, V22.2D, #3
+	sli	V6.2D, V31.2D, #20
+	sli	V9.2D, V22.2D, #61
+	eor	V31.16B, V14.16B, V29.16B
+	eor	V20.16B, V20.16B, V25.16B
+	ushr	V22.2D, V31.2D, #25
+	ushr	V14.2D, V20.2D, #46
+	sli	V22.2D, V31.2D, #39
+	sli	V14.2D, V20.2D, #18
+	eor	V31.16B, V2.16B, V27.16B
+	eor	V12.16B, V12.16B, V27.16B
+	ushr	V20.2D, V31.2D, #2
+	ushr	V2.2D, V12.2D, #21
+	sli	V20.2D, V31.2D, #62
+	sli	V2.2D, V12.2D, #43
+	eor	V31.16B, V13.16B, V28.16B
+	eor	V19.16B, V19.16B, V29.16B
+	ushr	V12.2D, V31.2D, #39
+	ushr	V13.2D, V19.2D, #56
+	sli	V12.2D, V31.2D, #25
+	sli	V13.2D, V19.2D, #8
+	eor	V31.16B, V23.16B, V28.16B
+	eor	V15.16B, V15.16B, V25.16B
+	ushr	V19.2D, V31.2D, #8
+	ushr	V23.2D, V15.2D, #23
+	sli	V19.2D, V31.2D, #56
+	sli	V23.2D, V15.2D, #41
+	eor	V31.16B, V4.16B, V29.16B
+	eor	V24.16B, V24.16B, V29.16B
+	ushr	V15.2D, V31.2D, #37
+	ushr	V4.2D, V24.2D, #50
+	sli	V15.2D, V31.2D, #27
+	sli	V4.2D, V24.2D, #14
+	eor	V31.16B, V21.16B, V26.16B
+	eor	V8.16B, V8.16B, V28.16B
+	ushr	V24.2D, V31.2D, #62
+	ushr	V21.2D, V8.2D, #9
+	sli	V24.2D, V31.2D, #2
+	sli	V21.2D, V8.2D, #55
+	eor	V31.16B, V16.16B, V26.16B
+	eor	V5.16B, V5.16B, V25.16B
+	ushr	V8.2D, V31.2D, #19
+	ushr	V16.2D, V5.2D, #28
+	sli	V8.2D, V31.2D, #45
+	sli	V16.2D, V5.2D, #36
+	eor	V31.16B, V3.16B, V28.16B
+	eor	V18.16B, V18.16B, V28.16B
+	ushr	V5.2D, V31.2D, #36
+	ushr	V3.2D, V18.2D, #43
+	sli	V5.2D, V31.2D, #28
+	sli	V3.2D, V18.2D, #21
+	eor	V31.16B, V17.16B, V27.16B
+	eor	V11.16B, V11.16B, V26.16B
+	ushr	V18.2D, V31.2D, #49
+	ushr	V17.2D, V11.2D, #54
+	sli	V18.2D, V31.2D, #15
+	sli	V17.2D, V11.2D, #10
+	eor	V31.16B, V7.16B, V27.16B
+	eor	V10.16B, V10.16B, V25.16B
+	ushr	V11.2D, V31.2D, #58
+	ushr	V7.2D, V10.2D, #61
+	sli	V11.2D, V31.2D, #6
+	sli	V7.2D, V10.2D, #3
+	; Row Mix NEON
+	bic	V25.16B, V2.16B, V1.16B
+	bic	V26.16B, V3.16B, V2.16B
+	bic	V27.16B, V4.16B, V3.16B
+	bic	V28.16B, V0.16B, V4.16B
+	bic	V29.16B, V1.16B, V0.16B
+	eor	V0.16B, V0.16B, V25.16B
+	eor	V1.16B, V1.16B, V26.16B
+	eor	V2.16B, V2.16B, V27.16B
+	eor	V3.16B, V3.16B, V28.16B
+	eor	V4.16B, V4.16B, V29.16B
+	bic	V25.16B, V7.16B, V6.16B
+	bic	V26.16B, V8.16B, V7.16B
+	bic	V27.16B, V9.16B, V8.16B
+	bic	V28.16B, V5.16B, V9.16B
+	bic	V29.16B, V6.16B, V5.16B
+	eor	V5.16B, V5.16B, V25.16B
+	eor	V6.16B, V6.16B, V26.16B
+	eor	V7.16B, V7.16B, V27.16B
+	eor	V8.16B, V8.16B, V28.16B
+	eor	V9.16B, V9.16B, V29.16B
+	bic	V25.16B, V12.16B, V11.16B
+	bic	V26.16B, V13.16B, V12.16B
+	bic	V27.16B, V14.16B, V13.16B
+	bic	V28.16B, V30.16B, V14.16B
+	bic	V29.16B, V11.16B, V30.16B
+	eor	V10.16B, V30.16B, V25.16B
+	eor	V11.16B, V11.16B, V26.16B
+	eor	V12.16B, V12.16B, V27.16B
+	eor	V13.16B, V13.16B, V28.16B
+	eor	V14.16B, V14.16B, V29.16B
+	bic	V25.16B, V17.16B, V16.16B
+	bic	V26.16B, V18.16B, V17.16B
+	bic	V27.16B, V19.16B, V18.16B
+	bic	V28.16B, V15.16B, V19.16B
+	bic	V29.16B, V16.16B, V15.16B
+	eor	V15.16B, V15.16B, V25.16B
+	eor	V16.16B, V16.16B, V26.16B
+	eor	V17.16B, V17.16B, V27.16B
+	eor	V18.16B, V18.16B, V28.16B
+	eor	V19.16B, V19.16B, V29.16B
+	bic	V25.16B, V22.16B, V21.16B
+	bic	V26.16B, V23.16B, V22.16B
+	bic	V27.16B, V24.16B, V23.16B
+	bic	V28.16B, V20.16B, V24.16B
+	bic	V29.16B, V21.16B, V20.16B
+	eor	V20.16B, V20.16B, V25.16B
+	eor	V21.16B, V21.16B, V26.16B
+	eor	V22.16B, V22.16B, V27.16B
+	eor	V23.16B, V23.16B, V28.16B
+	eor	V24.16B, V24.16B, V29.16B
+	ldr	x3, [x1], #8
+	subs	x2, x2, #1
+	mov	V30.D[0], x3
+	mov	V30.D[1], x3
+	eor	V0.16B, V0.16B, V30.16B
+	bne	L_frodokem_sha3_x2_neon_begin
+	st4	{V0.D, V1.D, V2.D, V3.D}[0], [x0], #32
+	st4	{V4.D, V5.D, V6.D, V7.D}[0], [x0], #32
+	st4	{V8.D, V9.D, V10.D, V11.D}[0], [x0], #32
+	st4	{V12.D, V13.D, V14.D, V15.D}[0], [x0], #32
+	st4	{V16.D, V17.D, V18.D, V19.D}[0], [x0], #32
+	st4	{V20.D, V21.D, V22.D, V23.D}[0], [x0], #32
+	st1	{V24.D}[0], [x0]
+	add	x0, x0, #8
+	st4	{V0.D, V1.D, V2.D, V3.D}[1], [x0], #32
+	st4	{V4.D, V5.D, V6.D, V7.D}[1], [x0], #32
+	st4	{V8.D, V9.D, V10.D, V11.D}[1], [x0], #32
+	st4	{V12.D, V13.D, V14.D, V15.D}[1], [x0], #32
+	st4	{V16.D, V17.D, V18.D, V19.D}[1], [x0], #32
+	st4	{V20.D, V21.D, V22.D, V23.D}[1], [x0], #32
+	st1	{V24.D}[1], [x0]
+	add	x0, x0, #8
+	ldp	D8, D9, [x29, #16]
+	ldp	D10, D11, [x29, #32]
+	ldp	D12, D13, [x29, #48]
+	ldp	D14, D15, [x29, #64]
+	ldp	x29, x30, [sp], #0x50
+	ret
+	ENDP
+	IF :DEF:WOLFSSL_ARMASM_CRYPTO_SHA3
+; 	.arch_extension	sha3
+	AREA	|.text|, CODE, READONLY
+	ALIGN	4
+	EXPORT	frodokem_sha3_x2_crypto
+frodokem_sha3_x2_crypto PROC
+	stp	x29, x30, [sp, #-80]!
+	add	x29, sp, #0
+	stp	D8, D9, [x29, #16]
+	stp	D10, D11, [x29, #32]
+	stp	D12, D13, [x29, #48]
+	stp	D14, D15, [x29, #64]
+	adrp	x1, L_sha3_aarch64_r
+	add	x1, x1, L_sha3_aarch64_r
+	ld4	{V0.D, V1.D, V2.D, V3.D}[0], [x0], #32
+	ld4	{V4.D, V5.D, V6.D, V7.D}[0], [x0], #32
+	ld4	{V8.D, V9.D, V10.D, V11.D}[0], [x0], #32
+	ld4	{V12.D, V13.D, V14.D, V15.D}[0], [x0], #32
+	ld4	{V16.D, V17.D, V18.D, V19.D}[0], [x0], #32
+	ld4	{V20.D, V21.D, V22.D, V23.D}[0], [x0], #32
+	ld1	{V24.D}[0], [x0]
+	add	x0, x0, #8
+	ld4	{V0.D, V1.D, V2.D, V3.D}[1], [x0], #32
+	ld4	{V4.D, V5.D, V6.D, V7.D}[1], [x0], #32
+	ld4	{V8.D, V9.D, V10.D, V11.D}[1], [x0], #32
+	ld4	{V12.D, V13.D, V14.D, V15.D}[1], [x0], #32
+	ld4	{V16.D, V17.D, V18.D, V19.D}[1], [x0], #32
+	ld4	{V20.D, V21.D, V22.D, V23.D}[1], [x0], #32
+	ld1	{V24.D}[1], [x0]
+	add	x0, x0, #8
+	sub	x0, x0, #0x190
+	mov	x2, #24
+L_frodokem_sha3_x2_crypto_begin
+	; Col Mix
+	eor3	V31.16B, V0.16B, V5.16B, V10.16B
+	eor3	V27.16B, V1.16B, V6.16B, V11.16B
+	eor3	V28.16B, V2.16B, V7.16B, V12.16B
+	eor3	V29.16B, V3.16B, V8.16B, V13.16B
+	eor3	V30.16B, V4.16B, V9.16B, V14.16B
+	eor3	V31.16B, V31.16B, V15.16B, V20.16B
+	eor3	V27.16B, V27.16B, V16.16B, V21.16B
+	eor3	V28.16B, V28.16B, V17.16B, V22.16B
+	eor3	V29.16B, V29.16B, V18.16B, V23.16B
+	eor3	V30.16B, V30.16B, V19.16B, V24.16B
+	rax1	V25.2D, V30.2D, V27.2D
+	rax1	V26.2D, V31.2D, V28.2D
+	rax1	V27.2D, V27.2D, V29.2D
+	rax1	V28.2D, V28.2D, V30.2D
+	rax1	V29.2D, V29.2D, V31.2D
+	eor	V0.16B, V0.16B, V25.16B
+	xar	V30.2D, V1.2D, V26.2D, #63
+	xar	V1.2D, V6.2D, V26.2D, #20
+	xar	V6.2D, V9.2D, V29.2D, #44
+	xar	V9.2D, V22.2D, V27.2D, #3
+	xar	V22.2D, V14.2D, V29.2D, #25
+	xar	V14.2D, V20.2D, V25.2D, #46
+	xar	V20.2D, V2.2D, V27.2D, #2
+	xar	V2.2D, V12.2D, V27.2D, #21
+	xar	V12.2D, V13.2D, V28.2D, #39
+	xar	V13.2D, V19.2D, V29.2D, #56
+	xar	V19.2D, V23.2D, V28.2D, #8
+	xar	V23.2D, V15.2D, V25.2D, #23
+	xar	V15.2D, V4.2D, V29.2D, #37
+	xar	V4.2D, V24.2D, V29.2D, #50
+	xar	V24.2D, V21.2D, V26.2D, #62
+	xar	V21.2D, V8.2D, V28.2D, #9
+	xar	V8.2D, V16.2D, V26.2D, #19
+	xar	V16.2D, V5.2D, V25.2D, #28
+	xar	V5.2D, V3.2D, V28.2D, #36
+	xar	V3.2D, V18.2D, V28.2D, #43
+	xar	V18.2D, V17.2D, V27.2D, #49
+	xar	V17.2D, V11.2D, V26.2D, #54
+	xar	V11.2D, V7.2D, V27.2D, #58
+	xar	V7.2D, V10.2D, V25.2D, #61
+	; Row Mix
+	mov	V25.16B, V0.16B
+	mov	V26.16B, V1.16B
+	bcax	V0.16B, V25.16B, V2.16B, V26.16B
+	bcax	V1.16B, V26.16B, V3.16B, V2.16B
+	bcax	V2.16B, V2.16B, V4.16B, V3.16B
+	bcax	V3.16B, V3.16B, V25.16B, V4.16B
+	bcax	V4.16B, V4.16B, V26.16B, V25.16B
+	mov	V25.16B, V5.16B
+	mov	V26.16B, V6.16B
+	bcax	V5.16B, V25.16B, V7.16B, V26.16B
+	bcax	V6.16B, V26.16B, V8.16B, V7.16B
+	bcax	V7.16B, V7.16B, V9.16B, V8.16B
+	bcax	V8.16B, V8.16B, V25.16B, V9.16B
+	bcax	V9.16B, V9.16B, V26.16B, V25.16B
+	mov	V26.16B, V11.16B
+	bcax	V10.16B, V30.16B, V12.16B, V26.16B
+	bcax	V11.16B, V26.16B, V13.16B, V12.16B
+	bcax	V12.16B, V12.16B, V14.16B, V13.16B
+	bcax	V13.16B, V13.16B, V30.16B, V14.16B
+	bcax	V14.16B, V14.16B, V26.16B, V30.16B
+	mov	V25.16B, V15.16B
+	mov	V26.16B, V16.16B
+	bcax	V15.16B, V25.16B, V17.16B, V26.16B
+	bcax	V16.16B, V26.16B, V18.16B, V17.16B
+	bcax	V17.16B, V17.16B, V19.16B, V18.16B
+	bcax	V18.16B, V18.16B, V25.16B, V19.16B
+	bcax	V19.16B, V19.16B, V26.16B, V25.16B
+	mov	V25.16B, V20.16B
+	mov	V26.16B, V21.16B
+	bcax	V20.16B, V25.16B, V22.16B, V26.16B
+	bcax	V21.16B, V26.16B, V23.16B, V22.16B
+	bcax	V22.16B, V22.16B, V24.16B, V23.16B
+	bcax	V23.16B, V23.16B, V25.16B, V24.16B
+	bcax	V24.16B, V24.16B, V26.16B, V25.16B
+	ldr	x3, [x1], #8
+	subs	x2, x2, #1
+	mov	V30.D[0], x3
+	mov	V30.D[1], x3
+	eor	V0.16B, V0.16B, V30.16B
+	bne	L_frodokem_sha3_x2_crypto_begin
+	st4	{V0.D, V1.D, V2.D, V3.D}[0], [x0], #32
+	st4	{V4.D, V5.D, V6.D, V7.D}[0], [x0], #32
+	st4	{V8.D, V9.D, V10.D, V11.D}[0], [x0], #32
+	st4	{V12.D, V13.D, V14.D, V15.D}[0], [x0], #32
+	st4	{V16.D, V17.D, V18.D, V19.D}[0], [x0], #32
+	st4	{V20.D, V21.D, V22.D, V23.D}[0], [x0], #32
+	st1	{V24.D}[0], [x0]
+	add	x0, x0, #8
+	st4	{V0.D, V1.D, V2.D, V3.D}[1], [x0], #32
+	st4	{V4.D, V5.D, V6.D, V7.D}[1], [x0], #32
+	st4	{V8.D, V9.D, V10.D, V11.D}[1], [x0], #32
+	st4	{V12.D, V13.D, V14.D, V15.D}[1], [x0], #32
+	st4	{V16.D, V17.D, V18.D, V19.D}[1], [x0], #32
+	st4	{V20.D, V21.D, V22.D, V23.D}[1], [x0], #32
+	st1	{V24.D}[1], [x0]
+	add	x0, x0, #8
+	ldp	D8, D9, [x29, #16]
+	ldp	D10, D11, [x29, #32]
+	ldp	D12, D13, [x29, #48]
+	ldp	D14, D15, [x29, #64]
+	ldp	x29, x30, [sp], #0x50
+	ret
+	ENDP
+	ENDIF
+	AREA	|.text|, CODE, READONLY
+	ALIGN	4
+	EXPORT	frodokem_add_neon
+frodokem_add_neon PROC
+	stp	x29, x30, [sp, #-80]!
+	add	x29, sp, #0
+	stp	D8, D9, [x29, #16]
+	stp	D10, D11, [x29, #32]
+	stp	D12, D13, [x29, #48]
+	stp	D14, D15, [x29, #64]
+	dup	V16.8H, w2
+	ld1	{V0.8H, V1.8H, V2.8H, V3.8H}, [x0], #0x40
+	ld1	{V4.8H, V5.8H, V6.8H, V7.8H}, [x0], #0x40
+	sub	x0, x0, #0x80
+	ld1	{V8.8H, V9.8H, V10.8H, V11.8H}, [x1], #0x40
+	ld1	{V12.8H, V13.8H, V14.8H, V15.8H}, [x1], #0x40
+	add	V0.8H, V0.8H, V8.8H
+	and	V0.16B, V0.16B, V16.16B
+	add	V1.8H, V1.8H, V9.8H
+	and	V1.16B, V1.16B, V16.16B
+	add	V2.8H, V2.8H, V10.8H
+	and	V2.16B, V2.16B, V16.16B
+	add	V3.8H, V3.8H, V11.8H
+	and	V3.16B, V3.16B, V16.16B
+	add	V4.8H, V4.8H, V12.8H
+	and	V4.16B, V4.16B, V16.16B
+	add	V5.8H, V5.8H, V13.8H
+	and	V5.16B, V5.16B, V16.16B
+	add	V6.8H, V6.8H, V14.8H
+	and	V6.16B, V6.16B, V16.16B
+	add	V7.8H, V7.8H, V15.8H
+	and	V7.16B, V7.16B, V16.16B
+	st1	{V0.8H, V1.8H, V2.8H, V3.8H}, [x0], #0x40
+	st1	{V4.8H, V5.8H, V6.8H, V7.8H}, [x0], #0x40
+	ldp	D8, D9, [x29, #16]
+	ldp	D10, D11, [x29, #32]
+	ldp	D12, D13, [x29, #48]
+	ldp	D14, D15, [x29, #64]
+	ldp	x29, x30, [sp], #0x50
+	ret
+	ENDP
+	AREA	|.text|, CODE, READONLY
+	ALIGN	4
+	EXPORT	frodokem_sample_neon
+frodokem_sample_neon PROC
+	stp	x29, x30, [sp, #-64]!
+	add	x29, sp, #0
+	stp	D8, D9, [x29, #16]
+	stp	D10, D11, [x29, #32]
+	stp	D12, D13, [x29, #48]
+	cmp	w1, #16
+	blt	L_frodokem_sample_neon_rem
+L_frodokem_sample_neon_blk16
+	ld1	{V0.8H}, [x0], #16
+	ld1	{V1.8H}, [x0], #16
+	sub	x0, x0, #32
+	ushr	V2.8H, V0.8H, #1
+	movi	V4.8H, #1
+	and	V4.16B, V4.16B, V0.16B
+	movi	V6.8H, #0
+	ushr	V3.8H, V1.8H, #1
+	movi	V5.8H, #1
+	and	V5.16B, V5.16B, V1.16B
+	movi	V7.8H, #0
+	mov	x4, x2
+	mov	w5, w3
+L_frodokem_sample_neon_cdf16
+	ld1r	{V12.8H}, [x4], #2
+	sub	V10.8H, V12.8H, V2.8H
+	ushr	V10.8H, V10.8H, #15
+	add	V6.8H, V6.8H, V10.8H
+	sub	V11.8H, V12.8H, V3.8H
+	ushr	V11.8H, V11.8H, #15
+	add	V7.8H, V7.8H, V11.8H
+	subs	w5, w5, #1
+	bne	L_frodokem_sample_neon_cdf16
+	neg	V8.8H, V4.8H
+	eor	V6.16B, V6.16B, V8.16B
+	add	V6.8H, V6.8H, V4.8H
+	st1	{V6.8H}, [x0], #16
+	neg	V9.8H, V5.8H
+	eor	V7.16B, V7.16B, V9.16B
+	add	V7.8H, V7.8H, V5.8H
+	st1	{V7.8H}, [x0], #16
+	subs	w1, w1, #16
+	cmp	w1, #16
+	bge	L_frodokem_sample_neon_blk16
+L_frodokem_sample_neon_rem
+	cmp	w1, #0
+	beq	L_frodokem_sample_neon_done
+L_frodokem_sample_neon_blk
+	ld1	{V0.8H}, [x0]
+	ushr	V2.8H, V0.8H, #1
+	movi	V4.8H, #1
+	and	V4.16B, V4.16B, V0.16B
+	movi	V6.8H, #0
+	mov	x4, x2
+	mov	w5, w3
+L_frodokem_sample_neon_cdf
+	ld1r	{V12.8H}, [x4], #2
+	sub	V12.8H, V12.8H, V2.8H
+	ushr	V12.8H, V12.8H, #15
+	add	V6.8H, V6.8H, V12.8H
+	subs	w5, w5, #1
+	bne	L_frodokem_sample_neon_cdf
+	neg	V8.8H, V4.8H
+	eor	V6.16B, V6.16B, V8.16B
+	add	V6.8H, V6.8H, V4.8H
+	st1	{V6.8H}, [x0], #16
+	subs	w1, w1, #8
+	bne	L_frodokem_sample_neon_blk
+L_frodokem_sample_neon_done
+	ldp	D8, D9, [x29, #16]
+	ldp	D10, D11, [x29, #32]
+	ldp	D12, D13, [x29, #48]
+	ldp	x29, x30, [sp], #0x40
+	ret
+	ENDP
+	AREA	|.text|, CODE, READONLY
+	ALIGN	4
+	EXPORT	frodokem_sa_accum_neon
+frodokem_sa_accum_neon PROC
+	stp	x29, x30, [sp, #-80]!
+	add	x29, sp, #0
+	str	x17, [x29, #24]
+	stp	D8, D9, [x29, #32]
+	stp	D10, D11, [x29, #48]
+	stp	D12, D13, [x29, #64]
+	sxtw	x5, w4
+	lsl	x5, x5, #1
+	lsl	x6, x5, #2
+	sxtw	x7, w3
+	lsl	x7, x7, #1
+	add	x7, x1, x7
+	mov	x8, x0
+	mov	x12, #2
+L_frodokem_sa_accum_neon_t
+	ldr	w13, [x7]
+	dup	V0.8H, w13
+	lsr	w13, w13, #16
+	dup	V4.8H, w13
+	add	x7, x7, x5
+	ldr	w13, [x7]
+	dup	V1.8H, w13
+	lsr	w13, w13, #16
+	dup	V5.8H, w13
+	add	x7, x7, x5
+	ldr	w13, [x7]
+	dup	V2.8H, w13
+	lsr	w13, w13, #16
+	dup	V6.8H, w13
+	add	x7, x7, x5
+	ldr	w13, [x7]
+	dup	V3.8H, w13
+	lsr	w13, w13, #16
+	dup	V7.8H, w13
+	add	x7, x7, x5
+	mov	x14, x8
+	add	x15, x14, x5
+	add	x16, x15, x5
+	add	x17, x16, x5
+	mov	x9, x2
+	add	x10, x2, x5
+	lsr	x11, x5, #4
+L_frodokem_sa_accum_neon_k
+	ld1	{V8.8H}, [x9], #16
+	ld1	{V9.8H}, [x10], #16
+	ld1	{V10.8H}, [x14]
+	mla	V10.8H, V8.8H, V0.8H
+	mla	V10.8H, V9.8H, V4.8H
+	st1	{V10.8H}, [x14], #16
+	ld1	{V11.8H}, [x15]
+	mla	V11.8H, V8.8H, V1.8H
+	mla	V11.8H, V9.8H, V5.8H
+	st1	{V11.8H}, [x15], #16
+	ld1	{V12.8H}, [x16]
+	mla	V12.8H, V8.8H, V2.8H
+	mla	V12.8H, V9.8H, V6.8H
+	st1	{V12.8H}, [x16], #16
+	ld1	{V13.8H}, [x17]
+	mla	V13.8H, V8.8H, V3.8H
+	mla	V13.8H, V9.8H, V7.8H
+	st1	{V13.8H}, [x17], #16
+	subs	x11, x11, #1
+	bne	L_frodokem_sa_accum_neon_k
+	add	x8, x8, x6
+	subs	x12, x12, #1
+	bne	L_frodokem_sa_accum_neon_t
+	ldr	x17, [x29, #24]
+	ldp	D8, D9, [x29, #32]
+	ldp	D10, D11, [x29, #48]
+	ldp	D12, D13, [x29, #64]
+	ldp	x29, x30, [sp], #0x50
+	ret
+	ENDP
+	AREA	|.text|, CODE, READONLY
+	ALIGN	4
+	EXPORT	frodokem_sa_accum_x4_neon
+frodokem_sa_accum_x4_neon PROC
+	stp	x29, x30, [sp, #-112]!
+	add	x29, sp, #0
+	stp	x17, x19, [x29, #24]
+	str	x20, [x29, #40]
+	stp	D8, D9, [x29, #48]
+	stp	D10, D11, [x29, #64]
+	stp	D12, D13, [x29, #80]
+	stp	D14, D15, [x29, #96]
+	sxtw	x5, w4
+	lsl	x5, x5, #1
+	lsl	x6, x5, #2
+	sxtw	x7, w3
+	lsl	x7, x7, #1
+	add	x7, x1, x7
+	mov	x8, x0
+	mov	x14, #2
+L_frodokem_sa_accum_x4_neon_t
+	ldr	x15, [x7]
+	dup	V0.8H, w15
+	lsr	x15, x15, #16
+	dup	V1.8H, w15
+	lsr	x15, x15, #16
+	dup	V2.8H, w15
+	lsr	x15, x15, #16
+	dup	V3.8H, w15
+	add	x7, x7, x5
+	ldr	x15, [x7]
+	dup	V4.8H, w15
+	lsr	x15, x15, #16
+	dup	V5.8H, w15
+	lsr	x15, x15, #16
+	dup	V6.8H, w15
+	lsr	x15, x15, #16
+	dup	V7.8H, w15
+	add	x7, x7, x5
+	ldr	x15, [x7]
+	dup	V8.8H, w15
+	lsr	x15, x15, #16
+	dup	V9.8H, w15
+	lsr	x15, x15, #16
+	dup	V10.8H, w15
+	lsr	x15, x15, #16
+	dup	V11.8H, w15
+	add	x7, x7, x5
+	ldr	x15, [x7]
+	dup	V12.8H, w15
+	lsr	x15, x15, #16
+	dup	V13.8H, w15
+	lsr	x15, x15, #16
+	dup	V14.8H, w15
+	lsr	x15, x15, #16
+	dup	V15.8H, w15
+	add	x7, x7, x5
+	mov	x16, x8
+	add	x17, x16, x5
+	add	x19, x17, x5
+	add	x20, x19, x5
+	mov	x9, x2
+	add	x10, x9, x5
+	add	x11, x10, x5
+	add	x12, x11, x5
+	lsr	x13, x5, #4
+L_frodokem_sa_accum_x4_neon_k
+	ld1	{V16.8H}, [x9], #16
+	ld1	{V17.8H}, [x10], #16
+	ld1	{V18.8H}, [x11], #16
+	ld1	{V19.8H}, [x12], #16
+	ld1	{V20.8H}, [x16]
+	mla	V20.8H, V16.8H, V0.8H
+	mla	V20.8H, V17.8H, V1.8H
+	mla	V20.8H, V18.8H, V2.8H
+	mla	V20.8H, V19.8H, V3.8H
+	st1	{V20.8H}, [x16], #16
+	ld1	{V21.8H}, [x17]
+	mla	V21.8H, V16.8H, V4.8H
+	mla	V21.8H, V17.8H, V5.8H
+	mla	V21.8H, V18.8H, V6.8H
+	mla	V21.8H, V19.8H, V7.8H
+	st1	{V21.8H}, [x17], #16
+	ld1	{V22.8H}, [x19]
+	mla	V22.8H, V16.8H, V8.8H
+	mla	V22.8H, V17.8H, V9.8H
+	mla	V22.8H, V18.8H, V10.8H
+	mla	V22.8H, V19.8H, V11.8H
+	st1	{V22.8H}, [x19], #16
+	ld1	{V23.8H}, [x20]
+	mla	V23.8H, V16.8H, V12.8H
+	mla	V23.8H, V17.8H, V13.8H
+	mla	V23.8H, V18.8H, V14.8H
+	mla	V23.8H, V19.8H, V15.8H
+	st1	{V23.8H}, [x20], #16
+	subs	x13, x13, #1
+	bne	L_frodokem_sa_accum_x4_neon_k
+	add	x8, x8, x6
+	subs	x14, x14, #1
+	bne	L_frodokem_sa_accum_x4_neon_t
+	ldp	x17, x19, [x29, #24]
+	ldr	x20, [x29, #40]
+	ldp	D8, D9, [x29, #48]
+	ldp	D10, D11, [x29, #64]
+	ldp	D12, D13, [x29, #80]
+	ldp	D14, D15, [x29, #96]
+	ldp	x29, x30, [sp], #0x70
+	ret
+	ENDP
+	AREA	|.text|, CODE, READONLY
+	ALIGN	4
+	EXPORT	frodokem_as_accum_neon
+frodokem_as_accum_neon PROC
+	stp	x29, x30, [sp, #-96]!
+	add	x29, sp, #0
+	stp	x17, x19, [x29, #24]
+	str	x20, [x29, #40]
+	stp	D8, D9, [x29, #48]
+	stp	D10, D11, [x29, #64]
+	stp	D12, D13, [x29, #80]
+	sxtw	x5, w4
+	lsl	x5, x5, #1
+	lsl	x6, x5, #2
+	sxtw	x15, w3
+	lsl	x15, x15, #4
+	add	x7, x0, x15
+	add	x8, x7, #16
+	mov	x9, x2
+	add	x10, x2, x5
+	mov	x13, x1
+	mov	x16, x13
+	add	x17, x16, x5
+	add	x19, x17, x5
+	add	x20, x19, x5
+	mov	x11, x9
+	mov	x12, x10
+	movi	V0.8H, #0
+	movi	V1.8H, #0
+	movi	V2.8H, #0
+	movi	V3.8H, #0
+	movi	V4.8H, #0
+	movi	V5.8H, #0
+	movi	V6.8H, #0
+	movi	V7.8H, #0
+	lsr	x14, x5, #4
+L_frodokem_as_accum_neon_k0
+	ld1	{V8.8H}, [x11], #16
+	ld1	{V9.8H}, [x12], #16
+	ld1	{V10.8H}, [x16], #16
+	mla	V0.8H, V8.8H, V10.8H
+	mla	V4.8H, V9.8H, V10.8H
+	ld1	{V10.8H}, [x17], #16
+	mla	V1.8H, V8.8H, V10.8H
+	mla	V5.8H, V9.8H, V10.8H
+	ld1	{V10.8H}, [x19], #16
+	mla	V2.8H, V8.8H, V10.8H
+	mla	V6.8H, V9.8H, V10.8H
+	ld1	{V10.8H}, [x20], #16
+	mla	V3.8H, V8.8H, V10.8H
+	mla	V7.8H, V9.8H, V10.8H
+	subs	x14, x14, #1
+	bne	L_frodokem_as_accum_neon_k0
+	addv	H13, V0.8H
+	ins	V11.H[0], V13.H[0]
+	addv	H13, V4.8H
+	ins	V12.H[0], V13.H[0]
+	addv	H13, V1.8H
+	ins	V11.H[1], V13.H[0]
+	addv	H13, V5.8H
+	ins	V12.H[1], V13.H[0]
+	addv	H13, V2.8H
+	ins	V11.H[2], V13.H[0]
+	addv	H13, V6.8H
+	ins	V12.H[2], V13.H[0]
+	addv	H13, V3.8H
+	ins	V11.H[3], V13.H[0]
+	addv	H13, V7.8H
+	ins	V12.H[3], V13.H[0]
+	add	x13, x13, x6
+	mov	x16, x13
+	add	x17, x16, x5
+	add	x19, x17, x5
+	add	x20, x19, x5
+	mov	x11, x9
+	mov	x12, x10
+	movi	V0.8H, #0
+	movi	V1.8H, #0
+	movi	V2.8H, #0
+	movi	V3.8H, #0
+	movi	V4.8H, #0
+	movi	V5.8H, #0
+	movi	V6.8H, #0
+	movi	V7.8H, #0
+	lsr	x14, x5, #4
+L_frodokem_as_accum_neon_k1
+	ld1	{V8.8H}, [x11], #16
+	ld1	{V9.8H}, [x12], #16
+	ld1	{V10.8H}, [x16], #16
+	mla	V0.8H, V8.8H, V10.8H
+	mla	V4.8H, V9.8H, V10.8H
+	ld1	{V10.8H}, [x17], #16
+	mla	V1.8H, V8.8H, V10.8H
+	mla	V5.8H, V9.8H, V10.8H
+	ld1	{V10.8H}, [x19], #16
+	mla	V2.8H, V8.8H, V10.8H
+	mla	V6.8H, V9.8H, V10.8H
+	ld1	{V10.8H}, [x20], #16
+	mla	V3.8H, V8.8H, V10.8H
+	mla	V7.8H, V9.8H, V10.8H
+	subs	x14, x14, #1
+	bne	L_frodokem_as_accum_neon_k1
+	addv	H13, V0.8H
+	ins	V11.H[4], V13.H[0]
+	addv	H13, V4.8H
+	ins	V12.H[4], V13.H[0]
+	addv	H13, V1.8H
+	ins	V11.H[5], V13.H[0]
+	addv	H13, V5.8H
+	ins	V12.H[5], V13.H[0]
+	addv	H13, V2.8H
+	ins	V11.H[6], V13.H[0]
+	addv	H13, V6.8H
+	ins	V12.H[6], V13.H[0]
+	addv	H13, V3.8H
+	ins	V11.H[7], V13.H[0]
+	addv	H13, V7.8H
+	ins	V12.H[7], V13.H[0]
+	add	x13, x13, x6
+	ld1	{V8.8H}, [x7]
+	add	V11.8H, V11.8H, V8.8H
+	st1	{V11.8H}, [x7], #16
+	ld1	{V9.8H}, [x8]
+	add	V12.8H, V12.8H, V9.8H
+	st1	{V12.8H}, [x8], #16
+	ldp	x17, x19, [x29, #24]
+	ldr	x20, [x29, #40]
+	ldp	D8, D9, [x29, #48]
+	ldp	D10, D11, [x29, #64]
+	ldp	D12, D13, [x29, #80]
+	ldp	x29, x30, [sp], #0x60
+	ret
+	ENDP
+	AREA	|.text|, CODE, READONLY
+	ALIGN	4
+	EXPORT	frodokem_as_accum_x4_neon
+frodokem_as_accum_x4_neon PROC
+	stp	x29, x30, [sp, #-160]!
+	add	x29, sp, #0
+	stp	x17, x19, [x29, #24]
+	stp	x20, x21, [x29, #40]
+	stp	x22, x23, [x29, #56]
+	stp	x24, x25, [x29, #72]
+	str	x26, [x29, #88]
+	stp	D8, D9, [x29, #96]
+	stp	D10, D11, [x29, #112]
+	stp	D12, D13, [x29, #128]
+	stp	D14, D15, [x29, #144]
+	sxtw	x5, w4
+	lsl	x5, x5, #1
+	lsl	x6, x5, #2
+	sxtw	x22, w3
+	lsl	x22, x22, #4
+	add	x7, x0, x22
+	add	x8, x7, #16
+	add	x9, x8, #16
+	add	x10, x9, #16
+	mov	x11, x2
+	add	x12, x11, x5
+	add	x13, x12, x5
+	add	x14, x13, x5
+	mov	x20, x1
+	mov	x23, x20
+	add	x24, x23, x5
+	add	x25, x24, x5
+	add	x26, x25, x5
+	mov	x15, x11
+	mov	x16, x12
+	mov	x17, x13
+	mov	x19, x14
+	movi	V0.8H, #0
+	movi	V1.8H, #0
+	movi	V2.8H, #0
+	movi	V3.8H, #0
+	movi	V4.8H, #0
+	movi	V5.8H, #0
+	movi	V6.8H, #0
+	movi	V7.8H, #0
+	movi	V8.8H, #0
+	movi	V9.8H, #0
+	movi	V10.8H, #0
+	movi	V11.8H, #0
+	movi	V12.8H, #0
+	movi	V13.8H, #0
+	movi	V14.8H, #0
+	movi	V15.8H, #0
+	lsr	x21, x5, #4
+L_frodokem_as_accum_x4_neon_k0
+	ld1	{V16.8H}, [x15], #16
+	ld1	{V17.8H}, [x16], #16
+	ld1	{V18.8H}, [x17], #16
+	ld1	{V19.8H}, [x19], #16
+	ld1	{V20.8H}, [x23], #16
+	mla	V0.8H, V16.8H, V20.8H
+	mla	V4.8H, V17.8H, V20.8H
+	mla	V8.8H, V18.8H, V20.8H
+	mla	V12.8H, V19.8H, V20.8H
+	ld1	{V20.8H}, [x24], #16
+	mla	V1.8H, V16.8H, V20.8H
+	mla	V5.8H, V17.8H, V20.8H
+	mla	V9.8H, V18.8H, V20.8H
+	mla	V13.8H, V19.8H, V20.8H
+	ld1	{V20.8H}, [x25], #16
+	mla	V2.8H, V16.8H, V20.8H
+	mla	V6.8H, V17.8H, V20.8H
+	mla	V10.8H, V18.8H, V20.8H
+	mla	V14.8H, V19.8H, V20.8H
+	ld1	{V20.8H}, [x26], #16
+	mla	V3.8H, V16.8H, V20.8H
+	mla	V7.8H, V17.8H, V20.8H
+	mla	V11.8H, V18.8H, V20.8H
+	mla	V15.8H, V19.8H, V20.8H
+	subs	x21, x21, #1
+	bne	L_frodokem_as_accum_x4_neon_k0
+	addv	H25, V0.8H
+	ins	V21.H[0], V25.H[0]
+	addv	H25, V4.8H
+	ins	V22.H[0], V25.H[0]
+	addv	H25, V8.8H
+	ins	V23.H[0], V25.H[0]
+	addv	H25, V12.8H
+	ins	V24.H[0], V25.H[0]
+	addv	H25, V1.8H
+	ins	V21.H[1], V25.H[0]
+	addv	H25, V5.8H
+	ins	V22.H[1], V25.H[0]
+	addv	H25, V9.8H
+	ins	V23.H[1], V25.H[0]
+	addv	H25, V13.8H
+	ins	V24.H[1], V25.H[0]
+	addv	H25, V2.8H
+	ins	V21.H[2], V25.H[0]
+	addv	H25, V6.8H
+	ins	V22.H[2], V25.H[0]
+	addv	H25, V10.8H
+	ins	V23.H[2], V25.H[0]
+	addv	H25, V14.8H
+	ins	V24.H[2], V25.H[0]
+	addv	H25, V3.8H
+	ins	V21.H[3], V25.H[0]
+	addv	H25, V7.8H
+	ins	V22.H[3], V25.H[0]
+	addv	H25, V11.8H
+	ins	V23.H[3], V25.H[0]
+	addv	H25, V15.8H
+	ins	V24.H[3], V25.H[0]
+	add	x20, x20, x6
+	mov	x23, x20
+	add	x24, x23, x5
+	add	x25, x24, x5
+	add	x26, x25, x5
+	mov	x15, x11
+	mov	x16, x12
+	mov	x17, x13
+	mov	x19, x14
+	movi	V0.8H, #0
+	movi	V1.8H, #0
+	movi	V2.8H, #0
+	movi	V3.8H, #0
+	movi	V4.8H, #0
+	movi	V5.8H, #0
+	movi	V6.8H, #0
+	movi	V7.8H, #0
+	movi	V8.8H, #0
+	movi	V9.8H, #0
+	movi	V10.8H, #0
+	movi	V11.8H, #0
+	movi	V12.8H, #0
+	movi	V13.8H, #0
+	movi	V14.8H, #0
+	movi	V15.8H, #0
+	lsr	x21, x5, #4
+L_frodokem_as_accum_x4_neon_k1
+	ld1	{V16.8H}, [x15], #16
+	ld1	{V17.8H}, [x16], #16
+	ld1	{V18.8H}, [x17], #16
+	ld1	{V19.8H}, [x19], #16
+	ld1	{V20.8H}, [x23], #16
+	mla	V0.8H, V16.8H, V20.8H
+	mla	V4.8H, V17.8H, V20.8H
+	mla	V8.8H, V18.8H, V20.8H
+	mla	V12.8H, V19.8H, V20.8H
+	ld1	{V20.8H}, [x24], #16
+	mla	V1.8H, V16.8H, V20.8H
+	mla	V5.8H, V17.8H, V20.8H
+	mla	V9.8H, V18.8H, V20.8H
+	mla	V13.8H, V19.8H, V20.8H
+	ld1	{V20.8H}, [x25], #16
+	mla	V2.8H, V16.8H, V20.8H
+	mla	V6.8H, V17.8H, V20.8H
+	mla	V10.8H, V18.8H, V20.8H
+	mla	V14.8H, V19.8H, V20.8H
+	ld1	{V20.8H}, [x26], #16
+	mla	V3.8H, V16.8H, V20.8H
+	mla	V7.8H, V17.8H, V20.8H
+	mla	V11.8H, V18.8H, V20.8H
+	mla	V15.8H, V19.8H, V20.8H
+	subs	x21, x21, #1
+	bne	L_frodokem_as_accum_x4_neon_k1
+	addv	H25, V0.8H
+	ins	V21.H[4], V25.H[0]
+	addv	H25, V4.8H
+	ins	V22.H[4], V25.H[0]
+	addv	H25, V8.8H
+	ins	V23.H[4], V25.H[0]
+	addv	H25, V12.8H
+	ins	V24.H[4], V25.H[0]
+	addv	H25, V1.8H
+	ins	V21.H[5], V25.H[0]
+	addv	H25, V5.8H
+	ins	V22.H[5], V25.H[0]
+	addv	H25, V9.8H
+	ins	V23.H[5], V25.H[0]
+	addv	H25, V13.8H
+	ins	V24.H[5], V25.H[0]
+	addv	H25, V2.8H
+	ins	V21.H[6], V25.H[0]
+	addv	H25, V6.8H
+	ins	V22.H[6], V25.H[0]
+	addv	H25, V10.8H
+	ins	V23.H[6], V25.H[0]
+	addv	H25, V14.8H
+	ins	V24.H[6], V25.H[0]
+	addv	H25, V3.8H
+	ins	V21.H[7], V25.H[0]
+	addv	H25, V7.8H
+	ins	V22.H[7], V25.H[0]
+	addv	H25, V11.8H
+	ins	V23.H[7], V25.H[0]
+	addv	H25, V15.8H
+	ins	V24.H[7], V25.H[0]
+	add	x20, x20, x6
+	ld1	{V16.8H}, [x7]
+	add	V21.8H, V21.8H, V16.8H
+	st1	{V21.8H}, [x7], #16
+	ld1	{V16.8H}, [x8]
+	add	V22.8H, V22.8H, V16.8H
+	st1	{V22.8H}, [x8], #16
+	ld1	{V16.8H}, [x9]
+	add	V23.8H, V23.8H, V16.8H
+	st1	{V23.8H}, [x9], #16
+	ld1	{V16.8H}, [x10]
+	add	V24.8H, V24.8H, V16.8H
+	st1	{V24.8H}, [x10], #16
+	ldp	x17, x19, [x29, #24]
+	ldp	x20, x21, [x29, #40]
+	ldp	x22, x23, [x29, #56]
+	ldp	x24, x25, [x29, #72]
+	ldr	x26, [x29, #88]
+	ldp	D8, D9, [x29, #96]
+	ldp	D10, D11, [x29, #112]
+	ldp	D12, D13, [x29, #128]
+	ldp	D14, D15, [x29, #144]
+	ldp	x29, x30, [sp], #0xa0
+	ret
+	ENDP
+	AREA	|.text|, CODE, READONLY
+	ALIGN	4
+	EXPORT	frodokem_mul_bs_neon
+frodokem_mul_bs_neon PROC
+	stp	x29, x30, [sp, #-160]!
+	add	x29, sp, #0
+	stp	x17, x19, [x29, #16]
+	stp	x20, x21, [x29, #32]
+	stp	x22, x23, [x29, #48]
+	stp	x24, x25, [x29, #64]
+	stp	x26, x27, [x29, #80]
+	stp	D8, D9, [x29, #96]
+	stp	D10, D11, [x29, #112]
+	stp	D12, D13, [x29, #128]
+	stp	D14, D15, [x29, #144]
+	sxtw	x5, w3
+	lsl	x5, x5, #1
+	lsl	x6, x5, #2
+	dup	V26.8H, w4
+	mov	x7, x0
+	add	x8, x7, #16
+	add	x9, x8, #16
+	add	x10, x9, #16
+	mov	x11, x1
+	mov	x23, #2
+L_frodokem_mul_bs_neon_p
+	mov	x12, x11
+	add	x13, x12, x5
+	add	x14, x13, x5
+	add	x15, x14, x5
+	mov	x21, x2
+	mov	x24, x21
+	add	x25, x24, x5
+	add	x26, x25, x5
+	add	x27, x26, x5
+	mov	x16, x12
+	mov	x17, x13
+	mov	x19, x14
+	mov	x20, x15
+	movi	V0.8H, #0
+	movi	V1.8H, #0
+	movi	V2.8H, #0
+	movi	V3.8H, #0
+	movi	V4.8H, #0
+	movi	V5.8H, #0
+	movi	V6.8H, #0
+	movi	V7.8H, #0
+	movi	V8.8H, #0
+	movi	V9.8H, #0
+	movi	V10.8H, #0
+	movi	V11.8H, #0
+	movi	V12.8H, #0
+	movi	V13.8H, #0
+	movi	V14.8H, #0
+	movi	V15.8H, #0
+	lsr	x22, x5, #4
+L_frodokem_mul_bs_neon_k0
+	ld1	{V16.8H}, [x16], #16
+	ld1	{V17.8H}, [x17], #16
+	ld1	{V18.8H}, [x19], #16
+	ld1	{V19.8H}, [x20], #16
+	ld1	{V20.8H}, [x24], #16
+	mla	V0.8H, V16.8H, V20.8H
+	mla	V4.8H, V17.8H, V20.8H
+	mla	V8.8H, V18.8H, V20.8H
+	mla	V12.8H, V19.8H, V20.8H
+	ld1	{V20.8H}, [x25], #16
+	mla	V1.8H, V16.8H, V20.8H
+	mla	V5.8H, V17.8H, V20.8H
+	mla	V9.8H, V18.8H, V20.8H
+	mla	V13.8H, V19.8H, V20.8H
+	ld1	{V20.8H}, [x26], #16
+	mla	V2.8H, V16.8H, V20.8H
+	mla	V6.8H, V17.8H, V20.8H
+	mla	V10.8H, V18.8H, V20.8H
+	mla	V14.8H, V19.8H, V20.8H
+	ld1	{V20.8H}, [x27], #16
+	mla	V3.8H, V16.8H, V20.8H
+	mla	V7.8H, V17.8H, V20.8H
+	mla	V11.8H, V18.8H, V20.8H
+	mla	V15.8H, V19.8H, V20.8H
+	subs	x22, x22, #1
+	bne	L_frodokem_mul_bs_neon_k0
+	addv	H25, V0.8H
+	ins	V21.H[0], V25.H[0]
+	addv	H25, V4.8H
+	ins	V22.H[0], V25.H[0]
+	addv	H25, V8.8H
+	ins	V23.H[0], V25.H[0]
+	addv	H25, V12.8H
+	ins	V24.H[0], V25.H[0]
+	addv	H25, V1.8H
+	ins	V21.H[1], V25.H[0]
+	addv	H25, V5.8H
+	ins	V22.H[1], V25.H[0]
+	addv	H25, V9.8H
+	ins	V23.H[1], V25.H[0]
+	addv	H25, V13.8H
+	ins	V24.H[1], V25.H[0]
+	addv	H25, V2.8H
+	ins	V21.H[2], V25.H[0]
+	addv	H25, V6.8H
+	ins	V22.H[2], V25.H[0]
+	addv	H25, V10.8H
+	ins	V23.H[2], V25.H[0]
+	addv	H25, V14.8H
+	ins	V24.H[2], V25.H[0]
+	addv	H25, V3.8H
+	ins	V21.H[3], V25.H[0]
+	addv	H25, V7.8H
+	ins	V22.H[3], V25.H[0]
+	addv	H25, V11.8H
+	ins	V23.H[3], V25.H[0]
+	addv	H25, V15.8H
+	ins	V24.H[3], V25.H[0]
+	add	x21, x21, x6
+	mov	x24, x21
+	add	x25, x24, x5
+	add	x26, x25, x5
+	add	x27, x26, x5
+	mov	x16, x12
+	mov	x17, x13
+	mov	x19, x14
+	mov	x20, x15
+	movi	V0.8H, #0
+	movi	V1.8H, #0
+	movi	V2.8H, #0
+	movi	V3.8H, #0
+	movi	V4.8H, #0
+	movi	V5.8H, #0
+	movi	V6.8H, #0
+	movi	V7.8H, #0
+	movi	V8.8H, #0
+	movi	V9.8H, #0
+	movi	V10.8H, #0
+	movi	V11.8H, #0
+	movi	V12.8H, #0
+	movi	V13.8H, #0
+	movi	V14.8H, #0
+	movi	V15.8H, #0
+	lsr	x22, x5, #4
+L_frodokem_mul_bs_neon_k1
+	ld1	{V16.8H}, [x16], #16
+	ld1	{V17.8H}, [x17], #16
+	ld1	{V18.8H}, [x19], #16
+	ld1	{V19.8H}, [x20], #16
+	ld1	{V20.8H}, [x24], #16
+	mla	V0.8H, V16.8H, V20.8H
+	mla	V4.8H, V17.8H, V20.8H
+	mla	V8.8H, V18.8H, V20.8H
+	mla	V12.8H, V19.8H, V20.8H
+	ld1	{V20.8H}, [x25], #16
+	mla	V1.8H, V16.8H, V20.8H
+	mla	V5.8H, V17.8H, V20.8H
+	mla	V9.8H, V18.8H, V20.8H
+	mla	V13.8H, V19.8H, V20.8H
+	ld1	{V20.8H}, [x26], #16
+	mla	V2.8H, V16.8H, V20.8H
+	mla	V6.8H, V17.8H, V20.8H
+	mla	V10.8H, V18.8H, V20.8H
+	mla	V14.8H, V19.8H, V20.8H
+	ld1	{V20.8H}, [x27], #16
+	mla	V3.8H, V16.8H, V20.8H
+	mla	V7.8H, V17.8H, V20.8H
+	mla	V11.8H, V18.8H, V20.8H
+	mla	V15.8H, V19.8H, V20.8H
+	subs	x22, x22, #1
+	bne	L_frodokem_mul_bs_neon_k1
+	addv	H25, V0.8H
+	ins	V21.H[4], V25.H[0]
+	addv	H25, V4.8H
+	ins	V22.H[4], V25.H[0]
+	addv	H25, V8.8H
+	ins	V23.H[4], V25.H[0]
+	addv	H25, V12.8H
+	ins	V24.H[4], V25.H[0]
+	addv	H25, V1.8H
+	ins	V21.H[5], V25.H[0]
+	addv	H25, V5.8H
+	ins	V22.H[5], V25.H[0]
+	addv	H25, V9.8H
+	ins	V23.H[5], V25.H[0]
+	addv	H25, V13.8H
+	ins	V24.H[5], V25.H[0]
+	addv	H25, V2.8H
+	ins	V21.H[6], V25.H[0]
+	addv	H25, V6.8H
+	ins	V22.H[6], V25.H[0]
+	addv	H25, V10.8H
+	ins	V23.H[6], V25.H[0]
+	addv	H25, V14.8H
+	ins	V24.H[6], V25.H[0]
+	addv	H25, V3.8H
+	ins	V21.H[7], V25.H[0]
+	addv	H25, V7.8H
+	ins	V22.H[7], V25.H[0]
+	addv	H25, V11.8H
+	ins	V23.H[7], V25.H[0]
+	addv	H25, V15.8H
+	ins	V24.H[7], V25.H[0]
+	add	x21, x21, x6
+	and	V21.16B, V21.16B, V26.16B
+	st1	{V21.8H}, [x7]
+	and	V22.16B, V22.16B, V26.16B
+	st1	{V22.8H}, [x8]
+	and	V23.16B, V23.16B, V26.16B
+	st1	{V23.8H}, [x9]
+	and	V24.16B, V24.16B, V26.16B
+	st1	{V24.8H}, [x10]
+	add	x11, x11, x6
+	add	x7, x7, #0x40
+	add	x8, x8, #0x40
+	add	x9, x9, #0x40
+	add	x10, x10, #0x40
+	subs	x23, x23, #1
+	bne	L_frodokem_mul_bs_neon_p
+	ldp	x17, x19, [x29, #16]
+	ldp	x20, x21, [x29, #32]
+	ldp	x22, x23, [x29, #48]
+	ldp	x24, x25, [x29, #64]
+	ldp	x26, x27, [x29, #80]
+	ldp	D8, D9, [x29, #96]
+	ldp	D10, D11, [x29, #112]
+	ldp	D12, D13, [x29, #128]
+	ldp	D14, D15, [x29, #144]
+	ldp	x29, x30, [sp], #0xa0
+	ret
+	ENDP
+	AREA	|.text|, CODE, READONLY
+	ALIGN	4
+	EXPORT	frodokem_mul_add_sb_plus_e_neon
+frodokem_mul_add_sb_plus_e_neon PROC
+	stp	x29, x30, [sp, #-48]!
+	add	x29, sp, #0
+	stp	D8, D9, [x29, #16]
+	stp	D10, D11, [x29, #32]
+	sxtw	x5, w3
+	lsl	x5, x5, #1
+	dup	V10.8H, w4
+	mov	x6, x2
+	add	x7, x6, x5
+	add	x8, x7, x5
+	add	x9, x8, x5
+	add	x10, x9, x5
+	add	x11, x10, x5
+	add	x12, x11, x5
+	add	x13, x12, x5
+	mov	x14, x1
+	mov	x16, x0
+	ld1	{V0.8H, V1.8H, V2.8H, V3.8H}, [x16], #0x40
+	ld1	{V4.8H, V5.8H, V6.8H, V7.8H}, [x16], #0x40
+	sub	x16, x16, #0x80
+	sxtw	x15, w3
+L_frodokem_mul_add_sb_plus_e_neon_j
+	ld1	{V8.8H}, [x14], #16
+	ld1r	{V9.8H}, [x6], #2
+	mla	V0.8H, V8.8H, V9.8H
+	ld1r	{V9.8H}, [x7], #2
+	mla	V1.8H, V8.8H, V9.8H
+	ld1r	{V9.8H}, [x8], #2
+	mla	V2.8H, V8.8H, V9.8H
+	ld1r	{V9.8H}, [x9], #2
+	mla	V3.8H, V8.8H, V9.8H
+	ld1r	{V9.8H}, [x10], #2
+	mla	V4.8H, V8.8H, V9.8H
+	ld1r	{V9.8H}, [x11], #2
+	mla	V5.8H, V8.8H, V9.8H
+	ld1r	{V9.8H}, [x12], #2
+	mla	V6.8H, V8.8H, V9.8H
+	ld1r	{V9.8H}, [x13], #2
+	mla	V7.8H, V8.8H, V9.8H
+	subs	x15, x15, #1
+	bne	L_frodokem_mul_add_sb_plus_e_neon_j
+	and	V0.16B, V0.16B, V10.16B
+	and	V1.16B, V1.16B, V10.16B
+	and	V2.16B, V2.16B, V10.16B
+	and	V3.16B, V3.16B, V10.16B
+	and	V4.16B, V4.16B, V10.16B
+	and	V5.16B, V5.16B, V10.16B
+	and	V6.16B, V6.16B, V10.16B
+	and	V7.16B, V7.16B, V10.16B
+	st1	{V0.8H, V1.8H, V2.8H, V3.8H}, [x16], #0x40
+	st1	{V4.8H, V5.8H, V6.8H, V7.8H}, [x16], #0x40
+	ldp	D8, D9, [x29, #16]
+	ldp	D10, D11, [x29, #32]
+	ldp	x29, x30, [sp], #48
+	ret
+	ENDP
+	ENDIF
+	IF :DEF:WOLFSSL_HAVE_FRODOKEM
+; 	.arch	armv8-a+crypto
+	AREA	|.text|, CODE, READONLY
+	ALIGN	4
+	EXPORT	frodokem_gen_a_rows_aes_arm64
+frodokem_gen_a_rows_aes_arm64 PROC
+	stp	x29, x30, [sp, #-80]!
+	add	x29, sp, #0
+	stp	D8, D9, [x29, #16]
+	stp	D10, D11, [x29, #32]
+	stp	D12, D13, [x29, #48]
+	stp	D14, D15, [x29, #64]
+	ld1	{V0.16B, V1.16B, V2.16B, V3.16B}, [x2], #0x40
+	ld1	{V4.16B, V5.16B, V6.16B, V7.16B}, [x2], #0x40
+	ld1	{V8.16B, V9.16B, V10.16B}, [x2], #48
+	dup	V19.8H, w6
+	sxtw	x13, w3
+	sxtw	x7, w5
+	lsl	x7, x7, #1
+	sxtw	x8, w4
+	mul	x8, x8, x7
+	movz	w14, #8, lsl 16
+	mov	x9, x0
+L_frodokem_gen_a_rows_aes_arm64_row
+	mov	w11, w13
+	add	x10, x9, x7
+L_frodokem_gen_a_rows_aes_arm64_blk
+	add	x12, x11, x14
+	stp	x11, xzr, [x9], #16
+	stp	x12, xzr, [x9], #16
+	add	x11, x12, x14
+	cmp	x9, x10
+	blt	L_frodokem_gen_a_rows_aes_arm64_blk
+	add	x13, x13, #1
+	add	x10, x0, x8
+	cmp	x9, x10
+	blt	L_frodokem_gen_a_rows_aes_arm64_row
+	cmp	x8, #0x80
+	blt	L_frodokem_gen_a_rows_aes_arm64_tail
+L_frodokem_gen_a_rows_aes_arm64_aes
+	ld1	{V11.16B, V12.16B, V13.16B, V14.16B}, [x0], #0x40
+	ld1	{V15.16B, V16.16B, V17.16B, V18.16B}, [x0], #0x40
+	aese	V11.16B, V0.16B
+	aesmc	V11.16B, V11.16B
+	aese	V12.16B, V0.16B
+	aesmc	V12.16B, V12.16B
+	aese	V13.16B, V0.16B
+	aesmc	V13.16B, V13.16B
+	aese	V14.16B, V0.16B
+	aesmc	V14.16B, V14.16B
+	aese	V15.16B, V0.16B
+	aesmc	V15.16B, V15.16B
+	aese	V16.16B, V0.16B
+	aesmc	V16.16B, V16.16B
+	aese	V17.16B, V0.16B
+	aesmc	V17.16B, V17.16B
+	aese	V18.16B, V0.16B
+	aesmc	V18.16B, V18.16B
+	aese	V11.16B, V1.16B
+	aesmc	V11.16B, V11.16B
+	aese	V12.16B, V1.16B
+	aesmc	V12.16B, V12.16B
+	aese	V13.16B, V1.16B
+	aesmc	V13.16B, V13.16B
+	aese	V14.16B, V1.16B
+	aesmc	V14.16B, V14.16B
+	aese	V15.16B, V1.16B
+	aesmc	V15.16B, V15.16B
+	aese	V16.16B, V1.16B
+	aesmc	V16.16B, V16.16B
+	aese	V17.16B, V1.16B
+	aesmc	V17.16B, V17.16B
+	aese	V18.16B, V1.16B
+	aesmc	V18.16B, V18.16B
+	aese	V11.16B, V2.16B
+	aesmc	V11.16B, V11.16B
+	aese	V12.16B, V2.16B
+	aesmc	V12.16B, V12.16B
+	aese	V13.16B, V2.16B
+	aesmc	V13.16B, V13.16B
+	aese	V14.16B, V2.16B
+	aesmc	V14.16B, V14.16B
+	aese	V15.16B, V2.16B
+	aesmc	V15.16B, V15.16B
+	aese	V16.16B, V2.16B
+	aesmc	V16.16B, V16.16B
+	aese	V17.16B, V2.16B
+	aesmc	V17.16B, V17.16B
+	aese	V18.16B, V2.16B
+	aesmc	V18.16B, V18.16B
+	aese	V11.16B, V3.16B
+	aesmc	V11.16B, V11.16B
+	aese	V12.16B, V3.16B
+	aesmc	V12.16B, V12.16B
+	aese	V13.16B, V3.16B
+	aesmc	V13.16B, V13.16B
+	aese	V14.16B, V3.16B
+	aesmc	V14.16B, V14.16B
+	aese	V15.16B, V3.16B
+	aesmc	V15.16B, V15.16B
+	aese	V16.16B, V3.16B
+	aesmc	V16.16B, V16.16B
+	aese	V17.16B, V3.16B
+	aesmc	V17.16B, V17.16B
+	aese	V18.16B, V3.16B
+	aesmc	V18.16B, V18.16B
+	aese	V11.16B, V4.16B
+	aesmc	V11.16B, V11.16B
+	aese	V12.16B, V4.16B
+	aesmc	V12.16B, V12.16B
+	aese	V13.16B, V4.16B
+	aesmc	V13.16B, V13.16B
+	aese	V14.16B, V4.16B
+	aesmc	V14.16B, V14.16B
+	aese	V15.16B, V4.16B
+	aesmc	V15.16B, V15.16B
+	aese	V16.16B, V4.16B
+	aesmc	V16.16B, V16.16B
+	aese	V17.16B, V4.16B
+	aesmc	V17.16B, V17.16B
+	aese	V18.16B, V4.16B
+	aesmc	V18.16B, V18.16B
+	aese	V11.16B, V5.16B
+	aesmc	V11.16B, V11.16B
+	aese	V12.16B, V5.16B
+	aesmc	V12.16B, V12.16B
+	aese	V13.16B, V5.16B
+	aesmc	V13.16B, V13.16B
+	aese	V14.16B, V5.16B
+	aesmc	V14.16B, V14.16B
+	aese	V15.16B, V5.16B
+	aesmc	V15.16B, V15.16B
+	aese	V16.16B, V5.16B
+	aesmc	V16.16B, V16.16B
+	aese	V17.16B, V5.16B
+	aesmc	V17.16B, V17.16B
+	aese	V18.16B, V5.16B
+	aesmc	V18.16B, V18.16B
+	aese	V11.16B, V6.16B
+	aesmc	V11.16B, V11.16B
+	aese	V12.16B, V6.16B
+	aesmc	V12.16B, V12.16B
+	aese	V13.16B, V6.16B
+	aesmc	V13.16B, V13.16B
+	aese	V14.16B, V6.16B
+	aesmc	V14.16B, V14.16B
+	aese	V15.16B, V6.16B
+	aesmc	V15.16B, V15.16B
+	aese	V16.16B, V6.16B
+	aesmc	V16.16B, V16.16B
+	aese	V17.16B, V6.16B
+	aesmc	V17.16B, V17.16B
+	aese	V18.16B, V6.16B
+	aesmc	V18.16B, V18.16B
+	aese	V11.16B, V7.16B
+	aesmc	V11.16B, V11.16B
+	aese	V12.16B, V7.16B
+	aesmc	V12.16B, V12.16B
+	aese	V13.16B, V7.16B
+	aesmc	V13.16B, V13.16B
+	aese	V14.16B, V7.16B
+	aesmc	V14.16B, V14.16B
+	aese	V15.16B, V7.16B
+	aesmc	V15.16B, V15.16B
+	aese	V16.16B, V7.16B
+	aesmc	V16.16B, V16.16B
+	aese	V17.16B, V7.16B
+	aesmc	V17.16B, V17.16B
+	aese	V18.16B, V7.16B
+	aesmc	V18.16B, V18.16B
+	aese	V11.16B, V8.16B
+	aesmc	V11.16B, V11.16B
+	aese	V12.16B, V8.16B
+	aesmc	V12.16B, V12.16B
+	aese	V13.16B, V8.16B
+	aesmc	V13.16B, V13.16B
+	aese	V14.16B, V8.16B
+	aesmc	V14.16B, V14.16B
+	aese	V15.16B, V8.16B
+	aesmc	V15.16B, V15.16B
+	aese	V16.16B, V8.16B
+	aesmc	V16.16B, V16.16B
+	aese	V17.16B, V8.16B
+	aesmc	V17.16B, V17.16B
+	aese	V18.16B, V8.16B
+	aesmc	V18.16B, V18.16B
+	aese	V11.16B, V9.16B
+	aese	V12.16B, V9.16B
+	aese	V13.16B, V9.16B
+	aese	V14.16B, V9.16B
+	aese	V15.16B, V9.16B
+	aese	V16.16B, V9.16B
+	aese	V17.16B, V9.16B
+	aese	V18.16B, V9.16B
+	eor	V11.16B, V11.16B, V10.16B
+	eor	V12.16B, V12.16B, V10.16B
+	eor	V13.16B, V13.16B, V10.16B
+	eor	V14.16B, V14.16B, V10.16B
+	eor	V15.16B, V15.16B, V10.16B
+	eor	V16.16B, V16.16B, V10.16B
+	eor	V17.16B, V17.16B, V10.16B
+	eor	V18.16B, V18.16B, V10.16B
+	and	V11.16B, V11.16B, V19.16B
+	and	V12.16B, V12.16B, V19.16B
+	and	V13.16B, V13.16B, V19.16B
+	and	V14.16B, V14.16B, V19.16B
+	and	V15.16B, V15.16B, V19.16B
+	and	V16.16B, V16.16B, V19.16B
+	and	V17.16B, V17.16B, V19.16B
+	and	V18.16B, V18.16B, V19.16B
+	st1	{V11.16B, V12.16B, V13.16B, V14.16B}, [x1], #0x40
+	st1	{V15.16B, V16.16B, V17.16B, V18.16B}, [x1], #0x40
+	subs	x8, x8, #0x80
+	cmp	x8, #0x80
+	bge	L_frodokem_gen_a_rows_aes_arm64_aes
+L_frodokem_gen_a_rows_aes_arm64_tail
+	cmp	x8, #0
+	beq	L_frodokem_gen_a_rows_aes_arm64_done
+	ld1	{V11.16B, V12.16B, V13.16B, V14.16B}, [x0], #0x40
+	aese	V11.16B, V0.16B
+	aesmc	V11.16B, V11.16B
+	aese	V12.16B, V0.16B
+	aesmc	V12.16B, V12.16B
+	aese	V13.16B, V0.16B
+	aesmc	V13.16B, V13.16B
+	aese	V14.16B, V0.16B
+	aesmc	V14.16B, V14.16B
+	aese	V11.16B, V1.16B
+	aesmc	V11.16B, V11.16B
+	aese	V12.16B, V1.16B
+	aesmc	V12.16B, V12.16B
+	aese	V13.16B, V1.16B
+	aesmc	V13.16B, V13.16B
+	aese	V14.16B, V1.16B
+	aesmc	V14.16B, V14.16B
+	aese	V11.16B, V2.16B
+	aesmc	V11.16B, V11.16B
+	aese	V12.16B, V2.16B
+	aesmc	V12.16B, V12.16B
+	aese	V13.16B, V2.16B
+	aesmc	V13.16B, V13.16B
+	aese	V14.16B, V2.16B
+	aesmc	V14.16B, V14.16B
+	aese	V11.16B, V3.16B
+	aesmc	V11.16B, V11.16B
+	aese	V12.16B, V3.16B
+	aesmc	V12.16B, V12.16B
+	aese	V13.16B, V3.16B
+	aesmc	V13.16B, V13.16B
+	aese	V14.16B, V3.16B
+	aesmc	V14.16B, V14.16B
+	aese	V11.16B, V4.16B
+	aesmc	V11.16B, V11.16B
+	aese	V12.16B, V4.16B
+	aesmc	V12.16B, V12.16B
+	aese	V13.16B, V4.16B
+	aesmc	V13.16B, V13.16B
+	aese	V14.16B, V4.16B
+	aesmc	V14.16B, V14.16B
+	aese	V11.16B, V5.16B
+	aesmc	V11.16B, V11.16B
+	aese	V12.16B, V5.16B
+	aesmc	V12.16B, V12.16B
+	aese	V13.16B, V5.16B
+	aesmc	V13.16B, V13.16B
+	aese	V14.16B, V5.16B
+	aesmc	V14.16B, V14.16B
+	aese	V11.16B, V6.16B
+	aesmc	V11.16B, V11.16B
+	aese	V12.16B, V6.16B
+	aesmc	V12.16B, V12.16B
+	aese	V13.16B, V6.16B
+	aesmc	V13.16B, V13.16B
+	aese	V14.16B, V6.16B
+	aesmc	V14.16B, V14.16B
+	aese	V11.16B, V7.16B
+	aesmc	V11.16B, V11.16B
+	aese	V12.16B, V7.16B
+	aesmc	V12.16B, V12.16B
+	aese	V13.16B, V7.16B
+	aesmc	V13.16B, V13.16B
+	aese	V14.16B, V7.16B
+	aesmc	V14.16B, V14.16B
+	aese	V11.16B, V8.16B
+	aesmc	V11.16B, V11.16B
+	aese	V12.16B, V8.16B
+	aesmc	V12.16B, V12.16B
+	aese	V13.16B, V8.16B
+	aesmc	V13.16B, V13.16B
+	aese	V14.16B, V8.16B
+	aesmc	V14.16B, V14.16B
+	aese	V11.16B, V9.16B
+	aese	V12.16B, V9.16B
+	aese	V13.16B, V9.16B
+	aese	V14.16B, V9.16B
+	eor	V11.16B, V11.16B, V10.16B
+	eor	V12.16B, V12.16B, V10.16B
+	eor	V13.16B, V13.16B, V10.16B
+	eor	V14.16B, V14.16B, V10.16B
+	and	V11.16B, V11.16B, V19.16B
+	and	V12.16B, V12.16B, V19.16B
+	and	V13.16B, V13.16B, V19.16B
+	and	V14.16B, V14.16B, V19.16B
+	st1	{V11.16B, V12.16B, V13.16B, V14.16B}, [x1], #0x40
+L_frodokem_gen_a_rows_aes_arm64_done
+	ldp	D8, D9, [x29, #16]
+	ldp	D10, D11, [x29, #32]
+	ldp	D12, D13, [x29, #48]
+	ldp	D14, D15, [x29, #64]
+	ldp	x29, x30, [sp], #0x50
+	ret
+	ENDP
+	ENDIF
+	END
diff --git a/wolfcrypt/src/port/arm/armv8-frodokem-asm_c.c b/wolfcrypt/src/port/arm/armv8-frodokem-asm_c.c
new file mode 100644
index 0000000000..7074cd9f8d
--- /dev/null
+++ b/wolfcrypt/src/port/arm/armv8-frodokem-asm_c.c
@@ -0,0 +1,1315 @@
+/* armv8-frodokem-asm
+ *
+ * Copyright (C) 2006-2026 wolfSSL Inc.
+ *
+ * This file is part of wolfSSL.
+ *
+ * wolfSSL is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * wolfSSL is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335, USA
+ */
+
+#include <wolfssl/wolfcrypt/libwolfssl_sources_asm.h>
+#include <wolfssl/wolfcrypt/error-crypt.h>
+
+/* Generated using (from wolfssl):
+ *   cd ../scripts
+ *   ruby ./frodokem/frodokem.rb arm64 \
+ *       ../wolfssl/wolfcrypt/src/port/arm/armv8-frodokem-asm.c
+ */
+#ifdef WOLFSSL_ARMASM
+#ifdef __aarch64__
+#ifdef WOLFSSL_ARMASM_INLINE
+#include <wolfssl/wolfcrypt/wc_frodokem_mat.h>
+
+#ifdef WOLFSSL_HAVE_FRODOKEM
+XALIGNED(16) static const word64 L_sha3_aarch64_r[] = {
+    0x0000000000000001UL, 0x0000000000008082UL,
+    0x800000000000808aUL, 0x8000000080008000UL,
+    0x000000000000808bUL, 0x0000000080000001UL,
+    0x8000000080008081UL, 0x8000000000008009UL,
+    0x000000000000008aUL, 0x0000000000000088UL,
+    0x0000000080008009UL, 0x000000008000000aUL,
+    0x000000008000808bUL, 0x800000000000008bUL,
+    0x8000000000008089UL, 0x8000000000008003UL,
+    0x8000000000008002UL, 0x8000000000000080UL,
+    0x000000000000800aUL, 0x800000008000000aUL,
+    0x8000000080008081UL, 0x8000000000008080UL,
+    0x0000000080000001UL, 0x8000000080008008UL,
+};
+
+void frodokem_sha3_x2_neon(word64* state)
+{
+    const word64* r = L_sha3_aarch64_r;
+    __asm__ __volatile__ (
+        "ld4	{v0.d, v1.d, v2.d, v3.d}[0], [%x[state]], #32\n\t"
+        "ld4	{v4.d, v5.d, v6.d, v7.d}[0], [%x[state]], #32\n\t"
+        "ld4	{v8.d, v9.d, v10.d, v11.d}[0], [%x[state]], #32\n\t"
+        "ld4	{v12.d, v13.d, v14.d, v15.d}[0], [%x[state]], #32\n\t"
+        "ld4	{v16.d, v17.d, v18.d, v19.d}[0], [%x[state]], #32\n\t"
+        "ld4	{v20.d, v21.d, v22.d, v23.d}[0], [%x[state]], #32\n\t"
+        "ld1	{v24.d}[0], [%x[state]]\n\t"
+        "add	%x[state], %x[state], #8\n\t"
+        "ld4	{v0.d, v1.d, v2.d, v3.d}[1], [%x[state]], #32\n\t"
+        "ld4	{v4.d, v5.d, v6.d, v7.d}[1], [%x[state]], #32\n\t"
+        "ld4	{v8.d, v9.d, v10.d, v11.d}[1], [%x[state]], #32\n\t"
+        "ld4	{v12.d, v13.d, v14.d, v15.d}[1], [%x[state]], #32\n\t"
+        "ld4	{v16.d, v17.d, v18.d, v19.d}[1], [%x[state]], #32\n\t"
+        "ld4	{v20.d, v21.d, v22.d, v23.d}[1], [%x[state]], #32\n\t"
+        "ld1	{v24.d}[1], [%x[state]]\n\t"
+        "add	%x[state], %x[state], #8\n\t"
+        "sub	%x[state], %x[state], #0x190\n\t"
+        "mov	x2, #24\n\t"
+        "\n"
+    "L_frodokem_sha3_x2_neon_begin_%=:\n\t"
+        /* Col Mix NEON */
+        "eor	v30.16b, v4.16b, v9.16b\n\t"
+        "eor	v27.16b, v1.16b, v6.16b\n\t"
+        "eor	v30.16b, v30.16b, v14.16b\n\t"
+        "eor	v27.16b, v27.16b, v11.16b\n\t"
+        "eor	v30.16b, v30.16b, v19.16b\n\t"
+        "eor	v27.16b, v27.16b, v16.16b\n\t"
+        "eor	v30.16b, v30.16b, v24.16b\n\t"
+        "eor	v27.16b, v27.16b, v21.16b\n\t"
+        "ushr	v25.2d, v27.2d, #63\n\t"
+        "sli	v25.2d, v27.2d, #1\n\t"
+        "eor	v25.16b, v25.16b, v30.16b\n\t"
+        "eor	v31.16b, v0.16b, v5.16b\n\t"
+        "eor	v28.16b, v2.16b, v7.16b\n\t"
+        "eor	v31.16b, v31.16b, v10.16b\n\t"
+        "eor	v28.16b, v28.16b, v12.16b\n\t"
+        "eor	v31.16b, v31.16b, v15.16b\n\t"
+        "eor	v28.16b, v28.16b, v17.16b\n\t"
+        "eor	v31.16b, v31.16b, v20.16b\n\t"
+        "eor	v28.16b, v28.16b, v22.16b\n\t"
+        "ushr	v29.2d, v30.2d, #63\n\t"
+        "ushr	v26.2d, v28.2d, #63\n\t"
+        "sli	v29.2d, v30.2d, #1\n\t"
+        "sli	v26.2d, v28.2d, #1\n\t"
+        "eor	v28.16b, v28.16b, v29.16b\n\t"
+        "eor	v29.16b, v3.16b, v8.16b\n\t"
+        "eor	v26.16b, v26.16b, v31.16b\n\t"
+        "eor	v29.16b, v29.16b, v13.16b\n\t"
+        "eor	v29.16b, v29.16b, v18.16b\n\t"
+        "eor	v29.16b, v29.16b, v23.16b\n\t"
+        "ushr	v30.2d, v29.2d, #63\n\t"
+        "sli	v30.2d, v29.2d, #1\n\t"
+        "eor	v27.16b, v27.16b, v30.16b\n\t"
+        "ushr	v30.2d, v31.2d, #63\n\t"
+        "sli	v30.2d, v31.2d, #1\n\t"
+        "eor	v29.16b, v29.16b, v30.16b\n\t"
+        /* Swap Rotate NEON */
+        "eor	v0.16b, v0.16b, v25.16b\n\t"
+        "eor	v31.16b, v1.16b, v26.16b\n\t"
+        "eor	v6.16b, v6.16b, v26.16b\n\t"
+        "ushr	v30.2d, v31.2d, #63\n\t"
+        "ushr	v1.2d, v6.2d, #20\n\t"
+        "sli	v30.2d, v31.2d, #1\n\t"
+        "sli	v1.2d, v6.2d, #44\n\t"
+        "eor	v31.16b, v9.16b, v29.16b\n\t"
+        "eor	v22.16b, v22.16b, v27.16b\n\t"
+        "ushr	v6.2d, v31.2d, #44\n\t"
+        "ushr	v9.2d, v22.2d, #3\n\t"
+        "sli	v6.2d, v31.2d, #20\n\t"
+        "sli	v9.2d, v22.2d, #61\n\t"
+        "eor	v31.16b, v14.16b, v29.16b\n\t"
+        "eor	v20.16b, v20.16b, v25.16b\n\t"
+        "ushr	v22.2d, v31.2d, #25\n\t"
+        "ushr	v14.2d, v20.2d, #46\n\t"
+        "sli	v22.2d, v31.2d, #39\n\t"
+        "sli	v14.2d, v20.2d, #18\n\t"
+        "eor	v31.16b, v2.16b, v27.16b\n\t"
+        "eor	v12.16b, v12.16b, v27.16b\n\t"
+        "ushr	v20.2d, v31.2d, #2\n\t"
+        "ushr	v2.2d, v12.2d, #21\n\t"
+        "sli	v20.2d, v31.2d, #62\n\t"
+        "sli	v2.2d, v12.2d, #43\n\t"
+        "eor	v31.16b, v13.16b, v28.16b\n\t"
+        "eor	v19.16b, v19.16b, v29.16b\n\t"
+        "ushr	v12.2d, v31.2d, #39\n\t"
+        "ushr	v13.2d, v19.2d, #56\n\t"
+        "sli	v12.2d, v31.2d, #25\n\t"
+        "sli	v13.2d, v19.2d, #8\n\t"
+        "eor	v31.16b, v23.16b, v28.16b\n\t"
+        "eor	v15.16b, v15.16b, v25.16b\n\t"
+        "ushr	v19.2d, v31.2d, #8\n\t"
+        "ushr	v23.2d, v15.2d, #23\n\t"
+        "sli	v19.2d, v31.2d, #56\n\t"
+        "sli	v23.2d, v15.2d, #41\n\t"
+        "eor	v31.16b, v4.16b, v29.16b\n\t"
+        "eor	v24.16b, v24.16b, v29.16b\n\t"
+        "ushr	v15.2d, v31.2d, #37\n\t"
+        "ushr	v4.2d, v24.2d, #50\n\t"
+        "sli	v15.2d, v31.2d, #27\n\t"
+        "sli	v4.2d, v24.2d, #14\n\t"
+        "eor	v31.16b, v21.16b, v26.16b\n\t"
+        "eor	v8.16b, v8.16b, v28.16b\n\t"
+        "ushr	v24.2d, v31.2d, #62\n\t"
+        "ushr	v21.2d, v8.2d, #9\n\t"
+        "sli	v24.2d, v31.2d, #2\n\t"
+        "sli	v21.2d, v8.2d, #55\n\t"
+        "eor	v31.16b, v16.16b, v26.16b\n\t"
+        "eor	v5.16b, v5.16b, v25.16b\n\t"
+        "ushr	v8.2d, v31.2d, #19\n\t"
+        "ushr	v16.2d, v5.2d, #28\n\t"
+        "sli	v8.2d, v31.2d, #45\n\t"
+        "sli	v16.2d, v5.2d, #36\n\t"
+        "eor	v31.16b, v3.16b, v28.16b\n\t"
+        "eor	v18.16b, v18.16b, v28.16b\n\t"
+        "ushr	v5.2d, v31.2d, #36\n\t"
+        "ushr	v3.2d, v18.2d, #43\n\t"
+        "sli	v5.2d, v31.2d, #28\n\t"
+        "sli	v3.2d, v18.2d, #21\n\t"
+        "eor	v31.16b, v17.16b, v27.16b\n\t"
+        "eor	v11.16b, v11.16b, v26.16b\n\t"
+        "ushr	v18.2d, v31.2d, #49\n\t"
+        "ushr	v17.2d, v11.2d, #54\n\t"
+        "sli	v18.2d, v31.2d, #15\n\t"
+        "sli	v17.2d, v11.2d, #10\n\t"
+        "eor	v31.16b, v7.16b, v27.16b\n\t"
+        "eor	v10.16b, v10.16b, v25.16b\n\t"
+        "ushr	v11.2d, v31.2d, #58\n\t"
+        "ushr	v7.2d, v10.2d, #61\n\t"
+        "sli	v11.2d, v31.2d, #6\n\t"
+        "sli	v7.2d, v10.2d, #3\n\t"
+        /* Row Mix NEON */
+        "bic	v25.16b, v2.16b, v1.16b\n\t"
+        "bic	v26.16b, v3.16b, v2.16b\n\t"
+        "bic	v27.16b, v4.16b, v3.16b\n\t"
+        "bic	v28.16b, v0.16b, v4.16b\n\t"
+        "bic	v29.16b, v1.16b, v0.16b\n\t"
+        "eor	v0.16b, v0.16b, v25.16b\n\t"
+        "eor	v1.16b, v1.16b, v26.16b\n\t"
+        "eor	v2.16b, v2.16b, v27.16b\n\t"
+        "eor	v3.16b, v3.16b, v28.16b\n\t"
+        "eor	v4.16b, v4.16b, v29.16b\n\t"
+        "bic	v25.16b, v7.16b, v6.16b\n\t"
+        "bic	v26.16b, v8.16b, v7.16b\n\t"
+        "bic	v27.16b, v9.16b, v8.16b\n\t"
+        "bic	v28.16b, v5.16b, v9.16b\n\t"
+        "bic	v29.16b, v6.16b, v5.16b\n\t"
+        "eor	v5.16b, v5.16b, v25.16b\n\t"
+        "eor	v6.16b, v6.16b, v26.16b\n\t"
+        "eor	v7.16b, v7.16b, v27.16b\n\t"
+        "eor	v8.16b, v8.16b, v28.16b\n\t"
+        "eor	v9.16b, v9.16b, v29.16b\n\t"
+        "bic	v25.16b, v12.16b, v11.16b\n\t"
+        "bic	v26.16b, v13.16b, v12.16b\n\t"
+        "bic	v27.16b, v14.16b, v13.16b\n\t"
+        "bic	v28.16b, v30.16b, v14.16b\n\t"
+        "bic	v29.16b, v11.16b, v30.16b\n\t"
+        "eor	v10.16b, v30.16b, v25.16b\n\t"
+        "eor	v11.16b, v11.16b, v26.16b\n\t"
+        "eor	v12.16b, v12.16b, v27.16b\n\t"
+        "eor	v13.16b, v13.16b, v28.16b\n\t"
+        "eor	v14.16b, v14.16b, v29.16b\n\t"
+        "bic	v25.16b, v17.16b, v16.16b\n\t"
+        "bic	v26.16b, v18.16b, v17.16b\n\t"
+        "bic	v27.16b, v19.16b, v18.16b\n\t"
+        "bic	v28.16b, v15.16b, v19.16b\n\t"
+        "bic	v29.16b, v16.16b, v15.16b\n\t"
+        "eor	v15.16b, v15.16b, v25.16b\n\t"
+        "eor	v16.16b, v16.16b, v26.16b\n\t"
+        "eor	v17.16b, v17.16b, v27.16b\n\t"
+        "eor	v18.16b, v18.16b, v28.16b\n\t"
+        "eor	v19.16b, v19.16b, v29.16b\n\t"
+        "bic	v25.16b, v22.16b, v21.16b\n\t"
+        "bic	v26.16b, v23.16b, v22.16b\n\t"
+        "bic	v27.16b, v24.16b, v23.16b\n\t"
+        "bic	v28.16b, v20.16b, v24.16b\n\t"
+        "bic	v29.16b, v21.16b, v20.16b\n\t"
+        "eor	v20.16b, v20.16b, v25.16b\n\t"
+        "eor	v21.16b, v21.16b, v26.16b\n\t"
+        "eor	v22.16b, v22.16b, v27.16b\n\t"
+        "eor	v23.16b, v23.16b, v28.16b\n\t"
+        "eor	v24.16b, v24.16b, v29.16b\n\t"
+        "ldr	x3, [%[r]], #8\n\t"
+        "subs	x2, x2, #1\n\t"
+        "mov	v30.d[0], x3\n\t"
+        "mov	v30.d[1], x3\n\t"
+        "eor	v0.16b, v0.16b, v30.16b\n\t"
+        "b.ne	L_frodokem_sha3_x2_neon_begin_%=\n\t"
+        "st4	{v0.d, v1.d, v2.d, v3.d}[0], [%x[state]], #32\n\t"
+        "st4	{v4.d, v5.d, v6.d, v7.d}[0], [%x[state]], #32\n\t"
+        "st4	{v8.d, v9.d, v10.d, v11.d}[0], [%x[state]], #32\n\t"
+        "st4	{v12.d, v13.d, v14.d, v15.d}[0], [%x[state]], #32\n\t"
+        "st4	{v16.d, v17.d, v18.d, v19.d}[0], [%x[state]], #32\n\t"
+        "st4	{v20.d, v21.d, v22.d, v23.d}[0], [%x[state]], #32\n\t"
+        "st1	{v24.d}[0], [%x[state]]\n\t"
+        "add	%x[state], %x[state], #8\n\t"
+        "st4	{v0.d, v1.d, v2.d, v3.d}[1], [%x[state]], #32\n\t"
+        "st4	{v4.d, v5.d, v6.d, v7.d}[1], [%x[state]], #32\n\t"
+        "st4	{v8.d, v9.d, v10.d, v11.d}[1], [%x[state]], #32\n\t"
+        "st4	{v12.d, v13.d, v14.d, v15.d}[1], [%x[state]], #32\n\t"
+        "st4	{v16.d, v17.d, v18.d, v19.d}[1], [%x[state]], #32\n\t"
+        "st4	{v20.d, v21.d, v22.d, v23.d}[1], [%x[state]], #32\n\t"
+        "st1	{v24.d}[1], [%x[state]]\n\t"
+        "add	%x[state], %x[state], #8\n\t"
+        : [state] "+r" (state)
+        : [r] "r" (r)
+        : "memory", "cc", "x2", "x3", "v0", "v1", "v2", "v3", "v4", "v5", "v6",
+            "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16",
+            "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25",
+            "v26", "v27", "v28", "v29", "v30", "v31"
+    );
+}
+
+#ifdef WOLFSSL_ARMASM_CRYPTO_SHA3
+    "	.arch_extension	sha3\n\t"
+void frodokem_sha3_x2_crypto(word64* state)
+{
+    const word64* r = L_sha3_aarch64_r;
+    __asm__ __volatile__ (
+        "ld4	{v0.d, v1.d, v2.d, v3.d}[0], [%x[state]], #32\n\t"
+        "ld4	{v4.d, v5.d, v6.d, v7.d}[0], [%x[state]], #32\n\t"
+        "ld4	{v8.d, v9.d, v10.d, v11.d}[0], [%x[state]], #32\n\t"
+        "ld4	{v12.d, v13.d, v14.d, v15.d}[0], [%x[state]], #32\n\t"
+        "ld4	{v16.d, v17.d, v18.d, v19.d}[0], [%x[state]], #32\n\t"
+        "ld4	{v20.d, v21.d, v22.d, v23.d}[0], [%x[state]], #32\n\t"
+        "ld1	{v24.d}[0], [%x[state]]\n\t"
+        "add	%x[state], %x[state], #8\n\t"
+        "ld4	{v0.d, v1.d, v2.d, v3.d}[1], [%x[state]], #32\n\t"
+        "ld4	{v4.d, v5.d, v6.d, v7.d}[1], [%x[state]], #32\n\t"
+        "ld4	{v8.d, v9.d, v10.d, v11.d}[1], [%x[state]], #32\n\t"
+        "ld4	{v12.d, v13.d, v14.d, v15.d}[1], [%x[state]], #32\n\t"
+        "ld4	{v16.d, v17.d, v18.d, v19.d}[1], [%x[state]], #32\n\t"
+        "ld4	{v20.d, v21.d, v22.d, v23.d}[1], [%x[state]], #32\n\t"
+        "ld1	{v24.d}[1], [%x[state]]\n\t"
+        "add	%x[state], %x[state], #8\n\t"
+        "sub	%x[state], %x[state], #0x190\n\t"
+        "mov	x2, #24\n\t"
+        "\n"
+    "L_frodokem_sha3_x2_crypto_begin_%=:\n\t"
+        /* Col Mix */
+        "eor3	v31.16b, v0.16b, v5.16b, v10.16b\n\t"
+        "eor3	v27.16b, v1.16b, v6.16b, v11.16b\n\t"
+        "eor3	v28.16b, v2.16b, v7.16b, v12.16b\n\t"
+        "eor3	v29.16b, v3.16b, v8.16b, v13.16b\n\t"
+        "eor3	v30.16b, v4.16b, v9.16b, v14.16b\n\t"
+        "eor3	v31.16b, v31.16b, v15.16b, v20.16b\n\t"
+        "eor3	v27.16b, v27.16b, v16.16b, v21.16b\n\t"
+        "eor3	v28.16b, v28.16b, v17.16b, v22.16b\n\t"
+        "eor3	v29.16b, v29.16b, v18.16b, v23.16b\n\t"
+        "eor3	v30.16b, v30.16b, v19.16b, v24.16b\n\t"
+        "rax1	v25.2d, v30.2d, v27.2d\n\t"
+        "rax1	v26.2d, v31.2d, v28.2d\n\t"
+        "rax1	v27.2d, v27.2d, v29.2d\n\t"
+        "rax1	v28.2d, v28.2d, v30.2d\n\t"
+        "rax1	v29.2d, v29.2d, v31.2d\n\t"
+        "eor	v0.16b, v0.16b, v25.16b\n\t"
+        "xar	v30.2d, v1.2d, v26.2d, #63\n\t"
+        "xar	v1.2d, v6.2d, v26.2d, #20\n\t"
+        "xar	v6.2d, v9.2d, v29.2d, #44\n\t"
+        "xar	v9.2d, v22.2d, v27.2d, #3\n\t"
+        "xar	v22.2d, v14.2d, v29.2d, #25\n\t"
+        "xar	v14.2d, v20.2d, v25.2d, #46\n\t"
+        "xar	v20.2d, v2.2d, v27.2d, #2\n\t"
+        "xar	v2.2d, v12.2d, v27.2d, #21\n\t"
+        "xar	v12.2d, v13.2d, v28.2d, #39\n\t"
+        "xar	v13.2d, v19.2d, v29.2d, #56\n\t"
+        "xar	v19.2d, v23.2d, v28.2d, #8\n\t"
+        "xar	v23.2d, v15.2d, v25.2d, #23\n\t"
+        "xar	v15.2d, v4.2d, v29.2d, #37\n\t"
+        "xar	v4.2d, v24.2d, v29.2d, #50\n\t"
+        "xar	v24.2d, v21.2d, v26.2d, #62\n\t"
+        "xar	v21.2d, v8.2d, v28.2d, #9\n\t"
+        "xar	v8.2d, v16.2d, v26.2d, #19\n\t"
+        "xar	v16.2d, v5.2d, v25.2d, #28\n\t"
+        "xar	v5.2d, v3.2d, v28.2d, #36\n\t"
+        "xar	v3.2d, v18.2d, v28.2d, #43\n\t"
+        "xar	v18.2d, v17.2d, v27.2d, #49\n\t"
+        "xar	v17.2d, v11.2d, v26.2d, #54\n\t"
+        "xar	v11.2d, v7.2d, v27.2d, #58\n\t"
+        "xar	v7.2d, v10.2d, v25.2d, #61\n\t"
+        /* Row Mix */
+        "mov	v25.16b, v0.16b\n\t"
+        "mov	v26.16b, v1.16b\n\t"
+        "bcax	v0.16b, v25.16b, v2.16b, v26.16b\n\t"
+        "bcax	v1.16b, v26.16b, v3.16b, v2.16b\n\t"
+        "bcax	v2.16b, v2.16b, v4.16b, v3.16b\n\t"
+        "bcax	v3.16b, v3.16b, v25.16b, v4.16b\n\t"
+        "bcax	v4.16b, v4.16b, v26.16b, v25.16b\n\t"
+        "mov	v25.16b, v5.16b\n\t"
+        "mov	v26.16b, v6.16b\n\t"
+        "bcax	v5.16b, v25.16b, v7.16b, v26.16b\n\t"
+        "bcax	v6.16b, v26.16b, v8.16b, v7.16b\n\t"
+        "bcax	v7.16b, v7.16b, v9.16b, v8.16b\n\t"
+        "bcax	v8.16b, v8.16b, v25.16b, v9.16b\n\t"
+        "bcax	v9.16b, v9.16b, v26.16b, v25.16b\n\t"
+        "mov	v26.16b, v11.16b\n\t"
+        "bcax	v10.16b, v30.16b, v12.16b, v26.16b\n\t"
+        "bcax	v11.16b, v26.16b, v13.16b, v12.16b\n\t"
+        "bcax	v12.16b, v12.16b, v14.16b, v13.16b\n\t"
+        "bcax	v13.16b, v13.16b, v30.16b, v14.16b\n\t"
+        "bcax	v14.16b, v14.16b, v26.16b, v30.16b\n\t"
+        "mov	v25.16b, v15.16b\n\t"
+        "mov	v26.16b, v16.16b\n\t"
+        "bcax	v15.16b, v25.16b, v17.16b, v26.16b\n\t"
+        "bcax	v16.16b, v26.16b, v18.16b, v17.16b\n\t"
+        "bcax	v17.16b, v17.16b, v19.16b, v18.16b\n\t"
+        "bcax	v18.16b, v18.16b, v25.16b, v19.16b\n\t"
+        "bcax	v19.16b, v19.16b, v26.16b, v25.16b\n\t"
+        "mov	v25.16b, v20.16b\n\t"
+        "mov	v26.16b, v21.16b\n\t"
+        "bcax	v20.16b, v25.16b, v22.16b, v26.16b\n\t"
+        "bcax	v21.16b, v26.16b, v23.16b, v22.16b\n\t"
+        "bcax	v22.16b, v22.16b, v24.16b, v23.16b\n\t"
+        "bcax	v23.16b, v23.16b, v25.16b, v24.16b\n\t"
+        "bcax	v24.16b, v24.16b, v26.16b, v25.16b\n\t"
+        "ldr	x3, [%[r]], #8\n\t"
+        "subs	x2, x2, #1\n\t"
+        "mov	v30.d[0], x3\n\t"
+        "mov	v30.d[1], x3\n\t"
+        "eor	v0.16b, v0.16b, v30.16b\n\t"
+        "b.ne	L_frodokem_sha3_x2_crypto_begin_%=\n\t"
+        "st4	{v0.d, v1.d, v2.d, v3.d}[0], [%x[state]], #32\n\t"
+        "st4	{v4.d, v5.d, v6.d, v7.d}[0], [%x[state]], #32\n\t"
+        "st4	{v8.d, v9.d, v10.d, v11.d}[0], [%x[state]], #32\n\t"
+        "st4	{v12.d, v13.d, v14.d, v15.d}[0], [%x[state]], #32\n\t"
+        "st4	{v16.d, v17.d, v18.d, v19.d}[0], [%x[state]], #32\n\t"
+        "st4	{v20.d, v21.d, v22.d, v23.d}[0], [%x[state]], #32\n\t"
+        "st1	{v24.d}[0], [%x[state]]\n\t"
+        "add	%x[state], %x[state], #8\n\t"
+        "st4	{v0.d, v1.d, v2.d, v3.d}[1], [%x[state]], #32\n\t"
+        "st4	{v4.d, v5.d, v6.d, v7.d}[1], [%x[state]], #32\n\t"
+        "st4	{v8.d, v9.d, v10.d, v11.d}[1], [%x[state]], #32\n\t"
+        "st4	{v12.d, v13.d, v14.d, v15.d}[1], [%x[state]], #32\n\t"
+        "st4	{v16.d, v17.d, v18.d, v19.d}[1], [%x[state]], #32\n\t"
+        "st4	{v20.d, v21.d, v22.d, v23.d}[1], [%x[state]], #32\n\t"
+        "st1	{v24.d}[1], [%x[state]]\n\t"
+        "add	%x[state], %x[state], #8\n\t"
+        : [state] "+r" (state)
+        : [r] "r" (r)
+        : "memory", "cc", "x2", "x3", "v0", "v1", "v2", "v3", "v4", "v5", "v6",
+            "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16",
+            "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25",
+            "v26", "v27", "v28", "v29", "v30", "v31"
+    );
+}
+
+#endif /* WOLFSSL_ARMASM_CRYPTO_SHA3 */
+void frodokem_add_neon(word16* a, const word16* b, int qmask)
+{
+    __asm__ __volatile__ (
+        "dup	v16.8h, %w[qmask]\n\t"
+        "ld1	{v0.8h, v1.8h, v2.8h, v3.8h}, [%x[a]], #0x40\n\t"
+        "ld1	{v4.8h, v5.8h, v6.8h, v7.8h}, [%x[a]], #0x40\n\t"
+        "sub	%x[a], %x[a], #0x80\n\t"
+        "ld1	{v8.8h, v9.8h, v10.8h, v11.8h}, [%x[b]], #0x40\n\t"
+        "ld1	{v12.8h, v13.8h, v14.8h, v15.8h}, [%x[b]], #0x40\n\t"
+        "add	v0.8h, v0.8h, v8.8h\n\t"
+        "and	v0.16b, v0.16b, v16.16b\n\t"
+        "add	v1.8h, v1.8h, v9.8h\n\t"
+        "and	v1.16b, v1.16b, v16.16b\n\t"
+        "add	v2.8h, v2.8h, v10.8h\n\t"
+        "and	v2.16b, v2.16b, v16.16b\n\t"
+        "add	v3.8h, v3.8h, v11.8h\n\t"
+        "and	v3.16b, v3.16b, v16.16b\n\t"
+        "add	v4.8h, v4.8h, v12.8h\n\t"
+        "and	v4.16b, v4.16b, v16.16b\n\t"
+        "add	v5.8h, v5.8h, v13.8h\n\t"
+        "and	v5.16b, v5.16b, v16.16b\n\t"
+        "add	v6.8h, v6.8h, v14.8h\n\t"
+        "and	v6.16b, v6.16b, v16.16b\n\t"
+        "add	v7.8h, v7.8h, v15.8h\n\t"
+        "and	v7.16b, v7.16b, v16.16b\n\t"
+        "st1	{v0.8h, v1.8h, v2.8h, v3.8h}, [%x[a]], #0x40\n\t"
+        "st1	{v4.8h, v5.8h, v6.8h, v7.8h}, [%x[a]], #0x40\n\t"
+        : [a] "+r" (a), [qmask] "+r" (qmask)
+        : [b] "r" (b)
+        : "memory", "cc", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8",
+            "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16"
+    );
+}
+
+void frodokem_sample_neon(word16* mat, int cnt, const word16* cdf, int cdflen)
+{
+    __asm__ __volatile__ (
+        "cmp	%w[cnt], #16\n\t"
+        "b.lt	L_frodokem_sample_neon_rem_%=\n\t"
+        "\n"
+    "L_frodokem_sample_neon_blk16_%=:\n\t"
+        "ld1	{v0.8h}, [%x[mat]], #16\n\t"
+        "ld1	{v1.8h}, [%x[mat]], #16\n\t"
+        "sub	%x[mat], %x[mat], #32\n\t"
+        "ushr	v2.8h, v0.8h, #1\n\t"
+        "movi	v4.8h, #1\n\t"
+        "and	v4.16b, v4.16b, v0.16b\n\t"
+        "movi	v6.8h, #0\n\t"
+        "ushr	v3.8h, v1.8h, #1\n\t"
+        "movi	v5.8h, #1\n\t"
+        "and	v5.16b, v5.16b, v1.16b\n\t"
+        "movi	v7.8h, #0\n\t"
+        "mov	x4, %x[cdf]\n\t"
+        "mov	w5, %w[cdflen]\n\t"
+        "\n"
+    "L_frodokem_sample_neon_cdf16_%=:\n\t"
+        "ld1r	{v12.8h}, [x4], #2\n\t"
+        "sub	v10.8h, v12.8h, v2.8h\n\t"
+        "ushr	v10.8h, v10.8h, #15\n\t"
+        "add	v6.8h, v6.8h, v10.8h\n\t"
+        "sub	v11.8h, v12.8h, v3.8h\n\t"
+        "ushr	v11.8h, v11.8h, #15\n\t"
+        "add	v7.8h, v7.8h, v11.8h\n\t"
+        "subs	w5, w5, #1\n\t"
+        "b.ne	L_frodokem_sample_neon_cdf16_%=\n\t"
+        "neg	v8.8h, v4.8h\n\t"
+        "eor	v6.16b, v6.16b, v8.16b\n\t"
+        "add	v6.8h, v6.8h, v4.8h\n\t"
+        "st1	{v6.8h}, [%x[mat]], #16\n\t"
+        "neg	v9.8h, v5.8h\n\t"
+        "eor	v7.16b, v7.16b, v9.16b\n\t"
+        "add	v7.8h, v7.8h, v5.8h\n\t"
+        "st1	{v7.8h}, [%x[mat]], #16\n\t"
+        "subs	%w[cnt], %w[cnt], #16\n\t"
+        "cmp	%w[cnt], #16\n\t"
+        "b.ge	L_frodokem_sample_neon_blk16_%=\n\t"
+        "\n"
+    "L_frodokem_sample_neon_rem_%=:\n\t"
+        "cmp	%w[cnt], #0\n\t"
+        "b.eq	L_frodokem_sample_neon_done_%=\n\t"
+        "\n"
+    "L_frodokem_sample_neon_blk_%=:\n\t"
+        "ld1	{v0.8h}, [%x[mat]]\n\t"
+        "ushr	v2.8h, v0.8h, #1\n\t"
+        "movi	v4.8h, #1\n\t"
+        "and	v4.16b, v4.16b, v0.16b\n\t"
+        "movi	v6.8h, #0\n\t"
+        "mov	x4, %x[cdf]\n\t"
+        "mov	w5, %w[cdflen]\n\t"
+        "\n"
+    "L_frodokem_sample_neon_cdf_%=:\n\t"
+        "ld1r	{v12.8h}, [x4], #2\n\t"
+        "sub	v12.8h, v12.8h, v2.8h\n\t"
+        "ushr	v12.8h, v12.8h, #15\n\t"
+        "add	v6.8h, v6.8h, v12.8h\n\t"
+        "subs	w5, w5, #1\n\t"
+        "b.ne	L_frodokem_sample_neon_cdf_%=\n\t"
+        "neg	v8.8h, v4.8h\n\t"
+        "eor	v6.16b, v6.16b, v8.16b\n\t"
+        "add	v6.8h, v6.8h, v4.8h\n\t"
+        "st1	{v6.8h}, [%x[mat]], #16\n\t"
+        "subs	%w[cnt], %w[cnt], #8\n\t"
+        "b.ne	L_frodokem_sample_neon_blk_%=\n\t"
+        "\n"
+    "L_frodokem_sample_neon_done_%=:\n\t"
+        : [mat] "+r" (mat), [cnt] "+r" (cnt), [cdflen] "+r" (cdflen)
+        : [cdf] "r" (cdf)
+        : "memory", "cc", "x4", "x5", "v0", "v1", "v2", "v3", "v4", "v5", "v6",
+            "v7", "v8", "v9", "v10", "v11", "v12"
+    );
+}
+
+void frodokem_sa_accum_neon(word16* out, const word16* s, const word16* row,
+    int j, int n)
+{
+    __asm__ __volatile__ (
+        "sxtw	x5, %w[n]\n\t"
+        "lsl	x5, x5, #1\n\t"
+        "lsl	x6, x5, #2\n\t"
+        "sxtw	x7, %w[j]\n\t"
+        "lsl	x7, x7, #1\n\t"
+        "add	x7, %x[s], x7\n\t"
+        "mov	x8, %x[out]\n\t"
+        "mov	x12, #2\n\t"
+        "\n"
+    "L_frodokem_sa_accum_neon_t_%=:\n\t"
+        "ldr	w13, [x7]\n\t"
+        "dup	v0.8h, w13\n\t"
+        "lsr	w13, w13, #16\n\t"
+        "dup	v4.8h, w13\n\t"
+        "add	x7, x7, x5\n\t"
+        "ldr	w13, [x7]\n\t"
+        "dup	v1.8h, w13\n\t"
+        "lsr	w13, w13, #16\n\t"
+        "dup	v5.8h, w13\n\t"
+        "add	x7, x7, x5\n\t"
+        "ldr	w13, [x7]\n\t"
+        "dup	v2.8h, w13\n\t"
+        "lsr	w13, w13, #16\n\t"
+        "dup	v6.8h, w13\n\t"
+        "add	x7, x7, x5\n\t"
+        "ldr	w13, [x7]\n\t"
+        "dup	v3.8h, w13\n\t"
+        "lsr	w13, w13, #16\n\t"
+        "dup	v7.8h, w13\n\t"
+        "add	x7, x7, x5\n\t"
+        "mov	x14, x8\n\t"
+        "add	x15, x14, x5\n\t"
+        "add	x16, x15, x5\n\t"
+        "add	x17, x16, x5\n\t"
+        "mov	x9, %x[row]\n\t"
+        "add	x10, %x[row], x5\n\t"
+        "lsr	x11, x5, #4\n\t"
+        "\n"
+    "L_frodokem_sa_accum_neon_k_%=:\n\t"
+        "ld1	{v8.8h}, [x9], #16\n\t"
+        "ld1	{v9.8h}, [x10], #16\n\t"
+        "ld1	{v10.8h}, [x14]\n\t"
+        "mla	v10.8h, v8.8h, v0.8h\n\t"
+        "mla	v10.8h, v9.8h, v4.8h\n\t"
+        "st1	{v10.8h}, [x14], #16\n\t"
+        "ld1	{v11.8h}, [x15]\n\t"
+        "mla	v11.8h, v8.8h, v1.8h\n\t"
+        "mla	v11.8h, v9.8h, v5.8h\n\t"
+        "st1	{v11.8h}, [x15], #16\n\t"
+        "ld1	{v12.8h}, [x16]\n\t"
+        "mla	v12.8h, v8.8h, v2.8h\n\t"
+        "mla	v12.8h, v9.8h, v6.8h\n\t"
+        "st1	{v12.8h}, [x16], #16\n\t"
+        "ld1	{v13.8h}, [x17]\n\t"
+        "mla	v13.8h, v8.8h, v3.8h\n\t"
+        "mla	v13.8h, v9.8h, v7.8h\n\t"
+        "st1	{v13.8h}, [x17], #16\n\t"
+        "subs	x11, x11, #1\n\t"
+        "b.ne	L_frodokem_sa_accum_neon_k_%=\n\t"
+        "add	x8, x8, x6\n\t"
+        "subs	x12, x12, #1\n\t"
+        "b.ne	L_frodokem_sa_accum_neon_t_%=\n\t"
+        : [out] "+r" (out), [j] "+r" (j), [n] "+r" (n)
+        : [s] "r" (s), [row] "r" (row)
+        : "memory", "cc", "x5", "x6", "x7", "x8", "x9", "x10", "x11", "x12",
+            "x13", "x14", "x15", "x16", "x17", "v0", "v1", "v2", "v3", "v4",
+            "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13"
+    );
+}
+
+void frodokem_sa_accum_x4_neon(word16* out, const word16* s, const word16* row,
+    int j, int n)
+{
+    __asm__ __volatile__ (
+        "sxtw	x5, %w[n]\n\t"
+        "lsl	x5, x5, #1\n\t"
+        "lsl	x6, x5, #2\n\t"
+        "sxtw	x7, %w[j]\n\t"
+        "lsl	x7, x7, #1\n\t"
+        "add	x7, %x[s], x7\n\t"
+        "mov	x8, %x[out]\n\t"
+        "mov	x14, #2\n\t"
+        "\n"
+    "L_frodokem_sa_accum_x4_neon_t_%=:\n\t"
+        "ldr	x15, [x7]\n\t"
+        "dup	v0.8h, w15\n\t"
+        "lsr	x15, x15, #16\n\t"
+        "dup	v1.8h, w15\n\t"
+        "lsr	x15, x15, #16\n\t"
+        "dup	v2.8h, w15\n\t"
+        "lsr	x15, x15, #16\n\t"
+        "dup	v3.8h, w15\n\t"
+        "add	x7, x7, x5\n\t"
+        "ldr	x15, [x7]\n\t"
+        "dup	v4.8h, w15\n\t"
+        "lsr	x15, x15, #16\n\t"
+        "dup	v5.8h, w15\n\t"
+        "lsr	x15, x15, #16\n\t"
+        "dup	v6.8h, w15\n\t"
+        "lsr	x15, x15, #16\n\t"
+        "dup	v7.8h, w15\n\t"
+        "add	x7, x7, x5\n\t"
+        "ldr	x15, [x7]\n\t"
+        "dup	v8.8h, w15\n\t"
+        "lsr	x15, x15, #16\n\t"
+        "dup	v9.8h, w15\n\t"
+        "lsr	x15, x15, #16\n\t"
+        "dup	v10.8h, w15\n\t"
+        "lsr	x15, x15, #16\n\t"
+        "dup	v11.8h, w15\n\t"
+        "add	x7, x7, x5\n\t"
+        "ldr	x15, [x7]\n\t"
+        "dup	v12.8h, w15\n\t"
+        "lsr	x15, x15, #16\n\t"
+        "dup	v13.8h, w15\n\t"
+        "lsr	x15, x15, #16\n\t"
+        "dup	v14.8h, w15\n\t"
+        "lsr	x15, x15, #16\n\t"
+        "dup	v15.8h, w15\n\t"
+        "add	x7, x7, x5\n\t"
+        "mov	x16, x8\n\t"
+        "add	x17, x16, x5\n\t"
+        "add	x19, x17, x5\n\t"
+        "add	x20, x19, x5\n\t"
+        "mov	x9, %x[row]\n\t"
+        "add	x10, x9, x5\n\t"
+        "add	x11, x10, x5\n\t"
+        "add	x12, x11, x5\n\t"
+        "lsr	x13, x5, #4\n\t"
+        "\n"
+    "L_frodokem_sa_accum_x4_neon_k_%=:\n\t"
+        "ld1	{v16.8h}, [x9], #16\n\t"
+        "ld1	{v17.8h}, [x10], #16\n\t"
+        "ld1	{v18.8h}, [x11], #16\n\t"
+        "ld1	{v19.8h}, [x12], #16\n\t"
+        "ld1	{v20.8h}, [x16]\n\t"
+        "mla	v20.8h, v16.8h, v0.8h\n\t"
+        "mla	v20.8h, v17.8h, v1.8h\n\t"
+        "mla	v20.8h, v18.8h, v2.8h\n\t"
+        "mla	v20.8h, v19.8h, v3.8h\n\t"
+        "st1	{v20.8h}, [x16], #16\n\t"
+        "ld1	{v21.8h}, [x17]\n\t"
+        "mla	v21.8h, v16.8h, v4.8h\n\t"
+        "mla	v21.8h, v17.8h, v5.8h\n\t"
+        "mla	v21.8h, v18.8h, v6.8h\n\t"
+        "mla	v21.8h, v19.8h, v7.8h\n\t"
+        "st1	{v21.8h}, [x17], #16\n\t"
+        "ld1	{v22.8h}, [x19]\n\t"
+        "mla	v22.8h, v16.8h, v8.8h\n\t"
+        "mla	v22.8h, v17.8h, v9.8h\n\t"
+        "mla	v22.8h, v18.8h, v10.8h\n\t"
+        "mla	v22.8h, v19.8h, v11.8h\n\t"
+        "st1	{v22.8h}, [x19], #16\n\t"
+        "ld1	{v23.8h}, [x20]\n\t"
+        "mla	v23.8h, v16.8h, v12.8h\n\t"
+        "mla	v23.8h, v17.8h, v13.8h\n\t"
+        "mla	v23.8h, v18.8h, v14.8h\n\t"
+        "mla	v23.8h, v19.8h, v15.8h\n\t"
+        "st1	{v23.8h}, [x20], #16\n\t"
+        "subs	x13, x13, #1\n\t"
+        "b.ne	L_frodokem_sa_accum_x4_neon_k_%=\n\t"
+        "add	x8, x8, x6\n\t"
+        "subs	x14, x14, #1\n\t"
+        "b.ne	L_frodokem_sa_accum_x4_neon_t_%=\n\t"
+        : [out] "+r" (out), [j] "+r" (j), [n] "+r" (n)
+        : [s] "r" (s), [row] "r" (row)
+        : "memory", "cc", "x5", "x6", "x7", "x8", "x9", "x10", "x11", "x12",
+            "x13", "x14", "x15", "x16", "x17", "x19", "x20", "v0", "v1", "v2",
+            "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12",
+            "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21",
+            "v22", "v23"
+    );
+}
+
+void frodokem_as_accum_neon(word16* out, const word16* s, const word16* row,
+    int i, int n)
+{
+    __asm__ __volatile__ (
+        "sxtw	x5, %w[n]\n\t"
+        "lsl	x5, x5, #1\n\t"
+        "lsl	x6, x5, #2\n\t"
+        "sxtw	x15, %w[i]\n\t"
+        "lsl	x15, x15, #4\n\t"
+        "add	x7, %x[out], x15\n\t"
+        "add	x8, x7, #16\n\t"
+        "mov	x9, %x[row]\n\t"
+        "add	x10, %x[row], x5\n\t"
+        "mov	x13, %x[s]\n\t"
+        "mov	x16, x13\n\t"
+        "add	x17, x16, x5\n\t"
+        "add	x19, x17, x5\n\t"
+        "add	x20, x19, x5\n\t"
+        "mov	x11, x9\n\t"
+        "mov	x12, x10\n\t"
+        "movi	v0.8h, #0\n\t"
+        "movi	v1.8h, #0\n\t"
+        "movi	v2.8h, #0\n\t"
+        "movi	v3.8h, #0\n\t"
+        "movi	v4.8h, #0\n\t"
+        "movi	v5.8h, #0\n\t"
+        "movi	v6.8h, #0\n\t"
+        "movi	v7.8h, #0\n\t"
+        "lsr	x14, x5, #4\n\t"
+        "\n"
+    "L_frodokem_as_accum_neon_k0_%=:\n\t"
+        "ld1	{v8.8h}, [x11], #16\n\t"
+        "ld1	{v9.8h}, [x12], #16\n\t"
+        "ld1	{v10.8h}, [x16], #16\n\t"
+        "mla	v0.8h, v8.8h, v10.8h\n\t"
+        "mla	v4.8h, v9.8h, v10.8h\n\t"
+        "ld1	{v10.8h}, [x17], #16\n\t"
+        "mla	v1.8h, v8.8h, v10.8h\n\t"
+        "mla	v5.8h, v9.8h, v10.8h\n\t"
+        "ld1	{v10.8h}, [x19], #16\n\t"
+        "mla	v2.8h, v8.8h, v10.8h\n\t"
+        "mla	v6.8h, v9.8h, v10.8h\n\t"
+        "ld1	{v10.8h}, [x20], #16\n\t"
+        "mla	v3.8h, v8.8h, v10.8h\n\t"
+        "mla	v7.8h, v9.8h, v10.8h\n\t"
+        "subs	x14, x14, #1\n\t"
+        "b.ne	L_frodokem_as_accum_neon_k0_%=\n\t"
+        "addv	h13, v0.8h\n\t"
+        "ins	v11.h[0], v13.h[0]\n\t"
+        "addv	h13, v4.8h\n\t"
+        "ins	v12.h[0], v13.h[0]\n\t"
+        "addv	h13, v1.8h\n\t"
+        "ins	v11.h[1], v13.h[0]\n\t"
+        "addv	h13, v5.8h\n\t"
+        "ins	v12.h[1], v13.h[0]\n\t"
+        "addv	h13, v2.8h\n\t"
+        "ins	v11.h[2], v13.h[0]\n\t"
+        "addv	h13, v6.8h\n\t"
+        "ins	v12.h[2], v13.h[0]\n\t"
+        "addv	h13, v3.8h\n\t"
+        "ins	v11.h[3], v13.h[0]\n\t"
+        "addv	h13, v7.8h\n\t"
+        "ins	v12.h[3], v13.h[0]\n\t"
+        "add	x13, x13, x6\n\t"
+        "mov	x16, x13\n\t"
+        "add	x17, x16, x5\n\t"
+        "add	x19, x17, x5\n\t"
+        "add	x20, x19, x5\n\t"
+        "mov	x11, x9\n\t"
+        "mov	x12, x10\n\t"
+        "movi	v0.8h, #0\n\t"
+        "movi	v1.8h, #0\n\t"
+        "movi	v2.8h, #0\n\t"
+        "movi	v3.8h, #0\n\t"
+        "movi	v4.8h, #0\n\t"
+        "movi	v5.8h, #0\n\t"
+        "movi	v6.8h, #0\n\t"
+        "movi	v7.8h, #0\n\t"
+        "lsr	x14, x5, #4\n\t"
+        "\n"
+    "L_frodokem_as_accum_neon_k1_%=:\n\t"
+        "ld1	{v8.8h}, [x11], #16\n\t"
+        "ld1	{v9.8h}, [x12], #16\n\t"
+        "ld1	{v10.8h}, [x16], #16\n\t"
+        "mla	v0.8h, v8.8h, v10.8h\n\t"
+        "mla	v4.8h, v9.8h, v10.8h\n\t"
+        "ld1	{v10.8h}, [x17], #16\n\t"
+        "mla	v1.8h, v8.8h, v10.8h\n\t"
+        "mla	v5.8h, v9.8h, v10.8h\n\t"
+        "ld1	{v10.8h}, [x19], #16\n\t"
+        "mla	v2.8h, v8.8h, v10.8h\n\t"
+        "mla	v6.8h, v9.8h, v10.8h\n\t"
+        "ld1	{v10.8h}, [x20], #16\n\t"
+        "mla	v3.8h, v8.8h, v10.8h\n\t"
+        "mla	v7.8h, v9.8h, v10.8h\n\t"
+        "subs	x14, x14, #1\n\t"
+        "b.ne	L_frodokem_as_accum_neon_k1_%=\n\t"
+        "addv	h13, v0.8h\n\t"
+        "ins	v11.h[4], v13.h[0]\n\t"
+        "addv	h13, v4.8h\n\t"
+        "ins	v12.h[4], v13.h[0]\n\t"
+        "addv	h13, v1.8h\n\t"
+        "ins	v11.h[5], v13.h[0]\n\t"
+        "addv	h13, v5.8h\n\t"
+        "ins	v12.h[5], v13.h[0]\n\t"
+        "addv	h13, v2.8h\n\t"
+        "ins	v11.h[6], v13.h[0]\n\t"
+        "addv	h13, v6.8h\n\t"
+        "ins	v12.h[6], v13.h[0]\n\t"
+        "addv	h13, v3.8h\n\t"
+        "ins	v11.h[7], v13.h[0]\n\t"
+        "addv	h13, v7.8h\n\t"
+        "ins	v12.h[7], v13.h[0]\n\t"
+        "add	x13, x13, x6\n\t"
+        "ld1	{v8.8h}, [x7]\n\t"
+        "add	v11.8h, v11.8h, v8.8h\n\t"
+        "st1	{v11.8h}, [x7], #16\n\t"
+        "ld1	{v9.8h}, [x8]\n\t"
+        "add	v12.8h, v12.8h, v9.8h\n\t"
+        "st1	{v12.8h}, [x8], #16\n\t"
+        : [out] "+r" (out), [i] "+r" (i), [n] "+r" (n)
+        : [s] "r" (s), [row] "r" (row)
+        : "memory", "cc", "x5", "x6", "x7", "x8", "x9", "x10", "x11", "x12",
+            "x13", "x14", "x15", "x16", "x17", "x19", "x20", "v0", "v1", "v2",
+            "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12",
+            "v13"
+    );
+}
+
+void frodokem_as_accum_x4_neon(word16* out, const word16* s, const word16* row,
+    int i, int n)
+{
+    __asm__ __volatile__ (
+        "sxtw	x5, %w[n]\n\t"
+        "lsl	x5, x5, #1\n\t"
+        "lsl	x6, x5, #2\n\t"
+        "sxtw	x22, %w[i]\n\t"
+        "lsl	x22, x22, #4\n\t"
+        "add	x7, %x[out], x22\n\t"
+        "add	x8, x7, #16\n\t"
+        "add	x9, x8, #16\n\t"
+        "add	x10, x9, #16\n\t"
+        "mov	x11, %x[row]\n\t"
+        "add	x12, x11, x5\n\t"
+        "add	x13, x12, x5\n\t"
+        "add	x14, x13, x5\n\t"
+        "mov	x20, %x[s]\n\t"
+        "mov	x23, x20\n\t"
+        "add	x24, x23, x5\n\t"
+        "add	x25, x24, x5\n\t"
+        "add	x26, x25, x5\n\t"
+        "mov	x15, x11\n\t"
+        "mov	x16, x12\n\t"
+        "mov	x17, x13\n\t"
+        "mov	x19, x14\n\t"
+        "movi	v0.8h, #0\n\t"
+        "movi	v1.8h, #0\n\t"
+        "movi	v2.8h, #0\n\t"
+        "movi	v3.8h, #0\n\t"
+        "movi	v4.8h, #0\n\t"
+        "movi	v5.8h, #0\n\t"
+        "movi	v6.8h, #0\n\t"
+        "movi	v7.8h, #0\n\t"
+        "movi	v8.8h, #0\n\t"
+        "movi	v9.8h, #0\n\t"
+        "movi	v10.8h, #0\n\t"
+        "movi	v11.8h, #0\n\t"
+        "movi	v12.8h, #0\n\t"
+        "movi	v13.8h, #0\n\t"
+        "movi	v14.8h, #0\n\t"
+        "movi	v15.8h, #0\n\t"
+        "lsr	x21, x5, #4\n\t"
+        "\n"
+    "L_frodokem_as_accum_x4_neon_k0_%=:\n\t"
+        "ld1	{v16.8h}, [x15], #16\n\t"
+        "ld1	{v17.8h}, [x16], #16\n\t"
+        "ld1	{v18.8h}, [x17], #16\n\t"
+        "ld1	{v19.8h}, [x19], #16\n\t"
+        "ld1	{v20.8h}, [x23], #16\n\t"
+        "mla	v0.8h, v16.8h, v20.8h\n\t"
+        "mla	v4.8h, v17.8h, v20.8h\n\t"
+        "mla	v8.8h, v18.8h, v20.8h\n\t"
+        "mla	v12.8h, v19.8h, v20.8h\n\t"
+        "ld1	{v20.8h}, [x24], #16\n\t"
+        "mla	v1.8h, v16.8h, v20.8h\n\t"
+        "mla	v5.8h, v17.8h, v20.8h\n\t"
+        "mla	v9.8h, v18.8h, v20.8h\n\t"
+        "mla	v13.8h, v19.8h, v20.8h\n\t"
+        "ld1	{v20.8h}, [x25], #16\n\t"
+        "mla	v2.8h, v16.8h, v20.8h\n\t"
+        "mla	v6.8h, v17.8h, v20.8h\n\t"
+        "mla	v10.8h, v18.8h, v20.8h\n\t"
+        "mla	v14.8h, v19.8h, v20.8h\n\t"
+        "ld1	{v20.8h}, [x26], #16\n\t"
+        "mla	v3.8h, v16.8h, v20.8h\n\t"
+        "mla	v7.8h, v17.8h, v20.8h\n\t"
+        "mla	v11.8h, v18.8h, v20.8h\n\t"
+        "mla	v15.8h, v19.8h, v20.8h\n\t"
+        "subs	x21, x21, #1\n\t"
+        "b.ne	L_frodokem_as_accum_x4_neon_k0_%=\n\t"
+        "addv	h25, v0.8h\n\t"
+        "ins	v21.h[0], v25.h[0]\n\t"
+        "addv	h25, v4.8h\n\t"
+        "ins	v22.h[0], v25.h[0]\n\t"
+        "addv	h25, v8.8h\n\t"
+        "ins	v23.h[0], v25.h[0]\n\t"
+        "addv	h25, v12.8h\n\t"
+        "ins	v24.h[0], v25.h[0]\n\t"
+        "addv	h25, v1.8h\n\t"
+        "ins	v21.h[1], v25.h[0]\n\t"
+        "addv	h25, v5.8h\n\t"
+        "ins	v22.h[1], v25.h[0]\n\t"
+        "addv	h25, v9.8h\n\t"
+        "ins	v23.h[1], v25.h[0]\n\t"
+        "addv	h25, v13.8h\n\t"
+        "ins	v24.h[1], v25.h[0]\n\t"
+        "addv	h25, v2.8h\n\t"
+        "ins	v21.h[2], v25.h[0]\n\t"
+        "addv	h25, v6.8h\n\t"
+        "ins	v22.h[2], v25.h[0]\n\t"
+        "addv	h25, v10.8h\n\t"
+        "ins	v23.h[2], v25.h[0]\n\t"
+        "addv	h25, v14.8h\n\t"
+        "ins	v24.h[2], v25.h[0]\n\t"
+        "addv	h25, v3.8h\n\t"
+        "ins	v21.h[3], v25.h[0]\n\t"
+        "addv	h25, v7.8h\n\t"
+        "ins	v22.h[3], v25.h[0]\n\t"
+        "addv	h25, v11.8h\n\t"
+        "ins	v23.h[3], v25.h[0]\n\t"
+        "addv	h25, v15.8h\n\t"
+        "ins	v24.h[3], v25.h[0]\n\t"
+        "add	x20, x20, x6\n\t"
+        "mov	x23, x20\n\t"
+        "add	x24, x23, x5\n\t"
+        "add	x25, x24, x5\n\t"
+        "add	x26, x25, x5\n\t"
+        "mov	x15, x11\n\t"
+        "mov	x16, x12\n\t"
+        "mov	x17, x13\n\t"
+        "mov	x19, x14\n\t"
+        "movi	v0.8h, #0\n\t"
+        "movi	v1.8h, #0\n\t"
+        "movi	v2.8h, #0\n\t"
+        "movi	v3.8h, #0\n\t"
+        "movi	v4.8h, #0\n\t"
+        "movi	v5.8h, #0\n\t"
+        "movi	v6.8h, #0\n\t"
+        "movi	v7.8h, #0\n\t"
+        "movi	v8.8h, #0\n\t"
+        "movi	v9.8h, #0\n\t"
+        "movi	v10.8h, #0\n\t"
+        "movi	v11.8h, #0\n\t"
+        "movi	v12.8h, #0\n\t"
+        "movi	v13.8h, #0\n\t"
+        "movi	v14.8h, #0\n\t"
+        "movi	v15.8h, #0\n\t"
+        "lsr	x21, x5, #4\n\t"
+        "\n"
+    "L_frodokem_as_accum_x4_neon_k1_%=:\n\t"
+        "ld1	{v16.8h}, [x15], #16\n\t"
+        "ld1	{v17.8h}, [x16], #16\n\t"
+        "ld1	{v18.8h}, [x17], #16\n\t"
+        "ld1	{v19.8h}, [x19], #16\n\t"
+        "ld1	{v20.8h}, [x23], #16\n\t"
+        "mla	v0.8h, v16.8h, v20.8h\n\t"
+        "mla	v4.8h, v17.8h, v20.8h\n\t"
+        "mla	v8.8h, v18.8h, v20.8h\n\t"
+        "mla	v12.8h, v19.8h, v20.8h\n\t"
+        "ld1	{v20.8h}, [x24], #16\n\t"
+        "mla	v1.8h, v16.8h, v20.8h\n\t"
+        "mla	v5.8h, v17.8h, v20.8h\n\t"
+        "mla	v9.8h, v18.8h, v20.8h\n\t"
+        "mla	v13.8h, v19.8h, v20.8h\n\t"
+        "ld1	{v20.8h}, [x25], #16\n\t"
+        "mla	v2.8h, v16.8h, v20.8h\n\t"
+        "mla	v6.8h, v17.8h, v20.8h\n\t"
+        "mla	v10.8h, v18.8h, v20.8h\n\t"
+        "mla	v14.8h, v19.8h, v20.8h\n\t"
+        "ld1	{v20.8h}, [x26], #16\n\t"
+        "mla	v3.8h, v16.8h, v20.8h\n\t"
+        "mla	v7.8h, v17.8h, v20.8h\n\t"
+        "mla	v11.8h, v18.8h, v20.8h\n\t"
+        "mla	v15.8h, v19.8h, v20.8h\n\t"
+        "subs	x21, x21, #1\n\t"
+        "b.ne	L_frodokem_as_accum_x4_neon_k1_%=\n\t"
+        "addv	h25, v0.8h\n\t"
+        "ins	v21.h[4], v25.h[0]\n\t"
+        "addv	h25, v4.8h\n\t"
+        "ins	v22.h[4], v25.h[0]\n\t"
+        "addv	h25, v8.8h\n\t"
+        "ins	v23.h[4], v25.h[0]\n\t"
+        "addv	h25, v12.8h\n\t"
+        "ins	v24.h[4], v25.h[0]\n\t"
+        "addv	h25, v1.8h\n\t"
+        "ins	v21.h[5], v25.h[0]\n\t"
+        "addv	h25, v5.8h\n\t"
+        "ins	v22.h[5], v25.h[0]\n\t"
+        "addv	h25, v9.8h\n\t"
+        "ins	v23.h[5], v25.h[0]\n\t"
+        "addv	h25, v13.8h\n\t"
+        "ins	v24.h[5], v25.h[0]\n\t"
+        "addv	h25, v2.8h\n\t"
+        "ins	v21.h[6], v25.h[0]\n\t"
+        "addv	h25, v6.8h\n\t"
+        "ins	v22.h[6], v25.h[0]\n\t"
+        "addv	h25, v10.8h\n\t"
+        "ins	v23.h[6], v25.h[0]\n\t"
+        "addv	h25, v14.8h\n\t"
+        "ins	v24.h[6], v25.h[0]\n\t"
+        "addv	h25, v3.8h\n\t"
+        "ins	v21.h[7], v25.h[0]\n\t"
+        "addv	h25, v7.8h\n\t"
+        "ins	v22.h[7], v25.h[0]\n\t"
+        "addv	h25, v11.8h\n\t"
+        "ins	v23.h[7], v25.h[0]\n\t"
+        "addv	h25, v15.8h\n\t"
+        "ins	v24.h[7], v25.h[0]\n\t"
+        "add	x20, x20, x6\n\t"
+        "ld1	{v16.8h}, [x7]\n\t"
+        "add	v21.8h, v21.8h, v16.8h\n\t"
+        "st1	{v21.8h}, [x7], #16\n\t"
+        "ld1	{v16.8h}, [x8]\n\t"
+        "add	v22.8h, v22.8h, v16.8h\n\t"
+        "st1	{v22.8h}, [x8], #16\n\t"
+        "ld1	{v16.8h}, [x9]\n\t"
+        "add	v23.8h, v23.8h, v16.8h\n\t"
+        "st1	{v23.8h}, [x9], #16\n\t"
+        "ld1	{v16.8h}, [x10]\n\t"
+        "add	v24.8h, v24.8h, v16.8h\n\t"
+        "st1	{v24.8h}, [x10], #16\n\t"
+        : [out] "+r" (out), [i] "+r" (i), [n] "+r" (n)
+        : [s] "r" (s), [row] "r" (row)
+        : "memory", "cc", "x5", "x6", "x7", "x8", "x9", "x10", "x11", "x12",
+            "x13", "x14", "x15", "x16", "x17", "x19", "x20", "x21", "x22",
+            "x23", "x24", "x25", "x26", "v0", "v1", "v2", "v3", "v4", "v5",
+            "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15",
+            "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24",
+            "v25"
+    );
+}
+
+void frodokem_mul_bs_neon(word16* out, const word16* b, const word16* s, int n,
+    int qmask)
+{
+    __asm__ __volatile__ (
+        "sxtw	x5, %w[n]\n\t"
+        "lsl	x5, x5, #1\n\t"
+        "lsl	x6, x5, #2\n\t"
+        "dup	v26.8h, %w[qmask]\n\t"
+        "mov	x7, %x[out]\n\t"
+        "add	x8, x7, #16\n\t"
+        "add	x9, x8, #16\n\t"
+        "add	x10, x9, #16\n\t"
+        "mov	x11, %x[b]\n\t"
+        "mov	x23, #2\n\t"
+        "\n"
+    "L_frodokem_mul_bs_neon_p_%=:\n\t"
+        "mov	x12, x11\n\t"
+        "add	x13, x12, x5\n\t"
+        "add	x14, x13, x5\n\t"
+        "add	x15, x14, x5\n\t"
+        "mov	x21, %x[s]\n\t"
+        "mov	x24, x21\n\t"
+        "add	x25, x24, x5\n\t"
+        "add	x26, x25, x5\n\t"
+        "add	x27, x26, x5\n\t"
+        "mov	x16, x12\n\t"
+        "mov	x17, x13\n\t"
+        "mov	x19, x14\n\t"
+        "mov	x20, x15\n\t"
+        "movi	v0.8h, #0\n\t"
+        "movi	v1.8h, #0\n\t"
+        "movi	v2.8h, #0\n\t"
+        "movi	v3.8h, #0\n\t"
+        "movi	v4.8h, #0\n\t"
+        "movi	v5.8h, #0\n\t"
+        "movi	v6.8h, #0\n\t"
+        "movi	v7.8h, #0\n\t"
+        "movi	v8.8h, #0\n\t"
+        "movi	v9.8h, #0\n\t"
+        "movi	v10.8h, #0\n\t"
+        "movi	v11.8h, #0\n\t"
+        "movi	v12.8h, #0\n\t"
+        "movi	v13.8h, #0\n\t"
+        "movi	v14.8h, #0\n\t"
+        "movi	v15.8h, #0\n\t"
+        "lsr	x22, x5, #4\n\t"
+        "\n"
+    "L_frodokem_mul_bs_neon_k0_%=:\n\t"
+        "ld1	{v16.8h}, [x16], #16\n\t"
+        "ld1	{v17.8h}, [x17], #16\n\t"
+        "ld1	{v18.8h}, [x19], #16\n\t"
+        "ld1	{v19.8h}, [x20], #16\n\t"
+        "ld1	{v20.8h}, [x24], #16\n\t"
+        "mla	v0.8h, v16.8h, v20.8h\n\t"
+        "mla	v4.8h, v17.8h, v20.8h\n\t"
+        "mla	v8.8h, v18.8h, v20.8h\n\t"
+        "mla	v12.8h, v19.8h, v20.8h\n\t"
+        "ld1	{v20.8h}, [x25], #16\n\t"
+        "mla	v1.8h, v16.8h, v20.8h\n\t"
+        "mla	v5.8h, v17.8h, v20.8h\n\t"
+        "mla	v9.8h, v18.8h, v20.8h\n\t"
+        "mla	v13.8h, v19.8h, v20.8h\n\t"
+        "ld1	{v20.8h}, [x26], #16\n\t"
+        "mla	v2.8h, v16.8h, v20.8h\n\t"
+        "mla	v6.8h, v17.8h, v20.8h\n\t"
+        "mla	v10.8h, v18.8h, v20.8h\n\t"
+        "mla	v14.8h, v19.8h, v20.8h\n\t"
+        "ld1	{v20.8h}, [x27], #16\n\t"
+        "mla	v3.8h, v16.8h, v20.8h\n\t"
+        "mla	v7.8h, v17.8h, v20.8h\n\t"
+        "mla	v11.8h, v18.8h, v20.8h\n\t"
+        "mla	v15.8h, v19.8h, v20.8h\n\t"
+        "subs	x22, x22, #1\n\t"
+        "b.ne	L_frodokem_mul_bs_neon_k0_%=\n\t"
+        "addv	h25, v0.8h\n\t"
+        "ins	v21.h[0], v25.h[0]\n\t"
+        "addv	h25, v4.8h\n\t"
+        "ins	v22.h[0], v25.h[0]\n\t"
+        "addv	h25, v8.8h\n\t"
+        "ins	v23.h[0], v25.h[0]\n\t"
+        "addv	h25, v12.8h\n\t"
+        "ins	v24.h[0], v25.h[0]\n\t"
+        "addv	h25, v1.8h\n\t"
+        "ins	v21.h[1], v25.h[0]\n\t"
+        "addv	h25, v5.8h\n\t"
+        "ins	v22.h[1], v25.h[0]\n\t"
+        "addv	h25, v9.8h\n\t"
+        "ins	v23.h[1], v25.h[0]\n\t"
+        "addv	h25, v13.8h\n\t"
+        "ins	v24.h[1], v25.h[0]\n\t"
+        "addv	h25, v2.8h\n\t"
+        "ins	v21.h[2], v25.h[0]\n\t"
+        "addv	h25, v6.8h\n\t"
+        "ins	v22.h[2], v25.h[0]\n\t"
+        "addv	h25, v10.8h\n\t"
+        "ins	v23.h[2], v25.h[0]\n\t"
+        "addv	h25, v14.8h\n\t"
+        "ins	v24.h[2], v25.h[0]\n\t"
+        "addv	h25, v3.8h\n\t"
+        "ins	v21.h[3], v25.h[0]\n\t"
+        "addv	h25, v7.8h\n\t"
+        "ins	v22.h[3], v25.h[0]\n\t"
+        "addv	h25, v11.8h\n\t"
+        "ins	v23.h[3], v25.h[0]\n\t"
+        "addv	h25, v15.8h\n\t"
+        "ins	v24.h[3], v25.h[0]\n\t"
+        "add	x21, x21, x6\n\t"
+        "mov	x24, x21\n\t"
+        "add	x25, x24, x5\n\t"
+        "add	x26, x25, x5\n\t"
+        "add	x27, x26, x5\n\t"
+        "mov	x16, x12\n\t"
+        "mov	x17, x13\n\t"
+        "mov	x19, x14\n\t"
+        "mov	x20, x15\n\t"
+        "movi	v0.8h, #0\n\t"
+        "movi	v1.8h, #0\n\t"
+        "movi	v2.8h, #0\n\t"
+        "movi	v3.8h, #0\n\t"
+        "movi	v4.8h, #0\n\t"
+        "movi	v5.8h, #0\n\t"
+        "movi	v6.8h, #0\n\t"
+        "movi	v7.8h, #0\n\t"
+        "movi	v8.8h, #0\n\t"
+        "movi	v9.8h, #0\n\t"
+        "movi	v10.8h, #0\n\t"
+        "movi	v11.8h, #0\n\t"
+        "movi	v12.8h, #0\n\t"
+        "movi	v13.8h, #0\n\t"
+        "movi	v14.8h, #0\n\t"
+        "movi	v15.8h, #0\n\t"
+        "lsr	x22, x5, #4\n\t"
+        "\n"
+    "L_frodokem_mul_bs_neon_k1_%=:\n\t"
+        "ld1	{v16.8h}, [x16], #16\n\t"
+        "ld1	{v17.8h}, [x17], #16\n\t"
+        "ld1	{v18.8h}, [x19], #16\n\t"
+        "ld1	{v19.8h}, [x20], #16\n\t"
+        "ld1	{v20.8h}, [x24], #16\n\t"
+        "mla	v0.8h, v16.8h, v20.8h\n\t"
+        "mla	v4.8h, v17.8h, v20.8h\n\t"
+        "mla	v8.8h, v18.8h, v20.8h\n\t"
+        "mla	v12.8h, v19.8h, v20.8h\n\t"
+        "ld1	{v20.8h}, [x25], #16\n\t"
+        "mla	v1.8h, v16.8h, v20.8h\n\t"
+        "mla	v5.8h, v17.8h, v20.8h\n\t"
+        "mla	v9.8h, v18.8h, v20.8h\n\t"
+        "mla	v13.8h, v19.8h, v20.8h\n\t"
+        "ld1	{v20.8h}, [x26], #16\n\t"
+        "mla	v2.8h, v16.8h, v20.8h\n\t"
+        "mla	v6.8h, v17.8h, v20.8h\n\t"
+        "mla	v10.8h, v18.8h, v20.8h\n\t"
+        "mla	v14.8h, v19.8h, v20.8h\n\t"
+        "ld1	{v20.8h}, [x27], #16\n\t"
+        "mla	v3.8h, v16.8h, v20.8h\n\t"
+        "mla	v7.8h, v17.8h, v20.8h\n\t"
+        "mla	v11.8h, v18.8h, v20.8h\n\t"
+        "mla	v15.8h, v19.8h, v20.8h\n\t"
+        "subs	x22, x22, #1\n\t"
+        "b.ne	L_frodokem_mul_bs_neon_k1_%=\n\t"
+        "addv	h25, v0.8h\n\t"
+        "ins	v21.h[4], v25.h[0]\n\t"
+        "addv	h25, v4.8h\n\t"
+        "ins	v22.h[4], v25.h[0]\n\t"
+        "addv	h25, v8.8h\n\t"
+        "ins	v23.h[4], v25.h[0]\n\t"
+        "addv	h25, v12.8h\n\t"
+        "ins	v24.h[4], v25.h[0]\n\t"
+        "addv	h25, v1.8h\n\t"
+        "ins	v21.h[5], v25.h[0]\n\t"
+        "addv	h25, v5.8h\n\t"
+        "ins	v22.h[5], v25.h[0]\n\t"
+        "addv	h25, v9.8h\n\t"
+        "ins	v23.h[5], v25.h[0]\n\t"
+        "addv	h25, v13.8h\n\t"
+        "ins	v24.h[5], v25.h[0]\n\t"
+        "addv	h25, v2.8h\n\t"
+        "ins	v21.h[6], v25.h[0]\n\t"
+        "addv	h25, v6.8h\n\t"
+        "ins	v22.h[6], v25.h[0]\n\t"
+        "addv	h25, v10.8h\n\t"
+        "ins	v23.h[6], v25.h[0]\n\t"
+        "addv	h25, v14.8h\n\t"
+        "ins	v24.h[6], v25.h[0]\n\t"
+        "addv	h25, v3.8h\n\t"
+        "ins	v21.h[7], v25.h[0]\n\t"
+        "addv	h25, v7.8h\n\t"
+        "ins	v22.h[7], v25.h[0]\n\t"
+        "addv	h25, v11.8h\n\t"
+        "ins	v23.h[7], v25.h[0]\n\t"
+        "addv	h25, v15.8h\n\t"
+        "ins	v24.h[7], v25.h[0]\n\t"
+        "add	x21, x21, x6\n\t"
+        "and	v21.16b, v21.16b, v26.16b\n\t"
+        "st1	{v21.8h}, [x7]\n\t"
+        "and	v22.16b, v22.16b, v26.16b\n\t"
+        "st1	{v22.8h}, [x8]\n\t"
+        "and	v23.16b, v23.16b, v26.16b\n\t"
+        "st1	{v23.8h}, [x9]\n\t"
+        "and	v24.16b, v24.16b, v26.16b\n\t"
+        "st1	{v24.8h}, [x10]\n\t"
+        "add	x11, x11, x6\n\t"
+        "add	x7, x7, #0x40\n\t"
+        "add	x8, x8, #0x40\n\t"
+        "add	x9, x9, #0x40\n\t"
+        "add	x10, x10, #0x40\n\t"
+        "subs	x23, x23, #1\n\t"
+        "b.ne	L_frodokem_mul_bs_neon_p_%=\n\t"
+        : [out] "+r" (out), [n] "+r" (n), [qmask] "+r" (qmask)
+        : [b] "r" (b), [s] "r" (s)
+        : "memory", "cc", "x5", "x6", "x7", "x8", "x9", "x10", "x11", "x12",
+            "x13", "x14", "x15", "x16", "x17", "x19", "x20", "x21", "x22",
+            "x23", "x24", "x25", "x26", "x27", "v0", "v1", "v2", "v3", "v4",
+            "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14",
+            "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23",
+            "v24", "v25", "v26"
+    );
+}
+
+void frodokem_mul_add_sb_plus_e_neon(word16* out, const word16* b,
+    const word16* s, int n, int qmask)
+{
+    __asm__ __volatile__ (
+        "sxtw	x5, %w[n]\n\t"
+        "lsl	x5, x5, #1\n\t"
+        "dup	v10.8h, %w[qmask]\n\t"
+        "mov	x6, %x[s]\n\t"
+        "add	x7, x6, x5\n\t"
+        "add	x8, x7, x5\n\t"
+        "add	x9, x8, x5\n\t"
+        "add	x10, x9, x5\n\t"
+        "add	x11, x10, x5\n\t"
+        "add	x12, x11, x5\n\t"
+        "add	x13, x12, x5\n\t"
+        "mov	x14, %x[b]\n\t"
+        "mov	x16, %x[out]\n\t"
+        "ld1	{v0.8h, v1.8h, v2.8h, v3.8h}, [x16], #0x40\n\t"
+        "ld1	{v4.8h, v5.8h, v6.8h, v7.8h}, [x16], #0x40\n\t"
+        "sub	x16, x16, #0x80\n\t"
+        "sxtw	x15, %w[n]\n\t"
+        "\n"
+    "L_frodokem_mul_add_sb_plus_e_neon_j_%=:\n\t"
+        "ld1	{v8.8h}, [x14], #16\n\t"
+        "ld1r	{v9.8h}, [x6], #2\n\t"
+        "mla	v0.8h, v8.8h, v9.8h\n\t"
+        "ld1r	{v9.8h}, [x7], #2\n\t"
+        "mla	v1.8h, v8.8h, v9.8h\n\t"
+        "ld1r	{v9.8h}, [x8], #2\n\t"
+        "mla	v2.8h, v8.8h, v9.8h\n\t"
+        "ld1r	{v9.8h}, [x9], #2\n\t"
+        "mla	v3.8h, v8.8h, v9.8h\n\t"
+        "ld1r	{v9.8h}, [x10], #2\n\t"
+        "mla	v4.8h, v8.8h, v9.8h\n\t"
+        "ld1r	{v9.8h}, [x11], #2\n\t"
+        "mla	v5.8h, v8.8h, v9.8h\n\t"
+        "ld1r	{v9.8h}, [x12], #2\n\t"
+        "mla	v6.8h, v8.8h, v9.8h\n\t"
+        "ld1r	{v9.8h}, [x13], #2\n\t"
+        "mla	v7.8h, v8.8h, v9.8h\n\t"
+        "subs	x15, x15, #1\n\t"
+        "b.ne	L_frodokem_mul_add_sb_plus_e_neon_j_%=\n\t"
+        "and	v0.16b, v0.16b, v10.16b\n\t"
+        "and	v1.16b, v1.16b, v10.16b\n\t"
+        "and	v2.16b, v2.16b, v10.16b\n\t"
+        "and	v3.16b, v3.16b, v10.16b\n\t"
+        "and	v4.16b, v4.16b, v10.16b\n\t"
+        "and	v5.16b, v5.16b, v10.16b\n\t"
+        "and	v6.16b, v6.16b, v10.16b\n\t"
+        "and	v7.16b, v7.16b, v10.16b\n\t"
+        "st1	{v0.8h, v1.8h, v2.8h, v3.8h}, [x16], #0x40\n\t"
+        "st1	{v4.8h, v5.8h, v6.8h, v7.8h}, [x16], #0x40\n\t"
+        : [out] "+r" (out), [n] "+r" (n), [qmask] "+r" (qmask)
+        : [b] "r" (b), [s] "r" (s)
+        : "memory", "cc", "x5", "x6", "x7", "x8", "x9", "x10", "x11", "x12",
+            "x13", "x14", "x15", "x16", "v0", "v1", "v2", "v3", "v4", "v5",
+            "v6", "v7", "v8", "v9", "v10"
+    );
+}
+
+#endif /* WOLFSSL_HAVE_FRODOKEM */
+#endif /* __aarch64__ */
+#endif /* WOLFSSL_ARMASM */
+#endif /* WOLFSSL_ARMASM_INLINE */
diff --git a/wolfcrypt/src/port/arm/thumb2-frodokem-asm.S b/wolfcrypt/src/port/arm/thumb2-frodokem-asm.S
new file mode 100644
index 0000000000..893d6dfb66
--- /dev/null
+++ b/wolfcrypt/src/port/arm/thumb2-frodokem-asm.S
@@ -0,0 +1,264 @@
+/* thumb2-frodokem-asm
+ *
+ * Copyright (C) 2006-2026 wolfSSL Inc.
+ *
+ * This file is part of wolfSSL.
+ *
+ * wolfSSL is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * wolfSSL is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335, USA
+ */
+
+/* Generated using (from wolfssl):
+ *   cd ../scripts
+ *   ruby ./frodokem/frodokem.rb \
+ *       thumb2 ../wolfssl/wolfcrypt/src/port/arm/thumb2-frodokem-asm.S
+ */
+
+#include <wolfssl/wolfcrypt/libwolfssl_sources_asm.h>
+
+#ifdef WOLFSSL_ARMASM
+#ifdef WOLFSSL_ARMASM_THUMB2
+#ifndef WOLFSSL_ARMASM_INLINE
+	.thumb
+	.syntax unified
+#ifdef WOLFSSL_HAVE_FRODOKEM
+	.text
+	.align	4
+	.globl	frodokem_add_thumb2
+	.type	frodokem_add_thumb2, %function
+frodokem_add_thumb2:
+	PUSH	{r4, r5, lr}
+	ORR	r2, r2, r2, LSL #16
+	MOV	r3, #32
+L_frodokem_add_thumb2_blk:
+	LDR	r4, [r0]
+	LDR	r5, [r1], #4
+	SADD16	r4, r4, r5
+	AND	r4, r4, r2
+	STR	r4, [r0], #4
+	SUBS	r3, r3, #1
+#if defined(__GNUC__) || defined(__ICCARM__) || defined(__IAR_SYSTEMS_ICC__)
+	BNE	L_frodokem_add_thumb2_blk
+#else
+	BNE.N	L_frodokem_add_thumb2_blk
+#endif
+	POP	{r4, r5, pc}
+	/* Cycle Count = 23 */
+	.size	frodokem_add_thumb2,.-frodokem_add_thumb2
+	.text
+	.align	4
+	.globl	frodokem_sa_accum_thumb2
+	.type	frodokem_sa_accum_thumb2, %function
+frodokem_sa_accum_thumb2:
+	PUSH	{r4, r5, r6, r7, r8, r9, r10, r11, lr}
+	LDR	r4, [sp, #36]
+	LSL	r4, r4, #1
+	ADD	r5, r1, r3, LSL #1
+	MOV	r8, #8
+L_frodokem_sa_accum_thumb2_i:
+	LDRH	r9, [r5]
+	ADD	r5, r5, r4
+	MOV	r6, r2
+	LSR	r7, r4, #2
+L_frodokem_sa_accum_thumb2_k:
+	LDR	r11, [r0]
+	LDR	r10, [r6], #4
+	SMULBB	r12, r9, r10
+	SMULBT	lr, r9, r10
+	PKHBT	r12, r12, lr, LSL #16
+	SADD16	r11, r11, r12
+	STR	r11, [r0], #4
+	SUBS	r7, r7, #1
+#if defined(__GNUC__) || defined(__ICCARM__) || defined(__IAR_SYSTEMS_ICC__)
+	BNE	L_frodokem_sa_accum_thumb2_k
+#else
+	BNE.N	L_frodokem_sa_accum_thumb2_k
+#endif
+	SUBS	r8, r8, #1
+#if defined(__GNUC__) || defined(__ICCARM__) || defined(__IAR_SYSTEMS_ICC__)
+	BNE	L_frodokem_sa_accum_thumb2_i
+#else
+	BNE.N	L_frodokem_sa_accum_thumb2_i
+#endif
+	POP	{r4, r5, r6, r7, r8, r9, r10, r11, pc}
+	/* Cycle Count = 50 */
+	.size	frodokem_sa_accum_thumb2,.-frodokem_sa_accum_thumb2
+	.text
+	.align	4
+	.globl	frodokem_as_accum_thumb2
+	.type	frodokem_as_accum_thumb2, %function
+frodokem_as_accum_thumb2:
+	PUSH	{r4, r5, r6, r7, r8, r9, r10, r11, lr}
+	LDR	r4, [sp, #36]
+	LSL	r4, r4, #1
+	ADD	r5, r0, r3, LSL #4
+	MOV	r6, r1
+	MOV	r9, #8
+L_frodokem_as_accum_thumb2_c:
+	MOV	r11, #0
+	MOV	r7, r2
+	MOV	r8, r6
+	LSR	r10, r4, #2
+L_frodokem_as_accum_thumb2_j:
+	LDR	r12, [r7], #4
+	LDR	lr, [r8], #4
+	SMLAD	r11, r12, lr, r11
+	SUBS	r10, r10, #1
+#if defined(__GNUC__) || defined(__ICCARM__) || defined(__IAR_SYSTEMS_ICC__)
+	BNE	L_frodokem_as_accum_thumb2_j
+#else
+	BNE.N	L_frodokem_as_accum_thumb2_j
+#endif
+	LDRH	r12, [r5]
+	ADD	r12, r12, r11
+	STRH	r12, [r5]
+	ADD	r5, r5, #2
+	ADD	r6, r6, r4
+	SUBS	r9, r9, #1
+#if defined(__GNUC__) || defined(__ICCARM__) || defined(__IAR_SYSTEMS_ICC__)
+	BNE	L_frodokem_as_accum_thumb2_c
+#else
+	BNE.N	L_frodokem_as_accum_thumb2_c
+#endif
+	POP	{r4, r5, r6, r7, r8, r9, r10, r11, pc}
+	/* Cycle Count = 52 */
+	.size	frodokem_as_accum_thumb2,.-frodokem_as_accum_thumb2
+	.text
+	.align	4
+	.globl	frodokem_mul_bs_thumb2
+	.type	frodokem_mul_bs_thumb2, %function
+frodokem_mul_bs_thumb2:
+	PUSH	{r4, r5, r6, r7, r8, r9, r10, r11, lr}
+	MOV	r4, r1
+	MOV	r8, #8
+L_frodokem_mul_bs_thumb2_i:
+	MOV	r5, r2
+	MOV	r9, #8
+L_frodokem_mul_bs_thumb2_c:
+	MOV	r11, #0
+	MOV	r6, r4
+	MOV	r7, r5
+	LSR	r10, r3, #1
+L_frodokem_mul_bs_thumb2_j:
+	LDR	r12, [r6], #4
+	LDR	lr, [r7], #4
+	SMLAD	r11, r12, lr, r11
+	SUBS	r10, r10, #1
+#if defined(__GNUC__) || defined(__ICCARM__) || defined(__IAR_SYSTEMS_ICC__)
+	BNE	L_frodokem_mul_bs_thumb2_j
+#else
+	BNE.N	L_frodokem_mul_bs_thumb2_j
+#endif
+	LDR	r12, [sp, #36]
+	ORR	r12, r12, r12, LSL #16
+	AND	r11, r11, r12
+	STRH	r11, [r0]
+	ADD	r0, r0, #2
+	ADD	r5, r5, r3, LSL #1
+	SUBS	r9, r9, #1
+#if defined(__GNUC__) || defined(__ICCARM__) || defined(__IAR_SYSTEMS_ICC__)
+	BNE	L_frodokem_mul_bs_thumb2_c
+#else
+	BNE.N	L_frodokem_mul_bs_thumb2_c
+#endif
+	ADD	r4, r4, r3, LSL #1
+	SUBS	r8, r8, #1
+#if defined(__GNUC__) || defined(__ICCARM__) || defined(__IAR_SYSTEMS_ICC__)
+	BNE	L_frodokem_mul_bs_thumb2_i
+#else
+	BNE.N	L_frodokem_mul_bs_thumb2_i
+#endif
+	POP	{r4, r5, r6, r7, r8, r9, r10, r11, pc}
+	/* Cycle Count = 57 */
+	.size	frodokem_mul_bs_thumb2,.-frodokem_mul_bs_thumb2
+	.text
+	.align	4
+	.globl	frodokem_mul_add_sb_plus_e_thumb2
+	.type	frodokem_mul_add_sb_plus_e_thumb2, %function
+frodokem_mul_add_sb_plus_e_thumb2:
+	PUSH	{r4, r5, r6, r7, r8, r9, r10, r11, lr}
+	SUB	sp, sp, #8
+	LSL	r11, r3, #1
+	STR	r11, [sp, #4]
+	STR	r1, [sp]
+	MOV	r4, #8
+L_frodokem_mul_add_sb_plus_e_thumb2_i:
+	LDR	r11, [sp, #4]
+	ADD	r3, r2, r11
+	LDR	r1, [sp]
+	MOV	r5, r2
+	LDR	r6, [r0]
+	LDR	r7, [r0, #4]
+	LDR	r8, [r0, #8]
+	LDR	r9, [r0, #12]
+L_frodokem_mul_add_sb_plus_e_thumb2_j:
+	LDRH	r10, [r5]
+	ADD	r5, r5, #2
+	LDR	r11, [r1]
+	SMULBB	r12, r10, r11
+	SMULBT	lr, r10, r11
+	PKHBT	r12, r12, lr, LSL #16
+	SADD16	r6, r6, r12
+	LDR	r11, [r1, #4]
+	SMULBB	r12, r10, r11
+	SMULBT	lr, r10, r11
+	PKHBT	r12, r12, lr, LSL #16
+	SADD16	r7, r7, r12
+	LDR	r11, [r1, #8]
+	SMULBB	r12, r10, r11
+	SMULBT	lr, r10, r11
+	PKHBT	r12, r12, lr, LSL #16
+	SADD16	r8, r8, r12
+	LDR	r11, [r1, #12]
+	SMULBB	r12, r10, r11
+	SMULBT	lr, r10, r11
+	PKHBT	r12, r12, lr, LSL #16
+	SADD16	r9, r9, r12
+	ADD	r1, r1, #16
+	CMP	r5, r3
+#if defined(__GNUC__) || defined(__ICCARM__) || defined(__IAR_SYSTEMS_ICC__)
+	BNE	L_frodokem_mul_add_sb_plus_e_thumb2_j
+#else
+	BNE.N	L_frodokem_mul_add_sb_plus_e_thumb2_j
+#endif
+	LDR	r11, [sp, #44]
+	ORR	r11, r11, r11, LSL #16
+	AND	r6, r6, r11
+	AND	r7, r7, r11
+	AND	r8, r8, r11
+	AND	r9, r9, r11
+	STR	r6, [r0]
+	STR	r7, [r0, #4]
+	STR	r8, [r0, #8]
+	STR	r9, [r0, #12]
+	ADD	r0, r0, #16
+	MOV	r2, r3
+	SUBS	r4, r4, #1
+#if defined(__GNUC__) || defined(__ICCARM__) || defined(__IAR_SYSTEMS_ICC__)
+	BNE	L_frodokem_mul_add_sb_plus_e_thumb2_i
+#else
+	BNE.N	L_frodokem_mul_add_sb_plus_e_thumb2_i
+#endif
+	ADD	sp, sp, #8
+	POP	{r4, r5, r6, r7, r8, r9, r10, r11, pc}
+	/* Cycle Count = 97 */
+	.size	frodokem_mul_add_sb_plus_e_thumb2,.-frodokem_mul_add_sb_plus_e_thumb2
+#endif /* WOLFSSL_HAVE_FRODOKEM */
+#endif /* WOLFSSL_ARMASM_THUMB2 */
+#endif /* WOLFSSL_ARMASM */
+
+#if defined(__linux__) && defined(__ELF__)
+.section	.note.GNU-stack,"",%progbits
+#endif
+#endif /* !WOLFSSL_ARMASM_INLINE */
diff --git a/wolfcrypt/src/port/arm/thumb2-frodokem-asm_c.c b/wolfcrypt/src/port/arm/thumb2-frodokem-asm_c.c
new file mode 100644
index 0000000000..6ec19c72de
--- /dev/null
+++ b/wolfcrypt/src/port/arm/thumb2-frodokem-asm_c.c
@@ -0,0 +1,467 @@
+/* thumb2-frodokem-asm
+ *
+ * Copyright (C) 2006-2026 wolfSSL Inc.
+ *
+ * This file is part of wolfSSL.
+ *
+ * wolfSSL is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * wolfSSL is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335, USA
+ */
+
+/* Generated using (from wolfssl):
+ *   cd ../scripts
+ *   ruby ./frodokem/frodokem.rb \
+ *       thumb2 ../wolfssl/wolfcrypt/src/port/arm/thumb2-frodokem-asm.c
+ */
+
+#include <wolfssl/wolfcrypt/libwolfssl_sources_asm.h>
+#include <wolfssl/wolfcrypt/error-crypt.h>
+
+#ifdef WOLFSSL_ARMASM
+#ifdef WOLFSSL_ARMASM_THUMB2
+#ifdef WOLFSSL_ARMASM_INLINE
+
+#ifdef __IAR_SYSTEMS_ICC__
+#define __asm__        asm
+#define __volatile__   volatile
+#define WOLFSSL_NO_VAR_ASSIGN_REG
+#endif /* __IAR_SYSTEMS_ICC__ */
+#ifdef __KEIL__
+#define __asm__        __asm
+#define __volatile__   volatile
+#endif /* __KEIL__ */
+#ifdef __ghs__
+#define __asm__        __asm
+#define __volatile__
+#define WOLFSSL_NO_VAR_ASSIGN_REG
+#endif /* __ghs__ */
+
+#include <wolfssl/wolfcrypt/wc_frodokem_mat.h>
+
+#ifdef WOLFSSL_HAVE_FRODOKEM
+#ifndef WOLFSSL_NO_VAR_ASSIGN_REG
+WC_OMIT_FRAME_POINTER void frodokem_add_thumb2(word16* a_p, const word16* b_p,
+    int qmask_p)
+#else
+WC_OMIT_FRAME_POINTER void frodokem_add_thumb2(word16* a, const word16* b,
+    int qmask)
+#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */
+{
+#ifndef WOLFSSL_NO_VAR_ASSIGN_REG
+    register word16* a __asm__ ("r0") = (word16*)a_p;
+    register const word16* b __asm__ ("r1") = (const word16*)b_p;
+    register int qmask __asm__ ("r2") = (int)qmask_p;
+#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */
+
+    __asm__ __volatile__ (
+        "ORR	%[qmask], %[qmask], %[qmask], LSL #16\n\t"
+        "MOV	r3, #32\n\t"
+        "\n"
+#if defined(__IAR_SYSTEMS_ICC__) && (__VER__ < 9000000)
+    "L_frodokem_add_thumb2_blk:\n\t"
+#else
+    "L_frodokem_add_thumb2_blk_%=:\n\t"
+#endif
+        "LDR	r4, [%[a]]\n\t"
+        "LDR	r5, [%[b]], #4\n\t"
+        "SADD16	r4, r4, r5\n\t"
+        "AND	r4, r4, %[qmask]\n\t"
+        "STR	r4, [%[a]], #4\n\t"
+        "SUBS	r3, r3, #1\n\t"
+#if defined(__GNUC__)
+        "BNE	L_frodokem_add_thumb2_blk_%=\n\t"
+#elif defined(__IAR_SYSTEMS_ICC__) && (__VER__ < 9000000)
+        "BNE.N	L_frodokem_add_thumb2_blk\n\t"
+#else
+        "BNE.N	L_frodokem_add_thumb2_blk_%=\n\t"
+#endif
+#ifndef WOLFSSL_NO_VAR_ASSIGN_REG
+        : [a] "+r" (a), [b] "+r" (b), [qmask] "+r" (qmask)
+        :
+#else
+        :
+        : [a] "r" (a), [b] "r" (b), [qmask] "r" (qmask)
+#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */
+        : "memory", "cc", "r3", "r4", "r5"
+    );
+}
+
+#ifndef WOLFSSL_NO_VAR_ASSIGN_REG
+WC_OMIT_FRAME_POINTER void frodokem_sa_accum_thumb2(word16* out_p,
+    const word16* s_p, const word16* row_p, int j_p, int n_p)
+#else
+WC_OMIT_FRAME_POINTER void frodokem_sa_accum_thumb2(word16* out,
+    const word16* s, const word16* row, int j, int n)
+#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */
+{
+#ifndef WOLFSSL_NO_VAR_ASSIGN_REG
+    register word16* out __asm__ ("r0") = (word16*)out_p;
+    register const word16* s __asm__ ("r1") = (const word16*)s_p;
+    register const word16* row __asm__ ("r2") = (const word16*)row_p;
+    register int j __asm__ ("r3") = (int)j_p;
+    register int n __asm__ ("r4") = (int)n_p;
+#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */
+
+    __asm__ __volatile__ (
+#ifndef WOLFSSL_NO_VAR_ASSIGN_REG
+#else
+        "MOV	r4, %[n]\n\t"
+#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */
+        "LSL	r4, r4, #1\n\t"
+        "ADD	r5, %[s], %[j], LSL #1\n\t"
+        "MOV	r8, #8\n\t"
+        "\n"
+#if defined(__IAR_SYSTEMS_ICC__) && (__VER__ < 9000000)
+    "L_frodokem_sa_accum_thumb2_i:\n\t"
+#else
+    "L_frodokem_sa_accum_thumb2_i_%=:\n\t"
+#endif
+        "LDRH	r9, [r5]\n\t"
+        "ADD	r5, r5, r4\n\t"
+        "MOV	r6, %[row]\n\t"
+        "LSR	r7, r4, #2\n\t"
+        "\n"
+#if defined(__IAR_SYSTEMS_ICC__) && (__VER__ < 9000000)
+    "L_frodokem_sa_accum_thumb2_k:\n\t"
+#else
+    "L_frodokem_sa_accum_thumb2_k_%=:\n\t"
+#endif
+        "LDR	r11, [%[out]]\n\t"
+        "LDR	r10, [r6], #4\n\t"
+        "SMULBB	r12, r9, r10\n\t"
+        "SMULBT	lr, r9, r10\n\t"
+        "PKHBT	r12, r12, lr, LSL #16\n\t"
+        "SADD16	r11, r11, r12\n\t"
+        "STR	r11, [%[out]], #4\n\t"
+        "SUBS	r7, r7, #1\n\t"
+#if defined(__GNUC__)
+        "BNE	L_frodokem_sa_accum_thumb2_k_%=\n\t"
+#elif defined(__IAR_SYSTEMS_ICC__) && (__VER__ < 9000000)
+        "BNE.N	L_frodokem_sa_accum_thumb2_k\n\t"
+#else
+        "BNE.N	L_frodokem_sa_accum_thumb2_k_%=\n\t"
+#endif
+        "SUBS	r8, r8, #1\n\t"
+#if defined(__GNUC__)
+        "BNE	L_frodokem_sa_accum_thumb2_i_%=\n\t"
+#elif defined(__IAR_SYSTEMS_ICC__) && (__VER__ < 9000000)
+        "BNE.N	L_frodokem_sa_accum_thumb2_i\n\t"
+#else
+        "BNE.N	L_frodokem_sa_accum_thumb2_i_%=\n\t"
+#endif
+#ifndef WOLFSSL_NO_VAR_ASSIGN_REG
+        : [out] "+r" (out), [s] "+r" (s), [row] "+r" (row), [j] "+r" (j),
+          [n] "+r" (n)
+        :
+#else
+        :
+        : [out] "r" (out), [s] "r" (s), [row] "r" (row), [j] "r" (j),
+          [n] "r" (n)
+#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */
+        : "memory", "cc", "r5", "r6", "r7", "r8", "r9", "r10", "r11", "r12",
+            "lr"
+    );
+}
+
+#ifndef WOLFSSL_NO_VAR_ASSIGN_REG
+WC_OMIT_FRAME_POINTER void frodokem_as_accum_thumb2(word16* out_p,
+    const word16* s_p, const word16* row_p, int i_p, int n_p)
+#else
+WC_OMIT_FRAME_POINTER void frodokem_as_accum_thumb2(word16* out,
+    const word16* s, const word16* row, int i, int n)
+#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */
+{
+#ifndef WOLFSSL_NO_VAR_ASSIGN_REG
+    register word16* out __asm__ ("r0") = (word16*)out_p;
+    register const word16* s __asm__ ("r1") = (const word16*)s_p;
+    register const word16* row __asm__ ("r2") = (const word16*)row_p;
+    register int i __asm__ ("r3") = (int)i_p;
+    register int n __asm__ ("r4") = (int)n_p;
+#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */
+
+    __asm__ __volatile__ (
+#ifndef WOLFSSL_NO_VAR_ASSIGN_REG
+#else
+        "MOV	r4, %[n]\n\t"
+#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */
+        "LSL	r4, r4, #1\n\t"
+        "ADD	r5, %[out], %[i], LSL #4\n\t"
+        "MOV	r6, %[s]\n\t"
+        "MOV	r9, #8\n\t"
+        "\n"
+#if defined(__IAR_SYSTEMS_ICC__) && (__VER__ < 9000000)
+    "L_frodokem_as_accum_thumb2_c:\n\t"
+#else
+    "L_frodokem_as_accum_thumb2_c_%=:\n\t"
+#endif
+        "MOV	r11, #0\n\t"
+        "MOV	r7, %[row]\n\t"
+        "MOV	r8, r6\n\t"
+        "LSR	r10, r4, #2\n\t"
+        "\n"
+#if defined(__IAR_SYSTEMS_ICC__) && (__VER__ < 9000000)
+    "L_frodokem_as_accum_thumb2_j:\n\t"
+#else
+    "L_frodokem_as_accum_thumb2_j_%=:\n\t"
+#endif
+        "LDR	r12, [r7], #4\n\t"
+        "LDR	lr, [r8], #4\n\t"
+        "SMLAD	r11, r12, lr, r11\n\t"
+        "SUBS	r10, r10, #1\n\t"
+#if defined(__GNUC__)
+        "BNE	L_frodokem_as_accum_thumb2_j_%=\n\t"
+#elif defined(__IAR_SYSTEMS_ICC__) && (__VER__ < 9000000)
+        "BNE.N	L_frodokem_as_accum_thumb2_j\n\t"
+#else
+        "BNE.N	L_frodokem_as_accum_thumb2_j_%=\n\t"
+#endif
+        "LDRH	r12, [r5]\n\t"
+        "ADD	r12, r12, r11\n\t"
+        "STRH	r12, [r5]\n\t"
+        "ADD	r5, r5, #2\n\t"
+        "ADD	r6, r6, r4\n\t"
+        "SUBS	r9, r9, #1\n\t"
+#if defined(__GNUC__)
+        "BNE	L_frodokem_as_accum_thumb2_c_%=\n\t"
+#elif defined(__IAR_SYSTEMS_ICC__) && (__VER__ < 9000000)
+        "BNE.N	L_frodokem_as_accum_thumb2_c\n\t"
+#else
+        "BNE.N	L_frodokem_as_accum_thumb2_c_%=\n\t"
+#endif
+#ifndef WOLFSSL_NO_VAR_ASSIGN_REG
+        : [out] "+r" (out), [s] "+r" (s), [row] "+r" (row), [i] "+r" (i),
+          [n] "+r" (n)
+        :
+#else
+        :
+        : [out] "r" (out), [s] "r" (s), [row] "r" (row), [i] "r" (i),
+          [n] "r" (n)
+#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */
+        : "memory", "cc", "r5", "r6", "r7", "r8", "r9", "r10", "r11", "r12",
+            "lr"
+    );
+}
+
+#ifndef WOLFSSL_NO_VAR_ASSIGN_REG
+WC_OMIT_FRAME_POINTER void frodokem_mul_bs_thumb2(word16* out_p,
+    const word16* b_p, const word16* s_p, int n_p, int qmask_p)
+#else
+WC_OMIT_FRAME_POINTER void frodokem_mul_bs_thumb2(word16* out, const word16* b,
+    const word16* s, int n, int qmask)
+#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */
+{
+#ifndef WOLFSSL_NO_VAR_ASSIGN_REG
+    register word16* out __asm__ ("r0") = (word16*)out_p;
+    register const word16* b __asm__ ("r1") = (const word16*)b_p;
+    register const word16* s __asm__ ("r2") = (const word16*)s_p;
+    register int n __asm__ ("r3") = (int)n_p;
+    register int qmask __asm__ ("r4") = (int)qmask_p;
+#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */
+
+    __asm__ __volatile__ (
+        "MOV	r4, %[b]\n\t"
+        "MOV	r8, #8\n\t"
+        "\n"
+#if defined(__IAR_SYSTEMS_ICC__) && (__VER__ < 9000000)
+    "L_frodokem_mul_bs_thumb2_i:\n\t"
+#else
+    "L_frodokem_mul_bs_thumb2_i_%=:\n\t"
+#endif
+        "MOV	r5, %[s]\n\t"
+        "MOV	r9, #8\n\t"
+        "\n"
+#if defined(__IAR_SYSTEMS_ICC__) && (__VER__ < 9000000)
+    "L_frodokem_mul_bs_thumb2_c:\n\t"
+#else
+    "L_frodokem_mul_bs_thumb2_c_%=:\n\t"
+#endif
+        "MOV	r11, #0\n\t"
+        "MOV	r6, r4\n\t"
+        "MOV	r7, r5\n\t"
+        "LSR	r10, %[n], #1\n\t"
+        "\n"
+#if defined(__IAR_SYSTEMS_ICC__) && (__VER__ < 9000000)
+    "L_frodokem_mul_bs_thumb2_j:\n\t"
+#else
+    "L_frodokem_mul_bs_thumb2_j_%=:\n\t"
+#endif
+        "LDR	r12, [r6], #4\n\t"
+        "LDR	lr, [r7], #4\n\t"
+        "SMLAD	r11, r12, lr, r11\n\t"
+        "SUBS	r10, r10, #1\n\t"
+#if defined(__GNUC__)
+        "BNE	L_frodokem_mul_bs_thumb2_j_%=\n\t"
+#elif defined(__IAR_SYSTEMS_ICC__) && (__VER__ < 9000000)
+        "BNE.N	L_frodokem_mul_bs_thumb2_j\n\t"
+#else
+        "BNE.N	L_frodokem_mul_bs_thumb2_j_%=\n\t"
+#endif
+#ifndef WOLFSSL_NO_VAR_ASSIGN_REG
+        "MOV	r12, r4\n\t"
+#else
+        "MOV	r12, %[qmask]\n\t"
+#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */
+        "ORR	r12, r12, r12, LSL #16\n\t"
+        "AND	r11, r11, r12\n\t"
+        "STRH	r11, [%[out]]\n\t"
+        "ADD	%[out], %[out], #2\n\t"
+        "ADD	r5, r5, %[n], LSL #1\n\t"
+        "SUBS	r9, r9, #1\n\t"
+#if defined(__GNUC__)
+        "BNE	L_frodokem_mul_bs_thumb2_c_%=\n\t"
+#elif defined(__IAR_SYSTEMS_ICC__) && (__VER__ < 9000000)
+        "BNE.N	L_frodokem_mul_bs_thumb2_c\n\t"
+#else
+        "BNE.N	L_frodokem_mul_bs_thumb2_c_%=\n\t"
+#endif
+        "ADD	r4, r4, %[n], LSL #1\n\t"
+        "SUBS	r8, r8, #1\n\t"
+#if defined(__GNUC__)
+        "BNE	L_frodokem_mul_bs_thumb2_i_%=\n\t"
+#elif defined(__IAR_SYSTEMS_ICC__) && (__VER__ < 9000000)
+        "BNE.N	L_frodokem_mul_bs_thumb2_i\n\t"
+#else
+        "BNE.N	L_frodokem_mul_bs_thumb2_i_%=\n\t"
+#endif
+#ifndef WOLFSSL_NO_VAR_ASSIGN_REG
+        : [out] "+r" (out), [b] "+r" (b), [s] "+r" (s), [n] "+r" (n),
+          [qmask] "+r" (qmask)
+        :
+#else
+        :
+        : [out] "r" (out), [b] "r" (b), [s] "r" (s), [n] "r" (n),
+          [qmask] "r" (qmask)
+#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */
+        : "memory", "cc", "r5", "r6", "r7", "r8", "r9", "r10", "r11", "r12",
+            "lr"
+    );
+}
+
+#ifndef WOLFSSL_NO_VAR_ASSIGN_REG
+WC_OMIT_FRAME_POINTER void frodokem_mul_add_sb_plus_e_thumb2(word16* out_p,
+    const word16* b_p, const word16* s_p, int n_p, int qmask_p)
+#else
+WC_OMIT_FRAME_POINTER void frodokem_mul_add_sb_plus_e_thumb2(word16* out,
+    const word16* b, const word16* s, int n, int qmask)
+#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */
+{
+#ifndef WOLFSSL_NO_VAR_ASSIGN_REG
+    register word16* out __asm__ ("r0") = (word16*)out_p;
+    register const word16* b __asm__ ("r1") = (const word16*)b_p;
+    register const word16* s __asm__ ("r2") = (const word16*)s_p;
+    register int n __asm__ ("r3") = (int)n_p;
+    register int qmask __asm__ ("r4") = (int)qmask_p;
+#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */
+
+    __asm__ __volatile__ (
+        "SUB	sp, sp, #8\n\t"
+        "LSL	r11, %[n], #1\n\t"
+        "STR	r11, [sp, #4]\n\t"
+        "STR	%[b], [sp]\n\t"
+        "MOV	r4, #8\n\t"
+        "\n"
+#if defined(__IAR_SYSTEMS_ICC__) && (__VER__ < 9000000)
+    "L_frodokem_mul_add_sb_plus_e_thumb2_i:\n\t"
+#else
+    "L_frodokem_mul_add_sb_plus_e_thumb2_i_%=:\n\t"
+#endif
+        "LDR	r11, [sp, #4]\n\t"
+        "ADD	%[n], %[s], r11\n\t"
+        "LDR	%[b], [sp]\n\t"
+        "MOV	r5, %[s]\n\t"
+        "LDR	r6, [%[out]]\n\t"
+        "LDR	r7, [%[out], #4]\n\t"
+        "LDR	r8, [%[out], #8]\n\t"
+        "LDR	r9, [%[out], #12]\n\t"
+        "\n"
+#if defined(__IAR_SYSTEMS_ICC__) && (__VER__ < 9000000)
+    "L_frodokem_mul_add_sb_plus_e_thumb2_j:\n\t"
+#else
+    "L_frodokem_mul_add_sb_plus_e_thumb2_j_%=:\n\t"
+#endif
+        "LDRH	r10, [r5]\n\t"
+        "ADD	r5, r5, #2\n\t"
+        "LDR	r11, [%[b]]\n\t"
+        "SMULBB	r12, r10, r11\n\t"
+        "SMULBT	lr, r10, r11\n\t"
+        "PKHBT	r12, r12, lr, LSL #16\n\t"
+        "SADD16	r6, r6, r12\n\t"
+        "LDR	r11, [%[b], #4]\n\t"
+        "SMULBB	r12, r10, r11\n\t"
+        "SMULBT	lr, r10, r11\n\t"
+        "PKHBT	r12, r12, lr, LSL #16\n\t"
+        "SADD16	r7, r7, r12\n\t"
+        "LDR	r11, [%[b], #8]\n\t"
+        "SMULBB	r12, r10, r11\n\t"
+        "SMULBT	lr, r10, r11\n\t"
+        "PKHBT	r12, r12, lr, LSL #16\n\t"
+        "SADD16	r8, r8, r12\n\t"
+        "LDR	r11, [%[b], #12]\n\t"
+        "SMULBB	r12, r10, r11\n\t"
+        "SMULBT	lr, r10, r11\n\t"
+        "PKHBT	r12, r12, lr, LSL #16\n\t"
+        "SADD16	r9, r9, r12\n\t"
+        "ADD	%[b], %[b], #16\n\t"
+        "CMP	r5, %[n]\n\t"
+#if defined(__GNUC__)
+        "BNE	L_frodokem_mul_add_sb_plus_e_thumb2_j_%=\n\t"
+#elif defined(__IAR_SYSTEMS_ICC__) && (__VER__ < 9000000)
+        "BNE.N	L_frodokem_mul_add_sb_plus_e_thumb2_j\n\t"
+#else
+        "BNE.N	L_frodokem_mul_add_sb_plus_e_thumb2_j_%=\n\t"
+#endif
+#ifndef WOLFSSL_NO_VAR_ASSIGN_REG
+        "MOV	r11, r4\n\t"
+#else
+        "MOV	r11, %[qmask]\n\t"
+#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */
+        "ORR	r11, r11, r11, LSL #16\n\t"
+        "AND	r6, r6, r11\n\t"
+        "AND	r7, r7, r11\n\t"
+        "AND	r8, r8, r11\n\t"
+        "AND	r9, r9, r11\n\t"
+        "STR	r6, [%[out]]\n\t"
+        "STR	r7, [%[out], #4]\n\t"
+        "STR	r8, [%[out], #8]\n\t"
+        "STR	r9, [%[out], #12]\n\t"
+        "ADD	%[out], %[out], #16\n\t"
+        "MOV	%[s], %[n]\n\t"
+        "SUBS	r4, r4, #1\n\t"
+#if defined(__GNUC__)
+        "BNE	L_frodokem_mul_add_sb_plus_e_thumb2_i_%=\n\t"
+#elif defined(__IAR_SYSTEMS_ICC__) && (__VER__ < 9000000)
+        "BNE.N	L_frodokem_mul_add_sb_plus_e_thumb2_i\n\t"
+#else
+        "BNE.N	L_frodokem_mul_add_sb_plus_e_thumb2_i_%=\n\t"
+#endif
+        "ADD	sp, sp, #8\n\t"
+#ifndef WOLFSSL_NO_VAR_ASSIGN_REG
+        : [out] "+r" (out), [b] "+r" (b), [s] "+r" (s), [n] "+r" (n),
+          [qmask] "+r" (qmask)
+        :
+#else
+        :
+        : [out] "r" (out), [b] "r" (b), [s] "r" (s), [n] "r" (n),
+          [qmask] "r" (qmask)
+#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */
+        : "memory", "cc", "r5", "r6", "r7", "r8", "r9", "r10", "r11", "r12",
+            "lr"
+    );
+}
+
+#endif /* WOLFSSL_HAVE_FRODOKEM */
+#endif /* WOLFSSL_ARMASM_THUMB2 */
+#endif /* WOLFSSL_ARMASM */
+
+#endif /* WOLFSSL_ARMASM_INLINE */
diff --git a/wolfcrypt/src/sha3_asm.S b/wolfcrypt/src/sha3_asm.S
index 810a1c6743..ce57189e8e 100644
--- a/wolfcrypt/src/sha3_asm.S
+++ b/wolfcrypt/src/sha3_asm.S
@@ -9938,7 +9938,7 @@ L_sha3_block_n_avx2_rounds:
 #ifndef __APPLE__
 .size	sha3_block_n_avx2,.-sha3_block_n_avx2
 #endif /* __APPLE__ */
-#if defined(WOLFSSL_HAVE_MLKEM) || defined(WOLFSSL_HAVE_MLDSA) || defined(WOLFSSL_HAVE_SLHDSA)
+#if defined(WOLFSSL_HAVE_MLKEM) || defined(WOLFSSL_HAVE_MLDSA) || defined(WOLFSSL_HAVE_SLHDSA) || defined(WOLFSSL_HAVE_FRODOKEM)
 #ifndef __APPLE__
 .text
 .globl	sha3_blocksx4_avx2
@@ -15297,6 +15297,8 @@ _sha3_blocksx4_avx2:
 #ifndef __APPLE__
 .size	sha3_blocksx4_avx2,.-sha3_blocksx4_avx2
 #endif /* __APPLE__ */
+#endif /* defined(WOLFSSL_HAVE_MLKEM) || defined(WOLFSSL_HAVE_MLDSA) || defined(WOLFSSL_HAVE_SLHDSA) || defined(WOLFSSL_HAVE_FRODOKEM) */
+#if defined(WOLFSSL_HAVE_MLKEM) || defined(WOLFSSL_HAVE_MLDSA) || defined(WOLFSSL_HAVE_SLHDSA)
 #ifndef __APPLE__
 .data
 #else
@@ -20675,72 +20677,65 @@ _sha3_128_blocksx4_seed_avx2:
 .size	sha3_128_blocksx4_seed_avx2,.-sha3_128_blocksx4_seed_avx2
 #endif /* __APPLE__ */
 #endif /* defined(WOLFSSL_HAVE_MLKEM) || defined(WOLFSSL_HAVE_MLDSA) || defined(WOLFSSL_HAVE_SLHDSA) */
-#ifdef WOLFSSL_HAVE_MLKEM
-#ifndef __APPLE__
-.data
-#else
-.section	__DATA,__data
-#endif /* __APPLE__ */
-#ifndef __APPLE__
-.align	32
-#else
-.p2align	5
-#endif /* __APPLE__ */
-L_sha3_256_blockx4_seed_avx2_end_mark:
-.quad	0x8000000000000000,0x8000000000000000
-.quad	0x8000000000000000,0x8000000000000000
+#ifdef WOLFSSL_HAVE_FRODOKEM
 #ifndef __APPLE__
 .text
-.globl	sha3_256_blocksx4_seed_avx2
-.type	sha3_256_blocksx4_seed_avx2,@function
+.globl	sha3_blocksx4_out_avx2
+.type	sha3_blocksx4_out_avx2,@function
 .align	16
-sha3_256_blocksx4_seed_avx2:
+sha3_blocksx4_out_avx2:
 #else
 .section	__TEXT,__text
-.globl	_sha3_256_blocksx4_seed_avx2
+.globl	_sha3_blocksx4_out_avx2
 .p2align	4
-_sha3_256_blocksx4_seed_avx2:
+_sha3_blocksx4_out_avx2:
 #endif /* __APPLE__ */
-        leaq	L_sha3_x4_avx2_r(%rip), %rdx
-        movq	%rdi, %rax
+        pushq	%r12
+        pushq	%r13
+        pushq	%r14
+        pushq	%r15
+        leaq	L_sha3_x4_avx2_r(%rip), %rax
+        movl	%edx, %r12d
+        movq	%rsi, %r9
+        addq	%r12, %r9
+        movq	%r9, %r10
+        addq	%r12, %r10
+        movq	%r10, %r11
+        addq	%r12, %r11
+        shrq	$3, %r12
+L_sha3_blocksx4_out_avx2_block:
+        vmovdqu	(%rdi), %ymm15
         movq	%rdi, %rcx
-        vpbroadcastq	(%rsi), %ymm15
+        movq	%rdi, %r8
         addq	$0x80, %rdi
-        vpbroadcastq	8(%rsi), %ymm11
-        addq	$0x180, %rax
-        vpbroadcastq	16(%rsi), %ymm12
-        addq	$0x280, %rcx
-        vpbroadcastq	24(%rsi), %ymm13
-        vmovdqu	L_sha3_256_blockx4_seed_avx2_end_mark(%rip), %ymm5
-        vpxor	%ymm6, %ymm6, %ymm6
-        vmovdqu	%ymm11, -96(%rdi)
-        vmovdqu	%ymm12, -64(%rdi)
-        vmovdqu	%ymm13, -32(%rdi)
-        vmovdqu	(%rdi), %ymm14
-        vmovdqu	%ymm6, 32(%rdi)
-        vmovdqu	%ymm6, 64(%rdi)
-        vmovdqu	%ymm6, 96(%rdi)
-        vmovdqu	%ymm6, 128(%rdi)
-        vmovdqu	%ymm6, -96(%rax)
-        vmovdqu	%ymm6, -64(%rax)
-        vmovdqu	%ymm6, -32(%rax)
-        vmovdqu	%ymm6, (%rax)
-        vmovdqu	%ymm6, 32(%rax)
-        vmovdqu	%ymm6, 64(%rax)
-        vmovdqu	%ymm6, 96(%rax)
-        vmovdqu	%ymm5, 128(%rax)
-        vmovdqu	%ymm6, -96(%rcx)
-        vmovdqu	%ymm6, -64(%rcx)
-        vmovdqu	%ymm6, -32(%rcx)
-        vmovdqu	%ymm6, (%rcx)
-        vmovdqu	%ymm6, 32(%rcx)
-        vmovdqu	%ymm6, 64(%rcx)
-        vmovdqu	%ymm6, 96(%rcx)
-        vmovdqu	%ymm6, 128(%rcx)
-        vmovdqu	%ymm15, %ymm10
-        vpxor	%ymm5, %ymm11, %ymm11
+        addq	$0x180, %rcx
+        addq	$0x280, %r8
         # Round 0
         # Calc b[0..4]
+        vmovdqu	-96(%rdi), %ymm11
+        vmovdqu	-64(%rdi), %ymm12
+        vmovdqu	-32(%rdi), %ymm13
+        vmovdqu	(%rdi), %ymm14
+        vpxor	32(%rdi), %ymm15, %ymm10
+        vpxor	64(%rdi), %ymm11, %ymm11
+        vpxor	96(%rdi), %ymm12, %ymm12
+        vpxor	128(%rdi), %ymm13, %ymm13
+        vpxor	-96(%rcx), %ymm14, %ymm14
+        vpxor	-64(%rcx), %ymm10, %ymm10
+        vpxor	-32(%rcx), %ymm11, %ymm11
+        vpxor	(%rcx), %ymm12, %ymm12
+        vpxor	32(%rcx), %ymm13, %ymm13
+        vpxor	64(%rcx), %ymm14, %ymm14
+        vpxor	96(%rcx), %ymm10, %ymm10
+        vpxor	128(%rcx), %ymm11, %ymm11
+        vpxor	-96(%r8), %ymm12, %ymm12
+        vpxor	-64(%r8), %ymm13, %ymm13
+        vpxor	-32(%r8), %ymm14, %ymm14
+        vpxor	(%r8), %ymm10, %ymm10
+        vpxor	32(%r8), %ymm11, %ymm11
+        vpxor	64(%r8), %ymm12, %ymm12
+        vpxor	96(%r8), %ymm13, %ymm13
+        vpxor	128(%r8), %ymm14, %ymm14
         # Calc t[0..4]
         vpsrlq	$63, %ymm11, %ymm0
         vpsrlq	$63, %ymm12, %ymm1
@@ -20766,9 +20761,9 @@ _sha3_256_blocksx4_seed_avx2:
         # Row 0
         vpxor	%ymm15, %ymm5, %ymm10
         vpxor	64(%rdi), %ymm6, %ymm11
-        vpxor	(%rax), %ymm7, %ymm12
-        vpxor	-64(%rcx), %ymm8, %ymm13
-        vpxor	128(%rcx), %ymm9, %ymm14
+        vpxor	(%rcx), %ymm7, %ymm12
+        vpxor	-64(%r8), %ymm8, %ymm13
+        vpxor	128(%r8), %ymm9, %ymm14
         vpsrlq	$20, %ymm11, %ymm0
         vpsrlq	$21, %ymm12, %ymm1
         vpsrlq	$43, %ymm13, %ymm2
@@ -20790,19 +20785,19 @@ _sha3_256_blocksx4_seed_avx2:
         vpxor	%ymm11, %ymm1, %ymm1
         vpxor	%ymm12, %ymm2, %ymm2
         # XOR in constant
-        vpxor	(%rdx), %ymm15, %ymm15
+        vpxor	(%rax), %ymm15, %ymm15
         vpxor	%ymm13, %ymm3, %ymm3
         vpxor	%ymm14, %ymm4, %ymm4
         vmovdqu	%ymm1, 64(%rdi)
-        vmovdqu	%ymm2, (%rax)
-        vmovdqu	%ymm3, -64(%rcx)
-        vmovdqu	%ymm4, 128(%rcx)
+        vmovdqu	%ymm2, (%rcx)
+        vmovdqu	%ymm3, -64(%r8)
+        vmovdqu	%ymm4, 128(%r8)
         # Row 1
         vpxor	-32(%rdi), %ymm8, %ymm10
-        vpxor	-96(%rax), %ymm9, %ymm11
-        vpxor	-64(%rax), %ymm5, %ymm12
-        vpxor	128(%rax), %ymm6, %ymm13
-        vpxor	64(%rcx), %ymm7, %ymm14
+        vpxor	-96(%rcx), %ymm9, %ymm11
+        vpxor	-64(%rcx), %ymm5, %ymm12
+        vpxor	128(%rcx), %ymm6, %ymm13
+        vpxor	64(%r8), %ymm7, %ymm14
         vpsrlq	$36, %ymm10, %ymm0
         vpsrlq	$44, %ymm11, %ymm1
         vpsrlq	$61, %ymm12, %ymm2
@@ -20829,16 +20824,16 @@ _sha3_256_blocksx4_seed_avx2:
         vpxor	%ymm13, %ymm3, %ymm3
         vpxor	%ymm14, %ymm4, %ymm4
         vmovdqu	%ymm0, -32(%rdi)
-        vmovdqu	%ymm1, -96(%rax)
-        vmovdqu	%ymm2, -64(%rax)
-        vmovdqu	%ymm3, 128(%rax)
-        vmovdqu	%ymm4, 64(%rcx)
+        vmovdqu	%ymm1, -96(%rcx)
+        vmovdqu	%ymm2, -64(%rcx)
+        vmovdqu	%ymm3, 128(%rcx)
+        vmovdqu	%ymm4, 64(%r8)
         # Row 2
         vpxor	-96(%rdi), %ymm6, %ymm10
         vpxor	96(%rdi), %ymm7, %ymm11
-        vpxor	32(%rax), %ymm8, %ymm12
-        vpxor	-32(%rcx), %ymm9, %ymm13
-        vpxor	(%rcx), %ymm5, %ymm14
+        vpxor	32(%rcx), %ymm8, %ymm12
+        vpxor	-32(%r8), %ymm9, %ymm13
+        vpxor	(%r8), %ymm5, %ymm14
         vpsrlq	$63, %ymm10, %ymm0
         vpsrlq	$58, %ymm11, %ymm1
         vpsrlq	$39, %ymm12, %ymm2
@@ -20866,15 +20861,15 @@ _sha3_256_blocksx4_seed_avx2:
         vpxor	%ymm14, %ymm4, %ymm4
         vmovdqu	%ymm0, -96(%rdi)
         vmovdqu	%ymm1, 96(%rdi)
-        vmovdqu	%ymm2, 32(%rax)
-        vmovdqu	%ymm3, -32(%rcx)
-        vmovdqu	%ymm4, (%rcx)
+        vmovdqu	%ymm2, 32(%rcx)
+        vmovdqu	%ymm3, -32(%r8)
+        vmovdqu	%ymm4, (%r8)
         # Row 3
         vpxor	(%rdi), %ymm9, %ymm10
         vpxor	32(%rdi), %ymm5, %ymm11
-        vpxor	-32(%rax), %ymm6, %ymm12
-        vpxor	-96(%rcx), %ymm7, %ymm13
-        vpxor	96(%rcx), %ymm8, %ymm14
+        vpxor	-32(%rcx), %ymm6, %ymm12
+        vpxor	-96(%r8), %ymm7, %ymm13
+        vpxor	96(%r8), %ymm8, %ymm14
         vpsrlq	$37, %ymm10, %ymm0
         vpsrlq	$28, %ymm11, %ymm1
         vpsrlq	$54, %ymm12, %ymm2
@@ -20902,15 +20897,15 @@ _sha3_256_blocksx4_seed_avx2:
         vpxor	%ymm14, %ymm4, %ymm4
         vmovdqu	%ymm0, (%rdi)
         vmovdqu	%ymm1, 32(%rdi)
-        vmovdqu	%ymm2, -32(%rax)
-        vmovdqu	%ymm3, -96(%rcx)
-        vmovdqu	%ymm4, 96(%rcx)
+        vmovdqu	%ymm2, -32(%rcx)
+        vmovdqu	%ymm3, -96(%r8)
+        vmovdqu	%ymm4, 96(%r8)
         # Row 4
         vpxor	-64(%rdi), %ymm7, %ymm10
         vpxor	128(%rdi), %ymm8, %ymm11
-        vpxor	64(%rax), %ymm9, %ymm12
-        vpxor	96(%rax), %ymm5, %ymm13
-        vpxor	32(%rcx), %ymm6, %ymm14
+        vpxor	64(%rcx), %ymm9, %ymm12
+        vpxor	96(%rcx), %ymm5, %ymm13
+        vpxor	32(%r8), %ymm6, %ymm14
         vpsrlq	$2, %ymm10, %ymm0
         vpsrlq	$9, %ymm11, %ymm1
         vpsrlq	$25, %ymm12, %ymm2
@@ -20938,9 +20933,9 @@ _sha3_256_blocksx4_seed_avx2:
         vpxor	%ymm14, %ymm4, %ymm4
         vmovdqu	%ymm0, -64(%rdi)
         vmovdqu	%ymm1, 128(%rdi)
-        vmovdqu	%ymm2, 64(%rax)
-        vmovdqu	%ymm3, 96(%rax)
-        vmovdqu	%ymm4, 32(%rcx)
+        vmovdqu	%ymm2, 64(%rcx)
+        vmovdqu	%ymm3, 96(%rcx)
+        vmovdqu	%ymm4, 32(%r8)
         # Round 1
         # Calc b[0..4]
         vpxor	%ymm15, %ymm0, %ymm10
@@ -20950,19 +20945,19 @@ _sha3_256_blocksx4_seed_avx2:
         vpxor	32(%rdi), %ymm1, %ymm11
         vpxor	64(%rdi), %ymm11, %ymm11
         vpxor	96(%rdi), %ymm11, %ymm11
-        vpxor	-96(%rax), %ymm11, %ymm11
-        vpxor	-64(%rax), %ymm2, %ymm12
-        vpxor	-32(%rax), %ymm12, %ymm12
-        vpxor	(%rax), %ymm12, %ymm12
-        vpxor	32(%rax), %ymm12, %ymm12
-        vpxor	128(%rax), %ymm3, %ymm13
-        vpxor	-96(%rcx), %ymm13, %ymm13
-        vpxor	-64(%rcx), %ymm13, %ymm13
-        vpxor	-32(%rcx), %ymm13, %ymm13
-        vpxor	(%rcx), %ymm4, %ymm14
-        vpxor	64(%rcx), %ymm14, %ymm14
-        vpxor	96(%rcx), %ymm14, %ymm14
-        vpxor	128(%rcx), %ymm14, %ymm14
+        vpxor	-96(%rcx), %ymm11, %ymm11
+        vpxor	-64(%rcx), %ymm2, %ymm12
+        vpxor	-32(%rcx), %ymm12, %ymm12
+        vpxor	(%rcx), %ymm12, %ymm12
+        vpxor	32(%rcx), %ymm12, %ymm12
+        vpxor	128(%rcx), %ymm3, %ymm13
+        vpxor	-96(%r8), %ymm13, %ymm13
+        vpxor	-64(%r8), %ymm13, %ymm13
+        vpxor	-32(%r8), %ymm13, %ymm13
+        vpxor	(%r8), %ymm4, %ymm14
+        vpxor	64(%r8), %ymm14, %ymm14
+        vpxor	96(%r8), %ymm14, %ymm14
+        vpxor	128(%r8), %ymm14, %ymm14
         # Calc t[0..4]
         vpsrlq	$63, %ymm11, %ymm0
         vpsrlq	$63, %ymm12, %ymm1
@@ -20987,10 +20982,10 @@ _sha3_256_blocksx4_seed_avx2:
         # Row Mix
         # Row 0
         vpxor	%ymm15, %ymm5, %ymm10
-        vpxor	-96(%rax), %ymm6, %ymm11
-        vpxor	32(%rax), %ymm7, %ymm12
-        vpxor	-96(%rcx), %ymm8, %ymm13
-        vpxor	32(%rcx), %ymm9, %ymm14
+        vpxor	-96(%rcx), %ymm6, %ymm11
+        vpxor	32(%rcx), %ymm7, %ymm12
+        vpxor	-96(%r8), %ymm8, %ymm13
+        vpxor	32(%r8), %ymm9, %ymm14
         vpsrlq	$20, %ymm11, %ymm0
         vpsrlq	$21, %ymm12, %ymm1
         vpsrlq	$43, %ymm13, %ymm2
@@ -21012,19 +21007,19 @@ _sha3_256_blocksx4_seed_avx2:
         vpxor	%ymm11, %ymm1, %ymm1
         vpxor	%ymm12, %ymm2, %ymm2
         # XOR in constant
-        vpxor	32(%rdx), %ymm15, %ymm15
+        vpxor	32(%rax), %ymm15, %ymm15
         vpxor	%ymm13, %ymm3, %ymm3
         vpxor	%ymm14, %ymm4, %ymm4
-        vmovdqu	%ymm1, -96(%rax)
-        vmovdqu	%ymm2, 32(%rax)
-        vmovdqu	%ymm3, -96(%rcx)
-        vmovdqu	%ymm4, 32(%rcx)
+        vmovdqu	%ymm1, -96(%rcx)
+        vmovdqu	%ymm2, 32(%rcx)
+        vmovdqu	%ymm3, -96(%r8)
+        vmovdqu	%ymm4, 32(%r8)
         # Row 1
-        vpxor	-64(%rcx), %ymm8, %ymm10
-        vpxor	64(%rcx), %ymm9, %ymm11
+        vpxor	-64(%r8), %ymm8, %ymm10
+        vpxor	64(%r8), %ymm9, %ymm11
         vpxor	-96(%rdi), %ymm5, %ymm12
         vpxor	32(%rdi), %ymm6, %ymm13
-        vpxor	64(%rax), %ymm7, %ymm14
+        vpxor	64(%rcx), %ymm7, %ymm14
         vpsrlq	$36, %ymm10, %ymm0
         vpsrlq	$44, %ymm11, %ymm1
         vpsrlq	$61, %ymm12, %ymm2
@@ -21050,16 +21045,16 @@ _sha3_256_blocksx4_seed_avx2:
         vpxor	%ymm12, %ymm2, %ymm2
         vpxor	%ymm13, %ymm3, %ymm3
         vpxor	%ymm14, %ymm4, %ymm4
-        vmovdqu	%ymm0, -64(%rcx)
-        vmovdqu	%ymm1, 64(%rcx)
+        vmovdqu	%ymm0, -64(%r8)
+        vmovdqu	%ymm1, 64(%r8)
         vmovdqu	%ymm2, -96(%rdi)
         vmovdqu	%ymm3, 32(%rdi)
-        vmovdqu	%ymm4, 64(%rax)
+        vmovdqu	%ymm4, 64(%rcx)
         # Row 2
         vpxor	64(%rdi), %ymm6, %ymm10
-        vpxor	-64(%rax), %ymm7, %ymm11
-        vpxor	-32(%rcx), %ymm8, %ymm12
-        vpxor	96(%rcx), %ymm9, %ymm13
+        vpxor	-64(%rcx), %ymm7, %ymm11
+        vpxor	-32(%r8), %ymm8, %ymm12
+        vpxor	96(%r8), %ymm9, %ymm13
         vpxor	-64(%rdi), %ymm5, %ymm14
         vpsrlq	$63, %ymm10, %ymm0
         vpsrlq	$58, %ymm11, %ymm1
@@ -21087,16 +21082,16 @@ _sha3_256_blocksx4_seed_avx2:
         vpxor	%ymm13, %ymm3, %ymm3
         vpxor	%ymm14, %ymm4, %ymm4
         vmovdqu	%ymm0, 64(%rdi)
-        vmovdqu	%ymm1, -64(%rax)
-        vmovdqu	%ymm2, -32(%rcx)
-        vmovdqu	%ymm3, 96(%rcx)
+        vmovdqu	%ymm1, -64(%rcx)
+        vmovdqu	%ymm2, -32(%r8)
+        vmovdqu	%ymm3, 96(%r8)
         vmovdqu	%ymm4, -64(%rdi)
         # Row 3
-        vpxor	128(%rcx), %ymm9, %ymm10
+        vpxor	128(%r8), %ymm9, %ymm10
         vpxor	-32(%rdi), %ymm5, %ymm11
         vpxor	96(%rdi), %ymm6, %ymm12
-        vpxor	-32(%rax), %ymm7, %ymm13
-        vpxor	96(%rax), %ymm8, %ymm14
+        vpxor	-32(%rcx), %ymm7, %ymm13
+        vpxor	96(%rcx), %ymm8, %ymm14
         vpsrlq	$37, %ymm10, %ymm0
         vpsrlq	$28, %ymm11, %ymm1
         vpsrlq	$54, %ymm12, %ymm2
@@ -21122,15 +21117,15 @@ _sha3_256_blocksx4_seed_avx2:
         vpxor	%ymm12, %ymm2, %ymm2
         vpxor	%ymm13, %ymm3, %ymm3
         vpxor	%ymm14, %ymm4, %ymm4
-        vmovdqu	%ymm0, 128(%rcx)
+        vmovdqu	%ymm0, 128(%r8)
         vmovdqu	%ymm1, -32(%rdi)
         vmovdqu	%ymm2, 96(%rdi)
-        vmovdqu	%ymm3, -32(%rax)
-        vmovdqu	%ymm4, 96(%rax)
+        vmovdqu	%ymm3, -32(%rcx)
+        vmovdqu	%ymm4, 96(%rcx)
         # Row 4
-        vpxor	(%rax), %ymm7, %ymm10
-        vpxor	128(%rax), %ymm8, %ymm11
-        vpxor	(%rcx), %ymm9, %ymm12
+        vpxor	(%rcx), %ymm7, %ymm10
+        vpxor	128(%rcx), %ymm8, %ymm11
+        vpxor	(%r8), %ymm9, %ymm12
         vpxor	(%rdi), %ymm5, %ymm13
         vpxor	128(%rdi), %ymm6, %ymm14
         vpsrlq	$2, %ymm10, %ymm0
@@ -21158,9 +21153,9 @@ _sha3_256_blocksx4_seed_avx2:
         vpxor	%ymm12, %ymm2, %ymm2
         vpxor	%ymm13, %ymm3, %ymm3
         vpxor	%ymm14, %ymm4, %ymm4
-        vmovdqu	%ymm0, (%rax)
-        vmovdqu	%ymm1, 128(%rax)
-        vmovdqu	%ymm2, (%rcx)
+        vmovdqu	%ymm0, (%rcx)
+        vmovdqu	%ymm1, 128(%rcx)
+        vmovdqu	%ymm2, (%r8)
         vmovdqu	%ymm3, (%rdi)
         vmovdqu	%ymm4, 128(%rdi)
         # Round 2
@@ -21172,19 +21167,19 @@ _sha3_256_blocksx4_seed_avx2:
         vpxor	32(%rdi), %ymm3, %ymm13
         vpxor	64(%rdi), %ymm10, %ymm10
         vpxor	96(%rdi), %ymm12, %ymm12
-        vpxor	-96(%rax), %ymm11, %ymm11
-        vpxor	-64(%rax), %ymm11, %ymm11
-        vpxor	-32(%rax), %ymm13, %ymm13
-        vpxor	32(%rax), %ymm12, %ymm12
-        vpxor	64(%rax), %ymm14, %ymm14
-        vpxor	96(%rax), %ymm14, %ymm14
-        vpxor	-96(%rcx), %ymm13, %ymm13
-        vpxor	-64(%rcx), %ymm10, %ymm10
-        vpxor	-32(%rcx), %ymm12, %ymm12
-        vpxor	32(%rcx), %ymm14, %ymm14
-        vpxor	64(%rcx), %ymm11, %ymm11
-        vpxor	96(%rcx), %ymm13, %ymm13
-        vpxor	128(%rcx), %ymm10, %ymm10
+        vpxor	-96(%rcx), %ymm11, %ymm11
+        vpxor	-64(%rcx), %ymm11, %ymm11
+        vpxor	-32(%rcx), %ymm13, %ymm13
+        vpxor	32(%rcx), %ymm12, %ymm12
+        vpxor	64(%rcx), %ymm14, %ymm14
+        vpxor	96(%rcx), %ymm14, %ymm14
+        vpxor	-96(%r8), %ymm13, %ymm13
+        vpxor	-64(%r8), %ymm10, %ymm10
+        vpxor	-32(%r8), %ymm12, %ymm12
+        vpxor	32(%r8), %ymm14, %ymm14
+        vpxor	64(%r8), %ymm11, %ymm11
+        vpxor	96(%r8), %ymm13, %ymm13
+        vpxor	128(%r8), %ymm10, %ymm10
         # Calc t[0..4]
         vpsrlq	$63, %ymm11, %ymm0
         vpsrlq	$63, %ymm12, %ymm1
@@ -21209,9 +21204,9 @@ _sha3_256_blocksx4_seed_avx2:
         # Row Mix
         # Row 0
         vpxor	%ymm15, %ymm5, %ymm10
-        vpxor	64(%rcx), %ymm6, %ymm11
-        vpxor	-32(%rcx), %ymm7, %ymm12
-        vpxor	-32(%rax), %ymm8, %ymm13
+        vpxor	64(%r8), %ymm6, %ymm11
+        vpxor	-32(%r8), %ymm7, %ymm12
+        vpxor	-32(%rcx), %ymm8, %ymm13
         vpxor	128(%rdi), %ymm9, %ymm14
         vpsrlq	$20, %ymm11, %ymm0
         vpsrlq	$21, %ymm12, %ymm1
@@ -21234,19 +21229,19 @@ _sha3_256_blocksx4_seed_avx2:
         vpxor	%ymm11, %ymm1, %ymm1
         vpxor	%ymm12, %ymm2, %ymm2
         # XOR in constant
-        vpxor	64(%rdx), %ymm15, %ymm15
+        vpxor	64(%rax), %ymm15, %ymm15
         vpxor	%ymm13, %ymm3, %ymm3
         vpxor	%ymm14, %ymm4, %ymm4
-        vmovdqu	%ymm1, 64(%rcx)
-        vmovdqu	%ymm2, -32(%rcx)
-        vmovdqu	%ymm3, -32(%rax)
+        vmovdqu	%ymm1, 64(%r8)
+        vmovdqu	%ymm2, -32(%r8)
+        vmovdqu	%ymm3, -32(%rcx)
         vmovdqu	%ymm4, 128(%rdi)
         # Row 1
-        vpxor	-96(%rcx), %ymm8, %ymm10
-        vpxor	64(%rax), %ymm9, %ymm11
+        vpxor	-96(%r8), %ymm8, %ymm10
+        vpxor	64(%rcx), %ymm9, %ymm11
         vpxor	64(%rdi), %ymm5, %ymm12
         vpxor	-32(%rdi), %ymm6, %ymm13
-        vpxor	(%rcx), %ymm7, %ymm14
+        vpxor	(%r8), %ymm7, %ymm14
         vpsrlq	$36, %ymm10, %ymm0
         vpsrlq	$44, %ymm11, %ymm1
         vpsrlq	$61, %ymm12, %ymm2
@@ -21272,17 +21267,17 @@ _sha3_256_blocksx4_seed_avx2:
         vpxor	%ymm12, %ymm2, %ymm2
         vpxor	%ymm13, %ymm3, %ymm3
         vpxor	%ymm14, %ymm4, %ymm4
-        vmovdqu	%ymm0, -96(%rcx)
-        vmovdqu	%ymm1, 64(%rax)
+        vmovdqu	%ymm0, -96(%r8)
+        vmovdqu	%ymm1, 64(%rcx)
         vmovdqu	%ymm2, 64(%rdi)
         vmovdqu	%ymm3, -32(%rdi)
-        vmovdqu	%ymm4, (%rcx)
+        vmovdqu	%ymm4, (%r8)
         # Row 2
-        vpxor	-96(%rax), %ymm6, %ymm10
+        vpxor	-96(%rcx), %ymm6, %ymm10
         vpxor	-96(%rdi), %ymm7, %ymm11
-        vpxor	96(%rcx), %ymm8, %ymm12
-        vpxor	96(%rax), %ymm9, %ymm13
-        vpxor	(%rax), %ymm5, %ymm14
+        vpxor	96(%r8), %ymm8, %ymm12
+        vpxor	96(%rcx), %ymm9, %ymm13
+        vpxor	(%rcx), %ymm5, %ymm14
         vpsrlq	$63, %ymm10, %ymm0
         vpsrlq	$58, %ymm11, %ymm1
         vpsrlq	$39, %ymm12, %ymm2
@@ -21308,15 +21303,15 @@ _sha3_256_blocksx4_seed_avx2:
         vpxor	%ymm12, %ymm2, %ymm2
         vpxor	%ymm13, %ymm3, %ymm3
         vpxor	%ymm14, %ymm4, %ymm4
-        vmovdqu	%ymm0, -96(%rax)
+        vmovdqu	%ymm0, -96(%rcx)
         vmovdqu	%ymm1, -96(%rdi)
-        vmovdqu	%ymm2, 96(%rcx)
-        vmovdqu	%ymm3, 96(%rax)
-        vmovdqu	%ymm4, (%rax)
+        vmovdqu	%ymm2, 96(%r8)
+        vmovdqu	%ymm3, 96(%rcx)
+        vmovdqu	%ymm4, (%rcx)
         # Row 3
-        vpxor	32(%rcx), %ymm9, %ymm10
-        vpxor	-64(%rcx), %ymm5, %ymm11
-        vpxor	-64(%rax), %ymm6, %ymm12
+        vpxor	32(%r8), %ymm9, %ymm10
+        vpxor	-64(%r8), %ymm5, %ymm11
+        vpxor	-64(%rcx), %ymm6, %ymm12
         vpxor	96(%rdi), %ymm7, %ymm13
         vpxor	(%rdi), %ymm8, %ymm14
         vpsrlq	$37, %ymm10, %ymm0
@@ -21344,17 +21339,5423 @@ _sha3_256_blocksx4_seed_avx2:
         vpxor	%ymm12, %ymm2, %ymm2
         vpxor	%ymm13, %ymm3, %ymm3
         vpxor	%ymm14, %ymm4, %ymm4
-        vmovdqu	%ymm0, 32(%rcx)
-        vmovdqu	%ymm1, -64(%rcx)
-        vmovdqu	%ymm2, -64(%rax)
+        vmovdqu	%ymm0, 32(%r8)
+        vmovdqu	%ymm1, -64(%r8)
+        vmovdqu	%ymm2, -64(%rcx)
         vmovdqu	%ymm3, 96(%rdi)
         vmovdqu	%ymm4, (%rdi)
         # Row 4
-        vpxor	32(%rax), %ymm7, %ymm10
+        vpxor	32(%rcx), %ymm7, %ymm10
         vpxor	32(%rdi), %ymm8, %ymm11
         vpxor	-64(%rdi), %ymm9, %ymm12
-        vpxor	128(%rcx), %ymm5, %ymm13
-        vpxor	128(%rax), %ymm6, %ymm14
+        vpxor	128(%r8), %ymm5, %ymm13
+        vpxor	128(%rcx), %ymm6, %ymm14
+        vpsrlq	$2, %ymm10, %ymm0
+        vpsrlq	$9, %ymm11, %ymm1
+        vpsrlq	$25, %ymm12, %ymm2
+        vpsrlq	$23, %ymm13, %ymm3
+        vpsrlq	$62, %ymm14, %ymm4
+        vpsllq	$62, %ymm10, %ymm10
+        vpsllq	$55, %ymm11, %ymm11
+        vpsllq	$39, %ymm12, %ymm12
+        vpsllq	$41, %ymm13, %ymm13
+        vpsllq	$2, %ymm14, %ymm14
+        vpor	%ymm0, %ymm10, %ymm10
+        vpor	%ymm1, %ymm11, %ymm11
+        vpor	%ymm2, %ymm12, %ymm12
+        vpor	%ymm3, %ymm13, %ymm13
+        vpor	%ymm4, %ymm14, %ymm14
+        vpandn	%ymm12, %ymm11, %ymm0
+        vpandn	%ymm13, %ymm12, %ymm1
+        vpandn	%ymm14, %ymm13, %ymm2
+        vpandn	%ymm10, %ymm14, %ymm3
+        vpandn	%ymm11, %ymm10, %ymm4
+        vpxor	%ymm10, %ymm0, %ymm0
+        vpxor	%ymm11, %ymm1, %ymm1
+        vpxor	%ymm12, %ymm2, %ymm2
+        vpxor	%ymm13, %ymm3, %ymm3
+        vpxor	%ymm14, %ymm4, %ymm4
+        vmovdqu	%ymm0, 32(%rcx)
+        vmovdqu	%ymm1, 32(%rdi)
+        vmovdqu	%ymm2, -64(%rdi)
+        vmovdqu	%ymm3, 128(%r8)
+        vmovdqu	%ymm4, 128(%rcx)
+        # Round 3
+        # Calc b[0..4]
+        vpxor	%ymm15, %ymm0, %ymm10
+        vpxor	-96(%rdi), %ymm1, %ymm11
+        vpxor	-32(%rdi), %ymm3, %ymm13
+        vpxor	(%rdi), %ymm4, %ymm14
+        vpxor	64(%rdi), %ymm2, %ymm12
+        vpxor	96(%rdi), %ymm13, %ymm13
+        vpxor	128(%rdi), %ymm14, %ymm14
+        vpxor	-96(%rcx), %ymm10, %ymm10
+        vpxor	-64(%rcx), %ymm12, %ymm12
+        vpxor	-32(%rcx), %ymm13, %ymm13
+        vpxor	(%rcx), %ymm14, %ymm14
+        vpxor	64(%rcx), %ymm11, %ymm11
+        vpxor	96(%rcx), %ymm13, %ymm13
+        vpxor	-96(%r8), %ymm10, %ymm10
+        vpxor	-64(%r8), %ymm11, %ymm11
+        vpxor	-32(%r8), %ymm12, %ymm12
+        vpxor	(%r8), %ymm14, %ymm14
+        vpxor	32(%r8), %ymm10, %ymm10
+        vpxor	64(%r8), %ymm11, %ymm11
+        vpxor	96(%r8), %ymm12, %ymm12
+        # Calc t[0..4]
+        vpsrlq	$63, %ymm11, %ymm0
+        vpsrlq	$63, %ymm12, %ymm1
+        vpsrlq	$63, %ymm13, %ymm2
+        vpsrlq	$63, %ymm14, %ymm3
+        vpsrlq	$63, %ymm10, %ymm4
+        vpaddq	%ymm11, %ymm11, %ymm5
+        vpaddq	%ymm12, %ymm12, %ymm6
+        vpaddq	%ymm13, %ymm13, %ymm7
+        vpaddq	%ymm14, %ymm14, %ymm8
+        vpaddq	%ymm10, %ymm10, %ymm9
+        vpor	%ymm0, %ymm5, %ymm5
+        vpor	%ymm1, %ymm6, %ymm6
+        vpor	%ymm2, %ymm7, %ymm7
+        vpor	%ymm3, %ymm8, %ymm8
+        vpor	%ymm4, %ymm9, %ymm9
+        vpxor	%ymm14, %ymm5, %ymm5
+        vpxor	%ymm10, %ymm6, %ymm6
+        vpxor	%ymm11, %ymm7, %ymm7
+        vpxor	%ymm12, %ymm8, %ymm8
+        vpxor	%ymm13, %ymm9, %ymm9
+        # Row Mix
+        # Row 0
+        vpxor	%ymm15, %ymm5, %ymm10
+        vpxor	64(%rcx), %ymm6, %ymm11
+        vpxor	96(%r8), %ymm7, %ymm12
+        vpxor	96(%rdi), %ymm8, %ymm13
+        vpxor	128(%rcx), %ymm9, %ymm14
+        vpsrlq	$20, %ymm11, %ymm0
+        vpsrlq	$21, %ymm12, %ymm1
+        vpsrlq	$43, %ymm13, %ymm2
+        vpsrlq	$50, %ymm14, %ymm3
+        vpsllq	$44, %ymm11, %ymm11
+        vpsllq	$43, %ymm12, %ymm12
+        vpsllq	$21, %ymm13, %ymm13
+        vpsllq	$14, %ymm14, %ymm14
+        vpor	%ymm0, %ymm11, %ymm11
+        vpor	%ymm1, %ymm12, %ymm12
+        vpor	%ymm2, %ymm13, %ymm13
+        vpor	%ymm3, %ymm14, %ymm14
+        vpandn	%ymm12, %ymm11, %ymm15
+        vpandn	%ymm13, %ymm12, %ymm1
+        vpandn	%ymm14, %ymm13, %ymm2
+        vpandn	%ymm10, %ymm14, %ymm3
+        vpandn	%ymm11, %ymm10, %ymm4
+        vpxor	%ymm10, %ymm15, %ymm15
+        vpxor	%ymm11, %ymm1, %ymm1
+        vpxor	%ymm12, %ymm2, %ymm2
+        # XOR in constant
+        vpxor	96(%rax), %ymm15, %ymm15
+        vpxor	%ymm13, %ymm3, %ymm3
+        vpxor	%ymm14, %ymm4, %ymm4
+        vmovdqu	%ymm1, 64(%rcx)
+        vmovdqu	%ymm2, 96(%r8)
+        vmovdqu	%ymm3, 96(%rdi)
+        vmovdqu	%ymm4, 128(%rcx)
+        # Row 1
+        vpxor	-32(%rcx), %ymm8, %ymm10
+        vpxor	(%r8), %ymm9, %ymm11
+        vpxor	-96(%rcx), %ymm5, %ymm12
+        vpxor	-64(%r8), %ymm6, %ymm13
+        vpxor	-64(%rdi), %ymm7, %ymm14
+        vpsrlq	$36, %ymm10, %ymm0
+        vpsrlq	$44, %ymm11, %ymm1
+        vpsrlq	$61, %ymm12, %ymm2
+        vpsrlq	$19, %ymm13, %ymm3
+        vpsrlq	$3, %ymm14, %ymm4
+        vpsllq	$28, %ymm10, %ymm10
+        vpsllq	$20, %ymm11, %ymm11
+        vpsllq	$3, %ymm12, %ymm12
+        vpsllq	$45, %ymm13, %ymm13
+        vpsllq	$61, %ymm14, %ymm14
+        vpor	%ymm0, %ymm10, %ymm10
+        vpor	%ymm1, %ymm11, %ymm11
+        vpor	%ymm2, %ymm12, %ymm12
+        vpor	%ymm3, %ymm13, %ymm13
+        vpor	%ymm4, %ymm14, %ymm14
+        vpandn	%ymm12, %ymm11, %ymm0
+        vpandn	%ymm13, %ymm12, %ymm1
+        vpandn	%ymm14, %ymm13, %ymm2
+        vpandn	%ymm10, %ymm14, %ymm3
+        vpandn	%ymm11, %ymm10, %ymm4
+        vpxor	%ymm10, %ymm0, %ymm0
+        vpxor	%ymm11, %ymm1, %ymm1
+        vpxor	%ymm12, %ymm2, %ymm2
+        vpxor	%ymm13, %ymm3, %ymm3
+        vpxor	%ymm14, %ymm4, %ymm4
+        vmovdqu	%ymm0, -32(%rcx)
+        vmovdqu	%ymm1, (%r8)
+        vmovdqu	%ymm2, -96(%rcx)
+        vmovdqu	%ymm3, -64(%r8)
+        vmovdqu	%ymm4, -64(%rdi)
+        # Row 2
+        vpxor	64(%r8), %ymm6, %ymm10
+        vpxor	64(%rdi), %ymm7, %ymm11
+        vpxor	96(%rcx), %ymm8, %ymm12
+        vpxor	(%rdi), %ymm9, %ymm13
+        vpxor	32(%rcx), %ymm5, %ymm14
+        vpsrlq	$63, %ymm10, %ymm0
+        vpsrlq	$58, %ymm11, %ymm1
+        vpsrlq	$39, %ymm12, %ymm2
+        vpsrlq	$56, %ymm13, %ymm3
+        vpsrlq	$46, %ymm14, %ymm4
+        vpaddq	%ymm10, %ymm10, %ymm10
+        vpsllq	$6, %ymm11, %ymm11
+        vpsllq	$25, %ymm12, %ymm12
+        vpsllq	$8, %ymm13, %ymm13
+        vpsllq	$18, %ymm14, %ymm14
+        vpor	%ymm0, %ymm10, %ymm10
+        vpor	%ymm1, %ymm11, %ymm11
+        vpor	%ymm2, %ymm12, %ymm12
+        vpor	%ymm3, %ymm13, %ymm13
+        vpor	%ymm4, %ymm14, %ymm14
+        vpandn	%ymm12, %ymm11, %ymm0
+        vpandn	%ymm13, %ymm12, %ymm1
+        vpandn	%ymm14, %ymm13, %ymm2
+        vpandn	%ymm10, %ymm14, %ymm3
+        vpandn	%ymm11, %ymm10, %ymm4
+        vpxor	%ymm10, %ymm0, %ymm0
+        vpxor	%ymm11, %ymm1, %ymm1
+        vpxor	%ymm12, %ymm2, %ymm2
+        vpxor	%ymm13, %ymm3, %ymm3
+        vpxor	%ymm14, %ymm4, %ymm4
+        vmovdqu	%ymm0, 64(%r8)
+        vmovdqu	%ymm1, 64(%rdi)
+        vmovdqu	%ymm2, 96(%rcx)
+        vmovdqu	%ymm3, (%rdi)
+        vmovdqu	%ymm4, 32(%rcx)
+        # Row 3
+        vpxor	128(%rdi), %ymm9, %ymm10
+        vpxor	-96(%r8), %ymm5, %ymm11
+        vpxor	-96(%rdi), %ymm6, %ymm12
+        vpxor	-64(%rcx), %ymm7, %ymm13
+        vpxor	128(%r8), %ymm8, %ymm14
+        vpsrlq	$37, %ymm10, %ymm0
+        vpsrlq	$28, %ymm11, %ymm1
+        vpsrlq	$54, %ymm12, %ymm2
+        vpsrlq	$49, %ymm13, %ymm3
+        vpsrlq	$8, %ymm14, %ymm4
+        vpsllq	$27, %ymm10, %ymm10
+        vpsllq	$36, %ymm11, %ymm11
+        vpsllq	$10, %ymm12, %ymm12
+        vpsllq	$15, %ymm13, %ymm13
+        vpsllq	$56, %ymm14, %ymm14
+        vpor	%ymm0, %ymm10, %ymm10
+        vpor	%ymm1, %ymm11, %ymm11
+        vpor	%ymm2, %ymm12, %ymm12
+        vpor	%ymm3, %ymm13, %ymm13
+        vpor	%ymm4, %ymm14, %ymm14
+        vpandn	%ymm12, %ymm11, %ymm0
+        vpandn	%ymm13, %ymm12, %ymm1
+        vpandn	%ymm14, %ymm13, %ymm2
+        vpandn	%ymm10, %ymm14, %ymm3
+        vpandn	%ymm11, %ymm10, %ymm4
+        vpxor	%ymm10, %ymm0, %ymm0
+        vpxor	%ymm11, %ymm1, %ymm1
+        vpxor	%ymm12, %ymm2, %ymm2
+        vpxor	%ymm13, %ymm3, %ymm3
+        vpxor	%ymm14, %ymm4, %ymm4
+        vmovdqu	%ymm0, 128(%rdi)
+        vmovdqu	%ymm1, -96(%r8)
+        vmovdqu	%ymm2, -96(%rdi)
+        vmovdqu	%ymm3, -64(%rcx)
+        vmovdqu	%ymm4, 128(%r8)
+        # Row 4
+        vpxor	-32(%r8), %ymm7, %ymm10
+        vpxor	-32(%rdi), %ymm8, %ymm11
+        vpxor	(%rcx), %ymm9, %ymm12
+        vpxor	32(%r8), %ymm5, %ymm13
+        vpxor	32(%rdi), %ymm6, %ymm14
+        vpsrlq	$2, %ymm10, %ymm0
+        vpsrlq	$9, %ymm11, %ymm1
+        vpsrlq	$25, %ymm12, %ymm2
+        vpsrlq	$23, %ymm13, %ymm3
+        vpsrlq	$62, %ymm14, %ymm4
+        vpsllq	$62, %ymm10, %ymm10
+        vpsllq	$55, %ymm11, %ymm11
+        vpsllq	$39, %ymm12, %ymm12
+        vpsllq	$41, %ymm13, %ymm13
+        vpsllq	$2, %ymm14, %ymm14
+        vpor	%ymm0, %ymm10, %ymm10
+        vpor	%ymm1, %ymm11, %ymm11
+        vpor	%ymm2, %ymm12, %ymm12
+        vpor	%ymm3, %ymm13, %ymm13
+        vpor	%ymm4, %ymm14, %ymm14
+        vpandn	%ymm12, %ymm11, %ymm0
+        vpandn	%ymm13, %ymm12, %ymm1
+        vpandn	%ymm14, %ymm13, %ymm2
+        vpandn	%ymm10, %ymm14, %ymm3
+        vpandn	%ymm11, %ymm10, %ymm4
+        vpxor	%ymm10, %ymm0, %ymm0
+        vpxor	%ymm11, %ymm1, %ymm1
+        vpxor	%ymm12, %ymm2, %ymm2
+        vpxor	%ymm13, %ymm3, %ymm3
+        vpxor	%ymm14, %ymm4, %ymm4
+        vmovdqu	%ymm0, -32(%r8)
+        vmovdqu	%ymm1, -32(%rdi)
+        vmovdqu	%ymm2, (%rcx)
+        vmovdqu	%ymm3, 32(%r8)
+        vmovdqu	%ymm4, 32(%rdi)
+        # Round 4
+        # Calc b[0..4]
+        vpxor	%ymm15, %ymm0, %ymm10
+        vpxor	-96(%rdi), %ymm2, %ymm12
+        vpxor	-64(%rdi), %ymm4, %ymm14
+        vpxor	(%rdi), %ymm3, %ymm13
+        vpxor	64(%rdi), %ymm1, %ymm11
+        vpxor	96(%rdi), %ymm13, %ymm13
+        vpxor	128(%rdi), %ymm10, %ymm10
+        vpxor	-96(%rcx), %ymm12, %ymm12
+        vpxor	-64(%rcx), %ymm13, %ymm13
+        vpxor	-32(%rcx), %ymm10, %ymm10
+        vpxor	32(%rcx), %ymm14, %ymm14
+        vpxor	64(%rcx), %ymm11, %ymm11
+        vpxor	96(%rcx), %ymm12, %ymm12
+        vpxor	128(%rcx), %ymm14, %ymm14
+        vpxor	-96(%r8), %ymm11, %ymm11
+        vpxor	-64(%r8), %ymm13, %ymm13
+        vpxor	(%r8), %ymm11, %ymm11
+        vpxor	64(%r8), %ymm10, %ymm10
+        vpxor	96(%r8), %ymm12, %ymm12
+        vpxor	128(%r8), %ymm14, %ymm14
+        # Calc t[0..4]
+        vpsrlq	$63, %ymm11, %ymm0
+        vpsrlq	$63, %ymm12, %ymm1
+        vpsrlq	$63, %ymm13, %ymm2
+        vpsrlq	$63, %ymm14, %ymm3
+        vpsrlq	$63, %ymm10, %ymm4
+        vpaddq	%ymm11, %ymm11, %ymm5
+        vpaddq	%ymm12, %ymm12, %ymm6
+        vpaddq	%ymm13, %ymm13, %ymm7
+        vpaddq	%ymm14, %ymm14, %ymm8
+        vpaddq	%ymm10, %ymm10, %ymm9
+        vpor	%ymm0, %ymm5, %ymm5
+        vpor	%ymm1, %ymm6, %ymm6
+        vpor	%ymm2, %ymm7, %ymm7
+        vpor	%ymm3, %ymm8, %ymm8
+        vpor	%ymm4, %ymm9, %ymm9
+        vpxor	%ymm14, %ymm5, %ymm5
+        vpxor	%ymm10, %ymm6, %ymm6
+        vpxor	%ymm11, %ymm7, %ymm7
+        vpxor	%ymm12, %ymm8, %ymm8
+        vpxor	%ymm13, %ymm9, %ymm9
+        # Row Mix
+        # Row 0
+        vpxor	%ymm15, %ymm5, %ymm10
+        vpxor	(%r8), %ymm6, %ymm11
+        vpxor	96(%rcx), %ymm7, %ymm12
+        vpxor	-64(%rcx), %ymm8, %ymm13
+        vpxor	32(%rdi), %ymm9, %ymm14
+        vpsrlq	$20, %ymm11, %ymm0
+        vpsrlq	$21, %ymm12, %ymm1
+        vpsrlq	$43, %ymm13, %ymm2
+        vpsrlq	$50, %ymm14, %ymm3
+        vpsllq	$44, %ymm11, %ymm11
+        vpsllq	$43, %ymm12, %ymm12
+        vpsllq	$21, %ymm13, %ymm13
+        vpsllq	$14, %ymm14, %ymm14
+        vpor	%ymm0, %ymm11, %ymm11
+        vpor	%ymm1, %ymm12, %ymm12
+        vpor	%ymm2, %ymm13, %ymm13
+        vpor	%ymm3, %ymm14, %ymm14
+        vpandn	%ymm12, %ymm11, %ymm15
+        vpandn	%ymm13, %ymm12, %ymm1
+        vpandn	%ymm14, %ymm13, %ymm2
+        vpandn	%ymm10, %ymm14, %ymm3
+        vpandn	%ymm11, %ymm10, %ymm4
+        vpxor	%ymm10, %ymm15, %ymm15
+        vpxor	%ymm11, %ymm1, %ymm1
+        vpxor	%ymm12, %ymm2, %ymm2
+        # XOR in constant
+        vpxor	128(%rax), %ymm15, %ymm15
+        vpxor	%ymm13, %ymm3, %ymm3
+        vpxor	%ymm14, %ymm4, %ymm4
+        vmovdqu	%ymm1, (%r8)
+        vmovdqu	%ymm2, 96(%rcx)
+        vmovdqu	%ymm3, -64(%rcx)
+        vmovdqu	%ymm4, 32(%rdi)
+        # Row 1
+        vpxor	96(%rdi), %ymm8, %ymm10
+        vpxor	-64(%rdi), %ymm9, %ymm11
+        vpxor	64(%r8), %ymm5, %ymm12
+        vpxor	-96(%r8), %ymm6, %ymm13
+        vpxor	(%rcx), %ymm7, %ymm14
+        vpsrlq	$36, %ymm10, %ymm0
+        vpsrlq	$44, %ymm11, %ymm1
+        vpsrlq	$61, %ymm12, %ymm2
+        vpsrlq	$19, %ymm13, %ymm3
+        vpsrlq	$3, %ymm14, %ymm4
+        vpsllq	$28, %ymm10, %ymm10
+        vpsllq	$20, %ymm11, %ymm11
+        vpsllq	$3, %ymm12, %ymm12
+        vpsllq	$45, %ymm13, %ymm13
+        vpsllq	$61, %ymm14, %ymm14
+        vpor	%ymm0, %ymm10, %ymm10
+        vpor	%ymm1, %ymm11, %ymm11
+        vpor	%ymm2, %ymm12, %ymm12
+        vpor	%ymm3, %ymm13, %ymm13
+        vpor	%ymm4, %ymm14, %ymm14
+        vpandn	%ymm12, %ymm11, %ymm0
+        vpandn	%ymm13, %ymm12, %ymm1
+        vpandn	%ymm14, %ymm13, %ymm2
+        vpandn	%ymm10, %ymm14, %ymm3
+        vpandn	%ymm11, %ymm10, %ymm4
+        vpxor	%ymm10, %ymm0, %ymm0
+        vpxor	%ymm11, %ymm1, %ymm1
+        vpxor	%ymm12, %ymm2, %ymm2
+        vpxor	%ymm13, %ymm3, %ymm3
+        vpxor	%ymm14, %ymm4, %ymm4
+        vmovdqu	%ymm0, 96(%rdi)
+        vmovdqu	%ymm1, -64(%rdi)
+        vmovdqu	%ymm2, 64(%r8)
+        vmovdqu	%ymm3, -96(%r8)
+        vmovdqu	%ymm4, (%rcx)
+        # Row 2
+        vpxor	64(%rcx), %ymm6, %ymm10
+        vpxor	-96(%rcx), %ymm7, %ymm11
+        vpxor	(%rdi), %ymm8, %ymm12
+        vpxor	128(%r8), %ymm9, %ymm13
+        vpxor	-32(%r8), %ymm5, %ymm14
+        vpsrlq	$63, %ymm10, %ymm0
+        vpsrlq	$58, %ymm11, %ymm1
+        vpsrlq	$39, %ymm12, %ymm2
+        vpsrlq	$56, %ymm13, %ymm3
+        vpsrlq	$46, %ymm14, %ymm4
+        vpaddq	%ymm10, %ymm10, %ymm10
+        vpsllq	$6, %ymm11, %ymm11
+        vpsllq	$25, %ymm12, %ymm12
+        vpsllq	$8, %ymm13, %ymm13
+        vpsllq	$18, %ymm14, %ymm14
+        vpor	%ymm0, %ymm10, %ymm10
+        vpor	%ymm1, %ymm11, %ymm11
+        vpor	%ymm2, %ymm12, %ymm12
+        vpor	%ymm3, %ymm13, %ymm13
+        vpor	%ymm4, %ymm14, %ymm14
+        vpandn	%ymm12, %ymm11, %ymm0
+        vpandn	%ymm13, %ymm12, %ymm1
+        vpandn	%ymm14, %ymm13, %ymm2
+        vpandn	%ymm10, %ymm14, %ymm3
+        vpandn	%ymm11, %ymm10, %ymm4
+        vpxor	%ymm10, %ymm0, %ymm0
+        vpxor	%ymm11, %ymm1, %ymm1
+        vpxor	%ymm12, %ymm2, %ymm2
+        vpxor	%ymm13, %ymm3, %ymm3
+        vpxor	%ymm14, %ymm4, %ymm4
+        vmovdqu	%ymm0, 64(%rcx)
+        vmovdqu	%ymm1, -96(%rcx)
+        vmovdqu	%ymm2, (%rdi)
+        vmovdqu	%ymm3, 128(%r8)
+        vmovdqu	%ymm4, -32(%r8)
+        # Row 3
+        vpxor	128(%rcx), %ymm9, %ymm10
+        vpxor	-32(%rcx), %ymm5, %ymm11
+        vpxor	64(%rdi), %ymm6, %ymm12
+        vpxor	-96(%rdi), %ymm7, %ymm13
+        vpxor	32(%r8), %ymm8, %ymm14
+        vpsrlq	$37, %ymm10, %ymm0
+        vpsrlq	$28, %ymm11, %ymm1
+        vpsrlq	$54, %ymm12, %ymm2
+        vpsrlq	$49, %ymm13, %ymm3
+        vpsrlq	$8, %ymm14, %ymm4
+        vpsllq	$27, %ymm10, %ymm10
+        vpsllq	$36, %ymm11, %ymm11
+        vpsllq	$10, %ymm12, %ymm12
+        vpsllq	$15, %ymm13, %ymm13
+        vpsllq	$56, %ymm14, %ymm14
+        vpor	%ymm0, %ymm10, %ymm10
+        vpor	%ymm1, %ymm11, %ymm11
+        vpor	%ymm2, %ymm12, %ymm12
+        vpor	%ymm3, %ymm13, %ymm13
+        vpor	%ymm4, %ymm14, %ymm14
+        vpandn	%ymm12, %ymm11, %ymm0
+        vpandn	%ymm13, %ymm12, %ymm1
+        vpandn	%ymm14, %ymm13, %ymm2
+        vpandn	%ymm10, %ymm14, %ymm3
+        vpandn	%ymm11, %ymm10, %ymm4
+        vpxor	%ymm10, %ymm0, %ymm0
+        vpxor	%ymm11, %ymm1, %ymm1
+        vpxor	%ymm12, %ymm2, %ymm2
+        vpxor	%ymm13, %ymm3, %ymm3
+        vpxor	%ymm14, %ymm4, %ymm4
+        vmovdqu	%ymm0, 128(%rcx)
+        vmovdqu	%ymm1, -32(%rcx)
+        vmovdqu	%ymm2, 64(%rdi)
+        vmovdqu	%ymm3, -96(%rdi)
+        vmovdqu	%ymm4, 32(%r8)
+        # Row 4
+        vpxor	96(%r8), %ymm7, %ymm10
+        vpxor	-64(%r8), %ymm8, %ymm11
+        vpxor	32(%rcx), %ymm9, %ymm12
+        vpxor	128(%rdi), %ymm5, %ymm13
+        vpxor	-32(%rdi), %ymm6, %ymm14
+        vpsrlq	$2, %ymm10, %ymm0
+        vpsrlq	$9, %ymm11, %ymm1
+        vpsrlq	$25, %ymm12, %ymm2
+        vpsrlq	$23, %ymm13, %ymm3
+        vpsrlq	$62, %ymm14, %ymm4
+        vpsllq	$62, %ymm10, %ymm10
+        vpsllq	$55, %ymm11, %ymm11
+        vpsllq	$39, %ymm12, %ymm12
+        vpsllq	$41, %ymm13, %ymm13
+        vpsllq	$2, %ymm14, %ymm14
+        vpor	%ymm0, %ymm10, %ymm10
+        vpor	%ymm1, %ymm11, %ymm11
+        vpor	%ymm2, %ymm12, %ymm12
+        vpor	%ymm3, %ymm13, %ymm13
+        vpor	%ymm4, %ymm14, %ymm14
+        vpandn	%ymm12, %ymm11, %ymm0
+        vpandn	%ymm13, %ymm12, %ymm1
+        vpandn	%ymm14, %ymm13, %ymm2
+        vpandn	%ymm10, %ymm14, %ymm3
+        vpandn	%ymm11, %ymm10, %ymm4
+        vpxor	%ymm10, %ymm0, %ymm0
+        vpxor	%ymm11, %ymm1, %ymm1
+        vpxor	%ymm12, %ymm2, %ymm2
+        vpxor	%ymm13, %ymm3, %ymm3
+        vpxor	%ymm14, %ymm4, %ymm4
+        vmovdqu	%ymm0, 96(%r8)
+        vmovdqu	%ymm1, -64(%r8)
+        vmovdqu	%ymm2, 32(%rcx)
+        vmovdqu	%ymm3, 128(%rdi)
+        vmovdqu	%ymm4, -32(%rdi)
+        # Round 5
+        # Calc b[0..4]
+        vpxor	%ymm15, %ymm0, %ymm10
+        vpxor	-96(%rdi), %ymm3, %ymm13
+        vpxor	-64(%rdi), %ymm1, %ymm11
+        vpxor	(%rdi), %ymm2, %ymm12
+        vpxor	32(%rdi), %ymm4, %ymm14
+        vpxor	64(%rdi), %ymm12, %ymm12
+        vpxor	96(%rdi), %ymm10, %ymm10
+        vpxor	-96(%rcx), %ymm11, %ymm11
+        vpxor	-64(%rcx), %ymm13, %ymm13
+        vpxor	-32(%rcx), %ymm11, %ymm11
+        vpxor	(%rcx), %ymm14, %ymm14
+        vpxor	64(%rcx), %ymm10, %ymm10
+        vpxor	96(%rcx), %ymm12, %ymm12
+        vpxor	128(%rcx), %ymm10, %ymm10
+        vpxor	-96(%r8), %ymm13, %ymm13
+        vpxor	-32(%r8), %ymm14, %ymm14
+        vpxor	(%r8), %ymm11, %ymm11
+        vpxor	32(%r8), %ymm14, %ymm14
+        vpxor	64(%r8), %ymm12, %ymm12
+        vpxor	128(%r8), %ymm13, %ymm13
+        # Calc t[0..4]
+        vpsrlq	$63, %ymm11, %ymm0
+        vpsrlq	$63, %ymm12, %ymm1
+        vpsrlq	$63, %ymm13, %ymm2
+        vpsrlq	$63, %ymm14, %ymm3
+        vpsrlq	$63, %ymm10, %ymm4
+        vpaddq	%ymm11, %ymm11, %ymm5
+        vpaddq	%ymm12, %ymm12, %ymm6
+        vpaddq	%ymm13, %ymm13, %ymm7
+        vpaddq	%ymm14, %ymm14, %ymm8
+        vpaddq	%ymm10, %ymm10, %ymm9
+        vpor	%ymm0, %ymm5, %ymm5
+        vpor	%ymm1, %ymm6, %ymm6
+        vpor	%ymm2, %ymm7, %ymm7
+        vpor	%ymm3, %ymm8, %ymm8
+        vpor	%ymm4, %ymm9, %ymm9
+        vpxor	%ymm14, %ymm5, %ymm5
+        vpxor	%ymm10, %ymm6, %ymm6
+        vpxor	%ymm11, %ymm7, %ymm7
+        vpxor	%ymm12, %ymm8, %ymm8
+        vpxor	%ymm13, %ymm9, %ymm9
+        # Row Mix
+        # Row 0
+        vpxor	%ymm15, %ymm5, %ymm10
+        vpxor	-64(%rdi), %ymm6, %ymm11
+        vpxor	(%rdi), %ymm7, %ymm12
+        vpxor	-96(%rdi), %ymm8, %ymm13
+        vpxor	-32(%rdi), %ymm9, %ymm14
+        vpsrlq	$20, %ymm11, %ymm0
+        vpsrlq	$21, %ymm12, %ymm1
+        vpsrlq	$43, %ymm13, %ymm2
+        vpsrlq	$50, %ymm14, %ymm3
+        vpsllq	$44, %ymm11, %ymm11
+        vpsllq	$43, %ymm12, %ymm12
+        vpsllq	$21, %ymm13, %ymm13
+        vpsllq	$14, %ymm14, %ymm14
+        vpor	%ymm0, %ymm11, %ymm11
+        vpor	%ymm1, %ymm12, %ymm12
+        vpor	%ymm2, %ymm13, %ymm13
+        vpor	%ymm3, %ymm14, %ymm14
+        vpandn	%ymm12, %ymm11, %ymm15
+        vpandn	%ymm13, %ymm12, %ymm1
+        vpandn	%ymm14, %ymm13, %ymm2
+        vpandn	%ymm10, %ymm14, %ymm3
+        vpandn	%ymm11, %ymm10, %ymm4
+        vpxor	%ymm10, %ymm15, %ymm15
+        vpxor	%ymm11, %ymm1, %ymm1
+        vpxor	%ymm12, %ymm2, %ymm2
+        # XOR in constant
+        vpxor	160(%rax), %ymm15, %ymm15
+        vpxor	%ymm13, %ymm3, %ymm3
+        vpxor	%ymm14, %ymm4, %ymm4
+        vmovdqu	%ymm1, -64(%rdi)
+        vmovdqu	%ymm2, (%rdi)
+        vmovdqu	%ymm3, -96(%rdi)
+        vmovdqu	%ymm4, -32(%rdi)
+        # Row 1
+        vpxor	-64(%rcx), %ymm8, %ymm10
+        vpxor	(%rcx), %ymm9, %ymm11
+        vpxor	64(%rcx), %ymm5, %ymm12
+        vpxor	-32(%rcx), %ymm6, %ymm13
+        vpxor	32(%rcx), %ymm7, %ymm14
+        vpsrlq	$36, %ymm10, %ymm0
+        vpsrlq	$44, %ymm11, %ymm1
+        vpsrlq	$61, %ymm12, %ymm2
+        vpsrlq	$19, %ymm13, %ymm3
+        vpsrlq	$3, %ymm14, %ymm4
+        vpsllq	$28, %ymm10, %ymm10
+        vpsllq	$20, %ymm11, %ymm11
+        vpsllq	$3, %ymm12, %ymm12
+        vpsllq	$45, %ymm13, %ymm13
+        vpsllq	$61, %ymm14, %ymm14
+        vpor	%ymm0, %ymm10, %ymm10
+        vpor	%ymm1, %ymm11, %ymm11
+        vpor	%ymm2, %ymm12, %ymm12
+        vpor	%ymm3, %ymm13, %ymm13
+        vpor	%ymm4, %ymm14, %ymm14
+        vpandn	%ymm12, %ymm11, %ymm0
+        vpandn	%ymm13, %ymm12, %ymm1
+        vpandn	%ymm14, %ymm13, %ymm2
+        vpandn	%ymm10, %ymm14, %ymm3
+        vpandn	%ymm11, %ymm10, %ymm4
+        vpxor	%ymm10, %ymm0, %ymm0
+        vpxor	%ymm11, %ymm1, %ymm1
+        vpxor	%ymm12, %ymm2, %ymm2
+        vpxor	%ymm13, %ymm3, %ymm3
+        vpxor	%ymm14, %ymm4, %ymm4
+        vmovdqu	%ymm0, -64(%rcx)
+        vmovdqu	%ymm1, (%rcx)
+        vmovdqu	%ymm2, 64(%rcx)
+        vmovdqu	%ymm3, -32(%rcx)
+        vmovdqu	%ymm4, 32(%rcx)
+        # Row 2
+        vpxor	(%r8), %ymm6, %ymm10
+        vpxor	64(%r8), %ymm7, %ymm11
+        vpxor	128(%r8), %ymm8, %ymm12
+        vpxor	32(%r8), %ymm9, %ymm13
+        vpxor	96(%r8), %ymm5, %ymm14
+        vpsrlq	$63, %ymm10, %ymm0
+        vpsrlq	$58, %ymm11, %ymm1
+        vpsrlq	$39, %ymm12, %ymm2
+        vpsrlq	$56, %ymm13, %ymm3
+        vpsrlq	$46, %ymm14, %ymm4
+        vpaddq	%ymm10, %ymm10, %ymm10
+        vpsllq	$6, %ymm11, %ymm11
+        vpsllq	$25, %ymm12, %ymm12
+        vpsllq	$8, %ymm13, %ymm13
+        vpsllq	$18, %ymm14, %ymm14
+        vpor	%ymm0, %ymm10, %ymm10
+        vpor	%ymm1, %ymm11, %ymm11
+        vpor	%ymm2, %ymm12, %ymm12
+        vpor	%ymm3, %ymm13, %ymm13
+        vpor	%ymm4, %ymm14, %ymm14
+        vpandn	%ymm12, %ymm11, %ymm0
+        vpandn	%ymm13, %ymm12, %ymm1
+        vpandn	%ymm14, %ymm13, %ymm2
+        vpandn	%ymm10, %ymm14, %ymm3
+        vpandn	%ymm11, %ymm10, %ymm4
+        vpxor	%ymm10, %ymm0, %ymm0
+        vpxor	%ymm11, %ymm1, %ymm1
+        vpxor	%ymm12, %ymm2, %ymm2
+        vpxor	%ymm13, %ymm3, %ymm3
+        vpxor	%ymm14, %ymm4, %ymm4
+        vmovdqu	%ymm0, (%r8)
+        vmovdqu	%ymm1, 64(%r8)
+        vmovdqu	%ymm2, 128(%r8)
+        vmovdqu	%ymm3, 32(%r8)
+        vmovdqu	%ymm4, 96(%r8)
+        # Row 3
+        vpxor	32(%rdi), %ymm9, %ymm10
+        vpxor	96(%rdi), %ymm5, %ymm11
+        vpxor	-96(%rcx), %ymm6, %ymm12
+        vpxor	64(%rdi), %ymm7, %ymm13
+        vpxor	128(%rdi), %ymm8, %ymm14
+        vpsrlq	$37, %ymm10, %ymm0
+        vpsrlq	$28, %ymm11, %ymm1
+        vpsrlq	$54, %ymm12, %ymm2
+        vpsrlq	$49, %ymm13, %ymm3
+        vpsrlq	$8, %ymm14, %ymm4
+        vpsllq	$27, %ymm10, %ymm10
+        vpsllq	$36, %ymm11, %ymm11
+        vpsllq	$10, %ymm12, %ymm12
+        vpsllq	$15, %ymm13, %ymm13
+        vpsllq	$56, %ymm14, %ymm14
+        vpor	%ymm0, %ymm10, %ymm10
+        vpor	%ymm1, %ymm11, %ymm11
+        vpor	%ymm2, %ymm12, %ymm12
+        vpor	%ymm3, %ymm13, %ymm13
+        vpor	%ymm4, %ymm14, %ymm14
+        vpandn	%ymm12, %ymm11, %ymm0
+        vpandn	%ymm13, %ymm12, %ymm1
+        vpandn	%ymm14, %ymm13, %ymm2
+        vpandn	%ymm10, %ymm14, %ymm3
+        vpandn	%ymm11, %ymm10, %ymm4
+        vpxor	%ymm10, %ymm0, %ymm0
+        vpxor	%ymm11, %ymm1, %ymm1
+        vpxor	%ymm12, %ymm2, %ymm2
+        vpxor	%ymm13, %ymm3, %ymm3
+        vpxor	%ymm14, %ymm4, %ymm4
+        vmovdqu	%ymm0, 32(%rdi)
+        vmovdqu	%ymm1, 96(%rdi)
+        vmovdqu	%ymm2, -96(%rcx)
+        vmovdqu	%ymm3, 64(%rdi)
+        vmovdqu	%ymm4, 128(%rdi)
+        # Row 4
+        vpxor	96(%rcx), %ymm7, %ymm10
+        vpxor	-96(%r8), %ymm8, %ymm11
+        vpxor	-32(%r8), %ymm9, %ymm12
+        vpxor	128(%rcx), %ymm5, %ymm13
+        vpxor	-64(%r8), %ymm6, %ymm14
+        vpsrlq	$2, %ymm10, %ymm0
+        vpsrlq	$9, %ymm11, %ymm1
+        vpsrlq	$25, %ymm12, %ymm2
+        vpsrlq	$23, %ymm13, %ymm3
+        vpsrlq	$62, %ymm14, %ymm4
+        vpsllq	$62, %ymm10, %ymm10
+        vpsllq	$55, %ymm11, %ymm11
+        vpsllq	$39, %ymm12, %ymm12
+        vpsllq	$41, %ymm13, %ymm13
+        vpsllq	$2, %ymm14, %ymm14
+        vpor	%ymm0, %ymm10, %ymm10
+        vpor	%ymm1, %ymm11, %ymm11
+        vpor	%ymm2, %ymm12, %ymm12
+        vpor	%ymm3, %ymm13, %ymm13
+        vpor	%ymm4, %ymm14, %ymm14
+        vpandn	%ymm12, %ymm11, %ymm0
+        vpandn	%ymm13, %ymm12, %ymm1
+        vpandn	%ymm14, %ymm13, %ymm2
+        vpandn	%ymm10, %ymm14, %ymm3
+        vpandn	%ymm11, %ymm10, %ymm4
+        vpxor	%ymm10, %ymm0, %ymm0
+        vpxor	%ymm11, %ymm1, %ymm1
+        vpxor	%ymm12, %ymm2, %ymm2
+        vpxor	%ymm13, %ymm3, %ymm3
+        vpxor	%ymm14, %ymm4, %ymm4
+        vmovdqu	%ymm0, 96(%rcx)
+        vmovdqu	%ymm1, -96(%r8)
+        vmovdqu	%ymm2, -32(%r8)
+        vmovdqu	%ymm3, 128(%rcx)
+        vmovdqu	%ymm4, -64(%r8)
+        # Round 6
+        # Calc b[0..4]
+        vpxor	%ymm15, %ymm0, %ymm10
+        vpxor	-96(%rdi), %ymm3, %ymm13
+        vpxor	-64(%rdi), %ymm1, %ymm11
+        vpxor	-32(%rdi), %ymm4, %ymm14
+        vpxor	(%rdi), %ymm2, %ymm12
+        vpxor	32(%rdi), %ymm10, %ymm10
+        vpxor	64(%rdi), %ymm13, %ymm13
+        vpxor	96(%rdi), %ymm11, %ymm11
+        vpxor	128(%rdi), %ymm14, %ymm14
+        vpxor	-96(%rcx), %ymm12, %ymm12
+        vpxor	-64(%rcx), %ymm10, %ymm10
+        vpxor	-32(%rcx), %ymm13, %ymm13
+        vpxor	(%rcx), %ymm11, %ymm11
+        vpxor	32(%rcx), %ymm14, %ymm14
+        vpxor	64(%rcx), %ymm12, %ymm12
+        vpxor	(%r8), %ymm10, %ymm10
+        vpxor	32(%r8), %ymm13, %ymm13
+        vpxor	64(%r8), %ymm11, %ymm11
+        vpxor	96(%r8), %ymm14, %ymm14
+        vpxor	128(%r8), %ymm12, %ymm12
+        # Calc t[0..4]
+        vpsrlq	$63, %ymm11, %ymm0
+        vpsrlq	$63, %ymm12, %ymm1
+        vpsrlq	$63, %ymm13, %ymm2
+        vpsrlq	$63, %ymm14, %ymm3
+        vpsrlq	$63, %ymm10, %ymm4
+        vpaddq	%ymm11, %ymm11, %ymm5
+        vpaddq	%ymm12, %ymm12, %ymm6
+        vpaddq	%ymm13, %ymm13, %ymm7
+        vpaddq	%ymm14, %ymm14, %ymm8
+        vpaddq	%ymm10, %ymm10, %ymm9
+        vpor	%ymm0, %ymm5, %ymm5
+        vpor	%ymm1, %ymm6, %ymm6
+        vpor	%ymm2, %ymm7, %ymm7
+        vpor	%ymm3, %ymm8, %ymm8
+        vpor	%ymm4, %ymm9, %ymm9
+        vpxor	%ymm14, %ymm5, %ymm5
+        vpxor	%ymm10, %ymm6, %ymm6
+        vpxor	%ymm11, %ymm7, %ymm7
+        vpxor	%ymm12, %ymm8, %ymm8
+        vpxor	%ymm13, %ymm9, %ymm9
+        # Row Mix
+        # Row 0
+        vpxor	%ymm15, %ymm5, %ymm10
+        vpxor	(%rcx), %ymm6, %ymm11
+        vpxor	128(%r8), %ymm7, %ymm12
+        vpxor	64(%rdi), %ymm8, %ymm13
+        vpxor	-64(%r8), %ymm9, %ymm14
+        vpsrlq	$20, %ymm11, %ymm0
+        vpsrlq	$21, %ymm12, %ymm1
+        vpsrlq	$43, %ymm13, %ymm2
+        vpsrlq	$50, %ymm14, %ymm3
+        vpsllq	$44, %ymm11, %ymm11
+        vpsllq	$43, %ymm12, %ymm12
+        vpsllq	$21, %ymm13, %ymm13
+        vpsllq	$14, %ymm14, %ymm14
+        vpor	%ymm0, %ymm11, %ymm11
+        vpor	%ymm1, %ymm12, %ymm12
+        vpor	%ymm2, %ymm13, %ymm13
+        vpor	%ymm3, %ymm14, %ymm14
+        vpandn	%ymm12, %ymm11, %ymm15
+        vpandn	%ymm13, %ymm12, %ymm1
+        vpandn	%ymm14, %ymm13, %ymm2
+        vpandn	%ymm10, %ymm14, %ymm3
+        vpandn	%ymm11, %ymm10, %ymm4
+        vpxor	%ymm10, %ymm15, %ymm15
+        vpxor	%ymm11, %ymm1, %ymm1
+        vpxor	%ymm12, %ymm2, %ymm2
+        # XOR in constant
+        vpxor	192(%rax), %ymm15, %ymm15
+        vpxor	%ymm13, %ymm3, %ymm3
+        vpxor	%ymm14, %ymm4, %ymm4
+        vmovdqu	%ymm1, (%rcx)
+        vmovdqu	%ymm2, 128(%r8)
+        vmovdqu	%ymm3, 64(%rdi)
+        vmovdqu	%ymm4, -64(%r8)
+        # Row 1
+        vpxor	-96(%rdi), %ymm8, %ymm10
+        vpxor	32(%rcx), %ymm9, %ymm11
+        vpxor	(%r8), %ymm5, %ymm12
+        vpxor	96(%rdi), %ymm6, %ymm13
+        vpxor	-32(%r8), %ymm7, %ymm14
+        vpsrlq	$36, %ymm10, %ymm0
+        vpsrlq	$44, %ymm11, %ymm1
+        vpsrlq	$61, %ymm12, %ymm2
+        vpsrlq	$19, %ymm13, %ymm3
+        vpsrlq	$3, %ymm14, %ymm4
+        vpsllq	$28, %ymm10, %ymm10
+        vpsllq	$20, %ymm11, %ymm11
+        vpsllq	$3, %ymm12, %ymm12
+        vpsllq	$45, %ymm13, %ymm13
+        vpsllq	$61, %ymm14, %ymm14
+        vpor	%ymm0, %ymm10, %ymm10
+        vpor	%ymm1, %ymm11, %ymm11
+        vpor	%ymm2, %ymm12, %ymm12
+        vpor	%ymm3, %ymm13, %ymm13
+        vpor	%ymm4, %ymm14, %ymm14
+        vpandn	%ymm12, %ymm11, %ymm0
+        vpandn	%ymm13, %ymm12, %ymm1
+        vpandn	%ymm14, %ymm13, %ymm2
+        vpandn	%ymm10, %ymm14, %ymm3
+        vpandn	%ymm11, %ymm10, %ymm4
+        vpxor	%ymm10, %ymm0, %ymm0
+        vpxor	%ymm11, %ymm1, %ymm1
+        vpxor	%ymm12, %ymm2, %ymm2
+        vpxor	%ymm13, %ymm3, %ymm3
+        vpxor	%ymm14, %ymm4, %ymm4
+        vmovdqu	%ymm0, -96(%rdi)
+        vmovdqu	%ymm1, 32(%rcx)
+        vmovdqu	%ymm2, (%r8)
+        vmovdqu	%ymm3, 96(%rdi)
+        vmovdqu	%ymm4, -32(%r8)
+        # Row 2
+        vpxor	-64(%rdi), %ymm6, %ymm10
+        vpxor	64(%rcx), %ymm7, %ymm11
+        vpxor	32(%r8), %ymm8, %ymm12
+        vpxor	128(%rdi), %ymm9, %ymm13
+        vpxor	96(%rcx), %ymm5, %ymm14
+        vpsrlq	$63, %ymm10, %ymm0
+        vpsrlq	$58, %ymm11, %ymm1
+        vpsrlq	$39, %ymm12, %ymm2
+        vpsrlq	$56, %ymm13, %ymm3
+        vpsrlq	$46, %ymm14, %ymm4
+        vpaddq	%ymm10, %ymm10, %ymm10
+        vpsllq	$6, %ymm11, %ymm11
+        vpsllq	$25, %ymm12, %ymm12
+        vpsllq	$8, %ymm13, %ymm13
+        vpsllq	$18, %ymm14, %ymm14
+        vpor	%ymm0, %ymm10, %ymm10
+        vpor	%ymm1, %ymm11, %ymm11
+        vpor	%ymm2, %ymm12, %ymm12
+        vpor	%ymm3, %ymm13, %ymm13
+        vpor	%ymm4, %ymm14, %ymm14
+        vpandn	%ymm12, %ymm11, %ymm0
+        vpandn	%ymm13, %ymm12, %ymm1
+        vpandn	%ymm14, %ymm13, %ymm2
+        vpandn	%ymm10, %ymm14, %ymm3
+        vpandn	%ymm11, %ymm10, %ymm4
+        vpxor	%ymm10, %ymm0, %ymm0
+        vpxor	%ymm11, %ymm1, %ymm1
+        vpxor	%ymm12, %ymm2, %ymm2
+        vpxor	%ymm13, %ymm3, %ymm3
+        vpxor	%ymm14, %ymm4, %ymm4
+        vmovdqu	%ymm0, -64(%rdi)
+        vmovdqu	%ymm1, 64(%rcx)
+        vmovdqu	%ymm2, 32(%r8)
+        vmovdqu	%ymm3, 128(%rdi)
+        vmovdqu	%ymm4, 96(%rcx)
+        # Row 3
+        vpxor	-32(%rdi), %ymm9, %ymm10
+        vpxor	-64(%rcx), %ymm5, %ymm11
+        vpxor	64(%r8), %ymm6, %ymm12
+        vpxor	-96(%rcx), %ymm7, %ymm13
+        vpxor	128(%rcx), %ymm8, %ymm14
+        vpsrlq	$37, %ymm10, %ymm0
+        vpsrlq	$28, %ymm11, %ymm1
+        vpsrlq	$54, %ymm12, %ymm2
+        vpsrlq	$49, %ymm13, %ymm3
+        vpsrlq	$8, %ymm14, %ymm4
+        vpsllq	$27, %ymm10, %ymm10
+        vpsllq	$36, %ymm11, %ymm11
+        vpsllq	$10, %ymm12, %ymm12
+        vpsllq	$15, %ymm13, %ymm13
+        vpsllq	$56, %ymm14, %ymm14
+        vpor	%ymm0, %ymm10, %ymm10
+        vpor	%ymm1, %ymm11, %ymm11
+        vpor	%ymm2, %ymm12, %ymm12
+        vpor	%ymm3, %ymm13, %ymm13
+        vpor	%ymm4, %ymm14, %ymm14
+        vpandn	%ymm12, %ymm11, %ymm0
+        vpandn	%ymm13, %ymm12, %ymm1
+        vpandn	%ymm14, %ymm13, %ymm2
+        vpandn	%ymm10, %ymm14, %ymm3
+        vpandn	%ymm11, %ymm10, %ymm4
+        vpxor	%ymm10, %ymm0, %ymm0
+        vpxor	%ymm11, %ymm1, %ymm1
+        vpxor	%ymm12, %ymm2, %ymm2
+        vpxor	%ymm13, %ymm3, %ymm3
+        vpxor	%ymm14, %ymm4, %ymm4
+        vmovdqu	%ymm0, -32(%rdi)
+        vmovdqu	%ymm1, -64(%rcx)
+        vmovdqu	%ymm2, 64(%r8)
+        vmovdqu	%ymm3, -96(%rcx)
+        vmovdqu	%ymm4, 128(%rcx)
+        # Row 4
+        vpxor	(%rdi), %ymm7, %ymm10
+        vpxor	-32(%rcx), %ymm8, %ymm11
+        vpxor	96(%r8), %ymm9, %ymm12
+        vpxor	32(%rdi), %ymm5, %ymm13
+        vpxor	-96(%r8), %ymm6, %ymm14
+        vpsrlq	$2, %ymm10, %ymm0
+        vpsrlq	$9, %ymm11, %ymm1
+        vpsrlq	$25, %ymm12, %ymm2
+        vpsrlq	$23, %ymm13, %ymm3
+        vpsrlq	$62, %ymm14, %ymm4
+        vpsllq	$62, %ymm10, %ymm10
+        vpsllq	$55, %ymm11, %ymm11
+        vpsllq	$39, %ymm12, %ymm12
+        vpsllq	$41, %ymm13, %ymm13
+        vpsllq	$2, %ymm14, %ymm14
+        vpor	%ymm0, %ymm10, %ymm10
+        vpor	%ymm1, %ymm11, %ymm11
+        vpor	%ymm2, %ymm12, %ymm12
+        vpor	%ymm3, %ymm13, %ymm13
+        vpor	%ymm4, %ymm14, %ymm14
+        vpandn	%ymm12, %ymm11, %ymm0
+        vpandn	%ymm13, %ymm12, %ymm1
+        vpandn	%ymm14, %ymm13, %ymm2
+        vpandn	%ymm10, %ymm14, %ymm3
+        vpandn	%ymm11, %ymm10, %ymm4
+        vpxor	%ymm10, %ymm0, %ymm0
+        vpxor	%ymm11, %ymm1, %ymm1
+        vpxor	%ymm12, %ymm2, %ymm2
+        vpxor	%ymm13, %ymm3, %ymm3
+        vpxor	%ymm14, %ymm4, %ymm4
+        vmovdqu	%ymm0, (%rdi)
+        vmovdqu	%ymm1, -32(%rcx)
+        vmovdqu	%ymm2, 96(%r8)
+        vmovdqu	%ymm3, 32(%rdi)
+        vmovdqu	%ymm4, -96(%r8)
+        # Round 7
+        # Calc b[0..4]
+        vpxor	%ymm15, %ymm0, %ymm10
+        vpxor	-96(%rdi), %ymm10, %ymm10
+        vpxor	-64(%rdi), %ymm10, %ymm10
+        vpxor	-32(%rdi), %ymm10, %ymm10
+        vpxor	64(%rdi), %ymm3, %ymm13
+        vpxor	96(%rdi), %ymm13, %ymm13
+        vpxor	128(%rdi), %ymm13, %ymm13
+        vpxor	-96(%rcx), %ymm13, %ymm13
+        vpxor	-64(%rcx), %ymm1, %ymm11
+        vpxor	(%rcx), %ymm11, %ymm11
+        vpxor	32(%rcx), %ymm11, %ymm11
+        vpxor	64(%rcx), %ymm11, %ymm11
+        vpxor	96(%rcx), %ymm4, %ymm14
+        vpxor	128(%rcx), %ymm14, %ymm14
+        vpxor	-64(%r8), %ymm14, %ymm14
+        vpxor	-32(%r8), %ymm14, %ymm14
+        vpxor	(%r8), %ymm2, %ymm12
+        vpxor	32(%r8), %ymm12, %ymm12
+        vpxor	64(%r8), %ymm12, %ymm12
+        vpxor	128(%r8), %ymm12, %ymm12
+        # Calc t[0..4]
+        vpsrlq	$63, %ymm11, %ymm0
+        vpsrlq	$63, %ymm12, %ymm1
+        vpsrlq	$63, %ymm13, %ymm2
+        vpsrlq	$63, %ymm14, %ymm3
+        vpsrlq	$63, %ymm10, %ymm4
+        vpaddq	%ymm11, %ymm11, %ymm5
+        vpaddq	%ymm12, %ymm12, %ymm6
+        vpaddq	%ymm13, %ymm13, %ymm7
+        vpaddq	%ymm14, %ymm14, %ymm8
+        vpaddq	%ymm10, %ymm10, %ymm9
+        vpor	%ymm0, %ymm5, %ymm5
+        vpor	%ymm1, %ymm6, %ymm6
+        vpor	%ymm2, %ymm7, %ymm7
+        vpor	%ymm3, %ymm8, %ymm8
+        vpor	%ymm4, %ymm9, %ymm9
+        vpxor	%ymm14, %ymm5, %ymm5
+        vpxor	%ymm10, %ymm6, %ymm6
+        vpxor	%ymm11, %ymm7, %ymm7
+        vpxor	%ymm12, %ymm8, %ymm8
+        vpxor	%ymm13, %ymm9, %ymm9
+        # Row Mix
+        # Row 0
+        vpxor	%ymm15, %ymm5, %ymm10
+        vpxor	32(%rcx), %ymm6, %ymm11
+        vpxor	32(%r8), %ymm7, %ymm12
+        vpxor	-96(%rcx), %ymm8, %ymm13
+        vpxor	-96(%r8), %ymm9, %ymm14
+        vpsrlq	$20, %ymm11, %ymm0
+        vpsrlq	$21, %ymm12, %ymm1
+        vpsrlq	$43, %ymm13, %ymm2
+        vpsrlq	$50, %ymm14, %ymm3
+        vpsllq	$44, %ymm11, %ymm11
+        vpsllq	$43, %ymm12, %ymm12
+        vpsllq	$21, %ymm13, %ymm13
+        vpsllq	$14, %ymm14, %ymm14
+        vpor	%ymm0, %ymm11, %ymm11
+        vpor	%ymm1, %ymm12, %ymm12
+        vpor	%ymm2, %ymm13, %ymm13
+        vpor	%ymm3, %ymm14, %ymm14
+        vpandn	%ymm12, %ymm11, %ymm15
+        vpandn	%ymm13, %ymm12, %ymm1
+        vpandn	%ymm14, %ymm13, %ymm2
+        vpandn	%ymm10, %ymm14, %ymm3
+        vpandn	%ymm11, %ymm10, %ymm4
+        vpxor	%ymm10, %ymm15, %ymm15
+        vpxor	%ymm11, %ymm1, %ymm1
+        vpxor	%ymm12, %ymm2, %ymm2
+        # XOR in constant
+        vpxor	224(%rax), %ymm15, %ymm15
+        vpxor	%ymm13, %ymm3, %ymm3
+        vpxor	%ymm14, %ymm4, %ymm4
+        vmovdqu	%ymm1, 32(%rcx)
+        vmovdqu	%ymm2, 32(%r8)
+        vmovdqu	%ymm3, -96(%rcx)
+        vmovdqu	%ymm4, -96(%r8)
+        # Row 1
+        vpxor	64(%rdi), %ymm8, %ymm10
+        vpxor	-32(%r8), %ymm9, %ymm11
+        vpxor	-64(%rdi), %ymm5, %ymm12
+        vpxor	-64(%rcx), %ymm6, %ymm13
+        vpxor	96(%r8), %ymm7, %ymm14
+        vpsrlq	$36, %ymm10, %ymm0
+        vpsrlq	$44, %ymm11, %ymm1
+        vpsrlq	$61, %ymm12, %ymm2
+        vpsrlq	$19, %ymm13, %ymm3
+        vpsrlq	$3, %ymm14, %ymm4
+        vpsllq	$28, %ymm10, %ymm10
+        vpsllq	$20, %ymm11, %ymm11
+        vpsllq	$3, %ymm12, %ymm12
+        vpsllq	$45, %ymm13, %ymm13
+        vpsllq	$61, %ymm14, %ymm14
+        vpor	%ymm0, %ymm10, %ymm10
+        vpor	%ymm1, %ymm11, %ymm11
+        vpor	%ymm2, %ymm12, %ymm12
+        vpor	%ymm3, %ymm13, %ymm13
+        vpor	%ymm4, %ymm14, %ymm14
+        vpandn	%ymm12, %ymm11, %ymm0
+        vpandn	%ymm13, %ymm12, %ymm1
+        vpandn	%ymm14, %ymm13, %ymm2
+        vpandn	%ymm10, %ymm14, %ymm3
+        vpandn	%ymm11, %ymm10, %ymm4
+        vpxor	%ymm10, %ymm0, %ymm0
+        vpxor	%ymm11, %ymm1, %ymm1
+        vpxor	%ymm12, %ymm2, %ymm2
+        vpxor	%ymm13, %ymm3, %ymm3
+        vpxor	%ymm14, %ymm4, %ymm4
+        vmovdqu	%ymm0, 64(%rdi)
+        vmovdqu	%ymm1, -32(%r8)
+        vmovdqu	%ymm2, -64(%rdi)
+        vmovdqu	%ymm3, -64(%rcx)
+        vmovdqu	%ymm4, 96(%r8)
+        # Row 2
+        vpxor	(%rcx), %ymm6, %ymm10
+        vpxor	(%r8), %ymm7, %ymm11
+        vpxor	128(%rdi), %ymm8, %ymm12
+        vpxor	128(%rcx), %ymm9, %ymm13
+        vpxor	(%rdi), %ymm5, %ymm14
+        vpsrlq	$63, %ymm10, %ymm0
+        vpsrlq	$58, %ymm11, %ymm1
+        vpsrlq	$39, %ymm12, %ymm2
+        vpsrlq	$56, %ymm13, %ymm3
+        vpsrlq	$46, %ymm14, %ymm4
+        vpaddq	%ymm10, %ymm10, %ymm10
+        vpsllq	$6, %ymm11, %ymm11
+        vpsllq	$25, %ymm12, %ymm12
+        vpsllq	$8, %ymm13, %ymm13
+        vpsllq	$18, %ymm14, %ymm14
+        vpor	%ymm0, %ymm10, %ymm10
+        vpor	%ymm1, %ymm11, %ymm11
+        vpor	%ymm2, %ymm12, %ymm12
+        vpor	%ymm3, %ymm13, %ymm13
+        vpor	%ymm4, %ymm14, %ymm14
+        vpandn	%ymm12, %ymm11, %ymm0
+        vpandn	%ymm13, %ymm12, %ymm1
+        vpandn	%ymm14, %ymm13, %ymm2
+        vpandn	%ymm10, %ymm14, %ymm3
+        vpandn	%ymm11, %ymm10, %ymm4
+        vpxor	%ymm10, %ymm0, %ymm0
+        vpxor	%ymm11, %ymm1, %ymm1
+        vpxor	%ymm12, %ymm2, %ymm2
+        vpxor	%ymm13, %ymm3, %ymm3
+        vpxor	%ymm14, %ymm4, %ymm4
+        vmovdqu	%ymm0, (%rcx)
+        vmovdqu	%ymm1, (%r8)
+        vmovdqu	%ymm2, 128(%rdi)
+        vmovdqu	%ymm3, 128(%rcx)
+        vmovdqu	%ymm4, (%rdi)
+        # Row 3
+        vpxor	-64(%r8), %ymm9, %ymm10
+        vpxor	-96(%rdi), %ymm5, %ymm11
+        vpxor	64(%rcx), %ymm6, %ymm12
+        vpxor	64(%r8), %ymm7, %ymm13
+        vpxor	32(%rdi), %ymm8, %ymm14
+        vpsrlq	$37, %ymm10, %ymm0
+        vpsrlq	$28, %ymm11, %ymm1
+        vpsrlq	$54, %ymm12, %ymm2
+        vpsrlq	$49, %ymm13, %ymm3
+        vpsrlq	$8, %ymm14, %ymm4
+        vpsllq	$27, %ymm10, %ymm10
+        vpsllq	$36, %ymm11, %ymm11
+        vpsllq	$10, %ymm12, %ymm12
+        vpsllq	$15, %ymm13, %ymm13
+        vpsllq	$56, %ymm14, %ymm14
+        vpor	%ymm0, %ymm10, %ymm10
+        vpor	%ymm1, %ymm11, %ymm11
+        vpor	%ymm2, %ymm12, %ymm12
+        vpor	%ymm3, %ymm13, %ymm13
+        vpor	%ymm4, %ymm14, %ymm14
+        vpandn	%ymm12, %ymm11, %ymm0
+        vpandn	%ymm13, %ymm12, %ymm1
+        vpandn	%ymm14, %ymm13, %ymm2
+        vpandn	%ymm10, %ymm14, %ymm3
+        vpandn	%ymm11, %ymm10, %ymm4
+        vpxor	%ymm10, %ymm0, %ymm0
+        vpxor	%ymm11, %ymm1, %ymm1
+        vpxor	%ymm12, %ymm2, %ymm2
+        vpxor	%ymm13, %ymm3, %ymm3
+        vpxor	%ymm14, %ymm4, %ymm4
+        vmovdqu	%ymm0, -64(%r8)
+        vmovdqu	%ymm1, -96(%rdi)
+        vmovdqu	%ymm2, 64(%rcx)
+        vmovdqu	%ymm3, 64(%r8)
+        vmovdqu	%ymm4, 32(%rdi)
+        # Row 4
+        vpxor	128(%r8), %ymm7, %ymm10
+        vpxor	96(%rdi), %ymm8, %ymm11
+        vpxor	96(%rcx), %ymm9, %ymm12
+        vpxor	-32(%rdi), %ymm5, %ymm13
+        vpxor	-32(%rcx), %ymm6, %ymm14
+        vpsrlq	$2, %ymm10, %ymm0
+        vpsrlq	$9, %ymm11, %ymm1
+        vpsrlq	$25, %ymm12, %ymm2
+        vpsrlq	$23, %ymm13, %ymm3
+        vpsrlq	$62, %ymm14, %ymm4
+        vpsllq	$62, %ymm10, %ymm10
+        vpsllq	$55, %ymm11, %ymm11
+        vpsllq	$39, %ymm12, %ymm12
+        vpsllq	$41, %ymm13, %ymm13
+        vpsllq	$2, %ymm14, %ymm14
+        vpor	%ymm0, %ymm10, %ymm10
+        vpor	%ymm1, %ymm11, %ymm11
+        vpor	%ymm2, %ymm12, %ymm12
+        vpor	%ymm3, %ymm13, %ymm13
+        vpor	%ymm4, %ymm14, %ymm14
+        vpandn	%ymm12, %ymm11, %ymm0
+        vpandn	%ymm13, %ymm12, %ymm1
+        vpandn	%ymm14, %ymm13, %ymm2
+        vpandn	%ymm10, %ymm14, %ymm3
+        vpandn	%ymm11, %ymm10, %ymm4
+        vpxor	%ymm10, %ymm0, %ymm0
+        vpxor	%ymm11, %ymm1, %ymm1
+        vpxor	%ymm12, %ymm2, %ymm2
+        vpxor	%ymm13, %ymm3, %ymm3
+        vpxor	%ymm14, %ymm4, %ymm4
+        vmovdqu	%ymm0, 128(%r8)
+        vmovdqu	%ymm1, 96(%rdi)
+        vmovdqu	%ymm2, 96(%rcx)
+        vmovdqu	%ymm3, -32(%rdi)
+        vmovdqu	%ymm4, -32(%rcx)
+        # Round 8
+        # Calc b[0..4]
+        vpxor	%ymm15, %ymm0, %ymm10
+        vpxor	-96(%rdi), %ymm1, %ymm11
+        vpxor	-64(%rdi), %ymm2, %ymm12
+        vpxor	(%rdi), %ymm4, %ymm14
+        vpxor	32(%rdi), %ymm14, %ymm14
+        vpxor	64(%rdi), %ymm10, %ymm10
+        vpxor	128(%rdi), %ymm12, %ymm12
+        vpxor	-96(%rcx), %ymm3, %ymm13
+        vpxor	-64(%rcx), %ymm13, %ymm13
+        vpxor	(%rcx), %ymm10, %ymm10
+        vpxor	32(%rcx), %ymm11, %ymm11
+        vpxor	64(%rcx), %ymm12, %ymm12
+        vpxor	128(%rcx), %ymm13, %ymm13
+        vpxor	-96(%r8), %ymm14, %ymm14
+        vpxor	-64(%r8), %ymm10, %ymm10
+        vpxor	-32(%r8), %ymm11, %ymm11
+        vpxor	(%r8), %ymm11, %ymm11
+        vpxor	32(%r8), %ymm12, %ymm12
+        vpxor	64(%r8), %ymm13, %ymm13
+        vpxor	96(%r8), %ymm14, %ymm14
+        # Calc t[0..4]
+        vpsrlq	$63, %ymm11, %ymm0
+        vpsrlq	$63, %ymm12, %ymm1
+        vpsrlq	$63, %ymm13, %ymm2
+        vpsrlq	$63, %ymm14, %ymm3
+        vpsrlq	$63, %ymm10, %ymm4
+        vpaddq	%ymm11, %ymm11, %ymm5
+        vpaddq	%ymm12, %ymm12, %ymm6
+        vpaddq	%ymm13, %ymm13, %ymm7
+        vpaddq	%ymm14, %ymm14, %ymm8
+        vpaddq	%ymm10, %ymm10, %ymm9
+        vpor	%ymm0, %ymm5, %ymm5
+        vpor	%ymm1, %ymm6, %ymm6
+        vpor	%ymm2, %ymm7, %ymm7
+        vpor	%ymm3, %ymm8, %ymm8
+        vpor	%ymm4, %ymm9, %ymm9
+        vpxor	%ymm14, %ymm5, %ymm5
+        vpxor	%ymm10, %ymm6, %ymm6
+        vpxor	%ymm11, %ymm7, %ymm7
+        vpxor	%ymm12, %ymm8, %ymm8
+        vpxor	%ymm13, %ymm9, %ymm9
+        # Row Mix
+        # Row 0
+        vpxor	%ymm15, %ymm5, %ymm10
+        vpxor	-32(%r8), %ymm6, %ymm11
+        vpxor	128(%rdi), %ymm7, %ymm12
+        vpxor	64(%r8), %ymm8, %ymm13
+        vpxor	-32(%rcx), %ymm9, %ymm14
+        vpsrlq	$20, %ymm11, %ymm0
+        vpsrlq	$21, %ymm12, %ymm1
+        vpsrlq	$43, %ymm13, %ymm2
+        vpsrlq	$50, %ymm14, %ymm3
+        vpsllq	$44, %ymm11, %ymm11
+        vpsllq	$43, %ymm12, %ymm12
+        vpsllq	$21, %ymm13, %ymm13
+        vpsllq	$14, %ymm14, %ymm14
+        vpor	%ymm0, %ymm11, %ymm11
+        vpor	%ymm1, %ymm12, %ymm12
+        vpor	%ymm2, %ymm13, %ymm13
+        vpor	%ymm3, %ymm14, %ymm14
+        vpandn	%ymm12, %ymm11, %ymm15
+        vpandn	%ymm13, %ymm12, %ymm1
+        vpandn	%ymm14, %ymm13, %ymm2
+        vpandn	%ymm10, %ymm14, %ymm3
+        vpandn	%ymm11, %ymm10, %ymm4
+        vpxor	%ymm10, %ymm15, %ymm15
+        vpxor	%ymm11, %ymm1, %ymm1
+        vpxor	%ymm12, %ymm2, %ymm2
+        # XOR in constant
+        vpxor	256(%rax), %ymm15, %ymm15
+        vpxor	%ymm13, %ymm3, %ymm3
+        vpxor	%ymm14, %ymm4, %ymm4
+        vmovdqu	%ymm1, -32(%r8)
+        vmovdqu	%ymm2, 128(%rdi)
+        vmovdqu	%ymm3, 64(%r8)
+        vmovdqu	%ymm4, -32(%rcx)
+        # Row 1
+        vpxor	-96(%rcx), %ymm8, %ymm10
+        vpxor	96(%r8), %ymm9, %ymm11
+        vpxor	(%rcx), %ymm5, %ymm12
+        vpxor	-96(%rdi), %ymm6, %ymm13
+        vpxor	96(%rcx), %ymm7, %ymm14
+        vpsrlq	$36, %ymm10, %ymm0
+        vpsrlq	$44, %ymm11, %ymm1
+        vpsrlq	$61, %ymm12, %ymm2
+        vpsrlq	$19, %ymm13, %ymm3
+        vpsrlq	$3, %ymm14, %ymm4
+        vpsllq	$28, %ymm10, %ymm10
+        vpsllq	$20, %ymm11, %ymm11
+        vpsllq	$3, %ymm12, %ymm12
+        vpsllq	$45, %ymm13, %ymm13
+        vpsllq	$61, %ymm14, %ymm14
+        vpor	%ymm0, %ymm10, %ymm10
+        vpor	%ymm1, %ymm11, %ymm11
+        vpor	%ymm2, %ymm12, %ymm12
+        vpor	%ymm3, %ymm13, %ymm13
+        vpor	%ymm4, %ymm14, %ymm14
+        vpandn	%ymm12, %ymm11, %ymm0
+        vpandn	%ymm13, %ymm12, %ymm1
+        vpandn	%ymm14, %ymm13, %ymm2
+        vpandn	%ymm10, %ymm14, %ymm3
+        vpandn	%ymm11, %ymm10, %ymm4
+        vpxor	%ymm10, %ymm0, %ymm0
+        vpxor	%ymm11, %ymm1, %ymm1
+        vpxor	%ymm12, %ymm2, %ymm2
+        vpxor	%ymm13, %ymm3, %ymm3
+        vpxor	%ymm14, %ymm4, %ymm4
+        vmovdqu	%ymm0, -96(%rcx)
+        vmovdqu	%ymm1, 96(%r8)
+        vmovdqu	%ymm2, (%rcx)
+        vmovdqu	%ymm3, -96(%rdi)
+        vmovdqu	%ymm4, 96(%rcx)
+        # Row 2
+        vpxor	32(%rcx), %ymm6, %ymm10
+        vpxor	-64(%rdi), %ymm7, %ymm11
+        vpxor	128(%rcx), %ymm8, %ymm12
+        vpxor	32(%rdi), %ymm9, %ymm13
+        vpxor	128(%r8), %ymm5, %ymm14
+        vpsrlq	$63, %ymm10, %ymm0
+        vpsrlq	$58, %ymm11, %ymm1
+        vpsrlq	$39, %ymm12, %ymm2
+        vpsrlq	$56, %ymm13, %ymm3
+        vpsrlq	$46, %ymm14, %ymm4
+        vpaddq	%ymm10, %ymm10, %ymm10
+        vpsllq	$6, %ymm11, %ymm11
+        vpsllq	$25, %ymm12, %ymm12
+        vpsllq	$8, %ymm13, %ymm13
+        vpsllq	$18, %ymm14, %ymm14
+        vpor	%ymm0, %ymm10, %ymm10
+        vpor	%ymm1, %ymm11, %ymm11
+        vpor	%ymm2, %ymm12, %ymm12
+        vpor	%ymm3, %ymm13, %ymm13
+        vpor	%ymm4, %ymm14, %ymm14
+        vpandn	%ymm12, %ymm11, %ymm0
+        vpandn	%ymm13, %ymm12, %ymm1
+        vpandn	%ymm14, %ymm13, %ymm2
+        vpandn	%ymm10, %ymm14, %ymm3
+        vpandn	%ymm11, %ymm10, %ymm4
+        vpxor	%ymm10, %ymm0, %ymm0
+        vpxor	%ymm11, %ymm1, %ymm1
+        vpxor	%ymm12, %ymm2, %ymm2
+        vpxor	%ymm13, %ymm3, %ymm3
+        vpxor	%ymm14, %ymm4, %ymm4
+        vmovdqu	%ymm0, 32(%rcx)
+        vmovdqu	%ymm1, -64(%rdi)
+        vmovdqu	%ymm2, 128(%rcx)
+        vmovdqu	%ymm3, 32(%rdi)
+        vmovdqu	%ymm4, 128(%r8)
+        # Row 3
+        vpxor	-96(%r8), %ymm9, %ymm10
+        vpxor	64(%rdi), %ymm5, %ymm11
+        vpxor	(%r8), %ymm6, %ymm12
+        vpxor	64(%rcx), %ymm7, %ymm13
+        vpxor	-32(%rdi), %ymm8, %ymm14
+        vpsrlq	$37, %ymm10, %ymm0
+        vpsrlq	$28, %ymm11, %ymm1
+        vpsrlq	$54, %ymm12, %ymm2
+        vpsrlq	$49, %ymm13, %ymm3
+        vpsrlq	$8, %ymm14, %ymm4
+        vpsllq	$27, %ymm10, %ymm10
+        vpsllq	$36, %ymm11, %ymm11
+        vpsllq	$10, %ymm12, %ymm12
+        vpsllq	$15, %ymm13, %ymm13
+        vpsllq	$56, %ymm14, %ymm14
+        vpor	%ymm0, %ymm10, %ymm10
+        vpor	%ymm1, %ymm11, %ymm11
+        vpor	%ymm2, %ymm12, %ymm12
+        vpor	%ymm3, %ymm13, %ymm13
+        vpor	%ymm4, %ymm14, %ymm14
+        vpandn	%ymm12, %ymm11, %ymm0
+        vpandn	%ymm13, %ymm12, %ymm1
+        vpandn	%ymm14, %ymm13, %ymm2
+        vpandn	%ymm10, %ymm14, %ymm3
+        vpandn	%ymm11, %ymm10, %ymm4
+        vpxor	%ymm10, %ymm0, %ymm0
+        vpxor	%ymm11, %ymm1, %ymm1
+        vpxor	%ymm12, %ymm2, %ymm2
+        vpxor	%ymm13, %ymm3, %ymm3
+        vpxor	%ymm14, %ymm4, %ymm4
+        vmovdqu	%ymm0, -96(%r8)
+        vmovdqu	%ymm1, 64(%rdi)
+        vmovdqu	%ymm2, (%r8)
+        vmovdqu	%ymm3, 64(%rcx)
+        vmovdqu	%ymm4, -32(%rdi)
+        # Row 4
+        vpxor	32(%r8), %ymm7, %ymm10
+        vpxor	-64(%rcx), %ymm8, %ymm11
+        vpxor	(%rdi), %ymm9, %ymm12
+        vpxor	-64(%r8), %ymm5, %ymm13
+        vpxor	96(%rdi), %ymm6, %ymm14
+        vpsrlq	$2, %ymm10, %ymm0
+        vpsrlq	$9, %ymm11, %ymm1
+        vpsrlq	$25, %ymm12, %ymm2
+        vpsrlq	$23, %ymm13, %ymm3
+        vpsrlq	$62, %ymm14, %ymm4
+        vpsllq	$62, %ymm10, %ymm10
+        vpsllq	$55, %ymm11, %ymm11
+        vpsllq	$39, %ymm12, %ymm12
+        vpsllq	$41, %ymm13, %ymm13
+        vpsllq	$2, %ymm14, %ymm14
+        vpor	%ymm0, %ymm10, %ymm10
+        vpor	%ymm1, %ymm11, %ymm11
+        vpor	%ymm2, %ymm12, %ymm12
+        vpor	%ymm3, %ymm13, %ymm13
+        vpor	%ymm4, %ymm14, %ymm14
+        vpandn	%ymm12, %ymm11, %ymm0
+        vpandn	%ymm13, %ymm12, %ymm1
+        vpandn	%ymm14, %ymm13, %ymm2
+        vpandn	%ymm10, %ymm14, %ymm3
+        vpandn	%ymm11, %ymm10, %ymm4
+        vpxor	%ymm10, %ymm0, %ymm0
+        vpxor	%ymm11, %ymm1, %ymm1
+        vpxor	%ymm12, %ymm2, %ymm2
+        vpxor	%ymm13, %ymm3, %ymm3
+        vpxor	%ymm14, %ymm4, %ymm4
+        vmovdqu	%ymm0, 32(%r8)
+        vmovdqu	%ymm1, -64(%rcx)
+        vmovdqu	%ymm2, (%rdi)
+        vmovdqu	%ymm3, -64(%r8)
+        vmovdqu	%ymm4, 96(%rdi)
+        # Round 9
+        # Calc b[0..4]
+        vpxor	%ymm15, %ymm0, %ymm10
+        vpxor	-96(%rdi), %ymm3, %ymm13
+        vpxor	-64(%rdi), %ymm1, %ymm11
+        vpxor	-32(%rdi), %ymm4, %ymm14
+        vpxor	32(%rdi), %ymm13, %ymm13
+        vpxor	64(%rdi), %ymm11, %ymm11
+        vpxor	128(%rdi), %ymm2, %ymm12
+        vpxor	-96(%rcx), %ymm10, %ymm10
+        vpxor	-32(%rcx), %ymm14, %ymm14
+        vpxor	(%rcx), %ymm12, %ymm12
+        vpxor	32(%rcx), %ymm10, %ymm10
+        vpxor	64(%rcx), %ymm13, %ymm13
+        vpxor	96(%rcx), %ymm14, %ymm14
+        vpxor	128(%rcx), %ymm12, %ymm12
+        vpxor	-96(%r8), %ymm10, %ymm10
+        vpxor	-32(%r8), %ymm11, %ymm11
+        vpxor	(%r8), %ymm12, %ymm12
+        vpxor	64(%r8), %ymm13, %ymm13
+        vpxor	96(%r8), %ymm11, %ymm11
+        vpxor	128(%r8), %ymm14, %ymm14
+        # Calc t[0..4]
+        vpsrlq	$63, %ymm11, %ymm0
+        vpsrlq	$63, %ymm12, %ymm1
+        vpsrlq	$63, %ymm13, %ymm2
+        vpsrlq	$63, %ymm14, %ymm3
+        vpsrlq	$63, %ymm10, %ymm4
+        vpaddq	%ymm11, %ymm11, %ymm5
+        vpaddq	%ymm12, %ymm12, %ymm6
+        vpaddq	%ymm13, %ymm13, %ymm7
+        vpaddq	%ymm14, %ymm14, %ymm8
+        vpaddq	%ymm10, %ymm10, %ymm9
+        vpor	%ymm0, %ymm5, %ymm5
+        vpor	%ymm1, %ymm6, %ymm6
+        vpor	%ymm2, %ymm7, %ymm7
+        vpor	%ymm3, %ymm8, %ymm8
+        vpor	%ymm4, %ymm9, %ymm9
+        vpxor	%ymm14, %ymm5, %ymm5
+        vpxor	%ymm10, %ymm6, %ymm6
+        vpxor	%ymm11, %ymm7, %ymm7
+        vpxor	%ymm12, %ymm8, %ymm8
+        vpxor	%ymm13, %ymm9, %ymm9
+        # Row Mix
+        # Row 0
+        vpxor	%ymm15, %ymm5, %ymm10
+        vpxor	96(%r8), %ymm6, %ymm11
+        vpxor	128(%rcx), %ymm7, %ymm12
+        vpxor	64(%rcx), %ymm8, %ymm13
+        vpxor	96(%rdi), %ymm9, %ymm14
+        vpsrlq	$20, %ymm11, %ymm0
+        vpsrlq	$21, %ymm12, %ymm1
+        vpsrlq	$43, %ymm13, %ymm2
+        vpsrlq	$50, %ymm14, %ymm3
+        vpsllq	$44, %ymm11, %ymm11
+        vpsllq	$43, %ymm12, %ymm12
+        vpsllq	$21, %ymm13, %ymm13
+        vpsllq	$14, %ymm14, %ymm14
+        vpor	%ymm0, %ymm11, %ymm11
+        vpor	%ymm1, %ymm12, %ymm12
+        vpor	%ymm2, %ymm13, %ymm13
+        vpor	%ymm3, %ymm14, %ymm14
+        vpandn	%ymm12, %ymm11, %ymm15
+        vpandn	%ymm13, %ymm12, %ymm1
+        vpandn	%ymm14, %ymm13, %ymm2
+        vpandn	%ymm10, %ymm14, %ymm3
+        vpandn	%ymm11, %ymm10, %ymm4
+        vpxor	%ymm10, %ymm15, %ymm15
+        vpxor	%ymm11, %ymm1, %ymm1
+        vpxor	%ymm12, %ymm2, %ymm2
+        # XOR in constant
+        vpxor	288(%rax), %ymm15, %ymm15
+        vpxor	%ymm13, %ymm3, %ymm3
+        vpxor	%ymm14, %ymm4, %ymm4
+        vmovdqu	%ymm1, 96(%r8)
+        vmovdqu	%ymm2, 128(%rcx)
+        vmovdqu	%ymm3, 64(%rcx)
+        vmovdqu	%ymm4, 96(%rdi)
+        # Row 1
+        vpxor	64(%r8), %ymm8, %ymm10
+        vpxor	96(%rcx), %ymm9, %ymm11
+        vpxor	32(%rcx), %ymm5, %ymm12
+        vpxor	64(%rdi), %ymm6, %ymm13
+        vpxor	(%rdi), %ymm7, %ymm14
+        vpsrlq	$36, %ymm10, %ymm0
+        vpsrlq	$44, %ymm11, %ymm1
+        vpsrlq	$61, %ymm12, %ymm2
+        vpsrlq	$19, %ymm13, %ymm3
+        vpsrlq	$3, %ymm14, %ymm4
+        vpsllq	$28, %ymm10, %ymm10
+        vpsllq	$20, %ymm11, %ymm11
+        vpsllq	$3, %ymm12, %ymm12
+        vpsllq	$45, %ymm13, %ymm13
+        vpsllq	$61, %ymm14, %ymm14
+        vpor	%ymm0, %ymm10, %ymm10
+        vpor	%ymm1, %ymm11, %ymm11
+        vpor	%ymm2, %ymm12, %ymm12
+        vpor	%ymm3, %ymm13, %ymm13
+        vpor	%ymm4, %ymm14, %ymm14
+        vpandn	%ymm12, %ymm11, %ymm0
+        vpandn	%ymm13, %ymm12, %ymm1
+        vpandn	%ymm14, %ymm13, %ymm2
+        vpandn	%ymm10, %ymm14, %ymm3
+        vpandn	%ymm11, %ymm10, %ymm4
+        vpxor	%ymm10, %ymm0, %ymm0
+        vpxor	%ymm11, %ymm1, %ymm1
+        vpxor	%ymm12, %ymm2, %ymm2
+        vpxor	%ymm13, %ymm3, %ymm3
+        vpxor	%ymm14, %ymm4, %ymm4
+        vmovdqu	%ymm0, 64(%r8)
+        vmovdqu	%ymm1, 96(%rcx)
+        vmovdqu	%ymm2, 32(%rcx)
+        vmovdqu	%ymm3, 64(%rdi)
+        vmovdqu	%ymm4, (%rdi)
+        # Row 2
+        vpxor	-32(%r8), %ymm6, %ymm10
+        vpxor	(%rcx), %ymm7, %ymm11
+        vpxor	32(%rdi), %ymm8, %ymm12
+        vpxor	-32(%rdi), %ymm9, %ymm13
+        vpxor	32(%r8), %ymm5, %ymm14
+        vpsrlq	$63, %ymm10, %ymm0
+        vpsrlq	$58, %ymm11, %ymm1
+        vpsrlq	$39, %ymm12, %ymm2
+        vpsrlq	$56, %ymm13, %ymm3
+        vpsrlq	$46, %ymm14, %ymm4
+        vpaddq	%ymm10, %ymm10, %ymm10
+        vpsllq	$6, %ymm11, %ymm11
+        vpsllq	$25, %ymm12, %ymm12
+        vpsllq	$8, %ymm13, %ymm13
+        vpsllq	$18, %ymm14, %ymm14
+        vpor	%ymm0, %ymm10, %ymm10
+        vpor	%ymm1, %ymm11, %ymm11
+        vpor	%ymm2, %ymm12, %ymm12
+        vpor	%ymm3, %ymm13, %ymm13
+        vpor	%ymm4, %ymm14, %ymm14
+        vpandn	%ymm12, %ymm11, %ymm0
+        vpandn	%ymm13, %ymm12, %ymm1
+        vpandn	%ymm14, %ymm13, %ymm2
+        vpandn	%ymm10, %ymm14, %ymm3
+        vpandn	%ymm11, %ymm10, %ymm4
+        vpxor	%ymm10, %ymm0, %ymm0
+        vpxor	%ymm11, %ymm1, %ymm1
+        vpxor	%ymm12, %ymm2, %ymm2
+        vpxor	%ymm13, %ymm3, %ymm3
+        vpxor	%ymm14, %ymm4, %ymm4
+        vmovdqu	%ymm0, -32(%r8)
+        vmovdqu	%ymm1, (%rcx)
+        vmovdqu	%ymm2, 32(%rdi)
+        vmovdqu	%ymm3, -32(%rdi)
+        vmovdqu	%ymm4, 32(%r8)
+        # Row 3
+        vpxor	-32(%rcx), %ymm9, %ymm10
+        vpxor	-96(%rcx), %ymm5, %ymm11
+        vpxor	-64(%rdi), %ymm6, %ymm12
+        vpxor	(%r8), %ymm7, %ymm13
+        vpxor	-64(%r8), %ymm8, %ymm14
+        vpsrlq	$37, %ymm10, %ymm0
+        vpsrlq	$28, %ymm11, %ymm1
+        vpsrlq	$54, %ymm12, %ymm2
+        vpsrlq	$49, %ymm13, %ymm3
+        vpsrlq	$8, %ymm14, %ymm4
+        vpsllq	$27, %ymm10, %ymm10
+        vpsllq	$36, %ymm11, %ymm11
+        vpsllq	$10, %ymm12, %ymm12
+        vpsllq	$15, %ymm13, %ymm13
+        vpsllq	$56, %ymm14, %ymm14
+        vpor	%ymm0, %ymm10, %ymm10
+        vpor	%ymm1, %ymm11, %ymm11
+        vpor	%ymm2, %ymm12, %ymm12
+        vpor	%ymm3, %ymm13, %ymm13
+        vpor	%ymm4, %ymm14, %ymm14
+        vpandn	%ymm12, %ymm11, %ymm0
+        vpandn	%ymm13, %ymm12, %ymm1
+        vpandn	%ymm14, %ymm13, %ymm2
+        vpandn	%ymm10, %ymm14, %ymm3
+        vpandn	%ymm11, %ymm10, %ymm4
+        vpxor	%ymm10, %ymm0, %ymm0
+        vpxor	%ymm11, %ymm1, %ymm1
+        vpxor	%ymm12, %ymm2, %ymm2
+        vpxor	%ymm13, %ymm3, %ymm3
+        vpxor	%ymm14, %ymm4, %ymm4
+        vmovdqu	%ymm0, -32(%rcx)
+        vmovdqu	%ymm1, -96(%rcx)
+        vmovdqu	%ymm2, -64(%rdi)
+        vmovdqu	%ymm3, (%r8)
+        vmovdqu	%ymm4, -64(%r8)
+        # Row 4
+        vpxor	128(%rdi), %ymm7, %ymm10
+        vpxor	-96(%rdi), %ymm8, %ymm11
+        vpxor	128(%r8), %ymm9, %ymm12
+        vpxor	-96(%r8), %ymm5, %ymm13
+        vpxor	-64(%rcx), %ymm6, %ymm14
+        vpsrlq	$2, %ymm10, %ymm0
+        vpsrlq	$9, %ymm11, %ymm1
+        vpsrlq	$25, %ymm12, %ymm2
+        vpsrlq	$23, %ymm13, %ymm3
+        vpsrlq	$62, %ymm14, %ymm4
+        vpsllq	$62, %ymm10, %ymm10
+        vpsllq	$55, %ymm11, %ymm11
+        vpsllq	$39, %ymm12, %ymm12
+        vpsllq	$41, %ymm13, %ymm13
+        vpsllq	$2, %ymm14, %ymm14
+        vpor	%ymm0, %ymm10, %ymm10
+        vpor	%ymm1, %ymm11, %ymm11
+        vpor	%ymm2, %ymm12, %ymm12
+        vpor	%ymm3, %ymm13, %ymm13
+        vpor	%ymm4, %ymm14, %ymm14
+        vpandn	%ymm12, %ymm11, %ymm0
+        vpandn	%ymm13, %ymm12, %ymm1
+        vpandn	%ymm14, %ymm13, %ymm2
+        vpandn	%ymm10, %ymm14, %ymm3
+        vpandn	%ymm11, %ymm10, %ymm4
+        vpxor	%ymm10, %ymm0, %ymm0
+        vpxor	%ymm11, %ymm1, %ymm1
+        vpxor	%ymm12, %ymm2, %ymm2
+        vpxor	%ymm13, %ymm3, %ymm3
+        vpxor	%ymm14, %ymm4, %ymm4
+        vmovdqu	%ymm0, 128(%rdi)
+        vmovdqu	%ymm1, -96(%rdi)
+        vmovdqu	%ymm2, 128(%r8)
+        vmovdqu	%ymm3, -96(%r8)
+        vmovdqu	%ymm4, -64(%rcx)
+        # Round 10
+        # Calc b[0..4]
+        vpxor	%ymm15, %ymm0, %ymm10
+        vpxor	-64(%rdi), %ymm2, %ymm12
+        vpxor	-32(%rdi), %ymm3, %ymm13
+        vpxor	(%rdi), %ymm4, %ymm14
+        vpxor	32(%rdi), %ymm12, %ymm12
+        vpxor	64(%rdi), %ymm13, %ymm13
+        vpxor	96(%rdi), %ymm14, %ymm14
+        vpxor	-96(%rcx), %ymm1, %ymm11
+        vpxor	-32(%rcx), %ymm10, %ymm10
+        vpxor	(%rcx), %ymm11, %ymm11
+        vpxor	32(%rcx), %ymm12, %ymm12
+        vpxor	64(%rcx), %ymm13, %ymm13
+        vpxor	96(%rcx), %ymm11, %ymm11
+        vpxor	128(%rcx), %ymm12, %ymm12
+        vpxor	-64(%r8), %ymm14, %ymm14
+        vpxor	-32(%r8), %ymm10, %ymm10
+        vpxor	(%r8), %ymm13, %ymm13
+        vpxor	32(%r8), %ymm14, %ymm14
+        vpxor	64(%r8), %ymm10, %ymm10
+        vpxor	96(%r8), %ymm11, %ymm11
+        # Calc t[0..4]
+        vpsrlq	$63, %ymm11, %ymm0
+        vpsrlq	$63, %ymm12, %ymm1
+        vpsrlq	$63, %ymm13, %ymm2
+        vpsrlq	$63, %ymm14, %ymm3
+        vpsrlq	$63, %ymm10, %ymm4
+        vpaddq	%ymm11, %ymm11, %ymm5
+        vpaddq	%ymm12, %ymm12, %ymm6
+        vpaddq	%ymm13, %ymm13, %ymm7
+        vpaddq	%ymm14, %ymm14, %ymm8
+        vpaddq	%ymm10, %ymm10, %ymm9
+        vpor	%ymm0, %ymm5, %ymm5
+        vpor	%ymm1, %ymm6, %ymm6
+        vpor	%ymm2, %ymm7, %ymm7
+        vpor	%ymm3, %ymm8, %ymm8
+        vpor	%ymm4, %ymm9, %ymm9
+        vpxor	%ymm14, %ymm5, %ymm5
+        vpxor	%ymm10, %ymm6, %ymm6
+        vpxor	%ymm11, %ymm7, %ymm7
+        vpxor	%ymm12, %ymm8, %ymm8
+        vpxor	%ymm13, %ymm9, %ymm9
+        # Row Mix
+        # Row 0
+        vpxor	%ymm15, %ymm5, %ymm10
+        vpxor	96(%rcx), %ymm6, %ymm11
+        vpxor	32(%rdi), %ymm7, %ymm12
+        vpxor	(%r8), %ymm8, %ymm13
+        vpxor	-64(%rcx), %ymm9, %ymm14
+        vpsrlq	$20, %ymm11, %ymm0
+        vpsrlq	$21, %ymm12, %ymm1
+        vpsrlq	$43, %ymm13, %ymm2
+        vpsrlq	$50, %ymm14, %ymm3
+        vpsllq	$44, %ymm11, %ymm11
+        vpsllq	$43, %ymm12, %ymm12
+        vpsllq	$21, %ymm13, %ymm13
+        vpsllq	$14, %ymm14, %ymm14
+        vpor	%ymm0, %ymm11, %ymm11
+        vpor	%ymm1, %ymm12, %ymm12
+        vpor	%ymm2, %ymm13, %ymm13
+        vpor	%ymm3, %ymm14, %ymm14
+        vpandn	%ymm12, %ymm11, %ymm15
+        vpandn	%ymm13, %ymm12, %ymm1
+        vpandn	%ymm14, %ymm13, %ymm2
+        vpandn	%ymm10, %ymm14, %ymm3
+        vpandn	%ymm11, %ymm10, %ymm4
+        vpxor	%ymm10, %ymm15, %ymm15
+        vpxor	%ymm11, %ymm1, %ymm1
+        vpxor	%ymm12, %ymm2, %ymm2
+        # XOR in constant
+        vpxor	320(%rax), %ymm15, %ymm15
+        vpxor	%ymm13, %ymm3, %ymm3
+        vpxor	%ymm14, %ymm4, %ymm4
+        vmovdqu	%ymm1, 96(%rcx)
+        vmovdqu	%ymm2, 32(%rdi)
+        vmovdqu	%ymm3, (%r8)
+        vmovdqu	%ymm4, -64(%rcx)
+        # Row 1
+        vpxor	64(%rcx), %ymm8, %ymm10
+        vpxor	(%rdi), %ymm9, %ymm11
+        vpxor	-32(%r8), %ymm5, %ymm12
+        vpxor	-96(%rcx), %ymm6, %ymm13
+        vpxor	128(%r8), %ymm7, %ymm14
+        vpsrlq	$36, %ymm10, %ymm0
+        vpsrlq	$44, %ymm11, %ymm1
+        vpsrlq	$61, %ymm12, %ymm2
+        vpsrlq	$19, %ymm13, %ymm3
+        vpsrlq	$3, %ymm14, %ymm4
+        vpsllq	$28, %ymm10, %ymm10
+        vpsllq	$20, %ymm11, %ymm11
+        vpsllq	$3, %ymm12, %ymm12
+        vpsllq	$45, %ymm13, %ymm13
+        vpsllq	$61, %ymm14, %ymm14
+        vpor	%ymm0, %ymm10, %ymm10
+        vpor	%ymm1, %ymm11, %ymm11
+        vpor	%ymm2, %ymm12, %ymm12
+        vpor	%ymm3, %ymm13, %ymm13
+        vpor	%ymm4, %ymm14, %ymm14
+        vpandn	%ymm12, %ymm11, %ymm0
+        vpandn	%ymm13, %ymm12, %ymm1
+        vpandn	%ymm14, %ymm13, %ymm2
+        vpandn	%ymm10, %ymm14, %ymm3
+        vpandn	%ymm11, %ymm10, %ymm4
+        vpxor	%ymm10, %ymm0, %ymm0
+        vpxor	%ymm11, %ymm1, %ymm1
+        vpxor	%ymm12, %ymm2, %ymm2
+        vpxor	%ymm13, %ymm3, %ymm3
+        vpxor	%ymm14, %ymm4, %ymm4
+        vmovdqu	%ymm0, 64(%rcx)
+        vmovdqu	%ymm1, (%rdi)
+        vmovdqu	%ymm2, -32(%r8)
+        vmovdqu	%ymm3, -96(%rcx)
+        vmovdqu	%ymm4, 128(%r8)
+        # Row 2
+        vpxor	96(%r8), %ymm6, %ymm10
+        vpxor	32(%rcx), %ymm7, %ymm11
+        vpxor	-32(%rdi), %ymm8, %ymm12
+        vpxor	-64(%r8), %ymm9, %ymm13
+        vpxor	128(%rdi), %ymm5, %ymm14
+        vpsrlq	$63, %ymm10, %ymm0
+        vpsrlq	$58, %ymm11, %ymm1
+        vpsrlq	$39, %ymm12, %ymm2
+        vpsrlq	$56, %ymm13, %ymm3
+        vpsrlq	$46, %ymm14, %ymm4
+        vpaddq	%ymm10, %ymm10, %ymm10
+        vpsllq	$6, %ymm11, %ymm11
+        vpsllq	$25, %ymm12, %ymm12
+        vpsllq	$8, %ymm13, %ymm13
+        vpsllq	$18, %ymm14, %ymm14
+        vpor	%ymm0, %ymm10, %ymm10
+        vpor	%ymm1, %ymm11, %ymm11
+        vpor	%ymm2, %ymm12, %ymm12
+        vpor	%ymm3, %ymm13, %ymm13
+        vpor	%ymm4, %ymm14, %ymm14
+        vpandn	%ymm12, %ymm11, %ymm0
+        vpandn	%ymm13, %ymm12, %ymm1
+        vpandn	%ymm14, %ymm13, %ymm2
+        vpandn	%ymm10, %ymm14, %ymm3
+        vpandn	%ymm11, %ymm10, %ymm4
+        vpxor	%ymm10, %ymm0, %ymm0
+        vpxor	%ymm11, %ymm1, %ymm1
+        vpxor	%ymm12, %ymm2, %ymm2
+        vpxor	%ymm13, %ymm3, %ymm3
+        vpxor	%ymm14, %ymm4, %ymm4
+        vmovdqu	%ymm0, 96(%r8)
+        vmovdqu	%ymm1, 32(%rcx)
+        vmovdqu	%ymm2, -32(%rdi)
+        vmovdqu	%ymm3, -64(%r8)
+        vmovdqu	%ymm4, 128(%rdi)
+        # Row 3
+        vpxor	96(%rdi), %ymm9, %ymm10
+        vpxor	64(%r8), %ymm5, %ymm11
+        vpxor	(%rcx), %ymm6, %ymm12
+        vpxor	-64(%rdi), %ymm7, %ymm13
+        vpxor	-96(%r8), %ymm8, %ymm14
+        vpsrlq	$37, %ymm10, %ymm0
+        vpsrlq	$28, %ymm11, %ymm1
+        vpsrlq	$54, %ymm12, %ymm2
+        vpsrlq	$49, %ymm13, %ymm3
+        vpsrlq	$8, %ymm14, %ymm4
+        vpsllq	$27, %ymm10, %ymm10
+        vpsllq	$36, %ymm11, %ymm11
+        vpsllq	$10, %ymm12, %ymm12
+        vpsllq	$15, %ymm13, %ymm13
+        vpsllq	$56, %ymm14, %ymm14
+        vpor	%ymm0, %ymm10, %ymm10
+        vpor	%ymm1, %ymm11, %ymm11
+        vpor	%ymm2, %ymm12, %ymm12
+        vpor	%ymm3, %ymm13, %ymm13
+        vpor	%ymm4, %ymm14, %ymm14
+        vpandn	%ymm12, %ymm11, %ymm0
+        vpandn	%ymm13, %ymm12, %ymm1
+        vpandn	%ymm14, %ymm13, %ymm2
+        vpandn	%ymm10, %ymm14, %ymm3
+        vpandn	%ymm11, %ymm10, %ymm4
+        vpxor	%ymm10, %ymm0, %ymm0
+        vpxor	%ymm11, %ymm1, %ymm1
+        vpxor	%ymm12, %ymm2, %ymm2
+        vpxor	%ymm13, %ymm3, %ymm3
+        vpxor	%ymm14, %ymm4, %ymm4
+        vmovdqu	%ymm0, 96(%rdi)
+        vmovdqu	%ymm1, 64(%r8)
+        vmovdqu	%ymm2, (%rcx)
+        vmovdqu	%ymm3, -64(%rdi)
+        vmovdqu	%ymm4, -96(%r8)
+        # Row 4
+        vpxor	128(%rcx), %ymm7, %ymm10
+        vpxor	64(%rdi), %ymm8, %ymm11
+        vpxor	32(%r8), %ymm9, %ymm12
+        vpxor	-32(%rcx), %ymm5, %ymm13
+        vpxor	-96(%rdi), %ymm6, %ymm14
+        vpsrlq	$2, %ymm10, %ymm0
+        vpsrlq	$9, %ymm11, %ymm1
+        vpsrlq	$25, %ymm12, %ymm2
+        vpsrlq	$23, %ymm13, %ymm3
+        vpsrlq	$62, %ymm14, %ymm4
+        vpsllq	$62, %ymm10, %ymm10
+        vpsllq	$55, %ymm11, %ymm11
+        vpsllq	$39, %ymm12, %ymm12
+        vpsllq	$41, %ymm13, %ymm13
+        vpsllq	$2, %ymm14, %ymm14
+        vpor	%ymm0, %ymm10, %ymm10
+        vpor	%ymm1, %ymm11, %ymm11
+        vpor	%ymm2, %ymm12, %ymm12
+        vpor	%ymm3, %ymm13, %ymm13
+        vpor	%ymm4, %ymm14, %ymm14
+        vpandn	%ymm12, %ymm11, %ymm0
+        vpandn	%ymm13, %ymm12, %ymm1
+        vpandn	%ymm14, %ymm13, %ymm2
+        vpandn	%ymm10, %ymm14, %ymm3
+        vpandn	%ymm11, %ymm10, %ymm4
+        vpxor	%ymm10, %ymm0, %ymm0
+        vpxor	%ymm11, %ymm1, %ymm1
+        vpxor	%ymm12, %ymm2, %ymm2
+        vpxor	%ymm13, %ymm3, %ymm3
+        vpxor	%ymm14, %ymm4, %ymm4
+        vmovdqu	%ymm0, 128(%rcx)
+        vmovdqu	%ymm1, 64(%rdi)
+        vmovdqu	%ymm2, 32(%r8)
+        vmovdqu	%ymm3, -32(%rcx)
+        vmovdqu	%ymm4, -96(%rdi)
+        # Round 11
+        # Calc b[0..4]
+        vpxor	%ymm15, %ymm0, %ymm10
+        vpxor	-64(%rdi), %ymm3, %ymm13
+        vpxor	-32(%rdi), %ymm2, %ymm12
+        vpxor	(%rdi), %ymm1, %ymm11
+        vpxor	32(%rdi), %ymm12, %ymm12
+        vpxor	96(%rdi), %ymm10, %ymm10
+        vpxor	128(%rdi), %ymm4, %ymm14
+        vpxor	-96(%rcx), %ymm13, %ymm13
+        vpxor	-64(%rcx), %ymm14, %ymm14
+        vpxor	(%rcx), %ymm12, %ymm12
+        vpxor	32(%rcx), %ymm11, %ymm11
+        vpxor	64(%rcx), %ymm10, %ymm10
+        vpxor	96(%rcx), %ymm11, %ymm11
+        vpxor	-96(%r8), %ymm14, %ymm14
+        vpxor	-64(%r8), %ymm13, %ymm13
+        vpxor	-32(%r8), %ymm12, %ymm12
+        vpxor	(%r8), %ymm13, %ymm13
+        vpxor	64(%r8), %ymm11, %ymm11
+        vpxor	96(%r8), %ymm10, %ymm10
+        vpxor	128(%r8), %ymm14, %ymm14
+        # Calc t[0..4]
+        vpsrlq	$63, %ymm11, %ymm0
+        vpsrlq	$63, %ymm12, %ymm1
+        vpsrlq	$63, %ymm13, %ymm2
+        vpsrlq	$63, %ymm14, %ymm3
+        vpsrlq	$63, %ymm10, %ymm4
+        vpaddq	%ymm11, %ymm11, %ymm5
+        vpaddq	%ymm12, %ymm12, %ymm6
+        vpaddq	%ymm13, %ymm13, %ymm7
+        vpaddq	%ymm14, %ymm14, %ymm8
+        vpaddq	%ymm10, %ymm10, %ymm9
+        vpor	%ymm0, %ymm5, %ymm5
+        vpor	%ymm1, %ymm6, %ymm6
+        vpor	%ymm2, %ymm7, %ymm7
+        vpor	%ymm3, %ymm8, %ymm8
+        vpor	%ymm4, %ymm9, %ymm9
+        vpxor	%ymm14, %ymm5, %ymm5
+        vpxor	%ymm10, %ymm6, %ymm6
+        vpxor	%ymm11, %ymm7, %ymm7
+        vpxor	%ymm12, %ymm8, %ymm8
+        vpxor	%ymm13, %ymm9, %ymm9
+        # Row Mix
+        # Row 0
+        vpxor	%ymm15, %ymm5, %ymm10
+        vpxor	(%rdi), %ymm6, %ymm11
+        vpxor	-32(%rdi), %ymm7, %ymm12
+        vpxor	-64(%rdi), %ymm8, %ymm13
+        vpxor	-96(%rdi), %ymm9, %ymm14
+        vpsrlq	$20, %ymm11, %ymm0
+        vpsrlq	$21, %ymm12, %ymm1
+        vpsrlq	$43, %ymm13, %ymm2
+        vpsrlq	$50, %ymm14, %ymm3
+        vpsllq	$44, %ymm11, %ymm11
+        vpsllq	$43, %ymm12, %ymm12
+        vpsllq	$21, %ymm13, %ymm13
+        vpsllq	$14, %ymm14, %ymm14
+        vpor	%ymm0, %ymm11, %ymm11
+        vpor	%ymm1, %ymm12, %ymm12
+        vpor	%ymm2, %ymm13, %ymm13
+        vpor	%ymm3, %ymm14, %ymm14
+        vpandn	%ymm12, %ymm11, %ymm15
+        vpandn	%ymm13, %ymm12, %ymm1
+        vpandn	%ymm14, %ymm13, %ymm2
+        vpandn	%ymm10, %ymm14, %ymm3
+        vpandn	%ymm11, %ymm10, %ymm4
+        vpxor	%ymm10, %ymm15, %ymm15
+        vpxor	%ymm11, %ymm1, %ymm1
+        vpxor	%ymm12, %ymm2, %ymm2
+        # XOR in constant
+        vpxor	352(%rax), %ymm15, %ymm15
+        vpxor	%ymm13, %ymm3, %ymm3
+        vpxor	%ymm14, %ymm4, %ymm4
+        vmovdqu	%ymm1, (%rdi)
+        vmovdqu	%ymm2, -32(%rdi)
+        vmovdqu	%ymm3, -64(%rdi)
+        vmovdqu	%ymm4, -96(%rdi)
+        # Row 1
+        vpxor	(%r8), %ymm8, %ymm10
+        vpxor	128(%r8), %ymm9, %ymm11
+        vpxor	96(%r8), %ymm5, %ymm12
+        vpxor	64(%r8), %ymm6, %ymm13
+        vpxor	32(%r8), %ymm7, %ymm14
+        vpsrlq	$36, %ymm10, %ymm0
+        vpsrlq	$44, %ymm11, %ymm1
+        vpsrlq	$61, %ymm12, %ymm2
+        vpsrlq	$19, %ymm13, %ymm3
+        vpsrlq	$3, %ymm14, %ymm4
+        vpsllq	$28, %ymm10, %ymm10
+        vpsllq	$20, %ymm11, %ymm11
+        vpsllq	$3, %ymm12, %ymm12
+        vpsllq	$45, %ymm13, %ymm13
+        vpsllq	$61, %ymm14, %ymm14
+        vpor	%ymm0, %ymm10, %ymm10
+        vpor	%ymm1, %ymm11, %ymm11
+        vpor	%ymm2, %ymm12, %ymm12
+        vpor	%ymm3, %ymm13, %ymm13
+        vpor	%ymm4, %ymm14, %ymm14
+        vpandn	%ymm12, %ymm11, %ymm0
+        vpandn	%ymm13, %ymm12, %ymm1
+        vpandn	%ymm14, %ymm13, %ymm2
+        vpandn	%ymm10, %ymm14, %ymm3
+        vpandn	%ymm11, %ymm10, %ymm4
+        vpxor	%ymm10, %ymm0, %ymm0
+        vpxor	%ymm11, %ymm1, %ymm1
+        vpxor	%ymm12, %ymm2, %ymm2
+        vpxor	%ymm13, %ymm3, %ymm3
+        vpxor	%ymm14, %ymm4, %ymm4
+        vmovdqu	%ymm0, (%r8)
+        vmovdqu	%ymm1, 128(%r8)
+        vmovdqu	%ymm2, 96(%r8)
+        vmovdqu	%ymm3, 64(%r8)
+        vmovdqu	%ymm4, 32(%r8)
+        # Row 2
+        vpxor	96(%rcx), %ymm6, %ymm10
+        vpxor	-32(%r8), %ymm7, %ymm11
+        vpxor	-64(%r8), %ymm8, %ymm12
+        vpxor	-96(%r8), %ymm9, %ymm13
+        vpxor	128(%rcx), %ymm5, %ymm14
+        vpsrlq	$63, %ymm10, %ymm0
+        vpsrlq	$58, %ymm11, %ymm1
+        vpsrlq	$39, %ymm12, %ymm2
+        vpsrlq	$56, %ymm13, %ymm3
+        vpsrlq	$46, %ymm14, %ymm4
+        vpaddq	%ymm10, %ymm10, %ymm10
+        vpsllq	$6, %ymm11, %ymm11
+        vpsllq	$25, %ymm12, %ymm12
+        vpsllq	$8, %ymm13, %ymm13
+        vpsllq	$18, %ymm14, %ymm14
+        vpor	%ymm0, %ymm10, %ymm10
+        vpor	%ymm1, %ymm11, %ymm11
+        vpor	%ymm2, %ymm12, %ymm12
+        vpor	%ymm3, %ymm13, %ymm13
+        vpor	%ymm4, %ymm14, %ymm14
+        vpandn	%ymm12, %ymm11, %ymm0
+        vpandn	%ymm13, %ymm12, %ymm1
+        vpandn	%ymm14, %ymm13, %ymm2
+        vpandn	%ymm10, %ymm14, %ymm3
+        vpandn	%ymm11, %ymm10, %ymm4
+        vpxor	%ymm10, %ymm0, %ymm0
+        vpxor	%ymm11, %ymm1, %ymm1
+        vpxor	%ymm12, %ymm2, %ymm2
+        vpxor	%ymm13, %ymm3, %ymm3
+        vpxor	%ymm14, %ymm4, %ymm4
+        vmovdqu	%ymm0, 96(%rcx)
+        vmovdqu	%ymm1, -32(%r8)
+        vmovdqu	%ymm2, -64(%r8)
+        vmovdqu	%ymm3, -96(%r8)
+        vmovdqu	%ymm4, 128(%rcx)
+        # Row 3
+        vpxor	-64(%rcx), %ymm9, %ymm10
+        vpxor	64(%rcx), %ymm5, %ymm11
+        vpxor	32(%rcx), %ymm6, %ymm12
+        vpxor	(%rcx), %ymm7, %ymm13
+        vpxor	-32(%rcx), %ymm8, %ymm14
+        vpsrlq	$37, %ymm10, %ymm0
+        vpsrlq	$28, %ymm11, %ymm1
+        vpsrlq	$54, %ymm12, %ymm2
+        vpsrlq	$49, %ymm13, %ymm3
+        vpsrlq	$8, %ymm14, %ymm4
+        vpsllq	$27, %ymm10, %ymm10
+        vpsllq	$36, %ymm11, %ymm11
+        vpsllq	$10, %ymm12, %ymm12
+        vpsllq	$15, %ymm13, %ymm13
+        vpsllq	$56, %ymm14, %ymm14
+        vpor	%ymm0, %ymm10, %ymm10
+        vpor	%ymm1, %ymm11, %ymm11
+        vpor	%ymm2, %ymm12, %ymm12
+        vpor	%ymm3, %ymm13, %ymm13
+        vpor	%ymm4, %ymm14, %ymm14
+        vpandn	%ymm12, %ymm11, %ymm0
+        vpandn	%ymm13, %ymm12, %ymm1
+        vpandn	%ymm14, %ymm13, %ymm2
+        vpandn	%ymm10, %ymm14, %ymm3
+        vpandn	%ymm11, %ymm10, %ymm4
+        vpxor	%ymm10, %ymm0, %ymm0
+        vpxor	%ymm11, %ymm1, %ymm1
+        vpxor	%ymm12, %ymm2, %ymm2
+        vpxor	%ymm13, %ymm3, %ymm3
+        vpxor	%ymm14, %ymm4, %ymm4
+        vmovdqu	%ymm0, -64(%rcx)
+        vmovdqu	%ymm1, 64(%rcx)
+        vmovdqu	%ymm2, 32(%rcx)
+        vmovdqu	%ymm3, (%rcx)
+        vmovdqu	%ymm4, -32(%rcx)
+        # Row 4
+        vpxor	32(%rdi), %ymm7, %ymm10
+        vpxor	-96(%rcx), %ymm8, %ymm11
+        vpxor	128(%rdi), %ymm9, %ymm12
+        vpxor	96(%rdi), %ymm5, %ymm13
+        vpxor	64(%rdi), %ymm6, %ymm14
+        vpsrlq	$2, %ymm10, %ymm0
+        vpsrlq	$9, %ymm11, %ymm1
+        vpsrlq	$25, %ymm12, %ymm2
+        vpsrlq	$23, %ymm13, %ymm3
+        vpsrlq	$62, %ymm14, %ymm4
+        vpsllq	$62, %ymm10, %ymm10
+        vpsllq	$55, %ymm11, %ymm11
+        vpsllq	$39, %ymm12, %ymm12
+        vpsllq	$41, %ymm13, %ymm13
+        vpsllq	$2, %ymm14, %ymm14
+        vpor	%ymm0, %ymm10, %ymm10
+        vpor	%ymm1, %ymm11, %ymm11
+        vpor	%ymm2, %ymm12, %ymm12
+        vpor	%ymm3, %ymm13, %ymm13
+        vpor	%ymm4, %ymm14, %ymm14
+        vpandn	%ymm12, %ymm11, %ymm0
+        vpandn	%ymm13, %ymm12, %ymm1
+        vpandn	%ymm14, %ymm13, %ymm2
+        vpandn	%ymm10, %ymm14, %ymm3
+        vpandn	%ymm11, %ymm10, %ymm4
+        vpxor	%ymm10, %ymm0, %ymm0
+        vpxor	%ymm11, %ymm1, %ymm1
+        vpxor	%ymm12, %ymm2, %ymm2
+        vpxor	%ymm13, %ymm3, %ymm3
+        vpxor	%ymm14, %ymm4, %ymm4
+        vmovdqu	%ymm0, 32(%rdi)
+        vmovdqu	%ymm1, -96(%rcx)
+        vmovdqu	%ymm2, 128(%rdi)
+        vmovdqu	%ymm3, 96(%rdi)
+        vmovdqu	%ymm4, 64(%rdi)
+        # Round 12
+        # Calc b[0..4]
+        vpxor	%ymm15, %ymm0, %ymm10
+        vpxor	-96(%rdi), %ymm4, %ymm14
+        vpxor	-64(%rdi), %ymm3, %ymm13
+        vpxor	-32(%rdi), %ymm2, %ymm12
+        vpxor	(%rdi), %ymm1, %ymm11
+        vpxor	-64(%rcx), %ymm10, %ymm10
+        vpxor	-32(%rcx), %ymm14, %ymm14
+        vpxor	(%rcx), %ymm13, %ymm13
+        vpxor	32(%rcx), %ymm12, %ymm12
+        vpxor	64(%rcx), %ymm11, %ymm11
+        vpxor	96(%rcx), %ymm10, %ymm10
+        vpxor	128(%rcx), %ymm14, %ymm14
+        vpxor	-96(%r8), %ymm13, %ymm13
+        vpxor	-64(%r8), %ymm12, %ymm12
+        vpxor	-32(%r8), %ymm11, %ymm11
+        vpxor	(%r8), %ymm10, %ymm10
+        vpxor	32(%r8), %ymm14, %ymm14
+        vpxor	64(%r8), %ymm13, %ymm13
+        vpxor	96(%r8), %ymm12, %ymm12
+        vpxor	128(%r8), %ymm11, %ymm11
+        # Calc t[0..4]
+        vpsrlq	$63, %ymm11, %ymm0
+        vpsrlq	$63, %ymm12, %ymm1
+        vpsrlq	$63, %ymm13, %ymm2
+        vpsrlq	$63, %ymm14, %ymm3
+        vpsrlq	$63, %ymm10, %ymm4
+        vpaddq	%ymm11, %ymm11, %ymm5
+        vpaddq	%ymm12, %ymm12, %ymm6
+        vpaddq	%ymm13, %ymm13, %ymm7
+        vpaddq	%ymm14, %ymm14, %ymm8
+        vpaddq	%ymm10, %ymm10, %ymm9
+        vpor	%ymm0, %ymm5, %ymm5
+        vpor	%ymm1, %ymm6, %ymm6
+        vpor	%ymm2, %ymm7, %ymm7
+        vpor	%ymm3, %ymm8, %ymm8
+        vpor	%ymm4, %ymm9, %ymm9
+        vpxor	%ymm14, %ymm5, %ymm5
+        vpxor	%ymm10, %ymm6, %ymm6
+        vpxor	%ymm11, %ymm7, %ymm7
+        vpxor	%ymm12, %ymm8, %ymm8
+        vpxor	%ymm13, %ymm9, %ymm9
+        # Row Mix
+        # Row 0
+        vpxor	%ymm15, %ymm5, %ymm10
+        vpxor	128(%r8), %ymm6, %ymm11
+        vpxor	-64(%r8), %ymm7, %ymm12
+        vpxor	(%rcx), %ymm8, %ymm13
+        vpxor	64(%rdi), %ymm9, %ymm14
+        vpsrlq	$20, %ymm11, %ymm0
+        vpsrlq	$21, %ymm12, %ymm1
+        vpsrlq	$43, %ymm13, %ymm2
+        vpsrlq	$50, %ymm14, %ymm3
+        vpsllq	$44, %ymm11, %ymm11
+        vpsllq	$43, %ymm12, %ymm12
+        vpsllq	$21, %ymm13, %ymm13
+        vpsllq	$14, %ymm14, %ymm14
+        vpor	%ymm0, %ymm11, %ymm11
+        vpor	%ymm1, %ymm12, %ymm12
+        vpor	%ymm2, %ymm13, %ymm13
+        vpor	%ymm3, %ymm14, %ymm14
+        vpandn	%ymm12, %ymm11, %ymm15
+        vpandn	%ymm13, %ymm12, %ymm1
+        vpandn	%ymm14, %ymm13, %ymm2
+        vpandn	%ymm10, %ymm14, %ymm3
+        vpandn	%ymm11, %ymm10, %ymm4
+        vpxor	%ymm10, %ymm15, %ymm15
+        vpxor	%ymm11, %ymm1, %ymm1
+        vpxor	%ymm12, %ymm2, %ymm2
+        # XOR in constant
+        vpxor	384(%rax), %ymm15, %ymm15
+        vpxor	%ymm13, %ymm3, %ymm3
+        vpxor	%ymm14, %ymm4, %ymm4
+        vmovdqu	%ymm1, 128(%r8)
+        vmovdqu	%ymm2, -64(%r8)
+        vmovdqu	%ymm3, (%rcx)
+        vmovdqu	%ymm4, 64(%rdi)
+        # Row 1
+        vpxor	-64(%rdi), %ymm8, %ymm10
+        vpxor	32(%r8), %ymm9, %ymm11
+        vpxor	96(%rcx), %ymm5, %ymm12
+        vpxor	64(%rcx), %ymm6, %ymm13
+        vpxor	128(%rdi), %ymm7, %ymm14
+        vpsrlq	$36, %ymm10, %ymm0
+        vpsrlq	$44, %ymm11, %ymm1
+        vpsrlq	$61, %ymm12, %ymm2
+        vpsrlq	$19, %ymm13, %ymm3
+        vpsrlq	$3, %ymm14, %ymm4
+        vpsllq	$28, %ymm10, %ymm10
+        vpsllq	$20, %ymm11, %ymm11
+        vpsllq	$3, %ymm12, %ymm12
+        vpsllq	$45, %ymm13, %ymm13
+        vpsllq	$61, %ymm14, %ymm14
+        vpor	%ymm0, %ymm10, %ymm10
+        vpor	%ymm1, %ymm11, %ymm11
+        vpor	%ymm2, %ymm12, %ymm12
+        vpor	%ymm3, %ymm13, %ymm13
+        vpor	%ymm4, %ymm14, %ymm14
+        vpandn	%ymm12, %ymm11, %ymm0
+        vpandn	%ymm13, %ymm12, %ymm1
+        vpandn	%ymm14, %ymm13, %ymm2
+        vpandn	%ymm10, %ymm14, %ymm3
+        vpandn	%ymm11, %ymm10, %ymm4
+        vpxor	%ymm10, %ymm0, %ymm0
+        vpxor	%ymm11, %ymm1, %ymm1
+        vpxor	%ymm12, %ymm2, %ymm2
+        vpxor	%ymm13, %ymm3, %ymm3
+        vpxor	%ymm14, %ymm4, %ymm4
+        vmovdqu	%ymm0, -64(%rdi)
+        vmovdqu	%ymm1, 32(%r8)
+        vmovdqu	%ymm2, 96(%rcx)
+        vmovdqu	%ymm3, 64(%rcx)
+        vmovdqu	%ymm4, 128(%rdi)
+        # Row 2
+        vpxor	(%rdi), %ymm6, %ymm10
+        vpxor	96(%r8), %ymm7, %ymm11
+        vpxor	-96(%r8), %ymm8, %ymm12
+        vpxor	-32(%rcx), %ymm9, %ymm13
+        vpxor	32(%rdi), %ymm5, %ymm14
+        vpsrlq	$63, %ymm10, %ymm0
+        vpsrlq	$58, %ymm11, %ymm1
+        vpsrlq	$39, %ymm12, %ymm2
+        vpsrlq	$56, %ymm13, %ymm3
+        vpsrlq	$46, %ymm14, %ymm4
+        vpaddq	%ymm10, %ymm10, %ymm10
+        vpsllq	$6, %ymm11, %ymm11
+        vpsllq	$25, %ymm12, %ymm12
+        vpsllq	$8, %ymm13, %ymm13
+        vpsllq	$18, %ymm14, %ymm14
+        vpor	%ymm0, %ymm10, %ymm10
+        vpor	%ymm1, %ymm11, %ymm11
+        vpor	%ymm2, %ymm12, %ymm12
+        vpor	%ymm3, %ymm13, %ymm13
+        vpor	%ymm4, %ymm14, %ymm14
+        vpandn	%ymm12, %ymm11, %ymm0
+        vpandn	%ymm13, %ymm12, %ymm1
+        vpandn	%ymm14, %ymm13, %ymm2
+        vpandn	%ymm10, %ymm14, %ymm3
+        vpandn	%ymm11, %ymm10, %ymm4
+        vpxor	%ymm10, %ymm0, %ymm0
+        vpxor	%ymm11, %ymm1, %ymm1
+        vpxor	%ymm12, %ymm2, %ymm2
+        vpxor	%ymm13, %ymm3, %ymm3
+        vpxor	%ymm14, %ymm4, %ymm4
+        vmovdqu	%ymm0, (%rdi)
+        vmovdqu	%ymm1, 96(%r8)
+        vmovdqu	%ymm2, -96(%r8)
+        vmovdqu	%ymm3, -32(%rcx)
+        vmovdqu	%ymm4, 32(%rdi)
+        # Row 3
+        vpxor	-96(%rdi), %ymm9, %ymm10
+        vpxor	(%r8), %ymm5, %ymm11
+        vpxor	-32(%r8), %ymm6, %ymm12
+        vpxor	32(%rcx), %ymm7, %ymm13
+        vpxor	96(%rdi), %ymm8, %ymm14
+        vpsrlq	$37, %ymm10, %ymm0
+        vpsrlq	$28, %ymm11, %ymm1
+        vpsrlq	$54, %ymm12, %ymm2
+        vpsrlq	$49, %ymm13, %ymm3
+        vpsrlq	$8, %ymm14, %ymm4
+        vpsllq	$27, %ymm10, %ymm10
+        vpsllq	$36, %ymm11, %ymm11
+        vpsllq	$10, %ymm12, %ymm12
+        vpsllq	$15, %ymm13, %ymm13
+        vpsllq	$56, %ymm14, %ymm14
+        vpor	%ymm0, %ymm10, %ymm10
+        vpor	%ymm1, %ymm11, %ymm11
+        vpor	%ymm2, %ymm12, %ymm12
+        vpor	%ymm3, %ymm13, %ymm13
+        vpor	%ymm4, %ymm14, %ymm14
+        vpandn	%ymm12, %ymm11, %ymm0
+        vpandn	%ymm13, %ymm12, %ymm1
+        vpandn	%ymm14, %ymm13, %ymm2
+        vpandn	%ymm10, %ymm14, %ymm3
+        vpandn	%ymm11, %ymm10, %ymm4
+        vpxor	%ymm10, %ymm0, %ymm0
+        vpxor	%ymm11, %ymm1, %ymm1
+        vpxor	%ymm12, %ymm2, %ymm2
+        vpxor	%ymm13, %ymm3, %ymm3
+        vpxor	%ymm14, %ymm4, %ymm4
+        vmovdqu	%ymm0, -96(%rdi)
+        vmovdqu	%ymm1, (%r8)
+        vmovdqu	%ymm2, -32(%r8)
+        vmovdqu	%ymm3, 32(%rcx)
+        vmovdqu	%ymm4, 96(%rdi)
+        # Row 4
+        vpxor	-32(%rdi), %ymm7, %ymm10
+        vpxor	64(%r8), %ymm8, %ymm11
+        vpxor	128(%rcx), %ymm9, %ymm12
+        vpxor	-64(%rcx), %ymm5, %ymm13
+        vpxor	-96(%rcx), %ymm6, %ymm14
+        vpsrlq	$2, %ymm10, %ymm0
+        vpsrlq	$9, %ymm11, %ymm1
+        vpsrlq	$25, %ymm12, %ymm2
+        vpsrlq	$23, %ymm13, %ymm3
+        vpsrlq	$62, %ymm14, %ymm4
+        vpsllq	$62, %ymm10, %ymm10
+        vpsllq	$55, %ymm11, %ymm11
+        vpsllq	$39, %ymm12, %ymm12
+        vpsllq	$41, %ymm13, %ymm13
+        vpsllq	$2, %ymm14, %ymm14
+        vpor	%ymm0, %ymm10, %ymm10
+        vpor	%ymm1, %ymm11, %ymm11
+        vpor	%ymm2, %ymm12, %ymm12
+        vpor	%ymm3, %ymm13, %ymm13
+        vpor	%ymm4, %ymm14, %ymm14
+        vpandn	%ymm12, %ymm11, %ymm0
+        vpandn	%ymm13, %ymm12, %ymm1
+        vpandn	%ymm14, %ymm13, %ymm2
+        vpandn	%ymm10, %ymm14, %ymm3
+        vpandn	%ymm11, %ymm10, %ymm4
+        vpxor	%ymm10, %ymm0, %ymm0
+        vpxor	%ymm11, %ymm1, %ymm1
+        vpxor	%ymm12, %ymm2, %ymm2
+        vpxor	%ymm13, %ymm3, %ymm3
+        vpxor	%ymm14, %ymm4, %ymm4
+        vmovdqu	%ymm0, -32(%rdi)
+        vmovdqu	%ymm1, 64(%r8)
+        vmovdqu	%ymm2, 128(%rcx)
+        vmovdqu	%ymm3, -64(%rcx)
+        vmovdqu	%ymm4, -96(%rcx)
+        # Round 13
+        # Calc b[0..4]
+        vpxor	%ymm15, %ymm0, %ymm10
+        vpxor	-96(%rdi), %ymm10, %ymm10
+        vpxor	-64(%rdi), %ymm10, %ymm10
+        vpxor	(%rdi), %ymm10, %ymm10
+        vpxor	32(%rdi), %ymm4, %ymm14
+        vpxor	64(%rdi), %ymm14, %ymm14
+        vpxor	96(%rdi), %ymm14, %ymm14
+        vpxor	128(%rdi), %ymm14, %ymm14
+        vpxor	-32(%rcx), %ymm3, %ymm13
+        vpxor	(%rcx), %ymm13, %ymm13
+        vpxor	32(%rcx), %ymm13, %ymm13
+        vpxor	64(%rcx), %ymm13, %ymm13
+        vpxor	96(%rcx), %ymm2, %ymm12
+        vpxor	-96(%r8), %ymm12, %ymm12
+        vpxor	-64(%r8), %ymm12, %ymm12
+        vpxor	-32(%r8), %ymm12, %ymm12
+        vpxor	(%r8), %ymm1, %ymm11
+        vpxor	32(%r8), %ymm11, %ymm11
+        vpxor	96(%r8), %ymm11, %ymm11
+        vpxor	128(%r8), %ymm11, %ymm11
+        # Calc t[0..4]
+        vpsrlq	$63, %ymm11, %ymm0
+        vpsrlq	$63, %ymm12, %ymm1
+        vpsrlq	$63, %ymm13, %ymm2
+        vpsrlq	$63, %ymm14, %ymm3
+        vpsrlq	$63, %ymm10, %ymm4
+        vpaddq	%ymm11, %ymm11, %ymm5
+        vpaddq	%ymm12, %ymm12, %ymm6
+        vpaddq	%ymm13, %ymm13, %ymm7
+        vpaddq	%ymm14, %ymm14, %ymm8
+        vpaddq	%ymm10, %ymm10, %ymm9
+        vpor	%ymm0, %ymm5, %ymm5
+        vpor	%ymm1, %ymm6, %ymm6
+        vpor	%ymm2, %ymm7, %ymm7
+        vpor	%ymm3, %ymm8, %ymm8
+        vpor	%ymm4, %ymm9, %ymm9
+        vpxor	%ymm14, %ymm5, %ymm5
+        vpxor	%ymm10, %ymm6, %ymm6
+        vpxor	%ymm11, %ymm7, %ymm7
+        vpxor	%ymm12, %ymm8, %ymm8
+        vpxor	%ymm13, %ymm9, %ymm9
+        # Row Mix
+        # Row 0
+        vpxor	%ymm15, %ymm5, %ymm10
+        vpxor	32(%r8), %ymm6, %ymm11
+        vpxor	-96(%r8), %ymm7, %ymm12
+        vpxor	32(%rcx), %ymm8, %ymm13
+        vpxor	-96(%rcx), %ymm9, %ymm14
+        vpsrlq	$20, %ymm11, %ymm0
+        vpsrlq	$21, %ymm12, %ymm1
+        vpsrlq	$43, %ymm13, %ymm2
+        vpsrlq	$50, %ymm14, %ymm3
+        vpsllq	$44, %ymm11, %ymm11
+        vpsllq	$43, %ymm12, %ymm12
+        vpsllq	$21, %ymm13, %ymm13
+        vpsllq	$14, %ymm14, %ymm14
+        vpor	%ymm0, %ymm11, %ymm11
+        vpor	%ymm1, %ymm12, %ymm12
+        vpor	%ymm2, %ymm13, %ymm13
+        vpor	%ymm3, %ymm14, %ymm14
+        vpandn	%ymm12, %ymm11, %ymm15
+        vpandn	%ymm13, %ymm12, %ymm1
+        vpandn	%ymm14, %ymm13, %ymm2
+        vpandn	%ymm10, %ymm14, %ymm3
+        vpandn	%ymm11, %ymm10, %ymm4
+        vpxor	%ymm10, %ymm15, %ymm15
+        vpxor	%ymm11, %ymm1, %ymm1
+        vpxor	%ymm12, %ymm2, %ymm2
+        # XOR in constant
+        vpxor	416(%rax), %ymm15, %ymm15
+        vpxor	%ymm13, %ymm3, %ymm3
+        vpxor	%ymm14, %ymm4, %ymm4
+        vmovdqu	%ymm1, 32(%r8)
+        vmovdqu	%ymm2, -96(%r8)
+        vmovdqu	%ymm3, 32(%rcx)
+        vmovdqu	%ymm4, -96(%rcx)
+        # Row 1
+        vpxor	(%rcx), %ymm8, %ymm10
+        vpxor	128(%rdi), %ymm9, %ymm11
+        vpxor	(%rdi), %ymm5, %ymm12
+        vpxor	(%r8), %ymm6, %ymm13
+        vpxor	128(%rcx), %ymm7, %ymm14
+        vpsrlq	$36, %ymm10, %ymm0
+        vpsrlq	$44, %ymm11, %ymm1
+        vpsrlq	$61, %ymm12, %ymm2
+        vpsrlq	$19, %ymm13, %ymm3
+        vpsrlq	$3, %ymm14, %ymm4
+        vpsllq	$28, %ymm10, %ymm10
+        vpsllq	$20, %ymm11, %ymm11
+        vpsllq	$3, %ymm12, %ymm12
+        vpsllq	$45, %ymm13, %ymm13
+        vpsllq	$61, %ymm14, %ymm14
+        vpor	%ymm0, %ymm10, %ymm10
+        vpor	%ymm1, %ymm11, %ymm11
+        vpor	%ymm2, %ymm12, %ymm12
+        vpor	%ymm3, %ymm13, %ymm13
+        vpor	%ymm4, %ymm14, %ymm14
+        vpandn	%ymm12, %ymm11, %ymm0
+        vpandn	%ymm13, %ymm12, %ymm1
+        vpandn	%ymm14, %ymm13, %ymm2
+        vpandn	%ymm10, %ymm14, %ymm3
+        vpandn	%ymm11, %ymm10, %ymm4
+        vpxor	%ymm10, %ymm0, %ymm0
+        vpxor	%ymm11, %ymm1, %ymm1
+        vpxor	%ymm12, %ymm2, %ymm2
+        vpxor	%ymm13, %ymm3, %ymm3
+        vpxor	%ymm14, %ymm4, %ymm4
+        vmovdqu	%ymm0, (%rcx)
+        vmovdqu	%ymm1, 128(%rdi)
+        vmovdqu	%ymm2, (%rdi)
+        vmovdqu	%ymm3, (%r8)
+        vmovdqu	%ymm4, 128(%rcx)
+        # Row 2
+        vpxor	128(%r8), %ymm6, %ymm10
+        vpxor	96(%rcx), %ymm7, %ymm11
+        vpxor	-32(%rcx), %ymm8, %ymm12
+        vpxor	96(%rdi), %ymm9, %ymm13
+        vpxor	-32(%rdi), %ymm5, %ymm14
+        vpsrlq	$63, %ymm10, %ymm0
+        vpsrlq	$58, %ymm11, %ymm1
+        vpsrlq	$39, %ymm12, %ymm2
+        vpsrlq	$56, %ymm13, %ymm3
+        vpsrlq	$46, %ymm14, %ymm4
+        vpaddq	%ymm10, %ymm10, %ymm10
+        vpsllq	$6, %ymm11, %ymm11
+        vpsllq	$25, %ymm12, %ymm12
+        vpsllq	$8, %ymm13, %ymm13
+        vpsllq	$18, %ymm14, %ymm14
+        vpor	%ymm0, %ymm10, %ymm10
+        vpor	%ymm1, %ymm11, %ymm11
+        vpor	%ymm2, %ymm12, %ymm12
+        vpor	%ymm3, %ymm13, %ymm13
+        vpor	%ymm4, %ymm14, %ymm14
+        vpandn	%ymm12, %ymm11, %ymm0
+        vpandn	%ymm13, %ymm12, %ymm1
+        vpandn	%ymm14, %ymm13, %ymm2
+        vpandn	%ymm10, %ymm14, %ymm3
+        vpandn	%ymm11, %ymm10, %ymm4
+        vpxor	%ymm10, %ymm0, %ymm0
+        vpxor	%ymm11, %ymm1, %ymm1
+        vpxor	%ymm12, %ymm2, %ymm2
+        vpxor	%ymm13, %ymm3, %ymm3
+        vpxor	%ymm14, %ymm4, %ymm4
+        vmovdqu	%ymm0, 128(%r8)
+        vmovdqu	%ymm1, 96(%rcx)
+        vmovdqu	%ymm2, -32(%rcx)
+        vmovdqu	%ymm3, 96(%rdi)
+        vmovdqu	%ymm4, -32(%rdi)
+        # Row 3
+        vpxor	64(%rdi), %ymm9, %ymm10
+        vpxor	-64(%rdi), %ymm5, %ymm11
+        vpxor	96(%r8), %ymm6, %ymm12
+        vpxor	-32(%r8), %ymm7, %ymm13
+        vpxor	-64(%rcx), %ymm8, %ymm14
+        vpsrlq	$37, %ymm10, %ymm0
+        vpsrlq	$28, %ymm11, %ymm1
+        vpsrlq	$54, %ymm12, %ymm2
+        vpsrlq	$49, %ymm13, %ymm3
+        vpsrlq	$8, %ymm14, %ymm4
+        vpsllq	$27, %ymm10, %ymm10
+        vpsllq	$36, %ymm11, %ymm11
+        vpsllq	$10, %ymm12, %ymm12
+        vpsllq	$15, %ymm13, %ymm13
+        vpsllq	$56, %ymm14, %ymm14
+        vpor	%ymm0, %ymm10, %ymm10
+        vpor	%ymm1, %ymm11, %ymm11
+        vpor	%ymm2, %ymm12, %ymm12
+        vpor	%ymm3, %ymm13, %ymm13
+        vpor	%ymm4, %ymm14, %ymm14
+        vpandn	%ymm12, %ymm11, %ymm0
+        vpandn	%ymm13, %ymm12, %ymm1
+        vpandn	%ymm14, %ymm13, %ymm2
+        vpandn	%ymm10, %ymm14, %ymm3
+        vpandn	%ymm11, %ymm10, %ymm4
+        vpxor	%ymm10, %ymm0, %ymm0
+        vpxor	%ymm11, %ymm1, %ymm1
+        vpxor	%ymm12, %ymm2, %ymm2
+        vpxor	%ymm13, %ymm3, %ymm3
+        vpxor	%ymm14, %ymm4, %ymm4
+        vmovdqu	%ymm0, 64(%rdi)
+        vmovdqu	%ymm1, -64(%rdi)
+        vmovdqu	%ymm2, 96(%r8)
+        vmovdqu	%ymm3, -32(%r8)
+        vmovdqu	%ymm4, -64(%rcx)
+        # Row 4
+        vpxor	-64(%r8), %ymm7, %ymm10
+        vpxor	64(%rcx), %ymm8, %ymm11
+        vpxor	32(%rdi), %ymm9, %ymm12
+        vpxor	-96(%rdi), %ymm5, %ymm13
+        vpxor	64(%r8), %ymm6, %ymm14
+        vpsrlq	$2, %ymm10, %ymm0
+        vpsrlq	$9, %ymm11, %ymm1
+        vpsrlq	$25, %ymm12, %ymm2
+        vpsrlq	$23, %ymm13, %ymm3
+        vpsrlq	$62, %ymm14, %ymm4
+        vpsllq	$62, %ymm10, %ymm10
+        vpsllq	$55, %ymm11, %ymm11
+        vpsllq	$39, %ymm12, %ymm12
+        vpsllq	$41, %ymm13, %ymm13
+        vpsllq	$2, %ymm14, %ymm14
+        vpor	%ymm0, %ymm10, %ymm10
+        vpor	%ymm1, %ymm11, %ymm11
+        vpor	%ymm2, %ymm12, %ymm12
+        vpor	%ymm3, %ymm13, %ymm13
+        vpor	%ymm4, %ymm14, %ymm14
+        vpandn	%ymm12, %ymm11, %ymm0
+        vpandn	%ymm13, %ymm12, %ymm1
+        vpandn	%ymm14, %ymm13, %ymm2
+        vpandn	%ymm10, %ymm14, %ymm3
+        vpandn	%ymm11, %ymm10, %ymm4
+        vpxor	%ymm10, %ymm0, %ymm0
+        vpxor	%ymm11, %ymm1, %ymm1
+        vpxor	%ymm12, %ymm2, %ymm2
+        vpxor	%ymm13, %ymm3, %ymm3
+        vpxor	%ymm14, %ymm4, %ymm4
+        vmovdqu	%ymm0, -64(%r8)
+        vmovdqu	%ymm1, 64(%rcx)
+        vmovdqu	%ymm2, 32(%rdi)
+        vmovdqu	%ymm3, -96(%rdi)
+        vmovdqu	%ymm4, 64(%r8)
+        # Round 14
+        # Calc b[0..4]
+        vpxor	%ymm15, %ymm0, %ymm10
+        vpxor	-64(%rdi), %ymm1, %ymm11
+        vpxor	-32(%rdi), %ymm4, %ymm14
+        vpxor	(%rdi), %ymm2, %ymm12
+        vpxor	64(%rdi), %ymm10, %ymm10
+        vpxor	96(%rdi), %ymm3, %ymm13
+        vpxor	128(%rdi), %ymm11, %ymm11
+        vpxor	-96(%rcx), %ymm14, %ymm14
+        vpxor	-64(%rcx), %ymm14, %ymm14
+        vpxor	-32(%rcx), %ymm12, %ymm12
+        vpxor	(%rcx), %ymm10, %ymm10
+        vpxor	32(%rcx), %ymm13, %ymm13
+        vpxor	96(%rcx), %ymm11, %ymm11
+        vpxor	128(%rcx), %ymm14, %ymm14
+        vpxor	-96(%r8), %ymm12, %ymm12
+        vpxor	-32(%r8), %ymm13, %ymm13
+        vpxor	(%r8), %ymm13, %ymm13
+        vpxor	32(%r8), %ymm11, %ymm11
+        vpxor	96(%r8), %ymm12, %ymm12
+        vpxor	128(%r8), %ymm10, %ymm10
+        # Calc t[0..4]
+        vpsrlq	$63, %ymm11, %ymm0
+        vpsrlq	$63, %ymm12, %ymm1
+        vpsrlq	$63, %ymm13, %ymm2
+        vpsrlq	$63, %ymm14, %ymm3
+        vpsrlq	$63, %ymm10, %ymm4
+        vpaddq	%ymm11, %ymm11, %ymm5
+        vpaddq	%ymm12, %ymm12, %ymm6
+        vpaddq	%ymm13, %ymm13, %ymm7
+        vpaddq	%ymm14, %ymm14, %ymm8
+        vpaddq	%ymm10, %ymm10, %ymm9
+        vpor	%ymm0, %ymm5, %ymm5
+        vpor	%ymm1, %ymm6, %ymm6
+        vpor	%ymm2, %ymm7, %ymm7
+        vpor	%ymm3, %ymm8, %ymm8
+        vpor	%ymm4, %ymm9, %ymm9
+        vpxor	%ymm14, %ymm5, %ymm5
+        vpxor	%ymm10, %ymm6, %ymm6
+        vpxor	%ymm11, %ymm7, %ymm7
+        vpxor	%ymm12, %ymm8, %ymm8
+        vpxor	%ymm13, %ymm9, %ymm9
+        # Row Mix
+        # Row 0
+        vpxor	%ymm15, %ymm5, %ymm10
+        vpxor	128(%rdi), %ymm6, %ymm11
+        vpxor	-32(%rcx), %ymm7, %ymm12
+        vpxor	-32(%r8), %ymm8, %ymm13
+        vpxor	64(%r8), %ymm9, %ymm14
+        vpsrlq	$20, %ymm11, %ymm0
+        vpsrlq	$21, %ymm12, %ymm1
+        vpsrlq	$43, %ymm13, %ymm2
+        vpsrlq	$50, %ymm14, %ymm3
+        vpsllq	$44, %ymm11, %ymm11
+        vpsllq	$43, %ymm12, %ymm12
+        vpsllq	$21, %ymm13, %ymm13
+        vpsllq	$14, %ymm14, %ymm14
+        vpor	%ymm0, %ymm11, %ymm11
+        vpor	%ymm1, %ymm12, %ymm12
+        vpor	%ymm2, %ymm13, %ymm13
+        vpor	%ymm3, %ymm14, %ymm14
+        vpandn	%ymm12, %ymm11, %ymm15
+        vpandn	%ymm13, %ymm12, %ymm1
+        vpandn	%ymm14, %ymm13, %ymm2
+        vpandn	%ymm10, %ymm14, %ymm3
+        vpandn	%ymm11, %ymm10, %ymm4
+        vpxor	%ymm10, %ymm15, %ymm15
+        vpxor	%ymm11, %ymm1, %ymm1
+        vpxor	%ymm12, %ymm2, %ymm2
+        # XOR in constant
+        vpxor	448(%rax), %ymm15, %ymm15
+        vpxor	%ymm13, %ymm3, %ymm3
+        vpxor	%ymm14, %ymm4, %ymm4
+        vmovdqu	%ymm1, 128(%rdi)
+        vmovdqu	%ymm2, -32(%rcx)
+        vmovdqu	%ymm3, -32(%r8)
+        vmovdqu	%ymm4, 64(%r8)
+        # Row 1
+        vpxor	32(%rcx), %ymm8, %ymm10
+        vpxor	128(%rcx), %ymm9, %ymm11
+        vpxor	128(%r8), %ymm5, %ymm12
+        vpxor	-64(%rdi), %ymm6, %ymm13
+        vpxor	32(%rdi), %ymm7, %ymm14
+        vpsrlq	$36, %ymm10, %ymm0
+        vpsrlq	$44, %ymm11, %ymm1
+        vpsrlq	$61, %ymm12, %ymm2
+        vpsrlq	$19, %ymm13, %ymm3
+        vpsrlq	$3, %ymm14, %ymm4
+        vpsllq	$28, %ymm10, %ymm10
+        vpsllq	$20, %ymm11, %ymm11
+        vpsllq	$3, %ymm12, %ymm12
+        vpsllq	$45, %ymm13, %ymm13
+        vpsllq	$61, %ymm14, %ymm14
+        vpor	%ymm0, %ymm10, %ymm10
+        vpor	%ymm1, %ymm11, %ymm11
+        vpor	%ymm2, %ymm12, %ymm12
+        vpor	%ymm3, %ymm13, %ymm13
+        vpor	%ymm4, %ymm14, %ymm14
+        vpandn	%ymm12, %ymm11, %ymm0
+        vpandn	%ymm13, %ymm12, %ymm1
+        vpandn	%ymm14, %ymm13, %ymm2
+        vpandn	%ymm10, %ymm14, %ymm3
+        vpandn	%ymm11, %ymm10, %ymm4
+        vpxor	%ymm10, %ymm0, %ymm0
+        vpxor	%ymm11, %ymm1, %ymm1
+        vpxor	%ymm12, %ymm2, %ymm2
+        vpxor	%ymm13, %ymm3, %ymm3
+        vpxor	%ymm14, %ymm4, %ymm4
+        vmovdqu	%ymm0, 32(%rcx)
+        vmovdqu	%ymm1, 128(%rcx)
+        vmovdqu	%ymm2, 128(%r8)
+        vmovdqu	%ymm3, -64(%rdi)
+        vmovdqu	%ymm4, 32(%rdi)
+        # Row 2
+        vpxor	32(%r8), %ymm6, %ymm10
+        vpxor	(%rdi), %ymm7, %ymm11
+        vpxor	96(%rdi), %ymm8, %ymm12
+        vpxor	-64(%rcx), %ymm9, %ymm13
+        vpxor	-64(%r8), %ymm5, %ymm14
+        vpsrlq	$63, %ymm10, %ymm0
+        vpsrlq	$58, %ymm11, %ymm1
+        vpsrlq	$39, %ymm12, %ymm2
+        vpsrlq	$56, %ymm13, %ymm3
+        vpsrlq	$46, %ymm14, %ymm4
+        vpaddq	%ymm10, %ymm10, %ymm10
+        vpsllq	$6, %ymm11, %ymm11
+        vpsllq	$25, %ymm12, %ymm12
+        vpsllq	$8, %ymm13, %ymm13
+        vpsllq	$18, %ymm14, %ymm14
+        vpor	%ymm0, %ymm10, %ymm10
+        vpor	%ymm1, %ymm11, %ymm11
+        vpor	%ymm2, %ymm12, %ymm12
+        vpor	%ymm3, %ymm13, %ymm13
+        vpor	%ymm4, %ymm14, %ymm14
+        vpandn	%ymm12, %ymm11, %ymm0
+        vpandn	%ymm13, %ymm12, %ymm1
+        vpandn	%ymm14, %ymm13, %ymm2
+        vpandn	%ymm10, %ymm14, %ymm3
+        vpandn	%ymm11, %ymm10, %ymm4
+        vpxor	%ymm10, %ymm0, %ymm0
+        vpxor	%ymm11, %ymm1, %ymm1
+        vpxor	%ymm12, %ymm2, %ymm2
+        vpxor	%ymm13, %ymm3, %ymm3
+        vpxor	%ymm14, %ymm4, %ymm4
+        vmovdqu	%ymm0, 32(%r8)
+        vmovdqu	%ymm1, (%rdi)
+        vmovdqu	%ymm2, 96(%rdi)
+        vmovdqu	%ymm3, -64(%rcx)
+        vmovdqu	%ymm4, -64(%r8)
+        # Row 3
+        vpxor	-96(%rcx), %ymm9, %ymm10
+        vpxor	(%rcx), %ymm5, %ymm11
+        vpxor	96(%rcx), %ymm6, %ymm12
+        vpxor	96(%r8), %ymm7, %ymm13
+        vpxor	-96(%rdi), %ymm8, %ymm14
+        vpsrlq	$37, %ymm10, %ymm0
+        vpsrlq	$28, %ymm11, %ymm1
+        vpsrlq	$54, %ymm12, %ymm2
+        vpsrlq	$49, %ymm13, %ymm3
+        vpsrlq	$8, %ymm14, %ymm4
+        vpsllq	$27, %ymm10, %ymm10
+        vpsllq	$36, %ymm11, %ymm11
+        vpsllq	$10, %ymm12, %ymm12
+        vpsllq	$15, %ymm13, %ymm13
+        vpsllq	$56, %ymm14, %ymm14
+        vpor	%ymm0, %ymm10, %ymm10
+        vpor	%ymm1, %ymm11, %ymm11
+        vpor	%ymm2, %ymm12, %ymm12
+        vpor	%ymm3, %ymm13, %ymm13
+        vpor	%ymm4, %ymm14, %ymm14
+        vpandn	%ymm12, %ymm11, %ymm0
+        vpandn	%ymm13, %ymm12, %ymm1
+        vpandn	%ymm14, %ymm13, %ymm2
+        vpandn	%ymm10, %ymm14, %ymm3
+        vpandn	%ymm11, %ymm10, %ymm4
+        vpxor	%ymm10, %ymm0, %ymm0
+        vpxor	%ymm11, %ymm1, %ymm1
+        vpxor	%ymm12, %ymm2, %ymm2
+        vpxor	%ymm13, %ymm3, %ymm3
+        vpxor	%ymm14, %ymm4, %ymm4
+        vmovdqu	%ymm0, -96(%rcx)
+        vmovdqu	%ymm1, (%rcx)
+        vmovdqu	%ymm2, 96(%rcx)
+        vmovdqu	%ymm3, 96(%r8)
+        vmovdqu	%ymm4, -96(%rdi)
+        # Row 4
+        vpxor	-96(%r8), %ymm7, %ymm10
+        vpxor	(%r8), %ymm8, %ymm11
+        vpxor	-32(%rdi), %ymm9, %ymm12
+        vpxor	64(%rdi), %ymm5, %ymm13
+        vpxor	64(%rcx), %ymm6, %ymm14
+        vpsrlq	$2, %ymm10, %ymm0
+        vpsrlq	$9, %ymm11, %ymm1
+        vpsrlq	$25, %ymm12, %ymm2
+        vpsrlq	$23, %ymm13, %ymm3
+        vpsrlq	$62, %ymm14, %ymm4
+        vpsllq	$62, %ymm10, %ymm10
+        vpsllq	$55, %ymm11, %ymm11
+        vpsllq	$39, %ymm12, %ymm12
+        vpsllq	$41, %ymm13, %ymm13
+        vpsllq	$2, %ymm14, %ymm14
+        vpor	%ymm0, %ymm10, %ymm10
+        vpor	%ymm1, %ymm11, %ymm11
+        vpor	%ymm2, %ymm12, %ymm12
+        vpor	%ymm3, %ymm13, %ymm13
+        vpor	%ymm4, %ymm14, %ymm14
+        vpandn	%ymm12, %ymm11, %ymm0
+        vpandn	%ymm13, %ymm12, %ymm1
+        vpandn	%ymm14, %ymm13, %ymm2
+        vpandn	%ymm10, %ymm14, %ymm3
+        vpandn	%ymm11, %ymm10, %ymm4
+        vpxor	%ymm10, %ymm0, %ymm0
+        vpxor	%ymm11, %ymm1, %ymm1
+        vpxor	%ymm12, %ymm2, %ymm2
+        vpxor	%ymm13, %ymm3, %ymm3
+        vpxor	%ymm14, %ymm4, %ymm4
+        vmovdqu	%ymm0, -96(%r8)
+        vmovdqu	%ymm1, (%r8)
+        vmovdqu	%ymm2, -32(%rdi)
+        vmovdqu	%ymm3, 64(%rdi)
+        vmovdqu	%ymm4, 64(%rcx)
+        # Round 15
+        # Calc b[0..4]
+        vpxor	%ymm15, %ymm0, %ymm10
+        vpxor	-96(%rdi), %ymm4, %ymm14
+        vpxor	-64(%rdi), %ymm3, %ymm13
+        vpxor	(%rdi), %ymm1, %ymm11
+        vpxor	32(%rdi), %ymm14, %ymm14
+        vpxor	96(%rdi), %ymm2, %ymm12
+        vpxor	128(%rdi), %ymm11, %ymm11
+        vpxor	-96(%rcx), %ymm10, %ymm10
+        vpxor	-64(%rcx), %ymm13, %ymm13
+        vpxor	-32(%rcx), %ymm12, %ymm12
+        vpxor	(%rcx), %ymm11, %ymm11
+        vpxor	32(%rcx), %ymm10, %ymm10
+        vpxor	96(%rcx), %ymm12, %ymm12
+        vpxor	128(%rcx), %ymm11, %ymm11
+        vpxor	-64(%r8), %ymm14, %ymm14
+        vpxor	-32(%r8), %ymm13, %ymm13
+        vpxor	32(%r8), %ymm10, %ymm10
+        vpxor	64(%r8), %ymm14, %ymm14
+        vpxor	96(%r8), %ymm13, %ymm13
+        vpxor	128(%r8), %ymm12, %ymm12
+        # Calc t[0..4]
+        vpsrlq	$63, %ymm11, %ymm0
+        vpsrlq	$63, %ymm12, %ymm1
+        vpsrlq	$63, %ymm13, %ymm2
+        vpsrlq	$63, %ymm14, %ymm3
+        vpsrlq	$63, %ymm10, %ymm4
+        vpaddq	%ymm11, %ymm11, %ymm5
+        vpaddq	%ymm12, %ymm12, %ymm6
+        vpaddq	%ymm13, %ymm13, %ymm7
+        vpaddq	%ymm14, %ymm14, %ymm8
+        vpaddq	%ymm10, %ymm10, %ymm9
+        vpor	%ymm0, %ymm5, %ymm5
+        vpor	%ymm1, %ymm6, %ymm6
+        vpor	%ymm2, %ymm7, %ymm7
+        vpor	%ymm3, %ymm8, %ymm8
+        vpor	%ymm4, %ymm9, %ymm9
+        vpxor	%ymm14, %ymm5, %ymm5
+        vpxor	%ymm10, %ymm6, %ymm6
+        vpxor	%ymm11, %ymm7, %ymm7
+        vpxor	%ymm12, %ymm8, %ymm8
+        vpxor	%ymm13, %ymm9, %ymm9
+        # Row Mix
+        # Row 0
+        vpxor	%ymm15, %ymm5, %ymm10
+        vpxor	128(%rcx), %ymm6, %ymm11
+        vpxor	96(%rdi), %ymm7, %ymm12
+        vpxor	96(%r8), %ymm8, %ymm13
+        vpxor	64(%rcx), %ymm9, %ymm14
+        vpsrlq	$20, %ymm11, %ymm0
+        vpsrlq	$21, %ymm12, %ymm1
+        vpsrlq	$43, %ymm13, %ymm2
+        vpsrlq	$50, %ymm14, %ymm3
+        vpsllq	$44, %ymm11, %ymm11
+        vpsllq	$43, %ymm12, %ymm12
+        vpsllq	$21, %ymm13, %ymm13
+        vpsllq	$14, %ymm14, %ymm14
+        vpor	%ymm0, %ymm11, %ymm11
+        vpor	%ymm1, %ymm12, %ymm12
+        vpor	%ymm2, %ymm13, %ymm13
+        vpor	%ymm3, %ymm14, %ymm14
+        vpandn	%ymm12, %ymm11, %ymm15
+        vpandn	%ymm13, %ymm12, %ymm1
+        vpandn	%ymm14, %ymm13, %ymm2
+        vpandn	%ymm10, %ymm14, %ymm3
+        vpandn	%ymm11, %ymm10, %ymm4
+        vpxor	%ymm10, %ymm15, %ymm15
+        vpxor	%ymm11, %ymm1, %ymm1
+        vpxor	%ymm12, %ymm2, %ymm2
+        # XOR in constant
+        vpxor	480(%rax), %ymm15, %ymm15
+        vpxor	%ymm13, %ymm3, %ymm3
+        vpxor	%ymm14, %ymm4, %ymm4
+        vmovdqu	%ymm1, 128(%rcx)
+        vmovdqu	%ymm2, 96(%rdi)
+        vmovdqu	%ymm3, 96(%r8)
+        vmovdqu	%ymm4, 64(%rcx)
+        # Row 1
+        vpxor	-32(%r8), %ymm8, %ymm10
+        vpxor	32(%rdi), %ymm9, %ymm11
+        vpxor	32(%r8), %ymm5, %ymm12
+        vpxor	(%rcx), %ymm6, %ymm13
+        vpxor	-32(%rdi), %ymm7, %ymm14
+        vpsrlq	$36, %ymm10, %ymm0
+        vpsrlq	$44, %ymm11, %ymm1
+        vpsrlq	$61, %ymm12, %ymm2
+        vpsrlq	$19, %ymm13, %ymm3
+        vpsrlq	$3, %ymm14, %ymm4
+        vpsllq	$28, %ymm10, %ymm10
+        vpsllq	$20, %ymm11, %ymm11
+        vpsllq	$3, %ymm12, %ymm12
+        vpsllq	$45, %ymm13, %ymm13
+        vpsllq	$61, %ymm14, %ymm14
+        vpor	%ymm0, %ymm10, %ymm10
+        vpor	%ymm1, %ymm11, %ymm11
+        vpor	%ymm2, %ymm12, %ymm12
+        vpor	%ymm3, %ymm13, %ymm13
+        vpor	%ymm4, %ymm14, %ymm14
+        vpandn	%ymm12, %ymm11, %ymm0
+        vpandn	%ymm13, %ymm12, %ymm1
+        vpandn	%ymm14, %ymm13, %ymm2
+        vpandn	%ymm10, %ymm14, %ymm3
+        vpandn	%ymm11, %ymm10, %ymm4
+        vpxor	%ymm10, %ymm0, %ymm0
+        vpxor	%ymm11, %ymm1, %ymm1
+        vpxor	%ymm12, %ymm2, %ymm2
+        vpxor	%ymm13, %ymm3, %ymm3
+        vpxor	%ymm14, %ymm4, %ymm4
+        vmovdqu	%ymm0, -32(%r8)
+        vmovdqu	%ymm1, 32(%rdi)
+        vmovdqu	%ymm2, 32(%r8)
+        vmovdqu	%ymm3, (%rcx)
+        vmovdqu	%ymm4, -32(%rdi)
+        # Row 2
+        vpxor	128(%rdi), %ymm6, %ymm10
+        vpxor	128(%r8), %ymm7, %ymm11
+        vpxor	-64(%rcx), %ymm8, %ymm12
+        vpxor	-96(%rdi), %ymm9, %ymm13
+        vpxor	-96(%r8), %ymm5, %ymm14
+        vpsrlq	$63, %ymm10, %ymm0
+        vpsrlq	$58, %ymm11, %ymm1
+        vpsrlq	$39, %ymm12, %ymm2
+        vpsrlq	$56, %ymm13, %ymm3
+        vpsrlq	$46, %ymm14, %ymm4
+        vpaddq	%ymm10, %ymm10, %ymm10
+        vpsllq	$6, %ymm11, %ymm11
+        vpsllq	$25, %ymm12, %ymm12
+        vpsllq	$8, %ymm13, %ymm13
+        vpsllq	$18, %ymm14, %ymm14
+        vpor	%ymm0, %ymm10, %ymm10
+        vpor	%ymm1, %ymm11, %ymm11
+        vpor	%ymm2, %ymm12, %ymm12
+        vpor	%ymm3, %ymm13, %ymm13
+        vpor	%ymm4, %ymm14, %ymm14
+        vpandn	%ymm12, %ymm11, %ymm0
+        vpandn	%ymm13, %ymm12, %ymm1
+        vpandn	%ymm14, %ymm13, %ymm2
+        vpandn	%ymm10, %ymm14, %ymm3
+        vpandn	%ymm11, %ymm10, %ymm4
+        vpxor	%ymm10, %ymm0, %ymm0
+        vpxor	%ymm11, %ymm1, %ymm1
+        vpxor	%ymm12, %ymm2, %ymm2
+        vpxor	%ymm13, %ymm3, %ymm3
+        vpxor	%ymm14, %ymm4, %ymm4
+        vmovdqu	%ymm0, 128(%rdi)
+        vmovdqu	%ymm1, 128(%r8)
+        vmovdqu	%ymm2, -64(%rcx)
+        vmovdqu	%ymm3, -96(%rdi)
+        vmovdqu	%ymm4, -96(%r8)
+        # Row 3
+        vpxor	64(%r8), %ymm9, %ymm10
+        vpxor	32(%rcx), %ymm5, %ymm11
+        vpxor	(%rdi), %ymm6, %ymm12
+        vpxor	96(%rcx), %ymm7, %ymm13
+        vpxor	64(%rdi), %ymm8, %ymm14
+        vpsrlq	$37, %ymm10, %ymm0
+        vpsrlq	$28, %ymm11, %ymm1
+        vpsrlq	$54, %ymm12, %ymm2
+        vpsrlq	$49, %ymm13, %ymm3
+        vpsrlq	$8, %ymm14, %ymm4
+        vpsllq	$27, %ymm10, %ymm10
+        vpsllq	$36, %ymm11, %ymm11
+        vpsllq	$10, %ymm12, %ymm12
+        vpsllq	$15, %ymm13, %ymm13
+        vpsllq	$56, %ymm14, %ymm14
+        vpor	%ymm0, %ymm10, %ymm10
+        vpor	%ymm1, %ymm11, %ymm11
+        vpor	%ymm2, %ymm12, %ymm12
+        vpor	%ymm3, %ymm13, %ymm13
+        vpor	%ymm4, %ymm14, %ymm14
+        vpandn	%ymm12, %ymm11, %ymm0
+        vpandn	%ymm13, %ymm12, %ymm1
+        vpandn	%ymm14, %ymm13, %ymm2
+        vpandn	%ymm10, %ymm14, %ymm3
+        vpandn	%ymm11, %ymm10, %ymm4
+        vpxor	%ymm10, %ymm0, %ymm0
+        vpxor	%ymm11, %ymm1, %ymm1
+        vpxor	%ymm12, %ymm2, %ymm2
+        vpxor	%ymm13, %ymm3, %ymm3
+        vpxor	%ymm14, %ymm4, %ymm4
+        vmovdqu	%ymm0, 64(%r8)
+        vmovdqu	%ymm1, 32(%rcx)
+        vmovdqu	%ymm2, (%rdi)
+        vmovdqu	%ymm3, 96(%rcx)
+        vmovdqu	%ymm4, 64(%rdi)
+        # Row 4
+        vpxor	-32(%rcx), %ymm7, %ymm10
+        vpxor	-64(%rdi), %ymm8, %ymm11
+        vpxor	-64(%r8), %ymm9, %ymm12
+        vpxor	-96(%rcx), %ymm5, %ymm13
+        vpxor	(%r8), %ymm6, %ymm14
+        vpsrlq	$2, %ymm10, %ymm0
+        vpsrlq	$9, %ymm11, %ymm1
+        vpsrlq	$25, %ymm12, %ymm2
+        vpsrlq	$23, %ymm13, %ymm3
+        vpsrlq	$62, %ymm14, %ymm4
+        vpsllq	$62, %ymm10, %ymm10
+        vpsllq	$55, %ymm11, %ymm11
+        vpsllq	$39, %ymm12, %ymm12
+        vpsllq	$41, %ymm13, %ymm13
+        vpsllq	$2, %ymm14, %ymm14
+        vpor	%ymm0, %ymm10, %ymm10
+        vpor	%ymm1, %ymm11, %ymm11
+        vpor	%ymm2, %ymm12, %ymm12
+        vpor	%ymm3, %ymm13, %ymm13
+        vpor	%ymm4, %ymm14, %ymm14
+        vpandn	%ymm12, %ymm11, %ymm0
+        vpandn	%ymm13, %ymm12, %ymm1
+        vpandn	%ymm14, %ymm13, %ymm2
+        vpandn	%ymm10, %ymm14, %ymm3
+        vpandn	%ymm11, %ymm10, %ymm4
+        vpxor	%ymm10, %ymm0, %ymm0
+        vpxor	%ymm11, %ymm1, %ymm1
+        vpxor	%ymm12, %ymm2, %ymm2
+        vpxor	%ymm13, %ymm3, %ymm3
+        vpxor	%ymm14, %ymm4, %ymm4
+        vmovdqu	%ymm0, -32(%rcx)
+        vmovdqu	%ymm1, -64(%rdi)
+        vmovdqu	%ymm2, -64(%r8)
+        vmovdqu	%ymm3, -96(%rcx)
+        vmovdqu	%ymm4, (%r8)
+        # Round 16
+        # Calc b[0..4]
+        vpxor	%ymm15, %ymm0, %ymm10
+        vpxor	-96(%rdi), %ymm3, %ymm13
+        vpxor	-32(%rdi), %ymm4, %ymm14
+        vpxor	(%rdi), %ymm2, %ymm12
+        vpxor	32(%rdi), %ymm1, %ymm11
+        vpxor	64(%rdi), %ymm14, %ymm14
+        vpxor	96(%rdi), %ymm12, %ymm12
+        vpxor	128(%rdi), %ymm10, %ymm10
+        vpxor	-64(%rcx), %ymm12, %ymm12
+        vpxor	(%rcx), %ymm13, %ymm13
+        vpxor	32(%rcx), %ymm11, %ymm11
+        vpxor	64(%rcx), %ymm14, %ymm14
+        vpxor	96(%rcx), %ymm13, %ymm13
+        vpxor	128(%rcx), %ymm11, %ymm11
+        vpxor	-96(%r8), %ymm14, %ymm14
+        vpxor	-32(%r8), %ymm10, %ymm10
+        vpxor	32(%r8), %ymm12, %ymm12
+        vpxor	64(%r8), %ymm10, %ymm10
+        vpxor	96(%r8), %ymm13, %ymm13
+        vpxor	128(%r8), %ymm11, %ymm11
+        # Calc t[0..4]
+        vpsrlq	$63, %ymm11, %ymm0
+        vpsrlq	$63, %ymm12, %ymm1
+        vpsrlq	$63, %ymm13, %ymm2
+        vpsrlq	$63, %ymm14, %ymm3
+        vpsrlq	$63, %ymm10, %ymm4
+        vpaddq	%ymm11, %ymm11, %ymm5
+        vpaddq	%ymm12, %ymm12, %ymm6
+        vpaddq	%ymm13, %ymm13, %ymm7
+        vpaddq	%ymm14, %ymm14, %ymm8
+        vpaddq	%ymm10, %ymm10, %ymm9
+        vpor	%ymm0, %ymm5, %ymm5
+        vpor	%ymm1, %ymm6, %ymm6
+        vpor	%ymm2, %ymm7, %ymm7
+        vpor	%ymm3, %ymm8, %ymm8
+        vpor	%ymm4, %ymm9, %ymm9
+        vpxor	%ymm14, %ymm5, %ymm5
+        vpxor	%ymm10, %ymm6, %ymm6
+        vpxor	%ymm11, %ymm7, %ymm7
+        vpxor	%ymm12, %ymm8, %ymm8
+        vpxor	%ymm13, %ymm9, %ymm9
+        # Row Mix
+        # Row 0
+        vpxor	%ymm15, %ymm5, %ymm10
+        vpxor	32(%rdi), %ymm6, %ymm11
+        vpxor	-64(%rcx), %ymm7, %ymm12
+        vpxor	96(%rcx), %ymm8, %ymm13
+        vpxor	(%r8), %ymm9, %ymm14
+        vpsrlq	$20, %ymm11, %ymm0
+        vpsrlq	$21, %ymm12, %ymm1
+        vpsrlq	$43, %ymm13, %ymm2
+        vpsrlq	$50, %ymm14, %ymm3
+        vpsllq	$44, %ymm11, %ymm11
+        vpsllq	$43, %ymm12, %ymm12
+        vpsllq	$21, %ymm13, %ymm13
+        vpsllq	$14, %ymm14, %ymm14
+        vpor	%ymm0, %ymm11, %ymm11
+        vpor	%ymm1, %ymm12, %ymm12
+        vpor	%ymm2, %ymm13, %ymm13
+        vpor	%ymm3, %ymm14, %ymm14
+        vpandn	%ymm12, %ymm11, %ymm15
+        vpandn	%ymm13, %ymm12, %ymm1
+        vpandn	%ymm14, %ymm13, %ymm2
+        vpandn	%ymm10, %ymm14, %ymm3
+        vpandn	%ymm11, %ymm10, %ymm4
+        vpxor	%ymm10, %ymm15, %ymm15
+        vpxor	%ymm11, %ymm1, %ymm1
+        vpxor	%ymm12, %ymm2, %ymm2
+        # XOR in constant
+        vpxor	512(%rax), %ymm15, %ymm15
+        vpxor	%ymm13, %ymm3, %ymm3
+        vpxor	%ymm14, %ymm4, %ymm4
+        vmovdqu	%ymm1, 32(%rdi)
+        vmovdqu	%ymm2, -64(%rcx)
+        vmovdqu	%ymm3, 96(%rcx)
+        vmovdqu	%ymm4, (%r8)
+        # Row 1
+        vpxor	96(%r8), %ymm8, %ymm10
+        vpxor	-32(%rdi), %ymm9, %ymm11
+        vpxor	128(%rdi), %ymm5, %ymm12
+        vpxor	32(%rcx), %ymm6, %ymm13
+        vpxor	-64(%r8), %ymm7, %ymm14
+        vpsrlq	$36, %ymm10, %ymm0
+        vpsrlq	$44, %ymm11, %ymm1
+        vpsrlq	$61, %ymm12, %ymm2
+        vpsrlq	$19, %ymm13, %ymm3
+        vpsrlq	$3, %ymm14, %ymm4
+        vpsllq	$28, %ymm10, %ymm10
+        vpsllq	$20, %ymm11, %ymm11
+        vpsllq	$3, %ymm12, %ymm12
+        vpsllq	$45, %ymm13, %ymm13
+        vpsllq	$61, %ymm14, %ymm14
+        vpor	%ymm0, %ymm10, %ymm10
+        vpor	%ymm1, %ymm11, %ymm11
+        vpor	%ymm2, %ymm12, %ymm12
+        vpor	%ymm3, %ymm13, %ymm13
+        vpor	%ymm4, %ymm14, %ymm14
+        vpandn	%ymm12, %ymm11, %ymm0
+        vpandn	%ymm13, %ymm12, %ymm1
+        vpandn	%ymm14, %ymm13, %ymm2
+        vpandn	%ymm10, %ymm14, %ymm3
+        vpandn	%ymm11, %ymm10, %ymm4
+        vpxor	%ymm10, %ymm0, %ymm0
+        vpxor	%ymm11, %ymm1, %ymm1
+        vpxor	%ymm12, %ymm2, %ymm2
+        vpxor	%ymm13, %ymm3, %ymm3
+        vpxor	%ymm14, %ymm4, %ymm4
+        vmovdqu	%ymm0, 96(%r8)
+        vmovdqu	%ymm1, -32(%rdi)
+        vmovdqu	%ymm2, 128(%rdi)
+        vmovdqu	%ymm3, 32(%rcx)
+        vmovdqu	%ymm4, -64(%r8)
+        # Row 2
+        vpxor	128(%rcx), %ymm6, %ymm10
+        vpxor	32(%r8), %ymm7, %ymm11
+        vpxor	-96(%rdi), %ymm8, %ymm12
+        vpxor	64(%rdi), %ymm9, %ymm13
+        vpxor	-32(%rcx), %ymm5, %ymm14
+        vpsrlq	$63, %ymm10, %ymm0
+        vpsrlq	$58, %ymm11, %ymm1
+        vpsrlq	$39, %ymm12, %ymm2
+        vpsrlq	$56, %ymm13, %ymm3
+        vpsrlq	$46, %ymm14, %ymm4
+        vpaddq	%ymm10, %ymm10, %ymm10
+        vpsllq	$6, %ymm11, %ymm11
+        vpsllq	$25, %ymm12, %ymm12
+        vpsllq	$8, %ymm13, %ymm13
+        vpsllq	$18, %ymm14, %ymm14
+        vpor	%ymm0, %ymm10, %ymm10
+        vpor	%ymm1, %ymm11, %ymm11
+        vpor	%ymm2, %ymm12, %ymm12
+        vpor	%ymm3, %ymm13, %ymm13
+        vpor	%ymm4, %ymm14, %ymm14
+        vpandn	%ymm12, %ymm11, %ymm0
+        vpandn	%ymm13, %ymm12, %ymm1
+        vpandn	%ymm14, %ymm13, %ymm2
+        vpandn	%ymm10, %ymm14, %ymm3
+        vpandn	%ymm11, %ymm10, %ymm4
+        vpxor	%ymm10, %ymm0, %ymm0
+        vpxor	%ymm11, %ymm1, %ymm1
+        vpxor	%ymm12, %ymm2, %ymm2
+        vpxor	%ymm13, %ymm3, %ymm3
+        vpxor	%ymm14, %ymm4, %ymm4
+        vmovdqu	%ymm0, 128(%rcx)
+        vmovdqu	%ymm1, 32(%r8)
+        vmovdqu	%ymm2, -96(%rdi)
+        vmovdqu	%ymm3, 64(%rdi)
+        vmovdqu	%ymm4, -32(%rcx)
+        # Row 3
+        vpxor	64(%rcx), %ymm9, %ymm10
+        vpxor	-32(%r8), %ymm5, %ymm11
+        vpxor	128(%r8), %ymm6, %ymm12
+        vpxor	(%rdi), %ymm7, %ymm13
+        vpxor	-96(%rcx), %ymm8, %ymm14
+        vpsrlq	$37, %ymm10, %ymm0
+        vpsrlq	$28, %ymm11, %ymm1
+        vpsrlq	$54, %ymm12, %ymm2
+        vpsrlq	$49, %ymm13, %ymm3
+        vpsrlq	$8, %ymm14, %ymm4
+        vpsllq	$27, %ymm10, %ymm10
+        vpsllq	$36, %ymm11, %ymm11
+        vpsllq	$10, %ymm12, %ymm12
+        vpsllq	$15, %ymm13, %ymm13
+        vpsllq	$56, %ymm14, %ymm14
+        vpor	%ymm0, %ymm10, %ymm10
+        vpor	%ymm1, %ymm11, %ymm11
+        vpor	%ymm2, %ymm12, %ymm12
+        vpor	%ymm3, %ymm13, %ymm13
+        vpor	%ymm4, %ymm14, %ymm14
+        vpandn	%ymm12, %ymm11, %ymm0
+        vpandn	%ymm13, %ymm12, %ymm1
+        vpandn	%ymm14, %ymm13, %ymm2
+        vpandn	%ymm10, %ymm14, %ymm3
+        vpandn	%ymm11, %ymm10, %ymm4
+        vpxor	%ymm10, %ymm0, %ymm0
+        vpxor	%ymm11, %ymm1, %ymm1
+        vpxor	%ymm12, %ymm2, %ymm2
+        vpxor	%ymm13, %ymm3, %ymm3
+        vpxor	%ymm14, %ymm4, %ymm4
+        vmovdqu	%ymm0, 64(%rcx)
+        vmovdqu	%ymm1, -32(%r8)
+        vmovdqu	%ymm2, 128(%r8)
+        vmovdqu	%ymm3, (%rdi)
+        vmovdqu	%ymm4, -96(%rcx)
+        # Row 4
+        vpxor	96(%rdi), %ymm7, %ymm10
+        vpxor	(%rcx), %ymm8, %ymm11
+        vpxor	-96(%r8), %ymm9, %ymm12
+        vpxor	64(%r8), %ymm5, %ymm13
+        vpxor	-64(%rdi), %ymm6, %ymm14
+        vpsrlq	$2, %ymm10, %ymm0
+        vpsrlq	$9, %ymm11, %ymm1
+        vpsrlq	$25, %ymm12, %ymm2
+        vpsrlq	$23, %ymm13, %ymm3
+        vpsrlq	$62, %ymm14, %ymm4
+        vpsllq	$62, %ymm10, %ymm10
+        vpsllq	$55, %ymm11, %ymm11
+        vpsllq	$39, %ymm12, %ymm12
+        vpsllq	$41, %ymm13, %ymm13
+        vpsllq	$2, %ymm14, %ymm14
+        vpor	%ymm0, %ymm10, %ymm10
+        vpor	%ymm1, %ymm11, %ymm11
+        vpor	%ymm2, %ymm12, %ymm12
+        vpor	%ymm3, %ymm13, %ymm13
+        vpor	%ymm4, %ymm14, %ymm14
+        vpandn	%ymm12, %ymm11, %ymm0
+        vpandn	%ymm13, %ymm12, %ymm1
+        vpandn	%ymm14, %ymm13, %ymm2
+        vpandn	%ymm10, %ymm14, %ymm3
+        vpandn	%ymm11, %ymm10, %ymm4
+        vpxor	%ymm10, %ymm0, %ymm0
+        vpxor	%ymm11, %ymm1, %ymm1
+        vpxor	%ymm12, %ymm2, %ymm2
+        vpxor	%ymm13, %ymm3, %ymm3
+        vpxor	%ymm14, %ymm4, %ymm4
+        vmovdqu	%ymm0, 96(%rdi)
+        vmovdqu	%ymm1, (%rcx)
+        vmovdqu	%ymm2, -96(%r8)
+        vmovdqu	%ymm3, 64(%r8)
+        vmovdqu	%ymm4, -64(%rdi)
+        # Round 17
+        # Calc b[0..4]
+        vpxor	%ymm15, %ymm0, %ymm10
+        vpxor	-96(%rdi), %ymm2, %ymm12
+        vpxor	-32(%rdi), %ymm1, %ymm11
+        vpxor	(%rdi), %ymm3, %ymm13
+        vpxor	32(%rdi), %ymm11, %ymm11
+        vpxor	64(%rdi), %ymm13, %ymm13
+        vpxor	128(%rdi), %ymm12, %ymm12
+        vpxor	-96(%rcx), %ymm4, %ymm14
+        vpxor	-64(%rcx), %ymm12, %ymm12
+        vpxor	-32(%rcx), %ymm14, %ymm14
+        vpxor	32(%rcx), %ymm13, %ymm13
+        vpxor	64(%rcx), %ymm10, %ymm10
+        vpxor	96(%rcx), %ymm13, %ymm13
+        vpxor	128(%rcx), %ymm10, %ymm10
+        vpxor	-64(%r8), %ymm14, %ymm14
+        vpxor	-32(%r8), %ymm11, %ymm11
+        vpxor	(%r8), %ymm14, %ymm14
+        vpxor	32(%r8), %ymm11, %ymm11
+        vpxor	96(%r8), %ymm10, %ymm10
+        vpxor	128(%r8), %ymm12, %ymm12
+        # Calc t[0..4]
+        vpsrlq	$63, %ymm11, %ymm0
+        vpsrlq	$63, %ymm12, %ymm1
+        vpsrlq	$63, %ymm13, %ymm2
+        vpsrlq	$63, %ymm14, %ymm3
+        vpsrlq	$63, %ymm10, %ymm4
+        vpaddq	%ymm11, %ymm11, %ymm5
+        vpaddq	%ymm12, %ymm12, %ymm6
+        vpaddq	%ymm13, %ymm13, %ymm7
+        vpaddq	%ymm14, %ymm14, %ymm8
+        vpaddq	%ymm10, %ymm10, %ymm9
+        vpor	%ymm0, %ymm5, %ymm5
+        vpor	%ymm1, %ymm6, %ymm6
+        vpor	%ymm2, %ymm7, %ymm7
+        vpor	%ymm3, %ymm8, %ymm8
+        vpor	%ymm4, %ymm9, %ymm9
+        vpxor	%ymm14, %ymm5, %ymm5
+        vpxor	%ymm10, %ymm6, %ymm6
+        vpxor	%ymm11, %ymm7, %ymm7
+        vpxor	%ymm12, %ymm8, %ymm8
+        vpxor	%ymm13, %ymm9, %ymm9
+        # Row Mix
+        # Row 0
+        vpxor	%ymm15, %ymm5, %ymm10
+        vpxor	-32(%rdi), %ymm6, %ymm11
+        vpxor	-96(%rdi), %ymm7, %ymm12
+        vpxor	(%rdi), %ymm8, %ymm13
+        vpxor	-64(%rdi), %ymm9, %ymm14
+        vpsrlq	$20, %ymm11, %ymm0
+        vpsrlq	$21, %ymm12, %ymm1
+        vpsrlq	$43, %ymm13, %ymm2
+        vpsrlq	$50, %ymm14, %ymm3
+        vpsllq	$44, %ymm11, %ymm11
+        vpsllq	$43, %ymm12, %ymm12
+        vpsllq	$21, %ymm13, %ymm13
+        vpsllq	$14, %ymm14, %ymm14
+        vpor	%ymm0, %ymm11, %ymm11
+        vpor	%ymm1, %ymm12, %ymm12
+        vpor	%ymm2, %ymm13, %ymm13
+        vpor	%ymm3, %ymm14, %ymm14
+        vpandn	%ymm12, %ymm11, %ymm15
+        vpandn	%ymm13, %ymm12, %ymm1
+        vpandn	%ymm14, %ymm13, %ymm2
+        vpandn	%ymm10, %ymm14, %ymm3
+        vpandn	%ymm11, %ymm10, %ymm4
+        vpxor	%ymm10, %ymm15, %ymm15
+        vpxor	%ymm11, %ymm1, %ymm1
+        vpxor	%ymm12, %ymm2, %ymm2
+        # XOR in constant
+        vpxor	544(%rax), %ymm15, %ymm15
+        vpxor	%ymm13, %ymm3, %ymm3
+        vpxor	%ymm14, %ymm4, %ymm4
+        vmovdqu	%ymm1, -32(%rdi)
+        vmovdqu	%ymm2, -96(%rdi)
+        vmovdqu	%ymm3, (%rdi)
+        vmovdqu	%ymm4, -64(%rdi)
+        # Row 1
+        vpxor	96(%rcx), %ymm8, %ymm10
+        vpxor	-64(%r8), %ymm9, %ymm11
+        vpxor	128(%rcx), %ymm5, %ymm12
+        vpxor	-32(%r8), %ymm6, %ymm13
+        vpxor	-96(%r8), %ymm7, %ymm14
+        vpsrlq	$36, %ymm10, %ymm0
+        vpsrlq	$44, %ymm11, %ymm1
+        vpsrlq	$61, %ymm12, %ymm2
+        vpsrlq	$19, %ymm13, %ymm3
+        vpsrlq	$3, %ymm14, %ymm4
+        vpsllq	$28, %ymm10, %ymm10
+        vpsllq	$20, %ymm11, %ymm11
+        vpsllq	$3, %ymm12, %ymm12
+        vpsllq	$45, %ymm13, %ymm13
+        vpsllq	$61, %ymm14, %ymm14
+        vpor	%ymm0, %ymm10, %ymm10
+        vpor	%ymm1, %ymm11, %ymm11
+        vpor	%ymm2, %ymm12, %ymm12
+        vpor	%ymm3, %ymm13, %ymm13
+        vpor	%ymm4, %ymm14, %ymm14
+        vpandn	%ymm12, %ymm11, %ymm0
+        vpandn	%ymm13, %ymm12, %ymm1
+        vpandn	%ymm14, %ymm13, %ymm2
+        vpandn	%ymm10, %ymm14, %ymm3
+        vpandn	%ymm11, %ymm10, %ymm4
+        vpxor	%ymm10, %ymm0, %ymm0
+        vpxor	%ymm11, %ymm1, %ymm1
+        vpxor	%ymm12, %ymm2, %ymm2
+        vpxor	%ymm13, %ymm3, %ymm3
+        vpxor	%ymm14, %ymm4, %ymm4
+        vmovdqu	%ymm0, 96(%rcx)
+        vmovdqu	%ymm1, -64(%r8)
+        vmovdqu	%ymm2, 128(%rcx)
+        vmovdqu	%ymm3, -32(%r8)
+        vmovdqu	%ymm4, -96(%r8)
+        # Row 2
+        vpxor	32(%rdi), %ymm6, %ymm10
+        vpxor	128(%rdi), %ymm7, %ymm11
+        vpxor	64(%rdi), %ymm8, %ymm12
+        vpxor	-96(%rcx), %ymm9, %ymm13
+        vpxor	96(%rdi), %ymm5, %ymm14
+        vpsrlq	$63, %ymm10, %ymm0
+        vpsrlq	$58, %ymm11, %ymm1
+        vpsrlq	$39, %ymm12, %ymm2
+        vpsrlq	$56, %ymm13, %ymm3
+        vpsrlq	$46, %ymm14, %ymm4
+        vpaddq	%ymm10, %ymm10, %ymm10
+        vpsllq	$6, %ymm11, %ymm11
+        vpsllq	$25, %ymm12, %ymm12
+        vpsllq	$8, %ymm13, %ymm13
+        vpsllq	$18, %ymm14, %ymm14
+        vpor	%ymm0, %ymm10, %ymm10
+        vpor	%ymm1, %ymm11, %ymm11
+        vpor	%ymm2, %ymm12, %ymm12
+        vpor	%ymm3, %ymm13, %ymm13
+        vpor	%ymm4, %ymm14, %ymm14
+        vpandn	%ymm12, %ymm11, %ymm0
+        vpandn	%ymm13, %ymm12, %ymm1
+        vpandn	%ymm14, %ymm13, %ymm2
+        vpandn	%ymm10, %ymm14, %ymm3
+        vpandn	%ymm11, %ymm10, %ymm4
+        vpxor	%ymm10, %ymm0, %ymm0
+        vpxor	%ymm11, %ymm1, %ymm1
+        vpxor	%ymm12, %ymm2, %ymm2
+        vpxor	%ymm13, %ymm3, %ymm3
+        vpxor	%ymm14, %ymm4, %ymm4
+        vmovdqu	%ymm0, 32(%rdi)
+        vmovdqu	%ymm1, 128(%rdi)
+        vmovdqu	%ymm2, 64(%rdi)
+        vmovdqu	%ymm3, -96(%rcx)
+        vmovdqu	%ymm4, 96(%rdi)
+        # Row 3
+        vpxor	(%r8), %ymm9, %ymm10
+        vpxor	96(%r8), %ymm5, %ymm11
+        vpxor	32(%r8), %ymm6, %ymm12
+        vpxor	128(%r8), %ymm7, %ymm13
+        vpxor	64(%r8), %ymm8, %ymm14
+        vpsrlq	$37, %ymm10, %ymm0
+        vpsrlq	$28, %ymm11, %ymm1
+        vpsrlq	$54, %ymm12, %ymm2
+        vpsrlq	$49, %ymm13, %ymm3
+        vpsrlq	$8, %ymm14, %ymm4
+        vpsllq	$27, %ymm10, %ymm10
+        vpsllq	$36, %ymm11, %ymm11
+        vpsllq	$10, %ymm12, %ymm12
+        vpsllq	$15, %ymm13, %ymm13
+        vpsllq	$56, %ymm14, %ymm14
+        vpor	%ymm0, %ymm10, %ymm10
+        vpor	%ymm1, %ymm11, %ymm11
+        vpor	%ymm2, %ymm12, %ymm12
+        vpor	%ymm3, %ymm13, %ymm13
+        vpor	%ymm4, %ymm14, %ymm14
+        vpandn	%ymm12, %ymm11, %ymm0
+        vpandn	%ymm13, %ymm12, %ymm1
+        vpandn	%ymm14, %ymm13, %ymm2
+        vpandn	%ymm10, %ymm14, %ymm3
+        vpandn	%ymm11, %ymm10, %ymm4
+        vpxor	%ymm10, %ymm0, %ymm0
+        vpxor	%ymm11, %ymm1, %ymm1
+        vpxor	%ymm12, %ymm2, %ymm2
+        vpxor	%ymm13, %ymm3, %ymm3
+        vpxor	%ymm14, %ymm4, %ymm4
+        vmovdqu	%ymm0, (%r8)
+        vmovdqu	%ymm1, 96(%r8)
+        vmovdqu	%ymm2, 32(%r8)
+        vmovdqu	%ymm3, 128(%r8)
+        vmovdqu	%ymm4, 64(%r8)
+        # Row 4
+        vpxor	-64(%rcx), %ymm7, %ymm10
+        vpxor	32(%rcx), %ymm8, %ymm11
+        vpxor	-32(%rcx), %ymm9, %ymm12
+        vpxor	64(%rcx), %ymm5, %ymm13
+        vpxor	(%rcx), %ymm6, %ymm14
+        vpsrlq	$2, %ymm10, %ymm0
+        vpsrlq	$9, %ymm11, %ymm1
+        vpsrlq	$25, %ymm12, %ymm2
+        vpsrlq	$23, %ymm13, %ymm3
+        vpsrlq	$62, %ymm14, %ymm4
+        vpsllq	$62, %ymm10, %ymm10
+        vpsllq	$55, %ymm11, %ymm11
+        vpsllq	$39, %ymm12, %ymm12
+        vpsllq	$41, %ymm13, %ymm13
+        vpsllq	$2, %ymm14, %ymm14
+        vpor	%ymm0, %ymm10, %ymm10
+        vpor	%ymm1, %ymm11, %ymm11
+        vpor	%ymm2, %ymm12, %ymm12
+        vpor	%ymm3, %ymm13, %ymm13
+        vpor	%ymm4, %ymm14, %ymm14
+        vpandn	%ymm12, %ymm11, %ymm0
+        vpandn	%ymm13, %ymm12, %ymm1
+        vpandn	%ymm14, %ymm13, %ymm2
+        vpandn	%ymm10, %ymm14, %ymm3
+        vpandn	%ymm11, %ymm10, %ymm4
+        vpxor	%ymm10, %ymm0, %ymm0
+        vpxor	%ymm11, %ymm1, %ymm1
+        vpxor	%ymm12, %ymm2, %ymm2
+        vpxor	%ymm13, %ymm3, %ymm3
+        vpxor	%ymm14, %ymm4, %ymm4
+        vmovdqu	%ymm0, -64(%rcx)
+        vmovdqu	%ymm1, 32(%rcx)
+        vmovdqu	%ymm2, -32(%rcx)
+        vmovdqu	%ymm3, 64(%rcx)
+        vmovdqu	%ymm4, (%rcx)
+        # Round 18
+        # Calc b[0..4]
+        vpxor	%ymm15, %ymm0, %ymm10
+        vpxor	-96(%rdi), %ymm2, %ymm12
+        vpxor	-64(%rdi), %ymm4, %ymm14
+        vpxor	-32(%rdi), %ymm1, %ymm11
+        vpxor	(%rdi), %ymm3, %ymm13
+        vpxor	32(%rdi), %ymm10, %ymm10
+        vpxor	64(%rdi), %ymm12, %ymm12
+        vpxor	96(%rdi), %ymm14, %ymm14
+        vpxor	128(%rdi), %ymm11, %ymm11
+        vpxor	-96(%rcx), %ymm13, %ymm13
+        vpxor	96(%rcx), %ymm10, %ymm10
+        vpxor	128(%rcx), %ymm12, %ymm12
+        vpxor	-96(%r8), %ymm14, %ymm14
+        vpxor	-64(%r8), %ymm11, %ymm11
+        vpxor	-32(%r8), %ymm13, %ymm13
+        vpxor	(%r8), %ymm10, %ymm10
+        vpxor	32(%r8), %ymm12, %ymm12
+        vpxor	64(%r8), %ymm14, %ymm14
+        vpxor	96(%r8), %ymm11, %ymm11
+        vpxor	128(%r8), %ymm13, %ymm13
+        # Calc t[0..4]
+        vpsrlq	$63, %ymm11, %ymm0
+        vpsrlq	$63, %ymm12, %ymm1
+        vpsrlq	$63, %ymm13, %ymm2
+        vpsrlq	$63, %ymm14, %ymm3
+        vpsrlq	$63, %ymm10, %ymm4
+        vpaddq	%ymm11, %ymm11, %ymm5
+        vpaddq	%ymm12, %ymm12, %ymm6
+        vpaddq	%ymm13, %ymm13, %ymm7
+        vpaddq	%ymm14, %ymm14, %ymm8
+        vpaddq	%ymm10, %ymm10, %ymm9
+        vpor	%ymm0, %ymm5, %ymm5
+        vpor	%ymm1, %ymm6, %ymm6
+        vpor	%ymm2, %ymm7, %ymm7
+        vpor	%ymm3, %ymm8, %ymm8
+        vpor	%ymm4, %ymm9, %ymm9
+        vpxor	%ymm14, %ymm5, %ymm5
+        vpxor	%ymm10, %ymm6, %ymm6
+        vpxor	%ymm11, %ymm7, %ymm7
+        vpxor	%ymm12, %ymm8, %ymm8
+        vpxor	%ymm13, %ymm9, %ymm9
+        # Row Mix
+        # Row 0
+        vpxor	%ymm15, %ymm5, %ymm10
+        vpxor	-64(%r8), %ymm6, %ymm11
+        vpxor	64(%rdi), %ymm7, %ymm12
+        vpxor	128(%r8), %ymm8, %ymm13
+        vpxor	(%rcx), %ymm9, %ymm14
+        vpsrlq	$20, %ymm11, %ymm0
+        vpsrlq	$21, %ymm12, %ymm1
+        vpsrlq	$43, %ymm13, %ymm2
+        vpsrlq	$50, %ymm14, %ymm3
+        vpsllq	$44, %ymm11, %ymm11
+        vpsllq	$43, %ymm12, %ymm12
+        vpsllq	$21, %ymm13, %ymm13
+        vpsllq	$14, %ymm14, %ymm14
+        vpor	%ymm0, %ymm11, %ymm11
+        vpor	%ymm1, %ymm12, %ymm12
+        vpor	%ymm2, %ymm13, %ymm13
+        vpor	%ymm3, %ymm14, %ymm14
+        vpandn	%ymm12, %ymm11, %ymm15
+        vpandn	%ymm13, %ymm12, %ymm1
+        vpandn	%ymm14, %ymm13, %ymm2
+        vpandn	%ymm10, %ymm14, %ymm3
+        vpandn	%ymm11, %ymm10, %ymm4
+        vpxor	%ymm10, %ymm15, %ymm15
+        vpxor	%ymm11, %ymm1, %ymm1
+        vpxor	%ymm12, %ymm2, %ymm2
+        # XOR in constant
+        vpxor	576(%rax), %ymm15, %ymm15
+        vpxor	%ymm13, %ymm3, %ymm3
+        vpxor	%ymm14, %ymm4, %ymm4
+        vmovdqu	%ymm1, -64(%r8)
+        vmovdqu	%ymm2, 64(%rdi)
+        vmovdqu	%ymm3, 128(%r8)
+        vmovdqu	%ymm4, (%rcx)
+        # Row 1
+        vpxor	(%rdi), %ymm8, %ymm10
+        vpxor	-96(%r8), %ymm9, %ymm11
+        vpxor	32(%rdi), %ymm5, %ymm12
+        vpxor	96(%r8), %ymm6, %ymm13
+        vpxor	-32(%rcx), %ymm7, %ymm14
+        vpsrlq	$36, %ymm10, %ymm0
+        vpsrlq	$44, %ymm11, %ymm1
+        vpsrlq	$61, %ymm12, %ymm2
+        vpsrlq	$19, %ymm13, %ymm3
+        vpsrlq	$3, %ymm14, %ymm4
+        vpsllq	$28, %ymm10, %ymm10
+        vpsllq	$20, %ymm11, %ymm11
+        vpsllq	$3, %ymm12, %ymm12
+        vpsllq	$45, %ymm13, %ymm13
+        vpsllq	$61, %ymm14, %ymm14
+        vpor	%ymm0, %ymm10, %ymm10
+        vpor	%ymm1, %ymm11, %ymm11
+        vpor	%ymm2, %ymm12, %ymm12
+        vpor	%ymm3, %ymm13, %ymm13
+        vpor	%ymm4, %ymm14, %ymm14
+        vpandn	%ymm12, %ymm11, %ymm0
+        vpandn	%ymm13, %ymm12, %ymm1
+        vpandn	%ymm14, %ymm13, %ymm2
+        vpandn	%ymm10, %ymm14, %ymm3
+        vpandn	%ymm11, %ymm10, %ymm4
+        vpxor	%ymm10, %ymm0, %ymm0
+        vpxor	%ymm11, %ymm1, %ymm1
+        vpxor	%ymm12, %ymm2, %ymm2
+        vpxor	%ymm13, %ymm3, %ymm3
+        vpxor	%ymm14, %ymm4, %ymm4
+        vmovdqu	%ymm0, (%rdi)
+        vmovdqu	%ymm1, -96(%r8)
+        vmovdqu	%ymm2, 32(%rdi)
+        vmovdqu	%ymm3, 96(%r8)
+        vmovdqu	%ymm4, -32(%rcx)
+        # Row 2
+        vpxor	-32(%rdi), %ymm6, %ymm10
+        vpxor	128(%rcx), %ymm7, %ymm11
+        vpxor	-96(%rcx), %ymm8, %ymm12
+        vpxor	64(%r8), %ymm9, %ymm13
+        vpxor	-64(%rcx), %ymm5, %ymm14
+        vpsrlq	$63, %ymm10, %ymm0
+        vpsrlq	$58, %ymm11, %ymm1
+        vpsrlq	$39, %ymm12, %ymm2
+        vpsrlq	$56, %ymm13, %ymm3
+        vpsrlq	$46, %ymm14, %ymm4
+        vpaddq	%ymm10, %ymm10, %ymm10
+        vpsllq	$6, %ymm11, %ymm11
+        vpsllq	$25, %ymm12, %ymm12
+        vpsllq	$8, %ymm13, %ymm13
+        vpsllq	$18, %ymm14, %ymm14
+        vpor	%ymm0, %ymm10, %ymm10
+        vpor	%ymm1, %ymm11, %ymm11
+        vpor	%ymm2, %ymm12, %ymm12
+        vpor	%ymm3, %ymm13, %ymm13
+        vpor	%ymm4, %ymm14, %ymm14
+        vpandn	%ymm12, %ymm11, %ymm0
+        vpandn	%ymm13, %ymm12, %ymm1
+        vpandn	%ymm14, %ymm13, %ymm2
+        vpandn	%ymm10, %ymm14, %ymm3
+        vpandn	%ymm11, %ymm10, %ymm4
+        vpxor	%ymm10, %ymm0, %ymm0
+        vpxor	%ymm11, %ymm1, %ymm1
+        vpxor	%ymm12, %ymm2, %ymm2
+        vpxor	%ymm13, %ymm3, %ymm3
+        vpxor	%ymm14, %ymm4, %ymm4
+        vmovdqu	%ymm0, -32(%rdi)
+        vmovdqu	%ymm1, 128(%rcx)
+        vmovdqu	%ymm2, -96(%rcx)
+        vmovdqu	%ymm3, 64(%r8)
+        vmovdqu	%ymm4, -64(%rcx)
+        # Row 3
+        vpxor	-64(%rdi), %ymm9, %ymm10
+        vpxor	96(%rcx), %ymm5, %ymm11
+        vpxor	128(%rdi), %ymm6, %ymm12
+        vpxor	32(%r8), %ymm7, %ymm13
+        vpxor	64(%rcx), %ymm8, %ymm14
+        vpsrlq	$37, %ymm10, %ymm0
+        vpsrlq	$28, %ymm11, %ymm1
+        vpsrlq	$54, %ymm12, %ymm2
+        vpsrlq	$49, %ymm13, %ymm3
+        vpsrlq	$8, %ymm14, %ymm4
+        vpsllq	$27, %ymm10, %ymm10
+        vpsllq	$36, %ymm11, %ymm11
+        vpsllq	$10, %ymm12, %ymm12
+        vpsllq	$15, %ymm13, %ymm13
+        vpsllq	$56, %ymm14, %ymm14
+        vpor	%ymm0, %ymm10, %ymm10
+        vpor	%ymm1, %ymm11, %ymm11
+        vpor	%ymm2, %ymm12, %ymm12
+        vpor	%ymm3, %ymm13, %ymm13
+        vpor	%ymm4, %ymm14, %ymm14
+        vpandn	%ymm12, %ymm11, %ymm0
+        vpandn	%ymm13, %ymm12, %ymm1
+        vpandn	%ymm14, %ymm13, %ymm2
+        vpandn	%ymm10, %ymm14, %ymm3
+        vpandn	%ymm11, %ymm10, %ymm4
+        vpxor	%ymm10, %ymm0, %ymm0
+        vpxor	%ymm11, %ymm1, %ymm1
+        vpxor	%ymm12, %ymm2, %ymm2
+        vpxor	%ymm13, %ymm3, %ymm3
+        vpxor	%ymm14, %ymm4, %ymm4
+        vmovdqu	%ymm0, -64(%rdi)
+        vmovdqu	%ymm1, 96(%rcx)
+        vmovdqu	%ymm2, 128(%rdi)
+        vmovdqu	%ymm3, 32(%r8)
+        vmovdqu	%ymm4, 64(%rcx)
+        # Row 4
+        vpxor	-96(%rdi), %ymm7, %ymm10
+        vpxor	-32(%r8), %ymm8, %ymm11
+        vpxor	96(%rdi), %ymm9, %ymm12
+        vpxor	(%r8), %ymm5, %ymm13
+        vpxor	32(%rcx), %ymm6, %ymm14
+        vpsrlq	$2, %ymm10, %ymm0
+        vpsrlq	$9, %ymm11, %ymm1
+        vpsrlq	$25, %ymm12, %ymm2
+        vpsrlq	$23, %ymm13, %ymm3
+        vpsrlq	$62, %ymm14, %ymm4
+        vpsllq	$62, %ymm10, %ymm10
+        vpsllq	$55, %ymm11, %ymm11
+        vpsllq	$39, %ymm12, %ymm12
+        vpsllq	$41, %ymm13, %ymm13
+        vpsllq	$2, %ymm14, %ymm14
+        vpor	%ymm0, %ymm10, %ymm10
+        vpor	%ymm1, %ymm11, %ymm11
+        vpor	%ymm2, %ymm12, %ymm12
+        vpor	%ymm3, %ymm13, %ymm13
+        vpor	%ymm4, %ymm14, %ymm14
+        vpandn	%ymm12, %ymm11, %ymm0
+        vpandn	%ymm13, %ymm12, %ymm1
+        vpandn	%ymm14, %ymm13, %ymm2
+        vpandn	%ymm10, %ymm14, %ymm3
+        vpandn	%ymm11, %ymm10, %ymm4
+        vpxor	%ymm10, %ymm0, %ymm0
+        vpxor	%ymm11, %ymm1, %ymm1
+        vpxor	%ymm12, %ymm2, %ymm2
+        vpxor	%ymm13, %ymm3, %ymm3
+        vpxor	%ymm14, %ymm4, %ymm4
+        vmovdqu	%ymm0, -96(%rdi)
+        vmovdqu	%ymm1, -32(%r8)
+        vmovdqu	%ymm2, 96(%rdi)
+        vmovdqu	%ymm3, (%r8)
+        vmovdqu	%ymm4, 32(%rcx)
+        # Round 19
+        # Calc b[0..4]
+        vpxor	%ymm15, %ymm0, %ymm10
+        vpxor	-64(%rdi), %ymm10, %ymm10
+        vpxor	-32(%rdi), %ymm10, %ymm10
+        vpxor	(%rdi), %ymm10, %ymm10
+        vpxor	32(%rdi), %ymm2, %ymm12
+        vpxor	64(%rdi), %ymm12, %ymm12
+        vpxor	128(%rdi), %ymm12, %ymm12
+        vpxor	-96(%rcx), %ymm12, %ymm12
+        vpxor	-64(%rcx), %ymm4, %ymm14
+        vpxor	-32(%rcx), %ymm14, %ymm14
+        vpxor	(%rcx), %ymm14, %ymm14
+        vpxor	64(%rcx), %ymm14, %ymm14
+        vpxor	96(%rcx), %ymm1, %ymm11
+        vpxor	128(%rcx), %ymm11, %ymm11
+        vpxor	-96(%r8), %ymm11, %ymm11
+        vpxor	-64(%r8), %ymm11, %ymm11
+        vpxor	32(%r8), %ymm3, %ymm13
+        vpxor	64(%r8), %ymm13, %ymm13
+        vpxor	96(%r8), %ymm13, %ymm13
+        vpxor	128(%r8), %ymm13, %ymm13
+        # Calc t[0..4]
+        vpsrlq	$63, %ymm11, %ymm0
+        vpsrlq	$63, %ymm12, %ymm1
+        vpsrlq	$63, %ymm13, %ymm2
+        vpsrlq	$63, %ymm14, %ymm3
+        vpsrlq	$63, %ymm10, %ymm4
+        vpaddq	%ymm11, %ymm11, %ymm5
+        vpaddq	%ymm12, %ymm12, %ymm6
+        vpaddq	%ymm13, %ymm13, %ymm7
+        vpaddq	%ymm14, %ymm14, %ymm8
+        vpaddq	%ymm10, %ymm10, %ymm9
+        vpor	%ymm0, %ymm5, %ymm5
+        vpor	%ymm1, %ymm6, %ymm6
+        vpor	%ymm2, %ymm7, %ymm7
+        vpor	%ymm3, %ymm8, %ymm8
+        vpor	%ymm4, %ymm9, %ymm9
+        vpxor	%ymm14, %ymm5, %ymm5
+        vpxor	%ymm10, %ymm6, %ymm6
+        vpxor	%ymm11, %ymm7, %ymm7
+        vpxor	%ymm12, %ymm8, %ymm8
+        vpxor	%ymm13, %ymm9, %ymm9
+        # Row Mix
+        # Row 0
+        vpxor	%ymm15, %ymm5, %ymm10
+        vpxor	-96(%r8), %ymm6, %ymm11
+        vpxor	-96(%rcx), %ymm7, %ymm12
+        vpxor	32(%r8), %ymm8, %ymm13
+        vpxor	32(%rcx), %ymm9, %ymm14
+        vpsrlq	$20, %ymm11, %ymm0
+        vpsrlq	$21, %ymm12, %ymm1
+        vpsrlq	$43, %ymm13, %ymm2
+        vpsrlq	$50, %ymm14, %ymm3
+        vpsllq	$44, %ymm11, %ymm11
+        vpsllq	$43, %ymm12, %ymm12
+        vpsllq	$21, %ymm13, %ymm13
+        vpsllq	$14, %ymm14, %ymm14
+        vpor	%ymm0, %ymm11, %ymm11
+        vpor	%ymm1, %ymm12, %ymm12
+        vpor	%ymm2, %ymm13, %ymm13
+        vpor	%ymm3, %ymm14, %ymm14
+        vpandn	%ymm12, %ymm11, %ymm15
+        vpandn	%ymm13, %ymm12, %ymm1
+        vpandn	%ymm14, %ymm13, %ymm2
+        vpandn	%ymm10, %ymm14, %ymm3
+        vpandn	%ymm11, %ymm10, %ymm4
+        vpxor	%ymm10, %ymm15, %ymm15
+        vpxor	%ymm11, %ymm1, %ymm1
+        vpxor	%ymm12, %ymm2, %ymm2
+        # XOR in constant
+        vpxor	608(%rax), %ymm15, %ymm15
+        vpxor	%ymm13, %ymm3, %ymm3
+        vpxor	%ymm14, %ymm4, %ymm4
+        vmovdqu	%ymm1, -96(%r8)
+        vmovdqu	%ymm2, -96(%rcx)
+        vmovdqu	%ymm3, 32(%r8)
+        vmovdqu	%ymm4, 32(%rcx)
+        # Row 1
+        vpxor	128(%r8), %ymm8, %ymm10
+        vpxor	-32(%rcx), %ymm9, %ymm11
+        vpxor	-32(%rdi), %ymm5, %ymm12
+        vpxor	96(%rcx), %ymm6, %ymm13
+        vpxor	96(%rdi), %ymm7, %ymm14
+        vpsrlq	$36, %ymm10, %ymm0
+        vpsrlq	$44, %ymm11, %ymm1
+        vpsrlq	$61, %ymm12, %ymm2
+        vpsrlq	$19, %ymm13, %ymm3
+        vpsrlq	$3, %ymm14, %ymm4
+        vpsllq	$28, %ymm10, %ymm10
+        vpsllq	$20, %ymm11, %ymm11
+        vpsllq	$3, %ymm12, %ymm12
+        vpsllq	$45, %ymm13, %ymm13
+        vpsllq	$61, %ymm14, %ymm14
+        vpor	%ymm0, %ymm10, %ymm10
+        vpor	%ymm1, %ymm11, %ymm11
+        vpor	%ymm2, %ymm12, %ymm12
+        vpor	%ymm3, %ymm13, %ymm13
+        vpor	%ymm4, %ymm14, %ymm14
+        vpandn	%ymm12, %ymm11, %ymm0
+        vpandn	%ymm13, %ymm12, %ymm1
+        vpandn	%ymm14, %ymm13, %ymm2
+        vpandn	%ymm10, %ymm14, %ymm3
+        vpandn	%ymm11, %ymm10, %ymm4
+        vpxor	%ymm10, %ymm0, %ymm0
+        vpxor	%ymm11, %ymm1, %ymm1
+        vpxor	%ymm12, %ymm2, %ymm2
+        vpxor	%ymm13, %ymm3, %ymm3
+        vpxor	%ymm14, %ymm4, %ymm4
+        vmovdqu	%ymm0, 128(%r8)
+        vmovdqu	%ymm1, -32(%rcx)
+        vmovdqu	%ymm2, -32(%rdi)
+        vmovdqu	%ymm3, 96(%rcx)
+        vmovdqu	%ymm4, 96(%rdi)
+        # Row 2
+        vpxor	-64(%r8), %ymm6, %ymm10
+        vpxor	32(%rdi), %ymm7, %ymm11
+        vpxor	64(%r8), %ymm8, %ymm12
+        vpxor	64(%rcx), %ymm9, %ymm13
+        vpxor	-96(%rdi), %ymm5, %ymm14
+        vpsrlq	$63, %ymm10, %ymm0
+        vpsrlq	$58, %ymm11, %ymm1
+        vpsrlq	$39, %ymm12, %ymm2
+        vpsrlq	$56, %ymm13, %ymm3
+        vpsrlq	$46, %ymm14, %ymm4
+        vpaddq	%ymm10, %ymm10, %ymm10
+        vpsllq	$6, %ymm11, %ymm11
+        vpsllq	$25, %ymm12, %ymm12
+        vpsllq	$8, %ymm13, %ymm13
+        vpsllq	$18, %ymm14, %ymm14
+        vpor	%ymm0, %ymm10, %ymm10
+        vpor	%ymm1, %ymm11, %ymm11
+        vpor	%ymm2, %ymm12, %ymm12
+        vpor	%ymm3, %ymm13, %ymm13
+        vpor	%ymm4, %ymm14, %ymm14
+        vpandn	%ymm12, %ymm11, %ymm0
+        vpandn	%ymm13, %ymm12, %ymm1
+        vpandn	%ymm14, %ymm13, %ymm2
+        vpandn	%ymm10, %ymm14, %ymm3
+        vpandn	%ymm11, %ymm10, %ymm4
+        vpxor	%ymm10, %ymm0, %ymm0
+        vpxor	%ymm11, %ymm1, %ymm1
+        vpxor	%ymm12, %ymm2, %ymm2
+        vpxor	%ymm13, %ymm3, %ymm3
+        vpxor	%ymm14, %ymm4, %ymm4
+        vmovdqu	%ymm0, -64(%r8)
+        vmovdqu	%ymm1, 32(%rdi)
+        vmovdqu	%ymm2, 64(%r8)
+        vmovdqu	%ymm3, 64(%rcx)
+        vmovdqu	%ymm4, -96(%rdi)
+        # Row 3
+        vpxor	(%rcx), %ymm9, %ymm10
+        vpxor	(%rdi), %ymm5, %ymm11
+        vpxor	128(%rcx), %ymm6, %ymm12
+        vpxor	128(%rdi), %ymm7, %ymm13
+        vpxor	(%r8), %ymm8, %ymm14
+        vpsrlq	$37, %ymm10, %ymm0
+        vpsrlq	$28, %ymm11, %ymm1
+        vpsrlq	$54, %ymm12, %ymm2
+        vpsrlq	$49, %ymm13, %ymm3
+        vpsrlq	$8, %ymm14, %ymm4
+        vpsllq	$27, %ymm10, %ymm10
+        vpsllq	$36, %ymm11, %ymm11
+        vpsllq	$10, %ymm12, %ymm12
+        vpsllq	$15, %ymm13, %ymm13
+        vpsllq	$56, %ymm14, %ymm14
+        vpor	%ymm0, %ymm10, %ymm10
+        vpor	%ymm1, %ymm11, %ymm11
+        vpor	%ymm2, %ymm12, %ymm12
+        vpor	%ymm3, %ymm13, %ymm13
+        vpor	%ymm4, %ymm14, %ymm14
+        vpandn	%ymm12, %ymm11, %ymm0
+        vpandn	%ymm13, %ymm12, %ymm1
+        vpandn	%ymm14, %ymm13, %ymm2
+        vpandn	%ymm10, %ymm14, %ymm3
+        vpandn	%ymm11, %ymm10, %ymm4
+        vpxor	%ymm10, %ymm0, %ymm0
+        vpxor	%ymm11, %ymm1, %ymm1
+        vpxor	%ymm12, %ymm2, %ymm2
+        vpxor	%ymm13, %ymm3, %ymm3
+        vpxor	%ymm14, %ymm4, %ymm4
+        vmovdqu	%ymm0, (%rcx)
+        vmovdqu	%ymm1, (%rdi)
+        vmovdqu	%ymm2, 128(%rcx)
+        vmovdqu	%ymm3, 128(%rdi)
+        vmovdqu	%ymm4, (%r8)
+        # Row 4
+        vpxor	64(%rdi), %ymm7, %ymm10
+        vpxor	96(%r8), %ymm8, %ymm11
+        vpxor	-64(%rcx), %ymm9, %ymm12
+        vpxor	-64(%rdi), %ymm5, %ymm13
+        vpxor	-32(%r8), %ymm6, %ymm14
+        vpsrlq	$2, %ymm10, %ymm0
+        vpsrlq	$9, %ymm11, %ymm1
+        vpsrlq	$25, %ymm12, %ymm2
+        vpsrlq	$23, %ymm13, %ymm3
+        vpsrlq	$62, %ymm14, %ymm4
+        vpsllq	$62, %ymm10, %ymm10
+        vpsllq	$55, %ymm11, %ymm11
+        vpsllq	$39, %ymm12, %ymm12
+        vpsllq	$41, %ymm13, %ymm13
+        vpsllq	$2, %ymm14, %ymm14
+        vpor	%ymm0, %ymm10, %ymm10
+        vpor	%ymm1, %ymm11, %ymm11
+        vpor	%ymm2, %ymm12, %ymm12
+        vpor	%ymm3, %ymm13, %ymm13
+        vpor	%ymm4, %ymm14, %ymm14
+        vpandn	%ymm12, %ymm11, %ymm0
+        vpandn	%ymm13, %ymm12, %ymm1
+        vpandn	%ymm14, %ymm13, %ymm2
+        vpandn	%ymm10, %ymm14, %ymm3
+        vpandn	%ymm11, %ymm10, %ymm4
+        vpxor	%ymm10, %ymm0, %ymm0
+        vpxor	%ymm11, %ymm1, %ymm1
+        vpxor	%ymm12, %ymm2, %ymm2
+        vpxor	%ymm13, %ymm3, %ymm3
+        vpxor	%ymm14, %ymm4, %ymm4
+        vmovdqu	%ymm0, 64(%rdi)
+        vmovdqu	%ymm1, 96(%r8)
+        vmovdqu	%ymm2, -64(%rcx)
+        vmovdqu	%ymm3, -64(%rdi)
+        vmovdqu	%ymm4, -32(%r8)
+        # Round 20
+        # Calc b[0..4]
+        vpxor	%ymm15, %ymm0, %ymm10
+        vpxor	-96(%rdi), %ymm4, %ymm14
+        vpxor	-32(%rdi), %ymm2, %ymm12
+        vpxor	(%rdi), %ymm1, %ymm11
+        vpxor	32(%rdi), %ymm11, %ymm11
+        vpxor	96(%rdi), %ymm14, %ymm14
+        vpxor	128(%rdi), %ymm3, %ymm13
+        vpxor	-96(%rcx), %ymm12, %ymm12
+        vpxor	-32(%rcx), %ymm11, %ymm11
+        vpxor	(%rcx), %ymm10, %ymm10
+        vpxor	32(%rcx), %ymm14, %ymm14
+        vpxor	64(%rcx), %ymm13, %ymm13
+        vpxor	96(%rcx), %ymm13, %ymm13
+        vpxor	128(%rcx), %ymm12, %ymm12
+        vpxor	-96(%r8), %ymm11, %ymm11
+        vpxor	-64(%r8), %ymm10, %ymm10
+        vpxor	(%r8), %ymm14, %ymm14
+        vpxor	32(%r8), %ymm13, %ymm13
+        vpxor	64(%r8), %ymm12, %ymm12
+        vpxor	128(%r8), %ymm10, %ymm10
+        # Calc t[0..4]
+        vpsrlq	$63, %ymm11, %ymm0
+        vpsrlq	$63, %ymm12, %ymm1
+        vpsrlq	$63, %ymm13, %ymm2
+        vpsrlq	$63, %ymm14, %ymm3
+        vpsrlq	$63, %ymm10, %ymm4
+        vpaddq	%ymm11, %ymm11, %ymm5
+        vpaddq	%ymm12, %ymm12, %ymm6
+        vpaddq	%ymm13, %ymm13, %ymm7
+        vpaddq	%ymm14, %ymm14, %ymm8
+        vpaddq	%ymm10, %ymm10, %ymm9
+        vpor	%ymm0, %ymm5, %ymm5
+        vpor	%ymm1, %ymm6, %ymm6
+        vpor	%ymm2, %ymm7, %ymm7
+        vpor	%ymm3, %ymm8, %ymm8
+        vpor	%ymm4, %ymm9, %ymm9
+        vpxor	%ymm14, %ymm5, %ymm5
+        vpxor	%ymm10, %ymm6, %ymm6
+        vpxor	%ymm11, %ymm7, %ymm7
+        vpxor	%ymm12, %ymm8, %ymm8
+        vpxor	%ymm13, %ymm9, %ymm9
+        # Row Mix
+        # Row 0
+        vpxor	%ymm15, %ymm5, %ymm10
+        vpxor	-32(%rcx), %ymm6, %ymm11
+        vpxor	64(%r8), %ymm7, %ymm12
+        vpxor	128(%rdi), %ymm8, %ymm13
+        vpxor	-32(%r8), %ymm9, %ymm14
+        vpsrlq	$20, %ymm11, %ymm0
+        vpsrlq	$21, %ymm12, %ymm1
+        vpsrlq	$43, %ymm13, %ymm2
+        vpsrlq	$50, %ymm14, %ymm3
+        vpsllq	$44, %ymm11, %ymm11
+        vpsllq	$43, %ymm12, %ymm12
+        vpsllq	$21, %ymm13, %ymm13
+        vpsllq	$14, %ymm14, %ymm14
+        vpor	%ymm0, %ymm11, %ymm11
+        vpor	%ymm1, %ymm12, %ymm12
+        vpor	%ymm2, %ymm13, %ymm13
+        vpor	%ymm3, %ymm14, %ymm14
+        vpandn	%ymm12, %ymm11, %ymm15
+        vpandn	%ymm13, %ymm12, %ymm1
+        vpandn	%ymm14, %ymm13, %ymm2
+        vpandn	%ymm10, %ymm14, %ymm3
+        vpandn	%ymm11, %ymm10, %ymm4
+        vpxor	%ymm10, %ymm15, %ymm15
+        vpxor	%ymm11, %ymm1, %ymm1
+        vpxor	%ymm12, %ymm2, %ymm2
+        # XOR in constant
+        vpxor	640(%rax), %ymm15, %ymm15
+        vpxor	%ymm13, %ymm3, %ymm3
+        vpxor	%ymm14, %ymm4, %ymm4
+        vmovdqu	%ymm1, -32(%rcx)
+        vmovdqu	%ymm2, 64(%r8)
+        vmovdqu	%ymm3, 128(%rdi)
+        vmovdqu	%ymm4, -32(%r8)
+        # Row 1
+        vpxor	32(%r8), %ymm8, %ymm10
+        vpxor	96(%rdi), %ymm9, %ymm11
+        vpxor	-64(%r8), %ymm5, %ymm12
+        vpxor	(%rdi), %ymm6, %ymm13
+        vpxor	-64(%rcx), %ymm7, %ymm14
+        vpsrlq	$36, %ymm10, %ymm0
+        vpsrlq	$44, %ymm11, %ymm1
+        vpsrlq	$61, %ymm12, %ymm2
+        vpsrlq	$19, %ymm13, %ymm3
+        vpsrlq	$3, %ymm14, %ymm4
+        vpsllq	$28, %ymm10, %ymm10
+        vpsllq	$20, %ymm11, %ymm11
+        vpsllq	$3, %ymm12, %ymm12
+        vpsllq	$45, %ymm13, %ymm13
+        vpsllq	$61, %ymm14, %ymm14
+        vpor	%ymm0, %ymm10, %ymm10
+        vpor	%ymm1, %ymm11, %ymm11
+        vpor	%ymm2, %ymm12, %ymm12
+        vpor	%ymm3, %ymm13, %ymm13
+        vpor	%ymm4, %ymm14, %ymm14
+        vpandn	%ymm12, %ymm11, %ymm0
+        vpandn	%ymm13, %ymm12, %ymm1
+        vpandn	%ymm14, %ymm13, %ymm2
+        vpandn	%ymm10, %ymm14, %ymm3
+        vpandn	%ymm11, %ymm10, %ymm4
+        vpxor	%ymm10, %ymm0, %ymm0
+        vpxor	%ymm11, %ymm1, %ymm1
+        vpxor	%ymm12, %ymm2, %ymm2
+        vpxor	%ymm13, %ymm3, %ymm3
+        vpxor	%ymm14, %ymm4, %ymm4
+        vmovdqu	%ymm0, 32(%r8)
+        vmovdqu	%ymm1, 96(%rdi)
+        vmovdqu	%ymm2, -64(%r8)
+        vmovdqu	%ymm3, (%rdi)
+        vmovdqu	%ymm4, -64(%rcx)
+        # Row 2
+        vpxor	-96(%r8), %ymm6, %ymm10
+        vpxor	-32(%rdi), %ymm7, %ymm11
+        vpxor	64(%rcx), %ymm8, %ymm12
+        vpxor	(%r8), %ymm9, %ymm13
+        vpxor	64(%rdi), %ymm5, %ymm14
+        vpsrlq	$63, %ymm10, %ymm0
+        vpsrlq	$58, %ymm11, %ymm1
+        vpsrlq	$39, %ymm12, %ymm2
+        vpsrlq	$56, %ymm13, %ymm3
+        vpsrlq	$46, %ymm14, %ymm4
+        vpaddq	%ymm10, %ymm10, %ymm10
+        vpsllq	$6, %ymm11, %ymm11
+        vpsllq	$25, %ymm12, %ymm12
+        vpsllq	$8, %ymm13, %ymm13
+        vpsllq	$18, %ymm14, %ymm14
+        vpor	%ymm0, %ymm10, %ymm10
+        vpor	%ymm1, %ymm11, %ymm11
+        vpor	%ymm2, %ymm12, %ymm12
+        vpor	%ymm3, %ymm13, %ymm13
+        vpor	%ymm4, %ymm14, %ymm14
+        vpandn	%ymm12, %ymm11, %ymm0
+        vpandn	%ymm13, %ymm12, %ymm1
+        vpandn	%ymm14, %ymm13, %ymm2
+        vpandn	%ymm10, %ymm14, %ymm3
+        vpandn	%ymm11, %ymm10, %ymm4
+        vpxor	%ymm10, %ymm0, %ymm0
+        vpxor	%ymm11, %ymm1, %ymm1
+        vpxor	%ymm12, %ymm2, %ymm2
+        vpxor	%ymm13, %ymm3, %ymm3
+        vpxor	%ymm14, %ymm4, %ymm4
+        vmovdqu	%ymm0, -96(%r8)
+        vmovdqu	%ymm1, -32(%rdi)
+        vmovdqu	%ymm2, 64(%rcx)
+        vmovdqu	%ymm3, (%r8)
+        vmovdqu	%ymm4, 64(%rdi)
+        # Row 3
+        vpxor	32(%rcx), %ymm9, %ymm10
+        vpxor	128(%r8), %ymm5, %ymm11
+        vpxor	32(%rdi), %ymm6, %ymm12
+        vpxor	128(%rcx), %ymm7, %ymm13
+        vpxor	-64(%rdi), %ymm8, %ymm14
+        vpsrlq	$37, %ymm10, %ymm0
+        vpsrlq	$28, %ymm11, %ymm1
+        vpsrlq	$54, %ymm12, %ymm2
+        vpsrlq	$49, %ymm13, %ymm3
+        vpsrlq	$8, %ymm14, %ymm4
+        vpsllq	$27, %ymm10, %ymm10
+        vpsllq	$36, %ymm11, %ymm11
+        vpsllq	$10, %ymm12, %ymm12
+        vpsllq	$15, %ymm13, %ymm13
+        vpsllq	$56, %ymm14, %ymm14
+        vpor	%ymm0, %ymm10, %ymm10
+        vpor	%ymm1, %ymm11, %ymm11
+        vpor	%ymm2, %ymm12, %ymm12
+        vpor	%ymm3, %ymm13, %ymm13
+        vpor	%ymm4, %ymm14, %ymm14
+        vpandn	%ymm12, %ymm11, %ymm0
+        vpandn	%ymm13, %ymm12, %ymm1
+        vpandn	%ymm14, %ymm13, %ymm2
+        vpandn	%ymm10, %ymm14, %ymm3
+        vpandn	%ymm11, %ymm10, %ymm4
+        vpxor	%ymm10, %ymm0, %ymm0
+        vpxor	%ymm11, %ymm1, %ymm1
+        vpxor	%ymm12, %ymm2, %ymm2
+        vpxor	%ymm13, %ymm3, %ymm3
+        vpxor	%ymm14, %ymm4, %ymm4
+        vmovdqu	%ymm0, 32(%rcx)
+        vmovdqu	%ymm1, 128(%r8)
+        vmovdqu	%ymm2, 32(%rdi)
+        vmovdqu	%ymm3, 128(%rcx)
+        vmovdqu	%ymm4, -64(%rdi)
+        # Row 4
+        vpxor	-96(%rcx), %ymm7, %ymm10
+        vpxor	96(%rcx), %ymm8, %ymm11
+        vpxor	-96(%rdi), %ymm9, %ymm12
+        vpxor	(%rcx), %ymm5, %ymm13
+        vpxor	96(%r8), %ymm6, %ymm14
+        vpsrlq	$2, %ymm10, %ymm0
+        vpsrlq	$9, %ymm11, %ymm1
+        vpsrlq	$25, %ymm12, %ymm2
+        vpsrlq	$23, %ymm13, %ymm3
+        vpsrlq	$62, %ymm14, %ymm4
+        vpsllq	$62, %ymm10, %ymm10
+        vpsllq	$55, %ymm11, %ymm11
+        vpsllq	$39, %ymm12, %ymm12
+        vpsllq	$41, %ymm13, %ymm13
+        vpsllq	$2, %ymm14, %ymm14
+        vpor	%ymm0, %ymm10, %ymm10
+        vpor	%ymm1, %ymm11, %ymm11
+        vpor	%ymm2, %ymm12, %ymm12
+        vpor	%ymm3, %ymm13, %ymm13
+        vpor	%ymm4, %ymm14, %ymm14
+        vpandn	%ymm12, %ymm11, %ymm0
+        vpandn	%ymm13, %ymm12, %ymm1
+        vpandn	%ymm14, %ymm13, %ymm2
+        vpandn	%ymm10, %ymm14, %ymm3
+        vpandn	%ymm11, %ymm10, %ymm4
+        vpxor	%ymm10, %ymm0, %ymm0
+        vpxor	%ymm11, %ymm1, %ymm1
+        vpxor	%ymm12, %ymm2, %ymm2
+        vpxor	%ymm13, %ymm3, %ymm3
+        vpxor	%ymm14, %ymm4, %ymm4
+        vmovdqu	%ymm0, -96(%rcx)
+        vmovdqu	%ymm1, 96(%rcx)
+        vmovdqu	%ymm2, -96(%rdi)
+        vmovdqu	%ymm3, (%rcx)
+        vmovdqu	%ymm4, 96(%r8)
+        # Round 21
+        # Calc b[0..4]
+        vpxor	%ymm15, %ymm0, %ymm10
+        vpxor	-64(%rdi), %ymm4, %ymm14
+        vpxor	-32(%rdi), %ymm1, %ymm11
+        vpxor	(%rdi), %ymm3, %ymm13
+        vpxor	32(%rdi), %ymm2, %ymm12
+        vpxor	64(%rdi), %ymm14, %ymm14
+        vpxor	96(%rdi), %ymm11, %ymm11
+        vpxor	128(%rdi), %ymm13, %ymm13
+        vpxor	-64(%rcx), %ymm14, %ymm14
+        vpxor	-32(%rcx), %ymm11, %ymm11
+        vpxor	32(%rcx), %ymm10, %ymm10
+        vpxor	64(%rcx), %ymm12, %ymm12
+        vpxor	128(%rcx), %ymm13, %ymm13
+        vpxor	-96(%r8), %ymm10, %ymm10
+        vpxor	-64(%r8), %ymm12, %ymm12
+        vpxor	-32(%r8), %ymm14, %ymm14
+        vpxor	(%r8), %ymm13, %ymm13
+        vpxor	32(%r8), %ymm10, %ymm10
+        vpxor	64(%r8), %ymm12, %ymm12
+        vpxor	128(%r8), %ymm11, %ymm11
+        # Calc t[0..4]
+        vpsrlq	$63, %ymm11, %ymm0
+        vpsrlq	$63, %ymm12, %ymm1
+        vpsrlq	$63, %ymm13, %ymm2
+        vpsrlq	$63, %ymm14, %ymm3
+        vpsrlq	$63, %ymm10, %ymm4
+        vpaddq	%ymm11, %ymm11, %ymm5
+        vpaddq	%ymm12, %ymm12, %ymm6
+        vpaddq	%ymm13, %ymm13, %ymm7
+        vpaddq	%ymm14, %ymm14, %ymm8
+        vpaddq	%ymm10, %ymm10, %ymm9
+        vpor	%ymm0, %ymm5, %ymm5
+        vpor	%ymm1, %ymm6, %ymm6
+        vpor	%ymm2, %ymm7, %ymm7
+        vpor	%ymm3, %ymm8, %ymm8
+        vpor	%ymm4, %ymm9, %ymm9
+        vpxor	%ymm14, %ymm5, %ymm5
+        vpxor	%ymm10, %ymm6, %ymm6
+        vpxor	%ymm11, %ymm7, %ymm7
+        vpxor	%ymm12, %ymm8, %ymm8
+        vpxor	%ymm13, %ymm9, %ymm9
+        # Row Mix
+        # Row 0
+        vpxor	%ymm15, %ymm5, %ymm10
+        vpxor	96(%rdi), %ymm6, %ymm11
+        vpxor	64(%rcx), %ymm7, %ymm12
+        vpxor	128(%rcx), %ymm8, %ymm13
+        vpxor	96(%r8), %ymm9, %ymm14
+        vpsrlq	$20, %ymm11, %ymm0
+        vpsrlq	$21, %ymm12, %ymm1
+        vpsrlq	$43, %ymm13, %ymm2
+        vpsrlq	$50, %ymm14, %ymm3
+        vpsllq	$44, %ymm11, %ymm11
+        vpsllq	$43, %ymm12, %ymm12
+        vpsllq	$21, %ymm13, %ymm13
+        vpsllq	$14, %ymm14, %ymm14
+        vpor	%ymm0, %ymm11, %ymm11
+        vpor	%ymm1, %ymm12, %ymm12
+        vpor	%ymm2, %ymm13, %ymm13
+        vpor	%ymm3, %ymm14, %ymm14
+        vpandn	%ymm12, %ymm11, %ymm15
+        vpandn	%ymm13, %ymm12, %ymm1
+        vpandn	%ymm14, %ymm13, %ymm2
+        vpandn	%ymm10, %ymm14, %ymm3
+        vpandn	%ymm11, %ymm10, %ymm4
+        vpxor	%ymm10, %ymm15, %ymm15
+        vpxor	%ymm11, %ymm1, %ymm1
+        vpxor	%ymm12, %ymm2, %ymm2
+        # XOR in constant
+        vpxor	672(%rax), %ymm15, %ymm15
+        vpxor	%ymm13, %ymm3, %ymm3
+        vpxor	%ymm14, %ymm4, %ymm4
+        vmovdqu	%ymm1, 96(%rdi)
+        vmovdqu	%ymm2, 64(%rcx)
+        vmovdqu	%ymm3, 128(%rcx)
+        vmovdqu	%ymm4, 96(%r8)
+        # Row 1
+        vpxor	128(%rdi), %ymm8, %ymm10
+        vpxor	-64(%rcx), %ymm9, %ymm11
+        vpxor	-96(%r8), %ymm5, %ymm12
+        vpxor	128(%r8), %ymm6, %ymm13
+        vpxor	-96(%rdi), %ymm7, %ymm14
+        vpsrlq	$36, %ymm10, %ymm0
+        vpsrlq	$44, %ymm11, %ymm1
+        vpsrlq	$61, %ymm12, %ymm2
+        vpsrlq	$19, %ymm13, %ymm3
+        vpsrlq	$3, %ymm14, %ymm4
+        vpsllq	$28, %ymm10, %ymm10
+        vpsllq	$20, %ymm11, %ymm11
+        vpsllq	$3, %ymm12, %ymm12
+        vpsllq	$45, %ymm13, %ymm13
+        vpsllq	$61, %ymm14, %ymm14
+        vpor	%ymm0, %ymm10, %ymm10
+        vpor	%ymm1, %ymm11, %ymm11
+        vpor	%ymm2, %ymm12, %ymm12
+        vpor	%ymm3, %ymm13, %ymm13
+        vpor	%ymm4, %ymm14, %ymm14
+        vpandn	%ymm12, %ymm11, %ymm0
+        vpandn	%ymm13, %ymm12, %ymm1
+        vpandn	%ymm14, %ymm13, %ymm2
+        vpandn	%ymm10, %ymm14, %ymm3
+        vpandn	%ymm11, %ymm10, %ymm4
+        vpxor	%ymm10, %ymm0, %ymm0
+        vpxor	%ymm11, %ymm1, %ymm1
+        vpxor	%ymm12, %ymm2, %ymm2
+        vpxor	%ymm13, %ymm3, %ymm3
+        vpxor	%ymm14, %ymm4, %ymm4
+        vmovdqu	%ymm0, 128(%rdi)
+        vmovdqu	%ymm1, -64(%rcx)
+        vmovdqu	%ymm2, -96(%r8)
+        vmovdqu	%ymm3, 128(%r8)
+        vmovdqu	%ymm4, -96(%rdi)
+        # Row 2
+        vpxor	-32(%rcx), %ymm6, %ymm10
+        vpxor	-64(%r8), %ymm7, %ymm11
+        vpxor	(%r8), %ymm8, %ymm12
+        vpxor	-64(%rdi), %ymm9, %ymm13
+        vpxor	-96(%rcx), %ymm5, %ymm14
+        vpsrlq	$63, %ymm10, %ymm0
+        vpsrlq	$58, %ymm11, %ymm1
+        vpsrlq	$39, %ymm12, %ymm2
+        vpsrlq	$56, %ymm13, %ymm3
+        vpsrlq	$46, %ymm14, %ymm4
+        vpaddq	%ymm10, %ymm10, %ymm10
+        vpsllq	$6, %ymm11, %ymm11
+        vpsllq	$25, %ymm12, %ymm12
+        vpsllq	$8, %ymm13, %ymm13
+        vpsllq	$18, %ymm14, %ymm14
+        vpor	%ymm0, %ymm10, %ymm10
+        vpor	%ymm1, %ymm11, %ymm11
+        vpor	%ymm2, %ymm12, %ymm12
+        vpor	%ymm3, %ymm13, %ymm13
+        vpor	%ymm4, %ymm14, %ymm14
+        vpandn	%ymm12, %ymm11, %ymm0
+        vpandn	%ymm13, %ymm12, %ymm1
+        vpandn	%ymm14, %ymm13, %ymm2
+        vpandn	%ymm10, %ymm14, %ymm3
+        vpandn	%ymm11, %ymm10, %ymm4
+        vpxor	%ymm10, %ymm0, %ymm0
+        vpxor	%ymm11, %ymm1, %ymm1
+        vpxor	%ymm12, %ymm2, %ymm2
+        vpxor	%ymm13, %ymm3, %ymm3
+        vpxor	%ymm14, %ymm4, %ymm4
+        vmovdqu	%ymm0, -32(%rcx)
+        vmovdqu	%ymm1, -64(%r8)
+        vmovdqu	%ymm2, (%r8)
+        vmovdqu	%ymm3, -64(%rdi)
+        vmovdqu	%ymm4, -96(%rcx)
+        # Row 3
+        vpxor	-32(%r8), %ymm9, %ymm10
+        vpxor	32(%r8), %ymm5, %ymm11
+        vpxor	-32(%rdi), %ymm6, %ymm12
+        vpxor	32(%rdi), %ymm7, %ymm13
+        vpxor	(%rcx), %ymm8, %ymm14
+        vpsrlq	$37, %ymm10, %ymm0
+        vpsrlq	$28, %ymm11, %ymm1
+        vpsrlq	$54, %ymm12, %ymm2
+        vpsrlq	$49, %ymm13, %ymm3
+        vpsrlq	$8, %ymm14, %ymm4
+        vpsllq	$27, %ymm10, %ymm10
+        vpsllq	$36, %ymm11, %ymm11
+        vpsllq	$10, %ymm12, %ymm12
+        vpsllq	$15, %ymm13, %ymm13
+        vpsllq	$56, %ymm14, %ymm14
+        vpor	%ymm0, %ymm10, %ymm10
+        vpor	%ymm1, %ymm11, %ymm11
+        vpor	%ymm2, %ymm12, %ymm12
+        vpor	%ymm3, %ymm13, %ymm13
+        vpor	%ymm4, %ymm14, %ymm14
+        vpandn	%ymm12, %ymm11, %ymm0
+        vpandn	%ymm13, %ymm12, %ymm1
+        vpandn	%ymm14, %ymm13, %ymm2
+        vpandn	%ymm10, %ymm14, %ymm3
+        vpandn	%ymm11, %ymm10, %ymm4
+        vpxor	%ymm10, %ymm0, %ymm0
+        vpxor	%ymm11, %ymm1, %ymm1
+        vpxor	%ymm12, %ymm2, %ymm2
+        vpxor	%ymm13, %ymm3, %ymm3
+        vpxor	%ymm14, %ymm4, %ymm4
+        vmovdqu	%ymm0, -32(%r8)
+        vmovdqu	%ymm1, 32(%r8)
+        vmovdqu	%ymm2, -32(%rdi)
+        vmovdqu	%ymm3, 32(%rdi)
+        vmovdqu	%ymm4, (%rcx)
+        # Row 4
+        vpxor	64(%r8), %ymm7, %ymm10
+        vpxor	(%rdi), %ymm8, %ymm11
+        vpxor	64(%rdi), %ymm9, %ymm12
+        vpxor	32(%rcx), %ymm5, %ymm13
+        vpxor	96(%rcx), %ymm6, %ymm14
+        vpsrlq	$2, %ymm10, %ymm0
+        vpsrlq	$9, %ymm11, %ymm1
+        vpsrlq	$25, %ymm12, %ymm2
+        vpsrlq	$23, %ymm13, %ymm3
+        vpsrlq	$62, %ymm14, %ymm4
+        vpsllq	$62, %ymm10, %ymm10
+        vpsllq	$55, %ymm11, %ymm11
+        vpsllq	$39, %ymm12, %ymm12
+        vpsllq	$41, %ymm13, %ymm13
+        vpsllq	$2, %ymm14, %ymm14
+        vpor	%ymm0, %ymm10, %ymm10
+        vpor	%ymm1, %ymm11, %ymm11
+        vpor	%ymm2, %ymm12, %ymm12
+        vpor	%ymm3, %ymm13, %ymm13
+        vpor	%ymm4, %ymm14, %ymm14
+        vpandn	%ymm12, %ymm11, %ymm0
+        vpandn	%ymm13, %ymm12, %ymm1
+        vpandn	%ymm14, %ymm13, %ymm2
+        vpandn	%ymm10, %ymm14, %ymm3
+        vpandn	%ymm11, %ymm10, %ymm4
+        vpxor	%ymm10, %ymm0, %ymm0
+        vpxor	%ymm11, %ymm1, %ymm1
+        vpxor	%ymm12, %ymm2, %ymm2
+        vpxor	%ymm13, %ymm3, %ymm3
+        vpxor	%ymm14, %ymm4, %ymm4
+        vmovdqu	%ymm0, 64(%r8)
+        vmovdqu	%ymm1, (%rdi)
+        vmovdqu	%ymm2, 64(%rdi)
+        vmovdqu	%ymm3, 32(%rcx)
+        vmovdqu	%ymm4, 96(%rcx)
+        # Round 22
+        # Calc b[0..4]
+        vpxor	%ymm15, %ymm0, %ymm10
+        vpxor	-96(%rdi), %ymm4, %ymm14
+        vpxor	-64(%rdi), %ymm3, %ymm13
+        vpxor	-32(%rdi), %ymm2, %ymm12
+        vpxor	32(%rdi), %ymm13, %ymm13
+        vpxor	96(%rdi), %ymm1, %ymm11
+        vpxor	128(%rdi), %ymm10, %ymm10
+        vpxor	-96(%rcx), %ymm14, %ymm14
+        vpxor	-64(%rcx), %ymm11, %ymm11
+        vpxor	-32(%rcx), %ymm10, %ymm10
+        vpxor	(%rcx), %ymm14, %ymm14
+        vpxor	64(%rcx), %ymm12, %ymm12
+        vpxor	128(%rcx), %ymm13, %ymm13
+        vpxor	-96(%r8), %ymm12, %ymm12
+        vpxor	-64(%r8), %ymm11, %ymm11
+        vpxor	-32(%r8), %ymm10, %ymm10
+        vpxor	(%r8), %ymm12, %ymm12
+        vpxor	32(%r8), %ymm11, %ymm11
+        vpxor	96(%r8), %ymm14, %ymm14
+        vpxor	128(%r8), %ymm13, %ymm13
+        # Calc t[0..4]
+        vpsrlq	$63, %ymm11, %ymm0
+        vpsrlq	$63, %ymm12, %ymm1
+        vpsrlq	$63, %ymm13, %ymm2
+        vpsrlq	$63, %ymm14, %ymm3
+        vpsrlq	$63, %ymm10, %ymm4
+        vpaddq	%ymm11, %ymm11, %ymm5
+        vpaddq	%ymm12, %ymm12, %ymm6
+        vpaddq	%ymm13, %ymm13, %ymm7
+        vpaddq	%ymm14, %ymm14, %ymm8
+        vpaddq	%ymm10, %ymm10, %ymm9
+        vpor	%ymm0, %ymm5, %ymm5
+        vpor	%ymm1, %ymm6, %ymm6
+        vpor	%ymm2, %ymm7, %ymm7
+        vpor	%ymm3, %ymm8, %ymm8
+        vpor	%ymm4, %ymm9, %ymm9
+        vpxor	%ymm14, %ymm5, %ymm5
+        vpxor	%ymm10, %ymm6, %ymm6
+        vpxor	%ymm11, %ymm7, %ymm7
+        vpxor	%ymm12, %ymm8, %ymm8
+        vpxor	%ymm13, %ymm9, %ymm9
+        # Row Mix
+        # Row 0
+        vpxor	%ymm15, %ymm5, %ymm10
+        vpxor	-64(%rcx), %ymm6, %ymm11
+        vpxor	(%r8), %ymm7, %ymm12
+        vpxor	32(%rdi), %ymm8, %ymm13
+        vpxor	96(%rcx), %ymm9, %ymm14
+        vpsrlq	$20, %ymm11, %ymm0
+        vpsrlq	$21, %ymm12, %ymm1
+        vpsrlq	$43, %ymm13, %ymm2
+        vpsrlq	$50, %ymm14, %ymm3
+        vpsllq	$44, %ymm11, %ymm11
+        vpsllq	$43, %ymm12, %ymm12
+        vpsllq	$21, %ymm13, %ymm13
+        vpsllq	$14, %ymm14, %ymm14
+        vpor	%ymm0, %ymm11, %ymm11
+        vpor	%ymm1, %ymm12, %ymm12
+        vpor	%ymm2, %ymm13, %ymm13
+        vpor	%ymm3, %ymm14, %ymm14
+        vpandn	%ymm12, %ymm11, %ymm15
+        vpandn	%ymm13, %ymm12, %ymm1
+        vpandn	%ymm14, %ymm13, %ymm2
+        vpandn	%ymm10, %ymm14, %ymm3
+        vpandn	%ymm11, %ymm10, %ymm4
+        vpxor	%ymm10, %ymm15, %ymm15
+        vpxor	%ymm11, %ymm1, %ymm1
+        vpxor	%ymm12, %ymm2, %ymm2
+        # XOR in constant
+        vpxor	704(%rax), %ymm15, %ymm15
+        vpxor	%ymm13, %ymm3, %ymm3
+        vpxor	%ymm14, %ymm4, %ymm4
+        vmovdqu	%ymm1, -64(%rcx)
+        vmovdqu	%ymm2, (%r8)
+        vmovdqu	%ymm3, 32(%rdi)
+        vmovdqu	%ymm4, 96(%rcx)
+        # Row 1
+        vpxor	128(%rcx), %ymm8, %ymm10
+        vpxor	-96(%rdi), %ymm9, %ymm11
+        vpxor	-32(%rcx), %ymm5, %ymm12
+        vpxor	32(%r8), %ymm6, %ymm13
+        vpxor	64(%rdi), %ymm7, %ymm14
+        vpsrlq	$36, %ymm10, %ymm0
+        vpsrlq	$44, %ymm11, %ymm1
+        vpsrlq	$61, %ymm12, %ymm2
+        vpsrlq	$19, %ymm13, %ymm3
+        vpsrlq	$3, %ymm14, %ymm4
+        vpsllq	$28, %ymm10, %ymm10
+        vpsllq	$20, %ymm11, %ymm11
+        vpsllq	$3, %ymm12, %ymm12
+        vpsllq	$45, %ymm13, %ymm13
+        vpsllq	$61, %ymm14, %ymm14
+        vpor	%ymm0, %ymm10, %ymm10
+        vpor	%ymm1, %ymm11, %ymm11
+        vpor	%ymm2, %ymm12, %ymm12
+        vpor	%ymm3, %ymm13, %ymm13
+        vpor	%ymm4, %ymm14, %ymm14
+        vpandn	%ymm12, %ymm11, %ymm0
+        vpandn	%ymm13, %ymm12, %ymm1
+        vpandn	%ymm14, %ymm13, %ymm2
+        vpandn	%ymm10, %ymm14, %ymm3
+        vpandn	%ymm11, %ymm10, %ymm4
+        vpxor	%ymm10, %ymm0, %ymm0
+        vpxor	%ymm11, %ymm1, %ymm1
+        vpxor	%ymm12, %ymm2, %ymm2
+        vpxor	%ymm13, %ymm3, %ymm3
+        vpxor	%ymm14, %ymm4, %ymm4
+        vmovdqu	%ymm0, 128(%rcx)
+        vmovdqu	%ymm1, -96(%rdi)
+        vmovdqu	%ymm2, -32(%rcx)
+        vmovdqu	%ymm3, 32(%r8)
+        vmovdqu	%ymm4, 64(%rdi)
+        # Row 2
+        vpxor	96(%rdi), %ymm6, %ymm10
+        vpxor	-96(%r8), %ymm7, %ymm11
+        vpxor	-64(%rdi), %ymm8, %ymm12
+        vpxor	(%rcx), %ymm9, %ymm13
+        vpxor	64(%r8), %ymm5, %ymm14
+        vpsrlq	$63, %ymm10, %ymm0
+        vpsrlq	$58, %ymm11, %ymm1
+        vpsrlq	$39, %ymm12, %ymm2
+        vpsrlq	$56, %ymm13, %ymm3
+        vpsrlq	$46, %ymm14, %ymm4
+        vpaddq	%ymm10, %ymm10, %ymm10
+        vpsllq	$6, %ymm11, %ymm11
+        vpsllq	$25, %ymm12, %ymm12
+        vpsllq	$8, %ymm13, %ymm13
+        vpsllq	$18, %ymm14, %ymm14
+        vpor	%ymm0, %ymm10, %ymm10
+        vpor	%ymm1, %ymm11, %ymm11
+        vpor	%ymm2, %ymm12, %ymm12
+        vpor	%ymm3, %ymm13, %ymm13
+        vpor	%ymm4, %ymm14, %ymm14
+        vpandn	%ymm12, %ymm11, %ymm0
+        vpandn	%ymm13, %ymm12, %ymm1
+        vpandn	%ymm14, %ymm13, %ymm2
+        vpandn	%ymm10, %ymm14, %ymm3
+        vpandn	%ymm11, %ymm10, %ymm4
+        vpxor	%ymm10, %ymm0, %ymm0
+        vpxor	%ymm11, %ymm1, %ymm1
+        vpxor	%ymm12, %ymm2, %ymm2
+        vpxor	%ymm13, %ymm3, %ymm3
+        vpxor	%ymm14, %ymm4, %ymm4
+        vmovdqu	%ymm0, 96(%rdi)
+        vmovdqu	%ymm1, -96(%r8)
+        vmovdqu	%ymm2, -64(%rdi)
+        vmovdqu	%ymm3, (%rcx)
+        vmovdqu	%ymm4, 64(%r8)
+        # Row 3
+        vpxor	96(%r8), %ymm9, %ymm10
+        vpxor	128(%rdi), %ymm5, %ymm11
+        vpxor	-64(%r8), %ymm6, %ymm12
+        vpxor	-32(%rdi), %ymm7, %ymm13
+        vpxor	32(%rcx), %ymm8, %ymm14
+        vpsrlq	$37, %ymm10, %ymm0
+        vpsrlq	$28, %ymm11, %ymm1
+        vpsrlq	$54, %ymm12, %ymm2
+        vpsrlq	$49, %ymm13, %ymm3
+        vpsrlq	$8, %ymm14, %ymm4
+        vpsllq	$27, %ymm10, %ymm10
+        vpsllq	$36, %ymm11, %ymm11
+        vpsllq	$10, %ymm12, %ymm12
+        vpsllq	$15, %ymm13, %ymm13
+        vpsllq	$56, %ymm14, %ymm14
+        vpor	%ymm0, %ymm10, %ymm10
+        vpor	%ymm1, %ymm11, %ymm11
+        vpor	%ymm2, %ymm12, %ymm12
+        vpor	%ymm3, %ymm13, %ymm13
+        vpor	%ymm4, %ymm14, %ymm14
+        vpandn	%ymm12, %ymm11, %ymm0
+        vpandn	%ymm13, %ymm12, %ymm1
+        vpandn	%ymm14, %ymm13, %ymm2
+        vpandn	%ymm10, %ymm14, %ymm3
+        vpandn	%ymm11, %ymm10, %ymm4
+        vpxor	%ymm10, %ymm0, %ymm0
+        vpxor	%ymm11, %ymm1, %ymm1
+        vpxor	%ymm12, %ymm2, %ymm2
+        vpxor	%ymm13, %ymm3, %ymm3
+        vpxor	%ymm14, %ymm4, %ymm4
+        vmovdqu	%ymm0, 96(%r8)
+        vmovdqu	%ymm1, 128(%rdi)
+        vmovdqu	%ymm2, -64(%r8)
+        vmovdqu	%ymm3, -32(%rdi)
+        vmovdqu	%ymm4, 32(%rcx)
+        # Row 4
+        vpxor	64(%rcx), %ymm7, %ymm10
+        vpxor	128(%r8), %ymm8, %ymm11
+        vpxor	-96(%rcx), %ymm9, %ymm12
+        vpxor	-32(%r8), %ymm5, %ymm13
+        vpxor	(%rdi), %ymm6, %ymm14
+        vpsrlq	$2, %ymm10, %ymm0
+        vpsrlq	$9, %ymm11, %ymm1
+        vpsrlq	$25, %ymm12, %ymm2
+        vpsrlq	$23, %ymm13, %ymm3
+        vpsrlq	$62, %ymm14, %ymm4
+        vpsllq	$62, %ymm10, %ymm10
+        vpsllq	$55, %ymm11, %ymm11
+        vpsllq	$39, %ymm12, %ymm12
+        vpsllq	$41, %ymm13, %ymm13
+        vpsllq	$2, %ymm14, %ymm14
+        vpor	%ymm0, %ymm10, %ymm10
+        vpor	%ymm1, %ymm11, %ymm11
+        vpor	%ymm2, %ymm12, %ymm12
+        vpor	%ymm3, %ymm13, %ymm13
+        vpor	%ymm4, %ymm14, %ymm14
+        vpandn	%ymm12, %ymm11, %ymm0
+        vpandn	%ymm13, %ymm12, %ymm1
+        vpandn	%ymm14, %ymm13, %ymm2
+        vpandn	%ymm10, %ymm14, %ymm3
+        vpandn	%ymm11, %ymm10, %ymm4
+        vpxor	%ymm10, %ymm0, %ymm0
+        vpxor	%ymm11, %ymm1, %ymm1
+        vpxor	%ymm12, %ymm2, %ymm2
+        vpxor	%ymm13, %ymm3, %ymm3
+        vpxor	%ymm14, %ymm4, %ymm4
+        vmovdqu	%ymm0, 64(%rcx)
+        vmovdqu	%ymm1, 128(%r8)
+        vmovdqu	%ymm2, -96(%rcx)
+        vmovdqu	%ymm3, -32(%r8)
+        vmovdqu	%ymm4, (%rdi)
+        # Round 23
+        # Calc b[0..4]
+        vpxor	%ymm15, %ymm0, %ymm10
+        vpxor	-96(%rdi), %ymm1, %ymm11
+        vpxor	-64(%rdi), %ymm2, %ymm12
+        vpxor	-32(%rdi), %ymm3, %ymm13
+        vpxor	32(%rdi), %ymm13, %ymm13
+        vpxor	64(%rdi), %ymm4, %ymm14
+        vpxor	96(%rdi), %ymm10, %ymm10
+        vpxor	128(%rdi), %ymm11, %ymm11
+        vpxor	-64(%rcx), %ymm11, %ymm11
+        vpxor	-32(%rcx), %ymm12, %ymm12
+        vpxor	(%rcx), %ymm13, %ymm13
+        vpxor	32(%rcx), %ymm14, %ymm14
+        vpxor	96(%rcx), %ymm14, %ymm14
+        vpxor	128(%rcx), %ymm10, %ymm10
+        vpxor	-96(%r8), %ymm11, %ymm11
+        vpxor	-64(%r8), %ymm12, %ymm12
+        vpxor	(%r8), %ymm12, %ymm12
+        vpxor	32(%r8), %ymm13, %ymm13
+        vpxor	64(%r8), %ymm14, %ymm14
+        vpxor	96(%r8), %ymm10, %ymm10
+        # Calc t[0..4]
+        vpsrlq	$63, %ymm11, %ymm0
+        vpsrlq	$63, %ymm12, %ymm1
+        vpsrlq	$63, %ymm13, %ymm2
+        vpsrlq	$63, %ymm14, %ymm3
+        vpsrlq	$63, %ymm10, %ymm4
+        vpaddq	%ymm11, %ymm11, %ymm5
+        vpaddq	%ymm12, %ymm12, %ymm6
+        vpaddq	%ymm13, %ymm13, %ymm7
+        vpaddq	%ymm14, %ymm14, %ymm8
+        vpaddq	%ymm10, %ymm10, %ymm9
+        vpor	%ymm0, %ymm5, %ymm5
+        vpor	%ymm1, %ymm6, %ymm6
+        vpor	%ymm2, %ymm7, %ymm7
+        vpor	%ymm3, %ymm8, %ymm8
+        vpor	%ymm4, %ymm9, %ymm9
+        vpxor	%ymm14, %ymm5, %ymm5
+        vpxor	%ymm10, %ymm6, %ymm6
+        vpxor	%ymm11, %ymm7, %ymm7
+        vpxor	%ymm12, %ymm8, %ymm8
+        vpxor	%ymm13, %ymm9, %ymm9
+        # Row Mix
+        # Row 0
+        vpxor	%ymm15, %ymm5, %ymm10
+        vpxor	-96(%rdi), %ymm6, %ymm11
+        vpxor	-64(%rdi), %ymm7, %ymm12
+        vpxor	-32(%rdi), %ymm8, %ymm13
+        vpxor	(%rdi), %ymm9, %ymm14
+        vpsrlq	$20, %ymm11, %ymm0
+        vpsrlq	$21, %ymm12, %ymm1
+        vpsrlq	$43, %ymm13, %ymm2
+        vpsrlq	$50, %ymm14, %ymm3
+        vpsllq	$44, %ymm11, %ymm11
+        vpsllq	$43, %ymm12, %ymm12
+        vpsllq	$21, %ymm13, %ymm13
+        vpsllq	$14, %ymm14, %ymm14
+        vpor	%ymm0, %ymm11, %ymm11
+        vpor	%ymm1, %ymm12, %ymm12
+        vpor	%ymm2, %ymm13, %ymm13
+        vpor	%ymm3, %ymm14, %ymm14
+        vpandn	%ymm12, %ymm11, %ymm15
+        vpandn	%ymm13, %ymm12, %ymm1
+        vpandn	%ymm14, %ymm13, %ymm2
+        vpandn	%ymm10, %ymm14, %ymm3
+        vpandn	%ymm11, %ymm10, %ymm4
+        vpxor	%ymm10, %ymm15, %ymm15
+        vpxor	%ymm11, %ymm1, %ymm1
+        vpxor	%ymm12, %ymm2, %ymm2
+        # XOR in constant
+        vpxor	736(%rax), %ymm15, %ymm15
+        vpxor	%ymm13, %ymm3, %ymm3
+        vpxor	%ymm14, %ymm4, %ymm4
+        vmovdqu	%ymm1, -96(%rdi)
+        vmovdqu	%ymm2, -64(%rdi)
+        vmovdqu	%ymm3, -32(%rdi)
+        vmovdqu	%ymm4, (%rdi)
+        # Row 1
+        vpxor	32(%rdi), %ymm8, %ymm10
+        vpxor	64(%rdi), %ymm9, %ymm11
+        vpxor	96(%rdi), %ymm5, %ymm12
+        vpxor	128(%rdi), %ymm6, %ymm13
+        vpxor	-96(%rcx), %ymm7, %ymm14
+        vpsrlq	$36, %ymm10, %ymm0
+        vpsrlq	$44, %ymm11, %ymm1
+        vpsrlq	$61, %ymm12, %ymm2
+        vpsrlq	$19, %ymm13, %ymm3
+        vpsrlq	$3, %ymm14, %ymm4
+        vpsllq	$28, %ymm10, %ymm10
+        vpsllq	$20, %ymm11, %ymm11
+        vpsllq	$3, %ymm12, %ymm12
+        vpsllq	$45, %ymm13, %ymm13
+        vpsllq	$61, %ymm14, %ymm14
+        vpor	%ymm0, %ymm10, %ymm10
+        vpor	%ymm1, %ymm11, %ymm11
+        vpor	%ymm2, %ymm12, %ymm12
+        vpor	%ymm3, %ymm13, %ymm13
+        vpor	%ymm4, %ymm14, %ymm14
+        vpandn	%ymm12, %ymm11, %ymm0
+        vpandn	%ymm13, %ymm12, %ymm1
+        vpandn	%ymm14, %ymm13, %ymm2
+        vpandn	%ymm10, %ymm14, %ymm3
+        vpandn	%ymm11, %ymm10, %ymm4
+        vpxor	%ymm10, %ymm0, %ymm0
+        vpxor	%ymm11, %ymm1, %ymm1
+        vpxor	%ymm12, %ymm2, %ymm2
+        vpxor	%ymm13, %ymm3, %ymm3
+        vpxor	%ymm14, %ymm4, %ymm4
+        vmovdqu	%ymm0, 32(%rdi)
+        vmovdqu	%ymm1, 64(%rdi)
+        vmovdqu	%ymm2, 96(%rdi)
+        vmovdqu	%ymm3, 128(%rdi)
+        vmovdqu	%ymm4, -96(%rcx)
+        # Row 2
+        vpxor	-64(%rcx), %ymm6, %ymm10
+        vpxor	-32(%rcx), %ymm7, %ymm11
+        vpxor	(%rcx), %ymm8, %ymm12
+        vpxor	32(%rcx), %ymm9, %ymm13
+        vpxor	64(%rcx), %ymm5, %ymm14
+        vpsrlq	$63, %ymm10, %ymm0
+        vpsrlq	$58, %ymm11, %ymm1
+        vpsrlq	$39, %ymm12, %ymm2
+        vpsrlq	$56, %ymm13, %ymm3
+        vpsrlq	$46, %ymm14, %ymm4
+        vpaddq	%ymm10, %ymm10, %ymm10
+        vpsllq	$6, %ymm11, %ymm11
+        vpsllq	$25, %ymm12, %ymm12
+        vpsllq	$8, %ymm13, %ymm13
+        vpsllq	$18, %ymm14, %ymm14
+        vpor	%ymm0, %ymm10, %ymm10
+        vpor	%ymm1, %ymm11, %ymm11
+        vpor	%ymm2, %ymm12, %ymm12
+        vpor	%ymm3, %ymm13, %ymm13
+        vpor	%ymm4, %ymm14, %ymm14
+        vpandn	%ymm12, %ymm11, %ymm0
+        vpandn	%ymm13, %ymm12, %ymm1
+        vpandn	%ymm14, %ymm13, %ymm2
+        vpandn	%ymm10, %ymm14, %ymm3
+        vpandn	%ymm11, %ymm10, %ymm4
+        vpxor	%ymm10, %ymm0, %ymm0
+        vpxor	%ymm11, %ymm1, %ymm1
+        vpxor	%ymm12, %ymm2, %ymm2
+        vpxor	%ymm13, %ymm3, %ymm3
+        vpxor	%ymm14, %ymm4, %ymm4
+        vmovdqu	%ymm0, -64(%rcx)
+        vmovdqu	%ymm1, -32(%rcx)
+        vmovdqu	%ymm2, (%rcx)
+        vmovdqu	%ymm3, 32(%rcx)
+        vmovdqu	%ymm4, 64(%rcx)
+        # Row 3
+        vpxor	96(%rcx), %ymm9, %ymm10
+        vpxor	128(%rcx), %ymm5, %ymm11
+        vpxor	-96(%r8), %ymm6, %ymm12
+        vpxor	-64(%r8), %ymm7, %ymm13
+        vpxor	-32(%r8), %ymm8, %ymm14
+        vpsrlq	$37, %ymm10, %ymm0
+        vpsrlq	$28, %ymm11, %ymm1
+        vpsrlq	$54, %ymm12, %ymm2
+        vpsrlq	$49, %ymm13, %ymm3
+        vpsrlq	$8, %ymm14, %ymm4
+        vpsllq	$27, %ymm10, %ymm10
+        vpsllq	$36, %ymm11, %ymm11
+        vpsllq	$10, %ymm12, %ymm12
+        vpsllq	$15, %ymm13, %ymm13
+        vpsllq	$56, %ymm14, %ymm14
+        vpor	%ymm0, %ymm10, %ymm10
+        vpor	%ymm1, %ymm11, %ymm11
+        vpor	%ymm2, %ymm12, %ymm12
+        vpor	%ymm3, %ymm13, %ymm13
+        vpor	%ymm4, %ymm14, %ymm14
+        vpandn	%ymm12, %ymm11, %ymm0
+        vpandn	%ymm13, %ymm12, %ymm1
+        vpandn	%ymm14, %ymm13, %ymm2
+        vpandn	%ymm10, %ymm14, %ymm3
+        vpandn	%ymm11, %ymm10, %ymm4
+        vpxor	%ymm10, %ymm0, %ymm0
+        vpxor	%ymm11, %ymm1, %ymm1
+        vpxor	%ymm12, %ymm2, %ymm2
+        vpxor	%ymm13, %ymm3, %ymm3
+        vpxor	%ymm14, %ymm4, %ymm4
+        vmovdqu	%ymm0, 96(%rcx)
+        vmovdqu	%ymm1, 128(%rcx)
+        vmovdqu	%ymm2, -96(%r8)
+        vmovdqu	%ymm3, -64(%r8)
+        vmovdqu	%ymm4, -32(%r8)
+        # Row 4
+        vpxor	(%r8), %ymm7, %ymm10
+        vpxor	32(%r8), %ymm8, %ymm11
+        vpxor	64(%r8), %ymm9, %ymm12
+        vpxor	96(%r8), %ymm5, %ymm13
+        vpxor	128(%r8), %ymm6, %ymm14
+        vpsrlq	$2, %ymm10, %ymm0
+        vpsrlq	$9, %ymm11, %ymm1
+        vpsrlq	$25, %ymm12, %ymm2
+        vpsrlq	$23, %ymm13, %ymm3
+        vpsrlq	$62, %ymm14, %ymm4
+        vpsllq	$62, %ymm10, %ymm10
+        vpsllq	$55, %ymm11, %ymm11
+        vpsllq	$39, %ymm12, %ymm12
+        vpsllq	$41, %ymm13, %ymm13
+        vpsllq	$2, %ymm14, %ymm14
+        vpor	%ymm0, %ymm10, %ymm10
+        vpor	%ymm1, %ymm11, %ymm11
+        vpor	%ymm2, %ymm12, %ymm12
+        vpor	%ymm3, %ymm13, %ymm13
+        vpor	%ymm4, %ymm14, %ymm14
+        vpandn	%ymm12, %ymm11, %ymm0
+        vpandn	%ymm13, %ymm12, %ymm1
+        vpandn	%ymm14, %ymm13, %ymm2
+        vpandn	%ymm10, %ymm14, %ymm3
+        vpandn	%ymm11, %ymm10, %ymm4
+        vpxor	%ymm10, %ymm0, %ymm0
+        vpxor	%ymm11, %ymm1, %ymm1
+        vpxor	%ymm12, %ymm2, %ymm2
+        vpxor	%ymm13, %ymm3, %ymm3
+        vpxor	%ymm14, %ymm4, %ymm4
+        vmovdqu	%ymm0, (%r8)
+        vmovdqu	%ymm1, 32(%r8)
+        vmovdqu	%ymm2, 64(%r8)
+        vmovdqu	%ymm3, 96(%r8)
+        vmovdqu	%ymm4, 128(%r8)
+        subq	$0x80, %rdi
+        vmovdqu	%ymm15, (%rdi)
+        leaq	(%rdi), %r14
+        movq	$21, %r13
+L_sha3_blocksx4_out_avx2_word:
+        movq	(%r14), %r15
+        movq	%r15, (%rsi)
+        movq	8(%r14), %r15
+        movq	%r15, (%r9)
+        movq	16(%r14), %r15
+        movq	%r15, (%r10)
+        movq	24(%r14), %r15
+        movq	%r15, (%r11)
+        addq	$32, %r14
+        addq	$8, %rsi
+        addq	$8, %r9
+        addq	$8, %r10
+        addq	$8, %r11
+        subq	$0x01, %r12
+        jz	L_sha3_blocksx4_out_avx2_done
+        subq	$0x01, %r13
+        jnz	L_sha3_blocksx4_out_avx2_word
+        jmp	L_sha3_blocksx4_out_avx2_block
+L_sha3_blocksx4_out_avx2_done:
+        vzeroupper
+        popq	%r15
+        popq	%r14
+        popq	%r13
+        popq	%r12
+        repz retq
+#ifndef __APPLE__
+.size	sha3_blocksx4_out_avx2,.-sha3_blocksx4_out_avx2
+#endif /* __APPLE__ */
+#endif /* WOLFSSL_HAVE_FRODOKEM */
+#ifdef WOLFSSL_HAVE_MLKEM
+#ifndef __APPLE__
+.data
+#else
+.section	__DATA,__data
+#endif /* __APPLE__ */
+#ifndef __APPLE__
+.align	32
+#else
+.p2align	5
+#endif /* __APPLE__ */
+L_sha3_256_blockx4_seed_avx2_end_mark:
+.quad	0x8000000000000000,0x8000000000000000
+.quad	0x8000000000000000,0x8000000000000000
+#ifndef __APPLE__
+.text
+.globl	sha3_256_blocksx4_seed_avx2
+.type	sha3_256_blocksx4_seed_avx2,@function
+.align	16
+sha3_256_blocksx4_seed_avx2:
+#else
+.section	__TEXT,__text
+.globl	_sha3_256_blocksx4_seed_avx2
+.p2align	4
+_sha3_256_blocksx4_seed_avx2:
+#endif /* __APPLE__ */
+        leaq	L_sha3_x4_avx2_r(%rip), %rdx
+        movq	%rdi, %rax
+        movq	%rdi, %rcx
+        vpbroadcastq	(%rsi), %ymm15
+        addq	$0x80, %rdi
+        vpbroadcastq	8(%rsi), %ymm11
+        addq	$0x180, %rax
+        vpbroadcastq	16(%rsi), %ymm12
+        addq	$0x280, %rcx
+        vpbroadcastq	24(%rsi), %ymm13
+        vmovdqu	L_sha3_256_blockx4_seed_avx2_end_mark(%rip), %ymm5
+        vpxor	%ymm6, %ymm6, %ymm6
+        vmovdqu	%ymm11, -96(%rdi)
+        vmovdqu	%ymm12, -64(%rdi)
+        vmovdqu	%ymm13, -32(%rdi)
+        vmovdqu	(%rdi), %ymm14
+        vmovdqu	%ymm6, 32(%rdi)
+        vmovdqu	%ymm6, 64(%rdi)
+        vmovdqu	%ymm6, 96(%rdi)
+        vmovdqu	%ymm6, 128(%rdi)
+        vmovdqu	%ymm6, -96(%rax)
+        vmovdqu	%ymm6, -64(%rax)
+        vmovdqu	%ymm6, -32(%rax)
+        vmovdqu	%ymm6, (%rax)
+        vmovdqu	%ymm6, 32(%rax)
+        vmovdqu	%ymm6, 64(%rax)
+        vmovdqu	%ymm6, 96(%rax)
+        vmovdqu	%ymm5, 128(%rax)
+        vmovdqu	%ymm6, -96(%rcx)
+        vmovdqu	%ymm6, -64(%rcx)
+        vmovdqu	%ymm6, -32(%rcx)
+        vmovdqu	%ymm6, (%rcx)
+        vmovdqu	%ymm6, 32(%rcx)
+        vmovdqu	%ymm6, 64(%rcx)
+        vmovdqu	%ymm6, 96(%rcx)
+        vmovdqu	%ymm6, 128(%rcx)
+        vmovdqu	%ymm15, %ymm10
+        vpxor	%ymm5, %ymm11, %ymm11
+        # Round 0
+        # Calc b[0..4]
+        # Calc t[0..4]
+        vpsrlq	$63, %ymm11, %ymm0
+        vpsrlq	$63, %ymm12, %ymm1
+        vpsrlq	$63, %ymm13, %ymm2
+        vpsrlq	$63, %ymm14, %ymm3
+        vpsrlq	$63, %ymm10, %ymm4
+        vpaddq	%ymm11, %ymm11, %ymm5
+        vpaddq	%ymm12, %ymm12, %ymm6
+        vpaddq	%ymm13, %ymm13, %ymm7
+        vpaddq	%ymm14, %ymm14, %ymm8
+        vpaddq	%ymm10, %ymm10, %ymm9
+        vpor	%ymm0, %ymm5, %ymm5
+        vpor	%ymm1, %ymm6, %ymm6
+        vpor	%ymm2, %ymm7, %ymm7
+        vpor	%ymm3, %ymm8, %ymm8
+        vpor	%ymm4, %ymm9, %ymm9
+        vpxor	%ymm14, %ymm5, %ymm5
+        vpxor	%ymm10, %ymm6, %ymm6
+        vpxor	%ymm11, %ymm7, %ymm7
+        vpxor	%ymm12, %ymm8, %ymm8
+        vpxor	%ymm13, %ymm9, %ymm9
+        # Row Mix
+        # Row 0
+        vpxor	%ymm15, %ymm5, %ymm10
+        vpxor	64(%rdi), %ymm6, %ymm11
+        vpxor	(%rax), %ymm7, %ymm12
+        vpxor	-64(%rcx), %ymm8, %ymm13
+        vpxor	128(%rcx), %ymm9, %ymm14
+        vpsrlq	$20, %ymm11, %ymm0
+        vpsrlq	$21, %ymm12, %ymm1
+        vpsrlq	$43, %ymm13, %ymm2
+        vpsrlq	$50, %ymm14, %ymm3
+        vpsllq	$44, %ymm11, %ymm11
+        vpsllq	$43, %ymm12, %ymm12
+        vpsllq	$21, %ymm13, %ymm13
+        vpsllq	$14, %ymm14, %ymm14
+        vpor	%ymm0, %ymm11, %ymm11
+        vpor	%ymm1, %ymm12, %ymm12
+        vpor	%ymm2, %ymm13, %ymm13
+        vpor	%ymm3, %ymm14, %ymm14
+        vpandn	%ymm12, %ymm11, %ymm15
+        vpandn	%ymm13, %ymm12, %ymm1
+        vpandn	%ymm14, %ymm13, %ymm2
+        vpandn	%ymm10, %ymm14, %ymm3
+        vpandn	%ymm11, %ymm10, %ymm4
+        vpxor	%ymm10, %ymm15, %ymm15
+        vpxor	%ymm11, %ymm1, %ymm1
+        vpxor	%ymm12, %ymm2, %ymm2
+        # XOR in constant
+        vpxor	(%rdx), %ymm15, %ymm15
+        vpxor	%ymm13, %ymm3, %ymm3
+        vpxor	%ymm14, %ymm4, %ymm4
+        vmovdqu	%ymm1, 64(%rdi)
+        vmovdqu	%ymm2, (%rax)
+        vmovdqu	%ymm3, -64(%rcx)
+        vmovdqu	%ymm4, 128(%rcx)
+        # Row 1
+        vpxor	-32(%rdi), %ymm8, %ymm10
+        vpxor	-96(%rax), %ymm9, %ymm11
+        vpxor	-64(%rax), %ymm5, %ymm12
+        vpxor	128(%rax), %ymm6, %ymm13
+        vpxor	64(%rcx), %ymm7, %ymm14
+        vpsrlq	$36, %ymm10, %ymm0
+        vpsrlq	$44, %ymm11, %ymm1
+        vpsrlq	$61, %ymm12, %ymm2
+        vpsrlq	$19, %ymm13, %ymm3
+        vpsrlq	$3, %ymm14, %ymm4
+        vpsllq	$28, %ymm10, %ymm10
+        vpsllq	$20, %ymm11, %ymm11
+        vpsllq	$3, %ymm12, %ymm12
+        vpsllq	$45, %ymm13, %ymm13
+        vpsllq	$61, %ymm14, %ymm14
+        vpor	%ymm0, %ymm10, %ymm10
+        vpor	%ymm1, %ymm11, %ymm11
+        vpor	%ymm2, %ymm12, %ymm12
+        vpor	%ymm3, %ymm13, %ymm13
+        vpor	%ymm4, %ymm14, %ymm14
+        vpandn	%ymm12, %ymm11, %ymm0
+        vpandn	%ymm13, %ymm12, %ymm1
+        vpandn	%ymm14, %ymm13, %ymm2
+        vpandn	%ymm10, %ymm14, %ymm3
+        vpandn	%ymm11, %ymm10, %ymm4
+        vpxor	%ymm10, %ymm0, %ymm0
+        vpxor	%ymm11, %ymm1, %ymm1
+        vpxor	%ymm12, %ymm2, %ymm2
+        vpxor	%ymm13, %ymm3, %ymm3
+        vpxor	%ymm14, %ymm4, %ymm4
+        vmovdqu	%ymm0, -32(%rdi)
+        vmovdqu	%ymm1, -96(%rax)
+        vmovdqu	%ymm2, -64(%rax)
+        vmovdqu	%ymm3, 128(%rax)
+        vmovdqu	%ymm4, 64(%rcx)
+        # Row 2
+        vpxor	-96(%rdi), %ymm6, %ymm10
+        vpxor	96(%rdi), %ymm7, %ymm11
+        vpxor	32(%rax), %ymm8, %ymm12
+        vpxor	-32(%rcx), %ymm9, %ymm13
+        vpxor	(%rcx), %ymm5, %ymm14
+        vpsrlq	$63, %ymm10, %ymm0
+        vpsrlq	$58, %ymm11, %ymm1
+        vpsrlq	$39, %ymm12, %ymm2
+        vpsrlq	$56, %ymm13, %ymm3
+        vpsrlq	$46, %ymm14, %ymm4
+        vpaddq	%ymm10, %ymm10, %ymm10
+        vpsllq	$6, %ymm11, %ymm11
+        vpsllq	$25, %ymm12, %ymm12
+        vpsllq	$8, %ymm13, %ymm13
+        vpsllq	$18, %ymm14, %ymm14
+        vpor	%ymm0, %ymm10, %ymm10
+        vpor	%ymm1, %ymm11, %ymm11
+        vpor	%ymm2, %ymm12, %ymm12
+        vpor	%ymm3, %ymm13, %ymm13
+        vpor	%ymm4, %ymm14, %ymm14
+        vpandn	%ymm12, %ymm11, %ymm0
+        vpandn	%ymm13, %ymm12, %ymm1
+        vpandn	%ymm14, %ymm13, %ymm2
+        vpandn	%ymm10, %ymm14, %ymm3
+        vpandn	%ymm11, %ymm10, %ymm4
+        vpxor	%ymm10, %ymm0, %ymm0
+        vpxor	%ymm11, %ymm1, %ymm1
+        vpxor	%ymm12, %ymm2, %ymm2
+        vpxor	%ymm13, %ymm3, %ymm3
+        vpxor	%ymm14, %ymm4, %ymm4
+        vmovdqu	%ymm0, -96(%rdi)
+        vmovdqu	%ymm1, 96(%rdi)
+        vmovdqu	%ymm2, 32(%rax)
+        vmovdqu	%ymm3, -32(%rcx)
+        vmovdqu	%ymm4, (%rcx)
+        # Row 3
+        vpxor	(%rdi), %ymm9, %ymm10
+        vpxor	32(%rdi), %ymm5, %ymm11
+        vpxor	-32(%rax), %ymm6, %ymm12
+        vpxor	-96(%rcx), %ymm7, %ymm13
+        vpxor	96(%rcx), %ymm8, %ymm14
+        vpsrlq	$37, %ymm10, %ymm0
+        vpsrlq	$28, %ymm11, %ymm1
+        vpsrlq	$54, %ymm12, %ymm2
+        vpsrlq	$49, %ymm13, %ymm3
+        vpsrlq	$8, %ymm14, %ymm4
+        vpsllq	$27, %ymm10, %ymm10
+        vpsllq	$36, %ymm11, %ymm11
+        vpsllq	$10, %ymm12, %ymm12
+        vpsllq	$15, %ymm13, %ymm13
+        vpsllq	$56, %ymm14, %ymm14
+        vpor	%ymm0, %ymm10, %ymm10
+        vpor	%ymm1, %ymm11, %ymm11
+        vpor	%ymm2, %ymm12, %ymm12
+        vpor	%ymm3, %ymm13, %ymm13
+        vpor	%ymm4, %ymm14, %ymm14
+        vpandn	%ymm12, %ymm11, %ymm0
+        vpandn	%ymm13, %ymm12, %ymm1
+        vpandn	%ymm14, %ymm13, %ymm2
+        vpandn	%ymm10, %ymm14, %ymm3
+        vpandn	%ymm11, %ymm10, %ymm4
+        vpxor	%ymm10, %ymm0, %ymm0
+        vpxor	%ymm11, %ymm1, %ymm1
+        vpxor	%ymm12, %ymm2, %ymm2
+        vpxor	%ymm13, %ymm3, %ymm3
+        vpxor	%ymm14, %ymm4, %ymm4
+        vmovdqu	%ymm0, (%rdi)
+        vmovdqu	%ymm1, 32(%rdi)
+        vmovdqu	%ymm2, -32(%rax)
+        vmovdqu	%ymm3, -96(%rcx)
+        vmovdqu	%ymm4, 96(%rcx)
+        # Row 4
+        vpxor	-64(%rdi), %ymm7, %ymm10
+        vpxor	128(%rdi), %ymm8, %ymm11
+        vpxor	64(%rax), %ymm9, %ymm12
+        vpxor	96(%rax), %ymm5, %ymm13
+        vpxor	32(%rcx), %ymm6, %ymm14
+        vpsrlq	$2, %ymm10, %ymm0
+        vpsrlq	$9, %ymm11, %ymm1
+        vpsrlq	$25, %ymm12, %ymm2
+        vpsrlq	$23, %ymm13, %ymm3
+        vpsrlq	$62, %ymm14, %ymm4
+        vpsllq	$62, %ymm10, %ymm10
+        vpsllq	$55, %ymm11, %ymm11
+        vpsllq	$39, %ymm12, %ymm12
+        vpsllq	$41, %ymm13, %ymm13
+        vpsllq	$2, %ymm14, %ymm14
+        vpor	%ymm0, %ymm10, %ymm10
+        vpor	%ymm1, %ymm11, %ymm11
+        vpor	%ymm2, %ymm12, %ymm12
+        vpor	%ymm3, %ymm13, %ymm13
+        vpor	%ymm4, %ymm14, %ymm14
+        vpandn	%ymm12, %ymm11, %ymm0
+        vpandn	%ymm13, %ymm12, %ymm1
+        vpandn	%ymm14, %ymm13, %ymm2
+        vpandn	%ymm10, %ymm14, %ymm3
+        vpandn	%ymm11, %ymm10, %ymm4
+        vpxor	%ymm10, %ymm0, %ymm0
+        vpxor	%ymm11, %ymm1, %ymm1
+        vpxor	%ymm12, %ymm2, %ymm2
+        vpxor	%ymm13, %ymm3, %ymm3
+        vpxor	%ymm14, %ymm4, %ymm4
+        vmovdqu	%ymm0, -64(%rdi)
+        vmovdqu	%ymm1, 128(%rdi)
+        vmovdqu	%ymm2, 64(%rax)
+        vmovdqu	%ymm3, 96(%rax)
+        vmovdqu	%ymm4, 32(%rcx)
+        # Round 1
+        # Calc b[0..4]
+        vpxor	%ymm15, %ymm0, %ymm10
+        vpxor	-96(%rdi), %ymm10, %ymm10
+        vpxor	-32(%rdi), %ymm10, %ymm10
+        vpxor	(%rdi), %ymm10, %ymm10
+        vpxor	32(%rdi), %ymm1, %ymm11
+        vpxor	64(%rdi), %ymm11, %ymm11
+        vpxor	96(%rdi), %ymm11, %ymm11
+        vpxor	-96(%rax), %ymm11, %ymm11
+        vpxor	-64(%rax), %ymm2, %ymm12
+        vpxor	-32(%rax), %ymm12, %ymm12
+        vpxor	(%rax), %ymm12, %ymm12
+        vpxor	32(%rax), %ymm12, %ymm12
+        vpxor	128(%rax), %ymm3, %ymm13
+        vpxor	-96(%rcx), %ymm13, %ymm13
+        vpxor	-64(%rcx), %ymm13, %ymm13
+        vpxor	-32(%rcx), %ymm13, %ymm13
+        vpxor	(%rcx), %ymm4, %ymm14
+        vpxor	64(%rcx), %ymm14, %ymm14
+        vpxor	96(%rcx), %ymm14, %ymm14
+        vpxor	128(%rcx), %ymm14, %ymm14
+        # Calc t[0..4]
+        vpsrlq	$63, %ymm11, %ymm0
+        vpsrlq	$63, %ymm12, %ymm1
+        vpsrlq	$63, %ymm13, %ymm2
+        vpsrlq	$63, %ymm14, %ymm3
+        vpsrlq	$63, %ymm10, %ymm4
+        vpaddq	%ymm11, %ymm11, %ymm5
+        vpaddq	%ymm12, %ymm12, %ymm6
+        vpaddq	%ymm13, %ymm13, %ymm7
+        vpaddq	%ymm14, %ymm14, %ymm8
+        vpaddq	%ymm10, %ymm10, %ymm9
+        vpor	%ymm0, %ymm5, %ymm5
+        vpor	%ymm1, %ymm6, %ymm6
+        vpor	%ymm2, %ymm7, %ymm7
+        vpor	%ymm3, %ymm8, %ymm8
+        vpor	%ymm4, %ymm9, %ymm9
+        vpxor	%ymm14, %ymm5, %ymm5
+        vpxor	%ymm10, %ymm6, %ymm6
+        vpxor	%ymm11, %ymm7, %ymm7
+        vpxor	%ymm12, %ymm8, %ymm8
+        vpxor	%ymm13, %ymm9, %ymm9
+        # Row Mix
+        # Row 0
+        vpxor	%ymm15, %ymm5, %ymm10
+        vpxor	-96(%rax), %ymm6, %ymm11
+        vpxor	32(%rax), %ymm7, %ymm12
+        vpxor	-96(%rcx), %ymm8, %ymm13
+        vpxor	32(%rcx), %ymm9, %ymm14
+        vpsrlq	$20, %ymm11, %ymm0
+        vpsrlq	$21, %ymm12, %ymm1
+        vpsrlq	$43, %ymm13, %ymm2
+        vpsrlq	$50, %ymm14, %ymm3
+        vpsllq	$44, %ymm11, %ymm11
+        vpsllq	$43, %ymm12, %ymm12
+        vpsllq	$21, %ymm13, %ymm13
+        vpsllq	$14, %ymm14, %ymm14
+        vpor	%ymm0, %ymm11, %ymm11
+        vpor	%ymm1, %ymm12, %ymm12
+        vpor	%ymm2, %ymm13, %ymm13
+        vpor	%ymm3, %ymm14, %ymm14
+        vpandn	%ymm12, %ymm11, %ymm15
+        vpandn	%ymm13, %ymm12, %ymm1
+        vpandn	%ymm14, %ymm13, %ymm2
+        vpandn	%ymm10, %ymm14, %ymm3
+        vpandn	%ymm11, %ymm10, %ymm4
+        vpxor	%ymm10, %ymm15, %ymm15
+        vpxor	%ymm11, %ymm1, %ymm1
+        vpxor	%ymm12, %ymm2, %ymm2
+        # XOR in constant
+        vpxor	32(%rdx), %ymm15, %ymm15
+        vpxor	%ymm13, %ymm3, %ymm3
+        vpxor	%ymm14, %ymm4, %ymm4
+        vmovdqu	%ymm1, -96(%rax)
+        vmovdqu	%ymm2, 32(%rax)
+        vmovdqu	%ymm3, -96(%rcx)
+        vmovdqu	%ymm4, 32(%rcx)
+        # Row 1
+        vpxor	-64(%rcx), %ymm8, %ymm10
+        vpxor	64(%rcx), %ymm9, %ymm11
+        vpxor	-96(%rdi), %ymm5, %ymm12
+        vpxor	32(%rdi), %ymm6, %ymm13
+        vpxor	64(%rax), %ymm7, %ymm14
+        vpsrlq	$36, %ymm10, %ymm0
+        vpsrlq	$44, %ymm11, %ymm1
+        vpsrlq	$61, %ymm12, %ymm2
+        vpsrlq	$19, %ymm13, %ymm3
+        vpsrlq	$3, %ymm14, %ymm4
+        vpsllq	$28, %ymm10, %ymm10
+        vpsllq	$20, %ymm11, %ymm11
+        vpsllq	$3, %ymm12, %ymm12
+        vpsllq	$45, %ymm13, %ymm13
+        vpsllq	$61, %ymm14, %ymm14
+        vpor	%ymm0, %ymm10, %ymm10
+        vpor	%ymm1, %ymm11, %ymm11
+        vpor	%ymm2, %ymm12, %ymm12
+        vpor	%ymm3, %ymm13, %ymm13
+        vpor	%ymm4, %ymm14, %ymm14
+        vpandn	%ymm12, %ymm11, %ymm0
+        vpandn	%ymm13, %ymm12, %ymm1
+        vpandn	%ymm14, %ymm13, %ymm2
+        vpandn	%ymm10, %ymm14, %ymm3
+        vpandn	%ymm11, %ymm10, %ymm4
+        vpxor	%ymm10, %ymm0, %ymm0
+        vpxor	%ymm11, %ymm1, %ymm1
+        vpxor	%ymm12, %ymm2, %ymm2
+        vpxor	%ymm13, %ymm3, %ymm3
+        vpxor	%ymm14, %ymm4, %ymm4
+        vmovdqu	%ymm0, -64(%rcx)
+        vmovdqu	%ymm1, 64(%rcx)
+        vmovdqu	%ymm2, -96(%rdi)
+        vmovdqu	%ymm3, 32(%rdi)
+        vmovdqu	%ymm4, 64(%rax)
+        # Row 2
+        vpxor	64(%rdi), %ymm6, %ymm10
+        vpxor	-64(%rax), %ymm7, %ymm11
+        vpxor	-32(%rcx), %ymm8, %ymm12
+        vpxor	96(%rcx), %ymm9, %ymm13
+        vpxor	-64(%rdi), %ymm5, %ymm14
+        vpsrlq	$63, %ymm10, %ymm0
+        vpsrlq	$58, %ymm11, %ymm1
+        vpsrlq	$39, %ymm12, %ymm2
+        vpsrlq	$56, %ymm13, %ymm3
+        vpsrlq	$46, %ymm14, %ymm4
+        vpaddq	%ymm10, %ymm10, %ymm10
+        vpsllq	$6, %ymm11, %ymm11
+        vpsllq	$25, %ymm12, %ymm12
+        vpsllq	$8, %ymm13, %ymm13
+        vpsllq	$18, %ymm14, %ymm14
+        vpor	%ymm0, %ymm10, %ymm10
+        vpor	%ymm1, %ymm11, %ymm11
+        vpor	%ymm2, %ymm12, %ymm12
+        vpor	%ymm3, %ymm13, %ymm13
+        vpor	%ymm4, %ymm14, %ymm14
+        vpandn	%ymm12, %ymm11, %ymm0
+        vpandn	%ymm13, %ymm12, %ymm1
+        vpandn	%ymm14, %ymm13, %ymm2
+        vpandn	%ymm10, %ymm14, %ymm3
+        vpandn	%ymm11, %ymm10, %ymm4
+        vpxor	%ymm10, %ymm0, %ymm0
+        vpxor	%ymm11, %ymm1, %ymm1
+        vpxor	%ymm12, %ymm2, %ymm2
+        vpxor	%ymm13, %ymm3, %ymm3
+        vpxor	%ymm14, %ymm4, %ymm4
+        vmovdqu	%ymm0, 64(%rdi)
+        vmovdqu	%ymm1, -64(%rax)
+        vmovdqu	%ymm2, -32(%rcx)
+        vmovdqu	%ymm3, 96(%rcx)
+        vmovdqu	%ymm4, -64(%rdi)
+        # Row 3
+        vpxor	128(%rcx), %ymm9, %ymm10
+        vpxor	-32(%rdi), %ymm5, %ymm11
+        vpxor	96(%rdi), %ymm6, %ymm12
+        vpxor	-32(%rax), %ymm7, %ymm13
+        vpxor	96(%rax), %ymm8, %ymm14
+        vpsrlq	$37, %ymm10, %ymm0
+        vpsrlq	$28, %ymm11, %ymm1
+        vpsrlq	$54, %ymm12, %ymm2
+        vpsrlq	$49, %ymm13, %ymm3
+        vpsrlq	$8, %ymm14, %ymm4
+        vpsllq	$27, %ymm10, %ymm10
+        vpsllq	$36, %ymm11, %ymm11
+        vpsllq	$10, %ymm12, %ymm12
+        vpsllq	$15, %ymm13, %ymm13
+        vpsllq	$56, %ymm14, %ymm14
+        vpor	%ymm0, %ymm10, %ymm10
+        vpor	%ymm1, %ymm11, %ymm11
+        vpor	%ymm2, %ymm12, %ymm12
+        vpor	%ymm3, %ymm13, %ymm13
+        vpor	%ymm4, %ymm14, %ymm14
+        vpandn	%ymm12, %ymm11, %ymm0
+        vpandn	%ymm13, %ymm12, %ymm1
+        vpandn	%ymm14, %ymm13, %ymm2
+        vpandn	%ymm10, %ymm14, %ymm3
+        vpandn	%ymm11, %ymm10, %ymm4
+        vpxor	%ymm10, %ymm0, %ymm0
+        vpxor	%ymm11, %ymm1, %ymm1
+        vpxor	%ymm12, %ymm2, %ymm2
+        vpxor	%ymm13, %ymm3, %ymm3
+        vpxor	%ymm14, %ymm4, %ymm4
+        vmovdqu	%ymm0, 128(%rcx)
+        vmovdqu	%ymm1, -32(%rdi)
+        vmovdqu	%ymm2, 96(%rdi)
+        vmovdqu	%ymm3, -32(%rax)
+        vmovdqu	%ymm4, 96(%rax)
+        # Row 4
+        vpxor	(%rax), %ymm7, %ymm10
+        vpxor	128(%rax), %ymm8, %ymm11
+        vpxor	(%rcx), %ymm9, %ymm12
+        vpxor	(%rdi), %ymm5, %ymm13
+        vpxor	128(%rdi), %ymm6, %ymm14
+        vpsrlq	$2, %ymm10, %ymm0
+        vpsrlq	$9, %ymm11, %ymm1
+        vpsrlq	$25, %ymm12, %ymm2
+        vpsrlq	$23, %ymm13, %ymm3
+        vpsrlq	$62, %ymm14, %ymm4
+        vpsllq	$62, %ymm10, %ymm10
+        vpsllq	$55, %ymm11, %ymm11
+        vpsllq	$39, %ymm12, %ymm12
+        vpsllq	$41, %ymm13, %ymm13
+        vpsllq	$2, %ymm14, %ymm14
+        vpor	%ymm0, %ymm10, %ymm10
+        vpor	%ymm1, %ymm11, %ymm11
+        vpor	%ymm2, %ymm12, %ymm12
+        vpor	%ymm3, %ymm13, %ymm13
+        vpor	%ymm4, %ymm14, %ymm14
+        vpandn	%ymm12, %ymm11, %ymm0
+        vpandn	%ymm13, %ymm12, %ymm1
+        vpandn	%ymm14, %ymm13, %ymm2
+        vpandn	%ymm10, %ymm14, %ymm3
+        vpandn	%ymm11, %ymm10, %ymm4
+        vpxor	%ymm10, %ymm0, %ymm0
+        vpxor	%ymm11, %ymm1, %ymm1
+        vpxor	%ymm12, %ymm2, %ymm2
+        vpxor	%ymm13, %ymm3, %ymm3
+        vpxor	%ymm14, %ymm4, %ymm4
+        vmovdqu	%ymm0, (%rax)
+        vmovdqu	%ymm1, 128(%rax)
+        vmovdqu	%ymm2, (%rcx)
+        vmovdqu	%ymm3, (%rdi)
+        vmovdqu	%ymm4, 128(%rdi)
+        # Round 2
+        # Calc b[0..4]
+        vpxor	%ymm15, %ymm0, %ymm10
+        vpxor	-96(%rdi), %ymm2, %ymm12
+        vpxor	-64(%rdi), %ymm4, %ymm14
+        vpxor	-32(%rdi), %ymm1, %ymm11
+        vpxor	32(%rdi), %ymm3, %ymm13
+        vpxor	64(%rdi), %ymm10, %ymm10
+        vpxor	96(%rdi), %ymm12, %ymm12
+        vpxor	-96(%rax), %ymm11, %ymm11
+        vpxor	-64(%rax), %ymm11, %ymm11
+        vpxor	-32(%rax), %ymm13, %ymm13
+        vpxor	32(%rax), %ymm12, %ymm12
+        vpxor	64(%rax), %ymm14, %ymm14
+        vpxor	96(%rax), %ymm14, %ymm14
+        vpxor	-96(%rcx), %ymm13, %ymm13
+        vpxor	-64(%rcx), %ymm10, %ymm10
+        vpxor	-32(%rcx), %ymm12, %ymm12
+        vpxor	32(%rcx), %ymm14, %ymm14
+        vpxor	64(%rcx), %ymm11, %ymm11
+        vpxor	96(%rcx), %ymm13, %ymm13
+        vpxor	128(%rcx), %ymm10, %ymm10
+        # Calc t[0..4]
+        vpsrlq	$63, %ymm11, %ymm0
+        vpsrlq	$63, %ymm12, %ymm1
+        vpsrlq	$63, %ymm13, %ymm2
+        vpsrlq	$63, %ymm14, %ymm3
+        vpsrlq	$63, %ymm10, %ymm4
+        vpaddq	%ymm11, %ymm11, %ymm5
+        vpaddq	%ymm12, %ymm12, %ymm6
+        vpaddq	%ymm13, %ymm13, %ymm7
+        vpaddq	%ymm14, %ymm14, %ymm8
+        vpaddq	%ymm10, %ymm10, %ymm9
+        vpor	%ymm0, %ymm5, %ymm5
+        vpor	%ymm1, %ymm6, %ymm6
+        vpor	%ymm2, %ymm7, %ymm7
+        vpor	%ymm3, %ymm8, %ymm8
+        vpor	%ymm4, %ymm9, %ymm9
+        vpxor	%ymm14, %ymm5, %ymm5
+        vpxor	%ymm10, %ymm6, %ymm6
+        vpxor	%ymm11, %ymm7, %ymm7
+        vpxor	%ymm12, %ymm8, %ymm8
+        vpxor	%ymm13, %ymm9, %ymm9
+        # Row Mix
+        # Row 0
+        vpxor	%ymm15, %ymm5, %ymm10
+        vpxor	64(%rcx), %ymm6, %ymm11
+        vpxor	-32(%rcx), %ymm7, %ymm12
+        vpxor	-32(%rax), %ymm8, %ymm13
+        vpxor	128(%rdi), %ymm9, %ymm14
+        vpsrlq	$20, %ymm11, %ymm0
+        vpsrlq	$21, %ymm12, %ymm1
+        vpsrlq	$43, %ymm13, %ymm2
+        vpsrlq	$50, %ymm14, %ymm3
+        vpsllq	$44, %ymm11, %ymm11
+        vpsllq	$43, %ymm12, %ymm12
+        vpsllq	$21, %ymm13, %ymm13
+        vpsllq	$14, %ymm14, %ymm14
+        vpor	%ymm0, %ymm11, %ymm11
+        vpor	%ymm1, %ymm12, %ymm12
+        vpor	%ymm2, %ymm13, %ymm13
+        vpor	%ymm3, %ymm14, %ymm14
+        vpandn	%ymm12, %ymm11, %ymm15
+        vpandn	%ymm13, %ymm12, %ymm1
+        vpandn	%ymm14, %ymm13, %ymm2
+        vpandn	%ymm10, %ymm14, %ymm3
+        vpandn	%ymm11, %ymm10, %ymm4
+        vpxor	%ymm10, %ymm15, %ymm15
+        vpxor	%ymm11, %ymm1, %ymm1
+        vpxor	%ymm12, %ymm2, %ymm2
+        # XOR in constant
+        vpxor	64(%rdx), %ymm15, %ymm15
+        vpxor	%ymm13, %ymm3, %ymm3
+        vpxor	%ymm14, %ymm4, %ymm4
+        vmovdqu	%ymm1, 64(%rcx)
+        vmovdqu	%ymm2, -32(%rcx)
+        vmovdqu	%ymm3, -32(%rax)
+        vmovdqu	%ymm4, 128(%rdi)
+        # Row 1
+        vpxor	-96(%rcx), %ymm8, %ymm10
+        vpxor	64(%rax), %ymm9, %ymm11
+        vpxor	64(%rdi), %ymm5, %ymm12
+        vpxor	-32(%rdi), %ymm6, %ymm13
+        vpxor	(%rcx), %ymm7, %ymm14
+        vpsrlq	$36, %ymm10, %ymm0
+        vpsrlq	$44, %ymm11, %ymm1
+        vpsrlq	$61, %ymm12, %ymm2
+        vpsrlq	$19, %ymm13, %ymm3
+        vpsrlq	$3, %ymm14, %ymm4
+        vpsllq	$28, %ymm10, %ymm10
+        vpsllq	$20, %ymm11, %ymm11
+        vpsllq	$3, %ymm12, %ymm12
+        vpsllq	$45, %ymm13, %ymm13
+        vpsllq	$61, %ymm14, %ymm14
+        vpor	%ymm0, %ymm10, %ymm10
+        vpor	%ymm1, %ymm11, %ymm11
+        vpor	%ymm2, %ymm12, %ymm12
+        vpor	%ymm3, %ymm13, %ymm13
+        vpor	%ymm4, %ymm14, %ymm14
+        vpandn	%ymm12, %ymm11, %ymm0
+        vpandn	%ymm13, %ymm12, %ymm1
+        vpandn	%ymm14, %ymm13, %ymm2
+        vpandn	%ymm10, %ymm14, %ymm3
+        vpandn	%ymm11, %ymm10, %ymm4
+        vpxor	%ymm10, %ymm0, %ymm0
+        vpxor	%ymm11, %ymm1, %ymm1
+        vpxor	%ymm12, %ymm2, %ymm2
+        vpxor	%ymm13, %ymm3, %ymm3
+        vpxor	%ymm14, %ymm4, %ymm4
+        vmovdqu	%ymm0, -96(%rcx)
+        vmovdqu	%ymm1, 64(%rax)
+        vmovdqu	%ymm2, 64(%rdi)
+        vmovdqu	%ymm3, -32(%rdi)
+        vmovdqu	%ymm4, (%rcx)
+        # Row 2
+        vpxor	-96(%rax), %ymm6, %ymm10
+        vpxor	-96(%rdi), %ymm7, %ymm11
+        vpxor	96(%rcx), %ymm8, %ymm12
+        vpxor	96(%rax), %ymm9, %ymm13
+        vpxor	(%rax), %ymm5, %ymm14
+        vpsrlq	$63, %ymm10, %ymm0
+        vpsrlq	$58, %ymm11, %ymm1
+        vpsrlq	$39, %ymm12, %ymm2
+        vpsrlq	$56, %ymm13, %ymm3
+        vpsrlq	$46, %ymm14, %ymm4
+        vpaddq	%ymm10, %ymm10, %ymm10
+        vpsllq	$6, %ymm11, %ymm11
+        vpsllq	$25, %ymm12, %ymm12
+        vpsllq	$8, %ymm13, %ymm13
+        vpsllq	$18, %ymm14, %ymm14
+        vpor	%ymm0, %ymm10, %ymm10
+        vpor	%ymm1, %ymm11, %ymm11
+        vpor	%ymm2, %ymm12, %ymm12
+        vpor	%ymm3, %ymm13, %ymm13
+        vpor	%ymm4, %ymm14, %ymm14
+        vpandn	%ymm12, %ymm11, %ymm0
+        vpandn	%ymm13, %ymm12, %ymm1
+        vpandn	%ymm14, %ymm13, %ymm2
+        vpandn	%ymm10, %ymm14, %ymm3
+        vpandn	%ymm11, %ymm10, %ymm4
+        vpxor	%ymm10, %ymm0, %ymm0
+        vpxor	%ymm11, %ymm1, %ymm1
+        vpxor	%ymm12, %ymm2, %ymm2
+        vpxor	%ymm13, %ymm3, %ymm3
+        vpxor	%ymm14, %ymm4, %ymm4
+        vmovdqu	%ymm0, -96(%rax)
+        vmovdqu	%ymm1, -96(%rdi)
+        vmovdqu	%ymm2, 96(%rcx)
+        vmovdqu	%ymm3, 96(%rax)
+        vmovdqu	%ymm4, (%rax)
+        # Row 3
+        vpxor	32(%rcx), %ymm9, %ymm10
+        vpxor	-64(%rcx), %ymm5, %ymm11
+        vpxor	-64(%rax), %ymm6, %ymm12
+        vpxor	96(%rdi), %ymm7, %ymm13
+        vpxor	(%rdi), %ymm8, %ymm14
+        vpsrlq	$37, %ymm10, %ymm0
+        vpsrlq	$28, %ymm11, %ymm1
+        vpsrlq	$54, %ymm12, %ymm2
+        vpsrlq	$49, %ymm13, %ymm3
+        vpsrlq	$8, %ymm14, %ymm4
+        vpsllq	$27, %ymm10, %ymm10
+        vpsllq	$36, %ymm11, %ymm11
+        vpsllq	$10, %ymm12, %ymm12
+        vpsllq	$15, %ymm13, %ymm13
+        vpsllq	$56, %ymm14, %ymm14
+        vpor	%ymm0, %ymm10, %ymm10
+        vpor	%ymm1, %ymm11, %ymm11
+        vpor	%ymm2, %ymm12, %ymm12
+        vpor	%ymm3, %ymm13, %ymm13
+        vpor	%ymm4, %ymm14, %ymm14
+        vpandn	%ymm12, %ymm11, %ymm0
+        vpandn	%ymm13, %ymm12, %ymm1
+        vpandn	%ymm14, %ymm13, %ymm2
+        vpandn	%ymm10, %ymm14, %ymm3
+        vpandn	%ymm11, %ymm10, %ymm4
+        vpxor	%ymm10, %ymm0, %ymm0
+        vpxor	%ymm11, %ymm1, %ymm1
+        vpxor	%ymm12, %ymm2, %ymm2
+        vpxor	%ymm13, %ymm3, %ymm3
+        vpxor	%ymm14, %ymm4, %ymm4
+        vmovdqu	%ymm0, 32(%rcx)
+        vmovdqu	%ymm1, -64(%rcx)
+        vmovdqu	%ymm2, -64(%rax)
+        vmovdqu	%ymm3, 96(%rdi)
+        vmovdqu	%ymm4, (%rdi)
+        # Row 4
+        vpxor	32(%rax), %ymm7, %ymm10
+        vpxor	32(%rdi), %ymm8, %ymm11
+        vpxor	-64(%rdi), %ymm9, %ymm12
+        vpxor	128(%rcx), %ymm5, %ymm13
+        vpxor	128(%rax), %ymm6, %ymm14
         vpsrlq	$2, %ymm10, %ymm0
         vpsrlq	$9, %ymm11, %ymm1
         vpsrlq	$25, %ymm12, %ymm2
@@ -31459,6 +36860,3056 @@ _sha3_256_blocksx4_seed_64_avx2:
 .size	sha3_256_blocksx4_seed_64_avx2,.-sha3_256_blocksx4_seed_64_avx2
 #endif /* __APPLE__ */
 #endif /* WOLFSSL_HAVE_MLDSA */
+#ifdef HAVE_INTEL_AVX512
+#ifdef WOLFSSL_HAVE_FRODOKEM
+#ifndef __APPLE__
+.data
+#else
+.section	__DATA,__data
+#endif /* __APPLE__ */
+#ifndef __APPLE__
+.align	32
+#else
+.p2align	5
+#endif /* __APPLE__ */
+L_sha3_x8_avx512_r:
+.quad	0x0000000000000001,0x0000000000000001
+.quad	0x0000000000000001,0x0000000000000001
+.quad	0x0000000000000001,0x0000000000000001
+.quad	0x0000000000000001,0x0000000000000001
+.quad	0x0000000000008082,0x0000000000008082
+.quad	0x0000000000008082,0x0000000000008082
+.quad	0x0000000000008082,0x0000000000008082
+.quad	0x0000000000008082,0x0000000000008082
+.quad	0x800000000000808a,0x800000000000808a
+.quad	0x800000000000808a,0x800000000000808a
+.quad	0x800000000000808a,0x800000000000808a
+.quad	0x800000000000808a,0x800000000000808a
+.quad	0x8000000080008000,0x8000000080008000
+.quad	0x8000000080008000,0x8000000080008000
+.quad	0x8000000080008000,0x8000000080008000
+.quad	0x8000000080008000,0x8000000080008000
+.quad	0x000000000000808b,0x000000000000808b
+.quad	0x000000000000808b,0x000000000000808b
+.quad	0x000000000000808b,0x000000000000808b
+.quad	0x000000000000808b,0x000000000000808b
+.quad	0x0000000080000001,0x0000000080000001
+.quad	0x0000000080000001,0x0000000080000001
+.quad	0x0000000080000001,0x0000000080000001
+.quad	0x0000000080000001,0x0000000080000001
+.quad	0x8000000080008081,0x8000000080008081
+.quad	0x8000000080008081,0x8000000080008081
+.quad	0x8000000080008081,0x8000000080008081
+.quad	0x8000000080008081,0x8000000080008081
+.quad	0x8000000000008009,0x8000000000008009
+.quad	0x8000000000008009,0x8000000000008009
+.quad	0x8000000000008009,0x8000000000008009
+.quad	0x8000000000008009,0x8000000000008009
+.quad	0x000000000000008a,0x000000000000008a
+.quad	0x000000000000008a,0x000000000000008a
+.quad	0x000000000000008a,0x000000000000008a
+.quad	0x000000000000008a,0x000000000000008a
+.quad	0x0000000000000088,0x0000000000000088
+.quad	0x0000000000000088,0x0000000000000088
+.quad	0x0000000000000088,0x0000000000000088
+.quad	0x0000000000000088,0x0000000000000088
+.quad	0x0000000080008009,0x0000000080008009
+.quad	0x0000000080008009,0x0000000080008009
+.quad	0x0000000080008009,0x0000000080008009
+.quad	0x0000000080008009,0x0000000080008009
+.quad	0x000000008000000a,0x000000008000000a
+.quad	0x000000008000000a,0x000000008000000a
+.quad	0x000000008000000a,0x000000008000000a
+.quad	0x000000008000000a,0x000000008000000a
+.quad	0x000000008000808b,0x000000008000808b
+.quad	0x000000008000808b,0x000000008000808b
+.quad	0x000000008000808b,0x000000008000808b
+.quad	0x000000008000808b,0x000000008000808b
+.quad	0x800000000000008b,0x800000000000008b
+.quad	0x800000000000008b,0x800000000000008b
+.quad	0x800000000000008b,0x800000000000008b
+.quad	0x800000000000008b,0x800000000000008b
+.quad	0x8000000000008089,0x8000000000008089
+.quad	0x8000000000008089,0x8000000000008089
+.quad	0x8000000000008089,0x8000000000008089
+.quad	0x8000000000008089,0x8000000000008089
+.quad	0x8000000000008003,0x8000000000008003
+.quad	0x8000000000008003,0x8000000000008003
+.quad	0x8000000000008003,0x8000000000008003
+.quad	0x8000000000008003,0x8000000000008003
+.quad	0x8000000000008002,0x8000000000008002
+.quad	0x8000000000008002,0x8000000000008002
+.quad	0x8000000000008002,0x8000000000008002
+.quad	0x8000000000008002,0x8000000000008002
+.quad	0x8000000000000080,0x8000000000000080
+.quad	0x8000000000000080,0x8000000000000080
+.quad	0x8000000000000080,0x8000000000000080
+.quad	0x8000000000000080,0x8000000000000080
+.quad	0x000000000000800a,0x000000000000800a
+.quad	0x000000000000800a,0x000000000000800a
+.quad	0x000000000000800a,0x000000000000800a
+.quad	0x000000000000800a,0x000000000000800a
+.quad	0x800000008000000a,0x800000008000000a
+.quad	0x800000008000000a,0x800000008000000a
+.quad	0x800000008000000a,0x800000008000000a
+.quad	0x800000008000000a,0x800000008000000a
+.quad	0x8000000080008081,0x8000000080008081
+.quad	0x8000000080008081,0x8000000080008081
+.quad	0x8000000080008081,0x8000000080008081
+.quad	0x8000000080008081,0x8000000080008081
+.quad	0x8000000000008080,0x8000000000008080
+.quad	0x8000000000008080,0x8000000000008080
+.quad	0x8000000000008080,0x8000000000008080
+.quad	0x8000000000008080,0x8000000000008080
+.quad	0x0000000080000001,0x0000000080000001
+.quad	0x0000000080000001,0x0000000080000001
+.quad	0x0000000080000001,0x0000000080000001
+.quad	0x0000000080000001,0x0000000080000001
+.quad	0x8000000080008008,0x8000000080008008
+.quad	0x8000000080008008,0x8000000080008008
+.quad	0x8000000080008008,0x8000000080008008
+.quad	0x8000000080008008,0x8000000080008008
+#ifndef __APPLE__
+.text
+.globl	sha3_blocksx8_out_avx512
+.type	sha3_blocksx8_out_avx512,@function
+.align	16
+sha3_blocksx8_out_avx512:
+#else
+.section	__TEXT,__text
+.globl	_sha3_blocksx8_out_avx512
+.p2align	4
+_sha3_blocksx8_out_avx512:
+#endif /* __APPLE__ */
+        pushq	%r12
+        pushq	%r13
+        pushq	%r14
+        pushq	%r15
+        pushq	%rbx
+        pushq	%rbp
+        leaq	L_sha3_x8_avx512_r(%rip), %rax
+        movl	%edx, %r14d
+        movq	%rsi, %rcx
+        addq	%r14, %rcx
+        movq	%rcx, %r8
+        addq	%r14, %r8
+        movq	%r8, %r9
+        addq	%r14, %r9
+        movq	%r9, %r10
+        addq	%r14, %r10
+        movq	%r10, %r11
+        addq	%r14, %r11
+        movq	%r11, %r12
+        addq	%r14, %r12
+        movq	%r12, %r13
+        addq	%r14, %r13
+        shrq	$3, %r14
+        vmovdqu64	(%rdi), %zmm0
+        vmovdqu64	64(%rdi), %zmm1
+        vmovdqu64	128(%rdi), %zmm2
+        vmovdqu64	192(%rdi), %zmm3
+        vmovdqu64	256(%rdi), %zmm4
+        vmovdqu64	320(%rdi), %zmm5
+        vmovdqu64	384(%rdi), %zmm6
+        vmovdqu64	448(%rdi), %zmm7
+        vmovdqu64	512(%rdi), %zmm8
+        vmovdqu64	576(%rdi), %zmm9
+        vmovdqu64	640(%rdi), %zmm10
+        vmovdqu64	704(%rdi), %zmm11
+        vmovdqu64	768(%rdi), %zmm12
+        vmovdqu64	832(%rdi), %zmm13
+        vmovdqu64	896(%rdi), %zmm14
+        vmovdqu64	960(%rdi), %zmm15
+        vmovdqu64	1024(%rdi), %zmm16
+        vmovdqu64	1088(%rdi), %zmm17
+        vmovdqu64	1152(%rdi), %zmm18
+        vmovdqu64	1216(%rdi), %zmm19
+        vmovdqu64	1280(%rdi), %zmm20
+        vmovdqu64	1344(%rdi), %zmm21
+        vmovdqu64	1408(%rdi), %zmm22
+        vmovdqu64	1472(%rdi), %zmm23
+        vmovdqu64	1536(%rdi), %zmm24
+L_sha3_blocksx8_out_avx512_block:
+        # Round 0
+        # Theta - column parity
+        vmovdqu64	%zmm0, %zmm25
+        vpternlogq	$0x96, %zmm10, %zmm5, %zmm25
+        vpternlogq	$0x96, %zmm20, %zmm15, %zmm25
+        vmovdqu64	%zmm1, %zmm26
+        vpternlogq	$0x96, %zmm11, %zmm6, %zmm26
+        vpternlogq	$0x96, %zmm21, %zmm16, %zmm26
+        vmovdqu64	%zmm2, %zmm27
+        vpternlogq	$0x96, %zmm12, %zmm7, %zmm27
+        vpternlogq	$0x96, %zmm22, %zmm17, %zmm27
+        vmovdqu64	%zmm3, %zmm28
+        vpternlogq	$0x96, %zmm13, %zmm8, %zmm28
+        vpternlogq	$0x96, %zmm23, %zmm18, %zmm28
+        vmovdqu64	%zmm4, %zmm29
+        vpternlogq	$0x96, %zmm14, %zmm9, %zmm29
+        vpternlogq	$0x96, %zmm24, %zmm19, %zmm29
+        # Theta - D and apply in place
+        vprolq	$0x01, %zmm26, %zmm30
+        vpxorq	%zmm29, %zmm30, %zmm30
+        vpxorq	%zmm30, %zmm0, %zmm0
+        vpxorq	%zmm30, %zmm5, %zmm5
+        vpxorq	%zmm30, %zmm10, %zmm10
+        vpxorq	%zmm30, %zmm15, %zmm15
+        vpxorq	%zmm30, %zmm20, %zmm20
+        vprolq	$0x01, %zmm27, %zmm30
+        vpxorq	%zmm25, %zmm30, %zmm30
+        vpxorq	%zmm30, %zmm1, %zmm1
+        vpxorq	%zmm30, %zmm6, %zmm6
+        vpxorq	%zmm30, %zmm11, %zmm11
+        vpxorq	%zmm30, %zmm16, %zmm16
+        vpxorq	%zmm30, %zmm21, %zmm21
+        vprolq	$0x01, %zmm28, %zmm30
+        vpxorq	%zmm26, %zmm30, %zmm30
+        vpxorq	%zmm30, %zmm2, %zmm2
+        vpxorq	%zmm30, %zmm7, %zmm7
+        vpxorq	%zmm30, %zmm12, %zmm12
+        vpxorq	%zmm30, %zmm17, %zmm17
+        vpxorq	%zmm30, %zmm22, %zmm22
+        vprolq	$0x01, %zmm29, %zmm30
+        vpxorq	%zmm27, %zmm30, %zmm30
+        vpxorq	%zmm30, %zmm3, %zmm3
+        vpxorq	%zmm30, %zmm8, %zmm8
+        vpxorq	%zmm30, %zmm13, %zmm13
+        vpxorq	%zmm30, %zmm18, %zmm18
+        vpxorq	%zmm30, %zmm23, %zmm23
+        vprolq	$0x01, %zmm25, %zmm30
+        vpxorq	%zmm28, %zmm30, %zmm30
+        vpxorq	%zmm30, %zmm4, %zmm4
+        vpxorq	%zmm30, %zmm9, %zmm9
+        vpxorq	%zmm30, %zmm14, %zmm14
+        vpxorq	%zmm30, %zmm19, %zmm19
+        vpxorq	%zmm30, %zmm24, %zmm24
+        # Rho - rotate each lane in place
+        vprolq	$0x01, %zmm1, %zmm1
+        vprolq	$62, %zmm2, %zmm2
+        vprolq	$28, %zmm3, %zmm3
+        vprolq	$27, %zmm4, %zmm4
+        vprolq	$36, %zmm5, %zmm5
+        vprolq	$44, %zmm6, %zmm6
+        vprolq	$6, %zmm7, %zmm7
+        vprolq	$55, %zmm8, %zmm8
+        vprolq	$20, %zmm9, %zmm9
+        vprolq	$3, %zmm10, %zmm10
+        vprolq	$10, %zmm11, %zmm11
+        vprolq	$43, %zmm12, %zmm12
+        vprolq	$25, %zmm13, %zmm13
+        vprolq	$39, %zmm14, %zmm14
+        vprolq	$41, %zmm15, %zmm15
+        vprolq	$45, %zmm16, %zmm16
+        vprolq	$15, %zmm17, %zmm17
+        vprolq	$21, %zmm18, %zmm18
+        vprolq	$8, %zmm19, %zmm19
+        vprolq	$18, %zmm20, %zmm20
+        vprolq	$2, %zmm21, %zmm21
+        vprolq	$61, %zmm22, %zmm22
+        vprolq	$56, %zmm23, %zmm23
+        vprolq	$14, %zmm24, %zmm24
+        # Pi - relabel logical -> physical
+        # Chi - per row, in place, fused with vpternlogq (imm 0xd2)
+        vmovdqu64	%zmm0, %zmm30
+        vmovdqu64	%zmm6, %zmm31
+        vpternlogq	$0xd2, %zmm12, %zmm6, %zmm0
+        vpternlogq	$0xd2, %zmm18, %zmm12, %zmm6
+        vpternlogq	$0xd2, %zmm24, %zmm18, %zmm12
+        vpternlogq	$0xd2, %zmm30, %zmm24, %zmm18
+        vpternlogq	$0xd2, %zmm31, %zmm30, %zmm24
+        vmovdqu64	%zmm3, %zmm30
+        vmovdqu64	%zmm9, %zmm31
+        vpternlogq	$0xd2, %zmm10, %zmm9, %zmm3
+        vpternlogq	$0xd2, %zmm16, %zmm10, %zmm9
+        vpternlogq	$0xd2, %zmm22, %zmm16, %zmm10
+        vpternlogq	$0xd2, %zmm30, %zmm22, %zmm16
+        vpternlogq	$0xd2, %zmm31, %zmm30, %zmm22
+        vmovdqu64	%zmm1, %zmm30
+        vmovdqu64	%zmm7, %zmm31
+        vpternlogq	$0xd2, %zmm13, %zmm7, %zmm1
+        vpternlogq	$0xd2, %zmm19, %zmm13, %zmm7
+        vpternlogq	$0xd2, %zmm20, %zmm19, %zmm13
+        vpternlogq	$0xd2, %zmm30, %zmm20, %zmm19
+        vpternlogq	$0xd2, %zmm31, %zmm30, %zmm20
+        vmovdqu64	%zmm4, %zmm30
+        vmovdqu64	%zmm5, %zmm31
+        vpternlogq	$0xd2, %zmm11, %zmm5, %zmm4
+        vpternlogq	$0xd2, %zmm17, %zmm11, %zmm5
+        vpternlogq	$0xd2, %zmm23, %zmm17, %zmm11
+        vpternlogq	$0xd2, %zmm30, %zmm23, %zmm17
+        vpternlogq	$0xd2, %zmm31, %zmm30, %zmm23
+        vmovdqu64	%zmm2, %zmm30
+        vmovdqu64	%zmm8, %zmm31
+        vpternlogq	$0xd2, %zmm14, %zmm8, %zmm2
+        vpternlogq	$0xd2, %zmm15, %zmm14, %zmm8
+        vpternlogq	$0xd2, %zmm21, %zmm15, %zmm14
+        vpternlogq	$0xd2, %zmm30, %zmm21, %zmm15
+        vpternlogq	$0xd2, %zmm31, %zmm30, %zmm21
+        # Iota
+        vpxorq	(%rax), %zmm0, %zmm0
+        # Round 1
+        # Theta - column parity
+        vmovdqu64	%zmm0, %zmm25
+        vpternlogq	$0x96, %zmm1, %zmm3, %zmm25
+        vpternlogq	$0x96, %zmm2, %zmm4, %zmm25
+        vmovdqu64	%zmm6, %zmm26
+        vpternlogq	$0x96, %zmm7, %zmm9, %zmm26
+        vpternlogq	$0x96, %zmm8, %zmm5, %zmm26
+        vmovdqu64	%zmm12, %zmm27
+        vpternlogq	$0x96, %zmm13, %zmm10, %zmm27
+        vpternlogq	$0x96, %zmm14, %zmm11, %zmm27
+        vmovdqu64	%zmm18, %zmm28
+        vpternlogq	$0x96, %zmm19, %zmm16, %zmm28
+        vpternlogq	$0x96, %zmm15, %zmm17, %zmm28
+        vmovdqu64	%zmm24, %zmm29
+        vpternlogq	$0x96, %zmm20, %zmm22, %zmm29
+        vpternlogq	$0x96, %zmm21, %zmm23, %zmm29
+        # Theta - D and apply in place
+        vprolq	$0x01, %zmm26, %zmm30
+        vpxorq	%zmm29, %zmm30, %zmm30
+        vpxorq	%zmm30, %zmm0, %zmm0
+        vpxorq	%zmm30, %zmm3, %zmm3
+        vpxorq	%zmm30, %zmm1, %zmm1
+        vpxorq	%zmm30, %zmm4, %zmm4
+        vpxorq	%zmm30, %zmm2, %zmm2
+        vprolq	$0x01, %zmm27, %zmm30
+        vpxorq	%zmm25, %zmm30, %zmm30
+        vpxorq	%zmm30, %zmm6, %zmm6
+        vpxorq	%zmm30, %zmm9, %zmm9
+        vpxorq	%zmm30, %zmm7, %zmm7
+        vpxorq	%zmm30, %zmm5, %zmm5
+        vpxorq	%zmm30, %zmm8, %zmm8
+        vprolq	$0x01, %zmm28, %zmm30
+        vpxorq	%zmm26, %zmm30, %zmm30
+        vpxorq	%zmm30, %zmm12, %zmm12
+        vpxorq	%zmm30, %zmm10, %zmm10
+        vpxorq	%zmm30, %zmm13, %zmm13
+        vpxorq	%zmm30, %zmm11, %zmm11
+        vpxorq	%zmm30, %zmm14, %zmm14
+        vprolq	$0x01, %zmm29, %zmm30
+        vpxorq	%zmm27, %zmm30, %zmm30
+        vpxorq	%zmm30, %zmm18, %zmm18
+        vpxorq	%zmm30, %zmm16, %zmm16
+        vpxorq	%zmm30, %zmm19, %zmm19
+        vpxorq	%zmm30, %zmm17, %zmm17
+        vpxorq	%zmm30, %zmm15, %zmm15
+        vprolq	$0x01, %zmm25, %zmm30
+        vpxorq	%zmm28, %zmm30, %zmm30
+        vpxorq	%zmm30, %zmm24, %zmm24
+        vpxorq	%zmm30, %zmm22, %zmm22
+        vpxorq	%zmm30, %zmm20, %zmm20
+        vpxorq	%zmm30, %zmm23, %zmm23
+        vpxorq	%zmm30, %zmm21, %zmm21
+        # Rho - rotate each lane in place
+        vprolq	$0x01, %zmm6, %zmm6
+        vprolq	$62, %zmm12, %zmm12
+        vprolq	$28, %zmm18, %zmm18
+        vprolq	$27, %zmm24, %zmm24
+        vprolq	$36, %zmm3, %zmm3
+        vprolq	$44, %zmm9, %zmm9
+        vprolq	$6, %zmm10, %zmm10
+        vprolq	$55, %zmm16, %zmm16
+        vprolq	$20, %zmm22, %zmm22
+        vprolq	$3, %zmm1, %zmm1
+        vprolq	$10, %zmm7, %zmm7
+        vprolq	$43, %zmm13, %zmm13
+        vprolq	$25, %zmm19, %zmm19
+        vprolq	$39, %zmm20, %zmm20
+        vprolq	$41, %zmm4, %zmm4
+        vprolq	$45, %zmm5, %zmm5
+        vprolq	$15, %zmm11, %zmm11
+        vprolq	$21, %zmm17, %zmm17
+        vprolq	$8, %zmm23, %zmm23
+        vprolq	$18, %zmm2, %zmm2
+        vprolq	$2, %zmm8, %zmm8
+        vprolq	$61, %zmm14, %zmm14
+        vprolq	$56, %zmm15, %zmm15
+        vprolq	$14, %zmm21, %zmm21
+        # Pi - relabel logical -> physical
+        # Chi - per row, in place, fused with vpternlogq (imm 0xd2)
+        vmovdqu64	%zmm0, %zmm30
+        vmovdqu64	%zmm9, %zmm31
+        vpternlogq	$0xd2, %zmm13, %zmm9, %zmm0
+        vpternlogq	$0xd2, %zmm17, %zmm13, %zmm9
+        vpternlogq	$0xd2, %zmm21, %zmm17, %zmm13
+        vpternlogq	$0xd2, %zmm30, %zmm21, %zmm17
+        vpternlogq	$0xd2, %zmm31, %zmm30, %zmm21
+        vmovdqu64	%zmm18, %zmm30
+        vmovdqu64	%zmm22, %zmm31
+        vpternlogq	$0xd2, %zmm1, %zmm22, %zmm18
+        vpternlogq	$0xd2, %zmm5, %zmm1, %zmm22
+        vpternlogq	$0xd2, %zmm14, %zmm5, %zmm1
+        vpternlogq	$0xd2, %zmm30, %zmm14, %zmm5
+        vpternlogq	$0xd2, %zmm31, %zmm30, %zmm14
+        vmovdqu64	%zmm6, %zmm30
+        vmovdqu64	%zmm10, %zmm31
+        vpternlogq	$0xd2, %zmm19, %zmm10, %zmm6
+        vpternlogq	$0xd2, %zmm23, %zmm19, %zmm10
+        vpternlogq	$0xd2, %zmm2, %zmm23, %zmm19
+        vpternlogq	$0xd2, %zmm30, %zmm2, %zmm23
+        vpternlogq	$0xd2, %zmm31, %zmm30, %zmm2
+        vmovdqu64	%zmm24, %zmm30
+        vmovdqu64	%zmm3, %zmm31
+        vpternlogq	$0xd2, %zmm7, %zmm3, %zmm24
+        vpternlogq	$0xd2, %zmm11, %zmm7, %zmm3
+        vpternlogq	$0xd2, %zmm15, %zmm11, %zmm7
+        vpternlogq	$0xd2, %zmm30, %zmm15, %zmm11
+        vpternlogq	$0xd2, %zmm31, %zmm30, %zmm15
+        vmovdqu64	%zmm12, %zmm30
+        vmovdqu64	%zmm16, %zmm31
+        vpternlogq	$0xd2, %zmm20, %zmm16, %zmm12
+        vpternlogq	$0xd2, %zmm4, %zmm20, %zmm16
+        vpternlogq	$0xd2, %zmm8, %zmm4, %zmm20
+        vpternlogq	$0xd2, %zmm30, %zmm8, %zmm4
+        vpternlogq	$0xd2, %zmm31, %zmm30, %zmm8
+        # Iota
+        vpxorq	64(%rax), %zmm0, %zmm0
+        # Round 2
+        # Theta - column parity
+        vmovdqu64	%zmm0, %zmm25
+        vpternlogq	$0x96, %zmm6, %zmm18, %zmm25
+        vpternlogq	$0x96, %zmm12, %zmm24, %zmm25
+        vmovdqu64	%zmm9, %zmm26
+        vpternlogq	$0x96, %zmm10, %zmm22, %zmm26
+        vpternlogq	$0x96, %zmm16, %zmm3, %zmm26
+        vmovdqu64	%zmm13, %zmm27
+        vpternlogq	$0x96, %zmm19, %zmm1, %zmm27
+        vpternlogq	$0x96, %zmm20, %zmm7, %zmm27
+        vmovdqu64	%zmm17, %zmm28
+        vpternlogq	$0x96, %zmm23, %zmm5, %zmm28
+        vpternlogq	$0x96, %zmm4, %zmm11, %zmm28
+        vmovdqu64	%zmm21, %zmm29
+        vpternlogq	$0x96, %zmm2, %zmm14, %zmm29
+        vpternlogq	$0x96, %zmm8, %zmm15, %zmm29
+        # Theta - D and apply in place
+        vprolq	$0x01, %zmm26, %zmm30
+        vpxorq	%zmm29, %zmm30, %zmm30
+        vpxorq	%zmm30, %zmm0, %zmm0
+        vpxorq	%zmm30, %zmm18, %zmm18
+        vpxorq	%zmm30, %zmm6, %zmm6
+        vpxorq	%zmm30, %zmm24, %zmm24
+        vpxorq	%zmm30, %zmm12, %zmm12
+        vprolq	$0x01, %zmm27, %zmm30
+        vpxorq	%zmm25, %zmm30, %zmm30
+        vpxorq	%zmm30, %zmm9, %zmm9
+        vpxorq	%zmm30, %zmm22, %zmm22
+        vpxorq	%zmm30, %zmm10, %zmm10
+        vpxorq	%zmm30, %zmm3, %zmm3
+        vpxorq	%zmm30, %zmm16, %zmm16
+        vprolq	$0x01, %zmm28, %zmm30
+        vpxorq	%zmm26, %zmm30, %zmm30
+        vpxorq	%zmm30, %zmm13, %zmm13
+        vpxorq	%zmm30, %zmm1, %zmm1
+        vpxorq	%zmm30, %zmm19, %zmm19
+        vpxorq	%zmm30, %zmm7, %zmm7
+        vpxorq	%zmm30, %zmm20, %zmm20
+        vprolq	$0x01, %zmm29, %zmm30
+        vpxorq	%zmm27, %zmm30, %zmm30
+        vpxorq	%zmm30, %zmm17, %zmm17
+        vpxorq	%zmm30, %zmm5, %zmm5
+        vpxorq	%zmm30, %zmm23, %zmm23
+        vpxorq	%zmm30, %zmm11, %zmm11
+        vpxorq	%zmm30, %zmm4, %zmm4
+        vprolq	$0x01, %zmm25, %zmm30
+        vpxorq	%zmm28, %zmm30, %zmm30
+        vpxorq	%zmm30, %zmm21, %zmm21
+        vpxorq	%zmm30, %zmm14, %zmm14
+        vpxorq	%zmm30, %zmm2, %zmm2
+        vpxorq	%zmm30, %zmm15, %zmm15
+        vpxorq	%zmm30, %zmm8, %zmm8
+        # Rho - rotate each lane in place
+        vprolq	$0x01, %zmm9, %zmm9
+        vprolq	$62, %zmm13, %zmm13
+        vprolq	$28, %zmm17, %zmm17
+        vprolq	$27, %zmm21, %zmm21
+        vprolq	$36, %zmm18, %zmm18
+        vprolq	$44, %zmm22, %zmm22
+        vprolq	$6, %zmm1, %zmm1
+        vprolq	$55, %zmm5, %zmm5
+        vprolq	$20, %zmm14, %zmm14
+        vprolq	$3, %zmm6, %zmm6
+        vprolq	$10, %zmm10, %zmm10
+        vprolq	$43, %zmm19, %zmm19
+        vprolq	$25, %zmm23, %zmm23
+        vprolq	$39, %zmm2, %zmm2
+        vprolq	$41, %zmm24, %zmm24
+        vprolq	$45, %zmm3, %zmm3
+        vprolq	$15, %zmm7, %zmm7
+        vprolq	$21, %zmm11, %zmm11
+        vprolq	$8, %zmm15, %zmm15
+        vprolq	$18, %zmm12, %zmm12
+        vprolq	$2, %zmm16, %zmm16
+        vprolq	$61, %zmm20, %zmm20
+        vprolq	$56, %zmm4, %zmm4
+        vprolq	$14, %zmm8, %zmm8
+        # Pi - relabel logical -> physical
+        # Chi - per row, in place, fused with vpternlogq (imm 0xd2)
+        vmovdqu64	%zmm0, %zmm30
+        vmovdqu64	%zmm22, %zmm31
+        vpternlogq	$0xd2, %zmm19, %zmm22, %zmm0
+        vpternlogq	$0xd2, %zmm11, %zmm19, %zmm22
+        vpternlogq	$0xd2, %zmm8, %zmm11, %zmm19
+        vpternlogq	$0xd2, %zmm30, %zmm8, %zmm11
+        vpternlogq	$0xd2, %zmm31, %zmm30, %zmm8
+        vmovdqu64	%zmm17, %zmm30
+        vmovdqu64	%zmm14, %zmm31
+        vpternlogq	$0xd2, %zmm6, %zmm14, %zmm17
+        vpternlogq	$0xd2, %zmm3, %zmm6, %zmm14
+        vpternlogq	$0xd2, %zmm20, %zmm3, %zmm6
+        vpternlogq	$0xd2, %zmm30, %zmm20, %zmm3
+        vpternlogq	$0xd2, %zmm31, %zmm30, %zmm20
+        vmovdqu64	%zmm9, %zmm30
+        vmovdqu64	%zmm1, %zmm31
+        vpternlogq	$0xd2, %zmm23, %zmm1, %zmm9
+        vpternlogq	$0xd2, %zmm15, %zmm23, %zmm1
+        vpternlogq	$0xd2, %zmm12, %zmm15, %zmm23
+        vpternlogq	$0xd2, %zmm30, %zmm12, %zmm15
+        vpternlogq	$0xd2, %zmm31, %zmm30, %zmm12
+        vmovdqu64	%zmm21, %zmm30
+        vmovdqu64	%zmm18, %zmm31
+        vpternlogq	$0xd2, %zmm10, %zmm18, %zmm21
+        vpternlogq	$0xd2, %zmm7, %zmm10, %zmm18
+        vpternlogq	$0xd2, %zmm4, %zmm7, %zmm10
+        vpternlogq	$0xd2, %zmm30, %zmm4, %zmm7
+        vpternlogq	$0xd2, %zmm31, %zmm30, %zmm4
+        vmovdqu64	%zmm13, %zmm30
+        vmovdqu64	%zmm5, %zmm31
+        vpternlogq	$0xd2, %zmm2, %zmm5, %zmm13
+        vpternlogq	$0xd2, %zmm24, %zmm2, %zmm5
+        vpternlogq	$0xd2, %zmm16, %zmm24, %zmm2
+        vpternlogq	$0xd2, %zmm30, %zmm16, %zmm24
+        vpternlogq	$0xd2, %zmm31, %zmm30, %zmm16
+        # Iota
+        vpxorq	128(%rax), %zmm0, %zmm0
+        # Round 3
+        # Theta - column parity
+        vmovdqu64	%zmm0, %zmm25
+        vpternlogq	$0x96, %zmm9, %zmm17, %zmm25
+        vpternlogq	$0x96, %zmm13, %zmm21, %zmm25
+        vmovdqu64	%zmm22, %zmm26
+        vpternlogq	$0x96, %zmm1, %zmm14, %zmm26
+        vpternlogq	$0x96, %zmm5, %zmm18, %zmm26
+        vmovdqu64	%zmm19, %zmm27
+        vpternlogq	$0x96, %zmm23, %zmm6, %zmm27
+        vpternlogq	$0x96, %zmm2, %zmm10, %zmm27
+        vmovdqu64	%zmm11, %zmm28
+        vpternlogq	$0x96, %zmm15, %zmm3, %zmm28
+        vpternlogq	$0x96, %zmm24, %zmm7, %zmm28
+        vmovdqu64	%zmm8, %zmm29
+        vpternlogq	$0x96, %zmm12, %zmm20, %zmm29
+        vpternlogq	$0x96, %zmm16, %zmm4, %zmm29
+        # Theta - D and apply in place
+        vprolq	$0x01, %zmm26, %zmm30
+        vpxorq	%zmm29, %zmm30, %zmm30
+        vpxorq	%zmm30, %zmm0, %zmm0
+        vpxorq	%zmm30, %zmm17, %zmm17
+        vpxorq	%zmm30, %zmm9, %zmm9
+        vpxorq	%zmm30, %zmm21, %zmm21
+        vpxorq	%zmm30, %zmm13, %zmm13
+        vprolq	$0x01, %zmm27, %zmm30
+        vpxorq	%zmm25, %zmm30, %zmm30
+        vpxorq	%zmm30, %zmm22, %zmm22
+        vpxorq	%zmm30, %zmm14, %zmm14
+        vpxorq	%zmm30, %zmm1, %zmm1
+        vpxorq	%zmm30, %zmm18, %zmm18
+        vpxorq	%zmm30, %zmm5, %zmm5
+        vprolq	$0x01, %zmm28, %zmm30
+        vpxorq	%zmm26, %zmm30, %zmm30
+        vpxorq	%zmm30, %zmm19, %zmm19
+        vpxorq	%zmm30, %zmm6, %zmm6
+        vpxorq	%zmm30, %zmm23, %zmm23
+        vpxorq	%zmm30, %zmm10, %zmm10
+        vpxorq	%zmm30, %zmm2, %zmm2
+        vprolq	$0x01, %zmm29, %zmm30
+        vpxorq	%zmm27, %zmm30, %zmm30
+        vpxorq	%zmm30, %zmm11, %zmm11
+        vpxorq	%zmm30, %zmm3, %zmm3
+        vpxorq	%zmm30, %zmm15, %zmm15
+        vpxorq	%zmm30, %zmm7, %zmm7
+        vpxorq	%zmm30, %zmm24, %zmm24
+        vprolq	$0x01, %zmm25, %zmm30
+        vpxorq	%zmm28, %zmm30, %zmm30
+        vpxorq	%zmm30, %zmm8, %zmm8
+        vpxorq	%zmm30, %zmm20, %zmm20
+        vpxorq	%zmm30, %zmm12, %zmm12
+        vpxorq	%zmm30, %zmm4, %zmm4
+        vpxorq	%zmm30, %zmm16, %zmm16
+        # Rho - rotate each lane in place
+        vprolq	$0x01, %zmm22, %zmm22
+        vprolq	$62, %zmm19, %zmm19
+        vprolq	$28, %zmm11, %zmm11
+        vprolq	$27, %zmm8, %zmm8
+        vprolq	$36, %zmm17, %zmm17
+        vprolq	$44, %zmm14, %zmm14
+        vprolq	$6, %zmm6, %zmm6
+        vprolq	$55, %zmm3, %zmm3
+        vprolq	$20, %zmm20, %zmm20
+        vprolq	$3, %zmm9, %zmm9
+        vprolq	$10, %zmm1, %zmm1
+        vprolq	$43, %zmm23, %zmm23
+        vprolq	$25, %zmm15, %zmm15
+        vprolq	$39, %zmm12, %zmm12
+        vprolq	$41, %zmm21, %zmm21
+        vprolq	$45, %zmm18, %zmm18
+        vprolq	$15, %zmm10, %zmm10
+        vprolq	$21, %zmm7, %zmm7
+        vprolq	$8, %zmm4, %zmm4
+        vprolq	$18, %zmm13, %zmm13
+        vprolq	$2, %zmm5, %zmm5
+        vprolq	$61, %zmm2, %zmm2
+        vprolq	$56, %zmm24, %zmm24
+        vprolq	$14, %zmm16, %zmm16
+        # Pi - relabel logical -> physical
+        # Chi - per row, in place, fused with vpternlogq (imm 0xd2)
+        vmovdqu64	%zmm0, %zmm30
+        vmovdqu64	%zmm14, %zmm31
+        vpternlogq	$0xd2, %zmm23, %zmm14, %zmm0
+        vpternlogq	$0xd2, %zmm7, %zmm23, %zmm14
+        vpternlogq	$0xd2, %zmm16, %zmm7, %zmm23
+        vpternlogq	$0xd2, %zmm30, %zmm16, %zmm7
+        vpternlogq	$0xd2, %zmm31, %zmm30, %zmm16
+        vmovdqu64	%zmm11, %zmm30
+        vmovdqu64	%zmm20, %zmm31
+        vpternlogq	$0xd2, %zmm9, %zmm20, %zmm11
+        vpternlogq	$0xd2, %zmm18, %zmm9, %zmm20
+        vpternlogq	$0xd2, %zmm2, %zmm18, %zmm9
+        vpternlogq	$0xd2, %zmm30, %zmm2, %zmm18
+        vpternlogq	$0xd2, %zmm31, %zmm30, %zmm2
+        vmovdqu64	%zmm22, %zmm30
+        vmovdqu64	%zmm6, %zmm31
+        vpternlogq	$0xd2, %zmm15, %zmm6, %zmm22
+        vpternlogq	$0xd2, %zmm4, %zmm15, %zmm6
+        vpternlogq	$0xd2, %zmm13, %zmm4, %zmm15
+        vpternlogq	$0xd2, %zmm30, %zmm13, %zmm4
+        vpternlogq	$0xd2, %zmm31, %zmm30, %zmm13
+        vmovdqu64	%zmm8, %zmm30
+        vmovdqu64	%zmm17, %zmm31
+        vpternlogq	$0xd2, %zmm1, %zmm17, %zmm8
+        vpternlogq	$0xd2, %zmm10, %zmm1, %zmm17
+        vpternlogq	$0xd2, %zmm24, %zmm10, %zmm1
+        vpternlogq	$0xd2, %zmm30, %zmm24, %zmm10
+        vpternlogq	$0xd2, %zmm31, %zmm30, %zmm24
+        vmovdqu64	%zmm19, %zmm30
+        vmovdqu64	%zmm3, %zmm31
+        vpternlogq	$0xd2, %zmm12, %zmm3, %zmm19
+        vpternlogq	$0xd2, %zmm21, %zmm12, %zmm3
+        vpternlogq	$0xd2, %zmm5, %zmm21, %zmm12
+        vpternlogq	$0xd2, %zmm30, %zmm5, %zmm21
+        vpternlogq	$0xd2, %zmm31, %zmm30, %zmm5
+        # Iota
+        vpxorq	192(%rax), %zmm0, %zmm0
+        # Round 4
+        # Theta - column parity
+        vmovdqu64	%zmm0, %zmm25
+        vpternlogq	$0x96, %zmm22, %zmm11, %zmm25
+        vpternlogq	$0x96, %zmm19, %zmm8, %zmm25
+        vmovdqu64	%zmm14, %zmm26
+        vpternlogq	$0x96, %zmm6, %zmm20, %zmm26
+        vpternlogq	$0x96, %zmm3, %zmm17, %zmm26
+        vmovdqu64	%zmm23, %zmm27
+        vpternlogq	$0x96, %zmm15, %zmm9, %zmm27
+        vpternlogq	$0x96, %zmm12, %zmm1, %zmm27
+        vmovdqu64	%zmm7, %zmm28
+        vpternlogq	$0x96, %zmm4, %zmm18, %zmm28
+        vpternlogq	$0x96, %zmm21, %zmm10, %zmm28
+        vmovdqu64	%zmm16, %zmm29
+        vpternlogq	$0x96, %zmm13, %zmm2, %zmm29
+        vpternlogq	$0x96, %zmm5, %zmm24, %zmm29
+        # Theta - D and apply in place
+        vprolq	$0x01, %zmm26, %zmm30
+        vpxorq	%zmm29, %zmm30, %zmm30
+        vpxorq	%zmm30, %zmm0, %zmm0
+        vpxorq	%zmm30, %zmm11, %zmm11
+        vpxorq	%zmm30, %zmm22, %zmm22
+        vpxorq	%zmm30, %zmm8, %zmm8
+        vpxorq	%zmm30, %zmm19, %zmm19
+        vprolq	$0x01, %zmm27, %zmm30
+        vpxorq	%zmm25, %zmm30, %zmm30
+        vpxorq	%zmm30, %zmm14, %zmm14
+        vpxorq	%zmm30, %zmm20, %zmm20
+        vpxorq	%zmm30, %zmm6, %zmm6
+        vpxorq	%zmm30, %zmm17, %zmm17
+        vpxorq	%zmm30, %zmm3, %zmm3
+        vprolq	$0x01, %zmm28, %zmm30
+        vpxorq	%zmm26, %zmm30, %zmm30
+        vpxorq	%zmm30, %zmm23, %zmm23
+        vpxorq	%zmm30, %zmm9, %zmm9
+        vpxorq	%zmm30, %zmm15, %zmm15
+        vpxorq	%zmm30, %zmm1, %zmm1
+        vpxorq	%zmm30, %zmm12, %zmm12
+        vprolq	$0x01, %zmm29, %zmm30
+        vpxorq	%zmm27, %zmm30, %zmm30
+        vpxorq	%zmm30, %zmm7, %zmm7
+        vpxorq	%zmm30, %zmm18, %zmm18
+        vpxorq	%zmm30, %zmm4, %zmm4
+        vpxorq	%zmm30, %zmm10, %zmm10
+        vpxorq	%zmm30, %zmm21, %zmm21
+        vprolq	$0x01, %zmm25, %zmm30
+        vpxorq	%zmm28, %zmm30, %zmm30
+        vpxorq	%zmm30, %zmm16, %zmm16
+        vpxorq	%zmm30, %zmm2, %zmm2
+        vpxorq	%zmm30, %zmm13, %zmm13
+        vpxorq	%zmm30, %zmm24, %zmm24
+        vpxorq	%zmm30, %zmm5, %zmm5
+        # Rho - rotate each lane in place
+        vprolq	$0x01, %zmm14, %zmm14
+        vprolq	$62, %zmm23, %zmm23
+        vprolq	$28, %zmm7, %zmm7
+        vprolq	$27, %zmm16, %zmm16
+        vprolq	$36, %zmm11, %zmm11
+        vprolq	$44, %zmm20, %zmm20
+        vprolq	$6, %zmm9, %zmm9
+        vprolq	$55, %zmm18, %zmm18
+        vprolq	$20, %zmm2, %zmm2
+        vprolq	$3, %zmm22, %zmm22
+        vprolq	$10, %zmm6, %zmm6
+        vprolq	$43, %zmm15, %zmm15
+        vprolq	$25, %zmm4, %zmm4
+        vprolq	$39, %zmm13, %zmm13
+        vprolq	$41, %zmm8, %zmm8
+        vprolq	$45, %zmm17, %zmm17
+        vprolq	$15, %zmm1, %zmm1
+        vprolq	$21, %zmm10, %zmm10
+        vprolq	$8, %zmm24, %zmm24
+        vprolq	$18, %zmm19, %zmm19
+        vprolq	$2, %zmm3, %zmm3
+        vprolq	$61, %zmm12, %zmm12
+        vprolq	$56, %zmm21, %zmm21
+        vprolq	$14, %zmm5, %zmm5
+        # Pi - relabel logical -> physical
+        # Chi - per row, in place, fused with vpternlogq (imm 0xd2)
+        vmovdqu64	%zmm0, %zmm30
+        vmovdqu64	%zmm20, %zmm31
+        vpternlogq	$0xd2, %zmm15, %zmm20, %zmm0
+        vpternlogq	$0xd2, %zmm10, %zmm15, %zmm20
+        vpternlogq	$0xd2, %zmm5, %zmm10, %zmm15
+        vpternlogq	$0xd2, %zmm30, %zmm5, %zmm10
+        vpternlogq	$0xd2, %zmm31, %zmm30, %zmm5
+        vmovdqu64	%zmm7, %zmm30
+        vmovdqu64	%zmm2, %zmm31
+        vpternlogq	$0xd2, %zmm22, %zmm2, %zmm7
+        vpternlogq	$0xd2, %zmm17, %zmm22, %zmm2
+        vpternlogq	$0xd2, %zmm12, %zmm17, %zmm22
+        vpternlogq	$0xd2, %zmm30, %zmm12, %zmm17
+        vpternlogq	$0xd2, %zmm31, %zmm30, %zmm12
+        vmovdqu64	%zmm14, %zmm30
+        vmovdqu64	%zmm9, %zmm31
+        vpternlogq	$0xd2, %zmm4, %zmm9, %zmm14
+        vpternlogq	$0xd2, %zmm24, %zmm4, %zmm9
+        vpternlogq	$0xd2, %zmm19, %zmm24, %zmm4
+        vpternlogq	$0xd2, %zmm30, %zmm19, %zmm24
+        vpternlogq	$0xd2, %zmm31, %zmm30, %zmm19
+        vmovdqu64	%zmm16, %zmm30
+        vmovdqu64	%zmm11, %zmm31
+        vpternlogq	$0xd2, %zmm6, %zmm11, %zmm16
+        vpternlogq	$0xd2, %zmm1, %zmm6, %zmm11
+        vpternlogq	$0xd2, %zmm21, %zmm1, %zmm6
+        vpternlogq	$0xd2, %zmm30, %zmm21, %zmm1
+        vpternlogq	$0xd2, %zmm31, %zmm30, %zmm21
+        vmovdqu64	%zmm23, %zmm30
+        vmovdqu64	%zmm18, %zmm31
+        vpternlogq	$0xd2, %zmm13, %zmm18, %zmm23
+        vpternlogq	$0xd2, %zmm8, %zmm13, %zmm18
+        vpternlogq	$0xd2, %zmm3, %zmm8, %zmm13
+        vpternlogq	$0xd2, %zmm30, %zmm3, %zmm8
+        vpternlogq	$0xd2, %zmm31, %zmm30, %zmm3
+        # Iota
+        vpxorq	256(%rax), %zmm0, %zmm0
+        # Round 5
+        # Theta - column parity
+        vmovdqu64	%zmm0, %zmm25
+        vpternlogq	$0x96, %zmm14, %zmm7, %zmm25
+        vpternlogq	$0x96, %zmm23, %zmm16, %zmm25
+        vmovdqu64	%zmm20, %zmm26
+        vpternlogq	$0x96, %zmm9, %zmm2, %zmm26
+        vpternlogq	$0x96, %zmm18, %zmm11, %zmm26
+        vmovdqu64	%zmm15, %zmm27
+        vpternlogq	$0x96, %zmm4, %zmm22, %zmm27
+        vpternlogq	$0x96, %zmm13, %zmm6, %zmm27
+        vmovdqu64	%zmm10, %zmm28
+        vpternlogq	$0x96, %zmm24, %zmm17, %zmm28
+        vpternlogq	$0x96, %zmm8, %zmm1, %zmm28
+        vmovdqu64	%zmm5, %zmm29
+        vpternlogq	$0x96, %zmm19, %zmm12, %zmm29
+        vpternlogq	$0x96, %zmm3, %zmm21, %zmm29
+        # Theta - D and apply in place
+        vprolq	$0x01, %zmm26, %zmm30
+        vpxorq	%zmm29, %zmm30, %zmm30
+        vpxorq	%zmm30, %zmm0, %zmm0
+        vpxorq	%zmm30, %zmm7, %zmm7
+        vpxorq	%zmm30, %zmm14, %zmm14
+        vpxorq	%zmm30, %zmm16, %zmm16
+        vpxorq	%zmm30, %zmm23, %zmm23
+        vprolq	$0x01, %zmm27, %zmm30
+        vpxorq	%zmm25, %zmm30, %zmm30
+        vpxorq	%zmm30, %zmm20, %zmm20
+        vpxorq	%zmm30, %zmm2, %zmm2
+        vpxorq	%zmm30, %zmm9, %zmm9
+        vpxorq	%zmm30, %zmm11, %zmm11
+        vpxorq	%zmm30, %zmm18, %zmm18
+        vprolq	$0x01, %zmm28, %zmm30
+        vpxorq	%zmm26, %zmm30, %zmm30
+        vpxorq	%zmm30, %zmm15, %zmm15
+        vpxorq	%zmm30, %zmm22, %zmm22
+        vpxorq	%zmm30, %zmm4, %zmm4
+        vpxorq	%zmm30, %zmm6, %zmm6
+        vpxorq	%zmm30, %zmm13, %zmm13
+        vprolq	$0x01, %zmm29, %zmm30
+        vpxorq	%zmm27, %zmm30, %zmm30
+        vpxorq	%zmm30, %zmm10, %zmm10
+        vpxorq	%zmm30, %zmm17, %zmm17
+        vpxorq	%zmm30, %zmm24, %zmm24
+        vpxorq	%zmm30, %zmm1, %zmm1
+        vpxorq	%zmm30, %zmm8, %zmm8
+        vprolq	$0x01, %zmm25, %zmm30
+        vpxorq	%zmm28, %zmm30, %zmm30
+        vpxorq	%zmm30, %zmm5, %zmm5
+        vpxorq	%zmm30, %zmm12, %zmm12
+        vpxorq	%zmm30, %zmm19, %zmm19
+        vpxorq	%zmm30, %zmm21, %zmm21
+        vpxorq	%zmm30, %zmm3, %zmm3
+        # Rho - rotate each lane in place
+        vprolq	$0x01, %zmm20, %zmm20
+        vprolq	$62, %zmm15, %zmm15
+        vprolq	$28, %zmm10, %zmm10
+        vprolq	$27, %zmm5, %zmm5
+        vprolq	$36, %zmm7, %zmm7
+        vprolq	$44, %zmm2, %zmm2
+        vprolq	$6, %zmm22, %zmm22
+        vprolq	$55, %zmm17, %zmm17
+        vprolq	$20, %zmm12, %zmm12
+        vprolq	$3, %zmm14, %zmm14
+        vprolq	$10, %zmm9, %zmm9
+        vprolq	$43, %zmm4, %zmm4
+        vprolq	$25, %zmm24, %zmm24
+        vprolq	$39, %zmm19, %zmm19
+        vprolq	$41, %zmm16, %zmm16
+        vprolq	$45, %zmm11, %zmm11
+        vprolq	$15, %zmm6, %zmm6
+        vprolq	$21, %zmm1, %zmm1
+        vprolq	$8, %zmm21, %zmm21
+        vprolq	$18, %zmm23, %zmm23
+        vprolq	$2, %zmm18, %zmm18
+        vprolq	$61, %zmm13, %zmm13
+        vprolq	$56, %zmm8, %zmm8
+        vprolq	$14, %zmm3, %zmm3
+        # Pi - relabel logical -> physical
+        # Chi - per row, in place, fused with vpternlogq (imm 0xd2)
+        vmovdqu64	%zmm0, %zmm30
+        vmovdqu64	%zmm2, %zmm31
+        vpternlogq	$0xd2, %zmm4, %zmm2, %zmm0
+        vpternlogq	$0xd2, %zmm1, %zmm4, %zmm2
+        vpternlogq	$0xd2, %zmm3, %zmm1, %zmm4
+        vpternlogq	$0xd2, %zmm30, %zmm3, %zmm1
+        vpternlogq	$0xd2, %zmm31, %zmm30, %zmm3
+        vmovdqu64	%zmm10, %zmm30
+        vmovdqu64	%zmm12, %zmm31
+        vpternlogq	$0xd2, %zmm14, %zmm12, %zmm10
+        vpternlogq	$0xd2, %zmm11, %zmm14, %zmm12
+        vpternlogq	$0xd2, %zmm13, %zmm11, %zmm14
+        vpternlogq	$0xd2, %zmm30, %zmm13, %zmm11
+        vpternlogq	$0xd2, %zmm31, %zmm30, %zmm13
+        vmovdqu64	%zmm20, %zmm30
+        vmovdqu64	%zmm22, %zmm31
+        vpternlogq	$0xd2, %zmm24, %zmm22, %zmm20
+        vpternlogq	$0xd2, %zmm21, %zmm24, %zmm22
+        vpternlogq	$0xd2, %zmm23, %zmm21, %zmm24
+        vpternlogq	$0xd2, %zmm30, %zmm23, %zmm21
+        vpternlogq	$0xd2, %zmm31, %zmm30, %zmm23
+        vmovdqu64	%zmm5, %zmm30
+        vmovdqu64	%zmm7, %zmm31
+        vpternlogq	$0xd2, %zmm9, %zmm7, %zmm5
+        vpternlogq	$0xd2, %zmm6, %zmm9, %zmm7
+        vpternlogq	$0xd2, %zmm8, %zmm6, %zmm9
+        vpternlogq	$0xd2, %zmm30, %zmm8, %zmm6
+        vpternlogq	$0xd2, %zmm31, %zmm30, %zmm8
+        vmovdqu64	%zmm15, %zmm30
+        vmovdqu64	%zmm17, %zmm31
+        vpternlogq	$0xd2, %zmm19, %zmm17, %zmm15
+        vpternlogq	$0xd2, %zmm16, %zmm19, %zmm17
+        vpternlogq	$0xd2, %zmm18, %zmm16, %zmm19
+        vpternlogq	$0xd2, %zmm30, %zmm18, %zmm16
+        vpternlogq	$0xd2, %zmm31, %zmm30, %zmm18
+        # Iota
+        vpxorq	320(%rax), %zmm0, %zmm0
+        # Round 6
+        # Theta - column parity
+        vmovdqu64	%zmm0, %zmm25
+        vpternlogq	$0x96, %zmm20, %zmm10, %zmm25
+        vpternlogq	$0x96, %zmm15, %zmm5, %zmm25
+        vmovdqu64	%zmm2, %zmm26
+        vpternlogq	$0x96, %zmm22, %zmm12, %zmm26
+        vpternlogq	$0x96, %zmm17, %zmm7, %zmm26
+        vmovdqu64	%zmm4, %zmm27
+        vpternlogq	$0x96, %zmm24, %zmm14, %zmm27
+        vpternlogq	$0x96, %zmm19, %zmm9, %zmm27
+        vmovdqu64	%zmm1, %zmm28
+        vpternlogq	$0x96, %zmm21, %zmm11, %zmm28
+        vpternlogq	$0x96, %zmm16, %zmm6, %zmm28
+        vmovdqu64	%zmm3, %zmm29
+        vpternlogq	$0x96, %zmm23, %zmm13, %zmm29
+        vpternlogq	$0x96, %zmm18, %zmm8, %zmm29
+        # Theta - D and apply in place
+        vprolq	$0x01, %zmm26, %zmm30
+        vpxorq	%zmm29, %zmm30, %zmm30
+        vpxorq	%zmm30, %zmm0, %zmm0
+        vpxorq	%zmm30, %zmm10, %zmm10
+        vpxorq	%zmm30, %zmm20, %zmm20
+        vpxorq	%zmm30, %zmm5, %zmm5
+        vpxorq	%zmm30, %zmm15, %zmm15
+        vprolq	$0x01, %zmm27, %zmm30
+        vpxorq	%zmm25, %zmm30, %zmm30
+        vpxorq	%zmm30, %zmm2, %zmm2
+        vpxorq	%zmm30, %zmm12, %zmm12
+        vpxorq	%zmm30, %zmm22, %zmm22
+        vpxorq	%zmm30, %zmm7, %zmm7
+        vpxorq	%zmm30, %zmm17, %zmm17
+        vprolq	$0x01, %zmm28, %zmm30
+        vpxorq	%zmm26, %zmm30, %zmm30
+        vpxorq	%zmm30, %zmm4, %zmm4
+        vpxorq	%zmm30, %zmm14, %zmm14
+        vpxorq	%zmm30, %zmm24, %zmm24
+        vpxorq	%zmm30, %zmm9, %zmm9
+        vpxorq	%zmm30, %zmm19, %zmm19
+        vprolq	$0x01, %zmm29, %zmm30
+        vpxorq	%zmm27, %zmm30, %zmm30
+        vpxorq	%zmm30, %zmm1, %zmm1
+        vpxorq	%zmm30, %zmm11, %zmm11
+        vpxorq	%zmm30, %zmm21, %zmm21
+        vpxorq	%zmm30, %zmm6, %zmm6
+        vpxorq	%zmm30, %zmm16, %zmm16
+        vprolq	$0x01, %zmm25, %zmm30
+        vpxorq	%zmm28, %zmm30, %zmm30
+        vpxorq	%zmm30, %zmm3, %zmm3
+        vpxorq	%zmm30, %zmm13, %zmm13
+        vpxorq	%zmm30, %zmm23, %zmm23
+        vpxorq	%zmm30, %zmm8, %zmm8
+        vpxorq	%zmm30, %zmm18, %zmm18
+        # Rho - rotate each lane in place
+        vprolq	$0x01, %zmm2, %zmm2
+        vprolq	$62, %zmm4, %zmm4
+        vprolq	$28, %zmm1, %zmm1
+        vprolq	$27, %zmm3, %zmm3
+        vprolq	$36, %zmm10, %zmm10
+        vprolq	$44, %zmm12, %zmm12
+        vprolq	$6, %zmm14, %zmm14
+        vprolq	$55, %zmm11, %zmm11
+        vprolq	$20, %zmm13, %zmm13
+        vprolq	$3, %zmm20, %zmm20
+        vprolq	$10, %zmm22, %zmm22
+        vprolq	$43, %zmm24, %zmm24
+        vprolq	$25, %zmm21, %zmm21
+        vprolq	$39, %zmm23, %zmm23
+        vprolq	$41, %zmm5, %zmm5
+        vprolq	$45, %zmm7, %zmm7
+        vprolq	$15, %zmm9, %zmm9
+        vprolq	$21, %zmm6, %zmm6
+        vprolq	$8, %zmm8, %zmm8
+        vprolq	$18, %zmm15, %zmm15
+        vprolq	$2, %zmm17, %zmm17
+        vprolq	$61, %zmm19, %zmm19
+        vprolq	$56, %zmm16, %zmm16
+        vprolq	$14, %zmm18, %zmm18
+        # Pi - relabel logical -> physical
+        # Chi - per row, in place, fused with vpternlogq (imm 0xd2)
+        vmovdqu64	%zmm0, %zmm30
+        vmovdqu64	%zmm12, %zmm31
+        vpternlogq	$0xd2, %zmm24, %zmm12, %zmm0
+        vpternlogq	$0xd2, %zmm6, %zmm24, %zmm12
+        vpternlogq	$0xd2, %zmm18, %zmm6, %zmm24
+        vpternlogq	$0xd2, %zmm30, %zmm18, %zmm6
+        vpternlogq	$0xd2, %zmm31, %zmm30, %zmm18
+        vmovdqu64	%zmm1, %zmm30
+        vmovdqu64	%zmm13, %zmm31
+        vpternlogq	$0xd2, %zmm20, %zmm13, %zmm1
+        vpternlogq	$0xd2, %zmm7, %zmm20, %zmm13
+        vpternlogq	$0xd2, %zmm19, %zmm7, %zmm20
+        vpternlogq	$0xd2, %zmm30, %zmm19, %zmm7
+        vpternlogq	$0xd2, %zmm31, %zmm30, %zmm19
+        vmovdqu64	%zmm2, %zmm30
+        vmovdqu64	%zmm14, %zmm31
+        vpternlogq	$0xd2, %zmm21, %zmm14, %zmm2
+        vpternlogq	$0xd2, %zmm8, %zmm21, %zmm14
+        vpternlogq	$0xd2, %zmm15, %zmm8, %zmm21
+        vpternlogq	$0xd2, %zmm30, %zmm15, %zmm8
+        vpternlogq	$0xd2, %zmm31, %zmm30, %zmm15
+        vmovdqu64	%zmm3, %zmm30
+        vmovdqu64	%zmm10, %zmm31
+        vpternlogq	$0xd2, %zmm22, %zmm10, %zmm3
+        vpternlogq	$0xd2, %zmm9, %zmm22, %zmm10
+        vpternlogq	$0xd2, %zmm16, %zmm9, %zmm22
+        vpternlogq	$0xd2, %zmm30, %zmm16, %zmm9
+        vpternlogq	$0xd2, %zmm31, %zmm30, %zmm16
+        vmovdqu64	%zmm4, %zmm30
+        vmovdqu64	%zmm11, %zmm31
+        vpternlogq	$0xd2, %zmm23, %zmm11, %zmm4
+        vpternlogq	$0xd2, %zmm5, %zmm23, %zmm11
+        vpternlogq	$0xd2, %zmm17, %zmm5, %zmm23
+        vpternlogq	$0xd2, %zmm30, %zmm17, %zmm5
+        vpternlogq	$0xd2, %zmm31, %zmm30, %zmm17
+        # Iota
+        vpxorq	384(%rax), %zmm0, %zmm0
+        # Round 7
+        # Theta - column parity
+        vmovdqu64	%zmm0, %zmm25
+        vpternlogq	$0x96, %zmm2, %zmm1, %zmm25
+        vpternlogq	$0x96, %zmm4, %zmm3, %zmm25
+        vmovdqu64	%zmm12, %zmm26
+        vpternlogq	$0x96, %zmm14, %zmm13, %zmm26
+        vpternlogq	$0x96, %zmm11, %zmm10, %zmm26
+        vmovdqu64	%zmm24, %zmm27
+        vpternlogq	$0x96, %zmm21, %zmm20, %zmm27
+        vpternlogq	$0x96, %zmm23, %zmm22, %zmm27
+        vmovdqu64	%zmm6, %zmm28
+        vpternlogq	$0x96, %zmm8, %zmm7, %zmm28
+        vpternlogq	$0x96, %zmm5, %zmm9, %zmm28
+        vmovdqu64	%zmm18, %zmm29
+        vpternlogq	$0x96, %zmm15, %zmm19, %zmm29
+        vpternlogq	$0x96, %zmm17, %zmm16, %zmm29
+        # Theta - D and apply in place
+        vprolq	$0x01, %zmm26, %zmm30
+        vpxorq	%zmm29, %zmm30, %zmm30
+        vpxorq	%zmm30, %zmm0, %zmm0
+        vpxorq	%zmm30, %zmm1, %zmm1
+        vpxorq	%zmm30, %zmm2, %zmm2
+        vpxorq	%zmm30, %zmm3, %zmm3
+        vpxorq	%zmm30, %zmm4, %zmm4
+        vprolq	$0x01, %zmm27, %zmm30
+        vpxorq	%zmm25, %zmm30, %zmm30
+        vpxorq	%zmm30, %zmm12, %zmm12
+        vpxorq	%zmm30, %zmm13, %zmm13
+        vpxorq	%zmm30, %zmm14, %zmm14
+        vpxorq	%zmm30, %zmm10, %zmm10
+        vpxorq	%zmm30, %zmm11, %zmm11
+        vprolq	$0x01, %zmm28, %zmm30
+        vpxorq	%zmm26, %zmm30, %zmm30
+        vpxorq	%zmm30, %zmm24, %zmm24
+        vpxorq	%zmm30, %zmm20, %zmm20
+        vpxorq	%zmm30, %zmm21, %zmm21
+        vpxorq	%zmm30, %zmm22, %zmm22
+        vpxorq	%zmm30, %zmm23, %zmm23
+        vprolq	$0x01, %zmm29, %zmm30
+        vpxorq	%zmm27, %zmm30, %zmm30
+        vpxorq	%zmm30, %zmm6, %zmm6
+        vpxorq	%zmm30, %zmm7, %zmm7
+        vpxorq	%zmm30, %zmm8, %zmm8
+        vpxorq	%zmm30, %zmm9, %zmm9
+        vpxorq	%zmm30, %zmm5, %zmm5
+        vprolq	$0x01, %zmm25, %zmm30
+        vpxorq	%zmm28, %zmm30, %zmm30
+        vpxorq	%zmm30, %zmm18, %zmm18
+        vpxorq	%zmm30, %zmm19, %zmm19
+        vpxorq	%zmm30, %zmm15, %zmm15
+        vpxorq	%zmm30, %zmm16, %zmm16
+        vpxorq	%zmm30, %zmm17, %zmm17
+        # Rho - rotate each lane in place
+        vprolq	$0x01, %zmm12, %zmm12
+        vprolq	$62, %zmm24, %zmm24
+        vprolq	$28, %zmm6, %zmm6
+        vprolq	$27, %zmm18, %zmm18
+        vprolq	$36, %zmm1, %zmm1
+        vprolq	$44, %zmm13, %zmm13
+        vprolq	$6, %zmm20, %zmm20
+        vprolq	$55, %zmm7, %zmm7
+        vprolq	$20, %zmm19, %zmm19
+        vprolq	$3, %zmm2, %zmm2
+        vprolq	$10, %zmm14, %zmm14
+        vprolq	$43, %zmm21, %zmm21
+        vprolq	$25, %zmm8, %zmm8
+        vprolq	$39, %zmm15, %zmm15
+        vprolq	$41, %zmm3, %zmm3
+        vprolq	$45, %zmm10, %zmm10
+        vprolq	$15, %zmm22, %zmm22
+        vprolq	$21, %zmm9, %zmm9
+        vprolq	$8, %zmm16, %zmm16
+        vprolq	$18, %zmm4, %zmm4
+        vprolq	$2, %zmm11, %zmm11
+        vprolq	$61, %zmm23, %zmm23
+        vprolq	$56, %zmm5, %zmm5
+        vprolq	$14, %zmm17, %zmm17
+        # Pi - relabel logical -> physical
+        # Chi - per row, in place, fused with vpternlogq (imm 0xd2)
+        vmovdqu64	%zmm0, %zmm30
+        vmovdqu64	%zmm13, %zmm31
+        vpternlogq	$0xd2, %zmm21, %zmm13, %zmm0
+        vpternlogq	$0xd2, %zmm9, %zmm21, %zmm13
+        vpternlogq	$0xd2, %zmm17, %zmm9, %zmm21
+        vpternlogq	$0xd2, %zmm30, %zmm17, %zmm9
+        vpternlogq	$0xd2, %zmm31, %zmm30, %zmm17
+        vmovdqu64	%zmm6, %zmm30
+        vmovdqu64	%zmm19, %zmm31
+        vpternlogq	$0xd2, %zmm2, %zmm19, %zmm6
+        vpternlogq	$0xd2, %zmm10, %zmm2, %zmm19
+        vpternlogq	$0xd2, %zmm23, %zmm10, %zmm2
+        vpternlogq	$0xd2, %zmm30, %zmm23, %zmm10
+        vpternlogq	$0xd2, %zmm31, %zmm30, %zmm23
+        vmovdqu64	%zmm12, %zmm30
+        vmovdqu64	%zmm20, %zmm31
+        vpternlogq	$0xd2, %zmm8, %zmm20, %zmm12
+        vpternlogq	$0xd2, %zmm16, %zmm8, %zmm20
+        vpternlogq	$0xd2, %zmm4, %zmm16, %zmm8
+        vpternlogq	$0xd2, %zmm30, %zmm4, %zmm16
+        vpternlogq	$0xd2, %zmm31, %zmm30, %zmm4
+        vmovdqu64	%zmm18, %zmm30
+        vmovdqu64	%zmm1, %zmm31
+        vpternlogq	$0xd2, %zmm14, %zmm1, %zmm18
+        vpternlogq	$0xd2, %zmm22, %zmm14, %zmm1
+        vpternlogq	$0xd2, %zmm5, %zmm22, %zmm14
+        vpternlogq	$0xd2, %zmm30, %zmm5, %zmm22
+        vpternlogq	$0xd2, %zmm31, %zmm30, %zmm5
+        vmovdqu64	%zmm24, %zmm30
+        vmovdqu64	%zmm7, %zmm31
+        vpternlogq	$0xd2, %zmm15, %zmm7, %zmm24
+        vpternlogq	$0xd2, %zmm3, %zmm15, %zmm7
+        vpternlogq	$0xd2, %zmm11, %zmm3, %zmm15
+        vpternlogq	$0xd2, %zmm30, %zmm11, %zmm3
+        vpternlogq	$0xd2, %zmm31, %zmm30, %zmm11
+        # Iota
+        vpxorq	448(%rax), %zmm0, %zmm0
+        # Round 8
+        # Theta - column parity
+        vmovdqu64	%zmm0, %zmm25
+        vpternlogq	$0x96, %zmm12, %zmm6, %zmm25
+        vpternlogq	$0x96, %zmm24, %zmm18, %zmm25
+        vmovdqu64	%zmm13, %zmm26
+        vpternlogq	$0x96, %zmm20, %zmm19, %zmm26
+        vpternlogq	$0x96, %zmm7, %zmm1, %zmm26
+        vmovdqu64	%zmm21, %zmm27
+        vpternlogq	$0x96, %zmm8, %zmm2, %zmm27
+        vpternlogq	$0x96, %zmm15, %zmm14, %zmm27
+        vmovdqu64	%zmm9, %zmm28
+        vpternlogq	$0x96, %zmm16, %zmm10, %zmm28
+        vpternlogq	$0x96, %zmm3, %zmm22, %zmm28
+        vmovdqu64	%zmm17, %zmm29
+        vpternlogq	$0x96, %zmm4, %zmm23, %zmm29
+        vpternlogq	$0x96, %zmm11, %zmm5, %zmm29
+        # Theta - D and apply in place
+        vprolq	$0x01, %zmm26, %zmm30
+        vpxorq	%zmm29, %zmm30, %zmm30
+        vpxorq	%zmm30, %zmm0, %zmm0
+        vpxorq	%zmm30, %zmm6, %zmm6
+        vpxorq	%zmm30, %zmm12, %zmm12
+        vpxorq	%zmm30, %zmm18, %zmm18
+        vpxorq	%zmm30, %zmm24, %zmm24
+        vprolq	$0x01, %zmm27, %zmm30
+        vpxorq	%zmm25, %zmm30, %zmm30
+        vpxorq	%zmm30, %zmm13, %zmm13
+        vpxorq	%zmm30, %zmm19, %zmm19
+        vpxorq	%zmm30, %zmm20, %zmm20
+        vpxorq	%zmm30, %zmm1, %zmm1
+        vpxorq	%zmm30, %zmm7, %zmm7
+        vprolq	$0x01, %zmm28, %zmm30
+        vpxorq	%zmm26, %zmm30, %zmm30
+        vpxorq	%zmm30, %zmm21, %zmm21
+        vpxorq	%zmm30, %zmm2, %zmm2
+        vpxorq	%zmm30, %zmm8, %zmm8
+        vpxorq	%zmm30, %zmm14, %zmm14
+        vpxorq	%zmm30, %zmm15, %zmm15
+        vprolq	$0x01, %zmm29, %zmm30
+        vpxorq	%zmm27, %zmm30, %zmm30
+        vpxorq	%zmm30, %zmm9, %zmm9
+        vpxorq	%zmm30, %zmm10, %zmm10
+        vpxorq	%zmm30, %zmm16, %zmm16
+        vpxorq	%zmm30, %zmm22, %zmm22
+        vpxorq	%zmm30, %zmm3, %zmm3
+        vprolq	$0x01, %zmm25, %zmm30
+        vpxorq	%zmm28, %zmm30, %zmm30
+        vpxorq	%zmm30, %zmm17, %zmm17
+        vpxorq	%zmm30, %zmm23, %zmm23
+        vpxorq	%zmm30, %zmm4, %zmm4
+        vpxorq	%zmm30, %zmm5, %zmm5
+        vpxorq	%zmm30, %zmm11, %zmm11
+        # Rho - rotate each lane in place
+        vprolq	$0x01, %zmm13, %zmm13
+        vprolq	$62, %zmm21, %zmm21
+        vprolq	$28, %zmm9, %zmm9
+        vprolq	$27, %zmm17, %zmm17
+        vprolq	$36, %zmm6, %zmm6
+        vprolq	$44, %zmm19, %zmm19
+        vprolq	$6, %zmm2, %zmm2
+        vprolq	$55, %zmm10, %zmm10
+        vprolq	$20, %zmm23, %zmm23
+        vprolq	$3, %zmm12, %zmm12
+        vprolq	$10, %zmm20, %zmm20
+        vprolq	$43, %zmm8, %zmm8
+        vprolq	$25, %zmm16, %zmm16
+        vprolq	$39, %zmm4, %zmm4
+        vprolq	$41, %zmm18, %zmm18
+        vprolq	$45, %zmm1, %zmm1
+        vprolq	$15, %zmm14, %zmm14
+        vprolq	$21, %zmm22, %zmm22
+        vprolq	$8, %zmm5, %zmm5
+        vprolq	$18, %zmm24, %zmm24
+        vprolq	$2, %zmm7, %zmm7
+        vprolq	$61, %zmm15, %zmm15
+        vprolq	$56, %zmm3, %zmm3
+        vprolq	$14, %zmm11, %zmm11
+        # Pi - relabel logical -> physical
+        # Chi - per row, in place, fused with vpternlogq (imm 0xd2)
+        vmovdqu64	%zmm0, %zmm30
+        vmovdqu64	%zmm19, %zmm31
+        vpternlogq	$0xd2, %zmm8, %zmm19, %zmm0
+        vpternlogq	$0xd2, %zmm22, %zmm8, %zmm19
+        vpternlogq	$0xd2, %zmm11, %zmm22, %zmm8
+        vpternlogq	$0xd2, %zmm30, %zmm11, %zmm22
+        vpternlogq	$0xd2, %zmm31, %zmm30, %zmm11
+        vmovdqu64	%zmm9, %zmm30
+        vmovdqu64	%zmm23, %zmm31
+        vpternlogq	$0xd2, %zmm12, %zmm23, %zmm9
+        vpternlogq	$0xd2, %zmm1, %zmm12, %zmm23
+        vpternlogq	$0xd2, %zmm15, %zmm1, %zmm12
+        vpternlogq	$0xd2, %zmm30, %zmm15, %zmm1
+        vpternlogq	$0xd2, %zmm31, %zmm30, %zmm15
+        vmovdqu64	%zmm13, %zmm30
+        vmovdqu64	%zmm2, %zmm31
+        vpternlogq	$0xd2, %zmm16, %zmm2, %zmm13
+        vpternlogq	$0xd2, %zmm5, %zmm16, %zmm2
+        vpternlogq	$0xd2, %zmm24, %zmm5, %zmm16
+        vpternlogq	$0xd2, %zmm30, %zmm24, %zmm5
+        vpternlogq	$0xd2, %zmm31, %zmm30, %zmm24
+        vmovdqu64	%zmm17, %zmm30
+        vmovdqu64	%zmm6, %zmm31
+        vpternlogq	$0xd2, %zmm20, %zmm6, %zmm17
+        vpternlogq	$0xd2, %zmm14, %zmm20, %zmm6
+        vpternlogq	$0xd2, %zmm3, %zmm14, %zmm20
+        vpternlogq	$0xd2, %zmm30, %zmm3, %zmm14
+        vpternlogq	$0xd2, %zmm31, %zmm30, %zmm3
+        vmovdqu64	%zmm21, %zmm30
+        vmovdqu64	%zmm10, %zmm31
+        vpternlogq	$0xd2, %zmm4, %zmm10, %zmm21
+        vpternlogq	$0xd2, %zmm18, %zmm4, %zmm10
+        vpternlogq	$0xd2, %zmm7, %zmm18, %zmm4
+        vpternlogq	$0xd2, %zmm30, %zmm7, %zmm18
+        vpternlogq	$0xd2, %zmm31, %zmm30, %zmm7
+        # Iota
+        vpxorq	512(%rax), %zmm0, %zmm0
+        # Round 9
+        # Theta - column parity
+        vmovdqu64	%zmm0, %zmm25
+        vpternlogq	$0x96, %zmm13, %zmm9, %zmm25
+        vpternlogq	$0x96, %zmm21, %zmm17, %zmm25
+        vmovdqu64	%zmm19, %zmm26
+        vpternlogq	$0x96, %zmm2, %zmm23, %zmm26
+        vpternlogq	$0x96, %zmm10, %zmm6, %zmm26
+        vmovdqu64	%zmm8, %zmm27
+        vpternlogq	$0x96, %zmm16, %zmm12, %zmm27
+        vpternlogq	$0x96, %zmm4, %zmm20, %zmm27
+        vmovdqu64	%zmm22, %zmm28
+        vpternlogq	$0x96, %zmm5, %zmm1, %zmm28
+        vpternlogq	$0x96, %zmm18, %zmm14, %zmm28
+        vmovdqu64	%zmm11, %zmm29
+        vpternlogq	$0x96, %zmm24, %zmm15, %zmm29
+        vpternlogq	$0x96, %zmm7, %zmm3, %zmm29
+        # Theta - D and apply in place
+        vprolq	$0x01, %zmm26, %zmm30
+        vpxorq	%zmm29, %zmm30, %zmm30
+        vpxorq	%zmm30, %zmm0, %zmm0
+        vpxorq	%zmm30, %zmm9, %zmm9
+        vpxorq	%zmm30, %zmm13, %zmm13
+        vpxorq	%zmm30, %zmm17, %zmm17
+        vpxorq	%zmm30, %zmm21, %zmm21
+        vprolq	$0x01, %zmm27, %zmm30
+        vpxorq	%zmm25, %zmm30, %zmm30
+        vpxorq	%zmm30, %zmm19, %zmm19
+        vpxorq	%zmm30, %zmm23, %zmm23
+        vpxorq	%zmm30, %zmm2, %zmm2
+        vpxorq	%zmm30, %zmm6, %zmm6
+        vpxorq	%zmm30, %zmm10, %zmm10
+        vprolq	$0x01, %zmm28, %zmm30
+        vpxorq	%zmm26, %zmm30, %zmm30
+        vpxorq	%zmm30, %zmm8, %zmm8
+        vpxorq	%zmm30, %zmm12, %zmm12
+        vpxorq	%zmm30, %zmm16, %zmm16
+        vpxorq	%zmm30, %zmm20, %zmm20
+        vpxorq	%zmm30, %zmm4, %zmm4
+        vprolq	$0x01, %zmm29, %zmm30
+        vpxorq	%zmm27, %zmm30, %zmm30
+        vpxorq	%zmm30, %zmm22, %zmm22
+        vpxorq	%zmm30, %zmm1, %zmm1
+        vpxorq	%zmm30, %zmm5, %zmm5
+        vpxorq	%zmm30, %zmm14, %zmm14
+        vpxorq	%zmm30, %zmm18, %zmm18
+        vprolq	$0x01, %zmm25, %zmm30
+        vpxorq	%zmm28, %zmm30, %zmm30
+        vpxorq	%zmm30, %zmm11, %zmm11
+        vpxorq	%zmm30, %zmm15, %zmm15
+        vpxorq	%zmm30, %zmm24, %zmm24
+        vpxorq	%zmm30, %zmm3, %zmm3
+        vpxorq	%zmm30, %zmm7, %zmm7
+        # Rho - rotate each lane in place
+        vprolq	$0x01, %zmm19, %zmm19
+        vprolq	$62, %zmm8, %zmm8
+        vprolq	$28, %zmm22, %zmm22
+        vprolq	$27, %zmm11, %zmm11
+        vprolq	$36, %zmm9, %zmm9
+        vprolq	$44, %zmm23, %zmm23
+        vprolq	$6, %zmm12, %zmm12
+        vprolq	$55, %zmm1, %zmm1
+        vprolq	$20, %zmm15, %zmm15
+        vprolq	$3, %zmm13, %zmm13
+        vprolq	$10, %zmm2, %zmm2
+        vprolq	$43, %zmm16, %zmm16
+        vprolq	$25, %zmm5, %zmm5
+        vprolq	$39, %zmm24, %zmm24
+        vprolq	$41, %zmm17, %zmm17
+        vprolq	$45, %zmm6, %zmm6
+        vprolq	$15, %zmm20, %zmm20
+        vprolq	$21, %zmm14, %zmm14
+        vprolq	$8, %zmm3, %zmm3
+        vprolq	$18, %zmm21, %zmm21
+        vprolq	$2, %zmm10, %zmm10
+        vprolq	$61, %zmm4, %zmm4
+        vprolq	$56, %zmm18, %zmm18
+        vprolq	$14, %zmm7, %zmm7
+        # Pi - relabel logical -> physical
+        # Chi - per row, in place, fused with vpternlogq (imm 0xd2)
+        vmovdqu64	%zmm0, %zmm30
+        vmovdqu64	%zmm23, %zmm31
+        vpternlogq	$0xd2, %zmm16, %zmm23, %zmm0
+        vpternlogq	$0xd2, %zmm14, %zmm16, %zmm23
+        vpternlogq	$0xd2, %zmm7, %zmm14, %zmm16
+        vpternlogq	$0xd2, %zmm30, %zmm7, %zmm14
+        vpternlogq	$0xd2, %zmm31, %zmm30, %zmm7
+        vmovdqu64	%zmm22, %zmm30
+        vmovdqu64	%zmm15, %zmm31
+        vpternlogq	$0xd2, %zmm13, %zmm15, %zmm22
+        vpternlogq	$0xd2, %zmm6, %zmm13, %zmm15
+        vpternlogq	$0xd2, %zmm4, %zmm6, %zmm13
+        vpternlogq	$0xd2, %zmm30, %zmm4, %zmm6
+        vpternlogq	$0xd2, %zmm31, %zmm30, %zmm4
+        vmovdqu64	%zmm19, %zmm30
+        vmovdqu64	%zmm12, %zmm31
+        vpternlogq	$0xd2, %zmm5, %zmm12, %zmm19
+        vpternlogq	$0xd2, %zmm3, %zmm5, %zmm12
+        vpternlogq	$0xd2, %zmm21, %zmm3, %zmm5
+        vpternlogq	$0xd2, %zmm30, %zmm21, %zmm3
+        vpternlogq	$0xd2, %zmm31, %zmm30, %zmm21
+        vmovdqu64	%zmm11, %zmm30
+        vmovdqu64	%zmm9, %zmm31
+        vpternlogq	$0xd2, %zmm2, %zmm9, %zmm11
+        vpternlogq	$0xd2, %zmm20, %zmm2, %zmm9
+        vpternlogq	$0xd2, %zmm18, %zmm20, %zmm2
+        vpternlogq	$0xd2, %zmm30, %zmm18, %zmm20
+        vpternlogq	$0xd2, %zmm31, %zmm30, %zmm18
+        vmovdqu64	%zmm8, %zmm30
+        vmovdqu64	%zmm1, %zmm31
+        vpternlogq	$0xd2, %zmm24, %zmm1, %zmm8
+        vpternlogq	$0xd2, %zmm17, %zmm24, %zmm1
+        vpternlogq	$0xd2, %zmm10, %zmm17, %zmm24
+        vpternlogq	$0xd2, %zmm30, %zmm10, %zmm17
+        vpternlogq	$0xd2, %zmm31, %zmm30, %zmm10
+        # Iota
+        vpxorq	576(%rax), %zmm0, %zmm0
+        # Round 10
+        # Theta - column parity
+        vmovdqu64	%zmm0, %zmm25
+        vpternlogq	$0x96, %zmm19, %zmm22, %zmm25
+        vpternlogq	$0x96, %zmm8, %zmm11, %zmm25
+        vmovdqu64	%zmm23, %zmm26
+        vpternlogq	$0x96, %zmm12, %zmm15, %zmm26
+        vpternlogq	$0x96, %zmm1, %zmm9, %zmm26
+        vmovdqu64	%zmm16, %zmm27
+        vpternlogq	$0x96, %zmm5, %zmm13, %zmm27
+        vpternlogq	$0x96, %zmm24, %zmm2, %zmm27
+        vmovdqu64	%zmm14, %zmm28
+        vpternlogq	$0x96, %zmm3, %zmm6, %zmm28
+        vpternlogq	$0x96, %zmm17, %zmm20, %zmm28
+        vmovdqu64	%zmm7, %zmm29
+        vpternlogq	$0x96, %zmm21, %zmm4, %zmm29
+        vpternlogq	$0x96, %zmm10, %zmm18, %zmm29
+        # Theta - D and apply in place
+        vprolq	$0x01, %zmm26, %zmm30
+        vpxorq	%zmm29, %zmm30, %zmm30
+        vpxorq	%zmm30, %zmm0, %zmm0
+        vpxorq	%zmm30, %zmm22, %zmm22
+        vpxorq	%zmm30, %zmm19, %zmm19
+        vpxorq	%zmm30, %zmm11, %zmm11
+        vpxorq	%zmm30, %zmm8, %zmm8
+        vprolq	$0x01, %zmm27, %zmm30
+        vpxorq	%zmm25, %zmm30, %zmm30
+        vpxorq	%zmm30, %zmm23, %zmm23
+        vpxorq	%zmm30, %zmm15, %zmm15
+        vpxorq	%zmm30, %zmm12, %zmm12
+        vpxorq	%zmm30, %zmm9, %zmm9
+        vpxorq	%zmm30, %zmm1, %zmm1
+        vprolq	$0x01, %zmm28, %zmm30
+        vpxorq	%zmm26, %zmm30, %zmm30
+        vpxorq	%zmm30, %zmm16, %zmm16
+        vpxorq	%zmm30, %zmm13, %zmm13
+        vpxorq	%zmm30, %zmm5, %zmm5
+        vpxorq	%zmm30, %zmm2, %zmm2
+        vpxorq	%zmm30, %zmm24, %zmm24
+        vprolq	$0x01, %zmm29, %zmm30
+        vpxorq	%zmm27, %zmm30, %zmm30
+        vpxorq	%zmm30, %zmm14, %zmm14
+        vpxorq	%zmm30, %zmm6, %zmm6
+        vpxorq	%zmm30, %zmm3, %zmm3
+        vpxorq	%zmm30, %zmm20, %zmm20
+        vpxorq	%zmm30, %zmm17, %zmm17
+        vprolq	$0x01, %zmm25, %zmm30
+        vpxorq	%zmm28, %zmm30, %zmm30
+        vpxorq	%zmm30, %zmm7, %zmm7
+        vpxorq	%zmm30, %zmm4, %zmm4
+        vpxorq	%zmm30, %zmm21, %zmm21
+        vpxorq	%zmm30, %zmm18, %zmm18
+        vpxorq	%zmm30, %zmm10, %zmm10
+        # Rho - rotate each lane in place
+        vprolq	$0x01, %zmm23, %zmm23
+        vprolq	$62, %zmm16, %zmm16
+        vprolq	$28, %zmm14, %zmm14
+        vprolq	$27, %zmm7, %zmm7
+        vprolq	$36, %zmm22, %zmm22
+        vprolq	$44, %zmm15, %zmm15
+        vprolq	$6, %zmm13, %zmm13
+        vprolq	$55, %zmm6, %zmm6
+        vprolq	$20, %zmm4, %zmm4
+        vprolq	$3, %zmm19, %zmm19
+        vprolq	$10, %zmm12, %zmm12
+        vprolq	$43, %zmm5, %zmm5
+        vprolq	$25, %zmm3, %zmm3
+        vprolq	$39, %zmm21, %zmm21
+        vprolq	$41, %zmm11, %zmm11
+        vprolq	$45, %zmm9, %zmm9
+        vprolq	$15, %zmm2, %zmm2
+        vprolq	$21, %zmm20, %zmm20
+        vprolq	$8, %zmm18, %zmm18
+        vprolq	$18, %zmm8, %zmm8
+        vprolq	$2, %zmm1, %zmm1
+        vprolq	$61, %zmm24, %zmm24
+        vprolq	$56, %zmm17, %zmm17
+        vprolq	$14, %zmm10, %zmm10
+        # Pi - relabel logical -> physical
+        # Chi - per row, in place, fused with vpternlogq (imm 0xd2)
+        vmovdqu64	%zmm0, %zmm30
+        vmovdqu64	%zmm15, %zmm31
+        vpternlogq	$0xd2, %zmm5, %zmm15, %zmm0
+        vpternlogq	$0xd2, %zmm20, %zmm5, %zmm15
+        vpternlogq	$0xd2, %zmm10, %zmm20, %zmm5
+        vpternlogq	$0xd2, %zmm30, %zmm10, %zmm20
+        vpternlogq	$0xd2, %zmm31, %zmm30, %zmm10
+        vmovdqu64	%zmm14, %zmm30
+        vmovdqu64	%zmm4, %zmm31
+        vpternlogq	$0xd2, %zmm19, %zmm4, %zmm14
+        vpternlogq	$0xd2, %zmm9, %zmm19, %zmm4
+        vpternlogq	$0xd2, %zmm24, %zmm9, %zmm19
+        vpternlogq	$0xd2, %zmm30, %zmm24, %zmm9
+        vpternlogq	$0xd2, %zmm31, %zmm30, %zmm24
+        vmovdqu64	%zmm23, %zmm30
+        vmovdqu64	%zmm13, %zmm31
+        vpternlogq	$0xd2, %zmm3, %zmm13, %zmm23
+        vpternlogq	$0xd2, %zmm18, %zmm3, %zmm13
+        vpternlogq	$0xd2, %zmm8, %zmm18, %zmm3
+        vpternlogq	$0xd2, %zmm30, %zmm8, %zmm18
+        vpternlogq	$0xd2, %zmm31, %zmm30, %zmm8
+        vmovdqu64	%zmm7, %zmm30
+        vmovdqu64	%zmm22, %zmm31
+        vpternlogq	$0xd2, %zmm12, %zmm22, %zmm7
+        vpternlogq	$0xd2, %zmm2, %zmm12, %zmm22
+        vpternlogq	$0xd2, %zmm17, %zmm2, %zmm12
+        vpternlogq	$0xd2, %zmm30, %zmm17, %zmm2
+        vpternlogq	$0xd2, %zmm31, %zmm30, %zmm17
+        vmovdqu64	%zmm16, %zmm30
+        vmovdqu64	%zmm6, %zmm31
+        vpternlogq	$0xd2, %zmm21, %zmm6, %zmm16
+        vpternlogq	$0xd2, %zmm11, %zmm21, %zmm6
+        vpternlogq	$0xd2, %zmm1, %zmm11, %zmm21
+        vpternlogq	$0xd2, %zmm30, %zmm1, %zmm11
+        vpternlogq	$0xd2, %zmm31, %zmm30, %zmm1
+        # Iota
+        vpxorq	640(%rax), %zmm0, %zmm0
+        # Round 11
+        # Theta - column parity
+        vmovdqu64	%zmm0, %zmm25
+        vpternlogq	$0x96, %zmm23, %zmm14, %zmm25
+        vpternlogq	$0x96, %zmm16, %zmm7, %zmm25
+        vmovdqu64	%zmm15, %zmm26
+        vpternlogq	$0x96, %zmm13, %zmm4, %zmm26
+        vpternlogq	$0x96, %zmm6, %zmm22, %zmm26
+        vmovdqu64	%zmm5, %zmm27
+        vpternlogq	$0x96, %zmm3, %zmm19, %zmm27
+        vpternlogq	$0x96, %zmm21, %zmm12, %zmm27
+        vmovdqu64	%zmm20, %zmm28
+        vpternlogq	$0x96, %zmm18, %zmm9, %zmm28
+        vpternlogq	$0x96, %zmm11, %zmm2, %zmm28
+        vmovdqu64	%zmm10, %zmm29
+        vpternlogq	$0x96, %zmm8, %zmm24, %zmm29
+        vpternlogq	$0x96, %zmm1, %zmm17, %zmm29
+        # Theta - D and apply in place
+        vprolq	$0x01, %zmm26, %zmm30
+        vpxorq	%zmm29, %zmm30, %zmm30
+        vpxorq	%zmm30, %zmm0, %zmm0
+        vpxorq	%zmm30, %zmm14, %zmm14
+        vpxorq	%zmm30, %zmm23, %zmm23
+        vpxorq	%zmm30, %zmm7, %zmm7
+        vpxorq	%zmm30, %zmm16, %zmm16
+        vprolq	$0x01, %zmm27, %zmm30
+        vpxorq	%zmm25, %zmm30, %zmm30
+        vpxorq	%zmm30, %zmm15, %zmm15
+        vpxorq	%zmm30, %zmm4, %zmm4
+        vpxorq	%zmm30, %zmm13, %zmm13
+        vpxorq	%zmm30, %zmm22, %zmm22
+        vpxorq	%zmm30, %zmm6, %zmm6
+        vprolq	$0x01, %zmm28, %zmm30
+        vpxorq	%zmm26, %zmm30, %zmm30
+        vpxorq	%zmm30, %zmm5, %zmm5
+        vpxorq	%zmm30, %zmm19, %zmm19
+        vpxorq	%zmm30, %zmm3, %zmm3
+        vpxorq	%zmm30, %zmm12, %zmm12
+        vpxorq	%zmm30, %zmm21, %zmm21
+        vprolq	$0x01, %zmm29, %zmm30
+        vpxorq	%zmm27, %zmm30, %zmm30
+        vpxorq	%zmm30, %zmm20, %zmm20
+        vpxorq	%zmm30, %zmm9, %zmm9
+        vpxorq	%zmm30, %zmm18, %zmm18
+        vpxorq	%zmm30, %zmm2, %zmm2
+        vpxorq	%zmm30, %zmm11, %zmm11
+        vprolq	$0x01, %zmm25, %zmm30
+        vpxorq	%zmm28, %zmm30, %zmm30
+        vpxorq	%zmm30, %zmm10, %zmm10
+        vpxorq	%zmm30, %zmm24, %zmm24
+        vpxorq	%zmm30, %zmm8, %zmm8
+        vpxorq	%zmm30, %zmm17, %zmm17
+        vpxorq	%zmm30, %zmm1, %zmm1
+        # Rho - rotate each lane in place
+        vprolq	$0x01, %zmm15, %zmm15
+        vprolq	$62, %zmm5, %zmm5
+        vprolq	$28, %zmm20, %zmm20
+        vprolq	$27, %zmm10, %zmm10
+        vprolq	$36, %zmm14, %zmm14
+        vprolq	$44, %zmm4, %zmm4
+        vprolq	$6, %zmm19, %zmm19
+        vprolq	$55, %zmm9, %zmm9
+        vprolq	$20, %zmm24, %zmm24
+        vprolq	$3, %zmm23, %zmm23
+        vprolq	$10, %zmm13, %zmm13
+        vprolq	$43, %zmm3, %zmm3
+        vprolq	$25, %zmm18, %zmm18
+        vprolq	$39, %zmm8, %zmm8
+        vprolq	$41, %zmm7, %zmm7
+        vprolq	$45, %zmm22, %zmm22
+        vprolq	$15, %zmm12, %zmm12
+        vprolq	$21, %zmm2, %zmm2
+        vprolq	$8, %zmm17, %zmm17
+        vprolq	$18, %zmm16, %zmm16
+        vprolq	$2, %zmm6, %zmm6
+        vprolq	$61, %zmm21, %zmm21
+        vprolq	$56, %zmm11, %zmm11
+        vprolq	$14, %zmm1, %zmm1
+        # Pi - relabel logical -> physical
+        # Chi - per row, in place, fused with vpternlogq (imm 0xd2)
+        vmovdqu64	%zmm0, %zmm30
+        vmovdqu64	%zmm4, %zmm31
+        vpternlogq	$0xd2, %zmm3, %zmm4, %zmm0
+        vpternlogq	$0xd2, %zmm2, %zmm3, %zmm4
+        vpternlogq	$0xd2, %zmm1, %zmm2, %zmm3
+        vpternlogq	$0xd2, %zmm30, %zmm1, %zmm2
+        vpternlogq	$0xd2, %zmm31, %zmm30, %zmm1
+        vmovdqu64	%zmm20, %zmm30
+        vmovdqu64	%zmm24, %zmm31
+        vpternlogq	$0xd2, %zmm23, %zmm24, %zmm20
+        vpternlogq	$0xd2, %zmm22, %zmm23, %zmm24
+        vpternlogq	$0xd2, %zmm21, %zmm22, %zmm23
+        vpternlogq	$0xd2, %zmm30, %zmm21, %zmm22
+        vpternlogq	$0xd2, %zmm31, %zmm30, %zmm21
+        vmovdqu64	%zmm15, %zmm30
+        vmovdqu64	%zmm19, %zmm31
+        vpternlogq	$0xd2, %zmm18, %zmm19, %zmm15
+        vpternlogq	$0xd2, %zmm17, %zmm18, %zmm19
+        vpternlogq	$0xd2, %zmm16, %zmm17, %zmm18
+        vpternlogq	$0xd2, %zmm30, %zmm16, %zmm17
+        vpternlogq	$0xd2, %zmm31, %zmm30, %zmm16
+        vmovdqu64	%zmm10, %zmm30
+        vmovdqu64	%zmm14, %zmm31
+        vpternlogq	$0xd2, %zmm13, %zmm14, %zmm10
+        vpternlogq	$0xd2, %zmm12, %zmm13, %zmm14
+        vpternlogq	$0xd2, %zmm11, %zmm12, %zmm13
+        vpternlogq	$0xd2, %zmm30, %zmm11, %zmm12
+        vpternlogq	$0xd2, %zmm31, %zmm30, %zmm11
+        vmovdqu64	%zmm5, %zmm30
+        vmovdqu64	%zmm9, %zmm31
+        vpternlogq	$0xd2, %zmm8, %zmm9, %zmm5
+        vpternlogq	$0xd2, %zmm7, %zmm8, %zmm9
+        vpternlogq	$0xd2, %zmm6, %zmm7, %zmm8
+        vpternlogq	$0xd2, %zmm30, %zmm6, %zmm7
+        vpternlogq	$0xd2, %zmm31, %zmm30, %zmm6
+        # Iota
+        vpxorq	704(%rax), %zmm0, %zmm0
+        # Round 12
+        # Theta - column parity
+        vmovdqu64	%zmm0, %zmm25
+        vpternlogq	$0x96, %zmm15, %zmm20, %zmm25
+        vpternlogq	$0x96, %zmm5, %zmm10, %zmm25
+        vmovdqu64	%zmm4, %zmm26
+        vpternlogq	$0x96, %zmm19, %zmm24, %zmm26
+        vpternlogq	$0x96, %zmm9, %zmm14, %zmm26
+        vmovdqu64	%zmm3, %zmm27
+        vpternlogq	$0x96, %zmm18, %zmm23, %zmm27
+        vpternlogq	$0x96, %zmm8, %zmm13, %zmm27
+        vmovdqu64	%zmm2, %zmm28
+        vpternlogq	$0x96, %zmm17, %zmm22, %zmm28
+        vpternlogq	$0x96, %zmm7, %zmm12, %zmm28
+        vmovdqu64	%zmm1, %zmm29
+        vpternlogq	$0x96, %zmm16, %zmm21, %zmm29
+        vpternlogq	$0x96, %zmm6, %zmm11, %zmm29
+        # Theta - D and apply in place
+        vprolq	$0x01, %zmm26, %zmm30
+        vpxorq	%zmm29, %zmm30, %zmm30
+        vpxorq	%zmm30, %zmm0, %zmm0
+        vpxorq	%zmm30, %zmm20, %zmm20
+        vpxorq	%zmm30, %zmm15, %zmm15
+        vpxorq	%zmm30, %zmm10, %zmm10
+        vpxorq	%zmm30, %zmm5, %zmm5
+        vprolq	$0x01, %zmm27, %zmm30
+        vpxorq	%zmm25, %zmm30, %zmm30
+        vpxorq	%zmm30, %zmm4, %zmm4
+        vpxorq	%zmm30, %zmm24, %zmm24
+        vpxorq	%zmm30, %zmm19, %zmm19
+        vpxorq	%zmm30, %zmm14, %zmm14
+        vpxorq	%zmm30, %zmm9, %zmm9
+        vprolq	$0x01, %zmm28, %zmm30
+        vpxorq	%zmm26, %zmm30, %zmm30
+        vpxorq	%zmm30, %zmm3, %zmm3
+        vpxorq	%zmm30, %zmm23, %zmm23
+        vpxorq	%zmm30, %zmm18, %zmm18
+        vpxorq	%zmm30, %zmm13, %zmm13
+        vpxorq	%zmm30, %zmm8, %zmm8
+        vprolq	$0x01, %zmm29, %zmm30
+        vpxorq	%zmm27, %zmm30, %zmm30
+        vpxorq	%zmm30, %zmm2, %zmm2
+        vpxorq	%zmm30, %zmm22, %zmm22
+        vpxorq	%zmm30, %zmm17, %zmm17
+        vpxorq	%zmm30, %zmm12, %zmm12
+        vpxorq	%zmm30, %zmm7, %zmm7
+        vprolq	$0x01, %zmm25, %zmm30
+        vpxorq	%zmm28, %zmm30, %zmm30
+        vpxorq	%zmm30, %zmm1, %zmm1
+        vpxorq	%zmm30, %zmm21, %zmm21
+        vpxorq	%zmm30, %zmm16, %zmm16
+        vpxorq	%zmm30, %zmm11, %zmm11
+        vpxorq	%zmm30, %zmm6, %zmm6
+        # Rho - rotate each lane in place
+        vprolq	$0x01, %zmm4, %zmm4
+        vprolq	$62, %zmm3, %zmm3
+        vprolq	$28, %zmm2, %zmm2
+        vprolq	$27, %zmm1, %zmm1
+        vprolq	$36, %zmm20, %zmm20
+        vprolq	$44, %zmm24, %zmm24
+        vprolq	$6, %zmm23, %zmm23
+        vprolq	$55, %zmm22, %zmm22
+        vprolq	$20, %zmm21, %zmm21
+        vprolq	$3, %zmm15, %zmm15
+        vprolq	$10, %zmm19, %zmm19
+        vprolq	$43, %zmm18, %zmm18
+        vprolq	$25, %zmm17, %zmm17
+        vprolq	$39, %zmm16, %zmm16
+        vprolq	$41, %zmm10, %zmm10
+        vprolq	$45, %zmm14, %zmm14
+        vprolq	$15, %zmm13, %zmm13
+        vprolq	$21, %zmm12, %zmm12
+        vprolq	$8, %zmm11, %zmm11
+        vprolq	$18, %zmm5, %zmm5
+        vprolq	$2, %zmm9, %zmm9
+        vprolq	$61, %zmm8, %zmm8
+        vprolq	$56, %zmm7, %zmm7
+        vprolq	$14, %zmm6, %zmm6
+        # Pi - relabel logical -> physical
+        # Chi - per row, in place, fused with vpternlogq (imm 0xd2)
+        vmovdqu64	%zmm0, %zmm30
+        vmovdqu64	%zmm24, %zmm31
+        vpternlogq	$0xd2, %zmm18, %zmm24, %zmm0
+        vpternlogq	$0xd2, %zmm12, %zmm18, %zmm24
+        vpternlogq	$0xd2, %zmm6, %zmm12, %zmm18
+        vpternlogq	$0xd2, %zmm30, %zmm6, %zmm12
+        vpternlogq	$0xd2, %zmm31, %zmm30, %zmm6
+        vmovdqu64	%zmm2, %zmm30
+        vmovdqu64	%zmm21, %zmm31
+        vpternlogq	$0xd2, %zmm15, %zmm21, %zmm2
+        vpternlogq	$0xd2, %zmm14, %zmm15, %zmm21
+        vpternlogq	$0xd2, %zmm8, %zmm14, %zmm15
+        vpternlogq	$0xd2, %zmm30, %zmm8, %zmm14
+        vpternlogq	$0xd2, %zmm31, %zmm30, %zmm8
+        vmovdqu64	%zmm4, %zmm30
+        vmovdqu64	%zmm23, %zmm31
+        vpternlogq	$0xd2, %zmm17, %zmm23, %zmm4
+        vpternlogq	$0xd2, %zmm11, %zmm17, %zmm23
+        vpternlogq	$0xd2, %zmm5, %zmm11, %zmm17
+        vpternlogq	$0xd2, %zmm30, %zmm5, %zmm11
+        vpternlogq	$0xd2, %zmm31, %zmm30, %zmm5
+        vmovdqu64	%zmm1, %zmm30
+        vmovdqu64	%zmm20, %zmm31
+        vpternlogq	$0xd2, %zmm19, %zmm20, %zmm1
+        vpternlogq	$0xd2, %zmm13, %zmm19, %zmm20
+        vpternlogq	$0xd2, %zmm7, %zmm13, %zmm19
+        vpternlogq	$0xd2, %zmm30, %zmm7, %zmm13
+        vpternlogq	$0xd2, %zmm31, %zmm30, %zmm7
+        vmovdqu64	%zmm3, %zmm30
+        vmovdqu64	%zmm22, %zmm31
+        vpternlogq	$0xd2, %zmm16, %zmm22, %zmm3
+        vpternlogq	$0xd2, %zmm10, %zmm16, %zmm22
+        vpternlogq	$0xd2, %zmm9, %zmm10, %zmm16
+        vpternlogq	$0xd2, %zmm30, %zmm9, %zmm10
+        vpternlogq	$0xd2, %zmm31, %zmm30, %zmm9
+        # Iota
+        vpxorq	768(%rax), %zmm0, %zmm0
+        # Round 13
+        # Theta - column parity
+        vmovdqu64	%zmm0, %zmm25
+        vpternlogq	$0x96, %zmm4, %zmm2, %zmm25
+        vpternlogq	$0x96, %zmm3, %zmm1, %zmm25
+        vmovdqu64	%zmm24, %zmm26
+        vpternlogq	$0x96, %zmm23, %zmm21, %zmm26
+        vpternlogq	$0x96, %zmm22, %zmm20, %zmm26
+        vmovdqu64	%zmm18, %zmm27
+        vpternlogq	$0x96, %zmm17, %zmm15, %zmm27
+        vpternlogq	$0x96, %zmm16, %zmm19, %zmm27
+        vmovdqu64	%zmm12, %zmm28
+        vpternlogq	$0x96, %zmm11, %zmm14, %zmm28
+        vpternlogq	$0x96, %zmm10, %zmm13, %zmm28
+        vmovdqu64	%zmm6, %zmm29
+        vpternlogq	$0x96, %zmm5, %zmm8, %zmm29
+        vpternlogq	$0x96, %zmm9, %zmm7, %zmm29
+        # Theta - D and apply in place
+        vprolq	$0x01, %zmm26, %zmm30
+        vpxorq	%zmm29, %zmm30, %zmm30
+        vpxorq	%zmm30, %zmm0, %zmm0
+        vpxorq	%zmm30, %zmm2, %zmm2
+        vpxorq	%zmm30, %zmm4, %zmm4
+        vpxorq	%zmm30, %zmm1, %zmm1
+        vpxorq	%zmm30, %zmm3, %zmm3
+        vprolq	$0x01, %zmm27, %zmm30
+        vpxorq	%zmm25, %zmm30, %zmm30
+        vpxorq	%zmm30, %zmm24, %zmm24
+        vpxorq	%zmm30, %zmm21, %zmm21
+        vpxorq	%zmm30, %zmm23, %zmm23
+        vpxorq	%zmm30, %zmm20, %zmm20
+        vpxorq	%zmm30, %zmm22, %zmm22
+        vprolq	$0x01, %zmm28, %zmm30
+        vpxorq	%zmm26, %zmm30, %zmm30
+        vpxorq	%zmm30, %zmm18, %zmm18
+        vpxorq	%zmm30, %zmm15, %zmm15
+        vpxorq	%zmm30, %zmm17, %zmm17
+        vpxorq	%zmm30, %zmm19, %zmm19
+        vpxorq	%zmm30, %zmm16, %zmm16
+        vprolq	$0x01, %zmm29, %zmm30
+        vpxorq	%zmm27, %zmm30, %zmm30
+        vpxorq	%zmm30, %zmm12, %zmm12
+        vpxorq	%zmm30, %zmm14, %zmm14
+        vpxorq	%zmm30, %zmm11, %zmm11
+        vpxorq	%zmm30, %zmm13, %zmm13
+        vpxorq	%zmm30, %zmm10, %zmm10
+        vprolq	$0x01, %zmm25, %zmm30
+        vpxorq	%zmm28, %zmm30, %zmm30
+        vpxorq	%zmm30, %zmm6, %zmm6
+        vpxorq	%zmm30, %zmm8, %zmm8
+        vpxorq	%zmm30, %zmm5, %zmm5
+        vpxorq	%zmm30, %zmm7, %zmm7
+        vpxorq	%zmm30, %zmm9, %zmm9
+        # Rho - rotate each lane in place
+        vprolq	$0x01, %zmm24, %zmm24
+        vprolq	$62, %zmm18, %zmm18
+        vprolq	$28, %zmm12, %zmm12
+        vprolq	$27, %zmm6, %zmm6
+        vprolq	$36, %zmm2, %zmm2
+        vprolq	$44, %zmm21, %zmm21
+        vprolq	$6, %zmm15, %zmm15
+        vprolq	$55, %zmm14, %zmm14
+        vprolq	$20, %zmm8, %zmm8
+        vprolq	$3, %zmm4, %zmm4
+        vprolq	$10, %zmm23, %zmm23
+        vprolq	$43, %zmm17, %zmm17
+        vprolq	$25, %zmm11, %zmm11
+        vprolq	$39, %zmm5, %zmm5
+        vprolq	$41, %zmm1, %zmm1
+        vprolq	$45, %zmm20, %zmm20
+        vprolq	$15, %zmm19, %zmm19
+        vprolq	$21, %zmm13, %zmm13
+        vprolq	$8, %zmm7, %zmm7
+        vprolq	$18, %zmm3, %zmm3
+        vprolq	$2, %zmm22, %zmm22
+        vprolq	$61, %zmm16, %zmm16
+        vprolq	$56, %zmm10, %zmm10
+        vprolq	$14, %zmm9, %zmm9
+        # Pi - relabel logical -> physical
+        # Chi - per row, in place, fused with vpternlogq (imm 0xd2)
+        vmovdqu64	%zmm0, %zmm30
+        vmovdqu64	%zmm21, %zmm31
+        vpternlogq	$0xd2, %zmm17, %zmm21, %zmm0
+        vpternlogq	$0xd2, %zmm13, %zmm17, %zmm21
+        vpternlogq	$0xd2, %zmm9, %zmm13, %zmm17
+        vpternlogq	$0xd2, %zmm30, %zmm9, %zmm13
+        vpternlogq	$0xd2, %zmm31, %zmm30, %zmm9
+        vmovdqu64	%zmm12, %zmm30
+        vmovdqu64	%zmm8, %zmm31
+        vpternlogq	$0xd2, %zmm4, %zmm8, %zmm12
+        vpternlogq	$0xd2, %zmm20, %zmm4, %zmm8
+        vpternlogq	$0xd2, %zmm16, %zmm20, %zmm4
+        vpternlogq	$0xd2, %zmm30, %zmm16, %zmm20
+        vpternlogq	$0xd2, %zmm31, %zmm30, %zmm16
+        vmovdqu64	%zmm24, %zmm30
+        vmovdqu64	%zmm15, %zmm31
+        vpternlogq	$0xd2, %zmm11, %zmm15, %zmm24
+        vpternlogq	$0xd2, %zmm7, %zmm11, %zmm15
+        vpternlogq	$0xd2, %zmm3, %zmm7, %zmm11
+        vpternlogq	$0xd2, %zmm30, %zmm3, %zmm7
+        vpternlogq	$0xd2, %zmm31, %zmm30, %zmm3
+        vmovdqu64	%zmm6, %zmm30
+        vmovdqu64	%zmm2, %zmm31
+        vpternlogq	$0xd2, %zmm23, %zmm2, %zmm6
+        vpternlogq	$0xd2, %zmm19, %zmm23, %zmm2
+        vpternlogq	$0xd2, %zmm10, %zmm19, %zmm23
+        vpternlogq	$0xd2, %zmm30, %zmm10, %zmm19
+        vpternlogq	$0xd2, %zmm31, %zmm30, %zmm10
+        vmovdqu64	%zmm18, %zmm30
+        vmovdqu64	%zmm14, %zmm31
+        vpternlogq	$0xd2, %zmm5, %zmm14, %zmm18
+        vpternlogq	$0xd2, %zmm1, %zmm5, %zmm14
+        vpternlogq	$0xd2, %zmm22, %zmm1, %zmm5
+        vpternlogq	$0xd2, %zmm30, %zmm22, %zmm1
+        vpternlogq	$0xd2, %zmm31, %zmm30, %zmm22
+        # Iota
+        vpxorq	832(%rax), %zmm0, %zmm0
+        # Round 14
+        # Theta - column parity
+        vmovdqu64	%zmm0, %zmm25
+        vpternlogq	$0x96, %zmm24, %zmm12, %zmm25
+        vpternlogq	$0x96, %zmm18, %zmm6, %zmm25
+        vmovdqu64	%zmm21, %zmm26
+        vpternlogq	$0x96, %zmm15, %zmm8, %zmm26
+        vpternlogq	$0x96, %zmm14, %zmm2, %zmm26
+        vmovdqu64	%zmm17, %zmm27
+        vpternlogq	$0x96, %zmm11, %zmm4, %zmm27
+        vpternlogq	$0x96, %zmm5, %zmm23, %zmm27
+        vmovdqu64	%zmm13, %zmm28
+        vpternlogq	$0x96, %zmm7, %zmm20, %zmm28
+        vpternlogq	$0x96, %zmm1, %zmm19, %zmm28
+        vmovdqu64	%zmm9, %zmm29
+        vpternlogq	$0x96, %zmm3, %zmm16, %zmm29
+        vpternlogq	$0x96, %zmm22, %zmm10, %zmm29
+        # Theta - D and apply in place
+        vprolq	$0x01, %zmm26, %zmm30
+        vpxorq	%zmm29, %zmm30, %zmm30
+        vpxorq	%zmm30, %zmm0, %zmm0
+        vpxorq	%zmm30, %zmm12, %zmm12
+        vpxorq	%zmm30, %zmm24, %zmm24
+        vpxorq	%zmm30, %zmm6, %zmm6
+        vpxorq	%zmm30, %zmm18, %zmm18
+        vprolq	$0x01, %zmm27, %zmm30
+        vpxorq	%zmm25, %zmm30, %zmm30
+        vpxorq	%zmm30, %zmm21, %zmm21
+        vpxorq	%zmm30, %zmm8, %zmm8
+        vpxorq	%zmm30, %zmm15, %zmm15
+        vpxorq	%zmm30, %zmm2, %zmm2
+        vpxorq	%zmm30, %zmm14, %zmm14
+        vprolq	$0x01, %zmm28, %zmm30
+        vpxorq	%zmm26, %zmm30, %zmm30
+        vpxorq	%zmm30, %zmm17, %zmm17
+        vpxorq	%zmm30, %zmm4, %zmm4
+        vpxorq	%zmm30, %zmm11, %zmm11
+        vpxorq	%zmm30, %zmm23, %zmm23
+        vpxorq	%zmm30, %zmm5, %zmm5
+        vprolq	$0x01, %zmm29, %zmm30
+        vpxorq	%zmm27, %zmm30, %zmm30
+        vpxorq	%zmm30, %zmm13, %zmm13
+        vpxorq	%zmm30, %zmm20, %zmm20
+        vpxorq	%zmm30, %zmm7, %zmm7
+        vpxorq	%zmm30, %zmm19, %zmm19
+        vpxorq	%zmm30, %zmm1, %zmm1
+        vprolq	$0x01, %zmm25, %zmm30
+        vpxorq	%zmm28, %zmm30, %zmm30
+        vpxorq	%zmm30, %zmm9, %zmm9
+        vpxorq	%zmm30, %zmm16, %zmm16
+        vpxorq	%zmm30, %zmm3, %zmm3
+        vpxorq	%zmm30, %zmm10, %zmm10
+        vpxorq	%zmm30, %zmm22, %zmm22
+        # Rho - rotate each lane in place
+        vprolq	$0x01, %zmm21, %zmm21
+        vprolq	$62, %zmm17, %zmm17
+        vprolq	$28, %zmm13, %zmm13
+        vprolq	$27, %zmm9, %zmm9
+        vprolq	$36, %zmm12, %zmm12
+        vprolq	$44, %zmm8, %zmm8
+        vprolq	$6, %zmm4, %zmm4
+        vprolq	$55, %zmm20, %zmm20
+        vprolq	$20, %zmm16, %zmm16
+        vprolq	$3, %zmm24, %zmm24
+        vprolq	$10, %zmm15, %zmm15
+        vprolq	$43, %zmm11, %zmm11
+        vprolq	$25, %zmm7, %zmm7
+        vprolq	$39, %zmm3, %zmm3
+        vprolq	$41, %zmm6, %zmm6
+        vprolq	$45, %zmm2, %zmm2
+        vprolq	$15, %zmm23, %zmm23
+        vprolq	$21, %zmm19, %zmm19
+        vprolq	$8, %zmm10, %zmm10
+        vprolq	$18, %zmm18, %zmm18
+        vprolq	$2, %zmm14, %zmm14
+        vprolq	$61, %zmm5, %zmm5
+        vprolq	$56, %zmm1, %zmm1
+        vprolq	$14, %zmm22, %zmm22
+        # Pi - relabel logical -> physical
+        # Chi - per row, in place, fused with vpternlogq (imm 0xd2)
+        vmovdqu64	%zmm0, %zmm30
+        vmovdqu64	%zmm8, %zmm31
+        vpternlogq	$0xd2, %zmm11, %zmm8, %zmm0
+        vpternlogq	$0xd2, %zmm19, %zmm11, %zmm8
+        vpternlogq	$0xd2, %zmm22, %zmm19, %zmm11
+        vpternlogq	$0xd2, %zmm30, %zmm22, %zmm19
+        vpternlogq	$0xd2, %zmm31, %zmm30, %zmm22
+        vmovdqu64	%zmm13, %zmm30
+        vmovdqu64	%zmm16, %zmm31
+        vpternlogq	$0xd2, %zmm24, %zmm16, %zmm13
+        vpternlogq	$0xd2, %zmm2, %zmm24, %zmm16
+        vpternlogq	$0xd2, %zmm5, %zmm2, %zmm24
+        vpternlogq	$0xd2, %zmm30, %zmm5, %zmm2
+        vpternlogq	$0xd2, %zmm31, %zmm30, %zmm5
+        vmovdqu64	%zmm21, %zmm30
+        vmovdqu64	%zmm4, %zmm31
+        vpternlogq	$0xd2, %zmm7, %zmm4, %zmm21
+        vpternlogq	$0xd2, %zmm10, %zmm7, %zmm4
+        vpternlogq	$0xd2, %zmm18, %zmm10, %zmm7
+        vpternlogq	$0xd2, %zmm30, %zmm18, %zmm10
+        vpternlogq	$0xd2, %zmm31, %zmm30, %zmm18
+        vmovdqu64	%zmm9, %zmm30
+        vmovdqu64	%zmm12, %zmm31
+        vpternlogq	$0xd2, %zmm15, %zmm12, %zmm9
+        vpternlogq	$0xd2, %zmm23, %zmm15, %zmm12
+        vpternlogq	$0xd2, %zmm1, %zmm23, %zmm15
+        vpternlogq	$0xd2, %zmm30, %zmm1, %zmm23
+        vpternlogq	$0xd2, %zmm31, %zmm30, %zmm1
+        vmovdqu64	%zmm17, %zmm30
+        vmovdqu64	%zmm20, %zmm31
+        vpternlogq	$0xd2, %zmm3, %zmm20, %zmm17
+        vpternlogq	$0xd2, %zmm6, %zmm3, %zmm20
+        vpternlogq	$0xd2, %zmm14, %zmm6, %zmm3
+        vpternlogq	$0xd2, %zmm30, %zmm14, %zmm6
+        vpternlogq	$0xd2, %zmm31, %zmm30, %zmm14
+        # Iota
+        vpxorq	896(%rax), %zmm0, %zmm0
+        # Round 15
+        # Theta - column parity
+        vmovdqu64	%zmm0, %zmm25
+        vpternlogq	$0x96, %zmm21, %zmm13, %zmm25
+        vpternlogq	$0x96, %zmm17, %zmm9, %zmm25
+        vmovdqu64	%zmm8, %zmm26
+        vpternlogq	$0x96, %zmm4, %zmm16, %zmm26
+        vpternlogq	$0x96, %zmm20, %zmm12, %zmm26
+        vmovdqu64	%zmm11, %zmm27
+        vpternlogq	$0x96, %zmm7, %zmm24, %zmm27
+        vpternlogq	$0x96, %zmm3, %zmm15, %zmm27
+        vmovdqu64	%zmm19, %zmm28
+        vpternlogq	$0x96, %zmm10, %zmm2, %zmm28
+        vpternlogq	$0x96, %zmm6, %zmm23, %zmm28
+        vmovdqu64	%zmm22, %zmm29
+        vpternlogq	$0x96, %zmm18, %zmm5, %zmm29
+        vpternlogq	$0x96, %zmm14, %zmm1, %zmm29
+        # Theta - D and apply in place
+        vprolq	$0x01, %zmm26, %zmm30
+        vpxorq	%zmm29, %zmm30, %zmm30
+        vpxorq	%zmm30, %zmm0, %zmm0
+        vpxorq	%zmm30, %zmm13, %zmm13
+        vpxorq	%zmm30, %zmm21, %zmm21
+        vpxorq	%zmm30, %zmm9, %zmm9
+        vpxorq	%zmm30, %zmm17, %zmm17
+        vprolq	$0x01, %zmm27, %zmm30
+        vpxorq	%zmm25, %zmm30, %zmm30
+        vpxorq	%zmm30, %zmm8, %zmm8
+        vpxorq	%zmm30, %zmm16, %zmm16
+        vpxorq	%zmm30, %zmm4, %zmm4
+        vpxorq	%zmm30, %zmm12, %zmm12
+        vpxorq	%zmm30, %zmm20, %zmm20
+        vprolq	$0x01, %zmm28, %zmm30
+        vpxorq	%zmm26, %zmm30, %zmm30
+        vpxorq	%zmm30, %zmm11, %zmm11
+        vpxorq	%zmm30, %zmm24, %zmm24
+        vpxorq	%zmm30, %zmm7, %zmm7
+        vpxorq	%zmm30, %zmm15, %zmm15
+        vpxorq	%zmm30, %zmm3, %zmm3
+        vprolq	$0x01, %zmm29, %zmm30
+        vpxorq	%zmm27, %zmm30, %zmm30
+        vpxorq	%zmm30, %zmm19, %zmm19
+        vpxorq	%zmm30, %zmm2, %zmm2
+        vpxorq	%zmm30, %zmm10, %zmm10
+        vpxorq	%zmm30, %zmm23, %zmm23
+        vpxorq	%zmm30, %zmm6, %zmm6
+        vprolq	$0x01, %zmm25, %zmm30
+        vpxorq	%zmm28, %zmm30, %zmm30
+        vpxorq	%zmm30, %zmm22, %zmm22
+        vpxorq	%zmm30, %zmm5, %zmm5
+        vpxorq	%zmm30, %zmm18, %zmm18
+        vpxorq	%zmm30, %zmm1, %zmm1
+        vpxorq	%zmm30, %zmm14, %zmm14
+        # Rho - rotate each lane in place
+        vprolq	$0x01, %zmm8, %zmm8
+        vprolq	$62, %zmm11, %zmm11
+        vprolq	$28, %zmm19, %zmm19
+        vprolq	$27, %zmm22, %zmm22
+        vprolq	$36, %zmm13, %zmm13
+        vprolq	$44, %zmm16, %zmm16
+        vprolq	$6, %zmm24, %zmm24
+        vprolq	$55, %zmm2, %zmm2
+        vprolq	$20, %zmm5, %zmm5
+        vprolq	$3, %zmm21, %zmm21
+        vprolq	$10, %zmm4, %zmm4
+        vprolq	$43, %zmm7, %zmm7
+        vprolq	$25, %zmm10, %zmm10
+        vprolq	$39, %zmm18, %zmm18
+        vprolq	$41, %zmm9, %zmm9
+        vprolq	$45, %zmm12, %zmm12
+        vprolq	$15, %zmm15, %zmm15
+        vprolq	$21, %zmm23, %zmm23
+        vprolq	$8, %zmm1, %zmm1
+        vprolq	$18, %zmm17, %zmm17
+        vprolq	$2, %zmm20, %zmm20
+        vprolq	$61, %zmm3, %zmm3
+        vprolq	$56, %zmm6, %zmm6
+        vprolq	$14, %zmm14, %zmm14
+        # Pi - relabel logical -> physical
+        # Chi - per row, in place, fused with vpternlogq (imm 0xd2)
+        vmovdqu64	%zmm0, %zmm30
+        vmovdqu64	%zmm16, %zmm31
+        vpternlogq	$0xd2, %zmm7, %zmm16, %zmm0
+        vpternlogq	$0xd2, %zmm23, %zmm7, %zmm16
+        vpternlogq	$0xd2, %zmm14, %zmm23, %zmm7
+        vpternlogq	$0xd2, %zmm30, %zmm14, %zmm23
+        vpternlogq	$0xd2, %zmm31, %zmm30, %zmm14
+        vmovdqu64	%zmm19, %zmm30
+        vmovdqu64	%zmm5, %zmm31
+        vpternlogq	$0xd2, %zmm21, %zmm5, %zmm19
+        vpternlogq	$0xd2, %zmm12, %zmm21, %zmm5
+        vpternlogq	$0xd2, %zmm3, %zmm12, %zmm21
+        vpternlogq	$0xd2, %zmm30, %zmm3, %zmm12
+        vpternlogq	$0xd2, %zmm31, %zmm30, %zmm3
+        vmovdqu64	%zmm8, %zmm30
+        vmovdqu64	%zmm24, %zmm31
+        vpternlogq	$0xd2, %zmm10, %zmm24, %zmm8
+        vpternlogq	$0xd2, %zmm1, %zmm10, %zmm24
+        vpternlogq	$0xd2, %zmm17, %zmm1, %zmm10
+        vpternlogq	$0xd2, %zmm30, %zmm17, %zmm1
+        vpternlogq	$0xd2, %zmm31, %zmm30, %zmm17
+        vmovdqu64	%zmm22, %zmm30
+        vmovdqu64	%zmm13, %zmm31
+        vpternlogq	$0xd2, %zmm4, %zmm13, %zmm22
+        vpternlogq	$0xd2, %zmm15, %zmm4, %zmm13
+        vpternlogq	$0xd2, %zmm6, %zmm15, %zmm4
+        vpternlogq	$0xd2, %zmm30, %zmm6, %zmm15
+        vpternlogq	$0xd2, %zmm31, %zmm30, %zmm6
+        vmovdqu64	%zmm11, %zmm30
+        vmovdqu64	%zmm2, %zmm31
+        vpternlogq	$0xd2, %zmm18, %zmm2, %zmm11
+        vpternlogq	$0xd2, %zmm9, %zmm18, %zmm2
+        vpternlogq	$0xd2, %zmm20, %zmm9, %zmm18
+        vpternlogq	$0xd2, %zmm30, %zmm20, %zmm9
+        vpternlogq	$0xd2, %zmm31, %zmm30, %zmm20
+        # Iota
+        vpxorq	960(%rax), %zmm0, %zmm0
+        # Round 16
+        # Theta - column parity
+        vmovdqu64	%zmm0, %zmm25
+        vpternlogq	$0x96, %zmm8, %zmm19, %zmm25
+        vpternlogq	$0x96, %zmm11, %zmm22, %zmm25
+        vmovdqu64	%zmm16, %zmm26
+        vpternlogq	$0x96, %zmm24, %zmm5, %zmm26
+        vpternlogq	$0x96, %zmm2, %zmm13, %zmm26
+        vmovdqu64	%zmm7, %zmm27
+        vpternlogq	$0x96, %zmm10, %zmm21, %zmm27
+        vpternlogq	$0x96, %zmm18, %zmm4, %zmm27
+        vmovdqu64	%zmm23, %zmm28
+        vpternlogq	$0x96, %zmm1, %zmm12, %zmm28
+        vpternlogq	$0x96, %zmm9, %zmm15, %zmm28
+        vmovdqu64	%zmm14, %zmm29
+        vpternlogq	$0x96, %zmm17, %zmm3, %zmm29
+        vpternlogq	$0x96, %zmm20, %zmm6, %zmm29
+        # Theta - D and apply in place
+        vprolq	$0x01, %zmm26, %zmm30
+        vpxorq	%zmm29, %zmm30, %zmm30
+        vpxorq	%zmm30, %zmm0, %zmm0
+        vpxorq	%zmm30, %zmm19, %zmm19
+        vpxorq	%zmm30, %zmm8, %zmm8
+        vpxorq	%zmm30, %zmm22, %zmm22
+        vpxorq	%zmm30, %zmm11, %zmm11
+        vprolq	$0x01, %zmm27, %zmm30
+        vpxorq	%zmm25, %zmm30, %zmm30
+        vpxorq	%zmm30, %zmm16, %zmm16
+        vpxorq	%zmm30, %zmm5, %zmm5
+        vpxorq	%zmm30, %zmm24, %zmm24
+        vpxorq	%zmm30, %zmm13, %zmm13
+        vpxorq	%zmm30, %zmm2, %zmm2
+        vprolq	$0x01, %zmm28, %zmm30
+        vpxorq	%zmm26, %zmm30, %zmm30
+        vpxorq	%zmm30, %zmm7, %zmm7
+        vpxorq	%zmm30, %zmm21, %zmm21
+        vpxorq	%zmm30, %zmm10, %zmm10
+        vpxorq	%zmm30, %zmm4, %zmm4
+        vpxorq	%zmm30, %zmm18, %zmm18
+        vprolq	$0x01, %zmm29, %zmm30
+        vpxorq	%zmm27, %zmm30, %zmm30
+        vpxorq	%zmm30, %zmm23, %zmm23
+        vpxorq	%zmm30, %zmm12, %zmm12
+        vpxorq	%zmm30, %zmm1, %zmm1
+        vpxorq	%zmm30, %zmm15, %zmm15
+        vpxorq	%zmm30, %zmm9, %zmm9
+        vprolq	$0x01, %zmm25, %zmm30
+        vpxorq	%zmm28, %zmm30, %zmm30
+        vpxorq	%zmm30, %zmm14, %zmm14
+        vpxorq	%zmm30, %zmm3, %zmm3
+        vpxorq	%zmm30, %zmm17, %zmm17
+        vpxorq	%zmm30, %zmm6, %zmm6
+        vpxorq	%zmm30, %zmm20, %zmm20
+        # Rho - rotate each lane in place
+        vprolq	$0x01, %zmm16, %zmm16
+        vprolq	$62, %zmm7, %zmm7
+        vprolq	$28, %zmm23, %zmm23
+        vprolq	$27, %zmm14, %zmm14
+        vprolq	$36, %zmm19, %zmm19
+        vprolq	$44, %zmm5, %zmm5
+        vprolq	$6, %zmm21, %zmm21
+        vprolq	$55, %zmm12, %zmm12
+        vprolq	$20, %zmm3, %zmm3
+        vprolq	$3, %zmm8, %zmm8
+        vprolq	$10, %zmm24, %zmm24
+        vprolq	$43, %zmm10, %zmm10
+        vprolq	$25, %zmm1, %zmm1
+        vprolq	$39, %zmm17, %zmm17
+        vprolq	$41, %zmm22, %zmm22
+        vprolq	$45, %zmm13, %zmm13
+        vprolq	$15, %zmm4, %zmm4
+        vprolq	$21, %zmm15, %zmm15
+        vprolq	$8, %zmm6, %zmm6
+        vprolq	$18, %zmm11, %zmm11
+        vprolq	$2, %zmm2, %zmm2
+        vprolq	$61, %zmm18, %zmm18
+        vprolq	$56, %zmm9, %zmm9
+        vprolq	$14, %zmm20, %zmm20
+        # Pi - relabel logical -> physical
+        # Chi - per row, in place, fused with vpternlogq (imm 0xd2)
+        vmovdqu64	%zmm0, %zmm30
+        vmovdqu64	%zmm5, %zmm31
+        vpternlogq	$0xd2, %zmm10, %zmm5, %zmm0
+        vpternlogq	$0xd2, %zmm15, %zmm10, %zmm5
+        vpternlogq	$0xd2, %zmm20, %zmm15, %zmm10
+        vpternlogq	$0xd2, %zmm30, %zmm20, %zmm15
+        vpternlogq	$0xd2, %zmm31, %zmm30, %zmm20
+        vmovdqu64	%zmm23, %zmm30
+        vmovdqu64	%zmm3, %zmm31
+        vpternlogq	$0xd2, %zmm8, %zmm3, %zmm23
+        vpternlogq	$0xd2, %zmm13, %zmm8, %zmm3
+        vpternlogq	$0xd2, %zmm18, %zmm13, %zmm8
+        vpternlogq	$0xd2, %zmm30, %zmm18, %zmm13
+        vpternlogq	$0xd2, %zmm31, %zmm30, %zmm18
+        vmovdqu64	%zmm16, %zmm30
+        vmovdqu64	%zmm21, %zmm31
+        vpternlogq	$0xd2, %zmm1, %zmm21, %zmm16
+        vpternlogq	$0xd2, %zmm6, %zmm1, %zmm21
+        vpternlogq	$0xd2, %zmm11, %zmm6, %zmm1
+        vpternlogq	$0xd2, %zmm30, %zmm11, %zmm6
+        vpternlogq	$0xd2, %zmm31, %zmm30, %zmm11
+        vmovdqu64	%zmm14, %zmm30
+        vmovdqu64	%zmm19, %zmm31
+        vpternlogq	$0xd2, %zmm24, %zmm19, %zmm14
+        vpternlogq	$0xd2, %zmm4, %zmm24, %zmm19
+        vpternlogq	$0xd2, %zmm9, %zmm4, %zmm24
+        vpternlogq	$0xd2, %zmm30, %zmm9, %zmm4
+        vpternlogq	$0xd2, %zmm31, %zmm30, %zmm9
+        vmovdqu64	%zmm7, %zmm30
+        vmovdqu64	%zmm12, %zmm31
+        vpternlogq	$0xd2, %zmm17, %zmm12, %zmm7
+        vpternlogq	$0xd2, %zmm22, %zmm17, %zmm12
+        vpternlogq	$0xd2, %zmm2, %zmm22, %zmm17
+        vpternlogq	$0xd2, %zmm30, %zmm2, %zmm22
+        vpternlogq	$0xd2, %zmm31, %zmm30, %zmm2
+        # Iota
+        vpxorq	1024(%rax), %zmm0, %zmm0
+        # Round 17
+        # Theta - column parity
+        vmovdqu64	%zmm0, %zmm25
+        vpternlogq	$0x96, %zmm16, %zmm23, %zmm25
+        vpternlogq	$0x96, %zmm7, %zmm14, %zmm25
+        vmovdqu64	%zmm5, %zmm26
+        vpternlogq	$0x96, %zmm21, %zmm3, %zmm26
+        vpternlogq	$0x96, %zmm12, %zmm19, %zmm26
+        vmovdqu64	%zmm10, %zmm27
+        vpternlogq	$0x96, %zmm1, %zmm8, %zmm27
+        vpternlogq	$0x96, %zmm17, %zmm24, %zmm27
+        vmovdqu64	%zmm15, %zmm28
+        vpternlogq	$0x96, %zmm6, %zmm13, %zmm28
+        vpternlogq	$0x96, %zmm22, %zmm4, %zmm28
+        vmovdqu64	%zmm20, %zmm29
+        vpternlogq	$0x96, %zmm11, %zmm18, %zmm29
+        vpternlogq	$0x96, %zmm2, %zmm9, %zmm29
+        # Theta - D and apply in place
+        vprolq	$0x01, %zmm26, %zmm30
+        vpxorq	%zmm29, %zmm30, %zmm30
+        vpxorq	%zmm30, %zmm0, %zmm0
+        vpxorq	%zmm30, %zmm23, %zmm23
+        vpxorq	%zmm30, %zmm16, %zmm16
+        vpxorq	%zmm30, %zmm14, %zmm14
+        vpxorq	%zmm30, %zmm7, %zmm7
+        vprolq	$0x01, %zmm27, %zmm30
+        vpxorq	%zmm25, %zmm30, %zmm30
+        vpxorq	%zmm30, %zmm5, %zmm5
+        vpxorq	%zmm30, %zmm3, %zmm3
+        vpxorq	%zmm30, %zmm21, %zmm21
+        vpxorq	%zmm30, %zmm19, %zmm19
+        vpxorq	%zmm30, %zmm12, %zmm12
+        vprolq	$0x01, %zmm28, %zmm30
+        vpxorq	%zmm26, %zmm30, %zmm30
+        vpxorq	%zmm30, %zmm10, %zmm10
+        vpxorq	%zmm30, %zmm8, %zmm8
+        vpxorq	%zmm30, %zmm1, %zmm1
+        vpxorq	%zmm30, %zmm24, %zmm24
+        vpxorq	%zmm30, %zmm17, %zmm17
+        vprolq	$0x01, %zmm29, %zmm30
+        vpxorq	%zmm27, %zmm30, %zmm30
+        vpxorq	%zmm30, %zmm15, %zmm15
+        vpxorq	%zmm30, %zmm13, %zmm13
+        vpxorq	%zmm30, %zmm6, %zmm6
+        vpxorq	%zmm30, %zmm4, %zmm4
+        vpxorq	%zmm30, %zmm22, %zmm22
+        vprolq	$0x01, %zmm25, %zmm30
+        vpxorq	%zmm28, %zmm30, %zmm30
+        vpxorq	%zmm30, %zmm20, %zmm20
+        vpxorq	%zmm30, %zmm18, %zmm18
+        vpxorq	%zmm30, %zmm11, %zmm11
+        vpxorq	%zmm30, %zmm9, %zmm9
+        vpxorq	%zmm30, %zmm2, %zmm2
+        # Rho - rotate each lane in place
+        vprolq	$0x01, %zmm5, %zmm5
+        vprolq	$62, %zmm10, %zmm10
+        vprolq	$28, %zmm15, %zmm15
+        vprolq	$27, %zmm20, %zmm20
+        vprolq	$36, %zmm23, %zmm23
+        vprolq	$44, %zmm3, %zmm3
+        vprolq	$6, %zmm8, %zmm8
+        vprolq	$55, %zmm13, %zmm13
+        vprolq	$20, %zmm18, %zmm18
+        vprolq	$3, %zmm16, %zmm16
+        vprolq	$10, %zmm21, %zmm21
+        vprolq	$43, %zmm1, %zmm1
+        vprolq	$25, %zmm6, %zmm6
+        vprolq	$39, %zmm11, %zmm11
+        vprolq	$41, %zmm14, %zmm14
+        vprolq	$45, %zmm19, %zmm19
+        vprolq	$15, %zmm24, %zmm24
+        vprolq	$21, %zmm4, %zmm4
+        vprolq	$8, %zmm9, %zmm9
+        vprolq	$18, %zmm7, %zmm7
+        vprolq	$2, %zmm12, %zmm12
+        vprolq	$61, %zmm17, %zmm17
+        vprolq	$56, %zmm22, %zmm22
+        vprolq	$14, %zmm2, %zmm2
+        # Pi - relabel logical -> physical
+        # Chi - per row, in place, fused with vpternlogq (imm 0xd2)
+        vmovdqu64	%zmm0, %zmm30
+        vmovdqu64	%zmm3, %zmm31
+        vpternlogq	$0xd2, %zmm1, %zmm3, %zmm0
+        vpternlogq	$0xd2, %zmm4, %zmm1, %zmm3
+        vpternlogq	$0xd2, %zmm2, %zmm4, %zmm1
+        vpternlogq	$0xd2, %zmm30, %zmm2, %zmm4
+        vpternlogq	$0xd2, %zmm31, %zmm30, %zmm2
+        vmovdqu64	%zmm15, %zmm30
+        vmovdqu64	%zmm18, %zmm31
+        vpternlogq	$0xd2, %zmm16, %zmm18, %zmm15
+        vpternlogq	$0xd2, %zmm19, %zmm16, %zmm18
+        vpternlogq	$0xd2, %zmm17, %zmm19, %zmm16
+        vpternlogq	$0xd2, %zmm30, %zmm17, %zmm19
+        vpternlogq	$0xd2, %zmm31, %zmm30, %zmm17
+        vmovdqu64	%zmm5, %zmm30
+        vmovdqu64	%zmm8, %zmm31
+        vpternlogq	$0xd2, %zmm6, %zmm8, %zmm5
+        vpternlogq	$0xd2, %zmm9, %zmm6, %zmm8
+        vpternlogq	$0xd2, %zmm7, %zmm9, %zmm6
+        vpternlogq	$0xd2, %zmm30, %zmm7, %zmm9
+        vpternlogq	$0xd2, %zmm31, %zmm30, %zmm7
+        vmovdqu64	%zmm20, %zmm30
+        vmovdqu64	%zmm23, %zmm31
+        vpternlogq	$0xd2, %zmm21, %zmm23, %zmm20
+        vpternlogq	$0xd2, %zmm24, %zmm21, %zmm23
+        vpternlogq	$0xd2, %zmm22, %zmm24, %zmm21
+        vpternlogq	$0xd2, %zmm30, %zmm22, %zmm24
+        vpternlogq	$0xd2, %zmm31, %zmm30, %zmm22
+        vmovdqu64	%zmm10, %zmm30
+        vmovdqu64	%zmm13, %zmm31
+        vpternlogq	$0xd2, %zmm11, %zmm13, %zmm10
+        vpternlogq	$0xd2, %zmm14, %zmm11, %zmm13
+        vpternlogq	$0xd2, %zmm12, %zmm14, %zmm11
+        vpternlogq	$0xd2, %zmm30, %zmm12, %zmm14
+        vpternlogq	$0xd2, %zmm31, %zmm30, %zmm12
+        # Iota
+        vpxorq	1088(%rax), %zmm0, %zmm0
+        # Round 18
+        # Theta - column parity
+        vmovdqu64	%zmm0, %zmm25
+        vpternlogq	$0x96, %zmm5, %zmm15, %zmm25
+        vpternlogq	$0x96, %zmm10, %zmm20, %zmm25
+        vmovdqu64	%zmm3, %zmm26
+        vpternlogq	$0x96, %zmm8, %zmm18, %zmm26
+        vpternlogq	$0x96, %zmm13, %zmm23, %zmm26
+        vmovdqu64	%zmm1, %zmm27
+        vpternlogq	$0x96, %zmm6, %zmm16, %zmm27
+        vpternlogq	$0x96, %zmm11, %zmm21, %zmm27
+        vmovdqu64	%zmm4, %zmm28
+        vpternlogq	$0x96, %zmm9, %zmm19, %zmm28
+        vpternlogq	$0x96, %zmm14, %zmm24, %zmm28
+        vmovdqu64	%zmm2, %zmm29
+        vpternlogq	$0x96, %zmm7, %zmm17, %zmm29
+        vpternlogq	$0x96, %zmm12, %zmm22, %zmm29
+        # Theta - D and apply in place
+        vprolq	$0x01, %zmm26, %zmm30
+        vpxorq	%zmm29, %zmm30, %zmm30
+        vpxorq	%zmm30, %zmm0, %zmm0
+        vpxorq	%zmm30, %zmm15, %zmm15
+        vpxorq	%zmm30, %zmm5, %zmm5
+        vpxorq	%zmm30, %zmm20, %zmm20
+        vpxorq	%zmm30, %zmm10, %zmm10
+        vprolq	$0x01, %zmm27, %zmm30
+        vpxorq	%zmm25, %zmm30, %zmm30
+        vpxorq	%zmm30, %zmm3, %zmm3
+        vpxorq	%zmm30, %zmm18, %zmm18
+        vpxorq	%zmm30, %zmm8, %zmm8
+        vpxorq	%zmm30, %zmm23, %zmm23
+        vpxorq	%zmm30, %zmm13, %zmm13
+        vprolq	$0x01, %zmm28, %zmm30
+        vpxorq	%zmm26, %zmm30, %zmm30
+        vpxorq	%zmm30, %zmm1, %zmm1
+        vpxorq	%zmm30, %zmm16, %zmm16
+        vpxorq	%zmm30, %zmm6, %zmm6
+        vpxorq	%zmm30, %zmm21, %zmm21
+        vpxorq	%zmm30, %zmm11, %zmm11
+        vprolq	$0x01, %zmm29, %zmm30
+        vpxorq	%zmm27, %zmm30, %zmm30
+        vpxorq	%zmm30, %zmm4, %zmm4
+        vpxorq	%zmm30, %zmm19, %zmm19
+        vpxorq	%zmm30, %zmm9, %zmm9
+        vpxorq	%zmm30, %zmm24, %zmm24
+        vpxorq	%zmm30, %zmm14, %zmm14
+        vprolq	$0x01, %zmm25, %zmm30
+        vpxorq	%zmm28, %zmm30, %zmm30
+        vpxorq	%zmm30, %zmm2, %zmm2
+        vpxorq	%zmm30, %zmm17, %zmm17
+        vpxorq	%zmm30, %zmm7, %zmm7
+        vpxorq	%zmm30, %zmm22, %zmm22
+        vpxorq	%zmm30, %zmm12, %zmm12
+        # Rho - rotate each lane in place
+        vprolq	$0x01, %zmm3, %zmm3
+        vprolq	$62, %zmm1, %zmm1
+        vprolq	$28, %zmm4, %zmm4
+        vprolq	$27, %zmm2, %zmm2
+        vprolq	$36, %zmm15, %zmm15
+        vprolq	$44, %zmm18, %zmm18
+        vprolq	$6, %zmm16, %zmm16
+        vprolq	$55, %zmm19, %zmm19
+        vprolq	$20, %zmm17, %zmm17
+        vprolq	$3, %zmm5, %zmm5
+        vprolq	$10, %zmm8, %zmm8
+        vprolq	$43, %zmm6, %zmm6
+        vprolq	$25, %zmm9, %zmm9
+        vprolq	$39, %zmm7, %zmm7
+        vprolq	$41, %zmm20, %zmm20
+        vprolq	$45, %zmm23, %zmm23
+        vprolq	$15, %zmm21, %zmm21
+        vprolq	$21, %zmm24, %zmm24
+        vprolq	$8, %zmm22, %zmm22
+        vprolq	$18, %zmm10, %zmm10
+        vprolq	$2, %zmm13, %zmm13
+        vprolq	$61, %zmm11, %zmm11
+        vprolq	$56, %zmm14, %zmm14
+        vprolq	$14, %zmm12, %zmm12
+        # Pi - relabel logical -> physical
+        # Chi - per row, in place, fused with vpternlogq (imm 0xd2)
+        vmovdqu64	%zmm0, %zmm30
+        vmovdqu64	%zmm18, %zmm31
+        vpternlogq	$0xd2, %zmm6, %zmm18, %zmm0
+        vpternlogq	$0xd2, %zmm24, %zmm6, %zmm18
+        vpternlogq	$0xd2, %zmm12, %zmm24, %zmm6
+        vpternlogq	$0xd2, %zmm30, %zmm12, %zmm24
+        vpternlogq	$0xd2, %zmm31, %zmm30, %zmm12
+        vmovdqu64	%zmm4, %zmm30
+        vmovdqu64	%zmm17, %zmm31
+        vpternlogq	$0xd2, %zmm5, %zmm17, %zmm4
+        vpternlogq	$0xd2, %zmm23, %zmm5, %zmm17
+        vpternlogq	$0xd2, %zmm11, %zmm23, %zmm5
+        vpternlogq	$0xd2, %zmm30, %zmm11, %zmm23
+        vpternlogq	$0xd2, %zmm31, %zmm30, %zmm11
+        vmovdqu64	%zmm3, %zmm30
+        vmovdqu64	%zmm16, %zmm31
+        vpternlogq	$0xd2, %zmm9, %zmm16, %zmm3
+        vpternlogq	$0xd2, %zmm22, %zmm9, %zmm16
+        vpternlogq	$0xd2, %zmm10, %zmm22, %zmm9
+        vpternlogq	$0xd2, %zmm30, %zmm10, %zmm22
+        vpternlogq	$0xd2, %zmm31, %zmm30, %zmm10
+        vmovdqu64	%zmm2, %zmm30
+        vmovdqu64	%zmm15, %zmm31
+        vpternlogq	$0xd2, %zmm8, %zmm15, %zmm2
+        vpternlogq	$0xd2, %zmm21, %zmm8, %zmm15
+        vpternlogq	$0xd2, %zmm14, %zmm21, %zmm8
+        vpternlogq	$0xd2, %zmm30, %zmm14, %zmm21
+        vpternlogq	$0xd2, %zmm31, %zmm30, %zmm14
+        vmovdqu64	%zmm1, %zmm30
+        vmovdqu64	%zmm19, %zmm31
+        vpternlogq	$0xd2, %zmm7, %zmm19, %zmm1
+        vpternlogq	$0xd2, %zmm20, %zmm7, %zmm19
+        vpternlogq	$0xd2, %zmm13, %zmm20, %zmm7
+        vpternlogq	$0xd2, %zmm30, %zmm13, %zmm20
+        vpternlogq	$0xd2, %zmm31, %zmm30, %zmm13
+        # Iota
+        vpxorq	1152(%rax), %zmm0, %zmm0
+        # Round 19
+        # Theta - column parity
+        vmovdqu64	%zmm0, %zmm25
+        vpternlogq	$0x96, %zmm3, %zmm4, %zmm25
+        vpternlogq	$0x96, %zmm1, %zmm2, %zmm25
+        vmovdqu64	%zmm18, %zmm26
+        vpternlogq	$0x96, %zmm16, %zmm17, %zmm26
+        vpternlogq	$0x96, %zmm19, %zmm15, %zmm26
+        vmovdqu64	%zmm6, %zmm27
+        vpternlogq	$0x96, %zmm9, %zmm5, %zmm27
+        vpternlogq	$0x96, %zmm7, %zmm8, %zmm27
+        vmovdqu64	%zmm24, %zmm28
+        vpternlogq	$0x96, %zmm22, %zmm23, %zmm28
+        vpternlogq	$0x96, %zmm20, %zmm21, %zmm28
+        vmovdqu64	%zmm12, %zmm29
+        vpternlogq	$0x96, %zmm10, %zmm11, %zmm29
+        vpternlogq	$0x96, %zmm13, %zmm14, %zmm29
+        # Theta - D and apply in place
+        vprolq	$0x01, %zmm26, %zmm30
+        vpxorq	%zmm29, %zmm30, %zmm30
+        vpxorq	%zmm30, %zmm0, %zmm0
+        vpxorq	%zmm30, %zmm4, %zmm4
+        vpxorq	%zmm30, %zmm3, %zmm3
+        vpxorq	%zmm30, %zmm2, %zmm2
+        vpxorq	%zmm30, %zmm1, %zmm1
+        vprolq	$0x01, %zmm27, %zmm30
+        vpxorq	%zmm25, %zmm30, %zmm30
+        vpxorq	%zmm30, %zmm18, %zmm18
+        vpxorq	%zmm30, %zmm17, %zmm17
+        vpxorq	%zmm30, %zmm16, %zmm16
+        vpxorq	%zmm30, %zmm15, %zmm15
+        vpxorq	%zmm30, %zmm19, %zmm19
+        vprolq	$0x01, %zmm28, %zmm30
+        vpxorq	%zmm26, %zmm30, %zmm30
+        vpxorq	%zmm30, %zmm6, %zmm6
+        vpxorq	%zmm30, %zmm5, %zmm5
+        vpxorq	%zmm30, %zmm9, %zmm9
+        vpxorq	%zmm30, %zmm8, %zmm8
+        vpxorq	%zmm30, %zmm7, %zmm7
+        vprolq	$0x01, %zmm29, %zmm30
+        vpxorq	%zmm27, %zmm30, %zmm30
+        vpxorq	%zmm30, %zmm24, %zmm24
+        vpxorq	%zmm30, %zmm23, %zmm23
+        vpxorq	%zmm30, %zmm22, %zmm22
+        vpxorq	%zmm30, %zmm21, %zmm21
+        vpxorq	%zmm30, %zmm20, %zmm20
+        vprolq	$0x01, %zmm25, %zmm30
+        vpxorq	%zmm28, %zmm30, %zmm30
+        vpxorq	%zmm30, %zmm12, %zmm12
+        vpxorq	%zmm30, %zmm11, %zmm11
+        vpxorq	%zmm30, %zmm10, %zmm10
+        vpxorq	%zmm30, %zmm14, %zmm14
+        vpxorq	%zmm30, %zmm13, %zmm13
+        # Rho - rotate each lane in place
+        vprolq	$0x01, %zmm18, %zmm18
+        vprolq	$62, %zmm6, %zmm6
+        vprolq	$28, %zmm24, %zmm24
+        vprolq	$27, %zmm12, %zmm12
+        vprolq	$36, %zmm4, %zmm4
+        vprolq	$44, %zmm17, %zmm17
+        vprolq	$6, %zmm5, %zmm5
+        vprolq	$55, %zmm23, %zmm23
+        vprolq	$20, %zmm11, %zmm11
+        vprolq	$3, %zmm3, %zmm3
+        vprolq	$10, %zmm16, %zmm16
+        vprolq	$43, %zmm9, %zmm9
+        vprolq	$25, %zmm22, %zmm22
+        vprolq	$39, %zmm10, %zmm10
+        vprolq	$41, %zmm2, %zmm2
+        vprolq	$45, %zmm15, %zmm15
+        vprolq	$15, %zmm8, %zmm8
+        vprolq	$21, %zmm21, %zmm21
+        vprolq	$8, %zmm14, %zmm14
+        vprolq	$18, %zmm1, %zmm1
+        vprolq	$2, %zmm19, %zmm19
+        vprolq	$61, %zmm7, %zmm7
+        vprolq	$56, %zmm20, %zmm20
+        vprolq	$14, %zmm13, %zmm13
+        # Pi - relabel logical -> physical
+        # Chi - per row, in place, fused with vpternlogq (imm 0xd2)
+        vmovdqu64	%zmm0, %zmm30
+        vmovdqu64	%zmm17, %zmm31
+        vpternlogq	$0xd2, %zmm9, %zmm17, %zmm0
+        vpternlogq	$0xd2, %zmm21, %zmm9, %zmm17
+        vpternlogq	$0xd2, %zmm13, %zmm21, %zmm9
+        vpternlogq	$0xd2, %zmm30, %zmm13, %zmm21
+        vpternlogq	$0xd2, %zmm31, %zmm30, %zmm13
+        vmovdqu64	%zmm24, %zmm30
+        vmovdqu64	%zmm11, %zmm31
+        vpternlogq	$0xd2, %zmm3, %zmm11, %zmm24
+        vpternlogq	$0xd2, %zmm15, %zmm3, %zmm11
+        vpternlogq	$0xd2, %zmm7, %zmm15, %zmm3
+        vpternlogq	$0xd2, %zmm30, %zmm7, %zmm15
+        vpternlogq	$0xd2, %zmm31, %zmm30, %zmm7
+        vmovdqu64	%zmm18, %zmm30
+        vmovdqu64	%zmm5, %zmm31
+        vpternlogq	$0xd2, %zmm22, %zmm5, %zmm18
+        vpternlogq	$0xd2, %zmm14, %zmm22, %zmm5
+        vpternlogq	$0xd2, %zmm1, %zmm14, %zmm22
+        vpternlogq	$0xd2, %zmm30, %zmm1, %zmm14
+        vpternlogq	$0xd2, %zmm31, %zmm30, %zmm1
+        vmovdqu64	%zmm12, %zmm30
+        vmovdqu64	%zmm4, %zmm31
+        vpternlogq	$0xd2, %zmm16, %zmm4, %zmm12
+        vpternlogq	$0xd2, %zmm8, %zmm16, %zmm4
+        vpternlogq	$0xd2, %zmm20, %zmm8, %zmm16
+        vpternlogq	$0xd2, %zmm30, %zmm20, %zmm8
+        vpternlogq	$0xd2, %zmm31, %zmm30, %zmm20
+        vmovdqu64	%zmm6, %zmm30
+        vmovdqu64	%zmm23, %zmm31
+        vpternlogq	$0xd2, %zmm10, %zmm23, %zmm6
+        vpternlogq	$0xd2, %zmm2, %zmm10, %zmm23
+        vpternlogq	$0xd2, %zmm19, %zmm2, %zmm10
+        vpternlogq	$0xd2, %zmm30, %zmm19, %zmm2
+        vpternlogq	$0xd2, %zmm31, %zmm30, %zmm19
+        # Iota
+        vpxorq	1216(%rax), %zmm0, %zmm0
+        # Round 20
+        # Theta - column parity
+        vmovdqu64	%zmm0, %zmm25
+        vpternlogq	$0x96, %zmm18, %zmm24, %zmm25
+        vpternlogq	$0x96, %zmm6, %zmm12, %zmm25
+        vmovdqu64	%zmm17, %zmm26
+        vpternlogq	$0x96, %zmm5, %zmm11, %zmm26
+        vpternlogq	$0x96, %zmm23, %zmm4, %zmm26
+        vmovdqu64	%zmm9, %zmm27
+        vpternlogq	$0x96, %zmm22, %zmm3, %zmm27
+        vpternlogq	$0x96, %zmm10, %zmm16, %zmm27
+        vmovdqu64	%zmm21, %zmm28
+        vpternlogq	$0x96, %zmm14, %zmm15, %zmm28
+        vpternlogq	$0x96, %zmm2, %zmm8, %zmm28
+        vmovdqu64	%zmm13, %zmm29
+        vpternlogq	$0x96, %zmm1, %zmm7, %zmm29
+        vpternlogq	$0x96, %zmm19, %zmm20, %zmm29
+        # Theta - D and apply in place
+        vprolq	$0x01, %zmm26, %zmm30
+        vpxorq	%zmm29, %zmm30, %zmm30
+        vpxorq	%zmm30, %zmm0, %zmm0
+        vpxorq	%zmm30, %zmm24, %zmm24
+        vpxorq	%zmm30, %zmm18, %zmm18
+        vpxorq	%zmm30, %zmm12, %zmm12
+        vpxorq	%zmm30, %zmm6, %zmm6
+        vprolq	$0x01, %zmm27, %zmm30
+        vpxorq	%zmm25, %zmm30, %zmm30
+        vpxorq	%zmm30, %zmm17, %zmm17
+        vpxorq	%zmm30, %zmm11, %zmm11
+        vpxorq	%zmm30, %zmm5, %zmm5
+        vpxorq	%zmm30, %zmm4, %zmm4
+        vpxorq	%zmm30, %zmm23, %zmm23
+        vprolq	$0x01, %zmm28, %zmm30
+        vpxorq	%zmm26, %zmm30, %zmm30
+        vpxorq	%zmm30, %zmm9, %zmm9
+        vpxorq	%zmm30, %zmm3, %zmm3
+        vpxorq	%zmm30, %zmm22, %zmm22
+        vpxorq	%zmm30, %zmm16, %zmm16
+        vpxorq	%zmm30, %zmm10, %zmm10
+        vprolq	$0x01, %zmm29, %zmm30
+        vpxorq	%zmm27, %zmm30, %zmm30
+        vpxorq	%zmm30, %zmm21, %zmm21
+        vpxorq	%zmm30, %zmm15, %zmm15
+        vpxorq	%zmm30, %zmm14, %zmm14
+        vpxorq	%zmm30, %zmm8, %zmm8
+        vpxorq	%zmm30, %zmm2, %zmm2
+        vprolq	$0x01, %zmm25, %zmm30
+        vpxorq	%zmm28, %zmm30, %zmm30
+        vpxorq	%zmm30, %zmm13, %zmm13
+        vpxorq	%zmm30, %zmm7, %zmm7
+        vpxorq	%zmm30, %zmm1, %zmm1
+        vpxorq	%zmm30, %zmm20, %zmm20
+        vpxorq	%zmm30, %zmm19, %zmm19
+        # Rho - rotate each lane in place
+        vprolq	$0x01, %zmm17, %zmm17
+        vprolq	$62, %zmm9, %zmm9
+        vprolq	$28, %zmm21, %zmm21
+        vprolq	$27, %zmm13, %zmm13
+        vprolq	$36, %zmm24, %zmm24
+        vprolq	$44, %zmm11, %zmm11
+        vprolq	$6, %zmm3, %zmm3
+        vprolq	$55, %zmm15, %zmm15
+        vprolq	$20, %zmm7, %zmm7
+        vprolq	$3, %zmm18, %zmm18
+        vprolq	$10, %zmm5, %zmm5
+        vprolq	$43, %zmm22, %zmm22
+        vprolq	$25, %zmm14, %zmm14
+        vprolq	$39, %zmm1, %zmm1
+        vprolq	$41, %zmm12, %zmm12
+        vprolq	$45, %zmm4, %zmm4
+        vprolq	$15, %zmm16, %zmm16
+        vprolq	$21, %zmm8, %zmm8
+        vprolq	$8, %zmm20, %zmm20
+        vprolq	$18, %zmm6, %zmm6
+        vprolq	$2, %zmm23, %zmm23
+        vprolq	$61, %zmm10, %zmm10
+        vprolq	$56, %zmm2, %zmm2
+        vprolq	$14, %zmm19, %zmm19
+        # Pi - relabel logical -> physical
+        # Chi - per row, in place, fused with vpternlogq (imm 0xd2)
+        vmovdqu64	%zmm0, %zmm30
+        vmovdqu64	%zmm11, %zmm31
+        vpternlogq	$0xd2, %zmm22, %zmm11, %zmm0
+        vpternlogq	$0xd2, %zmm8, %zmm22, %zmm11
+        vpternlogq	$0xd2, %zmm19, %zmm8, %zmm22
+        vpternlogq	$0xd2, %zmm30, %zmm19, %zmm8
+        vpternlogq	$0xd2, %zmm31, %zmm30, %zmm19
+        vmovdqu64	%zmm21, %zmm30
+        vmovdqu64	%zmm7, %zmm31
+        vpternlogq	$0xd2, %zmm18, %zmm7, %zmm21
+        vpternlogq	$0xd2, %zmm4, %zmm18, %zmm7
+        vpternlogq	$0xd2, %zmm10, %zmm4, %zmm18
+        vpternlogq	$0xd2, %zmm30, %zmm10, %zmm4
+        vpternlogq	$0xd2, %zmm31, %zmm30, %zmm10
+        vmovdqu64	%zmm17, %zmm30
+        vmovdqu64	%zmm3, %zmm31
+        vpternlogq	$0xd2, %zmm14, %zmm3, %zmm17
+        vpternlogq	$0xd2, %zmm20, %zmm14, %zmm3
+        vpternlogq	$0xd2, %zmm6, %zmm20, %zmm14
+        vpternlogq	$0xd2, %zmm30, %zmm6, %zmm20
+        vpternlogq	$0xd2, %zmm31, %zmm30, %zmm6
+        vmovdqu64	%zmm13, %zmm30
+        vmovdqu64	%zmm24, %zmm31
+        vpternlogq	$0xd2, %zmm5, %zmm24, %zmm13
+        vpternlogq	$0xd2, %zmm16, %zmm5, %zmm24
+        vpternlogq	$0xd2, %zmm2, %zmm16, %zmm5
+        vpternlogq	$0xd2, %zmm30, %zmm2, %zmm16
+        vpternlogq	$0xd2, %zmm31, %zmm30, %zmm2
+        vmovdqu64	%zmm9, %zmm30
+        vmovdqu64	%zmm15, %zmm31
+        vpternlogq	$0xd2, %zmm1, %zmm15, %zmm9
+        vpternlogq	$0xd2, %zmm12, %zmm1, %zmm15
+        vpternlogq	$0xd2, %zmm23, %zmm12, %zmm1
+        vpternlogq	$0xd2, %zmm30, %zmm23, %zmm12
+        vpternlogq	$0xd2, %zmm31, %zmm30, %zmm23
+        # Iota
+        vpxorq	1280(%rax), %zmm0, %zmm0
+        # Round 21
+        # Theta - column parity
+        vmovdqu64	%zmm0, %zmm25
+        vpternlogq	$0x96, %zmm17, %zmm21, %zmm25
+        vpternlogq	$0x96, %zmm9, %zmm13, %zmm25
+        vmovdqu64	%zmm11, %zmm26
+        vpternlogq	$0x96, %zmm3, %zmm7, %zmm26
+        vpternlogq	$0x96, %zmm15, %zmm24, %zmm26
+        vmovdqu64	%zmm22, %zmm27
+        vpternlogq	$0x96, %zmm14, %zmm18, %zmm27
+        vpternlogq	$0x96, %zmm1, %zmm5, %zmm27
+        vmovdqu64	%zmm8, %zmm28
+        vpternlogq	$0x96, %zmm20, %zmm4, %zmm28
+        vpternlogq	$0x96, %zmm12, %zmm16, %zmm28
+        vmovdqu64	%zmm19, %zmm29
+        vpternlogq	$0x96, %zmm6, %zmm10, %zmm29
+        vpternlogq	$0x96, %zmm23, %zmm2, %zmm29
+        # Theta - D and apply in place
+        vprolq	$0x01, %zmm26, %zmm30
+        vpxorq	%zmm29, %zmm30, %zmm30
+        vpxorq	%zmm30, %zmm0, %zmm0
+        vpxorq	%zmm30, %zmm21, %zmm21
+        vpxorq	%zmm30, %zmm17, %zmm17
+        vpxorq	%zmm30, %zmm13, %zmm13
+        vpxorq	%zmm30, %zmm9, %zmm9
+        vprolq	$0x01, %zmm27, %zmm30
+        vpxorq	%zmm25, %zmm30, %zmm30
+        vpxorq	%zmm30, %zmm11, %zmm11
+        vpxorq	%zmm30, %zmm7, %zmm7
+        vpxorq	%zmm30, %zmm3, %zmm3
+        vpxorq	%zmm30, %zmm24, %zmm24
+        vpxorq	%zmm30, %zmm15, %zmm15
+        vprolq	$0x01, %zmm28, %zmm30
+        vpxorq	%zmm26, %zmm30, %zmm30
+        vpxorq	%zmm30, %zmm22, %zmm22
+        vpxorq	%zmm30, %zmm18, %zmm18
+        vpxorq	%zmm30, %zmm14, %zmm14
+        vpxorq	%zmm30, %zmm5, %zmm5
+        vpxorq	%zmm30, %zmm1, %zmm1
+        vprolq	$0x01, %zmm29, %zmm30
+        vpxorq	%zmm27, %zmm30, %zmm30
+        vpxorq	%zmm30, %zmm8, %zmm8
+        vpxorq	%zmm30, %zmm4, %zmm4
+        vpxorq	%zmm30, %zmm20, %zmm20
+        vpxorq	%zmm30, %zmm16, %zmm16
+        vpxorq	%zmm30, %zmm12, %zmm12
+        vprolq	$0x01, %zmm25, %zmm30
+        vpxorq	%zmm28, %zmm30, %zmm30
+        vpxorq	%zmm30, %zmm19, %zmm19
+        vpxorq	%zmm30, %zmm10, %zmm10
+        vpxorq	%zmm30, %zmm6, %zmm6
+        vpxorq	%zmm30, %zmm2, %zmm2
+        vpxorq	%zmm30, %zmm23, %zmm23
+        # Rho - rotate each lane in place
+        vprolq	$0x01, %zmm11, %zmm11
+        vprolq	$62, %zmm22, %zmm22
+        vprolq	$28, %zmm8, %zmm8
+        vprolq	$27, %zmm19, %zmm19
+        vprolq	$36, %zmm21, %zmm21
+        vprolq	$44, %zmm7, %zmm7
+        vprolq	$6, %zmm18, %zmm18
+        vprolq	$55, %zmm4, %zmm4
+        vprolq	$20, %zmm10, %zmm10
+        vprolq	$3, %zmm17, %zmm17
+        vprolq	$10, %zmm3, %zmm3
+        vprolq	$43, %zmm14, %zmm14
+        vprolq	$25, %zmm20, %zmm20
+        vprolq	$39, %zmm6, %zmm6
+        vprolq	$41, %zmm13, %zmm13
+        vprolq	$45, %zmm24, %zmm24
+        vprolq	$15, %zmm5, %zmm5
+        vprolq	$21, %zmm16, %zmm16
+        vprolq	$8, %zmm2, %zmm2
+        vprolq	$18, %zmm9, %zmm9
+        vprolq	$2, %zmm15, %zmm15
+        vprolq	$61, %zmm1, %zmm1
+        vprolq	$56, %zmm12, %zmm12
+        vprolq	$14, %zmm23, %zmm23
+        # Pi - relabel logical -> physical
+        # Chi - per row, in place, fused with vpternlogq (imm 0xd2)
+        vmovdqu64	%zmm0, %zmm30
+        vmovdqu64	%zmm7, %zmm31
+        vpternlogq	$0xd2, %zmm14, %zmm7, %zmm0
+        vpternlogq	$0xd2, %zmm16, %zmm14, %zmm7
+        vpternlogq	$0xd2, %zmm23, %zmm16, %zmm14
+        vpternlogq	$0xd2, %zmm30, %zmm23, %zmm16
+        vpternlogq	$0xd2, %zmm31, %zmm30, %zmm23
+        vmovdqu64	%zmm8, %zmm30
+        vmovdqu64	%zmm10, %zmm31
+        vpternlogq	$0xd2, %zmm17, %zmm10, %zmm8
+        vpternlogq	$0xd2, %zmm24, %zmm17, %zmm10
+        vpternlogq	$0xd2, %zmm1, %zmm24, %zmm17
+        vpternlogq	$0xd2, %zmm30, %zmm1, %zmm24
+        vpternlogq	$0xd2, %zmm31, %zmm30, %zmm1
+        vmovdqu64	%zmm11, %zmm30
+        vmovdqu64	%zmm18, %zmm31
+        vpternlogq	$0xd2, %zmm20, %zmm18, %zmm11
+        vpternlogq	$0xd2, %zmm2, %zmm20, %zmm18
+        vpternlogq	$0xd2, %zmm9, %zmm2, %zmm20
+        vpternlogq	$0xd2, %zmm30, %zmm9, %zmm2
+        vpternlogq	$0xd2, %zmm31, %zmm30, %zmm9
+        vmovdqu64	%zmm19, %zmm30
+        vmovdqu64	%zmm21, %zmm31
+        vpternlogq	$0xd2, %zmm3, %zmm21, %zmm19
+        vpternlogq	$0xd2, %zmm5, %zmm3, %zmm21
+        vpternlogq	$0xd2, %zmm12, %zmm5, %zmm3
+        vpternlogq	$0xd2, %zmm30, %zmm12, %zmm5
+        vpternlogq	$0xd2, %zmm31, %zmm30, %zmm12
+        vmovdqu64	%zmm22, %zmm30
+        vmovdqu64	%zmm4, %zmm31
+        vpternlogq	$0xd2, %zmm6, %zmm4, %zmm22
+        vpternlogq	$0xd2, %zmm13, %zmm6, %zmm4
+        vpternlogq	$0xd2, %zmm15, %zmm13, %zmm6
+        vpternlogq	$0xd2, %zmm30, %zmm15, %zmm13
+        vpternlogq	$0xd2, %zmm31, %zmm30, %zmm15
+        # Iota
+        vpxorq	1344(%rax), %zmm0, %zmm0
+        # Round 22
+        # Theta - column parity
+        vmovdqu64	%zmm0, %zmm25
+        vpternlogq	$0x96, %zmm11, %zmm8, %zmm25
+        vpternlogq	$0x96, %zmm22, %zmm19, %zmm25
+        vmovdqu64	%zmm7, %zmm26
+        vpternlogq	$0x96, %zmm18, %zmm10, %zmm26
+        vpternlogq	$0x96, %zmm4, %zmm21, %zmm26
+        vmovdqu64	%zmm14, %zmm27
+        vpternlogq	$0x96, %zmm20, %zmm17, %zmm27
+        vpternlogq	$0x96, %zmm6, %zmm3, %zmm27
+        vmovdqu64	%zmm16, %zmm28
+        vpternlogq	$0x96, %zmm2, %zmm24, %zmm28
+        vpternlogq	$0x96, %zmm13, %zmm5, %zmm28
+        vmovdqu64	%zmm23, %zmm29
+        vpternlogq	$0x96, %zmm9, %zmm1, %zmm29
+        vpternlogq	$0x96, %zmm15, %zmm12, %zmm29
+        # Theta - D and apply in place
+        vprolq	$0x01, %zmm26, %zmm30
+        vpxorq	%zmm29, %zmm30, %zmm30
+        vpxorq	%zmm30, %zmm0, %zmm0
+        vpxorq	%zmm30, %zmm8, %zmm8
+        vpxorq	%zmm30, %zmm11, %zmm11
+        vpxorq	%zmm30, %zmm19, %zmm19
+        vpxorq	%zmm30, %zmm22, %zmm22
+        vprolq	$0x01, %zmm27, %zmm30
+        vpxorq	%zmm25, %zmm30, %zmm30
+        vpxorq	%zmm30, %zmm7, %zmm7
+        vpxorq	%zmm30, %zmm10, %zmm10
+        vpxorq	%zmm30, %zmm18, %zmm18
+        vpxorq	%zmm30, %zmm21, %zmm21
+        vpxorq	%zmm30, %zmm4, %zmm4
+        vprolq	$0x01, %zmm28, %zmm30
+        vpxorq	%zmm26, %zmm30, %zmm30
+        vpxorq	%zmm30, %zmm14, %zmm14
+        vpxorq	%zmm30, %zmm17, %zmm17
+        vpxorq	%zmm30, %zmm20, %zmm20
+        vpxorq	%zmm30, %zmm3, %zmm3
+        vpxorq	%zmm30, %zmm6, %zmm6
+        vprolq	$0x01, %zmm29, %zmm30
+        vpxorq	%zmm27, %zmm30, %zmm30
+        vpxorq	%zmm30, %zmm16, %zmm16
+        vpxorq	%zmm30, %zmm24, %zmm24
+        vpxorq	%zmm30, %zmm2, %zmm2
+        vpxorq	%zmm30, %zmm5, %zmm5
+        vpxorq	%zmm30, %zmm13, %zmm13
+        vprolq	$0x01, %zmm25, %zmm30
+        vpxorq	%zmm28, %zmm30, %zmm30
+        vpxorq	%zmm30, %zmm23, %zmm23
+        vpxorq	%zmm30, %zmm1, %zmm1
+        vpxorq	%zmm30, %zmm9, %zmm9
+        vpxorq	%zmm30, %zmm12, %zmm12
+        vpxorq	%zmm30, %zmm15, %zmm15
+        # Rho - rotate each lane in place
+        vprolq	$0x01, %zmm7, %zmm7
+        vprolq	$62, %zmm14, %zmm14
+        vprolq	$28, %zmm16, %zmm16
+        vprolq	$27, %zmm23, %zmm23
+        vprolq	$36, %zmm8, %zmm8
+        vprolq	$44, %zmm10, %zmm10
+        vprolq	$6, %zmm17, %zmm17
+        vprolq	$55, %zmm24, %zmm24
+        vprolq	$20, %zmm1, %zmm1
+        vprolq	$3, %zmm11, %zmm11
+        vprolq	$10, %zmm18, %zmm18
+        vprolq	$43, %zmm20, %zmm20
+        vprolq	$25, %zmm2, %zmm2
+        vprolq	$39, %zmm9, %zmm9
+        vprolq	$41, %zmm19, %zmm19
+        vprolq	$45, %zmm21, %zmm21
+        vprolq	$15, %zmm3, %zmm3
+        vprolq	$21, %zmm5, %zmm5
+        vprolq	$8, %zmm12, %zmm12
+        vprolq	$18, %zmm22, %zmm22
+        vprolq	$2, %zmm4, %zmm4
+        vprolq	$61, %zmm6, %zmm6
+        vprolq	$56, %zmm13, %zmm13
+        vprolq	$14, %zmm15, %zmm15
+        # Pi - relabel logical -> physical
+        # Chi - per row, in place, fused with vpternlogq (imm 0xd2)
+        vmovdqu64	%zmm0, %zmm30
+        vmovdqu64	%zmm10, %zmm31
+        vpternlogq	$0xd2, %zmm20, %zmm10, %zmm0
+        vpternlogq	$0xd2, %zmm5, %zmm20, %zmm10
+        vpternlogq	$0xd2, %zmm15, %zmm5, %zmm20
+        vpternlogq	$0xd2, %zmm30, %zmm15, %zmm5
+        vpternlogq	$0xd2, %zmm31, %zmm30, %zmm15
+        vmovdqu64	%zmm16, %zmm30
+        vmovdqu64	%zmm1, %zmm31
+        vpternlogq	$0xd2, %zmm11, %zmm1, %zmm16
+        vpternlogq	$0xd2, %zmm21, %zmm11, %zmm1
+        vpternlogq	$0xd2, %zmm6, %zmm21, %zmm11
+        vpternlogq	$0xd2, %zmm30, %zmm6, %zmm21
+        vpternlogq	$0xd2, %zmm31, %zmm30, %zmm6
+        vmovdqu64	%zmm7, %zmm30
+        vmovdqu64	%zmm17, %zmm31
+        vpternlogq	$0xd2, %zmm2, %zmm17, %zmm7
+        vpternlogq	$0xd2, %zmm12, %zmm2, %zmm17
+        vpternlogq	$0xd2, %zmm22, %zmm12, %zmm2
+        vpternlogq	$0xd2, %zmm30, %zmm22, %zmm12
+        vpternlogq	$0xd2, %zmm31, %zmm30, %zmm22
+        vmovdqu64	%zmm23, %zmm30
+        vmovdqu64	%zmm8, %zmm31
+        vpternlogq	$0xd2, %zmm18, %zmm8, %zmm23
+        vpternlogq	$0xd2, %zmm3, %zmm18, %zmm8
+        vpternlogq	$0xd2, %zmm13, %zmm3, %zmm18
+        vpternlogq	$0xd2, %zmm30, %zmm13, %zmm3
+        vpternlogq	$0xd2, %zmm31, %zmm30, %zmm13
+        vmovdqu64	%zmm14, %zmm30
+        vmovdqu64	%zmm24, %zmm31
+        vpternlogq	$0xd2, %zmm9, %zmm24, %zmm14
+        vpternlogq	$0xd2, %zmm19, %zmm9, %zmm24
+        vpternlogq	$0xd2, %zmm4, %zmm19, %zmm9
+        vpternlogq	$0xd2, %zmm30, %zmm4, %zmm19
+        vpternlogq	$0xd2, %zmm31, %zmm30, %zmm4
+        # Iota
+        vpxorq	1408(%rax), %zmm0, %zmm0
+        # Round 23
+        # Theta - column parity
+        vmovdqu64	%zmm0, %zmm25
+        vpternlogq	$0x96, %zmm7, %zmm16, %zmm25
+        vpternlogq	$0x96, %zmm14, %zmm23, %zmm25
+        vmovdqu64	%zmm10, %zmm26
+        vpternlogq	$0x96, %zmm17, %zmm1, %zmm26
+        vpternlogq	$0x96, %zmm24, %zmm8, %zmm26
+        vmovdqu64	%zmm20, %zmm27
+        vpternlogq	$0x96, %zmm2, %zmm11, %zmm27
+        vpternlogq	$0x96, %zmm9, %zmm18, %zmm27
+        vmovdqu64	%zmm5, %zmm28
+        vpternlogq	$0x96, %zmm12, %zmm21, %zmm28
+        vpternlogq	$0x96, %zmm19, %zmm3, %zmm28
+        vmovdqu64	%zmm15, %zmm29
+        vpternlogq	$0x96, %zmm22, %zmm6, %zmm29
+        vpternlogq	$0x96, %zmm4, %zmm13, %zmm29
+        # Theta - D and apply in place
+        vprolq	$0x01, %zmm26, %zmm30
+        vpxorq	%zmm29, %zmm30, %zmm30
+        vpxorq	%zmm30, %zmm0, %zmm0
+        vpxorq	%zmm30, %zmm16, %zmm16
+        vpxorq	%zmm30, %zmm7, %zmm7
+        vpxorq	%zmm30, %zmm23, %zmm23
+        vpxorq	%zmm30, %zmm14, %zmm14
+        vprolq	$0x01, %zmm27, %zmm30
+        vpxorq	%zmm25, %zmm30, %zmm30
+        vpxorq	%zmm30, %zmm10, %zmm10
+        vpxorq	%zmm30, %zmm1, %zmm1
+        vpxorq	%zmm30, %zmm17, %zmm17
+        vpxorq	%zmm30, %zmm8, %zmm8
+        vpxorq	%zmm30, %zmm24, %zmm24
+        vprolq	$0x01, %zmm28, %zmm30
+        vpxorq	%zmm26, %zmm30, %zmm30
+        vpxorq	%zmm30, %zmm20, %zmm20
+        vpxorq	%zmm30, %zmm11, %zmm11
+        vpxorq	%zmm30, %zmm2, %zmm2
+        vpxorq	%zmm30, %zmm18, %zmm18
+        vpxorq	%zmm30, %zmm9, %zmm9
+        vprolq	$0x01, %zmm29, %zmm30
+        vpxorq	%zmm27, %zmm30, %zmm30
+        vpxorq	%zmm30, %zmm5, %zmm5
+        vpxorq	%zmm30, %zmm21, %zmm21
+        vpxorq	%zmm30, %zmm12, %zmm12
+        vpxorq	%zmm30, %zmm3, %zmm3
+        vpxorq	%zmm30, %zmm19, %zmm19
+        vprolq	$0x01, %zmm25, %zmm30
+        vpxorq	%zmm28, %zmm30, %zmm30
+        vpxorq	%zmm30, %zmm15, %zmm15
+        vpxorq	%zmm30, %zmm6, %zmm6
+        vpxorq	%zmm30, %zmm22, %zmm22
+        vpxorq	%zmm30, %zmm13, %zmm13
+        vpxorq	%zmm30, %zmm4, %zmm4
+        # Rho - rotate each lane in place
+        vprolq	$0x01, %zmm10, %zmm10
+        vprolq	$62, %zmm20, %zmm20
+        vprolq	$28, %zmm5, %zmm5
+        vprolq	$27, %zmm15, %zmm15
+        vprolq	$36, %zmm16, %zmm16
+        vprolq	$44, %zmm1, %zmm1
+        vprolq	$6, %zmm11, %zmm11
+        vprolq	$55, %zmm21, %zmm21
+        vprolq	$20, %zmm6, %zmm6
+        vprolq	$3, %zmm7, %zmm7
+        vprolq	$10, %zmm17, %zmm17
+        vprolq	$43, %zmm2, %zmm2
+        vprolq	$25, %zmm12, %zmm12
+        vprolq	$39, %zmm22, %zmm22
+        vprolq	$41, %zmm23, %zmm23
+        vprolq	$45, %zmm8, %zmm8
+        vprolq	$15, %zmm18, %zmm18
+        vprolq	$21, %zmm3, %zmm3
+        vprolq	$8, %zmm13, %zmm13
+        vprolq	$18, %zmm14, %zmm14
+        vprolq	$2, %zmm24, %zmm24
+        vprolq	$61, %zmm9, %zmm9
+        vprolq	$56, %zmm19, %zmm19
+        vprolq	$14, %zmm4, %zmm4
+        # Pi - relabel logical -> physical
+        # Chi - per row, in place, fused with vpternlogq (imm 0xd2)
+        vmovdqu64	%zmm0, %zmm30
+        vmovdqu64	%zmm1, %zmm31
+        vpternlogq	$0xd2, %zmm2, %zmm1, %zmm0
+        vpternlogq	$0xd2, %zmm3, %zmm2, %zmm1
+        vpternlogq	$0xd2, %zmm4, %zmm3, %zmm2
+        vpternlogq	$0xd2, %zmm30, %zmm4, %zmm3
+        vpternlogq	$0xd2, %zmm31, %zmm30, %zmm4
+        vmovdqu64	%zmm5, %zmm30
+        vmovdqu64	%zmm6, %zmm31
+        vpternlogq	$0xd2, %zmm7, %zmm6, %zmm5
+        vpternlogq	$0xd2, %zmm8, %zmm7, %zmm6
+        vpternlogq	$0xd2, %zmm9, %zmm8, %zmm7
+        vpternlogq	$0xd2, %zmm30, %zmm9, %zmm8
+        vpternlogq	$0xd2, %zmm31, %zmm30, %zmm9
+        vmovdqu64	%zmm10, %zmm30
+        vmovdqu64	%zmm11, %zmm31
+        vpternlogq	$0xd2, %zmm12, %zmm11, %zmm10
+        vpternlogq	$0xd2, %zmm13, %zmm12, %zmm11
+        vpternlogq	$0xd2, %zmm14, %zmm13, %zmm12
+        vpternlogq	$0xd2, %zmm30, %zmm14, %zmm13
+        vpternlogq	$0xd2, %zmm31, %zmm30, %zmm14
+        vmovdqu64	%zmm15, %zmm30
+        vmovdqu64	%zmm16, %zmm31
+        vpternlogq	$0xd2, %zmm17, %zmm16, %zmm15
+        vpternlogq	$0xd2, %zmm18, %zmm17, %zmm16
+        vpternlogq	$0xd2, %zmm19, %zmm18, %zmm17
+        vpternlogq	$0xd2, %zmm30, %zmm19, %zmm18
+        vpternlogq	$0xd2, %zmm31, %zmm30, %zmm19
+        vmovdqu64	%zmm20, %zmm30
+        vmovdqu64	%zmm21, %zmm31
+        vpternlogq	$0xd2, %zmm22, %zmm21, %zmm20
+        vpternlogq	$0xd2, %zmm23, %zmm22, %zmm21
+        vpternlogq	$0xd2, %zmm24, %zmm23, %zmm22
+        vpternlogq	$0xd2, %zmm30, %zmm24, %zmm23
+        vpternlogq	$0xd2, %zmm31, %zmm30, %zmm24
+        # Iota
+        vpxorq	1472(%rax), %zmm0, %zmm0
+        vmovdqu64	%zmm0, (%rdi)
+        vmovdqu64	%zmm1, 64(%rdi)
+        vmovdqu64	%zmm2, 128(%rdi)
+        vmovdqu64	%zmm3, 192(%rdi)
+        vmovdqu64	%zmm4, 256(%rdi)
+        vmovdqu64	%zmm5, 320(%rdi)
+        vmovdqu64	%zmm6, 384(%rdi)
+        vmovdqu64	%zmm7, 448(%rdi)
+        vmovdqu64	%zmm8, 512(%rdi)
+        vmovdqu64	%zmm9, 576(%rdi)
+        vmovdqu64	%zmm10, 640(%rdi)
+        vmovdqu64	%zmm11, 704(%rdi)
+        vmovdqu64	%zmm12, 768(%rdi)
+        vmovdqu64	%zmm13, 832(%rdi)
+        vmovdqu64	%zmm14, 896(%rdi)
+        vmovdqu64	%zmm15, 960(%rdi)
+        vmovdqu64	%zmm16, 1024(%rdi)
+        vmovdqu64	%zmm17, 1088(%rdi)
+        vmovdqu64	%zmm18, 1152(%rdi)
+        vmovdqu64	%zmm19, 1216(%rdi)
+        vmovdqu64	%zmm20, 1280(%rdi)
+        vmovdqu64	%zmm21, 1344(%rdi)
+        vmovdqu64	%zmm22, 1408(%rdi)
+        vmovdqu64	%zmm23, 1472(%rdi)
+        vmovdqu64	%zmm24, 1536(%rdi)
+        movq	%rdi, %rbx
+        movq	$21, %r15
+L_sha3_blocksx8_out_avx512_word:
+        movq	(%rbx), %rbp
+        movq	%rbp, (%rsi)
+        movq	8(%rbx), %rbp
+        movq	%rbp, (%rcx)
+        movq	16(%rbx), %rbp
+        movq	%rbp, (%r8)
+        movq	24(%rbx), %rbp
+        movq	%rbp, (%r9)
+        movq	32(%rbx), %rbp
+        movq	%rbp, (%r10)
+        movq	40(%rbx), %rbp
+        movq	%rbp, (%r11)
+        movq	48(%rbx), %rbp
+        movq	%rbp, (%r12)
+        movq	56(%rbx), %rbp
+        movq	%rbp, (%r13)
+        addq	$0x40, %rbx
+        addq	$8, %rsi
+        addq	$8, %rcx
+        addq	$8, %r8
+        addq	$8, %r9
+        addq	$8, %r10
+        addq	$8, %r11
+        addq	$8, %r12
+        addq	$8, %r13
+        subq	$0x01, %r14
+        jz	L_sha3_blocksx8_out_avx512_done
+        subq	$0x01, %r15
+        jnz	L_sha3_blocksx8_out_avx512_word
+        jmp	L_sha3_blocksx8_out_avx512_block
+L_sha3_blocksx8_out_avx512_done:
+        vzeroupper
+        popq	%rbp
+        popq	%rbx
+        popq	%r15
+        popq	%r14
+        popq	%r13
+        popq	%r12
+        repz retq
+#ifndef __APPLE__
+.size	sha3_blocksx8_out_avx512,.-sha3_blocksx8_out_avx512
+#endif /* __APPLE__ */
+#endif /* WOLFSSL_HAVE_FRODOKEM */
+#endif /* HAVE_INTEL_AVX512 */
 #endif /* HAVE_INTEL_AVX2 */
 
 #if defined(__linux__) && defined(__ELF__)
diff --git a/wolfcrypt/src/sha3_asm.asm b/wolfcrypt/src/sha3_asm.asm
index 8f4db30ff5..874875b3df 100644
--- a/wolfcrypt/src/sha3_asm.asm
+++ b/wolfcrypt/src/sha3_asm.asm
@@ -9900,6 +9900,9 @@ ENDIF
 IFDEF WOLFSSL_HAVE_SLHDSA
 wc_masm_cond_0 = 1
 ENDIF
+IFDEF WOLFSSL_HAVE_FRODOKEM
+wc_masm_cond_0 = 1
+ENDIF
 IF wc_masm_cond_0
 _TEXT SEGMENT READONLY PARA
 sha3_blocksx4_avx2 PROC
@@ -15270,6 +15273,18 @@ sha3_blocksx4_avx2 PROC
         ret
 sha3_blocksx4_avx2 ENDP
 _TEXT ENDS
+ENDIF
+wc_masm_cond_1 = 0
+IFDEF WOLFSSL_HAVE_MLKEM
+wc_masm_cond_1 = 1
+ENDIF
+IFDEF WOLFSSL_HAVE_MLDSA
+wc_masm_cond_1 = 1
+ENDIF
+IFDEF WOLFSSL_HAVE_SLHDSA
+wc_masm_cond_1 = 1
+ENDIF
+IF wc_masm_cond_1
 _DATA SEGMENT
 ALIGN 16
 L_sha3_128_blockx4_seed_avx2_end_mark QWORD 8000000000000000h, 8000000000000000h
@@ -20652,15 +20667,15 @@ sha3_128_blocksx4_seed_avx2 PROC
 sha3_128_blocksx4_seed_avx2 ENDP
 _TEXT ENDS
 ENDIF
-IFDEF WOLFSSL_HAVE_MLKEM
-_DATA SEGMENT
-ALIGN 16
-L_sha3_256_blockx4_seed_avx2_end_mark QWORD 8000000000000000h, 8000000000000000h
-        QWORD 8000000000000000h, 8000000000000000h
-ptr_L_sha3_256_blockx4_seed_avx2_end_mark QWORD L_sha3_256_blockx4_seed_avx2_end_mark
-_DATA ENDS
+IFDEF WOLFSSL_HAVE_FRODOKEM
 _TEXT SEGMENT READONLY PARA
-sha3_256_blocksx4_seed_avx2 PROC
+sha3_blocksx4_out_avx2 PROC
+        push	r12
+        push	r13
+        push	r14
+        push	r15
+        push	rdi
+        push	rsi
         sub	rsp, 160
         vmovdqu	OWORD PTR [rsp], xmm6
         vmovdqu	OWORD PTR [rsp+16], xmm7
@@ -20673,45 +20688,47 @@ sha3_256_blocksx4_seed_avx2 PROC
         vmovdqu	OWORD PTR [rsp+128], xmm14
         vmovdqu	OWORD PTR [rsp+144], xmm15
         mov	rax, QWORD PTR [ptr_L_sha3_x4_avx2_r]
-        mov	r8, rcx
+        mov	r14d, r8d
+        mov	r11, rdx
+        add	r11, r14
+        mov	r12, r11
+        add	r12, r14
+        mov	r13, r12
+        add	r13, r14
+        shr	r14, 3
+L_sha3_blocksx4_out_avx2_block:
+        vmovdqu	ymm15, YMMWORD PTR [rcx]
         mov	r9, rcx
-        vpbroadcastq	ymm15, QWORD PTR [rdx]
+        mov	r10, rcx
         add	rcx, 128
-        vpbroadcastq	ymm11, QWORD PTR [rdx+8]
-        add	r8, 384
-        vpbroadcastq	ymm12, QWORD PTR [rdx+16]
-        add	r9, 640
-        vpbroadcastq	ymm13, QWORD PTR [rdx+24]
-        vmovdqu	ymm5, YMMWORD PTR L_sha3_256_blockx4_seed_avx2_end_mark
-        vpxor	ymm6, ymm6, ymm6
-        vmovdqu	YMMWORD PTR [rcx+-96], ymm11
-        vmovdqu	YMMWORD PTR [rcx+-64], ymm12
-        vmovdqu	YMMWORD PTR [rcx+-32], ymm13
-        vmovdqu	ymm14, YMMWORD PTR [rcx]
-        vmovdqu	YMMWORD PTR [rcx+32], ymm6
-        vmovdqu	YMMWORD PTR [rcx+64], ymm6
-        vmovdqu	YMMWORD PTR [rcx+96], ymm6
-        vmovdqu	YMMWORD PTR [rcx+128], ymm6
-        vmovdqu	YMMWORD PTR [r8+-96], ymm6
-        vmovdqu	YMMWORD PTR [r8+-64], ymm6
-        vmovdqu	YMMWORD PTR [r8+-32], ymm6
-        vmovdqu	YMMWORD PTR [r8], ymm6
-        vmovdqu	YMMWORD PTR [r8+32], ymm6
-        vmovdqu	YMMWORD PTR [r8+64], ymm6
-        vmovdqu	YMMWORD PTR [r8+96], ymm6
-        vmovdqu	YMMWORD PTR [r8+128], ymm5
-        vmovdqu	YMMWORD PTR [r9+-96], ymm6
-        vmovdqu	YMMWORD PTR [r9+-64], ymm6
-        vmovdqu	YMMWORD PTR [r9+-32], ymm6
-        vmovdqu	YMMWORD PTR [r9], ymm6
-        vmovdqu	YMMWORD PTR [r9+32], ymm6
-        vmovdqu	YMMWORD PTR [r9+64], ymm6
-        vmovdqu	YMMWORD PTR [r9+96], ymm6
-        vmovdqu	YMMWORD PTR [r9+128], ymm6
-        vmovdqu	ymm10, ymm15
-        vpxor	ymm11, ymm11, ymm5
+        add	r9, 384
+        add	r10, 640
         ; Round 0
         ; Calc b[0..4]
+        vmovdqu	ymm11, YMMWORD PTR [rcx+-96]
+        vmovdqu	ymm12, YMMWORD PTR [rcx+-64]
+        vmovdqu	ymm13, YMMWORD PTR [rcx+-32]
+        vmovdqu	ymm14, YMMWORD PTR [rcx]
+        vpxor	ymm10, ymm15, [rcx+32]
+        vpxor	ymm11, ymm11, [rcx+64]
+        vpxor	ymm12, ymm12, [rcx+96]
+        vpxor	ymm13, ymm13, [rcx+128]
+        vpxor	ymm14, ymm14, [r9+-96]
+        vpxor	ymm10, ymm10, [r9+-64]
+        vpxor	ymm11, ymm11, [r9+-32]
+        vpxor	ymm12, ymm12, [r9]
+        vpxor	ymm13, ymm13, [r9+32]
+        vpxor	ymm14, ymm14, [r9+64]
+        vpxor	ymm10, ymm10, [r9+96]
+        vpxor	ymm11, ymm11, [r9+128]
+        vpxor	ymm12, ymm12, [r10+-96]
+        vpxor	ymm13, ymm13, [r10+-64]
+        vpxor	ymm14, ymm14, [r10+-32]
+        vpxor	ymm10, ymm10, [r10]
+        vpxor	ymm11, ymm11, [r10+32]
+        vpxor	ymm12, ymm12, [r10+64]
+        vpxor	ymm13, ymm13, [r10+96]
+        vpxor	ymm14, ymm14, [r10+128]
         ; Calc t[0..4]
         vpsrlq	ymm0, ymm11, 63
         vpsrlq	ymm1, ymm12, 63
@@ -20737,9 +20754,9 @@ sha3_256_blocksx4_seed_avx2 PROC
         ; Row 0
         vpxor	ymm10, ymm5, ymm15
         vpxor	ymm11, ymm6, [rcx+64]
-        vpxor	ymm12, ymm7, [r8]
-        vpxor	ymm13, ymm8, [r9+-64]
-        vpxor	ymm14, ymm9, [r9+128]
+        vpxor	ymm12, ymm7, [r9]
+        vpxor	ymm13, ymm8, [r10+-64]
+        vpxor	ymm14, ymm9, [r10+128]
         vpsrlq	ymm0, ymm11, 20
         vpsrlq	ymm1, ymm12, 21
         vpsrlq	ymm2, ymm13, 43
@@ -20765,15 +20782,15 @@ sha3_256_blocksx4_seed_avx2 PROC
         vpxor	ymm3, ymm3, ymm13
         vpxor	ymm4, ymm4, ymm14
         vmovdqu	YMMWORD PTR [rcx+64], ymm1
-        vmovdqu	YMMWORD PTR [r8], ymm2
-        vmovdqu	YMMWORD PTR [r9+-64], ymm3
-        vmovdqu	YMMWORD PTR [r9+128], ymm4
+        vmovdqu	YMMWORD PTR [r9], ymm2
+        vmovdqu	YMMWORD PTR [r10+-64], ymm3
+        vmovdqu	YMMWORD PTR [r10+128], ymm4
         ; Row 1
         vpxor	ymm10, ymm8, [rcx+-32]
-        vpxor	ymm11, ymm9, [r8+-96]
-        vpxor	ymm12, ymm5, [r8+-64]
-        vpxor	ymm13, ymm6, [r8+128]
-        vpxor	ymm14, ymm7, [r9+64]
+        vpxor	ymm11, ymm9, [r9+-96]
+        vpxor	ymm12, ymm5, [r9+-64]
+        vpxor	ymm13, ymm6, [r9+128]
+        vpxor	ymm14, ymm7, [r10+64]
         vpsrlq	ymm0, ymm10, 36
         vpsrlq	ymm1, ymm11, 44
         vpsrlq	ymm2, ymm12, 61
@@ -20800,16 +20817,16 @@ sha3_256_blocksx4_seed_avx2 PROC
         vpxor	ymm3, ymm3, ymm13
         vpxor	ymm4, ymm4, ymm14
         vmovdqu	YMMWORD PTR [rcx+-32], ymm0
-        vmovdqu	YMMWORD PTR [r8+-96], ymm1
-        vmovdqu	YMMWORD PTR [r8+-64], ymm2
-        vmovdqu	YMMWORD PTR [r8+128], ymm3
-        vmovdqu	YMMWORD PTR [r9+64], ymm4
+        vmovdqu	YMMWORD PTR [r9+-96], ymm1
+        vmovdqu	YMMWORD PTR [r9+-64], ymm2
+        vmovdqu	YMMWORD PTR [r9+128], ymm3
+        vmovdqu	YMMWORD PTR [r10+64], ymm4
         ; Row 2
         vpxor	ymm10, ymm6, [rcx+-96]
         vpxor	ymm11, ymm7, [rcx+96]
-        vpxor	ymm12, ymm8, [r8+32]
-        vpxor	ymm13, ymm9, [r9+-32]
-        vpxor	ymm14, ymm5, [r9]
+        vpxor	ymm12, ymm8, [r9+32]
+        vpxor	ymm13, ymm9, [r10+-32]
+        vpxor	ymm14, ymm5, [r10]
         vpsrlq	ymm0, ymm10, 63
         vpsrlq	ymm1, ymm11, 58
         vpsrlq	ymm2, ymm12, 39
@@ -20837,15 +20854,15 @@ sha3_256_blocksx4_seed_avx2 PROC
         vpxor	ymm4, ymm4, ymm14
         vmovdqu	YMMWORD PTR [rcx+-96], ymm0
         vmovdqu	YMMWORD PTR [rcx+96], ymm1
-        vmovdqu	YMMWORD PTR [r8+32], ymm2
-        vmovdqu	YMMWORD PTR [r9+-32], ymm3
-        vmovdqu	YMMWORD PTR [r9], ymm4
+        vmovdqu	YMMWORD PTR [r9+32], ymm2
+        vmovdqu	YMMWORD PTR [r10+-32], ymm3
+        vmovdqu	YMMWORD PTR [r10], ymm4
         ; Row 3
         vpxor	ymm10, ymm9, [rcx]
         vpxor	ymm11, ymm5, [rcx+32]
-        vpxor	ymm12, ymm6, [r8+-32]
-        vpxor	ymm13, ymm7, [r9+-96]
-        vpxor	ymm14, ymm8, [r9+96]
+        vpxor	ymm12, ymm6, [r9+-32]
+        vpxor	ymm13, ymm7, [r10+-96]
+        vpxor	ymm14, ymm8, [r10+96]
         vpsrlq	ymm0, ymm10, 37
         vpsrlq	ymm1, ymm11, 28
         vpsrlq	ymm2, ymm12, 54
@@ -20873,15 +20890,15 @@ sha3_256_blocksx4_seed_avx2 PROC
         vpxor	ymm4, ymm4, ymm14
         vmovdqu	YMMWORD PTR [rcx], ymm0
         vmovdqu	YMMWORD PTR [rcx+32], ymm1
-        vmovdqu	YMMWORD PTR [r8+-32], ymm2
-        vmovdqu	YMMWORD PTR [r9+-96], ymm3
-        vmovdqu	YMMWORD PTR [r9+96], ymm4
+        vmovdqu	YMMWORD PTR [r9+-32], ymm2
+        vmovdqu	YMMWORD PTR [r10+-96], ymm3
+        vmovdqu	YMMWORD PTR [r10+96], ymm4
         ; Row 4
         vpxor	ymm10, ymm7, [rcx+-64]
         vpxor	ymm11, ymm8, [rcx+128]
-        vpxor	ymm12, ymm9, [r8+64]
-        vpxor	ymm13, ymm5, [r8+96]
-        vpxor	ymm14, ymm6, [r9+32]
+        vpxor	ymm12, ymm9, [r9+64]
+        vpxor	ymm13, ymm5, [r9+96]
+        vpxor	ymm14, ymm6, [r10+32]
         vpsrlq	ymm0, ymm10, 2
         vpsrlq	ymm1, ymm11, 9
         vpsrlq	ymm2, ymm12, 25
@@ -20909,9 +20926,9 @@ sha3_256_blocksx4_seed_avx2 PROC
         vpxor	ymm4, ymm4, ymm14
         vmovdqu	YMMWORD PTR [rcx+-64], ymm0
         vmovdqu	YMMWORD PTR [rcx+128], ymm1
-        vmovdqu	YMMWORD PTR [r8+64], ymm2
-        vmovdqu	YMMWORD PTR [r8+96], ymm3
-        vmovdqu	YMMWORD PTR [r9+32], ymm4
+        vmovdqu	YMMWORD PTR [r9+64], ymm2
+        vmovdqu	YMMWORD PTR [r9+96], ymm3
+        vmovdqu	YMMWORD PTR [r10+32], ymm4
         ; Round 1
         ; Calc b[0..4]
         vpxor	ymm10, ymm0, ymm15
@@ -20921,19 +20938,19 @@ sha3_256_blocksx4_seed_avx2 PROC
         vpxor	ymm11, ymm1, [rcx+32]
         vpxor	ymm11, ymm11, [rcx+64]
         vpxor	ymm11, ymm11, [rcx+96]
-        vpxor	ymm11, ymm11, [r8+-96]
-        vpxor	ymm12, ymm2, [r8+-64]
-        vpxor	ymm12, ymm12, [r8+-32]
-        vpxor	ymm12, ymm12, [r8]
-        vpxor	ymm12, ymm12, [r8+32]
-        vpxor	ymm13, ymm3, [r8+128]
-        vpxor	ymm13, ymm13, [r9+-96]
-        vpxor	ymm13, ymm13, [r9+-64]
-        vpxor	ymm13, ymm13, [r9+-32]
-        vpxor	ymm14, ymm4, [r9]
-        vpxor	ymm14, ymm14, [r9+64]
-        vpxor	ymm14, ymm14, [r9+96]
-        vpxor	ymm14, ymm14, [r9+128]
+        vpxor	ymm11, ymm11, [r9+-96]
+        vpxor	ymm12, ymm2, [r9+-64]
+        vpxor	ymm12, ymm12, [r9+-32]
+        vpxor	ymm12, ymm12, [r9]
+        vpxor	ymm12, ymm12, [r9+32]
+        vpxor	ymm13, ymm3, [r9+128]
+        vpxor	ymm13, ymm13, [r10+-96]
+        vpxor	ymm13, ymm13, [r10+-64]
+        vpxor	ymm13, ymm13, [r10+-32]
+        vpxor	ymm14, ymm4, [r10]
+        vpxor	ymm14, ymm14, [r10+64]
+        vpxor	ymm14, ymm14, [r10+96]
+        vpxor	ymm14, ymm14, [r10+128]
         ; Calc t[0..4]
         vpsrlq	ymm0, ymm11, 63
         vpsrlq	ymm1, ymm12, 63
@@ -20958,10 +20975,10 @@ sha3_256_blocksx4_seed_avx2 PROC
         ; Row Mix
         ; Row 0
         vpxor	ymm10, ymm5, ymm15
-        vpxor	ymm11, ymm6, [r8+-96]
-        vpxor	ymm12, ymm7, [r8+32]
-        vpxor	ymm13, ymm8, [r9+-96]
-        vpxor	ymm14, ymm9, [r9+32]
+        vpxor	ymm11, ymm6, [r9+-96]
+        vpxor	ymm12, ymm7, [r9+32]
+        vpxor	ymm13, ymm8, [r10+-96]
+        vpxor	ymm14, ymm9, [r10+32]
         vpsrlq	ymm0, ymm11, 20
         vpsrlq	ymm1, ymm12, 21
         vpsrlq	ymm2, ymm13, 43
@@ -20986,16 +21003,16 @@ sha3_256_blocksx4_seed_avx2 PROC
         vpxor	ymm15, ymm15, [rax+32]
         vpxor	ymm3, ymm3, ymm13
         vpxor	ymm4, ymm4, ymm14
-        vmovdqu	YMMWORD PTR [r8+-96], ymm1
-        vmovdqu	YMMWORD PTR [r8+32], ymm2
-        vmovdqu	YMMWORD PTR [r9+-96], ymm3
-        vmovdqu	YMMWORD PTR [r9+32], ymm4
+        vmovdqu	YMMWORD PTR [r9+-96], ymm1
+        vmovdqu	YMMWORD PTR [r9+32], ymm2
+        vmovdqu	YMMWORD PTR [r10+-96], ymm3
+        vmovdqu	YMMWORD PTR [r10+32], ymm4
         ; Row 1
-        vpxor	ymm10, ymm8, [r9+-64]
-        vpxor	ymm11, ymm9, [r9+64]
+        vpxor	ymm10, ymm8, [r10+-64]
+        vpxor	ymm11, ymm9, [r10+64]
         vpxor	ymm12, ymm5, [rcx+-96]
         vpxor	ymm13, ymm6, [rcx+32]
-        vpxor	ymm14, ymm7, [r8+64]
+        vpxor	ymm14, ymm7, [r9+64]
         vpsrlq	ymm0, ymm10, 36
         vpsrlq	ymm1, ymm11, 44
         vpsrlq	ymm2, ymm12, 61
@@ -21021,16 +21038,16 @@ sha3_256_blocksx4_seed_avx2 PROC
         vpxor	ymm2, ymm2, ymm12
         vpxor	ymm3, ymm3, ymm13
         vpxor	ymm4, ymm4, ymm14
-        vmovdqu	YMMWORD PTR [r9+-64], ymm0
-        vmovdqu	YMMWORD PTR [r9+64], ymm1
+        vmovdqu	YMMWORD PTR [r10+-64], ymm0
+        vmovdqu	YMMWORD PTR [r10+64], ymm1
         vmovdqu	YMMWORD PTR [rcx+-96], ymm2
         vmovdqu	YMMWORD PTR [rcx+32], ymm3
-        vmovdqu	YMMWORD PTR [r8+64], ymm4
+        vmovdqu	YMMWORD PTR [r9+64], ymm4
         ; Row 2
         vpxor	ymm10, ymm6, [rcx+64]
-        vpxor	ymm11, ymm7, [r8+-64]
-        vpxor	ymm12, ymm8, [r9+-32]
-        vpxor	ymm13, ymm9, [r9+96]
+        vpxor	ymm11, ymm7, [r9+-64]
+        vpxor	ymm12, ymm8, [r10+-32]
+        vpxor	ymm13, ymm9, [r10+96]
         vpxor	ymm14, ymm5, [rcx+-64]
         vpsrlq	ymm0, ymm10, 63
         vpsrlq	ymm1, ymm11, 58
@@ -21058,16 +21075,16 @@ sha3_256_blocksx4_seed_avx2 PROC
         vpxor	ymm3, ymm3, ymm13
         vpxor	ymm4, ymm4, ymm14
         vmovdqu	YMMWORD PTR [rcx+64], ymm0
-        vmovdqu	YMMWORD PTR [r8+-64], ymm1
-        vmovdqu	YMMWORD PTR [r9+-32], ymm2
-        vmovdqu	YMMWORD PTR [r9+96], ymm3
+        vmovdqu	YMMWORD PTR [r9+-64], ymm1
+        vmovdqu	YMMWORD PTR [r10+-32], ymm2
+        vmovdqu	YMMWORD PTR [r10+96], ymm3
         vmovdqu	YMMWORD PTR [rcx+-64], ymm4
         ; Row 3
-        vpxor	ymm10, ymm9, [r9+128]
+        vpxor	ymm10, ymm9, [r10+128]
         vpxor	ymm11, ymm5, [rcx+-32]
         vpxor	ymm12, ymm6, [rcx+96]
-        vpxor	ymm13, ymm7, [r8+-32]
-        vpxor	ymm14, ymm8, [r8+96]
+        vpxor	ymm13, ymm7, [r9+-32]
+        vpxor	ymm14, ymm8, [r9+96]
         vpsrlq	ymm0, ymm10, 37
         vpsrlq	ymm1, ymm11, 28
         vpsrlq	ymm2, ymm12, 54
@@ -21093,15 +21110,15 @@ sha3_256_blocksx4_seed_avx2 PROC
         vpxor	ymm2, ymm2, ymm12
         vpxor	ymm3, ymm3, ymm13
         vpxor	ymm4, ymm4, ymm14
-        vmovdqu	YMMWORD PTR [r9+128], ymm0
+        vmovdqu	YMMWORD PTR [r10+128], ymm0
         vmovdqu	YMMWORD PTR [rcx+-32], ymm1
         vmovdqu	YMMWORD PTR [rcx+96], ymm2
-        vmovdqu	YMMWORD PTR [r8+-32], ymm3
-        vmovdqu	YMMWORD PTR [r8+96], ymm4
+        vmovdqu	YMMWORD PTR [r9+-32], ymm3
+        vmovdqu	YMMWORD PTR [r9+96], ymm4
         ; Row 4
-        vpxor	ymm10, ymm7, [r8]
-        vpxor	ymm11, ymm8, [r8+128]
-        vpxor	ymm12, ymm9, [r9]
+        vpxor	ymm10, ymm7, [r9]
+        vpxor	ymm11, ymm8, [r9+128]
+        vpxor	ymm12, ymm9, [r10]
         vpxor	ymm13, ymm5, [rcx]
         vpxor	ymm14, ymm6, [rcx+128]
         vpsrlq	ymm0, ymm10, 2
@@ -21129,9 +21146,9 @@ sha3_256_blocksx4_seed_avx2 PROC
         vpxor	ymm2, ymm2, ymm12
         vpxor	ymm3, ymm3, ymm13
         vpxor	ymm4, ymm4, ymm14
-        vmovdqu	YMMWORD PTR [r8], ymm0
-        vmovdqu	YMMWORD PTR [r8+128], ymm1
-        vmovdqu	YMMWORD PTR [r9], ymm2
+        vmovdqu	YMMWORD PTR [r9], ymm0
+        vmovdqu	YMMWORD PTR [r9+128], ymm1
+        vmovdqu	YMMWORD PTR [r10], ymm2
         vmovdqu	YMMWORD PTR [rcx], ymm3
         vmovdqu	YMMWORD PTR [rcx+128], ymm4
         ; Round 2
@@ -21143,19 +21160,19 @@ sha3_256_blocksx4_seed_avx2 PROC
         vpxor	ymm13, ymm3, [rcx+32]
         vpxor	ymm10, ymm10, [rcx+64]
         vpxor	ymm12, ymm12, [rcx+96]
-        vpxor	ymm11, ymm11, [r8+-96]
-        vpxor	ymm11, ymm11, [r8+-64]
-        vpxor	ymm13, ymm13, [r8+-32]
-        vpxor	ymm12, ymm12, [r8+32]
-        vpxor	ymm14, ymm14, [r8+64]
-        vpxor	ymm14, ymm14, [r8+96]
-        vpxor	ymm13, ymm13, [r9+-96]
-        vpxor	ymm10, ymm10, [r9+-64]
-        vpxor	ymm12, ymm12, [r9+-32]
-        vpxor	ymm14, ymm14, [r9+32]
-        vpxor	ymm11, ymm11, [r9+64]
-        vpxor	ymm13, ymm13, [r9+96]
-        vpxor	ymm10, ymm10, [r9+128]
+        vpxor	ymm11, ymm11, [r9+-96]
+        vpxor	ymm11, ymm11, [r9+-64]
+        vpxor	ymm13, ymm13, [r9+-32]
+        vpxor	ymm12, ymm12, [r9+32]
+        vpxor	ymm14, ymm14, [r9+64]
+        vpxor	ymm14, ymm14, [r9+96]
+        vpxor	ymm13, ymm13, [r10+-96]
+        vpxor	ymm10, ymm10, [r10+-64]
+        vpxor	ymm12, ymm12, [r10+-32]
+        vpxor	ymm14, ymm14, [r10+32]
+        vpxor	ymm11, ymm11, [r10+64]
+        vpxor	ymm13, ymm13, [r10+96]
+        vpxor	ymm10, ymm10, [r10+128]
         ; Calc t[0..4]
         vpsrlq	ymm0, ymm11, 63
         vpsrlq	ymm1, ymm12, 63
@@ -21180,9 +21197,9 @@ sha3_256_blocksx4_seed_avx2 PROC
         ; Row Mix
         ; Row 0
         vpxor	ymm10, ymm5, ymm15
-        vpxor	ymm11, ymm6, [r9+64]
-        vpxor	ymm12, ymm7, [r9+-32]
-        vpxor	ymm13, ymm8, [r8+-32]
+        vpxor	ymm11, ymm6, [r10+64]
+        vpxor	ymm12, ymm7, [r10+-32]
+        vpxor	ymm13, ymm8, [r9+-32]
         vpxor	ymm14, ymm9, [rcx+128]
         vpsrlq	ymm0, ymm11, 20
         vpsrlq	ymm1, ymm12, 21
@@ -21208,16 +21225,16 @@ sha3_256_blocksx4_seed_avx2 PROC
         vpxor	ymm15, ymm15, [rax+64]
         vpxor	ymm3, ymm3, ymm13
         vpxor	ymm4, ymm4, ymm14
-        vmovdqu	YMMWORD PTR [r9+64], ymm1
-        vmovdqu	YMMWORD PTR [r9+-32], ymm2
-        vmovdqu	YMMWORD PTR [r8+-32], ymm3
+        vmovdqu	YMMWORD PTR [r10+64], ymm1
+        vmovdqu	YMMWORD PTR [r10+-32], ymm2
+        vmovdqu	YMMWORD PTR [r9+-32], ymm3
         vmovdqu	YMMWORD PTR [rcx+128], ymm4
         ; Row 1
-        vpxor	ymm10, ymm8, [r9+-96]
-        vpxor	ymm11, ymm9, [r8+64]
+        vpxor	ymm10, ymm8, [r10+-96]
+        vpxor	ymm11, ymm9, [r9+64]
         vpxor	ymm12, ymm5, [rcx+64]
         vpxor	ymm13, ymm6, [rcx+-32]
-        vpxor	ymm14, ymm7, [r9]
+        vpxor	ymm14, ymm7, [r10]
         vpsrlq	ymm0, ymm10, 36
         vpsrlq	ymm1, ymm11, 44
         vpsrlq	ymm2, ymm12, 61
@@ -21243,17 +21260,17 @@ sha3_256_blocksx4_seed_avx2 PROC
         vpxor	ymm2, ymm2, ymm12
         vpxor	ymm3, ymm3, ymm13
         vpxor	ymm4, ymm4, ymm14
-        vmovdqu	YMMWORD PTR [r9+-96], ymm0
-        vmovdqu	YMMWORD PTR [r8+64], ymm1
+        vmovdqu	YMMWORD PTR [r10+-96], ymm0
+        vmovdqu	YMMWORD PTR [r9+64], ymm1
         vmovdqu	YMMWORD PTR [rcx+64], ymm2
         vmovdqu	YMMWORD PTR [rcx+-32], ymm3
-        vmovdqu	YMMWORD PTR [r9], ymm4
+        vmovdqu	YMMWORD PTR [r10], ymm4
         ; Row 2
-        vpxor	ymm10, ymm6, [r8+-96]
+        vpxor	ymm10, ymm6, [r9+-96]
         vpxor	ymm11, ymm7, [rcx+-96]
-        vpxor	ymm12, ymm8, [r9+96]
-        vpxor	ymm13, ymm9, [r8+96]
-        vpxor	ymm14, ymm5, [r8]
+        vpxor	ymm12, ymm8, [r10+96]
+        vpxor	ymm13, ymm9, [r9+96]
+        vpxor	ymm14, ymm5, [r9]
         vpsrlq	ymm0, ymm10, 63
         vpsrlq	ymm1, ymm11, 58
         vpsrlq	ymm2, ymm12, 39
@@ -21279,15 +21296,15 @@ sha3_256_blocksx4_seed_avx2 PROC
         vpxor	ymm2, ymm2, ymm12
         vpxor	ymm3, ymm3, ymm13
         vpxor	ymm4, ymm4, ymm14
-        vmovdqu	YMMWORD PTR [r8+-96], ymm0
+        vmovdqu	YMMWORD PTR [r9+-96], ymm0
         vmovdqu	YMMWORD PTR [rcx+-96], ymm1
-        vmovdqu	YMMWORD PTR [r9+96], ymm2
-        vmovdqu	YMMWORD PTR [r8+96], ymm3
-        vmovdqu	YMMWORD PTR [r8], ymm4
+        vmovdqu	YMMWORD PTR [r10+96], ymm2
+        vmovdqu	YMMWORD PTR [r9+96], ymm3
+        vmovdqu	YMMWORD PTR [r9], ymm4
         ; Row 3
-        vpxor	ymm10, ymm9, [r9+32]
-        vpxor	ymm11, ymm5, [r9+-64]
-        vpxor	ymm12, ymm6, [r8+-64]
+        vpxor	ymm10, ymm9, [r10+32]
+        vpxor	ymm11, ymm5, [r10+-64]
+        vpxor	ymm12, ymm6, [r9+-64]
         vpxor	ymm13, ymm7, [rcx+96]
         vpxor	ymm14, ymm8, [rcx]
         vpsrlq	ymm0, ymm10, 37
@@ -21315,17 +21332,17 @@ sha3_256_blocksx4_seed_avx2 PROC
         vpxor	ymm2, ymm2, ymm12
         vpxor	ymm3, ymm3, ymm13
         vpxor	ymm4, ymm4, ymm14
-        vmovdqu	YMMWORD PTR [r9+32], ymm0
-        vmovdqu	YMMWORD PTR [r9+-64], ymm1
-        vmovdqu	YMMWORD PTR [r8+-64], ymm2
+        vmovdqu	YMMWORD PTR [r10+32], ymm0
+        vmovdqu	YMMWORD PTR [r10+-64], ymm1
+        vmovdqu	YMMWORD PTR [r9+-64], ymm2
         vmovdqu	YMMWORD PTR [rcx+96], ymm3
         vmovdqu	YMMWORD PTR [rcx], ymm4
         ; Row 4
-        vpxor	ymm10, ymm7, [r8+32]
+        vpxor	ymm10, ymm7, [r9+32]
         vpxor	ymm11, ymm8, [rcx+32]
         vpxor	ymm12, ymm9, [rcx+-64]
-        vpxor	ymm13, ymm5, [r9+128]
-        vpxor	ymm14, ymm6, [r8+128]
+        vpxor	ymm13, ymm5, [r10+128]
+        vpxor	ymm14, ymm6, [r9+128]
         vpsrlq	ymm0, ymm10, 2
         vpsrlq	ymm1, ymm11, 9
         vpsrlq	ymm2, ymm12, 25
@@ -21351,11 +21368,11 @@ sha3_256_blocksx4_seed_avx2 PROC
         vpxor	ymm2, ymm2, ymm12
         vpxor	ymm3, ymm3, ymm13
         vpxor	ymm4, ymm4, ymm14
-        vmovdqu	YMMWORD PTR [r8+32], ymm0
+        vmovdqu	YMMWORD PTR [r9+32], ymm0
         vmovdqu	YMMWORD PTR [rcx+32], ymm1
         vmovdqu	YMMWORD PTR [rcx+-64], ymm2
-        vmovdqu	YMMWORD PTR [r9+128], ymm3
-        vmovdqu	YMMWORD PTR [r8+128], ymm4
+        vmovdqu	YMMWORD PTR [r10+128], ymm3
+        vmovdqu	YMMWORD PTR [r9+128], ymm4
         ; Round 3
         ; Calc b[0..4]
         vpxor	ymm10, ymm0, ymm15
@@ -21365,7 +21382,5419 @@ sha3_256_blocksx4_seed_avx2 PROC
         vpxor	ymm12, ymm2, [rcx+64]
         vpxor	ymm13, ymm13, [rcx+96]
         vpxor	ymm14, ymm14, [rcx+128]
-        vpxor	ymm10, ymm10, [r8+-96]
+        vpxor	ymm10, ymm10, [r9+-96]
+        vpxor	ymm12, ymm12, [r9+-64]
+        vpxor	ymm13, ymm13, [r9+-32]
+        vpxor	ymm14, ymm14, [r9]
+        vpxor	ymm11, ymm11, [r9+64]
+        vpxor	ymm13, ymm13, [r9+96]
+        vpxor	ymm10, ymm10, [r10+-96]
+        vpxor	ymm11, ymm11, [r10+-64]
+        vpxor	ymm12, ymm12, [r10+-32]
+        vpxor	ymm14, ymm14, [r10]
+        vpxor	ymm10, ymm10, [r10+32]
+        vpxor	ymm11, ymm11, [r10+64]
+        vpxor	ymm12, ymm12, [r10+96]
+        ; Calc t[0..4]
+        vpsrlq	ymm0, ymm11, 63
+        vpsrlq	ymm1, ymm12, 63
+        vpsrlq	ymm2, ymm13, 63
+        vpsrlq	ymm3, ymm14, 63
+        vpsrlq	ymm4, ymm10, 63
+        vpaddq	ymm5, ymm11, ymm11
+        vpaddq	ymm6, ymm12, ymm12
+        vpaddq	ymm7, ymm13, ymm13
+        vpaddq	ymm8, ymm14, ymm14
+        vpaddq	ymm9, ymm10, ymm10
+        vpor	ymm5, ymm5, ymm0
+        vpor	ymm6, ymm6, ymm1
+        vpor	ymm7, ymm7, ymm2
+        vpor	ymm8, ymm8, ymm3
+        vpor	ymm9, ymm9, ymm4
+        vpxor	ymm5, ymm5, ymm14
+        vpxor	ymm6, ymm6, ymm10
+        vpxor	ymm7, ymm7, ymm11
+        vpxor	ymm8, ymm8, ymm12
+        vpxor	ymm9, ymm9, ymm13
+        ; Row Mix
+        ; Row 0
+        vpxor	ymm10, ymm5, ymm15
+        vpxor	ymm11, ymm6, [r9+64]
+        vpxor	ymm12, ymm7, [r10+96]
+        vpxor	ymm13, ymm8, [rcx+96]
+        vpxor	ymm14, ymm9, [r9+128]
+        vpsrlq	ymm0, ymm11, 20
+        vpsrlq	ymm1, ymm12, 21
+        vpsrlq	ymm2, ymm13, 43
+        vpsrlq	ymm3, ymm14, 50
+        vpsllq	ymm11, ymm11, 44
+        vpsllq	ymm12, ymm12, 43
+        vpsllq	ymm13, ymm13, 21
+        vpsllq	ymm14, ymm14, 14
+        vpor	ymm11, ymm11, ymm0
+        vpor	ymm12, ymm12, ymm1
+        vpor	ymm13, ymm13, ymm2
+        vpor	ymm14, ymm14, ymm3
+        vpandn	ymm15, ymm11, ymm12
+        vpandn	ymm1, ymm12, ymm13
+        vpandn	ymm2, ymm13, ymm14
+        vpandn	ymm3, ymm14, ymm10
+        vpandn	ymm4, ymm10, ymm11
+        vpxor	ymm15, ymm15, ymm10
+        vpxor	ymm1, ymm1, ymm11
+        vpxor	ymm2, ymm2, ymm12
+        ; XOR in constant
+        vpxor	ymm15, ymm15, [rax+96]
+        vpxor	ymm3, ymm3, ymm13
+        vpxor	ymm4, ymm4, ymm14
+        vmovdqu	YMMWORD PTR [r9+64], ymm1
+        vmovdqu	YMMWORD PTR [r10+96], ymm2
+        vmovdqu	YMMWORD PTR [rcx+96], ymm3
+        vmovdqu	YMMWORD PTR [r9+128], ymm4
+        ; Row 1
+        vpxor	ymm10, ymm8, [r9+-32]
+        vpxor	ymm11, ymm9, [r10]
+        vpxor	ymm12, ymm5, [r9+-96]
+        vpxor	ymm13, ymm6, [r10+-64]
+        vpxor	ymm14, ymm7, [rcx+-64]
+        vpsrlq	ymm0, ymm10, 36
+        vpsrlq	ymm1, ymm11, 44
+        vpsrlq	ymm2, ymm12, 61
+        vpsrlq	ymm3, ymm13, 19
+        vpsrlq	ymm4, ymm14, 3
+        vpsllq	ymm10, ymm10, 28
+        vpsllq	ymm11, ymm11, 20
+        vpsllq	ymm12, ymm12, 3
+        vpsllq	ymm13, ymm13, 45
+        vpsllq	ymm14, ymm14, 61
+        vpor	ymm10, ymm10, ymm0
+        vpor	ymm11, ymm11, ymm1
+        vpor	ymm12, ymm12, ymm2
+        vpor	ymm13, ymm13, ymm3
+        vpor	ymm14, ymm14, ymm4
+        vpandn	ymm0, ymm11, ymm12
+        vpandn	ymm1, ymm12, ymm13
+        vpandn	ymm2, ymm13, ymm14
+        vpandn	ymm3, ymm14, ymm10
+        vpandn	ymm4, ymm10, ymm11
+        vpxor	ymm0, ymm0, ymm10
+        vpxor	ymm1, ymm1, ymm11
+        vpxor	ymm2, ymm2, ymm12
+        vpxor	ymm3, ymm3, ymm13
+        vpxor	ymm4, ymm4, ymm14
+        vmovdqu	YMMWORD PTR [r9+-32], ymm0
+        vmovdqu	YMMWORD PTR [r10], ymm1
+        vmovdqu	YMMWORD PTR [r9+-96], ymm2
+        vmovdqu	YMMWORD PTR [r10+-64], ymm3
+        vmovdqu	YMMWORD PTR [rcx+-64], ymm4
+        ; Row 2
+        vpxor	ymm10, ymm6, [r10+64]
+        vpxor	ymm11, ymm7, [rcx+64]
+        vpxor	ymm12, ymm8, [r9+96]
+        vpxor	ymm13, ymm9, [rcx]
+        vpxor	ymm14, ymm5, [r9+32]
+        vpsrlq	ymm0, ymm10, 63
+        vpsrlq	ymm1, ymm11, 58
+        vpsrlq	ymm2, ymm12, 39
+        vpsrlq	ymm3, ymm13, 56
+        vpsrlq	ymm4, ymm14, 46
+        vpaddq	ymm10, ymm10, ymm10
+        vpsllq	ymm11, ymm11, 6
+        vpsllq	ymm12, ymm12, 25
+        vpsllq	ymm13, ymm13, 8
+        vpsllq	ymm14, ymm14, 18
+        vpor	ymm10, ymm10, ymm0
+        vpor	ymm11, ymm11, ymm1
+        vpor	ymm12, ymm12, ymm2
+        vpor	ymm13, ymm13, ymm3
+        vpor	ymm14, ymm14, ymm4
+        vpandn	ymm0, ymm11, ymm12
+        vpandn	ymm1, ymm12, ymm13
+        vpandn	ymm2, ymm13, ymm14
+        vpandn	ymm3, ymm14, ymm10
+        vpandn	ymm4, ymm10, ymm11
+        vpxor	ymm0, ymm0, ymm10
+        vpxor	ymm1, ymm1, ymm11
+        vpxor	ymm2, ymm2, ymm12
+        vpxor	ymm3, ymm3, ymm13
+        vpxor	ymm4, ymm4, ymm14
+        vmovdqu	YMMWORD PTR [r10+64], ymm0
+        vmovdqu	YMMWORD PTR [rcx+64], ymm1
+        vmovdqu	YMMWORD PTR [r9+96], ymm2
+        vmovdqu	YMMWORD PTR [rcx], ymm3
+        vmovdqu	YMMWORD PTR [r9+32], ymm4
+        ; Row 3
+        vpxor	ymm10, ymm9, [rcx+128]
+        vpxor	ymm11, ymm5, [r10+-96]
+        vpxor	ymm12, ymm6, [rcx+-96]
+        vpxor	ymm13, ymm7, [r9+-64]
+        vpxor	ymm14, ymm8, [r10+128]
+        vpsrlq	ymm0, ymm10, 37
+        vpsrlq	ymm1, ymm11, 28
+        vpsrlq	ymm2, ymm12, 54
+        vpsrlq	ymm3, ymm13, 49
+        vpsrlq	ymm4, ymm14, 8
+        vpsllq	ymm10, ymm10, 27
+        vpsllq	ymm11, ymm11, 36
+        vpsllq	ymm12, ymm12, 10
+        vpsllq	ymm13, ymm13, 15
+        vpsllq	ymm14, ymm14, 56
+        vpor	ymm10, ymm10, ymm0
+        vpor	ymm11, ymm11, ymm1
+        vpor	ymm12, ymm12, ymm2
+        vpor	ymm13, ymm13, ymm3
+        vpor	ymm14, ymm14, ymm4
+        vpandn	ymm0, ymm11, ymm12
+        vpandn	ymm1, ymm12, ymm13
+        vpandn	ymm2, ymm13, ymm14
+        vpandn	ymm3, ymm14, ymm10
+        vpandn	ymm4, ymm10, ymm11
+        vpxor	ymm0, ymm0, ymm10
+        vpxor	ymm1, ymm1, ymm11
+        vpxor	ymm2, ymm2, ymm12
+        vpxor	ymm3, ymm3, ymm13
+        vpxor	ymm4, ymm4, ymm14
+        vmovdqu	YMMWORD PTR [rcx+128], ymm0
+        vmovdqu	YMMWORD PTR [r10+-96], ymm1
+        vmovdqu	YMMWORD PTR [rcx+-96], ymm2
+        vmovdqu	YMMWORD PTR [r9+-64], ymm3
+        vmovdqu	YMMWORD PTR [r10+128], ymm4
+        ; Row 4
+        vpxor	ymm10, ymm7, [r10+-32]
+        vpxor	ymm11, ymm8, [rcx+-32]
+        vpxor	ymm12, ymm9, [r9]
+        vpxor	ymm13, ymm5, [r10+32]
+        vpxor	ymm14, ymm6, [rcx+32]
+        vpsrlq	ymm0, ymm10, 2
+        vpsrlq	ymm1, ymm11, 9
+        vpsrlq	ymm2, ymm12, 25
+        vpsrlq	ymm3, ymm13, 23
+        vpsrlq	ymm4, ymm14, 62
+        vpsllq	ymm10, ymm10, 62
+        vpsllq	ymm11, ymm11, 55
+        vpsllq	ymm12, ymm12, 39
+        vpsllq	ymm13, ymm13, 41
+        vpsllq	ymm14, ymm14, 2
+        vpor	ymm10, ymm10, ymm0
+        vpor	ymm11, ymm11, ymm1
+        vpor	ymm12, ymm12, ymm2
+        vpor	ymm13, ymm13, ymm3
+        vpor	ymm14, ymm14, ymm4
+        vpandn	ymm0, ymm11, ymm12
+        vpandn	ymm1, ymm12, ymm13
+        vpandn	ymm2, ymm13, ymm14
+        vpandn	ymm3, ymm14, ymm10
+        vpandn	ymm4, ymm10, ymm11
+        vpxor	ymm0, ymm0, ymm10
+        vpxor	ymm1, ymm1, ymm11
+        vpxor	ymm2, ymm2, ymm12
+        vpxor	ymm3, ymm3, ymm13
+        vpxor	ymm4, ymm4, ymm14
+        vmovdqu	YMMWORD PTR [r10+-32], ymm0
+        vmovdqu	YMMWORD PTR [rcx+-32], ymm1
+        vmovdqu	YMMWORD PTR [r9], ymm2
+        vmovdqu	YMMWORD PTR [r10+32], ymm3
+        vmovdqu	YMMWORD PTR [rcx+32], ymm4
+        ; Round 4
+        ; Calc b[0..4]
+        vpxor	ymm10, ymm0, ymm15
+        vpxor	ymm12, ymm2, [rcx+-96]
+        vpxor	ymm14, ymm4, [rcx+-64]
+        vpxor	ymm13, ymm3, [rcx]
+        vpxor	ymm11, ymm1, [rcx+64]
+        vpxor	ymm13, ymm13, [rcx+96]
+        vpxor	ymm10, ymm10, [rcx+128]
+        vpxor	ymm12, ymm12, [r9+-96]
+        vpxor	ymm13, ymm13, [r9+-64]
+        vpxor	ymm10, ymm10, [r9+-32]
+        vpxor	ymm14, ymm14, [r9+32]
+        vpxor	ymm11, ymm11, [r9+64]
+        vpxor	ymm12, ymm12, [r9+96]
+        vpxor	ymm14, ymm14, [r9+128]
+        vpxor	ymm11, ymm11, [r10+-96]
+        vpxor	ymm13, ymm13, [r10+-64]
+        vpxor	ymm11, ymm11, [r10]
+        vpxor	ymm10, ymm10, [r10+64]
+        vpxor	ymm12, ymm12, [r10+96]
+        vpxor	ymm14, ymm14, [r10+128]
+        ; Calc t[0..4]
+        vpsrlq	ymm0, ymm11, 63
+        vpsrlq	ymm1, ymm12, 63
+        vpsrlq	ymm2, ymm13, 63
+        vpsrlq	ymm3, ymm14, 63
+        vpsrlq	ymm4, ymm10, 63
+        vpaddq	ymm5, ymm11, ymm11
+        vpaddq	ymm6, ymm12, ymm12
+        vpaddq	ymm7, ymm13, ymm13
+        vpaddq	ymm8, ymm14, ymm14
+        vpaddq	ymm9, ymm10, ymm10
+        vpor	ymm5, ymm5, ymm0
+        vpor	ymm6, ymm6, ymm1
+        vpor	ymm7, ymm7, ymm2
+        vpor	ymm8, ymm8, ymm3
+        vpor	ymm9, ymm9, ymm4
+        vpxor	ymm5, ymm5, ymm14
+        vpxor	ymm6, ymm6, ymm10
+        vpxor	ymm7, ymm7, ymm11
+        vpxor	ymm8, ymm8, ymm12
+        vpxor	ymm9, ymm9, ymm13
+        ; Row Mix
+        ; Row 0
+        vpxor	ymm10, ymm5, ymm15
+        vpxor	ymm11, ymm6, [r10]
+        vpxor	ymm12, ymm7, [r9+96]
+        vpxor	ymm13, ymm8, [r9+-64]
+        vpxor	ymm14, ymm9, [rcx+32]
+        vpsrlq	ymm0, ymm11, 20
+        vpsrlq	ymm1, ymm12, 21
+        vpsrlq	ymm2, ymm13, 43
+        vpsrlq	ymm3, ymm14, 50
+        vpsllq	ymm11, ymm11, 44
+        vpsllq	ymm12, ymm12, 43
+        vpsllq	ymm13, ymm13, 21
+        vpsllq	ymm14, ymm14, 14
+        vpor	ymm11, ymm11, ymm0
+        vpor	ymm12, ymm12, ymm1
+        vpor	ymm13, ymm13, ymm2
+        vpor	ymm14, ymm14, ymm3
+        vpandn	ymm15, ymm11, ymm12
+        vpandn	ymm1, ymm12, ymm13
+        vpandn	ymm2, ymm13, ymm14
+        vpandn	ymm3, ymm14, ymm10
+        vpandn	ymm4, ymm10, ymm11
+        vpxor	ymm15, ymm15, ymm10
+        vpxor	ymm1, ymm1, ymm11
+        vpxor	ymm2, ymm2, ymm12
+        ; XOR in constant
+        vpxor	ymm15, ymm15, [rax+128]
+        vpxor	ymm3, ymm3, ymm13
+        vpxor	ymm4, ymm4, ymm14
+        vmovdqu	YMMWORD PTR [r10], ymm1
+        vmovdqu	YMMWORD PTR [r9+96], ymm2
+        vmovdqu	YMMWORD PTR [r9+-64], ymm3
+        vmovdqu	YMMWORD PTR [rcx+32], ymm4
+        ; Row 1
+        vpxor	ymm10, ymm8, [rcx+96]
+        vpxor	ymm11, ymm9, [rcx+-64]
+        vpxor	ymm12, ymm5, [r10+64]
+        vpxor	ymm13, ymm6, [r10+-96]
+        vpxor	ymm14, ymm7, [r9]
+        vpsrlq	ymm0, ymm10, 36
+        vpsrlq	ymm1, ymm11, 44
+        vpsrlq	ymm2, ymm12, 61
+        vpsrlq	ymm3, ymm13, 19
+        vpsrlq	ymm4, ymm14, 3
+        vpsllq	ymm10, ymm10, 28
+        vpsllq	ymm11, ymm11, 20
+        vpsllq	ymm12, ymm12, 3
+        vpsllq	ymm13, ymm13, 45
+        vpsllq	ymm14, ymm14, 61
+        vpor	ymm10, ymm10, ymm0
+        vpor	ymm11, ymm11, ymm1
+        vpor	ymm12, ymm12, ymm2
+        vpor	ymm13, ymm13, ymm3
+        vpor	ymm14, ymm14, ymm4
+        vpandn	ymm0, ymm11, ymm12
+        vpandn	ymm1, ymm12, ymm13
+        vpandn	ymm2, ymm13, ymm14
+        vpandn	ymm3, ymm14, ymm10
+        vpandn	ymm4, ymm10, ymm11
+        vpxor	ymm0, ymm0, ymm10
+        vpxor	ymm1, ymm1, ymm11
+        vpxor	ymm2, ymm2, ymm12
+        vpxor	ymm3, ymm3, ymm13
+        vpxor	ymm4, ymm4, ymm14
+        vmovdqu	YMMWORD PTR [rcx+96], ymm0
+        vmovdqu	YMMWORD PTR [rcx+-64], ymm1
+        vmovdqu	YMMWORD PTR [r10+64], ymm2
+        vmovdqu	YMMWORD PTR [r10+-96], ymm3
+        vmovdqu	YMMWORD PTR [r9], ymm4
+        ; Row 2
+        vpxor	ymm10, ymm6, [r9+64]
+        vpxor	ymm11, ymm7, [r9+-96]
+        vpxor	ymm12, ymm8, [rcx]
+        vpxor	ymm13, ymm9, [r10+128]
+        vpxor	ymm14, ymm5, [r10+-32]
+        vpsrlq	ymm0, ymm10, 63
+        vpsrlq	ymm1, ymm11, 58
+        vpsrlq	ymm2, ymm12, 39
+        vpsrlq	ymm3, ymm13, 56
+        vpsrlq	ymm4, ymm14, 46
+        vpaddq	ymm10, ymm10, ymm10
+        vpsllq	ymm11, ymm11, 6
+        vpsllq	ymm12, ymm12, 25
+        vpsllq	ymm13, ymm13, 8
+        vpsllq	ymm14, ymm14, 18
+        vpor	ymm10, ymm10, ymm0
+        vpor	ymm11, ymm11, ymm1
+        vpor	ymm12, ymm12, ymm2
+        vpor	ymm13, ymm13, ymm3
+        vpor	ymm14, ymm14, ymm4
+        vpandn	ymm0, ymm11, ymm12
+        vpandn	ymm1, ymm12, ymm13
+        vpandn	ymm2, ymm13, ymm14
+        vpandn	ymm3, ymm14, ymm10
+        vpandn	ymm4, ymm10, ymm11
+        vpxor	ymm0, ymm0, ymm10
+        vpxor	ymm1, ymm1, ymm11
+        vpxor	ymm2, ymm2, ymm12
+        vpxor	ymm3, ymm3, ymm13
+        vpxor	ymm4, ymm4, ymm14
+        vmovdqu	YMMWORD PTR [r9+64], ymm0
+        vmovdqu	YMMWORD PTR [r9+-96], ymm1
+        vmovdqu	YMMWORD PTR [rcx], ymm2
+        vmovdqu	YMMWORD PTR [r10+128], ymm3
+        vmovdqu	YMMWORD PTR [r10+-32], ymm4
+        ; Row 3
+        vpxor	ymm10, ymm9, [r9+128]
+        vpxor	ymm11, ymm5, [r9+-32]
+        vpxor	ymm12, ymm6, [rcx+64]
+        vpxor	ymm13, ymm7, [rcx+-96]
+        vpxor	ymm14, ymm8, [r10+32]
+        vpsrlq	ymm0, ymm10, 37
+        vpsrlq	ymm1, ymm11, 28
+        vpsrlq	ymm2, ymm12, 54
+        vpsrlq	ymm3, ymm13, 49
+        vpsrlq	ymm4, ymm14, 8
+        vpsllq	ymm10, ymm10, 27
+        vpsllq	ymm11, ymm11, 36
+        vpsllq	ymm12, ymm12, 10
+        vpsllq	ymm13, ymm13, 15
+        vpsllq	ymm14, ymm14, 56
+        vpor	ymm10, ymm10, ymm0
+        vpor	ymm11, ymm11, ymm1
+        vpor	ymm12, ymm12, ymm2
+        vpor	ymm13, ymm13, ymm3
+        vpor	ymm14, ymm14, ymm4
+        vpandn	ymm0, ymm11, ymm12
+        vpandn	ymm1, ymm12, ymm13
+        vpandn	ymm2, ymm13, ymm14
+        vpandn	ymm3, ymm14, ymm10
+        vpandn	ymm4, ymm10, ymm11
+        vpxor	ymm0, ymm0, ymm10
+        vpxor	ymm1, ymm1, ymm11
+        vpxor	ymm2, ymm2, ymm12
+        vpxor	ymm3, ymm3, ymm13
+        vpxor	ymm4, ymm4, ymm14
+        vmovdqu	YMMWORD PTR [r9+128], ymm0
+        vmovdqu	YMMWORD PTR [r9+-32], ymm1
+        vmovdqu	YMMWORD PTR [rcx+64], ymm2
+        vmovdqu	YMMWORD PTR [rcx+-96], ymm3
+        vmovdqu	YMMWORD PTR [r10+32], ymm4
+        ; Row 4
+        vpxor	ymm10, ymm7, [r10+96]
+        vpxor	ymm11, ymm8, [r10+-64]
+        vpxor	ymm12, ymm9, [r9+32]
+        vpxor	ymm13, ymm5, [rcx+128]
+        vpxor	ymm14, ymm6, [rcx+-32]
+        vpsrlq	ymm0, ymm10, 2
+        vpsrlq	ymm1, ymm11, 9
+        vpsrlq	ymm2, ymm12, 25
+        vpsrlq	ymm3, ymm13, 23
+        vpsrlq	ymm4, ymm14, 62
+        vpsllq	ymm10, ymm10, 62
+        vpsllq	ymm11, ymm11, 55
+        vpsllq	ymm12, ymm12, 39
+        vpsllq	ymm13, ymm13, 41
+        vpsllq	ymm14, ymm14, 2
+        vpor	ymm10, ymm10, ymm0
+        vpor	ymm11, ymm11, ymm1
+        vpor	ymm12, ymm12, ymm2
+        vpor	ymm13, ymm13, ymm3
+        vpor	ymm14, ymm14, ymm4
+        vpandn	ymm0, ymm11, ymm12
+        vpandn	ymm1, ymm12, ymm13
+        vpandn	ymm2, ymm13, ymm14
+        vpandn	ymm3, ymm14, ymm10
+        vpandn	ymm4, ymm10, ymm11
+        vpxor	ymm0, ymm0, ymm10
+        vpxor	ymm1, ymm1, ymm11
+        vpxor	ymm2, ymm2, ymm12
+        vpxor	ymm3, ymm3, ymm13
+        vpxor	ymm4, ymm4, ymm14
+        vmovdqu	YMMWORD PTR [r10+96], ymm0
+        vmovdqu	YMMWORD PTR [r10+-64], ymm1
+        vmovdqu	YMMWORD PTR [r9+32], ymm2
+        vmovdqu	YMMWORD PTR [rcx+128], ymm3
+        vmovdqu	YMMWORD PTR [rcx+-32], ymm4
+        ; Round 5
+        ; Calc b[0..4]
+        vpxor	ymm10, ymm0, ymm15
+        vpxor	ymm13, ymm3, [rcx+-96]
+        vpxor	ymm11, ymm1, [rcx+-64]
+        vpxor	ymm12, ymm2, [rcx]
+        vpxor	ymm14, ymm4, [rcx+32]
+        vpxor	ymm12, ymm12, [rcx+64]
+        vpxor	ymm10, ymm10, [rcx+96]
+        vpxor	ymm11, ymm11, [r9+-96]
+        vpxor	ymm13, ymm13, [r9+-64]
+        vpxor	ymm11, ymm11, [r9+-32]
+        vpxor	ymm14, ymm14, [r9]
+        vpxor	ymm10, ymm10, [r9+64]
+        vpxor	ymm12, ymm12, [r9+96]
+        vpxor	ymm10, ymm10, [r9+128]
+        vpxor	ymm13, ymm13, [r10+-96]
+        vpxor	ymm14, ymm14, [r10+-32]
+        vpxor	ymm11, ymm11, [r10]
+        vpxor	ymm14, ymm14, [r10+32]
+        vpxor	ymm12, ymm12, [r10+64]
+        vpxor	ymm13, ymm13, [r10+128]
+        ; Calc t[0..4]
+        vpsrlq	ymm0, ymm11, 63
+        vpsrlq	ymm1, ymm12, 63
+        vpsrlq	ymm2, ymm13, 63
+        vpsrlq	ymm3, ymm14, 63
+        vpsrlq	ymm4, ymm10, 63
+        vpaddq	ymm5, ymm11, ymm11
+        vpaddq	ymm6, ymm12, ymm12
+        vpaddq	ymm7, ymm13, ymm13
+        vpaddq	ymm8, ymm14, ymm14
+        vpaddq	ymm9, ymm10, ymm10
+        vpor	ymm5, ymm5, ymm0
+        vpor	ymm6, ymm6, ymm1
+        vpor	ymm7, ymm7, ymm2
+        vpor	ymm8, ymm8, ymm3
+        vpor	ymm9, ymm9, ymm4
+        vpxor	ymm5, ymm5, ymm14
+        vpxor	ymm6, ymm6, ymm10
+        vpxor	ymm7, ymm7, ymm11
+        vpxor	ymm8, ymm8, ymm12
+        vpxor	ymm9, ymm9, ymm13
+        ; Row Mix
+        ; Row 0
+        vpxor	ymm10, ymm5, ymm15
+        vpxor	ymm11, ymm6, [rcx+-64]
+        vpxor	ymm12, ymm7, [rcx]
+        vpxor	ymm13, ymm8, [rcx+-96]
+        vpxor	ymm14, ymm9, [rcx+-32]
+        vpsrlq	ymm0, ymm11, 20
+        vpsrlq	ymm1, ymm12, 21
+        vpsrlq	ymm2, ymm13, 43
+        vpsrlq	ymm3, ymm14, 50
+        vpsllq	ymm11, ymm11, 44
+        vpsllq	ymm12, ymm12, 43
+        vpsllq	ymm13, ymm13, 21
+        vpsllq	ymm14, ymm14, 14
+        vpor	ymm11, ymm11, ymm0
+        vpor	ymm12, ymm12, ymm1
+        vpor	ymm13, ymm13, ymm2
+        vpor	ymm14, ymm14, ymm3
+        vpandn	ymm15, ymm11, ymm12
+        vpandn	ymm1, ymm12, ymm13
+        vpandn	ymm2, ymm13, ymm14
+        vpandn	ymm3, ymm14, ymm10
+        vpandn	ymm4, ymm10, ymm11
+        vpxor	ymm15, ymm15, ymm10
+        vpxor	ymm1, ymm1, ymm11
+        vpxor	ymm2, ymm2, ymm12
+        ; XOR in constant
+        vpxor	ymm15, ymm15, [rax+160]
+        vpxor	ymm3, ymm3, ymm13
+        vpxor	ymm4, ymm4, ymm14
+        vmovdqu	YMMWORD PTR [rcx+-64], ymm1
+        vmovdqu	YMMWORD PTR [rcx], ymm2
+        vmovdqu	YMMWORD PTR [rcx+-96], ymm3
+        vmovdqu	YMMWORD PTR [rcx+-32], ymm4
+        ; Row 1
+        vpxor	ymm10, ymm8, [r9+-64]
+        vpxor	ymm11, ymm9, [r9]
+        vpxor	ymm12, ymm5, [r9+64]
+        vpxor	ymm13, ymm6, [r9+-32]
+        vpxor	ymm14, ymm7, [r9+32]
+        vpsrlq	ymm0, ymm10, 36
+        vpsrlq	ymm1, ymm11, 44
+        vpsrlq	ymm2, ymm12, 61
+        vpsrlq	ymm3, ymm13, 19
+        vpsrlq	ymm4, ymm14, 3
+        vpsllq	ymm10, ymm10, 28
+        vpsllq	ymm11, ymm11, 20
+        vpsllq	ymm12, ymm12, 3
+        vpsllq	ymm13, ymm13, 45
+        vpsllq	ymm14, ymm14, 61
+        vpor	ymm10, ymm10, ymm0
+        vpor	ymm11, ymm11, ymm1
+        vpor	ymm12, ymm12, ymm2
+        vpor	ymm13, ymm13, ymm3
+        vpor	ymm14, ymm14, ymm4
+        vpandn	ymm0, ymm11, ymm12
+        vpandn	ymm1, ymm12, ymm13
+        vpandn	ymm2, ymm13, ymm14
+        vpandn	ymm3, ymm14, ymm10
+        vpandn	ymm4, ymm10, ymm11
+        vpxor	ymm0, ymm0, ymm10
+        vpxor	ymm1, ymm1, ymm11
+        vpxor	ymm2, ymm2, ymm12
+        vpxor	ymm3, ymm3, ymm13
+        vpxor	ymm4, ymm4, ymm14
+        vmovdqu	YMMWORD PTR [r9+-64], ymm0
+        vmovdqu	YMMWORD PTR [r9], ymm1
+        vmovdqu	YMMWORD PTR [r9+64], ymm2
+        vmovdqu	YMMWORD PTR [r9+-32], ymm3
+        vmovdqu	YMMWORD PTR [r9+32], ymm4
+        ; Row 2
+        vpxor	ymm10, ymm6, [r10]
+        vpxor	ymm11, ymm7, [r10+64]
+        vpxor	ymm12, ymm8, [r10+128]
+        vpxor	ymm13, ymm9, [r10+32]
+        vpxor	ymm14, ymm5, [r10+96]
+        vpsrlq	ymm0, ymm10, 63
+        vpsrlq	ymm1, ymm11, 58
+        vpsrlq	ymm2, ymm12, 39
+        vpsrlq	ymm3, ymm13, 56
+        vpsrlq	ymm4, ymm14, 46
+        vpaddq	ymm10, ymm10, ymm10
+        vpsllq	ymm11, ymm11, 6
+        vpsllq	ymm12, ymm12, 25
+        vpsllq	ymm13, ymm13, 8
+        vpsllq	ymm14, ymm14, 18
+        vpor	ymm10, ymm10, ymm0
+        vpor	ymm11, ymm11, ymm1
+        vpor	ymm12, ymm12, ymm2
+        vpor	ymm13, ymm13, ymm3
+        vpor	ymm14, ymm14, ymm4
+        vpandn	ymm0, ymm11, ymm12
+        vpandn	ymm1, ymm12, ymm13
+        vpandn	ymm2, ymm13, ymm14
+        vpandn	ymm3, ymm14, ymm10
+        vpandn	ymm4, ymm10, ymm11
+        vpxor	ymm0, ymm0, ymm10
+        vpxor	ymm1, ymm1, ymm11
+        vpxor	ymm2, ymm2, ymm12
+        vpxor	ymm3, ymm3, ymm13
+        vpxor	ymm4, ymm4, ymm14
+        vmovdqu	YMMWORD PTR [r10], ymm0
+        vmovdqu	YMMWORD PTR [r10+64], ymm1
+        vmovdqu	YMMWORD PTR [r10+128], ymm2
+        vmovdqu	YMMWORD PTR [r10+32], ymm3
+        vmovdqu	YMMWORD PTR [r10+96], ymm4
+        ; Row 3
+        vpxor	ymm10, ymm9, [rcx+32]
+        vpxor	ymm11, ymm5, [rcx+96]
+        vpxor	ymm12, ymm6, [r9+-96]
+        vpxor	ymm13, ymm7, [rcx+64]
+        vpxor	ymm14, ymm8, [rcx+128]
+        vpsrlq	ymm0, ymm10, 37
+        vpsrlq	ymm1, ymm11, 28
+        vpsrlq	ymm2, ymm12, 54
+        vpsrlq	ymm3, ymm13, 49
+        vpsrlq	ymm4, ymm14, 8
+        vpsllq	ymm10, ymm10, 27
+        vpsllq	ymm11, ymm11, 36
+        vpsllq	ymm12, ymm12, 10
+        vpsllq	ymm13, ymm13, 15
+        vpsllq	ymm14, ymm14, 56
+        vpor	ymm10, ymm10, ymm0
+        vpor	ymm11, ymm11, ymm1
+        vpor	ymm12, ymm12, ymm2
+        vpor	ymm13, ymm13, ymm3
+        vpor	ymm14, ymm14, ymm4
+        vpandn	ymm0, ymm11, ymm12
+        vpandn	ymm1, ymm12, ymm13
+        vpandn	ymm2, ymm13, ymm14
+        vpandn	ymm3, ymm14, ymm10
+        vpandn	ymm4, ymm10, ymm11
+        vpxor	ymm0, ymm0, ymm10
+        vpxor	ymm1, ymm1, ymm11
+        vpxor	ymm2, ymm2, ymm12
+        vpxor	ymm3, ymm3, ymm13
+        vpxor	ymm4, ymm4, ymm14
+        vmovdqu	YMMWORD PTR [rcx+32], ymm0
+        vmovdqu	YMMWORD PTR [rcx+96], ymm1
+        vmovdqu	YMMWORD PTR [r9+-96], ymm2
+        vmovdqu	YMMWORD PTR [rcx+64], ymm3
+        vmovdqu	YMMWORD PTR [rcx+128], ymm4
+        ; Row 4
+        vpxor	ymm10, ymm7, [r9+96]
+        vpxor	ymm11, ymm8, [r10+-96]
+        vpxor	ymm12, ymm9, [r10+-32]
+        vpxor	ymm13, ymm5, [r9+128]
+        vpxor	ymm14, ymm6, [r10+-64]
+        vpsrlq	ymm0, ymm10, 2
+        vpsrlq	ymm1, ymm11, 9
+        vpsrlq	ymm2, ymm12, 25
+        vpsrlq	ymm3, ymm13, 23
+        vpsrlq	ymm4, ymm14, 62
+        vpsllq	ymm10, ymm10, 62
+        vpsllq	ymm11, ymm11, 55
+        vpsllq	ymm12, ymm12, 39
+        vpsllq	ymm13, ymm13, 41
+        vpsllq	ymm14, ymm14, 2
+        vpor	ymm10, ymm10, ymm0
+        vpor	ymm11, ymm11, ymm1
+        vpor	ymm12, ymm12, ymm2
+        vpor	ymm13, ymm13, ymm3
+        vpor	ymm14, ymm14, ymm4
+        vpandn	ymm0, ymm11, ymm12
+        vpandn	ymm1, ymm12, ymm13
+        vpandn	ymm2, ymm13, ymm14
+        vpandn	ymm3, ymm14, ymm10
+        vpandn	ymm4, ymm10, ymm11
+        vpxor	ymm0, ymm0, ymm10
+        vpxor	ymm1, ymm1, ymm11
+        vpxor	ymm2, ymm2, ymm12
+        vpxor	ymm3, ymm3, ymm13
+        vpxor	ymm4, ymm4, ymm14
+        vmovdqu	YMMWORD PTR [r9+96], ymm0
+        vmovdqu	YMMWORD PTR [r10+-96], ymm1
+        vmovdqu	YMMWORD PTR [r10+-32], ymm2
+        vmovdqu	YMMWORD PTR [r9+128], ymm3
+        vmovdqu	YMMWORD PTR [r10+-64], ymm4
+        ; Round 6
+        ; Calc b[0..4]
+        vpxor	ymm10, ymm0, ymm15
+        vpxor	ymm13, ymm3, [rcx+-96]
+        vpxor	ymm11, ymm1, [rcx+-64]
+        vpxor	ymm14, ymm4, [rcx+-32]
+        vpxor	ymm12, ymm2, [rcx]
+        vpxor	ymm10, ymm10, [rcx+32]
+        vpxor	ymm13, ymm13, [rcx+64]
+        vpxor	ymm11, ymm11, [rcx+96]
+        vpxor	ymm14, ymm14, [rcx+128]
+        vpxor	ymm12, ymm12, [r9+-96]
+        vpxor	ymm10, ymm10, [r9+-64]
+        vpxor	ymm13, ymm13, [r9+-32]
+        vpxor	ymm11, ymm11, [r9]
+        vpxor	ymm14, ymm14, [r9+32]
+        vpxor	ymm12, ymm12, [r9+64]
+        vpxor	ymm10, ymm10, [r10]
+        vpxor	ymm13, ymm13, [r10+32]
+        vpxor	ymm11, ymm11, [r10+64]
+        vpxor	ymm14, ymm14, [r10+96]
+        vpxor	ymm12, ymm12, [r10+128]
+        ; Calc t[0..4]
+        vpsrlq	ymm0, ymm11, 63
+        vpsrlq	ymm1, ymm12, 63
+        vpsrlq	ymm2, ymm13, 63
+        vpsrlq	ymm3, ymm14, 63
+        vpsrlq	ymm4, ymm10, 63
+        vpaddq	ymm5, ymm11, ymm11
+        vpaddq	ymm6, ymm12, ymm12
+        vpaddq	ymm7, ymm13, ymm13
+        vpaddq	ymm8, ymm14, ymm14
+        vpaddq	ymm9, ymm10, ymm10
+        vpor	ymm5, ymm5, ymm0
+        vpor	ymm6, ymm6, ymm1
+        vpor	ymm7, ymm7, ymm2
+        vpor	ymm8, ymm8, ymm3
+        vpor	ymm9, ymm9, ymm4
+        vpxor	ymm5, ymm5, ymm14
+        vpxor	ymm6, ymm6, ymm10
+        vpxor	ymm7, ymm7, ymm11
+        vpxor	ymm8, ymm8, ymm12
+        vpxor	ymm9, ymm9, ymm13
+        ; Row Mix
+        ; Row 0
+        vpxor	ymm10, ymm5, ymm15
+        vpxor	ymm11, ymm6, [r9]
+        vpxor	ymm12, ymm7, [r10+128]
+        vpxor	ymm13, ymm8, [rcx+64]
+        vpxor	ymm14, ymm9, [r10+-64]
+        vpsrlq	ymm0, ymm11, 20
+        vpsrlq	ymm1, ymm12, 21
+        vpsrlq	ymm2, ymm13, 43
+        vpsrlq	ymm3, ymm14, 50
+        vpsllq	ymm11, ymm11, 44
+        vpsllq	ymm12, ymm12, 43
+        vpsllq	ymm13, ymm13, 21
+        vpsllq	ymm14, ymm14, 14
+        vpor	ymm11, ymm11, ymm0
+        vpor	ymm12, ymm12, ymm1
+        vpor	ymm13, ymm13, ymm2
+        vpor	ymm14, ymm14, ymm3
+        vpandn	ymm15, ymm11, ymm12
+        vpandn	ymm1, ymm12, ymm13
+        vpandn	ymm2, ymm13, ymm14
+        vpandn	ymm3, ymm14, ymm10
+        vpandn	ymm4, ymm10, ymm11
+        vpxor	ymm15, ymm15, ymm10
+        vpxor	ymm1, ymm1, ymm11
+        vpxor	ymm2, ymm2, ymm12
+        ; XOR in constant
+        vpxor	ymm15, ymm15, [rax+192]
+        vpxor	ymm3, ymm3, ymm13
+        vpxor	ymm4, ymm4, ymm14
+        vmovdqu	YMMWORD PTR [r9], ymm1
+        vmovdqu	YMMWORD PTR [r10+128], ymm2
+        vmovdqu	YMMWORD PTR [rcx+64], ymm3
+        vmovdqu	YMMWORD PTR [r10+-64], ymm4
+        ; Row 1
+        vpxor	ymm10, ymm8, [rcx+-96]
+        vpxor	ymm11, ymm9, [r9+32]
+        vpxor	ymm12, ymm5, [r10]
+        vpxor	ymm13, ymm6, [rcx+96]
+        vpxor	ymm14, ymm7, [r10+-32]
+        vpsrlq	ymm0, ymm10, 36
+        vpsrlq	ymm1, ymm11, 44
+        vpsrlq	ymm2, ymm12, 61
+        vpsrlq	ymm3, ymm13, 19
+        vpsrlq	ymm4, ymm14, 3
+        vpsllq	ymm10, ymm10, 28
+        vpsllq	ymm11, ymm11, 20
+        vpsllq	ymm12, ymm12, 3
+        vpsllq	ymm13, ymm13, 45
+        vpsllq	ymm14, ymm14, 61
+        vpor	ymm10, ymm10, ymm0
+        vpor	ymm11, ymm11, ymm1
+        vpor	ymm12, ymm12, ymm2
+        vpor	ymm13, ymm13, ymm3
+        vpor	ymm14, ymm14, ymm4
+        vpandn	ymm0, ymm11, ymm12
+        vpandn	ymm1, ymm12, ymm13
+        vpandn	ymm2, ymm13, ymm14
+        vpandn	ymm3, ymm14, ymm10
+        vpandn	ymm4, ymm10, ymm11
+        vpxor	ymm0, ymm0, ymm10
+        vpxor	ymm1, ymm1, ymm11
+        vpxor	ymm2, ymm2, ymm12
+        vpxor	ymm3, ymm3, ymm13
+        vpxor	ymm4, ymm4, ymm14
+        vmovdqu	YMMWORD PTR [rcx+-96], ymm0
+        vmovdqu	YMMWORD PTR [r9+32], ymm1
+        vmovdqu	YMMWORD PTR [r10], ymm2
+        vmovdqu	YMMWORD PTR [rcx+96], ymm3
+        vmovdqu	YMMWORD PTR [r10+-32], ymm4
+        ; Row 2
+        vpxor	ymm10, ymm6, [rcx+-64]
+        vpxor	ymm11, ymm7, [r9+64]
+        vpxor	ymm12, ymm8, [r10+32]
+        vpxor	ymm13, ymm9, [rcx+128]
+        vpxor	ymm14, ymm5, [r9+96]
+        vpsrlq	ymm0, ymm10, 63
+        vpsrlq	ymm1, ymm11, 58
+        vpsrlq	ymm2, ymm12, 39
+        vpsrlq	ymm3, ymm13, 56
+        vpsrlq	ymm4, ymm14, 46
+        vpaddq	ymm10, ymm10, ymm10
+        vpsllq	ymm11, ymm11, 6
+        vpsllq	ymm12, ymm12, 25
+        vpsllq	ymm13, ymm13, 8
+        vpsllq	ymm14, ymm14, 18
+        vpor	ymm10, ymm10, ymm0
+        vpor	ymm11, ymm11, ymm1
+        vpor	ymm12, ymm12, ymm2
+        vpor	ymm13, ymm13, ymm3
+        vpor	ymm14, ymm14, ymm4
+        vpandn	ymm0, ymm11, ymm12
+        vpandn	ymm1, ymm12, ymm13
+        vpandn	ymm2, ymm13, ymm14
+        vpandn	ymm3, ymm14, ymm10
+        vpandn	ymm4, ymm10, ymm11
+        vpxor	ymm0, ymm0, ymm10
+        vpxor	ymm1, ymm1, ymm11
+        vpxor	ymm2, ymm2, ymm12
+        vpxor	ymm3, ymm3, ymm13
+        vpxor	ymm4, ymm4, ymm14
+        vmovdqu	YMMWORD PTR [rcx+-64], ymm0
+        vmovdqu	YMMWORD PTR [r9+64], ymm1
+        vmovdqu	YMMWORD PTR [r10+32], ymm2
+        vmovdqu	YMMWORD PTR [rcx+128], ymm3
+        vmovdqu	YMMWORD PTR [r9+96], ymm4
+        ; Row 3
+        vpxor	ymm10, ymm9, [rcx+-32]
+        vpxor	ymm11, ymm5, [r9+-64]
+        vpxor	ymm12, ymm6, [r10+64]
+        vpxor	ymm13, ymm7, [r9+-96]
+        vpxor	ymm14, ymm8, [r9+128]
+        vpsrlq	ymm0, ymm10, 37
+        vpsrlq	ymm1, ymm11, 28
+        vpsrlq	ymm2, ymm12, 54
+        vpsrlq	ymm3, ymm13, 49
+        vpsrlq	ymm4, ymm14, 8
+        vpsllq	ymm10, ymm10, 27
+        vpsllq	ymm11, ymm11, 36
+        vpsllq	ymm12, ymm12, 10
+        vpsllq	ymm13, ymm13, 15
+        vpsllq	ymm14, ymm14, 56
+        vpor	ymm10, ymm10, ymm0
+        vpor	ymm11, ymm11, ymm1
+        vpor	ymm12, ymm12, ymm2
+        vpor	ymm13, ymm13, ymm3
+        vpor	ymm14, ymm14, ymm4
+        vpandn	ymm0, ymm11, ymm12
+        vpandn	ymm1, ymm12, ymm13
+        vpandn	ymm2, ymm13, ymm14
+        vpandn	ymm3, ymm14, ymm10
+        vpandn	ymm4, ymm10, ymm11
+        vpxor	ymm0, ymm0, ymm10
+        vpxor	ymm1, ymm1, ymm11
+        vpxor	ymm2, ymm2, ymm12
+        vpxor	ymm3, ymm3, ymm13
+        vpxor	ymm4, ymm4, ymm14
+        vmovdqu	YMMWORD PTR [rcx+-32], ymm0
+        vmovdqu	YMMWORD PTR [r9+-64], ymm1
+        vmovdqu	YMMWORD PTR [r10+64], ymm2
+        vmovdqu	YMMWORD PTR [r9+-96], ymm3
+        vmovdqu	YMMWORD PTR [r9+128], ymm4
+        ; Row 4
+        vpxor	ymm10, ymm7, [rcx]
+        vpxor	ymm11, ymm8, [r9+-32]
+        vpxor	ymm12, ymm9, [r10+96]
+        vpxor	ymm13, ymm5, [rcx+32]
+        vpxor	ymm14, ymm6, [r10+-96]
+        vpsrlq	ymm0, ymm10, 2
+        vpsrlq	ymm1, ymm11, 9
+        vpsrlq	ymm2, ymm12, 25
+        vpsrlq	ymm3, ymm13, 23
+        vpsrlq	ymm4, ymm14, 62
+        vpsllq	ymm10, ymm10, 62
+        vpsllq	ymm11, ymm11, 55
+        vpsllq	ymm12, ymm12, 39
+        vpsllq	ymm13, ymm13, 41
+        vpsllq	ymm14, ymm14, 2
+        vpor	ymm10, ymm10, ymm0
+        vpor	ymm11, ymm11, ymm1
+        vpor	ymm12, ymm12, ymm2
+        vpor	ymm13, ymm13, ymm3
+        vpor	ymm14, ymm14, ymm4
+        vpandn	ymm0, ymm11, ymm12
+        vpandn	ymm1, ymm12, ymm13
+        vpandn	ymm2, ymm13, ymm14
+        vpandn	ymm3, ymm14, ymm10
+        vpandn	ymm4, ymm10, ymm11
+        vpxor	ymm0, ymm0, ymm10
+        vpxor	ymm1, ymm1, ymm11
+        vpxor	ymm2, ymm2, ymm12
+        vpxor	ymm3, ymm3, ymm13
+        vpxor	ymm4, ymm4, ymm14
+        vmovdqu	YMMWORD PTR [rcx], ymm0
+        vmovdqu	YMMWORD PTR [r9+-32], ymm1
+        vmovdqu	YMMWORD PTR [r10+96], ymm2
+        vmovdqu	YMMWORD PTR [rcx+32], ymm3
+        vmovdqu	YMMWORD PTR [r10+-96], ymm4
+        ; Round 7
+        ; Calc b[0..4]
+        vpxor	ymm10, ymm0, ymm15
+        vpxor	ymm10, ymm10, [rcx+-96]
+        vpxor	ymm10, ymm10, [rcx+-64]
+        vpxor	ymm10, ymm10, [rcx+-32]
+        vpxor	ymm13, ymm3, [rcx+64]
+        vpxor	ymm13, ymm13, [rcx+96]
+        vpxor	ymm13, ymm13, [rcx+128]
+        vpxor	ymm13, ymm13, [r9+-96]
+        vpxor	ymm11, ymm1, [r9+-64]
+        vpxor	ymm11, ymm11, [r9]
+        vpxor	ymm11, ymm11, [r9+32]
+        vpxor	ymm11, ymm11, [r9+64]
+        vpxor	ymm14, ymm4, [r9+96]
+        vpxor	ymm14, ymm14, [r9+128]
+        vpxor	ymm14, ymm14, [r10+-64]
+        vpxor	ymm14, ymm14, [r10+-32]
+        vpxor	ymm12, ymm2, [r10]
+        vpxor	ymm12, ymm12, [r10+32]
+        vpxor	ymm12, ymm12, [r10+64]
+        vpxor	ymm12, ymm12, [r10+128]
+        ; Calc t[0..4]
+        vpsrlq	ymm0, ymm11, 63
+        vpsrlq	ymm1, ymm12, 63
+        vpsrlq	ymm2, ymm13, 63
+        vpsrlq	ymm3, ymm14, 63
+        vpsrlq	ymm4, ymm10, 63
+        vpaddq	ymm5, ymm11, ymm11
+        vpaddq	ymm6, ymm12, ymm12
+        vpaddq	ymm7, ymm13, ymm13
+        vpaddq	ymm8, ymm14, ymm14
+        vpaddq	ymm9, ymm10, ymm10
+        vpor	ymm5, ymm5, ymm0
+        vpor	ymm6, ymm6, ymm1
+        vpor	ymm7, ymm7, ymm2
+        vpor	ymm8, ymm8, ymm3
+        vpor	ymm9, ymm9, ymm4
+        vpxor	ymm5, ymm5, ymm14
+        vpxor	ymm6, ymm6, ymm10
+        vpxor	ymm7, ymm7, ymm11
+        vpxor	ymm8, ymm8, ymm12
+        vpxor	ymm9, ymm9, ymm13
+        ; Row Mix
+        ; Row 0
+        vpxor	ymm10, ymm5, ymm15
+        vpxor	ymm11, ymm6, [r9+32]
+        vpxor	ymm12, ymm7, [r10+32]
+        vpxor	ymm13, ymm8, [r9+-96]
+        vpxor	ymm14, ymm9, [r10+-96]
+        vpsrlq	ymm0, ymm11, 20
+        vpsrlq	ymm1, ymm12, 21
+        vpsrlq	ymm2, ymm13, 43
+        vpsrlq	ymm3, ymm14, 50
+        vpsllq	ymm11, ymm11, 44
+        vpsllq	ymm12, ymm12, 43
+        vpsllq	ymm13, ymm13, 21
+        vpsllq	ymm14, ymm14, 14
+        vpor	ymm11, ymm11, ymm0
+        vpor	ymm12, ymm12, ymm1
+        vpor	ymm13, ymm13, ymm2
+        vpor	ymm14, ymm14, ymm3
+        vpandn	ymm15, ymm11, ymm12
+        vpandn	ymm1, ymm12, ymm13
+        vpandn	ymm2, ymm13, ymm14
+        vpandn	ymm3, ymm14, ymm10
+        vpandn	ymm4, ymm10, ymm11
+        vpxor	ymm15, ymm15, ymm10
+        vpxor	ymm1, ymm1, ymm11
+        vpxor	ymm2, ymm2, ymm12
+        ; XOR in constant
+        vpxor	ymm15, ymm15, [rax+224]
+        vpxor	ymm3, ymm3, ymm13
+        vpxor	ymm4, ymm4, ymm14
+        vmovdqu	YMMWORD PTR [r9+32], ymm1
+        vmovdqu	YMMWORD PTR [r10+32], ymm2
+        vmovdqu	YMMWORD PTR [r9+-96], ymm3
+        vmovdqu	YMMWORD PTR [r10+-96], ymm4
+        ; Row 1
+        vpxor	ymm10, ymm8, [rcx+64]
+        vpxor	ymm11, ymm9, [r10+-32]
+        vpxor	ymm12, ymm5, [rcx+-64]
+        vpxor	ymm13, ymm6, [r9+-64]
+        vpxor	ymm14, ymm7, [r10+96]
+        vpsrlq	ymm0, ymm10, 36
+        vpsrlq	ymm1, ymm11, 44
+        vpsrlq	ymm2, ymm12, 61
+        vpsrlq	ymm3, ymm13, 19
+        vpsrlq	ymm4, ymm14, 3
+        vpsllq	ymm10, ymm10, 28
+        vpsllq	ymm11, ymm11, 20
+        vpsllq	ymm12, ymm12, 3
+        vpsllq	ymm13, ymm13, 45
+        vpsllq	ymm14, ymm14, 61
+        vpor	ymm10, ymm10, ymm0
+        vpor	ymm11, ymm11, ymm1
+        vpor	ymm12, ymm12, ymm2
+        vpor	ymm13, ymm13, ymm3
+        vpor	ymm14, ymm14, ymm4
+        vpandn	ymm0, ymm11, ymm12
+        vpandn	ymm1, ymm12, ymm13
+        vpandn	ymm2, ymm13, ymm14
+        vpandn	ymm3, ymm14, ymm10
+        vpandn	ymm4, ymm10, ymm11
+        vpxor	ymm0, ymm0, ymm10
+        vpxor	ymm1, ymm1, ymm11
+        vpxor	ymm2, ymm2, ymm12
+        vpxor	ymm3, ymm3, ymm13
+        vpxor	ymm4, ymm4, ymm14
+        vmovdqu	YMMWORD PTR [rcx+64], ymm0
+        vmovdqu	YMMWORD PTR [r10+-32], ymm1
+        vmovdqu	YMMWORD PTR [rcx+-64], ymm2
+        vmovdqu	YMMWORD PTR [r9+-64], ymm3
+        vmovdqu	YMMWORD PTR [r10+96], ymm4
+        ; Row 2
+        vpxor	ymm10, ymm6, [r9]
+        vpxor	ymm11, ymm7, [r10]
+        vpxor	ymm12, ymm8, [rcx+128]
+        vpxor	ymm13, ymm9, [r9+128]
+        vpxor	ymm14, ymm5, [rcx]
+        vpsrlq	ymm0, ymm10, 63
+        vpsrlq	ymm1, ymm11, 58
+        vpsrlq	ymm2, ymm12, 39
+        vpsrlq	ymm3, ymm13, 56
+        vpsrlq	ymm4, ymm14, 46
+        vpaddq	ymm10, ymm10, ymm10
+        vpsllq	ymm11, ymm11, 6
+        vpsllq	ymm12, ymm12, 25
+        vpsllq	ymm13, ymm13, 8
+        vpsllq	ymm14, ymm14, 18
+        vpor	ymm10, ymm10, ymm0
+        vpor	ymm11, ymm11, ymm1
+        vpor	ymm12, ymm12, ymm2
+        vpor	ymm13, ymm13, ymm3
+        vpor	ymm14, ymm14, ymm4
+        vpandn	ymm0, ymm11, ymm12
+        vpandn	ymm1, ymm12, ymm13
+        vpandn	ymm2, ymm13, ymm14
+        vpandn	ymm3, ymm14, ymm10
+        vpandn	ymm4, ymm10, ymm11
+        vpxor	ymm0, ymm0, ymm10
+        vpxor	ymm1, ymm1, ymm11
+        vpxor	ymm2, ymm2, ymm12
+        vpxor	ymm3, ymm3, ymm13
+        vpxor	ymm4, ymm4, ymm14
+        vmovdqu	YMMWORD PTR [r9], ymm0
+        vmovdqu	YMMWORD PTR [r10], ymm1
+        vmovdqu	YMMWORD PTR [rcx+128], ymm2
+        vmovdqu	YMMWORD PTR [r9+128], ymm3
+        vmovdqu	YMMWORD PTR [rcx], ymm4
+        ; Row 3
+        vpxor	ymm10, ymm9, [r10+-64]
+        vpxor	ymm11, ymm5, [rcx+-96]
+        vpxor	ymm12, ymm6, [r9+64]
+        vpxor	ymm13, ymm7, [r10+64]
+        vpxor	ymm14, ymm8, [rcx+32]
+        vpsrlq	ymm0, ymm10, 37
+        vpsrlq	ymm1, ymm11, 28
+        vpsrlq	ymm2, ymm12, 54
+        vpsrlq	ymm3, ymm13, 49
+        vpsrlq	ymm4, ymm14, 8
+        vpsllq	ymm10, ymm10, 27
+        vpsllq	ymm11, ymm11, 36
+        vpsllq	ymm12, ymm12, 10
+        vpsllq	ymm13, ymm13, 15
+        vpsllq	ymm14, ymm14, 56
+        vpor	ymm10, ymm10, ymm0
+        vpor	ymm11, ymm11, ymm1
+        vpor	ymm12, ymm12, ymm2
+        vpor	ymm13, ymm13, ymm3
+        vpor	ymm14, ymm14, ymm4
+        vpandn	ymm0, ymm11, ymm12
+        vpandn	ymm1, ymm12, ymm13
+        vpandn	ymm2, ymm13, ymm14
+        vpandn	ymm3, ymm14, ymm10
+        vpandn	ymm4, ymm10, ymm11
+        vpxor	ymm0, ymm0, ymm10
+        vpxor	ymm1, ymm1, ymm11
+        vpxor	ymm2, ymm2, ymm12
+        vpxor	ymm3, ymm3, ymm13
+        vpxor	ymm4, ymm4, ymm14
+        vmovdqu	YMMWORD PTR [r10+-64], ymm0
+        vmovdqu	YMMWORD PTR [rcx+-96], ymm1
+        vmovdqu	YMMWORD PTR [r9+64], ymm2
+        vmovdqu	YMMWORD PTR [r10+64], ymm3
+        vmovdqu	YMMWORD PTR [rcx+32], ymm4
+        ; Row 4
+        vpxor	ymm10, ymm7, [r10+128]
+        vpxor	ymm11, ymm8, [rcx+96]
+        vpxor	ymm12, ymm9, [r9+96]
+        vpxor	ymm13, ymm5, [rcx+-32]
+        vpxor	ymm14, ymm6, [r9+-32]
+        vpsrlq	ymm0, ymm10, 2
+        vpsrlq	ymm1, ymm11, 9
+        vpsrlq	ymm2, ymm12, 25
+        vpsrlq	ymm3, ymm13, 23
+        vpsrlq	ymm4, ymm14, 62
+        vpsllq	ymm10, ymm10, 62
+        vpsllq	ymm11, ymm11, 55
+        vpsllq	ymm12, ymm12, 39
+        vpsllq	ymm13, ymm13, 41
+        vpsllq	ymm14, ymm14, 2
+        vpor	ymm10, ymm10, ymm0
+        vpor	ymm11, ymm11, ymm1
+        vpor	ymm12, ymm12, ymm2
+        vpor	ymm13, ymm13, ymm3
+        vpor	ymm14, ymm14, ymm4
+        vpandn	ymm0, ymm11, ymm12
+        vpandn	ymm1, ymm12, ymm13
+        vpandn	ymm2, ymm13, ymm14
+        vpandn	ymm3, ymm14, ymm10
+        vpandn	ymm4, ymm10, ymm11
+        vpxor	ymm0, ymm0, ymm10
+        vpxor	ymm1, ymm1, ymm11
+        vpxor	ymm2, ymm2, ymm12
+        vpxor	ymm3, ymm3, ymm13
+        vpxor	ymm4, ymm4, ymm14
+        vmovdqu	YMMWORD PTR [r10+128], ymm0
+        vmovdqu	YMMWORD PTR [rcx+96], ymm1
+        vmovdqu	YMMWORD PTR [r9+96], ymm2
+        vmovdqu	YMMWORD PTR [rcx+-32], ymm3
+        vmovdqu	YMMWORD PTR [r9+-32], ymm4
+        ; Round 8
+        ; Calc b[0..4]
+        vpxor	ymm10, ymm0, ymm15
+        vpxor	ymm11, ymm1, [rcx+-96]
+        vpxor	ymm12, ymm2, [rcx+-64]
+        vpxor	ymm14, ymm4, [rcx]
+        vpxor	ymm14, ymm14, [rcx+32]
+        vpxor	ymm10, ymm10, [rcx+64]
+        vpxor	ymm12, ymm12, [rcx+128]
+        vpxor	ymm13, ymm3, [r9+-96]
+        vpxor	ymm13, ymm13, [r9+-64]
+        vpxor	ymm10, ymm10, [r9]
+        vpxor	ymm11, ymm11, [r9+32]
+        vpxor	ymm12, ymm12, [r9+64]
+        vpxor	ymm13, ymm13, [r9+128]
+        vpxor	ymm14, ymm14, [r10+-96]
+        vpxor	ymm10, ymm10, [r10+-64]
+        vpxor	ymm11, ymm11, [r10+-32]
+        vpxor	ymm11, ymm11, [r10]
+        vpxor	ymm12, ymm12, [r10+32]
+        vpxor	ymm13, ymm13, [r10+64]
+        vpxor	ymm14, ymm14, [r10+96]
+        ; Calc t[0..4]
+        vpsrlq	ymm0, ymm11, 63
+        vpsrlq	ymm1, ymm12, 63
+        vpsrlq	ymm2, ymm13, 63
+        vpsrlq	ymm3, ymm14, 63
+        vpsrlq	ymm4, ymm10, 63
+        vpaddq	ymm5, ymm11, ymm11
+        vpaddq	ymm6, ymm12, ymm12
+        vpaddq	ymm7, ymm13, ymm13
+        vpaddq	ymm8, ymm14, ymm14
+        vpaddq	ymm9, ymm10, ymm10
+        vpor	ymm5, ymm5, ymm0
+        vpor	ymm6, ymm6, ymm1
+        vpor	ymm7, ymm7, ymm2
+        vpor	ymm8, ymm8, ymm3
+        vpor	ymm9, ymm9, ymm4
+        vpxor	ymm5, ymm5, ymm14
+        vpxor	ymm6, ymm6, ymm10
+        vpxor	ymm7, ymm7, ymm11
+        vpxor	ymm8, ymm8, ymm12
+        vpxor	ymm9, ymm9, ymm13
+        ; Row Mix
+        ; Row 0
+        vpxor	ymm10, ymm5, ymm15
+        vpxor	ymm11, ymm6, [r10+-32]
+        vpxor	ymm12, ymm7, [rcx+128]
+        vpxor	ymm13, ymm8, [r10+64]
+        vpxor	ymm14, ymm9, [r9+-32]
+        vpsrlq	ymm0, ymm11, 20
+        vpsrlq	ymm1, ymm12, 21
+        vpsrlq	ymm2, ymm13, 43
+        vpsrlq	ymm3, ymm14, 50
+        vpsllq	ymm11, ymm11, 44
+        vpsllq	ymm12, ymm12, 43
+        vpsllq	ymm13, ymm13, 21
+        vpsllq	ymm14, ymm14, 14
+        vpor	ymm11, ymm11, ymm0
+        vpor	ymm12, ymm12, ymm1
+        vpor	ymm13, ymm13, ymm2
+        vpor	ymm14, ymm14, ymm3
+        vpandn	ymm15, ymm11, ymm12
+        vpandn	ymm1, ymm12, ymm13
+        vpandn	ymm2, ymm13, ymm14
+        vpandn	ymm3, ymm14, ymm10
+        vpandn	ymm4, ymm10, ymm11
+        vpxor	ymm15, ymm15, ymm10
+        vpxor	ymm1, ymm1, ymm11
+        vpxor	ymm2, ymm2, ymm12
+        ; XOR in constant
+        vpxor	ymm15, ymm15, [rax+256]
+        vpxor	ymm3, ymm3, ymm13
+        vpxor	ymm4, ymm4, ymm14
+        vmovdqu	YMMWORD PTR [r10+-32], ymm1
+        vmovdqu	YMMWORD PTR [rcx+128], ymm2
+        vmovdqu	YMMWORD PTR [r10+64], ymm3
+        vmovdqu	YMMWORD PTR [r9+-32], ymm4
+        ; Row 1
+        vpxor	ymm10, ymm8, [r9+-96]
+        vpxor	ymm11, ymm9, [r10+96]
+        vpxor	ymm12, ymm5, [r9]
+        vpxor	ymm13, ymm6, [rcx+-96]
+        vpxor	ymm14, ymm7, [r9+96]
+        vpsrlq	ymm0, ymm10, 36
+        vpsrlq	ymm1, ymm11, 44
+        vpsrlq	ymm2, ymm12, 61
+        vpsrlq	ymm3, ymm13, 19
+        vpsrlq	ymm4, ymm14, 3
+        vpsllq	ymm10, ymm10, 28
+        vpsllq	ymm11, ymm11, 20
+        vpsllq	ymm12, ymm12, 3
+        vpsllq	ymm13, ymm13, 45
+        vpsllq	ymm14, ymm14, 61
+        vpor	ymm10, ymm10, ymm0
+        vpor	ymm11, ymm11, ymm1
+        vpor	ymm12, ymm12, ymm2
+        vpor	ymm13, ymm13, ymm3
+        vpor	ymm14, ymm14, ymm4
+        vpandn	ymm0, ymm11, ymm12
+        vpandn	ymm1, ymm12, ymm13
+        vpandn	ymm2, ymm13, ymm14
+        vpandn	ymm3, ymm14, ymm10
+        vpandn	ymm4, ymm10, ymm11
+        vpxor	ymm0, ymm0, ymm10
+        vpxor	ymm1, ymm1, ymm11
+        vpxor	ymm2, ymm2, ymm12
+        vpxor	ymm3, ymm3, ymm13
+        vpxor	ymm4, ymm4, ymm14
+        vmovdqu	YMMWORD PTR [r9+-96], ymm0
+        vmovdqu	YMMWORD PTR [r10+96], ymm1
+        vmovdqu	YMMWORD PTR [r9], ymm2
+        vmovdqu	YMMWORD PTR [rcx+-96], ymm3
+        vmovdqu	YMMWORD PTR [r9+96], ymm4
+        ; Row 2
+        vpxor	ymm10, ymm6, [r9+32]
+        vpxor	ymm11, ymm7, [rcx+-64]
+        vpxor	ymm12, ymm8, [r9+128]
+        vpxor	ymm13, ymm9, [rcx+32]
+        vpxor	ymm14, ymm5, [r10+128]
+        vpsrlq	ymm0, ymm10, 63
+        vpsrlq	ymm1, ymm11, 58
+        vpsrlq	ymm2, ymm12, 39
+        vpsrlq	ymm3, ymm13, 56
+        vpsrlq	ymm4, ymm14, 46
+        vpaddq	ymm10, ymm10, ymm10
+        vpsllq	ymm11, ymm11, 6
+        vpsllq	ymm12, ymm12, 25
+        vpsllq	ymm13, ymm13, 8
+        vpsllq	ymm14, ymm14, 18
+        vpor	ymm10, ymm10, ymm0
+        vpor	ymm11, ymm11, ymm1
+        vpor	ymm12, ymm12, ymm2
+        vpor	ymm13, ymm13, ymm3
+        vpor	ymm14, ymm14, ymm4
+        vpandn	ymm0, ymm11, ymm12
+        vpandn	ymm1, ymm12, ymm13
+        vpandn	ymm2, ymm13, ymm14
+        vpandn	ymm3, ymm14, ymm10
+        vpandn	ymm4, ymm10, ymm11
+        vpxor	ymm0, ymm0, ymm10
+        vpxor	ymm1, ymm1, ymm11
+        vpxor	ymm2, ymm2, ymm12
+        vpxor	ymm3, ymm3, ymm13
+        vpxor	ymm4, ymm4, ymm14
+        vmovdqu	YMMWORD PTR [r9+32], ymm0
+        vmovdqu	YMMWORD PTR [rcx+-64], ymm1
+        vmovdqu	YMMWORD PTR [r9+128], ymm2
+        vmovdqu	YMMWORD PTR [rcx+32], ymm3
+        vmovdqu	YMMWORD PTR [r10+128], ymm4
+        ; Row 3
+        vpxor	ymm10, ymm9, [r10+-96]
+        vpxor	ymm11, ymm5, [rcx+64]
+        vpxor	ymm12, ymm6, [r10]
+        vpxor	ymm13, ymm7, [r9+64]
+        vpxor	ymm14, ymm8, [rcx+-32]
+        vpsrlq	ymm0, ymm10, 37
+        vpsrlq	ymm1, ymm11, 28
+        vpsrlq	ymm2, ymm12, 54
+        vpsrlq	ymm3, ymm13, 49
+        vpsrlq	ymm4, ymm14, 8
+        vpsllq	ymm10, ymm10, 27
+        vpsllq	ymm11, ymm11, 36
+        vpsllq	ymm12, ymm12, 10
+        vpsllq	ymm13, ymm13, 15
+        vpsllq	ymm14, ymm14, 56
+        vpor	ymm10, ymm10, ymm0
+        vpor	ymm11, ymm11, ymm1
+        vpor	ymm12, ymm12, ymm2
+        vpor	ymm13, ymm13, ymm3
+        vpor	ymm14, ymm14, ymm4
+        vpandn	ymm0, ymm11, ymm12
+        vpandn	ymm1, ymm12, ymm13
+        vpandn	ymm2, ymm13, ymm14
+        vpandn	ymm3, ymm14, ymm10
+        vpandn	ymm4, ymm10, ymm11
+        vpxor	ymm0, ymm0, ymm10
+        vpxor	ymm1, ymm1, ymm11
+        vpxor	ymm2, ymm2, ymm12
+        vpxor	ymm3, ymm3, ymm13
+        vpxor	ymm4, ymm4, ymm14
+        vmovdqu	YMMWORD PTR [r10+-96], ymm0
+        vmovdqu	YMMWORD PTR [rcx+64], ymm1
+        vmovdqu	YMMWORD PTR [r10], ymm2
+        vmovdqu	YMMWORD PTR [r9+64], ymm3
+        vmovdqu	YMMWORD PTR [rcx+-32], ymm4
+        ; Row 4
+        vpxor	ymm10, ymm7, [r10+32]
+        vpxor	ymm11, ymm8, [r9+-64]
+        vpxor	ymm12, ymm9, [rcx]
+        vpxor	ymm13, ymm5, [r10+-64]
+        vpxor	ymm14, ymm6, [rcx+96]
+        vpsrlq	ymm0, ymm10, 2
+        vpsrlq	ymm1, ymm11, 9
+        vpsrlq	ymm2, ymm12, 25
+        vpsrlq	ymm3, ymm13, 23
+        vpsrlq	ymm4, ymm14, 62
+        vpsllq	ymm10, ymm10, 62
+        vpsllq	ymm11, ymm11, 55
+        vpsllq	ymm12, ymm12, 39
+        vpsllq	ymm13, ymm13, 41
+        vpsllq	ymm14, ymm14, 2
+        vpor	ymm10, ymm10, ymm0
+        vpor	ymm11, ymm11, ymm1
+        vpor	ymm12, ymm12, ymm2
+        vpor	ymm13, ymm13, ymm3
+        vpor	ymm14, ymm14, ymm4
+        vpandn	ymm0, ymm11, ymm12
+        vpandn	ymm1, ymm12, ymm13
+        vpandn	ymm2, ymm13, ymm14
+        vpandn	ymm3, ymm14, ymm10
+        vpandn	ymm4, ymm10, ymm11
+        vpxor	ymm0, ymm0, ymm10
+        vpxor	ymm1, ymm1, ymm11
+        vpxor	ymm2, ymm2, ymm12
+        vpxor	ymm3, ymm3, ymm13
+        vpxor	ymm4, ymm4, ymm14
+        vmovdqu	YMMWORD PTR [r10+32], ymm0
+        vmovdqu	YMMWORD PTR [r9+-64], ymm1
+        vmovdqu	YMMWORD PTR [rcx], ymm2
+        vmovdqu	YMMWORD PTR [r10+-64], ymm3
+        vmovdqu	YMMWORD PTR [rcx+96], ymm4
+        ; Round 9
+        ; Calc b[0..4]
+        vpxor	ymm10, ymm0, ymm15
+        vpxor	ymm13, ymm3, [rcx+-96]
+        vpxor	ymm11, ymm1, [rcx+-64]
+        vpxor	ymm14, ymm4, [rcx+-32]
+        vpxor	ymm13, ymm13, [rcx+32]
+        vpxor	ymm11, ymm11, [rcx+64]
+        vpxor	ymm12, ymm2, [rcx+128]
+        vpxor	ymm10, ymm10, [r9+-96]
+        vpxor	ymm14, ymm14, [r9+-32]
+        vpxor	ymm12, ymm12, [r9]
+        vpxor	ymm10, ymm10, [r9+32]
+        vpxor	ymm13, ymm13, [r9+64]
+        vpxor	ymm14, ymm14, [r9+96]
+        vpxor	ymm12, ymm12, [r9+128]
+        vpxor	ymm10, ymm10, [r10+-96]
+        vpxor	ymm11, ymm11, [r10+-32]
+        vpxor	ymm12, ymm12, [r10]
+        vpxor	ymm13, ymm13, [r10+64]
+        vpxor	ymm11, ymm11, [r10+96]
+        vpxor	ymm14, ymm14, [r10+128]
+        ; Calc t[0..4]
+        vpsrlq	ymm0, ymm11, 63
+        vpsrlq	ymm1, ymm12, 63
+        vpsrlq	ymm2, ymm13, 63
+        vpsrlq	ymm3, ymm14, 63
+        vpsrlq	ymm4, ymm10, 63
+        vpaddq	ymm5, ymm11, ymm11
+        vpaddq	ymm6, ymm12, ymm12
+        vpaddq	ymm7, ymm13, ymm13
+        vpaddq	ymm8, ymm14, ymm14
+        vpaddq	ymm9, ymm10, ymm10
+        vpor	ymm5, ymm5, ymm0
+        vpor	ymm6, ymm6, ymm1
+        vpor	ymm7, ymm7, ymm2
+        vpor	ymm8, ymm8, ymm3
+        vpor	ymm9, ymm9, ymm4
+        vpxor	ymm5, ymm5, ymm14
+        vpxor	ymm6, ymm6, ymm10
+        vpxor	ymm7, ymm7, ymm11
+        vpxor	ymm8, ymm8, ymm12
+        vpxor	ymm9, ymm9, ymm13
+        ; Row Mix
+        ; Row 0
+        vpxor	ymm10, ymm5, ymm15
+        vpxor	ymm11, ymm6, [r10+96]
+        vpxor	ymm12, ymm7, [r9+128]
+        vpxor	ymm13, ymm8, [r9+64]
+        vpxor	ymm14, ymm9, [rcx+96]
+        vpsrlq	ymm0, ymm11, 20
+        vpsrlq	ymm1, ymm12, 21
+        vpsrlq	ymm2, ymm13, 43
+        vpsrlq	ymm3, ymm14, 50
+        vpsllq	ymm11, ymm11, 44
+        vpsllq	ymm12, ymm12, 43
+        vpsllq	ymm13, ymm13, 21
+        vpsllq	ymm14, ymm14, 14
+        vpor	ymm11, ymm11, ymm0
+        vpor	ymm12, ymm12, ymm1
+        vpor	ymm13, ymm13, ymm2
+        vpor	ymm14, ymm14, ymm3
+        vpandn	ymm15, ymm11, ymm12
+        vpandn	ymm1, ymm12, ymm13
+        vpandn	ymm2, ymm13, ymm14
+        vpandn	ymm3, ymm14, ymm10
+        vpandn	ymm4, ymm10, ymm11
+        vpxor	ymm15, ymm15, ymm10
+        vpxor	ymm1, ymm1, ymm11
+        vpxor	ymm2, ymm2, ymm12
+        ; XOR in constant
+        vpxor	ymm15, ymm15, [rax+288]
+        vpxor	ymm3, ymm3, ymm13
+        vpxor	ymm4, ymm4, ymm14
+        vmovdqu	YMMWORD PTR [r10+96], ymm1
+        vmovdqu	YMMWORD PTR [r9+128], ymm2
+        vmovdqu	YMMWORD PTR [r9+64], ymm3
+        vmovdqu	YMMWORD PTR [rcx+96], ymm4
+        ; Row 1
+        vpxor	ymm10, ymm8, [r10+64]
+        vpxor	ymm11, ymm9, [r9+96]
+        vpxor	ymm12, ymm5, [r9+32]
+        vpxor	ymm13, ymm6, [rcx+64]
+        vpxor	ymm14, ymm7, [rcx]
+        vpsrlq	ymm0, ymm10, 36
+        vpsrlq	ymm1, ymm11, 44
+        vpsrlq	ymm2, ymm12, 61
+        vpsrlq	ymm3, ymm13, 19
+        vpsrlq	ymm4, ymm14, 3
+        vpsllq	ymm10, ymm10, 28
+        vpsllq	ymm11, ymm11, 20
+        vpsllq	ymm12, ymm12, 3
+        vpsllq	ymm13, ymm13, 45
+        vpsllq	ymm14, ymm14, 61
+        vpor	ymm10, ymm10, ymm0
+        vpor	ymm11, ymm11, ymm1
+        vpor	ymm12, ymm12, ymm2
+        vpor	ymm13, ymm13, ymm3
+        vpor	ymm14, ymm14, ymm4
+        vpandn	ymm0, ymm11, ymm12
+        vpandn	ymm1, ymm12, ymm13
+        vpandn	ymm2, ymm13, ymm14
+        vpandn	ymm3, ymm14, ymm10
+        vpandn	ymm4, ymm10, ymm11
+        vpxor	ymm0, ymm0, ymm10
+        vpxor	ymm1, ymm1, ymm11
+        vpxor	ymm2, ymm2, ymm12
+        vpxor	ymm3, ymm3, ymm13
+        vpxor	ymm4, ymm4, ymm14
+        vmovdqu	YMMWORD PTR [r10+64], ymm0
+        vmovdqu	YMMWORD PTR [r9+96], ymm1
+        vmovdqu	YMMWORD PTR [r9+32], ymm2
+        vmovdqu	YMMWORD PTR [rcx+64], ymm3
+        vmovdqu	YMMWORD PTR [rcx], ymm4
+        ; Row 2
+        vpxor	ymm10, ymm6, [r10+-32]
+        vpxor	ymm11, ymm7, [r9]
+        vpxor	ymm12, ymm8, [rcx+32]
+        vpxor	ymm13, ymm9, [rcx+-32]
+        vpxor	ymm14, ymm5, [r10+32]
+        vpsrlq	ymm0, ymm10, 63
+        vpsrlq	ymm1, ymm11, 58
+        vpsrlq	ymm2, ymm12, 39
+        vpsrlq	ymm3, ymm13, 56
+        vpsrlq	ymm4, ymm14, 46
+        vpaddq	ymm10, ymm10, ymm10
+        vpsllq	ymm11, ymm11, 6
+        vpsllq	ymm12, ymm12, 25
+        vpsllq	ymm13, ymm13, 8
+        vpsllq	ymm14, ymm14, 18
+        vpor	ymm10, ymm10, ymm0
+        vpor	ymm11, ymm11, ymm1
+        vpor	ymm12, ymm12, ymm2
+        vpor	ymm13, ymm13, ymm3
+        vpor	ymm14, ymm14, ymm4
+        vpandn	ymm0, ymm11, ymm12
+        vpandn	ymm1, ymm12, ymm13
+        vpandn	ymm2, ymm13, ymm14
+        vpandn	ymm3, ymm14, ymm10
+        vpandn	ymm4, ymm10, ymm11
+        vpxor	ymm0, ymm0, ymm10
+        vpxor	ymm1, ymm1, ymm11
+        vpxor	ymm2, ymm2, ymm12
+        vpxor	ymm3, ymm3, ymm13
+        vpxor	ymm4, ymm4, ymm14
+        vmovdqu	YMMWORD PTR [r10+-32], ymm0
+        vmovdqu	YMMWORD PTR [r9], ymm1
+        vmovdqu	YMMWORD PTR [rcx+32], ymm2
+        vmovdqu	YMMWORD PTR [rcx+-32], ymm3
+        vmovdqu	YMMWORD PTR [r10+32], ymm4
+        ; Row 3
+        vpxor	ymm10, ymm9, [r9+-32]
+        vpxor	ymm11, ymm5, [r9+-96]
+        vpxor	ymm12, ymm6, [rcx+-64]
+        vpxor	ymm13, ymm7, [r10]
+        vpxor	ymm14, ymm8, [r10+-64]
+        vpsrlq	ymm0, ymm10, 37
+        vpsrlq	ymm1, ymm11, 28
+        vpsrlq	ymm2, ymm12, 54
+        vpsrlq	ymm3, ymm13, 49
+        vpsrlq	ymm4, ymm14, 8
+        vpsllq	ymm10, ymm10, 27
+        vpsllq	ymm11, ymm11, 36
+        vpsllq	ymm12, ymm12, 10
+        vpsllq	ymm13, ymm13, 15
+        vpsllq	ymm14, ymm14, 56
+        vpor	ymm10, ymm10, ymm0
+        vpor	ymm11, ymm11, ymm1
+        vpor	ymm12, ymm12, ymm2
+        vpor	ymm13, ymm13, ymm3
+        vpor	ymm14, ymm14, ymm4
+        vpandn	ymm0, ymm11, ymm12
+        vpandn	ymm1, ymm12, ymm13
+        vpandn	ymm2, ymm13, ymm14
+        vpandn	ymm3, ymm14, ymm10
+        vpandn	ymm4, ymm10, ymm11
+        vpxor	ymm0, ymm0, ymm10
+        vpxor	ymm1, ymm1, ymm11
+        vpxor	ymm2, ymm2, ymm12
+        vpxor	ymm3, ymm3, ymm13
+        vpxor	ymm4, ymm4, ymm14
+        vmovdqu	YMMWORD PTR [r9+-32], ymm0
+        vmovdqu	YMMWORD PTR [r9+-96], ymm1
+        vmovdqu	YMMWORD PTR [rcx+-64], ymm2
+        vmovdqu	YMMWORD PTR [r10], ymm3
+        vmovdqu	YMMWORD PTR [r10+-64], ymm4
+        ; Row 4
+        vpxor	ymm10, ymm7, [rcx+128]
+        vpxor	ymm11, ymm8, [rcx+-96]
+        vpxor	ymm12, ymm9, [r10+128]
+        vpxor	ymm13, ymm5, [r10+-96]
+        vpxor	ymm14, ymm6, [r9+-64]
+        vpsrlq	ymm0, ymm10, 2
+        vpsrlq	ymm1, ymm11, 9
+        vpsrlq	ymm2, ymm12, 25
+        vpsrlq	ymm3, ymm13, 23
+        vpsrlq	ymm4, ymm14, 62
+        vpsllq	ymm10, ymm10, 62
+        vpsllq	ymm11, ymm11, 55
+        vpsllq	ymm12, ymm12, 39
+        vpsllq	ymm13, ymm13, 41
+        vpsllq	ymm14, ymm14, 2
+        vpor	ymm10, ymm10, ymm0
+        vpor	ymm11, ymm11, ymm1
+        vpor	ymm12, ymm12, ymm2
+        vpor	ymm13, ymm13, ymm3
+        vpor	ymm14, ymm14, ymm4
+        vpandn	ymm0, ymm11, ymm12
+        vpandn	ymm1, ymm12, ymm13
+        vpandn	ymm2, ymm13, ymm14
+        vpandn	ymm3, ymm14, ymm10
+        vpandn	ymm4, ymm10, ymm11
+        vpxor	ymm0, ymm0, ymm10
+        vpxor	ymm1, ymm1, ymm11
+        vpxor	ymm2, ymm2, ymm12
+        vpxor	ymm3, ymm3, ymm13
+        vpxor	ymm4, ymm4, ymm14
+        vmovdqu	YMMWORD PTR [rcx+128], ymm0
+        vmovdqu	YMMWORD PTR [rcx+-96], ymm1
+        vmovdqu	YMMWORD PTR [r10+128], ymm2
+        vmovdqu	YMMWORD PTR [r10+-96], ymm3
+        vmovdqu	YMMWORD PTR [r9+-64], ymm4
+        ; Round 10
+        ; Calc b[0..4]
+        vpxor	ymm10, ymm0, ymm15
+        vpxor	ymm12, ymm2, [rcx+-64]
+        vpxor	ymm13, ymm3, [rcx+-32]
+        vpxor	ymm14, ymm4, [rcx]
+        vpxor	ymm12, ymm12, [rcx+32]
+        vpxor	ymm13, ymm13, [rcx+64]
+        vpxor	ymm14, ymm14, [rcx+96]
+        vpxor	ymm11, ymm1, [r9+-96]
+        vpxor	ymm10, ymm10, [r9+-32]
+        vpxor	ymm11, ymm11, [r9]
+        vpxor	ymm12, ymm12, [r9+32]
+        vpxor	ymm13, ymm13, [r9+64]
+        vpxor	ymm11, ymm11, [r9+96]
+        vpxor	ymm12, ymm12, [r9+128]
+        vpxor	ymm14, ymm14, [r10+-64]
+        vpxor	ymm10, ymm10, [r10+-32]
+        vpxor	ymm13, ymm13, [r10]
+        vpxor	ymm14, ymm14, [r10+32]
+        vpxor	ymm10, ymm10, [r10+64]
+        vpxor	ymm11, ymm11, [r10+96]
+        ; Calc t[0..4]
+        vpsrlq	ymm0, ymm11, 63
+        vpsrlq	ymm1, ymm12, 63
+        vpsrlq	ymm2, ymm13, 63
+        vpsrlq	ymm3, ymm14, 63
+        vpsrlq	ymm4, ymm10, 63
+        vpaddq	ymm5, ymm11, ymm11
+        vpaddq	ymm6, ymm12, ymm12
+        vpaddq	ymm7, ymm13, ymm13
+        vpaddq	ymm8, ymm14, ymm14
+        vpaddq	ymm9, ymm10, ymm10
+        vpor	ymm5, ymm5, ymm0
+        vpor	ymm6, ymm6, ymm1
+        vpor	ymm7, ymm7, ymm2
+        vpor	ymm8, ymm8, ymm3
+        vpor	ymm9, ymm9, ymm4
+        vpxor	ymm5, ymm5, ymm14
+        vpxor	ymm6, ymm6, ymm10
+        vpxor	ymm7, ymm7, ymm11
+        vpxor	ymm8, ymm8, ymm12
+        vpxor	ymm9, ymm9, ymm13
+        ; Row Mix
+        ; Row 0
+        vpxor	ymm10, ymm5, ymm15
+        vpxor	ymm11, ymm6, [r9+96]
+        vpxor	ymm12, ymm7, [rcx+32]
+        vpxor	ymm13, ymm8, [r10]
+        vpxor	ymm14, ymm9, [r9+-64]
+        vpsrlq	ymm0, ymm11, 20
+        vpsrlq	ymm1, ymm12, 21
+        vpsrlq	ymm2, ymm13, 43
+        vpsrlq	ymm3, ymm14, 50
+        vpsllq	ymm11, ymm11, 44
+        vpsllq	ymm12, ymm12, 43
+        vpsllq	ymm13, ymm13, 21
+        vpsllq	ymm14, ymm14, 14
+        vpor	ymm11, ymm11, ymm0
+        vpor	ymm12, ymm12, ymm1
+        vpor	ymm13, ymm13, ymm2
+        vpor	ymm14, ymm14, ymm3
+        vpandn	ymm15, ymm11, ymm12
+        vpandn	ymm1, ymm12, ymm13
+        vpandn	ymm2, ymm13, ymm14
+        vpandn	ymm3, ymm14, ymm10
+        vpandn	ymm4, ymm10, ymm11
+        vpxor	ymm15, ymm15, ymm10
+        vpxor	ymm1, ymm1, ymm11
+        vpxor	ymm2, ymm2, ymm12
+        ; XOR in constant
+        vpxor	ymm15, ymm15, [rax+320]
+        vpxor	ymm3, ymm3, ymm13
+        vpxor	ymm4, ymm4, ymm14
+        vmovdqu	YMMWORD PTR [r9+96], ymm1
+        vmovdqu	YMMWORD PTR [rcx+32], ymm2
+        vmovdqu	YMMWORD PTR [r10], ymm3
+        vmovdqu	YMMWORD PTR [r9+-64], ymm4
+        ; Row 1
+        vpxor	ymm10, ymm8, [r9+64]
+        vpxor	ymm11, ymm9, [rcx]
+        vpxor	ymm12, ymm5, [r10+-32]
+        vpxor	ymm13, ymm6, [r9+-96]
+        vpxor	ymm14, ymm7, [r10+128]
+        vpsrlq	ymm0, ymm10, 36
+        vpsrlq	ymm1, ymm11, 44
+        vpsrlq	ymm2, ymm12, 61
+        vpsrlq	ymm3, ymm13, 19
+        vpsrlq	ymm4, ymm14, 3
+        vpsllq	ymm10, ymm10, 28
+        vpsllq	ymm11, ymm11, 20
+        vpsllq	ymm12, ymm12, 3
+        vpsllq	ymm13, ymm13, 45
+        vpsllq	ymm14, ymm14, 61
+        vpor	ymm10, ymm10, ymm0
+        vpor	ymm11, ymm11, ymm1
+        vpor	ymm12, ymm12, ymm2
+        vpor	ymm13, ymm13, ymm3
+        vpor	ymm14, ymm14, ymm4
+        vpandn	ymm0, ymm11, ymm12
+        vpandn	ymm1, ymm12, ymm13
+        vpandn	ymm2, ymm13, ymm14
+        vpandn	ymm3, ymm14, ymm10
+        vpandn	ymm4, ymm10, ymm11
+        vpxor	ymm0, ymm0, ymm10
+        vpxor	ymm1, ymm1, ymm11
+        vpxor	ymm2, ymm2, ymm12
+        vpxor	ymm3, ymm3, ymm13
+        vpxor	ymm4, ymm4, ymm14
+        vmovdqu	YMMWORD PTR [r9+64], ymm0
+        vmovdqu	YMMWORD PTR [rcx], ymm1
+        vmovdqu	YMMWORD PTR [r10+-32], ymm2
+        vmovdqu	YMMWORD PTR [r9+-96], ymm3
+        vmovdqu	YMMWORD PTR [r10+128], ymm4
+        ; Row 2
+        vpxor	ymm10, ymm6, [r10+96]
+        vpxor	ymm11, ymm7, [r9+32]
+        vpxor	ymm12, ymm8, [rcx+-32]
+        vpxor	ymm13, ymm9, [r10+-64]
+        vpxor	ymm14, ymm5, [rcx+128]
+        vpsrlq	ymm0, ymm10, 63
+        vpsrlq	ymm1, ymm11, 58
+        vpsrlq	ymm2, ymm12, 39
+        vpsrlq	ymm3, ymm13, 56
+        vpsrlq	ymm4, ymm14, 46
+        vpaddq	ymm10, ymm10, ymm10
+        vpsllq	ymm11, ymm11, 6
+        vpsllq	ymm12, ymm12, 25
+        vpsllq	ymm13, ymm13, 8
+        vpsllq	ymm14, ymm14, 18
+        vpor	ymm10, ymm10, ymm0
+        vpor	ymm11, ymm11, ymm1
+        vpor	ymm12, ymm12, ymm2
+        vpor	ymm13, ymm13, ymm3
+        vpor	ymm14, ymm14, ymm4
+        vpandn	ymm0, ymm11, ymm12
+        vpandn	ymm1, ymm12, ymm13
+        vpandn	ymm2, ymm13, ymm14
+        vpandn	ymm3, ymm14, ymm10
+        vpandn	ymm4, ymm10, ymm11
+        vpxor	ymm0, ymm0, ymm10
+        vpxor	ymm1, ymm1, ymm11
+        vpxor	ymm2, ymm2, ymm12
+        vpxor	ymm3, ymm3, ymm13
+        vpxor	ymm4, ymm4, ymm14
+        vmovdqu	YMMWORD PTR [r10+96], ymm0
+        vmovdqu	YMMWORD PTR [r9+32], ymm1
+        vmovdqu	YMMWORD PTR [rcx+-32], ymm2
+        vmovdqu	YMMWORD PTR [r10+-64], ymm3
+        vmovdqu	YMMWORD PTR [rcx+128], ymm4
+        ; Row 3
+        vpxor	ymm10, ymm9, [rcx+96]
+        vpxor	ymm11, ymm5, [r10+64]
+        vpxor	ymm12, ymm6, [r9]
+        vpxor	ymm13, ymm7, [rcx+-64]
+        vpxor	ymm14, ymm8, [r10+-96]
+        vpsrlq	ymm0, ymm10, 37
+        vpsrlq	ymm1, ymm11, 28
+        vpsrlq	ymm2, ymm12, 54
+        vpsrlq	ymm3, ymm13, 49
+        vpsrlq	ymm4, ymm14, 8
+        vpsllq	ymm10, ymm10, 27
+        vpsllq	ymm11, ymm11, 36
+        vpsllq	ymm12, ymm12, 10
+        vpsllq	ymm13, ymm13, 15
+        vpsllq	ymm14, ymm14, 56
+        vpor	ymm10, ymm10, ymm0
+        vpor	ymm11, ymm11, ymm1
+        vpor	ymm12, ymm12, ymm2
+        vpor	ymm13, ymm13, ymm3
+        vpor	ymm14, ymm14, ymm4
+        vpandn	ymm0, ymm11, ymm12
+        vpandn	ymm1, ymm12, ymm13
+        vpandn	ymm2, ymm13, ymm14
+        vpandn	ymm3, ymm14, ymm10
+        vpandn	ymm4, ymm10, ymm11
+        vpxor	ymm0, ymm0, ymm10
+        vpxor	ymm1, ymm1, ymm11
+        vpxor	ymm2, ymm2, ymm12
+        vpxor	ymm3, ymm3, ymm13
+        vpxor	ymm4, ymm4, ymm14
+        vmovdqu	YMMWORD PTR [rcx+96], ymm0
+        vmovdqu	YMMWORD PTR [r10+64], ymm1
+        vmovdqu	YMMWORD PTR [r9], ymm2
+        vmovdqu	YMMWORD PTR [rcx+-64], ymm3
+        vmovdqu	YMMWORD PTR [r10+-96], ymm4
+        ; Row 4
+        vpxor	ymm10, ymm7, [r9+128]
+        vpxor	ymm11, ymm8, [rcx+64]
+        vpxor	ymm12, ymm9, [r10+32]
+        vpxor	ymm13, ymm5, [r9+-32]
+        vpxor	ymm14, ymm6, [rcx+-96]
+        vpsrlq	ymm0, ymm10, 2
+        vpsrlq	ymm1, ymm11, 9
+        vpsrlq	ymm2, ymm12, 25
+        vpsrlq	ymm3, ymm13, 23
+        vpsrlq	ymm4, ymm14, 62
+        vpsllq	ymm10, ymm10, 62
+        vpsllq	ymm11, ymm11, 55
+        vpsllq	ymm12, ymm12, 39
+        vpsllq	ymm13, ymm13, 41
+        vpsllq	ymm14, ymm14, 2
+        vpor	ymm10, ymm10, ymm0
+        vpor	ymm11, ymm11, ymm1
+        vpor	ymm12, ymm12, ymm2
+        vpor	ymm13, ymm13, ymm3
+        vpor	ymm14, ymm14, ymm4
+        vpandn	ymm0, ymm11, ymm12
+        vpandn	ymm1, ymm12, ymm13
+        vpandn	ymm2, ymm13, ymm14
+        vpandn	ymm3, ymm14, ymm10
+        vpandn	ymm4, ymm10, ymm11
+        vpxor	ymm0, ymm0, ymm10
+        vpxor	ymm1, ymm1, ymm11
+        vpxor	ymm2, ymm2, ymm12
+        vpxor	ymm3, ymm3, ymm13
+        vpxor	ymm4, ymm4, ymm14
+        vmovdqu	YMMWORD PTR [r9+128], ymm0
+        vmovdqu	YMMWORD PTR [rcx+64], ymm1
+        vmovdqu	YMMWORD PTR [r10+32], ymm2
+        vmovdqu	YMMWORD PTR [r9+-32], ymm3
+        vmovdqu	YMMWORD PTR [rcx+-96], ymm4
+        ; Round 11
+        ; Calc b[0..4]
+        vpxor	ymm10, ymm0, ymm15
+        vpxor	ymm13, ymm3, [rcx+-64]
+        vpxor	ymm12, ymm2, [rcx+-32]
+        vpxor	ymm11, ymm1, [rcx]
+        vpxor	ymm12, ymm12, [rcx+32]
+        vpxor	ymm10, ymm10, [rcx+96]
+        vpxor	ymm14, ymm4, [rcx+128]
+        vpxor	ymm13, ymm13, [r9+-96]
+        vpxor	ymm14, ymm14, [r9+-64]
+        vpxor	ymm12, ymm12, [r9]
+        vpxor	ymm11, ymm11, [r9+32]
+        vpxor	ymm10, ymm10, [r9+64]
+        vpxor	ymm11, ymm11, [r9+96]
+        vpxor	ymm14, ymm14, [r10+-96]
+        vpxor	ymm13, ymm13, [r10+-64]
+        vpxor	ymm12, ymm12, [r10+-32]
+        vpxor	ymm13, ymm13, [r10]
+        vpxor	ymm11, ymm11, [r10+64]
+        vpxor	ymm10, ymm10, [r10+96]
+        vpxor	ymm14, ymm14, [r10+128]
+        ; Calc t[0..4]
+        vpsrlq	ymm0, ymm11, 63
+        vpsrlq	ymm1, ymm12, 63
+        vpsrlq	ymm2, ymm13, 63
+        vpsrlq	ymm3, ymm14, 63
+        vpsrlq	ymm4, ymm10, 63
+        vpaddq	ymm5, ymm11, ymm11
+        vpaddq	ymm6, ymm12, ymm12
+        vpaddq	ymm7, ymm13, ymm13
+        vpaddq	ymm8, ymm14, ymm14
+        vpaddq	ymm9, ymm10, ymm10
+        vpor	ymm5, ymm5, ymm0
+        vpor	ymm6, ymm6, ymm1
+        vpor	ymm7, ymm7, ymm2
+        vpor	ymm8, ymm8, ymm3
+        vpor	ymm9, ymm9, ymm4
+        vpxor	ymm5, ymm5, ymm14
+        vpxor	ymm6, ymm6, ymm10
+        vpxor	ymm7, ymm7, ymm11
+        vpxor	ymm8, ymm8, ymm12
+        vpxor	ymm9, ymm9, ymm13
+        ; Row Mix
+        ; Row 0
+        vpxor	ymm10, ymm5, ymm15
+        vpxor	ymm11, ymm6, [rcx]
+        vpxor	ymm12, ymm7, [rcx+-32]
+        vpxor	ymm13, ymm8, [rcx+-64]
+        vpxor	ymm14, ymm9, [rcx+-96]
+        vpsrlq	ymm0, ymm11, 20
+        vpsrlq	ymm1, ymm12, 21
+        vpsrlq	ymm2, ymm13, 43
+        vpsrlq	ymm3, ymm14, 50
+        vpsllq	ymm11, ymm11, 44
+        vpsllq	ymm12, ymm12, 43
+        vpsllq	ymm13, ymm13, 21
+        vpsllq	ymm14, ymm14, 14
+        vpor	ymm11, ymm11, ymm0
+        vpor	ymm12, ymm12, ymm1
+        vpor	ymm13, ymm13, ymm2
+        vpor	ymm14, ymm14, ymm3
+        vpandn	ymm15, ymm11, ymm12
+        vpandn	ymm1, ymm12, ymm13
+        vpandn	ymm2, ymm13, ymm14
+        vpandn	ymm3, ymm14, ymm10
+        vpandn	ymm4, ymm10, ymm11
+        vpxor	ymm15, ymm15, ymm10
+        vpxor	ymm1, ymm1, ymm11
+        vpxor	ymm2, ymm2, ymm12
+        ; XOR in constant
+        vpxor	ymm15, ymm15, [rax+352]
+        vpxor	ymm3, ymm3, ymm13
+        vpxor	ymm4, ymm4, ymm14
+        vmovdqu	YMMWORD PTR [rcx], ymm1
+        vmovdqu	YMMWORD PTR [rcx+-32], ymm2
+        vmovdqu	YMMWORD PTR [rcx+-64], ymm3
+        vmovdqu	YMMWORD PTR [rcx+-96], ymm4
+        ; Row 1
+        vpxor	ymm10, ymm8, [r10]
+        vpxor	ymm11, ymm9, [r10+128]
+        vpxor	ymm12, ymm5, [r10+96]
+        vpxor	ymm13, ymm6, [r10+64]
+        vpxor	ymm14, ymm7, [r10+32]
+        vpsrlq	ymm0, ymm10, 36
+        vpsrlq	ymm1, ymm11, 44
+        vpsrlq	ymm2, ymm12, 61
+        vpsrlq	ymm3, ymm13, 19
+        vpsrlq	ymm4, ymm14, 3
+        vpsllq	ymm10, ymm10, 28
+        vpsllq	ymm11, ymm11, 20
+        vpsllq	ymm12, ymm12, 3
+        vpsllq	ymm13, ymm13, 45
+        vpsllq	ymm14, ymm14, 61
+        vpor	ymm10, ymm10, ymm0
+        vpor	ymm11, ymm11, ymm1
+        vpor	ymm12, ymm12, ymm2
+        vpor	ymm13, ymm13, ymm3
+        vpor	ymm14, ymm14, ymm4
+        vpandn	ymm0, ymm11, ymm12
+        vpandn	ymm1, ymm12, ymm13
+        vpandn	ymm2, ymm13, ymm14
+        vpandn	ymm3, ymm14, ymm10
+        vpandn	ymm4, ymm10, ymm11
+        vpxor	ymm0, ymm0, ymm10
+        vpxor	ymm1, ymm1, ymm11
+        vpxor	ymm2, ymm2, ymm12
+        vpxor	ymm3, ymm3, ymm13
+        vpxor	ymm4, ymm4, ymm14
+        vmovdqu	YMMWORD PTR [r10], ymm0
+        vmovdqu	YMMWORD PTR [r10+128], ymm1
+        vmovdqu	YMMWORD PTR [r10+96], ymm2
+        vmovdqu	YMMWORD PTR [r10+64], ymm3
+        vmovdqu	YMMWORD PTR [r10+32], ymm4
+        ; Row 2
+        vpxor	ymm10, ymm6, [r9+96]
+        vpxor	ymm11, ymm7, [r10+-32]
+        vpxor	ymm12, ymm8, [r10+-64]
+        vpxor	ymm13, ymm9, [r10+-96]
+        vpxor	ymm14, ymm5, [r9+128]
+        vpsrlq	ymm0, ymm10, 63
+        vpsrlq	ymm1, ymm11, 58
+        vpsrlq	ymm2, ymm12, 39
+        vpsrlq	ymm3, ymm13, 56
+        vpsrlq	ymm4, ymm14, 46
+        vpaddq	ymm10, ymm10, ymm10
+        vpsllq	ymm11, ymm11, 6
+        vpsllq	ymm12, ymm12, 25
+        vpsllq	ymm13, ymm13, 8
+        vpsllq	ymm14, ymm14, 18
+        vpor	ymm10, ymm10, ymm0
+        vpor	ymm11, ymm11, ymm1
+        vpor	ymm12, ymm12, ymm2
+        vpor	ymm13, ymm13, ymm3
+        vpor	ymm14, ymm14, ymm4
+        vpandn	ymm0, ymm11, ymm12
+        vpandn	ymm1, ymm12, ymm13
+        vpandn	ymm2, ymm13, ymm14
+        vpandn	ymm3, ymm14, ymm10
+        vpandn	ymm4, ymm10, ymm11
+        vpxor	ymm0, ymm0, ymm10
+        vpxor	ymm1, ymm1, ymm11
+        vpxor	ymm2, ymm2, ymm12
+        vpxor	ymm3, ymm3, ymm13
+        vpxor	ymm4, ymm4, ymm14
+        vmovdqu	YMMWORD PTR [r9+96], ymm0
+        vmovdqu	YMMWORD PTR [r10+-32], ymm1
+        vmovdqu	YMMWORD PTR [r10+-64], ymm2
+        vmovdqu	YMMWORD PTR [r10+-96], ymm3
+        vmovdqu	YMMWORD PTR [r9+128], ymm4
+        ; Row 3
+        vpxor	ymm10, ymm9, [r9+-64]
+        vpxor	ymm11, ymm5, [r9+64]
+        vpxor	ymm12, ymm6, [r9+32]
+        vpxor	ymm13, ymm7, [r9]
+        vpxor	ymm14, ymm8, [r9+-32]
+        vpsrlq	ymm0, ymm10, 37
+        vpsrlq	ymm1, ymm11, 28
+        vpsrlq	ymm2, ymm12, 54
+        vpsrlq	ymm3, ymm13, 49
+        vpsrlq	ymm4, ymm14, 8
+        vpsllq	ymm10, ymm10, 27
+        vpsllq	ymm11, ymm11, 36
+        vpsllq	ymm12, ymm12, 10
+        vpsllq	ymm13, ymm13, 15
+        vpsllq	ymm14, ymm14, 56
+        vpor	ymm10, ymm10, ymm0
+        vpor	ymm11, ymm11, ymm1
+        vpor	ymm12, ymm12, ymm2
+        vpor	ymm13, ymm13, ymm3
+        vpor	ymm14, ymm14, ymm4
+        vpandn	ymm0, ymm11, ymm12
+        vpandn	ymm1, ymm12, ymm13
+        vpandn	ymm2, ymm13, ymm14
+        vpandn	ymm3, ymm14, ymm10
+        vpandn	ymm4, ymm10, ymm11
+        vpxor	ymm0, ymm0, ymm10
+        vpxor	ymm1, ymm1, ymm11
+        vpxor	ymm2, ymm2, ymm12
+        vpxor	ymm3, ymm3, ymm13
+        vpxor	ymm4, ymm4, ymm14
+        vmovdqu	YMMWORD PTR [r9+-64], ymm0
+        vmovdqu	YMMWORD PTR [r9+64], ymm1
+        vmovdqu	YMMWORD PTR [r9+32], ymm2
+        vmovdqu	YMMWORD PTR [r9], ymm3
+        vmovdqu	YMMWORD PTR [r9+-32], ymm4
+        ; Row 4
+        vpxor	ymm10, ymm7, [rcx+32]
+        vpxor	ymm11, ymm8, [r9+-96]
+        vpxor	ymm12, ymm9, [rcx+128]
+        vpxor	ymm13, ymm5, [rcx+96]
+        vpxor	ymm14, ymm6, [rcx+64]
+        vpsrlq	ymm0, ymm10, 2
+        vpsrlq	ymm1, ymm11, 9
+        vpsrlq	ymm2, ymm12, 25
+        vpsrlq	ymm3, ymm13, 23
+        vpsrlq	ymm4, ymm14, 62
+        vpsllq	ymm10, ymm10, 62
+        vpsllq	ymm11, ymm11, 55
+        vpsllq	ymm12, ymm12, 39
+        vpsllq	ymm13, ymm13, 41
+        vpsllq	ymm14, ymm14, 2
+        vpor	ymm10, ymm10, ymm0
+        vpor	ymm11, ymm11, ymm1
+        vpor	ymm12, ymm12, ymm2
+        vpor	ymm13, ymm13, ymm3
+        vpor	ymm14, ymm14, ymm4
+        vpandn	ymm0, ymm11, ymm12
+        vpandn	ymm1, ymm12, ymm13
+        vpandn	ymm2, ymm13, ymm14
+        vpandn	ymm3, ymm14, ymm10
+        vpandn	ymm4, ymm10, ymm11
+        vpxor	ymm0, ymm0, ymm10
+        vpxor	ymm1, ymm1, ymm11
+        vpxor	ymm2, ymm2, ymm12
+        vpxor	ymm3, ymm3, ymm13
+        vpxor	ymm4, ymm4, ymm14
+        vmovdqu	YMMWORD PTR [rcx+32], ymm0
+        vmovdqu	YMMWORD PTR [r9+-96], ymm1
+        vmovdqu	YMMWORD PTR [rcx+128], ymm2
+        vmovdqu	YMMWORD PTR [rcx+96], ymm3
+        vmovdqu	YMMWORD PTR [rcx+64], ymm4
+        ; Round 12
+        ; Calc b[0..4]
+        vpxor	ymm10, ymm0, ymm15
+        vpxor	ymm14, ymm4, [rcx+-96]
+        vpxor	ymm13, ymm3, [rcx+-64]
+        vpxor	ymm12, ymm2, [rcx+-32]
+        vpxor	ymm11, ymm1, [rcx]
+        vpxor	ymm10, ymm10, [r9+-64]
+        vpxor	ymm14, ymm14, [r9+-32]
+        vpxor	ymm13, ymm13, [r9]
+        vpxor	ymm12, ymm12, [r9+32]
+        vpxor	ymm11, ymm11, [r9+64]
+        vpxor	ymm10, ymm10, [r9+96]
+        vpxor	ymm14, ymm14, [r9+128]
+        vpxor	ymm13, ymm13, [r10+-96]
+        vpxor	ymm12, ymm12, [r10+-64]
+        vpxor	ymm11, ymm11, [r10+-32]
+        vpxor	ymm10, ymm10, [r10]
+        vpxor	ymm14, ymm14, [r10+32]
+        vpxor	ymm13, ymm13, [r10+64]
+        vpxor	ymm12, ymm12, [r10+96]
+        vpxor	ymm11, ymm11, [r10+128]
+        ; Calc t[0..4]
+        vpsrlq	ymm0, ymm11, 63
+        vpsrlq	ymm1, ymm12, 63
+        vpsrlq	ymm2, ymm13, 63
+        vpsrlq	ymm3, ymm14, 63
+        vpsrlq	ymm4, ymm10, 63
+        vpaddq	ymm5, ymm11, ymm11
+        vpaddq	ymm6, ymm12, ymm12
+        vpaddq	ymm7, ymm13, ymm13
+        vpaddq	ymm8, ymm14, ymm14
+        vpaddq	ymm9, ymm10, ymm10
+        vpor	ymm5, ymm5, ymm0
+        vpor	ymm6, ymm6, ymm1
+        vpor	ymm7, ymm7, ymm2
+        vpor	ymm8, ymm8, ymm3
+        vpor	ymm9, ymm9, ymm4
+        vpxor	ymm5, ymm5, ymm14
+        vpxor	ymm6, ymm6, ymm10
+        vpxor	ymm7, ymm7, ymm11
+        vpxor	ymm8, ymm8, ymm12
+        vpxor	ymm9, ymm9, ymm13
+        ; Row Mix
+        ; Row 0
+        vpxor	ymm10, ymm5, ymm15
+        vpxor	ymm11, ymm6, [r10+128]
+        vpxor	ymm12, ymm7, [r10+-64]
+        vpxor	ymm13, ymm8, [r9]
+        vpxor	ymm14, ymm9, [rcx+64]
+        vpsrlq	ymm0, ymm11, 20
+        vpsrlq	ymm1, ymm12, 21
+        vpsrlq	ymm2, ymm13, 43
+        vpsrlq	ymm3, ymm14, 50
+        vpsllq	ymm11, ymm11, 44
+        vpsllq	ymm12, ymm12, 43
+        vpsllq	ymm13, ymm13, 21
+        vpsllq	ymm14, ymm14, 14
+        vpor	ymm11, ymm11, ymm0
+        vpor	ymm12, ymm12, ymm1
+        vpor	ymm13, ymm13, ymm2
+        vpor	ymm14, ymm14, ymm3
+        vpandn	ymm15, ymm11, ymm12
+        vpandn	ymm1, ymm12, ymm13
+        vpandn	ymm2, ymm13, ymm14
+        vpandn	ymm3, ymm14, ymm10
+        vpandn	ymm4, ymm10, ymm11
+        vpxor	ymm15, ymm15, ymm10
+        vpxor	ymm1, ymm1, ymm11
+        vpxor	ymm2, ymm2, ymm12
+        ; XOR in constant
+        vpxor	ymm15, ymm15, [rax+384]
+        vpxor	ymm3, ymm3, ymm13
+        vpxor	ymm4, ymm4, ymm14
+        vmovdqu	YMMWORD PTR [r10+128], ymm1
+        vmovdqu	YMMWORD PTR [r10+-64], ymm2
+        vmovdqu	YMMWORD PTR [r9], ymm3
+        vmovdqu	YMMWORD PTR [rcx+64], ymm4
+        ; Row 1
+        vpxor	ymm10, ymm8, [rcx+-64]
+        vpxor	ymm11, ymm9, [r10+32]
+        vpxor	ymm12, ymm5, [r9+96]
+        vpxor	ymm13, ymm6, [r9+64]
+        vpxor	ymm14, ymm7, [rcx+128]
+        vpsrlq	ymm0, ymm10, 36
+        vpsrlq	ymm1, ymm11, 44
+        vpsrlq	ymm2, ymm12, 61
+        vpsrlq	ymm3, ymm13, 19
+        vpsrlq	ymm4, ymm14, 3
+        vpsllq	ymm10, ymm10, 28
+        vpsllq	ymm11, ymm11, 20
+        vpsllq	ymm12, ymm12, 3
+        vpsllq	ymm13, ymm13, 45
+        vpsllq	ymm14, ymm14, 61
+        vpor	ymm10, ymm10, ymm0
+        vpor	ymm11, ymm11, ymm1
+        vpor	ymm12, ymm12, ymm2
+        vpor	ymm13, ymm13, ymm3
+        vpor	ymm14, ymm14, ymm4
+        vpandn	ymm0, ymm11, ymm12
+        vpandn	ymm1, ymm12, ymm13
+        vpandn	ymm2, ymm13, ymm14
+        vpandn	ymm3, ymm14, ymm10
+        vpandn	ymm4, ymm10, ymm11
+        vpxor	ymm0, ymm0, ymm10
+        vpxor	ymm1, ymm1, ymm11
+        vpxor	ymm2, ymm2, ymm12
+        vpxor	ymm3, ymm3, ymm13
+        vpxor	ymm4, ymm4, ymm14
+        vmovdqu	YMMWORD PTR [rcx+-64], ymm0
+        vmovdqu	YMMWORD PTR [r10+32], ymm1
+        vmovdqu	YMMWORD PTR [r9+96], ymm2
+        vmovdqu	YMMWORD PTR [r9+64], ymm3
+        vmovdqu	YMMWORD PTR [rcx+128], ymm4
+        ; Row 2
+        vpxor	ymm10, ymm6, [rcx]
+        vpxor	ymm11, ymm7, [r10+96]
+        vpxor	ymm12, ymm8, [r10+-96]
+        vpxor	ymm13, ymm9, [r9+-32]
+        vpxor	ymm14, ymm5, [rcx+32]
+        vpsrlq	ymm0, ymm10, 63
+        vpsrlq	ymm1, ymm11, 58
+        vpsrlq	ymm2, ymm12, 39
+        vpsrlq	ymm3, ymm13, 56
+        vpsrlq	ymm4, ymm14, 46
+        vpaddq	ymm10, ymm10, ymm10
+        vpsllq	ymm11, ymm11, 6
+        vpsllq	ymm12, ymm12, 25
+        vpsllq	ymm13, ymm13, 8
+        vpsllq	ymm14, ymm14, 18
+        vpor	ymm10, ymm10, ymm0
+        vpor	ymm11, ymm11, ymm1
+        vpor	ymm12, ymm12, ymm2
+        vpor	ymm13, ymm13, ymm3
+        vpor	ymm14, ymm14, ymm4
+        vpandn	ymm0, ymm11, ymm12
+        vpandn	ymm1, ymm12, ymm13
+        vpandn	ymm2, ymm13, ymm14
+        vpandn	ymm3, ymm14, ymm10
+        vpandn	ymm4, ymm10, ymm11
+        vpxor	ymm0, ymm0, ymm10
+        vpxor	ymm1, ymm1, ymm11
+        vpxor	ymm2, ymm2, ymm12
+        vpxor	ymm3, ymm3, ymm13
+        vpxor	ymm4, ymm4, ymm14
+        vmovdqu	YMMWORD PTR [rcx], ymm0
+        vmovdqu	YMMWORD PTR [r10+96], ymm1
+        vmovdqu	YMMWORD PTR [r10+-96], ymm2
+        vmovdqu	YMMWORD PTR [r9+-32], ymm3
+        vmovdqu	YMMWORD PTR [rcx+32], ymm4
+        ; Row 3
+        vpxor	ymm10, ymm9, [rcx+-96]
+        vpxor	ymm11, ymm5, [r10]
+        vpxor	ymm12, ymm6, [r10+-32]
+        vpxor	ymm13, ymm7, [r9+32]
+        vpxor	ymm14, ymm8, [rcx+96]
+        vpsrlq	ymm0, ymm10, 37
+        vpsrlq	ymm1, ymm11, 28
+        vpsrlq	ymm2, ymm12, 54
+        vpsrlq	ymm3, ymm13, 49
+        vpsrlq	ymm4, ymm14, 8
+        vpsllq	ymm10, ymm10, 27
+        vpsllq	ymm11, ymm11, 36
+        vpsllq	ymm12, ymm12, 10
+        vpsllq	ymm13, ymm13, 15
+        vpsllq	ymm14, ymm14, 56
+        vpor	ymm10, ymm10, ymm0
+        vpor	ymm11, ymm11, ymm1
+        vpor	ymm12, ymm12, ymm2
+        vpor	ymm13, ymm13, ymm3
+        vpor	ymm14, ymm14, ymm4
+        vpandn	ymm0, ymm11, ymm12
+        vpandn	ymm1, ymm12, ymm13
+        vpandn	ymm2, ymm13, ymm14
+        vpandn	ymm3, ymm14, ymm10
+        vpandn	ymm4, ymm10, ymm11
+        vpxor	ymm0, ymm0, ymm10
+        vpxor	ymm1, ymm1, ymm11
+        vpxor	ymm2, ymm2, ymm12
+        vpxor	ymm3, ymm3, ymm13
+        vpxor	ymm4, ymm4, ymm14
+        vmovdqu	YMMWORD PTR [rcx+-96], ymm0
+        vmovdqu	YMMWORD PTR [r10], ymm1
+        vmovdqu	YMMWORD PTR [r10+-32], ymm2
+        vmovdqu	YMMWORD PTR [r9+32], ymm3
+        vmovdqu	YMMWORD PTR [rcx+96], ymm4
+        ; Row 4
+        vpxor	ymm10, ymm7, [rcx+-32]
+        vpxor	ymm11, ymm8, [r10+64]
+        vpxor	ymm12, ymm9, [r9+128]
+        vpxor	ymm13, ymm5, [r9+-64]
+        vpxor	ymm14, ymm6, [r9+-96]
+        vpsrlq	ymm0, ymm10, 2
+        vpsrlq	ymm1, ymm11, 9
+        vpsrlq	ymm2, ymm12, 25
+        vpsrlq	ymm3, ymm13, 23
+        vpsrlq	ymm4, ymm14, 62
+        vpsllq	ymm10, ymm10, 62
+        vpsllq	ymm11, ymm11, 55
+        vpsllq	ymm12, ymm12, 39
+        vpsllq	ymm13, ymm13, 41
+        vpsllq	ymm14, ymm14, 2
+        vpor	ymm10, ymm10, ymm0
+        vpor	ymm11, ymm11, ymm1
+        vpor	ymm12, ymm12, ymm2
+        vpor	ymm13, ymm13, ymm3
+        vpor	ymm14, ymm14, ymm4
+        vpandn	ymm0, ymm11, ymm12
+        vpandn	ymm1, ymm12, ymm13
+        vpandn	ymm2, ymm13, ymm14
+        vpandn	ymm3, ymm14, ymm10
+        vpandn	ymm4, ymm10, ymm11
+        vpxor	ymm0, ymm0, ymm10
+        vpxor	ymm1, ymm1, ymm11
+        vpxor	ymm2, ymm2, ymm12
+        vpxor	ymm3, ymm3, ymm13
+        vpxor	ymm4, ymm4, ymm14
+        vmovdqu	YMMWORD PTR [rcx+-32], ymm0
+        vmovdqu	YMMWORD PTR [r10+64], ymm1
+        vmovdqu	YMMWORD PTR [r9+128], ymm2
+        vmovdqu	YMMWORD PTR [r9+-64], ymm3
+        vmovdqu	YMMWORD PTR [r9+-96], ymm4
+        ; Round 13
+        ; Calc b[0..4]
+        vpxor	ymm10, ymm0, ymm15
+        vpxor	ymm10, ymm10, [rcx+-96]
+        vpxor	ymm10, ymm10, [rcx+-64]
+        vpxor	ymm10, ymm10, [rcx]
+        vpxor	ymm14, ymm4, [rcx+32]
+        vpxor	ymm14, ymm14, [rcx+64]
+        vpxor	ymm14, ymm14, [rcx+96]
+        vpxor	ymm14, ymm14, [rcx+128]
+        vpxor	ymm13, ymm3, [r9+-32]
+        vpxor	ymm13, ymm13, [r9]
+        vpxor	ymm13, ymm13, [r9+32]
+        vpxor	ymm13, ymm13, [r9+64]
+        vpxor	ymm12, ymm2, [r9+96]
+        vpxor	ymm12, ymm12, [r10+-96]
+        vpxor	ymm12, ymm12, [r10+-64]
+        vpxor	ymm12, ymm12, [r10+-32]
+        vpxor	ymm11, ymm1, [r10]
+        vpxor	ymm11, ymm11, [r10+32]
+        vpxor	ymm11, ymm11, [r10+96]
+        vpxor	ymm11, ymm11, [r10+128]
+        ; Calc t[0..4]
+        vpsrlq	ymm0, ymm11, 63
+        vpsrlq	ymm1, ymm12, 63
+        vpsrlq	ymm2, ymm13, 63
+        vpsrlq	ymm3, ymm14, 63
+        vpsrlq	ymm4, ymm10, 63
+        vpaddq	ymm5, ymm11, ymm11
+        vpaddq	ymm6, ymm12, ymm12
+        vpaddq	ymm7, ymm13, ymm13
+        vpaddq	ymm8, ymm14, ymm14
+        vpaddq	ymm9, ymm10, ymm10
+        vpor	ymm5, ymm5, ymm0
+        vpor	ymm6, ymm6, ymm1
+        vpor	ymm7, ymm7, ymm2
+        vpor	ymm8, ymm8, ymm3
+        vpor	ymm9, ymm9, ymm4
+        vpxor	ymm5, ymm5, ymm14
+        vpxor	ymm6, ymm6, ymm10
+        vpxor	ymm7, ymm7, ymm11
+        vpxor	ymm8, ymm8, ymm12
+        vpxor	ymm9, ymm9, ymm13
+        ; Row Mix
+        ; Row 0
+        vpxor	ymm10, ymm5, ymm15
+        vpxor	ymm11, ymm6, [r10+32]
+        vpxor	ymm12, ymm7, [r10+-96]
+        vpxor	ymm13, ymm8, [r9+32]
+        vpxor	ymm14, ymm9, [r9+-96]
+        vpsrlq	ymm0, ymm11, 20
+        vpsrlq	ymm1, ymm12, 21
+        vpsrlq	ymm2, ymm13, 43
+        vpsrlq	ymm3, ymm14, 50
+        vpsllq	ymm11, ymm11, 44
+        vpsllq	ymm12, ymm12, 43
+        vpsllq	ymm13, ymm13, 21
+        vpsllq	ymm14, ymm14, 14
+        vpor	ymm11, ymm11, ymm0
+        vpor	ymm12, ymm12, ymm1
+        vpor	ymm13, ymm13, ymm2
+        vpor	ymm14, ymm14, ymm3
+        vpandn	ymm15, ymm11, ymm12
+        vpandn	ymm1, ymm12, ymm13
+        vpandn	ymm2, ymm13, ymm14
+        vpandn	ymm3, ymm14, ymm10
+        vpandn	ymm4, ymm10, ymm11
+        vpxor	ymm15, ymm15, ymm10
+        vpxor	ymm1, ymm1, ymm11
+        vpxor	ymm2, ymm2, ymm12
+        ; XOR in constant
+        vpxor	ymm15, ymm15, [rax+416]
+        vpxor	ymm3, ymm3, ymm13
+        vpxor	ymm4, ymm4, ymm14
+        vmovdqu	YMMWORD PTR [r10+32], ymm1
+        vmovdqu	YMMWORD PTR [r10+-96], ymm2
+        vmovdqu	YMMWORD PTR [r9+32], ymm3
+        vmovdqu	YMMWORD PTR [r9+-96], ymm4
+        ; Row 1
+        vpxor	ymm10, ymm8, [r9]
+        vpxor	ymm11, ymm9, [rcx+128]
+        vpxor	ymm12, ymm5, [rcx]
+        vpxor	ymm13, ymm6, [r10]
+        vpxor	ymm14, ymm7, [r9+128]
+        vpsrlq	ymm0, ymm10, 36
+        vpsrlq	ymm1, ymm11, 44
+        vpsrlq	ymm2, ymm12, 61
+        vpsrlq	ymm3, ymm13, 19
+        vpsrlq	ymm4, ymm14, 3
+        vpsllq	ymm10, ymm10, 28
+        vpsllq	ymm11, ymm11, 20
+        vpsllq	ymm12, ymm12, 3
+        vpsllq	ymm13, ymm13, 45
+        vpsllq	ymm14, ymm14, 61
+        vpor	ymm10, ymm10, ymm0
+        vpor	ymm11, ymm11, ymm1
+        vpor	ymm12, ymm12, ymm2
+        vpor	ymm13, ymm13, ymm3
+        vpor	ymm14, ymm14, ymm4
+        vpandn	ymm0, ymm11, ymm12
+        vpandn	ymm1, ymm12, ymm13
+        vpandn	ymm2, ymm13, ymm14
+        vpandn	ymm3, ymm14, ymm10
+        vpandn	ymm4, ymm10, ymm11
+        vpxor	ymm0, ymm0, ymm10
+        vpxor	ymm1, ymm1, ymm11
+        vpxor	ymm2, ymm2, ymm12
+        vpxor	ymm3, ymm3, ymm13
+        vpxor	ymm4, ymm4, ymm14
+        vmovdqu	YMMWORD PTR [r9], ymm0
+        vmovdqu	YMMWORD PTR [rcx+128], ymm1
+        vmovdqu	YMMWORD PTR [rcx], ymm2
+        vmovdqu	YMMWORD PTR [r10], ymm3
+        vmovdqu	YMMWORD PTR [r9+128], ymm4
+        ; Row 2
+        vpxor	ymm10, ymm6, [r10+128]
+        vpxor	ymm11, ymm7, [r9+96]
+        vpxor	ymm12, ymm8, [r9+-32]
+        vpxor	ymm13, ymm9, [rcx+96]
+        vpxor	ymm14, ymm5, [rcx+-32]
+        vpsrlq	ymm0, ymm10, 63
+        vpsrlq	ymm1, ymm11, 58
+        vpsrlq	ymm2, ymm12, 39
+        vpsrlq	ymm3, ymm13, 56
+        vpsrlq	ymm4, ymm14, 46
+        vpaddq	ymm10, ymm10, ymm10
+        vpsllq	ymm11, ymm11, 6
+        vpsllq	ymm12, ymm12, 25
+        vpsllq	ymm13, ymm13, 8
+        vpsllq	ymm14, ymm14, 18
+        vpor	ymm10, ymm10, ymm0
+        vpor	ymm11, ymm11, ymm1
+        vpor	ymm12, ymm12, ymm2
+        vpor	ymm13, ymm13, ymm3
+        vpor	ymm14, ymm14, ymm4
+        vpandn	ymm0, ymm11, ymm12
+        vpandn	ymm1, ymm12, ymm13
+        vpandn	ymm2, ymm13, ymm14
+        vpandn	ymm3, ymm14, ymm10
+        vpandn	ymm4, ymm10, ymm11
+        vpxor	ymm0, ymm0, ymm10
+        vpxor	ymm1, ymm1, ymm11
+        vpxor	ymm2, ymm2, ymm12
+        vpxor	ymm3, ymm3, ymm13
+        vpxor	ymm4, ymm4, ymm14
+        vmovdqu	YMMWORD PTR [r10+128], ymm0
+        vmovdqu	YMMWORD PTR [r9+96], ymm1
+        vmovdqu	YMMWORD PTR [r9+-32], ymm2
+        vmovdqu	YMMWORD PTR [rcx+96], ymm3
+        vmovdqu	YMMWORD PTR [rcx+-32], ymm4
+        ; Row 3
+        vpxor	ymm10, ymm9, [rcx+64]
+        vpxor	ymm11, ymm5, [rcx+-64]
+        vpxor	ymm12, ymm6, [r10+96]
+        vpxor	ymm13, ymm7, [r10+-32]
+        vpxor	ymm14, ymm8, [r9+-64]
+        vpsrlq	ymm0, ymm10, 37
+        vpsrlq	ymm1, ymm11, 28
+        vpsrlq	ymm2, ymm12, 54
+        vpsrlq	ymm3, ymm13, 49
+        vpsrlq	ymm4, ymm14, 8
+        vpsllq	ymm10, ymm10, 27
+        vpsllq	ymm11, ymm11, 36
+        vpsllq	ymm12, ymm12, 10
+        vpsllq	ymm13, ymm13, 15
+        vpsllq	ymm14, ymm14, 56
+        vpor	ymm10, ymm10, ymm0
+        vpor	ymm11, ymm11, ymm1
+        vpor	ymm12, ymm12, ymm2
+        vpor	ymm13, ymm13, ymm3
+        vpor	ymm14, ymm14, ymm4
+        vpandn	ymm0, ymm11, ymm12
+        vpandn	ymm1, ymm12, ymm13
+        vpandn	ymm2, ymm13, ymm14
+        vpandn	ymm3, ymm14, ymm10
+        vpandn	ymm4, ymm10, ymm11
+        vpxor	ymm0, ymm0, ymm10
+        vpxor	ymm1, ymm1, ymm11
+        vpxor	ymm2, ymm2, ymm12
+        vpxor	ymm3, ymm3, ymm13
+        vpxor	ymm4, ymm4, ymm14
+        vmovdqu	YMMWORD PTR [rcx+64], ymm0
+        vmovdqu	YMMWORD PTR [rcx+-64], ymm1
+        vmovdqu	YMMWORD PTR [r10+96], ymm2
+        vmovdqu	YMMWORD PTR [r10+-32], ymm3
+        vmovdqu	YMMWORD PTR [r9+-64], ymm4
+        ; Row 4
+        vpxor	ymm10, ymm7, [r10+-64]
+        vpxor	ymm11, ymm8, [r9+64]
+        vpxor	ymm12, ymm9, [rcx+32]
+        vpxor	ymm13, ymm5, [rcx+-96]
+        vpxor	ymm14, ymm6, [r10+64]
+        vpsrlq	ymm0, ymm10, 2
+        vpsrlq	ymm1, ymm11, 9
+        vpsrlq	ymm2, ymm12, 25
+        vpsrlq	ymm3, ymm13, 23
+        vpsrlq	ymm4, ymm14, 62
+        vpsllq	ymm10, ymm10, 62
+        vpsllq	ymm11, ymm11, 55
+        vpsllq	ymm12, ymm12, 39
+        vpsllq	ymm13, ymm13, 41
+        vpsllq	ymm14, ymm14, 2
+        vpor	ymm10, ymm10, ymm0
+        vpor	ymm11, ymm11, ymm1
+        vpor	ymm12, ymm12, ymm2
+        vpor	ymm13, ymm13, ymm3
+        vpor	ymm14, ymm14, ymm4
+        vpandn	ymm0, ymm11, ymm12
+        vpandn	ymm1, ymm12, ymm13
+        vpandn	ymm2, ymm13, ymm14
+        vpandn	ymm3, ymm14, ymm10
+        vpandn	ymm4, ymm10, ymm11
+        vpxor	ymm0, ymm0, ymm10
+        vpxor	ymm1, ymm1, ymm11
+        vpxor	ymm2, ymm2, ymm12
+        vpxor	ymm3, ymm3, ymm13
+        vpxor	ymm4, ymm4, ymm14
+        vmovdqu	YMMWORD PTR [r10+-64], ymm0
+        vmovdqu	YMMWORD PTR [r9+64], ymm1
+        vmovdqu	YMMWORD PTR [rcx+32], ymm2
+        vmovdqu	YMMWORD PTR [rcx+-96], ymm3
+        vmovdqu	YMMWORD PTR [r10+64], ymm4
+        ; Round 14
+        ; Calc b[0..4]
+        vpxor	ymm10, ymm0, ymm15
+        vpxor	ymm11, ymm1, [rcx+-64]
+        vpxor	ymm14, ymm4, [rcx+-32]
+        vpxor	ymm12, ymm2, [rcx]
+        vpxor	ymm10, ymm10, [rcx+64]
+        vpxor	ymm13, ymm3, [rcx+96]
+        vpxor	ymm11, ymm11, [rcx+128]
+        vpxor	ymm14, ymm14, [r9+-96]
+        vpxor	ymm14, ymm14, [r9+-64]
+        vpxor	ymm12, ymm12, [r9+-32]
+        vpxor	ymm10, ymm10, [r9]
+        vpxor	ymm13, ymm13, [r9+32]
+        vpxor	ymm11, ymm11, [r9+96]
+        vpxor	ymm14, ymm14, [r9+128]
+        vpxor	ymm12, ymm12, [r10+-96]
+        vpxor	ymm13, ymm13, [r10+-32]
+        vpxor	ymm13, ymm13, [r10]
+        vpxor	ymm11, ymm11, [r10+32]
+        vpxor	ymm12, ymm12, [r10+96]
+        vpxor	ymm10, ymm10, [r10+128]
+        ; Calc t[0..4]
+        vpsrlq	ymm0, ymm11, 63
+        vpsrlq	ymm1, ymm12, 63
+        vpsrlq	ymm2, ymm13, 63
+        vpsrlq	ymm3, ymm14, 63
+        vpsrlq	ymm4, ymm10, 63
+        vpaddq	ymm5, ymm11, ymm11
+        vpaddq	ymm6, ymm12, ymm12
+        vpaddq	ymm7, ymm13, ymm13
+        vpaddq	ymm8, ymm14, ymm14
+        vpaddq	ymm9, ymm10, ymm10
+        vpor	ymm5, ymm5, ymm0
+        vpor	ymm6, ymm6, ymm1
+        vpor	ymm7, ymm7, ymm2
+        vpor	ymm8, ymm8, ymm3
+        vpor	ymm9, ymm9, ymm4
+        vpxor	ymm5, ymm5, ymm14
+        vpxor	ymm6, ymm6, ymm10
+        vpxor	ymm7, ymm7, ymm11
+        vpxor	ymm8, ymm8, ymm12
+        vpxor	ymm9, ymm9, ymm13
+        ; Row Mix
+        ; Row 0
+        vpxor	ymm10, ymm5, ymm15
+        vpxor	ymm11, ymm6, [rcx+128]
+        vpxor	ymm12, ymm7, [r9+-32]
+        vpxor	ymm13, ymm8, [r10+-32]
+        vpxor	ymm14, ymm9, [r10+64]
+        vpsrlq	ymm0, ymm11, 20
+        vpsrlq	ymm1, ymm12, 21
+        vpsrlq	ymm2, ymm13, 43
+        vpsrlq	ymm3, ymm14, 50
+        vpsllq	ymm11, ymm11, 44
+        vpsllq	ymm12, ymm12, 43
+        vpsllq	ymm13, ymm13, 21
+        vpsllq	ymm14, ymm14, 14
+        vpor	ymm11, ymm11, ymm0
+        vpor	ymm12, ymm12, ymm1
+        vpor	ymm13, ymm13, ymm2
+        vpor	ymm14, ymm14, ymm3
+        vpandn	ymm15, ymm11, ymm12
+        vpandn	ymm1, ymm12, ymm13
+        vpandn	ymm2, ymm13, ymm14
+        vpandn	ymm3, ymm14, ymm10
+        vpandn	ymm4, ymm10, ymm11
+        vpxor	ymm15, ymm15, ymm10
+        vpxor	ymm1, ymm1, ymm11
+        vpxor	ymm2, ymm2, ymm12
+        ; XOR in constant
+        vpxor	ymm15, ymm15, [rax+448]
+        vpxor	ymm3, ymm3, ymm13
+        vpxor	ymm4, ymm4, ymm14
+        vmovdqu	YMMWORD PTR [rcx+128], ymm1
+        vmovdqu	YMMWORD PTR [r9+-32], ymm2
+        vmovdqu	YMMWORD PTR [r10+-32], ymm3
+        vmovdqu	YMMWORD PTR [r10+64], ymm4
+        ; Row 1
+        vpxor	ymm10, ymm8, [r9+32]
+        vpxor	ymm11, ymm9, [r9+128]
+        vpxor	ymm12, ymm5, [r10+128]
+        vpxor	ymm13, ymm6, [rcx+-64]
+        vpxor	ymm14, ymm7, [rcx+32]
+        vpsrlq	ymm0, ymm10, 36
+        vpsrlq	ymm1, ymm11, 44
+        vpsrlq	ymm2, ymm12, 61
+        vpsrlq	ymm3, ymm13, 19
+        vpsrlq	ymm4, ymm14, 3
+        vpsllq	ymm10, ymm10, 28
+        vpsllq	ymm11, ymm11, 20
+        vpsllq	ymm12, ymm12, 3
+        vpsllq	ymm13, ymm13, 45
+        vpsllq	ymm14, ymm14, 61
+        vpor	ymm10, ymm10, ymm0
+        vpor	ymm11, ymm11, ymm1
+        vpor	ymm12, ymm12, ymm2
+        vpor	ymm13, ymm13, ymm3
+        vpor	ymm14, ymm14, ymm4
+        vpandn	ymm0, ymm11, ymm12
+        vpandn	ymm1, ymm12, ymm13
+        vpandn	ymm2, ymm13, ymm14
+        vpandn	ymm3, ymm14, ymm10
+        vpandn	ymm4, ymm10, ymm11
+        vpxor	ymm0, ymm0, ymm10
+        vpxor	ymm1, ymm1, ymm11
+        vpxor	ymm2, ymm2, ymm12
+        vpxor	ymm3, ymm3, ymm13
+        vpxor	ymm4, ymm4, ymm14
+        vmovdqu	YMMWORD PTR [r9+32], ymm0
+        vmovdqu	YMMWORD PTR [r9+128], ymm1
+        vmovdqu	YMMWORD PTR [r10+128], ymm2
+        vmovdqu	YMMWORD PTR [rcx+-64], ymm3
+        vmovdqu	YMMWORD PTR [rcx+32], ymm4
+        ; Row 2
+        vpxor	ymm10, ymm6, [r10+32]
+        vpxor	ymm11, ymm7, [rcx]
+        vpxor	ymm12, ymm8, [rcx+96]
+        vpxor	ymm13, ymm9, [r9+-64]
+        vpxor	ymm14, ymm5, [r10+-64]
+        vpsrlq	ymm0, ymm10, 63
+        vpsrlq	ymm1, ymm11, 58
+        vpsrlq	ymm2, ymm12, 39
+        vpsrlq	ymm3, ymm13, 56
+        vpsrlq	ymm4, ymm14, 46
+        vpaddq	ymm10, ymm10, ymm10
+        vpsllq	ymm11, ymm11, 6
+        vpsllq	ymm12, ymm12, 25
+        vpsllq	ymm13, ymm13, 8
+        vpsllq	ymm14, ymm14, 18
+        vpor	ymm10, ymm10, ymm0
+        vpor	ymm11, ymm11, ymm1
+        vpor	ymm12, ymm12, ymm2
+        vpor	ymm13, ymm13, ymm3
+        vpor	ymm14, ymm14, ymm4
+        vpandn	ymm0, ymm11, ymm12
+        vpandn	ymm1, ymm12, ymm13
+        vpandn	ymm2, ymm13, ymm14
+        vpandn	ymm3, ymm14, ymm10
+        vpandn	ymm4, ymm10, ymm11
+        vpxor	ymm0, ymm0, ymm10
+        vpxor	ymm1, ymm1, ymm11
+        vpxor	ymm2, ymm2, ymm12
+        vpxor	ymm3, ymm3, ymm13
+        vpxor	ymm4, ymm4, ymm14
+        vmovdqu	YMMWORD PTR [r10+32], ymm0
+        vmovdqu	YMMWORD PTR [rcx], ymm1
+        vmovdqu	YMMWORD PTR [rcx+96], ymm2
+        vmovdqu	YMMWORD PTR [r9+-64], ymm3
+        vmovdqu	YMMWORD PTR [r10+-64], ymm4
+        ; Row 3
+        vpxor	ymm10, ymm9, [r9+-96]
+        vpxor	ymm11, ymm5, [r9]
+        vpxor	ymm12, ymm6, [r9+96]
+        vpxor	ymm13, ymm7, [r10+96]
+        vpxor	ymm14, ymm8, [rcx+-96]
+        vpsrlq	ymm0, ymm10, 37
+        vpsrlq	ymm1, ymm11, 28
+        vpsrlq	ymm2, ymm12, 54
+        vpsrlq	ymm3, ymm13, 49
+        vpsrlq	ymm4, ymm14, 8
+        vpsllq	ymm10, ymm10, 27
+        vpsllq	ymm11, ymm11, 36
+        vpsllq	ymm12, ymm12, 10
+        vpsllq	ymm13, ymm13, 15
+        vpsllq	ymm14, ymm14, 56
+        vpor	ymm10, ymm10, ymm0
+        vpor	ymm11, ymm11, ymm1
+        vpor	ymm12, ymm12, ymm2
+        vpor	ymm13, ymm13, ymm3
+        vpor	ymm14, ymm14, ymm4
+        vpandn	ymm0, ymm11, ymm12
+        vpandn	ymm1, ymm12, ymm13
+        vpandn	ymm2, ymm13, ymm14
+        vpandn	ymm3, ymm14, ymm10
+        vpandn	ymm4, ymm10, ymm11
+        vpxor	ymm0, ymm0, ymm10
+        vpxor	ymm1, ymm1, ymm11
+        vpxor	ymm2, ymm2, ymm12
+        vpxor	ymm3, ymm3, ymm13
+        vpxor	ymm4, ymm4, ymm14
+        vmovdqu	YMMWORD PTR [r9+-96], ymm0
+        vmovdqu	YMMWORD PTR [r9], ymm1
+        vmovdqu	YMMWORD PTR [r9+96], ymm2
+        vmovdqu	YMMWORD PTR [r10+96], ymm3
+        vmovdqu	YMMWORD PTR [rcx+-96], ymm4
+        ; Row 4
+        vpxor	ymm10, ymm7, [r10+-96]
+        vpxor	ymm11, ymm8, [r10]
+        vpxor	ymm12, ymm9, [rcx+-32]
+        vpxor	ymm13, ymm5, [rcx+64]
+        vpxor	ymm14, ymm6, [r9+64]
+        vpsrlq	ymm0, ymm10, 2
+        vpsrlq	ymm1, ymm11, 9
+        vpsrlq	ymm2, ymm12, 25
+        vpsrlq	ymm3, ymm13, 23
+        vpsrlq	ymm4, ymm14, 62
+        vpsllq	ymm10, ymm10, 62
+        vpsllq	ymm11, ymm11, 55
+        vpsllq	ymm12, ymm12, 39
+        vpsllq	ymm13, ymm13, 41
+        vpsllq	ymm14, ymm14, 2
+        vpor	ymm10, ymm10, ymm0
+        vpor	ymm11, ymm11, ymm1
+        vpor	ymm12, ymm12, ymm2
+        vpor	ymm13, ymm13, ymm3
+        vpor	ymm14, ymm14, ymm4
+        vpandn	ymm0, ymm11, ymm12
+        vpandn	ymm1, ymm12, ymm13
+        vpandn	ymm2, ymm13, ymm14
+        vpandn	ymm3, ymm14, ymm10
+        vpandn	ymm4, ymm10, ymm11
+        vpxor	ymm0, ymm0, ymm10
+        vpxor	ymm1, ymm1, ymm11
+        vpxor	ymm2, ymm2, ymm12
+        vpxor	ymm3, ymm3, ymm13
+        vpxor	ymm4, ymm4, ymm14
+        vmovdqu	YMMWORD PTR [r10+-96], ymm0
+        vmovdqu	YMMWORD PTR [r10], ymm1
+        vmovdqu	YMMWORD PTR [rcx+-32], ymm2
+        vmovdqu	YMMWORD PTR [rcx+64], ymm3
+        vmovdqu	YMMWORD PTR [r9+64], ymm4
+        ; Round 15
+        ; Calc b[0..4]
+        vpxor	ymm10, ymm0, ymm15
+        vpxor	ymm14, ymm4, [rcx+-96]
+        vpxor	ymm13, ymm3, [rcx+-64]
+        vpxor	ymm11, ymm1, [rcx]
+        vpxor	ymm14, ymm14, [rcx+32]
+        vpxor	ymm12, ymm2, [rcx+96]
+        vpxor	ymm11, ymm11, [rcx+128]
+        vpxor	ymm10, ymm10, [r9+-96]
+        vpxor	ymm13, ymm13, [r9+-64]
+        vpxor	ymm12, ymm12, [r9+-32]
+        vpxor	ymm11, ymm11, [r9]
+        vpxor	ymm10, ymm10, [r9+32]
+        vpxor	ymm12, ymm12, [r9+96]
+        vpxor	ymm11, ymm11, [r9+128]
+        vpxor	ymm14, ymm14, [r10+-64]
+        vpxor	ymm13, ymm13, [r10+-32]
+        vpxor	ymm10, ymm10, [r10+32]
+        vpxor	ymm14, ymm14, [r10+64]
+        vpxor	ymm13, ymm13, [r10+96]
+        vpxor	ymm12, ymm12, [r10+128]
+        ; Calc t[0..4]
+        vpsrlq	ymm0, ymm11, 63
+        vpsrlq	ymm1, ymm12, 63
+        vpsrlq	ymm2, ymm13, 63
+        vpsrlq	ymm3, ymm14, 63
+        vpsrlq	ymm4, ymm10, 63
+        vpaddq	ymm5, ymm11, ymm11
+        vpaddq	ymm6, ymm12, ymm12
+        vpaddq	ymm7, ymm13, ymm13
+        vpaddq	ymm8, ymm14, ymm14
+        vpaddq	ymm9, ymm10, ymm10
+        vpor	ymm5, ymm5, ymm0
+        vpor	ymm6, ymm6, ymm1
+        vpor	ymm7, ymm7, ymm2
+        vpor	ymm8, ymm8, ymm3
+        vpor	ymm9, ymm9, ymm4
+        vpxor	ymm5, ymm5, ymm14
+        vpxor	ymm6, ymm6, ymm10
+        vpxor	ymm7, ymm7, ymm11
+        vpxor	ymm8, ymm8, ymm12
+        vpxor	ymm9, ymm9, ymm13
+        ; Row Mix
+        ; Row 0
+        vpxor	ymm10, ymm5, ymm15
+        vpxor	ymm11, ymm6, [r9+128]
+        vpxor	ymm12, ymm7, [rcx+96]
+        vpxor	ymm13, ymm8, [r10+96]
+        vpxor	ymm14, ymm9, [r9+64]
+        vpsrlq	ymm0, ymm11, 20
+        vpsrlq	ymm1, ymm12, 21
+        vpsrlq	ymm2, ymm13, 43
+        vpsrlq	ymm3, ymm14, 50
+        vpsllq	ymm11, ymm11, 44
+        vpsllq	ymm12, ymm12, 43
+        vpsllq	ymm13, ymm13, 21
+        vpsllq	ymm14, ymm14, 14
+        vpor	ymm11, ymm11, ymm0
+        vpor	ymm12, ymm12, ymm1
+        vpor	ymm13, ymm13, ymm2
+        vpor	ymm14, ymm14, ymm3
+        vpandn	ymm15, ymm11, ymm12
+        vpandn	ymm1, ymm12, ymm13
+        vpandn	ymm2, ymm13, ymm14
+        vpandn	ymm3, ymm14, ymm10
+        vpandn	ymm4, ymm10, ymm11
+        vpxor	ymm15, ymm15, ymm10
+        vpxor	ymm1, ymm1, ymm11
+        vpxor	ymm2, ymm2, ymm12
+        ; XOR in constant
+        vpxor	ymm15, ymm15, [rax+480]
+        vpxor	ymm3, ymm3, ymm13
+        vpxor	ymm4, ymm4, ymm14
+        vmovdqu	YMMWORD PTR [r9+128], ymm1
+        vmovdqu	YMMWORD PTR [rcx+96], ymm2
+        vmovdqu	YMMWORD PTR [r10+96], ymm3
+        vmovdqu	YMMWORD PTR [r9+64], ymm4
+        ; Row 1
+        vpxor	ymm10, ymm8, [r10+-32]
+        vpxor	ymm11, ymm9, [rcx+32]
+        vpxor	ymm12, ymm5, [r10+32]
+        vpxor	ymm13, ymm6, [r9]
+        vpxor	ymm14, ymm7, [rcx+-32]
+        vpsrlq	ymm0, ymm10, 36
+        vpsrlq	ymm1, ymm11, 44
+        vpsrlq	ymm2, ymm12, 61
+        vpsrlq	ymm3, ymm13, 19
+        vpsrlq	ymm4, ymm14, 3
+        vpsllq	ymm10, ymm10, 28
+        vpsllq	ymm11, ymm11, 20
+        vpsllq	ymm12, ymm12, 3
+        vpsllq	ymm13, ymm13, 45
+        vpsllq	ymm14, ymm14, 61
+        vpor	ymm10, ymm10, ymm0
+        vpor	ymm11, ymm11, ymm1
+        vpor	ymm12, ymm12, ymm2
+        vpor	ymm13, ymm13, ymm3
+        vpor	ymm14, ymm14, ymm4
+        vpandn	ymm0, ymm11, ymm12
+        vpandn	ymm1, ymm12, ymm13
+        vpandn	ymm2, ymm13, ymm14
+        vpandn	ymm3, ymm14, ymm10
+        vpandn	ymm4, ymm10, ymm11
+        vpxor	ymm0, ymm0, ymm10
+        vpxor	ymm1, ymm1, ymm11
+        vpxor	ymm2, ymm2, ymm12
+        vpxor	ymm3, ymm3, ymm13
+        vpxor	ymm4, ymm4, ymm14
+        vmovdqu	YMMWORD PTR [r10+-32], ymm0
+        vmovdqu	YMMWORD PTR [rcx+32], ymm1
+        vmovdqu	YMMWORD PTR [r10+32], ymm2
+        vmovdqu	YMMWORD PTR [r9], ymm3
+        vmovdqu	YMMWORD PTR [rcx+-32], ymm4
+        ; Row 2
+        vpxor	ymm10, ymm6, [rcx+128]
+        vpxor	ymm11, ymm7, [r10+128]
+        vpxor	ymm12, ymm8, [r9+-64]
+        vpxor	ymm13, ymm9, [rcx+-96]
+        vpxor	ymm14, ymm5, [r10+-96]
+        vpsrlq	ymm0, ymm10, 63
+        vpsrlq	ymm1, ymm11, 58
+        vpsrlq	ymm2, ymm12, 39
+        vpsrlq	ymm3, ymm13, 56
+        vpsrlq	ymm4, ymm14, 46
+        vpaddq	ymm10, ymm10, ymm10
+        vpsllq	ymm11, ymm11, 6
+        vpsllq	ymm12, ymm12, 25
+        vpsllq	ymm13, ymm13, 8
+        vpsllq	ymm14, ymm14, 18
+        vpor	ymm10, ymm10, ymm0
+        vpor	ymm11, ymm11, ymm1
+        vpor	ymm12, ymm12, ymm2
+        vpor	ymm13, ymm13, ymm3
+        vpor	ymm14, ymm14, ymm4
+        vpandn	ymm0, ymm11, ymm12
+        vpandn	ymm1, ymm12, ymm13
+        vpandn	ymm2, ymm13, ymm14
+        vpandn	ymm3, ymm14, ymm10
+        vpandn	ymm4, ymm10, ymm11
+        vpxor	ymm0, ymm0, ymm10
+        vpxor	ymm1, ymm1, ymm11
+        vpxor	ymm2, ymm2, ymm12
+        vpxor	ymm3, ymm3, ymm13
+        vpxor	ymm4, ymm4, ymm14
+        vmovdqu	YMMWORD PTR [rcx+128], ymm0
+        vmovdqu	YMMWORD PTR [r10+128], ymm1
+        vmovdqu	YMMWORD PTR [r9+-64], ymm2
+        vmovdqu	YMMWORD PTR [rcx+-96], ymm3
+        vmovdqu	YMMWORD PTR [r10+-96], ymm4
+        ; Row 3
+        vpxor	ymm10, ymm9, [r10+64]
+        vpxor	ymm11, ymm5, [r9+32]
+        vpxor	ymm12, ymm6, [rcx]
+        vpxor	ymm13, ymm7, [r9+96]
+        vpxor	ymm14, ymm8, [rcx+64]
+        vpsrlq	ymm0, ymm10, 37
+        vpsrlq	ymm1, ymm11, 28
+        vpsrlq	ymm2, ymm12, 54
+        vpsrlq	ymm3, ymm13, 49
+        vpsrlq	ymm4, ymm14, 8
+        vpsllq	ymm10, ymm10, 27
+        vpsllq	ymm11, ymm11, 36
+        vpsllq	ymm12, ymm12, 10
+        vpsllq	ymm13, ymm13, 15
+        vpsllq	ymm14, ymm14, 56
+        vpor	ymm10, ymm10, ymm0
+        vpor	ymm11, ymm11, ymm1
+        vpor	ymm12, ymm12, ymm2
+        vpor	ymm13, ymm13, ymm3
+        vpor	ymm14, ymm14, ymm4
+        vpandn	ymm0, ymm11, ymm12
+        vpandn	ymm1, ymm12, ymm13
+        vpandn	ymm2, ymm13, ymm14
+        vpandn	ymm3, ymm14, ymm10
+        vpandn	ymm4, ymm10, ymm11
+        vpxor	ymm0, ymm0, ymm10
+        vpxor	ymm1, ymm1, ymm11
+        vpxor	ymm2, ymm2, ymm12
+        vpxor	ymm3, ymm3, ymm13
+        vpxor	ymm4, ymm4, ymm14
+        vmovdqu	YMMWORD PTR [r10+64], ymm0
+        vmovdqu	YMMWORD PTR [r9+32], ymm1
+        vmovdqu	YMMWORD PTR [rcx], ymm2
+        vmovdqu	YMMWORD PTR [r9+96], ymm3
+        vmovdqu	YMMWORD PTR [rcx+64], ymm4
+        ; Row 4
+        vpxor	ymm10, ymm7, [r9+-32]
+        vpxor	ymm11, ymm8, [rcx+-64]
+        vpxor	ymm12, ymm9, [r10+-64]
+        vpxor	ymm13, ymm5, [r9+-96]
+        vpxor	ymm14, ymm6, [r10]
+        vpsrlq	ymm0, ymm10, 2
+        vpsrlq	ymm1, ymm11, 9
+        vpsrlq	ymm2, ymm12, 25
+        vpsrlq	ymm3, ymm13, 23
+        vpsrlq	ymm4, ymm14, 62
+        vpsllq	ymm10, ymm10, 62
+        vpsllq	ymm11, ymm11, 55
+        vpsllq	ymm12, ymm12, 39
+        vpsllq	ymm13, ymm13, 41
+        vpsllq	ymm14, ymm14, 2
+        vpor	ymm10, ymm10, ymm0
+        vpor	ymm11, ymm11, ymm1
+        vpor	ymm12, ymm12, ymm2
+        vpor	ymm13, ymm13, ymm3
+        vpor	ymm14, ymm14, ymm4
+        vpandn	ymm0, ymm11, ymm12
+        vpandn	ymm1, ymm12, ymm13
+        vpandn	ymm2, ymm13, ymm14
+        vpandn	ymm3, ymm14, ymm10
+        vpandn	ymm4, ymm10, ymm11
+        vpxor	ymm0, ymm0, ymm10
+        vpxor	ymm1, ymm1, ymm11
+        vpxor	ymm2, ymm2, ymm12
+        vpxor	ymm3, ymm3, ymm13
+        vpxor	ymm4, ymm4, ymm14
+        vmovdqu	YMMWORD PTR [r9+-32], ymm0
+        vmovdqu	YMMWORD PTR [rcx+-64], ymm1
+        vmovdqu	YMMWORD PTR [r10+-64], ymm2
+        vmovdqu	YMMWORD PTR [r9+-96], ymm3
+        vmovdqu	YMMWORD PTR [r10], ymm4
+        ; Round 16
+        ; Calc b[0..4]
+        vpxor	ymm10, ymm0, ymm15
+        vpxor	ymm13, ymm3, [rcx+-96]
+        vpxor	ymm14, ymm4, [rcx+-32]
+        vpxor	ymm12, ymm2, [rcx]
+        vpxor	ymm11, ymm1, [rcx+32]
+        vpxor	ymm14, ymm14, [rcx+64]
+        vpxor	ymm12, ymm12, [rcx+96]
+        vpxor	ymm10, ymm10, [rcx+128]
+        vpxor	ymm12, ymm12, [r9+-64]
+        vpxor	ymm13, ymm13, [r9]
+        vpxor	ymm11, ymm11, [r9+32]
+        vpxor	ymm14, ymm14, [r9+64]
+        vpxor	ymm13, ymm13, [r9+96]
+        vpxor	ymm11, ymm11, [r9+128]
+        vpxor	ymm14, ymm14, [r10+-96]
+        vpxor	ymm10, ymm10, [r10+-32]
+        vpxor	ymm12, ymm12, [r10+32]
+        vpxor	ymm10, ymm10, [r10+64]
+        vpxor	ymm13, ymm13, [r10+96]
+        vpxor	ymm11, ymm11, [r10+128]
+        ; Calc t[0..4]
+        vpsrlq	ymm0, ymm11, 63
+        vpsrlq	ymm1, ymm12, 63
+        vpsrlq	ymm2, ymm13, 63
+        vpsrlq	ymm3, ymm14, 63
+        vpsrlq	ymm4, ymm10, 63
+        vpaddq	ymm5, ymm11, ymm11
+        vpaddq	ymm6, ymm12, ymm12
+        vpaddq	ymm7, ymm13, ymm13
+        vpaddq	ymm8, ymm14, ymm14
+        vpaddq	ymm9, ymm10, ymm10
+        vpor	ymm5, ymm5, ymm0
+        vpor	ymm6, ymm6, ymm1
+        vpor	ymm7, ymm7, ymm2
+        vpor	ymm8, ymm8, ymm3
+        vpor	ymm9, ymm9, ymm4
+        vpxor	ymm5, ymm5, ymm14
+        vpxor	ymm6, ymm6, ymm10
+        vpxor	ymm7, ymm7, ymm11
+        vpxor	ymm8, ymm8, ymm12
+        vpxor	ymm9, ymm9, ymm13
+        ; Row Mix
+        ; Row 0
+        vpxor	ymm10, ymm5, ymm15
+        vpxor	ymm11, ymm6, [rcx+32]
+        vpxor	ymm12, ymm7, [r9+-64]
+        vpxor	ymm13, ymm8, [r9+96]
+        vpxor	ymm14, ymm9, [r10]
+        vpsrlq	ymm0, ymm11, 20
+        vpsrlq	ymm1, ymm12, 21
+        vpsrlq	ymm2, ymm13, 43
+        vpsrlq	ymm3, ymm14, 50
+        vpsllq	ymm11, ymm11, 44
+        vpsllq	ymm12, ymm12, 43
+        vpsllq	ymm13, ymm13, 21
+        vpsllq	ymm14, ymm14, 14
+        vpor	ymm11, ymm11, ymm0
+        vpor	ymm12, ymm12, ymm1
+        vpor	ymm13, ymm13, ymm2
+        vpor	ymm14, ymm14, ymm3
+        vpandn	ymm15, ymm11, ymm12
+        vpandn	ymm1, ymm12, ymm13
+        vpandn	ymm2, ymm13, ymm14
+        vpandn	ymm3, ymm14, ymm10
+        vpandn	ymm4, ymm10, ymm11
+        vpxor	ymm15, ymm15, ymm10
+        vpxor	ymm1, ymm1, ymm11
+        vpxor	ymm2, ymm2, ymm12
+        ; XOR in constant
+        vpxor	ymm15, ymm15, [rax+512]
+        vpxor	ymm3, ymm3, ymm13
+        vpxor	ymm4, ymm4, ymm14
+        vmovdqu	YMMWORD PTR [rcx+32], ymm1
+        vmovdqu	YMMWORD PTR [r9+-64], ymm2
+        vmovdqu	YMMWORD PTR [r9+96], ymm3
+        vmovdqu	YMMWORD PTR [r10], ymm4
+        ; Row 1
+        vpxor	ymm10, ymm8, [r10+96]
+        vpxor	ymm11, ymm9, [rcx+-32]
+        vpxor	ymm12, ymm5, [rcx+128]
+        vpxor	ymm13, ymm6, [r9+32]
+        vpxor	ymm14, ymm7, [r10+-64]
+        vpsrlq	ymm0, ymm10, 36
+        vpsrlq	ymm1, ymm11, 44
+        vpsrlq	ymm2, ymm12, 61
+        vpsrlq	ymm3, ymm13, 19
+        vpsrlq	ymm4, ymm14, 3
+        vpsllq	ymm10, ymm10, 28
+        vpsllq	ymm11, ymm11, 20
+        vpsllq	ymm12, ymm12, 3
+        vpsllq	ymm13, ymm13, 45
+        vpsllq	ymm14, ymm14, 61
+        vpor	ymm10, ymm10, ymm0
+        vpor	ymm11, ymm11, ymm1
+        vpor	ymm12, ymm12, ymm2
+        vpor	ymm13, ymm13, ymm3
+        vpor	ymm14, ymm14, ymm4
+        vpandn	ymm0, ymm11, ymm12
+        vpandn	ymm1, ymm12, ymm13
+        vpandn	ymm2, ymm13, ymm14
+        vpandn	ymm3, ymm14, ymm10
+        vpandn	ymm4, ymm10, ymm11
+        vpxor	ymm0, ymm0, ymm10
+        vpxor	ymm1, ymm1, ymm11
+        vpxor	ymm2, ymm2, ymm12
+        vpxor	ymm3, ymm3, ymm13
+        vpxor	ymm4, ymm4, ymm14
+        vmovdqu	YMMWORD PTR [r10+96], ymm0
+        vmovdqu	YMMWORD PTR [rcx+-32], ymm1
+        vmovdqu	YMMWORD PTR [rcx+128], ymm2
+        vmovdqu	YMMWORD PTR [r9+32], ymm3
+        vmovdqu	YMMWORD PTR [r10+-64], ymm4
+        ; Row 2
+        vpxor	ymm10, ymm6, [r9+128]
+        vpxor	ymm11, ymm7, [r10+32]
+        vpxor	ymm12, ymm8, [rcx+-96]
+        vpxor	ymm13, ymm9, [rcx+64]
+        vpxor	ymm14, ymm5, [r9+-32]
+        vpsrlq	ymm0, ymm10, 63
+        vpsrlq	ymm1, ymm11, 58
+        vpsrlq	ymm2, ymm12, 39
+        vpsrlq	ymm3, ymm13, 56
+        vpsrlq	ymm4, ymm14, 46
+        vpaddq	ymm10, ymm10, ymm10
+        vpsllq	ymm11, ymm11, 6
+        vpsllq	ymm12, ymm12, 25
+        vpsllq	ymm13, ymm13, 8
+        vpsllq	ymm14, ymm14, 18
+        vpor	ymm10, ymm10, ymm0
+        vpor	ymm11, ymm11, ymm1
+        vpor	ymm12, ymm12, ymm2
+        vpor	ymm13, ymm13, ymm3
+        vpor	ymm14, ymm14, ymm4
+        vpandn	ymm0, ymm11, ymm12
+        vpandn	ymm1, ymm12, ymm13
+        vpandn	ymm2, ymm13, ymm14
+        vpandn	ymm3, ymm14, ymm10
+        vpandn	ymm4, ymm10, ymm11
+        vpxor	ymm0, ymm0, ymm10
+        vpxor	ymm1, ymm1, ymm11
+        vpxor	ymm2, ymm2, ymm12
+        vpxor	ymm3, ymm3, ymm13
+        vpxor	ymm4, ymm4, ymm14
+        vmovdqu	YMMWORD PTR [r9+128], ymm0
+        vmovdqu	YMMWORD PTR [r10+32], ymm1
+        vmovdqu	YMMWORD PTR [rcx+-96], ymm2
+        vmovdqu	YMMWORD PTR [rcx+64], ymm3
+        vmovdqu	YMMWORD PTR [r9+-32], ymm4
+        ; Row 3
+        vpxor	ymm10, ymm9, [r9+64]
+        vpxor	ymm11, ymm5, [r10+-32]
+        vpxor	ymm12, ymm6, [r10+128]
+        vpxor	ymm13, ymm7, [rcx]
+        vpxor	ymm14, ymm8, [r9+-96]
+        vpsrlq	ymm0, ymm10, 37
+        vpsrlq	ymm1, ymm11, 28
+        vpsrlq	ymm2, ymm12, 54
+        vpsrlq	ymm3, ymm13, 49
+        vpsrlq	ymm4, ymm14, 8
+        vpsllq	ymm10, ymm10, 27
+        vpsllq	ymm11, ymm11, 36
+        vpsllq	ymm12, ymm12, 10
+        vpsllq	ymm13, ymm13, 15
+        vpsllq	ymm14, ymm14, 56
+        vpor	ymm10, ymm10, ymm0
+        vpor	ymm11, ymm11, ymm1
+        vpor	ymm12, ymm12, ymm2
+        vpor	ymm13, ymm13, ymm3
+        vpor	ymm14, ymm14, ymm4
+        vpandn	ymm0, ymm11, ymm12
+        vpandn	ymm1, ymm12, ymm13
+        vpandn	ymm2, ymm13, ymm14
+        vpandn	ymm3, ymm14, ymm10
+        vpandn	ymm4, ymm10, ymm11
+        vpxor	ymm0, ymm0, ymm10
+        vpxor	ymm1, ymm1, ymm11
+        vpxor	ymm2, ymm2, ymm12
+        vpxor	ymm3, ymm3, ymm13
+        vpxor	ymm4, ymm4, ymm14
+        vmovdqu	YMMWORD PTR [r9+64], ymm0
+        vmovdqu	YMMWORD PTR [r10+-32], ymm1
+        vmovdqu	YMMWORD PTR [r10+128], ymm2
+        vmovdqu	YMMWORD PTR [rcx], ymm3
+        vmovdqu	YMMWORD PTR [r9+-96], ymm4
+        ; Row 4
+        vpxor	ymm10, ymm7, [rcx+96]
+        vpxor	ymm11, ymm8, [r9]
+        vpxor	ymm12, ymm9, [r10+-96]
+        vpxor	ymm13, ymm5, [r10+64]
+        vpxor	ymm14, ymm6, [rcx+-64]
+        vpsrlq	ymm0, ymm10, 2
+        vpsrlq	ymm1, ymm11, 9
+        vpsrlq	ymm2, ymm12, 25
+        vpsrlq	ymm3, ymm13, 23
+        vpsrlq	ymm4, ymm14, 62
+        vpsllq	ymm10, ymm10, 62
+        vpsllq	ymm11, ymm11, 55
+        vpsllq	ymm12, ymm12, 39
+        vpsllq	ymm13, ymm13, 41
+        vpsllq	ymm14, ymm14, 2
+        vpor	ymm10, ymm10, ymm0
+        vpor	ymm11, ymm11, ymm1
+        vpor	ymm12, ymm12, ymm2
+        vpor	ymm13, ymm13, ymm3
+        vpor	ymm14, ymm14, ymm4
+        vpandn	ymm0, ymm11, ymm12
+        vpandn	ymm1, ymm12, ymm13
+        vpandn	ymm2, ymm13, ymm14
+        vpandn	ymm3, ymm14, ymm10
+        vpandn	ymm4, ymm10, ymm11
+        vpxor	ymm0, ymm0, ymm10
+        vpxor	ymm1, ymm1, ymm11
+        vpxor	ymm2, ymm2, ymm12
+        vpxor	ymm3, ymm3, ymm13
+        vpxor	ymm4, ymm4, ymm14
+        vmovdqu	YMMWORD PTR [rcx+96], ymm0
+        vmovdqu	YMMWORD PTR [r9], ymm1
+        vmovdqu	YMMWORD PTR [r10+-96], ymm2
+        vmovdqu	YMMWORD PTR [r10+64], ymm3
+        vmovdqu	YMMWORD PTR [rcx+-64], ymm4
+        ; Round 17
+        ; Calc b[0..4]
+        vpxor	ymm10, ymm0, ymm15
+        vpxor	ymm12, ymm2, [rcx+-96]
+        vpxor	ymm11, ymm1, [rcx+-32]
+        vpxor	ymm13, ymm3, [rcx]
+        vpxor	ymm11, ymm11, [rcx+32]
+        vpxor	ymm13, ymm13, [rcx+64]
+        vpxor	ymm12, ymm12, [rcx+128]
+        vpxor	ymm14, ymm4, [r9+-96]
+        vpxor	ymm12, ymm12, [r9+-64]
+        vpxor	ymm14, ymm14, [r9+-32]
+        vpxor	ymm13, ymm13, [r9+32]
+        vpxor	ymm10, ymm10, [r9+64]
+        vpxor	ymm13, ymm13, [r9+96]
+        vpxor	ymm10, ymm10, [r9+128]
+        vpxor	ymm14, ymm14, [r10+-64]
+        vpxor	ymm11, ymm11, [r10+-32]
+        vpxor	ymm14, ymm14, [r10]
+        vpxor	ymm11, ymm11, [r10+32]
+        vpxor	ymm10, ymm10, [r10+96]
+        vpxor	ymm12, ymm12, [r10+128]
+        ; Calc t[0..4]
+        vpsrlq	ymm0, ymm11, 63
+        vpsrlq	ymm1, ymm12, 63
+        vpsrlq	ymm2, ymm13, 63
+        vpsrlq	ymm3, ymm14, 63
+        vpsrlq	ymm4, ymm10, 63
+        vpaddq	ymm5, ymm11, ymm11
+        vpaddq	ymm6, ymm12, ymm12
+        vpaddq	ymm7, ymm13, ymm13
+        vpaddq	ymm8, ymm14, ymm14
+        vpaddq	ymm9, ymm10, ymm10
+        vpor	ymm5, ymm5, ymm0
+        vpor	ymm6, ymm6, ymm1
+        vpor	ymm7, ymm7, ymm2
+        vpor	ymm8, ymm8, ymm3
+        vpor	ymm9, ymm9, ymm4
+        vpxor	ymm5, ymm5, ymm14
+        vpxor	ymm6, ymm6, ymm10
+        vpxor	ymm7, ymm7, ymm11
+        vpxor	ymm8, ymm8, ymm12
+        vpxor	ymm9, ymm9, ymm13
+        ; Row Mix
+        ; Row 0
+        vpxor	ymm10, ymm5, ymm15
+        vpxor	ymm11, ymm6, [rcx+-32]
+        vpxor	ymm12, ymm7, [rcx+-96]
+        vpxor	ymm13, ymm8, [rcx]
+        vpxor	ymm14, ymm9, [rcx+-64]
+        vpsrlq	ymm0, ymm11, 20
+        vpsrlq	ymm1, ymm12, 21
+        vpsrlq	ymm2, ymm13, 43
+        vpsrlq	ymm3, ymm14, 50
+        vpsllq	ymm11, ymm11, 44
+        vpsllq	ymm12, ymm12, 43
+        vpsllq	ymm13, ymm13, 21
+        vpsllq	ymm14, ymm14, 14
+        vpor	ymm11, ymm11, ymm0
+        vpor	ymm12, ymm12, ymm1
+        vpor	ymm13, ymm13, ymm2
+        vpor	ymm14, ymm14, ymm3
+        vpandn	ymm15, ymm11, ymm12
+        vpandn	ymm1, ymm12, ymm13
+        vpandn	ymm2, ymm13, ymm14
+        vpandn	ymm3, ymm14, ymm10
+        vpandn	ymm4, ymm10, ymm11
+        vpxor	ymm15, ymm15, ymm10
+        vpxor	ymm1, ymm1, ymm11
+        vpxor	ymm2, ymm2, ymm12
+        ; XOR in constant
+        vpxor	ymm15, ymm15, [rax+544]
+        vpxor	ymm3, ymm3, ymm13
+        vpxor	ymm4, ymm4, ymm14
+        vmovdqu	YMMWORD PTR [rcx+-32], ymm1
+        vmovdqu	YMMWORD PTR [rcx+-96], ymm2
+        vmovdqu	YMMWORD PTR [rcx], ymm3
+        vmovdqu	YMMWORD PTR [rcx+-64], ymm4
+        ; Row 1
+        vpxor	ymm10, ymm8, [r9+96]
+        vpxor	ymm11, ymm9, [r10+-64]
+        vpxor	ymm12, ymm5, [r9+128]
+        vpxor	ymm13, ymm6, [r10+-32]
+        vpxor	ymm14, ymm7, [r10+-96]
+        vpsrlq	ymm0, ymm10, 36
+        vpsrlq	ymm1, ymm11, 44
+        vpsrlq	ymm2, ymm12, 61
+        vpsrlq	ymm3, ymm13, 19
+        vpsrlq	ymm4, ymm14, 3
+        vpsllq	ymm10, ymm10, 28
+        vpsllq	ymm11, ymm11, 20
+        vpsllq	ymm12, ymm12, 3
+        vpsllq	ymm13, ymm13, 45
+        vpsllq	ymm14, ymm14, 61
+        vpor	ymm10, ymm10, ymm0
+        vpor	ymm11, ymm11, ymm1
+        vpor	ymm12, ymm12, ymm2
+        vpor	ymm13, ymm13, ymm3
+        vpor	ymm14, ymm14, ymm4
+        vpandn	ymm0, ymm11, ymm12
+        vpandn	ymm1, ymm12, ymm13
+        vpandn	ymm2, ymm13, ymm14
+        vpandn	ymm3, ymm14, ymm10
+        vpandn	ymm4, ymm10, ymm11
+        vpxor	ymm0, ymm0, ymm10
+        vpxor	ymm1, ymm1, ymm11
+        vpxor	ymm2, ymm2, ymm12
+        vpxor	ymm3, ymm3, ymm13
+        vpxor	ymm4, ymm4, ymm14
+        vmovdqu	YMMWORD PTR [r9+96], ymm0
+        vmovdqu	YMMWORD PTR [r10+-64], ymm1
+        vmovdqu	YMMWORD PTR [r9+128], ymm2
+        vmovdqu	YMMWORD PTR [r10+-32], ymm3
+        vmovdqu	YMMWORD PTR [r10+-96], ymm4
+        ; Row 2
+        vpxor	ymm10, ymm6, [rcx+32]
+        vpxor	ymm11, ymm7, [rcx+128]
+        vpxor	ymm12, ymm8, [rcx+64]
+        vpxor	ymm13, ymm9, [r9+-96]
+        vpxor	ymm14, ymm5, [rcx+96]
+        vpsrlq	ymm0, ymm10, 63
+        vpsrlq	ymm1, ymm11, 58
+        vpsrlq	ymm2, ymm12, 39
+        vpsrlq	ymm3, ymm13, 56
+        vpsrlq	ymm4, ymm14, 46
+        vpaddq	ymm10, ymm10, ymm10
+        vpsllq	ymm11, ymm11, 6
+        vpsllq	ymm12, ymm12, 25
+        vpsllq	ymm13, ymm13, 8
+        vpsllq	ymm14, ymm14, 18
+        vpor	ymm10, ymm10, ymm0
+        vpor	ymm11, ymm11, ymm1
+        vpor	ymm12, ymm12, ymm2
+        vpor	ymm13, ymm13, ymm3
+        vpor	ymm14, ymm14, ymm4
+        vpandn	ymm0, ymm11, ymm12
+        vpandn	ymm1, ymm12, ymm13
+        vpandn	ymm2, ymm13, ymm14
+        vpandn	ymm3, ymm14, ymm10
+        vpandn	ymm4, ymm10, ymm11
+        vpxor	ymm0, ymm0, ymm10
+        vpxor	ymm1, ymm1, ymm11
+        vpxor	ymm2, ymm2, ymm12
+        vpxor	ymm3, ymm3, ymm13
+        vpxor	ymm4, ymm4, ymm14
+        vmovdqu	YMMWORD PTR [rcx+32], ymm0
+        vmovdqu	YMMWORD PTR [rcx+128], ymm1
+        vmovdqu	YMMWORD PTR [rcx+64], ymm2
+        vmovdqu	YMMWORD PTR [r9+-96], ymm3
+        vmovdqu	YMMWORD PTR [rcx+96], ymm4
+        ; Row 3
+        vpxor	ymm10, ymm9, [r10]
+        vpxor	ymm11, ymm5, [r10+96]
+        vpxor	ymm12, ymm6, [r10+32]
+        vpxor	ymm13, ymm7, [r10+128]
+        vpxor	ymm14, ymm8, [r10+64]
+        vpsrlq	ymm0, ymm10, 37
+        vpsrlq	ymm1, ymm11, 28
+        vpsrlq	ymm2, ymm12, 54
+        vpsrlq	ymm3, ymm13, 49
+        vpsrlq	ymm4, ymm14, 8
+        vpsllq	ymm10, ymm10, 27
+        vpsllq	ymm11, ymm11, 36
+        vpsllq	ymm12, ymm12, 10
+        vpsllq	ymm13, ymm13, 15
+        vpsllq	ymm14, ymm14, 56
+        vpor	ymm10, ymm10, ymm0
+        vpor	ymm11, ymm11, ymm1
+        vpor	ymm12, ymm12, ymm2
+        vpor	ymm13, ymm13, ymm3
+        vpor	ymm14, ymm14, ymm4
+        vpandn	ymm0, ymm11, ymm12
+        vpandn	ymm1, ymm12, ymm13
+        vpandn	ymm2, ymm13, ymm14
+        vpandn	ymm3, ymm14, ymm10
+        vpandn	ymm4, ymm10, ymm11
+        vpxor	ymm0, ymm0, ymm10
+        vpxor	ymm1, ymm1, ymm11
+        vpxor	ymm2, ymm2, ymm12
+        vpxor	ymm3, ymm3, ymm13
+        vpxor	ymm4, ymm4, ymm14
+        vmovdqu	YMMWORD PTR [r10], ymm0
+        vmovdqu	YMMWORD PTR [r10+96], ymm1
+        vmovdqu	YMMWORD PTR [r10+32], ymm2
+        vmovdqu	YMMWORD PTR [r10+128], ymm3
+        vmovdqu	YMMWORD PTR [r10+64], ymm4
+        ; Row 4
+        vpxor	ymm10, ymm7, [r9+-64]
+        vpxor	ymm11, ymm8, [r9+32]
+        vpxor	ymm12, ymm9, [r9+-32]
+        vpxor	ymm13, ymm5, [r9+64]
+        vpxor	ymm14, ymm6, [r9]
+        vpsrlq	ymm0, ymm10, 2
+        vpsrlq	ymm1, ymm11, 9
+        vpsrlq	ymm2, ymm12, 25
+        vpsrlq	ymm3, ymm13, 23
+        vpsrlq	ymm4, ymm14, 62
+        vpsllq	ymm10, ymm10, 62
+        vpsllq	ymm11, ymm11, 55
+        vpsllq	ymm12, ymm12, 39
+        vpsllq	ymm13, ymm13, 41
+        vpsllq	ymm14, ymm14, 2
+        vpor	ymm10, ymm10, ymm0
+        vpor	ymm11, ymm11, ymm1
+        vpor	ymm12, ymm12, ymm2
+        vpor	ymm13, ymm13, ymm3
+        vpor	ymm14, ymm14, ymm4
+        vpandn	ymm0, ymm11, ymm12
+        vpandn	ymm1, ymm12, ymm13
+        vpandn	ymm2, ymm13, ymm14
+        vpandn	ymm3, ymm14, ymm10
+        vpandn	ymm4, ymm10, ymm11
+        vpxor	ymm0, ymm0, ymm10
+        vpxor	ymm1, ymm1, ymm11
+        vpxor	ymm2, ymm2, ymm12
+        vpxor	ymm3, ymm3, ymm13
+        vpxor	ymm4, ymm4, ymm14
+        vmovdqu	YMMWORD PTR [r9+-64], ymm0
+        vmovdqu	YMMWORD PTR [r9+32], ymm1
+        vmovdqu	YMMWORD PTR [r9+-32], ymm2
+        vmovdqu	YMMWORD PTR [r9+64], ymm3
+        vmovdqu	YMMWORD PTR [r9], ymm4
+        ; Round 18
+        ; Calc b[0..4]
+        vpxor	ymm10, ymm0, ymm15
+        vpxor	ymm12, ymm2, [rcx+-96]
+        vpxor	ymm14, ymm4, [rcx+-64]
+        vpxor	ymm11, ymm1, [rcx+-32]
+        vpxor	ymm13, ymm3, [rcx]
+        vpxor	ymm10, ymm10, [rcx+32]
+        vpxor	ymm12, ymm12, [rcx+64]
+        vpxor	ymm14, ymm14, [rcx+96]
+        vpxor	ymm11, ymm11, [rcx+128]
+        vpxor	ymm13, ymm13, [r9+-96]
+        vpxor	ymm10, ymm10, [r9+96]
+        vpxor	ymm12, ymm12, [r9+128]
+        vpxor	ymm14, ymm14, [r10+-96]
+        vpxor	ymm11, ymm11, [r10+-64]
+        vpxor	ymm13, ymm13, [r10+-32]
+        vpxor	ymm10, ymm10, [r10]
+        vpxor	ymm12, ymm12, [r10+32]
+        vpxor	ymm14, ymm14, [r10+64]
+        vpxor	ymm11, ymm11, [r10+96]
+        vpxor	ymm13, ymm13, [r10+128]
+        ; Calc t[0..4]
+        vpsrlq	ymm0, ymm11, 63
+        vpsrlq	ymm1, ymm12, 63
+        vpsrlq	ymm2, ymm13, 63
+        vpsrlq	ymm3, ymm14, 63
+        vpsrlq	ymm4, ymm10, 63
+        vpaddq	ymm5, ymm11, ymm11
+        vpaddq	ymm6, ymm12, ymm12
+        vpaddq	ymm7, ymm13, ymm13
+        vpaddq	ymm8, ymm14, ymm14
+        vpaddq	ymm9, ymm10, ymm10
+        vpor	ymm5, ymm5, ymm0
+        vpor	ymm6, ymm6, ymm1
+        vpor	ymm7, ymm7, ymm2
+        vpor	ymm8, ymm8, ymm3
+        vpor	ymm9, ymm9, ymm4
+        vpxor	ymm5, ymm5, ymm14
+        vpxor	ymm6, ymm6, ymm10
+        vpxor	ymm7, ymm7, ymm11
+        vpxor	ymm8, ymm8, ymm12
+        vpxor	ymm9, ymm9, ymm13
+        ; Row Mix
+        ; Row 0
+        vpxor	ymm10, ymm5, ymm15
+        vpxor	ymm11, ymm6, [r10+-64]
+        vpxor	ymm12, ymm7, [rcx+64]
+        vpxor	ymm13, ymm8, [r10+128]
+        vpxor	ymm14, ymm9, [r9]
+        vpsrlq	ymm0, ymm11, 20
+        vpsrlq	ymm1, ymm12, 21
+        vpsrlq	ymm2, ymm13, 43
+        vpsrlq	ymm3, ymm14, 50
+        vpsllq	ymm11, ymm11, 44
+        vpsllq	ymm12, ymm12, 43
+        vpsllq	ymm13, ymm13, 21
+        vpsllq	ymm14, ymm14, 14
+        vpor	ymm11, ymm11, ymm0
+        vpor	ymm12, ymm12, ymm1
+        vpor	ymm13, ymm13, ymm2
+        vpor	ymm14, ymm14, ymm3
+        vpandn	ymm15, ymm11, ymm12
+        vpandn	ymm1, ymm12, ymm13
+        vpandn	ymm2, ymm13, ymm14
+        vpandn	ymm3, ymm14, ymm10
+        vpandn	ymm4, ymm10, ymm11
+        vpxor	ymm15, ymm15, ymm10
+        vpxor	ymm1, ymm1, ymm11
+        vpxor	ymm2, ymm2, ymm12
+        ; XOR in constant
+        vpxor	ymm15, ymm15, [rax+576]
+        vpxor	ymm3, ymm3, ymm13
+        vpxor	ymm4, ymm4, ymm14
+        vmovdqu	YMMWORD PTR [r10+-64], ymm1
+        vmovdqu	YMMWORD PTR [rcx+64], ymm2
+        vmovdqu	YMMWORD PTR [r10+128], ymm3
+        vmovdqu	YMMWORD PTR [r9], ymm4
+        ; Row 1
+        vpxor	ymm10, ymm8, [rcx]
+        vpxor	ymm11, ymm9, [r10+-96]
+        vpxor	ymm12, ymm5, [rcx+32]
+        vpxor	ymm13, ymm6, [r10+96]
+        vpxor	ymm14, ymm7, [r9+-32]
+        vpsrlq	ymm0, ymm10, 36
+        vpsrlq	ymm1, ymm11, 44
+        vpsrlq	ymm2, ymm12, 61
+        vpsrlq	ymm3, ymm13, 19
+        vpsrlq	ymm4, ymm14, 3
+        vpsllq	ymm10, ymm10, 28
+        vpsllq	ymm11, ymm11, 20
+        vpsllq	ymm12, ymm12, 3
+        vpsllq	ymm13, ymm13, 45
+        vpsllq	ymm14, ymm14, 61
+        vpor	ymm10, ymm10, ymm0
+        vpor	ymm11, ymm11, ymm1
+        vpor	ymm12, ymm12, ymm2
+        vpor	ymm13, ymm13, ymm3
+        vpor	ymm14, ymm14, ymm4
+        vpandn	ymm0, ymm11, ymm12
+        vpandn	ymm1, ymm12, ymm13
+        vpandn	ymm2, ymm13, ymm14
+        vpandn	ymm3, ymm14, ymm10
+        vpandn	ymm4, ymm10, ymm11
+        vpxor	ymm0, ymm0, ymm10
+        vpxor	ymm1, ymm1, ymm11
+        vpxor	ymm2, ymm2, ymm12
+        vpxor	ymm3, ymm3, ymm13
+        vpxor	ymm4, ymm4, ymm14
+        vmovdqu	YMMWORD PTR [rcx], ymm0
+        vmovdqu	YMMWORD PTR [r10+-96], ymm1
+        vmovdqu	YMMWORD PTR [rcx+32], ymm2
+        vmovdqu	YMMWORD PTR [r10+96], ymm3
+        vmovdqu	YMMWORD PTR [r9+-32], ymm4
+        ; Row 2
+        vpxor	ymm10, ymm6, [rcx+-32]
+        vpxor	ymm11, ymm7, [r9+128]
+        vpxor	ymm12, ymm8, [r9+-96]
+        vpxor	ymm13, ymm9, [r10+64]
+        vpxor	ymm14, ymm5, [r9+-64]
+        vpsrlq	ymm0, ymm10, 63
+        vpsrlq	ymm1, ymm11, 58
+        vpsrlq	ymm2, ymm12, 39
+        vpsrlq	ymm3, ymm13, 56
+        vpsrlq	ymm4, ymm14, 46
+        vpaddq	ymm10, ymm10, ymm10
+        vpsllq	ymm11, ymm11, 6
+        vpsllq	ymm12, ymm12, 25
+        vpsllq	ymm13, ymm13, 8
+        vpsllq	ymm14, ymm14, 18
+        vpor	ymm10, ymm10, ymm0
+        vpor	ymm11, ymm11, ymm1
+        vpor	ymm12, ymm12, ymm2
+        vpor	ymm13, ymm13, ymm3
+        vpor	ymm14, ymm14, ymm4
+        vpandn	ymm0, ymm11, ymm12
+        vpandn	ymm1, ymm12, ymm13
+        vpandn	ymm2, ymm13, ymm14
+        vpandn	ymm3, ymm14, ymm10
+        vpandn	ymm4, ymm10, ymm11
+        vpxor	ymm0, ymm0, ymm10
+        vpxor	ymm1, ymm1, ymm11
+        vpxor	ymm2, ymm2, ymm12
+        vpxor	ymm3, ymm3, ymm13
+        vpxor	ymm4, ymm4, ymm14
+        vmovdqu	YMMWORD PTR [rcx+-32], ymm0
+        vmovdqu	YMMWORD PTR [r9+128], ymm1
+        vmovdqu	YMMWORD PTR [r9+-96], ymm2
+        vmovdqu	YMMWORD PTR [r10+64], ymm3
+        vmovdqu	YMMWORD PTR [r9+-64], ymm4
+        ; Row 3
+        vpxor	ymm10, ymm9, [rcx+-64]
+        vpxor	ymm11, ymm5, [r9+96]
+        vpxor	ymm12, ymm6, [rcx+128]
+        vpxor	ymm13, ymm7, [r10+32]
+        vpxor	ymm14, ymm8, [r9+64]
+        vpsrlq	ymm0, ymm10, 37
+        vpsrlq	ymm1, ymm11, 28
+        vpsrlq	ymm2, ymm12, 54
+        vpsrlq	ymm3, ymm13, 49
+        vpsrlq	ymm4, ymm14, 8
+        vpsllq	ymm10, ymm10, 27
+        vpsllq	ymm11, ymm11, 36
+        vpsllq	ymm12, ymm12, 10
+        vpsllq	ymm13, ymm13, 15
+        vpsllq	ymm14, ymm14, 56
+        vpor	ymm10, ymm10, ymm0
+        vpor	ymm11, ymm11, ymm1
+        vpor	ymm12, ymm12, ymm2
+        vpor	ymm13, ymm13, ymm3
+        vpor	ymm14, ymm14, ymm4
+        vpandn	ymm0, ymm11, ymm12
+        vpandn	ymm1, ymm12, ymm13
+        vpandn	ymm2, ymm13, ymm14
+        vpandn	ymm3, ymm14, ymm10
+        vpandn	ymm4, ymm10, ymm11
+        vpxor	ymm0, ymm0, ymm10
+        vpxor	ymm1, ymm1, ymm11
+        vpxor	ymm2, ymm2, ymm12
+        vpxor	ymm3, ymm3, ymm13
+        vpxor	ymm4, ymm4, ymm14
+        vmovdqu	YMMWORD PTR [rcx+-64], ymm0
+        vmovdqu	YMMWORD PTR [r9+96], ymm1
+        vmovdqu	YMMWORD PTR [rcx+128], ymm2
+        vmovdqu	YMMWORD PTR [r10+32], ymm3
+        vmovdqu	YMMWORD PTR [r9+64], ymm4
+        ; Row 4
+        vpxor	ymm10, ymm7, [rcx+-96]
+        vpxor	ymm11, ymm8, [r10+-32]
+        vpxor	ymm12, ymm9, [rcx+96]
+        vpxor	ymm13, ymm5, [r10]
+        vpxor	ymm14, ymm6, [r9+32]
+        vpsrlq	ymm0, ymm10, 2
+        vpsrlq	ymm1, ymm11, 9
+        vpsrlq	ymm2, ymm12, 25
+        vpsrlq	ymm3, ymm13, 23
+        vpsrlq	ymm4, ymm14, 62
+        vpsllq	ymm10, ymm10, 62
+        vpsllq	ymm11, ymm11, 55
+        vpsllq	ymm12, ymm12, 39
+        vpsllq	ymm13, ymm13, 41
+        vpsllq	ymm14, ymm14, 2
+        vpor	ymm10, ymm10, ymm0
+        vpor	ymm11, ymm11, ymm1
+        vpor	ymm12, ymm12, ymm2
+        vpor	ymm13, ymm13, ymm3
+        vpor	ymm14, ymm14, ymm4
+        vpandn	ymm0, ymm11, ymm12
+        vpandn	ymm1, ymm12, ymm13
+        vpandn	ymm2, ymm13, ymm14
+        vpandn	ymm3, ymm14, ymm10
+        vpandn	ymm4, ymm10, ymm11
+        vpxor	ymm0, ymm0, ymm10
+        vpxor	ymm1, ymm1, ymm11
+        vpxor	ymm2, ymm2, ymm12
+        vpxor	ymm3, ymm3, ymm13
+        vpxor	ymm4, ymm4, ymm14
+        vmovdqu	YMMWORD PTR [rcx+-96], ymm0
+        vmovdqu	YMMWORD PTR [r10+-32], ymm1
+        vmovdqu	YMMWORD PTR [rcx+96], ymm2
+        vmovdqu	YMMWORD PTR [r10], ymm3
+        vmovdqu	YMMWORD PTR [r9+32], ymm4
+        ; Round 19
+        ; Calc b[0..4]
+        vpxor	ymm10, ymm0, ymm15
+        vpxor	ymm10, ymm10, [rcx+-64]
+        vpxor	ymm10, ymm10, [rcx+-32]
+        vpxor	ymm10, ymm10, [rcx]
+        vpxor	ymm12, ymm2, [rcx+32]
+        vpxor	ymm12, ymm12, [rcx+64]
+        vpxor	ymm12, ymm12, [rcx+128]
+        vpxor	ymm12, ymm12, [r9+-96]
+        vpxor	ymm14, ymm4, [r9+-64]
+        vpxor	ymm14, ymm14, [r9+-32]
+        vpxor	ymm14, ymm14, [r9]
+        vpxor	ymm14, ymm14, [r9+64]
+        vpxor	ymm11, ymm1, [r9+96]
+        vpxor	ymm11, ymm11, [r9+128]
+        vpxor	ymm11, ymm11, [r10+-96]
+        vpxor	ymm11, ymm11, [r10+-64]
+        vpxor	ymm13, ymm3, [r10+32]
+        vpxor	ymm13, ymm13, [r10+64]
+        vpxor	ymm13, ymm13, [r10+96]
+        vpxor	ymm13, ymm13, [r10+128]
+        ; Calc t[0..4]
+        vpsrlq	ymm0, ymm11, 63
+        vpsrlq	ymm1, ymm12, 63
+        vpsrlq	ymm2, ymm13, 63
+        vpsrlq	ymm3, ymm14, 63
+        vpsrlq	ymm4, ymm10, 63
+        vpaddq	ymm5, ymm11, ymm11
+        vpaddq	ymm6, ymm12, ymm12
+        vpaddq	ymm7, ymm13, ymm13
+        vpaddq	ymm8, ymm14, ymm14
+        vpaddq	ymm9, ymm10, ymm10
+        vpor	ymm5, ymm5, ymm0
+        vpor	ymm6, ymm6, ymm1
+        vpor	ymm7, ymm7, ymm2
+        vpor	ymm8, ymm8, ymm3
+        vpor	ymm9, ymm9, ymm4
+        vpxor	ymm5, ymm5, ymm14
+        vpxor	ymm6, ymm6, ymm10
+        vpxor	ymm7, ymm7, ymm11
+        vpxor	ymm8, ymm8, ymm12
+        vpxor	ymm9, ymm9, ymm13
+        ; Row Mix
+        ; Row 0
+        vpxor	ymm10, ymm5, ymm15
+        vpxor	ymm11, ymm6, [r10+-96]
+        vpxor	ymm12, ymm7, [r9+-96]
+        vpxor	ymm13, ymm8, [r10+32]
+        vpxor	ymm14, ymm9, [r9+32]
+        vpsrlq	ymm0, ymm11, 20
+        vpsrlq	ymm1, ymm12, 21
+        vpsrlq	ymm2, ymm13, 43
+        vpsrlq	ymm3, ymm14, 50
+        vpsllq	ymm11, ymm11, 44
+        vpsllq	ymm12, ymm12, 43
+        vpsllq	ymm13, ymm13, 21
+        vpsllq	ymm14, ymm14, 14
+        vpor	ymm11, ymm11, ymm0
+        vpor	ymm12, ymm12, ymm1
+        vpor	ymm13, ymm13, ymm2
+        vpor	ymm14, ymm14, ymm3
+        vpandn	ymm15, ymm11, ymm12
+        vpandn	ymm1, ymm12, ymm13
+        vpandn	ymm2, ymm13, ymm14
+        vpandn	ymm3, ymm14, ymm10
+        vpandn	ymm4, ymm10, ymm11
+        vpxor	ymm15, ymm15, ymm10
+        vpxor	ymm1, ymm1, ymm11
+        vpxor	ymm2, ymm2, ymm12
+        ; XOR in constant
+        vpxor	ymm15, ymm15, [rax+608]
+        vpxor	ymm3, ymm3, ymm13
+        vpxor	ymm4, ymm4, ymm14
+        vmovdqu	YMMWORD PTR [r10+-96], ymm1
+        vmovdqu	YMMWORD PTR [r9+-96], ymm2
+        vmovdqu	YMMWORD PTR [r10+32], ymm3
+        vmovdqu	YMMWORD PTR [r9+32], ymm4
+        ; Row 1
+        vpxor	ymm10, ymm8, [r10+128]
+        vpxor	ymm11, ymm9, [r9+-32]
+        vpxor	ymm12, ymm5, [rcx+-32]
+        vpxor	ymm13, ymm6, [r9+96]
+        vpxor	ymm14, ymm7, [rcx+96]
+        vpsrlq	ymm0, ymm10, 36
+        vpsrlq	ymm1, ymm11, 44
+        vpsrlq	ymm2, ymm12, 61
+        vpsrlq	ymm3, ymm13, 19
+        vpsrlq	ymm4, ymm14, 3
+        vpsllq	ymm10, ymm10, 28
+        vpsllq	ymm11, ymm11, 20
+        vpsllq	ymm12, ymm12, 3
+        vpsllq	ymm13, ymm13, 45
+        vpsllq	ymm14, ymm14, 61
+        vpor	ymm10, ymm10, ymm0
+        vpor	ymm11, ymm11, ymm1
+        vpor	ymm12, ymm12, ymm2
+        vpor	ymm13, ymm13, ymm3
+        vpor	ymm14, ymm14, ymm4
+        vpandn	ymm0, ymm11, ymm12
+        vpandn	ymm1, ymm12, ymm13
+        vpandn	ymm2, ymm13, ymm14
+        vpandn	ymm3, ymm14, ymm10
+        vpandn	ymm4, ymm10, ymm11
+        vpxor	ymm0, ymm0, ymm10
+        vpxor	ymm1, ymm1, ymm11
+        vpxor	ymm2, ymm2, ymm12
+        vpxor	ymm3, ymm3, ymm13
+        vpxor	ymm4, ymm4, ymm14
+        vmovdqu	YMMWORD PTR [r10+128], ymm0
+        vmovdqu	YMMWORD PTR [r9+-32], ymm1
+        vmovdqu	YMMWORD PTR [rcx+-32], ymm2
+        vmovdqu	YMMWORD PTR [r9+96], ymm3
+        vmovdqu	YMMWORD PTR [rcx+96], ymm4
+        ; Row 2
+        vpxor	ymm10, ymm6, [r10+-64]
+        vpxor	ymm11, ymm7, [rcx+32]
+        vpxor	ymm12, ymm8, [r10+64]
+        vpxor	ymm13, ymm9, [r9+64]
+        vpxor	ymm14, ymm5, [rcx+-96]
+        vpsrlq	ymm0, ymm10, 63
+        vpsrlq	ymm1, ymm11, 58
+        vpsrlq	ymm2, ymm12, 39
+        vpsrlq	ymm3, ymm13, 56
+        vpsrlq	ymm4, ymm14, 46
+        vpaddq	ymm10, ymm10, ymm10
+        vpsllq	ymm11, ymm11, 6
+        vpsllq	ymm12, ymm12, 25
+        vpsllq	ymm13, ymm13, 8
+        vpsllq	ymm14, ymm14, 18
+        vpor	ymm10, ymm10, ymm0
+        vpor	ymm11, ymm11, ymm1
+        vpor	ymm12, ymm12, ymm2
+        vpor	ymm13, ymm13, ymm3
+        vpor	ymm14, ymm14, ymm4
+        vpandn	ymm0, ymm11, ymm12
+        vpandn	ymm1, ymm12, ymm13
+        vpandn	ymm2, ymm13, ymm14
+        vpandn	ymm3, ymm14, ymm10
+        vpandn	ymm4, ymm10, ymm11
+        vpxor	ymm0, ymm0, ymm10
+        vpxor	ymm1, ymm1, ymm11
+        vpxor	ymm2, ymm2, ymm12
+        vpxor	ymm3, ymm3, ymm13
+        vpxor	ymm4, ymm4, ymm14
+        vmovdqu	YMMWORD PTR [r10+-64], ymm0
+        vmovdqu	YMMWORD PTR [rcx+32], ymm1
+        vmovdqu	YMMWORD PTR [r10+64], ymm2
+        vmovdqu	YMMWORD PTR [r9+64], ymm3
+        vmovdqu	YMMWORD PTR [rcx+-96], ymm4
+        ; Row 3
+        vpxor	ymm10, ymm9, [r9]
+        vpxor	ymm11, ymm5, [rcx]
+        vpxor	ymm12, ymm6, [r9+128]
+        vpxor	ymm13, ymm7, [rcx+128]
+        vpxor	ymm14, ymm8, [r10]
+        vpsrlq	ymm0, ymm10, 37
+        vpsrlq	ymm1, ymm11, 28
+        vpsrlq	ymm2, ymm12, 54
+        vpsrlq	ymm3, ymm13, 49
+        vpsrlq	ymm4, ymm14, 8
+        vpsllq	ymm10, ymm10, 27
+        vpsllq	ymm11, ymm11, 36
+        vpsllq	ymm12, ymm12, 10
+        vpsllq	ymm13, ymm13, 15
+        vpsllq	ymm14, ymm14, 56
+        vpor	ymm10, ymm10, ymm0
+        vpor	ymm11, ymm11, ymm1
+        vpor	ymm12, ymm12, ymm2
+        vpor	ymm13, ymm13, ymm3
+        vpor	ymm14, ymm14, ymm4
+        vpandn	ymm0, ymm11, ymm12
+        vpandn	ymm1, ymm12, ymm13
+        vpandn	ymm2, ymm13, ymm14
+        vpandn	ymm3, ymm14, ymm10
+        vpandn	ymm4, ymm10, ymm11
+        vpxor	ymm0, ymm0, ymm10
+        vpxor	ymm1, ymm1, ymm11
+        vpxor	ymm2, ymm2, ymm12
+        vpxor	ymm3, ymm3, ymm13
+        vpxor	ymm4, ymm4, ymm14
+        vmovdqu	YMMWORD PTR [r9], ymm0
+        vmovdqu	YMMWORD PTR [rcx], ymm1
+        vmovdqu	YMMWORD PTR [r9+128], ymm2
+        vmovdqu	YMMWORD PTR [rcx+128], ymm3
+        vmovdqu	YMMWORD PTR [r10], ymm4
+        ; Row 4
+        vpxor	ymm10, ymm7, [rcx+64]
+        vpxor	ymm11, ymm8, [r10+96]
+        vpxor	ymm12, ymm9, [r9+-64]
+        vpxor	ymm13, ymm5, [rcx+-64]
+        vpxor	ymm14, ymm6, [r10+-32]
+        vpsrlq	ymm0, ymm10, 2
+        vpsrlq	ymm1, ymm11, 9
+        vpsrlq	ymm2, ymm12, 25
+        vpsrlq	ymm3, ymm13, 23
+        vpsrlq	ymm4, ymm14, 62
+        vpsllq	ymm10, ymm10, 62
+        vpsllq	ymm11, ymm11, 55
+        vpsllq	ymm12, ymm12, 39
+        vpsllq	ymm13, ymm13, 41
+        vpsllq	ymm14, ymm14, 2
+        vpor	ymm10, ymm10, ymm0
+        vpor	ymm11, ymm11, ymm1
+        vpor	ymm12, ymm12, ymm2
+        vpor	ymm13, ymm13, ymm3
+        vpor	ymm14, ymm14, ymm4
+        vpandn	ymm0, ymm11, ymm12
+        vpandn	ymm1, ymm12, ymm13
+        vpandn	ymm2, ymm13, ymm14
+        vpandn	ymm3, ymm14, ymm10
+        vpandn	ymm4, ymm10, ymm11
+        vpxor	ymm0, ymm0, ymm10
+        vpxor	ymm1, ymm1, ymm11
+        vpxor	ymm2, ymm2, ymm12
+        vpxor	ymm3, ymm3, ymm13
+        vpxor	ymm4, ymm4, ymm14
+        vmovdqu	YMMWORD PTR [rcx+64], ymm0
+        vmovdqu	YMMWORD PTR [r10+96], ymm1
+        vmovdqu	YMMWORD PTR [r9+-64], ymm2
+        vmovdqu	YMMWORD PTR [rcx+-64], ymm3
+        vmovdqu	YMMWORD PTR [r10+-32], ymm4
+        ; Round 20
+        ; Calc b[0..4]
+        vpxor	ymm10, ymm0, ymm15
+        vpxor	ymm14, ymm4, [rcx+-96]
+        vpxor	ymm12, ymm2, [rcx+-32]
+        vpxor	ymm11, ymm1, [rcx]
+        vpxor	ymm11, ymm11, [rcx+32]
+        vpxor	ymm14, ymm14, [rcx+96]
+        vpxor	ymm13, ymm3, [rcx+128]
+        vpxor	ymm12, ymm12, [r9+-96]
+        vpxor	ymm11, ymm11, [r9+-32]
+        vpxor	ymm10, ymm10, [r9]
+        vpxor	ymm14, ymm14, [r9+32]
+        vpxor	ymm13, ymm13, [r9+64]
+        vpxor	ymm13, ymm13, [r9+96]
+        vpxor	ymm12, ymm12, [r9+128]
+        vpxor	ymm11, ymm11, [r10+-96]
+        vpxor	ymm10, ymm10, [r10+-64]
+        vpxor	ymm14, ymm14, [r10]
+        vpxor	ymm13, ymm13, [r10+32]
+        vpxor	ymm12, ymm12, [r10+64]
+        vpxor	ymm10, ymm10, [r10+128]
+        ; Calc t[0..4]
+        vpsrlq	ymm0, ymm11, 63
+        vpsrlq	ymm1, ymm12, 63
+        vpsrlq	ymm2, ymm13, 63
+        vpsrlq	ymm3, ymm14, 63
+        vpsrlq	ymm4, ymm10, 63
+        vpaddq	ymm5, ymm11, ymm11
+        vpaddq	ymm6, ymm12, ymm12
+        vpaddq	ymm7, ymm13, ymm13
+        vpaddq	ymm8, ymm14, ymm14
+        vpaddq	ymm9, ymm10, ymm10
+        vpor	ymm5, ymm5, ymm0
+        vpor	ymm6, ymm6, ymm1
+        vpor	ymm7, ymm7, ymm2
+        vpor	ymm8, ymm8, ymm3
+        vpor	ymm9, ymm9, ymm4
+        vpxor	ymm5, ymm5, ymm14
+        vpxor	ymm6, ymm6, ymm10
+        vpxor	ymm7, ymm7, ymm11
+        vpxor	ymm8, ymm8, ymm12
+        vpxor	ymm9, ymm9, ymm13
+        ; Row Mix
+        ; Row 0
+        vpxor	ymm10, ymm5, ymm15
+        vpxor	ymm11, ymm6, [r9+-32]
+        vpxor	ymm12, ymm7, [r10+64]
+        vpxor	ymm13, ymm8, [rcx+128]
+        vpxor	ymm14, ymm9, [r10+-32]
+        vpsrlq	ymm0, ymm11, 20
+        vpsrlq	ymm1, ymm12, 21
+        vpsrlq	ymm2, ymm13, 43
+        vpsrlq	ymm3, ymm14, 50
+        vpsllq	ymm11, ymm11, 44
+        vpsllq	ymm12, ymm12, 43
+        vpsllq	ymm13, ymm13, 21
+        vpsllq	ymm14, ymm14, 14
+        vpor	ymm11, ymm11, ymm0
+        vpor	ymm12, ymm12, ymm1
+        vpor	ymm13, ymm13, ymm2
+        vpor	ymm14, ymm14, ymm3
+        vpandn	ymm15, ymm11, ymm12
+        vpandn	ymm1, ymm12, ymm13
+        vpandn	ymm2, ymm13, ymm14
+        vpandn	ymm3, ymm14, ymm10
+        vpandn	ymm4, ymm10, ymm11
+        vpxor	ymm15, ymm15, ymm10
+        vpxor	ymm1, ymm1, ymm11
+        vpxor	ymm2, ymm2, ymm12
+        ; XOR in constant
+        vpxor	ymm15, ymm15, [rax+640]
+        vpxor	ymm3, ymm3, ymm13
+        vpxor	ymm4, ymm4, ymm14
+        vmovdqu	YMMWORD PTR [r9+-32], ymm1
+        vmovdqu	YMMWORD PTR [r10+64], ymm2
+        vmovdqu	YMMWORD PTR [rcx+128], ymm3
+        vmovdqu	YMMWORD PTR [r10+-32], ymm4
+        ; Row 1
+        vpxor	ymm10, ymm8, [r10+32]
+        vpxor	ymm11, ymm9, [rcx+96]
+        vpxor	ymm12, ymm5, [r10+-64]
+        vpxor	ymm13, ymm6, [rcx]
+        vpxor	ymm14, ymm7, [r9+-64]
+        vpsrlq	ymm0, ymm10, 36
+        vpsrlq	ymm1, ymm11, 44
+        vpsrlq	ymm2, ymm12, 61
+        vpsrlq	ymm3, ymm13, 19
+        vpsrlq	ymm4, ymm14, 3
+        vpsllq	ymm10, ymm10, 28
+        vpsllq	ymm11, ymm11, 20
+        vpsllq	ymm12, ymm12, 3
+        vpsllq	ymm13, ymm13, 45
+        vpsllq	ymm14, ymm14, 61
+        vpor	ymm10, ymm10, ymm0
+        vpor	ymm11, ymm11, ymm1
+        vpor	ymm12, ymm12, ymm2
+        vpor	ymm13, ymm13, ymm3
+        vpor	ymm14, ymm14, ymm4
+        vpandn	ymm0, ymm11, ymm12
+        vpandn	ymm1, ymm12, ymm13
+        vpandn	ymm2, ymm13, ymm14
+        vpandn	ymm3, ymm14, ymm10
+        vpandn	ymm4, ymm10, ymm11
+        vpxor	ymm0, ymm0, ymm10
+        vpxor	ymm1, ymm1, ymm11
+        vpxor	ymm2, ymm2, ymm12
+        vpxor	ymm3, ymm3, ymm13
+        vpxor	ymm4, ymm4, ymm14
+        vmovdqu	YMMWORD PTR [r10+32], ymm0
+        vmovdqu	YMMWORD PTR [rcx+96], ymm1
+        vmovdqu	YMMWORD PTR [r10+-64], ymm2
+        vmovdqu	YMMWORD PTR [rcx], ymm3
+        vmovdqu	YMMWORD PTR [r9+-64], ymm4
+        ; Row 2
+        vpxor	ymm10, ymm6, [r10+-96]
+        vpxor	ymm11, ymm7, [rcx+-32]
+        vpxor	ymm12, ymm8, [r9+64]
+        vpxor	ymm13, ymm9, [r10]
+        vpxor	ymm14, ymm5, [rcx+64]
+        vpsrlq	ymm0, ymm10, 63
+        vpsrlq	ymm1, ymm11, 58
+        vpsrlq	ymm2, ymm12, 39
+        vpsrlq	ymm3, ymm13, 56
+        vpsrlq	ymm4, ymm14, 46
+        vpaddq	ymm10, ymm10, ymm10
+        vpsllq	ymm11, ymm11, 6
+        vpsllq	ymm12, ymm12, 25
+        vpsllq	ymm13, ymm13, 8
+        vpsllq	ymm14, ymm14, 18
+        vpor	ymm10, ymm10, ymm0
+        vpor	ymm11, ymm11, ymm1
+        vpor	ymm12, ymm12, ymm2
+        vpor	ymm13, ymm13, ymm3
+        vpor	ymm14, ymm14, ymm4
+        vpandn	ymm0, ymm11, ymm12
+        vpandn	ymm1, ymm12, ymm13
+        vpandn	ymm2, ymm13, ymm14
+        vpandn	ymm3, ymm14, ymm10
+        vpandn	ymm4, ymm10, ymm11
+        vpxor	ymm0, ymm0, ymm10
+        vpxor	ymm1, ymm1, ymm11
+        vpxor	ymm2, ymm2, ymm12
+        vpxor	ymm3, ymm3, ymm13
+        vpxor	ymm4, ymm4, ymm14
+        vmovdqu	YMMWORD PTR [r10+-96], ymm0
+        vmovdqu	YMMWORD PTR [rcx+-32], ymm1
+        vmovdqu	YMMWORD PTR [r9+64], ymm2
+        vmovdqu	YMMWORD PTR [r10], ymm3
+        vmovdqu	YMMWORD PTR [rcx+64], ymm4
+        ; Row 3
+        vpxor	ymm10, ymm9, [r9+32]
+        vpxor	ymm11, ymm5, [r10+128]
+        vpxor	ymm12, ymm6, [rcx+32]
+        vpxor	ymm13, ymm7, [r9+128]
+        vpxor	ymm14, ymm8, [rcx+-64]
+        vpsrlq	ymm0, ymm10, 37
+        vpsrlq	ymm1, ymm11, 28
+        vpsrlq	ymm2, ymm12, 54
+        vpsrlq	ymm3, ymm13, 49
+        vpsrlq	ymm4, ymm14, 8
+        vpsllq	ymm10, ymm10, 27
+        vpsllq	ymm11, ymm11, 36
+        vpsllq	ymm12, ymm12, 10
+        vpsllq	ymm13, ymm13, 15
+        vpsllq	ymm14, ymm14, 56
+        vpor	ymm10, ymm10, ymm0
+        vpor	ymm11, ymm11, ymm1
+        vpor	ymm12, ymm12, ymm2
+        vpor	ymm13, ymm13, ymm3
+        vpor	ymm14, ymm14, ymm4
+        vpandn	ymm0, ymm11, ymm12
+        vpandn	ymm1, ymm12, ymm13
+        vpandn	ymm2, ymm13, ymm14
+        vpandn	ymm3, ymm14, ymm10
+        vpandn	ymm4, ymm10, ymm11
+        vpxor	ymm0, ymm0, ymm10
+        vpxor	ymm1, ymm1, ymm11
+        vpxor	ymm2, ymm2, ymm12
+        vpxor	ymm3, ymm3, ymm13
+        vpxor	ymm4, ymm4, ymm14
+        vmovdqu	YMMWORD PTR [r9+32], ymm0
+        vmovdqu	YMMWORD PTR [r10+128], ymm1
+        vmovdqu	YMMWORD PTR [rcx+32], ymm2
+        vmovdqu	YMMWORD PTR [r9+128], ymm3
+        vmovdqu	YMMWORD PTR [rcx+-64], ymm4
+        ; Row 4
+        vpxor	ymm10, ymm7, [r9+-96]
+        vpxor	ymm11, ymm8, [r9+96]
+        vpxor	ymm12, ymm9, [rcx+-96]
+        vpxor	ymm13, ymm5, [r9]
+        vpxor	ymm14, ymm6, [r10+96]
+        vpsrlq	ymm0, ymm10, 2
+        vpsrlq	ymm1, ymm11, 9
+        vpsrlq	ymm2, ymm12, 25
+        vpsrlq	ymm3, ymm13, 23
+        vpsrlq	ymm4, ymm14, 62
+        vpsllq	ymm10, ymm10, 62
+        vpsllq	ymm11, ymm11, 55
+        vpsllq	ymm12, ymm12, 39
+        vpsllq	ymm13, ymm13, 41
+        vpsllq	ymm14, ymm14, 2
+        vpor	ymm10, ymm10, ymm0
+        vpor	ymm11, ymm11, ymm1
+        vpor	ymm12, ymm12, ymm2
+        vpor	ymm13, ymm13, ymm3
+        vpor	ymm14, ymm14, ymm4
+        vpandn	ymm0, ymm11, ymm12
+        vpandn	ymm1, ymm12, ymm13
+        vpandn	ymm2, ymm13, ymm14
+        vpandn	ymm3, ymm14, ymm10
+        vpandn	ymm4, ymm10, ymm11
+        vpxor	ymm0, ymm0, ymm10
+        vpxor	ymm1, ymm1, ymm11
+        vpxor	ymm2, ymm2, ymm12
+        vpxor	ymm3, ymm3, ymm13
+        vpxor	ymm4, ymm4, ymm14
+        vmovdqu	YMMWORD PTR [r9+-96], ymm0
+        vmovdqu	YMMWORD PTR [r9+96], ymm1
+        vmovdqu	YMMWORD PTR [rcx+-96], ymm2
+        vmovdqu	YMMWORD PTR [r9], ymm3
+        vmovdqu	YMMWORD PTR [r10+96], ymm4
+        ; Round 21
+        ; Calc b[0..4]
+        vpxor	ymm10, ymm0, ymm15
+        vpxor	ymm14, ymm4, [rcx+-64]
+        vpxor	ymm11, ymm1, [rcx+-32]
+        vpxor	ymm13, ymm3, [rcx]
+        vpxor	ymm12, ymm2, [rcx+32]
+        vpxor	ymm14, ymm14, [rcx+64]
+        vpxor	ymm11, ymm11, [rcx+96]
+        vpxor	ymm13, ymm13, [rcx+128]
+        vpxor	ymm14, ymm14, [r9+-64]
+        vpxor	ymm11, ymm11, [r9+-32]
+        vpxor	ymm10, ymm10, [r9+32]
+        vpxor	ymm12, ymm12, [r9+64]
+        vpxor	ymm13, ymm13, [r9+128]
+        vpxor	ymm10, ymm10, [r10+-96]
+        vpxor	ymm12, ymm12, [r10+-64]
+        vpxor	ymm14, ymm14, [r10+-32]
+        vpxor	ymm13, ymm13, [r10]
+        vpxor	ymm10, ymm10, [r10+32]
+        vpxor	ymm12, ymm12, [r10+64]
+        vpxor	ymm11, ymm11, [r10+128]
+        ; Calc t[0..4]
+        vpsrlq	ymm0, ymm11, 63
+        vpsrlq	ymm1, ymm12, 63
+        vpsrlq	ymm2, ymm13, 63
+        vpsrlq	ymm3, ymm14, 63
+        vpsrlq	ymm4, ymm10, 63
+        vpaddq	ymm5, ymm11, ymm11
+        vpaddq	ymm6, ymm12, ymm12
+        vpaddq	ymm7, ymm13, ymm13
+        vpaddq	ymm8, ymm14, ymm14
+        vpaddq	ymm9, ymm10, ymm10
+        vpor	ymm5, ymm5, ymm0
+        vpor	ymm6, ymm6, ymm1
+        vpor	ymm7, ymm7, ymm2
+        vpor	ymm8, ymm8, ymm3
+        vpor	ymm9, ymm9, ymm4
+        vpxor	ymm5, ymm5, ymm14
+        vpxor	ymm6, ymm6, ymm10
+        vpxor	ymm7, ymm7, ymm11
+        vpxor	ymm8, ymm8, ymm12
+        vpxor	ymm9, ymm9, ymm13
+        ; Row Mix
+        ; Row 0
+        vpxor	ymm10, ymm5, ymm15
+        vpxor	ymm11, ymm6, [rcx+96]
+        vpxor	ymm12, ymm7, [r9+64]
+        vpxor	ymm13, ymm8, [r9+128]
+        vpxor	ymm14, ymm9, [r10+96]
+        vpsrlq	ymm0, ymm11, 20
+        vpsrlq	ymm1, ymm12, 21
+        vpsrlq	ymm2, ymm13, 43
+        vpsrlq	ymm3, ymm14, 50
+        vpsllq	ymm11, ymm11, 44
+        vpsllq	ymm12, ymm12, 43
+        vpsllq	ymm13, ymm13, 21
+        vpsllq	ymm14, ymm14, 14
+        vpor	ymm11, ymm11, ymm0
+        vpor	ymm12, ymm12, ymm1
+        vpor	ymm13, ymm13, ymm2
+        vpor	ymm14, ymm14, ymm3
+        vpandn	ymm15, ymm11, ymm12
+        vpandn	ymm1, ymm12, ymm13
+        vpandn	ymm2, ymm13, ymm14
+        vpandn	ymm3, ymm14, ymm10
+        vpandn	ymm4, ymm10, ymm11
+        vpxor	ymm15, ymm15, ymm10
+        vpxor	ymm1, ymm1, ymm11
+        vpxor	ymm2, ymm2, ymm12
+        ; XOR in constant
+        vpxor	ymm15, ymm15, [rax+672]
+        vpxor	ymm3, ymm3, ymm13
+        vpxor	ymm4, ymm4, ymm14
+        vmovdqu	YMMWORD PTR [rcx+96], ymm1
+        vmovdqu	YMMWORD PTR [r9+64], ymm2
+        vmovdqu	YMMWORD PTR [r9+128], ymm3
+        vmovdqu	YMMWORD PTR [r10+96], ymm4
+        ; Row 1
+        vpxor	ymm10, ymm8, [rcx+128]
+        vpxor	ymm11, ymm9, [r9+-64]
+        vpxor	ymm12, ymm5, [r10+-96]
+        vpxor	ymm13, ymm6, [r10+128]
+        vpxor	ymm14, ymm7, [rcx+-96]
+        vpsrlq	ymm0, ymm10, 36
+        vpsrlq	ymm1, ymm11, 44
+        vpsrlq	ymm2, ymm12, 61
+        vpsrlq	ymm3, ymm13, 19
+        vpsrlq	ymm4, ymm14, 3
+        vpsllq	ymm10, ymm10, 28
+        vpsllq	ymm11, ymm11, 20
+        vpsllq	ymm12, ymm12, 3
+        vpsllq	ymm13, ymm13, 45
+        vpsllq	ymm14, ymm14, 61
+        vpor	ymm10, ymm10, ymm0
+        vpor	ymm11, ymm11, ymm1
+        vpor	ymm12, ymm12, ymm2
+        vpor	ymm13, ymm13, ymm3
+        vpor	ymm14, ymm14, ymm4
+        vpandn	ymm0, ymm11, ymm12
+        vpandn	ymm1, ymm12, ymm13
+        vpandn	ymm2, ymm13, ymm14
+        vpandn	ymm3, ymm14, ymm10
+        vpandn	ymm4, ymm10, ymm11
+        vpxor	ymm0, ymm0, ymm10
+        vpxor	ymm1, ymm1, ymm11
+        vpxor	ymm2, ymm2, ymm12
+        vpxor	ymm3, ymm3, ymm13
+        vpxor	ymm4, ymm4, ymm14
+        vmovdqu	YMMWORD PTR [rcx+128], ymm0
+        vmovdqu	YMMWORD PTR [r9+-64], ymm1
+        vmovdqu	YMMWORD PTR [r10+-96], ymm2
+        vmovdqu	YMMWORD PTR [r10+128], ymm3
+        vmovdqu	YMMWORD PTR [rcx+-96], ymm4
+        ; Row 2
+        vpxor	ymm10, ymm6, [r9+-32]
+        vpxor	ymm11, ymm7, [r10+-64]
+        vpxor	ymm12, ymm8, [r10]
+        vpxor	ymm13, ymm9, [rcx+-64]
+        vpxor	ymm14, ymm5, [r9+-96]
+        vpsrlq	ymm0, ymm10, 63
+        vpsrlq	ymm1, ymm11, 58
+        vpsrlq	ymm2, ymm12, 39
+        vpsrlq	ymm3, ymm13, 56
+        vpsrlq	ymm4, ymm14, 46
+        vpaddq	ymm10, ymm10, ymm10
+        vpsllq	ymm11, ymm11, 6
+        vpsllq	ymm12, ymm12, 25
+        vpsllq	ymm13, ymm13, 8
+        vpsllq	ymm14, ymm14, 18
+        vpor	ymm10, ymm10, ymm0
+        vpor	ymm11, ymm11, ymm1
+        vpor	ymm12, ymm12, ymm2
+        vpor	ymm13, ymm13, ymm3
+        vpor	ymm14, ymm14, ymm4
+        vpandn	ymm0, ymm11, ymm12
+        vpandn	ymm1, ymm12, ymm13
+        vpandn	ymm2, ymm13, ymm14
+        vpandn	ymm3, ymm14, ymm10
+        vpandn	ymm4, ymm10, ymm11
+        vpxor	ymm0, ymm0, ymm10
+        vpxor	ymm1, ymm1, ymm11
+        vpxor	ymm2, ymm2, ymm12
+        vpxor	ymm3, ymm3, ymm13
+        vpxor	ymm4, ymm4, ymm14
+        vmovdqu	YMMWORD PTR [r9+-32], ymm0
+        vmovdqu	YMMWORD PTR [r10+-64], ymm1
+        vmovdqu	YMMWORD PTR [r10], ymm2
+        vmovdqu	YMMWORD PTR [rcx+-64], ymm3
+        vmovdqu	YMMWORD PTR [r9+-96], ymm4
+        ; Row 3
+        vpxor	ymm10, ymm9, [r10+-32]
+        vpxor	ymm11, ymm5, [r10+32]
+        vpxor	ymm12, ymm6, [rcx+-32]
+        vpxor	ymm13, ymm7, [rcx+32]
+        vpxor	ymm14, ymm8, [r9]
+        vpsrlq	ymm0, ymm10, 37
+        vpsrlq	ymm1, ymm11, 28
+        vpsrlq	ymm2, ymm12, 54
+        vpsrlq	ymm3, ymm13, 49
+        vpsrlq	ymm4, ymm14, 8
+        vpsllq	ymm10, ymm10, 27
+        vpsllq	ymm11, ymm11, 36
+        vpsllq	ymm12, ymm12, 10
+        vpsllq	ymm13, ymm13, 15
+        vpsllq	ymm14, ymm14, 56
+        vpor	ymm10, ymm10, ymm0
+        vpor	ymm11, ymm11, ymm1
+        vpor	ymm12, ymm12, ymm2
+        vpor	ymm13, ymm13, ymm3
+        vpor	ymm14, ymm14, ymm4
+        vpandn	ymm0, ymm11, ymm12
+        vpandn	ymm1, ymm12, ymm13
+        vpandn	ymm2, ymm13, ymm14
+        vpandn	ymm3, ymm14, ymm10
+        vpandn	ymm4, ymm10, ymm11
+        vpxor	ymm0, ymm0, ymm10
+        vpxor	ymm1, ymm1, ymm11
+        vpxor	ymm2, ymm2, ymm12
+        vpxor	ymm3, ymm3, ymm13
+        vpxor	ymm4, ymm4, ymm14
+        vmovdqu	YMMWORD PTR [r10+-32], ymm0
+        vmovdqu	YMMWORD PTR [r10+32], ymm1
+        vmovdqu	YMMWORD PTR [rcx+-32], ymm2
+        vmovdqu	YMMWORD PTR [rcx+32], ymm3
+        vmovdqu	YMMWORD PTR [r9], ymm4
+        ; Row 4
+        vpxor	ymm10, ymm7, [r10+64]
+        vpxor	ymm11, ymm8, [rcx]
+        vpxor	ymm12, ymm9, [rcx+64]
+        vpxor	ymm13, ymm5, [r9+32]
+        vpxor	ymm14, ymm6, [r9+96]
+        vpsrlq	ymm0, ymm10, 2
+        vpsrlq	ymm1, ymm11, 9
+        vpsrlq	ymm2, ymm12, 25
+        vpsrlq	ymm3, ymm13, 23
+        vpsrlq	ymm4, ymm14, 62
+        vpsllq	ymm10, ymm10, 62
+        vpsllq	ymm11, ymm11, 55
+        vpsllq	ymm12, ymm12, 39
+        vpsllq	ymm13, ymm13, 41
+        vpsllq	ymm14, ymm14, 2
+        vpor	ymm10, ymm10, ymm0
+        vpor	ymm11, ymm11, ymm1
+        vpor	ymm12, ymm12, ymm2
+        vpor	ymm13, ymm13, ymm3
+        vpor	ymm14, ymm14, ymm4
+        vpandn	ymm0, ymm11, ymm12
+        vpandn	ymm1, ymm12, ymm13
+        vpandn	ymm2, ymm13, ymm14
+        vpandn	ymm3, ymm14, ymm10
+        vpandn	ymm4, ymm10, ymm11
+        vpxor	ymm0, ymm0, ymm10
+        vpxor	ymm1, ymm1, ymm11
+        vpxor	ymm2, ymm2, ymm12
+        vpxor	ymm3, ymm3, ymm13
+        vpxor	ymm4, ymm4, ymm14
+        vmovdqu	YMMWORD PTR [r10+64], ymm0
+        vmovdqu	YMMWORD PTR [rcx], ymm1
+        vmovdqu	YMMWORD PTR [rcx+64], ymm2
+        vmovdqu	YMMWORD PTR [r9+32], ymm3
+        vmovdqu	YMMWORD PTR [r9+96], ymm4
+        ; Round 22
+        ; Calc b[0..4]
+        vpxor	ymm10, ymm0, ymm15
+        vpxor	ymm14, ymm4, [rcx+-96]
+        vpxor	ymm13, ymm3, [rcx+-64]
+        vpxor	ymm12, ymm2, [rcx+-32]
+        vpxor	ymm13, ymm13, [rcx+32]
+        vpxor	ymm11, ymm1, [rcx+96]
+        vpxor	ymm10, ymm10, [rcx+128]
+        vpxor	ymm14, ymm14, [r9+-96]
+        vpxor	ymm11, ymm11, [r9+-64]
+        vpxor	ymm10, ymm10, [r9+-32]
+        vpxor	ymm14, ymm14, [r9]
+        vpxor	ymm12, ymm12, [r9+64]
+        vpxor	ymm13, ymm13, [r9+128]
+        vpxor	ymm12, ymm12, [r10+-96]
+        vpxor	ymm11, ymm11, [r10+-64]
+        vpxor	ymm10, ymm10, [r10+-32]
+        vpxor	ymm12, ymm12, [r10]
+        vpxor	ymm11, ymm11, [r10+32]
+        vpxor	ymm14, ymm14, [r10+96]
+        vpxor	ymm13, ymm13, [r10+128]
+        ; Calc t[0..4]
+        vpsrlq	ymm0, ymm11, 63
+        vpsrlq	ymm1, ymm12, 63
+        vpsrlq	ymm2, ymm13, 63
+        vpsrlq	ymm3, ymm14, 63
+        vpsrlq	ymm4, ymm10, 63
+        vpaddq	ymm5, ymm11, ymm11
+        vpaddq	ymm6, ymm12, ymm12
+        vpaddq	ymm7, ymm13, ymm13
+        vpaddq	ymm8, ymm14, ymm14
+        vpaddq	ymm9, ymm10, ymm10
+        vpor	ymm5, ymm5, ymm0
+        vpor	ymm6, ymm6, ymm1
+        vpor	ymm7, ymm7, ymm2
+        vpor	ymm8, ymm8, ymm3
+        vpor	ymm9, ymm9, ymm4
+        vpxor	ymm5, ymm5, ymm14
+        vpxor	ymm6, ymm6, ymm10
+        vpxor	ymm7, ymm7, ymm11
+        vpxor	ymm8, ymm8, ymm12
+        vpxor	ymm9, ymm9, ymm13
+        ; Row Mix
+        ; Row 0
+        vpxor	ymm10, ymm5, ymm15
+        vpxor	ymm11, ymm6, [r9+-64]
+        vpxor	ymm12, ymm7, [r10]
+        vpxor	ymm13, ymm8, [rcx+32]
+        vpxor	ymm14, ymm9, [r9+96]
+        vpsrlq	ymm0, ymm11, 20
+        vpsrlq	ymm1, ymm12, 21
+        vpsrlq	ymm2, ymm13, 43
+        vpsrlq	ymm3, ymm14, 50
+        vpsllq	ymm11, ymm11, 44
+        vpsllq	ymm12, ymm12, 43
+        vpsllq	ymm13, ymm13, 21
+        vpsllq	ymm14, ymm14, 14
+        vpor	ymm11, ymm11, ymm0
+        vpor	ymm12, ymm12, ymm1
+        vpor	ymm13, ymm13, ymm2
+        vpor	ymm14, ymm14, ymm3
+        vpandn	ymm15, ymm11, ymm12
+        vpandn	ymm1, ymm12, ymm13
+        vpandn	ymm2, ymm13, ymm14
+        vpandn	ymm3, ymm14, ymm10
+        vpandn	ymm4, ymm10, ymm11
+        vpxor	ymm15, ymm15, ymm10
+        vpxor	ymm1, ymm1, ymm11
+        vpxor	ymm2, ymm2, ymm12
+        ; XOR in constant
+        vpxor	ymm15, ymm15, [rax+704]
+        vpxor	ymm3, ymm3, ymm13
+        vpxor	ymm4, ymm4, ymm14
+        vmovdqu	YMMWORD PTR [r9+-64], ymm1
+        vmovdqu	YMMWORD PTR [r10], ymm2
+        vmovdqu	YMMWORD PTR [rcx+32], ymm3
+        vmovdqu	YMMWORD PTR [r9+96], ymm4
+        ; Row 1
+        vpxor	ymm10, ymm8, [r9+128]
+        vpxor	ymm11, ymm9, [rcx+-96]
+        vpxor	ymm12, ymm5, [r9+-32]
+        vpxor	ymm13, ymm6, [r10+32]
+        vpxor	ymm14, ymm7, [rcx+64]
+        vpsrlq	ymm0, ymm10, 36
+        vpsrlq	ymm1, ymm11, 44
+        vpsrlq	ymm2, ymm12, 61
+        vpsrlq	ymm3, ymm13, 19
+        vpsrlq	ymm4, ymm14, 3
+        vpsllq	ymm10, ymm10, 28
+        vpsllq	ymm11, ymm11, 20
+        vpsllq	ymm12, ymm12, 3
+        vpsllq	ymm13, ymm13, 45
+        vpsllq	ymm14, ymm14, 61
+        vpor	ymm10, ymm10, ymm0
+        vpor	ymm11, ymm11, ymm1
+        vpor	ymm12, ymm12, ymm2
+        vpor	ymm13, ymm13, ymm3
+        vpor	ymm14, ymm14, ymm4
+        vpandn	ymm0, ymm11, ymm12
+        vpandn	ymm1, ymm12, ymm13
+        vpandn	ymm2, ymm13, ymm14
+        vpandn	ymm3, ymm14, ymm10
+        vpandn	ymm4, ymm10, ymm11
+        vpxor	ymm0, ymm0, ymm10
+        vpxor	ymm1, ymm1, ymm11
+        vpxor	ymm2, ymm2, ymm12
+        vpxor	ymm3, ymm3, ymm13
+        vpxor	ymm4, ymm4, ymm14
+        vmovdqu	YMMWORD PTR [r9+128], ymm0
+        vmovdqu	YMMWORD PTR [rcx+-96], ymm1
+        vmovdqu	YMMWORD PTR [r9+-32], ymm2
+        vmovdqu	YMMWORD PTR [r10+32], ymm3
+        vmovdqu	YMMWORD PTR [rcx+64], ymm4
+        ; Row 2
+        vpxor	ymm10, ymm6, [rcx+96]
+        vpxor	ymm11, ymm7, [r10+-96]
+        vpxor	ymm12, ymm8, [rcx+-64]
+        vpxor	ymm13, ymm9, [r9]
+        vpxor	ymm14, ymm5, [r10+64]
+        vpsrlq	ymm0, ymm10, 63
+        vpsrlq	ymm1, ymm11, 58
+        vpsrlq	ymm2, ymm12, 39
+        vpsrlq	ymm3, ymm13, 56
+        vpsrlq	ymm4, ymm14, 46
+        vpaddq	ymm10, ymm10, ymm10
+        vpsllq	ymm11, ymm11, 6
+        vpsllq	ymm12, ymm12, 25
+        vpsllq	ymm13, ymm13, 8
+        vpsllq	ymm14, ymm14, 18
+        vpor	ymm10, ymm10, ymm0
+        vpor	ymm11, ymm11, ymm1
+        vpor	ymm12, ymm12, ymm2
+        vpor	ymm13, ymm13, ymm3
+        vpor	ymm14, ymm14, ymm4
+        vpandn	ymm0, ymm11, ymm12
+        vpandn	ymm1, ymm12, ymm13
+        vpandn	ymm2, ymm13, ymm14
+        vpandn	ymm3, ymm14, ymm10
+        vpandn	ymm4, ymm10, ymm11
+        vpxor	ymm0, ymm0, ymm10
+        vpxor	ymm1, ymm1, ymm11
+        vpxor	ymm2, ymm2, ymm12
+        vpxor	ymm3, ymm3, ymm13
+        vpxor	ymm4, ymm4, ymm14
+        vmovdqu	YMMWORD PTR [rcx+96], ymm0
+        vmovdqu	YMMWORD PTR [r10+-96], ymm1
+        vmovdqu	YMMWORD PTR [rcx+-64], ymm2
+        vmovdqu	YMMWORD PTR [r9], ymm3
+        vmovdqu	YMMWORD PTR [r10+64], ymm4
+        ; Row 3
+        vpxor	ymm10, ymm9, [r10+96]
+        vpxor	ymm11, ymm5, [rcx+128]
+        vpxor	ymm12, ymm6, [r10+-64]
+        vpxor	ymm13, ymm7, [rcx+-32]
+        vpxor	ymm14, ymm8, [r9+32]
+        vpsrlq	ymm0, ymm10, 37
+        vpsrlq	ymm1, ymm11, 28
+        vpsrlq	ymm2, ymm12, 54
+        vpsrlq	ymm3, ymm13, 49
+        vpsrlq	ymm4, ymm14, 8
+        vpsllq	ymm10, ymm10, 27
+        vpsllq	ymm11, ymm11, 36
+        vpsllq	ymm12, ymm12, 10
+        vpsllq	ymm13, ymm13, 15
+        vpsllq	ymm14, ymm14, 56
+        vpor	ymm10, ymm10, ymm0
+        vpor	ymm11, ymm11, ymm1
+        vpor	ymm12, ymm12, ymm2
+        vpor	ymm13, ymm13, ymm3
+        vpor	ymm14, ymm14, ymm4
+        vpandn	ymm0, ymm11, ymm12
+        vpandn	ymm1, ymm12, ymm13
+        vpandn	ymm2, ymm13, ymm14
+        vpandn	ymm3, ymm14, ymm10
+        vpandn	ymm4, ymm10, ymm11
+        vpxor	ymm0, ymm0, ymm10
+        vpxor	ymm1, ymm1, ymm11
+        vpxor	ymm2, ymm2, ymm12
+        vpxor	ymm3, ymm3, ymm13
+        vpxor	ymm4, ymm4, ymm14
+        vmovdqu	YMMWORD PTR [r10+96], ymm0
+        vmovdqu	YMMWORD PTR [rcx+128], ymm1
+        vmovdqu	YMMWORD PTR [r10+-64], ymm2
+        vmovdqu	YMMWORD PTR [rcx+-32], ymm3
+        vmovdqu	YMMWORD PTR [r9+32], ymm4
+        ; Row 4
+        vpxor	ymm10, ymm7, [r9+64]
+        vpxor	ymm11, ymm8, [r10+128]
+        vpxor	ymm12, ymm9, [r9+-96]
+        vpxor	ymm13, ymm5, [r10+-32]
+        vpxor	ymm14, ymm6, [rcx]
+        vpsrlq	ymm0, ymm10, 2
+        vpsrlq	ymm1, ymm11, 9
+        vpsrlq	ymm2, ymm12, 25
+        vpsrlq	ymm3, ymm13, 23
+        vpsrlq	ymm4, ymm14, 62
+        vpsllq	ymm10, ymm10, 62
+        vpsllq	ymm11, ymm11, 55
+        vpsllq	ymm12, ymm12, 39
+        vpsllq	ymm13, ymm13, 41
+        vpsllq	ymm14, ymm14, 2
+        vpor	ymm10, ymm10, ymm0
+        vpor	ymm11, ymm11, ymm1
+        vpor	ymm12, ymm12, ymm2
+        vpor	ymm13, ymm13, ymm3
+        vpor	ymm14, ymm14, ymm4
+        vpandn	ymm0, ymm11, ymm12
+        vpandn	ymm1, ymm12, ymm13
+        vpandn	ymm2, ymm13, ymm14
+        vpandn	ymm3, ymm14, ymm10
+        vpandn	ymm4, ymm10, ymm11
+        vpxor	ymm0, ymm0, ymm10
+        vpxor	ymm1, ymm1, ymm11
+        vpxor	ymm2, ymm2, ymm12
+        vpxor	ymm3, ymm3, ymm13
+        vpxor	ymm4, ymm4, ymm14
+        vmovdqu	YMMWORD PTR [r9+64], ymm0
+        vmovdqu	YMMWORD PTR [r10+128], ymm1
+        vmovdqu	YMMWORD PTR [r9+-96], ymm2
+        vmovdqu	YMMWORD PTR [r10+-32], ymm3
+        vmovdqu	YMMWORD PTR [rcx], ymm4
+        ; Round 23
+        ; Calc b[0..4]
+        vpxor	ymm10, ymm0, ymm15
+        vpxor	ymm11, ymm1, [rcx+-96]
+        vpxor	ymm12, ymm2, [rcx+-64]
+        vpxor	ymm13, ymm3, [rcx+-32]
+        vpxor	ymm13, ymm13, [rcx+32]
+        vpxor	ymm14, ymm4, [rcx+64]
+        vpxor	ymm10, ymm10, [rcx+96]
+        vpxor	ymm11, ymm11, [rcx+128]
+        vpxor	ymm11, ymm11, [r9+-64]
+        vpxor	ymm12, ymm12, [r9+-32]
+        vpxor	ymm13, ymm13, [r9]
+        vpxor	ymm14, ymm14, [r9+32]
+        vpxor	ymm14, ymm14, [r9+96]
+        vpxor	ymm10, ymm10, [r9+128]
+        vpxor	ymm11, ymm11, [r10+-96]
+        vpxor	ymm12, ymm12, [r10+-64]
+        vpxor	ymm12, ymm12, [r10]
+        vpxor	ymm13, ymm13, [r10+32]
+        vpxor	ymm14, ymm14, [r10+64]
+        vpxor	ymm10, ymm10, [r10+96]
+        ; Calc t[0..4]
+        vpsrlq	ymm0, ymm11, 63
+        vpsrlq	ymm1, ymm12, 63
+        vpsrlq	ymm2, ymm13, 63
+        vpsrlq	ymm3, ymm14, 63
+        vpsrlq	ymm4, ymm10, 63
+        vpaddq	ymm5, ymm11, ymm11
+        vpaddq	ymm6, ymm12, ymm12
+        vpaddq	ymm7, ymm13, ymm13
+        vpaddq	ymm8, ymm14, ymm14
+        vpaddq	ymm9, ymm10, ymm10
+        vpor	ymm5, ymm5, ymm0
+        vpor	ymm6, ymm6, ymm1
+        vpor	ymm7, ymm7, ymm2
+        vpor	ymm8, ymm8, ymm3
+        vpor	ymm9, ymm9, ymm4
+        vpxor	ymm5, ymm5, ymm14
+        vpxor	ymm6, ymm6, ymm10
+        vpxor	ymm7, ymm7, ymm11
+        vpxor	ymm8, ymm8, ymm12
+        vpxor	ymm9, ymm9, ymm13
+        ; Row Mix
+        ; Row 0
+        vpxor	ymm10, ymm5, ymm15
+        vpxor	ymm11, ymm6, [rcx+-96]
+        vpxor	ymm12, ymm7, [rcx+-64]
+        vpxor	ymm13, ymm8, [rcx+-32]
+        vpxor	ymm14, ymm9, [rcx]
+        vpsrlq	ymm0, ymm11, 20
+        vpsrlq	ymm1, ymm12, 21
+        vpsrlq	ymm2, ymm13, 43
+        vpsrlq	ymm3, ymm14, 50
+        vpsllq	ymm11, ymm11, 44
+        vpsllq	ymm12, ymm12, 43
+        vpsllq	ymm13, ymm13, 21
+        vpsllq	ymm14, ymm14, 14
+        vpor	ymm11, ymm11, ymm0
+        vpor	ymm12, ymm12, ymm1
+        vpor	ymm13, ymm13, ymm2
+        vpor	ymm14, ymm14, ymm3
+        vpandn	ymm15, ymm11, ymm12
+        vpandn	ymm1, ymm12, ymm13
+        vpandn	ymm2, ymm13, ymm14
+        vpandn	ymm3, ymm14, ymm10
+        vpandn	ymm4, ymm10, ymm11
+        vpxor	ymm15, ymm15, ymm10
+        vpxor	ymm1, ymm1, ymm11
+        vpxor	ymm2, ymm2, ymm12
+        ; XOR in constant
+        vpxor	ymm15, ymm15, [rax+736]
+        vpxor	ymm3, ymm3, ymm13
+        vpxor	ymm4, ymm4, ymm14
+        vmovdqu	YMMWORD PTR [rcx+-96], ymm1
+        vmovdqu	YMMWORD PTR [rcx+-64], ymm2
+        vmovdqu	YMMWORD PTR [rcx+-32], ymm3
+        vmovdqu	YMMWORD PTR [rcx], ymm4
+        ; Row 1
+        vpxor	ymm10, ymm8, [rcx+32]
+        vpxor	ymm11, ymm9, [rcx+64]
+        vpxor	ymm12, ymm5, [rcx+96]
+        vpxor	ymm13, ymm6, [rcx+128]
+        vpxor	ymm14, ymm7, [r9+-96]
+        vpsrlq	ymm0, ymm10, 36
+        vpsrlq	ymm1, ymm11, 44
+        vpsrlq	ymm2, ymm12, 61
+        vpsrlq	ymm3, ymm13, 19
+        vpsrlq	ymm4, ymm14, 3
+        vpsllq	ymm10, ymm10, 28
+        vpsllq	ymm11, ymm11, 20
+        vpsllq	ymm12, ymm12, 3
+        vpsllq	ymm13, ymm13, 45
+        vpsllq	ymm14, ymm14, 61
+        vpor	ymm10, ymm10, ymm0
+        vpor	ymm11, ymm11, ymm1
+        vpor	ymm12, ymm12, ymm2
+        vpor	ymm13, ymm13, ymm3
+        vpor	ymm14, ymm14, ymm4
+        vpandn	ymm0, ymm11, ymm12
+        vpandn	ymm1, ymm12, ymm13
+        vpandn	ymm2, ymm13, ymm14
+        vpandn	ymm3, ymm14, ymm10
+        vpandn	ymm4, ymm10, ymm11
+        vpxor	ymm0, ymm0, ymm10
+        vpxor	ymm1, ymm1, ymm11
+        vpxor	ymm2, ymm2, ymm12
+        vpxor	ymm3, ymm3, ymm13
+        vpxor	ymm4, ymm4, ymm14
+        vmovdqu	YMMWORD PTR [rcx+32], ymm0
+        vmovdqu	YMMWORD PTR [rcx+64], ymm1
+        vmovdqu	YMMWORD PTR [rcx+96], ymm2
+        vmovdqu	YMMWORD PTR [rcx+128], ymm3
+        vmovdqu	YMMWORD PTR [r9+-96], ymm4
+        ; Row 2
+        vpxor	ymm10, ymm6, [r9+-64]
+        vpxor	ymm11, ymm7, [r9+-32]
+        vpxor	ymm12, ymm8, [r9]
+        vpxor	ymm13, ymm9, [r9+32]
+        vpxor	ymm14, ymm5, [r9+64]
+        vpsrlq	ymm0, ymm10, 63
+        vpsrlq	ymm1, ymm11, 58
+        vpsrlq	ymm2, ymm12, 39
+        vpsrlq	ymm3, ymm13, 56
+        vpsrlq	ymm4, ymm14, 46
+        vpaddq	ymm10, ymm10, ymm10
+        vpsllq	ymm11, ymm11, 6
+        vpsllq	ymm12, ymm12, 25
+        vpsllq	ymm13, ymm13, 8
+        vpsllq	ymm14, ymm14, 18
+        vpor	ymm10, ymm10, ymm0
+        vpor	ymm11, ymm11, ymm1
+        vpor	ymm12, ymm12, ymm2
+        vpor	ymm13, ymm13, ymm3
+        vpor	ymm14, ymm14, ymm4
+        vpandn	ymm0, ymm11, ymm12
+        vpandn	ymm1, ymm12, ymm13
+        vpandn	ymm2, ymm13, ymm14
+        vpandn	ymm3, ymm14, ymm10
+        vpandn	ymm4, ymm10, ymm11
+        vpxor	ymm0, ymm0, ymm10
+        vpxor	ymm1, ymm1, ymm11
+        vpxor	ymm2, ymm2, ymm12
+        vpxor	ymm3, ymm3, ymm13
+        vpxor	ymm4, ymm4, ymm14
+        vmovdqu	YMMWORD PTR [r9+-64], ymm0
+        vmovdqu	YMMWORD PTR [r9+-32], ymm1
+        vmovdqu	YMMWORD PTR [r9], ymm2
+        vmovdqu	YMMWORD PTR [r9+32], ymm3
+        vmovdqu	YMMWORD PTR [r9+64], ymm4
+        ; Row 3
+        vpxor	ymm10, ymm9, [r9+96]
+        vpxor	ymm11, ymm5, [r9+128]
+        vpxor	ymm12, ymm6, [r10+-96]
+        vpxor	ymm13, ymm7, [r10+-64]
+        vpxor	ymm14, ymm8, [r10+-32]
+        vpsrlq	ymm0, ymm10, 37
+        vpsrlq	ymm1, ymm11, 28
+        vpsrlq	ymm2, ymm12, 54
+        vpsrlq	ymm3, ymm13, 49
+        vpsrlq	ymm4, ymm14, 8
+        vpsllq	ymm10, ymm10, 27
+        vpsllq	ymm11, ymm11, 36
+        vpsllq	ymm12, ymm12, 10
+        vpsllq	ymm13, ymm13, 15
+        vpsllq	ymm14, ymm14, 56
+        vpor	ymm10, ymm10, ymm0
+        vpor	ymm11, ymm11, ymm1
+        vpor	ymm12, ymm12, ymm2
+        vpor	ymm13, ymm13, ymm3
+        vpor	ymm14, ymm14, ymm4
+        vpandn	ymm0, ymm11, ymm12
+        vpandn	ymm1, ymm12, ymm13
+        vpandn	ymm2, ymm13, ymm14
+        vpandn	ymm3, ymm14, ymm10
+        vpandn	ymm4, ymm10, ymm11
+        vpxor	ymm0, ymm0, ymm10
+        vpxor	ymm1, ymm1, ymm11
+        vpxor	ymm2, ymm2, ymm12
+        vpxor	ymm3, ymm3, ymm13
+        vpxor	ymm4, ymm4, ymm14
+        vmovdqu	YMMWORD PTR [r9+96], ymm0
+        vmovdqu	YMMWORD PTR [r9+128], ymm1
+        vmovdqu	YMMWORD PTR [r10+-96], ymm2
+        vmovdqu	YMMWORD PTR [r10+-64], ymm3
+        vmovdqu	YMMWORD PTR [r10+-32], ymm4
+        ; Row 4
+        vpxor	ymm10, ymm7, [r10]
+        vpxor	ymm11, ymm8, [r10+32]
+        vpxor	ymm12, ymm9, [r10+64]
+        vpxor	ymm13, ymm5, [r10+96]
+        vpxor	ymm14, ymm6, [r10+128]
+        vpsrlq	ymm0, ymm10, 2
+        vpsrlq	ymm1, ymm11, 9
+        vpsrlq	ymm2, ymm12, 25
+        vpsrlq	ymm3, ymm13, 23
+        vpsrlq	ymm4, ymm14, 62
+        vpsllq	ymm10, ymm10, 62
+        vpsllq	ymm11, ymm11, 55
+        vpsllq	ymm12, ymm12, 39
+        vpsllq	ymm13, ymm13, 41
+        vpsllq	ymm14, ymm14, 2
+        vpor	ymm10, ymm10, ymm0
+        vpor	ymm11, ymm11, ymm1
+        vpor	ymm12, ymm12, ymm2
+        vpor	ymm13, ymm13, ymm3
+        vpor	ymm14, ymm14, ymm4
+        vpandn	ymm0, ymm11, ymm12
+        vpandn	ymm1, ymm12, ymm13
+        vpandn	ymm2, ymm13, ymm14
+        vpandn	ymm3, ymm14, ymm10
+        vpandn	ymm4, ymm10, ymm11
+        vpxor	ymm0, ymm0, ymm10
+        vpxor	ymm1, ymm1, ymm11
+        vpxor	ymm2, ymm2, ymm12
+        vpxor	ymm3, ymm3, ymm13
+        vpxor	ymm4, ymm4, ymm14
+        vmovdqu	YMMWORD PTR [r10], ymm0
+        vmovdqu	YMMWORD PTR [r10+32], ymm1
+        vmovdqu	YMMWORD PTR [r10+64], ymm2
+        vmovdqu	YMMWORD PTR [r10+96], ymm3
+        vmovdqu	YMMWORD PTR [r10+128], ymm4
+        sub	rcx, 128
+        vmovdqu	YMMWORD PTR [rcx], ymm15
+        lea	rdi, QWORD PTR [rcx]
+        mov	r15, 21
+L_sha3_blocksx4_out_avx2_word:
+        mov	rsi, QWORD PTR [rdi]
+        mov	QWORD PTR [rdx], rsi
+        mov	rsi, QWORD PTR [rdi+8]
+        mov	QWORD PTR [r11], rsi
+        mov	rsi, QWORD PTR [rdi+16]
+        mov	QWORD PTR [r12], rsi
+        mov	rsi, QWORD PTR [rdi+24]
+        mov	QWORD PTR [r13], rsi
+        add	rdi, 32
+        add	rdx, 8
+        add	r11, 8
+        add	r12, 8
+        add	r13, 8
+        sub	r14, 1
+        jz	L_sha3_blocksx4_out_avx2_done
+        sub	r15, 1
+        jnz	L_sha3_blocksx4_out_avx2_word
+        jmp	L_sha3_blocksx4_out_avx2_block
+L_sha3_blocksx4_out_avx2_done:
+        vzeroupper
+        vmovdqu	xmm6, OWORD PTR [rsp]
+        vmovdqu	xmm7, OWORD PTR [rsp+16]
+        vmovdqu	xmm8, OWORD PTR [rsp+32]
+        vmovdqu	xmm9, OWORD PTR [rsp+48]
+        vmovdqu	xmm10, OWORD PTR [rsp+64]
+        vmovdqu	xmm11, OWORD PTR [rsp+80]
+        vmovdqu	xmm12, OWORD PTR [rsp+96]
+        vmovdqu	xmm13, OWORD PTR [rsp+112]
+        vmovdqu	xmm14, OWORD PTR [rsp+128]
+        vmovdqu	xmm15, OWORD PTR [rsp+144]
+        add	rsp, 160
+        pop	rsi
+        pop	rdi
+        pop	r15
+        pop	r14
+        pop	r13
+        pop	r12
+        ret
+sha3_blocksx4_out_avx2 ENDP
+_TEXT ENDS
+ENDIF
+IFDEF WOLFSSL_HAVE_MLKEM
+_DATA SEGMENT
+ALIGN 16
+L_sha3_256_blockx4_seed_avx2_end_mark QWORD 8000000000000000h, 8000000000000000h
+        QWORD 8000000000000000h, 8000000000000000h
+ptr_L_sha3_256_blockx4_seed_avx2_end_mark QWORD L_sha3_256_blockx4_seed_avx2_end_mark
+_DATA ENDS
+_TEXT SEGMENT READONLY PARA
+sha3_256_blocksx4_seed_avx2 PROC
+        sub	rsp, 160
+        vmovdqu	OWORD PTR [rsp], xmm6
+        vmovdqu	OWORD PTR [rsp+16], xmm7
+        vmovdqu	OWORD PTR [rsp+32], xmm8
+        vmovdqu	OWORD PTR [rsp+48], xmm9
+        vmovdqu	OWORD PTR [rsp+64], xmm10
+        vmovdqu	OWORD PTR [rsp+80], xmm11
+        vmovdqu	OWORD PTR [rsp+96], xmm12
+        vmovdqu	OWORD PTR [rsp+112], xmm13
+        vmovdqu	OWORD PTR [rsp+128], xmm14
+        vmovdqu	OWORD PTR [rsp+144], xmm15
+        mov	rax, QWORD PTR [ptr_L_sha3_x4_avx2_r]
+        mov	r8, rcx
+        mov	r9, rcx
+        vpbroadcastq	ymm15, QWORD PTR [rdx]
+        add	rcx, 128
+        vpbroadcastq	ymm11, QWORD PTR [rdx+8]
+        add	r8, 384
+        vpbroadcastq	ymm12, QWORD PTR [rdx+16]
+        add	r9, 640
+        vpbroadcastq	ymm13, QWORD PTR [rdx+24]
+        vmovdqu	ymm5, YMMWORD PTR L_sha3_256_blockx4_seed_avx2_end_mark
+        vpxor	ymm6, ymm6, ymm6
+        vmovdqu	YMMWORD PTR [rcx+-96], ymm11
+        vmovdqu	YMMWORD PTR [rcx+-64], ymm12
+        vmovdqu	YMMWORD PTR [rcx+-32], ymm13
+        vmovdqu	ymm14, YMMWORD PTR [rcx]
+        vmovdqu	YMMWORD PTR [rcx+32], ymm6
+        vmovdqu	YMMWORD PTR [rcx+64], ymm6
+        vmovdqu	YMMWORD PTR [rcx+96], ymm6
+        vmovdqu	YMMWORD PTR [rcx+128], ymm6
+        vmovdqu	YMMWORD PTR [r8+-96], ymm6
+        vmovdqu	YMMWORD PTR [r8+-64], ymm6
+        vmovdqu	YMMWORD PTR [r8+-32], ymm6
+        vmovdqu	YMMWORD PTR [r8], ymm6
+        vmovdqu	YMMWORD PTR [r8+32], ymm6
+        vmovdqu	YMMWORD PTR [r8+64], ymm6
+        vmovdqu	YMMWORD PTR [r8+96], ymm6
+        vmovdqu	YMMWORD PTR [r8+128], ymm5
+        vmovdqu	YMMWORD PTR [r9+-96], ymm6
+        vmovdqu	YMMWORD PTR [r9+-64], ymm6
+        vmovdqu	YMMWORD PTR [r9+-32], ymm6
+        vmovdqu	YMMWORD PTR [r9], ymm6
+        vmovdqu	YMMWORD PTR [r9+32], ymm6
+        vmovdqu	YMMWORD PTR [r9+64], ymm6
+        vmovdqu	YMMWORD PTR [r9+96], ymm6
+        vmovdqu	YMMWORD PTR [r9+128], ymm6
+        vmovdqu	ymm10, ymm15
+        vpxor	ymm11, ymm11, ymm5
+        ; Round 0
+        ; Calc b[0..4]
+        ; Calc t[0..4]
+        vpsrlq	ymm0, ymm11, 63
+        vpsrlq	ymm1, ymm12, 63
+        vpsrlq	ymm2, ymm13, 63
+        vpsrlq	ymm3, ymm14, 63
+        vpsrlq	ymm4, ymm10, 63
+        vpaddq	ymm5, ymm11, ymm11
+        vpaddq	ymm6, ymm12, ymm12
+        vpaddq	ymm7, ymm13, ymm13
+        vpaddq	ymm8, ymm14, ymm14
+        vpaddq	ymm9, ymm10, ymm10
+        vpor	ymm5, ymm5, ymm0
+        vpor	ymm6, ymm6, ymm1
+        vpor	ymm7, ymm7, ymm2
+        vpor	ymm8, ymm8, ymm3
+        vpor	ymm9, ymm9, ymm4
+        vpxor	ymm5, ymm5, ymm14
+        vpxor	ymm6, ymm6, ymm10
+        vpxor	ymm7, ymm7, ymm11
+        vpxor	ymm8, ymm8, ymm12
+        vpxor	ymm9, ymm9, ymm13
+        ; Row Mix
+        ; Row 0
+        vpxor	ymm10, ymm5, ymm15
+        vpxor	ymm11, ymm6, [rcx+64]
+        vpxor	ymm12, ymm7, [r8]
+        vpxor	ymm13, ymm8, [r9+-64]
+        vpxor	ymm14, ymm9, [r9+128]
+        vpsrlq	ymm0, ymm11, 20
+        vpsrlq	ymm1, ymm12, 21
+        vpsrlq	ymm2, ymm13, 43
+        vpsrlq	ymm3, ymm14, 50
+        vpsllq	ymm11, ymm11, 44
+        vpsllq	ymm12, ymm12, 43
+        vpsllq	ymm13, ymm13, 21
+        vpsllq	ymm14, ymm14, 14
+        vpor	ymm11, ymm11, ymm0
+        vpor	ymm12, ymm12, ymm1
+        vpor	ymm13, ymm13, ymm2
+        vpor	ymm14, ymm14, ymm3
+        vpandn	ymm15, ymm11, ymm12
+        vpandn	ymm1, ymm12, ymm13
+        vpandn	ymm2, ymm13, ymm14
+        vpandn	ymm3, ymm14, ymm10
+        vpandn	ymm4, ymm10, ymm11
+        vpxor	ymm15, ymm15, ymm10
+        vpxor	ymm1, ymm1, ymm11
+        vpxor	ymm2, ymm2, ymm12
+        ; XOR in constant
+        vpxor	ymm15, ymm15, [rax]
+        vpxor	ymm3, ymm3, ymm13
+        vpxor	ymm4, ymm4, ymm14
+        vmovdqu	YMMWORD PTR [rcx+64], ymm1
+        vmovdqu	YMMWORD PTR [r8], ymm2
+        vmovdqu	YMMWORD PTR [r9+-64], ymm3
+        vmovdqu	YMMWORD PTR [r9+128], ymm4
+        ; Row 1
+        vpxor	ymm10, ymm8, [rcx+-32]
+        vpxor	ymm11, ymm9, [r8+-96]
+        vpxor	ymm12, ymm5, [r8+-64]
+        vpxor	ymm13, ymm6, [r8+128]
+        vpxor	ymm14, ymm7, [r9+64]
+        vpsrlq	ymm0, ymm10, 36
+        vpsrlq	ymm1, ymm11, 44
+        vpsrlq	ymm2, ymm12, 61
+        vpsrlq	ymm3, ymm13, 19
+        vpsrlq	ymm4, ymm14, 3
+        vpsllq	ymm10, ymm10, 28
+        vpsllq	ymm11, ymm11, 20
+        vpsllq	ymm12, ymm12, 3
+        vpsllq	ymm13, ymm13, 45
+        vpsllq	ymm14, ymm14, 61
+        vpor	ymm10, ymm10, ymm0
+        vpor	ymm11, ymm11, ymm1
+        vpor	ymm12, ymm12, ymm2
+        vpor	ymm13, ymm13, ymm3
+        vpor	ymm14, ymm14, ymm4
+        vpandn	ymm0, ymm11, ymm12
+        vpandn	ymm1, ymm12, ymm13
+        vpandn	ymm2, ymm13, ymm14
+        vpandn	ymm3, ymm14, ymm10
+        vpandn	ymm4, ymm10, ymm11
+        vpxor	ymm0, ymm0, ymm10
+        vpxor	ymm1, ymm1, ymm11
+        vpxor	ymm2, ymm2, ymm12
+        vpxor	ymm3, ymm3, ymm13
+        vpxor	ymm4, ymm4, ymm14
+        vmovdqu	YMMWORD PTR [rcx+-32], ymm0
+        vmovdqu	YMMWORD PTR [r8+-96], ymm1
+        vmovdqu	YMMWORD PTR [r8+-64], ymm2
+        vmovdqu	YMMWORD PTR [r8+128], ymm3
+        vmovdqu	YMMWORD PTR [r9+64], ymm4
+        ; Row 2
+        vpxor	ymm10, ymm6, [rcx+-96]
+        vpxor	ymm11, ymm7, [rcx+96]
+        vpxor	ymm12, ymm8, [r8+32]
+        vpxor	ymm13, ymm9, [r9+-32]
+        vpxor	ymm14, ymm5, [r9]
+        vpsrlq	ymm0, ymm10, 63
+        vpsrlq	ymm1, ymm11, 58
+        vpsrlq	ymm2, ymm12, 39
+        vpsrlq	ymm3, ymm13, 56
+        vpsrlq	ymm4, ymm14, 46
+        vpaddq	ymm10, ymm10, ymm10
+        vpsllq	ymm11, ymm11, 6
+        vpsllq	ymm12, ymm12, 25
+        vpsllq	ymm13, ymm13, 8
+        vpsllq	ymm14, ymm14, 18
+        vpor	ymm10, ymm10, ymm0
+        vpor	ymm11, ymm11, ymm1
+        vpor	ymm12, ymm12, ymm2
+        vpor	ymm13, ymm13, ymm3
+        vpor	ymm14, ymm14, ymm4
+        vpandn	ymm0, ymm11, ymm12
+        vpandn	ymm1, ymm12, ymm13
+        vpandn	ymm2, ymm13, ymm14
+        vpandn	ymm3, ymm14, ymm10
+        vpandn	ymm4, ymm10, ymm11
+        vpxor	ymm0, ymm0, ymm10
+        vpxor	ymm1, ymm1, ymm11
+        vpxor	ymm2, ymm2, ymm12
+        vpxor	ymm3, ymm3, ymm13
+        vpxor	ymm4, ymm4, ymm14
+        vmovdqu	YMMWORD PTR [rcx+-96], ymm0
+        vmovdqu	YMMWORD PTR [rcx+96], ymm1
+        vmovdqu	YMMWORD PTR [r8+32], ymm2
+        vmovdqu	YMMWORD PTR [r9+-32], ymm3
+        vmovdqu	YMMWORD PTR [r9], ymm4
+        ; Row 3
+        vpxor	ymm10, ymm9, [rcx]
+        vpxor	ymm11, ymm5, [rcx+32]
+        vpxor	ymm12, ymm6, [r8+-32]
+        vpxor	ymm13, ymm7, [r9+-96]
+        vpxor	ymm14, ymm8, [r9+96]
+        vpsrlq	ymm0, ymm10, 37
+        vpsrlq	ymm1, ymm11, 28
+        vpsrlq	ymm2, ymm12, 54
+        vpsrlq	ymm3, ymm13, 49
+        vpsrlq	ymm4, ymm14, 8
+        vpsllq	ymm10, ymm10, 27
+        vpsllq	ymm11, ymm11, 36
+        vpsllq	ymm12, ymm12, 10
+        vpsllq	ymm13, ymm13, 15
+        vpsllq	ymm14, ymm14, 56
+        vpor	ymm10, ymm10, ymm0
+        vpor	ymm11, ymm11, ymm1
+        vpor	ymm12, ymm12, ymm2
+        vpor	ymm13, ymm13, ymm3
+        vpor	ymm14, ymm14, ymm4
+        vpandn	ymm0, ymm11, ymm12
+        vpandn	ymm1, ymm12, ymm13
+        vpandn	ymm2, ymm13, ymm14
+        vpandn	ymm3, ymm14, ymm10
+        vpandn	ymm4, ymm10, ymm11
+        vpxor	ymm0, ymm0, ymm10
+        vpxor	ymm1, ymm1, ymm11
+        vpxor	ymm2, ymm2, ymm12
+        vpxor	ymm3, ymm3, ymm13
+        vpxor	ymm4, ymm4, ymm14
+        vmovdqu	YMMWORD PTR [rcx], ymm0
+        vmovdqu	YMMWORD PTR [rcx+32], ymm1
+        vmovdqu	YMMWORD PTR [r8+-32], ymm2
+        vmovdqu	YMMWORD PTR [r9+-96], ymm3
+        vmovdqu	YMMWORD PTR [r9+96], ymm4
+        ; Row 4
+        vpxor	ymm10, ymm7, [rcx+-64]
+        vpxor	ymm11, ymm8, [rcx+128]
+        vpxor	ymm12, ymm9, [r8+64]
+        vpxor	ymm13, ymm5, [r8+96]
+        vpxor	ymm14, ymm6, [r9+32]
+        vpsrlq	ymm0, ymm10, 2
+        vpsrlq	ymm1, ymm11, 9
+        vpsrlq	ymm2, ymm12, 25
+        vpsrlq	ymm3, ymm13, 23
+        vpsrlq	ymm4, ymm14, 62
+        vpsllq	ymm10, ymm10, 62
+        vpsllq	ymm11, ymm11, 55
+        vpsllq	ymm12, ymm12, 39
+        vpsllq	ymm13, ymm13, 41
+        vpsllq	ymm14, ymm14, 2
+        vpor	ymm10, ymm10, ymm0
+        vpor	ymm11, ymm11, ymm1
+        vpor	ymm12, ymm12, ymm2
+        vpor	ymm13, ymm13, ymm3
+        vpor	ymm14, ymm14, ymm4
+        vpandn	ymm0, ymm11, ymm12
+        vpandn	ymm1, ymm12, ymm13
+        vpandn	ymm2, ymm13, ymm14
+        vpandn	ymm3, ymm14, ymm10
+        vpandn	ymm4, ymm10, ymm11
+        vpxor	ymm0, ymm0, ymm10
+        vpxor	ymm1, ymm1, ymm11
+        vpxor	ymm2, ymm2, ymm12
+        vpxor	ymm3, ymm3, ymm13
+        vpxor	ymm4, ymm4, ymm14
+        vmovdqu	YMMWORD PTR [rcx+-64], ymm0
+        vmovdqu	YMMWORD PTR [rcx+128], ymm1
+        vmovdqu	YMMWORD PTR [r8+64], ymm2
+        vmovdqu	YMMWORD PTR [r8+96], ymm3
+        vmovdqu	YMMWORD PTR [r9+32], ymm4
+        ; Round 1
+        ; Calc b[0..4]
+        vpxor	ymm10, ymm0, ymm15
+        vpxor	ymm10, ymm10, [rcx+-96]
+        vpxor	ymm10, ymm10, [rcx+-32]
+        vpxor	ymm10, ymm10, [rcx]
+        vpxor	ymm11, ymm1, [rcx+32]
+        vpxor	ymm11, ymm11, [rcx+64]
+        vpxor	ymm11, ymm11, [rcx+96]
+        vpxor	ymm11, ymm11, [r8+-96]
+        vpxor	ymm12, ymm2, [r8+-64]
+        vpxor	ymm12, ymm12, [r8+-32]
+        vpxor	ymm12, ymm12, [r8]
+        vpxor	ymm12, ymm12, [r8+32]
+        vpxor	ymm13, ymm3, [r8+128]
+        vpxor	ymm13, ymm13, [r9+-96]
+        vpxor	ymm13, ymm13, [r9+-64]
+        vpxor	ymm13, ymm13, [r9+-32]
+        vpxor	ymm14, ymm4, [r9]
+        vpxor	ymm14, ymm14, [r9+64]
+        vpxor	ymm14, ymm14, [r9+96]
+        vpxor	ymm14, ymm14, [r9+128]
+        ; Calc t[0..4]
+        vpsrlq	ymm0, ymm11, 63
+        vpsrlq	ymm1, ymm12, 63
+        vpsrlq	ymm2, ymm13, 63
+        vpsrlq	ymm3, ymm14, 63
+        vpsrlq	ymm4, ymm10, 63
+        vpaddq	ymm5, ymm11, ymm11
+        vpaddq	ymm6, ymm12, ymm12
+        vpaddq	ymm7, ymm13, ymm13
+        vpaddq	ymm8, ymm14, ymm14
+        vpaddq	ymm9, ymm10, ymm10
+        vpor	ymm5, ymm5, ymm0
+        vpor	ymm6, ymm6, ymm1
+        vpor	ymm7, ymm7, ymm2
+        vpor	ymm8, ymm8, ymm3
+        vpor	ymm9, ymm9, ymm4
+        vpxor	ymm5, ymm5, ymm14
+        vpxor	ymm6, ymm6, ymm10
+        vpxor	ymm7, ymm7, ymm11
+        vpxor	ymm8, ymm8, ymm12
+        vpxor	ymm9, ymm9, ymm13
+        ; Row Mix
+        ; Row 0
+        vpxor	ymm10, ymm5, ymm15
+        vpxor	ymm11, ymm6, [r8+-96]
+        vpxor	ymm12, ymm7, [r8+32]
+        vpxor	ymm13, ymm8, [r9+-96]
+        vpxor	ymm14, ymm9, [r9+32]
+        vpsrlq	ymm0, ymm11, 20
+        vpsrlq	ymm1, ymm12, 21
+        vpsrlq	ymm2, ymm13, 43
+        vpsrlq	ymm3, ymm14, 50
+        vpsllq	ymm11, ymm11, 44
+        vpsllq	ymm12, ymm12, 43
+        vpsllq	ymm13, ymm13, 21
+        vpsllq	ymm14, ymm14, 14
+        vpor	ymm11, ymm11, ymm0
+        vpor	ymm12, ymm12, ymm1
+        vpor	ymm13, ymm13, ymm2
+        vpor	ymm14, ymm14, ymm3
+        vpandn	ymm15, ymm11, ymm12
+        vpandn	ymm1, ymm12, ymm13
+        vpandn	ymm2, ymm13, ymm14
+        vpandn	ymm3, ymm14, ymm10
+        vpandn	ymm4, ymm10, ymm11
+        vpxor	ymm15, ymm15, ymm10
+        vpxor	ymm1, ymm1, ymm11
+        vpxor	ymm2, ymm2, ymm12
+        ; XOR in constant
+        vpxor	ymm15, ymm15, [rax+32]
+        vpxor	ymm3, ymm3, ymm13
+        vpxor	ymm4, ymm4, ymm14
+        vmovdqu	YMMWORD PTR [r8+-96], ymm1
+        vmovdqu	YMMWORD PTR [r8+32], ymm2
+        vmovdqu	YMMWORD PTR [r9+-96], ymm3
+        vmovdqu	YMMWORD PTR [r9+32], ymm4
+        ; Row 1
+        vpxor	ymm10, ymm8, [r9+-64]
+        vpxor	ymm11, ymm9, [r9+64]
+        vpxor	ymm12, ymm5, [rcx+-96]
+        vpxor	ymm13, ymm6, [rcx+32]
+        vpxor	ymm14, ymm7, [r8+64]
+        vpsrlq	ymm0, ymm10, 36
+        vpsrlq	ymm1, ymm11, 44
+        vpsrlq	ymm2, ymm12, 61
+        vpsrlq	ymm3, ymm13, 19
+        vpsrlq	ymm4, ymm14, 3
+        vpsllq	ymm10, ymm10, 28
+        vpsllq	ymm11, ymm11, 20
+        vpsllq	ymm12, ymm12, 3
+        vpsllq	ymm13, ymm13, 45
+        vpsllq	ymm14, ymm14, 61
+        vpor	ymm10, ymm10, ymm0
+        vpor	ymm11, ymm11, ymm1
+        vpor	ymm12, ymm12, ymm2
+        vpor	ymm13, ymm13, ymm3
+        vpor	ymm14, ymm14, ymm4
+        vpandn	ymm0, ymm11, ymm12
+        vpandn	ymm1, ymm12, ymm13
+        vpandn	ymm2, ymm13, ymm14
+        vpandn	ymm3, ymm14, ymm10
+        vpandn	ymm4, ymm10, ymm11
+        vpxor	ymm0, ymm0, ymm10
+        vpxor	ymm1, ymm1, ymm11
+        vpxor	ymm2, ymm2, ymm12
+        vpxor	ymm3, ymm3, ymm13
+        vpxor	ymm4, ymm4, ymm14
+        vmovdqu	YMMWORD PTR [r9+-64], ymm0
+        vmovdqu	YMMWORD PTR [r9+64], ymm1
+        vmovdqu	YMMWORD PTR [rcx+-96], ymm2
+        vmovdqu	YMMWORD PTR [rcx+32], ymm3
+        vmovdqu	YMMWORD PTR [r8+64], ymm4
+        ; Row 2
+        vpxor	ymm10, ymm6, [rcx+64]
+        vpxor	ymm11, ymm7, [r8+-64]
+        vpxor	ymm12, ymm8, [r9+-32]
+        vpxor	ymm13, ymm9, [r9+96]
+        vpxor	ymm14, ymm5, [rcx+-64]
+        vpsrlq	ymm0, ymm10, 63
+        vpsrlq	ymm1, ymm11, 58
+        vpsrlq	ymm2, ymm12, 39
+        vpsrlq	ymm3, ymm13, 56
+        vpsrlq	ymm4, ymm14, 46
+        vpaddq	ymm10, ymm10, ymm10
+        vpsllq	ymm11, ymm11, 6
+        vpsllq	ymm12, ymm12, 25
+        vpsllq	ymm13, ymm13, 8
+        vpsllq	ymm14, ymm14, 18
+        vpor	ymm10, ymm10, ymm0
+        vpor	ymm11, ymm11, ymm1
+        vpor	ymm12, ymm12, ymm2
+        vpor	ymm13, ymm13, ymm3
+        vpor	ymm14, ymm14, ymm4
+        vpandn	ymm0, ymm11, ymm12
+        vpandn	ymm1, ymm12, ymm13
+        vpandn	ymm2, ymm13, ymm14
+        vpandn	ymm3, ymm14, ymm10
+        vpandn	ymm4, ymm10, ymm11
+        vpxor	ymm0, ymm0, ymm10
+        vpxor	ymm1, ymm1, ymm11
+        vpxor	ymm2, ymm2, ymm12
+        vpxor	ymm3, ymm3, ymm13
+        vpxor	ymm4, ymm4, ymm14
+        vmovdqu	YMMWORD PTR [rcx+64], ymm0
+        vmovdqu	YMMWORD PTR [r8+-64], ymm1
+        vmovdqu	YMMWORD PTR [r9+-32], ymm2
+        vmovdqu	YMMWORD PTR [r9+96], ymm3
+        vmovdqu	YMMWORD PTR [rcx+-64], ymm4
+        ; Row 3
+        vpxor	ymm10, ymm9, [r9+128]
+        vpxor	ymm11, ymm5, [rcx+-32]
+        vpxor	ymm12, ymm6, [rcx+96]
+        vpxor	ymm13, ymm7, [r8+-32]
+        vpxor	ymm14, ymm8, [r8+96]
+        vpsrlq	ymm0, ymm10, 37
+        vpsrlq	ymm1, ymm11, 28
+        vpsrlq	ymm2, ymm12, 54
+        vpsrlq	ymm3, ymm13, 49
+        vpsrlq	ymm4, ymm14, 8
+        vpsllq	ymm10, ymm10, 27
+        vpsllq	ymm11, ymm11, 36
+        vpsllq	ymm12, ymm12, 10
+        vpsllq	ymm13, ymm13, 15
+        vpsllq	ymm14, ymm14, 56
+        vpor	ymm10, ymm10, ymm0
+        vpor	ymm11, ymm11, ymm1
+        vpor	ymm12, ymm12, ymm2
+        vpor	ymm13, ymm13, ymm3
+        vpor	ymm14, ymm14, ymm4
+        vpandn	ymm0, ymm11, ymm12
+        vpandn	ymm1, ymm12, ymm13
+        vpandn	ymm2, ymm13, ymm14
+        vpandn	ymm3, ymm14, ymm10
+        vpandn	ymm4, ymm10, ymm11
+        vpxor	ymm0, ymm0, ymm10
+        vpxor	ymm1, ymm1, ymm11
+        vpxor	ymm2, ymm2, ymm12
+        vpxor	ymm3, ymm3, ymm13
+        vpxor	ymm4, ymm4, ymm14
+        vmovdqu	YMMWORD PTR [r9+128], ymm0
+        vmovdqu	YMMWORD PTR [rcx+-32], ymm1
+        vmovdqu	YMMWORD PTR [rcx+96], ymm2
+        vmovdqu	YMMWORD PTR [r8+-32], ymm3
+        vmovdqu	YMMWORD PTR [r8+96], ymm4
+        ; Row 4
+        vpxor	ymm10, ymm7, [r8]
+        vpxor	ymm11, ymm8, [r8+128]
+        vpxor	ymm12, ymm9, [r9]
+        vpxor	ymm13, ymm5, [rcx]
+        vpxor	ymm14, ymm6, [rcx+128]
+        vpsrlq	ymm0, ymm10, 2
+        vpsrlq	ymm1, ymm11, 9
+        vpsrlq	ymm2, ymm12, 25
+        vpsrlq	ymm3, ymm13, 23
+        vpsrlq	ymm4, ymm14, 62
+        vpsllq	ymm10, ymm10, 62
+        vpsllq	ymm11, ymm11, 55
+        vpsllq	ymm12, ymm12, 39
+        vpsllq	ymm13, ymm13, 41
+        vpsllq	ymm14, ymm14, 2
+        vpor	ymm10, ymm10, ymm0
+        vpor	ymm11, ymm11, ymm1
+        vpor	ymm12, ymm12, ymm2
+        vpor	ymm13, ymm13, ymm3
+        vpor	ymm14, ymm14, ymm4
+        vpandn	ymm0, ymm11, ymm12
+        vpandn	ymm1, ymm12, ymm13
+        vpandn	ymm2, ymm13, ymm14
+        vpandn	ymm3, ymm14, ymm10
+        vpandn	ymm4, ymm10, ymm11
+        vpxor	ymm0, ymm0, ymm10
+        vpxor	ymm1, ymm1, ymm11
+        vpxor	ymm2, ymm2, ymm12
+        vpxor	ymm3, ymm3, ymm13
+        vpxor	ymm4, ymm4, ymm14
+        vmovdqu	YMMWORD PTR [r8], ymm0
+        vmovdqu	YMMWORD PTR [r8+128], ymm1
+        vmovdqu	YMMWORD PTR [r9], ymm2
+        vmovdqu	YMMWORD PTR [rcx], ymm3
+        vmovdqu	YMMWORD PTR [rcx+128], ymm4
+        ; Round 2
+        ; Calc b[0..4]
+        vpxor	ymm10, ymm0, ymm15
+        vpxor	ymm12, ymm2, [rcx+-96]
+        vpxor	ymm14, ymm4, [rcx+-64]
+        vpxor	ymm11, ymm1, [rcx+-32]
+        vpxor	ymm13, ymm3, [rcx+32]
+        vpxor	ymm10, ymm10, [rcx+64]
+        vpxor	ymm12, ymm12, [rcx+96]
+        vpxor	ymm11, ymm11, [r8+-96]
+        vpxor	ymm11, ymm11, [r8+-64]
+        vpxor	ymm13, ymm13, [r8+-32]
+        vpxor	ymm12, ymm12, [r8+32]
+        vpxor	ymm14, ymm14, [r8+64]
+        vpxor	ymm14, ymm14, [r8+96]
+        vpxor	ymm13, ymm13, [r9+-96]
+        vpxor	ymm10, ymm10, [r9+-64]
+        vpxor	ymm12, ymm12, [r9+-32]
+        vpxor	ymm14, ymm14, [r9+32]
+        vpxor	ymm11, ymm11, [r9+64]
+        vpxor	ymm13, ymm13, [r9+96]
+        vpxor	ymm10, ymm10, [r9+128]
+        ; Calc t[0..4]
+        vpsrlq	ymm0, ymm11, 63
+        vpsrlq	ymm1, ymm12, 63
+        vpsrlq	ymm2, ymm13, 63
+        vpsrlq	ymm3, ymm14, 63
+        vpsrlq	ymm4, ymm10, 63
+        vpaddq	ymm5, ymm11, ymm11
+        vpaddq	ymm6, ymm12, ymm12
+        vpaddq	ymm7, ymm13, ymm13
+        vpaddq	ymm8, ymm14, ymm14
+        vpaddq	ymm9, ymm10, ymm10
+        vpor	ymm5, ymm5, ymm0
+        vpor	ymm6, ymm6, ymm1
+        vpor	ymm7, ymm7, ymm2
+        vpor	ymm8, ymm8, ymm3
+        vpor	ymm9, ymm9, ymm4
+        vpxor	ymm5, ymm5, ymm14
+        vpxor	ymm6, ymm6, ymm10
+        vpxor	ymm7, ymm7, ymm11
+        vpxor	ymm8, ymm8, ymm12
+        vpxor	ymm9, ymm9, ymm13
+        ; Row Mix
+        ; Row 0
+        vpxor	ymm10, ymm5, ymm15
+        vpxor	ymm11, ymm6, [r9+64]
+        vpxor	ymm12, ymm7, [r9+-32]
+        vpxor	ymm13, ymm8, [r8+-32]
+        vpxor	ymm14, ymm9, [rcx+128]
+        vpsrlq	ymm0, ymm11, 20
+        vpsrlq	ymm1, ymm12, 21
+        vpsrlq	ymm2, ymm13, 43
+        vpsrlq	ymm3, ymm14, 50
+        vpsllq	ymm11, ymm11, 44
+        vpsllq	ymm12, ymm12, 43
+        vpsllq	ymm13, ymm13, 21
+        vpsllq	ymm14, ymm14, 14
+        vpor	ymm11, ymm11, ymm0
+        vpor	ymm12, ymm12, ymm1
+        vpor	ymm13, ymm13, ymm2
+        vpor	ymm14, ymm14, ymm3
+        vpandn	ymm15, ymm11, ymm12
+        vpandn	ymm1, ymm12, ymm13
+        vpandn	ymm2, ymm13, ymm14
+        vpandn	ymm3, ymm14, ymm10
+        vpandn	ymm4, ymm10, ymm11
+        vpxor	ymm15, ymm15, ymm10
+        vpxor	ymm1, ymm1, ymm11
+        vpxor	ymm2, ymm2, ymm12
+        ; XOR in constant
+        vpxor	ymm15, ymm15, [rax+64]
+        vpxor	ymm3, ymm3, ymm13
+        vpxor	ymm4, ymm4, ymm14
+        vmovdqu	YMMWORD PTR [r9+64], ymm1
+        vmovdqu	YMMWORD PTR [r9+-32], ymm2
+        vmovdqu	YMMWORD PTR [r8+-32], ymm3
+        vmovdqu	YMMWORD PTR [rcx+128], ymm4
+        ; Row 1
+        vpxor	ymm10, ymm8, [r9+-96]
+        vpxor	ymm11, ymm9, [r8+64]
+        vpxor	ymm12, ymm5, [rcx+64]
+        vpxor	ymm13, ymm6, [rcx+-32]
+        vpxor	ymm14, ymm7, [r9]
+        vpsrlq	ymm0, ymm10, 36
+        vpsrlq	ymm1, ymm11, 44
+        vpsrlq	ymm2, ymm12, 61
+        vpsrlq	ymm3, ymm13, 19
+        vpsrlq	ymm4, ymm14, 3
+        vpsllq	ymm10, ymm10, 28
+        vpsllq	ymm11, ymm11, 20
+        vpsllq	ymm12, ymm12, 3
+        vpsllq	ymm13, ymm13, 45
+        vpsllq	ymm14, ymm14, 61
+        vpor	ymm10, ymm10, ymm0
+        vpor	ymm11, ymm11, ymm1
+        vpor	ymm12, ymm12, ymm2
+        vpor	ymm13, ymm13, ymm3
+        vpor	ymm14, ymm14, ymm4
+        vpandn	ymm0, ymm11, ymm12
+        vpandn	ymm1, ymm12, ymm13
+        vpandn	ymm2, ymm13, ymm14
+        vpandn	ymm3, ymm14, ymm10
+        vpandn	ymm4, ymm10, ymm11
+        vpxor	ymm0, ymm0, ymm10
+        vpxor	ymm1, ymm1, ymm11
+        vpxor	ymm2, ymm2, ymm12
+        vpxor	ymm3, ymm3, ymm13
+        vpxor	ymm4, ymm4, ymm14
+        vmovdqu	YMMWORD PTR [r9+-96], ymm0
+        vmovdqu	YMMWORD PTR [r8+64], ymm1
+        vmovdqu	YMMWORD PTR [rcx+64], ymm2
+        vmovdqu	YMMWORD PTR [rcx+-32], ymm3
+        vmovdqu	YMMWORD PTR [r9], ymm4
+        ; Row 2
+        vpxor	ymm10, ymm6, [r8+-96]
+        vpxor	ymm11, ymm7, [rcx+-96]
+        vpxor	ymm12, ymm8, [r9+96]
+        vpxor	ymm13, ymm9, [r8+96]
+        vpxor	ymm14, ymm5, [r8]
+        vpsrlq	ymm0, ymm10, 63
+        vpsrlq	ymm1, ymm11, 58
+        vpsrlq	ymm2, ymm12, 39
+        vpsrlq	ymm3, ymm13, 56
+        vpsrlq	ymm4, ymm14, 46
+        vpaddq	ymm10, ymm10, ymm10
+        vpsllq	ymm11, ymm11, 6
+        vpsllq	ymm12, ymm12, 25
+        vpsllq	ymm13, ymm13, 8
+        vpsllq	ymm14, ymm14, 18
+        vpor	ymm10, ymm10, ymm0
+        vpor	ymm11, ymm11, ymm1
+        vpor	ymm12, ymm12, ymm2
+        vpor	ymm13, ymm13, ymm3
+        vpor	ymm14, ymm14, ymm4
+        vpandn	ymm0, ymm11, ymm12
+        vpandn	ymm1, ymm12, ymm13
+        vpandn	ymm2, ymm13, ymm14
+        vpandn	ymm3, ymm14, ymm10
+        vpandn	ymm4, ymm10, ymm11
+        vpxor	ymm0, ymm0, ymm10
+        vpxor	ymm1, ymm1, ymm11
+        vpxor	ymm2, ymm2, ymm12
+        vpxor	ymm3, ymm3, ymm13
+        vpxor	ymm4, ymm4, ymm14
+        vmovdqu	YMMWORD PTR [r8+-96], ymm0
+        vmovdqu	YMMWORD PTR [rcx+-96], ymm1
+        vmovdqu	YMMWORD PTR [r9+96], ymm2
+        vmovdqu	YMMWORD PTR [r8+96], ymm3
+        vmovdqu	YMMWORD PTR [r8], ymm4
+        ; Row 3
+        vpxor	ymm10, ymm9, [r9+32]
+        vpxor	ymm11, ymm5, [r9+-64]
+        vpxor	ymm12, ymm6, [r8+-64]
+        vpxor	ymm13, ymm7, [rcx+96]
+        vpxor	ymm14, ymm8, [rcx]
+        vpsrlq	ymm0, ymm10, 37
+        vpsrlq	ymm1, ymm11, 28
+        vpsrlq	ymm2, ymm12, 54
+        vpsrlq	ymm3, ymm13, 49
+        vpsrlq	ymm4, ymm14, 8
+        vpsllq	ymm10, ymm10, 27
+        vpsllq	ymm11, ymm11, 36
+        vpsllq	ymm12, ymm12, 10
+        vpsllq	ymm13, ymm13, 15
+        vpsllq	ymm14, ymm14, 56
+        vpor	ymm10, ymm10, ymm0
+        vpor	ymm11, ymm11, ymm1
+        vpor	ymm12, ymm12, ymm2
+        vpor	ymm13, ymm13, ymm3
+        vpor	ymm14, ymm14, ymm4
+        vpandn	ymm0, ymm11, ymm12
+        vpandn	ymm1, ymm12, ymm13
+        vpandn	ymm2, ymm13, ymm14
+        vpandn	ymm3, ymm14, ymm10
+        vpandn	ymm4, ymm10, ymm11
+        vpxor	ymm0, ymm0, ymm10
+        vpxor	ymm1, ymm1, ymm11
+        vpxor	ymm2, ymm2, ymm12
+        vpxor	ymm3, ymm3, ymm13
+        vpxor	ymm4, ymm4, ymm14
+        vmovdqu	YMMWORD PTR [r9+32], ymm0
+        vmovdqu	YMMWORD PTR [r9+-64], ymm1
+        vmovdqu	YMMWORD PTR [r8+-64], ymm2
+        vmovdqu	YMMWORD PTR [rcx+96], ymm3
+        vmovdqu	YMMWORD PTR [rcx], ymm4
+        ; Row 4
+        vpxor	ymm10, ymm7, [r8+32]
+        vpxor	ymm11, ymm8, [rcx+32]
+        vpxor	ymm12, ymm9, [rcx+-64]
+        vpxor	ymm13, ymm5, [r9+128]
+        vpxor	ymm14, ymm6, [r8+128]
+        vpsrlq	ymm0, ymm10, 2
+        vpsrlq	ymm1, ymm11, 9
+        vpsrlq	ymm2, ymm12, 25
+        vpsrlq	ymm3, ymm13, 23
+        vpsrlq	ymm4, ymm14, 62
+        vpsllq	ymm10, ymm10, 62
+        vpsllq	ymm11, ymm11, 55
+        vpsllq	ymm12, ymm12, 39
+        vpsllq	ymm13, ymm13, 41
+        vpsllq	ymm14, ymm14, 2
+        vpor	ymm10, ymm10, ymm0
+        vpor	ymm11, ymm11, ymm1
+        vpor	ymm12, ymm12, ymm2
+        vpor	ymm13, ymm13, ymm3
+        vpor	ymm14, ymm14, ymm4
+        vpandn	ymm0, ymm11, ymm12
+        vpandn	ymm1, ymm12, ymm13
+        vpandn	ymm2, ymm13, ymm14
+        vpandn	ymm3, ymm14, ymm10
+        vpandn	ymm4, ymm10, ymm11
+        vpxor	ymm0, ymm0, ymm10
+        vpxor	ymm1, ymm1, ymm11
+        vpxor	ymm2, ymm2, ymm12
+        vpxor	ymm3, ymm3, ymm13
+        vpxor	ymm4, ymm4, ymm14
+        vmovdqu	YMMWORD PTR [r8+32], ymm0
+        vmovdqu	YMMWORD PTR [rcx+32], ymm1
+        vmovdqu	YMMWORD PTR [rcx+-64], ymm2
+        vmovdqu	YMMWORD PTR [r9+128], ymm3
+        vmovdqu	YMMWORD PTR [r8+128], ymm4
+        ; Round 3
+        ; Calc b[0..4]
+        vpxor	ymm10, ymm0, ymm15
+        vpxor	ymm11, ymm1, [rcx+-96]
+        vpxor	ymm13, ymm3, [rcx+-32]
+        vpxor	ymm14, ymm4, [rcx]
+        vpxor	ymm12, ymm2, [rcx+64]
+        vpxor	ymm13, ymm13, [rcx+96]
+        vpxor	ymm14, ymm14, [rcx+128]
+        vpxor	ymm10, ymm10, [r8+-96]
         vpxor	ymm12, ymm12, [r8+-64]
         vpxor	ymm13, ymm13, [r8+-32]
         vpxor	ymm14, ymm14, [r8]
@@ -31444,5 +36873,3063 @@ sha3_256_blocksx4_seed_64_avx2 PROC
 sha3_256_blocksx4_seed_64_avx2 ENDP
 _TEXT ENDS
 ENDIF
+IFDEF HAVE_INTEL_AVX512
+IFDEF WOLFSSL_HAVE_FRODOKEM
+_DATA SEGMENT
+ALIGN 16
+L_sha3_x8_avx512_r QWORD 0000000000000001h, 0000000000000001h
+        QWORD 0000000000000001h, 0000000000000001h
+        QWORD 0000000000000001h, 0000000000000001h
+        QWORD 0000000000000001h, 0000000000000001h
+        QWORD 0000000000008082h, 0000000000008082h
+        QWORD 0000000000008082h, 0000000000008082h
+        QWORD 0000000000008082h, 0000000000008082h
+        QWORD 0000000000008082h, 0000000000008082h
+        QWORD 800000000000808ah, 800000000000808ah
+        QWORD 800000000000808ah, 800000000000808ah
+        QWORD 800000000000808ah, 800000000000808ah
+        QWORD 800000000000808ah, 800000000000808ah
+        QWORD 8000000080008000h, 8000000080008000h
+        QWORD 8000000080008000h, 8000000080008000h
+        QWORD 8000000080008000h, 8000000080008000h
+        QWORD 8000000080008000h, 8000000080008000h
+        QWORD 000000000000808bh, 000000000000808bh
+        QWORD 000000000000808bh, 000000000000808bh
+        QWORD 000000000000808bh, 000000000000808bh
+        QWORD 000000000000808bh, 000000000000808bh
+        QWORD 0000000080000001h, 0000000080000001h
+        QWORD 0000000080000001h, 0000000080000001h
+        QWORD 0000000080000001h, 0000000080000001h
+        QWORD 0000000080000001h, 0000000080000001h
+        QWORD 8000000080008081h, 8000000080008081h
+        QWORD 8000000080008081h, 8000000080008081h
+        QWORD 8000000080008081h, 8000000080008081h
+        QWORD 8000000080008081h, 8000000080008081h
+        QWORD 8000000000008009h, 8000000000008009h
+        QWORD 8000000000008009h, 8000000000008009h
+        QWORD 8000000000008009h, 8000000000008009h
+        QWORD 8000000000008009h, 8000000000008009h
+        QWORD 000000000000008ah, 000000000000008ah
+        QWORD 000000000000008ah, 000000000000008ah
+        QWORD 000000000000008ah, 000000000000008ah
+        QWORD 000000000000008ah, 000000000000008ah
+        QWORD 0000000000000088h, 0000000000000088h
+        QWORD 0000000000000088h, 0000000000000088h
+        QWORD 0000000000000088h, 0000000000000088h
+        QWORD 0000000000000088h, 0000000000000088h
+        QWORD 0000000080008009h, 0000000080008009h
+        QWORD 0000000080008009h, 0000000080008009h
+        QWORD 0000000080008009h, 0000000080008009h
+        QWORD 0000000080008009h, 0000000080008009h
+        QWORD 000000008000000ah, 000000008000000ah
+        QWORD 000000008000000ah, 000000008000000ah
+        QWORD 000000008000000ah, 000000008000000ah
+        QWORD 000000008000000ah, 000000008000000ah
+        QWORD 000000008000808bh, 000000008000808bh
+        QWORD 000000008000808bh, 000000008000808bh
+        QWORD 000000008000808bh, 000000008000808bh
+        QWORD 000000008000808bh, 000000008000808bh
+        QWORD 800000000000008bh, 800000000000008bh
+        QWORD 800000000000008bh, 800000000000008bh
+        QWORD 800000000000008bh, 800000000000008bh
+        QWORD 800000000000008bh, 800000000000008bh
+        QWORD 8000000000008089h, 8000000000008089h
+        QWORD 8000000000008089h, 8000000000008089h
+        QWORD 8000000000008089h, 8000000000008089h
+        QWORD 8000000000008089h, 8000000000008089h
+        QWORD 8000000000008003h, 8000000000008003h
+        QWORD 8000000000008003h, 8000000000008003h
+        QWORD 8000000000008003h, 8000000000008003h
+        QWORD 8000000000008003h, 8000000000008003h
+        QWORD 8000000000008002h, 8000000000008002h
+        QWORD 8000000000008002h, 8000000000008002h
+        QWORD 8000000000008002h, 8000000000008002h
+        QWORD 8000000000008002h, 8000000000008002h
+        QWORD 8000000000000080h, 8000000000000080h
+        QWORD 8000000000000080h, 8000000000000080h
+        QWORD 8000000000000080h, 8000000000000080h
+        QWORD 8000000000000080h, 8000000000000080h
+        QWORD 000000000000800ah, 000000000000800ah
+        QWORD 000000000000800ah, 000000000000800ah
+        QWORD 000000000000800ah, 000000000000800ah
+        QWORD 000000000000800ah, 000000000000800ah
+        QWORD 800000008000000ah, 800000008000000ah
+        QWORD 800000008000000ah, 800000008000000ah
+        QWORD 800000008000000ah, 800000008000000ah
+        QWORD 800000008000000ah, 800000008000000ah
+        QWORD 8000000080008081h, 8000000080008081h
+        QWORD 8000000080008081h, 8000000080008081h
+        QWORD 8000000080008081h, 8000000080008081h
+        QWORD 8000000080008081h, 8000000080008081h
+        QWORD 8000000000008080h, 8000000000008080h
+        QWORD 8000000000008080h, 8000000000008080h
+        QWORD 8000000000008080h, 8000000000008080h
+        QWORD 8000000000008080h, 8000000000008080h
+        QWORD 0000000080000001h, 0000000080000001h
+        QWORD 0000000080000001h, 0000000080000001h
+        QWORD 0000000080000001h, 0000000080000001h
+        QWORD 0000000080000001h, 0000000080000001h
+        QWORD 8000000080008008h, 8000000080008008h
+        QWORD 8000000080008008h, 8000000080008008h
+        QWORD 8000000080008008h, 8000000080008008h
+        QWORD 8000000080008008h, 8000000080008008h
+ptr_L_sha3_x8_avx512_r QWORD L_sha3_x8_avx512_r
+_DATA ENDS
+_TEXT SEGMENT READONLY PARA
+sha3_blocksx8_out_avx512 PROC
+        push	r12
+        push	r13
+        push	r14
+        push	r15
+        push	rdi
+        push	rsi
+        push	rbx
+        push	rbp
+        sub	rsp, 160
+        vmovdqu	OWORD PTR [rsp], xmm6
+        vmovdqu	OWORD PTR [rsp+16], xmm7
+        vmovdqu	OWORD PTR [rsp+32], xmm8
+        vmovdqu	OWORD PTR [rsp+48], xmm9
+        vmovdqu	OWORD PTR [rsp+64], xmm10
+        vmovdqu	OWORD PTR [rsp+80], xmm11
+        vmovdqu	OWORD PTR [rsp+96], xmm12
+        vmovdqu	OWORD PTR [rsp+112], xmm13
+        vmovdqu	OWORD PTR [rsp+128], xmm14
+        vmovdqu	OWORD PTR [rsp+144], xmm15
+        mov	rax, QWORD PTR [ptr_L_sha3_x8_avx512_r]
+        mov	edi, r8d
+        mov	r9, rdx
+        add	r9, rdi
+        mov	r10, r9
+        add	r10, rdi
+        mov	r11, r10
+        add	r11, rdi
+        mov	r12, r11
+        add	r12, rdi
+        mov	r13, r12
+        add	r13, rdi
+        mov	r14, r13
+        add	r14, rdi
+        mov	r15, r14
+        add	r15, rdi
+        shr	rdi, 3
+        vmovdqu64	zmm0, [rcx]
+        vmovdqu64	zmm1, [rcx+64]
+        vmovdqu64	zmm2, [rcx+128]
+        vmovdqu64	zmm3, [rcx+192]
+        vmovdqu64	zmm4, [rcx+256]
+        vmovdqu64	zmm5, [rcx+320]
+        vmovdqu64	zmm6, [rcx+384]
+        vmovdqu64	zmm7, [rcx+448]
+        vmovdqu64	zmm8, [rcx+512]
+        vmovdqu64	zmm9, [rcx+576]
+        vmovdqu64	zmm10, [rcx+640]
+        vmovdqu64	zmm11, [rcx+704]
+        vmovdqu64	zmm12, [rcx+768]
+        vmovdqu64	zmm13, [rcx+832]
+        vmovdqu64	zmm14, [rcx+896]
+        vmovdqu64	zmm15, [rcx+960]
+        vmovdqu64	zmm16, [rcx+1024]
+        vmovdqu64	zmm17, [rcx+1088]
+        vmovdqu64	zmm18, [rcx+1152]
+        vmovdqu64	zmm19, [rcx+1216]
+        vmovdqu64	zmm20, [rcx+1280]
+        vmovdqu64	zmm21, [rcx+1344]
+        vmovdqu64	zmm22, [rcx+1408]
+        vmovdqu64	zmm23, [rcx+1472]
+        vmovdqu64	zmm24, [rcx+1536]
+L_sha3_blocksx8_out_avx512_block:
+        ; Round 0
+        ; Theta - column parity
+        vmovdqu64	zmm25, zmm0
+        vpternlogq	zmm25, zmm5, zmm10, 150
+        vpternlogq	zmm25, zmm15, zmm20, 150
+        vmovdqu64	zmm26, zmm1
+        vpternlogq	zmm26, zmm6, zmm11, 150
+        vpternlogq	zmm26, zmm16, zmm21, 150
+        vmovdqu64	zmm27, zmm2
+        vpternlogq	zmm27, zmm7, zmm12, 150
+        vpternlogq	zmm27, zmm17, zmm22, 150
+        vmovdqu64	zmm28, zmm3
+        vpternlogq	zmm28, zmm8, zmm13, 150
+        vpternlogq	zmm28, zmm18, zmm23, 150
+        vmovdqu64	zmm29, zmm4
+        vpternlogq	zmm29, zmm9, zmm14, 150
+        vpternlogq	zmm29, zmm19, zmm24, 150
+        ; Theta - D and apply in place
+        vprolq	zmm30, zmm26, 1
+        vpxorq	zmm30, zmm30, zmm29
+        vpxorq	zmm0, zmm0, zmm30
+        vpxorq	zmm5, zmm5, zmm30
+        vpxorq	zmm10, zmm10, zmm30
+        vpxorq	zmm15, zmm15, zmm30
+        vpxorq	zmm20, zmm20, zmm30
+        vprolq	zmm30, zmm27, 1
+        vpxorq	zmm30, zmm30, zmm25
+        vpxorq	zmm1, zmm1, zmm30
+        vpxorq	zmm6, zmm6, zmm30
+        vpxorq	zmm11, zmm11, zmm30
+        vpxorq	zmm16, zmm16, zmm30
+        vpxorq	zmm21, zmm21, zmm30
+        vprolq	zmm30, zmm28, 1
+        vpxorq	zmm30, zmm30, zmm26
+        vpxorq	zmm2, zmm2, zmm30
+        vpxorq	zmm7, zmm7, zmm30
+        vpxorq	zmm12, zmm12, zmm30
+        vpxorq	zmm17, zmm17, zmm30
+        vpxorq	zmm22, zmm22, zmm30
+        vprolq	zmm30, zmm29, 1
+        vpxorq	zmm30, zmm30, zmm27
+        vpxorq	zmm3, zmm3, zmm30
+        vpxorq	zmm8, zmm8, zmm30
+        vpxorq	zmm13, zmm13, zmm30
+        vpxorq	zmm18, zmm18, zmm30
+        vpxorq	zmm23, zmm23, zmm30
+        vprolq	zmm30, zmm25, 1
+        vpxorq	zmm30, zmm30, zmm28
+        vpxorq	zmm4, zmm4, zmm30
+        vpxorq	zmm9, zmm9, zmm30
+        vpxorq	zmm14, zmm14, zmm30
+        vpxorq	zmm19, zmm19, zmm30
+        vpxorq	zmm24, zmm24, zmm30
+        ; Rho - rotate each lane in place
+        vprolq	zmm1, zmm1, 1
+        vprolq	zmm2, zmm2, 62
+        vprolq	zmm3, zmm3, 28
+        vprolq	zmm4, zmm4, 27
+        vprolq	zmm5, zmm5, 36
+        vprolq	zmm6, zmm6, 44
+        vprolq	zmm7, zmm7, 6
+        vprolq	zmm8, zmm8, 55
+        vprolq	zmm9, zmm9, 20
+        vprolq	zmm10, zmm10, 3
+        vprolq	zmm11, zmm11, 10
+        vprolq	zmm12, zmm12, 43
+        vprolq	zmm13, zmm13, 25
+        vprolq	zmm14, zmm14, 39
+        vprolq	zmm15, zmm15, 41
+        vprolq	zmm16, zmm16, 45
+        vprolq	zmm17, zmm17, 15
+        vprolq	zmm18, zmm18, 21
+        vprolq	zmm19, zmm19, 8
+        vprolq	zmm20, zmm20, 18
+        vprolq	zmm21, zmm21, 2
+        vprolq	zmm22, zmm22, 61
+        vprolq	zmm23, zmm23, 56
+        vprolq	zmm24, zmm24, 14
+        ; Pi - relabel logical -> physical
+        ; Chi - per row, in place, fused with vpternlogq (imm 0xd2)
+        vmovdqu64	zmm30, zmm0
+        vmovdqu64	zmm31, zmm6
+        vpternlogq	zmm0, zmm6, zmm12, 210
+        vpternlogq	zmm6, zmm12, zmm18, 210
+        vpternlogq	zmm12, zmm18, zmm24, 210
+        vpternlogq	zmm18, zmm24, zmm30, 210
+        vpternlogq	zmm24, zmm30, zmm31, 210
+        vmovdqu64	zmm30, zmm3
+        vmovdqu64	zmm31, zmm9
+        vpternlogq	zmm3, zmm9, zmm10, 210
+        vpternlogq	zmm9, zmm10, zmm16, 210
+        vpternlogq	zmm10, zmm16, zmm22, 210
+        vpternlogq	zmm16, zmm22, zmm30, 210
+        vpternlogq	zmm22, zmm30, zmm31, 210
+        vmovdqu64	zmm30, zmm1
+        vmovdqu64	zmm31, zmm7
+        vpternlogq	zmm1, zmm7, zmm13, 210
+        vpternlogq	zmm7, zmm13, zmm19, 210
+        vpternlogq	zmm13, zmm19, zmm20, 210
+        vpternlogq	zmm19, zmm20, zmm30, 210
+        vpternlogq	zmm20, zmm30, zmm31, 210
+        vmovdqu64	zmm30, zmm4
+        vmovdqu64	zmm31, zmm5
+        vpternlogq	zmm4, zmm5, zmm11, 210
+        vpternlogq	zmm5, zmm11, zmm17, 210
+        vpternlogq	zmm11, zmm17, zmm23, 210
+        vpternlogq	zmm17, zmm23, zmm30, 210
+        vpternlogq	zmm23, zmm30, zmm31, 210
+        vmovdqu64	zmm30, zmm2
+        vmovdqu64	zmm31, zmm8
+        vpternlogq	zmm2, zmm8, zmm14, 210
+        vpternlogq	zmm8, zmm14, zmm15, 210
+        vpternlogq	zmm14, zmm15, zmm21, 210
+        vpternlogq	zmm15, zmm21, zmm30, 210
+        vpternlogq	zmm21, zmm30, zmm31, 210
+        ; Iota
+        vpxorq	zmm0, zmm0, [rax]
+        ; Round 1
+        ; Theta - column parity
+        vmovdqu64	zmm25, zmm0
+        vpternlogq	zmm25, zmm3, zmm1, 150
+        vpternlogq	zmm25, zmm4, zmm2, 150
+        vmovdqu64	zmm26, zmm6
+        vpternlogq	zmm26, zmm9, zmm7, 150
+        vpternlogq	zmm26, zmm5, zmm8, 150
+        vmovdqu64	zmm27, zmm12
+        vpternlogq	zmm27, zmm10, zmm13, 150
+        vpternlogq	zmm27, zmm11, zmm14, 150
+        vmovdqu64	zmm28, zmm18
+        vpternlogq	zmm28, zmm16, zmm19, 150
+        vpternlogq	zmm28, zmm17, zmm15, 150
+        vmovdqu64	zmm29, zmm24
+        vpternlogq	zmm29, zmm22, zmm20, 150
+        vpternlogq	zmm29, zmm23, zmm21, 150
+        ; Theta - D and apply in place
+        vprolq	zmm30, zmm26, 1
+        vpxorq	zmm30, zmm30, zmm29
+        vpxorq	zmm0, zmm0, zmm30
+        vpxorq	zmm3, zmm3, zmm30
+        vpxorq	zmm1, zmm1, zmm30
+        vpxorq	zmm4, zmm4, zmm30
+        vpxorq	zmm2, zmm2, zmm30
+        vprolq	zmm30, zmm27, 1
+        vpxorq	zmm30, zmm30, zmm25
+        vpxorq	zmm6, zmm6, zmm30
+        vpxorq	zmm9, zmm9, zmm30
+        vpxorq	zmm7, zmm7, zmm30
+        vpxorq	zmm5, zmm5, zmm30
+        vpxorq	zmm8, zmm8, zmm30
+        vprolq	zmm30, zmm28, 1
+        vpxorq	zmm30, zmm30, zmm26
+        vpxorq	zmm12, zmm12, zmm30
+        vpxorq	zmm10, zmm10, zmm30
+        vpxorq	zmm13, zmm13, zmm30
+        vpxorq	zmm11, zmm11, zmm30
+        vpxorq	zmm14, zmm14, zmm30
+        vprolq	zmm30, zmm29, 1
+        vpxorq	zmm30, zmm30, zmm27
+        vpxorq	zmm18, zmm18, zmm30
+        vpxorq	zmm16, zmm16, zmm30
+        vpxorq	zmm19, zmm19, zmm30
+        vpxorq	zmm17, zmm17, zmm30
+        vpxorq	zmm15, zmm15, zmm30
+        vprolq	zmm30, zmm25, 1
+        vpxorq	zmm30, zmm30, zmm28
+        vpxorq	zmm24, zmm24, zmm30
+        vpxorq	zmm22, zmm22, zmm30
+        vpxorq	zmm20, zmm20, zmm30
+        vpxorq	zmm23, zmm23, zmm30
+        vpxorq	zmm21, zmm21, zmm30
+        ; Rho - rotate each lane in place
+        vprolq	zmm6, zmm6, 1
+        vprolq	zmm12, zmm12, 62
+        vprolq	zmm18, zmm18, 28
+        vprolq	zmm24, zmm24, 27
+        vprolq	zmm3, zmm3, 36
+        vprolq	zmm9, zmm9, 44
+        vprolq	zmm10, zmm10, 6
+        vprolq	zmm16, zmm16, 55
+        vprolq	zmm22, zmm22, 20
+        vprolq	zmm1, zmm1, 3
+        vprolq	zmm7, zmm7, 10
+        vprolq	zmm13, zmm13, 43
+        vprolq	zmm19, zmm19, 25
+        vprolq	zmm20, zmm20, 39
+        vprolq	zmm4, zmm4, 41
+        vprolq	zmm5, zmm5, 45
+        vprolq	zmm11, zmm11, 15
+        vprolq	zmm17, zmm17, 21
+        vprolq	zmm23, zmm23, 8
+        vprolq	zmm2, zmm2, 18
+        vprolq	zmm8, zmm8, 2
+        vprolq	zmm14, zmm14, 61
+        vprolq	zmm15, zmm15, 56
+        vprolq	zmm21, zmm21, 14
+        ; Pi - relabel logical -> physical
+        ; Chi - per row, in place, fused with vpternlogq (imm 0xd2)
+        vmovdqu64	zmm30, zmm0
+        vmovdqu64	zmm31, zmm9
+        vpternlogq	zmm0, zmm9, zmm13, 210
+        vpternlogq	zmm9, zmm13, zmm17, 210
+        vpternlogq	zmm13, zmm17, zmm21, 210
+        vpternlogq	zmm17, zmm21, zmm30, 210
+        vpternlogq	zmm21, zmm30, zmm31, 210
+        vmovdqu64	zmm30, zmm18
+        vmovdqu64	zmm31, zmm22
+        vpternlogq	zmm18, zmm22, zmm1, 210
+        vpternlogq	zmm22, zmm1, zmm5, 210
+        vpternlogq	zmm1, zmm5, zmm14, 210
+        vpternlogq	zmm5, zmm14, zmm30, 210
+        vpternlogq	zmm14, zmm30, zmm31, 210
+        vmovdqu64	zmm30, zmm6
+        vmovdqu64	zmm31, zmm10
+        vpternlogq	zmm6, zmm10, zmm19, 210
+        vpternlogq	zmm10, zmm19, zmm23, 210
+        vpternlogq	zmm19, zmm23, zmm2, 210
+        vpternlogq	zmm23, zmm2, zmm30, 210
+        vpternlogq	zmm2, zmm30, zmm31, 210
+        vmovdqu64	zmm30, zmm24
+        vmovdqu64	zmm31, zmm3
+        vpternlogq	zmm24, zmm3, zmm7, 210
+        vpternlogq	zmm3, zmm7, zmm11, 210
+        vpternlogq	zmm7, zmm11, zmm15, 210
+        vpternlogq	zmm11, zmm15, zmm30, 210
+        vpternlogq	zmm15, zmm30, zmm31, 210
+        vmovdqu64	zmm30, zmm12
+        vmovdqu64	zmm31, zmm16
+        vpternlogq	zmm12, zmm16, zmm20, 210
+        vpternlogq	zmm16, zmm20, zmm4, 210
+        vpternlogq	zmm20, zmm4, zmm8, 210
+        vpternlogq	zmm4, zmm8, zmm30, 210
+        vpternlogq	zmm8, zmm30, zmm31, 210
+        ; Iota
+        vpxorq	zmm0, zmm0, [rax+64]
+        ; Round 2
+        ; Theta - column parity
+        vmovdqu64	zmm25, zmm0
+        vpternlogq	zmm25, zmm18, zmm6, 150
+        vpternlogq	zmm25, zmm24, zmm12, 150
+        vmovdqu64	zmm26, zmm9
+        vpternlogq	zmm26, zmm22, zmm10, 150
+        vpternlogq	zmm26, zmm3, zmm16, 150
+        vmovdqu64	zmm27, zmm13
+        vpternlogq	zmm27, zmm1, zmm19, 150
+        vpternlogq	zmm27, zmm7, zmm20, 150
+        vmovdqu64	zmm28, zmm17
+        vpternlogq	zmm28, zmm5, zmm23, 150
+        vpternlogq	zmm28, zmm11, zmm4, 150
+        vmovdqu64	zmm29, zmm21
+        vpternlogq	zmm29, zmm14, zmm2, 150
+        vpternlogq	zmm29, zmm15, zmm8, 150
+        ; Theta - D and apply in place
+        vprolq	zmm30, zmm26, 1
+        vpxorq	zmm30, zmm30, zmm29
+        vpxorq	zmm0, zmm0, zmm30
+        vpxorq	zmm18, zmm18, zmm30
+        vpxorq	zmm6, zmm6, zmm30
+        vpxorq	zmm24, zmm24, zmm30
+        vpxorq	zmm12, zmm12, zmm30
+        vprolq	zmm30, zmm27, 1
+        vpxorq	zmm30, zmm30, zmm25
+        vpxorq	zmm9, zmm9, zmm30
+        vpxorq	zmm22, zmm22, zmm30
+        vpxorq	zmm10, zmm10, zmm30
+        vpxorq	zmm3, zmm3, zmm30
+        vpxorq	zmm16, zmm16, zmm30
+        vprolq	zmm30, zmm28, 1
+        vpxorq	zmm30, zmm30, zmm26
+        vpxorq	zmm13, zmm13, zmm30
+        vpxorq	zmm1, zmm1, zmm30
+        vpxorq	zmm19, zmm19, zmm30
+        vpxorq	zmm7, zmm7, zmm30
+        vpxorq	zmm20, zmm20, zmm30
+        vprolq	zmm30, zmm29, 1
+        vpxorq	zmm30, zmm30, zmm27
+        vpxorq	zmm17, zmm17, zmm30
+        vpxorq	zmm5, zmm5, zmm30
+        vpxorq	zmm23, zmm23, zmm30
+        vpxorq	zmm11, zmm11, zmm30
+        vpxorq	zmm4, zmm4, zmm30
+        vprolq	zmm30, zmm25, 1
+        vpxorq	zmm30, zmm30, zmm28
+        vpxorq	zmm21, zmm21, zmm30
+        vpxorq	zmm14, zmm14, zmm30
+        vpxorq	zmm2, zmm2, zmm30
+        vpxorq	zmm15, zmm15, zmm30
+        vpxorq	zmm8, zmm8, zmm30
+        ; Rho - rotate each lane in place
+        vprolq	zmm9, zmm9, 1
+        vprolq	zmm13, zmm13, 62
+        vprolq	zmm17, zmm17, 28
+        vprolq	zmm21, zmm21, 27
+        vprolq	zmm18, zmm18, 36
+        vprolq	zmm22, zmm22, 44
+        vprolq	zmm1, zmm1, 6
+        vprolq	zmm5, zmm5, 55
+        vprolq	zmm14, zmm14, 20
+        vprolq	zmm6, zmm6, 3
+        vprolq	zmm10, zmm10, 10
+        vprolq	zmm19, zmm19, 43
+        vprolq	zmm23, zmm23, 25
+        vprolq	zmm2, zmm2, 39
+        vprolq	zmm24, zmm24, 41
+        vprolq	zmm3, zmm3, 45
+        vprolq	zmm7, zmm7, 15
+        vprolq	zmm11, zmm11, 21
+        vprolq	zmm15, zmm15, 8
+        vprolq	zmm12, zmm12, 18
+        vprolq	zmm16, zmm16, 2
+        vprolq	zmm20, zmm20, 61
+        vprolq	zmm4, zmm4, 56
+        vprolq	zmm8, zmm8, 14
+        ; Pi - relabel logical -> physical
+        ; Chi - per row, in place, fused with vpternlogq (imm 0xd2)
+        vmovdqu64	zmm30, zmm0
+        vmovdqu64	zmm31, zmm22
+        vpternlogq	zmm0, zmm22, zmm19, 210
+        vpternlogq	zmm22, zmm19, zmm11, 210
+        vpternlogq	zmm19, zmm11, zmm8, 210
+        vpternlogq	zmm11, zmm8, zmm30, 210
+        vpternlogq	zmm8, zmm30, zmm31, 210
+        vmovdqu64	zmm30, zmm17
+        vmovdqu64	zmm31, zmm14
+        vpternlogq	zmm17, zmm14, zmm6, 210
+        vpternlogq	zmm14, zmm6, zmm3, 210
+        vpternlogq	zmm6, zmm3, zmm20, 210
+        vpternlogq	zmm3, zmm20, zmm30, 210
+        vpternlogq	zmm20, zmm30, zmm31, 210
+        vmovdqu64	zmm30, zmm9
+        vmovdqu64	zmm31, zmm1
+        vpternlogq	zmm9, zmm1, zmm23, 210
+        vpternlogq	zmm1, zmm23, zmm15, 210
+        vpternlogq	zmm23, zmm15, zmm12, 210
+        vpternlogq	zmm15, zmm12, zmm30, 210
+        vpternlogq	zmm12, zmm30, zmm31, 210
+        vmovdqu64	zmm30, zmm21
+        vmovdqu64	zmm31, zmm18
+        vpternlogq	zmm21, zmm18, zmm10, 210
+        vpternlogq	zmm18, zmm10, zmm7, 210
+        vpternlogq	zmm10, zmm7, zmm4, 210
+        vpternlogq	zmm7, zmm4, zmm30, 210
+        vpternlogq	zmm4, zmm30, zmm31, 210
+        vmovdqu64	zmm30, zmm13
+        vmovdqu64	zmm31, zmm5
+        vpternlogq	zmm13, zmm5, zmm2, 210
+        vpternlogq	zmm5, zmm2, zmm24, 210
+        vpternlogq	zmm2, zmm24, zmm16, 210
+        vpternlogq	zmm24, zmm16, zmm30, 210
+        vpternlogq	zmm16, zmm30, zmm31, 210
+        ; Iota
+        vpxorq	zmm0, zmm0, [rax+128]
+        ; Round 3
+        ; Theta - column parity
+        vmovdqu64	zmm25, zmm0
+        vpternlogq	zmm25, zmm17, zmm9, 150
+        vpternlogq	zmm25, zmm21, zmm13, 150
+        vmovdqu64	zmm26, zmm22
+        vpternlogq	zmm26, zmm14, zmm1, 150
+        vpternlogq	zmm26, zmm18, zmm5, 150
+        vmovdqu64	zmm27, zmm19
+        vpternlogq	zmm27, zmm6, zmm23, 150
+        vpternlogq	zmm27, zmm10, zmm2, 150
+        vmovdqu64	zmm28, zmm11
+        vpternlogq	zmm28, zmm3, zmm15, 150
+        vpternlogq	zmm28, zmm7, zmm24, 150
+        vmovdqu64	zmm29, zmm8
+        vpternlogq	zmm29, zmm20, zmm12, 150
+        vpternlogq	zmm29, zmm4, zmm16, 150
+        ; Theta - D and apply in place
+        vprolq	zmm30, zmm26, 1
+        vpxorq	zmm30, zmm30, zmm29
+        vpxorq	zmm0, zmm0, zmm30
+        vpxorq	zmm17, zmm17, zmm30
+        vpxorq	zmm9, zmm9, zmm30
+        vpxorq	zmm21, zmm21, zmm30
+        vpxorq	zmm13, zmm13, zmm30
+        vprolq	zmm30, zmm27, 1
+        vpxorq	zmm30, zmm30, zmm25
+        vpxorq	zmm22, zmm22, zmm30
+        vpxorq	zmm14, zmm14, zmm30
+        vpxorq	zmm1, zmm1, zmm30
+        vpxorq	zmm18, zmm18, zmm30
+        vpxorq	zmm5, zmm5, zmm30
+        vprolq	zmm30, zmm28, 1
+        vpxorq	zmm30, zmm30, zmm26
+        vpxorq	zmm19, zmm19, zmm30
+        vpxorq	zmm6, zmm6, zmm30
+        vpxorq	zmm23, zmm23, zmm30
+        vpxorq	zmm10, zmm10, zmm30
+        vpxorq	zmm2, zmm2, zmm30
+        vprolq	zmm30, zmm29, 1
+        vpxorq	zmm30, zmm30, zmm27
+        vpxorq	zmm11, zmm11, zmm30
+        vpxorq	zmm3, zmm3, zmm30
+        vpxorq	zmm15, zmm15, zmm30
+        vpxorq	zmm7, zmm7, zmm30
+        vpxorq	zmm24, zmm24, zmm30
+        vprolq	zmm30, zmm25, 1
+        vpxorq	zmm30, zmm30, zmm28
+        vpxorq	zmm8, zmm8, zmm30
+        vpxorq	zmm20, zmm20, zmm30
+        vpxorq	zmm12, zmm12, zmm30
+        vpxorq	zmm4, zmm4, zmm30
+        vpxorq	zmm16, zmm16, zmm30
+        ; Rho - rotate each lane in place
+        vprolq	zmm22, zmm22, 1
+        vprolq	zmm19, zmm19, 62
+        vprolq	zmm11, zmm11, 28
+        vprolq	zmm8, zmm8, 27
+        vprolq	zmm17, zmm17, 36
+        vprolq	zmm14, zmm14, 44
+        vprolq	zmm6, zmm6, 6
+        vprolq	zmm3, zmm3, 55
+        vprolq	zmm20, zmm20, 20
+        vprolq	zmm9, zmm9, 3
+        vprolq	zmm1, zmm1, 10
+        vprolq	zmm23, zmm23, 43
+        vprolq	zmm15, zmm15, 25
+        vprolq	zmm12, zmm12, 39
+        vprolq	zmm21, zmm21, 41
+        vprolq	zmm18, zmm18, 45
+        vprolq	zmm10, zmm10, 15
+        vprolq	zmm7, zmm7, 21
+        vprolq	zmm4, zmm4, 8
+        vprolq	zmm13, zmm13, 18
+        vprolq	zmm5, zmm5, 2
+        vprolq	zmm2, zmm2, 61
+        vprolq	zmm24, zmm24, 56
+        vprolq	zmm16, zmm16, 14
+        ; Pi - relabel logical -> physical
+        ; Chi - per row, in place, fused with vpternlogq (imm 0xd2)
+        vmovdqu64	zmm30, zmm0
+        vmovdqu64	zmm31, zmm14
+        vpternlogq	zmm0, zmm14, zmm23, 210
+        vpternlogq	zmm14, zmm23, zmm7, 210
+        vpternlogq	zmm23, zmm7, zmm16, 210
+        vpternlogq	zmm7, zmm16, zmm30, 210
+        vpternlogq	zmm16, zmm30, zmm31, 210
+        vmovdqu64	zmm30, zmm11
+        vmovdqu64	zmm31, zmm20
+        vpternlogq	zmm11, zmm20, zmm9, 210
+        vpternlogq	zmm20, zmm9, zmm18, 210
+        vpternlogq	zmm9, zmm18, zmm2, 210
+        vpternlogq	zmm18, zmm2, zmm30, 210
+        vpternlogq	zmm2, zmm30, zmm31, 210
+        vmovdqu64	zmm30, zmm22
+        vmovdqu64	zmm31, zmm6
+        vpternlogq	zmm22, zmm6, zmm15, 210
+        vpternlogq	zmm6, zmm15, zmm4, 210
+        vpternlogq	zmm15, zmm4, zmm13, 210
+        vpternlogq	zmm4, zmm13, zmm30, 210
+        vpternlogq	zmm13, zmm30, zmm31, 210
+        vmovdqu64	zmm30, zmm8
+        vmovdqu64	zmm31, zmm17
+        vpternlogq	zmm8, zmm17, zmm1, 210
+        vpternlogq	zmm17, zmm1, zmm10, 210
+        vpternlogq	zmm1, zmm10, zmm24, 210
+        vpternlogq	zmm10, zmm24, zmm30, 210
+        vpternlogq	zmm24, zmm30, zmm31, 210
+        vmovdqu64	zmm30, zmm19
+        vmovdqu64	zmm31, zmm3
+        vpternlogq	zmm19, zmm3, zmm12, 210
+        vpternlogq	zmm3, zmm12, zmm21, 210
+        vpternlogq	zmm12, zmm21, zmm5, 210
+        vpternlogq	zmm21, zmm5, zmm30, 210
+        vpternlogq	zmm5, zmm30, zmm31, 210
+        ; Iota
+        vpxorq	zmm0, zmm0, [rax+192]
+        ; Round 4
+        ; Theta - column parity
+        vmovdqu64	zmm25, zmm0
+        vpternlogq	zmm25, zmm11, zmm22, 150
+        vpternlogq	zmm25, zmm8, zmm19, 150
+        vmovdqu64	zmm26, zmm14
+        vpternlogq	zmm26, zmm20, zmm6, 150
+        vpternlogq	zmm26, zmm17, zmm3, 150
+        vmovdqu64	zmm27, zmm23
+        vpternlogq	zmm27, zmm9, zmm15, 150
+        vpternlogq	zmm27, zmm1, zmm12, 150
+        vmovdqu64	zmm28, zmm7
+        vpternlogq	zmm28, zmm18, zmm4, 150
+        vpternlogq	zmm28, zmm10, zmm21, 150
+        vmovdqu64	zmm29, zmm16
+        vpternlogq	zmm29, zmm2, zmm13, 150
+        vpternlogq	zmm29, zmm24, zmm5, 150
+        ; Theta - D and apply in place
+        vprolq	zmm30, zmm26, 1
+        vpxorq	zmm30, zmm30, zmm29
+        vpxorq	zmm0, zmm0, zmm30
+        vpxorq	zmm11, zmm11, zmm30
+        vpxorq	zmm22, zmm22, zmm30
+        vpxorq	zmm8, zmm8, zmm30
+        vpxorq	zmm19, zmm19, zmm30
+        vprolq	zmm30, zmm27, 1
+        vpxorq	zmm30, zmm30, zmm25
+        vpxorq	zmm14, zmm14, zmm30
+        vpxorq	zmm20, zmm20, zmm30
+        vpxorq	zmm6, zmm6, zmm30
+        vpxorq	zmm17, zmm17, zmm30
+        vpxorq	zmm3, zmm3, zmm30
+        vprolq	zmm30, zmm28, 1
+        vpxorq	zmm30, zmm30, zmm26
+        vpxorq	zmm23, zmm23, zmm30
+        vpxorq	zmm9, zmm9, zmm30
+        vpxorq	zmm15, zmm15, zmm30
+        vpxorq	zmm1, zmm1, zmm30
+        vpxorq	zmm12, zmm12, zmm30
+        vprolq	zmm30, zmm29, 1
+        vpxorq	zmm30, zmm30, zmm27
+        vpxorq	zmm7, zmm7, zmm30
+        vpxorq	zmm18, zmm18, zmm30
+        vpxorq	zmm4, zmm4, zmm30
+        vpxorq	zmm10, zmm10, zmm30
+        vpxorq	zmm21, zmm21, zmm30
+        vprolq	zmm30, zmm25, 1
+        vpxorq	zmm30, zmm30, zmm28
+        vpxorq	zmm16, zmm16, zmm30
+        vpxorq	zmm2, zmm2, zmm30
+        vpxorq	zmm13, zmm13, zmm30
+        vpxorq	zmm24, zmm24, zmm30
+        vpxorq	zmm5, zmm5, zmm30
+        ; Rho - rotate each lane in place
+        vprolq	zmm14, zmm14, 1
+        vprolq	zmm23, zmm23, 62
+        vprolq	zmm7, zmm7, 28
+        vprolq	zmm16, zmm16, 27
+        vprolq	zmm11, zmm11, 36
+        vprolq	zmm20, zmm20, 44
+        vprolq	zmm9, zmm9, 6
+        vprolq	zmm18, zmm18, 55
+        vprolq	zmm2, zmm2, 20
+        vprolq	zmm22, zmm22, 3
+        vprolq	zmm6, zmm6, 10
+        vprolq	zmm15, zmm15, 43
+        vprolq	zmm4, zmm4, 25
+        vprolq	zmm13, zmm13, 39
+        vprolq	zmm8, zmm8, 41
+        vprolq	zmm17, zmm17, 45
+        vprolq	zmm1, zmm1, 15
+        vprolq	zmm10, zmm10, 21
+        vprolq	zmm24, zmm24, 8
+        vprolq	zmm19, zmm19, 18
+        vprolq	zmm3, zmm3, 2
+        vprolq	zmm12, zmm12, 61
+        vprolq	zmm21, zmm21, 56
+        vprolq	zmm5, zmm5, 14
+        ; Pi - relabel logical -> physical
+        ; Chi - per row, in place, fused with vpternlogq (imm 0xd2)
+        vmovdqu64	zmm30, zmm0
+        vmovdqu64	zmm31, zmm20
+        vpternlogq	zmm0, zmm20, zmm15, 210
+        vpternlogq	zmm20, zmm15, zmm10, 210
+        vpternlogq	zmm15, zmm10, zmm5, 210
+        vpternlogq	zmm10, zmm5, zmm30, 210
+        vpternlogq	zmm5, zmm30, zmm31, 210
+        vmovdqu64	zmm30, zmm7
+        vmovdqu64	zmm31, zmm2
+        vpternlogq	zmm7, zmm2, zmm22, 210
+        vpternlogq	zmm2, zmm22, zmm17, 210
+        vpternlogq	zmm22, zmm17, zmm12, 210
+        vpternlogq	zmm17, zmm12, zmm30, 210
+        vpternlogq	zmm12, zmm30, zmm31, 210
+        vmovdqu64	zmm30, zmm14
+        vmovdqu64	zmm31, zmm9
+        vpternlogq	zmm14, zmm9, zmm4, 210
+        vpternlogq	zmm9, zmm4, zmm24, 210
+        vpternlogq	zmm4, zmm24, zmm19, 210
+        vpternlogq	zmm24, zmm19, zmm30, 210
+        vpternlogq	zmm19, zmm30, zmm31, 210
+        vmovdqu64	zmm30, zmm16
+        vmovdqu64	zmm31, zmm11
+        vpternlogq	zmm16, zmm11, zmm6, 210
+        vpternlogq	zmm11, zmm6, zmm1, 210
+        vpternlogq	zmm6, zmm1, zmm21, 210
+        vpternlogq	zmm1, zmm21, zmm30, 210
+        vpternlogq	zmm21, zmm30, zmm31, 210
+        vmovdqu64	zmm30, zmm23
+        vmovdqu64	zmm31, zmm18
+        vpternlogq	zmm23, zmm18, zmm13, 210
+        vpternlogq	zmm18, zmm13, zmm8, 210
+        vpternlogq	zmm13, zmm8, zmm3, 210
+        vpternlogq	zmm8, zmm3, zmm30, 210
+        vpternlogq	zmm3, zmm30, zmm31, 210
+        ; Iota
+        vpxorq	zmm0, zmm0, [rax+256]
+        ; Round 5
+        ; Theta - column parity
+        vmovdqu64	zmm25, zmm0
+        vpternlogq	zmm25, zmm7, zmm14, 150
+        vpternlogq	zmm25, zmm16, zmm23, 150
+        vmovdqu64	zmm26, zmm20
+        vpternlogq	zmm26, zmm2, zmm9, 150
+        vpternlogq	zmm26, zmm11, zmm18, 150
+        vmovdqu64	zmm27, zmm15
+        vpternlogq	zmm27, zmm22, zmm4, 150
+        vpternlogq	zmm27, zmm6, zmm13, 150
+        vmovdqu64	zmm28, zmm10
+        vpternlogq	zmm28, zmm17, zmm24, 150
+        vpternlogq	zmm28, zmm1, zmm8, 150
+        vmovdqu64	zmm29, zmm5
+        vpternlogq	zmm29, zmm12, zmm19, 150
+        vpternlogq	zmm29, zmm21, zmm3, 150
+        ; Theta - D and apply in place
+        vprolq	zmm30, zmm26, 1
+        vpxorq	zmm30, zmm30, zmm29
+        vpxorq	zmm0, zmm0, zmm30
+        vpxorq	zmm7, zmm7, zmm30
+        vpxorq	zmm14, zmm14, zmm30
+        vpxorq	zmm16, zmm16, zmm30
+        vpxorq	zmm23, zmm23, zmm30
+        vprolq	zmm30, zmm27, 1
+        vpxorq	zmm30, zmm30, zmm25
+        vpxorq	zmm20, zmm20, zmm30
+        vpxorq	zmm2, zmm2, zmm30
+        vpxorq	zmm9, zmm9, zmm30
+        vpxorq	zmm11, zmm11, zmm30
+        vpxorq	zmm18, zmm18, zmm30
+        vprolq	zmm30, zmm28, 1
+        vpxorq	zmm30, zmm30, zmm26
+        vpxorq	zmm15, zmm15, zmm30
+        vpxorq	zmm22, zmm22, zmm30
+        vpxorq	zmm4, zmm4, zmm30
+        vpxorq	zmm6, zmm6, zmm30
+        vpxorq	zmm13, zmm13, zmm30
+        vprolq	zmm30, zmm29, 1
+        vpxorq	zmm30, zmm30, zmm27
+        vpxorq	zmm10, zmm10, zmm30
+        vpxorq	zmm17, zmm17, zmm30
+        vpxorq	zmm24, zmm24, zmm30
+        vpxorq	zmm1, zmm1, zmm30
+        vpxorq	zmm8, zmm8, zmm30
+        vprolq	zmm30, zmm25, 1
+        vpxorq	zmm30, zmm30, zmm28
+        vpxorq	zmm5, zmm5, zmm30
+        vpxorq	zmm12, zmm12, zmm30
+        vpxorq	zmm19, zmm19, zmm30
+        vpxorq	zmm21, zmm21, zmm30
+        vpxorq	zmm3, zmm3, zmm30
+        ; Rho - rotate each lane in place
+        vprolq	zmm20, zmm20, 1
+        vprolq	zmm15, zmm15, 62
+        vprolq	zmm10, zmm10, 28
+        vprolq	zmm5, zmm5, 27
+        vprolq	zmm7, zmm7, 36
+        vprolq	zmm2, zmm2, 44
+        vprolq	zmm22, zmm22, 6
+        vprolq	zmm17, zmm17, 55
+        vprolq	zmm12, zmm12, 20
+        vprolq	zmm14, zmm14, 3
+        vprolq	zmm9, zmm9, 10
+        vprolq	zmm4, zmm4, 43
+        vprolq	zmm24, zmm24, 25
+        vprolq	zmm19, zmm19, 39
+        vprolq	zmm16, zmm16, 41
+        vprolq	zmm11, zmm11, 45
+        vprolq	zmm6, zmm6, 15
+        vprolq	zmm1, zmm1, 21
+        vprolq	zmm21, zmm21, 8
+        vprolq	zmm23, zmm23, 18
+        vprolq	zmm18, zmm18, 2
+        vprolq	zmm13, zmm13, 61
+        vprolq	zmm8, zmm8, 56
+        vprolq	zmm3, zmm3, 14
+        ; Pi - relabel logical -> physical
+        ; Chi - per row, in place, fused with vpternlogq (imm 0xd2)
+        vmovdqu64	zmm30, zmm0
+        vmovdqu64	zmm31, zmm2
+        vpternlogq	zmm0, zmm2, zmm4, 210
+        vpternlogq	zmm2, zmm4, zmm1, 210
+        vpternlogq	zmm4, zmm1, zmm3, 210
+        vpternlogq	zmm1, zmm3, zmm30, 210
+        vpternlogq	zmm3, zmm30, zmm31, 210
+        vmovdqu64	zmm30, zmm10
+        vmovdqu64	zmm31, zmm12
+        vpternlogq	zmm10, zmm12, zmm14, 210
+        vpternlogq	zmm12, zmm14, zmm11, 210
+        vpternlogq	zmm14, zmm11, zmm13, 210
+        vpternlogq	zmm11, zmm13, zmm30, 210
+        vpternlogq	zmm13, zmm30, zmm31, 210
+        vmovdqu64	zmm30, zmm20
+        vmovdqu64	zmm31, zmm22
+        vpternlogq	zmm20, zmm22, zmm24, 210
+        vpternlogq	zmm22, zmm24, zmm21, 210
+        vpternlogq	zmm24, zmm21, zmm23, 210
+        vpternlogq	zmm21, zmm23, zmm30, 210
+        vpternlogq	zmm23, zmm30, zmm31, 210
+        vmovdqu64	zmm30, zmm5
+        vmovdqu64	zmm31, zmm7
+        vpternlogq	zmm5, zmm7, zmm9, 210
+        vpternlogq	zmm7, zmm9, zmm6, 210
+        vpternlogq	zmm9, zmm6, zmm8, 210
+        vpternlogq	zmm6, zmm8, zmm30, 210
+        vpternlogq	zmm8, zmm30, zmm31, 210
+        vmovdqu64	zmm30, zmm15
+        vmovdqu64	zmm31, zmm17
+        vpternlogq	zmm15, zmm17, zmm19, 210
+        vpternlogq	zmm17, zmm19, zmm16, 210
+        vpternlogq	zmm19, zmm16, zmm18, 210
+        vpternlogq	zmm16, zmm18, zmm30, 210
+        vpternlogq	zmm18, zmm30, zmm31, 210
+        ; Iota
+        vpxorq	zmm0, zmm0, [rax+320]
+        ; Round 6
+        ; Theta - column parity
+        vmovdqu64	zmm25, zmm0
+        vpternlogq	zmm25, zmm10, zmm20, 150
+        vpternlogq	zmm25, zmm5, zmm15, 150
+        vmovdqu64	zmm26, zmm2
+        vpternlogq	zmm26, zmm12, zmm22, 150
+        vpternlogq	zmm26, zmm7, zmm17, 150
+        vmovdqu64	zmm27, zmm4
+        vpternlogq	zmm27, zmm14, zmm24, 150
+        vpternlogq	zmm27, zmm9, zmm19, 150
+        vmovdqu64	zmm28, zmm1
+        vpternlogq	zmm28, zmm11, zmm21, 150
+        vpternlogq	zmm28, zmm6, zmm16, 150
+        vmovdqu64	zmm29, zmm3
+        vpternlogq	zmm29, zmm13, zmm23, 150
+        vpternlogq	zmm29, zmm8, zmm18, 150
+        ; Theta - D and apply in place
+        vprolq	zmm30, zmm26, 1
+        vpxorq	zmm30, zmm30, zmm29
+        vpxorq	zmm0, zmm0, zmm30
+        vpxorq	zmm10, zmm10, zmm30
+        vpxorq	zmm20, zmm20, zmm30
+        vpxorq	zmm5, zmm5, zmm30
+        vpxorq	zmm15, zmm15, zmm30
+        vprolq	zmm30, zmm27, 1
+        vpxorq	zmm30, zmm30, zmm25
+        vpxorq	zmm2, zmm2, zmm30
+        vpxorq	zmm12, zmm12, zmm30
+        vpxorq	zmm22, zmm22, zmm30
+        vpxorq	zmm7, zmm7, zmm30
+        vpxorq	zmm17, zmm17, zmm30
+        vprolq	zmm30, zmm28, 1
+        vpxorq	zmm30, zmm30, zmm26
+        vpxorq	zmm4, zmm4, zmm30
+        vpxorq	zmm14, zmm14, zmm30
+        vpxorq	zmm24, zmm24, zmm30
+        vpxorq	zmm9, zmm9, zmm30
+        vpxorq	zmm19, zmm19, zmm30
+        vprolq	zmm30, zmm29, 1
+        vpxorq	zmm30, zmm30, zmm27
+        vpxorq	zmm1, zmm1, zmm30
+        vpxorq	zmm11, zmm11, zmm30
+        vpxorq	zmm21, zmm21, zmm30
+        vpxorq	zmm6, zmm6, zmm30
+        vpxorq	zmm16, zmm16, zmm30
+        vprolq	zmm30, zmm25, 1
+        vpxorq	zmm30, zmm30, zmm28
+        vpxorq	zmm3, zmm3, zmm30
+        vpxorq	zmm13, zmm13, zmm30
+        vpxorq	zmm23, zmm23, zmm30
+        vpxorq	zmm8, zmm8, zmm30
+        vpxorq	zmm18, zmm18, zmm30
+        ; Rho - rotate each lane in place
+        vprolq	zmm2, zmm2, 1
+        vprolq	zmm4, zmm4, 62
+        vprolq	zmm1, zmm1, 28
+        vprolq	zmm3, zmm3, 27
+        vprolq	zmm10, zmm10, 36
+        vprolq	zmm12, zmm12, 44
+        vprolq	zmm14, zmm14, 6
+        vprolq	zmm11, zmm11, 55
+        vprolq	zmm13, zmm13, 20
+        vprolq	zmm20, zmm20, 3
+        vprolq	zmm22, zmm22, 10
+        vprolq	zmm24, zmm24, 43
+        vprolq	zmm21, zmm21, 25
+        vprolq	zmm23, zmm23, 39
+        vprolq	zmm5, zmm5, 41
+        vprolq	zmm7, zmm7, 45
+        vprolq	zmm9, zmm9, 15
+        vprolq	zmm6, zmm6, 21
+        vprolq	zmm8, zmm8, 8
+        vprolq	zmm15, zmm15, 18
+        vprolq	zmm17, zmm17, 2
+        vprolq	zmm19, zmm19, 61
+        vprolq	zmm16, zmm16, 56
+        vprolq	zmm18, zmm18, 14
+        ; Pi - relabel logical -> physical
+        ; Chi - per row, in place, fused with vpternlogq (imm 0xd2)
+        vmovdqu64	zmm30, zmm0
+        vmovdqu64	zmm31, zmm12
+        vpternlogq	zmm0, zmm12, zmm24, 210
+        vpternlogq	zmm12, zmm24, zmm6, 210
+        vpternlogq	zmm24, zmm6, zmm18, 210
+        vpternlogq	zmm6, zmm18, zmm30, 210
+        vpternlogq	zmm18, zmm30, zmm31, 210
+        vmovdqu64	zmm30, zmm1
+        vmovdqu64	zmm31, zmm13
+        vpternlogq	zmm1, zmm13, zmm20, 210
+        vpternlogq	zmm13, zmm20, zmm7, 210
+        vpternlogq	zmm20, zmm7, zmm19, 210
+        vpternlogq	zmm7, zmm19, zmm30, 210
+        vpternlogq	zmm19, zmm30, zmm31, 210
+        vmovdqu64	zmm30, zmm2
+        vmovdqu64	zmm31, zmm14
+        vpternlogq	zmm2, zmm14, zmm21, 210
+        vpternlogq	zmm14, zmm21, zmm8, 210
+        vpternlogq	zmm21, zmm8, zmm15, 210
+        vpternlogq	zmm8, zmm15, zmm30, 210
+        vpternlogq	zmm15, zmm30, zmm31, 210
+        vmovdqu64	zmm30, zmm3
+        vmovdqu64	zmm31, zmm10
+        vpternlogq	zmm3, zmm10, zmm22, 210
+        vpternlogq	zmm10, zmm22, zmm9, 210
+        vpternlogq	zmm22, zmm9, zmm16, 210
+        vpternlogq	zmm9, zmm16, zmm30, 210
+        vpternlogq	zmm16, zmm30, zmm31, 210
+        vmovdqu64	zmm30, zmm4
+        vmovdqu64	zmm31, zmm11
+        vpternlogq	zmm4, zmm11, zmm23, 210
+        vpternlogq	zmm11, zmm23, zmm5, 210
+        vpternlogq	zmm23, zmm5, zmm17, 210
+        vpternlogq	zmm5, zmm17, zmm30, 210
+        vpternlogq	zmm17, zmm30, zmm31, 210
+        ; Iota
+        vpxorq	zmm0, zmm0, [rax+384]
+        ; Round 7
+        ; Theta - column parity
+        vmovdqu64	zmm25, zmm0
+        vpternlogq	zmm25, zmm1, zmm2, 150
+        vpternlogq	zmm25, zmm3, zmm4, 150
+        vmovdqu64	zmm26, zmm12
+        vpternlogq	zmm26, zmm13, zmm14, 150
+        vpternlogq	zmm26, zmm10, zmm11, 150
+        vmovdqu64	zmm27, zmm24
+        vpternlogq	zmm27, zmm20, zmm21, 150
+        vpternlogq	zmm27, zmm22, zmm23, 150
+        vmovdqu64	zmm28, zmm6
+        vpternlogq	zmm28, zmm7, zmm8, 150
+        vpternlogq	zmm28, zmm9, zmm5, 150
+        vmovdqu64	zmm29, zmm18
+        vpternlogq	zmm29, zmm19, zmm15, 150
+        vpternlogq	zmm29, zmm16, zmm17, 150
+        ; Theta - D and apply in place
+        vprolq	zmm30, zmm26, 1
+        vpxorq	zmm30, zmm30, zmm29
+        vpxorq	zmm0, zmm0, zmm30
+        vpxorq	zmm1, zmm1, zmm30
+        vpxorq	zmm2, zmm2, zmm30
+        vpxorq	zmm3, zmm3, zmm30
+        vpxorq	zmm4, zmm4, zmm30
+        vprolq	zmm30, zmm27, 1
+        vpxorq	zmm30, zmm30, zmm25
+        vpxorq	zmm12, zmm12, zmm30
+        vpxorq	zmm13, zmm13, zmm30
+        vpxorq	zmm14, zmm14, zmm30
+        vpxorq	zmm10, zmm10, zmm30
+        vpxorq	zmm11, zmm11, zmm30
+        vprolq	zmm30, zmm28, 1
+        vpxorq	zmm30, zmm30, zmm26
+        vpxorq	zmm24, zmm24, zmm30
+        vpxorq	zmm20, zmm20, zmm30
+        vpxorq	zmm21, zmm21, zmm30
+        vpxorq	zmm22, zmm22, zmm30
+        vpxorq	zmm23, zmm23, zmm30
+        vprolq	zmm30, zmm29, 1
+        vpxorq	zmm30, zmm30, zmm27
+        vpxorq	zmm6, zmm6, zmm30
+        vpxorq	zmm7, zmm7, zmm30
+        vpxorq	zmm8, zmm8, zmm30
+        vpxorq	zmm9, zmm9, zmm30
+        vpxorq	zmm5, zmm5, zmm30
+        vprolq	zmm30, zmm25, 1
+        vpxorq	zmm30, zmm30, zmm28
+        vpxorq	zmm18, zmm18, zmm30
+        vpxorq	zmm19, zmm19, zmm30
+        vpxorq	zmm15, zmm15, zmm30
+        vpxorq	zmm16, zmm16, zmm30
+        vpxorq	zmm17, zmm17, zmm30
+        ; Rho - rotate each lane in place
+        vprolq	zmm12, zmm12, 1
+        vprolq	zmm24, zmm24, 62
+        vprolq	zmm6, zmm6, 28
+        vprolq	zmm18, zmm18, 27
+        vprolq	zmm1, zmm1, 36
+        vprolq	zmm13, zmm13, 44
+        vprolq	zmm20, zmm20, 6
+        vprolq	zmm7, zmm7, 55
+        vprolq	zmm19, zmm19, 20
+        vprolq	zmm2, zmm2, 3
+        vprolq	zmm14, zmm14, 10
+        vprolq	zmm21, zmm21, 43
+        vprolq	zmm8, zmm8, 25
+        vprolq	zmm15, zmm15, 39
+        vprolq	zmm3, zmm3, 41
+        vprolq	zmm10, zmm10, 45
+        vprolq	zmm22, zmm22, 15
+        vprolq	zmm9, zmm9, 21
+        vprolq	zmm16, zmm16, 8
+        vprolq	zmm4, zmm4, 18
+        vprolq	zmm11, zmm11, 2
+        vprolq	zmm23, zmm23, 61
+        vprolq	zmm5, zmm5, 56
+        vprolq	zmm17, zmm17, 14
+        ; Pi - relabel logical -> physical
+        ; Chi - per row, in place, fused with vpternlogq (imm 0xd2)
+        vmovdqu64	zmm30, zmm0
+        vmovdqu64	zmm31, zmm13
+        vpternlogq	zmm0, zmm13, zmm21, 210
+        vpternlogq	zmm13, zmm21, zmm9, 210
+        vpternlogq	zmm21, zmm9, zmm17, 210
+        vpternlogq	zmm9, zmm17, zmm30, 210
+        vpternlogq	zmm17, zmm30, zmm31, 210
+        vmovdqu64	zmm30, zmm6
+        vmovdqu64	zmm31, zmm19
+        vpternlogq	zmm6, zmm19, zmm2, 210
+        vpternlogq	zmm19, zmm2, zmm10, 210
+        vpternlogq	zmm2, zmm10, zmm23, 210
+        vpternlogq	zmm10, zmm23, zmm30, 210
+        vpternlogq	zmm23, zmm30, zmm31, 210
+        vmovdqu64	zmm30, zmm12
+        vmovdqu64	zmm31, zmm20
+        vpternlogq	zmm12, zmm20, zmm8, 210
+        vpternlogq	zmm20, zmm8, zmm16, 210
+        vpternlogq	zmm8, zmm16, zmm4, 210
+        vpternlogq	zmm16, zmm4, zmm30, 210
+        vpternlogq	zmm4, zmm30, zmm31, 210
+        vmovdqu64	zmm30, zmm18
+        vmovdqu64	zmm31, zmm1
+        vpternlogq	zmm18, zmm1, zmm14, 210
+        vpternlogq	zmm1, zmm14, zmm22, 210
+        vpternlogq	zmm14, zmm22, zmm5, 210
+        vpternlogq	zmm22, zmm5, zmm30, 210
+        vpternlogq	zmm5, zmm30, zmm31, 210
+        vmovdqu64	zmm30, zmm24
+        vmovdqu64	zmm31, zmm7
+        vpternlogq	zmm24, zmm7, zmm15, 210
+        vpternlogq	zmm7, zmm15, zmm3, 210
+        vpternlogq	zmm15, zmm3, zmm11, 210
+        vpternlogq	zmm3, zmm11, zmm30, 210
+        vpternlogq	zmm11, zmm30, zmm31, 210
+        ; Iota
+        vpxorq	zmm0, zmm0, [rax+448]
+        ; Round 8
+        ; Theta - column parity
+        vmovdqu64	zmm25, zmm0
+        vpternlogq	zmm25, zmm6, zmm12, 150
+        vpternlogq	zmm25, zmm18, zmm24, 150
+        vmovdqu64	zmm26, zmm13
+        vpternlogq	zmm26, zmm19, zmm20, 150
+        vpternlogq	zmm26, zmm1, zmm7, 150
+        vmovdqu64	zmm27, zmm21
+        vpternlogq	zmm27, zmm2, zmm8, 150
+        vpternlogq	zmm27, zmm14, zmm15, 150
+        vmovdqu64	zmm28, zmm9
+        vpternlogq	zmm28, zmm10, zmm16, 150
+        vpternlogq	zmm28, zmm22, zmm3, 150
+        vmovdqu64	zmm29, zmm17
+        vpternlogq	zmm29, zmm23, zmm4, 150
+        vpternlogq	zmm29, zmm5, zmm11, 150
+        ; Theta - D and apply in place
+        vprolq	zmm30, zmm26, 1
+        vpxorq	zmm30, zmm30, zmm29
+        vpxorq	zmm0, zmm0, zmm30
+        vpxorq	zmm6, zmm6, zmm30
+        vpxorq	zmm12, zmm12, zmm30
+        vpxorq	zmm18, zmm18, zmm30
+        vpxorq	zmm24, zmm24, zmm30
+        vprolq	zmm30, zmm27, 1
+        vpxorq	zmm30, zmm30, zmm25
+        vpxorq	zmm13, zmm13, zmm30
+        vpxorq	zmm19, zmm19, zmm30
+        vpxorq	zmm20, zmm20, zmm30
+        vpxorq	zmm1, zmm1, zmm30
+        vpxorq	zmm7, zmm7, zmm30
+        vprolq	zmm30, zmm28, 1
+        vpxorq	zmm30, zmm30, zmm26
+        vpxorq	zmm21, zmm21, zmm30
+        vpxorq	zmm2, zmm2, zmm30
+        vpxorq	zmm8, zmm8, zmm30
+        vpxorq	zmm14, zmm14, zmm30
+        vpxorq	zmm15, zmm15, zmm30
+        vprolq	zmm30, zmm29, 1
+        vpxorq	zmm30, zmm30, zmm27
+        vpxorq	zmm9, zmm9, zmm30
+        vpxorq	zmm10, zmm10, zmm30
+        vpxorq	zmm16, zmm16, zmm30
+        vpxorq	zmm22, zmm22, zmm30
+        vpxorq	zmm3, zmm3, zmm30
+        vprolq	zmm30, zmm25, 1
+        vpxorq	zmm30, zmm30, zmm28
+        vpxorq	zmm17, zmm17, zmm30
+        vpxorq	zmm23, zmm23, zmm30
+        vpxorq	zmm4, zmm4, zmm30
+        vpxorq	zmm5, zmm5, zmm30
+        vpxorq	zmm11, zmm11, zmm30
+        ; Rho - rotate each lane in place
+        vprolq	zmm13, zmm13, 1
+        vprolq	zmm21, zmm21, 62
+        vprolq	zmm9, zmm9, 28
+        vprolq	zmm17, zmm17, 27
+        vprolq	zmm6, zmm6, 36
+        vprolq	zmm19, zmm19, 44
+        vprolq	zmm2, zmm2, 6
+        vprolq	zmm10, zmm10, 55
+        vprolq	zmm23, zmm23, 20
+        vprolq	zmm12, zmm12, 3
+        vprolq	zmm20, zmm20, 10
+        vprolq	zmm8, zmm8, 43
+        vprolq	zmm16, zmm16, 25
+        vprolq	zmm4, zmm4, 39
+        vprolq	zmm18, zmm18, 41
+        vprolq	zmm1, zmm1, 45
+        vprolq	zmm14, zmm14, 15
+        vprolq	zmm22, zmm22, 21
+        vprolq	zmm5, zmm5, 8
+        vprolq	zmm24, zmm24, 18
+        vprolq	zmm7, zmm7, 2
+        vprolq	zmm15, zmm15, 61
+        vprolq	zmm3, zmm3, 56
+        vprolq	zmm11, zmm11, 14
+        ; Pi - relabel logical -> physical
+        ; Chi - per row, in place, fused with vpternlogq (imm 0xd2)
+        vmovdqu64	zmm30, zmm0
+        vmovdqu64	zmm31, zmm19
+        vpternlogq	zmm0, zmm19, zmm8, 210
+        vpternlogq	zmm19, zmm8, zmm22, 210
+        vpternlogq	zmm8, zmm22, zmm11, 210
+        vpternlogq	zmm22, zmm11, zmm30, 210
+        vpternlogq	zmm11, zmm30, zmm31, 210
+        vmovdqu64	zmm30, zmm9
+        vmovdqu64	zmm31, zmm23
+        vpternlogq	zmm9, zmm23, zmm12, 210
+        vpternlogq	zmm23, zmm12, zmm1, 210
+        vpternlogq	zmm12, zmm1, zmm15, 210
+        vpternlogq	zmm1, zmm15, zmm30, 210
+        vpternlogq	zmm15, zmm30, zmm31, 210
+        vmovdqu64	zmm30, zmm13
+        vmovdqu64	zmm31, zmm2
+        vpternlogq	zmm13, zmm2, zmm16, 210
+        vpternlogq	zmm2, zmm16, zmm5, 210
+        vpternlogq	zmm16, zmm5, zmm24, 210
+        vpternlogq	zmm5, zmm24, zmm30, 210
+        vpternlogq	zmm24, zmm30, zmm31, 210
+        vmovdqu64	zmm30, zmm17
+        vmovdqu64	zmm31, zmm6
+        vpternlogq	zmm17, zmm6, zmm20, 210
+        vpternlogq	zmm6, zmm20, zmm14, 210
+        vpternlogq	zmm20, zmm14, zmm3, 210
+        vpternlogq	zmm14, zmm3, zmm30, 210
+        vpternlogq	zmm3, zmm30, zmm31, 210
+        vmovdqu64	zmm30, zmm21
+        vmovdqu64	zmm31, zmm10
+        vpternlogq	zmm21, zmm10, zmm4, 210
+        vpternlogq	zmm10, zmm4, zmm18, 210
+        vpternlogq	zmm4, zmm18, zmm7, 210
+        vpternlogq	zmm18, zmm7, zmm30, 210
+        vpternlogq	zmm7, zmm30, zmm31, 210
+        ; Iota
+        vpxorq	zmm0, zmm0, [rax+512]
+        ; Round 9
+        ; Theta - column parity
+        vmovdqu64	zmm25, zmm0
+        vpternlogq	zmm25, zmm9, zmm13, 150
+        vpternlogq	zmm25, zmm17, zmm21, 150
+        vmovdqu64	zmm26, zmm19
+        vpternlogq	zmm26, zmm23, zmm2, 150
+        vpternlogq	zmm26, zmm6, zmm10, 150
+        vmovdqu64	zmm27, zmm8
+        vpternlogq	zmm27, zmm12, zmm16, 150
+        vpternlogq	zmm27, zmm20, zmm4, 150
+        vmovdqu64	zmm28, zmm22
+        vpternlogq	zmm28, zmm1, zmm5, 150
+        vpternlogq	zmm28, zmm14, zmm18, 150
+        vmovdqu64	zmm29, zmm11
+        vpternlogq	zmm29, zmm15, zmm24, 150
+        vpternlogq	zmm29, zmm3, zmm7, 150
+        ; Theta - D and apply in place
+        vprolq	zmm30, zmm26, 1
+        vpxorq	zmm30, zmm30, zmm29
+        vpxorq	zmm0, zmm0, zmm30
+        vpxorq	zmm9, zmm9, zmm30
+        vpxorq	zmm13, zmm13, zmm30
+        vpxorq	zmm17, zmm17, zmm30
+        vpxorq	zmm21, zmm21, zmm30
+        vprolq	zmm30, zmm27, 1
+        vpxorq	zmm30, zmm30, zmm25
+        vpxorq	zmm19, zmm19, zmm30
+        vpxorq	zmm23, zmm23, zmm30
+        vpxorq	zmm2, zmm2, zmm30
+        vpxorq	zmm6, zmm6, zmm30
+        vpxorq	zmm10, zmm10, zmm30
+        vprolq	zmm30, zmm28, 1
+        vpxorq	zmm30, zmm30, zmm26
+        vpxorq	zmm8, zmm8, zmm30
+        vpxorq	zmm12, zmm12, zmm30
+        vpxorq	zmm16, zmm16, zmm30
+        vpxorq	zmm20, zmm20, zmm30
+        vpxorq	zmm4, zmm4, zmm30
+        vprolq	zmm30, zmm29, 1
+        vpxorq	zmm30, zmm30, zmm27
+        vpxorq	zmm22, zmm22, zmm30
+        vpxorq	zmm1, zmm1, zmm30
+        vpxorq	zmm5, zmm5, zmm30
+        vpxorq	zmm14, zmm14, zmm30
+        vpxorq	zmm18, zmm18, zmm30
+        vprolq	zmm30, zmm25, 1
+        vpxorq	zmm30, zmm30, zmm28
+        vpxorq	zmm11, zmm11, zmm30
+        vpxorq	zmm15, zmm15, zmm30
+        vpxorq	zmm24, zmm24, zmm30
+        vpxorq	zmm3, zmm3, zmm30
+        vpxorq	zmm7, zmm7, zmm30
+        ; Rho - rotate each lane in place
+        vprolq	zmm19, zmm19, 1
+        vprolq	zmm8, zmm8, 62
+        vprolq	zmm22, zmm22, 28
+        vprolq	zmm11, zmm11, 27
+        vprolq	zmm9, zmm9, 36
+        vprolq	zmm23, zmm23, 44
+        vprolq	zmm12, zmm12, 6
+        vprolq	zmm1, zmm1, 55
+        vprolq	zmm15, zmm15, 20
+        vprolq	zmm13, zmm13, 3
+        vprolq	zmm2, zmm2, 10
+        vprolq	zmm16, zmm16, 43
+        vprolq	zmm5, zmm5, 25
+        vprolq	zmm24, zmm24, 39
+        vprolq	zmm17, zmm17, 41
+        vprolq	zmm6, zmm6, 45
+        vprolq	zmm20, zmm20, 15
+        vprolq	zmm14, zmm14, 21
+        vprolq	zmm3, zmm3, 8
+        vprolq	zmm21, zmm21, 18
+        vprolq	zmm10, zmm10, 2
+        vprolq	zmm4, zmm4, 61
+        vprolq	zmm18, zmm18, 56
+        vprolq	zmm7, zmm7, 14
+        ; Pi - relabel logical -> physical
+        ; Chi - per row, in place, fused with vpternlogq (imm 0xd2)
+        vmovdqu64	zmm30, zmm0
+        vmovdqu64	zmm31, zmm23
+        vpternlogq	zmm0, zmm23, zmm16, 210
+        vpternlogq	zmm23, zmm16, zmm14, 210
+        vpternlogq	zmm16, zmm14, zmm7, 210
+        vpternlogq	zmm14, zmm7, zmm30, 210
+        vpternlogq	zmm7, zmm30, zmm31, 210
+        vmovdqu64	zmm30, zmm22
+        vmovdqu64	zmm31, zmm15
+        vpternlogq	zmm22, zmm15, zmm13, 210
+        vpternlogq	zmm15, zmm13, zmm6, 210
+        vpternlogq	zmm13, zmm6, zmm4, 210
+        vpternlogq	zmm6, zmm4, zmm30, 210
+        vpternlogq	zmm4, zmm30, zmm31, 210
+        vmovdqu64	zmm30, zmm19
+        vmovdqu64	zmm31, zmm12
+        vpternlogq	zmm19, zmm12, zmm5, 210
+        vpternlogq	zmm12, zmm5, zmm3, 210
+        vpternlogq	zmm5, zmm3, zmm21, 210
+        vpternlogq	zmm3, zmm21, zmm30, 210
+        vpternlogq	zmm21, zmm30, zmm31, 210
+        vmovdqu64	zmm30, zmm11
+        vmovdqu64	zmm31, zmm9
+        vpternlogq	zmm11, zmm9, zmm2, 210
+        vpternlogq	zmm9, zmm2, zmm20, 210
+        vpternlogq	zmm2, zmm20, zmm18, 210
+        vpternlogq	zmm20, zmm18, zmm30, 210
+        vpternlogq	zmm18, zmm30, zmm31, 210
+        vmovdqu64	zmm30, zmm8
+        vmovdqu64	zmm31, zmm1
+        vpternlogq	zmm8, zmm1, zmm24, 210
+        vpternlogq	zmm1, zmm24, zmm17, 210
+        vpternlogq	zmm24, zmm17, zmm10, 210
+        vpternlogq	zmm17, zmm10, zmm30, 210
+        vpternlogq	zmm10, zmm30, zmm31, 210
+        ; Iota
+        vpxorq	zmm0, zmm0, [rax+576]
+        ; Round 10
+        ; Theta - column parity
+        vmovdqu64	zmm25, zmm0
+        vpternlogq	zmm25, zmm22, zmm19, 150
+        vpternlogq	zmm25, zmm11, zmm8, 150
+        vmovdqu64	zmm26, zmm23
+        vpternlogq	zmm26, zmm15, zmm12, 150
+        vpternlogq	zmm26, zmm9, zmm1, 150
+        vmovdqu64	zmm27, zmm16
+        vpternlogq	zmm27, zmm13, zmm5, 150
+        vpternlogq	zmm27, zmm2, zmm24, 150
+        vmovdqu64	zmm28, zmm14
+        vpternlogq	zmm28, zmm6, zmm3, 150
+        vpternlogq	zmm28, zmm20, zmm17, 150
+        vmovdqu64	zmm29, zmm7
+        vpternlogq	zmm29, zmm4, zmm21, 150
+        vpternlogq	zmm29, zmm18, zmm10, 150
+        ; Theta - D and apply in place
+        vprolq	zmm30, zmm26, 1
+        vpxorq	zmm30, zmm30, zmm29
+        vpxorq	zmm0, zmm0, zmm30
+        vpxorq	zmm22, zmm22, zmm30
+        vpxorq	zmm19, zmm19, zmm30
+        vpxorq	zmm11, zmm11, zmm30
+        vpxorq	zmm8, zmm8, zmm30
+        vprolq	zmm30, zmm27, 1
+        vpxorq	zmm30, zmm30, zmm25
+        vpxorq	zmm23, zmm23, zmm30
+        vpxorq	zmm15, zmm15, zmm30
+        vpxorq	zmm12, zmm12, zmm30
+        vpxorq	zmm9, zmm9, zmm30
+        vpxorq	zmm1, zmm1, zmm30
+        vprolq	zmm30, zmm28, 1
+        vpxorq	zmm30, zmm30, zmm26
+        vpxorq	zmm16, zmm16, zmm30
+        vpxorq	zmm13, zmm13, zmm30
+        vpxorq	zmm5, zmm5, zmm30
+        vpxorq	zmm2, zmm2, zmm30
+        vpxorq	zmm24, zmm24, zmm30
+        vprolq	zmm30, zmm29, 1
+        vpxorq	zmm30, zmm30, zmm27
+        vpxorq	zmm14, zmm14, zmm30
+        vpxorq	zmm6, zmm6, zmm30
+        vpxorq	zmm3, zmm3, zmm30
+        vpxorq	zmm20, zmm20, zmm30
+        vpxorq	zmm17, zmm17, zmm30
+        vprolq	zmm30, zmm25, 1
+        vpxorq	zmm30, zmm30, zmm28
+        vpxorq	zmm7, zmm7, zmm30
+        vpxorq	zmm4, zmm4, zmm30
+        vpxorq	zmm21, zmm21, zmm30
+        vpxorq	zmm18, zmm18, zmm30
+        vpxorq	zmm10, zmm10, zmm30
+        ; Rho - rotate each lane in place
+        vprolq	zmm23, zmm23, 1
+        vprolq	zmm16, zmm16, 62
+        vprolq	zmm14, zmm14, 28
+        vprolq	zmm7, zmm7, 27
+        vprolq	zmm22, zmm22, 36
+        vprolq	zmm15, zmm15, 44
+        vprolq	zmm13, zmm13, 6
+        vprolq	zmm6, zmm6, 55
+        vprolq	zmm4, zmm4, 20
+        vprolq	zmm19, zmm19, 3
+        vprolq	zmm12, zmm12, 10
+        vprolq	zmm5, zmm5, 43
+        vprolq	zmm3, zmm3, 25
+        vprolq	zmm21, zmm21, 39
+        vprolq	zmm11, zmm11, 41
+        vprolq	zmm9, zmm9, 45
+        vprolq	zmm2, zmm2, 15
+        vprolq	zmm20, zmm20, 21
+        vprolq	zmm18, zmm18, 8
+        vprolq	zmm8, zmm8, 18
+        vprolq	zmm1, zmm1, 2
+        vprolq	zmm24, zmm24, 61
+        vprolq	zmm17, zmm17, 56
+        vprolq	zmm10, zmm10, 14
+        ; Pi - relabel logical -> physical
+        ; Chi - per row, in place, fused with vpternlogq (imm 0xd2)
+        vmovdqu64	zmm30, zmm0
+        vmovdqu64	zmm31, zmm15
+        vpternlogq	zmm0, zmm15, zmm5, 210
+        vpternlogq	zmm15, zmm5, zmm20, 210
+        vpternlogq	zmm5, zmm20, zmm10, 210
+        vpternlogq	zmm20, zmm10, zmm30, 210
+        vpternlogq	zmm10, zmm30, zmm31, 210
+        vmovdqu64	zmm30, zmm14
+        vmovdqu64	zmm31, zmm4
+        vpternlogq	zmm14, zmm4, zmm19, 210
+        vpternlogq	zmm4, zmm19, zmm9, 210
+        vpternlogq	zmm19, zmm9, zmm24, 210
+        vpternlogq	zmm9, zmm24, zmm30, 210
+        vpternlogq	zmm24, zmm30, zmm31, 210
+        vmovdqu64	zmm30, zmm23
+        vmovdqu64	zmm31, zmm13
+        vpternlogq	zmm23, zmm13, zmm3, 210
+        vpternlogq	zmm13, zmm3, zmm18, 210
+        vpternlogq	zmm3, zmm18, zmm8, 210
+        vpternlogq	zmm18, zmm8, zmm30, 210
+        vpternlogq	zmm8, zmm30, zmm31, 210
+        vmovdqu64	zmm30, zmm7
+        vmovdqu64	zmm31, zmm22
+        vpternlogq	zmm7, zmm22, zmm12, 210
+        vpternlogq	zmm22, zmm12, zmm2, 210
+        vpternlogq	zmm12, zmm2, zmm17, 210
+        vpternlogq	zmm2, zmm17, zmm30, 210
+        vpternlogq	zmm17, zmm30, zmm31, 210
+        vmovdqu64	zmm30, zmm16
+        vmovdqu64	zmm31, zmm6
+        vpternlogq	zmm16, zmm6, zmm21, 210
+        vpternlogq	zmm6, zmm21, zmm11, 210
+        vpternlogq	zmm21, zmm11, zmm1, 210
+        vpternlogq	zmm11, zmm1, zmm30, 210
+        vpternlogq	zmm1, zmm30, zmm31, 210
+        ; Iota
+        vpxorq	zmm0, zmm0, [rax+640]
+        ; Round 11
+        ; Theta - column parity
+        vmovdqu64	zmm25, zmm0
+        vpternlogq	zmm25, zmm14, zmm23, 150
+        vpternlogq	zmm25, zmm7, zmm16, 150
+        vmovdqu64	zmm26, zmm15
+        vpternlogq	zmm26, zmm4, zmm13, 150
+        vpternlogq	zmm26, zmm22, zmm6, 150
+        vmovdqu64	zmm27, zmm5
+        vpternlogq	zmm27, zmm19, zmm3, 150
+        vpternlogq	zmm27, zmm12, zmm21, 150
+        vmovdqu64	zmm28, zmm20
+        vpternlogq	zmm28, zmm9, zmm18, 150
+        vpternlogq	zmm28, zmm2, zmm11, 150
+        vmovdqu64	zmm29, zmm10
+        vpternlogq	zmm29, zmm24, zmm8, 150
+        vpternlogq	zmm29, zmm17, zmm1, 150
+        ; Theta - D and apply in place
+        vprolq	zmm30, zmm26, 1
+        vpxorq	zmm30, zmm30, zmm29
+        vpxorq	zmm0, zmm0, zmm30
+        vpxorq	zmm14, zmm14, zmm30
+        vpxorq	zmm23, zmm23, zmm30
+        vpxorq	zmm7, zmm7, zmm30
+        vpxorq	zmm16, zmm16, zmm30
+        vprolq	zmm30, zmm27, 1
+        vpxorq	zmm30, zmm30, zmm25
+        vpxorq	zmm15, zmm15, zmm30
+        vpxorq	zmm4, zmm4, zmm30
+        vpxorq	zmm13, zmm13, zmm30
+        vpxorq	zmm22, zmm22, zmm30
+        vpxorq	zmm6, zmm6, zmm30
+        vprolq	zmm30, zmm28, 1
+        vpxorq	zmm30, zmm30, zmm26
+        vpxorq	zmm5, zmm5, zmm30
+        vpxorq	zmm19, zmm19, zmm30
+        vpxorq	zmm3, zmm3, zmm30
+        vpxorq	zmm12, zmm12, zmm30
+        vpxorq	zmm21, zmm21, zmm30
+        vprolq	zmm30, zmm29, 1
+        vpxorq	zmm30, zmm30, zmm27
+        vpxorq	zmm20, zmm20, zmm30
+        vpxorq	zmm9, zmm9, zmm30
+        vpxorq	zmm18, zmm18, zmm30
+        vpxorq	zmm2, zmm2, zmm30
+        vpxorq	zmm11, zmm11, zmm30
+        vprolq	zmm30, zmm25, 1
+        vpxorq	zmm30, zmm30, zmm28
+        vpxorq	zmm10, zmm10, zmm30
+        vpxorq	zmm24, zmm24, zmm30
+        vpxorq	zmm8, zmm8, zmm30
+        vpxorq	zmm17, zmm17, zmm30
+        vpxorq	zmm1, zmm1, zmm30
+        ; Rho - rotate each lane in place
+        vprolq	zmm15, zmm15, 1
+        vprolq	zmm5, zmm5, 62
+        vprolq	zmm20, zmm20, 28
+        vprolq	zmm10, zmm10, 27
+        vprolq	zmm14, zmm14, 36
+        vprolq	zmm4, zmm4, 44
+        vprolq	zmm19, zmm19, 6
+        vprolq	zmm9, zmm9, 55
+        vprolq	zmm24, zmm24, 20
+        vprolq	zmm23, zmm23, 3
+        vprolq	zmm13, zmm13, 10
+        vprolq	zmm3, zmm3, 43
+        vprolq	zmm18, zmm18, 25
+        vprolq	zmm8, zmm8, 39
+        vprolq	zmm7, zmm7, 41
+        vprolq	zmm22, zmm22, 45
+        vprolq	zmm12, zmm12, 15
+        vprolq	zmm2, zmm2, 21
+        vprolq	zmm17, zmm17, 8
+        vprolq	zmm16, zmm16, 18
+        vprolq	zmm6, zmm6, 2
+        vprolq	zmm21, zmm21, 61
+        vprolq	zmm11, zmm11, 56
+        vprolq	zmm1, zmm1, 14
+        ; Pi - relabel logical -> physical
+        ; Chi - per row, in place, fused with vpternlogq (imm 0xd2)
+        vmovdqu64	zmm30, zmm0
+        vmovdqu64	zmm31, zmm4
+        vpternlogq	zmm0, zmm4, zmm3, 210
+        vpternlogq	zmm4, zmm3, zmm2, 210
+        vpternlogq	zmm3, zmm2, zmm1, 210
+        vpternlogq	zmm2, zmm1, zmm30, 210
+        vpternlogq	zmm1, zmm30, zmm31, 210
+        vmovdqu64	zmm30, zmm20
+        vmovdqu64	zmm31, zmm24
+        vpternlogq	zmm20, zmm24, zmm23, 210
+        vpternlogq	zmm24, zmm23, zmm22, 210
+        vpternlogq	zmm23, zmm22, zmm21, 210
+        vpternlogq	zmm22, zmm21, zmm30, 210
+        vpternlogq	zmm21, zmm30, zmm31, 210
+        vmovdqu64	zmm30, zmm15
+        vmovdqu64	zmm31, zmm19
+        vpternlogq	zmm15, zmm19, zmm18, 210
+        vpternlogq	zmm19, zmm18, zmm17, 210
+        vpternlogq	zmm18, zmm17, zmm16, 210
+        vpternlogq	zmm17, zmm16, zmm30, 210
+        vpternlogq	zmm16, zmm30, zmm31, 210
+        vmovdqu64	zmm30, zmm10
+        vmovdqu64	zmm31, zmm14
+        vpternlogq	zmm10, zmm14, zmm13, 210
+        vpternlogq	zmm14, zmm13, zmm12, 210
+        vpternlogq	zmm13, zmm12, zmm11, 210
+        vpternlogq	zmm12, zmm11, zmm30, 210
+        vpternlogq	zmm11, zmm30, zmm31, 210
+        vmovdqu64	zmm30, zmm5
+        vmovdqu64	zmm31, zmm9
+        vpternlogq	zmm5, zmm9, zmm8, 210
+        vpternlogq	zmm9, zmm8, zmm7, 210
+        vpternlogq	zmm8, zmm7, zmm6, 210
+        vpternlogq	zmm7, zmm6, zmm30, 210
+        vpternlogq	zmm6, zmm30, zmm31, 210
+        ; Iota
+        vpxorq	zmm0, zmm0, [rax+704]
+        ; Round 12
+        ; Theta - column parity
+        vmovdqu64	zmm25, zmm0
+        vpternlogq	zmm25, zmm20, zmm15, 150
+        vpternlogq	zmm25, zmm10, zmm5, 150
+        vmovdqu64	zmm26, zmm4
+        vpternlogq	zmm26, zmm24, zmm19, 150
+        vpternlogq	zmm26, zmm14, zmm9, 150
+        vmovdqu64	zmm27, zmm3
+        vpternlogq	zmm27, zmm23, zmm18, 150
+        vpternlogq	zmm27, zmm13, zmm8, 150
+        vmovdqu64	zmm28, zmm2
+        vpternlogq	zmm28, zmm22, zmm17, 150
+        vpternlogq	zmm28, zmm12, zmm7, 150
+        vmovdqu64	zmm29, zmm1
+        vpternlogq	zmm29, zmm21, zmm16, 150
+        vpternlogq	zmm29, zmm11, zmm6, 150
+        ; Theta - D and apply in place
+        vprolq	zmm30, zmm26, 1
+        vpxorq	zmm30, zmm30, zmm29
+        vpxorq	zmm0, zmm0, zmm30
+        vpxorq	zmm20, zmm20, zmm30
+        vpxorq	zmm15, zmm15, zmm30
+        vpxorq	zmm10, zmm10, zmm30
+        vpxorq	zmm5, zmm5, zmm30
+        vprolq	zmm30, zmm27, 1
+        vpxorq	zmm30, zmm30, zmm25
+        vpxorq	zmm4, zmm4, zmm30
+        vpxorq	zmm24, zmm24, zmm30
+        vpxorq	zmm19, zmm19, zmm30
+        vpxorq	zmm14, zmm14, zmm30
+        vpxorq	zmm9, zmm9, zmm30
+        vprolq	zmm30, zmm28, 1
+        vpxorq	zmm30, zmm30, zmm26
+        vpxorq	zmm3, zmm3, zmm30
+        vpxorq	zmm23, zmm23, zmm30
+        vpxorq	zmm18, zmm18, zmm30
+        vpxorq	zmm13, zmm13, zmm30
+        vpxorq	zmm8, zmm8, zmm30
+        vprolq	zmm30, zmm29, 1
+        vpxorq	zmm30, zmm30, zmm27
+        vpxorq	zmm2, zmm2, zmm30
+        vpxorq	zmm22, zmm22, zmm30
+        vpxorq	zmm17, zmm17, zmm30
+        vpxorq	zmm12, zmm12, zmm30
+        vpxorq	zmm7, zmm7, zmm30
+        vprolq	zmm30, zmm25, 1
+        vpxorq	zmm30, zmm30, zmm28
+        vpxorq	zmm1, zmm1, zmm30
+        vpxorq	zmm21, zmm21, zmm30
+        vpxorq	zmm16, zmm16, zmm30
+        vpxorq	zmm11, zmm11, zmm30
+        vpxorq	zmm6, zmm6, zmm30
+        ; Rho - rotate each lane in place
+        vprolq	zmm4, zmm4, 1
+        vprolq	zmm3, zmm3, 62
+        vprolq	zmm2, zmm2, 28
+        vprolq	zmm1, zmm1, 27
+        vprolq	zmm20, zmm20, 36
+        vprolq	zmm24, zmm24, 44
+        vprolq	zmm23, zmm23, 6
+        vprolq	zmm22, zmm22, 55
+        vprolq	zmm21, zmm21, 20
+        vprolq	zmm15, zmm15, 3
+        vprolq	zmm19, zmm19, 10
+        vprolq	zmm18, zmm18, 43
+        vprolq	zmm17, zmm17, 25
+        vprolq	zmm16, zmm16, 39
+        vprolq	zmm10, zmm10, 41
+        vprolq	zmm14, zmm14, 45
+        vprolq	zmm13, zmm13, 15
+        vprolq	zmm12, zmm12, 21
+        vprolq	zmm11, zmm11, 8
+        vprolq	zmm5, zmm5, 18
+        vprolq	zmm9, zmm9, 2
+        vprolq	zmm8, zmm8, 61
+        vprolq	zmm7, zmm7, 56
+        vprolq	zmm6, zmm6, 14
+        ; Pi - relabel logical -> physical
+        ; Chi - per row, in place, fused with vpternlogq (imm 0xd2)
+        vmovdqu64	zmm30, zmm0
+        vmovdqu64	zmm31, zmm24
+        vpternlogq	zmm0, zmm24, zmm18, 210
+        vpternlogq	zmm24, zmm18, zmm12, 210
+        vpternlogq	zmm18, zmm12, zmm6, 210
+        vpternlogq	zmm12, zmm6, zmm30, 210
+        vpternlogq	zmm6, zmm30, zmm31, 210
+        vmovdqu64	zmm30, zmm2
+        vmovdqu64	zmm31, zmm21
+        vpternlogq	zmm2, zmm21, zmm15, 210
+        vpternlogq	zmm21, zmm15, zmm14, 210
+        vpternlogq	zmm15, zmm14, zmm8, 210
+        vpternlogq	zmm14, zmm8, zmm30, 210
+        vpternlogq	zmm8, zmm30, zmm31, 210
+        vmovdqu64	zmm30, zmm4
+        vmovdqu64	zmm31, zmm23
+        vpternlogq	zmm4, zmm23, zmm17, 210
+        vpternlogq	zmm23, zmm17, zmm11, 210
+        vpternlogq	zmm17, zmm11, zmm5, 210
+        vpternlogq	zmm11, zmm5, zmm30, 210
+        vpternlogq	zmm5, zmm30, zmm31, 210
+        vmovdqu64	zmm30, zmm1
+        vmovdqu64	zmm31, zmm20
+        vpternlogq	zmm1, zmm20, zmm19, 210
+        vpternlogq	zmm20, zmm19, zmm13, 210
+        vpternlogq	zmm19, zmm13, zmm7, 210
+        vpternlogq	zmm13, zmm7, zmm30, 210
+        vpternlogq	zmm7, zmm30, zmm31, 210
+        vmovdqu64	zmm30, zmm3
+        vmovdqu64	zmm31, zmm22
+        vpternlogq	zmm3, zmm22, zmm16, 210
+        vpternlogq	zmm22, zmm16, zmm10, 210
+        vpternlogq	zmm16, zmm10, zmm9, 210
+        vpternlogq	zmm10, zmm9, zmm30, 210
+        vpternlogq	zmm9, zmm30, zmm31, 210
+        ; Iota
+        vpxorq	zmm0, zmm0, [rax+768]
+        ; Round 13
+        ; Theta - column parity
+        vmovdqu64	zmm25, zmm0
+        vpternlogq	zmm25, zmm2, zmm4, 150
+        vpternlogq	zmm25, zmm1, zmm3, 150
+        vmovdqu64	zmm26, zmm24
+        vpternlogq	zmm26, zmm21, zmm23, 150
+        vpternlogq	zmm26, zmm20, zmm22, 150
+        vmovdqu64	zmm27, zmm18
+        vpternlogq	zmm27, zmm15, zmm17, 150
+        vpternlogq	zmm27, zmm19, zmm16, 150
+        vmovdqu64	zmm28, zmm12
+        vpternlogq	zmm28, zmm14, zmm11, 150
+        vpternlogq	zmm28, zmm13, zmm10, 150
+        vmovdqu64	zmm29, zmm6
+        vpternlogq	zmm29, zmm8, zmm5, 150
+        vpternlogq	zmm29, zmm7, zmm9, 150
+        ; Theta - D and apply in place
+        vprolq	zmm30, zmm26, 1
+        vpxorq	zmm30, zmm30, zmm29
+        vpxorq	zmm0, zmm0, zmm30
+        vpxorq	zmm2, zmm2, zmm30
+        vpxorq	zmm4, zmm4, zmm30
+        vpxorq	zmm1, zmm1, zmm30
+        vpxorq	zmm3, zmm3, zmm30
+        vprolq	zmm30, zmm27, 1
+        vpxorq	zmm30, zmm30, zmm25
+        vpxorq	zmm24, zmm24, zmm30
+        vpxorq	zmm21, zmm21, zmm30
+        vpxorq	zmm23, zmm23, zmm30
+        vpxorq	zmm20, zmm20, zmm30
+        vpxorq	zmm22, zmm22, zmm30
+        vprolq	zmm30, zmm28, 1
+        vpxorq	zmm30, zmm30, zmm26
+        vpxorq	zmm18, zmm18, zmm30
+        vpxorq	zmm15, zmm15, zmm30
+        vpxorq	zmm17, zmm17, zmm30
+        vpxorq	zmm19, zmm19, zmm30
+        vpxorq	zmm16, zmm16, zmm30
+        vprolq	zmm30, zmm29, 1
+        vpxorq	zmm30, zmm30, zmm27
+        vpxorq	zmm12, zmm12, zmm30
+        vpxorq	zmm14, zmm14, zmm30
+        vpxorq	zmm11, zmm11, zmm30
+        vpxorq	zmm13, zmm13, zmm30
+        vpxorq	zmm10, zmm10, zmm30
+        vprolq	zmm30, zmm25, 1
+        vpxorq	zmm30, zmm30, zmm28
+        vpxorq	zmm6, zmm6, zmm30
+        vpxorq	zmm8, zmm8, zmm30
+        vpxorq	zmm5, zmm5, zmm30
+        vpxorq	zmm7, zmm7, zmm30
+        vpxorq	zmm9, zmm9, zmm30
+        ; Rho - rotate each lane in place
+        vprolq	zmm24, zmm24, 1
+        vprolq	zmm18, zmm18, 62
+        vprolq	zmm12, zmm12, 28
+        vprolq	zmm6, zmm6, 27
+        vprolq	zmm2, zmm2, 36
+        vprolq	zmm21, zmm21, 44
+        vprolq	zmm15, zmm15, 6
+        vprolq	zmm14, zmm14, 55
+        vprolq	zmm8, zmm8, 20
+        vprolq	zmm4, zmm4, 3
+        vprolq	zmm23, zmm23, 10
+        vprolq	zmm17, zmm17, 43
+        vprolq	zmm11, zmm11, 25
+        vprolq	zmm5, zmm5, 39
+        vprolq	zmm1, zmm1, 41
+        vprolq	zmm20, zmm20, 45
+        vprolq	zmm19, zmm19, 15
+        vprolq	zmm13, zmm13, 21
+        vprolq	zmm7, zmm7, 8
+        vprolq	zmm3, zmm3, 18
+        vprolq	zmm22, zmm22, 2
+        vprolq	zmm16, zmm16, 61
+        vprolq	zmm10, zmm10, 56
+        vprolq	zmm9, zmm9, 14
+        ; Pi - relabel logical -> physical
+        ; Chi - per row, in place, fused with vpternlogq (imm 0xd2)
+        vmovdqu64	zmm30, zmm0
+        vmovdqu64	zmm31, zmm21
+        vpternlogq	zmm0, zmm21, zmm17, 210
+        vpternlogq	zmm21, zmm17, zmm13, 210
+        vpternlogq	zmm17, zmm13, zmm9, 210
+        vpternlogq	zmm13, zmm9, zmm30, 210
+        vpternlogq	zmm9, zmm30, zmm31, 210
+        vmovdqu64	zmm30, zmm12
+        vmovdqu64	zmm31, zmm8
+        vpternlogq	zmm12, zmm8, zmm4, 210
+        vpternlogq	zmm8, zmm4, zmm20, 210
+        vpternlogq	zmm4, zmm20, zmm16, 210
+        vpternlogq	zmm20, zmm16, zmm30, 210
+        vpternlogq	zmm16, zmm30, zmm31, 210
+        vmovdqu64	zmm30, zmm24
+        vmovdqu64	zmm31, zmm15
+        vpternlogq	zmm24, zmm15, zmm11, 210
+        vpternlogq	zmm15, zmm11, zmm7, 210
+        vpternlogq	zmm11, zmm7, zmm3, 210
+        vpternlogq	zmm7, zmm3, zmm30, 210
+        vpternlogq	zmm3, zmm30, zmm31, 210
+        vmovdqu64	zmm30, zmm6
+        vmovdqu64	zmm31, zmm2
+        vpternlogq	zmm6, zmm2, zmm23, 210
+        vpternlogq	zmm2, zmm23, zmm19, 210
+        vpternlogq	zmm23, zmm19, zmm10, 210
+        vpternlogq	zmm19, zmm10, zmm30, 210
+        vpternlogq	zmm10, zmm30, zmm31, 210
+        vmovdqu64	zmm30, zmm18
+        vmovdqu64	zmm31, zmm14
+        vpternlogq	zmm18, zmm14, zmm5, 210
+        vpternlogq	zmm14, zmm5, zmm1, 210
+        vpternlogq	zmm5, zmm1, zmm22, 210
+        vpternlogq	zmm1, zmm22, zmm30, 210
+        vpternlogq	zmm22, zmm30, zmm31, 210
+        ; Iota
+        vpxorq	zmm0, zmm0, [rax+832]
+        ; Round 14
+        ; Theta - column parity
+        vmovdqu64	zmm25, zmm0
+        vpternlogq	zmm25, zmm12, zmm24, 150
+        vpternlogq	zmm25, zmm6, zmm18, 150
+        vmovdqu64	zmm26, zmm21
+        vpternlogq	zmm26, zmm8, zmm15, 150
+        vpternlogq	zmm26, zmm2, zmm14, 150
+        vmovdqu64	zmm27, zmm17
+        vpternlogq	zmm27, zmm4, zmm11, 150
+        vpternlogq	zmm27, zmm23, zmm5, 150
+        vmovdqu64	zmm28, zmm13
+        vpternlogq	zmm28, zmm20, zmm7, 150
+        vpternlogq	zmm28, zmm19, zmm1, 150
+        vmovdqu64	zmm29, zmm9
+        vpternlogq	zmm29, zmm16, zmm3, 150
+        vpternlogq	zmm29, zmm10, zmm22, 150
+        ; Theta - D and apply in place
+        vprolq	zmm30, zmm26, 1
+        vpxorq	zmm30, zmm30, zmm29
+        vpxorq	zmm0, zmm0, zmm30
+        vpxorq	zmm12, zmm12, zmm30
+        vpxorq	zmm24, zmm24, zmm30
+        vpxorq	zmm6, zmm6, zmm30
+        vpxorq	zmm18, zmm18, zmm30
+        vprolq	zmm30, zmm27, 1
+        vpxorq	zmm30, zmm30, zmm25
+        vpxorq	zmm21, zmm21, zmm30
+        vpxorq	zmm8, zmm8, zmm30
+        vpxorq	zmm15, zmm15, zmm30
+        vpxorq	zmm2, zmm2, zmm30
+        vpxorq	zmm14, zmm14, zmm30
+        vprolq	zmm30, zmm28, 1
+        vpxorq	zmm30, zmm30, zmm26
+        vpxorq	zmm17, zmm17, zmm30
+        vpxorq	zmm4, zmm4, zmm30
+        vpxorq	zmm11, zmm11, zmm30
+        vpxorq	zmm23, zmm23, zmm30
+        vpxorq	zmm5, zmm5, zmm30
+        vprolq	zmm30, zmm29, 1
+        vpxorq	zmm30, zmm30, zmm27
+        vpxorq	zmm13, zmm13, zmm30
+        vpxorq	zmm20, zmm20, zmm30
+        vpxorq	zmm7, zmm7, zmm30
+        vpxorq	zmm19, zmm19, zmm30
+        vpxorq	zmm1, zmm1, zmm30
+        vprolq	zmm30, zmm25, 1
+        vpxorq	zmm30, zmm30, zmm28
+        vpxorq	zmm9, zmm9, zmm30
+        vpxorq	zmm16, zmm16, zmm30
+        vpxorq	zmm3, zmm3, zmm30
+        vpxorq	zmm10, zmm10, zmm30
+        vpxorq	zmm22, zmm22, zmm30
+        ; Rho - rotate each lane in place
+        vprolq	zmm21, zmm21, 1
+        vprolq	zmm17, zmm17, 62
+        vprolq	zmm13, zmm13, 28
+        vprolq	zmm9, zmm9, 27
+        vprolq	zmm12, zmm12, 36
+        vprolq	zmm8, zmm8, 44
+        vprolq	zmm4, zmm4, 6
+        vprolq	zmm20, zmm20, 55
+        vprolq	zmm16, zmm16, 20
+        vprolq	zmm24, zmm24, 3
+        vprolq	zmm15, zmm15, 10
+        vprolq	zmm11, zmm11, 43
+        vprolq	zmm7, zmm7, 25
+        vprolq	zmm3, zmm3, 39
+        vprolq	zmm6, zmm6, 41
+        vprolq	zmm2, zmm2, 45
+        vprolq	zmm23, zmm23, 15
+        vprolq	zmm19, zmm19, 21
+        vprolq	zmm10, zmm10, 8
+        vprolq	zmm18, zmm18, 18
+        vprolq	zmm14, zmm14, 2
+        vprolq	zmm5, zmm5, 61
+        vprolq	zmm1, zmm1, 56
+        vprolq	zmm22, zmm22, 14
+        ; Pi - relabel logical -> physical
+        ; Chi - per row, in place, fused with vpternlogq (imm 0xd2)
+        vmovdqu64	zmm30, zmm0
+        vmovdqu64	zmm31, zmm8
+        vpternlogq	zmm0, zmm8, zmm11, 210
+        vpternlogq	zmm8, zmm11, zmm19, 210
+        vpternlogq	zmm11, zmm19, zmm22, 210
+        vpternlogq	zmm19, zmm22, zmm30, 210
+        vpternlogq	zmm22, zmm30, zmm31, 210
+        vmovdqu64	zmm30, zmm13
+        vmovdqu64	zmm31, zmm16
+        vpternlogq	zmm13, zmm16, zmm24, 210
+        vpternlogq	zmm16, zmm24, zmm2, 210
+        vpternlogq	zmm24, zmm2, zmm5, 210
+        vpternlogq	zmm2, zmm5, zmm30, 210
+        vpternlogq	zmm5, zmm30, zmm31, 210
+        vmovdqu64	zmm30, zmm21
+        vmovdqu64	zmm31, zmm4
+        vpternlogq	zmm21, zmm4, zmm7, 210
+        vpternlogq	zmm4, zmm7, zmm10, 210
+        vpternlogq	zmm7, zmm10, zmm18, 210
+        vpternlogq	zmm10, zmm18, zmm30, 210
+        vpternlogq	zmm18, zmm30, zmm31, 210
+        vmovdqu64	zmm30, zmm9
+        vmovdqu64	zmm31, zmm12
+        vpternlogq	zmm9, zmm12, zmm15, 210
+        vpternlogq	zmm12, zmm15, zmm23, 210
+        vpternlogq	zmm15, zmm23, zmm1, 210
+        vpternlogq	zmm23, zmm1, zmm30, 210
+        vpternlogq	zmm1, zmm30, zmm31, 210
+        vmovdqu64	zmm30, zmm17
+        vmovdqu64	zmm31, zmm20
+        vpternlogq	zmm17, zmm20, zmm3, 210
+        vpternlogq	zmm20, zmm3, zmm6, 210
+        vpternlogq	zmm3, zmm6, zmm14, 210
+        vpternlogq	zmm6, zmm14, zmm30, 210
+        vpternlogq	zmm14, zmm30, zmm31, 210
+        ; Iota
+        vpxorq	zmm0, zmm0, [rax+896]
+        ; Round 15
+        ; Theta - column parity
+        vmovdqu64	zmm25, zmm0
+        vpternlogq	zmm25, zmm13, zmm21, 150
+        vpternlogq	zmm25, zmm9, zmm17, 150
+        vmovdqu64	zmm26, zmm8
+        vpternlogq	zmm26, zmm16, zmm4, 150
+        vpternlogq	zmm26, zmm12, zmm20, 150
+        vmovdqu64	zmm27, zmm11
+        vpternlogq	zmm27, zmm24, zmm7, 150
+        vpternlogq	zmm27, zmm15, zmm3, 150
+        vmovdqu64	zmm28, zmm19
+        vpternlogq	zmm28, zmm2, zmm10, 150
+        vpternlogq	zmm28, zmm23, zmm6, 150
+        vmovdqu64	zmm29, zmm22
+        vpternlogq	zmm29, zmm5, zmm18, 150
+        vpternlogq	zmm29, zmm1, zmm14, 150
+        ; Theta - D and apply in place
+        vprolq	zmm30, zmm26, 1
+        vpxorq	zmm30, zmm30, zmm29
+        vpxorq	zmm0, zmm0, zmm30
+        vpxorq	zmm13, zmm13, zmm30
+        vpxorq	zmm21, zmm21, zmm30
+        vpxorq	zmm9, zmm9, zmm30
+        vpxorq	zmm17, zmm17, zmm30
+        vprolq	zmm30, zmm27, 1
+        vpxorq	zmm30, zmm30, zmm25
+        vpxorq	zmm8, zmm8, zmm30
+        vpxorq	zmm16, zmm16, zmm30
+        vpxorq	zmm4, zmm4, zmm30
+        vpxorq	zmm12, zmm12, zmm30
+        vpxorq	zmm20, zmm20, zmm30
+        vprolq	zmm30, zmm28, 1
+        vpxorq	zmm30, zmm30, zmm26
+        vpxorq	zmm11, zmm11, zmm30
+        vpxorq	zmm24, zmm24, zmm30
+        vpxorq	zmm7, zmm7, zmm30
+        vpxorq	zmm15, zmm15, zmm30
+        vpxorq	zmm3, zmm3, zmm30
+        vprolq	zmm30, zmm29, 1
+        vpxorq	zmm30, zmm30, zmm27
+        vpxorq	zmm19, zmm19, zmm30
+        vpxorq	zmm2, zmm2, zmm30
+        vpxorq	zmm10, zmm10, zmm30
+        vpxorq	zmm23, zmm23, zmm30
+        vpxorq	zmm6, zmm6, zmm30
+        vprolq	zmm30, zmm25, 1
+        vpxorq	zmm30, zmm30, zmm28
+        vpxorq	zmm22, zmm22, zmm30
+        vpxorq	zmm5, zmm5, zmm30
+        vpxorq	zmm18, zmm18, zmm30
+        vpxorq	zmm1, zmm1, zmm30
+        vpxorq	zmm14, zmm14, zmm30
+        ; Rho - rotate each lane in place
+        vprolq	zmm8, zmm8, 1
+        vprolq	zmm11, zmm11, 62
+        vprolq	zmm19, zmm19, 28
+        vprolq	zmm22, zmm22, 27
+        vprolq	zmm13, zmm13, 36
+        vprolq	zmm16, zmm16, 44
+        vprolq	zmm24, zmm24, 6
+        vprolq	zmm2, zmm2, 55
+        vprolq	zmm5, zmm5, 20
+        vprolq	zmm21, zmm21, 3
+        vprolq	zmm4, zmm4, 10
+        vprolq	zmm7, zmm7, 43
+        vprolq	zmm10, zmm10, 25
+        vprolq	zmm18, zmm18, 39
+        vprolq	zmm9, zmm9, 41
+        vprolq	zmm12, zmm12, 45
+        vprolq	zmm15, zmm15, 15
+        vprolq	zmm23, zmm23, 21
+        vprolq	zmm1, zmm1, 8
+        vprolq	zmm17, zmm17, 18
+        vprolq	zmm20, zmm20, 2
+        vprolq	zmm3, zmm3, 61
+        vprolq	zmm6, zmm6, 56
+        vprolq	zmm14, zmm14, 14
+        ; Pi - relabel logical -> physical
+        ; Chi - per row, in place, fused with vpternlogq (imm 0xd2)
+        vmovdqu64	zmm30, zmm0
+        vmovdqu64	zmm31, zmm16
+        vpternlogq	zmm0, zmm16, zmm7, 210
+        vpternlogq	zmm16, zmm7, zmm23, 210
+        vpternlogq	zmm7, zmm23, zmm14, 210
+        vpternlogq	zmm23, zmm14, zmm30, 210
+        vpternlogq	zmm14, zmm30, zmm31, 210
+        vmovdqu64	zmm30, zmm19
+        vmovdqu64	zmm31, zmm5
+        vpternlogq	zmm19, zmm5, zmm21, 210
+        vpternlogq	zmm5, zmm21, zmm12, 210
+        vpternlogq	zmm21, zmm12, zmm3, 210
+        vpternlogq	zmm12, zmm3, zmm30, 210
+        vpternlogq	zmm3, zmm30, zmm31, 210
+        vmovdqu64	zmm30, zmm8
+        vmovdqu64	zmm31, zmm24
+        vpternlogq	zmm8, zmm24, zmm10, 210
+        vpternlogq	zmm24, zmm10, zmm1, 210
+        vpternlogq	zmm10, zmm1, zmm17, 210
+        vpternlogq	zmm1, zmm17, zmm30, 210
+        vpternlogq	zmm17, zmm30, zmm31, 210
+        vmovdqu64	zmm30, zmm22
+        vmovdqu64	zmm31, zmm13
+        vpternlogq	zmm22, zmm13, zmm4, 210
+        vpternlogq	zmm13, zmm4, zmm15, 210
+        vpternlogq	zmm4, zmm15, zmm6, 210
+        vpternlogq	zmm15, zmm6, zmm30, 210
+        vpternlogq	zmm6, zmm30, zmm31, 210
+        vmovdqu64	zmm30, zmm11
+        vmovdqu64	zmm31, zmm2
+        vpternlogq	zmm11, zmm2, zmm18, 210
+        vpternlogq	zmm2, zmm18, zmm9, 210
+        vpternlogq	zmm18, zmm9, zmm20, 210
+        vpternlogq	zmm9, zmm20, zmm30, 210
+        vpternlogq	zmm20, zmm30, zmm31, 210
+        ; Iota
+        vpxorq	zmm0, zmm0, [rax+960]
+        ; Round 16
+        ; Theta - column parity
+        vmovdqu64	zmm25, zmm0
+        vpternlogq	zmm25, zmm19, zmm8, 150
+        vpternlogq	zmm25, zmm22, zmm11, 150
+        vmovdqu64	zmm26, zmm16
+        vpternlogq	zmm26, zmm5, zmm24, 150
+        vpternlogq	zmm26, zmm13, zmm2, 150
+        vmovdqu64	zmm27, zmm7
+        vpternlogq	zmm27, zmm21, zmm10, 150
+        vpternlogq	zmm27, zmm4, zmm18, 150
+        vmovdqu64	zmm28, zmm23
+        vpternlogq	zmm28, zmm12, zmm1, 150
+        vpternlogq	zmm28, zmm15, zmm9, 150
+        vmovdqu64	zmm29, zmm14
+        vpternlogq	zmm29, zmm3, zmm17, 150
+        vpternlogq	zmm29, zmm6, zmm20, 150
+        ; Theta - D and apply in place
+        vprolq	zmm30, zmm26, 1
+        vpxorq	zmm30, zmm30, zmm29
+        vpxorq	zmm0, zmm0, zmm30
+        vpxorq	zmm19, zmm19, zmm30
+        vpxorq	zmm8, zmm8, zmm30
+        vpxorq	zmm22, zmm22, zmm30
+        vpxorq	zmm11, zmm11, zmm30
+        vprolq	zmm30, zmm27, 1
+        vpxorq	zmm30, zmm30, zmm25
+        vpxorq	zmm16, zmm16, zmm30
+        vpxorq	zmm5, zmm5, zmm30
+        vpxorq	zmm24, zmm24, zmm30
+        vpxorq	zmm13, zmm13, zmm30
+        vpxorq	zmm2, zmm2, zmm30
+        vprolq	zmm30, zmm28, 1
+        vpxorq	zmm30, zmm30, zmm26
+        vpxorq	zmm7, zmm7, zmm30
+        vpxorq	zmm21, zmm21, zmm30
+        vpxorq	zmm10, zmm10, zmm30
+        vpxorq	zmm4, zmm4, zmm30
+        vpxorq	zmm18, zmm18, zmm30
+        vprolq	zmm30, zmm29, 1
+        vpxorq	zmm30, zmm30, zmm27
+        vpxorq	zmm23, zmm23, zmm30
+        vpxorq	zmm12, zmm12, zmm30
+        vpxorq	zmm1, zmm1, zmm30
+        vpxorq	zmm15, zmm15, zmm30
+        vpxorq	zmm9, zmm9, zmm30
+        vprolq	zmm30, zmm25, 1
+        vpxorq	zmm30, zmm30, zmm28
+        vpxorq	zmm14, zmm14, zmm30
+        vpxorq	zmm3, zmm3, zmm30
+        vpxorq	zmm17, zmm17, zmm30
+        vpxorq	zmm6, zmm6, zmm30
+        vpxorq	zmm20, zmm20, zmm30
+        ; Rho - rotate each lane in place
+        vprolq	zmm16, zmm16, 1
+        vprolq	zmm7, zmm7, 62
+        vprolq	zmm23, zmm23, 28
+        vprolq	zmm14, zmm14, 27
+        vprolq	zmm19, zmm19, 36
+        vprolq	zmm5, zmm5, 44
+        vprolq	zmm21, zmm21, 6
+        vprolq	zmm12, zmm12, 55
+        vprolq	zmm3, zmm3, 20
+        vprolq	zmm8, zmm8, 3
+        vprolq	zmm24, zmm24, 10
+        vprolq	zmm10, zmm10, 43
+        vprolq	zmm1, zmm1, 25
+        vprolq	zmm17, zmm17, 39
+        vprolq	zmm22, zmm22, 41
+        vprolq	zmm13, zmm13, 45
+        vprolq	zmm4, zmm4, 15
+        vprolq	zmm15, zmm15, 21
+        vprolq	zmm6, zmm6, 8
+        vprolq	zmm11, zmm11, 18
+        vprolq	zmm2, zmm2, 2
+        vprolq	zmm18, zmm18, 61
+        vprolq	zmm9, zmm9, 56
+        vprolq	zmm20, zmm20, 14
+        ; Pi - relabel logical -> physical
+        ; Chi - per row, in place, fused with vpternlogq (imm 0xd2)
+        vmovdqu64	zmm30, zmm0
+        vmovdqu64	zmm31, zmm5
+        vpternlogq	zmm0, zmm5, zmm10, 210
+        vpternlogq	zmm5, zmm10, zmm15, 210
+        vpternlogq	zmm10, zmm15, zmm20, 210
+        vpternlogq	zmm15, zmm20, zmm30, 210
+        vpternlogq	zmm20, zmm30, zmm31, 210
+        vmovdqu64	zmm30, zmm23
+        vmovdqu64	zmm31, zmm3
+        vpternlogq	zmm23, zmm3, zmm8, 210
+        vpternlogq	zmm3, zmm8, zmm13, 210
+        vpternlogq	zmm8, zmm13, zmm18, 210
+        vpternlogq	zmm13, zmm18, zmm30, 210
+        vpternlogq	zmm18, zmm30, zmm31, 210
+        vmovdqu64	zmm30, zmm16
+        vmovdqu64	zmm31, zmm21
+        vpternlogq	zmm16, zmm21, zmm1, 210
+        vpternlogq	zmm21, zmm1, zmm6, 210
+        vpternlogq	zmm1, zmm6, zmm11, 210
+        vpternlogq	zmm6, zmm11, zmm30, 210
+        vpternlogq	zmm11, zmm30, zmm31, 210
+        vmovdqu64	zmm30, zmm14
+        vmovdqu64	zmm31, zmm19
+        vpternlogq	zmm14, zmm19, zmm24, 210
+        vpternlogq	zmm19, zmm24, zmm4, 210
+        vpternlogq	zmm24, zmm4, zmm9, 210
+        vpternlogq	zmm4, zmm9, zmm30, 210
+        vpternlogq	zmm9, zmm30, zmm31, 210
+        vmovdqu64	zmm30, zmm7
+        vmovdqu64	zmm31, zmm12
+        vpternlogq	zmm7, zmm12, zmm17, 210
+        vpternlogq	zmm12, zmm17, zmm22, 210
+        vpternlogq	zmm17, zmm22, zmm2, 210
+        vpternlogq	zmm22, zmm2, zmm30, 210
+        vpternlogq	zmm2, zmm30, zmm31, 210
+        ; Iota
+        vpxorq	zmm0, zmm0, [rax+1024]
+        ; Round 17
+        ; Theta - column parity
+        vmovdqu64	zmm25, zmm0
+        vpternlogq	zmm25, zmm23, zmm16, 150
+        vpternlogq	zmm25, zmm14, zmm7, 150
+        vmovdqu64	zmm26, zmm5
+        vpternlogq	zmm26, zmm3, zmm21, 150
+        vpternlogq	zmm26, zmm19, zmm12, 150
+        vmovdqu64	zmm27, zmm10
+        vpternlogq	zmm27, zmm8, zmm1, 150
+        vpternlogq	zmm27, zmm24, zmm17, 150
+        vmovdqu64	zmm28, zmm15
+        vpternlogq	zmm28, zmm13, zmm6, 150
+        vpternlogq	zmm28, zmm4, zmm22, 150
+        vmovdqu64	zmm29, zmm20
+        vpternlogq	zmm29, zmm18, zmm11, 150
+        vpternlogq	zmm29, zmm9, zmm2, 150
+        ; Theta - D and apply in place
+        vprolq	zmm30, zmm26, 1
+        vpxorq	zmm30, zmm30, zmm29
+        vpxorq	zmm0, zmm0, zmm30
+        vpxorq	zmm23, zmm23, zmm30
+        vpxorq	zmm16, zmm16, zmm30
+        vpxorq	zmm14, zmm14, zmm30
+        vpxorq	zmm7, zmm7, zmm30
+        vprolq	zmm30, zmm27, 1
+        vpxorq	zmm30, zmm30, zmm25
+        vpxorq	zmm5, zmm5, zmm30
+        vpxorq	zmm3, zmm3, zmm30
+        vpxorq	zmm21, zmm21, zmm30
+        vpxorq	zmm19, zmm19, zmm30
+        vpxorq	zmm12, zmm12, zmm30
+        vprolq	zmm30, zmm28, 1
+        vpxorq	zmm30, zmm30, zmm26
+        vpxorq	zmm10, zmm10, zmm30
+        vpxorq	zmm8, zmm8, zmm30
+        vpxorq	zmm1, zmm1, zmm30
+        vpxorq	zmm24, zmm24, zmm30
+        vpxorq	zmm17, zmm17, zmm30
+        vprolq	zmm30, zmm29, 1
+        vpxorq	zmm30, zmm30, zmm27
+        vpxorq	zmm15, zmm15, zmm30
+        vpxorq	zmm13, zmm13, zmm30
+        vpxorq	zmm6, zmm6, zmm30
+        vpxorq	zmm4, zmm4, zmm30
+        vpxorq	zmm22, zmm22, zmm30
+        vprolq	zmm30, zmm25, 1
+        vpxorq	zmm30, zmm30, zmm28
+        vpxorq	zmm20, zmm20, zmm30
+        vpxorq	zmm18, zmm18, zmm30
+        vpxorq	zmm11, zmm11, zmm30
+        vpxorq	zmm9, zmm9, zmm30
+        vpxorq	zmm2, zmm2, zmm30
+        ; Rho - rotate each lane in place
+        vprolq	zmm5, zmm5, 1
+        vprolq	zmm10, zmm10, 62
+        vprolq	zmm15, zmm15, 28
+        vprolq	zmm20, zmm20, 27
+        vprolq	zmm23, zmm23, 36
+        vprolq	zmm3, zmm3, 44
+        vprolq	zmm8, zmm8, 6
+        vprolq	zmm13, zmm13, 55
+        vprolq	zmm18, zmm18, 20
+        vprolq	zmm16, zmm16, 3
+        vprolq	zmm21, zmm21, 10
+        vprolq	zmm1, zmm1, 43
+        vprolq	zmm6, zmm6, 25
+        vprolq	zmm11, zmm11, 39
+        vprolq	zmm14, zmm14, 41
+        vprolq	zmm19, zmm19, 45
+        vprolq	zmm24, zmm24, 15
+        vprolq	zmm4, zmm4, 21
+        vprolq	zmm9, zmm9, 8
+        vprolq	zmm7, zmm7, 18
+        vprolq	zmm12, zmm12, 2
+        vprolq	zmm17, zmm17, 61
+        vprolq	zmm22, zmm22, 56
+        vprolq	zmm2, zmm2, 14
+        ; Pi - relabel logical -> physical
+        ; Chi - per row, in place, fused with vpternlogq (imm 0xd2)
+        vmovdqu64	zmm30, zmm0
+        vmovdqu64	zmm31, zmm3
+        vpternlogq	zmm0, zmm3, zmm1, 210
+        vpternlogq	zmm3, zmm1, zmm4, 210
+        vpternlogq	zmm1, zmm4, zmm2, 210
+        vpternlogq	zmm4, zmm2, zmm30, 210
+        vpternlogq	zmm2, zmm30, zmm31, 210
+        vmovdqu64	zmm30, zmm15
+        vmovdqu64	zmm31, zmm18
+        vpternlogq	zmm15, zmm18, zmm16, 210
+        vpternlogq	zmm18, zmm16, zmm19, 210
+        vpternlogq	zmm16, zmm19, zmm17, 210
+        vpternlogq	zmm19, zmm17, zmm30, 210
+        vpternlogq	zmm17, zmm30, zmm31, 210
+        vmovdqu64	zmm30, zmm5
+        vmovdqu64	zmm31, zmm8
+        vpternlogq	zmm5, zmm8, zmm6, 210
+        vpternlogq	zmm8, zmm6, zmm9, 210
+        vpternlogq	zmm6, zmm9, zmm7, 210
+        vpternlogq	zmm9, zmm7, zmm30, 210
+        vpternlogq	zmm7, zmm30, zmm31, 210
+        vmovdqu64	zmm30, zmm20
+        vmovdqu64	zmm31, zmm23
+        vpternlogq	zmm20, zmm23, zmm21, 210
+        vpternlogq	zmm23, zmm21, zmm24, 210
+        vpternlogq	zmm21, zmm24, zmm22, 210
+        vpternlogq	zmm24, zmm22, zmm30, 210
+        vpternlogq	zmm22, zmm30, zmm31, 210
+        vmovdqu64	zmm30, zmm10
+        vmovdqu64	zmm31, zmm13
+        vpternlogq	zmm10, zmm13, zmm11, 210
+        vpternlogq	zmm13, zmm11, zmm14, 210
+        vpternlogq	zmm11, zmm14, zmm12, 210
+        vpternlogq	zmm14, zmm12, zmm30, 210
+        vpternlogq	zmm12, zmm30, zmm31, 210
+        ; Iota
+        vpxorq	zmm0, zmm0, [rax+1088]
+        ; Round 18
+        ; Theta - column parity
+        vmovdqu64	zmm25, zmm0
+        vpternlogq	zmm25, zmm15, zmm5, 150
+        vpternlogq	zmm25, zmm20, zmm10, 150
+        vmovdqu64	zmm26, zmm3
+        vpternlogq	zmm26, zmm18, zmm8, 150
+        vpternlogq	zmm26, zmm23, zmm13, 150
+        vmovdqu64	zmm27, zmm1
+        vpternlogq	zmm27, zmm16, zmm6, 150
+        vpternlogq	zmm27, zmm21, zmm11, 150
+        vmovdqu64	zmm28, zmm4
+        vpternlogq	zmm28, zmm19, zmm9, 150
+        vpternlogq	zmm28, zmm24, zmm14, 150
+        vmovdqu64	zmm29, zmm2
+        vpternlogq	zmm29, zmm17, zmm7, 150
+        vpternlogq	zmm29, zmm22, zmm12, 150
+        ; Theta - D and apply in place
+        vprolq	zmm30, zmm26, 1
+        vpxorq	zmm30, zmm30, zmm29
+        vpxorq	zmm0, zmm0, zmm30
+        vpxorq	zmm15, zmm15, zmm30
+        vpxorq	zmm5, zmm5, zmm30
+        vpxorq	zmm20, zmm20, zmm30
+        vpxorq	zmm10, zmm10, zmm30
+        vprolq	zmm30, zmm27, 1
+        vpxorq	zmm30, zmm30, zmm25
+        vpxorq	zmm3, zmm3, zmm30
+        vpxorq	zmm18, zmm18, zmm30
+        vpxorq	zmm8, zmm8, zmm30
+        vpxorq	zmm23, zmm23, zmm30
+        vpxorq	zmm13, zmm13, zmm30
+        vprolq	zmm30, zmm28, 1
+        vpxorq	zmm30, zmm30, zmm26
+        vpxorq	zmm1, zmm1, zmm30
+        vpxorq	zmm16, zmm16, zmm30
+        vpxorq	zmm6, zmm6, zmm30
+        vpxorq	zmm21, zmm21, zmm30
+        vpxorq	zmm11, zmm11, zmm30
+        vprolq	zmm30, zmm29, 1
+        vpxorq	zmm30, zmm30, zmm27
+        vpxorq	zmm4, zmm4, zmm30
+        vpxorq	zmm19, zmm19, zmm30
+        vpxorq	zmm9, zmm9, zmm30
+        vpxorq	zmm24, zmm24, zmm30
+        vpxorq	zmm14, zmm14, zmm30
+        vprolq	zmm30, zmm25, 1
+        vpxorq	zmm30, zmm30, zmm28
+        vpxorq	zmm2, zmm2, zmm30
+        vpxorq	zmm17, zmm17, zmm30
+        vpxorq	zmm7, zmm7, zmm30
+        vpxorq	zmm22, zmm22, zmm30
+        vpxorq	zmm12, zmm12, zmm30
+        ; Rho - rotate each lane in place
+        vprolq	zmm3, zmm3, 1
+        vprolq	zmm1, zmm1, 62
+        vprolq	zmm4, zmm4, 28
+        vprolq	zmm2, zmm2, 27
+        vprolq	zmm15, zmm15, 36
+        vprolq	zmm18, zmm18, 44
+        vprolq	zmm16, zmm16, 6
+        vprolq	zmm19, zmm19, 55
+        vprolq	zmm17, zmm17, 20
+        vprolq	zmm5, zmm5, 3
+        vprolq	zmm8, zmm8, 10
+        vprolq	zmm6, zmm6, 43
+        vprolq	zmm9, zmm9, 25
+        vprolq	zmm7, zmm7, 39
+        vprolq	zmm20, zmm20, 41
+        vprolq	zmm23, zmm23, 45
+        vprolq	zmm21, zmm21, 15
+        vprolq	zmm24, zmm24, 21
+        vprolq	zmm22, zmm22, 8
+        vprolq	zmm10, zmm10, 18
+        vprolq	zmm13, zmm13, 2
+        vprolq	zmm11, zmm11, 61
+        vprolq	zmm14, zmm14, 56
+        vprolq	zmm12, zmm12, 14
+        ; Pi - relabel logical -> physical
+        ; Chi - per row, in place, fused with vpternlogq (imm 0xd2)
+        vmovdqu64	zmm30, zmm0
+        vmovdqu64	zmm31, zmm18
+        vpternlogq	zmm0, zmm18, zmm6, 210
+        vpternlogq	zmm18, zmm6, zmm24, 210
+        vpternlogq	zmm6, zmm24, zmm12, 210
+        vpternlogq	zmm24, zmm12, zmm30, 210
+        vpternlogq	zmm12, zmm30, zmm31, 210
+        vmovdqu64	zmm30, zmm4
+        vmovdqu64	zmm31, zmm17
+        vpternlogq	zmm4, zmm17, zmm5, 210
+        vpternlogq	zmm17, zmm5, zmm23, 210
+        vpternlogq	zmm5, zmm23, zmm11, 210
+        vpternlogq	zmm23, zmm11, zmm30, 210
+        vpternlogq	zmm11, zmm30, zmm31, 210
+        vmovdqu64	zmm30, zmm3
+        vmovdqu64	zmm31, zmm16
+        vpternlogq	zmm3, zmm16, zmm9, 210
+        vpternlogq	zmm16, zmm9, zmm22, 210
+        vpternlogq	zmm9, zmm22, zmm10, 210
+        vpternlogq	zmm22, zmm10, zmm30, 210
+        vpternlogq	zmm10, zmm30, zmm31, 210
+        vmovdqu64	zmm30, zmm2
+        vmovdqu64	zmm31, zmm15
+        vpternlogq	zmm2, zmm15, zmm8, 210
+        vpternlogq	zmm15, zmm8, zmm21, 210
+        vpternlogq	zmm8, zmm21, zmm14, 210
+        vpternlogq	zmm21, zmm14, zmm30, 210
+        vpternlogq	zmm14, zmm30, zmm31, 210
+        vmovdqu64	zmm30, zmm1
+        vmovdqu64	zmm31, zmm19
+        vpternlogq	zmm1, zmm19, zmm7, 210
+        vpternlogq	zmm19, zmm7, zmm20, 210
+        vpternlogq	zmm7, zmm20, zmm13, 210
+        vpternlogq	zmm20, zmm13, zmm30, 210
+        vpternlogq	zmm13, zmm30, zmm31, 210
+        ; Iota
+        vpxorq	zmm0, zmm0, [rax+1152]
+        ; Round 19
+        ; Theta - column parity
+        vmovdqu64	zmm25, zmm0
+        vpternlogq	zmm25, zmm4, zmm3, 150
+        vpternlogq	zmm25, zmm2, zmm1, 150
+        vmovdqu64	zmm26, zmm18
+        vpternlogq	zmm26, zmm17, zmm16, 150
+        vpternlogq	zmm26, zmm15, zmm19, 150
+        vmovdqu64	zmm27, zmm6
+        vpternlogq	zmm27, zmm5, zmm9, 150
+        vpternlogq	zmm27, zmm8, zmm7, 150
+        vmovdqu64	zmm28, zmm24
+        vpternlogq	zmm28, zmm23, zmm22, 150
+        vpternlogq	zmm28, zmm21, zmm20, 150
+        vmovdqu64	zmm29, zmm12
+        vpternlogq	zmm29, zmm11, zmm10, 150
+        vpternlogq	zmm29, zmm14, zmm13, 150
+        ; Theta - D and apply in place
+        vprolq	zmm30, zmm26, 1
+        vpxorq	zmm30, zmm30, zmm29
+        vpxorq	zmm0, zmm0, zmm30
+        vpxorq	zmm4, zmm4, zmm30
+        vpxorq	zmm3, zmm3, zmm30
+        vpxorq	zmm2, zmm2, zmm30
+        vpxorq	zmm1, zmm1, zmm30
+        vprolq	zmm30, zmm27, 1
+        vpxorq	zmm30, zmm30, zmm25
+        vpxorq	zmm18, zmm18, zmm30
+        vpxorq	zmm17, zmm17, zmm30
+        vpxorq	zmm16, zmm16, zmm30
+        vpxorq	zmm15, zmm15, zmm30
+        vpxorq	zmm19, zmm19, zmm30
+        vprolq	zmm30, zmm28, 1
+        vpxorq	zmm30, zmm30, zmm26
+        vpxorq	zmm6, zmm6, zmm30
+        vpxorq	zmm5, zmm5, zmm30
+        vpxorq	zmm9, zmm9, zmm30
+        vpxorq	zmm8, zmm8, zmm30
+        vpxorq	zmm7, zmm7, zmm30
+        vprolq	zmm30, zmm29, 1
+        vpxorq	zmm30, zmm30, zmm27
+        vpxorq	zmm24, zmm24, zmm30
+        vpxorq	zmm23, zmm23, zmm30
+        vpxorq	zmm22, zmm22, zmm30
+        vpxorq	zmm21, zmm21, zmm30
+        vpxorq	zmm20, zmm20, zmm30
+        vprolq	zmm30, zmm25, 1
+        vpxorq	zmm30, zmm30, zmm28
+        vpxorq	zmm12, zmm12, zmm30
+        vpxorq	zmm11, zmm11, zmm30
+        vpxorq	zmm10, zmm10, zmm30
+        vpxorq	zmm14, zmm14, zmm30
+        vpxorq	zmm13, zmm13, zmm30
+        ; Rho - rotate each lane in place
+        vprolq	zmm18, zmm18, 1
+        vprolq	zmm6, zmm6, 62
+        vprolq	zmm24, zmm24, 28
+        vprolq	zmm12, zmm12, 27
+        vprolq	zmm4, zmm4, 36
+        vprolq	zmm17, zmm17, 44
+        vprolq	zmm5, zmm5, 6
+        vprolq	zmm23, zmm23, 55
+        vprolq	zmm11, zmm11, 20
+        vprolq	zmm3, zmm3, 3
+        vprolq	zmm16, zmm16, 10
+        vprolq	zmm9, zmm9, 43
+        vprolq	zmm22, zmm22, 25
+        vprolq	zmm10, zmm10, 39
+        vprolq	zmm2, zmm2, 41
+        vprolq	zmm15, zmm15, 45
+        vprolq	zmm8, zmm8, 15
+        vprolq	zmm21, zmm21, 21
+        vprolq	zmm14, zmm14, 8
+        vprolq	zmm1, zmm1, 18
+        vprolq	zmm19, zmm19, 2
+        vprolq	zmm7, zmm7, 61
+        vprolq	zmm20, zmm20, 56
+        vprolq	zmm13, zmm13, 14
+        ; Pi - relabel logical -> physical
+        ; Chi - per row, in place, fused with vpternlogq (imm 0xd2)
+        vmovdqu64	zmm30, zmm0
+        vmovdqu64	zmm31, zmm17
+        vpternlogq	zmm0, zmm17, zmm9, 210
+        vpternlogq	zmm17, zmm9, zmm21, 210
+        vpternlogq	zmm9, zmm21, zmm13, 210
+        vpternlogq	zmm21, zmm13, zmm30, 210
+        vpternlogq	zmm13, zmm30, zmm31, 210
+        vmovdqu64	zmm30, zmm24
+        vmovdqu64	zmm31, zmm11
+        vpternlogq	zmm24, zmm11, zmm3, 210
+        vpternlogq	zmm11, zmm3, zmm15, 210
+        vpternlogq	zmm3, zmm15, zmm7, 210
+        vpternlogq	zmm15, zmm7, zmm30, 210
+        vpternlogq	zmm7, zmm30, zmm31, 210
+        vmovdqu64	zmm30, zmm18
+        vmovdqu64	zmm31, zmm5
+        vpternlogq	zmm18, zmm5, zmm22, 210
+        vpternlogq	zmm5, zmm22, zmm14, 210
+        vpternlogq	zmm22, zmm14, zmm1, 210
+        vpternlogq	zmm14, zmm1, zmm30, 210
+        vpternlogq	zmm1, zmm30, zmm31, 210
+        vmovdqu64	zmm30, zmm12
+        vmovdqu64	zmm31, zmm4
+        vpternlogq	zmm12, zmm4, zmm16, 210
+        vpternlogq	zmm4, zmm16, zmm8, 210
+        vpternlogq	zmm16, zmm8, zmm20, 210
+        vpternlogq	zmm8, zmm20, zmm30, 210
+        vpternlogq	zmm20, zmm30, zmm31, 210
+        vmovdqu64	zmm30, zmm6
+        vmovdqu64	zmm31, zmm23
+        vpternlogq	zmm6, zmm23, zmm10, 210
+        vpternlogq	zmm23, zmm10, zmm2, 210
+        vpternlogq	zmm10, zmm2, zmm19, 210
+        vpternlogq	zmm2, zmm19, zmm30, 210
+        vpternlogq	zmm19, zmm30, zmm31, 210
+        ; Iota
+        vpxorq	zmm0, zmm0, [rax+1216]
+        ; Round 20
+        ; Theta - column parity
+        vmovdqu64	zmm25, zmm0
+        vpternlogq	zmm25, zmm24, zmm18, 150
+        vpternlogq	zmm25, zmm12, zmm6, 150
+        vmovdqu64	zmm26, zmm17
+        vpternlogq	zmm26, zmm11, zmm5, 150
+        vpternlogq	zmm26, zmm4, zmm23, 150
+        vmovdqu64	zmm27, zmm9
+        vpternlogq	zmm27, zmm3, zmm22, 150
+        vpternlogq	zmm27, zmm16, zmm10, 150
+        vmovdqu64	zmm28, zmm21
+        vpternlogq	zmm28, zmm15, zmm14, 150
+        vpternlogq	zmm28, zmm8, zmm2, 150
+        vmovdqu64	zmm29, zmm13
+        vpternlogq	zmm29, zmm7, zmm1, 150
+        vpternlogq	zmm29, zmm20, zmm19, 150
+        ; Theta - D and apply in place
+        vprolq	zmm30, zmm26, 1
+        vpxorq	zmm30, zmm30, zmm29
+        vpxorq	zmm0, zmm0, zmm30
+        vpxorq	zmm24, zmm24, zmm30
+        vpxorq	zmm18, zmm18, zmm30
+        vpxorq	zmm12, zmm12, zmm30
+        vpxorq	zmm6, zmm6, zmm30
+        vprolq	zmm30, zmm27, 1
+        vpxorq	zmm30, zmm30, zmm25
+        vpxorq	zmm17, zmm17, zmm30
+        vpxorq	zmm11, zmm11, zmm30
+        vpxorq	zmm5, zmm5, zmm30
+        vpxorq	zmm4, zmm4, zmm30
+        vpxorq	zmm23, zmm23, zmm30
+        vprolq	zmm30, zmm28, 1
+        vpxorq	zmm30, zmm30, zmm26
+        vpxorq	zmm9, zmm9, zmm30
+        vpxorq	zmm3, zmm3, zmm30
+        vpxorq	zmm22, zmm22, zmm30
+        vpxorq	zmm16, zmm16, zmm30
+        vpxorq	zmm10, zmm10, zmm30
+        vprolq	zmm30, zmm29, 1
+        vpxorq	zmm30, zmm30, zmm27
+        vpxorq	zmm21, zmm21, zmm30
+        vpxorq	zmm15, zmm15, zmm30
+        vpxorq	zmm14, zmm14, zmm30
+        vpxorq	zmm8, zmm8, zmm30
+        vpxorq	zmm2, zmm2, zmm30
+        vprolq	zmm30, zmm25, 1
+        vpxorq	zmm30, zmm30, zmm28
+        vpxorq	zmm13, zmm13, zmm30
+        vpxorq	zmm7, zmm7, zmm30
+        vpxorq	zmm1, zmm1, zmm30
+        vpxorq	zmm20, zmm20, zmm30
+        vpxorq	zmm19, zmm19, zmm30
+        ; Rho - rotate each lane in place
+        vprolq	zmm17, zmm17, 1
+        vprolq	zmm9, zmm9, 62
+        vprolq	zmm21, zmm21, 28
+        vprolq	zmm13, zmm13, 27
+        vprolq	zmm24, zmm24, 36
+        vprolq	zmm11, zmm11, 44
+        vprolq	zmm3, zmm3, 6
+        vprolq	zmm15, zmm15, 55
+        vprolq	zmm7, zmm7, 20
+        vprolq	zmm18, zmm18, 3
+        vprolq	zmm5, zmm5, 10
+        vprolq	zmm22, zmm22, 43
+        vprolq	zmm14, zmm14, 25
+        vprolq	zmm1, zmm1, 39
+        vprolq	zmm12, zmm12, 41
+        vprolq	zmm4, zmm4, 45
+        vprolq	zmm16, zmm16, 15
+        vprolq	zmm8, zmm8, 21
+        vprolq	zmm20, zmm20, 8
+        vprolq	zmm6, zmm6, 18
+        vprolq	zmm23, zmm23, 2
+        vprolq	zmm10, zmm10, 61
+        vprolq	zmm2, zmm2, 56
+        vprolq	zmm19, zmm19, 14
+        ; Pi - relabel logical -> physical
+        ; Chi - per row, in place, fused with vpternlogq (imm 0xd2)
+        vmovdqu64	zmm30, zmm0
+        vmovdqu64	zmm31, zmm11
+        vpternlogq	zmm0, zmm11, zmm22, 210
+        vpternlogq	zmm11, zmm22, zmm8, 210
+        vpternlogq	zmm22, zmm8, zmm19, 210
+        vpternlogq	zmm8, zmm19, zmm30, 210
+        vpternlogq	zmm19, zmm30, zmm31, 210
+        vmovdqu64	zmm30, zmm21
+        vmovdqu64	zmm31, zmm7
+        vpternlogq	zmm21, zmm7, zmm18, 210
+        vpternlogq	zmm7, zmm18, zmm4, 210
+        vpternlogq	zmm18, zmm4, zmm10, 210
+        vpternlogq	zmm4, zmm10, zmm30, 210
+        vpternlogq	zmm10, zmm30, zmm31, 210
+        vmovdqu64	zmm30, zmm17
+        vmovdqu64	zmm31, zmm3
+        vpternlogq	zmm17, zmm3, zmm14, 210
+        vpternlogq	zmm3, zmm14, zmm20, 210
+        vpternlogq	zmm14, zmm20, zmm6, 210
+        vpternlogq	zmm20, zmm6, zmm30, 210
+        vpternlogq	zmm6, zmm30, zmm31, 210
+        vmovdqu64	zmm30, zmm13
+        vmovdqu64	zmm31, zmm24
+        vpternlogq	zmm13, zmm24, zmm5, 210
+        vpternlogq	zmm24, zmm5, zmm16, 210
+        vpternlogq	zmm5, zmm16, zmm2, 210
+        vpternlogq	zmm16, zmm2, zmm30, 210
+        vpternlogq	zmm2, zmm30, zmm31, 210
+        vmovdqu64	zmm30, zmm9
+        vmovdqu64	zmm31, zmm15
+        vpternlogq	zmm9, zmm15, zmm1, 210
+        vpternlogq	zmm15, zmm1, zmm12, 210
+        vpternlogq	zmm1, zmm12, zmm23, 210
+        vpternlogq	zmm12, zmm23, zmm30, 210
+        vpternlogq	zmm23, zmm30, zmm31, 210
+        ; Iota
+        vpxorq	zmm0, zmm0, [rax+1280]
+        ; Round 21
+        ; Theta - column parity
+        vmovdqu64	zmm25, zmm0
+        vpternlogq	zmm25, zmm21, zmm17, 150
+        vpternlogq	zmm25, zmm13, zmm9, 150
+        vmovdqu64	zmm26, zmm11
+        vpternlogq	zmm26, zmm7, zmm3, 150
+        vpternlogq	zmm26, zmm24, zmm15, 150
+        vmovdqu64	zmm27, zmm22
+        vpternlogq	zmm27, zmm18, zmm14, 150
+        vpternlogq	zmm27, zmm5, zmm1, 150
+        vmovdqu64	zmm28, zmm8
+        vpternlogq	zmm28, zmm4, zmm20, 150
+        vpternlogq	zmm28, zmm16, zmm12, 150
+        vmovdqu64	zmm29, zmm19
+        vpternlogq	zmm29, zmm10, zmm6, 150
+        vpternlogq	zmm29, zmm2, zmm23, 150
+        ; Theta - D and apply in place
+        vprolq	zmm30, zmm26, 1
+        vpxorq	zmm30, zmm30, zmm29
+        vpxorq	zmm0, zmm0, zmm30
+        vpxorq	zmm21, zmm21, zmm30
+        vpxorq	zmm17, zmm17, zmm30
+        vpxorq	zmm13, zmm13, zmm30
+        vpxorq	zmm9, zmm9, zmm30
+        vprolq	zmm30, zmm27, 1
+        vpxorq	zmm30, zmm30, zmm25
+        vpxorq	zmm11, zmm11, zmm30
+        vpxorq	zmm7, zmm7, zmm30
+        vpxorq	zmm3, zmm3, zmm30
+        vpxorq	zmm24, zmm24, zmm30
+        vpxorq	zmm15, zmm15, zmm30
+        vprolq	zmm30, zmm28, 1
+        vpxorq	zmm30, zmm30, zmm26
+        vpxorq	zmm22, zmm22, zmm30
+        vpxorq	zmm18, zmm18, zmm30
+        vpxorq	zmm14, zmm14, zmm30
+        vpxorq	zmm5, zmm5, zmm30
+        vpxorq	zmm1, zmm1, zmm30
+        vprolq	zmm30, zmm29, 1
+        vpxorq	zmm30, zmm30, zmm27
+        vpxorq	zmm8, zmm8, zmm30
+        vpxorq	zmm4, zmm4, zmm30
+        vpxorq	zmm20, zmm20, zmm30
+        vpxorq	zmm16, zmm16, zmm30
+        vpxorq	zmm12, zmm12, zmm30
+        vprolq	zmm30, zmm25, 1
+        vpxorq	zmm30, zmm30, zmm28
+        vpxorq	zmm19, zmm19, zmm30
+        vpxorq	zmm10, zmm10, zmm30
+        vpxorq	zmm6, zmm6, zmm30
+        vpxorq	zmm2, zmm2, zmm30
+        vpxorq	zmm23, zmm23, zmm30
+        ; Rho - rotate each lane in place
+        vprolq	zmm11, zmm11, 1
+        vprolq	zmm22, zmm22, 62
+        vprolq	zmm8, zmm8, 28
+        vprolq	zmm19, zmm19, 27
+        vprolq	zmm21, zmm21, 36
+        vprolq	zmm7, zmm7, 44
+        vprolq	zmm18, zmm18, 6
+        vprolq	zmm4, zmm4, 55
+        vprolq	zmm10, zmm10, 20
+        vprolq	zmm17, zmm17, 3
+        vprolq	zmm3, zmm3, 10
+        vprolq	zmm14, zmm14, 43
+        vprolq	zmm20, zmm20, 25
+        vprolq	zmm6, zmm6, 39
+        vprolq	zmm13, zmm13, 41
+        vprolq	zmm24, zmm24, 45
+        vprolq	zmm5, zmm5, 15
+        vprolq	zmm16, zmm16, 21
+        vprolq	zmm2, zmm2, 8
+        vprolq	zmm9, zmm9, 18
+        vprolq	zmm15, zmm15, 2
+        vprolq	zmm1, zmm1, 61
+        vprolq	zmm12, zmm12, 56
+        vprolq	zmm23, zmm23, 14
+        ; Pi - relabel logical -> physical
+        ; Chi - per row, in place, fused with vpternlogq (imm 0xd2)
+        vmovdqu64	zmm30, zmm0
+        vmovdqu64	zmm31, zmm7
+        vpternlogq	zmm0, zmm7, zmm14, 210
+        vpternlogq	zmm7, zmm14, zmm16, 210
+        vpternlogq	zmm14, zmm16, zmm23, 210
+        vpternlogq	zmm16, zmm23, zmm30, 210
+        vpternlogq	zmm23, zmm30, zmm31, 210
+        vmovdqu64	zmm30, zmm8
+        vmovdqu64	zmm31, zmm10
+        vpternlogq	zmm8, zmm10, zmm17, 210
+        vpternlogq	zmm10, zmm17, zmm24, 210
+        vpternlogq	zmm17, zmm24, zmm1, 210
+        vpternlogq	zmm24, zmm1, zmm30, 210
+        vpternlogq	zmm1, zmm30, zmm31, 210
+        vmovdqu64	zmm30, zmm11
+        vmovdqu64	zmm31, zmm18
+        vpternlogq	zmm11, zmm18, zmm20, 210
+        vpternlogq	zmm18, zmm20, zmm2, 210
+        vpternlogq	zmm20, zmm2, zmm9, 210
+        vpternlogq	zmm2, zmm9, zmm30, 210
+        vpternlogq	zmm9, zmm30, zmm31, 210
+        vmovdqu64	zmm30, zmm19
+        vmovdqu64	zmm31, zmm21
+        vpternlogq	zmm19, zmm21, zmm3, 210
+        vpternlogq	zmm21, zmm3, zmm5, 210
+        vpternlogq	zmm3, zmm5, zmm12, 210
+        vpternlogq	zmm5, zmm12, zmm30, 210
+        vpternlogq	zmm12, zmm30, zmm31, 210
+        vmovdqu64	zmm30, zmm22
+        vmovdqu64	zmm31, zmm4
+        vpternlogq	zmm22, zmm4, zmm6, 210
+        vpternlogq	zmm4, zmm6, zmm13, 210
+        vpternlogq	zmm6, zmm13, zmm15, 210
+        vpternlogq	zmm13, zmm15, zmm30, 210
+        vpternlogq	zmm15, zmm30, zmm31, 210
+        ; Iota
+        vpxorq	zmm0, zmm0, [rax+1344]
+        ; Round 22
+        ; Theta - column parity
+        vmovdqu64	zmm25, zmm0
+        vpternlogq	zmm25, zmm8, zmm11, 150
+        vpternlogq	zmm25, zmm19, zmm22, 150
+        vmovdqu64	zmm26, zmm7
+        vpternlogq	zmm26, zmm10, zmm18, 150
+        vpternlogq	zmm26, zmm21, zmm4, 150
+        vmovdqu64	zmm27, zmm14
+        vpternlogq	zmm27, zmm17, zmm20, 150
+        vpternlogq	zmm27, zmm3, zmm6, 150
+        vmovdqu64	zmm28, zmm16
+        vpternlogq	zmm28, zmm24, zmm2, 150
+        vpternlogq	zmm28, zmm5, zmm13, 150
+        vmovdqu64	zmm29, zmm23
+        vpternlogq	zmm29, zmm1, zmm9, 150
+        vpternlogq	zmm29, zmm12, zmm15, 150
+        ; Theta - D and apply in place
+        vprolq	zmm30, zmm26, 1
+        vpxorq	zmm30, zmm30, zmm29
+        vpxorq	zmm0, zmm0, zmm30
+        vpxorq	zmm8, zmm8, zmm30
+        vpxorq	zmm11, zmm11, zmm30
+        vpxorq	zmm19, zmm19, zmm30
+        vpxorq	zmm22, zmm22, zmm30
+        vprolq	zmm30, zmm27, 1
+        vpxorq	zmm30, zmm30, zmm25
+        vpxorq	zmm7, zmm7, zmm30
+        vpxorq	zmm10, zmm10, zmm30
+        vpxorq	zmm18, zmm18, zmm30
+        vpxorq	zmm21, zmm21, zmm30
+        vpxorq	zmm4, zmm4, zmm30
+        vprolq	zmm30, zmm28, 1
+        vpxorq	zmm30, zmm30, zmm26
+        vpxorq	zmm14, zmm14, zmm30
+        vpxorq	zmm17, zmm17, zmm30
+        vpxorq	zmm20, zmm20, zmm30
+        vpxorq	zmm3, zmm3, zmm30
+        vpxorq	zmm6, zmm6, zmm30
+        vprolq	zmm30, zmm29, 1
+        vpxorq	zmm30, zmm30, zmm27
+        vpxorq	zmm16, zmm16, zmm30
+        vpxorq	zmm24, zmm24, zmm30
+        vpxorq	zmm2, zmm2, zmm30
+        vpxorq	zmm5, zmm5, zmm30
+        vpxorq	zmm13, zmm13, zmm30
+        vprolq	zmm30, zmm25, 1
+        vpxorq	zmm30, zmm30, zmm28
+        vpxorq	zmm23, zmm23, zmm30
+        vpxorq	zmm1, zmm1, zmm30
+        vpxorq	zmm9, zmm9, zmm30
+        vpxorq	zmm12, zmm12, zmm30
+        vpxorq	zmm15, zmm15, zmm30
+        ; Rho - rotate each lane in place
+        vprolq	zmm7, zmm7, 1
+        vprolq	zmm14, zmm14, 62
+        vprolq	zmm16, zmm16, 28
+        vprolq	zmm23, zmm23, 27
+        vprolq	zmm8, zmm8, 36
+        vprolq	zmm10, zmm10, 44
+        vprolq	zmm17, zmm17, 6
+        vprolq	zmm24, zmm24, 55
+        vprolq	zmm1, zmm1, 20
+        vprolq	zmm11, zmm11, 3
+        vprolq	zmm18, zmm18, 10
+        vprolq	zmm20, zmm20, 43
+        vprolq	zmm2, zmm2, 25
+        vprolq	zmm9, zmm9, 39
+        vprolq	zmm19, zmm19, 41
+        vprolq	zmm21, zmm21, 45
+        vprolq	zmm3, zmm3, 15
+        vprolq	zmm5, zmm5, 21
+        vprolq	zmm12, zmm12, 8
+        vprolq	zmm22, zmm22, 18
+        vprolq	zmm4, zmm4, 2
+        vprolq	zmm6, zmm6, 61
+        vprolq	zmm13, zmm13, 56
+        vprolq	zmm15, zmm15, 14
+        ; Pi - relabel logical -> physical
+        ; Chi - per row, in place, fused with vpternlogq (imm 0xd2)
+        vmovdqu64	zmm30, zmm0
+        vmovdqu64	zmm31, zmm10
+        vpternlogq	zmm0, zmm10, zmm20, 210
+        vpternlogq	zmm10, zmm20, zmm5, 210
+        vpternlogq	zmm20, zmm5, zmm15, 210
+        vpternlogq	zmm5, zmm15, zmm30, 210
+        vpternlogq	zmm15, zmm30, zmm31, 210
+        vmovdqu64	zmm30, zmm16
+        vmovdqu64	zmm31, zmm1
+        vpternlogq	zmm16, zmm1, zmm11, 210
+        vpternlogq	zmm1, zmm11, zmm21, 210
+        vpternlogq	zmm11, zmm21, zmm6, 210
+        vpternlogq	zmm21, zmm6, zmm30, 210
+        vpternlogq	zmm6, zmm30, zmm31, 210
+        vmovdqu64	zmm30, zmm7
+        vmovdqu64	zmm31, zmm17
+        vpternlogq	zmm7, zmm17, zmm2, 210
+        vpternlogq	zmm17, zmm2, zmm12, 210
+        vpternlogq	zmm2, zmm12, zmm22, 210
+        vpternlogq	zmm12, zmm22, zmm30, 210
+        vpternlogq	zmm22, zmm30, zmm31, 210
+        vmovdqu64	zmm30, zmm23
+        vmovdqu64	zmm31, zmm8
+        vpternlogq	zmm23, zmm8, zmm18, 210
+        vpternlogq	zmm8, zmm18, zmm3, 210
+        vpternlogq	zmm18, zmm3, zmm13, 210
+        vpternlogq	zmm3, zmm13, zmm30, 210
+        vpternlogq	zmm13, zmm30, zmm31, 210
+        vmovdqu64	zmm30, zmm14
+        vmovdqu64	zmm31, zmm24
+        vpternlogq	zmm14, zmm24, zmm9, 210
+        vpternlogq	zmm24, zmm9, zmm19, 210
+        vpternlogq	zmm9, zmm19, zmm4, 210
+        vpternlogq	zmm19, zmm4, zmm30, 210
+        vpternlogq	zmm4, zmm30, zmm31, 210
+        ; Iota
+        vpxorq	zmm0, zmm0, [rax+1408]
+        ; Round 23
+        ; Theta - column parity
+        vmovdqu64	zmm25, zmm0
+        vpternlogq	zmm25, zmm16, zmm7, 150
+        vpternlogq	zmm25, zmm23, zmm14, 150
+        vmovdqu64	zmm26, zmm10
+        vpternlogq	zmm26, zmm1, zmm17, 150
+        vpternlogq	zmm26, zmm8, zmm24, 150
+        vmovdqu64	zmm27, zmm20
+        vpternlogq	zmm27, zmm11, zmm2, 150
+        vpternlogq	zmm27, zmm18, zmm9, 150
+        vmovdqu64	zmm28, zmm5
+        vpternlogq	zmm28, zmm21, zmm12, 150
+        vpternlogq	zmm28, zmm3, zmm19, 150
+        vmovdqu64	zmm29, zmm15
+        vpternlogq	zmm29, zmm6, zmm22, 150
+        vpternlogq	zmm29, zmm13, zmm4, 150
+        ; Theta - D and apply in place
+        vprolq	zmm30, zmm26, 1
+        vpxorq	zmm30, zmm30, zmm29
+        vpxorq	zmm0, zmm0, zmm30
+        vpxorq	zmm16, zmm16, zmm30
+        vpxorq	zmm7, zmm7, zmm30
+        vpxorq	zmm23, zmm23, zmm30
+        vpxorq	zmm14, zmm14, zmm30
+        vprolq	zmm30, zmm27, 1
+        vpxorq	zmm30, zmm30, zmm25
+        vpxorq	zmm10, zmm10, zmm30
+        vpxorq	zmm1, zmm1, zmm30
+        vpxorq	zmm17, zmm17, zmm30
+        vpxorq	zmm8, zmm8, zmm30
+        vpxorq	zmm24, zmm24, zmm30
+        vprolq	zmm30, zmm28, 1
+        vpxorq	zmm30, zmm30, zmm26
+        vpxorq	zmm20, zmm20, zmm30
+        vpxorq	zmm11, zmm11, zmm30
+        vpxorq	zmm2, zmm2, zmm30
+        vpxorq	zmm18, zmm18, zmm30
+        vpxorq	zmm9, zmm9, zmm30
+        vprolq	zmm30, zmm29, 1
+        vpxorq	zmm30, zmm30, zmm27
+        vpxorq	zmm5, zmm5, zmm30
+        vpxorq	zmm21, zmm21, zmm30
+        vpxorq	zmm12, zmm12, zmm30
+        vpxorq	zmm3, zmm3, zmm30
+        vpxorq	zmm19, zmm19, zmm30
+        vprolq	zmm30, zmm25, 1
+        vpxorq	zmm30, zmm30, zmm28
+        vpxorq	zmm15, zmm15, zmm30
+        vpxorq	zmm6, zmm6, zmm30
+        vpxorq	zmm22, zmm22, zmm30
+        vpxorq	zmm13, zmm13, zmm30
+        vpxorq	zmm4, zmm4, zmm30
+        ; Rho - rotate each lane in place
+        vprolq	zmm10, zmm10, 1
+        vprolq	zmm20, zmm20, 62
+        vprolq	zmm5, zmm5, 28
+        vprolq	zmm15, zmm15, 27
+        vprolq	zmm16, zmm16, 36
+        vprolq	zmm1, zmm1, 44
+        vprolq	zmm11, zmm11, 6
+        vprolq	zmm21, zmm21, 55
+        vprolq	zmm6, zmm6, 20
+        vprolq	zmm7, zmm7, 3
+        vprolq	zmm17, zmm17, 10
+        vprolq	zmm2, zmm2, 43
+        vprolq	zmm12, zmm12, 25
+        vprolq	zmm22, zmm22, 39
+        vprolq	zmm23, zmm23, 41
+        vprolq	zmm8, zmm8, 45
+        vprolq	zmm18, zmm18, 15
+        vprolq	zmm3, zmm3, 21
+        vprolq	zmm13, zmm13, 8
+        vprolq	zmm14, zmm14, 18
+        vprolq	zmm24, zmm24, 2
+        vprolq	zmm9, zmm9, 61
+        vprolq	zmm19, zmm19, 56
+        vprolq	zmm4, zmm4, 14
+        ; Pi - relabel logical -> physical
+        ; Chi - per row, in place, fused with vpternlogq (imm 0xd2)
+        vmovdqu64	zmm30, zmm0
+        vmovdqu64	zmm31, zmm1
+        vpternlogq	zmm0, zmm1, zmm2, 210
+        vpternlogq	zmm1, zmm2, zmm3, 210
+        vpternlogq	zmm2, zmm3, zmm4, 210
+        vpternlogq	zmm3, zmm4, zmm30, 210
+        vpternlogq	zmm4, zmm30, zmm31, 210
+        vmovdqu64	zmm30, zmm5
+        vmovdqu64	zmm31, zmm6
+        vpternlogq	zmm5, zmm6, zmm7, 210
+        vpternlogq	zmm6, zmm7, zmm8, 210
+        vpternlogq	zmm7, zmm8, zmm9, 210
+        vpternlogq	zmm8, zmm9, zmm30, 210
+        vpternlogq	zmm9, zmm30, zmm31, 210
+        vmovdqu64	zmm30, zmm10
+        vmovdqu64	zmm31, zmm11
+        vpternlogq	zmm10, zmm11, zmm12, 210
+        vpternlogq	zmm11, zmm12, zmm13, 210
+        vpternlogq	zmm12, zmm13, zmm14, 210
+        vpternlogq	zmm13, zmm14, zmm30, 210
+        vpternlogq	zmm14, zmm30, zmm31, 210
+        vmovdqu64	zmm30, zmm15
+        vmovdqu64	zmm31, zmm16
+        vpternlogq	zmm15, zmm16, zmm17, 210
+        vpternlogq	zmm16, zmm17, zmm18, 210
+        vpternlogq	zmm17, zmm18, zmm19, 210
+        vpternlogq	zmm18, zmm19, zmm30, 210
+        vpternlogq	zmm19, zmm30, zmm31, 210
+        vmovdqu64	zmm30, zmm20
+        vmovdqu64	zmm31, zmm21
+        vpternlogq	zmm20, zmm21, zmm22, 210
+        vpternlogq	zmm21, zmm22, zmm23, 210
+        vpternlogq	zmm22, zmm23, zmm24, 210
+        vpternlogq	zmm23, zmm24, zmm30, 210
+        vpternlogq	zmm24, zmm30, zmm31, 210
+        ; Iota
+        vpxorq	zmm0, zmm0, [rax+1472]
+        vmovdqu64	[rcx], zmm0
+        vmovdqu64	[rcx+64], zmm1
+        vmovdqu64	[rcx+128], zmm2
+        vmovdqu64	[rcx+192], zmm3
+        vmovdqu64	[rcx+256], zmm4
+        vmovdqu64	[rcx+320], zmm5
+        vmovdqu64	[rcx+384], zmm6
+        vmovdqu64	[rcx+448], zmm7
+        vmovdqu64	[rcx+512], zmm8
+        vmovdqu64	[rcx+576], zmm9
+        vmovdqu64	[rcx+640], zmm10
+        vmovdqu64	[rcx+704], zmm11
+        vmovdqu64	[rcx+768], zmm12
+        vmovdqu64	[rcx+832], zmm13
+        vmovdqu64	[rcx+896], zmm14
+        vmovdqu64	[rcx+960], zmm15
+        vmovdqu64	[rcx+1024], zmm16
+        vmovdqu64	[rcx+1088], zmm17
+        vmovdqu64	[rcx+1152], zmm18
+        vmovdqu64	[rcx+1216], zmm19
+        vmovdqu64	[rcx+1280], zmm20
+        vmovdqu64	[rcx+1344], zmm21
+        vmovdqu64	[rcx+1408], zmm22
+        vmovdqu64	[rcx+1472], zmm23
+        vmovdqu64	[rcx+1536], zmm24
+        mov	rbx, rcx
+        mov	rsi, 21
+L_sha3_blocksx8_out_avx512_word:
+        mov	rbp, QWORD PTR [rbx]
+        mov	QWORD PTR [rdx], rbp
+        mov	rbp, QWORD PTR [rbx+8]
+        mov	QWORD PTR [r9], rbp
+        mov	rbp, QWORD PTR [rbx+16]
+        mov	QWORD PTR [r10], rbp
+        mov	rbp, QWORD PTR [rbx+24]
+        mov	QWORD PTR [r11], rbp
+        mov	rbp, QWORD PTR [rbx+32]
+        mov	QWORD PTR [r12], rbp
+        mov	rbp, QWORD PTR [rbx+40]
+        mov	QWORD PTR [r13], rbp
+        mov	rbp, QWORD PTR [rbx+48]
+        mov	QWORD PTR [r14], rbp
+        mov	rbp, QWORD PTR [rbx+56]
+        mov	QWORD PTR [r15], rbp
+        add	rbx, 64
+        add	rdx, 8
+        add	r9, 8
+        add	r10, 8
+        add	r11, 8
+        add	r12, 8
+        add	r13, 8
+        add	r14, 8
+        add	r15, 8
+        sub	rdi, 1
+        jz	L_sha3_blocksx8_out_avx512_done
+        sub	rsi, 1
+        jnz	L_sha3_blocksx8_out_avx512_word
+        jmp	L_sha3_blocksx8_out_avx512_block
+L_sha3_blocksx8_out_avx512_done:
+        vzeroupper
+        vmovdqu	xmm6, OWORD PTR [rsp]
+        vmovdqu	xmm7, OWORD PTR [rsp+16]
+        vmovdqu	xmm8, OWORD PTR [rsp+32]
+        vmovdqu	xmm9, OWORD PTR [rsp+48]
+        vmovdqu	xmm10, OWORD PTR [rsp+64]
+        vmovdqu	xmm11, OWORD PTR [rsp+80]
+        vmovdqu	xmm12, OWORD PTR [rsp+96]
+        vmovdqu	xmm13, OWORD PTR [rsp+112]
+        vmovdqu	xmm14, OWORD PTR [rsp+128]
+        vmovdqu	xmm15, OWORD PTR [rsp+144]
+        add	rsp, 160
+        pop	rbp
+        pop	rbx
+        pop	rsi
+        pop	rdi
+        pop	r15
+        pop	r14
+        pop	r13
+        pop	r12
+        ret
+sha3_blocksx8_out_avx512 ENDP
+_TEXT ENDS
+ENDIF
+ENDIF
 ENDIF
 END
diff --git a/wolfcrypt/src/sp_x86_64_asm.asm b/wolfcrypt/src/sp_x86_64_asm.asm
index 4fcf7b20a9..7d847de179 100644
--- a/wolfcrypt/src/sp_x86_64_asm.asm
+++ b/wolfcrypt/src/sp_x86_64_asm.asm
@@ -45883,60 +45883,52 @@ _TEXT ENDS
 IFDEF HAVE_INTEL_AVX2
 _DATA SEGMENT
 ALIGN 16
-L_sp256_mod_inv_avx2_4_order DWORD \
-     00632551h,  01e84f3bh,  03bce6fah,  03ffffffh,
-     03ff0000h,  00000000h,  00000000h,  00000000h,
-     0272b0bfh,  02b69c5eh,  03ffffffh,  000003ffh,
-     003fffffh,  00000000h,  00000000h,  00000000h
+L_sp256_mod_inv_avx2_4_order DWORD 00632551h, 01e84f3bh, 03bce6fah, 03ffffffh
+        DWORD 03ff0000h, 00000000h, 00000000h, 00000000h
+        DWORD 0272b0bfh, 02b69c5eh, 03ffffffh, 000003ffh
+        DWORD 003fffffh, 00000000h, 00000000h, 00000000h
 ptr_L_sp256_mod_inv_avx2_4_order QWORD L_sp256_mod_inv_avx2_4_order
 _DATA ENDS
 _DATA SEGMENT
 ALIGN 16
-L_sp256_mod_inv_avx2_4_one QWORD \
-     0000000000000001h,  0000000000000000h,
-     0000000000000000h,  0000000000000000h
+L_sp256_mod_inv_avx2_4_one QWORD 0000000000000001h, 0000000000000000h
+        QWORD 0000000000000000h, 0000000000000000h
 ptr_L_sp256_mod_inv_avx2_4_one QWORD L_sp256_mod_inv_avx2_4_one
 _DATA ENDS
 _DATA SEGMENT
 ALIGN 16
-L_sp256_mod_inv_avx2_4_all_one DWORD \
-     00000001h,  00000001h,  00000001h,  00000001h,
-     00000001h,  00000001h,  00000001h,  00000001h
+L_sp256_mod_inv_avx2_4_all_one DWORD 00000001h, 00000001h, 00000001h, 00000001h
+        DWORD 00000001h, 00000001h, 00000001h, 00000001h
 ptr_L_sp256_mod_inv_avx2_4_all_one QWORD L_sp256_mod_inv_avx2_4_all_one
 _DATA ENDS
 _DATA SEGMENT
 ALIGN 16
-L_sp256_mod_inv_avx2_4_mask01111 DWORD \
-     00000000h,  00000001h,  00000001h,  00000001h,
-     00000001h,  00000000h,  00000000h,  00000000h
+L_sp256_mod_inv_avx2_4_mask01111 DWORD 00000000h, 00000001h, 00000001h, 00000001h
+        DWORD 00000001h, 00000000h, 00000000h, 00000000h
 ptr_L_sp256_mod_inv_avx2_4_mask01111 QWORD L_sp256_mod_inv_avx2_4_mask01111
 _DATA ENDS
 _DATA SEGMENT
 ALIGN 16
-L_sp256_mod_inv_avx2_4_down_one_dword DWORD \
-     00000001h,  00000002h,  00000003h,  00000004h,
-     00000005h,  00000006h,  00000007h,  00000007h
+L_sp256_mod_inv_avx2_4_down_one_dword DWORD 00000001h, 00000002h, 00000003h, 00000004h
+        DWORD 00000005h, 00000006h, 00000007h, 00000007h
 ptr_L_sp256_mod_inv_avx2_4_down_one_dword QWORD L_sp256_mod_inv_avx2_4_down_one_dword
 _DATA ENDS
 _DATA SEGMENT
 ALIGN 16
-L_sp256_mod_inv_avx2_4_neg DWORD \
-     00000000h,  00000000h,  00000000h,  00000000h,
-     80000000h,  00000000h,  00000000h,  00000000h
+L_sp256_mod_inv_avx2_4_neg DWORD 00000000h, 00000000h, 00000000h, 00000000h
+        DWORD 80000000h, 00000000h, 00000000h, 00000000h
 ptr_L_sp256_mod_inv_avx2_4_neg QWORD L_sp256_mod_inv_avx2_4_neg
 _DATA ENDS
 _DATA SEGMENT
 ALIGN 16
-L_sp256_mod_inv_avx2_4_up_one_dword DWORD \
-     00000007h,  00000000h,  00000001h,  00000002h,
-     00000003h,  00000007h,  00000007h,  00000007h
+L_sp256_mod_inv_avx2_4_up_one_dword DWORD 00000007h, 00000000h, 00000001h, 00000002h
+        DWORD 00000003h, 00000007h, 00000007h, 00000007h
 ptr_L_sp256_mod_inv_avx2_4_up_one_dword QWORD L_sp256_mod_inv_avx2_4_up_one_dword
 _DATA ENDS
 _DATA SEGMENT
 ALIGN 16
-L_sp256_mod_inv_avx2_4_mask26 DWORD \
-     03ffffffh,  03ffffffh,  03ffffffh,  03ffffffh,
-     03ffffffh,  00000000h,  00000000h,  00000000h
+L_sp256_mod_inv_avx2_4_mask26 DWORD 03ffffffh, 03ffffffh, 03ffffffh, 03ffffffh
+        DWORD 03ffffffh, 00000000h, 00000000h, 00000000h
 ptr_L_sp256_mod_inv_avx2_4_mask26 QWORD L_sp256_mod_inv_avx2_4_mask26
 _DATA ENDS
 ; /* Non-constant time modular inversion.
diff --git a/wolfcrypt/src/wc_frodokem.c b/wolfcrypt/src/wc_frodokem.c
new file mode 100644
index 0000000000..56f7303798
--- /dev/null
+++ b/wolfcrypt/src/wc_frodokem.c
@@ -0,0 +1,1421 @@
+/* wc_frodokem.c
+ *
+ * Copyright (C) 2006-2026 wolfSSL Inc.
+ *
+ * This file is part of wolfSSL.
+ *
+ * wolfSSL is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * wolfSSL is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335, USA
+ */
+
+/* Reference implementation of FrodoKEM.
+ *
+ * Implementation based on:
+ *   https://www.ietf.org/archive/id/draft-longa-cfrg-frodokem-03.txt
+ *
+ * Matrix A is generated using SHAKE128 (Section 6.7.2 of the draft) and the
+ * hashing function SHAKE is SHAKE128 for FrodoKEM-640 and SHAKE256 for
+ * FrodoKEM-976 and FrodoKEM-1344 (Table 1).  The "standard" (salted) FrodoKEM
+ * variants are implemented.
+ *
+ * The API mirrors the wolfCrypt ML-KEM API (see wc_mlkem.c), using FrodoKem in
+ * the names in place of MlKem.
+ */
+
+#define _WC_BUILDING_WC_FRODOKEM_C
+
+#include <wolfssl/wolfcrypt/libwolfssl_sources.h>
+
+#include <wolfssl/wolfcrypt/wc_frodokem_mat.h>
+#include <wolfssl/wolfcrypt/hash.h>
+#include <wolfssl/wolfcrypt/memory.h>
+
+#ifdef NO_INLINE
+    #include <wolfssl/wolfcrypt/misc.h>
+#else
+    #define WOLFSSL_MISC_INCLUDED
+    #include <wolfcrypt/src/misc.c>
+#endif
+
+#ifdef WOLFSSL_HAVE_FRODOKEM
+
+/* Domain separator prepended to seedSE when generating key generation noise. */
+#define FRODOKEM_DOMAIN_KEYGEN  0x5F
+/* Domain separator prepended to seedSE when generating encapsulation noise. */
+#define FRODOKEM_DOMAIN_ENCAPS  0x96
+
+/******************************************************************************/
+/* Parameter sets.                                                            */
+/******************************************************************************/
+
+#ifdef WOLFSSL_WC_FRODOKEM_640
+/* CDF table for FrodoKEM-640 (Table 5). */
+static const word16 frodokem_cdf_640[] = {
+    4643, 13363, 20579, 25843, 29227, 31145, 32103, 32525, 32689, 32745,
+    32762, 32766, 32767
+};
+#endif
+
+#ifdef WOLFSSL_WC_FRODOKEM_976
+/* CDF table for FrodoKEM-976 (Table 5). */
+static const word16 frodokem_cdf_976[] = {
+    5638, 15915, 23689, 28571, 31116, 32217, 32613, 32731, 32760, 32766, 32767
+};
+#endif
+
+#ifdef WOLFSSL_WC_FRODOKEM_1344
+/* CDF table for FrodoKEM-1344 (Table 5). */
+static const word16 frodokem_cdf_1344[] = {
+    9142, 23462, 30338, 32361, 32725, 32765, 32767
+};
+#endif
+
+/* Constant parameter sets.
+ *
+ * FrodoKemParams field order:
+ *   n, d, qMask, b, cdf, cdfLen, lenSec, useShake256, pkSize, skSize,
+ *   lenSE, lenSalt, ctSize, useAes
+ * A base macro supplies the fields common to all variants of a parameter set
+ * (through skSize); the per-type initializer appends lenSE, lenSalt, ctSize and
+ * useAes. Standard FrodoKEM uses lenSE = lenSalt = 2 * lenSec and a salted
+ * ciphertext; eFrodoKEM (ephemeral) uses lenSE = lenSec and no salt.
+ */
+#ifdef WOLFSSL_WC_FRODOKEM_640
+#define FRODOKEM_BASE_640 \
+    WC_FRODOKEM_640_N, WC_FRODOKEM_640_D, 0x7FFF, 2, frodokem_cdf_640, \
+    (int)(sizeof(frodokem_cdf_640) / sizeof(frodokem_cdf_640[0])), \
+    WC_FRODOKEM_640_LENSEC, 0, \
+    WC_FRODOKEM_640_PUBLIC_KEY_SIZE, WC_FRODOKEM_640_PRIVATE_KEY_SIZE
+#ifdef WOLFSSL_FRODOKEM_SHAKE
+static const FrodoKemParams frodokem_p_640_shake =
+    { FRODOKEM_BASE_640, WC_FRODOKEM_640_LENSE, WC_FRODOKEM_640_LENSALT,
+      WC_FRODOKEM_640_CIPHER_TEXT_SIZE, 0 };
+#ifdef WOLFSSL_FRODOKEM_EPHEMERAL
+static const FrodoKemParams frodokem_ep_640_shake =
+    { FRODOKEM_BASE_640, WC_FRODOKEM_640_LENSEC, 0,
+      FRODOKEM_CT_SZ(WC_FRODOKEM_640_N, WC_FRODOKEM_640_D, 0), 0 };
+#endif
+#endif
+#ifdef WOLFSSL_FRODOKEM_AES
+static const FrodoKemParams frodokem_p_640_aes =
+    { FRODOKEM_BASE_640, WC_FRODOKEM_640_LENSE, WC_FRODOKEM_640_LENSALT,
+      WC_FRODOKEM_640_CIPHER_TEXT_SIZE, 1 };
+#ifdef WOLFSSL_FRODOKEM_EPHEMERAL
+static const FrodoKemParams frodokem_ep_640_aes =
+    { FRODOKEM_BASE_640, WC_FRODOKEM_640_LENSEC, 0,
+      FRODOKEM_CT_SZ(WC_FRODOKEM_640_N, WC_FRODOKEM_640_D, 0), 1 };
+#endif
+#endif
+#endif /* WOLFSSL_WC_FRODOKEM_640 */
+
+#ifdef WOLFSSL_WC_FRODOKEM_976
+#define FRODOKEM_BASE_976 \
+    WC_FRODOKEM_976_N, WC_FRODOKEM_976_D, 0xFFFF, 3, frodokem_cdf_976, \
+    (int)(sizeof(frodokem_cdf_976) / sizeof(frodokem_cdf_976[0])), \
+    WC_FRODOKEM_976_LENSEC, 1, \
+    WC_FRODOKEM_976_PUBLIC_KEY_SIZE, WC_FRODOKEM_976_PRIVATE_KEY_SIZE
+#ifdef WOLFSSL_FRODOKEM_SHAKE
+static const FrodoKemParams frodokem_p_976_shake =
+    { FRODOKEM_BASE_976, WC_FRODOKEM_976_LENSE, WC_FRODOKEM_976_LENSALT,
+      WC_FRODOKEM_976_CIPHER_TEXT_SIZE, 0 };
+#ifdef WOLFSSL_FRODOKEM_EPHEMERAL
+static const FrodoKemParams frodokem_ep_976_shake =
+    { FRODOKEM_BASE_976, WC_FRODOKEM_976_LENSEC, 0,
+      FRODOKEM_CT_SZ(WC_FRODOKEM_976_N, WC_FRODOKEM_976_D, 0), 0 };
+#endif
+#endif
+#ifdef WOLFSSL_FRODOKEM_AES
+static const FrodoKemParams frodokem_p_976_aes =
+    { FRODOKEM_BASE_976, WC_FRODOKEM_976_LENSE, WC_FRODOKEM_976_LENSALT,
+      WC_FRODOKEM_976_CIPHER_TEXT_SIZE, 1 };
+#ifdef WOLFSSL_FRODOKEM_EPHEMERAL
+static const FrodoKemParams frodokem_ep_976_aes =
+    { FRODOKEM_BASE_976, WC_FRODOKEM_976_LENSEC, 0,
+      FRODOKEM_CT_SZ(WC_FRODOKEM_976_N, WC_FRODOKEM_976_D, 0), 1 };
+#endif
+#endif
+#endif /* WOLFSSL_WC_FRODOKEM_976 */
+
+#ifdef WOLFSSL_WC_FRODOKEM_1344
+#define FRODOKEM_BASE_1344 \
+    WC_FRODOKEM_1344_N, WC_FRODOKEM_1344_D, 0xFFFF, 4, frodokem_cdf_1344, \
+    (int)(sizeof(frodokem_cdf_1344) / sizeof(frodokem_cdf_1344[0])), \
+    WC_FRODOKEM_1344_LENSEC, 1, \
+    WC_FRODOKEM_1344_PUBLIC_KEY_SIZE, WC_FRODOKEM_1344_PRIVATE_KEY_SIZE
+#ifdef WOLFSSL_FRODOKEM_SHAKE
+static const FrodoKemParams frodokem_p_1344_shake =
+    { FRODOKEM_BASE_1344, WC_FRODOKEM_1344_LENSE, WC_FRODOKEM_1344_LENSALT,
+      WC_FRODOKEM_1344_CIPHER_TEXT_SIZE, 0 };
+#ifdef WOLFSSL_FRODOKEM_EPHEMERAL
+static const FrodoKemParams frodokem_ep_1344_shake =
+    { FRODOKEM_BASE_1344, WC_FRODOKEM_1344_LENSEC, 0,
+      FRODOKEM_CT_SZ(WC_FRODOKEM_1344_N, WC_FRODOKEM_1344_D, 0), 0 };
+#endif
+#endif
+#ifdef WOLFSSL_FRODOKEM_AES
+static const FrodoKemParams frodokem_p_1344_aes =
+    { FRODOKEM_BASE_1344, WC_FRODOKEM_1344_LENSE, WC_FRODOKEM_1344_LENSALT,
+      WC_FRODOKEM_1344_CIPHER_TEXT_SIZE, 1 };
+#ifdef WOLFSSL_FRODOKEM_EPHEMERAL
+static const FrodoKemParams frodokem_ep_1344_aes =
+    { FRODOKEM_BASE_1344, WC_FRODOKEM_1344_LENSEC, 0,
+      FRODOKEM_CT_SZ(WC_FRODOKEM_1344_N, WC_FRODOKEM_1344_D, 0), 1 };
+#endif
+#endif
+#endif /* WOLFSSL_WC_FRODOKEM_1344 */
+
+/* Get the constant parameters for a key type.
+ *
+ * A type is a base parameter set (WC_FRODOKEM_640/976/1344) optionally OR'd
+ * with FRODOKEM_AES (AES-128 matrix A) and/or FRODOKEM_EPHEMERAL (eFrodoKEM).
+ *
+ * @param  [in]  type  Type of key.
+ * @return  Pointer to the constant parameters, or NULL when the type or a
+ *          requested modifier is not compiled in.
+ */
+static const FrodoKemParams* frodokem_get_params(int type)
+{
+    const FrodoKemParams* p = NULL;
+
+    switch (type) {
+#ifdef WOLFSSL_WC_FRODOKEM_640
+#ifdef WOLFSSL_FRODOKEM_SHAKE
+    case WC_FRODOKEM_640_SHAKE:
+        p = &frodokem_p_640_shake;
+        break;
+#ifdef WOLFSSL_FRODOKEM_EPHEMERAL
+    case WC_EFRODOKEM_640_SHAKE:
+        p = &frodokem_ep_640_shake;
+        break;
+#endif
+#endif
+#ifdef WOLFSSL_FRODOKEM_AES
+    case WC_FRODOKEM_640_AES:
+        p = &frodokem_p_640_aes;
+        break;
+#ifdef WOLFSSL_FRODOKEM_EPHEMERAL
+    case WC_EFRODOKEM_640_AES:
+        p = &frodokem_ep_640_aes;
+        break;
+#endif
+#endif
+#endif /* WOLFSSL_WC_FRODOKEM_640 */
+#ifdef WOLFSSL_WC_FRODOKEM_976
+#ifdef WOLFSSL_FRODOKEM_SHAKE
+    case WC_FRODOKEM_976_SHAKE:
+        p = &frodokem_p_976_shake;
+        break;
+#ifdef WOLFSSL_FRODOKEM_EPHEMERAL
+    case WC_EFRODOKEM_976_SHAKE:
+        p = &frodokem_ep_976_shake;
+        break;
+#endif
+#endif
+#ifdef WOLFSSL_FRODOKEM_AES
+    case WC_FRODOKEM_976_AES:
+        p = &frodokem_p_976_aes;
+        break;
+#ifdef WOLFSSL_FRODOKEM_EPHEMERAL
+    case WC_EFRODOKEM_976_AES:
+        p = &frodokem_ep_976_aes;
+        break;
+#endif
+#endif
+#endif /* WOLFSSL_WC_FRODOKEM_976 */
+#ifdef WOLFSSL_WC_FRODOKEM_1344
+#ifdef WOLFSSL_FRODOKEM_SHAKE
+    case WC_FRODOKEM_1344_SHAKE:
+        p = &frodokem_p_1344_shake;
+        break;
+#ifdef WOLFSSL_FRODOKEM_EPHEMERAL
+    case WC_EFRODOKEM_1344_SHAKE:
+        p = &frodokem_ep_1344_shake;
+        break;
+#endif
+#endif
+#ifdef WOLFSSL_FRODOKEM_AES
+    case WC_FRODOKEM_1344_AES:
+        p = &frodokem_p_1344_aes;
+        break;
+#ifdef WOLFSSL_FRODOKEM_EPHEMERAL
+    case WC_EFRODOKEM_1344_AES:
+        p = &frodokem_ep_1344_aes;
+        break;
+#endif
+#endif
+#endif /* WOLFSSL_WC_FRODOKEM_1344 */
+    default:
+        break;
+    }
+
+    return p;
+}
+
+/******************************************************************************/
+/* Key object management.                                                     */
+/******************************************************************************/
+
+#ifndef WC_NO_CONSTRUCTORS
+/* Create a new FrodoKEM key object.
+ *
+ * @param  [in]  type   Type of key.
+ * @param  [in]  heap   Dynamic memory hint.
+ * @param  [in]  devId  Device Id.
+ * @return  Pointer to new FrodoKemKey object on success.
+ * @return  NULL on failure.
+ */
+FrodoKemKey* wc_FrodoKemKey_New(int type, void* heap, int devId)
+{
+    int ret;
+    FrodoKemKey* key;
+
+    key = (FrodoKemKey*)XMALLOC(sizeof(FrodoKemKey), heap,
+        DYNAMIC_TYPE_TMP_BUFFER);
+    if (key != NULL) {
+        ret = wc_FrodoKemKey_Init(key, type, heap, devId);
+        if (ret != 0) {
+            XFREE(key, heap, DYNAMIC_TYPE_TMP_BUFFER);
+            key = NULL;
+        }
+    }
+
+    return key;
+}
+
+/* Delete and free a FrodoKEM key object.
+ *
+ * @param  [in]       key    FrodoKEM key object to delete.
+ * @param  [in, out]  key_p  Pointer to key pointer to set to NULL.
+ * @return  0 on success.
+ * @return  BAD_FUNC_ARG when key is NULL.
+ */
+int wc_FrodoKemKey_Delete(FrodoKemKey* key, FrodoKemKey** key_p)
+{
+    int ret = 0;
+
+    if (key == NULL) {
+        ret = BAD_FUNC_ARG;
+    }
+    else {
+        void* heap = key->heap;
+
+        wc_FrodoKemKey_Free(key);
+        XFREE(key, heap, DYNAMIC_TYPE_TMP_BUFFER);
+        if (key_p != NULL) {
+            *key_p = NULL;
+        }
+    }
+
+    return ret;
+}
+#endif /* !WC_NO_CONSTRUCTORS */
+
+/* Initialize the FrodoKEM key.
+ *
+ * @param  [out]  key    FrodoKEM key object to initialize.
+ * @param  [in]   type   Type of key.
+ * @param  [in]   heap   Dynamic memory hint.
+ * @param  [in]   devId  Device Id.
+ * @return  0 on success.
+ * @return  BAD_FUNC_ARG when key is NULL or type has invalid bits set.
+ * @return  NOT_COMPILED_IN when the key type is not compiled in.
+ */
+int wc_FrodoKemKey_Init(FrodoKemKey* key, int type, void* heap, int devId)
+{
+    int ret = 0;
+    const FrodoKemParams* p = NULL;
+
+    if (key == NULL) {
+        ret = BAD_FUNC_ARG;
+    }
+    /* Reject types with bits outside the known base and modifier ranges. */
+    else if ((type & ~(FRODOKEM_BASE_MASK | FRODOKEM_AES | FRODOKEM_EPHEMERAL))
+            != 0) {
+        ret = BAD_FUNC_ARG;
+    }
+    else {
+        /* Resolve the constant parameters. NULL means the base parameter set
+         * or a requested modifier (AES / ephemeral) is not compiled in. */
+        p = frodokem_get_params(type);
+        if (p == NULL) {
+            ret = NOT_COMPILED_IN;
+        }
+    }
+
+    if (ret == 0) {
+        XMEMSET(key, 0, sizeof(*key));
+        key->type = type;
+        key->params = p;
+        key->heap = heap;
+        key->devId = devId;
+        key->flags = 0;
+
+        /* Cache CPU feature flags for SIMD dispatch in matrix-A generation. */
+        frodokem_init();
+
+        /* Initialize the reusable SHAKE object. Init is variant-agnostic and
+         * the hashing/matrix-A helpers re-initialize it as SHAKE-128 or -256
+         * as needed, so the always-present SHAKE-128 init is used here. */
+        ret = wc_InitShake128(&key->shake, NULL, INVALID_DEVID);
+#ifdef WOLFSSL_FRODOKEM_AES
+        /* Initialize the reusable AES object; re-keyed with seedA per op. */
+        if (ret == 0) {
+            ret = wc_AesInit(&key->aes, heap, INVALID_DEVID);
+        }
+#endif
+    }
+
+    return ret;
+}
+
+/* Free the FrodoKEM key.
+ *
+ * Zeroizes secret material.
+ *
+ * @param  [in, out]  key  FrodoKEM key object.
+ * @return  0 on success.
+ * @return  BAD_FUNC_ARG when key is NULL.
+ */
+int wc_FrodoKemKey_Free(FrodoKemKey* key)
+{
+    int ret = 0;
+
+    if (key == NULL) {
+        ret = BAD_FUNC_ARG;
+    }
+    else {
+        /* Zeroize secret material. */
+        ForceZero(key->s, sizeof(key->s));
+        ForceZero(key->sMat, sizeof(key->sMat));
+        ForceZero(key->pkh, sizeof(key->pkh));
+        /* Free the reusable SHAKE object (zeroizes its state). Free is variant
+         * agnostic; SHAKE-128 is always compiled in, so use it. */
+        wc_Shake128_Free(&key->shake);
+#ifdef WOLFSSL_FRODOKEM_AES
+        /* Free the reusable AES object (zeroizes its key schedule). */
+        wc_AesFree(&key->aes);
+#endif
+        key->flags = 0;
+    }
+
+    return ret;
+}
+
+/******************************************************************************/
+/* Sizes.                                                                     */
+/******************************************************************************/
+
+/* Get the size in bytes of the ciphertext.
+ *
+ * @param  [in]   key  FrodoKEM key object.
+ * @param  [out]  len  Ciphertext size in bytes.
+ * @return  0 on success.
+ * @return  BAD_FUNC_ARG when key or len is NULL.
+ * @return  NOT_COMPILED_IN when key type is not supported.
+ */
+int wc_FrodoKemKey_CipherTextSize(const FrodoKemKey* key, word32* len)
+{
+    int ret = 0;
+    const FrodoKemParams* p = NULL;
+
+    if ((key == NULL) || (len == NULL)) {
+        ret = BAD_FUNC_ARG;
+    }
+    else {
+        p = key->params;
+        if (p == NULL) {
+            ret = NOT_COMPILED_IN;
+        }
+    }
+
+    if (ret == 0) {
+        *len = (word32)p->ctSize;
+    }
+
+    return ret;
+}
+
+/* Get the size in bytes of the shared secret.
+ *
+ * @param  [in]   key  FrodoKEM key object.
+ * @param  [out]  len  Shared secret size in bytes.
+ * @return  0 on success.
+ * @return  BAD_FUNC_ARG when key or len is NULL.
+ * @return  NOT_COMPILED_IN when key type is not supported.
+ */
+int wc_FrodoKemKey_SharedSecretSize(const FrodoKemKey* key, word32* len)
+{
+    int ret = 0;
+    const FrodoKemParams* p = NULL;
+
+    if ((key == NULL) || (len == NULL)) {
+        ret = BAD_FUNC_ARG;
+    }
+    else {
+        p = key->params;
+        if (p == NULL) {
+            ret = NOT_COMPILED_IN;
+        }
+    }
+
+    if (ret == 0) {
+        *len = (word32)p->lenSec;
+    }
+
+    return ret;
+}
+
+/* Get the size in bytes of an encoded private key.
+ *
+ * @param  [in]   key  FrodoKEM key object.
+ * @param  [out]  len  Private key size in bytes.
+ * @return  0 on success.
+ * @return  BAD_FUNC_ARG when key or len is NULL.
+ * @return  NOT_COMPILED_IN when key type is not supported.
+ */
+int wc_FrodoKemKey_PrivateKeySize(const FrodoKemKey* key, word32* len)
+{
+    int ret = 0;
+    const FrodoKemParams* p = NULL;
+
+    if ((key == NULL) || (len == NULL)) {
+        ret = BAD_FUNC_ARG;
+    }
+    else {
+        p = key->params;
+        if (p == NULL) {
+            ret = NOT_COMPILED_IN;
+        }
+    }
+
+    if (ret == 0) {
+        *len = (word32)p->skSize;
+    }
+
+    return ret;
+}
+
+/* Get the size in bytes of an encoded public key.
+ *
+ * @param  [in]   key  FrodoKEM key object.
+ * @param  [out]  len  Public key size in bytes.
+ * @return  0 on success.
+ * @return  BAD_FUNC_ARG when key or len is NULL.
+ * @return  NOT_COMPILED_IN when key type is not supported.
+ */
+int wc_FrodoKemKey_PublicKeySize(const FrodoKemKey* key, word32* len)
+{
+    int ret = 0;
+    const FrodoKemParams* p = NULL;
+
+    if ((key == NULL) || (len == NULL)) {
+        ret = BAD_FUNC_ARG;
+    }
+    else {
+        p = key->params;
+        if (p == NULL) {
+            ret = NOT_COMPILED_IN;
+        }
+    }
+
+    if (ret == 0) {
+        *len = (word32)p->pkSize;
+    }
+
+    return ret;
+}
+
+/* Wipe the secret-derived Keccak state retained in the reusable SHAKE object
+ * after an operation.
+ *
+ * Encapsulation and decapsulation finish by hashing the shared-secret key
+ * material, leaving it in the reusable SHAKE state until the next use. Every
+ * SHAKE use re-initializes the object first, so re-initializing here zeroizes
+ * that residue (InitSha3 clears the state) and leaves it ready. Best-effort: an
+ * init failure is not propagated as the object is re-initialized on next use.
+ *
+ * @param  [in, out]  key  FrodoKEM key object whose SHAKE state is wiped.
+ */
+static void frodokem_wipe_shake(FrodoKemKey* key)
+{
+    if (key != NULL) {
+        (void)wc_InitShake128(&key->shake, NULL, INVALID_DEVID);
+    }
+}
+
+/******************************************************************************/
+/* Key generation.                                                            */
+/******************************************************************************/
+
+#ifndef WOLFSSL_FRODOKEM_NO_MAKE_KEY
+
+/* Make a FrodoKEM key object using random data.
+ *
+ * Implements FrodoKEM.KeyGenInternal(s, seedSE, z).  The random data is the
+ * concatenation s || seedSE || z.
+ *
+ * @param  [in, out]  key   FrodoKEM key object.
+ * @param  [in]       rand  Random data.
+ * @param  [in]       len   Length of random data in bytes.
+ * @return  0 on success.
+ * @return  BAD_FUNC_ARG when key or rand is NULL.
+ * @return  BUFFER_E when len is not the expected size.
+ * @return  NOT_COMPILED_IN when key type is not supported.
+ * @return  MEMORY_E when dynamic memory allocation failed.
+ */
+int wc_FrodoKemKey_MakeKeyWithRandom(FrodoKemKey* key,
+    const unsigned char* rand, int len)
+{
+    const FrodoKemParams* p = NULL;
+    int ret = 0;
+    int n = 0;
+    void* heap = NULL;
+    const byte* seedSE;
+    const byte* z;
+    /* One buffer holds E (B computed in place over it), a scratch slot of
+     * FRODOKEM_ROW_MULT A-rows for on-the-fly matrix-A generation, and seInput
+     * (the domain-prefixed seed) in a trailing byte region. */
+    word16* bMat = NULL;
+    word16* row = NULL;
+    byte* seInput = NULL;
+
+    if ((key == NULL) || (rand == NULL)) {
+        ret = BAD_FUNC_ARG;
+    }
+    else {
+        p = key->params;
+        if (p == NULL) {
+            ret = NOT_COMPILED_IN;
+        }
+        else if (len != p->lenSec + p->lenSE + FRODOKEM_SEEDA_SZ) {
+            ret = BUFFER_E;
+        }
+    }
+
+    if (ret == 0) {
+        n = p->n;
+        heap = key->heap;
+        seedSE = rand + p->lenSec;
+        z = rand + p->lenSec + p->lenSE;
+
+        /* Store s (implicit rejection value). */
+        XMEMCPY(key->s, rand, (size_t)p->lenSec);
+
+        /* seedA = SHAKE(z, lenA). */
+        ret = frodokem_shake_seeda(p, &key->shake, z, key->seedA);
+    }
+    if (ret == 0) {
+        bMat = (word16*)XMALLOC(
+            (size_t)(n * FRODOKEM_NBAR + FRODOKEM_ROW_MULT * n) *
+                sizeof(word16) + (size_t)(1 + p->lenSE),
+            heap, DYNAMIC_TYPE_TMP_BUFFER);
+        if (bMat == NULL) {
+            ret = MEMORY_E;
+        }
+        else {
+            row = bMat + n * FRODOKEM_NBAR;
+            /* seInput (domain || seedSE) follows the word16 arena. */
+            seInput = (byte*)(bMat + n * FRODOKEM_NBAR +
+                FRODOKEM_ROW_MULT * n);
+        }
+    }
+
+    if (ret == 0) {
+        /* Generate and sample S^T (nbar x n) into key->sMat and E (n x nbar)
+         * into bMat (separate buffers) from SHAKE(0x5F || seedSE). Assemble the
+         * domain-prefixed seed; the row scratch is free here, so use it as the
+         * SHAKE block scratch. */
+        seInput[0] = FRODOKEM_DOMAIN_KEYGEN;
+        XMEMCPY(seInput + 1, seedSE, (size_t)p->lenSE);
+        ret = frodokem_gen_noise(p, &key->shake, seInput, (byte*)row,
+            key->sMat, FRODOKEM_NBAR * n, bMat, n * FRODOKEM_NBAR);
+    }
+    if (ret == 0) {
+        /* B = A * S + E, accumulated in place over E in bMat. */
+        ret = frodokem_mul_add_as_plus_e(key, bMat, key->sMat, row);
+    }
+    if (ret == 0) {
+        /* b = Pack(B, n, nbar). */
+        frodokem_pack(key->b, bMat, n * FRODOKEM_NBAR, p->d);
+        /* pkh = SHAKE(pk, lensec). seedA and b are contiguous in the key so
+         * the public key is a single buffer of length pkSize. */
+        ret = frodokem_shake_oneshot(p, &key->shake, key->seedA,
+            (word32)p->pkSize, key->pkh, (word32)p->lenSec);
+    }
+    if (ret == 0) {
+        key->flags |= FRODOKEM_FLAG_PRIV_SET | FRODOKEM_FLAG_PUB_SET |
+            FRODOKEM_FLAG_PKH_SET;
+    }
+
+    /* bMat held the secret error matrix E and the seed (in seInput), so
+     * zeroize it before freeing. */
+    if (bMat != NULL) {
+        ForceZero(bMat,
+            (word32)(n * FRODOKEM_NBAR + FRODOKEM_ROW_MULT * n) *
+                sizeof(word16) + (word32)(1 + p->lenSE));
+        XFREE(bMat, heap, DYNAMIC_TYPE_TMP_BUFFER);
+    }
+
+    /* Wipe secret-derived residue from the reusable SHAKE state. */
+    frodokem_wipe_shake(key);
+
+    return ret;
+}
+
+/* Make a FrodoKEM key object using a random number generator.
+ *
+ * @param  [in, out]  key  FrodoKEM key object.
+ * @param  [in]       rng  Random number generator.
+ * @return  0 on success.
+ * @return  BAD_FUNC_ARG when key or rng is NULL.
+ * @return  NOT_COMPILED_IN when no RNG is compiled in or key type unsupported.
+ * @return  MEMORY_E when dynamic memory allocation fails.
+ */
+int wc_FrodoKemKey_MakeKey(FrodoKemKey* key, WC_RNG* rng)
+{
+    int ret = 0;
+#ifndef WC_NO_RNG
+    const FrodoKemParams* p = NULL;
+    int randLen = 0;
+#ifdef WOLFSSL_SMALL_STACK
+    byte* rand = NULL;
+#else
+    byte rand[WC_FRODOKEM_1344_MAKEKEY_RAND_SZ];
+#endif
+
+    if ((key == NULL) || (rng == NULL)) {
+        ret = BAD_FUNC_ARG;
+    }
+    else {
+        p = key->params;
+        if (p == NULL) {
+            ret = NOT_COMPILED_IN;
+        }
+    }
+
+    if (ret == 0) {
+        randLen = p->lenSec + p->lenSE + FRODOKEM_SEEDA_SZ;
+    }
+#ifdef WOLFSSL_SMALL_STACK
+    /* Keep the random data off the stack. */
+    if (ret == 0) {
+        rand = (byte*)XMALLOC((size_t)randLen, key->heap,
+            DYNAMIC_TYPE_TMP_BUFFER);
+        if (rand == NULL) {
+            ret = MEMORY_E;
+        }
+    }
+#endif
+    if (ret == 0) {
+        ret = wc_RNG_GenerateBlock(rng, rand, (word32)randLen);
+    }
+    if (ret == 0) {
+        ret = wc_FrodoKemKey_MakeKeyWithRandom(key, rand, randLen);
+    }
+
+#ifdef WOLFSSL_SMALL_STACK
+    if (rand != NULL) {
+        ForceZero(rand, (word32)randLen);
+        XFREE(rand, key->heap, DYNAMIC_TYPE_TMP_BUFFER);
+    }
+#else
+    ForceZero(rand, sizeof(rand));
+#endif
+#else
+    (void)key;
+    (void)rng;
+    ret = NOT_COMPILED_IN;
+#endif
+    return ret;
+}
+#endif /* !WOLFSSL_FRODOKEM_NO_MAKE_KEY */
+
+/******************************************************************************/
+/* Encapsulation.                                                             */
+/******************************************************************************/
+
+#ifndef WOLFSSL_FRODOKEM_NO_ENCAPSULATE
+
+/* Encapsulate to a FrodoKEM public key using random data.
+ *
+ * Implements FrodoKEM.Encaps with the random data being u || salt.
+ *
+ * @param  [in]   key   FrodoKEM key object with public key set.
+ * @param  [out]  ct    Ciphertext.
+ * @param  [out]  ss    Shared secret.
+ * @param  [in]   rand  Random data (u || salt).
+ * @param  [in]   len   Length of random data in bytes.
+ * @return  0 on success.
+ * @return  BAD_FUNC_ARG when a pointer is NULL.
+ * @return  BUFFER_E when len is not the expected size.
+ * @return  BAD_STATE_E when public key is not set.
+ * @return  NOT_COMPILED_IN when key type is not supported.
+ * @return  MEMORY_E when dynamic memory allocation failed.
+ */
+int wc_FrodoKemKey_EncapsulateWithRandom(FrodoKemKey* key, unsigned char* ct,
+    unsigned char* ss, const unsigned char* rand, int len)
+{
+    const FrodoKemParams* p = NULL;
+    int ret = 0;
+    int n = 0;
+    int nn = FRODOKEM_NBAR_SQ;
+    void* heap = NULL;
+    const byte* u;
+    const byte* salt;
+    byte* c1;
+    byte* c2;
+    byte* saltOut;
+    /* One arena holds S' (nbar x n), a reused slot (E' -> B' -> B) (nbar x n)
+     * and E'' (nbar x nbar), contiguous so their noise is one region, plus a
+     * FRODOKEM_ROW_MULT-row scratch slot for matrix-A generation. */
+    word16* mat = NULL;
+    word16* sp = NULL;
+    word16* work = NULL;
+    word16* epp = NULL;
+    word16* row = NULL;
+#ifdef WOLFSSL_SMALL_STACK
+    /* seedSEk and pkh are carved from the mat allocation to keep them off the
+     * stack. */
+    byte* seedSEk = NULL;
+    byte* pkh = NULL;
+#else
+    byte seedSEk[1 + FRODOKEM_MAX_LENSE + FRODOKEM_MAX_LENSEC];
+    byte pkh[FRODOKEM_MAX_LENSEC];
+#endif
+    byte* kVal;
+    size_t matSz = 0;
+
+    if ((key == NULL) || (ct == NULL) || (ss == NULL) || (rand == NULL)) {
+        ret = BAD_FUNC_ARG;
+    }
+    else {
+        p = key->params;
+        if (p == NULL) {
+            ret = NOT_COMPILED_IN;
+        }
+        else if ((key->flags & FRODOKEM_FLAG_PUB_SET) == 0) {
+            ret = BAD_STATE_E;
+        }
+        else if (len != p->lenSec + p->lenSalt) {
+            ret = BUFFER_E;
+        }
+    }
+
+    if (ret == 0) {
+        n = p->n;
+        heap = key->heap;
+        u = rand;
+        salt = rand + p->lenSec;
+        c1 = ct;
+        c2 = ct + (p->d * n);
+        saltOut = ct + (p->d * n) + (p->d * FRODOKEM_NBAR);
+
+        matSz = (size_t)(2 * FRODOKEM_NBAR * n + FRODOKEM_NBAR_SQ +
+            FRODOKEM_ROW_MULT * n) * sizeof(word16);
+#ifdef WOLFSSL_SMALL_STACK
+        /* seedSEk (1 + lenSE + lenSec) and pkh (lenSec) follow the arena. */
+        matSz += (size_t)(1 + p->lenSE + 2 * p->lenSec);
+#endif
+        mat = (word16*)XMALLOC(matSz, heap, DYNAMIC_TYPE_TMP_BUFFER);
+        if (mat == NULL) {
+            ret = MEMORY_E;
+        }
+        else {
+            sp = mat;
+            work = mat + FRODOKEM_NBAR * n;
+            epp = mat + 2 * FRODOKEM_NBAR * n;
+            row = mat + 2 * FRODOKEM_NBAR * n + FRODOKEM_NBAR_SQ;
+#ifdef WOLFSSL_SMALL_STACK
+            seedSEk = (byte*)(mat + 2 * FRODOKEM_NBAR * n + FRODOKEM_NBAR_SQ +
+                FRODOKEM_ROW_MULT * n);
+            pkh = seedSEk + (1 + p->lenSE + p->lenSec);
+#endif
+        }
+    }
+
+    if (ret == 0) {
+        /* pkh = SHAKE(pk, lensec). seedA and b are contiguous in the key so
+         * the public key is a single buffer of length pkSize. */
+        ret = frodokem_shake_oneshot(p, &key->shake, key->seedA,
+            (word32)p->pkSize, pkh, (word32)p->lenSec);
+    }
+    if (ret == 0) {
+        /* seedSEk = domain || seedSE || k: the leading byte is the noise domain
+         * and seedSE || k = SHAKE(pkh || u || salt, lenSE + lensec) is written
+         * after it. rand is the message u || salt, one contiguous buffer. */
+        seedSEk[0] = FRODOKEM_DOMAIN_ENCAPS;
+        ret = frodokem_gen_seedse_k(p, &key->shake, pkh, rand, seedSEk + 1);
+    }
+    if (ret == 0) {
+        kVal = seedSEk + 1 + p->lenSE;
+
+        /* Generate and sample S' | E' | E'' from SHAKE(domain || seedSE). The
+         * three are contiguous from sp, so they form a single noise region. The
+         * row scratch is free here, so use it as the SHAKE block scratch. */
+        ret = frodokem_gen_noise(p, &key->shake, seedSEk, (byte*)row,
+            sp, 2 * FRODOKEM_NBAR * n + FRODOKEM_NBAR_SQ, NULL, 0);
+    }
+    if (ret == 0) {
+        /* B' = S' * A + E', accumulated in place over E' in work. */
+        ret = frodokem_mul_add_sa_plus_e(key, work, sp, row);
+    }
+    if (ret == 0) {
+        /* c1 = Pack(B', nbar, n). */
+        frodokem_pack(c1, work, FRODOKEM_NBAR * n, p->d);
+        /* B' no longer needed: reuse work for B = Unpack(b, n, nbar). */
+        frodokem_unpack(work, key->b, n * FRODOKEM_NBAR, p->d);
+        /* V = S' * B + E'', accumulated in place over E'' in epp. */
+        frodokem_mul_add_sb_plus_e(epp, work, sp, p->n, (int)p->qMask);
+        /* C = V + Encode(u). B is finished, so reuse work for Encode(u). */
+        frodokem_key_encode(work, u, p->d, p->b);
+        frodokem_add(epp, work, (int)p->qMask);
+        /* c2 = Pack(C, nbar, nbar). */
+        frodokem_pack(c2, epp, nn, p->d);
+        /* salt is appended to the ciphertext. */
+        XMEMCPY(saltOut, salt, (size_t)p->lenSalt);
+
+        /* ss = SHAKE(c1 || c2 || salt || k, lensec). c1 || c2 || salt is the
+         * leading part of ct. */
+        ret = frodokem_shake(p, &key->shake, ct,
+            (word32)((p->d * n) + (p->d * FRODOKEM_NBAR) + p->lenSalt),
+            kVal, (word32)p->lenSec, ss, (word32)p->lenSec);
+    }
+
+#ifndef WOLFSSL_SMALL_STACK
+    ForceZero(pkh, sizeof(pkh));
+    ForceZero(seedSEk, sizeof(seedSEk));
+#endif
+    /* Encode(u), V/E'' and B' live in mat (as do seedSEk and pkh under small
+     * stack), all zeroized on free below. */
+    if (mat != NULL) {
+        ForceZero(mat, (word32)matSz);
+        XFREE(mat, heap, DYNAMIC_TYPE_TMP_BUFFER);
+    }
+
+    /* Wipe secret-derived residue from the reusable SHAKE state. */
+    frodokem_wipe_shake(key);
+
+    return ret;
+}
+
+/* Encapsulate to a FrodoKEM public key using a random number generator.
+ *
+ * @param  [in]   key  FrodoKEM key object with public key set.
+ * @param  [out]  ct   Ciphertext.
+ * @param  [out]  ss   Shared secret.
+ * @param  [in]   rng  Random number generator.
+ * @return  0 on success.
+ * @return  BAD_FUNC_ARG when a pointer is NULL.
+ * @return  NOT_COMPILED_IN when no RNG is compiled in or key type unsupported.
+ * @return  MEMORY_E when dynamic memory allocation fails.
+ */
+int wc_FrodoKemKey_Encapsulate(FrodoKemKey* key, unsigned char* ct,
+    unsigned char* ss, WC_RNG* rng)
+{
+    int ret = 0;
+#ifndef WC_NO_RNG
+    const FrodoKemParams* p = NULL;
+    int randLen = 0;
+#ifdef WOLFSSL_SMALL_STACK
+    byte* rand = NULL;
+#else
+    byte rand[WC_FRODOKEM_1344_ENC_RAND_SZ];
+#endif
+
+    if ((key == NULL) || (ct == NULL) || (ss == NULL) || (rng == NULL)) {
+        ret = BAD_FUNC_ARG;
+    }
+    else {
+        p = key->params;
+        if (p == NULL) {
+            ret = NOT_COMPILED_IN;
+        }
+    }
+
+    if (ret == 0) {
+        randLen = p->lenSec + p->lenSalt;
+    }
+#ifdef WOLFSSL_SMALL_STACK
+    /* Keep the random data off the stack. */
+    if (ret == 0) {
+        rand = (byte*)XMALLOC((size_t)randLen, key->heap,
+            DYNAMIC_TYPE_TMP_BUFFER);
+        if (rand == NULL) {
+            ret = MEMORY_E;
+        }
+    }
+#endif
+    if (ret == 0) {
+        ret = wc_RNG_GenerateBlock(rng, rand, (word32)randLen);
+    }
+    if (ret == 0) {
+        ret = wc_FrodoKemKey_EncapsulateWithRandom(key, ct, ss, rand, randLen);
+    }
+
+#ifdef WOLFSSL_SMALL_STACK
+    if (rand != NULL) {
+        ForceZero(rand, (word32)randLen);
+        XFREE(rand, key->heap, DYNAMIC_TYPE_TMP_BUFFER);
+    }
+#else
+    ForceZero(rand, sizeof(rand));
+#endif
+#else
+    (void)key;
+    (void)ct;
+    (void)ss;
+    (void)rng;
+    ret = NOT_COMPILED_IN;
+#endif
+    return ret;
+}
+#endif /* !WOLFSSL_FRODOKEM_NO_ENCAPSULATE */
+
+/******************************************************************************/
+/* Decapsulation.                                                             */
+/******************************************************************************/
+
+#ifndef WOLFSSL_FRODOKEM_NO_DECAPSULATE
+/* Decapsulate a FrodoKEM ciphertext.
+ *
+ * Implements FrodoKEM.Decaps with implicit rejection.
+ *
+ * @param  [in]   key  FrodoKEM key object with private key set.
+ * @param  [out]  ss   Shared secret.
+ * @param  [in]   ct   Ciphertext.
+ * @param  [in]   len  Length of ciphertext in bytes.
+ * @return  0 on success.
+ * @return  BAD_FUNC_ARG when a pointer is NULL.
+ * @return  BUFFER_E when len is not the expected size.
+ * @return  BAD_STATE_E when private key is not set.
+ * @return  NOT_COMPILED_IN when key type is not supported.
+ * @return  MEMORY_E when dynamic memory allocation failed.
+ */
+int wc_FrodoKemKey_Decapsulate(FrodoKemKey* key, unsigned char* ss,
+    const unsigned char* ct, word32 len)
+{
+    const FrodoKemParams* p = NULL;
+    int ret = 0;
+    int n = 0;
+    int i;
+    int nn = FRODOKEM_NBAR_SQ;
+    void* heap = NULL;
+    const byte* c1;
+    const byte* c2;
+    const byte* salt;
+    /* Arena: B' (kept for compare), then S' | E'(-> B'' -> B) | E'' contiguous
+     * so their noise is one region, plus a FRODOKEM_ROW_MULT-row scratch slot
+     * for matrix-A generation. */
+    word16* mat = NULL;
+    word16* bpIn = NULL;
+    word16* sp = NULL;
+    word16* work = NULL;
+    word16* epp = NULL;
+    word16* row = NULL;
+    /* uSalt holds the decoded u' followed by a copy of the ciphertext salt, so
+     * u' || salt is contiguous for gen_seedse_k. */
+#ifdef WOLFSSL_SMALL_STACK
+    /* cIn, v, cEnc, seedSEk and uSalt are carved from the mat allocation to
+     * keep them off the stack. */
+    word16* cIn = NULL;
+    word16* v = NULL;
+    word16* cEnc = NULL;
+    byte* seedSEk = NULL;
+    byte* uSalt = NULL;
+#else
+    byte seedSEk[1 + FRODOKEM_MAX_LENSE + FRODOKEM_MAX_LENSEC];
+    byte uSalt[FRODOKEM_MAX_LENSEC + FRODOKEM_MAX_LENSALT];
+    word16 cIn[FRODOKEM_NBAR_SQ];
+    word16 v[FRODOKEM_NBAR_SQ];
+    word16 cEnc[FRODOKEM_NBAR_SQ];
+#endif
+    byte* kPrime;
+    word16 diff = 0;
+    word32 isEq;
+    byte mask;
+    size_t matSz = 0;
+
+    if ((key == NULL) || (ss == NULL) || (ct == NULL)) {
+        ret = BAD_FUNC_ARG;
+    }
+    else {
+        p = key->params;
+        if (p == NULL) {
+            ret = NOT_COMPILED_IN;
+        }
+        else if ((key->flags & FRODOKEM_FLAG_PRIV_SET) == 0) {
+            ret = BAD_STATE_E;
+        }
+        else if (len != (word32)p->ctSize) {
+            ret = BUFFER_E;
+        }
+    }
+
+    if (ret == 0) {
+        n = p->n;
+        heap = key->heap;
+        c1 = ct;
+        c2 = ct + (p->d * n);
+        salt = ct + (p->d * n) + (p->d * FRODOKEM_NBAR);
+
+        matSz = (size_t)(3 * FRODOKEM_NBAR * n + FRODOKEM_NBAR_SQ +
+            FRODOKEM_ROW_MULT * n) * sizeof(word16);
+#ifdef WOLFSSL_SMALL_STACK
+        /* cIn, v, cEnc (3 * nbar^2 words), then seedSEk (1 + lenSE + lenSec)
+         * and uSalt (lenSec + lensalt) bytes, all follow the matrix arena. */
+        matSz += (size_t)(3 * FRODOKEM_NBAR_SQ) * sizeof(word16);
+        matSz += (size_t)(1 + p->lenSE + 2 * p->lenSec + p->lenSalt);
+#endif
+        mat = (word16*)XMALLOC(matSz, heap, DYNAMIC_TYPE_TMP_BUFFER);
+        if (mat == NULL) {
+            ret = MEMORY_E;
+        }
+        else {
+            bpIn = mat;
+            sp = mat + FRODOKEM_NBAR * n;
+            work = mat + 2 * FRODOKEM_NBAR * n;
+            epp = mat + 3 * FRODOKEM_NBAR * n;
+            row = mat + 3 * FRODOKEM_NBAR * n + FRODOKEM_NBAR_SQ;
+#ifdef WOLFSSL_SMALL_STACK
+            cIn = row + FRODOKEM_ROW_MULT * n;
+            v = cIn + FRODOKEM_NBAR_SQ;
+            cEnc = v + FRODOKEM_NBAR_SQ;
+            seedSEk = (byte*)(cEnc + FRODOKEM_NBAR_SQ);
+            uSalt = seedSEk + (1 + p->lenSE + p->lenSec);
+#endif
+        }
+    }
+
+    if (ret == 0) {
+        /* B' = Unpack(c1, nbar, n) and C = Unpack(c2, nbar, nbar). */
+        frodokem_unpack(bpIn, c1, FRODOKEM_NBAR * n, p->d);
+        frodokem_unpack(cIn, c2, nn, p->d);
+        /* M = C - B' * S, computed in place: v holds B' * S, then M. */
+        frodokem_mul_bs(v, bpIn, key->sMat, p->n, (int)p->qMask);
+        for (i = 0; i < nn; i++) {
+            v[i] = (word16)((cIn[i] - v[i]) & p->qMask);
+        }
+        /* u' = Decode(M), placed next to a copy of the ciphertext salt so
+         * u' || salt is contiguous for the hash below. */
+        frodokem_key_decode(uSalt, v, p);
+        XMEMCPY(uSalt + p->lenSec, salt, (size_t)p->lenSalt);
+
+        /* seedSEk = domain || seedSE' || k': seedSE' || k' = SHAKE(pkh || u' ||
+         * salt, lenSE + lensec) written after the leading domain byte. */
+        seedSEk[0] = FRODOKEM_DOMAIN_ENCAPS;
+        ret = frodokem_gen_seedse_k(p, &key->shake, key->pkh, uSalt,
+            seedSEk + 1);
+    }
+    if (ret == 0) {
+        /* Generate and sample S' | E' | E'' from SHAKE(domain || seedSE'). The
+         * three are contiguous from sp, so they form a single noise region. The
+         * row scratch is free here, so use it as the SHAKE block scratch. */
+        ret = frodokem_gen_noise(p, &key->shake, seedSEk, (byte*)row,
+            sp, 2 * FRODOKEM_NBAR * n + FRODOKEM_NBAR_SQ, NULL, 0);
+    }
+    if (ret == 0) {
+        /* B'' = S' * A + E', accumulated in place over E' in work. */
+        ret = frodokem_mul_add_sa_plus_e(key, work, sp, row);
+    }
+    if (ret == 0) {
+        /* Compare B' (bpIn) against B'' (work) before reusing work. */
+        for (i = 0; i < FRODOKEM_NBAR * n; i++) {
+            diff |= (word16)(bpIn[i] ^ work[i]);
+        }
+        /* B'' no longer needed: reuse work for B = Unpack(b, n, nbar). */
+        frodokem_unpack(work, key->b, n * FRODOKEM_NBAR, p->d);
+        /* V = S' * B + E'' (in place over E'' in epp); C' = V + Encode(u'). */
+        frodokem_mul_add_sb_plus_e(epp, work, sp, p->n, (int)p->qMask);
+        frodokem_key_encode(cEnc, uSalt, p->d, p->b);
+        frodokem_add(epp, cEnc, (int)p->qMask);
+        /* Compare C (cIn) against C' (epp). */
+        for (i = 0; i < nn; i++) {
+            diff |= (word16)(cIn[i] ^ epp[i]);
+        }
+
+        kPrime = seedSEk + 1 + p->lenSE;
+        /* isEq is 1 when diff == 0, else 0. */
+        isEq = (word32)(((word32)diff - 1) >> 31);
+        mask = (byte)(0 - isEq);
+
+        /* kHat = k' if ciphertext valid, else s (implicit rejection). Selected
+         * in place over k' (kPrime) in seedSEk. */
+        for (i = 0; i < p->lenSec; i++) {
+            kPrime[i] = (byte)((kPrime[i] & mask) | (key->s[i] & (byte)~mask));
+        }
+
+        /* ss = SHAKE(c1 || c2 || salt || kHat, lensec). c1 || c2 || salt is
+         * the leading part of ct. */
+        ret = frodokem_shake(p, &key->shake, ct,
+            (word32)((p->d * n) + (p->d * FRODOKEM_NBAR) + p->lenSalt),
+            kPrime, (word32)p->lenSec, ss, (word32)p->lenSec);
+    }
+
+#ifndef WOLFSSL_SMALL_STACK
+    ForceZero(uSalt, sizeof(uSalt));
+    ForceZero(seedSEk, sizeof(seedSEk));
+    /* Zeroize the secret-derived stack matrices (E'' lives in mat). */
+    ForceZero(v, sizeof(v));
+    ForceZero(cEnc, sizeof(cEnc));
+#endif
+    /* Under small stack uSalt, seedSEk, cIn, v and cEnc live in mat, zeroized
+     * below along with the rest of the arena. */
+    if (mat != NULL) {
+        ForceZero(mat, (word32)matSz);
+        XFREE(mat, heap, DYNAMIC_TYPE_TMP_BUFFER);
+    }
+
+    /* Wipe secret-derived residue from the reusable SHAKE state. */
+    frodokem_wipe_shake(key);
+
+    return ret;
+}
+#endif /* !WOLFSSL_FRODOKEM_NO_DECAPSULATE */
+
+/******************************************************************************/
+/* Key encoding and decoding.                                                 */
+/******************************************************************************/
+
+/* Encode the FrodoKEM public key: seedA || b.
+ *
+ * @param  [in]   key  FrodoKEM key object with public key set.
+ * @param  [out]  out  Output buffer.
+ * @param  [in]   len  Length of output buffer in bytes.
+ * @return  0 on success.
+ * @return  BAD_FUNC_ARG when key or out is NULL.
+ * @return  BAD_STATE_E when public key is not set.
+ * @return  BUFFER_E when len is not the expected size.
+ * @return  NOT_COMPILED_IN when key type is not supported.
+ */
+int wc_FrodoKemKey_EncodePublicKey(FrodoKemKey* key, unsigned char* out,
+    word32 len)
+{
+    int ret = 0;
+    const FrodoKemParams* p = NULL;
+
+    if ((key == NULL) || (out == NULL)) {
+        ret = BAD_FUNC_ARG;
+    }
+    else {
+        p = key->params;
+        if (p == NULL) {
+            ret = NOT_COMPILED_IN;
+        }
+        else if ((key->flags & FRODOKEM_FLAG_PUB_SET) == 0) {
+            ret = BAD_STATE_E;
+        }
+        else if (len != (word32)p->pkSize) {
+            ret = BUFFER_E;
+        }
+    }
+
+    if (ret == 0) {
+        XMEMCPY(out, key->seedA, FRODOKEM_SEEDA_SZ);
+        XMEMCPY(out + FRODOKEM_SEEDA_SZ, key->b, (size_t)(p->d * p->n));
+    }
+
+    return ret;
+}
+
+/* Decode a FrodoKEM public key: seedA || b.
+ *
+ * @param  [in, out]  key  FrodoKEM key object.
+ * @param  [in]       in   Encoded public key.
+ * @param  [in]       len  Length of encoded public key in bytes.
+ * @return  0 on success.
+ * @return  BAD_FUNC_ARG when key or in is NULL.
+ * @return  BUFFER_E when len is not the expected size.
+ * @return  NOT_COMPILED_IN when key type is not supported.
+ */
+int wc_FrodoKemKey_DecodePublicKey(FrodoKemKey* key, const unsigned char* in,
+    word32 len)
+{
+    int ret = 0;
+    const FrodoKemParams* p = NULL;
+
+    if ((key == NULL) || (in == NULL)) {
+        ret = BAD_FUNC_ARG;
+    }
+    else {
+        p = key->params;
+        if (p == NULL) {
+            ret = NOT_COMPILED_IN;
+        }
+        else if (len != (word32)p->pkSize) {
+            ret = BUFFER_E;
+        }
+    }
+
+    if (ret == 0) {
+        XMEMCPY(key->seedA, in, FRODOKEM_SEEDA_SZ);
+        XMEMCPY(key->b, in + FRODOKEM_SEEDA_SZ, (size_t)(p->d * p->n));
+        /* No value validation is required: seedA is an unconstrained seed and,
+         * because q is a power of two, every coefficient unpacked from b is
+         * already a valid residue in [0, q). No embedded hash to check. */
+        /* This object now holds only a public key: replace any prior state
+         * (e.g. from a private key) rather than OR-ing in, so the flags cannot
+         * describe a mismatched public/private pair. */
+        key->flags = FRODOKEM_FLAG_PUB_SET;
+    }
+
+    return ret;
+}
+
+/* Encode the FrodoKEM private key: s || seedA || b || S^T || pkh.
+ *
+ * The secret matrix S^T is stored as little-endian 16-bit residues.
+ *
+ * @param  [in]   key  FrodoKEM key object with private key set.
+ * @param  [out]  out  Output buffer.
+ * @param  [in]   len  Length of output buffer in bytes.
+ * @return  0 on success.
+ * @return  BAD_FUNC_ARG when key or out is NULL.
+ * @return  BAD_STATE_E when private key is not set.
+ * @return  BUFFER_E when len is not the expected size.
+ * @return  NOT_COMPILED_IN when key type is not supported.
+ */
+int wc_FrodoKemKey_EncodePrivateKey(FrodoKemKey* key, unsigned char* out,
+    word32 len)
+{
+    int ret = 0;
+    const FrodoKemParams* p = NULL;
+    int sCnt;
+    byte* o;
+
+    if ((key == NULL) || (out == NULL)) {
+        ret = BAD_FUNC_ARG;
+    }
+    else {
+        p = key->params;
+        if (p == NULL) {
+            ret = NOT_COMPILED_IN;
+        }
+        else if (((key->flags & FRODOKEM_FLAG_BOTH_SET) !=
+                FRODOKEM_FLAG_BOTH_SET) ||
+                ((key->flags & FRODOKEM_FLAG_PKH_SET) == 0)) {
+            ret = BAD_STATE_E;
+        }
+        else if (len != (word32)p->skSize) {
+            ret = BUFFER_E;
+        }
+    }
+
+    if (ret == 0) {
+        sCnt = FRODOKEM_NBAR * p->n;
+        o = out;
+        XMEMCPY(o, key->s, (size_t)p->lenSec);
+        o += p->lenSec;
+        XMEMCPY(o, key->seedA, FRODOKEM_SEEDA_SZ);
+        o += FRODOKEM_SEEDA_SZ;
+        XMEMCPY(o, key->b, (size_t)(p->d * p->n));
+        o += p->d * p->n;
+        frodokem_store_matrix(o, key->sMat, sCnt);
+        o += 2 * sCnt;
+        XMEMCPY(o, key->pkh, (size_t)p->lenSec);
+    }
+
+    return ret;
+}
+
+/* Validate a decoded FrodoKEM private key.
+ *
+ * The only decoded value with a condition on it is the stored public-key hash:
+ * it must equal SHAKE(seedA || b). (seedA, b and the secret matrix are
+ * otherwise unconstrained - q is a power of two, so every unpacked coefficient
+ * is already a valid residue in [0, q).)
+ *
+ * @param  [in]  p    FrodoKEM parameters.
+ * @param  [in]  key  Key with seedA, b and pkh populated.
+ * @return  0 when the stored hash matches.
+ * @return  WC_KEY_MISMATCH_E when the stored hash is inconsistent.
+ * @return  Negative on hash error.
+ */
+static int frodokem_check_priv_key(const FrodoKemParams* p, FrodoKemKey* key)
+{
+    byte pkh[FRODOKEM_MAX_LENSEC];
+    int ret;
+
+    ret = frodokem_shake_oneshot(p, &key->shake, key->seedA, (word32)p->pkSize,
+        pkh, (word32)p->lenSec);
+    if ((ret == 0) && (XMEMCMP(pkh, key->pkh, (size_t)p->lenSec) != 0)) {
+        ret = WC_KEY_MISMATCH_E;
+    }
+
+    return ret;
+}
+
+/* Decode a FrodoKEM private key: s || seedA || b || S^T || pkh.
+ *
+ * @param  [in, out]  key  FrodoKEM key object.
+ * @param  [in]       in   Encoded private key.
+ * @param  [in]       len  Length of encoded private key in bytes.
+ * @return  0 on success.
+ * @return  BAD_FUNC_ARG when key or in is NULL.
+ * @return  BUFFER_E when len is not the expected size.
+ * @return  NOT_COMPILED_IN when key type is not supported.
+ * @return  WC_KEY_MISMATCH_E when the embedded public-key hash is inconsistent.
+ */
+int wc_FrodoKemKey_DecodePrivateKey(FrodoKemKey* key, const unsigned char* in,
+    word32 len)
+{
+    const FrodoKemParams* p = NULL;
+    int ret = 0;
+    int sCnt;
+    const byte* o;
+
+    if ((key == NULL) || (in == NULL)) {
+        ret = BAD_FUNC_ARG;
+    }
+    else {
+        p = key->params;
+        if (p == NULL) {
+            ret = NOT_COMPILED_IN;
+        }
+        else if (len != (word32)p->skSize) {
+            ret = BUFFER_E;
+        }
+    }
+
+    if (ret == 0) {
+        sCnt = FRODOKEM_NBAR * p->n;
+        o = in;
+        XMEMCPY(key->s, o, (size_t)p->lenSec);
+        o += p->lenSec;
+        XMEMCPY(key->seedA, o, FRODOKEM_SEEDA_SZ);
+        o += FRODOKEM_SEEDA_SZ;
+        XMEMCPY(key->b, o, (size_t)(p->d * p->n));
+        o += p->d * p->n;
+        frodokem_load_matrix(key->sMat, o, sCnt);
+        o += 2 * sCnt;
+        XMEMCPY(key->pkh, o, (size_t)p->lenSec);
+
+        /* Validate the decoded data before marking the key usable. */
+        ret = frodokem_check_priv_key(p, key);
+        if (ret == 0) {
+            key->flags |= FRODOKEM_FLAG_PRIV_SET | FRODOKEM_FLAG_PUB_SET |
+                FRODOKEM_FLAG_PKH_SET;
+        }
+        else {
+            /* Leave the key object unusable and wipe the secret material that
+             * was copied in before validation failed. */
+            ForceZero(key->s, sizeof(key->s));
+            ForceZero(key->sMat, sizeof(key->sMat));
+            key->flags = 0;
+        }
+    }
+
+    return ret;
+}
+
+#endif /* WOLFSSL_HAVE_FRODOKEM */
diff --git a/wolfcrypt/src/wc_frodokem_asm.S b/wolfcrypt/src/wc_frodokem_asm.S
new file mode 100644
index 0000000000..95c4ddbe5e
--- /dev/null
+++ b/wolfcrypt/src/wc_frodokem_asm.S
@@ -0,0 +1,1590 @@
+/* wc_frodokem_asm.S */
+/*
+ * Copyright (C) 2006-2026 wolfSSL Inc.
+ *
+ * This file is part of wolfSSL.
+ *
+ * wolfSSL is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * wolfSSL is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335, USA
+ */
+
+#ifdef WOLFSSL_USER_SETTINGS
+#ifdef WOLFSSL_USER_SETTINGS_ASM
+/*
+ * user_settings_asm.h is a file generated by the script user_settings_asm.sh.
+ * The script takes in a user_settings.h and produces user_settings_asm.h, which
+ * is a stripped down version of user_settings.h containing only preprocessor
+ * directives. This makes the header safe to include in assembly (.S) files.
+ */
+#include "user_settings_asm.h"
+#else
+/*
+ * Note: if user_settings.h contains any C code (e.g. a typedef or function
+ * prototype), including it here in an assembly (.S) file will cause an
+ * assembler failure. See user_settings_asm.h above.
+ */
+#include "user_settings.h"
+#endif /* WOLFSSL_USER_SETTINGS_ASM */
+#endif /* WOLFSSL_USER_SETTINGS */
+
+#ifndef HAVE_INTEL_AVX1
+#define HAVE_INTEL_AVX1
+#endif /* HAVE_INTEL_AVX1 */
+#ifndef NO_AVX2_SUPPORT
+#ifndef HAVE_INTEL_AVX2
+#define HAVE_INTEL_AVX2
+#endif /* HAVE_INTEL_AVX2 */
+#endif /* NO_AVX2_SUPPORT */
+#ifndef NO_VAES_SUPPORT
+#ifndef HAVE_INTEL_VAES
+#define HAVE_INTEL_VAES
+#endif /* HAVE_INTEL_VAES */
+#endif /* NO_VAES_SUPPORT */
+#ifndef NO_AVX512_SUPPORT
+#ifndef HAVE_INTEL_AVX512
+#define HAVE_INTEL_AVX512
+#endif /* HAVE_INTEL_AVX512 */
+#endif /* NO_AVX512_SUPPORT */
+
+#ifdef WOLFSSL_HAVE_FRODOKEM
+#ifdef HAVE_INTEL_AVX2
+#ifndef __APPLE__
+.text
+.globl	frodokem_sa_accum_avx2
+.type	frodokem_sa_accum_avx2,@function
+.align	16
+frodokem_sa_accum_avx2:
+#else
+.section	__TEXT,__text
+.globl	_frodokem_sa_accum_avx2
+.p2align	4
+_frodokem_sa_accum_avx2:
+#endif /* __APPLE__ */
+        pushq	%r12
+        pushq	%r13
+        movslq	%ecx, %rcx
+        shlq	$1, %rcx
+        addq	%rcx, %rsi
+        movslq	%r8d, %rax
+        shlq	$1, %rax
+        movq	%rdx, %r9
+        addq	%rax, %r9
+        movq	%r9, %r10
+        addq	%rax, %r10
+        movq	%r10, %r11
+        addq	%rax, %r11
+        movq	$8, %r13
+L_frodokem_sa_accum_avx2_i:
+        vpbroadcastw	(%rsi), %ymm0
+        vpbroadcastw	2(%rsi), %ymm1
+        vpbroadcastw	4(%rsi), %ymm2
+        vpbroadcastw	6(%rsi), %ymm3
+        xorq	%r12, %r12
+L_frodokem_sa_accum_avx2_k:
+        vmovdqu	(%rdi,%r12,1), %ymm4
+        vpmullw	(%rdx,%r12,1), %ymm0, %ymm5
+        vpaddw	%ymm5, %ymm4, %ymm4
+        vpmullw	(%r9,%r12,1), %ymm1, %ymm5
+        vpaddw	%ymm5, %ymm4, %ymm4
+        vpmullw	(%r10,%r12,1), %ymm2, %ymm5
+        vpaddw	%ymm5, %ymm4, %ymm4
+        vpmullw	(%r11,%r12,1), %ymm3, %ymm5
+        vpaddw	%ymm5, %ymm4, %ymm4
+        vmovdqu	%ymm4, (%rdi,%r12,1)
+        addq	$32, %r12
+        cmpq	%rax, %r12
+        jl	L_frodokem_sa_accum_avx2_k
+        addq	%rax, %rdi
+        addq	%rax, %rsi
+        subq	$0x01, %r13
+        jnz	L_frodokem_sa_accum_avx2_i
+        vzeroupper
+        popq	%r13
+        popq	%r12
+        repz retq
+#ifndef __APPLE__
+.size	frodokem_sa_accum_avx2,.-frodokem_sa_accum_avx2
+#endif /* __APPLE__ */
+#ifndef __APPLE__
+.text
+.globl	frodokem_as_accum_avx2
+.type	frodokem_as_accum_avx2,@function
+.align	16
+frodokem_as_accum_avx2:
+#else
+.section	__TEXT,__text
+.globl	_frodokem_as_accum_avx2
+.p2align	4
+_frodokem_as_accum_avx2:
+#endif /* __APPLE__ */
+        pushq	%r12
+        pushq	%r13
+        pushq	%r14
+        pushq	%r15
+        pushq	%rbx
+        pushq	%rbp
+        movslq	%ecx, %rcx
+        shlq	$4, %rcx
+        addq	%rcx, %rdi
+        movslq	%r8d, %rax
+        shlq	$1, %rax
+        movq	%rsi, %r9
+        addq	%rax, %r9
+        movq	%r9, %r10
+        addq	%rax, %r10
+        movq	%r10, %r11
+        addq	%rax, %r11
+        movq	%r11, %r12
+        addq	%rax, %r12
+        movq	%r12, %r13
+        addq	%rax, %r13
+        movq	%r13, %r14
+        addq	%rax, %r14
+        movq	%r14, %r15
+        addq	%rax, %r15
+        movq	$4, %rbp
+        vpcmpeqd	%xmm12, %xmm12, %xmm12
+        vpsrld	$16, %xmm12, %xmm12
+L_frodokem_as_accum_avx2_r:
+        vpxor	%ymm0, %ymm0, %ymm0
+        vpxor	%ymm1, %ymm1, %ymm1
+        vpxor	%ymm2, %ymm2, %ymm2
+        vpxor	%ymm3, %ymm3, %ymm3
+        vpxor	%ymm4, %ymm4, %ymm4
+        vpxor	%ymm5, %ymm5, %ymm5
+        vpxor	%ymm6, %ymm6, %ymm6
+        vpxor	%ymm7, %ymm7, %ymm7
+        xorq	%rbx, %rbx
+L_frodokem_as_accum_avx2_j:
+        vmovdqu	(%rdx,%rbx,1), %ymm8
+        vpmaddwd	(%rsi,%rbx,1), %ymm8, %ymm10
+        vpaddd	%ymm10, %ymm0, %ymm0
+        vpmaddwd	(%r9,%rbx,1), %ymm8, %ymm10
+        vpaddd	%ymm10, %ymm1, %ymm1
+        vpmaddwd	(%r10,%rbx,1), %ymm8, %ymm10
+        vpaddd	%ymm10, %ymm2, %ymm2
+        vpmaddwd	(%r11,%rbx,1), %ymm8, %ymm10
+        vpaddd	%ymm10, %ymm3, %ymm3
+        vpmaddwd	(%r12,%rbx,1), %ymm8, %ymm10
+        vpaddd	%ymm10, %ymm4, %ymm4
+        vpmaddwd	(%r13,%rbx,1), %ymm8, %ymm10
+        vpaddd	%ymm10, %ymm5, %ymm5
+        vpmaddwd	(%r14,%rbx,1), %ymm8, %ymm10
+        vpaddd	%ymm10, %ymm6, %ymm6
+        vpmaddwd	(%r15,%rbx,1), %ymm8, %ymm10
+        vpaddd	%ymm10, %ymm7, %ymm7
+        addq	$32, %rbx
+        cmpq	%rax, %rbx
+        jl	L_frodokem_as_accum_avx2_j
+        vextracti128	$0x01, %ymm0, %xmm8
+        vpaddd	%xmm8, %xmm0, %xmm0
+        vextracti128	$0x01, %ymm1, %xmm8
+        vpaddd	%xmm8, %xmm1, %xmm1
+        vextracti128	$0x01, %ymm2, %xmm8
+        vpaddd	%xmm8, %xmm2, %xmm2
+        vextracti128	$0x01, %ymm3, %xmm8
+        vpaddd	%xmm8, %xmm3, %xmm3
+        vextracti128	$0x01, %ymm4, %xmm8
+        vpaddd	%xmm8, %xmm4, %xmm4
+        vextracti128	$0x01, %ymm5, %xmm8
+        vpaddd	%xmm8, %xmm5, %xmm5
+        vextracti128	$0x01, %ymm6, %xmm8
+        vpaddd	%xmm8, %xmm6, %xmm6
+        vextracti128	$0x01, %ymm7, %xmm8
+        vpaddd	%xmm8, %xmm7, %xmm7
+        vphaddd	%xmm1, %xmm0, %xmm10
+        vphaddd	%xmm3, %xmm2, %xmm11
+        vphaddd	%xmm11, %xmm10, %xmm9
+        vphaddd	%xmm5, %xmm4, %xmm10
+        vphaddd	%xmm7, %xmm6, %xmm11
+        vphaddd	%xmm11, %xmm10, %xmm10
+        vpand	%xmm12, %xmm9, %xmm9
+        vpand	%xmm12, %xmm10, %xmm10
+        vpackusdw	%xmm10, %xmm9, %xmm9
+        vpaddw	(%rdi), %xmm9, %xmm9
+        vmovdqu	%xmm9, (%rdi)
+        addq	%rax, %rdx
+        addq	$16, %rdi
+        subq	$0x01, %rbp
+        jnz	L_frodokem_as_accum_avx2_r
+        vzeroupper
+        popq	%rbp
+        popq	%rbx
+        popq	%r15
+        popq	%r14
+        popq	%r13
+        popq	%r12
+        repz retq
+#ifndef __APPLE__
+.size	frodokem_as_accum_avx2,.-frodokem_as_accum_avx2
+#endif /* __APPLE__ */
+#ifndef __APPLE__
+.text
+.globl	frodokem_mul_bs_avx2
+.type	frodokem_mul_bs_avx2,@function
+.align	16
+frodokem_mul_bs_avx2:
+#else
+.section	__TEXT,__text
+.globl	_frodokem_mul_bs_avx2
+.p2align	4
+_frodokem_mul_bs_avx2:
+#endif /* __APPLE__ */
+        pushq	%r12
+        pushq	%r13
+        pushq	%r14
+        pushq	%r15
+        pushq	%rbx
+        pushq	%rbp
+        movslq	%ecx, %rax
+        shlq	$1, %rax
+        movq	%rdx, %r9
+        addq	%rax, %r9
+        movq	%r9, %r10
+        addq	%rax, %r10
+        movq	%r10, %r11
+        addq	%rax, %r11
+        movq	%r11, %r12
+        addq	%rax, %r12
+        movq	%r12, %r13
+        addq	%rax, %r13
+        movq	%r13, %r14
+        addq	%rax, %r14
+        movq	%r14, %r15
+        addq	%rax, %r15
+        movq	$8, %rbp
+        vmovd	%r8, %xmm12
+        vpbroadcastd	%xmm12, %xmm12
+L_frodokem_mul_bs_avx2_r:
+        vpxor	%ymm0, %ymm0, %ymm0
+        vpxor	%ymm1, %ymm1, %ymm1
+        vpxor	%ymm2, %ymm2, %ymm2
+        vpxor	%ymm3, %ymm3, %ymm3
+        vpxor	%ymm4, %ymm4, %ymm4
+        vpxor	%ymm5, %ymm5, %ymm5
+        vpxor	%ymm6, %ymm6, %ymm6
+        vpxor	%ymm7, %ymm7, %ymm7
+        xorq	%rbx, %rbx
+L_frodokem_mul_bs_avx2_j:
+        vmovdqu	(%rsi,%rbx,1), %ymm8
+        vpmaddwd	(%rdx,%rbx,1), %ymm8, %ymm10
+        vpaddd	%ymm10, %ymm0, %ymm0
+        vpmaddwd	(%r9,%rbx,1), %ymm8, %ymm10
+        vpaddd	%ymm10, %ymm1, %ymm1
+        vpmaddwd	(%r10,%rbx,1), %ymm8, %ymm10
+        vpaddd	%ymm10, %ymm2, %ymm2
+        vpmaddwd	(%r11,%rbx,1), %ymm8, %ymm10
+        vpaddd	%ymm10, %ymm3, %ymm3
+        vpmaddwd	(%r12,%rbx,1), %ymm8, %ymm10
+        vpaddd	%ymm10, %ymm4, %ymm4
+        vpmaddwd	(%r13,%rbx,1), %ymm8, %ymm10
+        vpaddd	%ymm10, %ymm5, %ymm5
+        vpmaddwd	(%r14,%rbx,1), %ymm8, %ymm10
+        vpaddd	%ymm10, %ymm6, %ymm6
+        vpmaddwd	(%r15,%rbx,1), %ymm8, %ymm10
+        vpaddd	%ymm10, %ymm7, %ymm7
+        addq	$32, %rbx
+        cmpq	%rax, %rbx
+        jl	L_frodokem_mul_bs_avx2_j
+        vextracti128	$0x01, %ymm0, %xmm8
+        vpaddd	%xmm8, %xmm0, %xmm0
+        vextracti128	$0x01, %ymm1, %xmm8
+        vpaddd	%xmm8, %xmm1, %xmm1
+        vextracti128	$0x01, %ymm2, %xmm8
+        vpaddd	%xmm8, %xmm2, %xmm2
+        vextracti128	$0x01, %ymm3, %xmm8
+        vpaddd	%xmm8, %xmm3, %xmm3
+        vextracti128	$0x01, %ymm4, %xmm8
+        vpaddd	%xmm8, %xmm4, %xmm4
+        vextracti128	$0x01, %ymm5, %xmm8
+        vpaddd	%xmm8, %xmm5, %xmm5
+        vextracti128	$0x01, %ymm6, %xmm8
+        vpaddd	%xmm8, %xmm6, %xmm6
+        vextracti128	$0x01, %ymm7, %xmm8
+        vpaddd	%xmm8, %xmm7, %xmm7
+        vphaddd	%xmm1, %xmm0, %xmm10
+        vphaddd	%xmm3, %xmm2, %xmm11
+        vphaddd	%xmm11, %xmm10, %xmm9
+        vphaddd	%xmm5, %xmm4, %xmm10
+        vphaddd	%xmm7, %xmm6, %xmm11
+        vphaddd	%xmm11, %xmm10, %xmm10
+        vpand	%xmm12, %xmm9, %xmm9
+        vpand	%xmm12, %xmm10, %xmm10
+        vpackusdw	%xmm10, %xmm9, %xmm9
+        vmovdqu	%xmm9, (%rdi)
+        addq	%rax, %rsi
+        addq	$16, %rdi
+        subq	$0x01, %rbp
+        jnz	L_frodokem_mul_bs_avx2_r
+        vzeroupper
+        popq	%rbp
+        popq	%rbx
+        popq	%r15
+        popq	%r14
+        popq	%r13
+        popq	%r12
+        repz retq
+#ifndef __APPLE__
+.size	frodokem_mul_bs_avx2,.-frodokem_mul_bs_avx2
+#endif /* __APPLE__ */
+#ifndef __APPLE__
+.text
+.globl	frodokem_mul_add_sb_plus_e_avx2
+.type	frodokem_mul_add_sb_plus_e_avx2,@function
+.align	16
+frodokem_mul_add_sb_plus_e_avx2:
+#else
+.section	__TEXT,__text
+.globl	_frodokem_mul_add_sb_plus_e_avx2
+.p2align	4
+_frodokem_mul_add_sb_plus_e_avx2:
+#endif /* __APPLE__ */
+        pushq	%r12
+        pushq	%r13
+        pushq	%r14
+        pushq	%r15
+        pushq	%rbx
+        movslq	%ecx, %rax
+        shlq	$1, %rax
+        movq	%rdx, %r9
+        addq	%rax, %r9
+        movq	%r9, %r10
+        addq	%rax, %r10
+        movq	%r10, %r11
+        addq	%rax, %r11
+        movq	%r11, %r12
+        addq	%rax, %r12
+        movq	%r12, %r13
+        addq	%rax, %r13
+        movq	%r13, %r14
+        addq	%rax, %r14
+        movq	%r14, %r15
+        addq	%rax, %r15
+        vmovd	%r8, %xmm7
+        vpbroadcastw	%xmm7, %ymm7
+        vmovdqu	(%rdi), %ymm0
+        vmovdqu	32(%rdi), %ymm1
+        vmovdqu	64(%rdi), %ymm2
+        vmovdqu	96(%rdi), %ymm3
+        xorq	%rbx, %rbx
+L_frodokem_mul_add_sb_plus_e_avx2_j:
+        vbroadcasti128	(%rsi), %ymm4
+        vpbroadcastw	(%rdx,%rbx,1), %ymm5
+        vpbroadcastw	(%r9,%rbx,1), %ymm6
+        vinserti128	$0x01, %xmm6, %ymm5, %ymm5
+        vpmullw	%ymm4, %ymm5, %ymm6
+        vpaddw	%ymm6, %ymm0, %ymm0
+        vpbroadcastw	(%r10,%rbx,1), %ymm5
+        vpbroadcastw	(%r11,%rbx,1), %ymm6
+        vinserti128	$0x01, %xmm6, %ymm5, %ymm5
+        vpmullw	%ymm4, %ymm5, %ymm6
+        vpaddw	%ymm6, %ymm1, %ymm1
+        vpbroadcastw	(%r12,%rbx,1), %ymm5
+        vpbroadcastw	(%r13,%rbx,1), %ymm6
+        vinserti128	$0x01, %xmm6, %ymm5, %ymm5
+        vpmullw	%ymm4, %ymm5, %ymm6
+        vpaddw	%ymm6, %ymm2, %ymm2
+        vpbroadcastw	(%r14,%rbx,1), %ymm5
+        vpbroadcastw	(%r15,%rbx,1), %ymm6
+        vinserti128	$0x01, %xmm6, %ymm5, %ymm5
+        vpmullw	%ymm4, %ymm5, %ymm6
+        vpaddw	%ymm6, %ymm3, %ymm3
+        addq	$16, %rsi
+        addq	$2, %rbx
+        cmpq	%rax, %rbx
+        jl	L_frodokem_mul_add_sb_plus_e_avx2_j
+        vpand	%ymm7, %ymm0, %ymm0
+        vmovdqu	%ymm0, (%rdi)
+        vpand	%ymm7, %ymm1, %ymm1
+        vmovdqu	%ymm1, 32(%rdi)
+        vpand	%ymm7, %ymm2, %ymm2
+        vmovdqu	%ymm2, 64(%rdi)
+        vpand	%ymm7, %ymm3, %ymm3
+        vmovdqu	%ymm3, 96(%rdi)
+        vzeroupper
+        popq	%rbx
+        popq	%r15
+        popq	%r14
+        popq	%r13
+        popq	%r12
+        repz retq
+#ifndef __APPLE__
+.size	frodokem_mul_add_sb_plus_e_avx2,.-frodokem_mul_add_sb_plus_e_avx2
+#endif /* __APPLE__ */
+#ifndef __APPLE__
+.text
+.globl	frodokem_add_avx2
+.type	frodokem_add_avx2,@function
+.align	16
+frodokem_add_avx2:
+#else
+.section	__TEXT,__text
+.globl	_frodokem_add_avx2
+.p2align	4
+_frodokem_add_avx2:
+#endif /* __APPLE__ */
+        vmovd	%rdx, %xmm0
+        vpbroadcastw	%xmm0, %ymm0
+        vmovdqu	(%rdi), %ymm1
+        vpaddw	(%rsi), %ymm1, %ymm1
+        vpand	%ymm0, %ymm1, %ymm1
+        vmovdqu	%ymm1, (%rdi)
+        vmovdqu	32(%rdi), %ymm2
+        vpaddw	32(%rsi), %ymm2, %ymm2
+        vpand	%ymm0, %ymm2, %ymm2
+        vmovdqu	%ymm2, 32(%rdi)
+        vmovdqu	64(%rdi), %ymm3
+        vpaddw	64(%rsi), %ymm3, %ymm3
+        vpand	%ymm0, %ymm3, %ymm3
+        vmovdqu	%ymm3, 64(%rdi)
+        vmovdqu	96(%rdi), %ymm4
+        vpaddw	96(%rsi), %ymm4, %ymm4
+        vpand	%ymm0, %ymm4, %ymm4
+        vmovdqu	%ymm4, 96(%rdi)
+        vzeroupper
+        repz retq
+#ifndef __APPLE__
+.size	frodokem_add_avx2,.-frodokem_add_avx2
+#endif /* __APPLE__ */
+#ifndef __APPLE__
+.text
+.globl	frodokem_a_rows_reduce_avx2
+.type	frodokem_a_rows_reduce_avx2,@function
+.align	16
+frodokem_a_rows_reduce_avx2:
+#else
+.section	__TEXT,__text
+.globl	_frodokem_a_rows_reduce_avx2
+.p2align	4
+_frodokem_a_rows_reduce_avx2:
+#endif /* __APPLE__ */
+        vmovd	%rdx, %xmm0
+        vpbroadcastw	%xmm0, %ymm0
+        movl	%esi, %eax
+L_frodokem_a_rows_reduce_avx2_blk:
+        cmpq	$16, %rax
+        jl	L_frodokem_a_rows_reduce_avx2_tail
+        vpand	(%rdi), %ymm0, %ymm1
+        vmovdqu	%ymm1, (%rdi)
+        addq	$32, %rdi
+        subq	$16, %rax
+        jmp	L_frodokem_a_rows_reduce_avx2_blk
+L_frodokem_a_rows_reduce_avx2_tail:
+        cmpq	$0x00, %rax
+        jle	L_frodokem_a_rows_reduce_avx2_done
+L_frodokem_a_rows_reduce_avx2_word:
+        movzwl	(%rdi), %ecx
+        andl	%edx, %ecx
+        movw	%cx, (%rdi)
+        addq	$2, %rdi
+        subq	$0x01, %rax
+        jg	L_frodokem_a_rows_reduce_avx2_word
+L_frodokem_a_rows_reduce_avx2_done:
+        vzeroupper
+        repz retq
+#ifndef __APPLE__
+.size	frodokem_a_rows_reduce_avx2,.-frodokem_a_rows_reduce_avx2
+#endif /* __APPLE__ */
+#ifndef __APPLE__
+.text
+.globl	frodokem_sample_avx2
+.type	frodokem_sample_avx2,@function
+.align	16
+frodokem_sample_avx2:
+#else
+.section	__TEXT,__text
+.globl	_frodokem_sample_avx2
+.p2align	4
+_frodokem_sample_avx2:
+#endif /* __APPLE__ */
+        movslq	%esi, %rax
+        shlq	$1, %rax
+        vpcmpeqw	%ymm0, %ymm0, %ymm0
+        vpsrlw	$15, %ymm0, %ymm0
+        vpxor	%ymm1, %ymm1, %ymm1
+        xorq	%r8, %r8
+L_frodokem_sample_avx2_blk:
+        vmovdqu	(%rdi,%r8,1), %ymm2
+        vpsrlw	$0x01, %ymm2, %ymm3
+        vpand	%ymm0, %ymm2, %ymm4
+        vpxor	%ymm5, %ymm5, %ymm5
+        movq	%rdx, %r9
+        movl	%ecx, %r10d
+L_frodokem_sample_avx2_cdf:
+        vpbroadcastw	(%r9), %ymm6
+        vpsubw	%ymm3, %ymm6, %ymm6
+        vpsrlw	$15, %ymm6, %ymm6
+        vpaddw	%ymm6, %ymm5, %ymm5
+        addq	$2, %r9
+        subq	$0x01, %r10
+        jnz	L_frodokem_sample_avx2_cdf
+        vpsubw	%ymm4, %ymm1, %ymm7
+        vpxor	%ymm7, %ymm5, %ymm5
+        vpaddw	%ymm4, %ymm5, %ymm5
+        vmovdqu	%ymm5, (%rdi,%r8,1)
+        addq	$32, %r8
+        cmpq	%rax, %r8
+        jl	L_frodokem_sample_avx2_blk
+        vzeroupper
+        repz retq
+#ifndef __APPLE__
+.size	frodokem_sample_avx2,.-frodokem_sample_avx2
+#endif /* __APPLE__ */
+#ifndef __APPLE__
+.text
+.globl	frodokem_gen_a_rows_aes_avx2
+.type	frodokem_gen_a_rows_aes_avx2,@function
+.align	16
+frodokem_gen_a_rows_aes_avx2:
+#else
+.section	__TEXT,__text
+.globl	_frodokem_gen_a_rows_aes_avx2
+.p2align	4
+_frodokem_gen_a_rows_aes_avx2:
+#endif /* __APPLE__ */
+        pushq	%r12
+        pushq	%r13
+        pushq	%r14
+        pushq	%r15
+        movq	40(%rsp), %rax
+        vbroadcasti128	(%rdx), %ymm0
+        vbroadcasti128	16(%rdx), %ymm1
+        vbroadcasti128	32(%rdx), %ymm2
+        vbroadcasti128	48(%rdx), %ymm3
+        vbroadcasti128	64(%rdx), %ymm4
+        vbroadcasti128	80(%rdx), %ymm5
+        vbroadcasti128	96(%rdx), %ymm6
+        vbroadcasti128	112(%rdx), %ymm7
+        vbroadcasti128	128(%rdx), %ymm8
+        vbroadcasti128	144(%rdx), %ymm9
+        vbroadcasti128	160(%rdx), %ymm10
+        vmovd	%rax, %xmm15
+        vpbroadcastw	%xmm15, %ymm15
+        movslq	%r9d, %r10
+        shrq	$3, %r10
+        movslq	%r8d, %r11
+        movslq	%ecx, %rcx
+        xorq	%r12, %r12
+L_frodokem_gen_a_rows_aes_avx2_row:
+        movq	%rcx, %r14
+        addq	%r12, %r14
+        xorq	%r13, %r13
+L_frodokem_gen_a_rows_aes_avx2_blk:
+        leaq	8(%r13), %r15
+        cmpq	%r10, %r15
+        jg	L_frodokem_gen_a_rows_aes_avx2_tail
+        vmovd	%r14d, %xmm11
+        addq	$0x80000, %r14
+        vmovd	%r14d, %xmm0
+        addq	$0x80000, %r14
+        vinserti128	$0x01, %xmm0, %ymm11, %ymm11
+        vmovd	%r14d, %xmm12
+        addq	$0x80000, %r14
+        vmovd	%r14d, %xmm0
+        addq	$0x80000, %r14
+        vinserti128	$0x01, %xmm0, %ymm12, %ymm12
+        vmovd	%r14d, %xmm13
+        addq	$0x80000, %r14
+        vmovd	%r14d, %xmm0
+        addq	$0x80000, %r14
+        vinserti128	$0x01, %xmm0, %ymm13, %ymm13
+        vmovd	%r14d, %xmm14
+        addq	$0x80000, %r14
+        vmovd	%r14d, %xmm0
+        addq	$0x80000, %r14
+        vinserti128	$0x01, %xmm0, %ymm14, %ymm14
+        vbroadcasti128	(%rdx), %ymm0
+        vpxor	%ymm0, %ymm11, %ymm11
+        vpxor	%ymm0, %ymm12, %ymm12
+        vpxor	%ymm0, %ymm13, %ymm13
+        vpxor	%ymm0, %ymm14, %ymm14
+        vaesenc	%ymm1, %ymm11, %ymm11
+        vaesenc	%ymm1, %ymm12, %ymm12
+        vaesenc	%ymm1, %ymm13, %ymm13
+        vaesenc	%ymm1, %ymm14, %ymm14
+        vaesenc	%ymm2, %ymm11, %ymm11
+        vaesenc	%ymm2, %ymm12, %ymm12
+        vaesenc	%ymm2, %ymm13, %ymm13
+        vaesenc	%ymm2, %ymm14, %ymm14
+        vaesenc	%ymm3, %ymm11, %ymm11
+        vaesenc	%ymm3, %ymm12, %ymm12
+        vaesenc	%ymm3, %ymm13, %ymm13
+        vaesenc	%ymm3, %ymm14, %ymm14
+        vaesenc	%ymm4, %ymm11, %ymm11
+        vaesenc	%ymm4, %ymm12, %ymm12
+        vaesenc	%ymm4, %ymm13, %ymm13
+        vaesenc	%ymm4, %ymm14, %ymm14
+        vaesenc	%ymm5, %ymm11, %ymm11
+        vaesenc	%ymm5, %ymm12, %ymm12
+        vaesenc	%ymm5, %ymm13, %ymm13
+        vaesenc	%ymm5, %ymm14, %ymm14
+        vaesenc	%ymm6, %ymm11, %ymm11
+        vaesenc	%ymm6, %ymm12, %ymm12
+        vaesenc	%ymm6, %ymm13, %ymm13
+        vaesenc	%ymm6, %ymm14, %ymm14
+        vaesenc	%ymm7, %ymm11, %ymm11
+        vaesenc	%ymm7, %ymm12, %ymm12
+        vaesenc	%ymm7, %ymm13, %ymm13
+        vaesenc	%ymm7, %ymm14, %ymm14
+        vaesenc	%ymm8, %ymm11, %ymm11
+        vaesenc	%ymm8, %ymm12, %ymm12
+        vaesenc	%ymm8, %ymm13, %ymm13
+        vaesenc	%ymm8, %ymm14, %ymm14
+        vaesenc	%ymm9, %ymm11, %ymm11
+        vaesenc	%ymm9, %ymm12, %ymm12
+        vaesenc	%ymm9, %ymm13, %ymm13
+        vaesenc	%ymm9, %ymm14, %ymm14
+        vaesenclast	%ymm10, %ymm11, %ymm11
+        vaesenclast	%ymm10, %ymm12, %ymm12
+        vaesenclast	%ymm10, %ymm13, %ymm13
+        vaesenclast	%ymm10, %ymm14, %ymm14
+        vpand	%ymm15, %ymm11, %ymm11
+        vmovdqu	%ymm11, (%rsi)
+        vpand	%ymm15, %ymm12, %ymm12
+        vmovdqu	%ymm12, 32(%rsi)
+        vpand	%ymm15, %ymm13, %ymm13
+        vmovdqu	%ymm13, 64(%rsi)
+        vpand	%ymm15, %ymm14, %ymm14
+        vmovdqu	%ymm14, 96(%rsi)
+        addq	$0x80, %rsi
+        addq	$8, %r13
+        jmp	L_frodokem_gen_a_rows_aes_avx2_blk
+L_frodokem_gen_a_rows_aes_avx2_tail:
+        cmpq	%r10, %r13
+        jge	L_frodokem_gen_a_rows_aes_avx2_next
+        vmovd	%r14d, %xmm11
+        addq	$0x80000, %r14
+        vpxor	%xmm0, %xmm11, %xmm11
+        vaesenc	%xmm1, %xmm11, %xmm11
+        vaesenc	%xmm2, %xmm11, %xmm11
+        vaesenc	%xmm3, %xmm11, %xmm11
+        vaesenc	%xmm4, %xmm11, %xmm11
+        vaesenc	%xmm5, %xmm11, %xmm11
+        vaesenc	%xmm6, %xmm11, %xmm11
+        vaesenc	%xmm7, %xmm11, %xmm11
+        vaesenc	%xmm8, %xmm11, %xmm11
+        vaesenc	%xmm9, %xmm11, %xmm11
+        vaesenclast	%xmm10, %xmm11, %xmm11
+        vpand	%xmm15, %xmm11, %xmm11
+        vmovdqu	%xmm11, (%rsi)
+        addq	$16, %rsi
+        addq	$0x01, %r13
+        jmp	L_frodokem_gen_a_rows_aes_avx2_tail
+L_frodokem_gen_a_rows_aes_avx2_next:
+        addq	$0x01, %r12
+        cmpq	%r11, %r12
+        jl	L_frodokem_gen_a_rows_aes_avx2_row
+        vzeroupper
+        popq	%r15
+        popq	%r14
+        popq	%r13
+        popq	%r12
+        repz retq
+#ifndef __APPLE__
+.size	frodokem_gen_a_rows_aes_avx2,.-frodokem_gen_a_rows_aes_avx2
+#endif /* __APPLE__ */
+#ifndef __APPLE__
+.text
+.globl	frodokem_gen_a_rows_aes_aesni
+.type	frodokem_gen_a_rows_aes_aesni,@function
+.align	16
+frodokem_gen_a_rows_aes_aesni:
+#else
+.section	__TEXT,__text
+.globl	_frodokem_gen_a_rows_aes_aesni
+.p2align	4
+_frodokem_gen_a_rows_aes_aesni:
+#endif /* __APPLE__ */
+        pushq	%r12
+        pushq	%r13
+        pushq	%r14
+        pushq	%r15
+        movq	40(%rsp), %rax
+        vmovd	%rax, %xmm8
+        vpbroadcastw	%xmm8, %xmm8
+        movslq	%r9d, %r10
+        shrq	$3, %r10
+        movslq	%r8d, %r11
+        movslq	%ecx, %rcx
+        xorq	%r12, %r12
+L_frodokem_gen_a_rows_aes_aesni_row:
+        movq	%rcx, %r14
+        addq	%r12, %r14
+        xorq	%r13, %r13
+L_frodokem_gen_a_rows_aes_aesni_blk:
+        leaq	8(%r13), %r15
+        cmpq	%r10, %r15
+        jg	L_frodokem_gen_a_rows_aes_aesni_tail
+        vmovd	%r14d, %xmm0
+        addq	$0x80000, %r14
+        vmovd	%r14d, %xmm1
+        addq	$0x80000, %r14
+        vmovd	%r14d, %xmm2
+        addq	$0x80000, %r14
+        vmovd	%r14d, %xmm3
+        addq	$0x80000, %r14
+        vmovd	%r14d, %xmm4
+        addq	$0x80000, %r14
+        vmovd	%r14d, %xmm5
+        addq	$0x80000, %r14
+        vmovd	%r14d, %xmm6
+        addq	$0x80000, %r14
+        vmovd	%r14d, %xmm7
+        addq	$0x80000, %r14
+        vpxor	(%rdx), %xmm0, %xmm0
+        vpxor	(%rdx), %xmm1, %xmm1
+        vpxor	(%rdx), %xmm2, %xmm2
+        vpxor	(%rdx), %xmm3, %xmm3
+        vpxor	(%rdx), %xmm4, %xmm4
+        vpxor	(%rdx), %xmm5, %xmm5
+        vpxor	(%rdx), %xmm6, %xmm6
+        vpxor	(%rdx), %xmm7, %xmm7
+        vaesenc	16(%rdx), %xmm0, %xmm0
+        vaesenc	16(%rdx), %xmm1, %xmm1
+        vaesenc	16(%rdx), %xmm2, %xmm2
+        vaesenc	16(%rdx), %xmm3, %xmm3
+        vaesenc	16(%rdx), %xmm4, %xmm4
+        vaesenc	16(%rdx), %xmm5, %xmm5
+        vaesenc	16(%rdx), %xmm6, %xmm6
+        vaesenc	16(%rdx), %xmm7, %xmm7
+        vaesenc	32(%rdx), %xmm0, %xmm0
+        vaesenc	32(%rdx), %xmm1, %xmm1
+        vaesenc	32(%rdx), %xmm2, %xmm2
+        vaesenc	32(%rdx), %xmm3, %xmm3
+        vaesenc	32(%rdx), %xmm4, %xmm4
+        vaesenc	32(%rdx), %xmm5, %xmm5
+        vaesenc	32(%rdx), %xmm6, %xmm6
+        vaesenc	32(%rdx), %xmm7, %xmm7
+        vaesenc	48(%rdx), %xmm0, %xmm0
+        vaesenc	48(%rdx), %xmm1, %xmm1
+        vaesenc	48(%rdx), %xmm2, %xmm2
+        vaesenc	48(%rdx), %xmm3, %xmm3
+        vaesenc	48(%rdx), %xmm4, %xmm4
+        vaesenc	48(%rdx), %xmm5, %xmm5
+        vaesenc	48(%rdx), %xmm6, %xmm6
+        vaesenc	48(%rdx), %xmm7, %xmm7
+        vaesenc	64(%rdx), %xmm0, %xmm0
+        vaesenc	64(%rdx), %xmm1, %xmm1
+        vaesenc	64(%rdx), %xmm2, %xmm2
+        vaesenc	64(%rdx), %xmm3, %xmm3
+        vaesenc	64(%rdx), %xmm4, %xmm4
+        vaesenc	64(%rdx), %xmm5, %xmm5
+        vaesenc	64(%rdx), %xmm6, %xmm6
+        vaesenc	64(%rdx), %xmm7, %xmm7
+        vaesenc	80(%rdx), %xmm0, %xmm0
+        vaesenc	80(%rdx), %xmm1, %xmm1
+        vaesenc	80(%rdx), %xmm2, %xmm2
+        vaesenc	80(%rdx), %xmm3, %xmm3
+        vaesenc	80(%rdx), %xmm4, %xmm4
+        vaesenc	80(%rdx), %xmm5, %xmm5
+        vaesenc	80(%rdx), %xmm6, %xmm6
+        vaesenc	80(%rdx), %xmm7, %xmm7
+        vaesenc	96(%rdx), %xmm0, %xmm0
+        vaesenc	96(%rdx), %xmm1, %xmm1
+        vaesenc	96(%rdx), %xmm2, %xmm2
+        vaesenc	96(%rdx), %xmm3, %xmm3
+        vaesenc	96(%rdx), %xmm4, %xmm4
+        vaesenc	96(%rdx), %xmm5, %xmm5
+        vaesenc	96(%rdx), %xmm6, %xmm6
+        vaesenc	96(%rdx), %xmm7, %xmm7
+        vaesenc	112(%rdx), %xmm0, %xmm0
+        vaesenc	112(%rdx), %xmm1, %xmm1
+        vaesenc	112(%rdx), %xmm2, %xmm2
+        vaesenc	112(%rdx), %xmm3, %xmm3
+        vaesenc	112(%rdx), %xmm4, %xmm4
+        vaesenc	112(%rdx), %xmm5, %xmm5
+        vaesenc	112(%rdx), %xmm6, %xmm6
+        vaesenc	112(%rdx), %xmm7, %xmm7
+        vaesenc	128(%rdx), %xmm0, %xmm0
+        vaesenc	128(%rdx), %xmm1, %xmm1
+        vaesenc	128(%rdx), %xmm2, %xmm2
+        vaesenc	128(%rdx), %xmm3, %xmm3
+        vaesenc	128(%rdx), %xmm4, %xmm4
+        vaesenc	128(%rdx), %xmm5, %xmm5
+        vaesenc	128(%rdx), %xmm6, %xmm6
+        vaesenc	128(%rdx), %xmm7, %xmm7
+        vaesenc	144(%rdx), %xmm0, %xmm0
+        vaesenc	144(%rdx), %xmm1, %xmm1
+        vaesenc	144(%rdx), %xmm2, %xmm2
+        vaesenc	144(%rdx), %xmm3, %xmm3
+        vaesenc	144(%rdx), %xmm4, %xmm4
+        vaesenc	144(%rdx), %xmm5, %xmm5
+        vaesenc	144(%rdx), %xmm6, %xmm6
+        vaesenc	144(%rdx), %xmm7, %xmm7
+        vaesenclast	160(%rdx), %xmm0, %xmm0
+        vaesenclast	160(%rdx), %xmm1, %xmm1
+        vaesenclast	160(%rdx), %xmm2, %xmm2
+        vaesenclast	160(%rdx), %xmm3, %xmm3
+        vaesenclast	160(%rdx), %xmm4, %xmm4
+        vaesenclast	160(%rdx), %xmm5, %xmm5
+        vaesenclast	160(%rdx), %xmm6, %xmm6
+        vaesenclast	160(%rdx), %xmm7, %xmm7
+        vpand	%xmm8, %xmm0, %xmm0
+        vmovdqu	%xmm0, (%rsi)
+        vpand	%xmm8, %xmm1, %xmm1
+        vmovdqu	%xmm1, 16(%rsi)
+        vpand	%xmm8, %xmm2, %xmm2
+        vmovdqu	%xmm2, 32(%rsi)
+        vpand	%xmm8, %xmm3, %xmm3
+        vmovdqu	%xmm3, 48(%rsi)
+        vpand	%xmm8, %xmm4, %xmm4
+        vmovdqu	%xmm4, 64(%rsi)
+        vpand	%xmm8, %xmm5, %xmm5
+        vmovdqu	%xmm5, 80(%rsi)
+        vpand	%xmm8, %xmm6, %xmm6
+        vmovdqu	%xmm6, 96(%rsi)
+        vpand	%xmm8, %xmm7, %xmm7
+        vmovdqu	%xmm7, 112(%rsi)
+        addq	$0x80, %rsi
+        addq	$8, %r13
+        jmp	L_frodokem_gen_a_rows_aes_aesni_blk
+L_frodokem_gen_a_rows_aes_aesni_tail:
+        cmpq	%r10, %r13
+        jge	L_frodokem_gen_a_rows_aes_aesni_next
+        vmovd	%r14d, %xmm0
+        addq	$0x80000, %r14
+        vpxor	(%rdx), %xmm0, %xmm0
+        vaesenc	16(%rdx), %xmm0, %xmm0
+        vaesenc	32(%rdx), %xmm0, %xmm0
+        vaesenc	48(%rdx), %xmm0, %xmm0
+        vaesenc	64(%rdx), %xmm0, %xmm0
+        vaesenc	80(%rdx), %xmm0, %xmm0
+        vaesenc	96(%rdx), %xmm0, %xmm0
+        vaesenc	112(%rdx), %xmm0, %xmm0
+        vaesenc	128(%rdx), %xmm0, %xmm0
+        vaesenc	144(%rdx), %xmm0, %xmm0
+        vaesenclast	160(%rdx), %xmm0, %xmm0
+        vpand	%xmm8, %xmm0, %xmm0
+        vmovdqu	%xmm0, (%rsi)
+        addq	$16, %rsi
+        addq	$0x01, %r13
+        jmp	L_frodokem_gen_a_rows_aes_aesni_tail
+L_frodokem_gen_a_rows_aes_aesni_next:
+        addq	$0x01, %r12
+        cmpq	%r11, %r12
+        jl	L_frodokem_gen_a_rows_aes_aesni_row
+        vzeroupper
+        popq	%r15
+        popq	%r14
+        popq	%r13
+        popq	%r12
+        repz retq
+#ifndef __APPLE__
+.size	frodokem_gen_a_rows_aes_aesni,.-frodokem_gen_a_rows_aes_aesni
+#endif /* __APPLE__ */
+#endif /* HAVE_INTEL_AVX2 */
+#endif /* WOLFSSL_HAVE_FRODOKEM */
+#ifdef WOLFSSL_HAVE_FRODOKEM
+#ifdef HAVE_INTEL_AVX512
+#ifndef __APPLE__
+.text
+.globl	frodokem_sa_accum_avx512
+.type	frodokem_sa_accum_avx512,@function
+.align	16
+frodokem_sa_accum_avx512:
+#else
+.section	__TEXT,__text
+.globl	_frodokem_sa_accum_avx512
+.p2align	4
+_frodokem_sa_accum_avx512:
+#endif /* __APPLE__ */
+        pushq	%r12
+        pushq	%r13
+        pushq	%r14
+        pushq	%r15
+        pushq	%rbx
+        pushq	%rbp
+        movslq	%ecx, %rcx
+        shlq	$1, %rcx
+        addq	%rcx, %rsi
+        movslq	%r8d, %rax
+        shlq	$1, %rax
+        movq	%rax, %rcx
+        andq	$-64, %rcx
+        movq	%rdx, %r9
+        addq	%rax, %r9
+        movq	%r9, %r10
+        addq	%rax, %r10
+        movq	%r10, %r11
+        addq	%rax, %r11
+        movq	%r11, %r12
+        addq	%rax, %r12
+        movq	%r12, %r13
+        addq	%rax, %r13
+        movq	%r13, %r14
+        addq	%rax, %r14
+        movq	%r14, %r15
+        addq	%rax, %r15
+        movq	$8, %rbp
+L_frodokem_sa_accum_avx512_i:
+        vpbroadcastw	(%rsi), %zmm0
+        vpbroadcastw	2(%rsi), %zmm1
+        vpbroadcastw	4(%rsi), %zmm2
+        vpbroadcastw	6(%rsi), %zmm3
+        vpbroadcastw	8(%rsi), %zmm4
+        vpbroadcastw	10(%rsi), %zmm5
+        vpbroadcastw	12(%rsi), %zmm6
+        vpbroadcastw	14(%rsi), %zmm7
+        xorq	%rbx, %rbx
+L_frodokem_sa_accum_avx512_k:
+        vmovdqu64	(%rdi,%rbx,1), %zmm8
+        vpmullw	(%rdx,%rbx,1), %zmm0, %zmm9
+        vpaddw	%zmm9, %zmm8, %zmm8
+        vpmullw	(%r9,%rbx,1), %zmm1, %zmm9
+        vpaddw	%zmm9, %zmm8, %zmm8
+        vpmullw	(%r10,%rbx,1), %zmm2, %zmm9
+        vpaddw	%zmm9, %zmm8, %zmm8
+        vpmullw	(%r11,%rbx,1), %zmm3, %zmm9
+        vpaddw	%zmm9, %zmm8, %zmm8
+        vpmullw	(%r12,%rbx,1), %zmm4, %zmm9
+        vpaddw	%zmm9, %zmm8, %zmm8
+        vpmullw	(%r13,%rbx,1), %zmm5, %zmm9
+        vpaddw	%zmm9, %zmm8, %zmm8
+        vpmullw	(%r14,%rbx,1), %zmm6, %zmm9
+        vpaddw	%zmm9, %zmm8, %zmm8
+        vpmullw	(%r15,%rbx,1), %zmm7, %zmm9
+        vpaddw	%zmm9, %zmm8, %zmm8
+        vmovdqu64	%zmm8, (%rdi,%rbx,1)
+        addq	$0x40, %rbx
+        cmpq	%rcx, %rbx
+        jl	L_frodokem_sa_accum_avx512_k
+        cmpq	%rax, %rbx
+        jge	L_frodokem_sa_accum_avx512_tail
+        vmovdqu	(%rdi,%rbx,1), %ymm8
+        vpmullw	(%rdx,%rbx,1), %ymm0, %ymm9
+        vpaddw	%ymm9, %ymm8, %ymm8
+        vpmullw	(%r9,%rbx,1), %ymm1, %ymm9
+        vpaddw	%ymm9, %ymm8, %ymm8
+        vpmullw	(%r10,%rbx,1), %ymm2, %ymm9
+        vpaddw	%ymm9, %ymm8, %ymm8
+        vpmullw	(%r11,%rbx,1), %ymm3, %ymm9
+        vpaddw	%ymm9, %ymm8, %ymm8
+        vpmullw	(%r12,%rbx,1), %ymm4, %ymm9
+        vpaddw	%ymm9, %ymm8, %ymm8
+        vpmullw	(%r13,%rbx,1), %ymm5, %ymm9
+        vpaddw	%ymm9, %ymm8, %ymm8
+        vpmullw	(%r14,%rbx,1), %ymm6, %ymm9
+        vpaddw	%ymm9, %ymm8, %ymm8
+        vpmullw	(%r15,%rbx,1), %ymm7, %ymm9
+        vpaddw	%ymm9, %ymm8, %ymm8
+        vmovdqu	%ymm8, (%rdi,%rbx,1)
+L_frodokem_sa_accum_avx512_tail:
+        addq	%rax, %rdi
+        addq	%rax, %rsi
+        subq	$0x01, %rbp
+        jnz	L_frodokem_sa_accum_avx512_i
+        vzeroupper
+        popq	%rbp
+        popq	%rbx
+        popq	%r15
+        popq	%r14
+        popq	%r13
+        popq	%r12
+        repz retq
+#ifndef __APPLE__
+.size	frodokem_sa_accum_avx512,.-frodokem_sa_accum_avx512
+#endif /* __APPLE__ */
+#ifndef __APPLE__
+.text
+.globl	frodokem_as_accum_avx512
+.type	frodokem_as_accum_avx512,@function
+.align	16
+frodokem_as_accum_avx512:
+#else
+.section	__TEXT,__text
+.globl	_frodokem_as_accum_avx512
+.p2align	4
+_frodokem_as_accum_avx512:
+#endif /* __APPLE__ */
+        pushq	%r12
+        pushq	%r13
+        pushq	%r14
+        pushq	%r15
+        pushq	%rbx
+        pushq	%rbp
+        movslq	%ecx, %rcx
+        shlq	$4, %rcx
+        addq	%rcx, %rdi
+        movslq	%r8d, %rax
+        shlq	$1, %rax
+        movq	%rax, %rcx
+        andq	$-64, %rcx
+        movq	%rsi, %r9
+        addq	%rax, %r9
+        movq	%r9, %r10
+        addq	%rax, %r10
+        movq	%r10, %r11
+        addq	%rax, %r11
+        movq	%r11, %r12
+        addq	%rax, %r12
+        movq	%r12, %r13
+        addq	%rax, %r13
+        movq	%r13, %r14
+        addq	%rax, %r14
+        movq	%r14, %r15
+        addq	%rax, %r15
+        movq	$8, %rbp
+        vpcmpeqd	%xmm13, %xmm13, %xmm13
+        vpsrld	$16, %xmm13, %xmm13
+L_frodokem_as_accum_avx512_r:
+        vpxorq	%zmm14, %zmm14, %zmm14
+        vpxorq	%ymm22, %ymm22, %ymm22
+        vpxorq	%zmm15, %zmm15, %zmm15
+        vpxorq	%ymm23, %ymm23, %ymm23
+        vpxorq	%zmm16, %zmm16, %zmm16
+        vpxorq	%ymm24, %ymm24, %ymm24
+        vpxorq	%zmm17, %zmm17, %zmm17
+        vpxorq	%ymm25, %ymm25, %ymm25
+        vpxorq	%zmm18, %zmm18, %zmm18
+        vpxorq	%ymm26, %ymm26, %ymm26
+        vpxorq	%zmm19, %zmm19, %zmm19
+        vpxorq	%ymm27, %ymm27, %ymm27
+        vpxorq	%zmm20, %zmm20, %zmm20
+        vpxorq	%ymm28, %ymm28, %ymm28
+        vpxorq	%zmm21, %zmm21, %zmm21
+        vpxorq	%ymm29, %ymm29, %ymm29
+        xorq	%rbx, %rbx
+L_frodokem_as_accum_avx512_j:
+        vmovdqu64	(%rdx,%rbx,1), %zmm0
+        vpmaddwd	(%rsi,%rbx,1), %zmm0, %zmm1
+        vpaddd	%zmm1, %zmm14, %zmm14
+        vpmaddwd	(%r9,%rbx,1), %zmm0, %zmm1
+        vpaddd	%zmm1, %zmm15, %zmm15
+        vpmaddwd	(%r10,%rbx,1), %zmm0, %zmm1
+        vpaddd	%zmm1, %zmm16, %zmm16
+        vpmaddwd	(%r11,%rbx,1), %zmm0, %zmm1
+        vpaddd	%zmm1, %zmm17, %zmm17
+        vpmaddwd	(%r12,%rbx,1), %zmm0, %zmm1
+        vpaddd	%zmm1, %zmm18, %zmm18
+        vpmaddwd	(%r13,%rbx,1), %zmm0, %zmm1
+        vpaddd	%zmm1, %zmm19, %zmm19
+        vpmaddwd	(%r14,%rbx,1), %zmm0, %zmm1
+        vpaddd	%zmm1, %zmm20, %zmm20
+        vpmaddwd	(%r15,%rbx,1), %zmm0, %zmm1
+        vpaddd	%zmm1, %zmm21, %zmm21
+        addq	$0x40, %rbx
+        cmpq	%rcx, %rbx
+        jl	L_frodokem_as_accum_avx512_j
+        cmpq	%rax, %rbx
+        jge	L_frodokem_as_accum_avx512_tail
+        vmovdqu	(%rdx,%rbx,1), %ymm0
+        vpmaddwd	(%rsi,%rbx,1), %ymm0, %ymm22
+        vpmaddwd	(%r9,%rbx,1), %ymm0, %ymm23
+        vpmaddwd	(%r10,%rbx,1), %ymm0, %ymm24
+        vpmaddwd	(%r11,%rbx,1), %ymm0, %ymm25
+        vpmaddwd	(%r12,%rbx,1), %ymm0, %ymm26
+        vpmaddwd	(%r13,%rbx,1), %ymm0, %ymm27
+        vpmaddwd	(%r14,%rbx,1), %ymm0, %ymm28
+        vpmaddwd	(%r15,%rbx,1), %ymm0, %ymm29
+L_frodokem_as_accum_avx512_tail:
+        vextracti64x4	$0x01, %zmm14, %ymm1
+        vpaddd	%ymm1, %ymm14, %ymm1
+        vpaddd	%ymm22, %ymm1, %ymm1
+        vextracti128	$0x01, %ymm1, %xmm0
+        vpaddd	%xmm0, %xmm1, %xmm2
+        vextracti64x4	$0x01, %zmm15, %ymm1
+        vpaddd	%ymm1, %ymm15, %ymm1
+        vpaddd	%ymm23, %ymm1, %ymm1
+        vextracti128	$0x01, %ymm1, %xmm0
+        vpaddd	%xmm0, %xmm1, %xmm3
+        vextracti64x4	$0x01, %zmm16, %ymm1
+        vpaddd	%ymm1, %ymm16, %ymm1
+        vpaddd	%ymm24, %ymm1, %ymm1
+        vextracti128	$0x01, %ymm1, %xmm0
+        vpaddd	%xmm0, %xmm1, %xmm4
+        vextracti64x4	$0x01, %zmm17, %ymm1
+        vpaddd	%ymm1, %ymm17, %ymm1
+        vpaddd	%ymm25, %ymm1, %ymm1
+        vextracti128	$0x01, %ymm1, %xmm0
+        vpaddd	%xmm0, %xmm1, %xmm5
+        vextracti64x4	$0x01, %zmm18, %ymm1
+        vpaddd	%ymm1, %ymm18, %ymm1
+        vpaddd	%ymm26, %ymm1, %ymm1
+        vextracti128	$0x01, %ymm1, %xmm0
+        vpaddd	%xmm0, %xmm1, %xmm6
+        vextracti64x4	$0x01, %zmm19, %ymm1
+        vpaddd	%ymm1, %ymm19, %ymm1
+        vpaddd	%ymm27, %ymm1, %ymm1
+        vextracti128	$0x01, %ymm1, %xmm0
+        vpaddd	%xmm0, %xmm1, %xmm7
+        vextracti64x4	$0x01, %zmm20, %ymm1
+        vpaddd	%ymm1, %ymm20, %ymm1
+        vpaddd	%ymm28, %ymm1, %ymm1
+        vextracti128	$0x01, %ymm1, %xmm0
+        vpaddd	%xmm0, %xmm1, %xmm8
+        vextracti64x4	$0x01, %zmm21, %ymm1
+        vpaddd	%ymm1, %ymm21, %ymm1
+        vpaddd	%ymm29, %ymm1, %ymm1
+        vextracti128	$0x01, %ymm1, %xmm0
+        vpaddd	%xmm0, %xmm1, %xmm9
+        vphaddd	%xmm3, %xmm2, %xmm11
+        vphaddd	%xmm5, %xmm4, %xmm12
+        vphaddd	%xmm12, %xmm11, %xmm10
+        vphaddd	%xmm7, %xmm6, %xmm11
+        vphaddd	%xmm9, %xmm8, %xmm12
+        vphaddd	%xmm12, %xmm11, %xmm11
+        vpand	%xmm13, %xmm10, %xmm10
+        vpand	%xmm13, %xmm11, %xmm11
+        vpackusdw	%xmm11, %xmm10, %xmm10
+        vpaddw	(%rdi), %xmm10, %xmm10
+        vmovdqu	%xmm10, (%rdi)
+        addq	%rax, %rdx
+        addq	$16, %rdi
+        subq	$0x01, %rbp
+        jnz	L_frodokem_as_accum_avx512_r
+        vzeroupper
+        popq	%rbp
+        popq	%rbx
+        popq	%r15
+        popq	%r14
+        popq	%r13
+        popq	%r12
+        repz retq
+#ifndef __APPLE__
+.size	frodokem_as_accum_avx512,.-frodokem_as_accum_avx512
+#endif /* __APPLE__ */
+#ifndef __APPLE__
+.text
+.globl	frodokem_mul_bs_avx512
+.type	frodokem_mul_bs_avx512,@function
+.align	16
+frodokem_mul_bs_avx512:
+#else
+.section	__TEXT,__text
+.globl	_frodokem_mul_bs_avx512
+.p2align	4
+_frodokem_mul_bs_avx512:
+#endif /* __APPLE__ */
+        pushq	%r12
+        pushq	%r13
+        pushq	%r14
+        pushq	%r15
+        pushq	%rbx
+        pushq	%rbp
+        movslq	%ecx, %rax
+        shlq	$1, %rax
+        movq	%rax, %rcx
+        andq	$-64, %rcx
+        movq	%rdx, %r9
+        addq	%rax, %r9
+        movq	%r9, %r10
+        addq	%rax, %r10
+        movq	%r10, %r11
+        addq	%rax, %r11
+        movq	%r11, %r12
+        addq	%rax, %r12
+        movq	%r12, %r13
+        addq	%rax, %r13
+        movq	%r13, %r14
+        addq	%rax, %r14
+        movq	%r14, %r15
+        addq	%rax, %r15
+        movq	$8, %rbp
+        vmovd	%r8, %xmm13
+        vpbroadcastd	%xmm13, %xmm13
+L_frodokem_mul_bs_avx512_r:
+        vpxorq	%zmm14, %zmm14, %zmm14
+        vpxorq	%ymm22, %ymm22, %ymm22
+        vpxorq	%zmm15, %zmm15, %zmm15
+        vpxorq	%ymm23, %ymm23, %ymm23
+        vpxorq	%zmm16, %zmm16, %zmm16
+        vpxorq	%ymm24, %ymm24, %ymm24
+        vpxorq	%zmm17, %zmm17, %zmm17
+        vpxorq	%ymm25, %ymm25, %ymm25
+        vpxorq	%zmm18, %zmm18, %zmm18
+        vpxorq	%ymm26, %ymm26, %ymm26
+        vpxorq	%zmm19, %zmm19, %zmm19
+        vpxorq	%ymm27, %ymm27, %ymm27
+        vpxorq	%zmm20, %zmm20, %zmm20
+        vpxorq	%ymm28, %ymm28, %ymm28
+        vpxorq	%zmm21, %zmm21, %zmm21
+        vpxorq	%ymm29, %ymm29, %ymm29
+        xorq	%rbx, %rbx
+L_frodokem_mul_bs_avx512_j:
+        vmovdqu64	(%rsi,%rbx,1), %zmm0
+        vpmaddwd	(%rdx,%rbx,1), %zmm0, %zmm1
+        vpaddd	%zmm1, %zmm14, %zmm14
+        vpmaddwd	(%r9,%rbx,1), %zmm0, %zmm1
+        vpaddd	%zmm1, %zmm15, %zmm15
+        vpmaddwd	(%r10,%rbx,1), %zmm0, %zmm1
+        vpaddd	%zmm1, %zmm16, %zmm16
+        vpmaddwd	(%r11,%rbx,1), %zmm0, %zmm1
+        vpaddd	%zmm1, %zmm17, %zmm17
+        vpmaddwd	(%r12,%rbx,1), %zmm0, %zmm1
+        vpaddd	%zmm1, %zmm18, %zmm18
+        vpmaddwd	(%r13,%rbx,1), %zmm0, %zmm1
+        vpaddd	%zmm1, %zmm19, %zmm19
+        vpmaddwd	(%r14,%rbx,1), %zmm0, %zmm1
+        vpaddd	%zmm1, %zmm20, %zmm20
+        vpmaddwd	(%r15,%rbx,1), %zmm0, %zmm1
+        vpaddd	%zmm1, %zmm21, %zmm21
+        addq	$0x40, %rbx
+        cmpq	%rcx, %rbx
+        jl	L_frodokem_mul_bs_avx512_j
+        cmpq	%rax, %rbx
+        jge	L_frodokem_mul_bs_avx512_tail
+        vmovdqu	(%rsi,%rbx,1), %ymm0
+        vpmaddwd	(%rdx,%rbx,1), %ymm0, %ymm22
+        vpmaddwd	(%r9,%rbx,1), %ymm0, %ymm23
+        vpmaddwd	(%r10,%rbx,1), %ymm0, %ymm24
+        vpmaddwd	(%r11,%rbx,1), %ymm0, %ymm25
+        vpmaddwd	(%r12,%rbx,1), %ymm0, %ymm26
+        vpmaddwd	(%r13,%rbx,1), %ymm0, %ymm27
+        vpmaddwd	(%r14,%rbx,1), %ymm0, %ymm28
+        vpmaddwd	(%r15,%rbx,1), %ymm0, %ymm29
+L_frodokem_mul_bs_avx512_tail:
+        vextracti64x4	$0x01, %zmm14, %ymm1
+        vpaddd	%ymm1, %ymm14, %ymm1
+        vpaddd	%ymm22, %ymm1, %ymm1
+        vextracti128	$0x01, %ymm1, %xmm0
+        vpaddd	%xmm0, %xmm1, %xmm2
+        vextracti64x4	$0x01, %zmm15, %ymm1
+        vpaddd	%ymm1, %ymm15, %ymm1
+        vpaddd	%ymm23, %ymm1, %ymm1
+        vextracti128	$0x01, %ymm1, %xmm0
+        vpaddd	%xmm0, %xmm1, %xmm3
+        vextracti64x4	$0x01, %zmm16, %ymm1
+        vpaddd	%ymm1, %ymm16, %ymm1
+        vpaddd	%ymm24, %ymm1, %ymm1
+        vextracti128	$0x01, %ymm1, %xmm0
+        vpaddd	%xmm0, %xmm1, %xmm4
+        vextracti64x4	$0x01, %zmm17, %ymm1
+        vpaddd	%ymm1, %ymm17, %ymm1
+        vpaddd	%ymm25, %ymm1, %ymm1
+        vextracti128	$0x01, %ymm1, %xmm0
+        vpaddd	%xmm0, %xmm1, %xmm5
+        vextracti64x4	$0x01, %zmm18, %ymm1
+        vpaddd	%ymm1, %ymm18, %ymm1
+        vpaddd	%ymm26, %ymm1, %ymm1
+        vextracti128	$0x01, %ymm1, %xmm0
+        vpaddd	%xmm0, %xmm1, %xmm6
+        vextracti64x4	$0x01, %zmm19, %ymm1
+        vpaddd	%ymm1, %ymm19, %ymm1
+        vpaddd	%ymm27, %ymm1, %ymm1
+        vextracti128	$0x01, %ymm1, %xmm0
+        vpaddd	%xmm0, %xmm1, %xmm7
+        vextracti64x4	$0x01, %zmm20, %ymm1
+        vpaddd	%ymm1, %ymm20, %ymm1
+        vpaddd	%ymm28, %ymm1, %ymm1
+        vextracti128	$0x01, %ymm1, %xmm0
+        vpaddd	%xmm0, %xmm1, %xmm8
+        vextracti64x4	$0x01, %zmm21, %ymm1
+        vpaddd	%ymm1, %ymm21, %ymm1
+        vpaddd	%ymm29, %ymm1, %ymm1
+        vextracti128	$0x01, %ymm1, %xmm0
+        vpaddd	%xmm0, %xmm1, %xmm9
+        vphaddd	%xmm3, %xmm2, %xmm11
+        vphaddd	%xmm5, %xmm4, %xmm12
+        vphaddd	%xmm12, %xmm11, %xmm10
+        vphaddd	%xmm7, %xmm6, %xmm11
+        vphaddd	%xmm9, %xmm8, %xmm12
+        vphaddd	%xmm12, %xmm11, %xmm11
+        vpand	%xmm13, %xmm10, %xmm10
+        vpand	%xmm13, %xmm11, %xmm11
+        vpackusdw	%xmm11, %xmm10, %xmm10
+        vmovdqu	%xmm10, (%rdi)
+        addq	%rax, %rsi
+        addq	$16, %rdi
+        subq	$0x01, %rbp
+        jnz	L_frodokem_mul_bs_avx512_r
+        vzeroupper
+        popq	%rbp
+        popq	%rbx
+        popq	%r15
+        popq	%r14
+        popq	%r13
+        popq	%r12
+        repz retq
+#ifndef __APPLE__
+.size	frodokem_mul_bs_avx512,.-frodokem_mul_bs_avx512
+#endif /* __APPLE__ */
+#ifndef __APPLE__
+.text
+.globl	frodokem_mul_add_sb_plus_e_avx512
+.type	frodokem_mul_add_sb_plus_e_avx512,@function
+.align	16
+frodokem_mul_add_sb_plus_e_avx512:
+#else
+.section	__TEXT,__text
+.globl	_frodokem_mul_add_sb_plus_e_avx512
+.p2align	4
+_frodokem_mul_add_sb_plus_e_avx512:
+#endif /* __APPLE__ */
+        pushq	%r12
+        pushq	%r13
+        pushq	%r14
+        pushq	%r15
+        pushq	%rbx
+        movslq	%ecx, %rax
+        shlq	$1, %rax
+        movq	%rdx, %r9
+        addq	%rax, %r9
+        movq	%r9, %r10
+        addq	%rax, %r10
+        movq	%r10, %r11
+        addq	%rax, %r11
+        movq	%r11, %r12
+        addq	%rax, %r12
+        movq	%r12, %r13
+        addq	%rax, %r13
+        movq	%r13, %r14
+        addq	%rax, %r14
+        movq	%r14, %r15
+        addq	%rax, %r15
+        vmovd	%r8, %xmm5
+        vpbroadcastw	%xmm5, %zmm5
+        vmovdqu64	(%rdi), %zmm0
+        vmovdqu64	64(%rdi), %zmm1
+        xorq	%rbx, %rbx
+L_frodokem_mul_add_sb_plus_e_avx512_j:
+        vbroadcasti32x4	(%rsi), %zmm2
+        vpbroadcastw	(%rdx,%rbx,1), %zmm3
+        vpbroadcastw	(%r9,%rbx,1), %zmm4
+        vinserti32x4	$0x01, %xmm4, %zmm3, %zmm3
+        vpbroadcastw	(%r10,%rbx,1), %zmm4
+        vinserti32x4	$2, %xmm4, %zmm3, %zmm3
+        vpbroadcastw	(%r11,%rbx,1), %zmm4
+        vinserti32x4	$3, %xmm4, %zmm3, %zmm3
+        vpmullw	%zmm2, %zmm3, %zmm4
+        vpaddw	%zmm4, %zmm0, %zmm0
+        vpbroadcastw	(%r12,%rbx,1), %zmm3
+        vpbroadcastw	(%r13,%rbx,1), %zmm4
+        vinserti32x4	$0x01, %xmm4, %zmm3, %zmm3
+        vpbroadcastw	(%r14,%rbx,1), %zmm4
+        vinserti32x4	$2, %xmm4, %zmm3, %zmm3
+        vpbroadcastw	(%r15,%rbx,1), %zmm4
+        vinserti32x4	$3, %xmm4, %zmm3, %zmm3
+        vpmullw	%zmm2, %zmm3, %zmm4
+        vpaddw	%zmm4, %zmm1, %zmm1
+        addq	$16, %rsi
+        addq	$2, %rbx
+        cmpq	%rax, %rbx
+        jl	L_frodokem_mul_add_sb_plus_e_avx512_j
+        vpandq	%zmm5, %zmm0, %zmm0
+        vmovdqu64	%zmm0, (%rdi)
+        vpandq	%zmm5, %zmm1, %zmm1
+        vmovdqu64	%zmm1, 64(%rdi)
+        vzeroupper
+        popq	%rbx
+        popq	%r15
+        popq	%r14
+        popq	%r13
+        popq	%r12
+        repz retq
+#ifndef __APPLE__
+.size	frodokem_mul_add_sb_plus_e_avx512,.-frodokem_mul_add_sb_plus_e_avx512
+#endif /* __APPLE__ */
+#ifndef __APPLE__
+.text
+.globl	frodokem_add_avx512
+.type	frodokem_add_avx512,@function
+.align	16
+frodokem_add_avx512:
+#else
+.section	__TEXT,__text
+.globl	_frodokem_add_avx512
+.p2align	4
+_frodokem_add_avx512:
+#endif /* __APPLE__ */
+        vmovd	%rdx, %xmm0
+        vpbroadcastw	%xmm0, %zmm0
+        vmovdqu64	(%rdi), %zmm1
+        vpaddw	(%rsi), %zmm1, %zmm1
+        vpandq	%zmm0, %zmm1, %zmm1
+        vmovdqu64	%zmm1, (%rdi)
+        vmovdqu64	64(%rdi), %zmm2
+        vpaddw	64(%rsi), %zmm2, %zmm2
+        vpandq	%zmm0, %zmm2, %zmm2
+        vmovdqu64	%zmm2, 64(%rdi)
+        vzeroupper
+        repz retq
+#ifndef __APPLE__
+.size	frodokem_add_avx512,.-frodokem_add_avx512
+#endif /* __APPLE__ */
+#ifndef __APPLE__
+.text
+.globl	frodokem_sample_avx512
+.type	frodokem_sample_avx512,@function
+.align	16
+frodokem_sample_avx512:
+#else
+.section	__TEXT,__text
+.globl	_frodokem_sample_avx512
+.p2align	4
+_frodokem_sample_avx512:
+#endif /* __APPLE__ */
+        movslq	%esi, %rax
+        shlq	$1, %rax
+        vpternlogd	$0xff, %zmm0, %zmm0, %zmm0
+        vpsrlw	$15, %zmm0, %zmm0
+        vpxorq	%zmm1, %zmm1, %zmm1
+        xorq	%r8, %r8
+L_frodokem_sample_avx512_blk:
+        vmovdqu64	(%rdi,%r8,1), %zmm2
+        vpsrlw	$0x01, %zmm2, %zmm3
+        vpandq	%zmm0, %zmm2, %zmm4
+        vpxorq	%zmm5, %zmm5, %zmm5
+        movq	%rdx, %r9
+        movl	%ecx, %r10d
+L_frodokem_sample_avx512_cdf:
+        vpbroadcastw	(%r9), %zmm6
+        vpsubw	%zmm3, %zmm6, %zmm6
+        vpsrlw	$15, %zmm6, %zmm6
+        vpaddw	%zmm6, %zmm5, %zmm5
+        addq	$2, %r9
+        subq	$0x01, %r10
+        jnz	L_frodokem_sample_avx512_cdf
+        vpsubw	%zmm4, %zmm1, %zmm7
+        vpxorq	%zmm7, %zmm5, %zmm5
+        vpaddw	%zmm4, %zmm5, %zmm5
+        vmovdqu64	%zmm5, (%rdi,%r8,1)
+        addq	$0x40, %r8
+        cmpq	%rax, %r8
+        jl	L_frodokem_sample_avx512_blk
+        vzeroupper
+        repz retq
+#ifndef __APPLE__
+.size	frodokem_sample_avx512,.-frodokem_sample_avx512
+#endif /* __APPLE__ */
+#ifndef __APPLE__
+.text
+.globl	frodokem_gen_a_rows_aes_avx512
+.type	frodokem_gen_a_rows_aes_avx512,@function
+.align	16
+frodokem_gen_a_rows_aes_avx512:
+#else
+.section	__TEXT,__text
+.globl	_frodokem_gen_a_rows_aes_avx512
+.p2align	4
+_frodokem_gen_a_rows_aes_avx512:
+#endif /* __APPLE__ */
+        pushq	%r12
+        pushq	%r13
+        pushq	%r14
+        pushq	%r15
+        movq	40(%rsp), %rax
+        vbroadcasti32x4	(%rdx), %zmm0
+        vbroadcasti32x4	16(%rdx), %zmm1
+        vbroadcasti32x4	32(%rdx), %zmm2
+        vbroadcasti32x4	48(%rdx), %zmm3
+        vbroadcasti32x4	64(%rdx), %zmm4
+        vbroadcasti32x4	80(%rdx), %zmm5
+        vbroadcasti32x4	96(%rdx), %zmm6
+        vbroadcasti32x4	112(%rdx), %zmm7
+        vbroadcasti32x4	128(%rdx), %zmm8
+        vbroadcasti32x4	144(%rdx), %zmm9
+        vbroadcasti32x4	160(%rdx), %zmm10
+        vmovd	%rax, %xmm15
+        vpbroadcastw	%xmm15, %zmm15
+        movslq	%r9d, %r10
+        shlq	$1, %r10
+        movslq	%r8d, %r11
+        imulq	%r10, %r11
+        movslq	%ecx, %rcx
+        xorq	%r15, %r15
+        movq	%rdi, %r12
+L_frodokem_gen_a_rows_aes_avx512_row:
+        movl	%ecx, %r14d
+        movq	%r12, %r13
+        addq	%r10, %r13
+L_frodokem_gen_a_rows_aes_avx512_blk:
+        movq	%r14, (%r12)
+        movq	%r15, 8(%r12)
+        addq	$0x80000, %r14
+        addq	$16, %r12
+        cmpq	%r13, %r12
+        jl	L_frodokem_gen_a_rows_aes_avx512_blk
+        addq	$0x01, %rcx
+        movq	%rdi, %r13
+        addq	%r11, %r13
+        cmpq	%r13, %r12
+        jl	L_frodokem_gen_a_rows_aes_avx512_row
+        xorq	%r14, %r14
+L_frodokem_gen_a_rows_aes_avx512_aes:
+        vmovdqu64	(%rdi), %zmm11
+        vmovdqu64	64(%rdi), %zmm12
+        vmovdqu64	128(%rdi), %zmm13
+        vmovdqu64	192(%rdi), %zmm14
+        vpxorq	%zmm0, %zmm11, %zmm11
+        vpxorq	%zmm0, %zmm12, %zmm12
+        vpxorq	%zmm0, %zmm13, %zmm13
+        vpxorq	%zmm0, %zmm14, %zmm14
+        vaesenc	%zmm1, %zmm11, %zmm11
+        vaesenc	%zmm1, %zmm12, %zmm12
+        vaesenc	%zmm1, %zmm13, %zmm13
+        vaesenc	%zmm1, %zmm14, %zmm14
+        vaesenc	%zmm2, %zmm11, %zmm11
+        vaesenc	%zmm2, %zmm12, %zmm12
+        vaesenc	%zmm2, %zmm13, %zmm13
+        vaesenc	%zmm2, %zmm14, %zmm14
+        vaesenc	%zmm3, %zmm11, %zmm11
+        vaesenc	%zmm3, %zmm12, %zmm12
+        vaesenc	%zmm3, %zmm13, %zmm13
+        vaesenc	%zmm3, %zmm14, %zmm14
+        vaesenc	%zmm4, %zmm11, %zmm11
+        vaesenc	%zmm4, %zmm12, %zmm12
+        vaesenc	%zmm4, %zmm13, %zmm13
+        vaesenc	%zmm4, %zmm14, %zmm14
+        vaesenc	%zmm5, %zmm11, %zmm11
+        vaesenc	%zmm5, %zmm12, %zmm12
+        vaesenc	%zmm5, %zmm13, %zmm13
+        vaesenc	%zmm5, %zmm14, %zmm14
+        vaesenc	%zmm6, %zmm11, %zmm11
+        vaesenc	%zmm6, %zmm12, %zmm12
+        vaesenc	%zmm6, %zmm13, %zmm13
+        vaesenc	%zmm6, %zmm14, %zmm14
+        vaesenc	%zmm7, %zmm11, %zmm11
+        vaesenc	%zmm7, %zmm12, %zmm12
+        vaesenc	%zmm7, %zmm13, %zmm13
+        vaesenc	%zmm7, %zmm14, %zmm14
+        vaesenc	%zmm8, %zmm11, %zmm11
+        vaesenc	%zmm8, %zmm12, %zmm12
+        vaesenc	%zmm8, %zmm13, %zmm13
+        vaesenc	%zmm8, %zmm14, %zmm14
+        vaesenc	%zmm9, %zmm11, %zmm11
+        vaesenc	%zmm9, %zmm12, %zmm12
+        vaesenc	%zmm9, %zmm13, %zmm13
+        vaesenc	%zmm9, %zmm14, %zmm14
+        vaesenclast	%zmm10, %zmm11, %zmm11
+        vaesenclast	%zmm10, %zmm12, %zmm12
+        vaesenclast	%zmm10, %zmm13, %zmm13
+        vaesenclast	%zmm10, %zmm14, %zmm14
+        vpandq	%zmm15, %zmm11, %zmm11
+        vpandq	%zmm15, %zmm12, %zmm12
+        vpandq	%zmm15, %zmm13, %zmm13
+        vpandq	%zmm15, %zmm14, %zmm14
+        vmovdqu64	%zmm11, (%rsi)
+        vmovdqu64	%zmm12, 64(%rsi)
+        vmovdqu64	%zmm13, 128(%rsi)
+        vmovdqu64	%zmm14, 192(%rsi)
+        addq	$0x100, %rdi
+        addq	$0x100, %rsi
+        addq	$0x100, %r14
+        cmpq	%r11, %r14
+        jl	L_frodokem_gen_a_rows_aes_avx512_aes
+        vzeroupper
+        popq	%r15
+        popq	%r14
+        popq	%r13
+        popq	%r12
+        repz retq
+#ifndef __APPLE__
+.size	frodokem_gen_a_rows_aes_avx512,.-frodokem_gen_a_rows_aes_avx512
+#endif /* __APPLE__ */
+#endif /* HAVE_INTEL_AVX512 */
+#endif /* WOLFSSL_HAVE_FRODOKEM */
+
+#if defined(__linux__) && defined(__ELF__)
+.section	.note.GNU-stack,"",%progbits
+#endif
diff --git a/wolfcrypt/src/wc_frodokem_asm.asm b/wolfcrypt/src/wc_frodokem_asm.asm
new file mode 100644
index 0000000000..1ce1e92c2b
--- /dev/null
+++ b/wolfcrypt/src/wc_frodokem_asm.asm
@@ -0,0 +1,1613 @@
+; /* wc_frodokem_asm.asm */
+; /*
+;  * Copyright (C) 2006-2026 wolfSSL Inc.
+;  *
+;  * This file is part of wolfSSL.
+;  *
+;  * wolfSSL is free software; you can redistribute it and/or modify
+;  * it under the terms of the GNU General Public License as published by
+;  * the Free Software Foundation; either version 3 of the License, or
+;  * (at your option) any later version.
+;  *
+;  * wolfSSL is distributed in the hope that it will be useful,
+;  * but WITHOUT ANY WARRANTY; without even the implied warranty of
+;  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+;  * GNU General Public License for more details.
+;  *
+;  * You should have received a copy of the GNU General Public License
+;  * along with this program; if not, write to the Free Software
+;  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335, USA
+;  */
+
+IF @Version LT 1200
+; AVX2 instructions not recognized by old versions of MASM
+IFNDEF NO_AVX2_SUPPORT
+NO_AVX2_SUPPORT = 1
+ENDIF
+; MOVBE instruction not recognized by old versions of MASM
+IFNDEF NO_MOVBE_SUPPORT
+NO_MOVBE_SUPPORT = 1
+ENDIF
+ENDIF
+
+IFNDEF HAVE_INTEL_AVX1
+HAVE_INTEL_AVX1 = 1
+ENDIF
+IFNDEF NO_AVX2_SUPPORT
+HAVE_INTEL_AVX2 = 1
+ENDIF
+
+IFNDEF _WIN64
+_WIN64 = 1
+ENDIF
+
+IFDEF WOLFSSL_HAVE_FRODOKEM
+IFDEF HAVE_INTEL_AVX2
+_TEXT SEGMENT READONLY PARA
+frodokem_sa_accum_avx2 PROC
+        push	r12
+        push	r13
+        push	r14
+        push	r15
+        mov	rax, QWORD PTR [rsp+72]
+        movsxd	r9, r9d
+        shl	r9, 1
+        add	rdx, r9
+        movsxd	r10, eax
+        shl	r10, 1
+        mov	r11, r8
+        add	r11, r10
+        mov	r12, r11
+        add	r12, r10
+        mov	r13, r12
+        add	r13, r10
+        mov	r15, 8
+L_frodokem_sa_accum_avx2_i:
+        vpbroadcastw	ymm0, WORD PTR [rdx]
+        vpbroadcastw	ymm1, WORD PTR [rdx+2]
+        vpbroadcastw	ymm2, WORD PTR [rdx+4]
+        vpbroadcastw	ymm3, WORD PTR [rdx+6]
+        xor	r14, r14
+L_frodokem_sa_accum_avx2_k:
+        vmovdqu	ymm4, YMMWORD PTR [rcx+r14]
+        vpmullw	ymm5, ymm0, [r8+r14]
+        vpaddw	ymm4, ymm4, ymm5
+        vpmullw	ymm5, ymm1, [r11+r14]
+        vpaddw	ymm4, ymm4, ymm5
+        vpmullw	ymm5, ymm2, [r12+r14]
+        vpaddw	ymm4, ymm4, ymm5
+        vpmullw	ymm5, ymm3, [r13+r14]
+        vpaddw	ymm4, ymm4, ymm5
+        vmovdqu	YMMWORD PTR [rcx+r14], ymm4
+        add	r14, 32
+        cmp	r14, r10
+        jl	L_frodokem_sa_accum_avx2_k
+        add	rcx, r10
+        add	rdx, r10
+        sub	r15, 1
+        jnz	L_frodokem_sa_accum_avx2_i
+        vzeroupper
+        pop	r15
+        pop	r14
+        pop	r13
+        pop	r12
+        ret
+frodokem_sa_accum_avx2 ENDP
+_TEXT ENDS
+_TEXT SEGMENT READONLY PARA
+frodokem_as_accum_avx2 PROC
+        push	r12
+        push	r13
+        push	r14
+        push	r15
+        push	rdi
+        push	rsi
+        push	rbx
+        push	rbp
+        mov	rax, QWORD PTR [rsp+104]
+        sub	rsp, 112
+        vmovdqu	OWORD PTR [rsp], xmm6
+        vmovdqu	OWORD PTR [rsp+16], xmm7
+        vmovdqu	OWORD PTR [rsp+32], xmm8
+        vmovdqu	OWORD PTR [rsp+48], xmm9
+        vmovdqu	OWORD PTR [rsp+64], xmm10
+        vmovdqu	OWORD PTR [rsp+80], xmm11
+        vmovdqu	OWORD PTR [rsp+96], xmm12
+        movsxd	r9, r9d
+        shl	r9, 4
+        add	rcx, r9
+        movsxd	r10, eax
+        shl	r10, 1
+        mov	r11, rdx
+        add	r11, r10
+        mov	r12, r11
+        add	r12, r10
+        mov	r13, r12
+        add	r13, r10
+        mov	r14, r13
+        add	r14, r10
+        mov	r15, r14
+        add	r15, r10
+        mov	rdi, r15
+        add	rdi, r10
+        mov	rsi, rdi
+        add	rsi, r10
+        mov	rbp, 4
+        vpcmpeqd	xmm12, xmm12, xmm12
+        vpsrld	xmm12, xmm12, 16
+L_frodokem_as_accum_avx2_r:
+        vpxor	ymm0, ymm0, ymm0
+        vpxor	ymm1, ymm1, ymm1
+        vpxor	ymm2, ymm2, ymm2
+        vpxor	ymm3, ymm3, ymm3
+        vpxor	ymm4, ymm4, ymm4
+        vpxor	ymm5, ymm5, ymm5
+        vpxor	ymm6, ymm6, ymm6
+        vpxor	ymm7, ymm7, ymm7
+        xor	rbx, rbx
+L_frodokem_as_accum_avx2_j:
+        vmovdqu	ymm8, YMMWORD PTR [r8+rbx]
+        vpmaddwd	ymm10, ymm8, [rdx+rbx]
+        vpaddd	ymm0, ymm0, ymm10
+        vpmaddwd	ymm10, ymm8, [r11+rbx]
+        vpaddd	ymm1, ymm1, ymm10
+        vpmaddwd	ymm10, ymm8, [r12+rbx]
+        vpaddd	ymm2, ymm2, ymm10
+        vpmaddwd	ymm10, ymm8, [r13+rbx]
+        vpaddd	ymm3, ymm3, ymm10
+        vpmaddwd	ymm10, ymm8, [r14+rbx]
+        vpaddd	ymm4, ymm4, ymm10
+        vpmaddwd	ymm10, ymm8, [r15+rbx]
+        vpaddd	ymm5, ymm5, ymm10
+        vpmaddwd	ymm10, ymm8, [rdi+rbx]
+        vpaddd	ymm6, ymm6, ymm10
+        vpmaddwd	ymm10, ymm8, [rsi+rbx]
+        vpaddd	ymm7, ymm7, ymm10
+        add	rbx, 32
+        cmp	rbx, r10
+        jl	L_frodokem_as_accum_avx2_j
+        vextracti128	xmm8, ymm0, 1
+        vpaddd	xmm0, xmm0, xmm8
+        vextracti128	xmm8, ymm1, 1
+        vpaddd	xmm1, xmm1, xmm8
+        vextracti128	xmm8, ymm2, 1
+        vpaddd	xmm2, xmm2, xmm8
+        vextracti128	xmm8, ymm3, 1
+        vpaddd	xmm3, xmm3, xmm8
+        vextracti128	xmm8, ymm4, 1
+        vpaddd	xmm4, xmm4, xmm8
+        vextracti128	xmm8, ymm5, 1
+        vpaddd	xmm5, xmm5, xmm8
+        vextracti128	xmm8, ymm6, 1
+        vpaddd	xmm6, xmm6, xmm8
+        vextracti128	xmm8, ymm7, 1
+        vpaddd	xmm7, xmm7, xmm8
+        vphaddd	xmm10, xmm0, xmm1
+        vphaddd	xmm11, xmm2, xmm3
+        vphaddd	xmm9, xmm10, xmm11
+        vphaddd	xmm10, xmm4, xmm5
+        vphaddd	xmm11, xmm6, xmm7
+        vphaddd	xmm10, xmm10, xmm11
+        vpand	xmm9, xmm9, xmm12
+        vpand	xmm10, xmm10, xmm12
+        vpackusdw	xmm9, xmm9, xmm10
+        vpaddw	xmm9, xmm9, [rcx]
+        vmovdqu	OWORD PTR [rcx], xmm9
+        add	r8, r10
+        add	rcx, 16
+        sub	rbp, 1
+        jnz	L_frodokem_as_accum_avx2_r
+        vzeroupper
+        vmovdqu	xmm6, OWORD PTR [rsp]
+        vmovdqu	xmm7, OWORD PTR [rsp+16]
+        vmovdqu	xmm8, OWORD PTR [rsp+32]
+        vmovdqu	xmm9, OWORD PTR [rsp+48]
+        vmovdqu	xmm10, OWORD PTR [rsp+64]
+        vmovdqu	xmm11, OWORD PTR [rsp+80]
+        vmovdqu	xmm12, OWORD PTR [rsp+96]
+        add	rsp, 112
+        pop	rbp
+        pop	rbx
+        pop	rsi
+        pop	rdi
+        pop	r15
+        pop	r14
+        pop	r13
+        pop	r12
+        ret
+frodokem_as_accum_avx2 ENDP
+_TEXT ENDS
+_TEXT SEGMENT READONLY PARA
+frodokem_mul_bs_avx2 PROC
+        push	r12
+        push	r13
+        push	r14
+        push	r15
+        push	rdi
+        push	rsi
+        push	rbx
+        push	rbp
+        mov	rax, QWORD PTR [rsp+104]
+        sub	rsp, 112
+        vmovdqu	OWORD PTR [rsp], xmm6
+        vmovdqu	OWORD PTR [rsp+16], xmm7
+        vmovdqu	OWORD PTR [rsp+32], xmm8
+        vmovdqu	OWORD PTR [rsp+48], xmm9
+        vmovdqu	OWORD PTR [rsp+64], xmm10
+        vmovdqu	OWORD PTR [rsp+80], xmm11
+        vmovdqu	OWORD PTR [rsp+96], xmm12
+        movsxd	r10, r9d
+        shl	r10, 1
+        mov	r11, r8
+        add	r11, r10
+        mov	r12, r11
+        add	r12, r10
+        mov	r13, r12
+        add	r13, r10
+        mov	r14, r13
+        add	r14, r10
+        mov	r15, r14
+        add	r15, r10
+        mov	rdi, r15
+        add	rdi, r10
+        mov	rsi, rdi
+        add	rsi, r10
+        mov	rbp, 8
+        vmovd	xmm12, rax
+        vpbroadcastd	xmm12, xmm12
+L_frodokem_mul_bs_avx2_r:
+        vpxor	ymm0, ymm0, ymm0
+        vpxor	ymm1, ymm1, ymm1
+        vpxor	ymm2, ymm2, ymm2
+        vpxor	ymm3, ymm3, ymm3
+        vpxor	ymm4, ymm4, ymm4
+        vpxor	ymm5, ymm5, ymm5
+        vpxor	ymm6, ymm6, ymm6
+        vpxor	ymm7, ymm7, ymm7
+        xor	rbx, rbx
+L_frodokem_mul_bs_avx2_j:
+        vmovdqu	ymm8, YMMWORD PTR [rdx+rbx]
+        vpmaddwd	ymm10, ymm8, [r8+rbx]
+        vpaddd	ymm0, ymm0, ymm10
+        vpmaddwd	ymm10, ymm8, [r11+rbx]
+        vpaddd	ymm1, ymm1, ymm10
+        vpmaddwd	ymm10, ymm8, [r12+rbx]
+        vpaddd	ymm2, ymm2, ymm10
+        vpmaddwd	ymm10, ymm8, [r13+rbx]
+        vpaddd	ymm3, ymm3, ymm10
+        vpmaddwd	ymm10, ymm8, [r14+rbx]
+        vpaddd	ymm4, ymm4, ymm10
+        vpmaddwd	ymm10, ymm8, [r15+rbx]
+        vpaddd	ymm5, ymm5, ymm10
+        vpmaddwd	ymm10, ymm8, [rdi+rbx]
+        vpaddd	ymm6, ymm6, ymm10
+        vpmaddwd	ymm10, ymm8, [rsi+rbx]
+        vpaddd	ymm7, ymm7, ymm10
+        add	rbx, 32
+        cmp	rbx, r10
+        jl	L_frodokem_mul_bs_avx2_j
+        vextracti128	xmm8, ymm0, 1
+        vpaddd	xmm0, xmm0, xmm8
+        vextracti128	xmm8, ymm1, 1
+        vpaddd	xmm1, xmm1, xmm8
+        vextracti128	xmm8, ymm2, 1
+        vpaddd	xmm2, xmm2, xmm8
+        vextracti128	xmm8, ymm3, 1
+        vpaddd	xmm3, xmm3, xmm8
+        vextracti128	xmm8, ymm4, 1
+        vpaddd	xmm4, xmm4, xmm8
+        vextracti128	xmm8, ymm5, 1
+        vpaddd	xmm5, xmm5, xmm8
+        vextracti128	xmm8, ymm6, 1
+        vpaddd	xmm6, xmm6, xmm8
+        vextracti128	xmm8, ymm7, 1
+        vpaddd	xmm7, xmm7, xmm8
+        vphaddd	xmm10, xmm0, xmm1
+        vphaddd	xmm11, xmm2, xmm3
+        vphaddd	xmm9, xmm10, xmm11
+        vphaddd	xmm10, xmm4, xmm5
+        vphaddd	xmm11, xmm6, xmm7
+        vphaddd	xmm10, xmm10, xmm11
+        vpand	xmm9, xmm9, xmm12
+        vpand	xmm10, xmm10, xmm12
+        vpackusdw	xmm9, xmm9, xmm10
+        vmovdqu	OWORD PTR [rcx], xmm9
+        add	rdx, r10
+        add	rcx, 16
+        sub	rbp, 1
+        jnz	L_frodokem_mul_bs_avx2_r
+        vzeroupper
+        vmovdqu	xmm6, OWORD PTR [rsp]
+        vmovdqu	xmm7, OWORD PTR [rsp+16]
+        vmovdqu	xmm8, OWORD PTR [rsp+32]
+        vmovdqu	xmm9, OWORD PTR [rsp+48]
+        vmovdqu	xmm10, OWORD PTR [rsp+64]
+        vmovdqu	xmm11, OWORD PTR [rsp+80]
+        vmovdqu	xmm12, OWORD PTR [rsp+96]
+        add	rsp, 112
+        pop	rbp
+        pop	rbx
+        pop	rsi
+        pop	rdi
+        pop	r15
+        pop	r14
+        pop	r13
+        pop	r12
+        ret
+frodokem_mul_bs_avx2 ENDP
+_TEXT ENDS
+_TEXT SEGMENT READONLY PARA
+frodokem_mul_add_sb_plus_e_avx2 PROC
+        push	r12
+        push	r13
+        push	r14
+        push	r15
+        push	rdi
+        push	rsi
+        push	rbx
+        mov	rax, QWORD PTR [rsp+96]
+        sub	rsp, 32
+        vmovdqu	OWORD PTR [rsp], xmm6
+        vmovdqu	OWORD PTR [rsp+16], xmm7
+        movsxd	r10, r9d
+        shl	r10, 1
+        mov	r11, r8
+        add	r11, r10
+        mov	r12, r11
+        add	r12, r10
+        mov	r13, r12
+        add	r13, r10
+        mov	r14, r13
+        add	r14, r10
+        mov	r15, r14
+        add	r15, r10
+        mov	rdi, r15
+        add	rdi, r10
+        mov	rsi, rdi
+        add	rsi, r10
+        vmovd	xmm7, rax
+        vpbroadcastw	ymm7, xmm7
+        vmovdqu	ymm0, YMMWORD PTR [rcx]
+        vmovdqu	ymm1, YMMWORD PTR [rcx+32]
+        vmovdqu	ymm2, YMMWORD PTR [rcx+64]
+        vmovdqu	ymm3, YMMWORD PTR [rcx+96]
+        xor	rbx, rbx
+L_frodokem_mul_add_sb_plus_e_avx2_j:
+        vbroadcasti128	ymm4, OWORD PTR [rdx]
+        vpbroadcastw	ymm5, WORD PTR [r8+rbx]
+        vpbroadcastw	ymm6, WORD PTR [r11+rbx]
+        vinserti128	ymm5, ymm5, xmm6, 1
+        vpmullw	ymm6, ymm5, ymm4
+        vpaddw	ymm0, ymm0, ymm6
+        vpbroadcastw	ymm5, WORD PTR [r12+rbx]
+        vpbroadcastw	ymm6, WORD PTR [r13+rbx]
+        vinserti128	ymm5, ymm5, xmm6, 1
+        vpmullw	ymm6, ymm5, ymm4
+        vpaddw	ymm1, ymm1, ymm6
+        vpbroadcastw	ymm5, WORD PTR [r14+rbx]
+        vpbroadcastw	ymm6, WORD PTR [r15+rbx]
+        vinserti128	ymm5, ymm5, xmm6, 1
+        vpmullw	ymm6, ymm5, ymm4
+        vpaddw	ymm2, ymm2, ymm6
+        vpbroadcastw	ymm5, WORD PTR [rdi+rbx]
+        vpbroadcastw	ymm6, WORD PTR [rsi+rbx]
+        vinserti128	ymm5, ymm5, xmm6, 1
+        vpmullw	ymm6, ymm5, ymm4
+        vpaddw	ymm3, ymm3, ymm6
+        add	rdx, 16
+        add	rbx, 2
+        cmp	rbx, r10
+        jl	L_frodokem_mul_add_sb_plus_e_avx2_j
+        vpand	ymm0, ymm0, ymm7
+        vmovdqu	YMMWORD PTR [rcx], ymm0
+        vpand	ymm1, ymm1, ymm7
+        vmovdqu	YMMWORD PTR [rcx+32], ymm1
+        vpand	ymm2, ymm2, ymm7
+        vmovdqu	YMMWORD PTR [rcx+64], ymm2
+        vpand	ymm3, ymm3, ymm7
+        vmovdqu	YMMWORD PTR [rcx+96], ymm3
+        vzeroupper
+        vmovdqu	xmm6, OWORD PTR [rsp]
+        vmovdqu	xmm7, OWORD PTR [rsp+16]
+        add	rsp, 32
+        pop	rbx
+        pop	rsi
+        pop	rdi
+        pop	r15
+        pop	r14
+        pop	r13
+        pop	r12
+        ret
+frodokem_mul_add_sb_plus_e_avx2 ENDP
+_TEXT ENDS
+_TEXT SEGMENT READONLY PARA
+frodokem_add_avx2 PROC
+        vmovd	xmm0, r8
+        vpbroadcastw	ymm0, xmm0
+        vmovdqu	ymm1, YMMWORD PTR [rcx]
+        vpaddw	ymm1, ymm1, [rdx]
+        vpand	ymm1, ymm1, ymm0
+        vmovdqu	YMMWORD PTR [rcx], ymm1
+        vmovdqu	ymm2, YMMWORD PTR [rcx+32]
+        vpaddw	ymm2, ymm2, [rdx+32]
+        vpand	ymm2, ymm2, ymm0
+        vmovdqu	YMMWORD PTR [rcx+32], ymm2
+        vmovdqu	ymm3, YMMWORD PTR [rcx+64]
+        vpaddw	ymm3, ymm3, [rdx+64]
+        vpand	ymm3, ymm3, ymm0
+        vmovdqu	YMMWORD PTR [rcx+64], ymm3
+        vmovdqu	ymm4, YMMWORD PTR [rcx+96]
+        vpaddw	ymm4, ymm4, [rdx+96]
+        vpand	ymm4, ymm4, ymm0
+        vmovdqu	YMMWORD PTR [rcx+96], ymm4
+        vzeroupper
+        ret
+frodokem_add_avx2 ENDP
+_TEXT ENDS
+_TEXT SEGMENT READONLY PARA
+frodokem_a_rows_reduce_avx2 PROC
+        vmovd	xmm0, r8
+        vpbroadcastw	ymm0, xmm0
+        mov	eax, edx
+L_frodokem_a_rows_reduce_avx2_blk:
+        cmp	rax, 16
+        jl	L_frodokem_a_rows_reduce_avx2_tail
+        vpand	ymm1, ymm0, YMMWORD PTR [rcx]
+        vmovdqu	YMMWORD PTR [rcx], ymm1
+        add	rcx, 32
+        sub	rax, 16
+        jmp	L_frodokem_a_rows_reduce_avx2_blk
+L_frodokem_a_rows_reduce_avx2_tail:
+        cmp	rax, 0
+        jle	L_frodokem_a_rows_reduce_avx2_done
+L_frodokem_a_rows_reduce_avx2_word:
+        movzx	r9d, WORD PTR [rcx]
+        and	r9d, r8d
+        mov	WORD PTR [rcx], r9w
+        add	rcx, 2
+        sub	rax, 1
+        jg	L_frodokem_a_rows_reduce_avx2_word
+L_frodokem_a_rows_reduce_avx2_done:
+        vzeroupper
+        ret
+frodokem_a_rows_reduce_avx2 ENDP
+_TEXT ENDS
+_TEXT SEGMENT READONLY PARA
+frodokem_sample_avx2 PROC
+        push	r12
+        sub	rsp, 32
+        vmovdqu	OWORD PTR [rsp], xmm6
+        vmovdqu	OWORD PTR [rsp+16], xmm7
+        movsxd	rax, edx
+        shl	rax, 1
+        vpcmpeqw	ymm0, ymm0, ymm0
+        vpsrlw	ymm0, ymm0, 15
+        vpxor	ymm1, ymm1, ymm1
+        xor	r10, r10
+L_frodokem_sample_avx2_blk:
+        vmovdqu	ymm2, YMMWORD PTR [rcx+r10]
+        vpsrlw	ymm3, ymm2, 1
+        vpand	ymm4, ymm2, ymm0
+        vpxor	ymm5, ymm5, ymm5
+        mov	r11, r8
+        mov	r12d, r9d
+L_frodokem_sample_avx2_cdf:
+        vpbroadcastw	ymm6, WORD PTR [r11]
+        vpsubw	ymm6, ymm6, ymm3
+        vpsrlw	ymm6, ymm6, 15
+        vpaddw	ymm5, ymm5, ymm6
+        add	r11, 2
+        sub	r12, 1
+        jnz	L_frodokem_sample_avx2_cdf
+        vpsubw	ymm7, ymm1, ymm4
+        vpxor	ymm5, ymm5, ymm7
+        vpaddw	ymm5, ymm5, ymm4
+        vmovdqu	YMMWORD PTR [rcx+r10], ymm5
+        add	r10, 32
+        cmp	r10, rax
+        jl	L_frodokem_sample_avx2_blk
+        vzeroupper
+        vmovdqu	xmm6, OWORD PTR [rsp]
+        vmovdqu	xmm7, OWORD PTR [rsp+16]
+        add	rsp, 32
+        pop	r12
+        ret
+frodokem_sample_avx2 ENDP
+_TEXT ENDS
+_TEXT SEGMENT READONLY PARA
+frodokem_gen_a_rows_aes_avx2 PROC
+        push	r12
+        push	r13
+        push	r14
+        push	r15
+        push	rdi
+        push	rsi
+        mov	rax, QWORD PTR [rsp+88]
+        mov	r10, QWORD PTR [rsp+96]
+        mov	r11, QWORD PTR [rsp+104]
+        sub	rsp, 160
+        vmovdqu	OWORD PTR [rsp], xmm6
+        vmovdqu	OWORD PTR [rsp+16], xmm7
+        vmovdqu	OWORD PTR [rsp+32], xmm8
+        vmovdqu	OWORD PTR [rsp+48], xmm9
+        vmovdqu	OWORD PTR [rsp+64], xmm10
+        vmovdqu	OWORD PTR [rsp+80], xmm11
+        vmovdqu	OWORD PTR [rsp+96], xmm12
+        vmovdqu	OWORD PTR [rsp+112], xmm13
+        vmovdqu	OWORD PTR [rsp+128], xmm14
+        vmovdqu	OWORD PTR [rsp+144], xmm15
+        vbroadcasti128	ymm0, OWORD PTR [r8]
+        vbroadcasti128	ymm1, OWORD PTR [r8+16]
+        vbroadcasti128	ymm2, OWORD PTR [r8+32]
+        vbroadcasti128	ymm3, OWORD PTR [r8+48]
+        vbroadcasti128	ymm4, OWORD PTR [r8+64]
+        vbroadcasti128	ymm5, OWORD PTR [r8+80]
+        vbroadcasti128	ymm6, OWORD PTR [r8+96]
+        vbroadcasti128	ymm7, OWORD PTR [r8+112]
+        vbroadcasti128	ymm8, OWORD PTR [r8+128]
+        vbroadcasti128	ymm9, OWORD PTR [r8+144]
+        vbroadcasti128	ymm10, OWORD PTR [r8+160]
+        vmovd	xmm15, r11
+        vpbroadcastw	ymm15, xmm15
+        movsxd	r12, r10d
+        shr	r12, 3
+        movsxd	r13, eax
+        movsxd	r9, r9d
+        xor	r14, r14
+L_frodokem_gen_a_rows_aes_avx2_row:
+        mov	rdi, r9
+        add	rdi, r14
+        xor	r15, r15
+L_frodokem_gen_a_rows_aes_avx2_blk:
+        lea	rsi, QWORD PTR [r15+8]
+        cmp	rsi, r12
+        jg	L_frodokem_gen_a_rows_aes_avx2_tail
+        vmovd	xmm11, edi
+        add	rdi, 524288
+        vmovd	xmm0, edi
+        add	rdi, 524288
+        vinserti128	ymm11, ymm11, xmm0, 1
+        vmovd	xmm12, edi
+        add	rdi, 524288
+        vmovd	xmm0, edi
+        add	rdi, 524288
+        vinserti128	ymm12, ymm12, xmm0, 1
+        vmovd	xmm13, edi
+        add	rdi, 524288
+        vmovd	xmm0, edi
+        add	rdi, 524288
+        vinserti128	ymm13, ymm13, xmm0, 1
+        vmovd	xmm14, edi
+        add	rdi, 524288
+        vmovd	xmm0, edi
+        add	rdi, 524288
+        vinserti128	ymm14, ymm14, xmm0, 1
+        vbroadcasti128	ymm0, OWORD PTR [r8]
+        vpxor	ymm11, ymm11, ymm0
+        vpxor	ymm12, ymm12, ymm0
+        vpxor	ymm13, ymm13, ymm0
+        vpxor	ymm14, ymm14, ymm0
+        vaesenc	ymm11, ymm11, ymm1
+        vaesenc	ymm12, ymm12, ymm1
+        vaesenc	ymm13, ymm13, ymm1
+        vaesenc	ymm14, ymm14, ymm1
+        vaesenc	ymm11, ymm11, ymm2
+        vaesenc	ymm12, ymm12, ymm2
+        vaesenc	ymm13, ymm13, ymm2
+        vaesenc	ymm14, ymm14, ymm2
+        vaesenc	ymm11, ymm11, ymm3
+        vaesenc	ymm12, ymm12, ymm3
+        vaesenc	ymm13, ymm13, ymm3
+        vaesenc	ymm14, ymm14, ymm3
+        vaesenc	ymm11, ymm11, ymm4
+        vaesenc	ymm12, ymm12, ymm4
+        vaesenc	ymm13, ymm13, ymm4
+        vaesenc	ymm14, ymm14, ymm4
+        vaesenc	ymm11, ymm11, ymm5
+        vaesenc	ymm12, ymm12, ymm5
+        vaesenc	ymm13, ymm13, ymm5
+        vaesenc	ymm14, ymm14, ymm5
+        vaesenc	ymm11, ymm11, ymm6
+        vaesenc	ymm12, ymm12, ymm6
+        vaesenc	ymm13, ymm13, ymm6
+        vaesenc	ymm14, ymm14, ymm6
+        vaesenc	ymm11, ymm11, ymm7
+        vaesenc	ymm12, ymm12, ymm7
+        vaesenc	ymm13, ymm13, ymm7
+        vaesenc	ymm14, ymm14, ymm7
+        vaesenc	ymm11, ymm11, ymm8
+        vaesenc	ymm12, ymm12, ymm8
+        vaesenc	ymm13, ymm13, ymm8
+        vaesenc	ymm14, ymm14, ymm8
+        vaesenc	ymm11, ymm11, ymm9
+        vaesenc	ymm12, ymm12, ymm9
+        vaesenc	ymm13, ymm13, ymm9
+        vaesenc	ymm14, ymm14, ymm9
+        vaesenclast	ymm11, ymm11, ymm10
+        vaesenclast	ymm12, ymm12, ymm10
+        vaesenclast	ymm13, ymm13, ymm10
+        vaesenclast	ymm14, ymm14, ymm10
+        vpand	ymm11, ymm11, ymm15
+        vmovdqu	YMMWORD PTR [rdx], ymm11
+        vpand	ymm12, ymm12, ymm15
+        vmovdqu	YMMWORD PTR [rdx+32], ymm12
+        vpand	ymm13, ymm13, ymm15
+        vmovdqu	YMMWORD PTR [rdx+64], ymm13
+        vpand	ymm14, ymm14, ymm15
+        vmovdqu	YMMWORD PTR [rdx+96], ymm14
+        add	rdx, 128
+        add	r15, 8
+        jmp	L_frodokem_gen_a_rows_aes_avx2_blk
+L_frodokem_gen_a_rows_aes_avx2_tail:
+        cmp	r15, r12
+        jge	L_frodokem_gen_a_rows_aes_avx2_next
+        vmovd	xmm11, edi
+        add	rdi, 524288
+        vpxor	xmm11, xmm11, xmm0
+        vaesenc	xmm11, xmm11, xmm1
+        vaesenc	xmm11, xmm11, xmm2
+        vaesenc	xmm11, xmm11, xmm3
+        vaesenc	xmm11, xmm11, xmm4
+        vaesenc	xmm11, xmm11, xmm5
+        vaesenc	xmm11, xmm11, xmm6
+        vaesenc	xmm11, xmm11, xmm7
+        vaesenc	xmm11, xmm11, xmm8
+        vaesenc	xmm11, xmm11, xmm9
+        vaesenclast	xmm11, xmm11, xmm10
+        vpand	xmm11, xmm11, xmm15
+        vmovdqu	OWORD PTR [rdx], xmm11
+        add	rdx, 16
+        add	r15, 1
+        jmp	L_frodokem_gen_a_rows_aes_avx2_tail
+L_frodokem_gen_a_rows_aes_avx2_next:
+        add	r14, 1
+        cmp	r14, r13
+        jl	L_frodokem_gen_a_rows_aes_avx2_row
+        vzeroupper
+        vmovdqu	xmm6, OWORD PTR [rsp]
+        vmovdqu	xmm7, OWORD PTR [rsp+16]
+        vmovdqu	xmm8, OWORD PTR [rsp+32]
+        vmovdqu	xmm9, OWORD PTR [rsp+48]
+        vmovdqu	xmm10, OWORD PTR [rsp+64]
+        vmovdqu	xmm11, OWORD PTR [rsp+80]
+        vmovdqu	xmm12, OWORD PTR [rsp+96]
+        vmovdqu	xmm13, OWORD PTR [rsp+112]
+        vmovdqu	xmm14, OWORD PTR [rsp+128]
+        vmovdqu	xmm15, OWORD PTR [rsp+144]
+        add	rsp, 160
+        pop	rsi
+        pop	rdi
+        pop	r15
+        pop	r14
+        pop	r13
+        pop	r12
+        ret
+frodokem_gen_a_rows_aes_avx2 ENDP
+_TEXT ENDS
+_TEXT SEGMENT READONLY PARA
+frodokem_gen_a_rows_aes_aesni PROC
+        push	r12
+        push	r13
+        push	r14
+        push	r15
+        push	rdi
+        push	rsi
+        mov	rax, QWORD PTR [rsp+88]
+        mov	r10, QWORD PTR [rsp+96]
+        mov	r11, QWORD PTR [rsp+104]
+        sub	rsp, 48
+        vmovdqu	OWORD PTR [rsp], xmm6
+        vmovdqu	OWORD PTR [rsp+16], xmm7
+        vmovdqu	OWORD PTR [rsp+32], xmm8
+        vmovd	xmm8, r11
+        vpbroadcastw	xmm8, xmm8
+        movsxd	r12, r10d
+        shr	r12, 3
+        movsxd	r13, eax
+        movsxd	r9, r9d
+        xor	r14, r14
+L_frodokem_gen_a_rows_aes_aesni_row:
+        mov	rdi, r9
+        add	rdi, r14
+        xor	r15, r15
+L_frodokem_gen_a_rows_aes_aesni_blk:
+        lea	rsi, QWORD PTR [r15+8]
+        cmp	rsi, r12
+        jg	L_frodokem_gen_a_rows_aes_aesni_tail
+        vmovd	xmm0, edi
+        add	rdi, 524288
+        vmovd	xmm1, edi
+        add	rdi, 524288
+        vmovd	xmm2, edi
+        add	rdi, 524288
+        vmovd	xmm3, edi
+        add	rdi, 524288
+        vmovd	xmm4, edi
+        add	rdi, 524288
+        vmovd	xmm5, edi
+        add	rdi, 524288
+        vmovd	xmm6, edi
+        add	rdi, 524288
+        vmovd	xmm7, edi
+        add	rdi, 524288
+        vpxor	xmm0, xmm0, [r8]
+        vpxor	xmm1, xmm1, [r8]
+        vpxor	xmm2, xmm2, [r8]
+        vpxor	xmm3, xmm3, [r8]
+        vpxor	xmm4, xmm4, [r8]
+        vpxor	xmm5, xmm5, [r8]
+        vpxor	xmm6, xmm6, [r8]
+        vpxor	xmm7, xmm7, [r8]
+        vaesenc	xmm0, xmm0, [r8+16]
+        vaesenc	xmm1, xmm1, [r8+16]
+        vaesenc	xmm2, xmm2, [r8+16]
+        vaesenc	xmm3, xmm3, [r8+16]
+        vaesenc	xmm4, xmm4, [r8+16]
+        vaesenc	xmm5, xmm5, [r8+16]
+        vaesenc	xmm6, xmm6, [r8+16]
+        vaesenc	xmm7, xmm7, [r8+16]
+        vaesenc	xmm0, xmm0, [r8+32]
+        vaesenc	xmm1, xmm1, [r8+32]
+        vaesenc	xmm2, xmm2, [r8+32]
+        vaesenc	xmm3, xmm3, [r8+32]
+        vaesenc	xmm4, xmm4, [r8+32]
+        vaesenc	xmm5, xmm5, [r8+32]
+        vaesenc	xmm6, xmm6, [r8+32]
+        vaesenc	xmm7, xmm7, [r8+32]
+        vaesenc	xmm0, xmm0, [r8+48]
+        vaesenc	xmm1, xmm1, [r8+48]
+        vaesenc	xmm2, xmm2, [r8+48]
+        vaesenc	xmm3, xmm3, [r8+48]
+        vaesenc	xmm4, xmm4, [r8+48]
+        vaesenc	xmm5, xmm5, [r8+48]
+        vaesenc	xmm6, xmm6, [r8+48]
+        vaesenc	xmm7, xmm7, [r8+48]
+        vaesenc	xmm0, xmm0, [r8+64]
+        vaesenc	xmm1, xmm1, [r8+64]
+        vaesenc	xmm2, xmm2, [r8+64]
+        vaesenc	xmm3, xmm3, [r8+64]
+        vaesenc	xmm4, xmm4, [r8+64]
+        vaesenc	xmm5, xmm5, [r8+64]
+        vaesenc	xmm6, xmm6, [r8+64]
+        vaesenc	xmm7, xmm7, [r8+64]
+        vaesenc	xmm0, xmm0, [r8+80]
+        vaesenc	xmm1, xmm1, [r8+80]
+        vaesenc	xmm2, xmm2, [r8+80]
+        vaesenc	xmm3, xmm3, [r8+80]
+        vaesenc	xmm4, xmm4, [r8+80]
+        vaesenc	xmm5, xmm5, [r8+80]
+        vaesenc	xmm6, xmm6, [r8+80]
+        vaesenc	xmm7, xmm7, [r8+80]
+        vaesenc	xmm0, xmm0, [r8+96]
+        vaesenc	xmm1, xmm1, [r8+96]
+        vaesenc	xmm2, xmm2, [r8+96]
+        vaesenc	xmm3, xmm3, [r8+96]
+        vaesenc	xmm4, xmm4, [r8+96]
+        vaesenc	xmm5, xmm5, [r8+96]
+        vaesenc	xmm6, xmm6, [r8+96]
+        vaesenc	xmm7, xmm7, [r8+96]
+        vaesenc	xmm0, xmm0, [r8+112]
+        vaesenc	xmm1, xmm1, [r8+112]
+        vaesenc	xmm2, xmm2, [r8+112]
+        vaesenc	xmm3, xmm3, [r8+112]
+        vaesenc	xmm4, xmm4, [r8+112]
+        vaesenc	xmm5, xmm5, [r8+112]
+        vaesenc	xmm6, xmm6, [r8+112]
+        vaesenc	xmm7, xmm7, [r8+112]
+        vaesenc	xmm0, xmm0, [r8+128]
+        vaesenc	xmm1, xmm1, [r8+128]
+        vaesenc	xmm2, xmm2, [r8+128]
+        vaesenc	xmm3, xmm3, [r8+128]
+        vaesenc	xmm4, xmm4, [r8+128]
+        vaesenc	xmm5, xmm5, [r8+128]
+        vaesenc	xmm6, xmm6, [r8+128]
+        vaesenc	xmm7, xmm7, [r8+128]
+        vaesenc	xmm0, xmm0, [r8+144]
+        vaesenc	xmm1, xmm1, [r8+144]
+        vaesenc	xmm2, xmm2, [r8+144]
+        vaesenc	xmm3, xmm3, [r8+144]
+        vaesenc	xmm4, xmm4, [r8+144]
+        vaesenc	xmm5, xmm5, [r8+144]
+        vaesenc	xmm6, xmm6, [r8+144]
+        vaesenc	xmm7, xmm7, [r8+144]
+        vaesenclast	xmm0, xmm0, [r8+160]
+        vaesenclast	xmm1, xmm1, [r8+160]
+        vaesenclast	xmm2, xmm2, [r8+160]
+        vaesenclast	xmm3, xmm3, [r8+160]
+        vaesenclast	xmm4, xmm4, [r8+160]
+        vaesenclast	xmm5, xmm5, [r8+160]
+        vaesenclast	xmm6, xmm6, [r8+160]
+        vaesenclast	xmm7, xmm7, [r8+160]
+        vpand	xmm0, xmm0, xmm8
+        vmovdqu	OWORD PTR [rdx], xmm0
+        vpand	xmm1, xmm1, xmm8
+        vmovdqu	OWORD PTR [rdx+16], xmm1
+        vpand	xmm2, xmm2, xmm8
+        vmovdqu	OWORD PTR [rdx+32], xmm2
+        vpand	xmm3, xmm3, xmm8
+        vmovdqu	OWORD PTR [rdx+48], xmm3
+        vpand	xmm4, xmm4, xmm8
+        vmovdqu	OWORD PTR [rdx+64], xmm4
+        vpand	xmm5, xmm5, xmm8
+        vmovdqu	OWORD PTR [rdx+80], xmm5
+        vpand	xmm6, xmm6, xmm8
+        vmovdqu	OWORD PTR [rdx+96], xmm6
+        vpand	xmm7, xmm7, xmm8
+        vmovdqu	OWORD PTR [rdx+112], xmm7
+        add	rdx, 128
+        add	r15, 8
+        jmp	L_frodokem_gen_a_rows_aes_aesni_blk
+L_frodokem_gen_a_rows_aes_aesni_tail:
+        cmp	r15, r12
+        jge	L_frodokem_gen_a_rows_aes_aesni_next
+        vmovd	xmm0, edi
+        add	rdi, 524288
+        vpxor	xmm0, xmm0, [r8]
+        vaesenc	xmm0, xmm0, [r8+16]
+        vaesenc	xmm0, xmm0, [r8+32]
+        vaesenc	xmm0, xmm0, [r8+48]
+        vaesenc	xmm0, xmm0, [r8+64]
+        vaesenc	xmm0, xmm0, [r8+80]
+        vaesenc	xmm0, xmm0, [r8+96]
+        vaesenc	xmm0, xmm0, [r8+112]
+        vaesenc	xmm0, xmm0, [r8+128]
+        vaesenc	xmm0, xmm0, [r8+144]
+        vaesenclast	xmm0, xmm0, [r8+160]
+        vpand	xmm0, xmm0, xmm8
+        vmovdqu	OWORD PTR [rdx], xmm0
+        add	rdx, 16
+        add	r15, 1
+        jmp	L_frodokem_gen_a_rows_aes_aesni_tail
+L_frodokem_gen_a_rows_aes_aesni_next:
+        add	r14, 1
+        cmp	r14, r13
+        jl	L_frodokem_gen_a_rows_aes_aesni_row
+        vzeroupper
+        vmovdqu	xmm6, OWORD PTR [rsp]
+        vmovdqu	xmm7, OWORD PTR [rsp+16]
+        vmovdqu	xmm8, OWORD PTR [rsp+32]
+        add	rsp, 48
+        pop	rsi
+        pop	rdi
+        pop	r15
+        pop	r14
+        pop	r13
+        pop	r12
+        ret
+frodokem_gen_a_rows_aes_aesni ENDP
+_TEXT ENDS
+ENDIF
+ENDIF
+IFDEF WOLFSSL_HAVE_FRODOKEM
+IFDEF HAVE_INTEL_AVX512
+_TEXT SEGMENT READONLY PARA
+frodokem_sa_accum_avx512 PROC
+        push	r12
+        push	r13
+        push	r14
+        push	r15
+        push	rdi
+        push	rsi
+        push	rbx
+        push	rbp
+        mov	rax, QWORD PTR [rsp+104]
+        sub	rsp, 64
+        vmovdqu	OWORD PTR [rsp], xmm6
+        vmovdqu	OWORD PTR [rsp+16], xmm7
+        vmovdqu	OWORD PTR [rsp+32], xmm8
+        vmovdqu	OWORD PTR [rsp+48], xmm9
+        movsxd	r9, r9d
+        shl	r9, 1
+        add	rdx, r9
+        movsxd	r10, eax
+        shl	r10, 1
+        mov	r9, r10
+        and	r9, -64
+        mov	r11, r8
+        add	r11, r10
+        mov	r12, r11
+        add	r12, r10
+        mov	r13, r12
+        add	r13, r10
+        mov	r14, r13
+        add	r14, r10
+        mov	r15, r14
+        add	r15, r10
+        mov	rdi, r15
+        add	rdi, r10
+        mov	rsi, rdi
+        add	rsi, r10
+        mov	rbp, 8
+L_frodokem_sa_accum_avx512_i:
+        vpbroadcastw	zmm0, WORD PTR [rdx]
+        vpbroadcastw	zmm1, WORD PTR [rdx+2]
+        vpbroadcastw	zmm2, WORD PTR [rdx+4]
+        vpbroadcastw	zmm3, WORD PTR [rdx+6]
+        vpbroadcastw	zmm4, WORD PTR [rdx+8]
+        vpbroadcastw	zmm5, WORD PTR [rdx+10]
+        vpbroadcastw	zmm6, WORD PTR [rdx+12]
+        vpbroadcastw	zmm7, WORD PTR [rdx+14]
+        xor	rbx, rbx
+L_frodokem_sa_accum_avx512_k:
+        vmovdqu64	zmm8, [rcx+rbx]
+        vpmullw	zmm9, zmm0, [r8+rbx]
+        vpaddw	zmm8, zmm8, zmm9
+        vpmullw	zmm9, zmm1, [r11+rbx]
+        vpaddw	zmm8, zmm8, zmm9
+        vpmullw	zmm9, zmm2, [r12+rbx]
+        vpaddw	zmm8, zmm8, zmm9
+        vpmullw	zmm9, zmm3, [r13+rbx]
+        vpaddw	zmm8, zmm8, zmm9
+        vpmullw	zmm9, zmm4, [r14+rbx]
+        vpaddw	zmm8, zmm8, zmm9
+        vpmullw	zmm9, zmm5, [r15+rbx]
+        vpaddw	zmm8, zmm8, zmm9
+        vpmullw	zmm9, zmm6, [rdi+rbx]
+        vpaddw	zmm8, zmm8, zmm9
+        vpmullw	zmm9, zmm7, [rsi+rbx]
+        vpaddw	zmm8, zmm8, zmm9
+        vmovdqu64	[rcx+rbx], zmm8
+        add	rbx, 64
+        cmp	rbx, r9
+        jl	L_frodokem_sa_accum_avx512_k
+        cmp	rbx, r10
+        jge	L_frodokem_sa_accum_avx512_tail
+        vmovdqu	ymm8, YMMWORD PTR [rcx+rbx]
+        vpmullw	ymm9, ymm0, [r8+rbx]
+        vpaddw	ymm8, ymm8, ymm9
+        vpmullw	ymm9, ymm1, [r11+rbx]
+        vpaddw	ymm8, ymm8, ymm9
+        vpmullw	ymm9, ymm2, [r12+rbx]
+        vpaddw	ymm8, ymm8, ymm9
+        vpmullw	ymm9, ymm3, [r13+rbx]
+        vpaddw	ymm8, ymm8, ymm9
+        vpmullw	ymm9, ymm4, [r14+rbx]
+        vpaddw	ymm8, ymm8, ymm9
+        vpmullw	ymm9, ymm5, [r15+rbx]
+        vpaddw	ymm8, ymm8, ymm9
+        vpmullw	ymm9, ymm6, [rdi+rbx]
+        vpaddw	ymm8, ymm8, ymm9
+        vpmullw	ymm9, ymm7, [rsi+rbx]
+        vpaddw	ymm8, ymm8, ymm9
+        vmovdqu	YMMWORD PTR [rcx+rbx], ymm8
+L_frodokem_sa_accum_avx512_tail:
+        add	rcx, r10
+        add	rdx, r10
+        sub	rbp, 1
+        jnz	L_frodokem_sa_accum_avx512_i
+        vzeroupper
+        vmovdqu	xmm6, OWORD PTR [rsp]
+        vmovdqu	xmm7, OWORD PTR [rsp+16]
+        vmovdqu	xmm8, OWORD PTR [rsp+32]
+        vmovdqu	xmm9, OWORD PTR [rsp+48]
+        add	rsp, 64
+        pop	rbp
+        pop	rbx
+        pop	rsi
+        pop	rdi
+        pop	r15
+        pop	r14
+        pop	r13
+        pop	r12
+        ret
+frodokem_sa_accum_avx512 ENDP
+_TEXT ENDS
+_TEXT SEGMENT READONLY PARA
+frodokem_as_accum_avx512 PROC
+        push	r12
+        push	r13
+        push	r14
+        push	r15
+        push	rdi
+        push	rsi
+        push	rbx
+        push	rbp
+        mov	rax, QWORD PTR [rsp+104]
+        sub	rsp, 160
+        vmovdqu	OWORD PTR [rsp], xmm6
+        vmovdqu	OWORD PTR [rsp+16], xmm7
+        vmovdqu	OWORD PTR [rsp+32], xmm8
+        vmovdqu	OWORD PTR [rsp+48], xmm9
+        vmovdqu	OWORD PTR [rsp+64], xmm10
+        vmovdqu	OWORD PTR [rsp+80], xmm11
+        vmovdqu	OWORD PTR [rsp+96], xmm12
+        vmovdqu	OWORD PTR [rsp+112], xmm13
+        vmovdqu	OWORD PTR [rsp+128], xmm14
+        vmovdqu	OWORD PTR [rsp+144], xmm15
+        movsxd	r9, r9d
+        shl	r9, 4
+        add	rcx, r9
+        movsxd	r10, eax
+        shl	r10, 1
+        mov	r9, r10
+        and	r9, -64
+        mov	r11, rdx
+        add	r11, r10
+        mov	r12, r11
+        add	r12, r10
+        mov	r13, r12
+        add	r13, r10
+        mov	r14, r13
+        add	r14, r10
+        mov	r15, r14
+        add	r15, r10
+        mov	rdi, r15
+        add	rdi, r10
+        mov	rsi, rdi
+        add	rsi, r10
+        mov	rbp, 8
+        vpcmpeqd	xmm13, xmm13, xmm13
+        vpsrld	xmm13, xmm13, 16
+L_frodokem_as_accum_avx512_r:
+        vpxorq	zmm14, zmm14, zmm14
+        vpxorq	ymm22, ymm22, ymm22
+        vpxorq	zmm15, zmm15, zmm15
+        vpxorq	ymm23, ymm23, ymm23
+        vpxorq	zmm16, zmm16, zmm16
+        vpxorq	ymm24, ymm24, ymm24
+        vpxorq	zmm17, zmm17, zmm17
+        vpxorq	ymm25, ymm25, ymm25
+        vpxorq	zmm18, zmm18, zmm18
+        vpxorq	ymm26, ymm26, ymm26
+        vpxorq	zmm19, zmm19, zmm19
+        vpxorq	ymm27, ymm27, ymm27
+        vpxorq	zmm20, zmm20, zmm20
+        vpxorq	ymm28, ymm28, ymm28
+        vpxorq	zmm21, zmm21, zmm21
+        vpxorq	ymm29, ymm29, ymm29
+        xor	rbx, rbx
+L_frodokem_as_accum_avx512_j:
+        vmovdqu64	zmm0, [r8+rbx]
+        vpmaddwd	zmm1, zmm0, [rdx+rbx]
+        vpaddd	zmm14, zmm14, zmm1
+        vpmaddwd	zmm1, zmm0, [r11+rbx]
+        vpaddd	zmm15, zmm15, zmm1
+        vpmaddwd	zmm1, zmm0, [r12+rbx]
+        vpaddd	zmm16, zmm16, zmm1
+        vpmaddwd	zmm1, zmm0, [r13+rbx]
+        vpaddd	zmm17, zmm17, zmm1
+        vpmaddwd	zmm1, zmm0, [r14+rbx]
+        vpaddd	zmm18, zmm18, zmm1
+        vpmaddwd	zmm1, zmm0, [r15+rbx]
+        vpaddd	zmm19, zmm19, zmm1
+        vpmaddwd	zmm1, zmm0, [rdi+rbx]
+        vpaddd	zmm20, zmm20, zmm1
+        vpmaddwd	zmm1, zmm0, [rsi+rbx]
+        vpaddd	zmm21, zmm21, zmm1
+        add	rbx, 64
+        cmp	rbx, r9
+        jl	L_frodokem_as_accum_avx512_j
+        cmp	rbx, r10
+        jge	L_frodokem_as_accum_avx512_tail
+        vmovdqu	ymm0, YMMWORD PTR [r8+rbx]
+        vpmaddwd	ymm22, ymm0, [rdx+rbx]
+        vpmaddwd	ymm23, ymm0, [r11+rbx]
+        vpmaddwd	ymm24, ymm0, [r12+rbx]
+        vpmaddwd	ymm25, ymm0, [r13+rbx]
+        vpmaddwd	ymm26, ymm0, [r14+rbx]
+        vpmaddwd	ymm27, ymm0, [r15+rbx]
+        vpmaddwd	ymm28, ymm0, [rdi+rbx]
+        vpmaddwd	ymm29, ymm0, [rsi+rbx]
+L_frodokem_as_accum_avx512_tail:
+        vextracti64x4	ymm1, zmm14, 1
+        vpaddd	ymm1, ymm14, ymm1
+        vpaddd	ymm1, ymm1, ymm22
+        vextracti128	xmm0, ymm1, 1
+        vpaddd	xmm2, xmm1, xmm0
+        vextracti64x4	ymm1, zmm15, 1
+        vpaddd	ymm1, ymm15, ymm1
+        vpaddd	ymm1, ymm1, ymm23
+        vextracti128	xmm0, ymm1, 1
+        vpaddd	xmm3, xmm1, xmm0
+        vextracti64x4	ymm1, zmm16, 1
+        vpaddd	ymm1, ymm16, ymm1
+        vpaddd	ymm1, ymm1, ymm24
+        vextracti128	xmm0, ymm1, 1
+        vpaddd	xmm4, xmm1, xmm0
+        vextracti64x4	ymm1, zmm17, 1
+        vpaddd	ymm1, ymm17, ymm1
+        vpaddd	ymm1, ymm1, ymm25
+        vextracti128	xmm0, ymm1, 1
+        vpaddd	xmm5, xmm1, xmm0
+        vextracti64x4	ymm1, zmm18, 1
+        vpaddd	ymm1, ymm18, ymm1
+        vpaddd	ymm1, ymm1, ymm26
+        vextracti128	xmm0, ymm1, 1
+        vpaddd	xmm6, xmm1, xmm0
+        vextracti64x4	ymm1, zmm19, 1
+        vpaddd	ymm1, ymm19, ymm1
+        vpaddd	ymm1, ymm1, ymm27
+        vextracti128	xmm0, ymm1, 1
+        vpaddd	xmm7, xmm1, xmm0
+        vextracti64x4	ymm1, zmm20, 1
+        vpaddd	ymm1, ymm20, ymm1
+        vpaddd	ymm1, ymm1, ymm28
+        vextracti128	xmm0, ymm1, 1
+        vpaddd	xmm8, xmm1, xmm0
+        vextracti64x4	ymm1, zmm21, 1
+        vpaddd	ymm1, ymm21, ymm1
+        vpaddd	ymm1, ymm1, ymm29
+        vextracti128	xmm0, ymm1, 1
+        vpaddd	xmm9, xmm1, xmm0
+        vphaddd	xmm11, xmm2, xmm3
+        vphaddd	xmm12, xmm4, xmm5
+        vphaddd	xmm10, xmm11, xmm12
+        vphaddd	xmm11, xmm6, xmm7
+        vphaddd	xmm12, xmm8, xmm9
+        vphaddd	xmm11, xmm11, xmm12
+        vpand	xmm10, xmm10, xmm13
+        vpand	xmm11, xmm11, xmm13
+        vpackusdw	xmm10, xmm10, xmm11
+        vpaddw	xmm10, xmm10, [rcx]
+        vmovdqu	OWORD PTR [rcx], xmm10
+        add	r8, r10
+        add	rcx, 16
+        sub	rbp, 1
+        jnz	L_frodokem_as_accum_avx512_r
+        vzeroupper
+        vmovdqu	xmm6, OWORD PTR [rsp]
+        vmovdqu	xmm7, OWORD PTR [rsp+16]
+        vmovdqu	xmm8, OWORD PTR [rsp+32]
+        vmovdqu	xmm9, OWORD PTR [rsp+48]
+        vmovdqu	xmm10, OWORD PTR [rsp+64]
+        vmovdqu	xmm11, OWORD PTR [rsp+80]
+        vmovdqu	xmm12, OWORD PTR [rsp+96]
+        vmovdqu	xmm13, OWORD PTR [rsp+112]
+        vmovdqu	xmm14, OWORD PTR [rsp+128]
+        vmovdqu	xmm15, OWORD PTR [rsp+144]
+        add	rsp, 160
+        pop	rbp
+        pop	rbx
+        pop	rsi
+        pop	rdi
+        pop	r15
+        pop	r14
+        pop	r13
+        pop	r12
+        ret
+frodokem_as_accum_avx512 ENDP
+_TEXT ENDS
+_TEXT SEGMENT READONLY PARA
+frodokem_mul_bs_avx512 PROC
+        push	r12
+        push	r13
+        push	r14
+        push	r15
+        push	rdi
+        push	rsi
+        push	rbx
+        push	rbp
+        mov	rax, QWORD PTR [rsp+104]
+        sub	rsp, 160
+        vmovdqu	OWORD PTR [rsp], xmm6
+        vmovdqu	OWORD PTR [rsp+16], xmm7
+        vmovdqu	OWORD PTR [rsp+32], xmm8
+        vmovdqu	OWORD PTR [rsp+48], xmm9
+        vmovdqu	OWORD PTR [rsp+64], xmm10
+        vmovdqu	OWORD PTR [rsp+80], xmm11
+        vmovdqu	OWORD PTR [rsp+96], xmm12
+        vmovdqu	OWORD PTR [rsp+112], xmm13
+        vmovdqu	OWORD PTR [rsp+128], xmm14
+        vmovdqu	OWORD PTR [rsp+144], xmm15
+        movsxd	r10, r9d
+        shl	r10, 1
+        mov	r9, r10
+        and	r9, -64
+        mov	r11, r8
+        add	r11, r10
+        mov	r12, r11
+        add	r12, r10
+        mov	r13, r12
+        add	r13, r10
+        mov	r14, r13
+        add	r14, r10
+        mov	r15, r14
+        add	r15, r10
+        mov	rdi, r15
+        add	rdi, r10
+        mov	rsi, rdi
+        add	rsi, r10
+        mov	rbp, 8
+        vmovd	xmm13, rax
+        vpbroadcastd	xmm13, xmm13
+L_frodokem_mul_bs_avx512_r:
+        vpxorq	zmm14, zmm14, zmm14
+        vpxorq	ymm22, ymm22, ymm22
+        vpxorq	zmm15, zmm15, zmm15
+        vpxorq	ymm23, ymm23, ymm23
+        vpxorq	zmm16, zmm16, zmm16
+        vpxorq	ymm24, ymm24, ymm24
+        vpxorq	zmm17, zmm17, zmm17
+        vpxorq	ymm25, ymm25, ymm25
+        vpxorq	zmm18, zmm18, zmm18
+        vpxorq	ymm26, ymm26, ymm26
+        vpxorq	zmm19, zmm19, zmm19
+        vpxorq	ymm27, ymm27, ymm27
+        vpxorq	zmm20, zmm20, zmm20
+        vpxorq	ymm28, ymm28, ymm28
+        vpxorq	zmm21, zmm21, zmm21
+        vpxorq	ymm29, ymm29, ymm29
+        xor	rbx, rbx
+L_frodokem_mul_bs_avx512_j:
+        vmovdqu64	zmm0, [rdx+rbx]
+        vpmaddwd	zmm1, zmm0, [r8+rbx]
+        vpaddd	zmm14, zmm14, zmm1
+        vpmaddwd	zmm1, zmm0, [r11+rbx]
+        vpaddd	zmm15, zmm15, zmm1
+        vpmaddwd	zmm1, zmm0, [r12+rbx]
+        vpaddd	zmm16, zmm16, zmm1
+        vpmaddwd	zmm1, zmm0, [r13+rbx]
+        vpaddd	zmm17, zmm17, zmm1
+        vpmaddwd	zmm1, zmm0, [r14+rbx]
+        vpaddd	zmm18, zmm18, zmm1
+        vpmaddwd	zmm1, zmm0, [r15+rbx]
+        vpaddd	zmm19, zmm19, zmm1
+        vpmaddwd	zmm1, zmm0, [rdi+rbx]
+        vpaddd	zmm20, zmm20, zmm1
+        vpmaddwd	zmm1, zmm0, [rsi+rbx]
+        vpaddd	zmm21, zmm21, zmm1
+        add	rbx, 64
+        cmp	rbx, r9
+        jl	L_frodokem_mul_bs_avx512_j
+        cmp	rbx, r10
+        jge	L_frodokem_mul_bs_avx512_tail
+        vmovdqu	ymm0, YMMWORD PTR [rdx+rbx]
+        vpmaddwd	ymm22, ymm0, [r8+rbx]
+        vpmaddwd	ymm23, ymm0, [r11+rbx]
+        vpmaddwd	ymm24, ymm0, [r12+rbx]
+        vpmaddwd	ymm25, ymm0, [r13+rbx]
+        vpmaddwd	ymm26, ymm0, [r14+rbx]
+        vpmaddwd	ymm27, ymm0, [r15+rbx]
+        vpmaddwd	ymm28, ymm0, [rdi+rbx]
+        vpmaddwd	ymm29, ymm0, [rsi+rbx]
+L_frodokem_mul_bs_avx512_tail:
+        vextracti64x4	ymm1, zmm14, 1
+        vpaddd	ymm1, ymm14, ymm1
+        vpaddd	ymm1, ymm1, ymm22
+        vextracti128	xmm0, ymm1, 1
+        vpaddd	xmm2, xmm1, xmm0
+        vextracti64x4	ymm1, zmm15, 1
+        vpaddd	ymm1, ymm15, ymm1
+        vpaddd	ymm1, ymm1, ymm23
+        vextracti128	xmm0, ymm1, 1
+        vpaddd	xmm3, xmm1, xmm0
+        vextracti64x4	ymm1, zmm16, 1
+        vpaddd	ymm1, ymm16, ymm1
+        vpaddd	ymm1, ymm1, ymm24
+        vextracti128	xmm0, ymm1, 1
+        vpaddd	xmm4, xmm1, xmm0
+        vextracti64x4	ymm1, zmm17, 1
+        vpaddd	ymm1, ymm17, ymm1
+        vpaddd	ymm1, ymm1, ymm25
+        vextracti128	xmm0, ymm1, 1
+        vpaddd	xmm5, xmm1, xmm0
+        vextracti64x4	ymm1, zmm18, 1
+        vpaddd	ymm1, ymm18, ymm1
+        vpaddd	ymm1, ymm1, ymm26
+        vextracti128	xmm0, ymm1, 1
+        vpaddd	xmm6, xmm1, xmm0
+        vextracti64x4	ymm1, zmm19, 1
+        vpaddd	ymm1, ymm19, ymm1
+        vpaddd	ymm1, ymm1, ymm27
+        vextracti128	xmm0, ymm1, 1
+        vpaddd	xmm7, xmm1, xmm0
+        vextracti64x4	ymm1, zmm20, 1
+        vpaddd	ymm1, ymm20, ymm1
+        vpaddd	ymm1, ymm1, ymm28
+        vextracti128	xmm0, ymm1, 1
+        vpaddd	xmm8, xmm1, xmm0
+        vextracti64x4	ymm1, zmm21, 1
+        vpaddd	ymm1, ymm21, ymm1
+        vpaddd	ymm1, ymm1, ymm29
+        vextracti128	xmm0, ymm1, 1
+        vpaddd	xmm9, xmm1, xmm0
+        vphaddd	xmm11, xmm2, xmm3
+        vphaddd	xmm12, xmm4, xmm5
+        vphaddd	xmm10, xmm11, xmm12
+        vphaddd	xmm11, xmm6, xmm7
+        vphaddd	xmm12, xmm8, xmm9
+        vphaddd	xmm11, xmm11, xmm12
+        vpand	xmm10, xmm10, xmm13
+        vpand	xmm11, xmm11, xmm13
+        vpackusdw	xmm10, xmm10, xmm11
+        vmovdqu	OWORD PTR [rcx], xmm10
+        add	rdx, r10
+        add	rcx, 16
+        sub	rbp, 1
+        jnz	L_frodokem_mul_bs_avx512_r
+        vzeroupper
+        vmovdqu	xmm6, OWORD PTR [rsp]
+        vmovdqu	xmm7, OWORD PTR [rsp+16]
+        vmovdqu	xmm8, OWORD PTR [rsp+32]
+        vmovdqu	xmm9, OWORD PTR [rsp+48]
+        vmovdqu	xmm10, OWORD PTR [rsp+64]
+        vmovdqu	xmm11, OWORD PTR [rsp+80]
+        vmovdqu	xmm12, OWORD PTR [rsp+96]
+        vmovdqu	xmm13, OWORD PTR [rsp+112]
+        vmovdqu	xmm14, OWORD PTR [rsp+128]
+        vmovdqu	xmm15, OWORD PTR [rsp+144]
+        add	rsp, 160
+        pop	rbp
+        pop	rbx
+        pop	rsi
+        pop	rdi
+        pop	r15
+        pop	r14
+        pop	r13
+        pop	r12
+        ret
+frodokem_mul_bs_avx512 ENDP
+_TEXT ENDS
+_TEXT SEGMENT READONLY PARA
+frodokem_mul_add_sb_plus_e_avx512 PROC
+        push	r12
+        push	r13
+        push	r14
+        push	r15
+        push	rdi
+        push	rsi
+        push	rbx
+        mov	rax, QWORD PTR [rsp+96]
+        movsxd	r10, r9d
+        shl	r10, 1
+        mov	r11, r8
+        add	r11, r10
+        mov	r12, r11
+        add	r12, r10
+        mov	r13, r12
+        add	r13, r10
+        mov	r14, r13
+        add	r14, r10
+        mov	r15, r14
+        add	r15, r10
+        mov	rdi, r15
+        add	rdi, r10
+        mov	rsi, rdi
+        add	rsi, r10
+        vmovd	xmm5, rax
+        vpbroadcastw	zmm5, xmm5
+        vmovdqu64	zmm0, [rcx]
+        vmovdqu64	zmm1, [rcx+64]
+        xor	rbx, rbx
+L_frodokem_mul_add_sb_plus_e_avx512_j:
+        vbroadcasti32x4	zmm2, OWORD PTR [rdx]
+        vpbroadcastw	zmm3, WORD PTR [r8+rbx]
+        vpbroadcastw	zmm4, WORD PTR [r11+rbx]
+        vinserti32x4	zmm3, zmm3, xmm4, 1
+        vpbroadcastw	zmm4, WORD PTR [r12+rbx]
+        vinserti32x4	zmm3, zmm3, xmm4, 2
+        vpbroadcastw	zmm4, WORD PTR [r13+rbx]
+        vinserti32x4	zmm3, zmm3, xmm4, 3
+        vpmullw	zmm4, zmm3, zmm2
+        vpaddw	zmm0, zmm0, zmm4
+        vpbroadcastw	zmm3, WORD PTR [r14+rbx]
+        vpbroadcastw	zmm4, WORD PTR [r15+rbx]
+        vinserti32x4	zmm3, zmm3, xmm4, 1
+        vpbroadcastw	zmm4, WORD PTR [rdi+rbx]
+        vinserti32x4	zmm3, zmm3, xmm4, 2
+        vpbroadcastw	zmm4, WORD PTR [rsi+rbx]
+        vinserti32x4	zmm3, zmm3, xmm4, 3
+        vpmullw	zmm4, zmm3, zmm2
+        vpaddw	zmm1, zmm1, zmm4
+        add	rdx, 16
+        add	rbx, 2
+        cmp	rbx, r10
+        jl	L_frodokem_mul_add_sb_plus_e_avx512_j
+        vpandq	zmm0, zmm0, zmm5
+        vmovdqu64	[rcx], zmm0
+        vpandq	zmm1, zmm1, zmm5
+        vmovdqu64	[rcx+64], zmm1
+        vzeroupper
+        pop	rbx
+        pop	rsi
+        pop	rdi
+        pop	r15
+        pop	r14
+        pop	r13
+        pop	r12
+        ret
+frodokem_mul_add_sb_plus_e_avx512 ENDP
+_TEXT ENDS
+_TEXT SEGMENT READONLY PARA
+frodokem_add_avx512 PROC
+        vmovd	xmm0, r8
+        vpbroadcastw	zmm0, xmm0
+        vmovdqu64	zmm1, [rcx]
+        vpaddw	zmm1, zmm1, [rdx]
+        vpandq	zmm1, zmm1, zmm0
+        vmovdqu64	[rcx], zmm1
+        vmovdqu64	zmm2, [rcx+64]
+        vpaddw	zmm2, zmm2, [rdx+64]
+        vpandq	zmm2, zmm2, zmm0
+        vmovdqu64	[rcx+64], zmm2
+        vzeroupper
+        ret
+frodokem_add_avx512 ENDP
+_TEXT ENDS
+_TEXT SEGMENT READONLY PARA
+frodokem_sample_avx512 PROC
+        push	r12
+        sub	rsp, 32
+        vmovdqu	OWORD PTR [rsp], xmm6
+        vmovdqu	OWORD PTR [rsp+16], xmm7
+        movsxd	rax, edx
+        shl	rax, 1
+        vpternlogd	zmm0, zmm0, zmm0, 255
+        vpsrlw	zmm0, zmm0, 15
+        vpxorq	zmm1, zmm1, zmm1
+        xor	r10, r10
+L_frodokem_sample_avx512_blk:
+        vmovdqu64	zmm2, [rcx+r10]
+        vpsrlw	zmm3, zmm2, 1
+        vpandq	zmm4, zmm2, zmm0
+        vpxorq	zmm5, zmm5, zmm5
+        mov	r11, r8
+        mov	r12d, r9d
+L_frodokem_sample_avx512_cdf:
+        vpbroadcastw	zmm6, WORD PTR [r11]
+        vpsubw	zmm6, zmm6, zmm3
+        vpsrlw	zmm6, zmm6, 15
+        vpaddw	zmm5, zmm5, zmm6
+        add	r11, 2
+        sub	r12, 1
+        jnz	L_frodokem_sample_avx512_cdf
+        vpsubw	zmm7, zmm1, zmm4
+        vpxorq	zmm5, zmm5, zmm7
+        vpaddw	zmm5, zmm5, zmm4
+        vmovdqu64	[rcx+r10], zmm5
+        add	r10, 64
+        cmp	r10, rax
+        jl	L_frodokem_sample_avx512_blk
+        vzeroupper
+        vmovdqu	xmm6, OWORD PTR [rsp]
+        vmovdqu	xmm7, OWORD PTR [rsp+16]
+        add	rsp, 32
+        pop	r12
+        ret
+frodokem_sample_avx512 ENDP
+_TEXT ENDS
+_TEXT SEGMENT READONLY PARA
+frodokem_gen_a_rows_aes_avx512 PROC
+        push	r12
+        push	r13
+        push	r14
+        push	r15
+        push	rdi
+        push	rsi
+        mov	rax, QWORD PTR [rsp+88]
+        mov	r10, QWORD PTR [rsp+96]
+        mov	r11, QWORD PTR [rsp+104]
+        sub	rsp, 160
+        vmovdqu	OWORD PTR [rsp], xmm6
+        vmovdqu	OWORD PTR [rsp+16], xmm7
+        vmovdqu	OWORD PTR [rsp+32], xmm8
+        vmovdqu	OWORD PTR [rsp+48], xmm9
+        vmovdqu	OWORD PTR [rsp+64], xmm10
+        vmovdqu	OWORD PTR [rsp+80], xmm11
+        vmovdqu	OWORD PTR [rsp+96], xmm12
+        vmovdqu	OWORD PTR [rsp+112], xmm13
+        vmovdqu	OWORD PTR [rsp+128], xmm14
+        vmovdqu	OWORD PTR [rsp+144], xmm15
+        vbroadcasti32x4	zmm0, OWORD PTR [r8]
+        vbroadcasti32x4	zmm1, OWORD PTR [r8+16]
+        vbroadcasti32x4	zmm2, OWORD PTR [r8+32]
+        vbroadcasti32x4	zmm3, OWORD PTR [r8+48]
+        vbroadcasti32x4	zmm4, OWORD PTR [r8+64]
+        vbroadcasti32x4	zmm5, OWORD PTR [r8+80]
+        vbroadcasti32x4	zmm6, OWORD PTR [r8+96]
+        vbroadcasti32x4	zmm7, OWORD PTR [r8+112]
+        vbroadcasti32x4	zmm8, OWORD PTR [r8+128]
+        vbroadcasti32x4	zmm9, OWORD PTR [r8+144]
+        vbroadcasti32x4	zmm10, OWORD PTR [r8+160]
+        vmovd	xmm15, r11
+        vpbroadcastw	zmm15, xmm15
+        movsxd	r12, r10d
+        shl	r12, 1
+        movsxd	r13, eax
+        imul	r13, r12
+        movsxd	r9, r9d
+        xor	rsi, rsi
+        mov	r14, rcx
+L_frodokem_gen_a_rows_aes_avx512_row:
+        mov	edi, r9d
+        mov	r15, r14
+        add	r15, r12
+L_frodokem_gen_a_rows_aes_avx512_blk:
+        mov	QWORD PTR [r14], rdi
+        mov	QWORD PTR [r14+8], rsi
+        add	rdi, 524288
+        add	r14, 16
+        cmp	r14, r15
+        jl	L_frodokem_gen_a_rows_aes_avx512_blk
+        add	r9, 1
+        mov	r15, rcx
+        add	r15, r13
+        cmp	r14, r15
+        jl	L_frodokem_gen_a_rows_aes_avx512_row
+        xor	rdi, rdi
+L_frodokem_gen_a_rows_aes_avx512_aes:
+        vmovdqu64	zmm11, [rcx]
+        vmovdqu64	zmm12, [rcx+64]
+        vmovdqu64	zmm13, [rcx+128]
+        vmovdqu64	zmm14, [rcx+192]
+        vpxorq	zmm11, zmm11, zmm0
+        vpxorq	zmm12, zmm12, zmm0
+        vpxorq	zmm13, zmm13, zmm0
+        vpxorq	zmm14, zmm14, zmm0
+        vaesenc	zmm11, zmm11, zmm1
+        vaesenc	zmm12, zmm12, zmm1
+        vaesenc	zmm13, zmm13, zmm1
+        vaesenc	zmm14, zmm14, zmm1
+        vaesenc	zmm11, zmm11, zmm2
+        vaesenc	zmm12, zmm12, zmm2
+        vaesenc	zmm13, zmm13, zmm2
+        vaesenc	zmm14, zmm14, zmm2
+        vaesenc	zmm11, zmm11, zmm3
+        vaesenc	zmm12, zmm12, zmm3
+        vaesenc	zmm13, zmm13, zmm3
+        vaesenc	zmm14, zmm14, zmm3
+        vaesenc	zmm11, zmm11, zmm4
+        vaesenc	zmm12, zmm12, zmm4
+        vaesenc	zmm13, zmm13, zmm4
+        vaesenc	zmm14, zmm14, zmm4
+        vaesenc	zmm11, zmm11, zmm5
+        vaesenc	zmm12, zmm12, zmm5
+        vaesenc	zmm13, zmm13, zmm5
+        vaesenc	zmm14, zmm14, zmm5
+        vaesenc	zmm11, zmm11, zmm6
+        vaesenc	zmm12, zmm12, zmm6
+        vaesenc	zmm13, zmm13, zmm6
+        vaesenc	zmm14, zmm14, zmm6
+        vaesenc	zmm11, zmm11, zmm7
+        vaesenc	zmm12, zmm12, zmm7
+        vaesenc	zmm13, zmm13, zmm7
+        vaesenc	zmm14, zmm14, zmm7
+        vaesenc	zmm11, zmm11, zmm8
+        vaesenc	zmm12, zmm12, zmm8
+        vaesenc	zmm13, zmm13, zmm8
+        vaesenc	zmm14, zmm14, zmm8
+        vaesenc	zmm11, zmm11, zmm9
+        vaesenc	zmm12, zmm12, zmm9
+        vaesenc	zmm13, zmm13, zmm9
+        vaesenc	zmm14, zmm14, zmm9
+        vaesenclast	zmm11, zmm11, zmm10
+        vaesenclast	zmm12, zmm12, zmm10
+        vaesenclast	zmm13, zmm13, zmm10
+        vaesenclast	zmm14, zmm14, zmm10
+        vpandq	zmm11, zmm11, zmm15
+        vpandq	zmm12, zmm12, zmm15
+        vpandq	zmm13, zmm13, zmm15
+        vpandq	zmm14, zmm14, zmm15
+        vmovdqu64	[rdx], zmm11
+        vmovdqu64	[rdx+64], zmm12
+        vmovdqu64	[rdx+128], zmm13
+        vmovdqu64	[rdx+192], zmm14
+        add	rcx, 256
+        add	rdx, 256
+        add	rdi, 256
+        cmp	rdi, r13
+        jl	L_frodokem_gen_a_rows_aes_avx512_aes
+        vzeroupper
+        vmovdqu	xmm6, OWORD PTR [rsp]
+        vmovdqu	xmm7, OWORD PTR [rsp+16]
+        vmovdqu	xmm8, OWORD PTR [rsp+32]
+        vmovdqu	xmm9, OWORD PTR [rsp+48]
+        vmovdqu	xmm10, OWORD PTR [rsp+64]
+        vmovdqu	xmm11, OWORD PTR [rsp+80]
+        vmovdqu	xmm12, OWORD PTR [rsp+96]
+        vmovdqu	xmm13, OWORD PTR [rsp+112]
+        vmovdqu	xmm14, OWORD PTR [rsp+128]
+        vmovdqu	xmm15, OWORD PTR [rsp+144]
+        add	rsp, 160
+        pop	rsi
+        pop	rdi
+        pop	r15
+        pop	r14
+        pop	r13
+        pop	r12
+        ret
+frodokem_gen_a_rows_aes_avx512 ENDP
+_TEXT ENDS
+ENDIF
+ENDIF
+END
diff --git a/wolfcrypt/src/wc_frodokem_mat.c b/wolfcrypt/src/wc_frodokem_mat.c
new file mode 100644
index 0000000000..9b01ed111e
--- /dev/null
+++ b/wolfcrypt/src/wc_frodokem_mat.c
@@ -0,0 +1,3014 @@
+/* wc_frodokem_mat.c
+ *
+ * Copyright (C) 2006-2026 wolfSSL Inc.
+ *
+ * This file is part of wolfSSL.
+ *
+ * wolfSSL is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * wolfSSL is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335, USA
+ */
+
+/* Low-level matrix arithmetic and encoding routines for the FrodoKEM reference
+ * implementation.
+ *
+ * Implementation based on:
+ *   https://www.ietf.org/archive/id/draft-longa-cfrg-frodokem-03.txt
+ *
+ * The matrix A is generated on the fly (a row at a time) using AES-128
+ * (Section 6.7.1) or SHAKE128 (Section 6.7.2) of the draft, selected per
+ * parameter set and build.  The full matrix is never stored, keeping memory
+ * usage independent of q and small relative to n^2.
+ */
+
+#define _WC_BUILDING_WC_FRODOKEM_MAT_C
+
+#include <wolfssl/wolfcrypt/libwolfssl_sources.h>
+
+#include <wolfssl/wolfcrypt/wc_frodokem_mat.h>
+#include <wolfssl/wolfcrypt/hash.h>
+#include <wolfssl/wolfcrypt/memory.h>
+#if defined(USE_INTEL_SPEEDUP) || defined(FRODOKEM_HAVE_SVE) || \
+    (defined(FRODOKEM_HAVE_NEON_ASM) && defined(__aarch64__))
+#include <wolfssl/wolfcrypt/cpuid.h>
+#endif
+#ifdef WOLFSSL_FRODOKEM_AES
+    #include <wolfssl/wolfcrypt/aes.h>
+    #if defined(NO_AES) || !defined(WOLFSSL_AES_DIRECT) || \
+        !defined(HAVE_AES_ECB)
+        #error "WOLFSSL_FRODOKEM_AES needs WOLFSSL_AES_DIRECT and HAVE_AES_ECB."
+    #endif
+#endif
+
+#ifdef NO_INLINE
+    #include <wolfssl/wolfcrypt/misc.h>
+#else
+    #define WOLFSSL_MISC_INCLUDED
+    #include <wolfcrypt/src/misc.c>
+#endif
+
+#ifdef WOLFSSL_HAVE_FRODOKEM
+
+/* On AArch64 the presence of NEON (Advanced SIMD) is verified at run time via
+ * IS_AARCH64_ASIMD, so the NEON asm and the portable-C matrix path are both
+ * compiled and the choice is made per call (matching the sha256/chacha
+ * pattern). AArch32 NEON and Thumb2 guarantee their SIMD at compile time
+ * (-mfpu / DSP) and always use the asm. */
+#if defined(FRODOKEM_HAVE_NEON_ASM) && defined(__aarch64__)
+    #define FRODOKEM_HAVE_NEON_RUNTIME
+#endif
+
+/* Prefix for the S * A dispatch: use SME when the CPU has it (a small S-column
+ * gather then the ZA-tile outer product), else fall through. Empty when SME is
+ * not compiled in. */
+#ifdef FRODOKEM_HAVE_SME
+#define FRODOKEM_SA_ACCUM_SME(out, s, row, j, n)                               \
+        if (frodokem_sme_svl_ok) {                                             \
+            frodokem_sa_accum_sme_wrap((out), (s), (row), (j), (n));           \
+        }                                                                      \
+        else
+#define FRODOKEM_AS_ACCUM_SME(out, s, row, i, n)                               \
+        if (frodokem_sme_svl_ok) {                                             \
+            frodokem_as_accum_sme_wrap((out), (s), (row), (i), (n));           \
+        }                                                                      \
+        else
+#else
+#define FRODOKEM_SA_ACCUM_SME(out, s, row, j, n) /* SME not compiled in */
+#define FRODOKEM_AS_ACCUM_SME(out, s, row, i, n) /* SME not compiled in */
+#endif
+
+/* Dispatch the S * A and A * S accumulates. SME > SVE > NEON (if Advanced SIMD)
+ * > portable C, chosen per call from the cached CPU flags. */
+#if defined(FRODOKEM_HAVE_SVE)
+#define FRODOKEM_SA_ACCUM(out, s, row, j, n)                                   \
+    do {                                                                       \
+        FRODOKEM_SA_ACCUM_SME(out, s, row, j, n)                               \
+        if (IS_AARCH64_SVE(cpuid_flags)) {                                     \
+            frodokem_sa_accum_sve((out), (s), (row), (j), (n));                \
+        }                                                                      \
+        else if (IS_AARCH64_ASIMD(cpuid_flags)) {                             \
+            frodokem_sa_accum_arm((out), (s), (row), (j), (n));                \
+        }                                                                      \
+        else {                                                                 \
+            frodokem_sa_accum((out), (s), (row), (j), (n));                    \
+        }                                                                      \
+    } while (0)
+#define FRODOKEM_AS_ACCUM(out, s, row, i, n)                                   \
+    do {                                                                       \
+        FRODOKEM_AS_ACCUM_SME(out, s, row, i, n)                               \
+        if (IS_AARCH64_SVE(cpuid_flags)) {                                     \
+            frodokem_as_accum_sve((out), (s), (row), (i), (n));                \
+        }                                                                      \
+        else if (IS_AARCH64_ASIMD(cpuid_flags)) {                             \
+            frodokem_as_accum_arm((out), (s), (row), (i), (n));                \
+        }                                                                      \
+        else {                                                                 \
+            frodokem_as_accum((out), (s), (row), (i), (n));                    \
+        }                                                                      \
+    } while (0)
+#elif defined(FRODOKEM_HAVE_NEON_RUNTIME)
+#define FRODOKEM_SA_ACCUM(out, s, row, j, n)                                   \
+    do {                                                                       \
+        FRODOKEM_SA_ACCUM_SME(out, s, row, j, n)                               \
+        if (IS_AARCH64_ASIMD(cpuid_flags)) {                                   \
+            frodokem_sa_accum_arm((out), (s), (row), (j), (n));                \
+        }                                                                      \
+        else {                                                                 \
+            frodokem_sa_accum((out), (s), (row), (j), (n));                    \
+        }                                                                      \
+    } while (0)
+#define FRODOKEM_AS_ACCUM(out, s, row, i, n)                                   \
+    do {                                                                       \
+        FRODOKEM_AS_ACCUM_SME(out, s, row, i, n)                               \
+        if (IS_AARCH64_ASIMD(cpuid_flags)) {                                   \
+            frodokem_as_accum_arm((out), (s), (row), (i), (n));                \
+        }                                                                      \
+        else {                                                                 \
+            frodokem_as_accum((out), (s), (row), (i), (n));                    \
+        }                                                                      \
+    } while (0)
+#elif defined(FRODOKEM_HAVE_ARM_ASM)
+#define FRODOKEM_SA_ACCUM(out, s, row, j, n)                                   \
+    frodokem_sa_accum_arm((out), (s), (row), (j), (n))
+#define FRODOKEM_AS_ACCUM(out, s, row, i, n)                                   \
+    frodokem_as_accum_arm((out), (s), (row), (i), (n))
+#endif
+
+/* Accumulate a whole FRODOKEM_ROW_MULT-row A*S batch (as generated per AES gen
+ * call). On AArch64 NEON without SVE/SME, FRODOKEM_ROW_MULT is 4 and a dedicated
+ * 4-row routine fuses all four rows so each S column block is loaded once and
+ * multiply-accumulated into all four output rows (halving S-load issue pressure).
+ * Otherwise accumulate FRODOKEM_AS_ACCUM_ROWS rows at a time with the base
+ * routine (which the macro dispatches to SME/SVE/NEON/portable at run time). */
+#if defined(FRODOKEM_HAVE_NEON_ASM) && defined(__aarch64__) && \
+    !defined(FRODOKEM_HAVE_SVE) && !defined(FRODOKEM_HAVE_SME)
+#define FRODOKEM_AS_ACCUM_BATCH(out, s, row, i, n)                             \
+    frodokem_as_accum_x4_neon((out), (s), (row), (i), (n))
+#define FRODOKEM_SA_ACCUM_BATCH(out, s, row, j, n)                             \
+    frodokem_sa_accum_x4_neon((out), (s), (row), (j), (n))
+#elif defined(FRODOKEM_HAVE_ARM_ASM)
+#define FRODOKEM_AS_ACCUM_BATCH(out, s, row, i, n)                             \
+    do {                                                                       \
+        int k_;                                                                \
+        for (k_ = 0; k_ < FRODOKEM_ROW_MULT; k_ += FRODOKEM_AS_ACCUM_ROWS) {   \
+            FRODOKEM_AS_ACCUM((out), (s), (row) + k_ * (n), (i) + k_, (n));     \
+        }                                                                      \
+    } while (0)
+#define FRODOKEM_SA_ACCUM_BATCH(out, s, row, j, n)                             \
+    do {                                                                       \
+        int k_;                                                                \
+        for (k_ = 0; k_ < FRODOKEM_ROW_MULT; k_ += FRODOKEM_AS_ACCUM_ROWS) {   \
+            FRODOKEM_SA_ACCUM((out), (s), (row) + k_ * (n), (j) + k_, (n));     \
+        }                                                                      \
+    } while (0)
+#endif
+
+/* FrodoKEM-976 and -1344 use D=16 packing and SHAKE-256 noise generation;
+ * FrodoKEM-640 uses D=15 packing and SHAKE-128. Only compile the code paths
+ * the enabled parameter sets can actually reach. */
+#if defined(WOLFSSL_WC_FRODOKEM_976) || defined(WOLFSSL_WC_FRODOKEM_1344)
+    #define FRODOKEM_D16_SHAKE256
+#endif
+#ifdef WOLFSSL_WC_FRODOKEM_640
+    #define FRODOKEM_D15_SHAKE128
+#endif
+
+/* restrict qualifier for the matrix multiply inner loops. The A-row scratch
+ * and the output-row slices never overlap, so flagging them lets the compiler
+ * vectorize each contiguous accumulation stream (without it the aliasing
+ * analysis gives up and the loops stay scalar). Falls back to nothing on
+ * compilers without a restrict keyword - the code stays correct either way. */
+#if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L)
+    #define FRODOKEM_RESTRICT restrict
+#elif defined(__GNUC__) || defined(__clang__) || defined(_MSC_VER)
+    #define FRODOKEM_RESTRICT __restrict
+#else
+    #define FRODOKEM_RESTRICT
+#endif
+
+#if defined(USE_INTEL_SPEEDUP) || defined(FRODOKEM_HAVE_SVE) || \
+    defined(FRODOKEM_HAVE_NEON_RUNTIME)
+/* Cached CPU feature flags used to select SIMD routines: AVX2 / BMI2 Keccak in
+ * matrix-A generation on Intel, and the SVE / NEON matrix ops on AArch64 (NEON
+ * is gated on Advanced SIMD). Populated once by frodokem_init(). */
+static cpuid_flags_t cpuid_flags = WC_CPUID_INITIALIZER;
+#endif
+
+#ifdef FRODOKEM_HAVE_SME
+/* The SME kernels compute the whole nbar x nbar (8 x 8) product in one ZA.S
+ * tile, which needs a streaming vector length (SVL) of at least 256 bits (32
+ * bytes). HWCAP2_SME alone does not guarantee that, so the SVL is measured and
+ * the SME path is selected only when it is wide enough (else SVE / NEON runs).
+ * Set by frodokem_init(); 0 until then, so SME stays off until measured. */
+static int frodokem_sme_svl_ok = 0;
+
+/* Streaming vector length in bytes (SVL / 8), read with RDSVL. Encoded as a raw
+ * instruction so no SME assembler support is required. Only called when SME is
+ * present (IS_AARCH64_SME), so RDSVL will not trap. */
+static WC_INLINE int frodokem_sme_svl_bytes(void)
+{
+    word64 svl;
+    __asm__ volatile(".inst 0x04bf5820\n\t"   /* rdsvl x0, #1 */
+                     "mov %0, x0"
+                     : "=r"(svl) : : "x0");
+    return (int)svl;
+}
+#endif
+
+/* Initialize FrodoKEM internal state. Caches the CPU feature flags used to
+ * dispatch the SIMD implementations. Called from wc_FrodoKemKey_Init; safe to
+ * call repeatedly (the flags are read from the CPU only once). */
+void frodokem_init(void)
+{
+#if defined(USE_INTEL_SPEEDUP) || defined(FRODOKEM_HAVE_SVE) || \
+    defined(FRODOKEM_HAVE_NEON_RUNTIME)
+    cpuid_get_flags_ex(&cpuid_flags);
+#endif
+#ifdef FRODOKEM_HAVE_SME
+    /* SME needs SVL >= 256 bits (>= 32 bytes) for the 8 x 8 ZA.S tile. */
+    frodokem_sme_svl_ok = IS_AARCH64_SME(cpuid_flags) &&
+        (frodokem_sme_svl_bytes() >= 32);
+#endif
+}
+
+#ifdef FRODOKEM_HAVE_SME
+/* Portable-C A * S accumulate (defined later): the SME fallback on allocation
+ * failure. Compiled here since FRODOKEM_HAVE_SME implies AArch64 NEON
+ * runtime. */
+static void frodokem_as_accum(word16* out, const word16* s, const word16* row,
+    int i, int n);
+
+/* Reshape a rows x n matrix (row-major) into the SME UMOPA interleaved layout:
+ * dst[t*rows*2 + 2*r + e] = m[r*n + 2*t + e].
+ *
+ * @param  [out]  dst   Interleaved output (rows * n word16).
+ * @param  [in]   m     Source matrix (rows * n, row-major).
+ * @param  [in]   rows  Number of rows in the source matrix.
+ * @param  [in]   n     Number of columns (a multiple of 2).
+ */
+static void frodokem_sme_interleave(word16* dst, const word16* m, int rows,
+    int n)
+{
+    int t;
+    int r;
+    for (t = 0; t < n / 2; t++) {
+        for (r = 0; r < rows; r++) {
+            dst[t * rows * 2 + 2 * r + 0] = m[r * n + 2 * t + 0];
+            dst[t * rows * 2 + 2 * r + 1] = m[r * n + 2 * t + 1];
+        }
+    }
+}
+
+/* Gather the S column pair for column j (sc[2*i + e] = s[i*n + j + e]) into a
+ * small stack buffer and run the SME S * A accumulate. The A-row interleave is
+ * done inside the asm, so no large scratch is needed.
+ *
+ * @param  [in, out]  out  S * A accumulator (nbar * n, row-major).
+ * @param  [in]       s    Matrix S (nbar * n, row-major).
+ * @param  [in]       row  The two generated A rows (2 * n coefficients).
+ * @param  [in]       j    First column of the S column pair.
+ * @param  [in]       n    Matrix dimension n.
+ */
+static void frodokem_sa_accum_sme_wrap(word16* out, const word16* s,
+    const word16* row, int j, int n)
+{
+    word16 sc[2 * FRODOKEM_NBAR];
+    int i;
+
+    for (i = 0; i < FRODOKEM_NBAR; i++) {
+        sc[2 * i + 0] = s[i * n + j + 0];
+        sc[2 * i + 1] = s[i * n + j + 1];
+    }
+    frodokem_sa_accum_sme(out, sc, row, n);
+}
+
+/* Transpose the two A rows and S into the SME interleaved layout, then run the
+ * SME A * S accumulate. Both need reshaping (unlike S * A), so this uses
+ * scratch and falls back to the portable-C accumulate if it cannot be had.
+ *
+ * @param  [in, out]  out  A * S accumulator (n * nbar, row-major).
+ * @param  [in]       s    Matrix S^T (nbar * n, row-major).
+ * @param  [in]       row  The two generated A rows (2 * n coefficients).
+ * @param  [in]       i    Index of the first of the two generated A rows.
+ * @param  [in]       n    Matrix dimension n.
+ */
+static void frodokem_as_accum_sme_wrap(word16* out, const word16* s,
+    const word16* row, int i, int n)
+{
+    word16* at;
+    word16* st;
+
+    at = (word16*)XMALLOC((size_t)2 * (size_t)n * sizeof(word16), NULL,
+        DYNAMIC_TYPE_TMP_BUFFER);
+    st = (word16*)XMALLOC((size_t)FRODOKEM_NBAR * (size_t)n * sizeof(word16),
+        NULL, DYNAMIC_TYPE_TMP_BUFFER);
+    if ((at != NULL) && (st != NULL)) {
+        frodokem_sme_interleave(at, row, 2, n);
+        frodokem_sme_interleave(st, s, FRODOKEM_NBAR, n);
+        frodokem_as_accum_sme(out, at, st, i, n);
+    }
+    else {
+        frodokem_as_accum(out, s, row, i, n);
+    }
+    XFREE(at, NULL, DYNAMIC_TYPE_TMP_BUFFER);
+    XFREE(st, NULL, DYNAMIC_TYPE_TMP_BUFFER);
+}
+#endif /* FRODOKEM_HAVE_SME */
+
+/******************************************************************************/
+/* Packing and unpacking of matrices to/from byte strings (Section 6.2).      */
+/******************************************************************************/
+
+/* Pack a matrix of nElem coefficients into a byte string.
+ *
+ * Each coefficient contributes its low 'd' bits, most-significant bit first,
+ * and the resulting bit string is packed into bytes most-significant bit first
+ * (Section 6.2).
+ *
+ * @param  [out]  out    Output byte string. Must be (nElem * d + 7) / 8 bytes.
+ * @param  [in]   in     Matrix coefficients in row-major order.
+ * @param  [in]   nElem  Number of coefficients.
+ * @param  [in]   d      Number of bits per coefficient.
+ */
+void frodokem_pack(byte* out, const word16* in, int nElem, int d)
+{
+#if !defined(FRODOKEM_D16_SHAKE256) || !defined(FRODOKEM_D15_SHAKE128)
+    (void)d;
+#endif
+
+#if defined(FRODOKEM_D16_SHAKE256) && defined(FRODOKEM_D15_SHAKE128)
+    if (d == 16)
+#endif
+#ifdef FRODOKEM_D16_SHAKE256
+    {
+        /* Coefficients are byte-aligned: each is stored as a big-endian 16-bit
+         * value (most-significant byte first). */
+#ifdef BIG_ENDIAN_ORDER
+        /* word16 memory is already big-endian: a straight copy. */
+        XMEMCPY(out, in, (size_t)nElem * sizeof(word16));
+#else
+        int i;
+
+        for (i = 0; i < nElem; i++) {
+            out[2 * i]     = (byte)(in[i] >> 8);
+            out[2 * i + 1] = (byte)(in[i] & 0xff);
+        }
+#endif
+    }
+#endif
+#if defined(FRODOKEM_D16_SHAKE256) && defined(FRODOKEM_D15_SHAKE128)
+    else
+#endif
+#ifdef FRODOKEM_D15_SHAKE128
+    {
+        /* d == 15 (FrodoKEM-640): stream 15 significant bits per coefficient,
+         * most-significant bit first, through a bit accumulator, emitting a
+         * byte whenever 8 bits are buffered. nElem is a multiple of nbar (8),
+         * so packing ends on a byte boundary and every output byte is fully
+         * assigned - no pre-clear needed. This is endian-independent. */
+        word32 acc = 0;
+        int accBits = 0;
+        int oi = 0;
+        int i;
+
+        for (i = 0; i < nElem; i++) {
+            acc = (acc << 15) | (word32)(in[i] & 0x7fff);
+            accBits += 15;
+            do {
+                accBits -= 8;
+                out[oi++] = (byte)(acc >> accBits);
+            } while (accBits >= 8);
+            acc &= ((word32)1 << accBits) - 1;
+        }
+    }
+#endif
+}
+
+/* Unpack a byte string into a matrix of nElem coefficients.
+ *
+ * The inverse of frodokem_pack().
+ *
+ * @param  [out]  out    Output matrix coefficients in row-major order.
+ * @param  [in]   in     Input byte string.
+ * @param  [in]   nElem  Number of coefficients.
+ * @param  [in]   d      Number of bits per coefficient.
+ */
+void frodokem_unpack(word16* out, const byte* in, int nElem, int d)
+{
+#if !defined(FRODOKEM_D16_SHAKE256) || !defined(FRODOKEM_D15_SHAKE128)
+    (void)d;
+#endif
+
+#if defined(FRODOKEM_D16_SHAKE256) && defined(FRODOKEM_D15_SHAKE128)
+    if (d == 16)
+#endif
+#ifdef FRODOKEM_D16_SHAKE256
+    {
+        /* Byte-aligned coefficients: inverse of the big-endian byte swap. */
+#ifdef BIG_ENDIAN_ORDER
+        /* word16 memory is already big-endian: a straight copy. */
+        XMEMCPY(out, in, (size_t)nElem * sizeof(word16));
+#else
+        int i;
+
+        for (i = 0; i < nElem; i++) {
+            out[i] = (word16)(((word16)in[2 * i] << 8) | in[2 * i + 1]);
+        }
+#endif
+    }
+#endif
+#if defined(FRODOKEM_D16_SHAKE256) && defined(FRODOKEM_D15_SHAKE128)
+    else
+#endif
+#ifdef FRODOKEM_D15_SHAKE128
+    {
+        /* d == 15 (FrodoKEM-640): read 15 bits per coefficient, most-
+         * significant bit first, through a bit accumulator fed a byte at a
+         * time. nElem is a multiple of nbar (8), so the stream is consumed
+         * exactly. This is endian-independent. */
+        word32 acc = 0;
+        int accBits = 0;
+        int ii = 0;
+        int i;
+
+        for (i = 0; i < nElem; i++) {
+            while (accBits < 15) {
+                acc = (acc << 8) | in[ii++];
+                accBits += 8;
+            }
+            accBits -= 15;
+            out[i] = (word16)((acc >> accBits) & 0x7fff);
+            acc &= ((word32)1 << accBits) - 1;
+        }
+    }
+#endif
+}
+
+/* Serialize a matrix of word16 coefficients as little-endian 16-bit values.
+ *
+ * Used for the secret matrix S^T in the private key, whose two's-complement
+ * samples are stored as full 16-bit little-endian words (Section 6.3).
+ *
+ * @param  [out]  out  Output byte string (2 * cnt bytes).
+ * @param  [in]   mat  Matrix coefficients.
+ * @param  [in]   cnt  Number of coefficients.
+ */
+void frodokem_store_matrix(byte* out, const word16* mat, int cnt)
+{
+#ifdef BIG_ENDIAN_ORDER
+    int i;
+
+    /* Swap each coefficient from big-endian memory to little-endian bytes. */
+    for (i = 0; i < cnt; i++) {
+        out[2 * i]     = (byte)(mat[i] & 0xff);
+        out[2 * i + 1] = (byte)((mat[i] >> 8) & 0xff);
+    }
+#else
+    /* Coefficients are already little-endian in memory: a straight copy. */
+    XMEMCPY(out, mat, (size_t)cnt * sizeof(word16));
+#endif
+}
+
+/* Deserialize little-endian 16-bit values into a matrix of word16 coefficients.
+ *
+ * The inverse of frodokem_store_matrix(): loads the secret matrix S^T from the
+ * private key.
+ *
+ * @param  [out]  mat  Output matrix coefficients.
+ * @param  [in]   in   Input byte string (2 * cnt bytes).
+ * @param  [in]   cnt  Number of coefficients.
+ */
+void frodokem_load_matrix(word16* mat, const byte* in, int cnt)
+{
+#ifdef BIG_ENDIAN_ORDER
+    int i;
+
+    /* Swap each coefficient from little-endian bytes to big-endian memory. */
+    for (i = 0; i < cnt; i++) {
+        mat[i] = (word16)(in[2 * i] | ((word16)in[2 * i + 1] << 8));
+    }
+#else
+    /* Coefficients are already little-endian in memory: a straight copy. */
+    XMEMCPY(mat, in, (size_t)cnt * sizeof(word16));
+#endif
+}
+
+/******************************************************************************/
+/* Encoding and decoding of messages to/from matrices (Section 6.1).          */
+/******************************************************************************/
+
+/* Encode a message bit string into an nbar x nbar matrix.
+ *
+ * Groups of B bits (least-significant bit of each byte first, Section 6.1) are
+ * turned into a coefficient scaled by q / 2^B, i.e. shifted left by (D - B).
+ *
+ * @param  [out]  c     Output matrix (nbar * nbar coefficients).
+ * @param  [in]   msg   Message of B * nbar * nbar bits (lenSec bytes).
+ * @param  [in]   d     Number of bits per coefficient (D).
+ * @param  [in]   bits  Number of message bits per coefficient (B).
+ */
+void frodokem_key_encode(word16* c, const byte* msg, int d, int bits)
+{
+    int idx;
+    int k;
+    word32 bitPos = 0;
+    int nElem = FRODOKEM_NBAR_SQ;
+    int shift = d - bits;
+
+    for (idx = 0; idx < nElem; idx++) {
+        unsigned int val = 0;
+
+        for (k = 0; k < bits; k++) {
+            /* Bits taken least-significant first within each byte. */
+            int bit = (msg[bitPos >> 3] >> (bitPos & 7)) & 1;
+            val |= ((unsigned int)bit) << k;
+            bitPos++;
+        }
+        c[idx] = (word16)(val << shift);
+    }
+}
+
+/* Decode an nbar x nbar matrix into a message bit string.
+ *
+ * The inverse of frodokem_key_encode(): each coefficient is rounded to
+ * the nearest multiple of q / 2^B and the resulting B bits are emitted
+ * least-significant bit first.
+ *
+ * @param  [out]  msg  Output message of B * nbar * nbar bits (lenSec bytes).
+ * @param  [in]   c    Input matrix (nbar * nbar coefficients).
+ * @param  [in]   p    FrodoKEM parameters.
+ */
+void frodokem_key_decode(byte* msg, const word16* c, const FrodoKemParams* p)
+{
+    int idx;
+    int k;
+    word32 bitPos = 0;
+    int nElem = FRODOKEM_NBAR_SQ;
+    int shift = p->d - p->b;
+    /* Rounding constant: q / 2^(B+1) = 2^(D - B - 1). */
+    unsigned int rnd = 1u << (shift - 1);
+    unsigned int mask = (1u << p->b) - 1;
+
+    XMEMSET(msg, 0, (size_t)p->lenSec);
+
+    for (idx = 0; idx < nElem; idx++) {
+        /* Reduce the coefficient modulo q before rounding. */
+        unsigned int v = (unsigned int)(c[idx] & p->qMask);
+        unsigned int dec = ((v + rnd) >> shift) & mask;
+
+        for (k = 0; k < p->b; k++) {
+            int bit = (dec >> k) & 1;
+            msg[bitPos >> 3] |= (byte)(bit << (bitPos & 7));
+            bitPos++;
+        }
+    }
+}
+
+/******************************************************************************/
+/* SHAKE hashing.                                                             */
+/******************************************************************************/
+
+#ifdef FRODOKEM_D15_SHAKE128
+/* Compute out = SHAKE128(in0 || in1, outLen). Used for the hashing function
+ * of FrodoKEM-640 (D=15).
+ *
+ * @param  [in]   shake   Reusable SHAKE object (re-initialized here).
+ * @param  [in]   in0     First input buffer.
+ * @param  [in]   len0    Length in bytes of the first input.
+ * @param  [in]   in1     Second input buffer.
+ * @param  [in]   len1    Length in bytes of the second input.
+ * @param  [out]  out     Output buffer.
+ * @param  [in]   outLen  Number of bytes to output.
+ * @return  0 on success, negative on hash error.
+ */
+static int frodokem_shake128(wc_Shake* shake, const byte* in0, word32 len0,
+    const byte* in1, word32 len1, byte* out, word32 outLen)
+{
+    int ret;
+
+    ret = wc_InitShake128(shake, NULL, INVALID_DEVID);
+    if (ret == 0) {
+        ret = wc_Shake128_Update(shake, in0, len0);
+    }
+    if (ret == 0) {
+        ret = wc_Shake128_Update(shake, in1, len1);
+    }
+    if (ret == 0) {
+        ret = wc_Shake128_Final(shake, out, outLen);
+    }
+
+    return ret;
+}
+#endif /* FRODOKEM_D15_SHAKE128 */
+
+#ifdef FRODOKEM_D16_SHAKE256
+/* Compute out = SHAKE256(in0 || in1, outLen). Used for the hashing function
+ * of FrodoKEM-976 / -1344 (D=16).
+ *
+ * @param  [in]   shake   Reusable SHAKE object (re-initialized here).
+ * @param  [in]   in0     First input buffer.
+ * @param  [in]   len0    Length in bytes of the first input.
+ * @param  [in]   in1     Second input buffer.
+ * @param  [in]   len1    Length in bytes of the second input.
+ * @param  [out]  out     Output buffer.
+ * @param  [in]   outLen  Number of bytes to output.
+ * @return  0 on success, negative on hash error.
+ */
+static int frodokem_shake256(wc_Shake* shake, const byte* in0, word32 len0,
+    const byte* in1, word32 len1, byte* out, word32 outLen)
+{
+    int ret;
+
+    ret = wc_InitShake256(shake, NULL, INVALID_DEVID);
+    if (ret == 0) {
+        ret = wc_Shake256_Update(shake, in0, len0);
+    }
+    if (ret == 0) {
+        ret = wc_Shake256_Update(shake, in1, len1);
+    }
+    if (ret == 0) {
+        ret = wc_Shake256_Final(shake, out, outLen);
+    }
+
+    return ret;
+}
+#endif /* FRODOKEM_D16_SHAKE256 */
+
+/* Compute out = SHAKE(in0 || in1, outLen), selecting SHAKE128 for FrodoKEM-640
+ * and SHAKE256 for FrodoKEM-976 / -1344.
+ *
+ * @param  [in]   p       FrodoKEM parameters.
+ * @param  [in]   shake   Reusable SHAKE object (re-initialized here).
+ * @param  [in]   in0     First input buffer.
+ * @param  [in]   len0    Length in bytes of the first input.
+ * @param  [in]   in1     Second input buffer.
+ * @param  [in]   len1    Length in bytes of the second input.
+ * @param  [out]  out     Output buffer.
+ * @param  [in]   outLen  Number of bytes to output.
+ * @return  0 on success, negative on hash error.
+ */
+int frodokem_shake(const FrodoKemParams* p, wc_Shake* shake, const byte* in0,
+    word32 len0, const byte* in1, word32 len1, byte* out, word32 outLen)
+{
+    int ret = 0;
+
+#if !defined(FRODOKEM_D16_SHAKE256) || !defined(FRODOKEM_D15_SHAKE128)
+    (void)p;
+#endif
+#if defined(FRODOKEM_D16_SHAKE256) && defined(FRODOKEM_D15_SHAKE128)
+    if (p->useShake256)
+#endif
+#ifdef FRODOKEM_D16_SHAKE256
+    {
+        ret = frodokem_shake256(shake, in0, len0, in1, len1, out, outLen);
+    }
+#endif
+#if defined(FRODOKEM_D16_SHAKE256) && defined(FRODOKEM_D15_SHAKE128)
+    else
+#endif
+#ifdef FRODOKEM_D15_SHAKE128
+    {
+        ret = frodokem_shake128(shake, in0, len0, in1, len1, out, outLen);
+    }
+#endif
+
+    return ret;
+}
+
+/* One-shot SHAKE over a single contiguous input buffer, selecting SHAKE128 for
+ * FrodoKEM-640 and SHAKE256 for FrodoKEM-976 / -1344.
+ *
+ * @param  [in]   p       FrodoKEM parameters.
+ * @param  [in]   shake   Reusable SHAKE object (re-initialized here).
+ * @param  [in]   in      Input buffer.
+ * @param  [in]   inLen   Length of input in bytes.
+ * @param  [out]  out     Output buffer.
+ * @param  [in]   outLen  Number of bytes to output.
+ * @return  0 on success, negative on hash error.
+ */
+int frodokem_shake_oneshot(const FrodoKemParams* p, wc_Shake* shake,
+    const byte* in, word32 inLen, byte* out, word32 outLen)
+{
+    int ret = 0;
+
+#if !defined(FRODOKEM_D16_SHAKE256) || !defined(FRODOKEM_D15_SHAKE128)
+    (void)p;
+#endif
+#if defined(FRODOKEM_D16_SHAKE256) && defined(FRODOKEM_D15_SHAKE128)
+    if (p->useShake256)
+#endif
+#ifdef FRODOKEM_D16_SHAKE256
+    {
+        ret = wc_InitShake256(shake, NULL, INVALID_DEVID);
+        if (ret == 0) {
+            ret = wc_Shake256_Update(shake, in, inLen);
+        }
+        if (ret == 0) {
+            ret = wc_Shake256_Final(shake, out, outLen);
+        }
+    }
+#endif
+#if defined(FRODOKEM_D16_SHAKE256) && defined(FRODOKEM_D15_SHAKE128)
+    else
+#endif
+#ifdef FRODOKEM_D15_SHAKE128
+    {
+        ret = wc_InitShake128(shake, NULL, INVALID_DEVID);
+        if (ret == 0) {
+            ret = wc_Shake128_Update(shake, in, inLen);
+        }
+        if (ret == 0) {
+            ret = wc_Shake128_Final(shake, out, outLen);
+        }
+    }
+#endif
+
+    return ret;
+}
+
+/******************************************************************************/
+/* Error sampling (Section 6.5).                                              */
+/******************************************************************************/
+
+/* The sign/magnitude split of the 16-bit random value: bit 0 is the sign, the
+ * remaining 15 bits form the magnitude looked up against the CDF table. */
+#define FRODOKEM_SAMPLE_PRND(val)  ((word16)((val) >> 1))
+#define FRODOKEM_SAMPLE_SIGN(val)  ((word16)((val) & 1))
+
+/* Apply the sign: e = (-1)^sign * e, held as a 16-bit two's complement value.
+ * It is deliberately NOT reduced modulo q: the secret and error matrices are
+ * stored (and serialized in the secret key) in this form, and all downstream
+ * matrix arithmetic reduces modulo q at the end. */
+#define FRODOKEM_SAMPLE_APPLY_SIGN(e, sign) \
+    ((word16)(((e) ^ (word16)(0 - (sign))) + (sign)))
+
+/* The scalar-C samplers are the fallback for CPUs without the SIMD sampler.
+ * On AArch64 they are also compiled as the run-time fallback used when the CPU
+ * lacks Advanced SIMD (NEON). */
+#if !defined(FRODOKEM_HAVE_NEON_ASM) || defined(FRODOKEM_HAVE_NEON_RUNTIME)
+#ifdef WOLFSSL_FRODOKEM_SMALL
+/* Sample a single value from the error distribution using the parameter set's
+ * CDF table (Section 6.5).
+ *
+ * @param  [in]  val  16-bit random value (little-endian bytes assembled by the
+ *                    caller: bit 0 is the sign, bits 1..15 the magnitude).
+ * @param  [in]  p    FrodoKEM parameters.
+ * @return  Sampled value (16-bit two's complement, not reduced modulo q).
+ */
+static word16 frodokem_sample(word16 val, const FrodoKemParams* p)
+{
+    int i;
+    word16 prnd = FRODOKEM_SAMPLE_PRND(val);
+    word16 sign = FRODOKEM_SAMPLE_SIGN(val);
+    word16 e = 0;
+
+    /* Constant-time count of table entries strictly less than prnd. The final
+     * table entry (2^15 - 1) can never be exceeded, so it contributes zero. */
+    for (i = 0; i < p->cdfLen; i++) {
+        e = (word16)(e + (word16)((word16)(p->cdf[i] - prnd) >> 15));
+    }
+
+    return FRODOKEM_SAMPLE_APPLY_SIGN(e, sign);
+}
+#else
+/* One constant-time CDF comparison: add 1 to e when table entry V < prnd, i.e.
+ * when the 16-bit difference V - prnd borrows and sets bit 15. */
+#define FRODOKEM_CDF_STEP(V) \
+    e = (word16)(e + (word16)((word16)((V) - prnd) >> 15))
+
+/* Per-parameter samplers with the CDF loop fully unrolled and the CDF values
+ * (Table 5) used as literals - no table load or loop overhead. Each performs
+ * the same constant-time count as the generic sampler above. */
+#ifdef WOLFSSL_WC_FRODOKEM_640
+/* Sample a single FrodoKEM-640 error value from its 13-entry CDF (Table 5).
+ *
+ * @param  [in]  val  16-bit random value (bit 0 is the sign, bits 1..15 the
+ *                    magnitude).
+ * @return  Sampled value (16-bit two's complement, not reduced modulo q).
+ */
+static word16 frodokem_sample_640(word16 val)
+{
+    word16 prnd = FRODOKEM_SAMPLE_PRND(val);
+    word16 sign = FRODOKEM_SAMPLE_SIGN(val);
+    word16 e = 0;
+
+    FRODOKEM_CDF_STEP(4643);
+    FRODOKEM_CDF_STEP(13363);
+    FRODOKEM_CDF_STEP(20579);
+    FRODOKEM_CDF_STEP(25843);
+    FRODOKEM_CDF_STEP(29227);
+    FRODOKEM_CDF_STEP(31145);
+    FRODOKEM_CDF_STEP(32103);
+    FRODOKEM_CDF_STEP(32525);
+    FRODOKEM_CDF_STEP(32689);
+    FRODOKEM_CDF_STEP(32745);
+    FRODOKEM_CDF_STEP(32762);
+    FRODOKEM_CDF_STEP(32766);
+    FRODOKEM_CDF_STEP(32767);
+
+    return FRODOKEM_SAMPLE_APPLY_SIGN(e, sign);
+}
+#endif
+#ifdef WOLFSSL_WC_FRODOKEM_976
+/* Sample a single FrodoKEM-976 error value from its 11-entry CDF (Table 5).
+ *
+ * @param  [in]  val  16-bit random value (bit 0 is the sign, bits 1..15 the
+ *                    magnitude).
+ * @return  Sampled value (16-bit two's complement, not reduced modulo q).
+ */
+static word16 frodokem_sample_976(word16 val)
+{
+    word16 prnd = FRODOKEM_SAMPLE_PRND(val);
+    word16 sign = FRODOKEM_SAMPLE_SIGN(val);
+    word16 e = 0;
+
+    FRODOKEM_CDF_STEP(5638);
+    FRODOKEM_CDF_STEP(15915);
+    FRODOKEM_CDF_STEP(23689);
+    FRODOKEM_CDF_STEP(28571);
+    FRODOKEM_CDF_STEP(31116);
+    FRODOKEM_CDF_STEP(32217);
+    FRODOKEM_CDF_STEP(32613);
+    FRODOKEM_CDF_STEP(32731);
+    FRODOKEM_CDF_STEP(32760);
+    FRODOKEM_CDF_STEP(32766);
+    FRODOKEM_CDF_STEP(32767);
+
+    return FRODOKEM_SAMPLE_APPLY_SIGN(e, sign);
+}
+#endif
+#ifdef WOLFSSL_WC_FRODOKEM_1344
+/* Sample a single FrodoKEM-1344 error value from its 7-entry CDF (Table 5).
+ *
+ * @param  [in]  val  16-bit random value (bit 0 is the sign, bits 1..15 the
+ *                    magnitude).
+ * @return  Sampled value (16-bit two's complement, not reduced modulo q).
+ */
+static word16 frodokem_sample_1344(word16 val)
+{
+    word16 prnd = FRODOKEM_SAMPLE_PRND(val);
+    word16 sign = FRODOKEM_SAMPLE_SIGN(val);
+    word16 e = 0;
+
+    FRODOKEM_CDF_STEP(9142);
+    FRODOKEM_CDF_STEP(23462);
+    FRODOKEM_CDF_STEP(30338);
+    FRODOKEM_CDF_STEP(32361);
+    FRODOKEM_CDF_STEP(32725);
+    FRODOKEM_CDF_STEP(32765);
+    FRODOKEM_CDF_STEP(32767);
+
+    return FRODOKEM_SAMPLE_APPLY_SIGN(e, sign);
+}
+#endif
+#undef FRODOKEM_CDF_STEP
+#endif /* WOLFSSL_FRODOKEM_SMALL */
+#endif /* !FRODOKEM_HAVE_NEON_ASM || FRODOKEM_HAVE_NEON_RUNTIME */
+
+/* The i-th 16-bit little-endian noise value. Sampling is in place on little-
+ * endian (r aliases mat), so the native word16 is read directly; on big-endian
+ * the two little-endian bytes are assembled. */
+#ifdef BIG_ENDIAN_ORDER
+    #define FRODOKEM_NOISE_VAL(mat, r, i) \
+        ((word16)((r)[2 * (i)] | ((word16)(r)[2 * (i) + 1] << 8)))
+#else
+    #define FRODOKEM_NOISE_VAL(mat, r, i)  ((mat)[i])
+#endif
+
+/* Sample cnt error coefficients from a buffer of random bytes.
+ *
+ * Sampling is elementwise, so the matrix shape is irrelevant - only the total
+ * coefficient count is needed. The per-parameter sampler is chosen once, ahead
+ * of the loop.
+ *
+ * @param  [in, out]  mat  Little-endian: raw noise bytes in place, replaced by
+ *                         the sampled coefficients. Big-endian: coefficients
+ *                         out (bytes come from r).
+ * @param  [in]       cnt  Number of coefficients.
+ * @param  [in]       r    Random bytes (2 * cnt); used on big-endian only.
+ * @param  [in]       p    FrodoKEM parameters.
+ */
+static void frodokem_sample_matrix(word16* mat, int cnt, const byte* r,
+    const FrodoKemParams* p)
+{
+    int i;
+
+#ifndef BIG_ENDIAN_ORDER
+    /* Little-endian reads the value straight from mat (sampled in place). */
+    (void)r;
+#endif
+
+#if defined(FRODOKEM_HAVE_NEON_ASM) && !defined(FRODOKEM_HAVE_NEON_RUNTIME)
+    (void)i;
+    frodokem_sample_neon(mat, cnt, p->cdf, p->cdfLen);
+#else
+#ifdef FRODOKEM_HAVE_NEON_RUNTIME
+    if (IS_AARCH64_ASIMD(cpuid_flags)) {
+        frodokem_sample_neon(mat, cnt, p->cdf, p->cdfLen);
+    }
+    else
+#endif
+#ifdef FRODOKEM_HAVE_MATRIX_ASM_AVX512
+    if (IS_INTEL_AVX512(cpuid_flags) && (SAVE_VECTOR_REGISTERS2() == 0)) {
+        frodokem_sample_avx512(mat, cnt, p->cdf, p->cdfLen);
+        RESTORE_VECTOR_REGISTERS();
+    }
+    else
+#endif
+#ifdef FRODOKEM_HAVE_MATRIX_ASM
+    if (IS_INTEL_AVX2(cpuid_flags) && (SAVE_VECTOR_REGISTERS2() == 0)) {
+        frodokem_sample_avx2(mat, cnt, p->cdf, p->cdfLen);
+        RESTORE_VECTOR_REGISTERS();
+    }
+    else
+#endif
+    {
+#ifdef WOLFSSL_FRODOKEM_SMALL
+    for (i = 0; i < cnt; i++) {
+        mat[i] = frodokem_sample(FRODOKEM_NOISE_VAL(mat, r, i), p);
+    }
+#else
+    switch (p->n) {
+#ifdef WOLFSSL_WC_FRODOKEM_640
+    case WC_FRODOKEM_640_N:
+        for (i = 0; i < cnt; i++) {
+            mat[i] = frodokem_sample_640(FRODOKEM_NOISE_VAL(mat, r, i));
+        }
+        break;
+#endif
+#ifdef WOLFSSL_WC_FRODOKEM_976
+    case WC_FRODOKEM_976_N:
+        for (i = 0; i < cnt; i++) {
+            mat[i] = frodokem_sample_976(FRODOKEM_NOISE_VAL(mat, r, i));
+        }
+        break;
+#endif
+#ifdef WOLFSSL_WC_FRODOKEM_1344
+    case WC_FRODOKEM_1344_N:
+        for (i = 0; i < cnt; i++) {
+            mat[i] = frodokem_sample_1344(FRODOKEM_NOISE_VAL(mat, r, i));
+        }
+        break;
+#endif
+    default:
+        break;
+    }
+#endif /* WOLFSSL_FRODOKEM_SMALL */
+    }
+#endif /* FRODOKEM_HAVE_NEON_ASM */
+}
+
+/* SHAKE-128 rate (block size) in bytes. */
+#define FRODOKEM_SHAKE128_RATE  168
+/* SHAKE-256 rate (block size) in bytes. */
+#define FRODOKEM_SHAKE256_RATE  136
+
+/* Generate and sample the noise matrices for one FrodoKEM operation.
+ *
+ * Absorbs the caller-assembled 'domain || seedSE' (seInput, 1 + lenSE bytes)
+ * into a SHAKE XOF and squeezes the noise stream. Whole SHAKE blocks are
+ * squeezed straight into the destination buffers; tmp holds only the final
+ * partial block of a region, whose unused tail continues the stream into the
+ * next region. This is byte-identical to a single large SHAKE squeeze but
+ * avoids copying full blocks. A region of cnt word16 occupies exactly 2 * cnt
+ * bytes - the same as its noise - so each region is sampled in place, needing
+ * no separate noise buffer. The second region is optional (cnt1 == 0 to skip
+ * it): MakeKey's two matrices live in separate buffers, whereas Encapsulate and
+ * Decapsulate pass everything as one region. Region 1, when present, is much
+ * larger than one SHAKE block. The SHAKE variant is chosen once; each branch
+ * uses only its own primitives - SHAKE-256 for FrodoKEM-976/-1344, SHAKE-128
+ * for -640.
+ *
+ * The caller owns seInput and tmp: seInput is a small buffer whose first byte
+ * holds the domain and whose next lenSE bytes hold the seed (so no seed copy or
+ * seed zeroize happens here); tmp is scratch of at least a SHAKE block, for
+ * which callers pass the matrix-A row scratch (unused during noise generation).
+ *
+ * @param  [in]   p        FrodoKEM parameters.
+ * @param  [in]   shake    Reusable SHAKE object (re-initialized here).
+ * @param  [in]   seInput  Domain byte followed by the seed (1 + lenSE bytes).
+ * @param  [out]  tmp      Scratch for one squeezed block (>= SHAKE rate bytes).
+ * @param  [out]  mat0     First region output buffer (cnt0 coefficients).
+ * @param  [in]   cnt0     Number of coefficients in the first region.
+ * @param  [out]  mat1     Second region output buffer, or NULL when cnt1 == 0.
+ * @param  [in]   cnt1     Number of coefficients in the second region, or 0.
+ * @return  0 on success, negative on error.
+ */
+int frodokem_gen_noise(const FrodoKemParams* p, wc_Shake* shake,
+    const byte* seInput, byte* tmp, word16* mat0, int cnt0, word16* mat1,
+    int cnt1)
+{
+    /* Initialized: with a single parameter set built, only one SHAKE branch is
+     * compiled and the compiler cannot see that p->useShake256 always selects
+     * it, so ret needs a defined value on the (unreachable) fall-through. */
+    int ret = 0;
+    word32 inLen;
+    byte* mat0_8 = (byte*)mat0;
+    byte* mat1_8 = (byte*)mat1;
+
+    inLen = (word32)(1 + p->lenSE);
+
+#ifdef FRODOKEM_D16_SHAKE256
+    if (p->useShake256 &&
+        ((ret = wc_InitShake256(shake, NULL, INVALID_DEVID)) == 0)) {
+        word32 len0 = (word32)(2 * cnt0);
+        word32 full = len0 / FRODOKEM_SHAKE256_RATE;
+
+        ret = wc_Shake256_Absorb(shake, seInput, inLen);
+        /* Whole blocks straight into mat0, final partial block via tmp. */
+        if (ret == 0) {
+            ret = wc_Shake256_SqueezeBlocks(shake, mat0_8, full);
+        }
+        if (ret == 0) {
+            ret = wc_Shake256_SqueezeBlocks(shake, tmp, 1);
+        }
+        if (ret == 0) {
+            len0 -= full * FRODOKEM_SHAKE256_RATE;
+            XMEMCPY(mat0_8 + full * FRODOKEM_SHAKE256_RATE, tmp, len0);
+            frodokem_sample_matrix(mat0, cnt0, mat0_8, p);
+        }
+        if ((ret == 0) && (cnt1 > 0)) {
+            /* tmp[len0 ..] is the start of region 1 (stream continues). The
+             * only cnt1 > 0 caller (MakeKey) passes a region 1 of many blocks,
+             * so 2 * cnt1 > lead and the unsigned rest cannot underflow. */
+            word32 lead = FRODOKEM_SHAKE256_RATE - len0;
+            word32 rest = (word32)(2 * cnt1) - lead;
+
+            XMEMCPY(mat1_8, tmp + len0, lead);
+            full = rest / FRODOKEM_SHAKE256_RATE;
+            ret = wc_Shake256_SqueezeBlocks(shake, mat1_8 + lead, full);
+            if (ret == 0) {
+                ret = wc_Shake256_SqueezeBlocks(shake, tmp, 1);
+                if (ret == 0) {
+                    rest -= full * FRODOKEM_SHAKE256_RATE;
+                    XMEMCPY(mat1_8 + lead + full * FRODOKEM_SHAKE256_RATE, tmp,
+                        rest);
+                }
+            }
+            if (ret == 0) {
+                frodokem_sample_matrix(mat1, cnt1, mat1_8, p);
+            }
+        }
+    }
+#endif
+#ifdef FRODOKEM_D15_SHAKE128
+    if ((!p->useShake256) &&
+        ((ret = wc_InitShake128(shake, NULL, INVALID_DEVID)) == 0)) {
+        word32 len0 = (word32)(2 * cnt0);
+        word32 full = len0 / FRODOKEM_SHAKE128_RATE;
+
+        ret = wc_Shake128_Absorb(shake, seInput, inLen);
+        /* Whole blocks straight into mat0, final partial block via tmp. */
+        if (ret == 0) {
+            ret = wc_Shake128_SqueezeBlocks(shake, mat0_8, full);
+        }
+        if (ret == 0) {
+            ret = wc_Shake128_SqueezeBlocks(shake, tmp, 1);
+        }
+        if (ret == 0) {
+            len0 -= full * FRODOKEM_SHAKE128_RATE;
+            XMEMCPY(mat0_8 + full * FRODOKEM_SHAKE128_RATE, tmp, len0);
+            frodokem_sample_matrix(mat0, cnt0, mat0_8, p);
+        }
+        if ((ret == 0) && (cnt1 > 0)) {
+            /* tmp[len0 ..] is the start of region 1 (stream continues). The
+             * only cnt1 > 0 caller (MakeKey) passes a region 1 of many blocks,
+             * so 2 * cnt1 > lead and the unsigned rest cannot underflow. */
+            word32 lead = FRODOKEM_SHAKE128_RATE - len0;
+            word32 rest = (word32)(2 * cnt1) - lead;
+
+            XMEMCPY(mat1_8, tmp + len0, lead);
+            full = rest / FRODOKEM_SHAKE128_RATE;
+            ret = wc_Shake128_SqueezeBlocks(shake, mat1_8 + lead, full);
+            if (ret == 0) {
+                ret = wc_Shake128_SqueezeBlocks(shake, tmp, 1);
+                if (ret == 0) {
+                    rest -= full * FRODOKEM_SHAKE128_RATE;
+                    XMEMCPY(mat1_8 + lead + full * FRODOKEM_SHAKE128_RATE, tmp,
+                        rest);
+                }
+            }
+            if (ret == 0) {
+                frodokem_sample_matrix(mat1, cnt1, mat1_8, p);
+            }
+        }
+    }
+#endif
+
+    return ret;
+}
+
+/******************************************************************************/
+/* Generation of matrix A and matrix multiplications (Sections 6.7, 6.4).     */
+/******************************************************************************/
+
+/* Reduce a freshly generated A row (n coefficients) in place, once the AES/
+ * SHAKE output has been written into row as bytes. On big-endian each little-
+ * endian 16-bit value is reassembled (no reduction). Then, only when q != 2^16,
+ * the values are reduced mod q with qmask - for q == 2^16 (FrodoKEM-976/-1344)
+ * the 16-bit values are already valid coefficients so no mask is applied. The
+ * nbar (== 8) step is unrolled; n is a multiple of nbar.
+ *
+ * @param  [in, out]  row    A row bytes in; reduced coefficients out (n).
+ * @param  [in]       n      Number of coefficients in the row.
+ * @param  [in]       qmask  Reduction mask (q - 1); 0xFFFF when q == 2^16.
+ */
+static void frodokem_a_row_reduce(word16* row, int n, int qmask)
+{
+#ifdef BIG_ENDIAN_ORDER
+    /* Reassemble each little-endian 16-bit value in place (no reduction). */
+    {
+        const byte* rowBytes = (const byte*)row;
+        int k;
+
+        for (k = 0; k < n; k += FRODOKEM_NBAR) {
+            row[k + 0] = (word16)(rowBytes[2 * (k + 0)] |
+                ((word16)rowBytes[2 * (k + 0) + 1] << 8));
+            row[k + 1] = (word16)(rowBytes[2 * (k + 1)] |
+                ((word16)rowBytes[2 * (k + 1) + 1] << 8));
+            row[k + 2] = (word16)(rowBytes[2 * (k + 2)] |
+                ((word16)rowBytes[2 * (k + 2) + 1] << 8));
+            row[k + 3] = (word16)(rowBytes[2 * (k + 3)] |
+                ((word16)rowBytes[2 * (k + 3) + 1] << 8));
+            row[k + 4] = (word16)(rowBytes[2 * (k + 4)] |
+                ((word16)rowBytes[2 * (k + 4) + 1] << 8));
+            row[k + 5] = (word16)(rowBytes[2 * (k + 5)] |
+                ((word16)rowBytes[2 * (k + 5) + 1] << 8));
+            row[k + 6] = (word16)(rowBytes[2 * (k + 6)] |
+                ((word16)rowBytes[2 * (k + 6) + 1] << 8));
+            row[k + 7] = (word16)(rowBytes[2 * (k + 7)] |
+                ((word16)rowBytes[2 * (k + 7) + 1] << 8));
+        }
+    }
+#endif
+
+    /* Reduce mod q only when necessary: q == 2^16 (976/1344) values are already
+     * valid coefficients; only q == 2^15 (640) needs the reduction. */
+#ifdef FRODOKEM_D15_SHAKE128
+#ifdef FRODOKEM_D16_SHAKE256
+    if (qmask != 0xffff)
+#endif
+    {
+        int k;
+
+        for (k = 0; k < n; k += FRODOKEM_NBAR) {
+            row[k + 0] = (word16)(row[k + 0] & qmask);
+            row[k + 1] = (word16)(row[k + 1] & qmask);
+            row[k + 2] = (word16)(row[k + 2] & qmask);
+            row[k + 3] = (word16)(row[k + 3] & qmask);
+            row[k + 4] = (word16)(row[k + 4] & qmask);
+            row[k + 5] = (word16)(row[k + 5] & qmask);
+            row[k + 6] = (word16)(row[k + 6] & qmask);
+            row[k + 7] = (word16)(row[k + 7] & qmask);
+        }
+    }
+#endif
+
+#ifndef FRODOKEM_D15_SHAKE128
+    /* q == 2^16 only: no reduction (qmask unused). */
+    (void)qmask;
+#ifndef BIG_ENDIAN_ORDER
+    /* Little-endian too: nothing at all to do. */
+    (void)row;
+    (void)n;
+#endif
+#endif
+}
+
+#ifdef WOLFSSL_FRODOKEM_AES
+#ifndef FRODOKEM_HAVE_ARM_AES_ASM
+/* Per-row C generator. On ARM with the crypto extension the whole batch is done
+ * by frodokem_gen_a_rows_aes_arm, so this fallback is compiled out there. */
+/* Generate cnt consecutive rows of matrix A (indices i .. i+cnt-1) using
+ * AES-128 (Section 6.7.1). Each row is built, ECB-encrypted and reduced before
+ * the next - keeping it hot in L1 - which measured faster than one ECB call
+ * over the whole group (that spills the working set to L2). Consuming cnt rows
+ * per accumulate call is still a win over one dispatch call per row.
+ *
+ * @param  [out]  rows  Output rows (cnt * n coefficients, contiguous).
+ * @param  [in]   aes   AES object keyed with seedA.
+ * @param  [in]   i     Index of the first row.
+ * @param  [in]   cnt   Number of consecutive rows to generate.
+ * @param  [in]   p     FrodoKEM parameters.
+ * @return  0 on success.
+ * @return  Negative on error from AES.
+ */
+static int frodokem_gen_a_rows_aes(word16* rows, Aes* aes, int i, int cnt,
+    const FrodoKemParams* p)
+{
+    int ret;
+    int r;
+    int j;
+    int n = p->n;
+    byte* rowBytes = (byte*)rows;
+
+    ret = 0;
+    /* Process one row at a time (build its input blocks, ECB the whole row,
+     * reduce) so each row stays hot in L1 through all three passes - measured
+     * faster than one big ECB over the whole group, whose larger working set
+     * spills to L2. For row i + r, block j is i+r || j || 0..0 (16-bit LE) and
+     * an AES block yields FRODOKEM_NBAR (== 8) coefficients, so it lands at
+     * rows[r*n] + 2*j = rowBytes + r*(2*n) + 2*j. In-place ECB is safe: each
+     * block's output depends only on its own input. */
+    for (r = 0; (ret == 0) && (r < cnt); r++) {
+        byte* rowR = rowBytes + r * (2 * n);
+        int idx = i + r;
+
+        for (j = 0; j < n; j += FRODOKEM_NBAR) {
+            byte* blk = rowR + 2 * j;
+
+            blk[0] = (byte)(idx & 0xff);
+            blk[1] = (byte)((idx >> 8) & 0xff);
+            blk[2] = (byte)(j & 0xff);
+            blk[3] = (byte)((j >> 8) & 0xff);
+            XMEMSET(blk + 4, 0, 12);
+        }
+
+        ret = wc_AesEcbEncrypt(aes, rowR, rowR, (word32)(2 * n));
+        if (ret == 0) {
+            frodokem_a_row_reduce(rows + r * n, n, (int)p->qMask);
+        }
+    }
+
+    return ret;
+}
+#endif /* !FRODOKEM_HAVE_ARM_AES_ASM */
+#endif /* WOLFSSL_FRODOKEM_AES */
+
+#ifdef WOLFSSL_FRODOKEM_SHAKE
+/* The single-row scalar SHAKE generator is the fallback used when the 2-way
+ * NEON permute is unavailable: on non-NEON targets, and on AArch64 at run time
+ * when the CPU lacks Advanced SIMD. */
+#if !defined(FRODOKEM_HAVE_SHAKE_NEON) || defined(FRODOKEM_HAVE_NEON_RUNTIME)
+/* Generate a single row of matrix A using SHAKE128 (Section 6.7.2).
+ *
+ * The SHAKE output is squeezed into the row buffer itself and converted to
+ * coefficients in place (ascending index), avoiding a separate scratch buffer.
+ * The caller initializes shake as SHAKE-128 once before its row loop; each
+ * wc_Shake128_Final here resets the state ready for the next row.
+ *
+ * @param  [in]   shake Reusable SHAKE object (SHAKE-128, ready to absorb).
+ * @param  [out]  row   Output row of n coefficients (2 * n bytes).
+ * @param  [in]   seedA Seed for A (FRODOKEM_SEEDA_SZ bytes).
+ * @param  [in]   i     Row index.
+ * @param  [in]   p     FrodoKEM parameters.
+ * @return  0 on success.
+ * @return  Negative on error from the hash function.
+ */
+static int frodokem_gen_a_row_shake(wc_Shake* shake, word16* row,
+    const byte* seedA, int i, const FrodoKemParams* p)
+{
+    int ret;
+#ifdef WOLFSSL_X86_64_BUILD
+    byte* rowBytes = (byte*)row;
+    word64* state = shake->s;
+    word8 *state8 = (word8*)state;
+    word16 l;
+    static word16 inc = WC_SHA3_128_BLOCK_SIZE;
+
+    state8[0] = (byte)(i & 0xff);
+    state8[1] = (byte)((i >> 8) & 0xff);
+    XMEMCPY(state8 + 2, seedA, FRODOKEM_SEEDA_SZ);
+    state8[2 + FRODOKEM_SEEDA_SZ] = 0x1f;
+    XMEMSET(state8 + 2 + FRODOKEM_SEEDA_SZ + 1, 0,
+        sizeof(shake->s) - (2 + FRODOKEM_SEEDA_SZ + 1));
+    state8[WC_SHA3_128_COUNT * 8 - 1] = 0x80;
+
+    for (l = 0; l + inc < 2 * p->n; l += inc) {
+        BlockSha3(state);
+        XMEMCPY(rowBytes + l, state8, WC_SHA3_128_BLOCK_SIZE);
+    }
+    BlockSha3(state);
+    XMEMCPY(rowBytes + l, state8, (word32)(2 * p->n - l));
+
+    ret = 0;
+#else
+    byte in[2 + FRODOKEM_SEEDA_SZ];
+    byte* rowBytes = (byte*)row;
+
+    /* Row index as 16-bit little-endian, followed by seedA. */
+    in[0] = (byte)(i & 0xff);
+    in[1] = (byte)((i >> 8) & 0xff);
+    XMEMCPY(in + 2, seedA, FRODOKEM_SEEDA_SZ);
+
+    ret = wc_Shake128_Update(shake, in, (word32)sizeof(in));
+    if (ret == 0) {
+        ret = wc_Shake128_Final(shake, rowBytes, (word32)(2 * p->n));
+    }
+#endif
+
+    if (ret == 0) {
+        /* Endian fix-up and mod-q reduction, done once over the whole row. */
+        frodokem_a_row_reduce(row, p->n, (int)p->qMask);
+    }
+
+    return ret;
+}
+#endif /* !FRODOKEM_HAVE_SHAKE_NEON || FRODOKEM_HAVE_NEON_RUNTIME */
+
+#ifdef FRODOKEM_HAVE_SHAKE_X4
+/* Generate four rows of matrix A at once (indices i, i+1, i+2, i+3) with the
+ * 4-way AVX2 SHAKE-128 permutation (Section 6.7.2). Byte-identical to four
+ * frodokem_gen_a_row_shake calls, just computed in parallel lanes. Only built
+ * on little-endian x86 with AVX2 (the layout below assumes little-endian).
+ *
+ * @param  [out]  rows   Four contiguous output rows (4 * n coefficients).
+ * @param  [in]   seedA  Seed for A (FRODOKEM_SEEDA_SZ bytes).
+ * @param  [in]   i      Index of the first of the four rows (i % 4 == 0).
+ * @param  [in]   p      FrodoKEM parameters.
+ * @return  0 on success.
+ */
+static int frodokem_gen_a_row_shake_x4(word16* rows, const byte* seedA,
+    int i, const FrodoKemParams* p)
+{
+    /* Four interleaved SHAKE-128 states (Keccak-1600 is 25 word64 each): state
+     * word m of lane k is at state[m * 4 + k]. */
+    word64 state[25 * 4];
+    int n = p->n;
+    int lane;
+    /* seedA-derived state words - the same in all four lanes. The bytes each
+     * lane absorbs are: i (16-bit LE) || seedA (16) || 0x1f domain byte. */
+    word64 w0 = ((word64)seedA[0] << 16) | ((word64)seedA[1] << 24) |
+                ((word64)seedA[2] << 32) | ((word64)seedA[3] << 40) |
+                ((word64)seedA[4] << 48) | ((word64)seedA[5] << 56);
+    word64 w1 = (word64)seedA[6] | ((word64)seedA[7] << 8) |
+                ((word64)seedA[8] << 16) | ((word64)seedA[9] << 24) |
+                ((word64)seedA[10] << 32) | ((word64)seedA[11] << 40) |
+                ((word64)seedA[12] << 48) | ((word64)seedA[13] << 56);
+    word64 w2 = (word64)seedA[14] | ((word64)seedA[15] << 8) |
+                ((word64)0x1f << 16);
+
+    XMEMSET(state, 0, sizeof(state));
+    for (lane = 0; lane < 4; lane++) {
+        state[0 * 4 + lane] = w0 | (word64)((word32)(i + lane) & 0xffffU);
+        state[1 * 4 + lane] = w1;
+        state[2 * 4 + lane] = w2;
+        /* pad10*1: top bit of last rate byte (byte 167 = word 20, byte 7). */
+        state[(WC_SHA3_128_COUNT - 1) * 4 + lane] = (word64)0x80 << 56;
+    }
+
+    /* Squeeze 2*n bytes per lane, de-interleaved into the four rows. The rows
+     * are contiguous, so lane k's output lands at (byte*)rows + k * (2 * n) =
+     * rows + k * n, matching the row layout. 2*n is a multiple of 8. */
+    sha3_blocksx4_out_avx2(state, (byte*)rows, (word32)(2 * n));
+
+    /* 640 (q == 2^15) needs the mod-q mask; 976/1344 (q == 2^16) are already
+     * valid coefficients. The four rows are contiguous, so mask all 4*n at
+     * once with a single AVX2 pass (faster than four scalar per-lane passes;
+     * we are already inside the AVX2 vector-register region). */
+    if (p->qMask != 0xffff) {
+        frodokem_a_rows_reduce_avx2(rows, (word32)(4 * n), (int)p->qMask);
+    }
+
+    return 0;
+}
+#endif /* FRODOKEM_HAVE_SHAKE_X4 */
+
+#ifdef FRODOKEM_HAVE_SHAKE_X8
+/* Generate eight rows of matrix A at once (indices i .. i+7) with the 8-way
+ * AVX512 SHAKE-128 permutation (Section 6.7.2). Byte-identical to eight
+ * frodokem_gen_a_row_shake calls, just computed in parallel lanes. Only built
+ * on little-endian x86 with AVX512 (the layout below assumes little-endian).
+ *
+ * @param  [out]  rows   Eight contiguous output rows (8 * n coefficients).
+ * @param  [in]   seedA  Seed for A (FRODOKEM_SEEDA_SZ bytes).
+ * @param  [in]   i      Index of the first of the eight rows (i % 8 == 0).
+ * @param  [in]   p      FrodoKEM parameters.
+ * @return  0 on success.
+ */
+static int frodokem_gen_a_row_shake_x8(word16* rows, const byte* seedA,
+    int i, const FrodoKemParams* p)
+{
+    /* Eight interleaved SHAKE-128 states (Keccak-1600 is 25 word64 each): state
+     * word m of lane k is at state[m * 8 + k]. */
+    word64 state[25 * 8];
+    int n = p->n;
+    int lane;
+    /* seedA-derived state words - the same in all eight lanes. The bytes each
+     * lane absorbs are: i (16-bit LE) || seedA (16) || 0x1f domain byte. */
+    word64 w0 = ((word64)seedA[0] << 16) | ((word64)seedA[1] << 24) |
+                ((word64)seedA[2] << 32) | ((word64)seedA[3] << 40) |
+                ((word64)seedA[4] << 48) | ((word64)seedA[5] << 56);
+    word64 w1 = (word64)seedA[6] | ((word64)seedA[7] << 8) |
+                ((word64)seedA[8] << 16) | ((word64)seedA[9] << 24) |
+                ((word64)seedA[10] << 32) | ((word64)seedA[11] << 40) |
+                ((word64)seedA[12] << 48) | ((word64)seedA[13] << 56);
+    word64 w2 = (word64)seedA[14] | ((word64)seedA[15] << 8) |
+                ((word64)0x1f << 16);
+
+    XMEMSET(state, 0, sizeof(state));
+    for (lane = 0; lane < 8; lane++) {
+        state[0 * 8 + lane] = w0 | (word64)((word32)(i + lane) & 0xffffU);
+        state[1 * 8 + lane] = w1;
+        state[2 * 8 + lane] = w2;
+        /* pad10*1: top bit of last rate byte (byte 167 = word 20, byte 7). */
+        state[(WC_SHA3_128_COUNT - 1) * 8 + lane] = (word64)0x80 << 56;
+    }
+
+    /* Squeeze 2*n bytes per lane, de-interleaved into the eight rows. Lane k's
+     * output lands at (byte*)rows + k * (2 * n) = rows + k * n. 2*n is a
+     * multiple of 8. */
+    sha3_blocksx8_out_avx512(state, (byte*)rows, (word32)(2 * n));
+
+    /* 640 (q == 2^15) needs the mod-q mask; 976/1344 (q == 2^16) are already
+     * valid coefficients. The eight rows are contiguous, so mask all 8*n at
+     * once (the AVX2 ymm pass runs fine inside this AVX512 region). */
+    if (p->qMask != 0xffff) {
+        frodokem_a_rows_reduce_avx2(rows, (word32)(8 * n), (int)p->qMask);
+    }
+
+    return 0;
+}
+#endif /* FRODOKEM_HAVE_SHAKE_X8 */
+
+#ifdef FRODOKEM_HAVE_SHAKE_NEON
+/* Generate two rows of matrix A at once (indices i, i+1) with the 2-way NEON
+ * SHAKE-128 permutation (Section 6.7.2). Byte-identical to two
+ * frodokem_gen_a_row_shake calls. The two Keccak states are laid out
+ * contiguously - state[0..24] for row i, state[25..49] for row i+1 - so each
+ * squeezed rate block copies straight to its row (no lane de-interleave).
+ *
+ * @param  [out]  rows   Two contiguous output rows (2 * n coefficients).
+ * @param  [in]   seedA  Seed for A (FRODOKEM_SEEDA_SZ bytes).
+ * @param  [in]   i      Index of the first of the two rows (i % 2 == 0).
+ * @param  [in]   p      FrodoKEM parameters.
+ * @return  0 on success.
+ */
+static int frodokem_gen_a_row_shake_x2(word16* rows, const byte* seedA,
+    int i, const FrodoKemParams* p)
+{
+    /* Two 25-word Keccak states, contiguous: A then B. */
+    word64 state[50];
+    int n = p->n;
+    int total = 2 * n;              /* output bytes per row */
+    int off;
+    byte* sA = (byte*)(state + 0);
+    byte* sB = (byte*)(state + 25);
+    byte* rowA = (byte*)(rows + 0);
+    byte* rowB = (byte*)(rows + n);
+
+    XMEMSET(state, 0, sizeof(state));
+    /* Absorb i || seedA || 0x1f pad, with pad10*1 top bit at the last rate
+     * byte (167). State A is index i, state B is index i+1. */
+    sA[0] = (byte)(i & 0xff);
+    sA[1] = (byte)((i >> 8) & 0xff);
+    XMEMCPY(sA + 2, seedA, FRODOKEM_SEEDA_SZ);
+    sA[2 + FRODOKEM_SEEDA_SZ] = 0x1f;
+    sA[WC_SHA3_128_COUNT * 8 - 1] = 0x80;
+    sB[0] = (byte)((i + 1) & 0xff);
+    sB[1] = (byte)(((i + 1) >> 8) & 0xff);
+    XMEMCPY(sB + 2, seedA, FRODOKEM_SEEDA_SZ);
+    sB[2 + FRODOKEM_SEEDA_SZ] = 0x1f;
+    sB[WC_SHA3_128_COUNT * 8 - 1] = 0x80;
+
+    /* Squeeze 2*n bytes per row, a rate block per permutation. */
+    for (off = 0; off < total; off += WC_SHA3_128_BLOCK_SIZE) {
+        int blk = total - off;
+
+        if (blk > WC_SHA3_128_BLOCK_SIZE) {
+            blk = WC_SHA3_128_BLOCK_SIZE;
+        }
+#ifdef WOLFSSL_ARMASM_CRYPTO_SHA3
+        /* Use the SHA3 crypto extension (EOR3/RAX1/XAR/BCAX) when present. */
+        if (IS_AARCH64_SHA3(cpuid_flags)) {
+            frodokem_sha3_x2_crypto(state);
+        }
+        else
+#endif
+        {
+            frodokem_sha3_x2_neon(state);
+        }
+        XMEMCPY(rowA + off, sA, (size_t)blk);
+        XMEMCPY(rowB + off, sB, (size_t)blk);
+    }
+
+    frodokem_a_row_reduce(rows + 0, n, (int)p->qMask);
+    frodokem_a_row_reduce(rows + n, n, (int)p->qMask);
+
+    return 0;
+}
+#endif /* FRODOKEM_HAVE_SHAKE_NEON */
+#endif /* WOLFSSL_FRODOKEM_SHAKE */
+
+/* The A-generating multiplies below have separate implementations per matrix-A
+ * method (AES-128 or SHAKE128) so the method is chosen once, not per row. */
+
+/* The scalar-C accumulates are the fallback for CPUs without the wide SIMD or
+ * packed accumulate. AArch32 NEON and Thumb2 always use their asm, so these are
+ * not compiled there; on AArch64 they are compiled as the run-time fallback for
+ * CPUs without Advanced SIMD (NEON). */
+#if !defined(FRODOKEM_HAVE_ARM_ASM) || defined(FRODOKEM_HAVE_NEON_RUNTIME)
+/* Accumulate the contribution of one generated A row (index j) into S * A:
+ * out[i*n + k] += s[i*n + j] * row[k] for i in [0, nbar), k in [0, n).
+ *
+ * Each output row is a separate contiguous accumulation stream. Flagging row
+ * and the output slice as non-aliasing (they never overlap) lets the compiler
+ * vectorize each stream; a form that updates all nbar rows in one fused loop
+ * has too many possibly-aliasing pointers and stays scalar. Shared by the AES
+ * and SHAKE paths (the arithmetic is identical; only A generation differs).
+ *
+ * @param  [in, out]  out  S * A accumulator (nbar * n, row-major).
+ * @param  [in]       s    Matrix S (nbar * n, row-major).
+ * @param  [in]       row  Generated A row j (n coefficients).
+ * @param  [in]       j    Index of the generated A row.
+ * @param  [in]       n    Matrix dimension n.
+ */
+static void frodokem_sa_accum(word16* out, const word16* s,
+    const word16* FRODOKEM_RESTRICT row, int j, int n)
+{
+    int k;
+#ifdef WOLFSSL_FRODOKEM_SMALL
+    int i;
+
+    for (i = 0; i < FRODOKEM_NBAR; i++) {
+        word16 sij = s[i * n + j];
+        word16* FRODOKEM_RESTRICT o = out + i * n;
+
+        for (k = 0; k < n; k++) {
+            o[k] = (word16)(o[k] + sij * row[k]);
+        }
+    }
+#else
+#if FRODOKEM_NBAR != 8
+    #error "Unrolled frodokem_sa_accum assumes FRODOKEM_NBAR == 8."
+#endif
+    /* Unroll the nbar (== 8) output rows into 8 independent contiguous streams.
+     * Each stays a single-stream accumulation the compiler vectorizes; the
+     * unroll drops the outer-loop overhead. A single advancing output pointer,
+     * with each S coefficient read just before its loop, keeps only one stream
+     * live at a time (low register pressure). The output slice, wherever o
+     * points, never overlaps row. */
+    word16* FRODOKEM_RESTRICT o = out;
+    word16 sij;
+
+    sij = s[0 * n + j];
+    for (k = 0; k < n; k++) {
+        o[k] = (word16)(o[k] + sij * row[k]);
+    }
+    o += n;
+    sij = s[1 * n + j];
+    for (k = 0; k < n; k++) {
+        o[k] = (word16)(o[k] + sij * row[k]);
+    }
+    o += n;
+    sij = s[2 * n + j];
+    for (k = 0; k < n; k++) {
+        o[k] = (word16)(o[k] + sij * row[k]);
+    }
+    o += n;
+    sij = s[3 * n + j];
+    for (k = 0; k < n; k++) {
+        o[k] = (word16)(o[k] + sij * row[k]);
+    }
+    o += n;
+    sij = s[4 * n + j];
+    for (k = 0; k < n; k++) {
+        o[k] = (word16)(o[k] + sij * row[k]);
+    }
+    o += n;
+    sij = s[5 * n + j];
+    for (k = 0; k < n; k++) {
+        o[k] = (word16)(o[k] + sij * row[k]);
+    }
+    o += n;
+    sij = s[6 * n + j];
+    for (k = 0; k < n; k++) {
+        o[k] = (word16)(o[k] + sij * row[k]);
+    }
+    o += n;
+    sij = s[7 * n + j];
+    for (k = 0; k < n; k++) {
+        o[k] = (word16)(o[k] + sij * row[k]);
+    }
+#endif
+}
+
+/* Accumulate the contribution of one generated A row (index i) into A * S:
+ * out[i*nbar + k] += sum_j row[j] * s[k*n + j] for k in [0, nbar). row is the
+ * generated A row i; column k of S is row k of S^T. out holds E on entry, so
+ * the accumulation is in place. Shared by the AES and SHAKE paths (the
+ * arithmetic is identical; only A generation differs).
+ *
+ * @param  [in, out]  out  A * S accumulator (n * nbar, row-major).
+ * @param  [in]       s    Matrix S^T (nbar * n, row-major).
+ * @param  [in]       row  Generated A row i (n coefficients).
+ * @param  [in]       i    Index of the generated A row.
+ * @param  [in]       n    Matrix dimension n.
+ */
+static void frodokem_as_accum(word16* out, const word16* s,
+    const word16* row, int i, int n)
+{
+    int j;
+#ifdef WOLFSSL_FRODOKEM_SMALL
+    int k;
+
+    for (k = 0; k < FRODOKEM_NBAR; k++) {
+        /* column k of S == row k of S^T. out holds E on entry; word16 wraps
+         * mod 2^16, reduced to mod q by the final mask. */
+        const word16* sCol = s + k * n;
+        word16 acc = out[i * FRODOKEM_NBAR + k];
+
+        for (j = 0; j < n; j++) {
+            acc = (word16)(acc + row[j] * sCol[j]);
+        }
+        out[i * FRODOKEM_NBAR + k] = acc;
+    }
+#else
+#if FRODOKEM_NBAR != 8
+    #error "Unrolled frodokem_as_accum assumes FRODOKEM_NBAR == 8."
+#endif
+    /* Unroll the nbar (== 8) output columns: keep 8 accumulators so each A
+     * element row[j] is loaded once and reused across all 8 columns (8x less
+     * row traffic than a separate dot product per column). Column k of S is
+     * row k of S^T (s + k*n); out holds E on entry. */
+    const word16* s0 = s;
+    const word16* s1 = s + n;
+    const word16* s2 = s + 2 * n;
+    const word16* s3 = s + 3 * n;
+    const word16* s4 = s + 4 * n;
+    const word16* s5 = s + 5 * n;
+    const word16* s6 = s + 6 * n;
+    const word16* s7 = s + 7 * n;
+    word16* outRow = out + i * FRODOKEM_NBAR;
+    word16 a0 = outRow[0];
+    word16 a1 = outRow[1];
+    word16 a2 = outRow[2];
+    word16 a3 = outRow[3];
+    word16 a4 = outRow[4];
+    word16 a5 = outRow[5];
+    word16 a6 = outRow[6];
+    word16 a7 = outRow[7];
+
+    for (j = 0; j < n; j++) {
+        word16 rj = row[j];
+
+        a0 = (word16)(a0 + rj * s0[j]);
+        a1 = (word16)(a1 + rj * s1[j]);
+        a2 = (word16)(a2 + rj * s2[j]);
+        a3 = (word16)(a3 + rj * s3[j]);
+        a4 = (word16)(a4 + rj * s4[j]);
+        a5 = (word16)(a5 + rj * s5[j]);
+        a6 = (word16)(a6 + rj * s6[j]);
+        a7 = (word16)(a7 + rj * s7[j]);
+    }
+
+    outRow[0] = a0;
+    outRow[1] = a1;
+    outRow[2] = a2;
+    outRow[3] = a3;
+    outRow[4] = a4;
+    outRow[5] = a5;
+    outRow[6] = a6;
+    outRow[7] = a7;
+#endif
+}
+#endif /* !FRODOKEM_HAVE_ARM_ASM || FRODOKEM_HAVE_NEON_RUNTIME */
+
+#ifdef WOLFSSL_FRODOKEM_AES
+/* Compute out += A * S (out holds E on entry) with A generated by AES-128.
+ * See frodokem_mul_add_as_plus_e for the interface.
+ *
+ * @param  [in, out]  out    E on entry; out + A*S on exit (n * nbar).
+ * @param  [in]       s      Secret matrix S^T (nbar * n, row-major).
+ * @param  [in]       seedA  Seed for matrix-A generation (FRODOKEM_SEEDA_SZ).
+ * @param  [in]       p      FrodoKEM parameters.
+ * @param  [out]      row    Scratch for the generated A rows (ROW_MULT * n).
+ * @param  [in]       aes    AES object for matrix-A generation.
+ * @return  0 on success, negative on error.
+ */
+static int frodokem_mul_add_as_plus_e_aes(word16* out, const word16* s,
+    const byte* seedA, const FrodoKemParams* p, word16* row, Aes* aes)
+{
+    int ret;
+    int i;
+    int n = p->n;
+
+    /* Key the reusable AES object with seedA (the object is wc_AesInit'd once
+     * in wc_FrodoKemKey_Init, so no per-call init/free here). */
+    ret = wc_AesSetKeyDirect(aes, seedA, FRODOKEM_SEEDA_SZ, NULL,
+        AES_ENCRYPTION);
+
+    /* Generate the A rows in batches matching the fused asm accumulate (eight
+     * with AVX512, four with AVX2); n is a multiple of 8 for every set. */
+#ifdef FRODOKEM_HAVE_MATRIX_ASM_AVX512
+    if ((ret == 0) && IS_INTEL_AVX512(cpuid_flags) &&
+            (SAVE_VECTOR_REGISTERS2() == 0)) {
+        for (i = 0; i < n; i += 8) {
+            /* Widest matrix-A generator available at run time (cf. aes.c):
+             * VAES (whole batch in one asm call), else the AES-NI register
+             * kernel, else the per-row C generator. */
+#ifdef FRODOKEM_HAVE_MATRIX_ASM_VAES
+            if (IS_INTEL_VAES(cpuid_flags)) {
+                frodokem_gen_a_rows_aes_avx512((byte*)row, row,
+                    (const byte*)aes->key, i, 8, n, (int)p->qMask);
+            }
+            else
+#endif
+            if (IS_INTEL_AESNI(cpuid_flags)) {
+                frodokem_gen_a_rows_aes_aesni((byte*)row, row,
+                    (const byte*)aes->key, i, 8, n, (int)p->qMask);
+            }
+            else {
+                ret = frodokem_gen_a_rows_aes(row, aes, i, 8, p);
+                if (ret != 0) {
+                    break;
+                }
+            }
+            frodokem_as_accum_avx512(out, s, row, i, n);
+        }
+        RESTORE_VECTOR_REGISTERS();
+    }
+    else
+#endif
+#ifdef FRODOKEM_HAVE_MATRIX_ASM
+    if ((ret == 0) && IS_INTEL_AVX2(cpuid_flags) &&
+            (SAVE_VECTOR_REGISTERS2() == 0)) {
+        for (i = 0; i < n; i += 4) {
+            /* Widest matrix-A generator available at run time (cf. aes.c):
+             * VAES (whole batch in one asm call), else the AES-NI register
+             * kernel, else the per-row C generator. */
+#ifdef FRODOKEM_HAVE_MATRIX_ASM_VAES
+            if (IS_INTEL_VAES(cpuid_flags)) {
+                frodokem_gen_a_rows_aes_avx2((byte*)row, row,
+                    (const byte*)aes->key, i, 4, n, (int)p->qMask);
+            }
+            else
+#endif
+            if (IS_INTEL_AESNI(cpuid_flags)) {
+                frodokem_gen_a_rows_aes_aesni((byte*)row, row,
+                    (const byte*)aes->key, i, 4, n, (int)p->qMask);
+            }
+            else {
+                ret = frodokem_gen_a_rows_aes(row, aes, i, 4, p);
+                if (ret != 0) {
+                    break;
+                }
+            }
+            frodokem_as_accum_avx2(out, s, row, i, n);
+        }
+        RESTORE_VECTOR_REGISTERS();
+    }
+    else
+#endif
+#ifdef FRODOKEM_HAVE_ARM_ASM
+    {
+        for (i = 0; i < n; i += FRODOKEM_ROW_MULT) {
+        #ifdef FRODOKEM_HAVE_ARM_AES_ASM
+            /* The whole build + AES-ECB + reduce is one crypto-extension asm
+             * call (in place). */
+            frodokem_gen_a_rows_aes_arm((byte*)row, row,
+                (const byte*)aes->key, i, FRODOKEM_ROW_MULT, n, (int)p->qMask);
+        #else
+            ret = frodokem_gen_a_rows_aes(row, aes, i, FRODOKEM_ROW_MULT, p);
+            if (ret != 0) {
+                break;
+            }
+        #endif
+            /* Accumulate the whole FRODOKEM_ROW_MULT-row batch just generated. */
+            FRODOKEM_AS_ACCUM_BATCH(out, s, row, i, n);
+        }
+    }
+#else
+    {
+        for (i = 0; (ret == 0) && (i < n); i++) {
+            ret = frodokem_gen_a_rows_aes(row, aes, i, 1, p);
+            if (ret == 0) {
+                frodokem_as_accum(out, s, row, i, n);
+            }
+        }
+    }
+#endif /* FRODOKEM_HAVE_ARM_ASM */
+
+#ifdef FRODOKEM_D15_SHAKE128
+    /* q == 2^15 (640) needs a final reduction mod q; for q == 2^16 (976/1344)
+     * the word16 accumulation already reduced. */
+#ifdef FRODOKEM_D16_SHAKE256
+    if ((ret == 0) && (p->qMask != 0xffff))
+#else
+    if (ret == 0)
+#endif
+    {
+        for (i = 0; i < FRODOKEM_NBAR * n; i++) {
+            out[i] = (word16)(out[i] & p->qMask);
+        }
+    }
+#endif /* FRODOKEM_D15_SHAKE128 */
+
+    return ret;
+}
+#endif /* WOLFSSL_FRODOKEM_AES */
+
+#ifdef WOLFSSL_FRODOKEM_SHAKE
+/* Compute out += A * S (out holds E on entry) with A generated by SHAKE128.
+ * See frodokem_mul_add_as_plus_e for the interface.
+ *
+ * @param  [in, out]  out    E on entry; out + A*S on exit (n * nbar).
+ * @param  [in]       s      Secret matrix S^T (nbar * n, row-major).
+ * @param  [in]       seedA  Seed for matrix-A generation (FRODOKEM_SEEDA_SZ).
+ * @param  [in]       p      FrodoKEM parameters.
+ * @param  [out]      row    Scratch for the generated A rows (ROW_MULT * n).
+ * @param  [in]       shake  SHAKE object for matrix-A generation.
+ * @return  0 on success, negative on error.
+ */
+static int frodokem_mul_add_as_plus_e_shake(word16* out, const word16* s,
+    const byte* seedA, const FrodoKemParams* p, word16* row, wc_Shake* shake)
+{
+    int ret;
+    int i;
+    int n = p->n;
+
+#ifdef FRODOKEM_HAVE_SHAKE_X8
+    /* When AVX512 is available, generate and consume eight A rows at a time
+     * with the 8-way SHAKE permutation. n is a multiple of 8 for every param
+     * set, so there is no remainder loop. */
+    if (IS_INTEL_AVX512(cpuid_flags) && (SAVE_VECTOR_REGISTERS2() == 0)) {
+        ret = 0;
+        for (i = 0; i < n; i += 8) {
+            ret = frodokem_gen_a_row_shake_x8(row, seedA, i, p);
+            if (ret != 0) {
+                break;
+            }
+            /* Fused 8-row A * S accumulate for all nbar output columns. */
+            frodokem_as_accum_avx512(out, s, row, i, n);
+        }
+        RESTORE_VECTOR_REGISTERS();
+    }
+    else
+#endif /* FRODOKEM_HAVE_SHAKE_X8 */
+#ifdef FRODOKEM_HAVE_SHAKE_X4
+    /* When AVX2 is available, generate and consume four A rows at a time with
+     * the 4-way SHAKE permutation. n is a multiple of 4 for every parameter
+     * set, so there is no remainder loop. */
+    if (IS_INTEL_AVX2(cpuid_flags) && (SAVE_VECTOR_REGISTERS2() == 0)) {
+        ret = 0;
+        for (i = 0; i < n; i += 4) {
+            ret = frodokem_gen_a_row_shake_x4(row, seedA, i, p);
+            if (ret != 0) {
+                break;
+            }
+            /* Fused 4-row A * S accumulate for all nbar output columns. */
+            frodokem_as_accum_avx2(out, s, row, i, n);
+        }
+        RESTORE_VECTOR_REGISTERS();
+    }
+    else
+#endif /* FRODOKEM_HAVE_SHAKE_X4 */
+#ifdef FRODOKEM_HAVE_SHAKE_NEON
+    /* AArch64: with Advanced SIMD, generate and consume two A rows at a time
+     * with the 2-way NEON SHAKE; without it, fall back to the scalar SHAKE and
+     * portable-C accumulate (the 2-way permute is itself NEON). n is a multiple
+     * of 2 for every set. */
+    if (IS_AARCH64_ASIMD(cpuid_flags)) {
+        (void)shake;
+        ret = 0;
+        for (i = 0; i < n; i += 2) {
+            ret = frodokem_gen_a_row_shake_x2(row, seedA, i, p);
+            if (ret != 0) {
+                break;
+            }
+            FRODOKEM_AS_ACCUM(out, s, row, i, n);
+        }
+    }
+    else {
+        ret = wc_InitShake128(shake, NULL, INVALID_DEVID);
+        for (i = 0; (ret == 0) && (i < n); i++) {
+            ret = frodokem_gen_a_row_shake(shake, row, seedA, i, p);
+            if (ret == 0) {
+                frodokem_as_accum(out, s, row, i, n);
+            }
+        }
+    }
+#elif defined(FRODOKEM_HAVE_ARM_ASM)
+    /* AArch32 (NEON or Thumb2): no parallel Keccak, so generate one A row at a
+     * time with SHAKE-128 and consume it with the AArch32 A * S accumulate. */
+    {
+        ret = wc_InitShake128(shake, NULL, INVALID_DEVID);
+
+        for (i = 0; (ret == 0) && (i < n); i++) {
+            ret = frodokem_gen_a_row_shake(shake, row, seedA, i, p);
+            if (ret == 0) {
+                FRODOKEM_AS_ACCUM(out, s, row, i, n);
+            }
+        }
+    }
+#else
+    {
+        /* Matrix A is generated with SHAKE-128 for every parameter set.
+         * Initialize the shared object once here; each row's Final resets it
+         * for the next. */
+        ret = wc_InitShake128(shake, NULL, INVALID_DEVID);
+
+        for (i = 0; (ret == 0) && (i < n); i++) {
+            ret = frodokem_gen_a_row_shake(shake, row, seedA, i, p);
+            if (ret == 0) {
+                frodokem_as_accum(out, s, row, i, n);
+            }
+        }
+    }
+#endif /* FRODOKEM_HAVE_SHAKE_NEON */
+
+#ifdef FRODOKEM_D15_SHAKE128
+    /* q == 2^15 (640) needs a final reduction mod q; for q == 2^16 (976/1344)
+     * the word16 accumulation already reduced. */
+#ifdef FRODOKEM_D16_SHAKE256
+    if ((ret == 0) && (p->qMask != 0xffff))
+#else
+    if (ret == 0)
+#endif
+    {
+        for (i = 0; i < FRODOKEM_NBAR * n; i++) {
+            out[i] = (word16)(out[i] & p->qMask);
+        }
+    }
+#endif /* FRODOKEM_D15_SHAKE128 */
+
+    return ret;
+}
+#endif /* WOLFSSL_FRODOKEM_SHAKE */
+
+/* Compute out = out + A * S, accumulating A * S into out which holds the error
+ * matrix E on entry (all n x nbar); A is n x n and S is n x nbar. Taking E in
+ * place saves the caller a copy (it samples E straight into out).
+ *
+ * S is supplied transposed (S^T, nbar x n) so column k of S is row k of s.
+ * seedA, the parameters and the SHAKE / AES objects for matrix-A generation are
+ * taken from key; dispatch is by the key parameters' useAes.
+ *
+ * @param  [in]       key    FrodoKEM key (provides seedA, params, SHAKE / AES).
+ * @param  [in, out]  out    E on entry; out + A*S on exit (n * nbar).
+ * @param  [in]       s      Secret matrix S^T (nbar * n, row-major).
+ * @param  [out]      row    Scratch for the generated A rows (ROW_MULT * n).
+ * @return  0 on success, negative on error.
+ */
+int frodokem_mul_add_as_plus_e(FrodoKemKey* key, word16* out, const word16* s,
+    word16* row)
+{
+    const FrodoKemParams* p = key->params;
+    int ret;
+
+#ifdef WOLFSSL_FRODOKEM_AES
+    if (p->useAes) {
+        ret = frodokem_mul_add_as_plus_e_aes(out, s, key->seedA, p, row,
+            &key->aes);
+    }
+    else
+#endif
+    {
+#ifdef WOLFSSL_FRODOKEM_SHAKE
+        ret = frodokem_mul_add_as_plus_e_shake(out, s, key->seedA, p, row,
+            &key->shake);
+#else
+        ret = NOT_COMPILED_IN;
+#endif
+    }
+
+    return ret;
+}
+
+#ifdef WOLFSSL_FRODOKEM_AES
+/* Compute out = S * A + E (out holds E on entry) with A generated by AES-128.
+ * See frodokem_mul_add_sa_plus_e for the interface.
+ *
+ * @param  [in, out]  out    E on entry; out + S*A on exit (nbar * n).
+ * @param  [in]       s      Matrix S (nbar * n, row-major).
+ * @param  [in]       seedA  Seed for matrix-A generation (FRODOKEM_SEEDA_SZ).
+ * @param  [in]       p      FrodoKEM parameters.
+ * @param  [out]      row    Scratch for the generated A rows (ROW_MULT * n).
+ * @param  [in]       aes    AES object for matrix-A generation.
+ * @return  0 on success, negative on error.
+ */
+static int frodokem_mul_add_sa_plus_e_aes(word16* out, const word16* s,
+    const byte* seedA, const FrodoKemParams* p, word16* row, Aes* aes)
+{
+    int ret;
+#ifdef FRODOKEM_D15_SHAKE128
+    int i;
+#endif
+    int j;
+    int n = p->n;
+
+    /* Key the reusable AES object with seedA (the object is wc_AesInit'd once
+     * in wc_FrodoKemKey_Init, so no per-call init/free here). */
+    ret = wc_AesSetKeyDirect(aes, seedA, FRODOKEM_SEEDA_SZ, NULL,
+        AES_ENCRYPTION);
+
+    /* Generate the A rows in batches matching the fused asm accumulate (eight
+     * with AVX512, four with AVX2); n is a multiple of 8 for every set. */
+#ifdef FRODOKEM_HAVE_MATRIX_ASM_AVX512
+    if ((ret == 0) && IS_INTEL_AVX512(cpuid_flags) &&
+            (SAVE_VECTOR_REGISTERS2() == 0)) {
+        for (j = 0; j < n; j += 8) {
+            /* Widest matrix-A generator available at run time (cf. aes.c):
+             * VAES (whole batch in one asm call), else the AES-NI register
+             * kernel, else the per-row C generator. */
+#ifdef FRODOKEM_HAVE_MATRIX_ASM_VAES
+            if (IS_INTEL_VAES(cpuid_flags)) {
+                frodokem_gen_a_rows_aes_avx512((byte*)row, row,
+                    (const byte*)aes->key, j, 8, n, (int)p->qMask);
+            }
+            else
+#endif
+            if (IS_INTEL_AESNI(cpuid_flags)) {
+                frodokem_gen_a_rows_aes_aesni((byte*)row, row,
+                    (const byte*)aes->key, j, 8, n, (int)p->qMask);
+            }
+            else {
+                ret = frodokem_gen_a_rows_aes(row, aes, j, 8, p);
+                if (ret != 0) {
+                    break;
+                }
+            }
+            frodokem_sa_accum_avx512(out, s, row, j, n);
+        }
+        RESTORE_VECTOR_REGISTERS();
+    }
+    else
+#endif
+#ifdef FRODOKEM_HAVE_MATRIX_ASM
+    if ((ret == 0) && IS_INTEL_AVX2(cpuid_flags) &&
+            (SAVE_VECTOR_REGISTERS2() == 0)) {
+        for (j = 0; j < n; j += 4) {
+            /* Widest matrix-A generator available at run time (cf. aes.c):
+             * VAES (whole batch in one asm call), else the AES-NI register
+             * kernel, else the per-row C generator. */
+#ifdef FRODOKEM_HAVE_MATRIX_ASM_VAES
+            if (IS_INTEL_VAES(cpuid_flags)) {
+                frodokem_gen_a_rows_aes_avx2((byte*)row, row,
+                    (const byte*)aes->key, j, 4, n, (int)p->qMask);
+            }
+            else
+#endif
+            if (IS_INTEL_AESNI(cpuid_flags)) {
+                frodokem_gen_a_rows_aes_aesni((byte*)row, row,
+                    (const byte*)aes->key, j, 4, n, (int)p->qMask);
+            }
+            else {
+                ret = frodokem_gen_a_rows_aes(row, aes, j, 4, p);
+                if (ret != 0) {
+                    break;
+                }
+            }
+            frodokem_sa_accum_avx2(out, s, row, j, n);
+        }
+        RESTORE_VECTOR_REGISTERS();
+    }
+    else
+#endif
+#ifdef FRODOKEM_HAVE_ARM_ASM
+    {
+        for (j = 0; j < n; j += FRODOKEM_ROW_MULT) {
+        #ifdef FRODOKEM_HAVE_ARM_AES_ASM
+            /* The whole build + AES-ECB + reduce is one crypto-extension asm
+             * call (in place). */
+            frodokem_gen_a_rows_aes_arm((byte*)row, row,
+                (const byte*)aes->key, j, FRODOKEM_ROW_MULT, n, (int)p->qMask);
+        #else
+            ret = frodokem_gen_a_rows_aes(row, aes, j, FRODOKEM_ROW_MULT, p);
+            if (ret != 0) {
+                break;
+            }
+        #endif
+            /* Accumulate the whole FRODOKEM_ROW_MULT-row batch just generated. */
+            FRODOKEM_SA_ACCUM_BATCH(out, s, row, j, n);
+        }
+    }
+#else
+    {
+        /* Generate A a row at a time (index j) and accumulate its part. */
+        for (j = 0; (ret == 0) && (j < n); j++) {
+            ret = frodokem_gen_a_rows_aes(row, aes, j, 1, p);
+            if (ret == 0) {
+                frodokem_sa_accum(out, s, row, j, n);
+            }
+        }
+    }
+#endif /* FRODOKEM_HAVE_ARM_ASM */
+
+#ifdef FRODOKEM_D15_SHAKE128
+    /* q == 2^15 (640) needs a final reduction mod q; for q == 2^16 (976/1344)
+     * the word16 accumulation already reduced. */
+#ifdef FRODOKEM_D16_SHAKE256
+    if ((ret == 0) && (p->qMask != 0xffff))
+#else
+    if (ret == 0)
+#endif
+    {
+        for (i = 0; i < FRODOKEM_NBAR * n; i++) {
+            out[i] = (word16)(out[i] & p->qMask);
+        }
+    }
+#endif /* FRODOKEM_D15_SHAKE128 */
+
+    return ret;
+}
+#endif /* WOLFSSL_FRODOKEM_AES */
+
+#ifdef WOLFSSL_FRODOKEM_SHAKE
+/* Compute out = S * A + E (out holds E on entry) with A generated by SHAKE128.
+ * See frodokem_mul_add_sa_plus_e for the interface.
+ *
+ * @param  [in, out]  out    E on entry; out + S*A on exit (nbar * n).
+ * @param  [in]       s      Matrix S (nbar * n, row-major).
+ * @param  [in]       seedA  Seed for matrix-A generation (FRODOKEM_SEEDA_SZ).
+ * @param  [in]       p      FrodoKEM parameters.
+ * @param  [out]      row    Scratch for the generated A rows (ROW_MULT * n).
+ * @param  [in]       shake  SHAKE object for matrix-A generation.
+ * @return  0 on success, negative on error.
+ */
+static int frodokem_mul_add_sa_plus_e_shake(word16* out, const word16* s,
+    const byte* seedA, const FrodoKemParams* p, word16* row, wc_Shake* shake)
+{
+    int ret;
+#ifdef FRODOKEM_D15_SHAKE128
+    int i;
+#endif
+    int j;
+    int n = p->n;
+
+#ifdef FRODOKEM_HAVE_SHAKE_X8
+    /* When AVX512 is available, generate and consume eight A rows at a time
+     * with the 8-way SHAKE permutation. n is a multiple of 8 for every param
+     * set, so there is no remainder loop. */
+    if (IS_INTEL_AVX512(cpuid_flags) && (SAVE_VECTOR_REGISTERS2() == 0)) {
+        ret = 0;
+        for (j = 0; j < n; j += 8) {
+            ret = frodokem_gen_a_row_shake_x8(row, seedA, j, p);
+            if (ret != 0) {
+                break;
+            }
+            /* Fused 8-row S * A accumulate for all nbar output rows. */
+            frodokem_sa_accum_avx512(out, s, row, j, n);
+        }
+        RESTORE_VECTOR_REGISTERS();
+    }
+    else
+#endif /* FRODOKEM_HAVE_SHAKE_X8 */
+#ifdef FRODOKEM_HAVE_SHAKE_X4
+    /* When AVX2 is available, generate and consume four A rows at a time with
+     * the 4-way SHAKE permutation. n is a multiple of 4 for every parameter
+     * set, so there is no remainder loop. */
+    if (IS_INTEL_AVX2(cpuid_flags) && (SAVE_VECTOR_REGISTERS2() == 0)) {
+        ret = 0;
+        for (j = 0; j < n; j += 4) {
+            ret = frodokem_gen_a_row_shake_x4(row, seedA, j, p);
+            if (ret != 0) {
+                break;
+            }
+            /* Fused 4-row S * A accumulate for all nbar output rows. */
+            frodokem_sa_accum_avx2(out, s, row, j, n);
+        }
+        RESTORE_VECTOR_REGISTERS();
+    }
+    else
+#endif /* FRODOKEM_HAVE_SHAKE_X4 */
+#ifdef FRODOKEM_HAVE_SHAKE_NEON
+    /* AArch64: with Advanced SIMD, generate and consume two A rows at a time
+     * with the 2-way NEON SHAKE (SVE or NEON accumulate); without it, fall back
+     * to the scalar SHAKE and portable-C accumulate (the 2-way permute is
+     * itself NEON). n is a multiple of 2 for every set. */
+    if (IS_AARCH64_ASIMD(cpuid_flags)) {
+        (void)shake;
+        ret = 0;
+        for (j = 0; j < n; j += 2) {
+            ret = frodokem_gen_a_row_shake_x2(row, seedA, j, p);
+            if (ret != 0) {
+                break;
+            }
+            FRODOKEM_SA_ACCUM(out, s, row, j, n);
+        }
+    }
+    else {
+        ret = wc_InitShake128(shake, NULL, INVALID_DEVID);
+        for (j = 0; (ret == 0) && (j < n); j++) {
+            ret = frodokem_gen_a_row_shake(shake, row, seedA, j, p);
+            if (ret == 0) {
+                frodokem_sa_accum(out, s, row, j, n);
+            }
+        }
+    }
+#elif defined(FRODOKEM_HAVE_ARM_ASM)
+    /* AArch32 (NEON or Thumb2): no parallel Keccak, so generate one A row at a
+     * time with SHAKE-128 and consume it with the AArch32 S * A accumulate. */
+    {
+        ret = wc_InitShake128(shake, NULL, INVALID_DEVID);
+
+        for (j = 0; (ret == 0) && (j < n); j++) {
+            ret = frodokem_gen_a_row_shake(shake, row, seedA, j, p);
+            if (ret == 0) {
+                FRODOKEM_SA_ACCUM(out, s, row, j, n);
+            }
+        }
+    }
+#else
+    {
+        /* Matrix A is generated with SHAKE-128 for every parameter set.
+         * Initialize the shared object once here; each row's Final resets it
+         * for the next. */
+        ret = wc_InitShake128(shake, NULL, INVALID_DEVID);
+
+        /* Generate A a row at a time (index j) and accumulate it. */
+        for (j = 0; (ret == 0) && (j < n); j++) {
+            ret = frodokem_gen_a_row_shake(shake, row, seedA, j, p);
+            if (ret == 0) {
+                frodokem_sa_accum(out, s, row, j, n);
+            }
+        }
+    }
+#endif /* FRODOKEM_HAVE_SHAKE_NEON */
+
+#ifdef FRODOKEM_D15_SHAKE128
+    /* q == 2^15 (640) needs a final reduction mod q; for q == 2^16 (976/1344)
+     * the word16 accumulation already reduced. */
+#ifdef FRODOKEM_D16_SHAKE256
+    if ((ret == 0) && (p->qMask != 0xffff))
+#else
+    if (ret == 0)
+#endif
+    {
+        for (i = 0; i < FRODOKEM_NBAR * n; i++) {
+            out[i] = (word16)(out[i] & p->qMask);
+        }
+    }
+#endif /* FRODOKEM_D15_SHAKE128 */
+
+    return ret;
+}
+#endif /* WOLFSSL_FRODOKEM_SHAKE */
+
+/* Compute out = out + S * A, accumulating S * A into out which holds the error
+ * matrix E on entry (all nbar x n); S is nbar x n and A is n x n. Taking E in
+ * place saves the caller a copy (it samples E' straight into out). seedA, the
+ * parameters and the SHAKE / AES objects for matrix-A generation are taken from
+ * key; dispatch is by the key parameters' useAes.
+ *
+ * @param  [in]       key    FrodoKEM key (provides seedA, params, SHAKE / AES).
+ * @param  [in, out]  out    E on entry; out + S*A on exit (nbar * n).
+ * @param  [in]       s      Matrix S (nbar * n, row-major).
+ * @param  [out]      row    Scratch for the generated A rows (ROW_MULT * n).
+ * @return  0 on success, negative on error.
+ */
+int frodokem_mul_add_sa_plus_e(FrodoKemKey* key, word16* out, const word16* s,
+    word16* row)
+{
+    const FrodoKemParams* p = key->params;
+    int ret;
+
+#ifdef WOLFSSL_FRODOKEM_AES
+    if (p->useAes) {
+        ret = frodokem_mul_add_sa_plus_e_aes(out, s, key->seedA, p, row,
+            &key->aes);
+    }
+    else
+#endif
+    {
+#ifdef WOLFSSL_FRODOKEM_SHAKE
+        ret = frodokem_mul_add_sa_plus_e_shake(out, s, key->seedA, p, row,
+            &key->shake);
+#else
+        ret = NOT_COMPILED_IN;
+#endif
+    }
+
+    return ret;
+}
+
+/* Compute out = out + S * B, accumulating S * B into out which holds the error
+ * matrix E on entry (all nbar x nbar). S is nbar x n and B is n x nbar. Taking
+ * E in place saves the caller a copy (it samples E'' straight into out).
+ *
+ * @param  [in, out]  out  E on entry; out + S*B on exit (nbar * nbar).
+ * @param  [in]       b    Matrix B (n * nbar, row-major).
+ * @param  [in]       s    Matrix S (nbar * n, row-major).
+ * @param  [in]       n      Matrix dimension n.
+ * @param  [in]       qmask  Reduction mask (q - 1).
+ */
+void frodokem_mul_add_sb_plus_e(word16* out, const word16* b, const word16* s,
+    int n, int qmask)
+{
+#if defined(FRODOKEM_HAVE_ARM_ASM) && !defined(FRODOKEM_HAVE_NEON_RUNTIME)
+    frodokem_mul_add_sb_plus_e_arm(out, b, s, n, qmask);
+#else
+#ifdef FRODOKEM_HAVE_NEON_RUNTIME
+    if (IS_AARCH64_ASIMD(cpuid_flags)) {
+        frodokem_mul_add_sb_plus_e_arm(out, b, s, n, qmask);
+    }
+    else
+#endif
+#ifdef FRODOKEM_HAVE_MATRIX_ASM_AVX512
+    if (IS_INTEL_AVX512(cpuid_flags) && (SAVE_VECTOR_REGISTERS2() == 0)) {
+        frodokem_mul_add_sb_plus_e_avx512(out, b, s, n, qmask);
+        RESTORE_VECTOR_REGISTERS();
+    }
+    else
+#endif
+#ifdef FRODOKEM_HAVE_MATRIX_ASM
+    if (IS_INTEL_AVX2(cpuid_flags) && (SAVE_VECTOR_REGISTERS2() == 0)) {
+        frodokem_mul_add_sb_plus_e_avx2(out, b, s, n, qmask);
+        RESTORE_VECTOR_REGISTERS();
+    }
+    else
+#endif
+    {
+#ifdef WOLFSSL_FRODOKEM_SMALL
+    int i;
+    int j;
+    int k;
+
+    /* Keep FRODOKEM_NBAR accumulators per row of S (one per output column), as
+     * in the FrodoKEM reference: the inner loop then reads a contiguous row of
+     * B (b[j*nbar + 0..nbar)) and reuses the single S coefficient, which is
+     * cache-friendly and lets the compiler vectorize the nbar-wide update.
+     * Accumulation order per output element is unchanged, so results match. */
+    for (i = 0; i < FRODOKEM_NBAR; i++) {
+        word16 acc[FRODOKEM_NBAR];
+        const word16* sRow = s + i * n;
+
+        for (k = 0; k < FRODOKEM_NBAR; k++) {
+            acc[k] = out[i * FRODOKEM_NBAR + k];
+        }
+        for (j = 0; j < n; j++) {
+            word16 sij = sRow[j];
+            const word16* bRow = b + j * FRODOKEM_NBAR;
+
+            for (k = 0; k < FRODOKEM_NBAR; k++) {
+                acc[k] = (word16)(acc[k] + sij * bRow[k]);
+            }
+        }
+        for (k = 0; k < FRODOKEM_NBAR; k++) {
+            out[i * FRODOKEM_NBAR + k] = (word16)(acc[k] & qmask);
+        }
+    }
+#else
+#if FRODOKEM_NBAR != 8
+    #error "Unrolled frodokem_mul_add_sb_plus_e assumes FRODOKEM_NBAR == 8."
+#endif
+    int j;
+    const word16* s0 = s;
+    const word16* s1 = s + n;
+    const word16* s2 = s + 2 * n;
+    const word16* s3 = s + 3 * n;
+    const word16* s4 = s + 4 * n;
+    const word16* s5 = s + 5 * n;
+    const word16* s6 = s + 6 * n;
+    const word16* s7 = s + 7 * n;
+
+    /* out holds E on entry. Both nbar dimensions are fully unrolled, leaving
+     * only the loop over n; each B row is loaded once and reused across all 8
+     * output rows. Accumulate directly into out - the word16 stores wrap mod
+     * 2^16, the final reduction for q == 2^16 (976/1344). For q == 2^15 (640)
+     * one &qMask pass at the very end reduces mod q (q divides 2^16). */
+    for (j = 0; j < n; j++) {
+        const word16* bRow = b + j * FRODOKEM_NBAR;
+        word16 b0 = bRow[0];
+        word16 b1 = bRow[1];
+        word16 b2 = bRow[2];
+        word16 b3 = bRow[3];
+        word16 b4 = bRow[4];
+        word16 b5 = bRow[5];
+        word16 b6 = bRow[6];
+        word16 b7 = bRow[7];
+        word16 sij;
+
+        sij = s0[j];
+        out[0]  = (word16)(out[0]  + sij * b0);
+        out[1]  = (word16)(out[1]  + sij * b1);
+        out[2]  = (word16)(out[2]  + sij * b2);
+        out[3]  = (word16)(out[3]  + sij * b3);
+        out[4]  = (word16)(out[4]  + sij * b4);
+        out[5]  = (word16)(out[5]  + sij * b5);
+        out[6]  = (word16)(out[6]  + sij * b6);
+        out[7]  = (word16)(out[7]  + sij * b7);
+        sij = s1[j];
+        out[8]  = (word16)(out[8]  + sij * b0);
+        out[9]  = (word16)(out[9]  + sij * b1);
+        out[10] = (word16)(out[10] + sij * b2);
+        out[11] = (word16)(out[11] + sij * b3);
+        out[12] = (word16)(out[12] + sij * b4);
+        out[13] = (word16)(out[13] + sij * b5);
+        out[14] = (word16)(out[14] + sij * b6);
+        out[15] = (word16)(out[15] + sij * b7);
+        sij = s2[j];
+        out[16] = (word16)(out[16] + sij * b0);
+        out[17] = (word16)(out[17] + sij * b1);
+        out[18] = (word16)(out[18] + sij * b2);
+        out[19] = (word16)(out[19] + sij * b3);
+        out[20] = (word16)(out[20] + sij * b4);
+        out[21] = (word16)(out[21] + sij * b5);
+        out[22] = (word16)(out[22] + sij * b6);
+        out[23] = (word16)(out[23] + sij * b7);
+        sij = s3[j];
+        out[24] = (word16)(out[24] + sij * b0);
+        out[25] = (word16)(out[25] + sij * b1);
+        out[26] = (word16)(out[26] + sij * b2);
+        out[27] = (word16)(out[27] + sij * b3);
+        out[28] = (word16)(out[28] + sij * b4);
+        out[29] = (word16)(out[29] + sij * b5);
+        out[30] = (word16)(out[30] + sij * b6);
+        out[31] = (word16)(out[31] + sij * b7);
+        sij = s4[j];
+        out[32] = (word16)(out[32] + sij * b0);
+        out[33] = (word16)(out[33] + sij * b1);
+        out[34] = (word16)(out[34] + sij * b2);
+        out[35] = (word16)(out[35] + sij * b3);
+        out[36] = (word16)(out[36] + sij * b4);
+        out[37] = (word16)(out[37] + sij * b5);
+        out[38] = (word16)(out[38] + sij * b6);
+        out[39] = (word16)(out[39] + sij * b7);
+        sij = s5[j];
+        out[40] = (word16)(out[40] + sij * b0);
+        out[41] = (word16)(out[41] + sij * b1);
+        out[42] = (word16)(out[42] + sij * b2);
+        out[43] = (word16)(out[43] + sij * b3);
+        out[44] = (word16)(out[44] + sij * b4);
+        out[45] = (word16)(out[45] + sij * b5);
+        out[46] = (word16)(out[46] + sij * b6);
+        out[47] = (word16)(out[47] + sij * b7);
+        sij = s6[j];
+        out[48] = (word16)(out[48] + sij * b0);
+        out[49] = (word16)(out[49] + sij * b1);
+        out[50] = (word16)(out[50] + sij * b2);
+        out[51] = (word16)(out[51] + sij * b3);
+        out[52] = (word16)(out[52] + sij * b4);
+        out[53] = (word16)(out[53] + sij * b5);
+        out[54] = (word16)(out[54] + sij * b6);
+        out[55] = (word16)(out[55] + sij * b7);
+        sij = s7[j];
+        out[56] = (word16)(out[56] + sij * b0);
+        out[57] = (word16)(out[57] + sij * b1);
+        out[58] = (word16)(out[58] + sij * b2);
+        out[59] = (word16)(out[59] + sij * b3);
+        out[60] = (word16)(out[60] + sij * b4);
+        out[61] = (word16)(out[61] + sij * b5);
+        out[62] = (word16)(out[62] + sij * b6);
+        out[63] = (word16)(out[63] + sij * b7);
+    }
+
+#ifdef FRODOKEM_D15_SHAKE128
+    /* q == 2^15 (640) needs a final reduction mod q; for q == 2^16 (976/1344)
+     * the word16 stores already reduced, so no mask. */
+#ifdef FRODOKEM_D16_SHAKE256
+    if (qmask != 0xffff)
+#endif
+    {
+        out[0] = (word16)(out[0] & qmask);
+        out[1] = (word16)(out[1] & qmask);
+        out[2] = (word16)(out[2] & qmask);
+        out[3] = (word16)(out[3] & qmask);
+        out[4] = (word16)(out[4] & qmask);
+        out[5] = (word16)(out[5] & qmask);
+        out[6] = (word16)(out[6] & qmask);
+        out[7] = (word16)(out[7] & qmask);
+        out[8] = (word16)(out[8] & qmask);
+        out[9] = (word16)(out[9] & qmask);
+        out[10] = (word16)(out[10] & qmask);
+        out[11] = (word16)(out[11] & qmask);
+        out[12] = (word16)(out[12] & qmask);
+        out[13] = (word16)(out[13] & qmask);
+        out[14] = (word16)(out[14] & qmask);
+        out[15] = (word16)(out[15] & qmask);
+        out[16] = (word16)(out[16] & qmask);
+        out[17] = (word16)(out[17] & qmask);
+        out[18] = (word16)(out[18] & qmask);
+        out[19] = (word16)(out[19] & qmask);
+        out[20] = (word16)(out[20] & qmask);
+        out[21] = (word16)(out[21] & qmask);
+        out[22] = (word16)(out[22] & qmask);
+        out[23] = (word16)(out[23] & qmask);
+        out[24] = (word16)(out[24] & qmask);
+        out[25] = (word16)(out[25] & qmask);
+        out[26] = (word16)(out[26] & qmask);
+        out[27] = (word16)(out[27] & qmask);
+        out[28] = (word16)(out[28] & qmask);
+        out[29] = (word16)(out[29] & qmask);
+        out[30] = (word16)(out[30] & qmask);
+        out[31] = (word16)(out[31] & qmask);
+        out[32] = (word16)(out[32] & qmask);
+        out[33] = (word16)(out[33] & qmask);
+        out[34] = (word16)(out[34] & qmask);
+        out[35] = (word16)(out[35] & qmask);
+        out[36] = (word16)(out[36] & qmask);
+        out[37] = (word16)(out[37] & qmask);
+        out[38] = (word16)(out[38] & qmask);
+        out[39] = (word16)(out[39] & qmask);
+        out[40] = (word16)(out[40] & qmask);
+        out[41] = (word16)(out[41] & qmask);
+        out[42] = (word16)(out[42] & qmask);
+        out[43] = (word16)(out[43] & qmask);
+        out[44] = (word16)(out[44] & qmask);
+        out[45] = (word16)(out[45] & qmask);
+        out[46] = (word16)(out[46] & qmask);
+        out[47] = (word16)(out[47] & qmask);
+        out[48] = (word16)(out[48] & qmask);
+        out[49] = (word16)(out[49] & qmask);
+        out[50] = (word16)(out[50] & qmask);
+        out[51] = (word16)(out[51] & qmask);
+        out[52] = (word16)(out[52] & qmask);
+        out[53] = (word16)(out[53] & qmask);
+        out[54] = (word16)(out[54] & qmask);
+        out[55] = (word16)(out[55] & qmask);
+        out[56] = (word16)(out[56] & qmask);
+        out[57] = (word16)(out[57] & qmask);
+        out[58] = (word16)(out[58] & qmask);
+        out[59] = (word16)(out[59] & qmask);
+        out[60] = (word16)(out[60] & qmask);
+        out[61] = (word16)(out[61] & qmask);
+        out[62] = (word16)(out[62] & qmask);
+        out[63] = (word16)(out[63] & qmask);
+    }
+#else
+    (void)qmask;
+#endif /* FRODOKEM_D15_SHAKE128 */
+#endif /* WOLFSSL_FRODOKEM_SMALL */
+    }
+#endif /* FRODOKEM_HAVE_ARM_ASM */
+}
+
+/* Compute out = B * S where B is nbar x n and S (supplied as S^T, nbar x n)
+ * is n x nbar.
+ *
+ * @param  [out]  out  Output matrix (nbar * nbar, row-major).
+ * @param  [in]   b      Matrix B (nbar * n, row-major).
+ * @param  [in]   s      Secret matrix S^T (nbar * n, row-major).
+ * @param  [in]   n      Matrix dimension n.
+ * @param  [in]   qmask  Reduction mask (q - 1).
+ */
+void frodokem_mul_bs(word16* out, const word16* b, const word16* s,
+    int n, int qmask)
+{
+#ifdef FRODOKEM_HAVE_SME
+    /* SME computes the whole nbar x nbar tile with the ZA UMOPA, but needs B
+     * and S in the interleaved MOPA layout, so transpose into scratch first.
+     * Falls through to SVE/NEON/C if SME is absent or scratch cannot be had. */
+    word16* bt = NULL;
+    word16* st = NULL;
+    int sme = 0;
+
+    if (frodokem_sme_svl_ok) {
+        size_t sz = (size_t)FRODOKEM_NBAR * (size_t)n * sizeof(word16);
+        bt = (word16*)XMALLOC(sz, NULL, DYNAMIC_TYPE_TMP_BUFFER);
+        st = (word16*)XMALLOC(sz, NULL, DYNAMIC_TYPE_TMP_BUFFER);
+        if ((bt != NULL) && (st != NULL)) {
+            frodokem_sme_interleave(bt, b, FRODOKEM_NBAR, n);
+            frodokem_sme_interleave(st, s, FRODOKEM_NBAR, n);
+            frodokem_mul_bs_sme(out, bt, st, n, qmask);
+            sme = 1;
+        }
+    }
+    if (!sme)
+#endif /* FRODOKEM_HAVE_SME */
+    {
+#if defined(FRODOKEM_HAVE_ARM_ASM) && !defined(FRODOKEM_HAVE_NEON_RUNTIME)
+    frodokem_mul_bs_arm(out, b, s, n, qmask);
+#else
+#ifdef FRODOKEM_HAVE_SVE
+    if (IS_AARCH64_SVE(cpuid_flags)) {
+        frodokem_mul_bs_sve(out, b, s, n, qmask);
+    }
+    else
+#endif
+#ifdef FRODOKEM_HAVE_NEON_RUNTIME
+    if (IS_AARCH64_ASIMD(cpuid_flags)) {
+        frodokem_mul_bs_arm(out, b, s, n, qmask);
+    }
+    else
+#endif
+#ifdef FRODOKEM_HAVE_MATRIX_ASM_AVX512
+    if (IS_INTEL_AVX512(cpuid_flags) && (SAVE_VECTOR_REGISTERS2() == 0)) {
+        frodokem_mul_bs_avx512(out, b, s, n, qmask);
+        RESTORE_VECTOR_REGISTERS();
+    }
+    else
+#endif
+#ifdef FRODOKEM_HAVE_MATRIX_ASM
+    if (IS_INTEL_AVX2(cpuid_flags) && (SAVE_VECTOR_REGISTERS2() == 0)) {
+        frodokem_mul_bs_avx2(out, b, s, n, qmask);
+        RESTORE_VECTOR_REGISTERS();
+    }
+    else
+#endif
+    {
+#ifdef WOLFSSL_FRODOKEM_SMALL
+    int i;
+    int j;
+    int k;
+
+    for (i = 0; i < FRODOKEM_NBAR; i++) {
+        for (k = 0; k < FRODOKEM_NBAR; k++) {
+            word16 acc = 0;
+            /* column k of S == row k of S^T. */
+            const word16* sCol = s + k * n;
+
+            for (j = 0; j < n; j++) {
+                acc = (word16)(acc + b[i * n + j] * sCol[j]);
+            }
+            out[i * FRODOKEM_NBAR + k] = (word16)(acc & qmask);
+        }
+    }
+#else
+#if FRODOKEM_NBAR != 8
+    #error "Unrolled frodokem_mul_bs assumes FRODOKEM_NBAR == 8."
+#endif
+    int j;
+    const word16* b0 = b;
+    const word16* b1 = b + n;
+    const word16* b2 = b + 2 * n;
+    const word16* b3 = b + 3 * n;
+    const word16* b4 = b + 4 * n;
+    const word16* b5 = b + 5 * n;
+    const word16* b6 = b + 6 * n;
+    const word16* b7 = b + 7 * n;
+    const word16* s0 = s;
+    const word16* s1 = s + n;
+    const word16* s2 = s + 2 * n;
+    const word16* s3 = s + 3 * n;
+    const word16* s4 = s + 4 * n;
+    const word16* s5 = s + 5 * n;
+    const word16* s6 = s + 6 * n;
+    const word16* s7 = s + 7 * n;
+
+    XMEMSET(out, 0, sizeof(word16) * FRODOKEM_NBAR_SQ);
+
+    /* Both nbar dimensions are fully unrolled, leaving only the loop over n.
+     * Each row of B and column of S (row of S^T) is loaded once per j and
+     * combined as an outer product accumulated into out. The word16 stores
+     * wrap mod 2^16, the final reduction for q == 2^16 (976/1344). For
+     * q == 2^15 (640) one &qMask pass at the very end reduces mod q (q
+     * divides 2^16). */
+    for (j = 0; j < n; j++) {
+        word16 sc0 = s0[j];
+        word16 sc1 = s1[j];
+        word16 sc2 = s2[j];
+        word16 sc3 = s3[j];
+        word16 sc4 = s4[j];
+        word16 sc5 = s5[j];
+        word16 sc6 = s6[j];
+        word16 sc7 = s7[j];
+        word16 bij;
+
+        bij = b0[j];
+        out[0]  = (word16)(out[0]  + bij * sc0);
+        out[1]  = (word16)(out[1]  + bij * sc1);
+        out[2]  = (word16)(out[2]  + bij * sc2);
+        out[3]  = (word16)(out[3]  + bij * sc3);
+        out[4]  = (word16)(out[4]  + bij * sc4);
+        out[5]  = (word16)(out[5]  + bij * sc5);
+        out[6]  = (word16)(out[6]  + bij * sc6);
+        out[7]  = (word16)(out[7]  + bij * sc7);
+        bij = b1[j];
+        out[8]  = (word16)(out[8]  + bij * sc0);
+        out[9]  = (word16)(out[9]  + bij * sc1);
+        out[10] = (word16)(out[10] + bij * sc2);
+        out[11] = (word16)(out[11] + bij * sc3);
+        out[12] = (word16)(out[12] + bij * sc4);
+        out[13] = (word16)(out[13] + bij * sc5);
+        out[14] = (word16)(out[14] + bij * sc6);
+        out[15] = (word16)(out[15] + bij * sc7);
+        bij = b2[j];
+        out[16] = (word16)(out[16] + bij * sc0);
+        out[17] = (word16)(out[17] + bij * sc1);
+        out[18] = (word16)(out[18] + bij * sc2);
+        out[19] = (word16)(out[19] + bij * sc3);
+        out[20] = (word16)(out[20] + bij * sc4);
+        out[21] = (word16)(out[21] + bij * sc5);
+        out[22] = (word16)(out[22] + bij * sc6);
+        out[23] = (word16)(out[23] + bij * sc7);
+        bij = b3[j];
+        out[24] = (word16)(out[24] + bij * sc0);
+        out[25] = (word16)(out[25] + bij * sc1);
+        out[26] = (word16)(out[26] + bij * sc2);
+        out[27] = (word16)(out[27] + bij * sc3);
+        out[28] = (word16)(out[28] + bij * sc4);
+        out[29] = (word16)(out[29] + bij * sc5);
+        out[30] = (word16)(out[30] + bij * sc6);
+        out[31] = (word16)(out[31] + bij * sc7);
+        bij = b4[j];
+        out[32] = (word16)(out[32] + bij * sc0);
+        out[33] = (word16)(out[33] + bij * sc1);
+        out[34] = (word16)(out[34] + bij * sc2);
+        out[35] = (word16)(out[35] + bij * sc3);
+        out[36] = (word16)(out[36] + bij * sc4);
+        out[37] = (word16)(out[37] + bij * sc5);
+        out[38] = (word16)(out[38] + bij * sc6);
+        out[39] = (word16)(out[39] + bij * sc7);
+        bij = b5[j];
+        out[40] = (word16)(out[40] + bij * sc0);
+        out[41] = (word16)(out[41] + bij * sc1);
+        out[42] = (word16)(out[42] + bij * sc2);
+        out[43] = (word16)(out[43] + bij * sc3);
+        out[44] = (word16)(out[44] + bij * sc4);
+        out[45] = (word16)(out[45] + bij * sc5);
+        out[46] = (word16)(out[46] + bij * sc6);
+        out[47] = (word16)(out[47] + bij * sc7);
+        bij = b6[j];
+        out[48] = (word16)(out[48] + bij * sc0);
+        out[49] = (word16)(out[49] + bij * sc1);
+        out[50] = (word16)(out[50] + bij * sc2);
+        out[51] = (word16)(out[51] + bij * sc3);
+        out[52] = (word16)(out[52] + bij * sc4);
+        out[53] = (word16)(out[53] + bij * sc5);
+        out[54] = (word16)(out[54] + bij * sc6);
+        out[55] = (word16)(out[55] + bij * sc7);
+        bij = b7[j];
+        out[56] = (word16)(out[56] + bij * sc0);
+        out[57] = (word16)(out[57] + bij * sc1);
+        out[58] = (word16)(out[58] + bij * sc2);
+        out[59] = (word16)(out[59] + bij * sc3);
+        out[60] = (word16)(out[60] + bij * sc4);
+        out[61] = (word16)(out[61] + bij * sc5);
+        out[62] = (word16)(out[62] + bij * sc6);
+        out[63] = (word16)(out[63] + bij * sc7);
+    }
+
+#ifdef FRODOKEM_D15_SHAKE128
+    /* q == 2^15 (640) needs a final reduction mod q; for q == 2^16 (976/1344)
+     * the word16 stores already reduced, so no mask. */
+#ifdef FRODOKEM_D16_SHAKE256
+    if (qmask != 0xffff)
+#endif
+    {
+        out[0] = (word16)(out[0] & qmask);
+        out[1] = (word16)(out[1] & qmask);
+        out[2] = (word16)(out[2] & qmask);
+        out[3] = (word16)(out[3] & qmask);
+        out[4] = (word16)(out[4] & qmask);
+        out[5] = (word16)(out[5] & qmask);
+        out[6] = (word16)(out[6] & qmask);
+        out[7] = (word16)(out[7] & qmask);
+        out[8] = (word16)(out[8] & qmask);
+        out[9] = (word16)(out[9] & qmask);
+        out[10] = (word16)(out[10] & qmask);
+        out[11] = (word16)(out[11] & qmask);
+        out[12] = (word16)(out[12] & qmask);
+        out[13] = (word16)(out[13] & qmask);
+        out[14] = (word16)(out[14] & qmask);
+        out[15] = (word16)(out[15] & qmask);
+        out[16] = (word16)(out[16] & qmask);
+        out[17] = (word16)(out[17] & qmask);
+        out[18] = (word16)(out[18] & qmask);
+        out[19] = (word16)(out[19] & qmask);
+        out[20] = (word16)(out[20] & qmask);
+        out[21] = (word16)(out[21] & qmask);
+        out[22] = (word16)(out[22] & qmask);
+        out[23] = (word16)(out[23] & qmask);
+        out[24] = (word16)(out[24] & qmask);
+        out[25] = (word16)(out[25] & qmask);
+        out[26] = (word16)(out[26] & qmask);
+        out[27] = (word16)(out[27] & qmask);
+        out[28] = (word16)(out[28] & qmask);
+        out[29] = (word16)(out[29] & qmask);
+        out[30] = (word16)(out[30] & qmask);
+        out[31] = (word16)(out[31] & qmask);
+        out[32] = (word16)(out[32] & qmask);
+        out[33] = (word16)(out[33] & qmask);
+        out[34] = (word16)(out[34] & qmask);
+        out[35] = (word16)(out[35] & qmask);
+        out[36] = (word16)(out[36] & qmask);
+        out[37] = (word16)(out[37] & qmask);
+        out[38] = (word16)(out[38] & qmask);
+        out[39] = (word16)(out[39] & qmask);
+        out[40] = (word16)(out[40] & qmask);
+        out[41] = (word16)(out[41] & qmask);
+        out[42] = (word16)(out[42] & qmask);
+        out[43] = (word16)(out[43] & qmask);
+        out[44] = (word16)(out[44] & qmask);
+        out[45] = (word16)(out[45] & qmask);
+        out[46] = (word16)(out[46] & qmask);
+        out[47] = (word16)(out[47] & qmask);
+        out[48] = (word16)(out[48] & qmask);
+        out[49] = (word16)(out[49] & qmask);
+        out[50] = (word16)(out[50] & qmask);
+        out[51] = (word16)(out[51] & qmask);
+        out[52] = (word16)(out[52] & qmask);
+        out[53] = (word16)(out[53] & qmask);
+        out[54] = (word16)(out[54] & qmask);
+        out[55] = (word16)(out[55] & qmask);
+        out[56] = (word16)(out[56] & qmask);
+        out[57] = (word16)(out[57] & qmask);
+        out[58] = (word16)(out[58] & qmask);
+        out[59] = (word16)(out[59] & qmask);
+        out[60] = (word16)(out[60] & qmask);
+        out[61] = (word16)(out[61] & qmask);
+        out[62] = (word16)(out[62] & qmask);
+        out[63] = (word16)(out[63] & qmask);
+    }
+#else
+    (void)qmask;
+#endif /* FRODOKEM_D15_SHAKE128 */
+#endif /* WOLFSSL_FRODOKEM_SMALL */
+    }
+#endif /* FRODOKEM_HAVE_ARM_ASM */
+    }
+#ifdef FRODOKEM_HAVE_SME
+    XFREE(bt, NULL, DYNAMIC_TYPE_TMP_BUFFER);
+    XFREE(st, NULL, DYNAMIC_TYPE_TMP_BUFFER);
+#endif /* FRODOKEM_HAVE_SME */
+}
+
+/* Add matrix b into matrix a modulo q, element-wise (nbar x nbar).
+ *
+ * Used to form C = V + Encode(u): a is V (updated in place), b is Encode(u).
+ *
+ * @param  [in, out]  a      Matrix accumulated into (nbar * nbar, row-major).
+ * @param  [in]       b      Matrix added to a (nbar * nbar, row-major).
+ * @param  [in]       qmask  Reduction mask (q - 1).
+ */
+void frodokem_add(word16* a, const word16* b, int qmask)
+{
+#if defined(FRODOKEM_HAVE_ARM_ASM) && !defined(FRODOKEM_HAVE_NEON_RUNTIME)
+    frodokem_add_arm(a, b, qmask);
+#else
+#ifdef FRODOKEM_HAVE_SVE
+    if (IS_AARCH64_SVE(cpuid_flags)) {
+        frodokem_add_sve(a, b, qmask);
+    }
+    else
+#endif
+#ifdef FRODOKEM_HAVE_NEON_RUNTIME
+    if (IS_AARCH64_ASIMD(cpuid_flags)) {
+        frodokem_add_arm(a, b, qmask);
+    }
+    else
+#endif
+#ifdef FRODOKEM_HAVE_MATRIX_ASM_AVX512
+    if (IS_INTEL_AVX512(cpuid_flags) && (SAVE_VECTOR_REGISTERS2() == 0)) {
+        frodokem_add_avx512(a, b, qmask);
+        RESTORE_VECTOR_REGISTERS();
+    }
+    else
+#endif
+#ifdef FRODOKEM_HAVE_MATRIX_ASM
+    if (IS_INTEL_AVX2(cpuid_flags) && (SAVE_VECTOR_REGISTERS2() == 0)) {
+        frodokem_add_avx2(a, b, qmask);
+        RESTORE_VECTOR_REGISTERS();
+    }
+    else
+#endif
+    {
+#ifdef WOLFSSL_FRODOKEM_SMALL
+    int i;
+
+    for (i = 0; i < FRODOKEM_NBAR_SQ; i++) {
+        a[i] = (word16)((a[i] + b[i]) & qmask);
+    }
+#else
+#if FRODOKEM_NBAR_SQ != 64
+    #error "Unrolled frodokem_add assumes FRODOKEM_NBAR_SQ == 64."
+#endif
+    /* nbar x nbar = FRODOKEM_NBAR_SQ (64) elements; the i loop is fully
+     * unrolled. word16 stores wrap mod 2^16, the reduction for q == 2^16
+     * (976/1344). */
+    a[0] = (word16)(a[0] + b[0]);
+    a[1] = (word16)(a[1] + b[1]);
+    a[2] = (word16)(a[2] + b[2]);
+    a[3] = (word16)(a[3] + b[3]);
+    a[4] = (word16)(a[4] + b[4]);
+    a[5] = (word16)(a[5] + b[5]);
+    a[6] = (word16)(a[6] + b[6]);
+    a[7] = (word16)(a[7] + b[7]);
+    a[8] = (word16)(a[8] + b[8]);
+    a[9] = (word16)(a[9] + b[9]);
+    a[10] = (word16)(a[10] + b[10]);
+    a[11] = (word16)(a[11] + b[11]);
+    a[12] = (word16)(a[12] + b[12]);
+    a[13] = (word16)(a[13] + b[13]);
+    a[14] = (word16)(a[14] + b[14]);
+    a[15] = (word16)(a[15] + b[15]);
+    a[16] = (word16)(a[16] + b[16]);
+    a[17] = (word16)(a[17] + b[17]);
+    a[18] = (word16)(a[18] + b[18]);
+    a[19] = (word16)(a[19] + b[19]);
+    a[20] = (word16)(a[20] + b[20]);
+    a[21] = (word16)(a[21] + b[21]);
+    a[22] = (word16)(a[22] + b[22]);
+    a[23] = (word16)(a[23] + b[23]);
+    a[24] = (word16)(a[24] + b[24]);
+    a[25] = (word16)(a[25] + b[25]);
+    a[26] = (word16)(a[26] + b[26]);
+    a[27] = (word16)(a[27] + b[27]);
+    a[28] = (word16)(a[28] + b[28]);
+    a[29] = (word16)(a[29] + b[29]);
+    a[30] = (word16)(a[30] + b[30]);
+    a[31] = (word16)(a[31] + b[31]);
+    a[32] = (word16)(a[32] + b[32]);
+    a[33] = (word16)(a[33] + b[33]);
+    a[34] = (word16)(a[34] + b[34]);
+    a[35] = (word16)(a[35] + b[35]);
+    a[36] = (word16)(a[36] + b[36]);
+    a[37] = (word16)(a[37] + b[37]);
+    a[38] = (word16)(a[38] + b[38]);
+    a[39] = (word16)(a[39] + b[39]);
+    a[40] = (word16)(a[40] + b[40]);
+    a[41] = (word16)(a[41] + b[41]);
+    a[42] = (word16)(a[42] + b[42]);
+    a[43] = (word16)(a[43] + b[43]);
+    a[44] = (word16)(a[44] + b[44]);
+    a[45] = (word16)(a[45] + b[45]);
+    a[46] = (word16)(a[46] + b[46]);
+    a[47] = (word16)(a[47] + b[47]);
+    a[48] = (word16)(a[48] + b[48]);
+    a[49] = (word16)(a[49] + b[49]);
+    a[50] = (word16)(a[50] + b[50]);
+    a[51] = (word16)(a[51] + b[51]);
+    a[52] = (word16)(a[52] + b[52]);
+    a[53] = (word16)(a[53] + b[53]);
+    a[54] = (word16)(a[54] + b[54]);
+    a[55] = (word16)(a[55] + b[55]);
+    a[56] = (word16)(a[56] + b[56]);
+    a[57] = (word16)(a[57] + b[57]);
+    a[58] = (word16)(a[58] + b[58]);
+    a[59] = (word16)(a[59] + b[59]);
+    a[60] = (word16)(a[60] + b[60]);
+    a[61] = (word16)(a[61] + b[61]);
+    a[62] = (word16)(a[62] + b[62]);
+    a[63] = (word16)(a[63] + b[63]);
+
+#ifdef FRODOKEM_D15_SHAKE128
+    /* q == 2^15 (640) needs a final reduction mod q. */
+#ifdef FRODOKEM_D16_SHAKE256
+    if (qmask != 0xffff)
+#endif
+    {
+        a[0] = (word16)(a[0] & qmask);
+        a[1] = (word16)(a[1] & qmask);
+        a[2] = (word16)(a[2] & qmask);
+        a[3] = (word16)(a[3] & qmask);
+        a[4] = (word16)(a[4] & qmask);
+        a[5] = (word16)(a[5] & qmask);
+        a[6] = (word16)(a[6] & qmask);
+        a[7] = (word16)(a[7] & qmask);
+        a[8] = (word16)(a[8] & qmask);
+        a[9] = (word16)(a[9] & qmask);
+        a[10] = (word16)(a[10] & qmask);
+        a[11] = (word16)(a[11] & qmask);
+        a[12] = (word16)(a[12] & qmask);
+        a[13] = (word16)(a[13] & qmask);
+        a[14] = (word16)(a[14] & qmask);
+        a[15] = (word16)(a[15] & qmask);
+        a[16] = (word16)(a[16] & qmask);
+        a[17] = (word16)(a[17] & qmask);
+        a[18] = (word16)(a[18] & qmask);
+        a[19] = (word16)(a[19] & qmask);
+        a[20] = (word16)(a[20] & qmask);
+        a[21] = (word16)(a[21] & qmask);
+        a[22] = (word16)(a[22] & qmask);
+        a[23] = (word16)(a[23] & qmask);
+        a[24] = (word16)(a[24] & qmask);
+        a[25] = (word16)(a[25] & qmask);
+        a[26] = (word16)(a[26] & qmask);
+        a[27] = (word16)(a[27] & qmask);
+        a[28] = (word16)(a[28] & qmask);
+        a[29] = (word16)(a[29] & qmask);
+        a[30] = (word16)(a[30] & qmask);
+        a[31] = (word16)(a[31] & qmask);
+        a[32] = (word16)(a[32] & qmask);
+        a[33] = (word16)(a[33] & qmask);
+        a[34] = (word16)(a[34] & qmask);
+        a[35] = (word16)(a[35] & qmask);
+        a[36] = (word16)(a[36] & qmask);
+        a[37] = (word16)(a[37] & qmask);
+        a[38] = (word16)(a[38] & qmask);
+        a[39] = (word16)(a[39] & qmask);
+        a[40] = (word16)(a[40] & qmask);
+        a[41] = (word16)(a[41] & qmask);
+        a[42] = (word16)(a[42] & qmask);
+        a[43] = (word16)(a[43] & qmask);
+        a[44] = (word16)(a[44] & qmask);
+        a[45] = (word16)(a[45] & qmask);
+        a[46] = (word16)(a[46] & qmask);
+        a[47] = (word16)(a[47] & qmask);
+        a[48] = (word16)(a[48] & qmask);
+        a[49] = (word16)(a[49] & qmask);
+        a[50] = (word16)(a[50] & qmask);
+        a[51] = (word16)(a[51] & qmask);
+        a[52] = (word16)(a[52] & qmask);
+        a[53] = (word16)(a[53] & qmask);
+        a[54] = (word16)(a[54] & qmask);
+        a[55] = (word16)(a[55] & qmask);
+        a[56] = (word16)(a[56] & qmask);
+        a[57] = (word16)(a[57] & qmask);
+        a[58] = (word16)(a[58] & qmask);
+        a[59] = (word16)(a[59] & qmask);
+        a[60] = (word16)(a[60] & qmask);
+        a[61] = (word16)(a[61] & qmask);
+        a[62] = (word16)(a[62] & qmask);
+        a[63] = (word16)(a[63] & qmask);
+    }
+#else
+    (void)qmask;
+#endif /* FRODOKEM_D15_SHAKE128 */
+#endif /* WOLFSSL_FRODOKEM_SMALL */
+    }
+#endif /* FRODOKEM_HAVE_ARM_ASM */
+}
+
+#endif /* WOLFSSL_HAVE_FRODOKEM */
diff --git a/wolfcrypt/test/test.c b/wolfcrypt/test/test.c
index c70617a9bd..37dc802587 100644
--- a/wolfcrypt/test/test.c
+++ b/wolfcrypt/test/test.c
@@ -395,6 +395,9 @@ static const byte const_byte_array[] = "A+Gd\0\0\0";
 #ifdef WOLFSSL_HAVE_MLKEM
     #include <wolfssl/wolfcrypt/wc_mlkem.h>
 #endif
+#ifdef WOLFSSL_HAVE_FRODOKEM
+    #include <wolfssl/wolfcrypt/wc_frodokem.h>
+#endif
 #ifdef WOLFSSL_HAVE_MLDSA
     #include <wolfssl/wolfcrypt/wc_mldsa.h>
 #endif
@@ -971,6 +974,9 @@ WOLFSSL_TEST_SUBROUTINE wc_test_ret_t scrypt_test(void);
 #ifdef WOLFSSL_HAVE_MLKEM
     WOLFSSL_TEST_SUBROUTINE wc_test_ret_t  mlkem_test(void);
 #endif
+#ifdef WOLFSSL_HAVE_FRODOKEM
+    WOLFSSL_TEST_SUBROUTINE wc_test_ret_t  frodokem_test(void);
+#endif
 #ifdef WOLFSSL_HAVE_MLDSA
     WOLFSSL_TEST_SUBROUTINE wc_test_ret_t  mldsa_test(void);
 #endif
@@ -3162,6 +3168,13 @@ options: [-s max_relative_stack_bytes] [-m max_relative_heap_memory_bytes]\n\
     PRIVATE_KEY_LOCK();
 #endif
 
+#ifdef WOLFSSL_HAVE_FRODOKEM
+    if ( (ret = frodokem_test()) != 0)
+        TEST_FAIL("FRODOKEM test failed!\n", ret);
+    else
+        TEST_PASS("FRODOKEM test passed!\n");
+#endif
+
 #ifdef WOLFSSL_HAVE_MLDSA
     PRIVATE_KEY_UNLOCK();
     if ( (ret = mldsa_test()) != 0)
@@ -53230,6 +53243,152 @@ WOLFSSL_TEST_SUBROUTINE wc_test_ret_t mlkem_test(void)
 }
 #endif /* WOLFSSL_HAVE_MLKEM */
 
+#ifdef WOLFSSL_HAVE_FRODOKEM
+/* Basic FrodoKEM test: for each compiled variant generate a key, encapsulate
+ * and decapsulate (shared secrets must match), then confirm an encode/decode
+ * round trip of the private key still decapsulates correctly. */
+WOLFSSL_TEST_SUBROUTINE wc_test_ret_t frodokem_test(void)
+{
+    wc_test_ret_t ret = 0;
+#if !defined(WC_NO_RNG) && !defined(WOLFSSL_FRODOKEM_NO_MAKE_KEY) && \
+    !defined(WOLFSSL_FRODOKEM_NO_ENCAPSULATE) && \
+    !defined(WOLFSSL_FRODOKEM_NO_DECAPSULATE)
+    static const int types[] = {
+    #ifdef WOLFSSL_FRODOKEM_SHAKE
+        #ifdef WOLFSSL_WC_FRODOKEM_640
+        WC_FRODOKEM_640_SHAKE,
+        #endif
+        #ifdef WOLFSSL_WC_FRODOKEM_976
+        WC_FRODOKEM_976_SHAKE,
+        #endif
+        #ifdef WOLFSSL_WC_FRODOKEM_1344
+        WC_FRODOKEM_1344_SHAKE,
+        #endif
+    #endif
+    #ifdef WOLFSSL_FRODOKEM_AES
+        #ifdef WOLFSSL_WC_FRODOKEM_640
+        WC_FRODOKEM_640_AES,
+        #endif
+        #ifdef WOLFSSL_WC_FRODOKEM_976
+        WC_FRODOKEM_976_AES,
+        #endif
+        #ifdef WOLFSSL_WC_FRODOKEM_1344
+        WC_FRODOKEM_1344_AES,
+        #endif
+    #endif
+    #ifdef WOLFSSL_FRODOKEM_EPHEMERAL
+        #ifdef WOLFSSL_FRODOKEM_SHAKE
+            #ifdef WOLFSSL_WC_FRODOKEM_640
+        WC_EFRODOKEM_640_SHAKE,
+            #endif
+            #ifdef WOLFSSL_WC_FRODOKEM_976
+        WC_EFRODOKEM_976_SHAKE,
+            #endif
+            #ifdef WOLFSSL_WC_FRODOKEM_1344
+        WC_EFRODOKEM_1344_SHAKE,
+            #endif
+        #endif
+        #ifdef WOLFSSL_FRODOKEM_AES
+            #ifdef WOLFSSL_WC_FRODOKEM_640
+        WC_EFRODOKEM_640_AES,
+            #endif
+            #ifdef WOLFSSL_WC_FRODOKEM_976
+        WC_EFRODOKEM_976_AES,
+            #endif
+            #ifdef WOLFSSL_WC_FRODOKEM_1344
+        WC_EFRODOKEM_1344_AES,
+            #endif
+        #endif
+    #endif
+        0 /* sentinel so the array is never empty */
+    };
+    int numTypes = (int)(sizeof(types) / sizeof(types[0])) - 1;
+    int i;
+    WC_RNG rng;
+    int rngInit = 0;
+    FrodoKemKey* key = NULL;
+    byte* ct = NULL;
+    byte* pk = NULL;
+    byte* sk = NULL;
+    byte ss[FRODOKEM_MAX_LENSEC];
+    byte ss2[FRODOKEM_MAX_LENSEC];
+    word32 pkLen = 0;
+    word32 skLen = 0;
+    word32 ctLen = 0;
+    word32 ssLen = 0;
+
+    key = (FrodoKemKey*)XMALLOC(sizeof(*key), HEAP_HINT,
+        DYNAMIC_TYPE_TMP_BUFFER);
+    ct = (byte*)XMALLOC(FRODOKEM_MAX_CIPHER_TEXT_SIZE, HEAP_HINT,
+        DYNAMIC_TYPE_TMP_BUFFER);
+    pk = (byte*)XMALLOC(FRODOKEM_MAX_PUBLIC_KEY_SIZE, HEAP_HINT,
+        DYNAMIC_TYPE_TMP_BUFFER);
+    sk = (byte*)XMALLOC(FRODOKEM_MAX_PRIVATE_KEY_SIZE, HEAP_HINT,
+        DYNAMIC_TYPE_TMP_BUFFER);
+    if ((key == NULL) || (ct == NULL) || (pk == NULL) || (sk == NULL)) {
+        ret = WC_TEST_RET_ENC_NC;
+        goto out;
+    }
+
+    ret = wc_InitRng_ex(&rng, HEAP_HINT, devId);
+    if (ret != 0) {
+        ret = WC_TEST_RET_ENC_EC(ret);
+        goto out;
+    }
+    rngInit = 1;
+
+    for (i = 0; i < numTypes; i++) {
+        ret = wc_FrodoKemKey_Init(key, types[i], HEAP_HINT, devId);
+        if (ret != 0) { ret = WC_TEST_RET_ENC_EC(ret); break; }
+        ret = wc_FrodoKemKey_PublicKeySize(key, &pkLen);
+        if (ret != 0) { ret = WC_TEST_RET_ENC_EC(ret); break; }
+        ret = wc_FrodoKemKey_PrivateKeySize(key, &skLen);
+        if (ret != 0) { ret = WC_TEST_RET_ENC_EC(ret); break; }
+        ret = wc_FrodoKemKey_CipherTextSize(key, &ctLen);
+        if (ret != 0) { ret = WC_TEST_RET_ENC_EC(ret); break; }
+        ret = wc_FrodoKemKey_SharedSecretSize(key, &ssLen);
+        if (ret != 0) { ret = WC_TEST_RET_ENC_EC(ret); break; }
+
+        ret = wc_FrodoKemKey_MakeKey(key, &rng);
+        if (ret != 0) { ret = WC_TEST_RET_ENC_EC(ret); break; }
+        ret = wc_FrodoKemKey_Encapsulate(key, ct, ss, &rng);
+        if (ret != 0) { ret = WC_TEST_RET_ENC_EC(ret); break; }
+        ret = wc_FrodoKemKey_Decapsulate(key, ss2, ct, ctLen);
+        if (ret != 0) { ret = WC_TEST_RET_ENC_EC(ret); break; }
+        if (XMEMCMP(ss, ss2, ssLen) != 0) {
+            ret = WC_TEST_RET_ENC_NC; break;
+        }
+
+        /* Encode then decode the private key and decapsulate again. */
+        ret = wc_FrodoKemKey_EncodePublicKey(key, pk, pkLen);
+        if (ret != 0) { ret = WC_TEST_RET_ENC_EC(ret); break; }
+        ret = wc_FrodoKemKey_EncodePrivateKey(key, sk, skLen);
+        if (ret != 0) { ret = WC_TEST_RET_ENC_EC(ret); break; }
+        ret = wc_FrodoKemKey_DecodePrivateKey(key, sk, skLen);
+        if (ret != 0) { ret = WC_TEST_RET_ENC_EC(ret); break; }
+        ret = wc_FrodoKemKey_Decapsulate(key, ss2, ct, ctLen);
+        if (ret != 0) { ret = WC_TEST_RET_ENC_EC(ret); break; }
+        if (XMEMCMP(ss, ss2, ssLen) != 0) {
+            ret = WC_TEST_RET_ENC_NC; break;
+        }
+
+        wc_FrodoKemKey_Free(key);
+    }
+
+out:
+    if (key != NULL)
+        wc_FrodoKemKey_Free(key);
+    if (rngInit)
+        wc_FreeRng(&rng);
+    XFREE(sk, HEAP_HINT, DYNAMIC_TYPE_TMP_BUFFER);
+    XFREE(pk, HEAP_HINT, DYNAMIC_TYPE_TMP_BUFFER);
+    XFREE(ct, HEAP_HINT, DYNAMIC_TYPE_TMP_BUFFER);
+    XFREE(key, HEAP_HINT, DYNAMIC_TYPE_TMP_BUFFER);
+#endif /* !WC_NO_RNG */
+    return ret;
+}
+#endif /* WOLFSSL_HAVE_FRODOKEM */
+
 #ifdef WOLFSSL_HAVE_MLDSA
 #ifndef WOLFSSL_MLDSA_NO_VERIFY
 static wc_test_ret_t mldsa_param_vfy_test(int param, const byte* pubKey,
@@ -77560,6 +77719,10 @@ WOLFSSL_TEST_SUBROUTINE wc_test_ret_t cryptocb_test(void)
     if (ret == 0)
         ret = mlkem_test();
 #endif
+#ifdef WOLFSSL_HAVE_FRODOKEM
+    if (ret == 0)
+        ret = frodokem_test();
+#endif
 #ifdef WOLFSSL_HAVE_MLDSA
     if (ret == 0)
         ret = mldsa_test();
diff --git a/wolfssl-VS2022.vcxproj b/wolfssl-VS2022.vcxproj
index b7178b0c48..d6cc484bf4 100644
--- a/wolfssl-VS2022.vcxproj
+++ b/wolfssl-VS2022.vcxproj
@@ -76,7 +76,7 @@
          the WOLFSSL_ARMASM* defines below); a symbol that is left undefined
          here simply drops the matching assembly (safe - the C caller then uses
          a C fallback), while a symbol enabled here but not in C is dead code. -->
-    <WolfSSLArmAsmDefs>-PreDefine "HAVE_AES_CBC SETL {TRUE}" -PreDefine "HAVE_AESCCM SETL {TRUE}" -PreDefine "HAVE_AESGCM SETL {TRUE}" -PreDefine "HAVE_AES_ECB SETL {TRUE}" -PreDefine "HAVE_AES_DECRYPT SETL {TRUE}" -PreDefine "WOLFSSL_AES_COUNTER SETL {TRUE}" -PreDefine "WOLFSSL_AES_DIRECT SETL {TRUE}" -PreDefine "WOLFSSL_AES_XTS SETL {TRUE}" -PreDefine "WOLFSSL_AESGCM_STREAM SETL {TRUE}" -PreDefine "WOLFSSL_SHA224 SETL {TRUE}" -PreDefine "WOLFSSL_SHA3 SETL {TRUE}" -PreDefine "WOLFSSL_ARMASM_CRYPTO_SHA3 SETL {TRUE}" -PreDefine "WOLFSSL_SHA384 SETL {TRUE}" -PreDefine "WOLFSSL_SHA512 SETL {TRUE}" -PreDefine "WOLFSSL_ARMASM_CRYPTO_SHA512 SETL {TRUE}" -PreDefine "HAVE_CHACHA SETL {TRUE}" -PreDefine "HAVE_CURVE25519 SETL {TRUE}" -PreDefine "HAVE_ED25519 SETL {TRUE}" -PreDefine "WOLFSSL_HAVE_MLKEM SETL {TRUE}"</WolfSSLArmAsmDefs>
+    <WolfSSLArmAsmDefs>-PreDefine "HAVE_AES_CBC SETL {TRUE}" -PreDefine "HAVE_AESCCM SETL {TRUE}" -PreDefine "HAVE_AESGCM SETL {TRUE}" -PreDefine "HAVE_AES_ECB SETL {TRUE}" -PreDefine "HAVE_AES_DECRYPT SETL {TRUE}" -PreDefine "WOLFSSL_AES_COUNTER SETL {TRUE}" -PreDefine "WOLFSSL_AES_DIRECT SETL {TRUE}" -PreDefine "WOLFSSL_AES_XTS SETL {TRUE}" -PreDefine "WOLFSSL_AESGCM_STREAM SETL {TRUE}" -PreDefine "WOLFSSL_SHA224 SETL {TRUE}" -PreDefine "WOLFSSL_SHA3 SETL {TRUE}" -PreDefine "WOLFSSL_ARMASM_CRYPTO_SHA3 SETL {TRUE}" -PreDefine "WOLFSSL_SHA384 SETL {TRUE}" -PreDefine "WOLFSSL_SHA512 SETL {TRUE}" -PreDefine "WOLFSSL_ARMASM_CRYPTO_SHA512 SETL {TRUE}" -PreDefine "HAVE_CHACHA SETL {TRUE}" -PreDefine "HAVE_CURVE25519 SETL {TRUE}" -PreDefine "HAVE_ED25519 SETL {TRUE}" -PreDefine "WOLFSSL_HAVE_MLKEM SETL {TRUE}" -PreDefine "WOLFSSL_HAVE_FRODOKEM SETL {TRUE}"</WolfSSLArmAsmDefs>
   </PropertyGroup>
   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
     <ConfigurationType>StaticLibrary</ConfigurationType>
@@ -462,6 +462,8 @@
     <ClCompile Include="wolfcrypt\src\kdf.c" />
     <ClCompile Include="wolfcrypt\src\wc_mlkem.c" />
     <ClCompile Include="wolfcrypt\src\wc_mlkem_poly.c" />
+    <ClCompile Include="wolfcrypt\src\wc_frodokem.c" />
+    <ClCompile Include="wolfcrypt\src\wc_frodokem_mat.c" />
     <ClCompile Include="wolfcrypt\src\logging.c" />
     <ClCompile Include="wolfcrypt\src\md2.c" />
     <ClCompile Include="wolfcrypt\src\md4.c" />
@@ -677,6 +679,12 @@
       <Command Condition="'$(WolfSSLAarch64Asm)'=='true' And '$(Platform)'=='ARM64'">armasm64.exe -o "$(IntDir)%(Filename).obj" $(WolfSSLArmAsmDefs) %(Identity)</Command>
       <Outputs Condition="'$(WolfSSLAarch64Asm)'=='true' And '$(Platform)'=='ARM64'">$(IntDir)%(Filename).obj</Outputs>
     </CustomBuild>
+    <CustomBuild Include="wolfcrypt\src\port\arm\armv8-frodokem-asm.asm">
+      <ExcludedFromBuild>true</ExcludedFromBuild>
+      <ExcludedFromBuild Condition="'$(WolfSSLAarch64Asm)'=='true' And '$(Platform)'=='ARM64'">false</ExcludedFromBuild>
+      <Command Condition="'$(WolfSSLAarch64Asm)'=='true' And '$(Platform)'=='ARM64'">armasm64.exe -o "$(IntDir)%(Filename).obj" $(WolfSSLArmAsmDefs) %(Identity)</Command>
+      <Outputs Condition="'$(WolfSSLAarch64Asm)'=='true' And '$(Platform)'=='ARM64'">$(IntDir)%(Filename).obj</Outputs>
+    </CustomBuild>
     <CustomBuild Include="wolfcrypt\src\port\arm\armv8-poly1305-asm.asm">
       <ExcludedFromBuild>true</ExcludedFromBuild>
       <ExcludedFromBuild Condition="'$(WolfSSLAarch64Asm)'=='true' And '$(Platform)'=='ARM64'">false</ExcludedFromBuild>
@@ -701,6 +709,12 @@
       <Command Condition="'$(WolfSSLAarch64Asm)'=='true' And '$(Platform)'=='ARM64'">armasm64.exe -o "$(IntDir)%(Filename).obj" $(WolfSSLArmAsmDefs) %(Identity)</Command>
       <Outputs Condition="'$(WolfSSLAarch64Asm)'=='true' And '$(Platform)'=='ARM64'">$(IntDir)%(Filename).obj</Outputs>
     </CustomBuild>
+    <CustomBuild Include="wolfcrypt\src\wc_frodokem_asm.asm">
+      <ExcludedFromBuild>true</ExcludedFromBuild>
+      <ExcludedFromBuild Condition="'$(WolfSSLIntelAsm)'=='true' And '$(Platform)'=='x64'">false</ExcludedFromBuild>
+      <Command Condition="'$(WolfSSLIntelAsm)'=='true' And '$(Platform)'=='x64'">ml64.exe /c /Zi /DWOLFSSL_X86_64_BUILD /DWOLFSSL_HAVE_FRODOKEM /Fo"$(IntDir)%(Filename).obj" %(Identity)</Command>
+      <Outputs Condition="'$(WolfSSLIntelAsm)'=='true' And '$(Platform)'=='x64'">$(IntDir)%(Filename).obj</Outputs>
+    </CustomBuild>
     <ClInclude Include="IDE\WIN\user_settings.h" />
   </ItemGroup>
   <ItemGroup>
diff --git a/wolfssl.vcxproj b/wolfssl.vcxproj
index 63b17ef9cc..1bd3220bb8 100644
--- a/wolfssl.vcxproj
+++ b/wolfssl.vcxproj
@@ -75,7 +75,7 @@
          the WOLFSSL_ARMASM* defines below); a symbol that is left undefined
          here simply drops the matching assembly (safe - the C caller then uses
          a C fallback), while a symbol enabled here but not in C is dead code. -->
-    <WolfSSLArmAsmDefs>-PreDefine "HAVE_AES_CBC SETL {TRUE}" -PreDefine "HAVE_AESCCM SETL {TRUE}" -PreDefine "HAVE_AESGCM SETL {TRUE}" -PreDefine "HAVE_AES_ECB SETL {TRUE}" -PreDefine "HAVE_AES_DECRYPT SETL {TRUE}" -PreDefine "WOLFSSL_AES_COUNTER SETL {TRUE}" -PreDefine "WOLFSSL_AES_DIRECT SETL {TRUE}" -PreDefine "WOLFSSL_AES_XTS SETL {TRUE}" -PreDefine "WOLFSSL_AESGCM_STREAM SETL {TRUE}" -PreDefine "WOLFSSL_SHA224 SETL {TRUE}" -PreDefine "WOLFSSL_SHA3 SETL {TRUE}" -PreDefine "WOLFSSL_ARMASM_CRYPTO_SHA3 SETL {TRUE}" -PreDefine "WOLFSSL_SHA384 SETL {TRUE}" -PreDefine "WOLFSSL_SHA512 SETL {TRUE}" -PreDefine "WOLFSSL_ARMASM_CRYPTO_SHA512 SETL {TRUE}" -PreDefine "HAVE_CHACHA SETL {TRUE}" -PreDefine "HAVE_CURVE25519 SETL {TRUE}" -PreDefine "HAVE_ED25519 SETL {TRUE}" -PreDefine "WOLFSSL_HAVE_MLKEM SETL {TRUE}"</WolfSSLArmAsmDefs>
+    <WolfSSLArmAsmDefs>-PreDefine "HAVE_AES_CBC SETL {TRUE}" -PreDefine "HAVE_AESCCM SETL {TRUE}" -PreDefine "HAVE_AESGCM SETL {TRUE}" -PreDefine "HAVE_AES_ECB SETL {TRUE}" -PreDefine "HAVE_AES_DECRYPT SETL {TRUE}" -PreDefine "WOLFSSL_AES_COUNTER SETL {TRUE}" -PreDefine "WOLFSSL_AES_DIRECT SETL {TRUE}" -PreDefine "WOLFSSL_AES_XTS SETL {TRUE}" -PreDefine "WOLFSSL_AESGCM_STREAM SETL {TRUE}" -PreDefine "WOLFSSL_SHA224 SETL {TRUE}" -PreDefine "WOLFSSL_SHA3 SETL {TRUE}" -PreDefine "WOLFSSL_ARMASM_CRYPTO_SHA3 SETL {TRUE}" -PreDefine "WOLFSSL_SHA384 SETL {TRUE}" -PreDefine "WOLFSSL_SHA512 SETL {TRUE}" -PreDefine "WOLFSSL_ARMASM_CRYPTO_SHA512 SETL {TRUE}" -PreDefine "HAVE_CHACHA SETL {TRUE}" -PreDefine "HAVE_CURVE25519 SETL {TRUE}" -PreDefine "HAVE_ED25519 SETL {TRUE}" -PreDefine "WOLFSSL_HAVE_MLKEM SETL {TRUE}" -PreDefine "WOLFSSL_HAVE_FRODOKEM SETL {TRUE}"</WolfSSLArmAsmDefs>
   </PropertyGroup>
   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
     <ConfigurationType>StaticLibrary</ConfigurationType>
@@ -462,6 +462,8 @@
     <ClCompile Include="wolfcrypt\src\kdf.c" />
     <ClCompile Include="wolfcrypt\src\wc_mlkem.c" />
     <ClCompile Include="wolfcrypt\src\wc_mlkem_poly.c" />
+    <ClCompile Include="wolfcrypt\src\wc_frodokem.c" />
+    <ClCompile Include="wolfcrypt\src\wc_frodokem_mat.c" />
     <ClCompile Include="wolfcrypt\src\logging.c" />
     <ClCompile Include="wolfcrypt\src\md2.c" />
     <ClCompile Include="wolfcrypt\src\md4.c" />
@@ -677,6 +679,12 @@
       <Command Condition="'$(WolfSSLAarch64Asm)'=='true' And '$(Platform)'=='ARM64'">armasm64.exe -o "$(IntDir)%(Filename).obj" $(WolfSSLArmAsmDefs) %(Identity)</Command>
       <Outputs Condition="'$(WolfSSLAarch64Asm)'=='true' And '$(Platform)'=='ARM64'">$(IntDir)%(Filename).obj</Outputs>
     </CustomBuild>
+    <CustomBuild Include="wolfcrypt\src\port\arm\armv8-frodokem-asm.asm">
+      <ExcludedFromBuild>true</ExcludedFromBuild>
+      <ExcludedFromBuild Condition="'$(WolfSSLAarch64Asm)'=='true' And '$(Platform)'=='ARM64'">false</ExcludedFromBuild>
+      <Command Condition="'$(WolfSSLAarch64Asm)'=='true' And '$(Platform)'=='ARM64'">armasm64.exe -o "$(IntDir)%(Filename).obj" $(WolfSSLArmAsmDefs) %(Identity)</Command>
+      <Outputs Condition="'$(WolfSSLAarch64Asm)'=='true' And '$(Platform)'=='ARM64'">$(IntDir)%(Filename).obj</Outputs>
+    </CustomBuild>
     <CustomBuild Include="wolfcrypt\src\port\arm\armv8-poly1305-asm.asm">
       <ExcludedFromBuild>true</ExcludedFromBuild>
       <ExcludedFromBuild Condition="'$(WolfSSLAarch64Asm)'=='true' And '$(Platform)'=='ARM64'">false</ExcludedFromBuild>
@@ -701,6 +709,12 @@
       <Command Condition="'$(WolfSSLAarch64Asm)'=='true' And '$(Platform)'=='ARM64'">armasm64.exe -o "$(IntDir)%(Filename).obj" $(WolfSSLArmAsmDefs) %(Identity)</Command>
       <Outputs Condition="'$(WolfSSLAarch64Asm)'=='true' And '$(Platform)'=='ARM64'">$(IntDir)%(Filename).obj</Outputs>
     </CustomBuild>
+    <CustomBuild Include="wolfcrypt\src\wc_frodokem_asm.asm">
+      <ExcludedFromBuild>true</ExcludedFromBuild>
+      <ExcludedFromBuild Condition="'$(WolfSSLIntelAsm)'=='true' And '$(Platform)'=='x64'">false</ExcludedFromBuild>
+      <Command Condition="'$(WolfSSLIntelAsm)'=='true' And '$(Platform)'=='x64'">ml64.exe /c /Zi /DWOLFSSL_X86_64_BUILD /DWOLFSSL_HAVE_FRODOKEM /Fo"$(IntDir)%(Filename).obj" %(Identity)</Command>
+      <Outputs Condition="'$(WolfSSLIntelAsm)'=='true' And '$(Platform)'=='x64'">$(IntDir)%(Filename).obj</Outputs>
+    </CustomBuild>
     <ClInclude Include="IDE\WIN\user_settings.h" />
   </ItemGroup>
   <ItemGroup>
diff --git a/wolfssl/wolfcrypt/cpuid.h b/wolfssl/wolfcrypt/cpuid.h
index b393100f97..106bd50909 100644
--- a/wolfssl/wolfcrypt/cpuid.h
+++ b/wolfssl/wolfcrypt/cpuid.h
@@ -99,6 +99,8 @@ typedef word32 cpuid_flags_t;
     #define CPUID_SM4         0x0080    /* SM4 enc/dec */
     #define CPUID_SB          0x0100    /* Speculation barrier */
     #define CPUID_ASIMD       0x0200    /* ASIMD - NEON */
+    #define CPUID_SVE         0x0400    /* Scalable Vector Extension */
+    #define CPUID_SME         0x0800    /* Scalable Matrix Extension */
 
     #define IS_AARCH64_AES(f)     (WOLFSSL_ATOMIC_COERCE_UINT(f) & CPUID_AES)
     #define IS_AARCH64_PMULL(f)   (WOLFSSL_ATOMIC_COERCE_UINT(f) & CPUID_PMULL)
@@ -110,6 +112,8 @@ typedef word32 cpuid_flags_t;
     #define IS_AARCH64_SM4(f)     (WOLFSSL_ATOMIC_COERCE_UINT(f) & CPUID_SM4)
     #define IS_AARCH64_SB(f)      (WOLFSSL_ATOMIC_COERCE_UINT(f) & CPUID_SB)
     #define IS_AARCH64_ASIMD(f)   (WOLFSSL_ATOMIC_COERCE_UINT(f) & CPUID_ASIMD)
+    #define IS_AARCH64_SVE(f)     (WOLFSSL_ATOMIC_COERCE_UINT(f) & CPUID_SVE)
+    #define IS_AARCH64_SME(f)     (WOLFSSL_ATOMIC_COERCE_UINT(f) & CPUID_SME)
 
 #elif defined(HAVE_CPUID_PPC64)
 
diff --git a/wolfssl/wolfcrypt/include.am b/wolfssl/wolfcrypt/include.am
index b6936f73c3..5339c71a64 100644
--- a/wolfssl/wolfcrypt/include.am
+++ b/wolfssl/wolfcrypt/include.am
@@ -79,6 +79,8 @@ nobase_include_HEADERS+= \
                          wolfssl/wolfcrypt/cryptocb.h \
                          wolfssl/wolfcrypt/wc_mldsa.h \
                          wolfssl/wolfcrypt/wc_mlkem.h \
+                         wolfssl/wolfcrypt/wc_frodokem.h \
+                         wolfssl/wolfcrypt/wc_frodokem_mat.h \
                          wolfssl/wolfcrypt/sm2.h \
                          wolfssl/wolfcrypt/sm3.h \
                          wolfssl/wolfcrypt/sm4.h \
diff --git a/wolfssl/wolfcrypt/sha3.h b/wolfssl/wolfcrypt/sha3.h
index e0c529b8af..ff187e6385 100644
--- a/wolfssl/wolfcrypt/sha3.h
+++ b/wolfssl/wolfcrypt/sha3.h
@@ -262,6 +262,10 @@ WOLFSSL_LOCAL void BlockSha3(word64 *s);
         word64 c);
     WOLFSSL_LOCAL void sha3_block_avx2(word64* s);
     WOLFSSL_LOCAL void sha3_blocksx4_avx2(word64* s);
+    WOLFSSL_LOCAL void sha3_blocksx4_out_avx2(word64* s, byte* out,
+        word32 len);
+    WOLFSSL_LOCAL void sha3_blocksx8_out_avx512(word64* s, byte* out,
+        word32 len);
 
     WOLFSSL_LOCAL void sha3_128_blocksx4_seed_avx2(word64* s, byte* seed);
     WOLFSSL_LOCAL void sha3_256_blocksx4_seed_avx2(word64* s, byte* seed);
diff --git a/wolfssl/wolfcrypt/wc_frodokem.h b/wolfssl/wolfcrypt/wc_frodokem.h
new file mode 100644
index 0000000000..264eb5dd2b
--- /dev/null
+++ b/wolfssl/wolfcrypt/wc_frodokem.h
@@ -0,0 +1,328 @@
+/* wc_frodokem.h
+ *
+ * Copyright (C) 2006-2026 wolfSSL Inc.
+ *
+ * This file is part of wolfSSL.
+ *
+ * wolfSSL is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * wolfSSL is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335, USA
+ */
+
+/*!
+    \file wolfssl/wolfcrypt/wc_frodokem.h
+*/
+
+#ifndef WOLF_CRYPT_WC_FRODOKEM_H
+#define WOLF_CRYPT_WC_FRODOKEM_H
+
+#include <wolfssl/wolfcrypt/types.h>
+#include <wolfssl/wolfcrypt/random.h>
+#include <wolfssl/wolfcrypt/sha3.h>
+#ifdef WOLFSSL_FRODOKEM_AES
+    #include <wolfssl/wolfcrypt/aes.h>
+#endif
+
+#ifdef WOLFSSL_HAVE_FRODOKEM
+
+#ifdef __cplusplus
+    extern "C" {
+#endif
+
+/* Define algorithm parameter sets when not excluded. */
+#ifndef WOLFSSL_NO_FRODOKEM_640
+    #define WOLFSSL_WC_FRODOKEM_640
+#endif
+#ifndef WOLFSSL_NO_FRODOKEM_976
+    #define WOLFSSL_WC_FRODOKEM_976
+#endif
+#ifndef WOLFSSL_NO_FRODOKEM_1344
+    #define WOLFSSL_WC_FRODOKEM_1344
+#endif
+
+#if !defined(WOLFSSL_WC_FRODOKEM_640) && !defined(WOLFSSL_WC_FRODOKEM_976) && \
+    !defined(WOLFSSL_WC_FRODOKEM_1344)
+    #error "No FrodoKEM parameter set chosen."
+#endif
+
+/* Matrix A generation method(s). AES-128 (Section 6.7.1) and SHAKE-128
+ * (Section 6.7.2) are independently selectable build options; at least one
+ * must be enabled. */
+#if !defined(WOLFSSL_FRODOKEM_AES) && !defined(WOLFSSL_FRODOKEM_SHAKE)
+    #error "FrodoKEM needs WOLFSSL_FRODOKEM_AES and/or WOLFSSL_FRODOKEM_SHAKE."
+#endif
+
+/* WOLFSSL_FRODOKEM_EPHEMERAL enables the eFrodoKEM (ephemeral, salt-less)
+ * parameter set types. */
+
+/* Operations are individually selectable: define WOLFSSL_FRODOKEM_NO_MAKE_KEY,
+ * WOLFSSL_FRODOKEM_NO_ENCAPSULATE and/or WOLFSSL_FRODOKEM_NO_DECAPSULATE to
+ * exclude them. At least one operation must remain. */
+#if defined(WOLFSSL_FRODOKEM_NO_MAKE_KEY) && \
+    defined(WOLFSSL_FRODOKEM_NO_ENCAPSULATE) && \
+    defined(WOLFSSL_FRODOKEM_NO_DECAPSULATE)
+    #error "No FrodoKEM operations to be built."
+#endif
+
+/* Number of columns in the "small" matrices - constant across all parameter
+ * sets (n-bar = m-bar = 8). */
+#define FRODOKEM_NBAR       8
+
+/* Number of elements in an nbar x nbar matrix. */
+#define FRODOKEM_NBAR_SQ    (FRODOKEM_NBAR * FRODOKEM_NBAR)
+
+/* Length of the seed for generating matrix A (lenA / 8 = 128 / 8). */
+#define FRODOKEM_SEEDA_SZ   16
+
+/* Size in bytes of a public key: seedA || b (b = nbar * n coefficients of D
+ * bits each = D * n bytes, nbar == 8). */
+#define FRODOKEM_PK_SZ(n, d)        (FRODOKEM_SEEDA_SZ + ((d) * (n)))
+/* Size in bytes of a private key: s || seedA || b || S^T || pkh.
+ * S^T holds nbar * n coefficients, each stored as 2 bytes. */
+#define FRODOKEM_SK_SZ(n, d, ls)    ((ls) + FRODOKEM_SEEDA_SZ + ((d) * (n)) + \
+                                     (2 * FRODOKEM_NBAR * (n)) + (ls))
+/* Size in bytes of a ciphertext: c1 || c2 || salt.
+ * c1 = D * nbar * n bits, c2 = D * nbar * nbar bits. */
+#define FRODOKEM_CT_SZ(n, d, lsalt) (((d) * (n)) + ((d) * FRODOKEM_NBAR) + \
+                                     (lsalt))
+
+/* FrodoKEM-640 parameters. */
+#define WC_FRODOKEM_640_N               640
+#define WC_FRODOKEM_640_D               15
+#define WC_FRODOKEM_640_LENSEC          16      /* lensec  / 8 = 128 / 8  */
+#define WC_FRODOKEM_640_LENSE           32      /* lenSE   / 8 = 256 / 8  */
+#define WC_FRODOKEM_640_LENSALT         32      /* lensalt / 8 = 256 / 8  */
+#define WC_FRODOKEM_640_PUBLIC_KEY_SIZE \
+    FRODOKEM_PK_SZ(WC_FRODOKEM_640_N, WC_FRODOKEM_640_D)
+#define WC_FRODOKEM_640_PRIVATE_KEY_SIZE \
+    FRODOKEM_SK_SZ(WC_FRODOKEM_640_N, WC_FRODOKEM_640_D, WC_FRODOKEM_640_LENSEC)
+#define WC_FRODOKEM_640_CIPHER_TEXT_SIZE \
+    FRODOKEM_CT_SZ(WC_FRODOKEM_640_N, WC_FRODOKEM_640_D, \
+        WC_FRODOKEM_640_LENSALT)
+#define WC_FRODOKEM_640_SS_SIZE         WC_FRODOKEM_640_LENSEC
+#define WC_FRODOKEM_640_MAKEKEY_RAND_SZ \
+    (WC_FRODOKEM_640_LENSEC + WC_FRODOKEM_640_LENSE + FRODOKEM_SEEDA_SZ)
+#define WC_FRODOKEM_640_ENC_RAND_SZ \
+    (WC_FRODOKEM_640_LENSEC + WC_FRODOKEM_640_LENSALT)
+
+/* FrodoKEM-976 parameters. */
+#define WC_FRODOKEM_976_N               976
+#define WC_FRODOKEM_976_D               16
+#define WC_FRODOKEM_976_LENSEC          24      /* lensec  / 8 = 192 / 8  */
+#define WC_FRODOKEM_976_LENSE           48      /* lenSE   / 8 = 384 / 8  */
+#define WC_FRODOKEM_976_LENSALT         48      /* lensalt / 8 = 384 / 8  */
+#define WC_FRODOKEM_976_PUBLIC_KEY_SIZE \
+    FRODOKEM_PK_SZ(WC_FRODOKEM_976_N, WC_FRODOKEM_976_D)
+#define WC_FRODOKEM_976_PRIVATE_KEY_SIZE \
+    FRODOKEM_SK_SZ(WC_FRODOKEM_976_N, WC_FRODOKEM_976_D, WC_FRODOKEM_976_LENSEC)
+#define WC_FRODOKEM_976_CIPHER_TEXT_SIZE \
+    FRODOKEM_CT_SZ(WC_FRODOKEM_976_N, WC_FRODOKEM_976_D, \
+        WC_FRODOKEM_976_LENSALT)
+#define WC_FRODOKEM_976_SS_SIZE         WC_FRODOKEM_976_LENSEC
+#define WC_FRODOKEM_976_MAKEKEY_RAND_SZ \
+    (WC_FRODOKEM_976_LENSEC + WC_FRODOKEM_976_LENSE + FRODOKEM_SEEDA_SZ)
+#define WC_FRODOKEM_976_ENC_RAND_SZ \
+    (WC_FRODOKEM_976_LENSEC + WC_FRODOKEM_976_LENSALT)
+
+/* FrodoKEM-1344 parameters. */
+#define WC_FRODOKEM_1344_N              1344
+#define WC_FRODOKEM_1344_D              16
+#define WC_FRODOKEM_1344_LENSEC         32      /* lensec  / 8 = 256 / 8  */
+#define WC_FRODOKEM_1344_LENSE          64      /* lenSE   / 8 = 512 / 8  */
+#define WC_FRODOKEM_1344_LENSALT        64      /* lensalt / 8 = 512 / 8  */
+#define WC_FRODOKEM_1344_PUBLIC_KEY_SIZE \
+    FRODOKEM_PK_SZ(WC_FRODOKEM_1344_N, WC_FRODOKEM_1344_D)
+#define WC_FRODOKEM_1344_PRIVATE_KEY_SIZE \
+    FRODOKEM_SK_SZ(WC_FRODOKEM_1344_N, WC_FRODOKEM_1344_D, \
+        WC_FRODOKEM_1344_LENSEC)
+#define WC_FRODOKEM_1344_CIPHER_TEXT_SIZE \
+    FRODOKEM_CT_SZ(WC_FRODOKEM_1344_N, WC_FRODOKEM_1344_D, \
+        WC_FRODOKEM_1344_LENSALT)
+#define WC_FRODOKEM_1344_SS_SIZE        WC_FRODOKEM_1344_LENSEC
+#define WC_FRODOKEM_1344_MAKEKEY_RAND_SZ \
+    (WC_FRODOKEM_1344_LENSEC + WC_FRODOKEM_1344_LENSE + FRODOKEM_SEEDA_SZ)
+#define WC_FRODOKEM_1344_ENC_RAND_SZ \
+    (WC_FRODOKEM_1344_LENSEC + WC_FRODOKEM_1344_LENSALT)
+
+/* Maximum dimensions and sizes over the enabled parameter sets. */
+#ifdef WOLFSSL_WC_FRODOKEM_1344
+    #define FRODOKEM_MAX_N                  WC_FRODOKEM_1344_N
+    #define FRODOKEM_MAX_LENSEC             WC_FRODOKEM_1344_LENSEC
+    #define FRODOKEM_MAX_LENSE              WC_FRODOKEM_1344_LENSE
+    #define FRODOKEM_MAX_LENSALT            WC_FRODOKEM_1344_LENSALT
+    #define FRODOKEM_MAX_D                  WC_FRODOKEM_1344_D
+    #define FRODOKEM_MAX_PUBLIC_KEY_SIZE    WC_FRODOKEM_1344_PUBLIC_KEY_SIZE
+    #define FRODOKEM_MAX_PRIVATE_KEY_SIZE   WC_FRODOKEM_1344_PRIVATE_KEY_SIZE
+    #define FRODOKEM_MAX_CIPHER_TEXT_SIZE   WC_FRODOKEM_1344_CIPHER_TEXT_SIZE
+#elif defined(WOLFSSL_WC_FRODOKEM_976)
+    #define FRODOKEM_MAX_N                  WC_FRODOKEM_976_N
+    #define FRODOKEM_MAX_LENSEC             WC_FRODOKEM_976_LENSEC
+    #define FRODOKEM_MAX_LENSE              WC_FRODOKEM_976_LENSE
+    #define FRODOKEM_MAX_LENSALT            WC_FRODOKEM_976_LENSALT
+    #define FRODOKEM_MAX_D                  WC_FRODOKEM_976_D
+    #define FRODOKEM_MAX_PUBLIC_KEY_SIZE    WC_FRODOKEM_976_PUBLIC_KEY_SIZE
+    #define FRODOKEM_MAX_PRIVATE_KEY_SIZE   WC_FRODOKEM_976_PRIVATE_KEY_SIZE
+    #define FRODOKEM_MAX_CIPHER_TEXT_SIZE   WC_FRODOKEM_976_CIPHER_TEXT_SIZE
+#else
+    #define FRODOKEM_MAX_N                  WC_FRODOKEM_640_N
+    #define FRODOKEM_MAX_LENSEC             WC_FRODOKEM_640_LENSEC
+    #define FRODOKEM_MAX_LENSE              WC_FRODOKEM_640_LENSE
+    #define FRODOKEM_MAX_LENSALT            WC_FRODOKEM_640_LENSALT
+    #define FRODOKEM_MAX_D                  WC_FRODOKEM_640_D
+    #define FRODOKEM_MAX_PUBLIC_KEY_SIZE    WC_FRODOKEM_640_PUBLIC_KEY_SIZE
+    #define FRODOKEM_MAX_PRIVATE_KEY_SIZE   WC_FRODOKEM_640_PRIVATE_KEY_SIZE
+    #define FRODOKEM_MAX_CIPHER_TEXT_SIZE   WC_FRODOKEM_640_CIPHER_TEXT_SIZE
+#endif
+
+/* Size in bytes of the packed public matrix b (D * n bits => D * n bytes as
+ * nbar == 8). */
+#define FRODOKEM_MAX_B_SZ       (FRODOKEM_MAX_D * FRODOKEM_MAX_N)
+/* Number of coefficients in the secret matrix S^T (nbar * n). */
+#define FRODOKEM_MAX_S_CNT      (FRODOKEM_NBAR * FRODOKEM_MAX_N)
+
+
+enum {
+    /* Base parameter sets. On their own these select the standard (salted)
+     * FrodoKEM using SHAKE-128 to generate matrix A. */
+    WC_FRODOKEM_640  = 0,
+    WC_FRODOKEM_976  = 1,
+    WC_FRODOKEM_1344 = 2,
+
+    /* Modifier bits OR'd with a base parameter set to form a key type. */
+    FRODOKEM_AES        = 0x10, /* Generate matrix A with AES-128. */
+    FRODOKEM_EPHEMERAL  = 0x20, /* eFrodoKEM: ephemeral, no salt. */
+
+    /* Mask to extract the base parameter set from a type. */
+    FRODOKEM_BASE_MASK  = 0x0F,
+
+    /* Explicit named types: FrodoKEM (standard, salted). */
+    WC_FRODOKEM_640_SHAKE   = WC_FRODOKEM_640,
+    WC_FRODOKEM_976_SHAKE   = WC_FRODOKEM_976,
+    WC_FRODOKEM_1344_SHAKE  = WC_FRODOKEM_1344,
+    WC_FRODOKEM_640_AES     = WC_FRODOKEM_640  | FRODOKEM_AES,
+    WC_FRODOKEM_976_AES     = WC_FRODOKEM_976  | FRODOKEM_AES,
+    WC_FRODOKEM_1344_AES    = WC_FRODOKEM_1344 | FRODOKEM_AES,
+
+    /* Explicit named types: eFrodoKEM (ephemeral). */
+    WC_EFRODOKEM_640_SHAKE  = WC_FRODOKEM_640  | FRODOKEM_EPHEMERAL,
+    WC_EFRODOKEM_976_SHAKE  = WC_FRODOKEM_976  | FRODOKEM_EPHEMERAL,
+    WC_EFRODOKEM_1344_SHAKE = WC_FRODOKEM_1344 | FRODOKEM_EPHEMERAL,
+    WC_EFRODOKEM_640_AES    = WC_EFRODOKEM_640_SHAKE  | FRODOKEM_AES,
+    WC_EFRODOKEM_976_AES    = WC_EFRODOKEM_976_SHAKE  | FRODOKEM_AES,
+    WC_EFRODOKEM_1344_AES   = WC_EFRODOKEM_1344_SHAKE | FRODOKEM_AES,
+
+    /* Flags indicating what is stored in a key. */
+    FRODOKEM_FLAG_PRIV_SET = 0x0001,
+    FRODOKEM_FLAG_PUB_SET  = 0x0002,
+    FRODOKEM_FLAG_BOTH_SET = 0x0003,
+    FRODOKEM_FLAG_PKH_SET  = 0x0004
+};
+
+
+/* Run-time parameters for a FrodoKEM parameter set. The definition is internal
+ * (see wc_frodokem_mat.h); a key only holds a pointer to a constant instance. */
+typedef struct FrodoKemParams FrodoKemParams;
+
+
+/* FrodoKEM key. */
+typedef struct FrodoKemKey {
+    /* Type of key: a base parameter set optionally OR'd with FRODOKEM_AES
+     * and/or FRODOKEM_EPHEMERAL. */
+    int type;
+    /* Parameters for this key type (points to a constant). Set by Init. */
+    const FrodoKemParams* params;
+    /* Dynamic memory allocation hint. */
+    void* heap;
+    /* Device Id. */
+    int devId;
+    /* Flags indicating what is stored in the key. */
+    int flags;
+
+    /* Secret value used for implicit rejection (length lenSec). */
+    byte s[FRODOKEM_MAX_LENSEC];
+    /* Secret matrix S^T (nbar x n) stored row-major as residues modulo q. */
+    word16 sMat[FRODOKEM_MAX_S_CNT];
+
+    /* Seed used to generate matrix A. */
+    byte seedA[FRODOKEM_SEEDA_SZ];
+    /* Packed public matrix B (D * n bytes). */
+    byte b[FRODOKEM_MAX_B_SZ];
+    /* Hash of the public key (length lenSec). */
+    byte pkh[FRODOKEM_MAX_LENSEC];
+
+    /* Reusable SHAKE object for all hashing and matrix-A generation. Embedded
+     * (not allocated); initialized by wc_FrodoKemKey_Init for the key's SHAKE
+     * variant. Matrix-A generation always uses SHAKE-128; the hashing SHAKE is
+     * SHAKE-128 for FrodoKEM-640 and SHAKE-256 for -976 / -1344. */
+    wc_Shake shake;
+#ifdef WOLFSSL_FRODOKEM_AES
+    /* Reusable AES-128 object for matrix-A generation (AES key type). Embedded
+     * (not allocated); wc_AesInit'd by wc_FrodoKemKey_Init and re-keyed with
+     * seedA per operation. */
+    Aes aes;
+#endif
+} FrodoKemKey;
+
+
+WOLFSSL_API FrodoKemKey* wc_FrodoKemKey_New(int type, void* heap, int devId);
+WOLFSSL_API int wc_FrodoKemKey_Delete(FrodoKemKey* key, FrodoKemKey** key_p);
+
+WOLFSSL_API int wc_FrodoKemKey_Init(FrodoKemKey* key, int type, void* heap,
+    int devId);
+WOLFSSL_API int wc_FrodoKemKey_Free(FrodoKemKey* key);
+
+#ifndef WOLFSSL_FRODOKEM_NO_MAKE_KEY
+WOLFSSL_API int wc_FrodoKemKey_MakeKey(FrodoKemKey* key, WC_RNG* rng);
+WOLFSSL_API int wc_FrodoKemKey_MakeKeyWithRandom(FrodoKemKey* key,
+    const unsigned char* rand, int len);
+#endif
+
+WOLFSSL_API int wc_FrodoKemKey_CipherTextSize(const FrodoKemKey* key,
+    word32* len);
+WOLFSSL_API int wc_FrodoKemKey_SharedSecretSize(const FrodoKemKey* key,
+    word32* len);
+
+#ifndef WOLFSSL_FRODOKEM_NO_ENCAPSULATE
+WOLFSSL_API int wc_FrodoKemKey_Encapsulate(FrodoKemKey* key, unsigned char* ct,
+    unsigned char* ss, WC_RNG* rng);
+WOLFSSL_API int wc_FrodoKemKey_EncapsulateWithRandom(FrodoKemKey* key,
+    unsigned char* ct, unsigned char* ss, const unsigned char* rand, int len);
+#endif
+#ifndef WOLFSSL_FRODOKEM_NO_DECAPSULATE
+WOLFSSL_API int wc_FrodoKemKey_Decapsulate(FrodoKemKey* key, unsigned char* ss,
+    const unsigned char* ct, word32 len);
+#endif
+
+WOLFSSL_API int wc_FrodoKemKey_DecodePrivateKey(FrodoKemKey* key,
+    const unsigned char* in, word32 len);
+WOLFSSL_API int wc_FrodoKemKey_DecodePublicKey(FrodoKemKey* key,
+    const unsigned char* in, word32 len);
+
+WOLFSSL_API int wc_FrodoKemKey_PrivateKeySize(const FrodoKemKey* key,
+    word32* len);
+WOLFSSL_API int wc_FrodoKemKey_PublicKeySize(const FrodoKemKey* key,
+    word32* len);
+WOLFSSL_API int wc_FrodoKemKey_EncodePrivateKey(FrodoKemKey* key,
+    unsigned char* out, word32 len);
+WOLFSSL_API int wc_FrodoKemKey_EncodePublicKey(FrodoKemKey* key,
+    unsigned char* out, word32 len);
+
+
+
+#ifdef __cplusplus
+    } /* extern "C" */
+#endif
+
+#endif /* WOLFSSL_HAVE_FRODOKEM */
+
+#endif /* WOLF_CRYPT_WC_FRODOKEM_H */
diff --git a/wolfssl/wolfcrypt/wc_frodokem_mat.h b/wolfssl/wolfcrypt/wc_frodokem_mat.h
new file mode 100644
index 0000000000..7ab86badd9
--- /dev/null
+++ b/wolfssl/wolfcrypt/wc_frodokem_mat.h
@@ -0,0 +1,466 @@
+/* wc_frodokem_mat.h
+ *
+ * Copyright (C) 2006-2026 wolfSSL Inc.
+ *
+ * This file is part of wolfSSL.
+ *
+ * wolfSSL is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * wolfSSL is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335, USA
+ */
+
+/*!
+    \file wolfssl/wolfcrypt/wc_frodokem_mat.h
+*/
+
+/* Internal-only declarations for the FrodoKEM reference implementation: the
+ * run-time parameter set, the assembly dispatch configuration and the low-level
+ * matrix / encoding operations shared by wc_frodokem.c and wc_frodokem_mat.c
+ * (and the generated inline-assembly _c.c files). The public API and the
+ * FrodoKemKey object live in wc_frodokem.h. */
+
+#ifndef WOLF_CRYPT_WC_FRODOKEM_MAT_H
+#define WOLF_CRYPT_WC_FRODOKEM_MAT_H
+
+#include <wolfssl/wolfcrypt/wc_frodokem.h>
+
+#ifdef WOLFSSL_HAVE_FRODOKEM
+
+#ifdef __cplusplus
+    extern "C" {
+#endif
+
+/* When the multi-way AVX2/AVX512 Keccak squeeze routines are compiled in,
+ * matrix-A rows for the SHAKE method are generated several at a time: four with
+ * AVX2 (sha3_blocksx4_out_avx2), eight with AVX512 (sha3_blocksx8_out_avx512).
+ * The one-A-row scratch buffer is widened to hold FRODOKEM_ROW_MULT rows (the
+ * widest path compiled in); the runtime path is chosen by CPU features. */
+#if defined(WOLFSSL_FRODOKEM_SHAKE) && defined(USE_INTEL_SPEEDUP) && \
+    !defined(WC_SHA3_NO_ASM)
+    #define FRODOKEM_HAVE_SHAKE_X4
+    #ifndef NO_AVX512_SUPPORT
+        #define FRODOKEM_HAVE_SHAKE_X8
+    #endif
+#endif
+
+/* AVX2/AVX512 assembly for the matrix operations: the fused S*A / A*S row
+ * accumulates, the small nbar x nbar ops (mul_bs, mul_add_sb_plus_e, add) and
+ * the CDF sampler. Available whenever the Intel assembly is built, independent
+ * of the matrix-A generation method (AES or SHAKE). */
+#if defined(USE_INTEL_SPEEDUP)
+    #define FRODOKEM_HAVE_MATRIX_ASM
+    #ifndef NO_AVX512_SUPPORT
+        #define FRODOKEM_HAVE_MATRIX_ASM_AVX512
+    #endif
+    /* VAES (vector AES over YMM/ZMM) matrix-A generation. Follows aes.c: the
+     * VAES asm is compiled and dispatched unless the build opts out with
+     * NO_VAES_SUPPORT, in which case the AES-NI kernel is the widest path. */
+    #ifndef NO_VAES_SUPPORT
+        #define FRODOKEM_HAVE_MATRIX_ASM_VAES
+    #endif
+#endif
+
+/* AArch64 NEON assembly: a 2-way SHAKE (two matrix-A rows at once) and NEON
+ * matrix ops. NEON is baseline on AArch64, so no runtime feature check. */
+#if defined(WOLFSSL_ARMASM) && defined(__aarch64__)
+    #define FRODOKEM_HAVE_NEON_ASM
+    #ifdef WOLFSSL_FRODOKEM_SHAKE
+        #define FRODOKEM_HAVE_SHAKE_NEON
+    #endif
+/* AArch32 NEON assembly: NEON matrix ops only. There is no 2-way NEON Keccak on
+ * AArch32 (the single-stream permutation already uses all 32 d-registers), so
+ * matrix A rows are generated one at a time (FRODOKEM_ROW_MULT is 1). Requires
+ * NEON (-mfpu=neon) - excludes Thumb2, ARMv6 and other non-NEON ARM targets. */
+#elif defined(WOLFSSL_ARMASM) && !defined(WOLFSSL_ARMASM_THUMB2) && \
+      !defined(WOLFSSL_ARMASM_NO_NEON) && \
+      (defined(__ARM_NEON) || defined(__ARM_NEON__))
+    #define FRODOKEM_HAVE_NEON_ASM
+/* ARMv7 Thumb2 assembly: matrix ops using the packed dual-16-bit DSP
+ * instructions (no NEON). Two coefficients per 32-bit register. Requires the
+ * DSP extension (__ARM_FEATURE_DSP, e.g. Cortex-M4/M7/R) - the CDF sampler
+ * stays on the C path. Matrix A rows are generated one at a time. */
+#elif defined(WOLFSSL_ARMASM) && defined(WOLFSSL_ARMASM_THUMB2) && \
+      defined(__ARM_FEATURE_DSP)
+    #define FRODOKEM_HAVE_THUMB2_ASM
+/* AArch32 (A32) scalar assembly: the same packed dual-16-bit matrix ops as the
+ * Thumb2 path, in A32 encoding, for ARMv7-A without NEON (reached only when the
+ * NEON branch above did not match, i.e. NEON is absent or disabled via
+ * WOLFSSL_ARMASM_NO_NEON) when the 32-bit SIMD instructions (SMLAD/SADD16/PKHBT)
+ * are present. The CDF sampler stays on the C path. Matrix A rows are generated
+ * one at a time. */
+#elif defined(WOLFSSL_ARMASM) && !defined(__aarch64__) && \
+      !defined(WOLFSSL_ARMASM_THUMB2) && defined(__ARM_FEATURE_SIMD32)
+    #define FRODOKEM_HAVE_ARM32_SIMD32_ASM
+#endif
+
+/* The ARM asm variants (AArch64 NEON, AArch32 NEON/Thumb2/SIMD32) present the
+ * same matrix-op interface. Alias them to arch-neutral names so the dispatch
+ * has a single ARM path; each call generates FRODOKEM_ROW_MULT A rows (four on
+ * AArch64 NEON, one on AArch32). The CDF sampler is NEON-only. */
+#if defined(FRODOKEM_HAVE_NEON_ASM)
+    #define FRODOKEM_HAVE_ARM_ASM
+    #define frodokem_add_arm                frodokem_add_neon
+    #define frodokem_sa_accum_arm           frodokem_sa_accum_neon
+    #define frodokem_as_accum_arm           frodokem_as_accum_neon
+    #define frodokem_mul_bs_arm             frodokem_mul_bs_neon
+    #define frodokem_mul_add_sb_plus_e_arm  frodokem_mul_add_sb_plus_e_neon
+#elif defined(FRODOKEM_HAVE_THUMB2_ASM)
+    #define FRODOKEM_HAVE_ARM_ASM
+    #define frodokem_add_arm                frodokem_add_thumb2
+    #define frodokem_sa_accum_arm           frodokem_sa_accum_thumb2
+    #define frodokem_as_accum_arm           frodokem_as_accum_thumb2
+    #define frodokem_mul_bs_arm             frodokem_mul_bs_thumb2
+    #define frodokem_mul_add_sb_plus_e_arm  frodokem_mul_add_sb_plus_e_thumb2
+#elif defined(FRODOKEM_HAVE_ARM32_SIMD32_ASM)
+    #define FRODOKEM_HAVE_ARM_ASM
+    #define frodokem_add_arm                frodokem_add_simd32
+    #define frodokem_sa_accum_arm           frodokem_sa_accum_simd32
+    #define frodokem_as_accum_arm           frodokem_as_accum_simd32
+    #define frodokem_mul_bs_arm             frodokem_mul_bs_simd32
+    #define frodokem_mul_add_sb_plus_e_arm  frodokem_mul_add_sb_plus_e_simd32
+#endif
+
+/* AArch64 SVE: a length-agnostic scalable-vector S * A accumulate, emitted
+ * into armv8-frodokem-asm.S alongside NEON. Compiled only when explicitly
+ * enabled (WOLFSSL_FRODOKEM_SVE) and selected at run time by an AT_HWCAP
+ * check, since SVE is optional per CPU (NEON is the always-present fallback).
+ * Not available under inline assembly (WOLFSSL_ARMASM_INLINE): the SVE
+ * routines are only emitted into the .S file, not the inline C twin, so inline
+ * builds fall back to NEON. */
+#if defined(FRODOKEM_HAVE_NEON_ASM) && defined(__aarch64__) && \
+    defined(WOLFSSL_FRODOKEM_SVE) && !defined(WOLFSSL_ARMASM_INLINE)
+    #define FRODOKEM_HAVE_SVE
+#endif
+
+/* AArch64 SME: the B * S product via the ZA-tile integer outer product (UMOPA),
+ * emitted into armv8-frodokem-asm.S alongside NEON. Compiled only when
+ * explicitly enabled (WOLFSSL_FRODOKEM_SME) and selected at run time by an
+ * AT_HWCAP2 check. The MOPA needs the B/S matrices in an interleaved layout, so
+ * the wrapper transposes into a scratch buffer first. Not available under
+ * inline assembly (see FRODOKEM_HAVE_SVE): the SME routines are only in the .S
+ * file. */
+#if defined(FRODOKEM_HAVE_NEON_ASM) && defined(__aarch64__) && \
+    defined(WOLFSSL_FRODOKEM_SME) && !defined(WOLFSSL_ARMASM_INLINE)
+    #define FRODOKEM_HAVE_SME
+#endif
+
+/* ARMv8 AES matrix-A generation via the crypto extension (AESE/AESMC), emitted
+ * into armv8-frodokem-asm.S (AArch64) or armv8-32-frodokem-asm.S (AArch32).
+ * Available whenever the ARMv8 AES hardware crypto is present and the NEON
+ * matrix path is compiled (AArch32 also requires NEON) - the crypto condition
+ * wc_AesEcbEncrypt uses. A single frodokem_gen_a_rows_aes_arm name aliases the
+ * arch-specific routine so the dispatch has one AES A-gen call. Not available
+ * under inline assembly (WOLFSSL_ARMASM_INLINE): the generator is only in the
+ * .S file, not the inline C twin, so inline builds fall back to the per-row C
+ * AES A-generator (frodokem_gen_a_rows_aes). */
+#if defined(FRODOKEM_HAVE_NEON_ASM) && defined(__aarch64__) && \
+    defined(WOLFSSL_FRODOKEM_AES) && !defined(WOLFSSL_ARMASM_NO_HW_CRYPTO) && \
+    !defined(WOLFSSL_ARMASM_INLINE)
+    #define FRODOKEM_HAVE_ARM64_AES_ASM
+    #define FRODOKEM_HAVE_ARM_AES_ASM
+    #define frodokem_gen_a_rows_aes_arm  frodokem_gen_a_rows_aes_arm64
+#elif defined(FRODOKEM_HAVE_NEON_ASM) && !defined(__aarch64__) && \
+    !defined(WOLFSSL_ARMASM_THUMB2) && defined(WOLFSSL_FRODOKEM_AES) && \
+    !defined(WOLFSSL_ARMASM_NO_HW_CRYPTO) && !defined(WOLFSSL_ARMASM_INLINE)
+    #define FRODOKEM_HAVE_ARM32_AES_ASM
+    #define FRODOKEM_HAVE_ARM_AES_ASM
+    #define frodokem_gen_a_rows_aes_arm  frodokem_gen_a_rows_aes_arm32
+#endif
+
+/* The A-row scratch buffer holds this many rows: the number of A rows generated
+ * per generator call (also the outer accumulate loop's row step), used by both
+ * the SHAKE and AES paths; the runtime path is chosen by CPU features. On x86
+ * this matches the fused accumulate width (8 with AVX512, 4 with AVX2). On
+ * AArch64 the AES generator batches four rows so its AES-ECB loop keeps enough
+ * blocks in flight to saturate the crypto unit; the driver then accumulates them
+ * FRODOKEM_AS_ACCUM_ROWS (two) at a time. */
+#if defined(FRODOKEM_HAVE_MATRIX_ASM_AVX512)
+    #define FRODOKEM_ROW_MULT   8
+#elif defined(USE_INTEL_SPEEDUP)
+    #define FRODOKEM_ROW_MULT   4
+#elif defined(FRODOKEM_HAVE_NEON_ASM) && defined(__aarch64__)
+    #define FRODOKEM_ROW_MULT   4
+#else
+    /* AArch32 NEON and the portable C path both consume one A row per call. */
+    #define FRODOKEM_ROW_MULT   1
+#endif
+
+/* Number of A rows a single fused A * S / S * A accumulate call consumes on the
+ * ARM paths. The AArch64 accumulate (NEON, and the runtime-interchangeable
+ * SVE/SME variants) fuses two rows; AArch32 and the portable C path do one.
+ * FRODOKEM_ROW_MULT is a multiple of this, so a generated batch of rows is
+ * accumulated this many at a time. */
+#if defined(FRODOKEM_HAVE_NEON_ASM) && defined(__aarch64__)
+    #define FRODOKEM_AS_ACCUM_ROWS  2
+#else
+    #define FRODOKEM_AS_ACCUM_ROWS  1
+#endif
+
+
+/* Run-time parameters for a FrodoKEM parameter set. */
+struct FrodoKemParams {
+    /* Number of rows/columns in the big matrix A. */
+    int n;
+    /* Number of bits per coefficient - log2 of the modulus q. */
+    int d;
+    /* Mask to reduce a value modulo q (q - 1). */
+    word16 qMask;
+    /* Number of bits extracted per coefficient in Encode/Decode. */
+    int b;
+    /* Cumulative distribution table for error sampling. */
+    const word16* cdf;
+    /* Number of entries in the CDF table. */
+    int cdfLen;
+    /* Length of s / pkh / shared secret / message in bytes (lensec / 8). */
+    int lenSec;
+    /* Use SHAKE256 (1) or SHAKE128 (0) for the hashing function SHAKE. */
+    int useShake256;
+    /* Encoded public key size in bytes (base parameter set only). */
+    int pkSize;
+    /* Encoded private key size in bytes (base parameter set only). */
+    int skSize;
+    /* Length of seedSE in bytes (lenSE / 8). */
+    int lenSE;
+    /* Length of salt in bytes (lensalt / 8). 0 for eFrodoKEM (ephemeral). */
+    int lenSalt;
+    /* Ciphertext size in bytes (depends on salt length). */
+    int ctSize;
+    /* Generate matrix A with AES-128 (1) or SHAKE-128 (0). */
+    int useAes;
+};
+
+
+/* Low-level matrix and encoding operations (wc_frodokem_mat.c). */
+
+WOLFSSL_LOCAL void frodokem_init(void);
+
+WOLFSSL_LOCAL void frodokem_pack(byte* out, const word16* in, int nElem, int d);
+WOLFSSL_LOCAL void frodokem_unpack(word16* out, const byte* in, int nElem,
+    int d);
+WOLFSSL_LOCAL void frodokem_store_matrix(byte* out, const word16* mat, int cnt);
+WOLFSSL_LOCAL void frodokem_load_matrix(word16* mat, const byte* in, int cnt);
+
+WOLFSSL_LOCAL void frodokem_key_encode(word16* c, const byte* msg, int d,
+    int bits);
+WOLFSSL_LOCAL void frodokem_key_decode(byte* msg, const word16* c,
+    const FrodoKemParams* p);
+
+WOLFSSL_LOCAL int frodokem_shake(const FrodoKemParams* p, wc_Shake* shake,
+    const byte* in0, word32 len0, const byte* in1, word32 len1, byte* out,
+    word32 outLen);
+WOLFSSL_LOCAL int frodokem_shake_oneshot(const FrodoKemParams* p,
+    wc_Shake* shake, const byte* in, word32 inLen, byte* out, word32 outLen);
+/* seedA = SHAKE(z); z and seedA are both FRODOKEM_SEEDA_SZ bytes, so this is
+ * just the one-shot path with those fixed lengths. */
+#define frodokem_shake_seeda(p, shake, in, out)                     \
+    frodokem_shake_oneshot(p, shake, in, FRODOKEM_SEEDA_SZ, out,     \
+        FRODOKEM_SEEDA_SZ)
+/* seedSE || k = SHAKE(pkh || u || salt, lenSE + lensec). pkh is lensec bytes;
+ * uSalt is u (lensec) followed by salt (lensalt). */
+#define frodokem_gen_seedse_k(p, shake, pkh, uSalt, out)            \
+    frodokem_shake(p, shake, pkh, (word32)(p)->lenSec, uSalt,       \
+        (word32)((p)->lenSec + (p)->lenSalt), out,                  \
+        (word32)((p)->lenSE + (p)->lenSec))
+
+WOLFSSL_LOCAL int frodokem_gen_noise(const FrodoKemParams* p, wc_Shake* shake,
+    const byte* seInput, byte* tmp, word16* mat0, int cnt0, word16* mat1,
+    int cnt1);
+
+/* Compute out += A * S + E (keygen) and out += S * A + E (encaps/decaps),
+ * generating matrix A a row (batch) at a time. seedA, the parameters and the
+ * SHAKE / AES objects for matrix-A generation are taken from key. */
+WOLFSSL_LOCAL int frodokem_mul_add_as_plus_e(FrodoKemKey* key, word16* out,
+    const word16* s, word16* row);
+WOLFSSL_LOCAL int frodokem_mul_add_sa_plus_e(FrodoKemKey* key, word16* out,
+    const word16* s, word16* row);
+WOLFSSL_LOCAL void frodokem_mul_add_sb_plus_e(word16* out, const word16* b,
+    const word16* s, int n, int qmask);
+WOLFSSL_LOCAL void frodokem_mul_bs(word16* out, const word16* b,
+    const word16* s, int n, int qmask);
+WOLFSSL_LOCAL void frodokem_add(word16* a, const word16* b, int qmask);
+
+#ifdef FRODOKEM_HAVE_MATRIX_ASM
+/* Mask cnt contiguous matrix-A coefficients with qmask in one AVX2 pass - the
+ * mod-q reduction for the 640 SHAKE path (q == 2^15). cnt is a coefficient
+ * count; any leftover after the 16-wide loop is masked one word at a time. */
+WOLFSSL_LOCAL void frodokem_a_rows_reduce_avx2(word16* rows, word32 cnt,
+    int qmask);
+/* Assembly S*A / A*S accumulate for four generated A rows at once (AVX2). */
+WOLFSSL_LOCAL void frodokem_sa_accum_avx2(word16* out, const word16* s,
+    const word16* row, int j, int n);
+WOLFSSL_LOCAL void frodokem_as_accum_avx2(word16* out, const word16* s,
+    const word16* row, int i, int n);
+#ifdef FRODOKEM_HAVE_MATRIX_ASM_VAES
+/* Assembly AES-128-ECB matrix-A row generation + mod-q reduction for cnt rows
+ * (AVX2 + VAES). Builds the counter blocks in registers, ECB-encrypts and
+ * masks; in is unused. ks is the eleven expanded AES-128 round keys (Aes.key).
+ * n must be a multiple of 8 (cnt need not be). */
+WOLFSSL_LOCAL void frodokem_gen_a_rows_aes_avx2(byte* in, word16* out,
+    const byte* ks, int i, int cnt, int n, int qmask);
+#endif
+/* Assembly AES-128-ECB matrix-A row generation + mod-q reduction for cnt rows
+ * (AVX + AES-NI, no VAES). Builds the AES-ECB counter blocks in registers (no
+ * memory build pass), encrypts eight at a time and folds the mask into the
+ * store. Used when AVX2 is present but VAES is not (pre-Ice-Lake Intel,
+ * pre-Zen4 AMD). in is unused (kept for signature parity); out receives the
+ * cnt*n coefficients. ks is the eleven expanded AES-128 round keys (Aes.key).
+ * Handles any n that is a multiple of 8 (cnt and n need not be 8-aligned). */
+WOLFSSL_LOCAL void frodokem_gen_a_rows_aes_aesni(byte* in, word16* out,
+    const byte* ks, int i, int cnt, int n, int qmask);
+#endif
+#ifdef FRODOKEM_HAVE_MATRIX_ASM_AVX512
+/* Assembly S * A / A * S accumulate for eight generated A rows at once
+ * (AVX512). */
+WOLFSSL_LOCAL void frodokem_sa_accum_avx512(word16* out, const word16* s,
+    const word16* row, int j, int n);
+WOLFSSL_LOCAL void frodokem_as_accum_avx512(word16* out, const word16* s,
+    const word16* row, int i, int n);
+#ifdef FRODOKEM_HAVE_MATRIX_ASM_VAES
+/* Assembly AES-128-ECB matrix-A row generation + mod-q reduction for cnt rows
+ * (AVX512 + VAES). Builds the input blocks, ECB-encrypts and masks; in may
+ * alias out. ks is the eleven expanded AES-128 round keys (Aes.key). cnt*2*n
+ * must be a multiple of 256 (holds for cnt == 8). */
+WOLFSSL_LOCAL void frodokem_gen_a_rows_aes_avx512(byte* in, word16* out,
+    const byte* ks, int i, int cnt, int n, int qmask);
+#endif
+#endif
+#ifdef FRODOKEM_HAVE_MATRIX_ASM
+/* AVX2 nbar x nbar matrix ops (B*S, S*B+E, add). */
+WOLFSSL_LOCAL void frodokem_mul_bs_avx2(word16* out, const word16* b,
+    const word16* s, int n, int qmask);
+WOLFSSL_LOCAL void frodokem_mul_add_sb_plus_e_avx2(word16* out, const word16* b,
+    const word16* s, int n, int qmask);
+WOLFSSL_LOCAL void frodokem_add_avx2(word16* a, const word16* b, int qmask);
+WOLFSSL_LOCAL void frodokem_sample_avx2(word16* mat, int cnt,
+    const word16* cdf, int cdflen);
+#endif
+#ifdef FRODOKEM_HAVE_MATRIX_ASM_AVX512
+/* AVX512 nbar x nbar matrix ops (B*S, S*B+E, add). */
+WOLFSSL_LOCAL void frodokem_mul_bs_avx512(word16* out, const word16* b,
+    const word16* s, int n, int qmask);
+WOLFSSL_LOCAL void frodokem_mul_add_sb_plus_e_avx512(word16* out,
+    const word16* b, const word16* s, int n, int qmask);
+WOLFSSL_LOCAL void frodokem_add_avx512(word16* a, const word16* b, int qmask);
+WOLFSSL_LOCAL void frodokem_sample_avx512(word16* mat, int cnt,
+    const word16* cdf, int cdflen);
+#endif
+#ifdef FRODOKEM_HAVE_SHAKE_NEON
+/* AArch64 NEON Keccak-f[1600] permutation on two interleaved states at once
+ * (state[0..24] and state[25..49]) - matrix-A SHAKE generates two rows at a
+ * time. */
+WOLFSSL_LOCAL void frodokem_sha3_x2_neon(word64* state);
+#ifdef WOLFSSL_ARMASM_CRYPTO_SHA3
+/* SHA3-crypto-extension twin of frodokem_sha3_x2_neon (EOR3/RAX1/XAR/BCAX),
+ * used at run time when the CPU reports FEAT_SHA3. */
+WOLFSSL_LOCAL void frodokem_sha3_x2_crypto(word64* state);
+#endif
+#endif
+#ifdef FRODOKEM_HAVE_NEON_ASM
+/* AArch64/AArch32 NEON matrix operations. sa/as accumulate FRODOKEM_ROW_MULT
+ * generated A rows per call (two on AArch64, one on AArch32); the rest match the
+ * Intel routines. */
+WOLFSSL_LOCAL void frodokem_sa_accum_neon(word16* out, const word16* s,
+    const word16* row, int j, int n);
+WOLFSSL_LOCAL void frodokem_sa_accum_x4_neon(word16* out, const word16* s,
+    const word16* row, int j, int n);
+WOLFSSL_LOCAL void frodokem_as_accum_neon(word16* out, const word16* s,
+    const word16* row, int i, int n);
+WOLFSSL_LOCAL void frodokem_as_accum_x4_neon(word16* out, const word16* s,
+    const word16* row, int i, int n);
+WOLFSSL_LOCAL void frodokem_mul_bs_neon(word16* out, const word16* b,
+    const word16* s, int n, int qmask);
+WOLFSSL_LOCAL void frodokem_mul_add_sb_plus_e_neon(word16* out,
+    const word16* b, const word16* s, int n, int qmask);
+WOLFSSL_LOCAL void frodokem_add_neon(word16* a, const word16* b, int qmask);
+WOLFSSL_LOCAL void frodokem_sample_neon(word16* mat, int cnt,
+    const word16* cdf, int cdflen);
+#endif
+#ifdef FRODOKEM_HAVE_ARM64_AES_ASM
+/* AArch64 AES-128-ECB matrix-A row generation + mod-q reduction for cnt rows
+ * (ARMv8 crypto extension). Builds the input blocks, ECB-encrypts and masks; in
+ * may alias out. ks is the eleven expanded AES-128 round keys (Aes.key). cnt*2*n
+ * must be a multiple of 64 (holds for cnt == 2). */
+WOLFSSL_LOCAL void frodokem_gen_a_rows_aes_arm64(byte* in, word16* out,
+    const byte* ks, int i, int cnt, int n, int qmask);
+#endif
+#ifdef FRODOKEM_HAVE_ARM32_AES_ASM
+/* AArch32 AES-128-ECB matrix-A row generation + mod-q reduction for cnt rows
+ * (ARMv8 crypto extension). Builds the input blocks, ECB-encrypts and masks; in
+ * may alias out. ks is the eleven expanded AES-128 round keys (Aes.key). cnt*2*n
+ * must be a multiple of 32 (holds for cnt == 1). */
+WOLFSSL_LOCAL void frodokem_gen_a_rows_aes_arm32(byte* in, word16* out,
+    const byte* ks, int i, int cnt, int n, int qmask);
+#endif
+#ifdef FRODOKEM_HAVE_THUMB2_ASM
+/* ARMv7 Thumb2 packed dual-16-bit matrix operations (no NEON). sa/as accumulate
+ * one A row per call; the CDF sampler stays on the C path. */
+WOLFSSL_LOCAL void frodokem_sa_accum_thumb2(word16* out, const word16* s,
+    const word16* row, int j, int n);
+WOLFSSL_LOCAL void frodokem_as_accum_thumb2(word16* out, const word16* s,
+    const word16* row, int i, int n);
+WOLFSSL_LOCAL void frodokem_mul_bs_thumb2(word16* out, const word16* b,
+    const word16* s, int n, int qmask);
+WOLFSSL_LOCAL void frodokem_mul_add_sb_plus_e_thumb2(word16* out,
+    const word16* b, const word16* s, int n, int qmask);
+WOLFSSL_LOCAL void frodokem_add_thumb2(word16* a, const word16* b, int qmask);
+#endif
+#ifdef FRODOKEM_HAVE_ARM32_SIMD32_ASM
+/* AArch32 (A32) packed dual-16-bit matrix operations (no NEON) using the 32-bit
+ * SIMD instructions (SMLAD/SADD16/PKHBT/...), the A32 encoding of the same
+ * kernels as the Thumb2 path. sa/as accumulate one A row per call; the CDF
+ * sampler stays on the C path. */
+WOLFSSL_LOCAL void frodokem_sa_accum_simd32(word16* out, const word16* s,
+    const word16* row, int j, int n);
+WOLFSSL_LOCAL void frodokem_as_accum_simd32(word16* out, const word16* s,
+    const word16* row, int i, int n);
+WOLFSSL_LOCAL void frodokem_mul_bs_simd32(word16* out, const word16* b,
+    const word16* s, int n, int qmask);
+WOLFSSL_LOCAL void frodokem_mul_add_sb_plus_e_simd32(word16* out,
+    const word16* b, const word16* s, int n, int qmask);
+WOLFSSL_LOCAL void frodokem_add_simd32(word16* a, const word16* b, int qmask);
+#endif
+#ifdef FRODOKEM_HAVE_SVE
+/* AArch64 SVE scalable-vector matrix accumulates (two A rows per call, matching
+ * the NEON interface). Length-agnostic: run on any SVE vector length. */
+WOLFSSL_LOCAL void frodokem_sa_accum_sve(word16* out, const word16* s,
+    const word16* row, int j, int n);
+WOLFSSL_LOCAL void frodokem_as_accum_sve(word16* out, const word16* s,
+    const word16* row, int i, int n);
+WOLFSSL_LOCAL void frodokem_mul_bs_sve(word16* out, const word16* b,
+    const word16* s, int n, int qmask);
+WOLFSSL_LOCAL void frodokem_add_sve(word16* a, const word16* b, int qmask);
+#endif
+#ifdef FRODOKEM_HAVE_SME
+/* AArch64 SME B * S (nbar x nbar) via the ZA-tile UMOPA. bt/st are B and S in
+ * the interleaved MOPA layout (bt[t*nbar*2 + 2*i + e] = b[i*n + 2*t + e]). */
+WOLFSSL_LOCAL void frodokem_mul_bs_sme(word16* out, const word16* bt,
+    const word16* st, int n, int qmask);
+/* AArch64 SME S * A accumulate (two A rows) via the ZA-tile UMOPA tiled over the
+ * n columns. sc is the S column pair (sc[2*i + e] = s[i*n + j + e], 2*nbar
+ * word16); row is the natural pair of A rows (the interleave is done in-asm). */
+WOLFSSL_LOCAL void frodokem_sa_accum_sme(word16* out, const word16* sc,
+    const word16* row, int n);
+/* AArch64 SME A * S accumulate (two A rows) via the ZA-tile UMOPA. at/st are the
+ * A rows and S in the interleaved MOPA layout (at[t*4 + 2*r + e] = row[r*n + 2*t
+ * + e], st[t*nbar*2 + 2*k + e] = s[k*n + 2*t + e]). */
+WOLFSSL_LOCAL void frodokem_as_accum_sme(word16* out, const word16* at,
+    const word16* st, int i, int n);
+#endif
+
+#ifdef __cplusplus
+    } /* extern "C" */
+#endif
+
+#endif /* WOLFSSL_HAVE_FRODOKEM */
+
+#endif /* WOLF_CRYPT_WC_FRODOKEM_MAT_H */